OpenConceptLab/ocl_issues#2840 | Mapper caches lookup 404s and drops bridge targets missing from the target version

src/components/map-projects/MapProject.jsx

@@ -117,7 +117,7 @@ import { getPreviewLimitError, isAIPreviewLimitError } from './previewLimits'
 import { formatRowNumbers, rowNumberOf, runMatchBatch, requestSingleMatch, runWithConcurrency, INTERACTIVE_WAIT_CAP_MS } from './matchBatch'
 import { getAIAssistantChoices, getModelUsed, getPromptTemplateKey, createLatestRequestGate } from './aiVisibility'
 import { DEFAULT_ENCODER_MODEL } from './rerankerModels'
-import { normalizeAlgorithmInvocation, hasSuccessfulAlgorithmResponse, getAlgorithmStagesFromResponses, lookupStatusRank, buildRecommendableConceptEntry, stripConstantClassAndDatatype, buildLookupConceptUrl, serializeCandidates, toStoredRowMatchState } from './normalizers'
+import { normalizeAlgorithmInvocation, hasSuccessfulAlgorithmResponse, getAlgorithmStagesFromResponses, lookupStatusRank, buildRecommendableConceptEntry, stripConstantClassAndDatatype, buildLookupConceptUrl, serializeCandidates, toStoredRowMatchState, dropMissingBridgeTargets } from './normalizers'
 import { parseConceptKey } from './conceptKey'
 import { normalizeLookupConfig } from './lookupConfigFields'
 import { getDeclaredLocales, getDefaultTargetRepoVersion, getProjectTargetRepoVersion, getSavedTargetRepoVersion, getTargetRepoVersionFromUrl, getTargetRepoVersionId } from './projectTargetRepo'
@@ -537,7 +537,11 @@ const MapProject = () => {
         : cr
     })
 
-    rowMatchStateRef.current = { ...prevAll, [rowIndex]: nextRow }
+    // A bridge can propose codes the target version lacks; once a lookup has
+    // found one missing, later rows don't propose it again (ocl_issues#2840).
+    const keptRow = dropMissingBridgeTargets(nextRow, missingConceptKeysRef.current)
+
+    rowMatchStateRef.current = { ...prevAll, [rowIndex]: keptRow }
     setRowMatchState(rowMatchStateRef.current)
 
     // Merge ConceptDefinitions into the project-wide conceptCache, keyed by
@@ -567,7 +571,7 @@ const MapProject = () => {
     // free. Forward-ref pattern because ensureLoaded is declared later.
     if(ensureLoadedRef.current) {
       const keysToLoad = concept_definitions
-        .filter(d => d && d.lookup_status !== 'full')
+        .filter(d => d && d.lookup_status !== 'full' && !missingConceptKeysRef.current.has(d.key))
         .map(d => d.key)
       if(keysToLoad.length) {
         ensureLoadedRef.current(keysToLoad, rowIndex, isRunTraffic).then(() => {
@@ -601,9 +605,13 @@ const MapProject = () => {
   // ocl-scispacy uses a hardcoded canonical 'http://loinc.org' while
   // ocl-bridge cascade targets use target_repo's canonical — they produce
   // different concept_keys but resolve to the same OCL URL). Failed/404
-  // entries are deleted so they remain retryable; successful entries persist
-  // for the session.
+  // entries are deleted so they remain retryable; successful and 404 entries
+  // persist for the session, so a missing code is fetched once
+  // (ocl_issues#2840).
   const urlFetchCacheRef = React.useRef(new Map())
+  // concept_keys whose lookup came back 404: bridge_child candidates for them
+  // are dropped (ocl_issues#2840).
+  const missingConceptKeysRef = React.useRef(new Set())
 
   // Rerank scheduling (plans/unified-mapper-model.md "Rerank — debounce +
   // in-flight check"). Replaces the legacy "wait for every algo to
@@ -5017,6 +5025,23 @@ const MapProject = () => {
     writeConceptCachePatch(key, { ...existing, lookup_status: 'failed' })
   }, [writeConceptCachePatch])
 
+  // A bridge_child whose concept isn't in the target version isn't a
+  // proposal: drop it from every row (ocl_issues#2840).
+  const dropMissingConcept = React.useCallback((key) => {
+    missingConceptKeysRef.current.add(key)
+    const prevAll = rowMatchStateRef.current
+    let changed = false
+    const nextAll = {}
+    Object.entries(prevAll).forEach(([idx, rowState]) => {
+      nextAll[idx] = dropMissingBridgeTargets(rowState, missingConceptKeysRef.current)
+      if(nextAll[idx] !== rowState) changed = true
+    })
+    if(changed) {
+      rowMatchStateRef.current = nextAll
+      setRowMatchState(nextAll)
+    }
+  }, [])
+
   const ensureLoaded = React.useCallback(async (conceptKeys, logRowIndex, isRunTraffic = false) => {
     if(!Array.isArray(conceptKeys) || conceptKeys.length === 0) return
     const ctx = buildProjectContext()
@@ -5118,7 +5143,11 @@ const MapProject = () => {
             if(err?.response?.status === 404)
               log({action: 'lookup', description: t('map_project.lookup_concept_404', {code, attempt: attempt + 1}), extras: {url: oclUrl, error: t('map_project.lookup_concept_404', {code, attempt: attempt + 1})}}, logRowIndex)
           },
-        }).catch(() => {
+        }).catch(err => {
+          // Still 404 after the retries: the code isn't there. Keep that for
+          // the session so other rows don't fetch it again (ocl_issues#2840).
+          if(err?.response?.status === 404)
+            return {notFound: true}
           urlFetchCacheRef.current.delete(oclUrl)
           return null
         })
@@ -5139,6 +5168,8 @@ const MapProject = () => {
           })
         } else {
           writeLookupFailure(key)
+          if(data?.notFound)
+            dropMissingConcept(key)
         }
       } catch (_) {
         writeLookupFailure(key)
@@ -5197,7 +5228,7 @@ const MapProject = () => {
     }
 
     await Promise.all([directPromise, resolvePromise, ...pending])
-  }, [buildProjectContext, writeConceptCachePatch, writeLookupFailure, lookupConfig?.token])
+  }, [buildProjectContext, writeConceptCachePatch, writeLookupFailure, dropMissingConcept, lookupConfig?.token])
 
   // Wire the forward-ref consumed by mergeIntoRowMatchState. useEffect
   // so the ref always points at the latest closure (ensureLoaded captures
@@ -5207,6 +5238,13 @@ const MapProject = () => {
     ensureLoadedRef.current = ensureLoaded
   }, [ensureLoaded])
 
+  // A code missing from one target version may be in another, so the
+  // session's 404s only hold for the version they were fetched against.
+  React.useEffect(() => {
+    missingConceptKeysRef.current = new Set()
+    urlFetchCacheRef.current = new Map()
+  }, [selectedTargetRepoVersion, lookupConfig?.url])
+
   // Lazily enrich a row's saved candidates when it is opened. The persisted
   // v2 candidates payload intentionally carries sparse ConceptDefinitions
   // ('partial'/'pending' — no descriptions, and for migrated projects no

src/components/map-projects/__tests__/normalizers.test.js

@@ -28,6 +28,7 @@ import {
   compactProperty,
   stripConstantClassAndDatatype,
   filterAndTrimNames,
+  dropMissingBridgeTargets,
   filterAndTrimDescriptions,
   primarySubtag,
   uniqByPrimarySubtag,
@@ -1784,3 +1785,42 @@ test('getAlgorithmStagesFromResponses: done (1) when any response succeeded, fai
   )), { 'ocl-semantic': 1, 'ocl-search': -2, 'custom-1': 1 })
   assert.deepEqual(getAlgorithmStagesFromResponses({}), {})
 })
+
+// ---------------------------------------------------------------------------
+// dropMissingBridgeTargets (ocl_issues#2840)
+// ---------------------------------------------------------------------------
+
+test('dropMissingBridgeTargets drops bridge_child candidates for missing codes and their concept_rows', () => {
+  const rowState = {
+    algorithm_responses: {r1: {id: 'r1'}},
+    candidates: {
+      p: {id: 'p', type: 'bridge', concept_key: 'ciel|1'},
+      a: {id: 'a', type: 'bridge_child', concept_key: 'loinc|missing'},
+      b: {id: 'b', type: 'bridge_child', concept_key: 'loinc|present'}
+    },
+    concept_rows: {'ciel|1': {}, 'loinc|missing': {}, 'loinc|present': {}}
+  }
+  const result = dropMissingBridgeTargets(rowState, new Set(['loinc|missing']))
+  assert.deepEqual(Object.keys(result.candidates), ['p', 'b'])
+  assert.deepEqual(Object.keys(result.concept_rows), ['ciel|1', 'loinc|present'])
+  assert.equal(result.algorithm_responses, rowState.algorithm_responses)
+})
+
+test('dropMissingBridgeTargets keeps a missing key another algorithm also proposed', () => {
+  const rowState = {
+    candidates: {
+      a: {id: 'a', type: 'bridge_child', concept_key: 'loinc|x'},
+      s: {id: 's', type: 'standard', concept_key: 'loinc|x'}
+    },
+    concept_rows: {'loinc|x': {}}
+  }
+  const result = dropMissingBridgeTargets(rowState, new Set(['loinc|x']))
+  assert.deepEqual(Object.keys(result.candidates), ['s'])
+  assert.deepEqual(Object.keys(result.concept_rows), ['loinc|x'])
+})
+
+test('dropMissingBridgeTargets returns the same row when nothing is missing', () => {
+  const rowState = {candidates: {a: {id: 'a', type: 'bridge_child', concept_key: 'loinc|y'}}, concept_rows: {'loinc|y': {}}}
+  assert.equal(dropMissingBridgeTargets(rowState, new Set(['loinc|z'])), rowState)
+  assert.equal(dropMissingBridgeTargets(rowState, new Set()), rowState)
+})

src/components/map-projects/normalizers.js

@@ -454,6 +454,33 @@ export const normalizeAlgorithmInvocation = (rawPayload, ctx = {}) => {
   }
 }
 
+/**
+ * Drop the bridge_child candidates whose concept the target version doesn't
+ * have (its lookup came back 404), and the concept_rows nothing references
+ * after that. The bridge filters cascade targets by the target repo, not its
+ * pinned version, so it can propose codes the version lacks (ocl_issues#2840).
+ * Returns the same rowState when nothing is dropped.
+ *
+ * @param {Object} rowState     {algorithm_responses, candidates, concept_rows}
+ * @param {Set} missingKeys     concept_keys known to be missing.
+ */
+export const dropMissingBridgeTargets = (rowState, missingKeys) => {
+  if (!rowState?.candidates || !missingKeys?.size) return rowState
+  const candidates = {}
+  let dropped = false
+  Object.entries(rowState.candidates).forEach(([id, c]) => {
+    if (c?.type === 'bridge_child' && missingKeys.has(c.concept_key)) dropped = true
+    else candidates[id] = c
+  })
+  if (!dropped) return rowState
+  const referencedKeys = new Set(Object.values(candidates).map(c => c?.concept_key))
+  const conceptRows = {}
+  Object.entries(rowState.concept_rows || {}).forEach(([key, cr]) => {
+    if (referencedKeys.has(key)) conceptRows[key] = cr
+  })
+  return { ...rowState, candidates, concept_rows: conceptRows }
+}
+
 const LOOKUP_RANK = { pending: 0, failed: 0, partial: 1, full: 2 }
 export const lookupStatusRank = (status) => LOOKUP_RANK[status] ?? 0