Report 12: resolve absolute IRIs in the research-gaps builder

changelog.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## fix/research-gaps-absolute-iris
+
+Report 12 reads the published record again. Since the canonical layout (#154) every
+`rdf:about` / `rdf:resource` in `ari_t1d.owl` is absolute, but `build_research_gaps.py` keyed
+nodes by `about.lstrip("#")` and matched types like `#AutoimmuneDisease`, so it found 0 diseases.
+
+- `build_research_gaps.py` resolves every IRI against the file's `xml:base` (as
+  `build_uberon_import.py` does) and compares full IRIs. It finds 228 diseases again.
+- `12_Research_Gaps.xlsx` regenerated. Against the 2026-10-16 report, every difference is
+  curation since then: one new disease (ARI:0001168 Primary immune deficiency), Evans syndrome's
+  synonyms now published (#156), and Secondary Raynaud's phenomenon has one cited-type item fewer.
+- `import_columns.py` and `import_gaps.py` are marked historical and refuse to run, as
+  `import_prevalence.py` already does. They edit `ari_t1d.owl` line by line around relative `#`
+  IRIs, so fixing their lookups alone would still leave a writer that breaks the canonical layout.
+  Their imports landed in #141.
+
 ## claude/ontology-file-structure-23209b
 
 Design for splitting the ontology into modules (core, disease, symptoms, anatomy, biomarkers), in

notebook/master-list-comparison/import_columns.py

@@ -1,11 +1,14 @@
-"""Import six immunology columns of the master list into ari_t1d.owl, for every disease that
+"""HISTORICAL - kept as the record of the 2026-10-08 import; it refuses to run.
+
+Import six immunology columns of the master list into ari_t1d.owl, for every disease that
 report 10 matched: Target Antigens, Target Cells, Antibodies, Self Reactive Cells, Molecular
 Mimicry and Environmental Triggers. Each line of a cell becomes one item, linked the way the
 editor reads that category. Items a disease already has are not added again.
 
-Written against the layout ari_t1d.owl had before the canonical formatter (relative `#` IRIs,
-one element per line). Check its lookups against today's file before re-running it, then run
-tools/format-owl.sh on the result.
+It reads and writes ari_t1d.owl line by line in the layout the file had before the canonical
+formatter (#154), with relative `#` IRIs. Since #154 every IRI is absolute, so its `#` lookups and
+the declaration lines it anchors on match nothing, and the items it writes would carry relative
+IRIs. Do not re-run it; add or change items in the editor instead.
 
 Usage: python import_columns.py <master.xlsx> <ari_t1d.owl> <10_Master_List_Comparison.xlsx>
 """
@@ -18,6 +21,9 @@
 
 from sheet_items import items, norm, same_item
 
+raise SystemExit('Historical: written for the pre-#154 layout of ari_t1d.owl. '
+                 'See the docstring.')
+
 MASTER, OWL_PATH, REPORT = sys.argv[1:4]
 SOURCE = 'ARI Master List 2026-10-08'
 CURATOR = 'Leon Islas'

notebook/master-list-comparison/import_gaps.py

@@ -1,4 +1,6 @@
-"""Import the master-list data that report 12 marks "In repo - import" into ari_t1d.owl, for every
+"""HISTORICAL - kept as the record of the 2026-10-08 import; it refuses to run.
+
+Import the master-list data that report 12 marks "In repo - import" into ari_t1d.owl, for every
 disease that report 10 matched. Only fields a disease does not yet have are filled:
 
   Definition                   -> rdfs:comment
@@ -14,9 +16,10 @@
 evidence, unconfirmed as the data model's grade), MeSH terms as synonyms (left to synonym review)
 and report 11 parents (all remaining ones were rejected in proposed_parents.tsv).
 
-Written against the layout ari_t1d.owl had before the canonical formatter (relative `#` IRIs,
-one element per line). Check its lookups against today's file before re-running it, then run
-tools/format-owl.sh on the result.
+It reads and writes ari_t1d.owl line by line in the layout the file had before the canonical
+formatter (#154), with relative `#` IRIs. Since #154 every IRI is absolute, so its `#` lookups and
+the declaration lines it anchors on match nothing, and the items it writes would carry relative
+IRIs. Do not re-run it; add or change items in the editor instead.
 
 Usage: python import_gaps.py <master.xlsx> <ari_t1d.owl> <10_Master_List_Comparison.xlsx> <ari.synonym_blocklist.tsv>
 """
@@ -29,6 +32,9 @@
 
 from sheet_items import items, norm
 
+raise SystemExit('Historical: written for the pre-#154 layout of ari_t1d.owl. '
+                 'See the docstring.')
+
 MASTER, OWL_PATH, REPORT, BLOCKLIST = sys.argv[1:5]
 SOURCE = 'ARI Master List 2026-10-08'
 CURATOR = 'Leon Islas'

notebook/research-gaps/build_research_gaps.py

@@ -14,6 +14,7 @@
 """
 import csv, os, re
 import xml.etree.ElementTree as ET
+from urllib.parse import urljoin
 from collections import Counter, defaultdict
 import openpyxl
 from openpyxl.styles import Alignment, Font, PatternFill
@@ -28,9 +29,11 @@
 PROPOSED_PARENTS = os.path.join(ROOT, "notebook/disease-hierarchy/proposed_parents.tsv")
 OUT = os.path.join(ROOT, "data/4-reports/12_Research_Gaps.xlsx")
 
-A = "{http://www.aurint.org/ontologies/ari-disease#}"
+ARI = "http://www.aurint.org/ontologies/ari-disease#"
+A = "{" + ARI + "}"
 RDF = "{http://www.w3.org/1999/02/22-rdf-syntax-ns#}"
 RDFS = "{http://www.w3.org/2000/01/rdf-schema#}"
+XML = "{http://www.w3.org/XML/1998/namespace}"
 
 PUBLISHED, PARTIAL, IN_REPO, RESEARCH = "Published", "Partial", "In repo - import", "Research"
 STATUSES = [PUBLISHED, PARTIAL, IN_REPO, RESEARCH]
@@ -45,23 +48,28 @@
 
 # ---- published record ----------------------------------------------------------------------
 root = ET.parse(ONTOLOGY).getroot()
-nodes = {}
-for el in root:
-    about = el.get(RDF + "about")
-    if about:
-        nodes[about.lstrip("#")] = el
+
+
+def iri(ref):
+    """Absolute IRI of an rdf:about / rdf:resource, which may be relative to the file's xml:base."""
+    return urljoin(root.get(XML + "base"), ref)
+
+
+nodes = {iri(el.get(RDF + "about")): el for el in root if el.get(RDF + "about")}
 
 
 def values(el, prop, ns=A):
-    return [(c.get(RDF + "resource") or c.text or "").strip() for c in el.findall(ns + prop)]
+    """Literal text, or the absolute IRI of an rdf:resource."""
+    return [iri(c.get(RDF + "resource")) if c.get(RDF + "resource") else (c.text or "").strip()
+            for c in el.findall(ns + prop)]
 
 
 def node_type(ref):
-    return {t.get(RDF + "resource").lstrip("#") for t in nodes[ref.lstrip("#")].findall(RDF + "type")}
+    return {iri(t.get(RDF + "resource")) for t in nodes[ref].findall(RDF + "type")}
 
 
 diseases = [el for el in root
-            if any(t.get(RDF + "resource") == "#AutoimmuneDisease" for t in el.findall(RDF + "type"))
+            if ARI + "AutoimmuneDisease" in {iri(t.get(RDF + "resource")) for t in el.findall(RDF + "type")}
             and "true" not in values(el, "ARI_Obsolete")]
 diseases.sort(key=lambda el: values(el, "ARI_ID")[0])
 
@@ -128,7 +136,7 @@ def short(text, n=90):
 # ---- field checks: each returns (status, note) ------------------------------------------------
 def linked(el, prop, cls=None):
     refs = values(el, prop)
-    return [r for r in refs if cls is None or cls in node_type(r)]
+    return [r for r in refs if cls is None or ARI + cls in node_type(r)]
 
 
 def by_link(prop, cls=None, repo_col=None, repo_what=None, note=""):
@@ -232,7 +240,7 @@ def case_reports(el, ari):
 
 def prevalence(el, ari):
     # Each rate is an EpidemiologyProfile figure (measure, region, sex, rate, year).
-    figures = [nodes[r.lstrip("#")] for r in linked(el, "hasEpidemiologyProfile", "EpidemiologyProfile")]
+    figures = [nodes[r] for r in linked(el, "hasEpidemiologyProfile", "EpidemiologyProfile")]
     rated = [f for f in figures if values(f, "epiRatePer100k")]
     if rated:
         return PUBLISHED, "; ".join(values(f, "label", RDFS)[0] for f in rated)
@@ -283,7 +291,7 @@ def check(el, ari):
 
 def systemic(el, ari):
     tissues = values(el, "targetsTissue")
-    if "#Tissue_Systemic" in tissues:
+    if ARI + "Tissue_Systemic" in tissues:
         return PARTIAL, "No field yet; targetsTissue is Systemic, so the answer is likely Yes"
     return RESEARCH, "No field yet (Yes / No / Variable)"
 
@@ -385,7 +393,7 @@ def synonyms(el, ari):
 
 def citations(el):
     refs = [r for p in CITED for r in values(el, p)]
-    cited = sum(1 for r in refs if nodes[r.lstrip("#")].find(A + "sourcePMID") is not None)
+    cited = sum(1 for r in refs if nodes[r].find(A + "sourcePMID") is not None)
     return len(refs), cited