diff --git a/changelog.md b/changelog.md index 89b6d29..1b9540a 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,21 @@ # Changelog +## fix/research-gaps-absolute-iris + +Report 12 reads the published record again. Since the canonical layout (#154) every +`rdf:about` / `rdf:resource` in `ari_t1d.owl` is absolute, but `build_research_gaps.py` keyed +nodes by `about.lstrip("#")` and matched types like `#AutoimmuneDisease`, so it found 0 diseases. + +- `build_research_gaps.py` resolves every IRI against the file's `xml:base` (as + `build_uberon_import.py` does) and compares full IRIs. It finds 228 diseases again. +- `12_Research_Gaps.xlsx` regenerated. Against the 2026-10-16 report, every difference is + curation since then: one new disease (ARI:0001168 Primary immune deficiency), Evans syndrome's + synonyms now published (#156), and Secondary Raynaud's phenomenon has one cited-type item fewer. +- `import_columns.py` and `import_gaps.py` are marked historical and refuse to run, as + `import_prevalence.py` already does. They edit `ari_t1d.owl` line by line around relative `#` + IRIs, so fixing their lookups alone would still leave a writer that breaks the canonical layout. + Their imports landed in #141. + ## claude/ontology-file-structure-23209b Design for splitting the ontology into modules (core, disease, symptoms, anatomy, biomarkers), in diff --git a/data/4-reports/12_Research_Gaps.xlsx b/data/4-reports/12_Research_Gaps.xlsx index 569bbf6..2b928f1 100644 Binary files a/data/4-reports/12_Research_Gaps.xlsx and b/data/4-reports/12_Research_Gaps.xlsx differ diff --git a/notebook/master-list-comparison/import_columns.py b/notebook/master-list-comparison/import_columns.py index fcc3891..c979a6c 100644 --- a/notebook/master-list-comparison/import_columns.py +++ b/notebook/master-list-comparison/import_columns.py @@ -1,11 +1,14 @@ -"""Import six immunology columns of the master list into ari_t1d.owl, for every disease that +"""HISTORICAL - kept as the record of the 2026-10-08 import; it refuses to run. + +Import six immunology columns of the master list into ari_t1d.owl, for every disease that report 10 matched: Target Antigens, Target Cells, Antibodies, Self Reactive Cells, Molecular Mimicry and Environmental Triggers. Each line of a cell becomes one item, linked the way the editor reads that category. Items a disease already has are not added again. -Written against the layout ari_t1d.owl had before the canonical formatter (relative `#` IRIs, -one element per line). Check its lookups against today's file before re-running it, then run -tools/format-owl.sh on the result. +It reads and writes ari_t1d.owl line by line in the layout the file had before the canonical +formatter (#154), with relative `#` IRIs. Since #154 every IRI is absolute, so its `#` lookups and +the declaration lines it anchors on match nothing, and the items it writes would carry relative +IRIs. Do not re-run it; add or change items in the editor instead. Usage: python import_columns.py <10_Master_List_Comparison.xlsx> """ @@ -18,6 +21,9 @@ from sheet_items import items, norm, same_item +raise SystemExit('Historical: written for the pre-#154 layout of ari_t1d.owl. ' + 'See the docstring.') + MASTER, OWL_PATH, REPORT = sys.argv[1:4] SOURCE = 'ARI Master List 2026-10-08' CURATOR = 'Leon Islas' diff --git a/notebook/master-list-comparison/import_gaps.py b/notebook/master-list-comparison/import_gaps.py index 52726a5..39d6f1b 100644 --- a/notebook/master-list-comparison/import_gaps.py +++ b/notebook/master-list-comparison/import_gaps.py @@ -1,4 +1,6 @@ -"""Import the master-list data that report 12 marks "In repo - import" into ari_t1d.owl, for every +"""HISTORICAL - kept as the record of the 2026-10-08 import; it refuses to run. + +Import the master-list data that report 12 marks "In repo - import" into ari_t1d.owl, for every disease that report 10 matched. Only fields a disease does not yet have are filled: Definition -> rdfs:comment @@ -14,9 +16,10 @@ evidence, unconfirmed as the data model's grade), MeSH terms as synonyms (left to synonym review) and report 11 parents (all remaining ones were rejected in proposed_parents.tsv). -Written against the layout ari_t1d.owl had before the canonical formatter (relative `#` IRIs, -one element per line). Check its lookups against today's file before re-running it, then run -tools/format-owl.sh on the result. +It reads and writes ari_t1d.owl line by line in the layout the file had before the canonical +formatter (#154), with relative `#` IRIs. Since #154 every IRI is absolute, so its `#` lookups and +the declaration lines it anchors on match nothing, and the items it writes would carry relative +IRIs. Do not re-run it; add or change items in the editor instead. Usage: python import_gaps.py <10_Master_List_Comparison.xlsx> """ @@ -29,6 +32,9 @@ from sheet_items import items, norm +raise SystemExit('Historical: written for the pre-#154 layout of ari_t1d.owl. ' + 'See the docstring.') + MASTER, OWL_PATH, REPORT, BLOCKLIST = sys.argv[1:5] SOURCE = 'ARI Master List 2026-10-08' CURATOR = 'Leon Islas' diff --git a/notebook/research-gaps/build_research_gaps.py b/notebook/research-gaps/build_research_gaps.py index a78b1bb..64c283a 100644 --- a/notebook/research-gaps/build_research_gaps.py +++ b/notebook/research-gaps/build_research_gaps.py @@ -14,6 +14,7 @@ """ import csv, os, re import xml.etree.ElementTree as ET +from urllib.parse import urljoin from collections import Counter, defaultdict import openpyxl from openpyxl.styles import Alignment, Font, PatternFill @@ -28,9 +29,11 @@ PROPOSED_PARENTS = os.path.join(ROOT, "notebook/disease-hierarchy/proposed_parents.tsv") OUT = os.path.join(ROOT, "data/4-reports/12_Research_Gaps.xlsx") -A = "{http://www.aurint.org/ontologies/ari-disease#}" +ARI = "http://www.aurint.org/ontologies/ari-disease#" +A = "{" + ARI + "}" RDF = "{http://www.w3.org/1999/02/22-rdf-syntax-ns#}" RDFS = "{http://www.w3.org/2000/01/rdf-schema#}" +XML = "{http://www.w3.org/XML/1998/namespace}" PUBLISHED, PARTIAL, IN_REPO, RESEARCH = "Published", "Partial", "In repo - import", "Research" STATUSES = [PUBLISHED, PARTIAL, IN_REPO, RESEARCH] @@ -45,23 +48,28 @@ # ---- published record ---------------------------------------------------------------------- root = ET.parse(ONTOLOGY).getroot() -nodes = {} -for el in root: - about = el.get(RDF + "about") - if about: - nodes[about.lstrip("#")] = el + + +def iri(ref): + """Absolute IRI of an rdf:about / rdf:resource, which may be relative to the file's xml:base.""" + return urljoin(root.get(XML + "base"), ref) + + +nodes = {iri(el.get(RDF + "about")): el for el in root if el.get(RDF + "about")} def values(el, prop, ns=A): - return [(c.get(RDF + "resource") or c.text or "").strip() for c in el.findall(ns + prop)] + """Literal text, or the absolute IRI of an rdf:resource.""" + return [iri(c.get(RDF + "resource")) if c.get(RDF + "resource") else (c.text or "").strip() + for c in el.findall(ns + prop)] def node_type(ref): - return {t.get(RDF + "resource").lstrip("#") for t in nodes[ref.lstrip("#")].findall(RDF + "type")} + return {iri(t.get(RDF + "resource")) for t in nodes[ref].findall(RDF + "type")} diseases = [el for el in root - if any(t.get(RDF + "resource") == "#AutoimmuneDisease" for t in el.findall(RDF + "type")) + if ARI + "AutoimmuneDisease" in {iri(t.get(RDF + "resource")) for t in el.findall(RDF + "type")} and "true" not in values(el, "ARI_Obsolete")] diseases.sort(key=lambda el: values(el, "ARI_ID")[0]) @@ -128,7 +136,7 @@ def short(text, n=90): # ---- field checks: each returns (status, note) ------------------------------------------------ def linked(el, prop, cls=None): refs = values(el, prop) - return [r for r in refs if cls is None or cls in node_type(r)] + return [r for r in refs if cls is None or ARI + cls in node_type(r)] def by_link(prop, cls=None, repo_col=None, repo_what=None, note=""): @@ -232,7 +240,7 @@ def case_reports(el, ari): def prevalence(el, ari): # Each rate is an EpidemiologyProfile figure (measure, region, sex, rate, year). - figures = [nodes[r.lstrip("#")] for r in linked(el, "hasEpidemiologyProfile", "EpidemiologyProfile")] + figures = [nodes[r] for r in linked(el, "hasEpidemiologyProfile", "EpidemiologyProfile")] rated = [f for f in figures if values(f, "epiRatePer100k")] if rated: return PUBLISHED, "; ".join(values(f, "label", RDFS)[0] for f in rated) @@ -283,7 +291,7 @@ def check(el, ari): def systemic(el, ari): tissues = values(el, "targetsTissue") - if "#Tissue_Systemic" in tissues: + if ARI + "Tissue_Systemic" in tissues: return PARTIAL, "No field yet; targetsTissue is Systemic, so the answer is likely Yes" return RESEARCH, "No field yet (Yes / No / Variable)" @@ -385,7 +393,7 @@ def synonyms(el, ari): def citations(el): refs = [r for p in CITED for r in values(el, p)] - cited = sum(1 for r in refs if nodes[r.lstrip("#")].find(A + "sourcePMID") is not None) + cited = sum(1 for r in refs if nodes[r].find(A + "sourcePMID") is not None) return len(refs), cited