Link each Symptom individual to its shared symptom (symptomConcept)

.github/scripts/validate_mappings.py

@@ -5,7 +5,9 @@
 predictor's `mappings/ari.synonym_blocklist.tsv` for
 structural problems, malformed identifiers and internal contradictions, that
 the two files agree with each other, and that both agree with the stored
-cross-references in `ontologies/ari_t1d.owl`.
+cross-references in `ontologies/ari_t1d.owl`. Also checks that every Symptom
+individual's `symptomConcept` agrees with
+`notebook/symptom-mapping/symptom_mapping.tsv`.
 
 Usage:
     python .github/scripts/validate_mappings.py                # audit everything
@@ -43,6 +45,15 @@
 # beside it is the generated evidence behind these decisions and is a report, so
 # nothing here validates it.
 BLOCKLIST_PATH = "mappings/ari.synonym_blocklist.tsv"
+# Curated symptom string -> shared ARI symptom (ARIS) table. Every Symptom
+# individual in the ontology links its row's ARIS class with symptomConcept,
+# which notebook/symptom-mapping/build_symptom_mapping.py writes.
+SYMPTOM_MAPPING_PATH = "notebook/symptom-mapping/symptom_mapping.tsv"
+SYMPTOM_MAPPING_COLUMNS = [
+    "Symptom ID", "Symptom", "Mapping", "SYMP ID", "SYMP Label",
+    "HPO ID", "HPO Label", "HPO Mapping", "Parent Symptom", "Merged Into",
+]
+ARIS_BASE = "http://www.aurint.org/ontologies/ari-symptoms#"
 
 SSSOM_COLUMNS = [
     "subject_id",
@@ -163,6 +174,7 @@
 ANNOTATION_RE = re.compile(r"<(ARI_\w+)[^>]*>(.*?)</\1>")
 LABEL_RE = re.compile(r"<rdfs:label[^>]*>(.*?)</rdfs:label>")
 CURIE_MAP_RE = re.compile(r"#\s{2,}([A-Za-z0-9_.]+):\s")
+SYMPTOM_CONCEPT_RE = re.compile(r'<symptomConcept rdf:resource="([^"]*)"/>')
 
 
 @dataclass(frozen=True)
@@ -1225,6 +1237,92 @@ def check_blocklist(rows: list[Row], diseases: dict[str, Disease] | None, report
             )
 
 
+def expected_symptom_concepts(rows: list[Row], report: Report) -> dict[str, str | None]:
+    """Symptom string -> the ARIS class IRI its individuals link to; None for `not a symptom`."""
+    by_name = {row.fields["Symptom"]: row for row in rows}
+    expected: dict[str, str | None] = {}
+    for name, row in by_name.items():
+        target = row.fields["Merged Into"]
+        if target and target not in by_name:
+            report.error(
+                "symptom-merge-target", SYMPTOM_MAPPING_PATH, row.line,
+                f"{name!r} is merged into {target!r}, which has no row.",
+            )
+            continue
+        symptom_id = by_name[target].fields["Symptom ID"] if target else row.fields["Symptom ID"]
+        expected[name] = ARIS_BASE + symptom_id.replace(":", "_") if symptom_id else None
+    return expected
+
+
+def check_symptom_concepts(text: str, rows: list[Row], report: Report) -> None:
+    """Every Symptom individual links the shared symptom its label maps to.
+
+    The label is the key: symptom_mapping.tsv owns which ARIS class a symptom
+    string is, and symptomConcept is that decision applied to each per-disease
+    individual. Findings sit on the label line, since a renamed label is what
+    leaves a link stale. A label with no row yet, or a mapped label not linked
+    yet, is a warning: the curator added a symptom and the mapping catches up.
+    A link that disagrees with the row is an error.
+    """
+    expected = expected_symptom_concepts(rows, report)
+    current: dict | None = None
+    for index, line in enumerate(text.replace("\r\n", "\n").split("\n"), start=1):
+        open_match = ENTITY_OPEN_RE.match(line)
+        if open_match and not line.rstrip().endswith("/>"):
+            current = {
+                "close": f"</{open_match.group(1)}>",
+                "symptom": open_match.group(1) == "Symptom",
+                "label": None,
+                "line": index,
+                "concepts": [],
+            }
+            continue
+        if current is None:
+            continue
+        if '<rdf:type rdf:resource="#Symptom"/>' in line:
+            current["symptom"] = True
+        label_match = LABEL_RE.search(line)
+        if label_match:
+            current["label"] = html.unescape(label_match.group(1)).strip()
+            current["line"] = index
+        current["concepts"].extend(SYMPTOM_CONCEPT_RE.findall(line))
+        if not line.startswith(current["close"]):
+            continue
+        entry, current = current, None
+        if not entry["symptom"]:
+            continue
+        label, line_no, concepts = entry["label"], entry["line"], entry["concepts"]
+        if len(concepts) > 1:
+            report.error(
+                "symptom-concept-repeated", ONTOLOGY_PATH, line_no,
+                f"Symptom {label!r} has {len(concepts)} symptomConcept values; it is one symptom.",
+            )
+            continue
+        have = concepts[0] if concepts else None
+        if label not in expected:
+            report.warning(
+                "symptom-unmapped", ONTOLOGY_PATH, line_no,
+                f"Symptom {label!r} has no row in {SYMPTOM_MAPPING_PATH}"
+                + (f", so its symptomConcept {have} is left from a former label" if have else "")
+                + ". Add a row and run notebook/symptom-mapping/build_symptom_mapping.py.",
+            )
+        elif have == expected[label]:
+            continue
+        elif have is None:
+            report.warning(
+                "symptom-concept-missing", ONTOLOGY_PATH, line_no,
+                f"Symptom {label!r} maps to {expected[label]} in {SYMPTOM_MAPPING_PATH} but has no "
+                "symptomConcept. Run notebook/symptom-mapping/build_symptom_mapping.py.",
+            )
+        else:
+            report.error(
+                "symptom-concept-mismatch", ONTOLOGY_PATH, line_no,
+                f"Symptom {label!r} links {have}, but {SYMPTOM_MAPPING_PATH} maps it to "
+                f"{expected[label] or 'not a symptom'}. Run "
+                "notebook/symptom-mapping/build_symptom_mapping.py.",
+            )
+
+
 def main() -> int:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument(
@@ -1260,6 +1358,10 @@ def main() -> int:
         check_against_ontology(sssom_rows, diseases, report)
         check_equiv_against_ontology(equiv_rows, diseases, report)
         check_ontology_values(diseases, report)
+        symptom_text = read_text(SYMPTOM_MAPPING_PATH, report)
+        if symptom_text:
+            symptom_rows = split_rows(symptom_text, SYMPTOM_MAPPING_PATH, SYMPTOM_MAPPING_COLUMNS, report)
+            check_symptom_concepts(current_text(ONTOLOGY_PATH), symptom_rows, report)
 
     findings = sorted(report.findings, key=Finding.sort_key)
     scope = "whole repository"

.github/workflows/mappings-validate.yml

@@ -10,6 +10,7 @@ on:
     paths:
       - 'mappings/**'
       - 'ontologies/**'
+      - 'notebook/symptom-mapping/symptom_mapping.tsv'
       - '.github/scripts/validate_mappings.py'
       - '.github/workflows/mappings-validate.yml'
 

changelog.md

@@ -1,5 +1,32 @@
 # Changelog
 
+## claude/symptom-concept-links
+
+Links each per-disease Symptom individual to its shared symptom. "Fatigue" in 76 diseases can now
+be queried as one thing, and the 76 individuals stay separate. PR #148 kept same-label individuals
+apart on purpose, because each one holds its own disease's likelihood, description and source. The
+editor also destroys an individual when it is removed from a disease, so merging them would remove
+or edit the symptom for every disease at once.
+
+- New annotation property `symptomConcept` in `ontologies/ari_t1d.owl`. Its value is the IRI of
+  the `ARIS_…` class in `notebook/symptom-mapping/ari_symptoms.owl`. It is an annotation property
+  rather than an object property because the target is a class in another file.
+- `symptom_mapping.tsv` owns the link, keyed by the symptom string, and
+  `build_symptom_mapping.py` writes it. A merged string links the class it merges into. A `not a
+  symptom` string links nothing. The script edits only these lines and gives the same result on a
+  second run.
+- 2,248 of the 2,294 Symptom individuals are linked: all 2,242 live ones and 6
+  obsolete ones whose labels map. The other 46 are obsolete `not a symptom` entries.
+- `ari_symptoms.owl` no longer carries the reverse links (`ariSymptomIndividual`), so the link is
+  stored in one direction only.
+- `validate_mappings.py` checks each link against the TSV, and the PR gate now also runs when the
+  TSV changes. A link that disagrees with the TSV is an error. A symptom with no row, or a mapped
+  symptom with no link yet, is a warning, so adding a symptom in the editor doesn't block the publish.
+- Report 4's *Diseases (ontology)* counts are corrected for 50 symptoms. The old counts still
+  included individuals retired or merged in #143, #144, #148 and #149, whose PRs didn't rebuild the report.
+- Still to do in the editor (KrishnaTO/ARI-metadata-manager): set `symptomConcept` from the TSV
+  when a curator adds or renames a symptom. Until then, those symptoms show the warnings above
+  until the next build.
 ## claude/uberon-tissue-systems-linking-1f66a5
 
 Links ARI's tissue and system classes to UBERON and imports the UBERON terms they link to.

notebook/symptom-mapping/README.md

@@ -62,10 +62,29 @@ It then writes:
 - **`ari_symptoms.owl`**: a class per exact or broader symptom (`ARIS_…`) under its parent. Each class has:
   - `skos:exactMatch` or `skos:broadMatch` to its SYMP term and to its HPO term
   - its merged strings as `skos:altLabel`
-  - `ariSymptomIndividual` links to the Symptom individuals in `ari_t1d.owl` that carry it
 
   The SYMP terms it references are copied in with their labels and is_a parents, up to *symptom*,
   so the file opens in Protégé without importing SYMP. HPO terms are referenced by IRI only.
 
-The OWL file shows the proposal for review. `ontologies/ari_t1d.owl` is still the record, and it
-is not changed here.
+- **`symptomConcept` in `ontologies/ari_t1d.owl`**: on every Symptom individual, the `ARIS_…` class
+  its label maps to. A merged label links the class of the symptom it merges into; a `not a
+  symptom` label links nothing. Every other line of the ontology is left as it is.
+
+### Symptom individuals and shared symptoms
+
+A Symptom individual in `ari_t1d.owl` is one symptom *in one disease*: it carries that disease's
+likelihood, description and source, and the editor deletes it when a curator removes it from that
+disease. So "Fatigue" stays 76 individuals, one per disease, and each points at the one shared
+**Fatigue** class here with `symptomConcept`. To ask which diseases list fatigue:
+
+```sparql
+PREFIX ari:  <http://www.aurint.org/ontologies/ari-disease#>
+PREFIX aris: <http://www.aurint.org/ontologies/ari-symptoms#>
+SELECT ?disease WHERE { ?disease ari:hasSymptom ?s . ?s ari:symptomConcept aris:ARIS_0000190 }
+```
+
+This table owns the link, keyed by the symptom string. The ontology only carries it.
+`.github/scripts/validate_mappings.py` checks every link against this table. A wrong link is an
+error. A symptom with no row yet, or a mapped symptom with no link yet, is a warning: it appears
+when a curator adds or renames a symptom in the editor. To clear it, add the row here and rerun
+the build.