From 0014cf408a87051fe9dfa2542b67a0050ba6b032 Mon Sep 17 00:00:00 2001 From: JianhaoYu-AI <139939479+JianhaoYu-AI@users.noreply.github.com> Date: Wed, 21 Oct 2026 15:03:29 +0800 Subject: [PATCH 1/2] feat: add offline reproducibility checker --- docs/guide/offline-reproducibility.md | 150 +++++++ mkdocs.yml | 1 + scripts/check_reproducibility.py | 33 ++ skillopt/utils/reproducibility.py | 311 ++++++++++++++ tests/fixtures/reproducibility/.gitattributes | 3 + .../reproducibility/eval_summary.json | 6 + tests/fixtures/reproducibility/evidence.json | 13 + tests/fixtures/reproducibility/results.jsonl | 2 + .../reproducibility/split_manifest.json | 9 + .../fixtures/reproducibility/test/items.json | 1 + .../fixtures/reproducibility/train/items.json | 1 + tests/fixtures/reproducibility/val/items.json | 1 + tests/test_reproducibility.py | 383 ++++++++++++++++++ 13 files changed, 914 insertions(+) create mode 100644 docs/guide/offline-reproducibility.md create mode 100644 scripts/check_reproducibility.py create mode 100644 skillopt/utils/reproducibility.py create mode 100644 tests/fixtures/reproducibility/.gitattributes create mode 100644 tests/fixtures/reproducibility/eval_summary.json create mode 100644 tests/fixtures/reproducibility/evidence.json create mode 100644 tests/fixtures/reproducibility/results.jsonl create mode 100644 tests/fixtures/reproducibility/split_manifest.json create mode 100644 tests/fixtures/reproducibility/test/items.json create mode 100644 tests/fixtures/reproducibility/train/items.json create mode 100644 tests/fixtures/reproducibility/val/items.json create mode 100644 tests/test_reproducibility.py diff --git a/docs/guide/offline-reproducibility.md b/docs/guide/offline-reproducibility.md new file mode 100644 index 000000000..e97536e37 --- /dev/null +++ b/docs/guide/offline-reproducibility.md @@ -0,0 +1,150 @@ +# Offline reproducibility checker + +The checker reads local split manifests, saved per-item scores, and evaluation +summaries without importing model backends or running benchmark evaluators. +It supports the remaining offline audit work in [Issue #158](https://github.com/microsoft/SkillOpt/issues/158). +It does not change the LiveMath metadata correction from PR #176. + +## Minimal example + +From the repository root, with Python 3.10+: + +```bash +python -m scripts.check_reproducibility tests/fixtures/reproducibility/split_manifest.json --results tests/fixtures/reproducibility/results.jsonl --summary tests/fixtures/reproducibility/eval_summary.json --evidence tests/fixtures/reproducibility/evidence.json +``` + +This tiny **synthetic** fixture has one train item, one validation item, and +two test items. Hard scores 0.25 and 0.75 average to 0.5; soft scores 0.4 and +0.8 average to approximately 0.6. Artifact and aggregation checks pass, but +the overall status is **MISSING** (exit 2): reconstructed data cannot establish +a historical experiment. The fixture's `source_revision` is explicitly a +synthetic label, not a historical Git commit. Do not relabel it as RECORDED. + +Use `--format json` for machine-readable output. Exit codes are 0 for PASS, +1 for FAIL, and 2 for MISSING. FAIL takes precedence when both failures and +missing evidence exist. PASS means the supplied evidence is internally +consistent; provenance declarations are not independent authentication. + +To inspect a released split without original run artifacts: + +```bash +python -m scripts.check_reproducibility data/docvqa_id_split/split_manifest.json +``` + +This reports split counts, IDs, declared dataset revision and local SHA-256 +digests, with MISSING for historical results. Manifests are lookup lists, not +full runnable benchmark payloads; the checker does not download raw data. + +## Supported inputs + +* Manifest: the repository's `split_manifest.json` with `counts.train`, + `counts.val`, `counts.test`, and `source_revision`. Adjacent + `train/items.json`, `val/items.json`, and `test/items.json` must be arrays + of objects with nonempty string or integer `id` values. Integer IDs are + compared as strings, matching the loaders' ID normalization. +* Results: the actual `results.jsonl` format written by benchmark rollouts, + one object per line with `id`, `hard`, `soft`, and optional other fields. +* Summary: `eval_summary.json` written by `scripts/eval_only.py`, containing + `split`, `n_items`, `hard`, and `soft`. Use `--split val` or `--split train` + when checking those canonical splits. The runtime aliases `valid_seen` + (val) and `valid_unseen` (test) are accepted in both CLI and summary. + Use `--split all` for `eval_only.py`'s default concatenation of train, + validation and test. Other aliases/subsets are unsupported. +* Evidence: an explicit JSON sidecar describing the origin and aggregation + of the supplied artifacts. Existing evaluation outputs do not contain all + this metadata; missing fields remain MISSING, never inferred. + +The checker calls `skillopt.utils.scoring.compute_score`: each metric is the +unweighted mean of **all** result rows. It preserves continuous hard scores. +Failed requests and timeouts are not implicitly excluded. Runtime missing +score keys default to zero, but an audit with missing/null scores withholds +the comparison rather than guessing whether the omission was intentional. +Nonfinite or nonnumeric scores are invalid. Empty results cannot establish +an evaluation, even though the runtime returns zero for an empty list. +An empty results file is MISSING, not a failed numerical comparison. With an +unknown filtering or subset protocol, ID coverage differences remain MISSING; +the checker does not assume that every manifest item was scheduled. Duplicate +result IDs still fail independently. With an explicitly declared all-items +protocol, incomplete coverage fails. + +## Evidence sidecar + +Use the fixture's `evidence.json` as a schema example, not historical evidence. +For actual artifacts provide: + +| Field | Meaning | +|---|---| +| `classification` | `RECORDED`, `RECONSTRUCTED`, or `MISSING` | +| `origin` | RECORDED: `historical` or `new_run`; RECONSTRUCTED: `synthetic` or `reconstruction` | +| `run_id` | Identifies the supplied run or explicitly synthetic example | +| `source_revision` | Actual run's code revision, distinct from the checker's current checkout SHA | +| `aggregation` | `single_run_unweighted_all_items` for the supported comparison protocol | +| `manifest_sha256` | Exact SHA-256 of the supplied manifest | +| `train_sha256`, `val_sha256`, `test_sha256` | Exact SHA-256 of each split item file | +| `results_sha256`, `summary_sha256` | Exact SHA-256 of the corresponding saved artifacts | +| `scorer_sha256` | Optional digest of the scorer; checked against the current scorer when supplied | + +Hashes must come from the artifacts whose relationship is being attested. +They bind the declaration to bytes, not to an independently verified timestamp +or author. For example, Python's `hashlib.sha256(Path(path).read_bytes()).hexdigest()` +computes a local file digest. Line-ending changes intentionally change the +digest. The checker never writes or repairs supplied evidence. +Available artifact bindings are checked even when results or summary evidence +is missing or empty. A missing summary withholds the numerical comparison but +does not hide a known hash mismatch in the supplied results or manifest. + +The report always includes the current Git HEAD and dirty state, the current +scorer digest, the manifest's declared dataset revision, and artifact digests. +A dirty checkout's HEAD alone does not identify all local code. Raw dataset +payloads and the actual historical source snapshot are not verified here. +Recorded new runs stay distinct from recorded historical runs. Synthetic or +new runs can pass a local aggregate comparison while historical evidence +remains MISSING. Even a RECORDED historical declaration cannot prove its own +authenticity or that a paper used those artifacts. + +## Aggregation and precision + +Numerical comparisons require exact result-ID coverage of the selected split, +unique result IDs, complete finite scores, matching artifact digests, a run ID +and run source revision, summary metadata, and the explicit supported protocol. +The report shows the row count, denominator, excluded count, and error-row +count. Unknown protocols have no claimed effective denominator or exclusions; +the all-row mean may still be shown as a local calculation. + +Row-level `weight`, `excluded`, or `run_id` fields require additional protocol +support and withhold comparison. Do not concatenate runs or silently filter +rows. Multi-run, weighted, category-averaged and subset results remain MISSING +until their actual protocol is supplied and supported. A mismatch between +supplied artifacts does not establish that a paper value is wrong. + +By default, summary values are fractions and the absolute comparison tolerance +is 1e-9. For a percentage reported to one decimal place, explicitly pass: + +```bash +python -m scripts.check_reproducibility path/to/split_manifest.json --results path/to/results.jsonl --summary path/to/eval_summary.json --evidence path/to/evidence.json --scale 100 --decimals 1 +``` + +This uses a conservative half-last-digit interval (0.05 percentage points, +plus four units in the last place of the recomputed floating-point value). +The slack follows the value's representable precision rather than imposing a +fixed 1e-9 floor on high-precision reports. It accepts either tie-rounding convention at +the boundary; it does not infer precision or scale from the reported number. +Metadata containing nonfinite numbers, including overflowing exponents such +as `1e309`, produces a structured FAIL report. Aggregation or scale overflow +also fails without emitting nonfinite JSON numbers. + +## Validation and limitations + +```bash +python -m pytest -q tests/test_reproducibility.py tests/test_scoring.py tests/test_data_manifests.py +python -m pytest -q +python -m mkdocs build --strict +``` + +The tests are offline and use tiny synthetic artifacts, including explicitly +mocked historical declarations to test classification branches. No real +historical run is bundled. The checker recomputes saved score aggregation, +not the correctness of model answers, evaluator judgments, or model settings. +It cannot prove missing historical results and cannot automatically resolve +[Issue #326](https://github.com/microsoft/SkillOpt/issues/326)'s request for +original per-item records, code snapshots, skill artifacts and runtime config. diff --git a/mkdocs.yml b/mkdocs.yml index b5934ad6d..43b9fc84b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -55,6 +55,7 @@ nav: - OpenAI-compatible Endpoints: sleep/openai-compatible-endpoints.md - Results: sleep/RESULTS.md - Extension Guides: + - Offline Reproducibility Checker: guide/offline-reproducibility.md - Add a New Benchmark: guide/new-benchmark.md - Local Environment Smoke Tests: guide/local-env-smoke.md - Add a New Model Backend: guide/new-backend.md diff --git a/scripts/check_reproducibility.py b/scripts/check_reproducibility.py new file mode 100644 index 000000000..ed85c6e3a --- /dev/null +++ b/scripts/check_reproducibility.py @@ -0,0 +1,33 @@ +"""Offline manifest and saved-score checker: python -m scripts.check_reproducibility.""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from skillopt.utils.reproducibility import SPLITS, audit, format_text + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("manifest", type=Path) + parser.add_argument("--source-root", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--results", type=Path) + parser.add_argument("--summary", type=Path) + parser.add_argument("--evidence", type=Path) + parser.add_argument("--split", choices=tuple(SPLITS), default="test") + parser.add_argument("--decimals", type=int, choices=range(13)) + parser.add_argument("--scale", type=int, choices=(1, 100), default=1) + parser.add_argument("--format", choices=("text", "json"), default="text") + args = parser.parse_args(argv) + report = audit( + args.manifest, source_root=args.source_root, results_path=args.results, + summary_path=args.summary, evidence_path=args.evidence, split=args.split, + decimals=args.decimals, scale=args.scale, + ) + print(json.dumps(report, indent=2, allow_nan=False) if args.format == "json" else format_text(report)) + return {"PASS": 0, "FAIL": 1, "MISSING": 2}[report["status"]] + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skillopt/utils/reproducibility.py b/skillopt/utils/reproducibility.py new file mode 100644 index 000000000..bcc1ecb78 --- /dev/null +++ b/skillopt/utils/reproducibility.py @@ -0,0 +1,311 @@ +"""Read-only audits of released split manifests and saved evaluation scores. + +No benchmark adapter is imported and no model or evaluator is executed. +""" +from __future__ import annotations + +import hashlib +import json +import math +import subprocess +from collections import Counter +from itertools import combinations +from pathlib import Path +from typing import Any + +from skillopt.datasets.base import SPLIT_NAMES +from skillopt.utils.scoring import compute_score + +PROTOCOL = "single_run_unweighted_all_items" +SPLITS = {"train": "train", "val": "val", "test": "test", "valid_seen": "val", "valid_unseen": "test", "all": "all"} +LIMITATIONS = [ + "Recomputes saved scores, not model outputs or benchmark judgments.", + "Hashes identify supplied bytes; provenance declarations are not independent authentication.", + "Current checkout and dataset metadata do not establish the original experiment versions.", + "Cannot recover missing historical artifacts or resolve Issue #326's artifact request.", + "Multi-run, weighted, filtered, and subset protocols require separate evidence and are not compared.", +] + + +def sha256(path: Path) -> str: + """Hash the exact local artifact bytes.""" + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _git(root: Path, *args: str) -> str | None: + try: + return subprocess.check_output( + ["git", "-C", str(root), *args], stderr=subprocess.DEVNULL, text=True, + ).strip() + except (OSError, subprocess.CalledProcessError): + return None + + +def _object(path: Path) -> dict[str, Any]: + def reject_constant(value: str) -> None: + raise ValueError(f"{path}: nonfinite JSON literal {value}") + + def finite_float(value: str) -> float: + number = float(value) + if not math.isfinite(number): + raise ValueError(f"{path}: nonfinite JSON number {value}") + return number + + value = json.loads( + path.read_text(encoding="utf-8"), parse_constant=reject_constant, parse_float=finite_float, + ) + if not isinstance(value, dict): + raise ValueError(f"{path}: expected a JSON object") + return value + + +def _ids(rows: list[Any], label: str) -> list[str]: + ids = [] + for index, row in enumerate(rows): + if not isinstance(row, dict) or not isinstance(row.get("id"), (str, int)) or isinstance(row["id"], bool): + raise ValueError(f"{label}[{index}]: expected a string or integer id") + value = str(row["id"]) + if not value.strip(): + raise ValueError(f"{label}[{index}]: empty id") + ids.append(value) + return ids + + +def _number(value: Any) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) + + +def _present(value: Any) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def audit( + manifest_path: Path, + *, + source_root: Path, + results_path: Path | None = None, + summary_path: Path | None = None, + evidence_path: Path | None = None, + split: str = "test", + decimals: int | None = None, + scale: int = 1, +) -> dict[str, Any]: + """Audit canonical train/val/test manifests and eval_only JSONL artifacts. + + Rounded reports are compared within half a unit in the explicitly supplied + last decimal place. Without precision, use absolute tolerance 1e-9. + Malformed input is FAIL; unavailable evidence is MISSING. + """ + if split not in SPLITS or scale not in (1, 100) or (decimals is not None and not 0 <= decimals <= 12): + raise ValueError("Expected supported split, scale 1 or 100, and decimals between 0 and 12") + split = SPLITS[split] + report: dict[str, Any] = { + "source_revision": _git(source_root, "rev-parse", "HEAD"), + "source_dirty": None, + "dataset_revision": None, + "artifacts": {}, + "evidence": {"classification": "MISSING", "origin": None}, + "split_counts": {}, + "duplicates": {}, + "overlaps": {}, + "aggregation": {}, + "checks": [], + "limitations": list(LIMITATIONS), + } + evidence: dict[str, Any] = {} + bindings_valid: bool | None = None + + def check(name: str, status: str, detail: str) -> None: + report["checks"].append({"name": name, "status": status, "detail": detail}) + + def artifact(path: Path, label: str) -> bool: + if not path.is_file(): + check(label, "MISSING", f"Artifact unavailable: {path}") + return False + try: + report["artifacts"][label] = {"path": str(path), "sha256": sha256(path)} + except OSError as exc: + check(label, "FAIL", f"Cannot read artifact {path}: {exc}") + return False + return True + + def check_bindings() -> bool: + nonlocal bindings_valid + if bindings_valid is None: + bindings_valid = True + labels = ["manifest", "train", "val", "test", "results", "summary"] + if "scorer_sha256" in evidence: + labels.append("scorer") + for label in labels: + expected_hash = evidence.get(f"{label}_sha256") + actual_hash = report["artifacts"].get(label, {}).get("sha256") + status = "MISSING" if not expected_hash or not actual_hash else "PASS" if expected_hash == actual_hash else "FAIL" + check(f"binding:{label}", status, f"{label}: supplied provenance digest compared with current bytes") + bindings_valid &= status == "PASS" + return bindings_valid + + def finish() -> dict[str, Any]: + # Integrity checks are independent of whether aggregation is possible. + check_bindings() + states = {item["status"] for item in report["checks"]} + report["status"] = "FAIL" if "FAIL" in states else "MISSING" if "MISSING" in states else "PASS" + report["missing_evidence"] = [c["detail"] for c in report["checks"] if c["status"] == "MISSING"] + return report + + dirty = _git(source_root, "status", "--porcelain", "--untracked-files=normal") + report["source_dirty"] = None if dirty is None else bool(dirty) + check("source", "PASS" if report["source_revision"] else "MISSING", "Current checkout SHA; not historical run SHA") + scorer = Path(__file__).with_name("scoring.py") + artifact(scorer, "scorer") + split_ids: dict[str, list[str]] = {} + try: + if not artifact(manifest_path, "manifest"): + return finish() + manifest = _object(manifest_path) + report["dataset_revision"] = manifest.get("source_revision") + check("dataset_revision", "PASS" if _present(manifest.get("source_revision")) else "MISSING", "Dataset revision declared by manifest; raw payload not verified") + counts = manifest.get("counts", {}) + if not isinstance(counts, dict): + raise ValueError("manifest counts must be an object") + for name in SPLIT_NAMES: + path = manifest_path.parent / name / "items.json" + if not artifact(path, name): + continue + rows = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(rows, list): + raise ValueError(f"{path}: expected a JSON array") + ids = _ids(rows, name) + split_ids[name] = ids + declared = counts.get(name) + report["split_counts"][name] = {"actual": len(rows), "declared": declared} + valid_count = isinstance(declared, int) and not isinstance(declared, bool) and declared >= 0 + check(f"count:{name}", "MISSING" if declared is None else "PASS" if valid_count and declared == len(rows) else "FAIL", f"{name}: declared={declared}, actual={len(rows)}") + duplicates = sorted(key for key, count in Counter(ids).items() if count > 1) + report["duplicates"][name] = duplicates + check(f"duplicates:{name}", "FAIL" if duplicates else "PASS", f"{name}: {duplicates}") + for left, right in combinations(split_ids, 2): + overlap = sorted(set(split_ids[left]) & set(split_ids[right])) + report["overlaps"][f"{left}/{right}"] = overlap + check(f"overlap:{left}/{right}", "FAIL" if overlap else "PASS", f"{left}/{right}: {overlap}") + except (ValueError, OSError) as exc: + check("manifest", "FAIL", str(exc)) + return finish() + + try: + if evidence_path is not None and artifact(evidence_path, "evidence"): + evidence = _object(evidence_path) + kind = evidence.get("classification", "MISSING") + origin = evidence.get("origin") + if not isinstance(kind, str) or kind not in {"RECORDED", "RECONSTRUCTED", "MISSING"}: + raise ValueError("classification must be RECORDED, RECONSTRUCTED, or MISSING") + if kind == "RECORDED" and origin not in ("historical", "new_run"): + raise ValueError("RECORDED requires origin historical or new_run; synthetic data is RECONSTRUCTED") + if kind == "RECONSTRUCTED" and origin not in ("synthetic", "reconstruction"): + raise ValueError("RECONSTRUCTED requires origin synthetic or reconstruction") + report["evidence"] = {"classification": kind, "origin": origin, "run_id": evidence.get("run_id"), "run_source_revision": evidence.get("source_revision")} + run_metadata = _present(evidence.get("run_id")) and _present(evidence.get("source_revision")) + historical = kind == "RECORDED" and origin == "historical" and run_metadata + check("historical_evidence", "PASS" if historical else "MISSING", "Historical origin, run ID and source revision are user declarations; absent or reconstructed artifacts cannot verify a historical result") + if kind != "MISSING": + check("run_metadata", "PASS" if run_metadata else "MISSING", "Run ID and actual run source revision required") + except (ValueError, OSError) as exc: + check("evidence", "FAIL", str(exc)) + return finish() + + try: + results_available = results_path is not None and artifact(results_path, "results") + summary_available = summary_path is not None and artifact(summary_path, "summary") + if not results_available: + check("per_item_results", "MISSING", "Per-item results.jsonl unavailable; no aggregate verdict") + return finish() + rows = [json.loads(line) for line in results_path.read_text(encoding="utf-8").splitlines() if line.strip()] + ids = _ids(rows, "results") + duplicates = sorted(key for key, count in Counter(ids).items() if count > 1) + check("result_duplicates", "FAIL" if duplicates else "PASS", str(duplicates)) + expected = split_ids.get(split) + if split == "all" and len(split_ids) == len(SPLIT_NAMES): + expected = [item for name in SPLIT_NAMES for item in split_ids[name]] + protocol = evidence.get("aggregation") == PROTOCOL + # No implicit dropping of errors: compute_score includes every row. + unsupported = any(any(key in row for key in ("weight", "excluded", "run_id")) for row in rows) + protocol &= not unsupported + check("protocol", "PASS" if protocol else "MISSING", f"Requires declared {PROTOCOL}; row-level run_id/weight/excluded needs separate protocol support") + coverage = expected is not None and Counter(ids) == Counter(expected) and not duplicates + coverage_status = "MISSING" if expected is None or not rows else "PASS" if coverage else "FAIL" if protocol else "MISSING" + check("result_coverage", coverage_status, f"split={split}; missing={sorted(set(expected or []) - set(ids))}; unexpected={sorted(set(ids) - set(expected or []))}; results={len(rows)}") + complete = True + for metric in ("hard", "soft"): + missing = [i for i, row in enumerate(rows) if metric not in row or row[metric] is None] + invalid = [i for i, row in enumerate(rows) if metric in row and row[metric] is not None and not _number(row[metric])] + check(f"values:{metric}", "FAIL" if invalid else "MISSING" if missing else "PASS", f"{metric}: missing/null rows={missing}, invalid rows={invalid}") + complete &= not missing and not invalid + report["aggregation"] = { + "n_rows": len(rows), "row_mean_denominator": len(rows), + "effective_denominator": len(rows) if protocol else None, + "excluded": 0 if protocol else None, + "missing_hard_defaults_to_zero_in_runtime": True, + "error_rows": sum(row.get("agent_ok") is False or row.get("phase") == "timeout" for row in rows), + "formula": "sum(metric)/len(results); all rows including errors; runtime missing keys default to zero", + "comparison_status": "MISSING", + } + if not rows: + check("nonempty_results", "MISSING", "Empty results cannot establish a reported evaluation") + return finish() + if complete: + hard, soft = compute_score(rows) + if math.isfinite(hard) and math.isfinite(soft): + report["aggregation"].update(hard=hard, soft=soft) + else: + check("aggregation_values", "FAIL", "Saved-score aggregation produced a nonfinite value") + complete = False + if not summary_available: + check("summary", "MISSING", "eval_summary.json unavailable; no reported-score comparison") + return finish() + summary = _object(summary_path) + bindings = check_bindings() + n_items = summary.get("n_items") + summary_split = summary.get("split") + normalized_split = SPLITS.get(summary_split) if isinstance(summary_split, str) else None + summary_metadata = isinstance(n_items, int) and not isinstance(n_items, bool) and normalized_split == split + check("summary_metadata", "MISSING" if n_items is None or normalized_split is None else "PASS" if summary_metadata else "FAIL", f"n_items={n_items}, split={summary_split}") + run_metadata = kind != "MISSING" and run_metadata + ready = complete and coverage and protocol and bindings and summary_metadata and run_metadata + if ready: + check("denominator", "PASS" if n_items == len(rows) else "FAIL", f"reported n_items={n_items}, actual denominator={len(rows)}") + outcomes = [] + for metric in ("hard", "soft"): + reported = summary.get(metric) + actual = report["aggregation"][metric] * scale + if not math.isfinite(actual): + check(f"aggregate:{metric}", "FAIL", f"{metric}: scaling by {scale} produced a nonfinite value") + outcomes.append("FAIL") + continue + tolerance = 1e-9 if decimals is None else 0.5 * 10 ** -decimals + 4 * math.ulp(actual) + status = "MISSING" if reported is None else "FAIL" if not _number(reported) else "PASS" if abs(actual - reported) <= tolerance else "FAIL" + check(f"aggregate:{metric}", status, f"recomputed={actual}, reported={reported}, scale={scale}, absolute tolerance={tolerance}") + outcomes.append(status) + report["aggregation"]["comparison_status"] = "FAIL" if "FAIL" in outcomes or n_items != len(rows) else "MISSING" if "MISSING" in outcomes else "PASS" + else: + check("aggregate_comparison", "MISSING", "Incomplete values, ID coverage, artifact bindings, summary metadata, or aggregation protocol; no numerical inconsistency verdict") + except (ValueError, OSError, OverflowError) as exc: + check("results", "FAIL", str(exc)) + return finish() + + +def format_text(report: dict[str, Any]) -> str: + """Render all findings, including missing evidence, without hiding failures.""" + lines = [ + f"Check status: {report['status']}", + f"Source revision: {report['source_revision']} (dirty={report['source_dirty']})", + f"Dataset/manifest revision: {report['dataset_revision']}", + f"Evidence: {json.dumps(report['evidence'], sort_keys=True)}", + f"Split counts: {json.dumps(report['split_counts'], sort_keys=True)}", + f"Duplicates: {json.dumps(report['duplicates'], sort_keys=True)}", + f"Overlaps: {json.dumps(report['overlaps'], sort_keys=True)}", + f"Hard/soft aggregation: {json.dumps(report['aggregation'], sort_keys=True)}", + "Artifact hashes: " + json.dumps(report['artifacts'], sort_keys=True), + ] + lines += [f"[{c['status']}] {c['name']}: {c['detail']}" for c in report["checks"]] + lines += ["Missing evidence: " + json.dumps(report["missing_evidence"]), "Reproduction limitations:"] + lines += [f"- {item}" for item in report["limitations"]] + return "\n".join(lines) diff --git a/tests/fixtures/reproducibility/.gitattributes b/tests/fixtures/reproducibility/.gitattributes new file mode 100644 index 000000000..5a7ced534 --- /dev/null +++ b/tests/fixtures/reproducibility/.gitattributes @@ -0,0 +1,3 @@ +# Evidence hashes identify exact bytes, including line endings. +*.json text eol=lf +*.jsonl text eol=lf diff --git a/tests/fixtures/reproducibility/eval_summary.json b/tests/fixtures/reproducibility/eval_summary.json new file mode 100644 index 000000000..baecd7d5e --- /dev/null +++ b/tests/fixtures/reproducibility/eval_summary.json @@ -0,0 +1,6 @@ +{ + "split": "test", + "n_items": 2, + "hard": 0.5, + "soft": 0.6 +} diff --git a/tests/fixtures/reproducibility/evidence.json b/tests/fixtures/reproducibility/evidence.json new file mode 100644 index 000000000..39bc2066e --- /dev/null +++ b/tests/fixtures/reproducibility/evidence.json @@ -0,0 +1,13 @@ +{ + "classification": "RECONSTRUCTED", + "origin": "synthetic", + "run_id": "synthetic-example-only", + "source_revision": "synthetic-example-not-a-historical-commit", + "aggregation": "single_run_unweighted_all_items", + "manifest_sha256": "9c7c3be628488fdf640ff7a4fe8cc6f8bf18fd6798e10ed571c4fbb9577d8c8e", + "results_sha256": "48c912fcd09985619bc01dbf84ac309ff53bbc3460bb66e59c577eba3f48d062", + "summary_sha256": "6c1c9209a2898fb1ac3c957c0a4e4e0eb537e35ccbd5df6ca7cdb23a1f594d05", + "train_sha256": "5e488fa13a3542d97577a48a657cd86788274d8a221b168fb193ef19220973af", + "val_sha256": "6a2bb60cf6b680d783fd4f547bb9ea7e9e63464d950169f420bca383399d96b6", + "test_sha256": "1985d8ec2557b474716b558c29911015af11b810f1540a70d78cdc36b5ede28f" +} diff --git a/tests/fixtures/reproducibility/results.jsonl b/tests/fixtures/reproducibility/results.jsonl new file mode 100644 index 000000000..6dd76bd17 --- /dev/null +++ b/tests/fixtures/reproducibility/results.jsonl @@ -0,0 +1,2 @@ +{"id":"a","hard":0.25,"soft":0.4} +{"id":"b","hard":0.75,"soft":0.8} diff --git a/tests/fixtures/reproducibility/split_manifest.json b/tests/fixtures/reproducibility/split_manifest.json new file mode 100644 index 000000000..3edfd5d5d --- /dev/null +++ b/tests/fixtures/reproducibility/split_manifest.json @@ -0,0 +1,9 @@ +{ + "benchmark": "SYNTHETIC - not a paper dataset", + "source_revision": "synthetic-v1", + "counts": { + "train": 1, + "val": 1, + "test": 2 + } +} diff --git a/tests/fixtures/reproducibility/test/items.json b/tests/fixtures/reproducibility/test/items.json new file mode 100644 index 000000000..e19936c54 --- /dev/null +++ b/tests/fixtures/reproducibility/test/items.json @@ -0,0 +1 @@ +[{"id": "a"}, {"id": "b"}] diff --git a/tests/fixtures/reproducibility/train/items.json b/tests/fixtures/reproducibility/train/items.json new file mode 100644 index 000000000..29a586973 --- /dev/null +++ b/tests/fixtures/reproducibility/train/items.json @@ -0,0 +1 @@ +[{"id": "train-1"}] diff --git a/tests/fixtures/reproducibility/val/items.json b/tests/fixtures/reproducibility/val/items.json new file mode 100644 index 000000000..4fb3867f7 --- /dev/null +++ b/tests/fixtures/reproducibility/val/items.json @@ -0,0 +1 @@ +[{"id": "val-1"}] diff --git a/tests/test_reproducibility.py b/tests/test_reproducibility.py new file mode 100644 index 000000000..ff9b8330f --- /dev/null +++ b/tests/test_reproducibility.py @@ -0,0 +1,383 @@ +"""Small offline fixtures; no original experiment records or model calls.""" +from __future__ import annotations + +import json +import shutil +from pathlib import Path + +import pytest + +from scripts.check_reproducibility import main +from skillopt.utils.reproducibility import PROTOCOL, audit, format_text, sha256 +from skillopt.utils.scoring import compute_score + +FIXTURE = Path(__file__).parent / "fixtures" / "reproducibility" +ROOT = Path(__file__).resolve().parents[1] + + +@pytest.fixture +def sample(tmp_path: Path) -> Path: + shutil.copytree(FIXTURE, tmp_path / "sample") + return tmp_path / "sample" + + +def write(path: Path, value: object) -> None: + path.write_text(json.dumps(value), encoding="utf-8") + + +def bind(sample: Path, **updates: object) -> None: + """Bind mutated test artifacts, never label these synthetic bytes historical.""" + evidence = json.loads((sample / "evidence.json").read_text(encoding="utf-8")) + for label, path in { + "manifest": sample / "split_manifest.json", + "results": sample / "results.jsonl", + "summary": sample / "eval_summary.json", + "scorer": ROOT / "skillopt" / "utils" / "scoring.py", + **{s: sample / s / "items.json" for s in ("train", "val", "test")}, + }.items(): + evidence[f"{label}_sha256"] = sha256(path) + evidence.update(updates) + write(sample / "evidence.json", evidence) + + +def run(sample: Path, **kwargs: object) -> dict: + return audit( + sample / "split_manifest.json", source_root=ROOT, + results_path=sample / "results.jsonl", summary_path=sample / "eval_summary.json", + evidence_path=sample / "evidence.json", **kwargs, + ) + + +def status(report: dict, name: str) -> str: + return next(c["status"] for c in report["checks"] if c["name"] == name) + + +def test_synthetic_fixture_is_consistent_but_not_historical(sample: Path) -> None: + report = run(sample) + assert report["status"] == "MISSING" + assert report["evidence"]["classification"] == "RECONSTRUCTED" + assert report["aggregation"]["comparison_status"] == "PASS" + assert report["aggregation"]["hard"] == 0.5 + assert not [c for c in report["checks"] if c["status"] == "FAIL"] + assert status(report, "historical_evidence") == "MISSING" + + +def test_recorded_classification_is_a_declaration_only(sample: Path) -> None: + # Exercise the recorded branch with explicitly MOCK metadata, not a fixture + # presented as an actual historical run. Hashes cannot authenticate origin. + bind(sample, classification="RECORDED", origin="historical", run_id="MOCK-ONLY") + report = run(sample) + assert report["status"] == "PASS" + assert report["evidence"]["run_id"] == "MOCK-ONLY" + assert any("not independent authentication" in text for text in report["limitations"]) + + +def test_duplicate_split_id(sample: Path) -> None: + write(sample / "test/items.json", [{"id": "a"}, {"id": "a"}]) + bind(sample) + report = run(sample) + assert status(report, "duplicates:test") == "FAIL" + assert report["duplicates"]["test"] == ["a"] + + +def test_cross_split_overlap(sample: Path) -> None: + write(sample / "train/items.json", [{"id": "a"}]) + bind(sample) + report = run(sample) + assert status(report, "overlap:train/test") == "FAIL" + assert report["overlaps"]["train/test"] == ["a"] + + +def test_manifest_count_mismatch(sample: Path) -> None: + manifest = json.loads((sample / "split_manifest.json").read_text()) + manifest["counts"]["test"] = 3 + write(sample / "split_manifest.json", manifest) + bind(sample) + assert status(run(sample), "count:test") == "FAIL" + + +@pytest.mark.parametrize("metric", ["hard", "soft"]) +def test_score_mismatch_with_complete_protocol(sample: Path, metric: str) -> None: + summary = json.loads((sample / "eval_summary.json").read_text()) + summary[metric] = 0.9 + write(sample / "eval_summary.json", summary) + bind(sample) + report = run(sample) + assert status(report, f"aggregate:{metric}") == "FAIL" + assert report["aggregation"]["comparison_status"] == "FAIL" + + +def test_nonbinary_scores_reuse_runtime_mean(sample: Path) -> None: + rows = [{"id": "a", "hard": 0.25, "soft": 0.4}, {"id": "b", "hard": 0.75, "soft": 0.8}] + (sample / "results.jsonl").write_text("\n".join(json.dumps(r) for r in rows)) + bind(sample) + report = run(sample) + assert report["aggregation"]["hard"] == compute_score(rows)[0] + assert report["aggregation"]["comparison_status"] == "PASS" + + +def test_missing_per_item_results(sample: Path) -> None: + (sample / "results.jsonl").unlink() + report = run(sample) + assert status(report, "per_item_results") == "MISSING" + assert report["aggregation"] == {} + + +@pytest.mark.parametrize("value,expected", [(66.7, "PASS"), (66.8, "FAIL")]) +def test_explicit_percentage_precision(sample: Path, value: float, expected: str) -> None: + rows = [{"id": "a", "hard": 1 / 3, "soft": 1 / 3}, {"id": "b", "hard": 1, "soft": 1}] + (sample / "results.jsonl").write_text("\n".join(json.dumps(r) for r in rows)) + write(sample / "eval_summary.json", {"n_items": 2, "split": "test", "hard": value, "soft": value}) + bind(sample) + assert status(run(sample, decimals=1, scale=100), "aggregate:hard") == expected + + +def test_new_recorded_run_does_not_prove_history(sample: Path) -> None: + bind(sample, classification="RECORDED", origin="new_run") + report = run(sample) + assert status(report, "historical_evidence") == "MISSING" + assert report["evidence"]["origin"] == "new_run" + + +@pytest.mark.parametrize("field", ["aggregation", "run_id", "source_revision", "results_sha256"]) +def test_missing_protocol_or_provenance(sample: Path, field: str) -> None: + evidence = json.loads((sample / "evidence.json").read_text()) + evidence.pop(field) + write(sample / "evidence.json", evidence) + report = run(sample) + assert report["status"] == "MISSING" + if field in {"aggregation", "results_sha256"}: + assert report["aggregation"]["comparison_status"] == "MISSING" + + +@pytest.mark.parametrize("metric_value", [None, "0.5", float("nan"), float("inf")]) +def test_missing_or_invalid_score_never_becomes_zero(sample: Path, metric_value: object) -> None: + rows = [{"id": "a", "hard": metric_value, "soft": 0.4}, {"id": "b", "hard": 1, "soft": 0.8}] + (sample / "results.jsonl").write_text("\n".join(json.dumps(r) for r in rows)) + bind(sample) + report = run(sample) + assert status(report, "values:hard") == ("MISSING" if metric_value is None else "FAIL") + assert report["aggregation"]["comparison_status"] == "MISSING" + + +def test_missing_key_and_failed_row_are_not_filtered(sample: Path) -> None: + rows = [{"id": "a", "soft": 0.4, "agent_ok": False}, {"id": "b", "hard": 1, "soft": 0.8}] + (sample / "results.jsonl").write_text("\n".join(json.dumps(r) for r in rows)) + bind(sample) + report = run(sample) + assert report["aggregation"]["effective_denominator"] == 2 + assert report["aggregation"]["error_rows"] == 1 + assert status(report, "values:hard") == "MISSING" + rows[0]["hard"] = 0 + (sample / "results.jsonl").write_text("\n".join(json.dumps(r) for r in rows)) + bind(sample) + assert run(sample)["aggregation"]["comparison_status"] == "PASS" + + +@pytest.mark.parametrize("extra", [{"weight": 2}, {"excluded": True}, {"run_id": "second-run"}]) +def test_unknown_aggregation_is_missing(sample: Path, extra: dict) -> None: + rows = [{"id": "a", "hard": 0, "soft": 0.4, **extra}, {"id": "b", "hard": 1, "soft": 0.8}] + (sample / "results.jsonl").write_text("\n".join(json.dumps(r) for r in rows)) + bind(sample) + report = run(sample) + assert status(report, "protocol") == "MISSING" + assert report["aggregation"]["comparison_status"] == "MISSING" + + +def test_duplicate_and_missing_result_ids(sample: Path) -> None: + (sample / "results.jsonl").write_text('{"id":"a","hard":0,"soft":0.4}\n' * 2) + bind(sample) + report = run(sample) + assert status(report, "result_duplicates") == "FAIL" + assert status(report, "result_coverage") == "FAIL" + assert report["aggregation"]["comparison_status"] == "MISSING" + + +def test_reported_denominator(sample: Path) -> None: + write(sample / "eval_summary.json", {"n_items": 1, "split": "test", "hard": 0.5, "soft": 0.6}) + bind(sample) + assert status(run(sample), "denominator") == "FAIL" + + +def test_changed_artifact_is_not_silently_trusted(sample: Path) -> None: + with (sample / "results.jsonl").open("a") as stream: + stream.write("\n") + report = run(sample) + assert status(report, "binding:results") == "FAIL" + assert report["aggregation"]["comparison_status"] == "MISSING" + + +@pytest.mark.parametrize("target", ["split_manifest.json", "test/items.json", "results.jsonl", "eval_summary.json", "evidence.json"]) +def test_malformed_json_is_readable_failure(sample: Path, target: str) -> None: + for invalid in ("{broken", '{"source_revision":NaN}', '{"source_revision":1e309}'): + (sample / target).write_text(invalid) + report = run(sample) + assert report["status"] == "FAIL" + assert "[FAIL]" in format_text(report) + json.dumps(report, allow_nan=False) + + +def test_unavailable_revision_and_manifest(sample: Path) -> None: + manifest = json.loads((sample / "split_manifest.json").read_text()) + manifest.pop("source_revision") + write(sample / "split_manifest.json", manifest) + bind(sample) + assert status(run(sample), "dataset_revision") == "MISSING" + report = audit(sample / "absent.json", source_root=sample) + assert report["source_revision"] is None + assert report["status"] == "MISSING" + + +def test_empty_results_are_missing(sample: Path) -> None: + (sample / "results.jsonl").write_text("") + bind(sample) + report = run(sample) + assert status(report, "nonempty_results") == "MISSING" + assert status(report, "result_coverage") == "MISSING" + assert report["status"] == "MISSING" + + +def test_cli_text_json_and_exit_codes(sample: Path, capsys: pytest.CaptureFixture) -> None: + args = [str(sample / "split_manifest.json"), "--results", str(sample / "results.jsonl"), "--summary", str(sample / "eval_summary.json"), "--evidence", str(sample / "evidence.json")] + assert main(args) == 2 + output = capsys.readouterr().out + assert "Check status: MISSING" in output + assert "Reproduction limitations:" in output + assert main(args + ["--format", "json"]) == 2 + assert json.loads(capsys.readouterr().out)["aggregation"]["comparison_status"] == "PASS" + bind(sample, classification="RECORDED", origin="historical", run_id="MOCK-ONLY") + assert main(args) == 0 + capsys.readouterr() + (sample / "results.jsonl").write_text("bad") + assert main(args) == 1 + + +def test_protocol_constant_matches_fixture() -> None: + evidence = json.loads((FIXTURE / "evidence.json").read_text()) + assert evidence["aggregation"] == PROTOCOL + paths = {"manifest": "split_manifest.json", "results": "results.jsonl", "summary": "eval_summary.json"} + paths.update({split: f"{split}/items.json" for split in ("train", "val", "test")}) + for label, relative in paths.items(): + path = FIXTURE / relative + assert b"\r\n" not in path.read_bytes() + assert sha256(path) == evidence[f"{label}_sha256"] + + +def test_runtime_split_alias(sample: Path) -> None: + write(sample / "eval_summary.json", {"split": "valid_unseen", "n_items": 2, "hard": 0.5, "soft": 0.6}) + bind(sample) + assert run(sample, split="valid_unseen")["aggregation"]["comparison_status"] == "PASS" + + +def test_runtime_all_splits(sample: Path) -> None: + rows = [{"id": key, "hard": 0.5, "soft": 0.6} for key in ("train-1", "val-1", "a", "b")] + (sample / "results.jsonl").write_text("\n".join(json.dumps(r) for r in rows)) + write(sample / "eval_summary.json", {"split": "all", "n_items": 4, "hard": 0.5, "soft": 0.6}) + bind(sample) + assert run(sample, split="all")["aggregation"]["comparison_status"] == "PASS" + + +def test_unknown_summary_split_is_missing(sample: Path) -> None: + write(sample / "eval_summary.json", {"split": "some-filtered-subset", "n_items": 2, "hard": 0.5, "soft": 0.6}) + bind(sample) + assert status(run(sample), "summary_metadata") == "MISSING" + + +@pytest.mark.parametrize("protocol", [None, "filtered_protocol_unavailable", "weighted_mean"]) +def test_unknown_subset_protocol_withholds_failure(sample: Path, protocol: str | None) -> None: + (sample / "results.jsonl").write_text('{"id":"a","hard":0.25,"soft":0.4}\n', encoding="utf-8") + write(sample / "eval_summary.json", {"split": "test", "n_items": 1, "hard": 0.25, "soft": 0.4}) + bind(sample, aggregation=protocol) + report = run(sample) + assert status(report, "result_coverage") == "MISSING" + assert report["status"] == "MISSING" + assert report["aggregation"]["comparison_status"] == "MISSING" + + +def test_declared_all_items_protocol_rejects_incomplete_coverage(sample: Path) -> None: + (sample / "results.jsonl").write_text('{"id":"a","hard":0.25,"soft":0.4}\n', encoding="utf-8") + bind(sample) + assert status(run(sample), "result_coverage") == "FAIL" + + +def test_unknown_protocol_still_rejects_duplicate_results(sample: Path) -> None: + (sample / "results.jsonl").write_text('{"id":"a","hard":0,"soft":0.4}\n' * 2, encoding="utf-8") + bind(sample, aggregation="unknown") + report = run(sample) + assert status(report, "result_duplicates") == "FAIL" + assert report["status"] == "FAIL" + + +@pytest.mark.parametrize("omit_argument", [False, True]) +def test_results_binding_checked_without_summary(sample: Path, omit_argument: bool) -> None: + with (sample / "results.jsonl").open("a", encoding="utf-8") as stream: + stream.write("\n") + (sample / "eval_summary.json").unlink() + report = audit( + sample / "split_manifest.json", source_root=ROOT, results_path=sample / "results.jsonl", + summary_path=None if omit_argument else sample / "eval_summary.json", evidence_path=sample / "evidence.json", + ) + assert status(report, "binding:results") == "FAIL" + assert report["status"] == "FAIL" + assert report["aggregation"]["comparison_status"] == "MISSING" + + +@pytest.mark.parametrize("empty_results", [False, True]) +def test_manifest_binding_checked_without_usable_results(sample: Path, empty_results: bool) -> None: + # Keep valid JSON and consistent counts while changing its attested bytes. + with (sample / "split_manifest.json").open("a", encoding="utf-8") as stream: + stream.write("\n") + if empty_results: + (sample / "results.jsonl").write_text("", encoding="utf-8") + else: + (sample / "results.jsonl").unlink() + assert status(run(sample), "binding:manifest") == "FAIL" + + +@pytest.mark.parametrize("decimals", [9, 10, 11, 12]) +def test_high_precision_rejects_distinct_rounded_scores(sample: Path, decimals: int) -> None: + summary = json.loads((sample / "eval_summary.json").read_text(encoding="utf-8")) + summary["hard"] = 0.5 + 2 * 10 ** -decimals + write(sample / "eval_summary.json", summary) + bind(sample) + assert status(run(sample, decimals=decimals), "aggregate:hard") == "FAIL" + + +@pytest.mark.parametrize("reported,expected", [(0.62, "PASS"), (0.63, "PASS"), (0.64, "FAIL")]) +def test_rounding_boundary_accepts_either_tie_convention(sample: Path, reported: float, expected: str) -> None: + rows = [{"id": key, "hard": 0.625, "soft": 0.6} for key in ("a", "b")] + (sample / "results.jsonl").write_text("\n".join(json.dumps(row) for row in rows), encoding="utf-8") + write(sample / "eval_summary.json", {"split": "test", "n_items": 2, "hard": reported, "soft": 0.6}) + bind(sample) + assert status(run(sample, decimals=2), "aggregate:hard") == expected + + +@pytest.mark.parametrize("target", ["split_manifest.json", "evidence.json", "eval_summary.json"]) +def test_exponent_overflow_emits_structured_cli_failure(sample: Path, target: str, capsys: pytest.CaptureFixture) -> None: + path = sample / target + value = json.loads(path.read_text(encoding="utf-8")) + if target == "split_manifest.json": + value["counts"]["train"] = float("inf") + elif target == "evidence.json": + value["source_revision"] = float("inf") + else: + value["hard"] = float("inf") + path.write_text(json.dumps(value).replace("Infinity", "1e309"), encoding="utf-8") + args = [str(sample / "split_manifest.json"), "--results", str(sample / "results.jsonl"), + "--summary", str(sample / "eval_summary.json"), "--evidence", str(sample / "evidence.json"), "--format", "json"] + assert main(args) == 1 + report = json.loads(capsys.readouterr().out) + assert report["status"] == "FAIL" + json.dumps(report, allow_nan=False) + + +@pytest.mark.parametrize("scale", [1, 100]) +def test_finite_scores_cannot_overflow_report(sample: Path, scale: int) -> None: + value = 1e308 if scale == 1 else 1e307 + rows = [{"id": key, "hard": value, "soft": 0.6} for key in ("a", "b")] + (sample / "results.jsonl").write_text("\n".join(json.dumps(row) for row in rows), encoding="utf-8") + bind(sample) + report = run(sample, scale=scale) + assert report["status"] == "FAIL" + assert report["aggregation"]["comparison_status"] != "PASS" + json.dumps(report, allow_nan=False) From 4de39e7975a6a17b84271641ba54b6f2659a70b8 Mon Sep 17 00:00:00 2001 From: JianhaoYu <139939479+JianhaoYu-AI@users.noreply.github.com> Date: Sat, 24 Oct 2026 10:13:24 +0800 Subject: [PATCH 2/2] fix: validate available reproducibility artifacts independently --- docs/guide/offline-reproducibility.md | 5 + skillopt/utils/reproducibility.py | 101 +++++++++++++------- tests/test_reproducibility.py | 130 ++++++++++++++++++++++++++ 3 files changed, 200 insertions(+), 36 deletions(-) diff --git a/docs/guide/offline-reproducibility.md b/docs/guide/offline-reproducibility.md index e97536e37..efaa7b62a 100644 --- a/docs/guide/offline-reproducibility.md +++ b/docs/guide/offline-reproducibility.md @@ -92,6 +92,11 @@ digest. The checker never writes or repairs supplied evidence. Available artifact bindings are checked even when results or summary evidence is missing or empty. A missing summary withholds the numerical comparison but does not hide a known hash mismatch in the supplied results or manifest. +Each available artifact is also parsed and validated independently. A missing +or malformed manifest does not skip the other supplied files, and a summary +with invalid fields fails even when results are unavailable or empty. Missing +inputs withhold the aggregate comparison; known failures still take precedence +over MISSING in the overall report. The report always includes the current Git HEAD and dirty state, the current scorer digest, the manifest's declared dataset revision, and artifact digests. diff --git a/skillopt/utils/reproducibility.py b/skillopt/utils/reproducibility.py index bcc1ecb78..f3d601ff9 100644 --- a/skillopt/utils/reproducibility.py +++ b/skillopt/utils/reproducibility.py @@ -157,20 +157,31 @@ def finish() -> dict[str, Any]: check("source", "PASS" if report["source_revision"] else "MISSING", "Current checkout SHA; not historical run SHA") scorer = Path(__file__).with_name("scoring.py") artifact(scorer, "scorer") + paths = { + "manifest": manifest_path, "evidence": evidence_path, + "results": results_path, "summary": summary_path, + **{name: manifest_path.parent / name / "items.json" for name in SPLIT_NAMES}, + } + available = {label: path is not None and artifact(path, label) for label, path in paths.items()} split_ids: dict[str, list[str]] = {} + counts: dict[str, Any] = {} + manifest_valid = False try: - if not artifact(manifest_path, "manifest"): - return finish() - manifest = _object(manifest_path) - report["dataset_revision"] = manifest.get("source_revision") - check("dataset_revision", "PASS" if _present(manifest.get("source_revision")) else "MISSING", "Dataset revision declared by manifest; raw payload not verified") - counts = manifest.get("counts", {}) - if not isinstance(counts, dict): - raise ValueError("manifest counts must be an object") - for name in SPLIT_NAMES: - path = manifest_path.parent / name / "items.json" - if not artifact(path, name): - continue + if available["manifest"]: + manifest = _object(manifest_path) + report["dataset_revision"] = manifest.get("source_revision") + check("dataset_revision", "PASS" if _present(manifest.get("source_revision")) else "MISSING", "Dataset revision declared by manifest; raw payload not verified") + if not isinstance(manifest.get("counts", {}), dict): + raise ValueError("manifest counts must be an object") + counts = manifest.get("counts", {}) + manifest_valid = True + except (ValueError, OSError) as exc: + check("manifest", "FAIL", str(exc)) + for name in SPLIT_NAMES: + if not available[name]: + continue + path = manifest_path.parent / name / "items.json" + try: rows = json.loads(path.read_text(encoding="utf-8")) if not isinstance(rows, list): raise ValueError(f"{path}: expected a JSON array") @@ -183,16 +194,16 @@ def finish() -> dict[str, Any]: duplicates = sorted(key for key, count in Counter(ids).items() if count > 1) report["duplicates"][name] = duplicates check(f"duplicates:{name}", "FAIL" if duplicates else "PASS", f"{name}: {duplicates}") - for left, right in combinations(split_ids, 2): - overlap = sorted(set(split_ids[left]) & set(split_ids[right])) - report["overlaps"][f"{left}/{right}"] = overlap - check(f"overlap:{left}/{right}", "FAIL" if overlap else "PASS", f"{left}/{right}: {overlap}") - except (ValueError, OSError) as exc: - check("manifest", "FAIL", str(exc)) - return finish() + except (ValueError, OSError) as exc: + check(name, "FAIL", str(exc)) + for left, right in combinations(split_ids, 2): + overlap = sorted(set(split_ids[left]) & set(split_ids[right])) + report["overlaps"][f"{left}/{right}"] = overlap + check(f"overlap:{left}/{right}", "FAIL" if overlap else "PASS", f"{left}/{right}: {overlap}") + run_metadata = False try: - if evidence_path is not None and artifact(evidence_path, "evidence"): + if available["evidence"]: evidence = _object(evidence_path) kind = evidence.get("classification", "MISSING") origin = evidence.get("origin") @@ -203,19 +214,47 @@ def finish() -> dict[str, Any]: if kind == "RECONSTRUCTED" and origin not in ("synthetic", "reconstruction"): raise ValueError("RECONSTRUCTED requires origin synthetic or reconstruction") report["evidence"] = {"classification": kind, "origin": origin, "run_id": evidence.get("run_id"), "run_source_revision": evidence.get("source_revision")} - run_metadata = _present(evidence.get("run_id")) and _present(evidence.get("source_revision")) + run_metadata = kind != "MISSING" and _present(evidence.get("run_id")) and _present(evidence.get("source_revision")) historical = kind == "RECORDED" and origin == "historical" and run_metadata check("historical_evidence", "PASS" if historical else "MISSING", "Historical origin, run ID and source revision are user declarations; absent or reconstructed artifacts cannot verify a historical result") if kind != "MISSING": check("run_metadata", "PASS" if run_metadata else "MISSING", "Run ID and actual run source revision required") except (ValueError, OSError) as exc: check("evidence", "FAIL", str(exc)) - return finish() + + summary: dict[str, Any] = {} + summary_metadata = False + summary_values = False + if available["summary"]: + try: + summary = _object(summary_path) + n_items = summary.get("n_items") + summary_split = summary.get("split") + normalized_split = SPLITS.get(summary_split) if isinstance(summary_split, str) else None + valid_count = isinstance(n_items, int) and not isinstance(n_items, bool) and n_items >= 0 + invalid_metadata = ( + n_items is not None and not valid_count + or summary_split is not None and not isinstance(summary_split, str) + or normalized_split is not None and normalized_split != split + ) + summary_metadata = valid_count and normalized_split == split + check("summary_metadata", "FAIL" if invalid_metadata else "PASS" if summary_metadata else "MISSING", f"n_items={n_items}, split={summary_split}") + summary_values = True + for metric in ("hard", "soft"): + value = summary.get(metric) + state = "MISSING" if value is None else "PASS" if _number(value) else "FAIL" + check(f"summary_values:{metric}", state, f"{metric}: reported={value}") + summary_values &= state == "PASS" + except (ValueError, OSError, OverflowError) as exc: + check("summary", "FAIL", str(exc)) + else: + check("summary", "MISSING", "eval_summary.json unavailable; no reported-score comparison") + + # All available bytes and provenance have been read before caching bindings. + bindings = check_bindings() try: - results_available = results_path is not None and artifact(results_path, "results") - summary_available = summary_path is not None and artifact(summary_path, "summary") - if not results_available: + if not available["results"]: check("per_item_results", "MISSING", "Per-item results.jsonl unavailable; no aggregate verdict") return finish() rows = [json.loads(line) for line in results_path.read_text(encoding="utf-8").splitlines() if line.strip()] @@ -258,18 +297,8 @@ def finish() -> dict[str, Any]: else: check("aggregation_values", "FAIL", "Saved-score aggregation produced a nonfinite value") complete = False - if not summary_available: - check("summary", "MISSING", "eval_summary.json unavailable; no reported-score comparison") - return finish() - summary = _object(summary_path) - bindings = check_bindings() n_items = summary.get("n_items") - summary_split = summary.get("split") - normalized_split = SPLITS.get(summary_split) if isinstance(summary_split, str) else None - summary_metadata = isinstance(n_items, int) and not isinstance(n_items, bool) and normalized_split == split - check("summary_metadata", "MISSING" if n_items is None or normalized_split is None else "PASS" if summary_metadata else "FAIL", f"n_items={n_items}, split={summary_split}") - run_metadata = kind != "MISSING" and run_metadata - ready = complete and coverage and protocol and bindings and summary_metadata and run_metadata + ready = manifest_valid and complete and coverage and protocol and bindings and summary_metadata and summary_values and run_metadata if ready: check("denominator", "PASS" if n_items == len(rows) else "FAIL", f"reported n_items={n_items}, actual denominator={len(rows)}") outcomes = [] diff --git a/tests/test_reproducibility.py b/tests/test_reproducibility.py index ff9b8330f..270f58935 100644 --- a/tests/test_reproducibility.py +++ b/tests/test_reproducibility.py @@ -123,6 +123,136 @@ def test_missing_per_item_results(sample: Path) -> None: assert report["aggregation"] == {} +@pytest.mark.parametrize("label,relative", [("results", "results.jsonl"), ("summary", "eval_summary.json")]) +def test_missing_manifest_preserves_available_digest_failure(sample: Path, label: str, relative: str) -> None: + bind(sample) + (sample / "split_manifest.json").unlink() + with (sample / relative).open("a", encoding="utf-8") as stream: + stream.write("\n") + report = run(sample) + assert status(report, f"binding:{label}") == "FAIL" + assert report["status"] == "FAIL" + assert report["aggregation"].get("comparison_status") != "PASS" + + +@pytest.mark.parametrize("contents", ["[]", "{broken"]) +def test_missing_results_still_validates_bound_summary(sample: Path, contents: str) -> None: + (sample / "eval_summary.json").write_text(contents, encoding="utf-8") + bind(sample) + (sample / "results.jsonl").unlink() + report = run(sample) + assert status(report, "binding:summary") == "PASS" + assert status(report, "summary") == "FAIL" + assert report["status"] == "FAIL" + assert report["aggregation"] == {} + + +@pytest.mark.parametrize("field,value", [ + ("hard", "0.5"), ("soft", True), ("n_items", "2"), + ("n_items", True), ("n_items", -1), ("split", []), +]) +@pytest.mark.parametrize("results_mode", ["missing", "omitted", "empty"]) +def test_partial_results_do_not_hide_invalid_summary_fields( + sample: Path, field: str, value: object, results_mode: str, +) -> None: + summary = {"split": "test", "n_items": 2, "hard": 0.5, "soft": 0.6, field: value} + write(sample / "eval_summary.json", summary) + if results_mode == "empty": + (sample / "results.jsonl").write_text("", encoding="utf-8") + bind(sample) + if results_mode == "missing": + (sample / "results.jsonl").unlink() + report = audit( + sample / "split_manifest.json", source_root=ROOT, + results_path=None if results_mode == "omitted" else sample / "results.jsonl", + summary_path=sample / "eval_summary.json", evidence_path=sample / "evidence.json", + ) + assert status(report, "binding:summary") == "PASS" + check = f"summary_values:{field}" if field in {"hard", "soft"} else "summary_metadata" + assert status(report, check) == "FAIL" + assert report["status"] == "FAIL" + assert report["aggregation"].get("comparison_status") != "PASS" + + +@pytest.mark.parametrize("field", ["hard", "soft", "n_items", "split"]) +def test_incomplete_summary_fields_remain_missing(sample: Path, field: str) -> None: + summary = {"split": "test", "n_items": 2, "hard": 0.5, "soft": 0.6} + summary[field] = None + write(sample / "eval_summary.json", summary) + bind(sample) + report = run(sample) + check = f"summary_values:{field}" if field in {"hard", "soft"} else "summary_metadata" + assert status(report, check) == "MISSING" + assert report["status"] == "MISSING" + assert report["aggregation"]["comparison_status"] == "MISSING" + + +@pytest.mark.parametrize("target", ["split_manifest.json", "evidence.json", "train/items.json"]) +def test_malformed_artifact_does_not_skip_other_artifacts(sample: Path, target: str) -> None: + (sample / target).write_text("{broken", encoding="utf-8") + (sample / "results.jsonl").write_text("{broken", encoding="utf-8") + write(sample / "eval_summary.json", []) + report = run(sample) + label = {"split_manifest.json": "manifest", "evidence.json": "evidence", "train/items.json": "train"}[target] + assert status(report, label) == "FAIL" + assert status(report, "results") == "FAIL" + assert status(report, "summary") == "FAIL" + assert report["status"] == "FAIL" + assert "test" in report["split_counts"] + + +def test_invalid_evidence_metadata_preserves_supplied_bindings(sample: Path) -> None: + bind(sample, classification="invalid") + with (sample / "results.jsonl").open("a", encoding="utf-8") as stream: + stream.write("\n") + report = run(sample) + assert status(report, "evidence") == "FAIL" + assert status(report, "binding:results") == "FAIL" + assert status(report, "summary_metadata") == "PASS" + assert report["aggregation"]["comparison_status"] == "MISSING" + + +@pytest.mark.parametrize("target", ["split_manifest.json", "results.jsonl", "eval_summary.json"]) +def test_missing_artifact_preserves_valid_available_checks(sample: Path, target: str) -> None: + bind(sample) + (sample / target).unlink() + report = run(sample) + assert report["status"] == "MISSING" + assert not [check for check in report["checks"] if check["status"] == "FAIL"] + assert report["aggregation"].get("comparison_status") != "PASS" + assert status(report, "duplicates:test") == "PASS" + assert status(report, "binding:train") == "PASS" + + +@pytest.mark.parametrize("case", ["missing_manifest", "missing_results", "only_missing"]) +def test_partial_input_cli_keeps_failures_and_input_bytes( + sample: Path, capsys: pytest.CaptureFixture, case: str, +) -> None: + if case == "missing_results": + write(sample / "eval_summary.json", []) + bind(sample) + if case == "missing_manifest": + (sample / "split_manifest.json").unlink() + with (sample / "results.jsonl").open("a", encoding="utf-8") as stream: + stream.write("\n") + else: + (sample / "results.jsonl").unlink() + before = {path: path.read_bytes() for path in sample.rglob("*") if path.is_file()} + args = [str(sample / "split_manifest.json"), "--results", str(sample / "results.jsonl"), + "--summary", str(sample / "eval_summary.json"), "--evidence", str(sample / "evidence.json")] + expected = "MISSING" if case == "only_missing" else "FAIL" + code = 2 if expected == "MISSING" else 1 + assert main(args + ["--format", "json"]) == code + report = json.loads(capsys.readouterr().out) + assert report["status"] == expected + assert report["aggregation"].get("comparison_status") != "PASS" + names = [check["name"] for check in report["checks"]] + assert len(names) == len(set(names)) + assert main(args) == code + assert f"Check status: {expected}" in capsys.readouterr().out + assert before == {path: path.read_bytes() for path in sample.rglob("*") if path.is_file()} + + @pytest.mark.parametrize("value,expected", [(66.7, "PASS"), (66.8, "FAIL")]) def test_explicit_percentage_precision(sample: Path, value: float, expected: str) -> None: rows = [{"id": "a", "hard": 1 / 3, "soft": 1 / 3}, {"id": "b", "hard": 1, "soft": 1}]