diff --git a/changelog.d/653.changed.md b/changelog.d/653.changed.md new file mode 100644 index 00000000..961782f1 --- /dev/null +++ b/changelog.d/653.changed.md @@ -0,0 +1 @@ +The UK replay qualification gate records narrowly scoped data-limited exceptions under policy d1344-replay-regressions-v2. A matching regression requires coherent zero candidate effects, approved input lineage, and digest-bound Budget-input and positive-input probe evidence from the same candidate-pair attempt. Named acceptance must restate the publication designation; unmatched failures still fail. diff --git a/changelog.d/654.fixed.md b/changelog.d/654.fixed.md new file mode 100644 index 00000000..890542bf --- /dev/null +++ b/changelog.d/654.fixed.md @@ -0,0 +1 @@ +Replay exceptions reject nonzero candidate effects at every reported fiscal level and retain cumulative usable-input history across certified pairs. Budget-input evidence is bound to the exact pair and attempt, status maps must agree with missing-input claims, and probe evidence must match the installed model and recorded baseline/reform arithmetic. diff --git a/scripts/check_uk_candidate.py b/scripts/check_uk_candidate.py index d07fdfe5..b82c432c 100644 --- a/scripts/check_uk_candidate.py +++ b/scripts/check_uk_candidate.py @@ -66,6 +66,7 @@ ResolvedCandidate, WheelFile, accept_review, + attempt_evidence_id, attempt_is_recoverable, attempt_is_waiting_for_replay, attempt_key, @@ -76,6 +77,8 @@ candidate_pr_body, expected_release_bundle, finalize_attempt, + load_attempt_budget_inputs, + load_attempt_replay_probe, packaged_manifest_sha256, plan_promotion, prepare_candidate_commit, @@ -369,6 +372,16 @@ def cmd_gates(args: argparse.Namespace) -> int: ledger_root=ledger_root, ) write_gate_results(attempt, results, evidence) + from policyengine.provenance.replay_probe import run_replay_probes + + probe_result, probe_evidence = run_replay_probes( + attempt_id=attempt_evidence_id(attempt), + candidate_model_version=payload["pair"]["model"]["version"], + ) + write_gate_results(attempt, [probe_result], {"replay_probe": probe_evidence}) + print( + f"{probe_result.gate_id:20s} {probe_result.status:13s} {probe_result.summary}" + ) for result in results: print(f"{result.gate_id:20s} {result.status:13s} {result.summary}") return 0 @@ -417,10 +430,19 @@ def cmd_poll_replay(args: argparse.Namespace) -> int: read_json(attempt / "replay" / "request.json") ) fetch = scorecard_file(settings["repository"]) + budget_result, budget_evidence = load_attempt_budget_inputs(attempt) + probe_result, probe_evidence = load_attempt_replay_probe(attempt) deadline = time.monotonic() + 60 * args.wait_minutes while True: gate, status, receipt = read_replay( - request, fetch=fetch, state_ref=settings["state_ref"] + request, + fetch=fetch, + state_ref=settings["state_ref"], + budget_inputs_result=budget_result, + budget_inputs_evidence=budget_evidence, + probe_result=probe_result, + probe_evidence=probe_evidence, + attempt_id=attempt_evidence_id(attempt), ) if (status is not None and status.terminal) or time.monotonic() >= deadline: break diff --git a/src/policyengine/data/qualification/replay_policy.md b/src/policyengine/data/qualification/replay_policy.md index bb6aa65e..87c55e16 100644 --- a/src/policyengine/data/qualification/replay_policy.md +++ b/src/policyengine/data/qualification/replay_policy.md @@ -1,4 +1,4 @@ -# Replay gate policy d1344-replay-regressions-v1 +# Replay gate policy d1344-replay-regressions-v2 Decided under cos d1344 by workstream W4 on 15 October 2026, after two independent reviews. The pull request that added this file records the @@ -7,6 +7,11 @@ October 2026, before first use, after the third to fifth reviews of policyengine.py#637: scorecard's own record of a mixed known gap no longer blocks, flag counters must equal the flags the rows carry, and any candidate value other than £0 at a known gap is a contradiction. +The data-limited exception amendment below was adopted by the Budget +coordinator following independent review agreement under cos d1344: +RULE AGREE (GPT-6.1 Sol, round 2, 17 October 2026, head 7f518ff61) and +CODE AGREE WITH NITS (round 3, head dbc1806df), with the P3 nit fixed in +this commit. The registry records this adopted decision and review status. `evaluate_replay_receipt` in `policyengine.provenance.qualification` applies this text. Each qualification receipt records the policy id and this file's SHA-256 in its `replay` gate details. @@ -20,23 +25,26 @@ every other flagged move to a named reviewer. - A rejected request, `no_supported_years`, integrity `failed` (a crash or missing outputs, persistent or not), or a receipt not bound to the - request. + request. Only the narrow data-limited exceptions below can make a failed + integrity reviewable. - A mapping flag. - A lost result: a coverage row with a base value and no candidate value, unless it is the record of a known gap (below). - A lost response: - - a measure-year the candidate lists as inert, unless it is a known gap; + - a measure-year the candidate lists as inert, unless it is a known gap + or a data-limited exception applies; - a measure-year total at exactly £0 where the base's absolute value is at least the diff's bound, max(£1m, 1% of the base's absolute value), - whether the base was positive or negative. A known gap never excuses - this: a base whose record lists the measure-year as a gap cannot also - hold a material value for it. + whether the base was positive or negative, unless a data-limited + exception applies. A known gap never excuses this: a base whose record + lists the measure-year as a gap cannot also hold a material value for + it, and a data-limited exception never excuses that contradiction. - An event whose supported grid is incomplete, unless every inert or - missing and supported measure-year in it is a known gap. An incomplete - event whose receipt does not list its missing measure-years fails, and so - does one with no inert and no missing and supported measure-year at all. - A receipt whose integrity is `incomplete` with no incomplete event also - fails. + missing and supported measure-year in it is a known gap or is accounted + for by a data-limited exception. An incomplete event whose receipt does + not list its missing measure-years fails, and so does one with no inert + and no missing and supported measure-year at all. A receipt whose + integrity is `incomplete` with no incomplete event also fails. - A receipt that contradicts itself, including a complete event that lists an inert, or missing and supported, measure-year; flag counters that differ, either way, from the flags its rows carry; and a candidate value @@ -66,6 +74,80 @@ even £0, is still a lost result, and any candidate value other than £0 at a known gap contradicts the candidate's own record, which lists the measure-year as inert (exactly £0) or missing (no value). +## Data-limited exceptions + +`uk_replay_exceptions.json` lists narrowly scoped exceptions for regressions +caused by inputs absent from the candidate's data. Missingness alone does +not establish cause. The registry is reviewed like code. Each entry records +an id, failure codes, event, measure key, years, required missing inputs, +cause, tracking issue, approved lineage, positive-input probe specification, +publication designation, governing decision and review status. + +An exception applies only when its code, event, measure key and year all +match, and the same qualification attempt's `budget_inputs` gate result +and its evidence report every required input as missing from the +candidate's data. A degenerate but present input cannot satisfy that +condition. The same evidence must carry the cumulative `ever_usable_inputs` +set from the previous certified pair, adding inputs usable in that pair and +in the candidate. Removing an input or making it formula-backed never erases +this history. An exception requires that the input has never been usable in +this chain and is not usable now. For evidence predating the cumulative field, +recompute the exact direct predecessor's usable inputs and record +`ever_usable_inputs_basis: "seeded from direct predecessor"`; the registry's +approved lineage records the human-verified history before that base. +A previous pair with both the variable +and column present requires a fresh reviewed registry decision and fails, +including when the previous column was degenerate. The approved +SDLT lineage is that no certified uk-data release through 1.58.2 supplied +the input. When the data gains an input, or the measure-year falls outside +the entry's years, the exception stops applying automatically. + +Every reported fiscal effect for an excepted measure-year must be exactly £0, +including fiscal heads, measure totals, source rows and any other reported +level. Any nonzero candidate effect is a contradiction and fails, +independently of any base known gap. This checks reported rows only. + +The gates job also runs the registry's positive-input probe directly on the +exact installed candidate model, using one synthetic household and no +survey data. For the SDLT exception, an England household buys an additional +residential property for £300,000 in 2026. The reform adds 0.02 to every +band of `gov.hmrc.stamp_duty.residential.purchase.additional.rate`. Its +`sdlt_on_residential_property_transactions` output must increase by £6,000 +within £0.10, and must be exactly unchanged when the purchase input is £0. +An import error, missing variable or parameter, or any other inability to +run the probe records failure. An exception requires a passing probe from +the same attempt (identified by both pair id and attempt key), bound to the +candidate model version and probe specification by digests. Finalization +re-reads the evidence and verifies its digest, as it does for `budget_inputs`. +The consumer also checks the installed model version and verifies that the +recorded baseline/reform outputs yield the claimed positive and zero deltas. +Budget-input evidence embeds and checks both pair and attempt identity; its +candidate status map must agree with the missing list, and its previous +status flags and cumulative history must be internally consistent. + +Matching `construction_inert` integrity failures and `lost_response` +regressions (candidate inertness or a measure-year total of £0 against a +material base value) become review items. Each names the exception, cause +and required missing inputs. A failed receipt proceeds to the ordinary +corpus and diff checks only if every integrity failure matches an +exception. Every unmatched failure, mapping flag, lost result and +contradiction retains its ordinary treatment. An exception accounts only +for its matched inert measure-years when checking grid completeness; it +does not excuse a missing result. + +An applied exception always requires review and can never qualify a +candidate automatically. Promotion requires a named reviewer to accept +the exact qualification receipt. Applied-exception review items carry the +registry's publication designation, and the named reviewer's acceptance +explanation must restate it. For SDLT the designation is: not computable, +in PolicyEngine/uk-autumn-budget-dashboard-2026 (data-limited handling, +PR #51; Budget-day runbook, PR #14). + +The gate records every applied exception's id, event, measure key, year +and matched codes, the registry's SHA-256, and the digests of the same +attempt's `budget_inputs` and probe results it relied on. Every result +that reads the registry records its digest. + ## Review A needs-review verdict blocks automatic promotion. It stays blocked until a diff --git a/src/policyengine/data/qualification/uk_replay_exceptions.json b/src/policyengine/data/qualification/uk_replay_exceptions.json new file mode 100644 index 00000000..93306475 --- /dev/null +++ b/src/policyengine/data/qualification/uk_replay_exceptions.json @@ -0,0 +1,42 @@ +{ + "_comment": "Narrow data-limited exceptions for the UK replay qualification gate, reviewed like code under cos d1344. An entry requires coherent zero candidate effects, never-usable input lineage from budget_inputs, and a passing positive-input probe from the same attempt. It always requires a named reviewer's acceptance. The 17 October 2026 decision is adopted by the Budget coordinator following independent review agreement under cos d1344: RULE AGREE (GPT-6.1 Sol, round 2, 17 October 2026, head 7f518ff61) and CODE AGREE WITH NITS (round 3, head dbc1806df), with the P3 nit fixed in this commit.", + "exceptions": [ + { + "id": "ab2024-sdlt-hrad-purchase-input", + "codes": [ + "construction_inert", + "lost_response" + ], + "event": "autumn_budget_2024", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "years": [ + 2025, + 2026, + 2027, + 2028, + 2029 + ], + "requires_missing_inputs": [ + "additional_residential_property_purchased" + ], + "cause": "policyengine-uk#2241 charges SDLT higher rates on purchases, which uk-data does not yet impute.", + "issue": "PolicyEngine/policyengine-uk-data#573 (cos d1354)", + "approved_lineage": "never supplied by any certified uk-data release up to and including 1.58.2.", + "publication": "not computable, in PolicyEngine/uk-autumn-budget-dashboard-2026 (data-limited handling, PR #51; Budget-day runbook, PR #14).", + "probe": { + "input_variable": "additional_residential_property_purchased", + "input_entity": "household", + "year": 2026, + "positive_input_gbp": 300000, + "output_variable": "sdlt_on_residential_property_transactions", + "parameter": "gov.hmrc.stamp_duty.residential.purchase.additional.rate", + "rate_increment": 0.02, + "expected_change_gbp": 6000, + "absolute_tolerance_gbp": 0.1, + "region": "LONDON" + }, + "decided_under": "cos d1344; Budget coordinator's data-limited exception proposal of 17 October 2026", + "review": "Adopted by the Budget coordinator following independent review agreement under cos d1344: RULE AGREE (GPT-6.1 Sol, round 2, 17 October 2026, head 7f518ff61) and CODE AGREE WITH NITS (round 3, head dbc1806df), with the P3 nit fixed in this commit." + } + ] +} diff --git a/src/policyengine/provenance/qualification.py b/src/policyengine/provenance/qualification.py index 29c505bd..31fc9344 100644 --- a/src/policyengine/provenance/qualification.py +++ b/src/policyengine/provenance/qualification.py @@ -31,6 +31,7 @@ import functools import hashlib import json +import math from collections.abc import Iterable, Mapping, Sequence from typing import Any, ClassVar, Literal, Optional @@ -62,6 +63,9 @@ "GateResult", "QualificationHarness", "QualificationReceipt", + "ReplayDataLimitedException", + "ReplayExceptionRegistry", + "ReplayExceptionProbe", "ReplayQualificationRequest", "ReplayReceipt", "ReplayStatus", @@ -108,8 +112,9 @@ #: identifies, call a gap known only on base-side evidence, and send every #: other flagged move to a named reviewer. Its text ships as #: ``data/qualification/replay_policy.md`` and each receipt records its digest. -REPLAY_POLICY_ID = "d1344-replay-regressions-v1" +REPLAY_POLICY_ID = "d1344-replay-regressions-v2" REPLAY_POLICY_FILE = "replay_policy.md" +REPLAY_EXCEPTIONS_FILE = "uk_replay_exceptions.json" #: Flags that fail whatever the row says. BLOCKING_REPLAY_FLAGS: frozenset[str] = frozenset({"mapping"}) #: Flags that need review unless the row is a lost result or lost response. @@ -180,6 +185,314 @@ class GateResult(_Contract): details: dict[str, Any] = Field(default_factory=dict) +class ReplayExceptionProbe(_Contract): + """A survey-free causal check on the exact candidate model.""" + + input_variable: str = Field(min_length=1) + input_entity: Literal["household"] + year: int + positive_input_gbp: float = Field(gt=0, allow_inf_nan=False) + output_variable: str = Field(min_length=1) + parameter: str = Field(min_length=1) + rate_increment: float = Field(gt=0, allow_inf_nan=False) + expected_change_gbp: float = Field(gt=0, allow_inf_nan=False) + absolute_tolerance_gbp: float = Field(ge=0, allow_inf_nan=False) + region: str = Field(min_length=1) + + +class ReplayDataLimitedException(_Contract): + """A reviewed, finite set of data-limited replay losses, never a pass. + + Only construction inertness and lost responses can be excepted. Crashes, + lost results, mapping failures and contradictions cannot be registered. + """ + + id: str = Field(min_length=1) + codes: list[Literal["construction_inert", "lost_response"]] = Field(min_length=1) + event: str = Field(min_length=1) + measure_key: str = Field(min_length=1) + years: list[int] = Field(min_length=1) + requires_missing_inputs: list[str] = Field(min_length=1) + cause: str = Field(min_length=1) + issue: str = Field(min_length=1) + decided_under: str = Field(min_length=1) + review: str = Field(min_length=1) + approved_lineage: str = Field(min_length=1) + publication: str = Field(min_length=1) + probe: ReplayExceptionProbe + + @model_validator(mode="after") + def _unique_items(self) -> ReplayDataLimitedException: + for name in ("codes", "years", "requires_missing_inputs"): + _unique(getattr(self, name), name) + if any(not name for name in self.requires_missing_inputs): + raise ValueError("Required missing input names must not be empty.") + if self.probe.input_variable not in self.requires_missing_inputs: + raise ValueError("The probe must exercise a required missing input.") + return self + + def matches( + self, + code: str, + event: Optional[str], + measure_key: Optional[str], + year: Optional[int], + missing_inputs: set[str], + ) -> bool: + return ( + code in self.codes + and event == self.event + and measure_key == self.measure_key + and year in self.years + and set(self.requires_missing_inputs) <= missing_inputs + ) + + +class ReplayExceptionRegistry(_Contract): + """Code-reviewed replay exceptions shipped with the gate's policy.""" + + model_config = ConfigDict(populate_by_name=True) + + comment: Optional[str] = Field(default=None, alias="_comment") + exceptions: list[ReplayDataLimitedException] = Field(default_factory=list) + + @model_validator(mode="after") + def _unique_ids(self) -> ReplayExceptionRegistry: + _unique([entry.id for entry in self.exceptions], "exception ids") + return self + + def canonical_payload(self) -> dict[str, Any]: + return self.model_dump(mode="json", by_alias=True, exclude_none=True) + + +def _replay_exceptions() -> tuple[ReplayExceptionRegistry, str]: + from importlib.resources import files + + raw = ( + files("policyengine.data.qualification") / REPLAY_EXCEPTIONS_FILE + ).read_bytes() + return ReplayExceptionRegistry.model_validate_json(raw), hashlib.sha256( + raw + ).hexdigest() + + +def _consistent_input_status(state: Any) -> bool: + """Validate model membership and usability facts at the evidence boundary.""" + + return ( + isinstance(state, Mapping) + and all( + type(state.get(key)) is bool + for key in ("in_model", "is_input", "in_data", "usable") + ) + and (not state["is_input"] or state["in_model"]) + and (not state["usable"] or (state["in_model"] and state["in_data"])) + ) + + +def _input_name_set(value: Any) -> Optional[set[str]]: + if ( + not isinstance(value, list) + or any(not isinstance(name, str) or not name for name in value) + or len(value) != len(set(value)) + ): + return None + return set(value) + + +def _replay_budget_identity_matches( + evidence: Mapping[str, Any], attempt_id: Optional[str] +) -> bool: + """The caller supplies the exact pair/attempt identity from the ledger.""" + + if not attempt_id or "/" not in attempt_id: + return False + pair_id = attempt_id.partition("/")[0] + return ( + len(pair_id) == 64 + and all(character in "0123456789abcdef" for character in pair_id) + and evidence.get("pair_id") == pair_id + and evidence.get("attempt_id") == attempt_id + ) + + +def _replay_missing_inputs( + result: Optional[GateResult], + evidence: Optional[Mapping[str, Any]], + attempt_id: Optional[str] = None, +) -> set[str]: + """Absent candidate columns, bound to the same attempt's gate result. + + ``details.missing_inputs`` is a count, so persistent missing inputs must + come from the evidence it hashes. Degenerate columns are not missing. + The attempt loader is responsible for reading only its own gate files. + """ + + if ( + result is None + or result.gate_id != "budget_inputs" + or result.status in ("error", "unavailable") + or not isinstance(evidence, Mapping) + or not _replay_budget_identity_matches(evidence, attempt_id) + ): + return set() + try: + if result.evidence_sha256 != content_sha256(evidence): + return set() + except (TypeError, ValueError): + return set() + # Rehashing edited evidence must not erase the producer's history marker. + recorded_ever = _input_name_set(result.details.get("ever_usable_inputs")) + if recorded_ever is None or recorded_ever != _input_name_set( + evidence.get("ever_usable_inputs") + ): + return set() + missing = _input_name_set(evidence.get("missing_inputs")) + newly_missing = _input_name_set(evidence.get("newly_missing_inputs")) + states = evidence.get("input_status") + if ( + missing is None + or newly_missing is None + or not newly_missing <= missing + or type(result.details.get("missing_inputs")) is not int + or result.details.get("missing_inputs") != len(missing) + or result.details.get("newly_missing_inputs") + != evidence["newly_missing_inputs"] + ): + return set() + if ( + not isinstance(states, Mapping) + or any( + not isinstance(name, str) or not name or not _consistent_input_status(state) + for name, state in states.items() + ) + or missing + != { + name + for name, state in states.items() + if state["is_input"] and not state["in_data"] + } + ): + return set() + return missing + + +def _replay_lineage_problem( + entry: ReplayDataLimitedException, evidence: Optional[Mapping[str, Any]] +) -> Optional[str]: + """Require absence throughout the cumulative certified comparison chain. + + The caller first verifies the budget-input evidence's digest. These are + the previous certified pair's model and data facts, not an inference from + the candidate's missing column or from the registry's approval text. A + supplied but degenerate column also needs a fresh registry decision. + """ + + previous = evidence.get("previous_input_status") if evidence is not None else None + if not isinstance(previous, Mapping): + return "the previous certified pair's input usability is unknown" + if any(not _consistent_input_status(state) for state in previous.values()): + return ( + "the previous certified pair's input usability is unknown or inconsistent; " + "a fresh registry decision is required" + ) + previous_ever = _input_name_set(evidence.get("previous_ever_usable_inputs")) + ever_usable_inputs = _input_name_set(evidence.get("ever_usable_inputs")) + if previous_ever is None or ever_usable_inputs is None: + return "the certified chain's cumulative input usability is unknown" + current = evidence["input_status"] + if not { + name for name, state in previous.items() if state["usable"] + } <= previous_ever or ever_usable_inputs != previous_ever | { + name for name, state in current.items() if state["usable"] + }: + return ( + "the certified chain's cumulative input usability is inconsistent; " + "a fresh registry decision is required" + ) + for name in entry.requires_missing_inputs: + state = previous.get(name) + if state is None: + return f"the previous certified pair's usability of {name} is unknown" + if name in ever_usable_inputs: + return ( + f"{name} was previously usable; a fresh registry decision is required" + ) + if state["in_model"] and state["in_data"]: + return ( + f"{name} was supplied by the previous certified pair; " + "a fresh registry decision is required" + ) + return None + + +def _replay_probe_passes( + entry: ReplayDataLimitedException, + request: ReplayQualificationRequest, + result: Optional[GateResult], + evidence: Optional[Mapping[str, Any]], + attempt_id: Optional[str], +) -> bool: + """Validate the exact-attempt, exact-spec positive and zero-input checks.""" + + if ( + result is None + or result.gate_id != "replay_probe" + or result.status != "pass" + or evidence is None + or not attempt_id + or evidence.get("attempt_id") != attempt_id + or evidence.get("candidate_model_version") + != request.candidate.policyengine_uk.version + ): + return False + try: + if result.evidence_sha256 != content_sha256(evidence): + return False + except (TypeError, ValueError): + return False + probes = evidence.get("probes") + if not isinstance(probes, list) or any( + not isinstance(probe, Mapping) for probe in probes + ): + return False + matching = [probe for probe in probes if probe.get("exception_id") == entry.id] + if len(matching) != 1: + return False + probe = matching[0] + if ( + probe.get("status") != "pass" + or probe.get("installed_model_version") + != request.candidate.policyengine_uk.version + or probe.get("spec_sha256") + != content_sha256(entry.probe.model_dump(mode="json")) + ): + return False + output_fields = ( + "positive_baseline_gbp", + "positive_reform_gbp", + "positive_change_gbp", + "zero_baseline_gbp", + "zero_reform_gbp", + "zero_change_gbp", + ) + if any( + type(probe.get(field)) not in (int, float) or not math.isfinite(probe[field]) + for field in output_fields + ): + return False + positive, zero = probe.get("positive_change_gbp"), probe.get("zero_change_gbp") + # The producer subtracts these exact recorded outputs to obtain each delta. + return ( + positive == probe["positive_reform_gbp"] - probe["positive_baseline_gbp"] + and zero == probe["zero_reform_gbp"] - probe["zero_baseline_gbp"] + and positive > 0 + and abs(positive - entry.probe.expected_change_gbp) + <= entry.probe.absolute_tolerance_gbp + and zero == 0 + ) + + def qualification_verdict(gates: Iterable[GateResult]) -> Verdict: """The verdict a set of gate results determines. @@ -880,12 +1193,19 @@ def evaluate_replay_receipt( request: ReplayQualificationRequest, status: ReplayStatus, receipt: ReplayReceipt, + *, + budget_inputs_result: Optional[GateResult] = None, + budget_inputs_evidence: Optional[Mapping[str, Any]] = None, + probe_result: Optional[GateResult] = None, + probe_evidence: Optional[Mapping[str, Any]] = None, + attempt_id: Optional[str] = None, + exception_registry: Optional[ReplayExceptionRegistry] = None, ) -> GateResult: """The ``replay`` gate for a terminal replay status and its receipt. The receipt must answer this exact request, be the bytes the status names, and report an execution of exactly the requested artifacts. Its - contents are then judged by policy ``d1344-replay-regressions-v1`` + contents are then judged by policy ``d1344-replay-regressions-v2`` (``data/qualification/replay_policy.md``): - fail on rejected or failed replays, mapping flags, lost results @@ -902,7 +1222,13 @@ def evaluate_replay_receipt( other than £0 at a known gap fails as a contradiction, and any other coverage row with a base value and no candidate value, even £0, is a lost result; - - send every other flagged move to review. + - send every other flagged move to review. A registered data-limited + construction failure or lost response also needs review, only while + the same attempt's digest-bound budget-input evidence reports every + required input missing and never previously usable, and an exact-model + positive-input probe demonstrates the reform's response. Excepted + measure-years must have zero effects at every reported fiscal level. + Every unmatched failure and contradiction retains its ordinary verdict. Unsupported years are reported, never failed on their own. The gate's details record the policy id, the policy text's SHA-256, the known gaps @@ -915,6 +1241,19 @@ def evaluate_replay_receipt( than the ``compare_to`` asked for). """ + if exception_registry is None: + exception_registry, registry_sha256 = _replay_exceptions() + else: + registry_sha256 = content_sha256( + exception_registry.model_dump(mode="json", by_alias=True, exclude_none=True) + ) + missing_inputs = _replay_missing_inputs( + budget_inputs_result, budget_inputs_evidence, attempt_id + ) + exceptions = sorted(exception_registry.exceptions, key=lambda entry: entry.id) + applied: dict[tuple[str, str, str, int], dict[str, Any]] = {} + exception_reviews: list[dict[str, Any]] = [] + exception_rejections: dict[str, str] = {} request_id = request.request_id problems: list[str] = [] if status.request_id != request_id or receipt.request_id != request_id: @@ -939,8 +1278,83 @@ def evaluate_replay_receipt( "known_gaps": [], "regressions": [], "review_items": [], + "exception_registry_sha256": registry_sha256, + "applied_exceptions": [], + "budget_inputs_result_sha256": None, } + def matching_exception( + code: str, event: Optional[str], measure_key: Optional[str], year: Optional[int] + ) -> Optional[ReplayDataLimitedException]: + for entry in exceptions: + if not entry.matches(code, event, measure_key, year, missing_inputs): + continue + lineage_problem = _replay_lineage_problem(entry, budget_inputs_evidence) + if lineage_problem is not None: + exception_rejections[entry.id] = lineage_problem + continue + if not _replay_probe_passes( + entry, request, probe_result, probe_evidence, attempt_id + ): + exception_rejections[entry.id] = ( + "a passing digest-bound probe from this attempt is required" + ) + continue + return entry + return None + + def except_regression(record: dict[str, Any], code: str) -> bool: + entry = matching_exception( + code, record.get("event"), record.get("measure_key"), record.get("year") + ) + if entry is None: + return False + where = (entry.id, record["event"], record["measure_key"], record["year"]) + if where not in applied: + applied[where] = { + "id": entry.id, + "event": record["event"], + "measure_key": record["measure_key"], + "year": record["year"], + "codes": [], + "cause": entry.cause, + "requires_missing_inputs": sorted(entry.requires_missing_inputs), + "issue": entry.issue, + "decided_under": entry.decided_under, + "review": entry.review, + "approved_lineage": entry.approved_lineage, + "publication": entry.publication, + "probe_evidence_sha256": ( + probe_result.evidence_sha256 if probe_result is not None else None + ), + } + applied[where]["codes"] = sorted(set(applied[where]["codes"]) | {code}) + details["applied_exceptions"] = [applied[key] for key in sorted(applied)] + assert budget_inputs_result is not None + details["budget_inputs_result_sha256"] = content_sha256( + budget_inputs_result.canonical_payload() + ) + assert probe_result is not None + details["probe_result_sha256"] = content_sha256( + probe_result.canonical_payload() + ) + details["probe_evidence_sha256"] = probe_result.evidence_sha256 + exception_reviews.append( + { + **record, + "kind": "data_limited_exception", + "code": code, + "exception_id": entry.id, + "cause": entry.cause, + "requires_missing_inputs": sorted(entry.requires_missing_inputs), + "approved_lineage": entry.approved_lineage, + "publication": entry.publication, + "probe_evidence_sha256": probe_result.evidence_sha256, + } + ) + details["review_items"] = exception_reviews + return True + def fail_early(kind: str, summary: str, **extra: Any) -> GateResult: """A failure decided before the receipt's contents are judged; the policy record lists it as the regression.""" @@ -1030,13 +1444,98 @@ def fail_early(kind: str, summary: str, **extra: Any) -> GateResult: execution_mismatches=mismatches, ) + # An inert construction has identical worlds: every reported fiscal-effect + # level must be zero. Check before granting any exception, + # including failures whose corpus omitted its inert list and known gaps. + exceptable_measure_years: set[tuple[str, str, int]] = set() + for failure in receipt.integrity.failures: + if ( + matching_exception( + failure.code, failure.event, failure.measure_key, failure.year + ) + is not None + ): + assert failure.event is not None + assert failure.measure_key is not None + assert failure.year is not None + exceptable_measure_years.add( + (failure.event, failure.measure_key, failure.year) + ) + if receipt.corpus is not None: + for event, grid in receipt.corpus.events.items(): + for inert_year in grid.inert_measure_years: + if ( + matching_exception( + "lost_response", event, inert_year.measure_key, inert_year.year + ) + is not None + ): + exceptable_measure_years.add( + (event, inert_year.measure_key, inert_year.year) + ) + for row in receipt.diff.flagged_rows if receipt.diff is not None else []: + if ( + row.level == "measure_year" + and row.head_gbp == 0 + and row.base_gbp is not None + and abs(row.base_gbp) + >= max(REPLAY_ABSOLUTE_BOUND_GBP, REPLAY_RELATIVE_BOUND * abs(row.base_gbp)) + and matching_exception( + "lost_response", row.event, row.measure_key, row.year + ) + is not None + ): + exceptable_measure_years.add((row.event, row.measure_key, row.year)) + incoherent_rows = [ + row + for row in (receipt.diff.flagged_rows if receipt.diff is not None else []) + if (row.event, row.measure_key, row.year) in exceptable_measure_years + and row.head_gbp not in (None, 0) + ] + if incoherent_rows: + return fail_early( + "contradiction", + "Replay receipt contradicts its inert construction: an excepted " + "measure-year has a nonzero candidate effect.", + contradictions=[row.document() for row in incoherent_rows], + ) + if receipt.status == "failed" or receipt.integrity.status == "failed": codes = sorted({failure.code for failure in receipt.integrity.failures}) - return fail_early( - "failed", - "Replay integrity failed: " + (", ".join(codes) or "no failure codes"), - integrity_failures=[f.document() for f in receipt.integrity.failures], + # Failed receipts may describe an early crash with no corpus or + # sealed result. Continue only with complete execution evidence and + # a nonempty failure list, all of whose members match exactly. + can_review = ( + receipt.status == "failed" + and receipt.integrity.status == "failed" + and receipt.corpus is not None + and receipt.variant_id is not None + and receipt.variant_identity_sha256 is not None + and receipt.results.manifest_sha256 is not None + and bool(receipt.integrity.failures) + and all( + f.code == "construction_inert" + and matching_exception(f.code, f.event, f.measure_key, f.year) + is not None + for f in receipt.integrity.failures + ) ) + if not can_review: + rejected_exceptions = "; ".join( + f"{entry_id}: {reason}" + for entry_id, reason in sorted(exception_rejections.items()) + ) + return fail_early( + "failed", + "Replay integrity failed: " + + (", ".join(codes) or "no failure codes") + + (f"; {rejected_exceptions}" if rejected_exceptions else ""), + integrity_failures=[f.document() for f in receipt.integrity.failures], + ) + for failure in receipt.integrity.failures: + except_regression( + {"kind": failure.code, **failure.document()}, failure.code + ) corpus = receipt.corpus assert corpus is not None incomplete_events = sorted( @@ -1064,7 +1563,7 @@ def fail_early(kind: str, summary: str, **extra: Any) -> GateResult: ) # A receipt whose parts disagree is not evidence of anything. contradictions: list[str] = [] - if receipt.integrity.failures: + if receipt.integrity.failures and receipt.integrity.status != "failed": codes = sorted({failure.code for failure in receipt.integrity.failures}) contradictions.append( f"integrity is complete but lists failures ({', '.join(codes)})" @@ -1193,16 +1692,19 @@ def item(kind: str, row: Any = None, **fields: Any) -> dict[str, Any]: record.update(fields) return record + excepted_inert: set[tuple[str, str, int]] = set() for event, measure_key, year in sorted(candidate_inert): - regressions.append( - item( - "lost_response", - event=event, - measure_key=measure_key, - year=year, - why="inert in the candidate", - ) + record = item( + "lost_response", + event=event, + measure_key=measure_key, + year=year, + why="inert in the candidate", ) + if except_regression(record, "lost_response"): + excepted_inert.add((event, measure_key, year)) + else: + regressions.append(record) for name in incomplete_events: event = corpus.events[name] missing = missing_supported(event) @@ -1217,7 +1719,12 @@ def item(kind: str, row: Any = None, **fields: Any) -> dict[str, Any]: ) continue gaps = pairs(event.inert_measure_years) | missing - unknown = sorted(gap for gap in gaps if (name, *gap) not in known) + unknown = sorted( + gap + for gap in gaps + if (name, *gap) not in known + and (gap in missing or (name, *gap) not in excepted_inert) + ) if not gaps or unknown: regressions.append( item( @@ -1285,7 +1792,9 @@ def item(kind: str, row: Any = None, **fields: Any) -> dict[str, Any]: # The candidate's own inert list decides: a lost response above. lost = True elif zero_total: - regressions.append(item("lost_response", row, why="£0 against the base")) + record = item("lost_response", row, why="£0 against the base") + if not except_regression(record, "lost_response"): + regressions.append(record) lost = True if where in listed_inert or ( gap is not None @@ -1315,6 +1824,9 @@ def item(kind: str, row: Any = None, **fields: Any) -> dict[str, Any]: else: kind = "flagged_move" review_items.append(item(kind, row)) + # Include exceptions found in corpus and diff evaluation as well as the + # integrity failures. A matched loss always blocks automatic promotion. + review_items = exception_reviews + review_items details.update( { "policy": REPLAY_POLICY_ID, diff --git a/src/policyengine/provenance/replay_probe.py b/src/policyengine/provenance/replay_probe.py new file mode 100644 index 00000000..1936e929 --- /dev/null +++ b/src/policyengine/provenance/replay_probe.py @@ -0,0 +1,157 @@ +"""Synthetic, survey-free evidence for data-limited replay exceptions. + +The country model is intentionally used directly: this probe establishes that +the exact installed candidate responds to a positive input, independently of +the survey data whose absence is under review. +""" + +from __future__ import annotations + +import math +from importlib.metadata import version +from typing import Any + +from policyengine.provenance.qualification import ( + GateResult, + ReplayExceptionProbe, + ReplayExceptionRegistry, + _replay_exceptions, +) +from policyengine.provenance.receipt import content_sha256 + + +def _run_probe(spec: ReplayExceptionProbe) -> dict[str, Any]: + from policyengine_uk import Simulation + from policyengine_uk.model_api import Reform + + period = str(spec.year) + reform_rates: list[dict[str, float]] = [] + + def modify_parameters(parameters): + scale = parameters + for name in spec.parameter.split("."): + scale = getattr(scale, name) + # SDLT uses a marginal rate scale; update every bracket, including + # those above the synthetic purchase price, against its actual value. + if not scale.brackets: + raise ValueError("Probe parameter has no rate brackets.") + rates = [] + for bracket in scale.brackets: + baseline = float(bracket.rate(period)) + changed = baseline + spec.rate_increment + bracket.rate.update(period=period, value=changed) + rates.append({"baseline": baseline, "reform": changed}) + if reform_rates and reform_rates != rates: + raise ValueError("Positive and zero probes used different reform rates.") + reform_rates[:] = rates + return parameters + + class ProbeReform(Reform): + def apply(self): + self.modify_parameters(modifier_function=modify_parameters) + + def output(price: float, reform=None) -> float: + situation = { + "people": {"person": {"age": {period: 40}}}, + "benunits": {"benunit": {"members": ["person"]}}, + "households": { + "household": { + "members": ["person"], + "region": {period: spec.region}, + spec.input_variable: {period: price}, + } + }, + } + simulation = Simulation(situation=situation, reform=reform) + values = simulation.calculate(spec.output_variable, period) + if len(values) != 1: + raise ValueError("Probe must produce exactly one household output.") + result = float(values[0]) + if not math.isfinite(result): + raise ValueError("Probe produced a non-finite output.") + return result + + positive_baseline = output(spec.positive_input_gbp) + positive_reform = output(spec.positive_input_gbp, ProbeReform) + zero_baseline = output(0) + zero_reform = output(0, ProbeReform) + positive_change = positive_reform - positive_baseline + zero_change = zero_reform - zero_baseline + passed = ( + positive_change > 0 + and abs(positive_change - spec.expected_change_gbp) + <= spec.absolute_tolerance_gbp + and zero_change == 0 + ) + return { + "status": "pass" if passed else "fail", + "positive_baseline_gbp": positive_baseline, + "positive_reform_gbp": positive_reform, + "positive_change_gbp": positive_change, + "zero_baseline_gbp": zero_baseline, + "zero_reform_gbp": zero_reform, + "zero_change_gbp": zero_change, + "rates": reform_rates, + } + + +def run_replay_probes( + *, + attempt_id: str, + candidate_model_version: str, + exception_registry: ReplayExceptionRegistry | None = None, +) -> tuple[GateResult, dict[str, Any]]: + """Run registry probes and seal their attempt-specific JSON evidence. + + Any import, version, variable, parameter or calculation failure becomes + failing evidence. The replay evaluator decides whether an exception needs + this evidence; this result is not an additional unconditional release gate. + """ + + if exception_registry is None: + registry, registry_sha256 = _replay_exceptions() + else: + registry = exception_registry + registry_sha256 = content_sha256(registry.canonical_payload()) + evidence: dict[str, Any] = { + "attempt_id": attempt_id, + "candidate_model_version": candidate_model_version, + "exception_registry_sha256": registry_sha256, + "probes": [], + } + for entry in registry.exceptions: + probe: dict[str, Any] = { + "exception_id": entry.id, + "spec_sha256": content_sha256(entry.probe.model_dump(mode="json")), + } + try: + installed_version = version("policyengine-uk") + probe["installed_model_version"] = installed_version + if installed_version != candidate_model_version: + raise ValueError( + f"Installed policyengine-uk {installed_version} does not match " + f"candidate {candidate_model_version}." + ) + probe.update(_run_probe(entry.probe)) + except Exception as error: + probe.update(status="fail", error=f"{type(error).__name__}: {error}") + evidence["probes"].append(probe) + failed = sum(probe["status"] != "pass" for probe in evidence["probes"]) + result = GateResult( + gate_id="replay_probe", + status="fail" if failed else "pass", + summary=( + f"{failed} of {len(evidence['probes'])} replay exception probes failed." + if failed + else f"All {len(evidence['probes'])} replay exception probes passed." + ), + evidence_sha256=content_sha256(evidence), + details={ + "attempt_id": attempt_id, + "candidate_model_version": candidate_model_version, + "exception_registry_sha256": registry_sha256, + "probe_count": len(evidence["probes"]), + "failed_probes": failed, + }, + ) + return result, evidence diff --git a/src/policyengine/provenance/uk_candidate.py b/src/policyengine/provenance/uk_candidate.py index e77ec711..4599babf 100644 --- a/src/policyengine/provenance/uk_candidate.py +++ b/src/policyengine/provenance/uk_candidate.py @@ -59,7 +59,7 @@ def _json_safe(value: Any) -> Any: numpy = sys.modules.get("numpy") if numpy is not None and isinstance( - value, (numpy.bool_, numpy.integer, numpy.floating) + value, numpy.bool_ | numpy.integer | numpy.floating ): value = value.item() if isinstance(value, float) and not math.isfinite(value): @@ -68,7 +68,7 @@ def _json_safe(value: Any) -> Any: ) if isinstance(value, Mapping): return {str(key): _json_safe(item) for key, item in value.items()} - if isinstance(value, (list, tuple)): + if isinstance(value, list | tuple): return [_json_safe(item) for item in value] return value @@ -543,6 +543,9 @@ def budget_inputs( accepted_gaps: Mapping[str, str], previous_missing_inputs: Optional[Sequence[str]], previous_unusable_critical: Optional[Sequence[str]], + model_variables: Optional[Iterable[str]] = None, + previous_input_status: Optional[Mapping[str, Mapping[str, bool]]] = None, + previous_ever_usable_inputs: Optional[Sequence[str]] = None, tables: Optional[Mapping[str, Any]] = None, ) -> GateOutcome: """Critical inputs are present and usable; no input goes newly missing.""" @@ -553,6 +556,7 @@ def budget_inputs( for column in table.columns: columns.setdefault(column, entity) inputs = set(model_input_variables) + model_names = inputs | set(model_variables or []) missing = sorted(inputs - set(columns)) newly_missing = ( sorted(set(missing) - set(previous_missing_inputs)) @@ -569,18 +573,56 @@ def budget_inputs( regressions = [(n, why) for n, why in unusable if n not in previous_bad] standing = [(n, why) for n, why in unusable if n in previous_bad] unaccepted_standing = [(n, why) for n, why in standing if n not in accepted_gaps] + input_status = { + name: { + "in_model": name in model_names, + "is_input": name in inputs, + "in_data": name in columns, + "usable": name in model_names + and name in columns + and not _degenerate(tables[columns[name]][name]), + } + for name in sorted(model_names | set(columns)) + } + # Keep the complete certified history even when an intermediate model + # removes a variable or turns it into a formula. Direct-predecessor + # coverage alone must never reactivate an expired exception. + previous_ever_usable = set(previous_ever_usable_inputs or []) | { + name + for name, status in (previous_input_status or {}).items() + if status.get("usable") is True + } + ever_usable = previous_ever_usable | { + name for name, status in input_status.items() if status["usable"] + } + if previous_ever_usable_inputs is not None: + lineage_basis = "carried from direct predecessor" + elif previous_input_status is not None: + lineage_basis = "seeded from direct predecessor" + else: + lineage_basis = "candidate inputs" evidence = { + "pair_id": pair.pair_id, "model_input_variables": len(inputs), + "model_variables": len(model_names), "dataset_columns": len(columns), "missing_inputs": missing, "newly_missing_inputs": newly_missing, "unusable_critical": [list(item) for item in unusable], "accepted_gaps": dict(accepted_gaps), + "input_status": input_status, + "previous_input_status": dict(previous_input_status or {}), + "previous_ever_usable_inputs": sorted(previous_ever_usable), + "ever_usable_inputs": sorted(ever_usable), + "ever_usable_inputs_basis": lineage_basis, } details = { "missing_inputs": len(missing), "newly_missing_inputs": newly_missing, "unusable_critical": [name for name, _ in unusable], + # This immutable receipt marker distinguishes legacy evidence from a + # damaged modern cache, which must never reset the comparison chain. + "ever_usable_inputs": sorted(ever_usable), } if regressions: return _result( @@ -1439,7 +1481,7 @@ def _excludes(row: Mapping[str, Any]) -> tuple[str, ...]: def _finite_number(value: Any) -> bool: return ( not isinstance(value, bool) - and isinstance(value, (int, float)) + and isinstance(value, int | float) and math.isfinite(value) ) @@ -2127,7 +2169,7 @@ def reform_responsiveness( changes[str(year)] = None if format_problems else value if ( isinstance(value, bool) - or not isinstance(value, (int, float)) + or not isinstance(value, int | float) or not math.isfinite(value) or not sign * value >= reform["minimum"] ): @@ -2269,10 +2311,13 @@ def reform_responsiveness( from policyengine.provenance.dataset_materialization import materialize_dataset path = str(materialize_dataset("uk", data_dir=Path(settings["data_dir"])).path) inputs = settings.get("model_input_variables") +model_names = settings.get("model_variables") if inputs is None: from policyengine_uk.system import system inputs = [name for name, variable in system.variables.items() if variable.is_input_variable()] + model_names = list(system.variables) inputs = set(inputs) +model_names = inputs | set(model_names or []) with pd.HDFStore(path, mode="r") as store: tables = {e: store[e] for e in ("person", "benunit", "household") if f"/{e}" in store} first_year = int(str(store["time_period"].iloc[0])[:4]) @@ -2304,9 +2349,21 @@ def degenerate(series): years = sorted(int(y) for y in years) out.write_text(json.dumps({ "dataset_path": path, + "model_input_variables": len(inputs), + "model_variables": len(model_names), + "dataset_columns": len(columns), "missing_inputs": sorted(inputs - set(columns)), "unusable_critical": unusable, "supported_years": [y for y in years if y >= first_year], + "input_status": { + name: { + "in_model": name in model_names, + "is_input": name in inputs, + "in_data": name in columns, + "usable": name in model_names and name in columns and not degenerate(tables[columns[name]][name]), + } + for name in sorted(model_names | set(columns)) + }, })) """ @@ -2319,6 +2376,7 @@ def run_inputs_snapshot( dataset_path: Optional[Path] = None, data_dir: Optional[Path] = None, model_input_variables: Optional[Sequence[str]] = None, + model_variables: Optional[Sequence[str]] = None, projected_years: Optional[Sequence[int]] = None, run: Optional[Callable[..., Any]] = None, ) -> dict[str, Any]: @@ -2338,6 +2396,9 @@ def run_inputs_snapshot( "model_input_variables": ( list(model_input_variables) if model_input_variables is not None else None ), + "model_variables": list(model_variables) + if model_variables is not None + else None, "projected_years": list(projected_years) if projected_years is not None else None, @@ -2420,7 +2481,7 @@ def load_standard_reforms(vendor_root: Path, filename: str) -> list[dict[str, An or not responds.get("variable") or responds.get("direction") not in ("up", "down") or isinstance(minimum, bool) - or not isinstance(minimum, (int, float)) + or not isinstance(minimum, int | float) or not minimum > 0 ): raise ValueError( @@ -2895,20 +2956,38 @@ def _previous_gate_evidence( ) -> dict[str, Any]: """Verified evidence of the exact certified pair and receipt's attempt. - Legacy certificates that cannot bind every wheel and data pin by pair ID, - or incomplete/corrupt evidence, fall back to recomputation by the caller. + Legacy certificates can fall back to recomputation. Once a certified + receipt records cumulative history, unavailable evidence fails closed. """ - if ledger_root is None or not previous: + if not previous: return {} pair_id = previous.get("qualification_pair_id") if not isinstance(pair_id, str) or not re.fullmatch(r"[a-f0-9]{64}", pair_id): return {} + if ledger_root is None: + raise PreviousPairUnavailable( + "The previous certified pair's receipt is unavailable; a fresh " + "registry decision is required." + ) receipt_path = ledger_root / "uk" / pair_id / "qualification_receipt.json" + certified_history_recorded = False + receipt_loaded = False try: receipt = QualificationReceipt.model_validate_json(receipt_path.read_text()) + receipt_loaded = True pair = receipt.pair + by_gate = {gate.gate_id: gate for gate in receipt.gates} + inputs_gate = by_gate.get("budget_inputs") + certified_history_recorded = ( + inputs_gate is not None and "ever_usable_inputs" in inputs_gate.details + ) if not receipt.promotable or receipt.pair_id != pair_id: + if certified_history_recorded: + raise PreviousPairUnavailable( + "The cumulative input history does not identify the " + "previous certified pair; a fresh registry decision is required." + ) return {} expected = ( (pair.model.version, previous.get("model_version")), @@ -2917,6 +2996,12 @@ def _previous_gate_evidence( (pair.data.dataset_sha256, previous.get("dataset_sha256")), ) if any(actual != certified for actual, certified in expected): + if certified_history_recorded: + raise PreviousPairUnavailable( + "The cumulative input history does not match the " + "previous certified pair's pins; a fresh registry decision " + "is required." + ) return {} # The pair ID binds all wheels and data pins even for certificates # whose bundle's package record does not repeat the core wheel hash. @@ -2925,6 +3010,12 @@ def _previous_gate_evidence( ("core_wheel_sha256", pair.core.wheel_sha256), ): if previous.get(field) is not None and previous[field] != actual: + if certified_history_recorded: + raise PreviousPairUnavailable( + "The cumulative input history does not match the " + "previous certified pair's wheel pins; a fresh registry " + "decision is required." + ) return {} for attempt_receipt in sorted( (receipt_path.parent / "attempts").glob("*/qualification_receipt.json") @@ -2939,26 +3030,75 @@ def _previous_gate_evidence( harness = json.loads((attempt_receipt.parent / "harness.json").read_text()) if harness != receipt.harness.canonical_payload(): continue - by_gate = {gate.gate_id: gate for gate in receipt.gates} - found: dict[str, Any] = {} - for name in ("supported_years", "budget_inputs"): - gate = by_gate.get(name) - evidence = json.loads( + found = { + name: json.loads( (attempt_receipt.parent / "gates" / f"{name}.json").read_text() ) + for name in ("budget_inputs", "supported_years") + } + has_history = ( + certified_history_recorded + or "ever_usable_inputs" in found["budget_inputs"] + ) + for name in ("supported_years", "budget_inputs"): + gate = by_gate.get(name) + evidence = found[name] if ( not gate or gate.status not in ("pass", "needs_review") or gate.evidence_sha256 != evidence_sha256(evidence) ): + if has_history: + raise PreviousPairUnavailable( + "The previous certified pair's cumulative input " + "history could not be verified; a fresh registry " + "decision is required." + ) return {} - found[name] = evidence + if certified_history_recorded and ( + found["budget_inputs"].get("ever_usable_inputs") + != inputs_gate.details["ever_usable_inputs"] + ): + raise PreviousPairUnavailable( + "The previous certified pair's cumulative input history " + "does not match its receipt; a fresh registry decision " + "is required." + ) + if has_history and ( + found["budget_inputs"].get("pair_id") != pair_id + or found["budget_inputs"].get("attempt_id") + != f"{pair_id}/{attempt_receipt.parent.name}" + ): + raise PreviousPairUnavailable( + "The previous certified pair's cumulative input history " + "belongs to another pair or attempt; a fresh registry " + "decision is required." + ) if _valid_previous_snapshot_evidence(found): return found + if "ever_usable_inputs" in found["budget_inputs"]: + # A certified cumulative history cannot be replaced by a + # fresh direct-predecessor seed when its evidence is damaged. + raise PreviousPairUnavailable( + "The previous certified pair's cumulative input history " + "is inconsistent; a fresh registry decision is required." + ) return {} - except (OSError, ValueError, TypeError, AttributeError): + except PreviousPairUnavailable: + raise + except (OSError, ValueError, TypeError, AttributeError) as exc: + if certified_history_recorded or not receipt_loaded: + raise PreviousPairUnavailable( + "The previous certified pair's cumulative input history " + "could not be read; a fresh registry decision is required." + ) from exc # An unavailable cache is not qualification evidence; recompute it. return {} + if certified_history_recorded: + raise PreviousPairUnavailable( + "The previous certified pair's cumulative input history has no " + "matching attempt; a fresh registry decision is required." + ) return {} @@ -2967,6 +3107,8 @@ def _valid_previous_snapshot_evidence(evidence: Mapping[str, Any]) -> bool: inputs = evidence["budget_inputs"] missing = inputs.get("missing_inputs") unusable = inputs.get("unusable_critical") + input_status = inputs.get("input_status") + ever_usable = inputs.get("ever_usable_inputs") return ( isinstance(years, list) and bool(years) @@ -2985,6 +3127,60 @@ def _valid_previous_snapshot_evidence(evidence: Mapping[str, Any]) -> bool: and all(isinstance(value, str) and bool(value) for value in row) for row in unusable ) + and ( + "ever_usable_inputs" not in inputs + or ( + isinstance(ever_usable, list) + and all(isinstance(name, str) and bool(name) for name in ever_usable) + and len(ever_usable) == len(set(ever_usable)) + and isinstance(input_status, dict) + and { + name + for name, state in input_status.items() + if isinstance(state, dict) and state.get("usable") is True + }.issubset(ever_usable) + ) + ) + and ( + input_status is None + or ( + isinstance(input_status, dict) + and all( + isinstance(name, str) + and bool(name) + and isinstance(state, dict) + and all( + type(state.get(key)) is bool + for key in ("in_model", "in_data", "usable", "is_input") + ) + and ( + not state["usable"] or (state["in_model"] and state["in_data"]) + ) + and (not state["is_input"] or state["in_model"]) + for name, state in input_status.items() + ) + and all( + type(inputs.get(key)) is int + for key in ( + "model_variables", + "model_input_variables", + "dataset_columns", + ) + ) + and inputs["model_variables"] + == sum(state["in_model"] for state in input_status.values()) + and inputs["model_input_variables"] + == sum(state["is_input"] for state in input_status.values()) + and inputs["dataset_columns"] + == sum(state["in_data"] for state in input_status.values()) + and set(missing) + == { + name + for name, state in input_status.items() + if state["is_input"] and not state["in_data"] + } + ) + ) ) @@ -3080,12 +3276,19 @@ def previous_snapshot() -> Optional[dict[str, Any]]: snapshot: Optional[dict[str, Any]] = None recorded_inputs = previous_evidence.get("budget_inputs") recorded_years = previous_evidence.get("supported_years") - if recorded_inputs and recorded_years: + if ( + recorded_inputs + and recorded_years + and isinstance(recorded_inputs.get("input_status"), dict) + and "ever_usable_inputs" in recorded_inputs + ): snapshot = { "missing_inputs": recorded_inputs.get("missing_inputs"), "unusable_critical": recorded_inputs.get("unusable_critical", []), "supported_years": recorded_years.get("supported_years"), "source": "ledger", + "input_status": recorded_inputs["input_status"], + "ever_usable_inputs": recorded_inputs["ever_usable_inputs"], } elif previous_certificate: previous_python, previous_dataset = previous_environment() @@ -3116,14 +3319,31 @@ def gate_inputs() -> GateOutcome: from policyengine_uk.system import system previous = previous_snapshot() + input_names = [ + name + for name, variable in system.variables.items() + if variable.is_input_variable() + ] + previous_status = previous.get("input_status") if previous else None + if isinstance(previous_status, dict): + # Do not project away variables removed or made formula-backed by + # the candidate: they may already have expired an exception. + previous_status = dict(previous_status) + for name in system.variables: + previous_status.setdefault( + name, + { + "in_model": False, + "is_input": False, + "in_data": False, + "usable": False, + }, + ) result, evidence = budget_inputs( pair, dataset(), - model_input_variables=[ - name - for name, variable in system.variables.items() - if variable.is_input_variable() - ], + model_input_variables=input_names, + model_variables=system.variables, critical_inputs=critical_spec["critical"], accepted_gaps=critical_spec.get("accepted_gaps", {}), previous_missing_inputs=previous.get("missing_inputs") @@ -3134,6 +3354,10 @@ def gate_inputs() -> GateOutcome: if previous else None ), + previous_input_status=previous_status, + previous_ever_usable_inputs=previous.get("ever_usable_inputs") + if previous + else None, ) evidence["previous_source"] = previous.get("source") if previous else None return result, evidence diff --git a/src/policyengine/provenance/uk_qualification.py b/src/policyengine/provenance/uk_qualification.py index 42b4b7e5..5e98532d 100644 --- a/src/policyengine/provenance/uk_qualification.py +++ b/src/policyengine/provenance/uk_qualification.py @@ -77,7 +77,7 @@ read_json, write_json, ) -from policyengine.provenance.receipt import certificate_sha256 +from policyengine.provenance.receipt import certificate_sha256, content_sha256 UK_MODEL_PACKAGE = "policyengine-uk" CORE_PACKAGE = "policyengine-core" @@ -927,6 +927,11 @@ def read_replay( *, fetch: Callable[[str, str], Optional[bytes]], state_ref: str, + budget_inputs_result: GateResult | None = None, + budget_inputs_evidence: Mapping[str, Any] | None = None, + probe_result: GateResult | None = None, + probe_evidence: Mapping[str, Any] | None = None, + attempt_id: str | None = None, ) -> tuple[GateResult, Optional[ReplayStatus], Optional[ReplayReceipt]]: """Read the scorecard's status and, when terminal, its receipt. @@ -959,7 +964,20 @@ def read_replay( None, ) receipt = ReplayReceipt.from_document(json.loads(raw_receipt)) - return evaluate_replay_receipt(request, status, receipt), status, receipt + return ( + evaluate_replay_receipt( + request, + status, + receipt, + budget_inputs_result=budget_inputs_result, + budget_inputs_evidence=budget_inputs_evidence, + probe_result=probe_result, + probe_evidence=probe_evidence, + attempt_id=attempt_id, + ), + status, + receipt, + ) # --- Finalizing --------------------------------------------------------------- @@ -1020,6 +1038,18 @@ def accept_review( "The acceptance names different receipt bytes " f"({accepted_receipt_sha256}, current {receipt.unreviewed_sha256})." ) + publications = { + item["publication"] + for gate in receipt.gates + if gate.gate_id == "replay" + for item in gate.details.get("applied_exceptions", []) + if isinstance(item.get("publication"), str) + } + if any(publication not in explanation for publication in publications): + raise ValueError( + "Acceptance must restate each data-limited exception's publication " + "designation: " + "; ".join(sorted(publications)) + ) return receipt.model_copy( update={ "review": ReviewAcceptance( @@ -1096,6 +1126,24 @@ def stage_attempt( def write_gate_results( attempt_dir: Path, results: Sequence[GateResult], evidence: Mapping[str, Any] ) -> None: + evidence = dict(evidence) + results = list(results) + if "budget_inputs" in evidence and (attempt_dir / "candidate.json").exists(): + item = dict(evidence["budget_inputs"]) + attempt_id = attempt_evidence_id(attempt_dir) + identity = {"pair_id": attempt_id.partition("/")[0], "attempt_id": attempt_id} + if any(key in item and item[key] != value for key, value in identity.items()): + raise ValueError( + "Budget-input evidence belongs to a different pair or attempt." + ) + item.update(identity) + evidence["budget_inputs"] = item + results = [ + result.model_copy(update={"evidence_sha256": content_sha256(item)}) + if result.gate_id == "budget_inputs" + else result + for result in results + ] for result in results: write_json( attempt_dir / "gates" / f"{result.gate_id}.result.json", @@ -1109,6 +1157,9 @@ def load_gate_results(attempt_dir: Path) -> list[GateResult]: results = [ GateResult.model_validate(read_json(path)) for path in sorted((attempt_dir / "gates").glob("*.result.json")) + # The synthetic probe authorizes an exception; it is not an additional + # independent gate for candidates that do not need an exception. + if path.name != "replay_probe.result.json" ] replay = attempt_dir / "replay" / "replay.result.json" if replay.exists(): @@ -1117,6 +1168,67 @@ def load_gate_results(attempt_dir: Path) -> list[GateResult]: return results +def load_attempt_budget_inputs( + attempt_dir: Path, +) -> tuple[GateResult | None, Mapping[str, Any] | None]: + """Load digest-bound input evidence from this qualification attempt only. + + Absent or altered evidence cannot authorize a replay exception. Legacy + attempts without these files retain the ordinary replay rules. + """ + + result_path = attempt_dir / "gates" / "budget_inputs.result.json" + evidence_path = attempt_dir / "gates" / "budget_inputs.json" + try: + result = GateResult.model_validate(read_json(result_path)) + evidence = read_json(evidence_path) + attempt_id = attempt_evidence_id(attempt_dir) + if ( + result.gate_id != "budget_inputs" + or not isinstance(evidence, Mapping) + or evidence.get("attempt_id") != attempt_id + or evidence.get("pair_id") != attempt_id.partition("/")[0] + or result.evidence_sha256 != content_sha256(evidence) + ): + return None, None + except (OSError, ValueError, TypeError, KeyError): + return None, None + return result, evidence + + +def attempt_evidence_id(attempt_dir: Path) -> str: + """Stable identity across job checkouts, including the exact candidate pair. + + Attempt directory names alone repeat across pairs at the same base commit. + """ + + candidate = read_json(attempt_dir / "candidate.json") + pair = CandidatePair.model_validate(candidate["pair"]) + return f"{pair.pair_id}/{attempt_dir.name}" + + +def load_attempt_replay_probe( + attempt_dir: Path, +) -> tuple[GateResult | None, Mapping[str, Any] | None]: + """Read this attempt's probe, never a cached result from another attempt.""" + + result_path = attempt_dir / "gates" / "replay_probe.result.json" + evidence_path = attempt_dir / "gates" / "replay_probe.json" + try: + result = GateResult.model_validate(read_json(result_path)) + evidence = read_json(evidence_path) + if ( + result.gate_id != "replay_probe" + or not isinstance(evidence, Mapping) + or evidence.get("attempt_id") != attempt_evidence_id(attempt_dir) + or result.evidence_sha256 != content_sha256(evidence) + ): + return None, None + except (OSError, ValueError, TypeError, KeyError): + return None, None + return result, evidence + + def finalize_attempt( attempt_dir: Path, ledger_root: Path, @@ -1141,6 +1253,7 @@ def finalize_attempt( read_json(attempt_dir / "harness.json") ) gates = load_gate_results(attempt_dir) + cached_replay = next((gate for gate in gates if gate.gate_id == "replay"), None) if not any(gate.gate_id == "replay" for gate in gates): gates.append(replay_gate_while_pending(None)) previous = ( @@ -1149,6 +1262,7 @@ def finalize_attempt( else None ) replay_reference = None + replay_reevaluated = False request_path = attempt_dir / "replay" / "request.json" if request_path.exists(): request = ReplayQualificationRequest.model_validate(read_json(request_path)) @@ -1158,11 +1272,61 @@ def finalize_attempt( if status_path.exists() else None ) + replay_receipt_path = attempt_dir / "replay" / "receipt.json" + if status is not None and status.terminal and replay_receipt_path.exists(): + # Reevaluate the received bytes with this attempt's input evidence; + # a cached replay result must not bypass its missing-input condition. + budget_result, budget_evidence = load_attempt_budget_inputs(attempt_dir) + probe_result, probe_evidence = load_attempt_replay_probe(attempt_dir) + replay_gate = evaluate_replay_receipt( + request, + status, + ReplayReceipt.from_document(read_json(replay_receipt_path)), + budget_inputs_result=budget_result, + budget_inputs_evidence=budget_evidence, + probe_result=probe_result, + probe_evidence=probe_evidence, + attempt_id=attempt_evidence_id(attempt_dir), + ) + replay_reevaluated = True + gates = [gate for gate in gates if gate.gate_id != "replay"] + gates.append(replay_gate) + write_json( + attempt_dir / "replay" / "replay.result.json", + replay_gate.canonical_payload(), + ) replay_reference = ReplayReference( request_id=request.request_id, receipt_sha256=status.receipt_sha256 if status else None, receipt_commit=status.receipt_commit if status else None, ) + if ( + not replay_reevaluated + and cached_replay is not None + and cached_replay.details.get("applied_exceptions") + ): + # Exceptions must be evaluated again against the attempt's current + # input evidence; a cached waiver cannot substitute for received bytes. + replay_gate = GateResult( + gate_id="replay", + status="fail", + summary=( + "Cached replay exceptions cannot be validated without this " + "attempt's request, terminal status and receipt." + ), + details={ + "cached_applied_exceptions": cached_replay.details[ + "applied_exceptions" + ], + "regressions": [{"kind": "unverifiable_exception"}], + }, + ) + gates = [gate for gate in gates if gate.gate_id != "replay"] + gates.append(replay_gate) + write_json( + attempt_dir / "replay" / "replay.result.json", + replay_gate.canonical_payload(), + ) receipt = assemble_receipt( pair, harness, diff --git a/tests/fixtures/replay_exceptions/README.md b/tests/fixtures/replay_exceptions/README.md new file mode 100644 index 00000000..3deabf4c --- /dev/null +++ b/tests/fixtures/replay_exceptions/README.md @@ -0,0 +1,21 @@ +# UK replay data-limited exception fixtures + +The four `*2.134.7*.json` files are byte-for-byte copies of the Budget +coordinator's 17 October 2026 decision fixtures. They contain national +aggregates and provenance only, with no survey-level records. The archived +qualification receipt retains its original v1 policy and digest; it is +historical evidence, not a payload produced by the new gate. + +The original replay request was not supplied. Tests preserve the archived +receipt and construct a request from its execution identity, then rebind and +reseal a copy. All integrity failures, corpus records and diff rows remain +unchanged. This tests the real regression evidence without claiming an +unavailable original request binding. + +`main_replay_policy.py.txt` freezes the evaluator and its gap helper from +commit `3a278490102f`; `main_policy_metadata.json` records that commit's +policy id and text digest. Differential tests compare the complete gate +result against that oracle, removing only the new exception provenance +fields and the deliberately changed policy id/digest. A subprocess also +checks every replay evaluation in the existing contract suite, including +Hypothesis-generated receipts. diff --git a/tests/fixtures/replay_exceptions/main_policy_metadata.json b/tests/fixtures/replay_exceptions/main_policy_metadata.json new file mode 100644 index 00000000..e9e156c4 --- /dev/null +++ b/tests/fixtures/replay_exceptions/main_policy_metadata.json @@ -0,0 +1,5 @@ +{ + "commit": "3a278490102f", + "policy": "d1344-replay-regressions-v1", + "policy_sha256": "65ecaf44289624b6aae001068099250e789424854ed2b29ead86b71845a7dfa0" +} diff --git a/tests/fixtures/replay_exceptions/main_replay_policy.py.txt b/tests/fixtures/replay_exceptions/main_replay_policy.py.txt new file mode 100644 index 00000000..25a27723 --- /dev/null +++ b/tests/fixtures/replay_exceptions/main_replay_policy.py.txt @@ -0,0 +1,502 @@ +# Frozen d1344-replay-regressions-v1 evaluator at 3a278490102f. +# Loaded only as a differential test oracle; production never imports this. +from __future__ import annotations + +def _is_the_gap(row: FlaggedRow, gap: Mapping[str, Any]) -> bool: + """Whether a coverage row is scorecard's own record of a mixed known gap. + + Where one side is inert and the other missing, scorecard compares an + inert artifact (identical worlds, a total of exactly £0) with none, and + writes a coverage row with £0 on the inert side and nothing on the + other. Any other shape at a known gap is judged as usual. + """ + + if gap["candidate"] == "inert" and gap["why"] == "missing in the base": + return row.base_gbp is None and row.head_gbp == 0 + if gap["candidate"] == "missing" and gap["why"] == "inert in the base": + return row.head_gbp is None and row.base_gbp == 0 + return False + +def evaluate_replay_receipt( + request: ReplayQualificationRequest, + status: ReplayStatus, + receipt: ReplayReceipt, +) -> GateResult: + """The ``replay`` gate for a terminal replay status and its receipt. + + The receipt must answer this exact request, be the bytes the status + names, and report an execution of exactly the requested artifacts. Its + contents are then judged by policy ``d1344-replay-regressions-v1`` + (``data/qualification/replay_policy.md``): + + - fail on rejected or failed replays, mapping flags, lost results + (a coverage row with a base value and no candidate value), lost + responses (a measure-year total of exactly £0 against a base at least + the diff's bound away, which a known gap never excuses, or a + measure-year the candidate lists as inert, unless it is a known gap) + and incomplete grids (unless every gap in them is known). A gap is + known when the base's own record (``diff.base_events``) lists the same + measure-year as inert, or missing and supported. A known gap does not + block, and neither do the rows by which scorecard records a mixed one + (``_is_the_gap``, and the base-only inertness on a valueless row where + the candidate is missing and the base inert); any candidate value + other than £0 at a known gap fails as a contradiction, and any other + coverage row with a base value and no candidate value, even £0, is a + lost result; + - send every other flagged move to review. + + Unsupported years are reported, never failed on their own. The gate's + details record the policy id, the policy text's SHA-256, the known gaps + and the rows that record them, the regressions and the review items. + + The receipt is read as a whole: a receipt that contradicts itself fails + (failures listed under a complete integrity, ``flag_counts`` that differ + from its rows' flags, supported years with no artifacts, nothing + replayed at all) or contradicts the request (a comparison base other + than the ``compare_to`` asked for). + """ + + request_id = request.request_id + problems: list[str] = [] + if status.request_id != request_id or receipt.request_id != request_id: + problems.append("status or receipt answers a different request") + if not status.terminal: + problems.append(f"replay is still {status.status}") + if status.status != receipt.status and status.terminal: + problems.append("status and receipt disagree on the outcome") + if receipt.computed_receipt_sha256 != receipt.receipt_sha256: + problems.append("receipt_sha256 does not match the receipt bytes") + if status.receipt_sha256 != receipt.receipt_sha256: + problems.append("status names different receipt bytes") + + details: dict[str, Any] = { + "request_id": request_id, + "receipt_sha256": receipt.receipt_sha256, + "receipt_commit": status.receipt_commit, + "status": receipt.status, + "integrity": receipt.integrity.status, + "policy": REPLAY_POLICY_ID, + "policy_sha256": replay_policy_sha256(), + "known_gaps": [], + "regressions": [], + "review_items": [], + } + + def fail_early(kind: str, summary: str, **extra: Any) -> GateResult: + """A failure decided before the receipt's contents are judged; the + policy record lists it as the regression.""" + + return GateResult( + gate_id="replay", + status="fail", + summary=summary, + details={ + **details, + **extra, + "regressions": [{"kind": kind, "why": summary}], + }, + ) + + if problems: + return fail_early( + "unbound", + "Replay receipt is not bound to this request: " + "; ".join(problems), + binding_problems=problems, + ) + if receipt.status == "rejected": + assert receipt.rejection is not None + return fail_early( + "rejected", + f"Replay rejected the candidate ({receipt.rejection.code}): " + f"{receipt.rejection.detail}", + rejection=receipt.rejection.document(), + ) + + execution = receipt.execution + if execution is None: + codes = sorted({failure.code for failure in receipt.integrity.failures}) + return fail_early( + "failed", + "Replay failed before the candidate was identified: " + + (", ".join(codes) or "no failure codes"), + integrity_failures=[ + failure.document() for failure in receipt.integrity.failures + ], + ) + candidate = request.candidate + mismatches: list[str] = [] + packages = { + "policyengine": ( + execution.packages.policyengine, + candidate.policyengine.version, + ( + candidate.policyengine.source.pypi.wheel_sha256 + if candidate.policyengine.source.pypi is not None + else None + ), + ), + "policyengine-uk": ( + execution.packages.policyengine_uk, + candidate.policyengine_uk.version, + candidate.policyengine_uk.wheel_sha256, + ), + "policyengine-core": ( + execution.packages.policyengine_core, + candidate.policyengine_core.version, + candidate.policyengine_core.wheel_sha256, + ), + } + for name, (installed, version, wheel) in packages.items(): + if installed.version != version: + mismatches.append(f"{name} {installed.version} ran, not {version}") + if wheel is not None and installed.wheel_sha256 != wheel: + mismatches.append( + f"{name} wheel hash is missing from the receipt" + if installed.wheel_sha256 is None + else f"{name} wheel hash differs from the request" + ) + if ( + execution.packaged_bundle_manifest_sha256 + != candidate.policyengine.packaged_bundle_manifest_sha256 + ): + mismatches.append("packaged bundle manifest differs from the request") + if execution.dataset_sha256 != candidate.data.sha256: + mismatches.append("dataset bytes differ from the request") + if execution.release_bundle != candidate.expected_release_bundle: + mismatches.append("release bundle differs from the expected release bundle") + if mismatches: + return fail_early( + "unbound", + "Replay ran something other than the candidate: " + "; ".join(mismatches), + execution_mismatches=mismatches, + ) + + if receipt.status == "failed" or receipt.integrity.status == "failed": + codes = sorted({failure.code for failure in receipt.integrity.failures}) + return fail_early( + "failed", + "Replay integrity failed: " + (", ".join(codes) or "no failure codes"), + integrity_failures=[f.document() for f in receipt.integrity.failures], + ) + corpus = receipt.corpus + assert corpus is not None + incomplete_events = sorted( + name + for name, event in corpus.events.items() + if not event.full_supported_grid_complete + ) + unsupported = sorted( + (name, item.year, item.reason) + for name, event in corpus.events.items() + for item in event.unsupported_years + ) + inert = sorted( + (name, item.measure_key, item.year) + for name, event in corpus.events.items() + for item in event.inert_measure_years + ) + details.update( + { + "corpus_version": corpus.corpus_version, + "events": sorted(corpus.events), + "unsupported_years": [list(item) for item in unsupported], + "inert_measure_years": [list(item) for item in inert], + } + ) + # A receipt whose parts disagree is not evidence of anything. + contradictions: list[str] = [] + if receipt.integrity.failures: + codes = sorted({failure.code for failure in receipt.integrity.failures}) + contradictions.append( + f"integrity is complete but lists failures ({', '.join(codes)})" + ) + replayed = 0 + for name, event in sorted(corpus.events.items()): + replayed += event.artifacts + listed_missing = { + (item.measure_key, item.year) for item in event.missing_measure_years or () + } - { + (item.measure_key, item.year) + for item in event.unsupported_measure_years or () + } + if event.full_supported_grid_complete and ( + event.inert_measure_years or listed_missing + ): + contradictions.append( + f"{name} is complete but lists inert or missing measure-years" + ) + if event.supported_years and event.artifacts == 0: + contradictions.append(f"{name} has supported years but no artifacts") + if not event.supported_years and not event.unsupported_years: + contradictions.append(f"{name} names no years at all") + if replayed == 0: + contradictions.append("no measure-year was replayed") + counts = receipt.diff.flag_counts.document() if receipt.diff is not None else {} + rows = receipt.diff.flagged_rows if receipt.diff is not None else [] + row_counts: dict[str, int] = {} + for row in rows: + for kind in row.flags: + row_counts[kind] = row_counts.get(kind, 0) + 1 + # Scorecard's counters are tallies of its rows' flags, so any difference, + # either way, is a receipt that contradicts itself. + for kind in sorted(set(row_counts) | {kind for kind, n in counts.items() if n}): + seen, counted = row_counts.get(kind, 0), counts.get(kind, 0) + if seen != counted: + contradictions.append( + f"{seen} flagged row(s) carry {kind} but flag_counts says {counted}" + ) + details["flag_counts"] = counts + details["flagged_row_counts"] = row_counts + if receipt.diff is not None: + details["base_variant_id"] = receipt.diff.base_variant_id + details["diff_sha256"] = receipt.diff.diff_sha256 + if request.compare_to != "latest_certified": + base = receipt.diff.base_variant_id if receipt.diff is not None else None + if base != request.compare_to: + contradictions.append( + f"the request asked for a comparison with {request.compare_to}, " + f"but the receipt compares with {base}" + ) + if contradictions: + return fail_early( + "contradiction", + "Replay receipt contradicts itself or the request: " + + "; ".join(contradictions), + contradictions=contradictions, + ) + # A gap is known only where the base's own grid record (scorecard#183) + # lists the same measure-year as inert, or missing and supported, for the + # same event. A legacy-layout base, or an event the base did not replay, + # has no record, so none of its gaps are known. + base_events = (receipt.diff.base_events or {}) if receipt.diff is not None else {} + + def pairs(items: Optional[Sequence[MeasureYear]]) -> set[tuple[str, int]]: + return {(item.measure_key, item.year) for item in items or ()} + + def missing_supported(record: Any) -> Optional[set[tuple[str, int]]]: + if record.missing_measure_years is None: + return None + return pairs(record.missing_measure_years) - pairs( + record.unsupported_measure_years + ) + + known_gaps: list[dict[str, Any]] = [] + for name, event in sorted(corpus.events.items()): + base = base_events.get(name) + if base is None: + continue + base_inert = pairs(base.inert_measure_years) + base_missing = missing_supported(base) or set() + event_inert = pairs(event.inert_measure_years) + for measure_key, year in sorted( + event_inert | (missing_supported(event) or set()) + ): + if (measure_key, year) in base_inert: + why = "inert in the base" + elif (measure_key, year) in base_missing: + why = "missing in the base" + else: + continue + known_gaps.append( + { + "event": name, + "measure_key": measure_key, + "year": year, + "candidate": "inert" + if (measure_key, year) in event_inert + else "missing", + "why": why, + "base_staging_manifest_sha256": base.staging_manifest_sha256, + } + ) + gaps_at = { + (gap["event"], gap["measure_key"], gap["year"]): gap for gap in known_gaps + } + known = set(gaps_at) + listed_inert = set(inert) + candidate_inert = {where for where in listed_inert if where not in known} + regressions: list[dict[str, Any]] = [] + review_items: list[dict[str, Any]] = [] + + def item(kind: str, row: Any = None, **fields: Any) -> dict[str, Any]: + record: dict[str, Any] = {"kind": kind} + if row is not None: + record.update( + event=row.event, + measure_key=row.measure_key, + year=row.year, + level=row.level, + key=row.key, + flags=sorted(row.flags), + base_gbp=row.base_gbp, + head_gbp=row.head_gbp, + ) + record.update(fields) + return record + + for event, measure_key, year in sorted(candidate_inert): + regressions.append( + item( + "lost_response", + event=event, + measure_key=measure_key, + year=year, + why="inert in the candidate", + ) + ) + for name in incomplete_events: + event = corpus.events[name] + missing = missing_supported(event) + if missing is None: + regressions.append( + item( + "incomplete", + event=name, + why="the supported grid is incomplete and the receipt does " + "not list its missing measure-years", + ) + ) + continue + gaps = pairs(event.inert_measure_years) | missing + unknown = sorted(gap for gap in gaps if (name, *gap) not in known) + if not gaps or unknown: + regressions.append( + item( + "incomplete", + event=name, + why="the supported grid is incomplete and not every inert " + "or missing measure-year in it is a known gap", + measure_years=[list(gap) for gap in unknown], + ) + ) + if receipt.integrity.status == "incomplete" and not incomplete_events: + regressions.append( + item("incomplete", why="integrity is incomplete with no incomplete event") + ) + gap_rows: list[dict[str, Any]] = [] + for row in rows: + flags = set(row.flags) + where = (row.event, row.measure_key, row.year) + gap = gaps_at.get(where) + lost = False + if gap is not None and "coverage" in flags and _is_the_gap(row, gap): + # Scorecard's own expression of a mixed known gap: one side has no + # artifact and the other an inert one (identical worlds, £0). + flags.discard("coverage") + if flags & BLOCKING_REPLAY_FLAGS: + regressions.append(item("mapping", row)) + lost = True + if "coverage" in flags and row.base_gbp is not None and row.head_gbp is None: + regressions.append(item("lost_result", row)) + lost = True + if gap is not None and row.head_gbp not in (None, 0): + # The candidate's record lists the measure-year as inert or + # missing, so no row can hold a candidate value for it (an inert + # artifact's effects are exactly £0; a missing one has none). + regressions.append( + item( + "contradiction", + row, + why="a candidate value for a measure-year its record lists " + "as a gap", + ) + ) + lost = True + zero_total = ( + row.level == "measure_year" + and row.head_gbp == 0 + and row.base_gbp is not None + and abs(row.base_gbp) + >= max(REPLAY_ABSOLUTE_BOUND_GBP, REPLAY_RELATIVE_BOUND * abs(row.base_gbp)) + ) + if zero_total and where in known: + # A base whose own record lists the measure-year as a gap cannot + # also hold a material value for it: the receipt contradicts + # itself, so the gap does not excuse the loss. + regressions.append( + item( + "lost_response", + row, + why="£0 against a material base value, though the base's " + "record lists the measure-year as a gap", + ) + ) + lost = True + elif where in listed_inert and zero_total: + # The candidate's own inert list decides: a lost response above. + lost = True + elif zero_total: + regressions.append(item("lost_response", row, why="£0 against the base")) + lost = True + if where in listed_inert or ( + gap is not None + and gap["candidate"] == "missing" + and gap["why"] == "inert in the base" + and row.base_gbp is None + and row.head_gbp is None + ): + # The candidate's inert list decides its inertness (a lost response + # above, or a known gap). Where the candidate is missing and the + # base inert, a base-only inertness flag on a row with no value on + # either side (as scorecard writes it) is the known gap itself. + # The row's other flags are judged as usual. + flags = flags - {"inertness"} + if gap is not None and flags != set(row.flags): + # A row the known gap accounts for, in whole or in part. + gap_rows.append(item("known_gap_row", row)) + if not lost and flags & REVIEW_REPLAY_FLAGS: + if "coverage" in flags and row.base_gbp is None: + kind = ( + "coverage_gain" + if row.head_gbp is not None + else "coverage_without_values" + ) + elif flags == {"inertness"}: + kind = "base_only_inertness" + else: + kind = "flagged_move" + review_items.append(item(kind, row)) + details.update( + { + "policy": REPLAY_POLICY_ID, + "policy_sha256": replay_policy_sha256(), + "known_gaps": known_gaps, + "known_gap_rows": gap_rows, + "regressions": regressions, + "review_items": review_items, + } + ) + + def tally(records: Sequence[dict[str, Any]]) -> str: + kinds: dict[str, int] = {} + for record in records: + kinds[record["kind"]] = kinds.get(record["kind"], 0) + 1 + return ", ".join(f"{kind} {n}" for kind, n in sorted(kinds.items())) + + if regressions: + return GateResult( + gate_id="replay", + status="fail", + summary="Replay regressions: " + tally(regressions), + details=details, + ) + if review_items: + return GateResult( + gate_id="replay", + status="needs_review", + summary="Replay moves need review: " + tally(review_items), + details=details, + ) + if receipt.diff is None: + return GateResult( + gate_id="replay", + status="needs_review", + summary="Replay completed but reported no diff against a certified variant.", + details=details, + ) + return GateResult( + gate_id="replay", + status="pass", + summary=f"Replayed {len(corpus.events)} event(s) with no flags against {receipt.diff.base_variant_id}.", + details=details, + ) diff --git a/tests/fixtures/replay_exceptions/replay_receipt_uk2.134.7_failed.json b/tests/fixtures/replay_exceptions/replay_receipt_uk2.134.7_failed.json new file mode 100644 index 00000000..0b85ee6c --- /dev/null +++ b/tests/fixtures/replay_exceptions/replay_receipt_uk2.134.7_failed.json @@ -0,0 +1,2994 @@ +{ + "corpus": { + "corpus_version": "uk-fiscal-event-replay/2", + "events": { + "autumn_budget_2024": { + "artifacts": 30, + "comparison_provenance_sha256": "3eb3c36139f0f186c1b051e6aa4790375f480ab857082f48d2309862c2cc0da0", + "full_supported_grid_complete": false, + "inert_measure_years": [ + { + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2025 + }, + { + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2026 + }, + { + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2027 + }, + { + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2028 + }, + { + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2029 + } + ], + "missing_measure_years": [], + "registry_sha256": "89784d34920fb2d6aae702fb4028f84d7bf87f1694df77a0c9c5d255edbcdd34", + "run_manifest_sha256": "293ae41959858e3d397a231930a57052fcc4db665daeb31c63d4a1f191a72b5d", + "staging_manifest_sha256": "9b2ea2ebf798d1ede7afaffbb2d203b82f1939e930683a5a10b24833892f0a74", + "supported_years": [ + 2024, + 2025, + 2026, + 2027, + 2028, + 2029 + ], + "unsupported_measure_years": [], + "unsupported_years": [] + }, + "autumn_budget_2025": { + "artifacts": 54, + "comparison_provenance_sha256": "25feff57d6902340cf788371a4110341795243392c0c0826937e0d0b463f9f04", + "full_supported_grid_complete": true, + "inert_measure_years": [], + "missing_measure_years": [], + "registry_sha256": "3da50f9af6c7986fd5fe453ae2dac34275f319d1f2ccd441e3c438f51a18656e", + "run_manifest_sha256": "4774ac2a195f589b2051f62101a521827cd272bf02c1a6159064e2a9d1c90494", + "staging_manifest_sha256": "11f5b7a3e6326de6d43097087dd84e988077dc0efe6f548432768b6a8d8d376e", + "supported_years": [ + 2025, + 2026, + 2027, + 2028, + 2029, + 2030 + ], + "unsupported_measure_years": [], + "unsupported_years": [] + }, + "autumn_statement_2023": { + "artifacts": 20, + "comparison_provenance_sha256": "48587c1a7cd4dbbe0eb91ecd5470a3a92cfbc33e4859f4eb6dc2d0e9871d0507", + "full_supported_grid_complete": true, + "inert_measure_years": [], + "missing_measure_years": [ + { + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2023 + }, + { + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2023 + }, + { + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2023 + }, + { + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2023 + } + ], + "registry_sha256": "197d5583f0171104c673ca1f40b402a77ca2b003f496de37d6bc913fb9fd1021", + "run_manifest_sha256": "32ffaa6282bfce8567937b26fa3503a583880f9ed8ffc313868a18567d1c1c4f", + "staging_manifest_sha256": "86008b7f6e609eceef5740528ba9393466ce62f250fbd1770fa6ba029dd69a6e", + "supported_years": [ + 2024, + 2025, + 2026, + 2027, + 2028 + ], + "unsupported_measure_years": [ + { + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2023 + }, + { + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2023 + }, + { + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2023 + }, + { + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2023 + } + ], + "unsupported_years": [ + { + "reason": "before_data_year", + "year": 2023 + } + ] + }, + "spring_budget_2023": { + "artifacts": 8, + "comparison_provenance_sha256": "bf29ce3837068aaf5ccdd9358d801ef337b6b9851e511c79acb19f830218d678", + "full_supported_grid_complete": true, + "inert_measure_years": [], + "missing_measure_years": [ + { + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2023 + }, + { + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2023 + } + ], + "registry_sha256": "0a78bfe32712bd30bdf9137bdf182e82a266c7da0e9885dd61cac6ef99a76357", + "run_manifest_sha256": "f36cd0908ad1b8b75edd14e83d0a08ee058b89bdb0db4f6508fcf63d79ec7280", + "staging_manifest_sha256": "2f06bc224c8f6376cb9063e21b3b15e02463304ab1a89afa322e5b701a810729", + "supported_years": [ + 2024, + 2025, + 2026, + 2027 + ], + "unsupported_measure_years": [ + { + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2023 + }, + { + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2023 + } + ], + "unsupported_years": [ + { + "reason": "before_data_year", + "year": 2023 + } + ] + }, + "spring_budget_2024": { + "artifacts": 15, + "comparison_provenance_sha256": "957b7a9038cba2722667a6b4f18c26b1848d5c3db0a9df4da5cf5d1f42bd980a", + "full_supported_grid_complete": true, + "inert_measure_years": [], + "missing_measure_years": [ + { + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2023 + }, + { + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2023 + }, + { + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2023 + } + ], + "registry_sha256": "64a3bccfd950e868ffc5d6261bf3903df88a3ebedc9b4444494acc717259817a", + "run_manifest_sha256": "3caf0a9e3dba2f9883a3b069816f23582e2f827a18a259aaa0e1d364ea347926", + "staging_manifest_sha256": "e763dfd510abc233adeaf5470f0c246accd22f18756e32cc21eb9152f27f2401", + "supported_years": [ + 2024, + 2025, + 2026, + 2027, + 2028 + ], + "unsupported_measure_years": [ + { + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2023 + }, + { + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2023 + }, + { + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2023 + } + ], + "unsupported_years": [ + { + "reason": "before_data_year", + "year": 2023 + } + ] + }, + "spring_statement_2025": { + "artifacts": 12, + "comparison_provenance_sha256": "67fc6be5ba9b1f5be57e11747c37750e749122144cf3b9d729d5be1d617cf36a", + "full_supported_grid_complete": true, + "inert_measure_years": [], + "missing_measure_years": [], + "registry_sha256": "d01ce0612f3f24830a8d8ea0abaaa6b2626963316b16750310d303f404fbab5b", + "run_manifest_sha256": "0f9083557962ab52336d5b30c750fafff37286f7c5b417438b3741b880c4d373", + "staging_manifest_sha256": "a54e47699cff2e43a409f95073c00921109559e721cb7b8122332e4b6d764e4e", + "supported_years": [ + 2024, + 2025, + 2026, + 2027, + 2028, + 2029 + ], + "unsupported_measure_years": [], + "unsupported_years": [] + } + } + }, + "diff": { + "base_events": { + "autumn_budget_2024": { + "full_supported_grid_complete": true, + "inert_measure_years": [], + "missing_measure_years": [], + "run_manifest_sha256": "23ab600e88ef0269fa7d73f5e403ec3500e7b0a7d5fcddd60ba14c8239e39fda", + "staging_manifest_sha256": "1d882c1d4eb7a13b523bf99ac31945e2d4a3ebf836d6789565a006d2b71e789e", + "supported_years": [ + 2024, + 2025, + 2026, + 2027, + 2028, + 2029 + ], + "unsupported_measure_years": [] + }, + "autumn_budget_2025": { + "full_supported_grid_complete": true, + "inert_measure_years": [], + "missing_measure_years": [], + "run_manifest_sha256": "413acf6342a0e0a3a277cf1f2e9c0bf3c4be1e103204201f9171b90ab21ff5c5", + "staging_manifest_sha256": "5e4df8946e6cea2fa728200d1af2e6803cb74020cac521dd9b99f8869f80d456", + "supported_years": [ + 2025, + 2026, + 2027, + 2028, + 2029, + 2030 + ], + "unsupported_measure_years": [] + }, + "autumn_statement_2023": { + "full_supported_grid_complete": true, + "inert_measure_years": [], + "missing_measure_years": [ + { + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2023 + }, + { + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2023 + }, + { + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2023 + }, + { + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2023 + } + ], + "run_manifest_sha256": "eb3705d7edb480a91eb37555c3fd52169861ceee84c5514b5529e248247d882a", + "staging_manifest_sha256": "239703b8c314f555fcaf96adaf573a7831962433fcb0e2b26713a217ec48ec7e", + "supported_years": [ + 2024, + 2025, + 2026, + 2027, + 2028 + ], + "unsupported_measure_years": [ + { + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2023 + }, + { + "measure_key": "autumn_statement_2023__class_2_self_employed_nics_abolition", + "year": 2023 + }, + { + "measure_key": "autumn_statement_2023__class_4_self_employed_nics_main_rate_cut_1p", + "year": 2023 + }, + { + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2023 + } + ] + }, + "spring_budget_2023": { + "full_supported_grid_complete": true, + "inert_measure_years": [], + "missing_measure_years": [ + { + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2023 + }, + { + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2023 + } + ], + "run_manifest_sha256": "29944c4e0393091d621f260f658f700549c207dda4996dce198907665a2b3baa", + "staging_manifest_sha256": "a298924c7ed24832820ddb95196db1d9e9b7b40d32103762e7a5ac59477ef8fd", + "supported_years": [ + 2024, + 2025, + 2026, + 2027 + ], + "unsupported_measure_years": [ + { + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2023 + }, + { + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2023 + } + ] + }, + "spring_budget_2024": { + "full_supported_grid_complete": true, + "inert_measure_years": [], + "missing_measure_years": [ + { + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2023 + }, + { + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2023 + }, + { + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2023 + } + ], + "run_manifest_sha256": "458f2f27eb954422bb8e8a3063b629b43f2ffae9cdf901680924c8846a3e9666", + "staging_manifest_sha256": "3679f6e865bd2068efe46db2ea3358f92d6be9d6ade8d2c1c901901ce9813d07", + "supported_years": [ + 2024, + 2025, + 2026, + 2027, + 2028 + ], + "unsupported_measure_years": [ + { + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2023 + }, + { + "measure_key": "spring_budget_2024__class_4_self_employed_nics_main_rate_cut_2pp", + "year": 2023 + }, + { + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2023 + } + ] + }, + "spring_statement_2025": { + "full_supported_grid_complete": true, + "inert_measure_years": [], + "missing_measure_years": [], + "run_manifest_sha256": "01d97cd339b9a389390e2de1eec9617ab4d3a0e0d3374cd2dd9d466bba717fe4", + "staging_manifest_sha256": "dcd2e1349596fa7cd26140ea2b4d29e3f6a2c2b7744359e93143af98d83bdb6c", + "supported_years": [ + 2024, + 2025, + 2026, + 2027, + 2028, + 2029 + ], + "unsupported_measure_years": [] + } + }, + "base_variant_id": "pe6.5.0_uk2.130.1_policyengine-uk-data-1.58.1", + "diff_sha256": "f73fe6b98811fa574e7178f7fe7ca0cf49bd58797b392df93ee56f3e245a8bf8", + "flag_counts": { + "construction": 0, + "coverage": 0, + "inertness": 10, + "mapping": 0, + "numerical": 200, + "structural": 10, + "support": 0 + }, + "flagged_rows": [ + { + "base_gbp": 956314309.095192, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 726838394.7580452, + "key": "obr_pmd_nov2025:11682:73295721bdfdb9e8", + "level": "source_row", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2024 + }, + { + "base_gbp": 956314309.095192, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 726838394.7580452, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2024 + }, + { + "base_gbp": 956314309.095192, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 726838394.7580452, + "key": "capital_gains_tax", + "level": "head", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2024 + }, + { + "base_gbp": 2321626476.090164, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1745149570.9462242, + "key": "obr_pmd_nov2025:11683:b9096ecd110093bf", + "level": "source_row", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2025 + }, + { + "base_gbp": 2321626476.090164, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1745149570.9462242, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2025 + }, + { + "base_gbp": 2321626476.090164, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1745149570.9462242, + "key": "capital_gains_tax", + "level": "head", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2025 + }, + { + "base_gbp": 2397857290.327587, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1801465870.3176498, + "key": "obr_pmd_nov2025:11684:0d39628e9b763f67", + "level": "source_row", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2026 + }, + { + "base_gbp": 2397857290.327587, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1801465870.3176498, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2026 + }, + { + "base_gbp": 2397857290.327587, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1801465870.3176498, + "key": "capital_gains_tax", + "level": "head", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2026 + }, + { + "base_gbp": 2484671863.66201, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1865611783.5399685, + "key": "obr_pmd_nov2025:11685:440412b0fb8c6e2d", + "level": "source_row", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2027 + }, + { + "base_gbp": 2484671863.66201, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1865611783.5399685, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2027 + }, + { + "base_gbp": 2484671863.66201, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1865611783.5399685, + "key": "capital_gains_tax", + "level": "head", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2027 + }, + { + "base_gbp": 2572403121.83226, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1930467713.429018, + "key": "obr_pmd_nov2025:11686:9ceb140875d859b8", + "level": "source_row", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2028 + }, + { + "base_gbp": 2572403121.83226, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1930467713.429018, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2028 + }, + { + "base_gbp": 2572403121.83226, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1930467713.429018, + "key": "capital_gains_tax", + "level": "head", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2028 + }, + { + "base_gbp": 2660288740.9552917, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1995518507.5508633, + "key": "obr_pmd_nov2025:11687:f738e039412d9872", + "level": "source_row", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2029 + }, + { + "base_gbp": 2660288740.9552917, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1995518507.5508633, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2029 + }, + { + "base_gbp": 2660288740.9552917, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1995518507.5508633, + "key": "capital_gains_tax", + "level": "head", + "measure_key": "autumn_budget_2024__capital_gains_main_rates_and_reliefs", + "year": 2029 + }, + { + "base_gbp": -7898227103.010376, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -7806967661.259277, + "key": "obr_pmd_nov2025:11668:a076a7d6b3992e7c", + "level": "source_row", + "measure_key": "autumn_budget_2024__employer_nics_package", + "year": 2028 + }, + { + "base_gbp": -7898227103.010376, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -7806967661.259277, + "key": "income_tax", + "level": "head", + "measure_key": "autumn_budget_2024__employer_nics_package", + "year": 2028 + }, + { + "base_gbp": 1940272255.14898, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1844618182.746337, + "key": "obr_pmd_nov2025:11617:414a87b10621e433", + "level": "source_row", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2025 + }, + { + "base_gbp": 1940272255.14898, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1844618182.746337, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2025 + }, + { + "base_gbp": 1940272255.14898, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1844618182.746337, + "key": "private_school_vat", + "level": "head", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2025 + }, + { + "base_gbp": 1966545651.307916, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1929337062.7593496, + "key": "obr_pmd_nov2025:11618:ad4256274cc146d3", + "level": "source_row", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2026 + }, + { + "base_gbp": 1966545651.307916, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1929337062.7593496, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2026 + }, + { + "base_gbp": 1966545651.307916, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1929337062.7593496, + "key": "private_school_vat", + "level": "head", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2026 + }, + { + "base_gbp": 2044395945.9878988, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1977114282.153624, + "key": "obr_pmd_nov2025:11619:8726edda9aa7b020", + "level": "source_row", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2027 + }, + { + "base_gbp": 2044395945.9878988, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1977114282.153624, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2027 + }, + { + "base_gbp": 2044395945.9878988, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 1977114282.153624, + "key": "private_school_vat", + "level": "head", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2027 + }, + { + "base_gbp": 2138101665.0296612, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2027518629.2448545, + "key": "obr_pmd_nov2025:11620:5446e6157f4ef4ec", + "level": "source_row", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2028 + }, + { + "base_gbp": 2138101665.0296612, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2027518629.2448545, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2028 + }, + { + "base_gbp": 2138101665.0296612, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2027518629.2448545, + "key": "private_school_vat", + "level": "head", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2028 + }, + { + "base_gbp": 2163020750.92759, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2078193996.5772798, + "key": "obr_pmd_nov2025:11621:8bf103a18fecf781", + "level": "source_row", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2029 + }, + { + "base_gbp": 2163020750.92759, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2078193996.5772798, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2029 + }, + { + "base_gbp": 2163020750.92759, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 2078193996.5772798, + "key": "private_school_vat", + "level": "head", + "measure_key": "autumn_budget_2024__private_school_vat_20pct", + "year": 2029 + }, + { + "base_gbp": 2182277423.2901382, + "event": "autumn_budget_2024", + "flags": [ + "inertness" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:11647:b269082735adb83b", + "level": "source_row", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2025 + }, + { + "base_gbp": 2182277423.2901382, + "event": "autumn_budget_2024", + "flags": [ + "numerical", + "structural", + "inertness" + ], + "head_gbp": 0.0, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2025 + }, + { + "base_gbp": 2182277423.2901382, + "event": "autumn_budget_2024", + "flags": [ + "numerical", + "structural" + ], + "head_gbp": 0.0, + "key": "stamp_duty_land_tax", + "level": "head", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2025 + }, + { + "base_gbp": 2254534269.309538, + "event": "autumn_budget_2024", + "flags": [ + "inertness" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:11648:48bd77ddd06dec27", + "level": "source_row", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2026 + }, + { + "base_gbp": 2254534269.309538, + "event": "autumn_budget_2024", + "flags": [ + "numerical", + "structural", + "inertness" + ], + "head_gbp": 0.0, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2026 + }, + { + "base_gbp": 2254534269.309538, + "event": "autumn_budget_2024", + "flags": [ + "numerical", + "structural" + ], + "head_gbp": 0.0, + "key": "stamp_duty_land_tax", + "level": "head", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2026 + }, + { + "base_gbp": 2335967764.6855392, + "event": "autumn_budget_2024", + "flags": [ + "inertness" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:11649:910077d14a180f90", + "level": "source_row", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2027 + }, + { + "base_gbp": 2335967764.6855392, + "event": "autumn_budget_2024", + "flags": [ + "numerical", + "structural", + "inertness" + ], + "head_gbp": 0.0, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2027 + }, + { + "base_gbp": 2335967764.6855392, + "event": "autumn_budget_2024", + "flags": [ + "numerical", + "structural" + ], + "head_gbp": 0.0, + "key": "stamp_duty_land_tax", + "level": "head", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2027 + }, + { + "base_gbp": 2418086716.0288925, + "event": "autumn_budget_2024", + "flags": [ + "inertness" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:11650:068acce8b0873200", + "level": "source_row", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2028 + }, + { + "base_gbp": 2418086716.0288925, + "event": "autumn_budget_2024", + "flags": [ + "numerical", + "structural", + "inertness" + ], + "head_gbp": 0.0, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2028 + }, + { + "base_gbp": 2418086716.0288925, + "event": "autumn_budget_2024", + "flags": [ + "numerical", + "structural" + ], + "head_gbp": 0.0, + "key": "stamp_duty_land_tax", + "level": "head", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2028 + }, + { + "base_gbp": 2500617321.3659096, + "event": "autumn_budget_2024", + "flags": [ + "inertness" + ], + "head_gbp": null, + "key": "obr_pmd_nov2025:11651:9a7327566a66890c", + "level": "source_row", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2029 + }, + { + "base_gbp": 2500617321.3659096, + "event": "autumn_budget_2024", + "flags": [ + "numerical", + "structural", + "inertness" + ], + "head_gbp": 0.0, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2029 + }, + { + "base_gbp": 2500617321.3659096, + "event": "autumn_budget_2024", + "flags": [ + "numerical", + "structural" + ], + "head_gbp": 0.0, + "key": "stamp_duty_land_tax", + "level": "head", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2029 + }, + { + "base_gbp": 175460925.4459455, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 180672779.3509035, + "key": "obr_pmd_nov2025:22854:edbde987f66af165", + "level": "source_row", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2025 + }, + { + "base_gbp": 175460925.4459455, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 180672779.3509035, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2025 + }, + { + "base_gbp": 175460925.4459455, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 180672779.3509035, + "key": "winter_fuel_allowance", + "level": "head", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2025 + }, + { + "base_gbp": 184684987.91397357, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 190211145.96311736, + "key": "obr_pmd_nov2025:22855:dbc5baf2a54734dc", + "level": "source_row", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2026 + }, + { + "base_gbp": 184684987.91397357, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 190211145.96311736, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2026 + }, + { + "base_gbp": 184684987.91397357, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 190211145.96311736, + "key": "winter_fuel_allowance", + "level": "head", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2026 + }, + { + "base_gbp": 216206005.17124224, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 219651206.51188183, + "key": "obr_pmd_nov2025:22856:c29981419df6c6f6", + "level": "source_row", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2027 + }, + { + "base_gbp": 216206005.17124224, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 219651206.51188183, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2027 + }, + { + "base_gbp": 216206005.17124224, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 219651206.51188183, + "key": "winter_fuel_allowance", + "level": "head", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2027 + }, + { + "base_gbp": 217874757.0607648, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 222857307.28443575, + "key": "obr_pmd_nov2025:22857:92bf18d3bd843235", + "level": "source_row", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2028 + }, + { + "base_gbp": 217874757.0607648, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 222857307.28443575, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2028 + }, + { + "base_gbp": 217874757.0607648, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 222857307.28443575, + "key": "winter_fuel_allowance", + "level": "head", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2028 + }, + { + "base_gbp": 228829386.94062686, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 232920109.30445337, + "key": "obr_pmd_nov2025:22858:6050e74df71edc1c", + "level": "source_row", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2029 + }, + { + "base_gbp": 228829386.94062686, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 232920109.30445337, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2029 + }, + { + "base_gbp": 228829386.94062686, + "event": "autumn_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 232920109.30445337, + "key": "winter_fuel_allowance", + "level": "head", + "measure_key": "autumn_budget_2024__winter_fuel_means_test", + "year": 2029 + }, + { + "base_gbp": 483900104.6233177, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": 458380794.9795723, + "key": "obr_pmd_nov2025:12375:297fca1f0cf23095", + "level": "source_row", + "measure_key": "ab2025__high_value_council_tax_surcharge", + "year": 2028 + }, + { + "base_gbp": 483900104.6233177, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": 458380794.9795723, + "key": "total", + "level": "measure_year", + "measure_key": "ab2025__high_value_council_tax_surcharge", + "year": 2028 + }, + { + "base_gbp": 483900104.6233177, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": 458380794.9795723, + "key": "high_value_council_tax_surcharge", + "level": "head", + "measure_key": "ab2025__high_value_council_tax_surcharge", + "year": 2028 + }, + { + "base_gbp": 495749379.82161856, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": 469605186.562956, + "key": "obr_pmd_nov2025:12376:d658edd74795036b", + "level": "source_row", + "measure_key": "ab2025__high_value_council_tax_surcharge", + "year": 2029 + }, + { + "base_gbp": 495749379.82161856, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": 469605186.562956, + "key": "total", + "level": "measure_year", + "measure_key": "ab2025__high_value_council_tax_surcharge", + "year": 2029 + }, + { + "base_gbp": 495749379.82161856, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": 469605186.562956, + "key": "high_value_council_tax_surcharge", + "level": "head", + "measure_key": "ab2025__high_value_council_tax_surcharge", + "year": 2029 + }, + { + "base_gbp": 507938784.7696847, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": 481151760.1865325, + "key": "obr_pmd_nov2025:12377:ca5b9686e623d09c", + "level": "source_row", + "measure_key": "ab2025__high_value_council_tax_surcharge", + "year": 2030 + }, + { + "base_gbp": 507938784.7696847, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": 481151760.1865325, + "key": "total", + "level": "measure_year", + "measure_key": "ab2025__high_value_council_tax_surcharge", + "year": 2030 + }, + { + "base_gbp": 507938784.7696847, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": 481151760.1865325, + "key": "high_value_council_tax_surcharge", + "level": "head", + "measure_key": "ab2025__high_value_council_tax_surcharge", + "year": 2030 + }, + { + "base_gbp": -75417252.38853455, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": -76486965.76553345, + "key": "obr_pmd_nov2025:23918:cb97d1297e767678", + "level": "source_row", + "measure_key": "ab2025__personal_tax_thresholds_freeze_to_2031", + "year": 2028 + }, + { + "base_gbp": -75417252.38853455, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": -76486965.76553345, + "key": "universal_credit", + "level": "head", + "measure_key": "ab2025__personal_tax_thresholds_freeze_to_2031", + "year": 2028 + }, + { + "base_gbp": 8078999312.888916, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": 7979163787.683044, + "key": "obr_pmd_nov2025:12238:ed7d488b306dcf83", + "level": "source_row", + "measure_key": "ab2025__personal_tax_thresholds_freeze_to_2031", + "year": 2029 + }, + { + "base_gbp": -150448797.6953888, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": -152521040.89421082, + "key": "obr_pmd_nov2025:23919:d57980273e85240e", + "level": "source_row", + "measure_key": "ab2025__personal_tax_thresholds_freeze_to_2031", + "year": 2029 + }, + { + "base_gbp": 7928550515.193527, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": 7826642746.788834, + "key": "total", + "level": "measure_year", + "measure_key": "ab2025__personal_tax_thresholds_freeze_to_2031", + "year": 2029 + }, + { + "base_gbp": 8078999312.888916, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": 7979163787.683044, + "key": "income_tax", + "level": "head", + "measure_key": "ab2025__personal_tax_thresholds_freeze_to_2031", + "year": 2029 + }, + { + "base_gbp": -150448797.6953888, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": -152521040.89421082, + "key": "universal_credit", + "level": "head", + "measure_key": "ab2025__personal_tax_thresholds_freeze_to_2031", + "year": 2029 + }, + { + "base_gbp": -225537908.19476318, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": -233216948.8221283, + "key": "obr_pmd_nov2025:23920:07fdfe7c4df486ae", + "level": "source_row", + "measure_key": "ab2025__personal_tax_thresholds_freeze_to_2031", + "year": 2030 + }, + { + "base_gbp": -225537908.19476318, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": -233216948.8221283, + "key": "universal_credit", + "level": "head", + "measure_key": "ab2025__personal_tax_thresholds_freeze_to_2031", + "year": 2030 + }, + { + "base_gbp": 119185198.37719727, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": 69459033.33605957, + "key": "obr_pmd_nov2025:12340:56a6872c6215ff54", + "level": "source_row", + "measure_key": "ab2025__salary_sacrifice_pension_nics_cap_2000", + "year": 2029 + }, + { + "base_gbp": 2250469902.8025227, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": 2198876402.449813, + "key": "total", + "level": "measure_year", + "measure_key": "ab2025__salary_sacrifice_pension_nics_cap_2000", + "year": 2029 + }, + { + "base_gbp": 119185198.37719727, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": 69459033.33605957, + "key": "income_tax", + "level": "head", + "measure_key": "ab2025__salary_sacrifice_pension_nics_cap_2000", + "year": 2029 + }, + { + "base_gbp": 263855675.31918335, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": 259418290.7624817, + "key": "ni_class_1_employee", + "level": "head", + "measure_key": "ab2025__salary_sacrifice_pension_nics_cap_2000", + "year": 2029 + }, + { + "base_gbp": 213117200.6135254, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": 140749488.84204102, + "key": "obr_pmd_nov2025:12341:ce174c39c117892d", + "level": "source_row", + "measure_key": "ab2025__salary_sacrifice_pension_nics_cap_2000", + "year": 2030 + }, + { + "base_gbp": 2342273218.03615, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": 2265225600.1851068, + "key": "total", + "level": "measure_year", + "measure_key": "ab2025__salary_sacrifice_pension_nics_cap_2000", + "year": 2030 + }, + { + "base_gbp": 213117200.6135254, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": 140749488.84204102, + "key": "income_tax", + "level": "head", + "measure_key": "ab2025__salary_sacrifice_pension_nics_cap_2000", + "year": 2030 + }, + { + "base_gbp": 260886140.45806122, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": 253634719.6460724, + "key": "ni_class_1_employee", + "level": "head", + "measure_key": "ab2025__salary_sacrifice_pension_nics_cap_2000", + "year": 2030 + }, + { + "base_gbp": -2694647923.6472473, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": -2758924251.607239, + "key": "obr_pmd_nov2025:23526:7a52217d035becaf", + "level": "source_row", + "measure_key": "ab2025__uc_child_element_remove_two_child_limit", + "year": 2026 + }, + { + "base_gbp": -2694647923.6472473, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": -2758924251.607239, + "key": "total", + "level": "measure_year", + "measure_key": "ab2025__uc_child_element_remove_two_child_limit", + "year": 2026 + }, + { + "base_gbp": -2694647923.6472473, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": -2758924251.607239, + "key": "universal_credit", + "level": "head", + "measure_key": "ab2025__uc_child_element_remove_two_child_limit", + "year": 2026 + }, + { + "base_gbp": -3154822181.9229736, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": -3200094615.9770355, + "key": "obr_pmd_nov2025:23528:35553a766a3ad3d3", + "level": "source_row", + "measure_key": "ab2025__uc_child_element_remove_two_child_limit", + "year": 2028 + }, + { + "base_gbp": -3154822181.9229736, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": -3200094615.9770355, + "key": "total", + "level": "measure_year", + "measure_key": "ab2025__uc_child_element_remove_two_child_limit", + "year": 2028 + }, + { + "base_gbp": -3154822181.9229736, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": -3200094615.9770355, + "key": "universal_credit", + "level": "head", + "measure_key": "ab2025__uc_child_element_remove_two_child_limit", + "year": 2028 + }, + { + "base_gbp": -3259218800.0862427, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": -3297122995.833664, + "key": "obr_pmd_nov2025:23529:1e1a237caba88918", + "level": "source_row", + "measure_key": "ab2025__uc_child_element_remove_two_child_limit", + "year": 2029 + }, + { + "base_gbp": -3259218800.0862427, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": -3297122995.833664, + "key": "total", + "level": "measure_year", + "measure_key": "ab2025__uc_child_element_remove_two_child_limit", + "year": 2029 + }, + { + "base_gbp": -3259218800.0862427, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": -3297122995.833664, + "key": "universal_credit", + "level": "head", + "measure_key": "ab2025__uc_child_element_remove_two_child_limit", + "year": 2029 + }, + { + "base_gbp": -3488191222.989334, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": -3562340543.2262115, + "key": "obr_pmd_nov2025:23530:d1bc796d9f235c4f", + "level": "source_row", + "measure_key": "ab2025__uc_child_element_remove_two_child_limit", + "year": 2030 + }, + { + "base_gbp": -3488191222.989334, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": -3562340543.2262115, + "key": "total", + "level": "measure_year", + "measure_key": "ab2025__uc_child_element_remove_two_child_limit", + "year": 2030 + }, + { + "base_gbp": -3488191222.989334, + "event": "autumn_budget_2025", + "flags": [ + "numerical" + ], + "head_gbp": -3562340543.2262115, + "key": "universal_credit", + "level": "head", + "measure_key": "ab2025__uc_child_element_remove_two_child_limit", + "year": 2030 + }, + { + "base_gbp": 241090644.10290527, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 245497292.5802765, + "key": "obr_pmd_nov2025:21462:c775903b43caf723", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2024 + }, + { + "base_gbp": 241090644.10290527, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 245497292.5802765, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2024 + }, + { + "base_gbp": 266181343.4298706, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 270589535.8361969, + "key": "obr_pmd_nov2025:21463:fd9bc11d19465ecb", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2025 + }, + { + "base_gbp": 266181343.4298706, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 270589535.8361969, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2025 + }, + { + "base_gbp": 282034438.7785034, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 286625968.6225281, + "key": "obr_pmd_nov2025:21464:190052623cdf04c0", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2026 + }, + { + "base_gbp": 282034438.7785034, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 286625968.6225281, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2026 + }, + { + "base_gbp": 292445236.05249023, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 297236767.7370758, + "key": "obr_pmd_nov2025:21465:6ea19652f3e3ea6b", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2027 + }, + { + "base_gbp": 292445236.05249023, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 297236767.7370758, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2027 + }, + { + "base_gbp": 303440070.03996277, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 309358037.8637085, + "key": "obr_pmd_nov2025:21466:43674024394e1bc0", + "level": "source_row", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2028 + }, + { + "base_gbp": 303440070.03996277, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": 309358037.8637085, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__class_1_employee_nics_main_rate_cut_2p", + "year": 2028 + }, + { + "base_gbp": -1687110967.884717, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -1747185557.3423557, + "key": "obr_pmd_nov2025:21630:bf9643f47e057983", + "level": "source_row", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2024 + }, + { + "base_gbp": -1687110967.884717, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -1747185557.3423557, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2024 + }, + { + "base_gbp": -132222067.57011127, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -136503630.25988197, + "key": "housing_benefit", + "level": "head", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2024 + }, + { + "base_gbp": -1554888900.3146057, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -1610681927.0824738, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2024 + }, + { + "base_gbp": -1883075660.240798, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -1928346474.5677586, + "key": "obr_pmd_nov2025:21631:75b9415771e2e20c", + "level": "source_row", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2025 + }, + { + "base_gbp": -1883075660.240798, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -1928346474.5677586, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2025 + }, + { + "base_gbp": -145070938.0732565, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -147247363.7580204, + "key": "housing_benefit", + "level": "head", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2025 + }, + { + "base_gbp": -1738004722.1675415, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -1781099110.8097382, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2025 + }, + { + "base_gbp": -1917164100.9136887, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -1959163536.443016, + "key": "obr_pmd_nov2025:21632:353c5d3f922bfc89", + "level": "source_row", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2026 + }, + { + "base_gbp": -1917164100.9136887, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -1959163536.443016, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2026 + }, + { + "base_gbp": -1767232278.0137787, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -1807857532.622879, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2026 + }, + { + "base_gbp": -1979148274.9173164, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -2013656625.9414272, + "key": "obr_pmd_nov2025:21633:5ce95c1559b8bc39", + "level": "source_row", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2027 + }, + { + "base_gbp": -1979148274.9173164, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -2013656625.9414272, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2027 + }, + { + "base_gbp": -1815586682.2787323, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -1851647008.492157, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2027 + }, + { + "base_gbp": -2000162838.6384335, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -2029628073.5191765, + "key": "obr_pmd_nov2025:21634:ff94f5b54b89d1ee", + "level": "source_row", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2028 + }, + { + "base_gbp": -2000162838.6384335, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -2029628073.5191765, + "key": "total", + "level": "measure_year", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2028 + }, + { + "base_gbp": -169179610.90740013, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -167055019.7617302, + "key": "housing_benefit", + "level": "head", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2028 + }, + { + "base_gbp": -1830983227.7310333, + "event": "autumn_statement_2023", + "flags": [ + "numerical" + ], + "head_gbp": -1862573053.7574463, + "key": "universal_credit", + "level": "head", + "measure_key": "autumn_statement_2023__lha_reset_to_30th_percentile", + "year": 2028 + }, + { + "base_gbp": -3004426501.3733215, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -3074285340.8434753, + "key": "obr_pmd_nov2025:10232:aa24b205227c35bd", + "level": "source_row", + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2024 + }, + { + "base_gbp": -3004426501.3733215, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -3074285340.8434753, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2024 + }, + { + "base_gbp": -3004426501.3733215, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -3074285340.8434753, + "key": "income_tax", + "level": "head", + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2024 + }, + { + "base_gbp": -3540262930.7841797, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -3416190748.439331, + "key": "obr_pmd_nov2025:10234:7fb22cdd33e49932", + "level": "source_row", + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2026 + }, + { + "base_gbp": -3540262930.7841797, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -3416190748.439331, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2026 + }, + { + "base_gbp": -3540262930.7841797, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -3416190748.439331, + "key": "income_tax", + "level": "head", + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2026 + }, + { + "base_gbp": -3725202663.6395264, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -3505566866.5803223, + "key": "obr_pmd_nov2025:10235:fbb8ccda250b78e9", + "level": "source_row", + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2027 + }, + { + "base_gbp": -3725202663.6395264, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -3505566866.5803223, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2027 + }, + { + "base_gbp": -3725202663.6395264, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -3505566866.5803223, + "key": "income_tax", + "level": "head", + "measure_key": "spring_budget_2023__pension_annual_allowance_package", + "year": 2027 + }, + { + "base_gbp": -239031104.5414734, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -233460988.38542175, + "key": "obr_pmd_nov2025:20833:270c17d92e54dca4", + "level": "source_row", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2024 + }, + { + "base_gbp": -239031104.5414734, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -233460988.38542175, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2024 + }, + { + "base_gbp": -239031104.5414734, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -233460988.38542175, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2024 + }, + { + "base_gbp": -251689860.3839264, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -246211570.27645874, + "key": "obr_pmd_nov2025:20834:e0b34fcdbf83f61a", + "level": "source_row", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2025 + }, + { + "base_gbp": -251689860.3839264, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -246211570.27645874, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2025 + }, + { + "base_gbp": -251689860.3839264, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -246211570.27645874, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2025 + }, + { + "base_gbp": -255168633.24157715, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -249406786.5783844, + "key": "obr_pmd_nov2025:20835:1e730fccbdf4c015", + "level": "source_row", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2026 + }, + { + "base_gbp": -255168633.24157715, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -249406786.5783844, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2026 + }, + { + "base_gbp": -255168633.24157715, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -249406786.5783844, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2026 + }, + { + "base_gbp": -266037263.24046326, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -260633714.00463867, + "key": "obr_pmd_nov2025:20836:76beca3818ed1fe1", + "level": "source_row", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2027 + }, + { + "base_gbp": -266037263.24046326, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -260633714.00463867, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2027 + }, + { + "base_gbp": -266037263.24046326, + "event": "spring_budget_2023", + "flags": [ + "numerical" + ], + "head_gbp": -260633714.00463867, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2027 + }, + { + "base_gbp": 241090644.10290527, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 245497292.5802765, + "key": "obr_pmd_nov2025:22170:45627d3976bd492a", + "level": "source_row", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2024 + }, + { + "base_gbp": 241090644.10290527, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 245497292.5802765, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2024 + }, + { + "base_gbp": 266181343.4298706, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 270589535.8361969, + "key": "obr_pmd_nov2025:22171:93d8943dfe121528", + "level": "source_row", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2025 + }, + { + "base_gbp": 266181343.4298706, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 270589535.8361969, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2025 + }, + { + "base_gbp": 282034438.7785034, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 286625968.6225281, + "key": "obr_pmd_nov2025:22172:8c021f0cf80fe71b", + "level": "source_row", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2026 + }, + { + "base_gbp": 282034438.7785034, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 286625968.6225281, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2026 + }, + { + "base_gbp": 292445236.05249023, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 297236767.7370758, + "key": "obr_pmd_nov2025:22173:3beb0f8e7601b1e2", + "level": "source_row", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2027 + }, + { + "base_gbp": 292445236.05249023, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 297236767.7370758, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2027 + }, + { + "base_gbp": 303440070.03996277, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 309358037.8637085, + "key": "obr_pmd_nov2025:22174:b9088212717e5ab4", + "level": "source_row", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2028 + }, + { + "base_gbp": 303440070.03996277, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": 309358037.8637085, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_budget_2024__class_1_employee_nics_main_rate_cut_2pp", + "year": 2028 + }, + { + "base_gbp": -779631545.703949, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -796179226.5253601, + "key": "obr_pmd_nov2025:10987:0f254c98c34bc660", + "level": "source_row", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2024 + }, + { + "base_gbp": -336165102.39004517, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -346389876.7350273, + "key": "obr_pmd_nov2025:22206:49a0caec037600d2", + "level": "source_row", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2024 + }, + { + "base_gbp": -1115796648.0939941, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -1142569103.2603874, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2024 + }, + { + "base_gbp": -336165102.39004517, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -346389876.7350273, + "key": "child_benefit", + "level": "head", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2024 + }, + { + "base_gbp": -779631545.703949, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -796179226.5253601, + "key": "income_tax", + "level": "head", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2024 + }, + { + "base_gbp": -282526333.77780914, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -291545336.7139454, + "key": "obr_pmd_nov2025:22207:01b31afdbe23dae6", + "level": "source_row", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2025 + }, + { + "base_gbp": -282526333.77780914, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -291545336.7139454, + "key": "child_benefit", + "level": "head", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2025 + }, + { + "base_gbp": -1131591240.399292, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -1108097441.7645874, + "key": "obr_pmd_nov2025:10989:43638ad22300c9c3", + "level": "source_row", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2026 + }, + { + "base_gbp": -307810876.10124016, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -315976928.4119816, + "key": "obr_pmd_nov2025:22208:6c19f800730945b5", + "level": "source_row", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2026 + }, + { + "base_gbp": -1439402116.5005322, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -1424074370.176569, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2026 + }, + { + "base_gbp": -307810876.10124016, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -315976928.4119816, + "key": "child_benefit", + "level": "head", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2026 + }, + { + "base_gbp": -1131591240.399292, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -1108097441.7645874, + "key": "income_tax", + "level": "head", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2026 + }, + { + "base_gbp": -1274651222.2009888, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -1230493883.52063, + "key": "obr_pmd_nov2025:10990:e86af1dbeb4da64e", + "level": "source_row", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2027 + }, + { + "base_gbp": -323556121.7815895, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -329556758.2181053, + "key": "obr_pmd_nov2025:22209:71f606ed0f936afe", + "level": "source_row", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2027 + }, + { + "base_gbp": -1598207343.9825783, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -1560050641.7387352, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2027 + }, + { + "base_gbp": -323556121.7815895, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -329556758.2181053, + "key": "child_benefit", + "level": "head", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2027 + }, + { + "base_gbp": -1274651222.2009888, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -1230493883.52063, + "key": "income_tax", + "level": "head", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2027 + }, + { + "base_gbp": -1394705720.206482, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -1328781780.8405762, + "key": "obr_pmd_nov2025:10991:11a476b01436c0db", + "level": "source_row", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2028 + }, + { + "base_gbp": -363073039.4392204, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -369300783.7877197, + "key": "obr_pmd_nov2025:22210:0ea61ade0bedb0f5", + "level": "source_row", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2028 + }, + { + "base_gbp": -1757778759.6457024, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -1698082564.628296, + "key": "total", + "level": "measure_year", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2028 + }, + { + "base_gbp": -363073039.4392204, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -369300783.7877197, + "key": "child_benefit", + "level": "head", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2028 + }, + { + "base_gbp": -1394705720.206482, + "event": "spring_budget_2024", + "flags": [ + "numerical" + ], + "head_gbp": -1328781780.8405762, + "key": "income_tax", + "level": "head", + "measure_key": "spring_budget_2024__hicbc_threshold_and_taper", + "year": 2028 + }, + { + "base_gbp": 743178827.6944885, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": 716742439.4071198, + "key": "obr_pmd_nov2025:23317:1b983354102bb14d", + "level": "source_row", + "measure_key": "spring_statement_2025__uc_health_element_freeze_and_new_claimant_cut", + "year": 2026 + }, + { + "base_gbp": 743178827.6944885, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": 716742439.4071198, + "key": "total", + "level": "measure_year", + "measure_key": "spring_statement_2025__uc_health_element_freeze_and_new_claimant_cut", + "year": 2026 + }, + { + "base_gbp": 743178827.6944885, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": 716742439.4071198, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_statement_2025__uc_health_element_freeze_and_new_claimant_cut", + "year": 2026 + }, + { + "base_gbp": 893600229.4221191, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": 866066283.9185486, + "key": "obr_pmd_nov2025:23318:dcc529f309fec572", + "level": "source_row", + "measure_key": "spring_statement_2025__uc_health_element_freeze_and_new_claimant_cut", + "year": 2027 + }, + { + "base_gbp": 893600229.4221191, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": 866066283.9185486, + "key": "total", + "level": "measure_year", + "measure_key": "spring_statement_2025__uc_health_element_freeze_and_new_claimant_cut", + "year": 2027 + }, + { + "base_gbp": 893600229.4221191, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": 866066283.9185486, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_statement_2025__uc_health_element_freeze_and_new_claimant_cut", + "year": 2027 + }, + { + "base_gbp": 1053047649.6080627, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": 1026780086.9030914, + "key": "obr_pmd_nov2025:23319:c6ce8500db20415e", + "level": "source_row", + "measure_key": "spring_statement_2025__uc_health_element_freeze_and_new_claimant_cut", + "year": 2028 + }, + { + "base_gbp": 1053047649.6080627, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": 1026780086.9030914, + "key": "total", + "level": "measure_year", + "measure_key": "spring_statement_2025__uc_health_element_freeze_and_new_claimant_cut", + "year": 2028 + }, + { + "base_gbp": 1053047649.6080627, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": 1026780086.9030914, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_statement_2025__uc_health_element_freeze_and_new_claimant_cut", + "year": 2028 + }, + { + "base_gbp": 1297158615.1423492, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": 1270196015.4547424, + "key": "obr_pmd_nov2025:23320:4c41d84635ed82ad", + "level": "source_row", + "measure_key": "spring_statement_2025__uc_health_element_freeze_and_new_claimant_cut", + "year": 2029 + }, + { + "base_gbp": 1297158615.1423492, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": 1270196015.4547424, + "key": "total", + "level": "measure_year", + "measure_key": "spring_statement_2025__uc_health_element_freeze_and_new_claimant_cut", + "year": 2029 + }, + { + "base_gbp": 1297158615.1423492, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": 1270196015.4547424, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_statement_2025__uc_health_element_freeze_and_new_claimant_cut", + "year": 2029 + }, + { + "base_gbp": -732669693.5323181, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": -924233495.3892365, + "key": "obr_pmd_nov2025:23336:5d8fea1a1f3c86d6", + "level": "source_row", + "measure_key": "spring_statement_2025__uc_standard_allowance_above_inflation", + "year": 2027 + }, + { + "base_gbp": -732669693.5323181, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": -924233495.3892365, + "key": "total", + "level": "measure_year", + "measure_key": "spring_statement_2025__uc_standard_allowance_above_inflation", + "year": 2027 + }, + { + "base_gbp": -732669693.5323181, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": -924233495.3892365, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_statement_2025__uc_standard_allowance_above_inflation", + "year": 2027 + }, + { + "base_gbp": -746613549.421051, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": -1152562763.0171204, + "key": "obr_pmd_nov2025:23337:31f0f1693e3bba3c", + "level": "source_row", + "measure_key": "spring_statement_2025__uc_standard_allowance_above_inflation", + "year": 2028 + }, + { + "base_gbp": -746613549.421051, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": -1152562763.0171204, + "key": "total", + "level": "measure_year", + "measure_key": "spring_statement_2025__uc_standard_allowance_above_inflation", + "year": 2028 + }, + { + "base_gbp": -746613549.421051, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": -1152562763.0171204, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_statement_2025__uc_standard_allowance_above_inflation", + "year": 2028 + }, + { + "base_gbp": -753234184.9562225, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": -1362077410.9302979, + "key": "obr_pmd_nov2025:23338:27e34c93a163b4f0", + "level": "source_row", + "measure_key": "spring_statement_2025__uc_standard_allowance_above_inflation", + "year": 2029 + }, + { + "base_gbp": -753234184.9562225, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": -1362077410.9302979, + "key": "total", + "level": "measure_year", + "measure_key": "spring_statement_2025__uc_standard_allowance_above_inflation", + "year": 2029 + }, + { + "base_gbp": -753234184.9562225, + "event": "spring_statement_2025", + "flags": [ + "numerical" + ], + "head_gbp": -1362077410.9302979, + "key": "universal_credit", + "level": "head", + "measure_key": "spring_statement_2025__uc_standard_allowance_above_inflation", + "year": 2029 + } + ] + }, + "execution": { + "dataset_sha256": "40804ba2540458cb6f3c476d29d3bb4bc88d9ac6b401fa285ed6e4232a55aa52", + "packaged_bundle_manifest_sha256": "17bdafce1d10094969972d09aa324551a20adfaffaa749f0edba6667d7031ffd", + "packages": { + "policyengine": { + "record_sha256": "e70d98bf2f3032131e8e7ef152bb9a17ce5abcf3a515f65c1156b556dbc006c7", + "version": "6.7.0" + }, + "policyengine-core": { + "record_sha256": "53a92f2e67982a16faeba61b8b3d145d3ab9a50d298e92c512d61166ffe5d5d4", + "version": "3.33.0", + "wheel_sha256": "2e32d02a8910ab683a2b7600987904b351d058fc2bd5910a8ea6641438bcd789" + }, + "policyengine-uk": { + "record_sha256": "b6330bc40d5938301a56a5f33f331dee9322beb3ed6c2e9feefb5197b891b698", + "version": "2.134.7", + "wheel_sha256": "7e1ec3a0469f7f3ae1956ad3165b8200ade69cb6039b6fb3bb1d6ad6979af2a5" + } + }, + "platform": "Linux-6.17.0-1022-azure-x86_64-with-glibc2.39", + "python": "3.12.3", + "release_bundle": { + "bundle_id": "uk-6.7.0", + "certified_by": "policyengine.py bundle certification", + "certified_data_artifact_sha256": "40804ba2540458cb6f3c476d29d3bb4bc88d9ac6b401fa285ed6e4232a55aa52", + "certified_data_build_id": "policyengine-uk-data-1.58.2", + "compatibility_basis": "qualified_exact_pair", + "country_id": "uk", + "data_build_fingerprint": "sha256:c094d7006fdd81df220090284212f16bbffb15624643bc442c4bb3e1b1e5dac8", + "data_build_model_git_sha": "a43e8c5196c7e7c09b6ecb4924f034444dd83b8f", + "data_build_model_version": "2.122.2", + "data_package": "policyengine-uk-data", + "data_version": "1.58.2", + "default_dataset": "enhanced_frs_2024_25", + "default_dataset_uri": "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.2", + "model_package": "policyengine-uk", + "model_version": "2.134.7", + "policyengine_version": "6.7.0" + }, + "requirements_sha256": "8e796c1d3af7e4a2c5bc98877c76541c6cfbe158a94fc619c68586fb71bdb0b8", + "runner": "github-actions Linux X64" + }, + "integrity": { + "failures": [ + { + "code": "construction_inert", + "detail": "identical heads although CONSTRUCTION_PREMISES.json sets gov.hmrc.stamp_duty.residential.purchase.additional.rate[0].rate to 0.03 in the baseline world and 0.05 in the reform world (and 4 more paths)", + "event": "autumn_budget_2024", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2025 + }, + { + "code": "construction_inert", + "detail": "identical heads although CONSTRUCTION_PREMISES.json sets gov.hmrc.stamp_duty.residential.purchase.additional.rate[0].rate to 0.03 in the baseline world and 0.05 in the reform world (and 4 more paths)", + "event": "autumn_budget_2024", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2026 + }, + { + "code": "construction_inert", + "detail": "identical heads although CONSTRUCTION_PREMISES.json sets gov.hmrc.stamp_duty.residential.purchase.additional.rate[0].rate to 0.03 in the baseline world and 0.05 in the reform world (and 4 more paths)", + "event": "autumn_budget_2024", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2027 + }, + { + "code": "construction_inert", + "detail": "identical heads although CONSTRUCTION_PREMISES.json sets gov.hmrc.stamp_duty.residential.purchase.additional.rate[0].rate to 0.03 in the baseline world and 0.05 in the reform world (and 4 more paths)", + "event": "autumn_budget_2024", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2028 + }, + { + "code": "construction_inert", + "detail": "identical heads although CONSTRUCTION_PREMISES.json sets gov.hmrc.stamp_duty.residential.purchase.additional.rate[0].rate to 0.03 in the baseline world and 0.05 in the reform world (and 4 more paths)", + "event": "autumn_budget_2024", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2029 + } + ], + "status": "failed" + }, + "receipt_sha256": "27d64a4742b37bebba23cd122889a2affbba79a4b217a584d203ab26b6717b8a", + "rejection": null, + "request_id": "b989d471f90e7cca4ee62f987a43a7255c6b9180d81f6bc2155e0919f6928b77", + "results": { + "manifest_sha256": "623480aedacc2272ec1b21d090ddbc602d65576a26b467726fbe1a88500e6514", + "path": "requests/b989d471f90e7cca4ee62f987a43a7255c6b9180d81f6bc2155e0919f6928b77/results/", + "ref": "replay-state", + "repository": "PolicyEngine/policyengine-scorecard" + }, + "schema": "policyengine.uk-replay-receipt/v1", + "status": "failed", + "variant_id": "pe6.7.0_uk2.134.7_policyengine-uk-data-1.58.2", + "variant_identity_sha256": "2251990830b3186bc668e01f5cb1ee5e0c3ed422ea0eba53fb2d323e789b2846" +} diff --git a/tests/fixtures/replay_exceptions/uk2.134.7_budget_inputs.json b/tests/fixtures/replay_exceptions/uk2.134.7_budget_inputs.json new file mode 100644 index 00000000..302e1123 --- /dev/null +++ b/tests/fixtures/replay_exceptions/uk2.134.7_budget_inputs.json @@ -0,0 +1,179 @@ +{ + "accepted_gaps": {}, + "dataset_columns": 180, + "missing_inputs": [ + "additional_residential_property_purchased", + "adult_dependants_grant_has_other_adult_dependant", + "adult_dependants_grant_other_adult_income", + "adult_dependants_grant_receives_postgraduate_loan", + "armed_forces_independence_payment", + "base_net_income", + "baseline_child_benefit_entitlement", + "baseline_income_support_entitlement", + "baseline_pension_credit_entitlement", + "beer_draught_share", + "beer_litres", + "benunit_reported_capital", + "blind_persons_allowance", + "bursary_fund_16_to_19_in_care_or_care_leaver", + "bursary_fund_16_to_19_participation_costs", + "bursary_fund_16_to_19_self_supporting", + "bus_fare_spending_reported", + "bus_in_london_trips", + "bus_pass_eligible", + "capital_allowances", + "capital_gains_badr", + "capital_gains_before_response", + "capital_gains_carried_interest", + "capital_gains_residential_property", + "car_co2_emissions", + "car_engine_size", + "car_first_registration_date", + "car_fuel_type", + "car_is_ved_exempt", + "car_list_price", + "care_hours", + "cash_isa", + "childcare_grant_child_has_special_educational_needs", + "childcare_grant_full_time_student", + "childcare_grant_provider_is_nanny", + "childcare_grant_receives_nhs_childcare_support", + "childcare_grant_receives_postgraduate_loan", + "cider_draught_share", + "cider_litres", + "cigarettes_per_week", + "cigars_grams_per_week", + "consumer_debt", + "covenanted_payments", + "cumulative_non_residential_rent", + "cumulative_residential_rent", + "deficiency_relief", + "disabled_students_allowance_receives_equivalent_support", + "ebr_energy_bills_credit_eligible", + "employee_pension_contributions_reported", + "employment_expenses", + "employment_income_before_lsr", + "families", + "free_school_milk", + "hand_rolling_tobacco_grams_per_week", + "heated_tobacco_grams_per_week", + "households", + "housing_benefit_board_and_attendance_determination", + "in_FE", + "in_HE", + "in_original_frs", + "in_rural_fuel_duty_relief_area", + "in_specified_or_temporary_accommodation", + "individual_savings_account_interest_income", + "is_about_to_start_paid_work", + "is_apprentice", + "is_blind", + "is_engaged_in_owned_company_trade", + "is_looked_after_by_local_authority", + "is_newly_self_employed", + "is_shared_accommodation", + "liable_for_share_of_household_rent", + "lifetime_isa_balance", + "local_authority", + "loss_relief", + "lpg_kg", + "married_couples_allowance", + "maternity_allowance_reported", + "max_free_entitlement_hours_used", + "meals_included_in_rent", + "mortgage_debt", + "natural_gas_kg", + "ni_class_3", + "non_residential_property_purchased", + "non_residential_rent", + "num_bedrooms", + "original_weight", + "other_deductions", + "other_fermented_draught_share", + "other_fermented_litres", + "other_investment_income", + "other_local_bus_trips", + "other_tax_credits", + "other_tobacco_grams_per_week", + "owned_company_capital", + "owned_company_carries_on_property_business", + "owned_company_carries_on_trade", + "owned_company_holding_value", + "owned_company_income_share", + "owned_company_intermediary_earnings_chapter", + "owned_company_intermediary_earnings_from_main_employment", + "owned_company_is_main_employment", + "owned_company_trade_assets", + "pension_income", + "people", + "person_benunit_role", + "person_household_role", + "private_pension_wealth", + "property_finance_costs", + "property_finance_costs_brought_forward", + "property_rental_income", + "raw_person_weight", + "receives_limited_capability_for_work_credits", + "rent_paid_as_boarder", + "rent_paid_as_lodger", + "role", + "scottish_funded_early_learning_childcare_start_date_reached", + "self_employment_gross_receipts", + "sparkling_cider_draught_share", + "sparkling_cider_litres", + "spi_imputed", + "spirits_draught_share", + "spirits_litres", + "stands_as_sole_owner_or_partner_of_company", + "statutory_paternity_pay", + "stocks_and_shares_isa", + "sublet_income", + "tax_free_childcare_meets_uk_connection", + "tax_free_childcare_on_adoption_leave", + "tax_free_childcare_on_carers_leave", + "tax_free_childcare_on_qualifying_leave", + "tax_free_childcare_on_shared_parental_leave", + "tax_free_childcare_self_employment_start_up_period", + "tax_free_childcare_uses_qualifying_provider", + "tax_reported", + "trading_loss", + "travel_grant_abroad_placement", + "travel_grant_clinical_placement", + "travel_grant_eligible_expenses", + "travel_grant_receives_means_tested_nhs_bursary", + "uc_has_offer_of_paid_work_starting_by_end_of_next_assessment_period", + "uc_is_adopter_in_first_year", + "uc_is_foster_parent_or_new_friend_or_family_carer", + "uc_is_in_pregnancy_or_post_confinement_period", + "uc_is_in_startup_period", + "uc_is_responsible_foster_parent_of_child_under_one", + "uc_is_student_with_no_work_related_requirements", + "uc_is_temporarily_absent_from_claimant_household", + "uc_migrated", + "uc_receives_new_claimant_health_element", + "unused_pension_annual_allowance", + "uses_energy_prepayment_meter", + "wine_draught_share", + "wine_litres", + "working_tax_credit_run_on", + "would_claim_adult_dependants_grant", + "would_claim_bursary_fund_16_to_19", + "would_claim_care_to_learn", + "would_claim_carers_allowance", + "would_claim_childcare_grant", + "would_claim_disabled_students_allowance", + "would_claim_parents_learning_allowance", + "would_claim_travel_grant", + "would_claim_uc_at_legacy_closure", + "would_claim_uc_childcare" + ], + "model_input_variables": 303, + "newly_missing_inputs": [ + "additional_residential_property_purchased", + "loss_relief", + "non_residential_property_purchased", + "unused_pension_annual_allowance" + ], + "previous_source": "computed with policyengine 6.7.0", + "unusable_critical": [] +} diff --git a/tests/fixtures/replay_exceptions/uk2.134.7_budget_inputs.result.json b/tests/fixtures/replay_exceptions/uk2.134.7_budget_inputs.result.json new file mode 100644 index 00000000..626f41eb --- /dev/null +++ b/tests/fixtures/replay_exceptions/uk2.134.7_budget_inputs.result.json @@ -0,0 +1,16 @@ +{ + "details": { + "missing_inputs": 164, + "newly_missing_inputs": [ + "additional_residential_property_purchased", + "loss_relief", + "non_residential_property_purchased", + "unused_pension_annual_allowance" + ], + "unusable_critical": [] + }, + "evidence_sha256": "f47e6dd61e6e2274faebbfa5be05eedbbf9969358cc5f2c629f4c52c5a979dd7", + "gate_id": "budget_inputs", + "status": "needs_review", + "summary": "4 model inputs are newly missing from the data (additional_residential_property_purchased, loss_relief, non_residential_property_purchased, unused_pension_annual_allowance)" +} diff --git a/tests/fixtures/replay_exceptions/uk2.134.7_qualification_receipt.json b/tests/fixtures/replay_exceptions/uk2.134.7_qualification_receipt.json new file mode 100644 index 00000000..35ca7ccc --- /dev/null +++ b/tests/fixtures/replay_exceptions/uk2.134.7_qualification_receipt.json @@ -0,0 +1,334 @@ +{ + "completed_at": "2026-10-17T00:41:01Z", + "gates": [ + { + "details": {}, + "evidence_sha256": "b0e2d172f030a45cf06a5dcc6ec8d82e75f48cb1719fdfcfe2e41fa9c7978625", + "gate_id": "installed_versions", + "status": "pass", + "summary": "policyengine 6.7.0, policyengine-uk 2.134.7 and policyengine-core 3.33.0 are installed and their requirements hold." + }, + { + "details": {}, + "evidence_sha256": "6403ded8cca3e9e63fd8e2ea3cc502ea046b402fb9aab16ba1e3f33795a0d6d8", + "gate_id": "wheel_hashes", + "status": "pass", + "summary": "The pinned PyPI wheels match their RECORDs and the installed files (policyengine-uk: 2751 wheel members, 2751 installed files, policyengine-core: 199 wheel members, 199 installed files)." + }, + { + "details": { + "publisher_claim_covers_core": false, + "publisher_claim_covers_model": false + }, + "evidence_sha256": "6264064dc0c3d3adeeddfacefc5bb2b9917f748bf61a3cf7222bfbfe85948f46", + "gate_id": "core_compatibility", + "status": "pass", + "summary": "policyengine-core 3.33.0 satisfies policyengine-uk>=3.32.27; certification will record an exact qualified pair, since the publisher's claims do not cover both." + }, + { + "details": {}, + "evidence_sha256": "9c0503f44f41642267643596582cd7286a1424b71cb7b2043ec0ee86d49fd4ae", + "gate_id": "data_access", + "status": "pass", + "summary": "Read the release manifest and 7 artifacts with the configured credential; anonymous access returned HTTP 401." + }, + { + "details": {}, + "evidence_sha256": "83546621dd62b8a2e4ce942e4a9a6dec5b52555a5be87d02ce63cef70cca02cf", + "gate_id": "dataset_bytes", + "status": "pass", + "summary": "enhanced_frs_2024_25 is the pinned 128,414,344 bytes with the UK single-year layout." + }, + { + "details": { + "outside_projection": [], + "supported_years": [ + 2024, + 2025, + 2026, + 2027, + 2028, + 2029, + 2030, + 2031, + 2032 + ] + }, + "evidence_sha256": "742a9ab0571fefd55495a9a09ac0f7180df43076a567769eae1a24ac5434c8e6", + "gate_id": "supported_years", + "status": "pass", + "summary": "Supports 2024–2032, covering the required and horizon years." + }, + { + "details": { + "missing_inputs": 164, + "newly_missing_inputs": [ + "additional_residential_property_purchased", + "loss_relief", + "non_residential_property_purchased", + "unused_pension_annual_allowance" + ], + "unusable_critical": [] + }, + "evidence_sha256": "f47e6dd61e6e2274faebbfa5be05eedbbf9969358cc5f2c629f4c52c5a979dd7", + "gate_id": "budget_inputs", + "status": "needs_review", + "summary": "4 model inputs are newly missing from the data (additional_residential_property_purchased, loss_relief, non_residential_property_purchased, unused_pension_annual_allowance)" + }, + { + "details": { + "contract_id": "policyengine-uk/budget-reform-contract", + "contract_version": "1.1.0", + "counts": { + "error": 15, + "fail": 1, + "pass": 722, + "pending": 23, + "unsupported": 62 + }, + "pending": [ + "case extended_freeze@country.reform_dict", + "case extended_freeze@country.reform_dict", + "case extended_freeze@country.scenario_parameter_changes", + "case extended_freeze@country.scenario_parameter_changes", + "case extended_freeze@country.reform_from_dict", + "case extended_freeze@country.reform_from_dict", + "case cgt_default_response@country.reform_dict", + "case cgt_default_response@country.reform_dict", + "case cgt_default_response@country.reform_dict", + "case cgt_default_response@country.reform_dict", + "case cgt_default_response@country.reform_dict", + "case cgt_default_response@country.scenario_parameter_changes", + "case cgt_default_response@country.scenario_parameter_changes", + "case cgt_default_response@country.scenario_parameter_changes", + "case cgt_default_response@country.scenario_parameter_changes", + "case cgt_default_response@country.scenario_parameter_changes", + "case cgt_default_response@country.reform_from_dict", + "case cgt_default_response@country.reform_from_dict", + "case cgt_default_response@country.reform_from_dict", + "case cgt_default_response@country.reform_from_dict", + "case cgt_default_response@country.reform_from_dict", + "case cgt_default_response@wrapper.policy", + "case cgt_default_response@wrapper.policy" + ], + "problems": [], + "report_sha256": "c95f2a933869e69e8d2594c0d914ffaa1a306a1fef8a9b9d7afbe06f11e56037", + "unsupported": [ + "case april_rate_change@wrapper.policy", + "case april_rate_change@wrapper.policy", + "case april_rate_change@wrapper.policy", + "case april_rate_change@api.sim_api", + "case finite_rate_change@wrapper.policy", + "case finite_rate_change@wrapper.policy", + "case finite_rate_change@api.sim_api", + "case extended_freeze@wrapper.policy", + "case extended_freeze@wrapper.policy", + "case extended_freeze@wrapper.policy", + "case extended_freeze@wrapper.policy", + "case extended_freeze@wrapper.policy", + "case extended_freeze@wrapper.policy", + "case extended_freeze@wrapper.policy", + "case extended_freeze@wrapper.policy", + "case extended_freeze@wrapper.policy", + "case extended_freeze@wrapper.policy", + "case extended_freeze@wrapper.policy", + "case extended_freeze@wrapper.policy", + "case extended_freeze@wrapper.policy", + "case extended_freeze@wrapper.policy", + "case extended_freeze@wrapper.policy", + "case extended_freeze@wrapper.policy", + "case extended_freeze@wrapper.policy", + "case extended_freeze@wrapper.policy", + "case extended_freeze@wrapper.policy", + "case extended_freeze@wrapper.policy", + "case extended_freeze@wrapper.policy", + "case extended_freeze@api.sim_api", + "case april_benefit_uprating@api.sim_api", + "case bounded_change_on_uprated_benefit@api.sim_api", + "case uc_award_follows_the_standard_allowance@api.sim_api", + "case new_state_pension_rate@api.sim_api", + "case mid_year_blended_rate@wrapper.policy", + "case mid_year_blended_rate@wrapper.policy", + "case mid_year_blended_rate@wrapper.policy", + "case mid_year_blended_rate@wrapper.policy", + "case mid_year_blended_rate@wrapper.policy", + "case mid_year_blended_rate@wrapper.policy", + "case mid_year_blended_rate@api.sim_api", + "case cgt_retention_response@wrapper.policy", + "case cgt_retention_response@wrapper.policy", + "case cgt_retention_response@wrapper.policy", + "case cgt_retention_response@wrapper.policy", + "case cgt_retention_response@api.sim_api", + "case cgt_default_response@wrapper.policy", + "case cgt_default_response@wrapper.policy", + "case cgt_default_response@wrapper.policy", + "case cgt_default_response@api.sim_api", + "case uc_rebalancing_reform_after_data_load@api.sim_api", + "case structural_switch@api.sim_api", + "case baseline_vat_is_current_law@wrapper.policy", + "case baseline_vat_is_current_law@api.sim_api", + "case modifier_then_parameter_changes@wrapper.policy", + "case modifier_then_parameter_changes@wrapper.policy", + "case modifier_then_parameter_changes@wrapper.policy", + "case modifier_then_parameter_changes@wrapper.policy", + "case parameter_changes_then_modifier@wrapper.policy", + "case parameter_changes_then_modifier@wrapper.policy", + "case parameter_changes_then_modifier@wrapper.policy", + "case parameter_changes_then_modifier@wrapper.policy", + "case unknown_path@api.sim_api" + ] + }, + "evidence_sha256": "c95f2a933869e69e8d2594c0d914ffaa1a306a1fef8a9b9d7afbe06f11e56037", + "gate_id": "reform_contract", + "status": "fail", + "summary": "16 contract row(s) failed: case april_rate_change@wrapper.household, case finite_rate_change@wrapper.household, case extended_freeze@wrapper.household, case april_benefit_uprating@wrapper.household, case bounded_change_on_uprated_benefit@wrapper.household, case uc_award_follows_the_standard_allowance@wrapper.household, case new_state_pension_rate@wrapper.household, case mid_year_blended_rate@wrapper.household" + }, + { + "details": { + "data_only": "needs_review", + "model_only": "needs_review" + }, + "evidence_sha256": "4cf32de2209d526e05bf3b0d75d277d21e69fdeb1e44be493e764d08cc59a064", + "gate_id": "metrics_review", + "status": "needs_review", + "summary": "model_only: 109 metric move(s) exceed their review bounds; data_only: 106 metric move(s) exceed their review bounds" + }, + { + "details": {}, + "evidence_sha256": "b47e4f5b97c2917bff418ecc267e1c53d9533d5a400797f2519b368d3759eea2", + "gate_id": "reform_responsiveness", + "status": "pass", + "summary": "All 10 standard reforms ran and moved their targets by at least their minimums in every year from 2025-26 to 2031-32." + }, + { + "details": { + "integrity": "failed", + "integrity_failures": [ + { + "code": "construction_inert", + "detail": "identical heads although CONSTRUCTION_PREMISES.json sets gov.hmrc.stamp_duty.residential.purchase.additional.rate[0].rate to 0.03 in the baseline world and 0.05 in the reform world (and 4 more paths)", + "event": "autumn_budget_2024", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2025 + }, + { + "code": "construction_inert", + "detail": "identical heads although CONSTRUCTION_PREMISES.json sets gov.hmrc.stamp_duty.residential.purchase.additional.rate[0].rate to 0.03 in the baseline world and 0.05 in the reform world (and 4 more paths)", + "event": "autumn_budget_2024", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2026 + }, + { + "code": "construction_inert", + "detail": "identical heads although CONSTRUCTION_PREMISES.json sets gov.hmrc.stamp_duty.residential.purchase.additional.rate[0].rate to 0.03 in the baseline world and 0.05 in the reform world (and 4 more paths)", + "event": "autumn_budget_2024", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2027 + }, + { + "code": "construction_inert", + "detail": "identical heads although CONSTRUCTION_PREMISES.json sets gov.hmrc.stamp_duty.residential.purchase.additional.rate[0].rate to 0.03 in the baseline world and 0.05 in the reform world (and 4 more paths)", + "event": "autumn_budget_2024", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2028 + }, + { + "code": "construction_inert", + "detail": "identical heads although CONSTRUCTION_PREMISES.json sets gov.hmrc.stamp_duty.residential.purchase.additional.rate[0].rate to 0.03 in the baseline world and 0.05 in the reform world (and 4 more paths)", + "event": "autumn_budget_2024", + "measure_key": "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp", + "year": 2029 + } + ], + "known_gaps": [], + "policy": "d1344-replay-regressions-v1", + "policy_sha256": "65ecaf44289624b6aae001068099250e789424854ed2b29ead86b71845a7dfa0", + "receipt_commit": "16439a6eb8411b013a48bddab0cd802c2433f04b", + "receipt_sha256": "27d64a4742b37bebba23cd122889a2affbba79a4b217a584d203ab26b6717b8a", + "regressions": [ + { + "kind": "failed", + "why": "Replay integrity failed: construction_inert" + } + ], + "request_id": "b989d471f90e7cca4ee62f987a43a7255c6b9180d81f6bc2155e0919f6928b77", + "review_items": [], + "status": "failed" + }, + "gate_id": "replay", + "status": "fail", + "summary": "Replay integrity failed: construction_inert" + } + ], + "harness": { + "git_commit": "7832b467fa845247e34c104187bdb096e2421101", + "packaged_bundle_manifest_sha256": "17bdafce1d10094969972d09aa324551a20adfaffaa749f0edba6667d7031ffd", + "policyengine_version": "6.7.0" + }, + "issued_by": { + "repository": "PolicyEngine/policyengine.py", + "run_attempt": 1, + "run_url": "https://github.com/PolicyEngine/policyengine.py/actions/runs/38976238397", + "workflow_run_id": 38976238397 + }, + "pair": { + "core": { + "name": "policyengine-core", + "version": "3.33.0", + "wheel_sha256": "2e32d02a8910ab683a2b7600987904b351d058fc2bd5910a8ea6641438bcd789" + }, + "country_id": "uk", + "data": { + "build_id": "policyengine-uk-data-1.58.2", + "data_package": "policyengine-uk-data", + "dataset": "enhanced_frs_2024_25", + "dataset_path": "enhanced_frs_2024_25.h5", + "dataset_revision": "1.58.2", + "dataset_sha256": "40804ba2540458cb6f3c476d29d3bb4bc88d9ac6b401fa285ed6e4232a55aa52", + "release_manifest_path": "releases/1.58.2/release_manifest.json", + "release_manifest_revision": "cd0856d995eeab2757a987ac2452d3cacce839ee", + "release_manifest_sha256": "6867e8a7bd8be6110816527191b2348f712c4ba2502886696142506bb59fbbff", + "release_version": "1.58.2", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "size_bytes": 128414344 + }, + "model": { + "name": "policyengine-uk", + "version": "2.134.7", + "wheel_sha256": "7e1ec3a0469f7f3ae1956ad3165b8200ade69cb6039b6fb3bb1d6ad6979af2a5" + } + }, + "pair_id": "fc9046aa4c796807e1efd3beb97a1f620fec5d6c4d7a8cff4860e43de7e62cea", + "previous_certificate": { + "bundle_version": "6.7.0", + "certificate_sha256": "565727f064cf6afb44782b69e4ba69bf872257828e6fa19e1edda59d2a876cae", + "core_version": "3.33.0", + "data_release_version": "1.58.1", + "dataset_name": "enhanced_frs_2024_25", + "dataset_reference": { + "path": "enhanced_frs_2024_25.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "1.58.1", + "sha256": "056e80fd8e9151d78f432a25cde863340baee81304c6216a37fb575ce8563784" + }, + "dataset_sha256": "056e80fd8e9151d78f432a25cde863340baee81304c6216a37fb575ce8563784", + "model_version": "2.130.1", + "model_wheel_sha256": "f8b1b8b917c9297ff66937811dbebd9a62d3b1796ce894c326e98051cac4c873", + "release_manifest_reference": { + "path": "releases/1.58.1/release_manifest.json", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "89be6b73fc1e6eeba6dae2f9400676d8510bcd51" + } + }, + "receipt_type": "policyengine.candidate_qualification_receipt", + "replay": { + "receipt_commit": "16439a6eb8411b013a48bddab0cd802c2433f04b", + "receipt_sha256": "27d64a4742b37bebba23cd122889a2affbba79a4b217a584d203ab26b6717b8a", + "request_id": "b989d471f90e7cca4ee62f987a43a7255c6b9180d81f6bc2155e0919f6928b77" + }, + "schema_version": 1, + "verdict": "rejected" +} diff --git a/tests/test_qualification_contracts.py b/tests/test_qualification_contracts.py index a81330bd..913de1fd 100644 --- a/tests/test_qualification_contracts.py +++ b/tests/test_qualification_contracts.py @@ -13,7 +13,7 @@ vendored JSON Schemas and reproduce its golden digests (differential); - a replay receipt counts only when bound to this request and these artifacts; -- the replay policy (d1344-replay-regressions-v1) fails exactly the +- the replay policy (d1344-replay-regressions-v2) fails exactly the regressions it names, sends every other flagged row to review, and is monotone: adding a flagged row never makes the verdict better. """ diff --git a/tests/test_replay_exception_coherence_round3.py b/tests/test_replay_exception_coherence_round3.py new file mode 100644 index 00000000..b159addb --- /dev/null +++ b/tests/test_replay_exception_coherence_round3.py @@ -0,0 +1,141 @@ +"""Independent review regressions reject incoherence at every reported level.""" + +from __future__ import annotations + +import copy + +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine.provenance import uk_qualification +from policyengine.provenance.qualification import ( + ReplayReceipt, + evaluate_replay_receipt, + qualification_verdict, +) +from policyengine.provenance.qualification_ledger import read_json, write_json +from tests.qualification_fixtures import ISSUED, gates +from tests.test_qualification_contracts import _base_event, _sealed, _status +from tests.test_replay_exception_finalize_review import _stage_real_failed_attempt +from tests.test_replay_exceptions import ( + EVENT, + MEASURE, + TEST_ATTEMPT_ID, + _failed_receipt, + _judge, + _probe, + _real_budget, + _real_receipt, +) + + +@pytest.mark.parametrize("level", ["measure_year", "source_row", "head"]) +@pytest.mark.parametrize("effect", [1_000_000_000, -1_000_000_000]) +@pytest.mark.parametrize("lists_inert", [False, True]) +@pytest.mark.parametrize("base_known_gap", [False, True]) +def test_every_receipt_row_level_rejects_nonzero_excepted_effect( + level, effect, lists_inert, base_known_gap +): + document = _failed_receipt() + row = copy.deepcopy(document["diff"]["flagged_rows"][0]) + row.update(level=level, key="income_tax" if level == "head" else "row-1") + row["head_gbp"] = effect + document["diff"]["flagged_rows"].append(row) + for flag in row["flags"]: + document["diff"]["flag_counts"][flag] += 1 + if not lists_inert: + document["corpus"]["events"][EVENT]["inert_measure_years"] = [] + document["corpus"]["events"][EVENT]["full_supported_grid_complete"] = True + if base_known_gap: + document["diff"]["base_events"] = {EVENT: _base_event(inert=[(MEASURE, 2026)])} + replay = _judge(_sealed(document)) + assert replay.status == "fail", replay.canonical_payload() + + +def test_finalizer_rejects_nonzero_fiscal_head_in_excepted_real_measure(tmp_path): + attempt, _, _ = _stage_real_failed_attempt(tmp_path) + document = read_json(attempt / "replay/receipt.json") + row = next( + row + for row in document["diff"]["flagged_rows"] + if row["level"] == "head" + and row["measure_key"] + == "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp" + and row["year"] == 2026 + ) + row["head_gbp"] = 1_000_000_000 + document = _sealed(document) + write_json(attempt / "replay/receipt.json", document) + write_json( + attempt / "replay/status.json", + _status(document["request_id"], document).document(), + ) + finalized = uk_qualification.finalize_attempt( + attempt, + tmp_path / "ledger", + issued_by=ISSUED, + completed_at="2026-10-17T10:00:00Z", + ) + assert finalized.verdict == "rejected", finalized.canonical_payload() + + +@pytest.mark.parametrize("level", ["measure_year", "source_row", "head"]) +@pytest.mark.parametrize("effect", [1_000_000_000, -1_000_000_000]) +def test_archived_real_receipt_rejects_nonzero_excepted_effect_at_every_level( + level, effect +): + request, document = _real_receipt() + row = next( + row + for row in document["diff"]["flagged_rows"] + if row["level"] == level + and row["measure_key"] + == "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp" + ) + row["head_gbp"] = effect + document = _sealed(document) + budget_result, budget_evidence = _real_budget() + probe_result, probe_evidence = _probe(request=request) + replay = evaluate_replay_receipt( + request, + _status(request.request_id, document), + ReplayReceipt.from_document(document), + budget_inputs_result=budget_result, + budget_inputs_evidence=budget_evidence, + probe_result=probe_result, + probe_evidence=probe_evidence, + attempt_id=TEST_ATTEMPT_ID, + ) + assert replay.status == "fail", replay.canonical_payload() + + +@settings(max_examples=75, deadline=None) +@given( + level=st.sampled_from(["head", "source_row", "measure_year"]), + effect=st.one_of( + st.floats(min_value=1e-12, max_value=1e15, allow_nan=False), + st.floats(min_value=-1e15, max_value=-1e-12, allow_nan=False), + ), + lists_inert=st.booleans(), + base_known_gap=st.booleans(), +) +def test_any_nonzero_reported_level_forbids_exception_and_qualification( + level, effect, lists_inert, base_known_gap +): + document = _failed_receipt() + row = copy.deepcopy(document["diff"]["flagged_rows"][0]) + row.update(level=level, key="income_tax" if level == "head" else "source-1") + row["head_gbp"] = effect + document["diff"]["flagged_rows"].append(row) + for flag in row["flags"]: + document["diff"]["flag_counts"][flag] += 1 + if not lists_inert: + document["corpus"]["events"][EVENT]["inert_measure_years"] = [] + document["corpus"]["events"][EVENT]["full_supported_grid_complete"] = True + if base_known_gap: + document["diff"]["base_events"] = {EVENT: _base_event(inert=[(MEASURE, 2026)])} + replay = _judge(_sealed(document)) + assert replay.status == "fail" + assert replay.details["applied_exceptions"] == [] + assert qualification_verdict(gates(replay=replay.status)) == "rejected" diff --git a/tests/test_replay_exception_finalize_review.py b/tests/test_replay_exception_finalize_review.py new file mode 100644 index 00000000..ed9ce510 --- /dev/null +++ b/tests/test_replay_exception_finalize_review.py @@ -0,0 +1,319 @@ +"""Independent adversarial review probes for replay exception finalization. + +The independent review's contradiction and stale-evidence reproducers now +assert fail-closed behavior for coherence, lineage and positive-input probes. +""" + +from __future__ import annotations + +import copy + +import pandas as pd +import pytest + +from policyengine.provenance import uk_candidate, uk_qualification +from policyengine.provenance.qualification import GateResult +from policyengine.provenance.qualification_ledger import read_json, write_json +from policyengine.provenance.receipt import content_sha256 +from tests.qualification_fixtures import HARNESS, ISSUED, gates, make_pair +from tests.test_qualification_contracts import _sealed, _status +from tests.test_replay_exceptions import ( + EVENT, + INPUT, + TEST_ATTEMPT_ID, + _failed_receipt, + _judge, + _legacy_result, + _probe, + _real_budget, + _real_receipt, +) + + +@pytest.mark.parametrize("head_gbp", [1_000_000_000, -1_000_000_000]) +@pytest.mark.parametrize("lists_inert", [False, True]) +def test_review_probe_nonzero_effect_with_construction_inert_failure( + head_gbp, lists_inert +): + """Identical construction worlds cannot produce a nonzero candidate effect.""" + + document = _failed_receipt( + rows=[ + { + "flags": ["structural", "inertness"], + "base_gbp": 2_500_000_000, + "head_gbp": head_gbp, + } + ] + ) + if not lists_inert: + document["corpus"]["events"][EVENT]["inert_measure_years"] = [] + document["corpus"]["events"][EVENT]["full_supported_grid_complete"] = True + document = _sealed(document) + before = _legacy_result(document) + after = _judge(document) + assert before.status == "fail" + assert after.status == "fail" + assert after.details["regressions"] + assert after.details["applied_exceptions"] == [] + + +@pytest.mark.parametrize("level", ["measure_year", "source_row"]) +def test_review_probe_production_registry_rejects_nonzero_inert_effect(level): + request, document = _real_receipt() + row = next( + row + for row in document["diff"]["flagged_rows"] + if row["level"] == level + and row["measure_key"] + == "autumn_budget_2024__sdlt_additional_dwelling_surcharge_2pp" + ) + row["head_gbp"] = 1_000_000_000 + document = _sealed(document) + budget_result, budget_evidence = _real_budget() + probe_result, probe_evidence = _probe(request=request) + actual = uk_qualification.evaluate_replay_receipt( + request, + _status(request.request_id, document), + uk_qualification.ReplayReceipt.from_document(document), + budget_inputs_result=budget_result, + budget_inputs_evidence=budget_evidence, + probe_result=probe_result, + probe_evidence=probe_evidence, + attempt_id=TEST_ATTEMPT_ID, + ) + assert _legacy_result(document, request).status == "fail" + assert actual.status == "fail" + assert actual.details["regressions"] + + +@pytest.mark.parametrize("values", [[0, 0], [1, 1], [None, None]]) +@pytest.mark.parametrize("critical", [False, True]) +def test_present_degenerate_input_expires_exception_using_real_budget_gate( + tmp_path, values, critical +): + result, evidence = uk_candidate.budget_inputs( + make_pair(), + tmp_path / "unused.h5", + model_input_variables=[INPUT], + critical_inputs=[INPUT] if critical else [], + accepted_gaps={}, + previous_missing_inputs=[INPUT], + previous_unusable_critical=[], + tables={"household": pd.DataFrame({INPUT: values})}, + ) + assert evidence["missing_inputs"] == [] + assert _judge(_failed_receipt(), budget=(result, evidence)).status == "fail" + + +def _stage_real_failed_attempt(tmp_path): + attempt = tmp_path / "attempt" + uk_qualification.stage_attempt( + attempt, + {"pair": make_pair().canonical_payload(), "previous_certificate": None}, + HARNESS, + ) + uk_qualification.write_gate_results(attempt, gates(), {}) + budget_result, budget_evidence = _real_budget( + attempt_id=uk_qualification.attempt_evidence_id(attempt) + ) + uk_qualification.write_gate_results( + attempt, [budget_result], {"budget_inputs": budget_evidence} + ) + request, document = _real_receipt() + probe_result, probe_evidence = _probe( + request=request, attempt_id=uk_qualification.attempt_evidence_id(attempt) + ) + uk_qualification.write_gate_results( + attempt, [probe_result], {"replay_probe": probe_evidence} + ) + write_json(attempt / "replay/request.json", request.document()) + write_json( + attempt / "replay/status.json", _status(request.request_id, document).document() + ) + write_json(attempt / "replay/receipt.json", document) + # Use a real production-registry evaluation rather than the synthetic + # registry _judge normally supplies. + cached = uk_qualification.evaluate_replay_receipt( + request, + _status(request.request_id, document), + uk_qualification.ReplayReceipt.from_document(document), + budget_inputs_result=budget_result, + budget_inputs_evidence=budget_evidence, + probe_result=probe_result, + probe_evidence=probe_evidence, + attempt_id=uk_qualification.attempt_evidence_id(attempt), + ) + assert cached.status == "needs_review" + write_json(attempt / "replay/replay.result.json", cached.canonical_payload()) + return attempt, budget_result, budget_evidence + + +@pytest.mark.parametrize( + "change", ["digest_mismatch", "present_with_new_digest", "missing_files"] +) +def test_finalize_rejects_stale_real_exception_when_input_evidence_changes( + tmp_path, change +): + attempt, result, evidence = _stage_real_failed_attempt(tmp_path) + if change == "missing_files": + (attempt / "gates/budget_inputs.json").unlink() + else: + evidence = copy.deepcopy(evidence) + evidence["missing_inputs"].remove(INPUT) + evidence["newly_missing_inputs"].remove(INPUT) + write_json(attempt / "gates/budget_inputs.json", evidence) + if change == "present_with_new_digest": + details = copy.deepcopy(result.details) + details["missing_inputs"] -= 1 + details["newly_missing_inputs"].remove(INPUT) + replacement = GateResult( + **{ + **result.canonical_payload(), + "evidence_sha256": content_sha256(evidence), + "details": details, + } + ) + write_json( + attempt / "gates/budget_inputs.result.json", + replacement.canonical_payload(), + ) + finalized = uk_qualification.finalize_attempt( + attempt, + tmp_path / "ledger", + issued_by=ISSUED, + completed_at="2026-10-17T10:00:00Z", + ) + actual = next(gate for gate in finalized.gates if gate.gate_id == "replay") + assert finalized.verdict == "rejected" + assert actual.status == "fail" + assert actual.details["applied_exceptions"] == [] + assert ( + read_json(attempt / "replay/replay.result.json") == actual.canonical_payload() + ) + + +@pytest.mark.parametrize("changed", ["receipt", "status"]) +def test_finalize_rejects_cached_exception_after_raw_binding_changes(tmp_path, changed): + attempt, _, _ = _stage_real_failed_attempt(tmp_path) + if changed == "receipt": + path = attempt / "replay/receipt.json" + document = read_json(path) + document["integrity"]["failures"][0]["detail"] += " changed after polling" + write_json(path, _sealed(document)) + else: + path = attempt / "replay/status.json" + document = read_json(path) + document["request_id"] = "0" * 64 + write_json(path, document) + finalized = uk_qualification.finalize_attempt( + attempt, + tmp_path / "ledger", + issued_by=ISSUED, + completed_at="2026-10-17T10:00:00Z", + ) + actual = next(gate for gate in finalized.gates if gate.gate_id == "replay") + assert finalized.verdict == "rejected" + assert actual.details["regressions"][0]["kind"] == "unbound" + + +@pytest.mark.parametrize( + "change", + ["digest_mismatch", "failed_with_new_digest", "wrong_attempt", "missing_files"], +) +def test_finalize_rechecks_probe_evidence_for_the_same_attempt(tmp_path, change): + attempt, _, _ = _stage_real_failed_attempt(tmp_path) + evidence_path = attempt / "gates/replay_probe.json" + result_path = attempt / "gates/replay_probe.result.json" + if change == "missing_files": + evidence_path.unlink() + else: + evidence = read_json(evidence_path) + if change == "wrong_attempt": + evidence["attempt_id"] = "previous-attempt" + else: + evidence["probes"][0]["status"] = "fail" + write_json(evidence_path, evidence) + if change != "digest_mismatch": + result = GateResult.model_validate(read_json(result_path)) + result = result.model_copy( + update={"evidence_sha256": content_sha256(evidence)} + ) + write_json(result_path, result.canonical_payload()) + finalized = uk_qualification.finalize_attempt( + attempt, + tmp_path / "ledger", + issued_by=ISSUED, + completed_at="2026-10-17T10:00:00Z", + ) + actual = next(gate for gate in finalized.gates if gate.gate_id == "replay") + assert finalized.verdict == "rejected" + assert actual.status == "fail" + assert actual.details["applied_exceptions"] == [] + assert ( + read_json(attempt / "replay/replay.result.json") == actual.canonical_payload() + ) + + +def test_finalize_rejects_a_newly_usable_previous_lineage(tmp_path): + attempt, result, evidence = _stage_real_failed_attempt(tmp_path) + evidence["previous_input_status"][INPUT] = { + "in_model": True, + "in_data": True, + "usable": True, + } + write_json(attempt / "gates/budget_inputs.json", evidence) + result = result.model_copy(update={"evidence_sha256": content_sha256(evidence)}) + write_json(attempt / "gates/budget_inputs.result.json", result.canonical_payload()) + finalized = uk_qualification.finalize_attempt( + attempt, + tmp_path / "ledger", + issued_by=ISSUED, + completed_at="2026-10-17T10:00:00Z", + ) + actual = next(gate for gate in finalized.gates if gate.gate_id == "replay") + assert finalized.verdict == "rejected" + assert actual.status == "fail" + assert "fresh registry decision" in str(actual.model_dump()).lower() + + +@pytest.mark.parametrize("history", ["never_usable", "ever_usable", "erased_rehashed"]) +def test_finalize_checks_the_producer_history_marker(tmp_path, history): + attempt, _, old_evidence = _stage_real_failed_attempt(tmp_path) + result, evidence = uk_candidate.budget_inputs( + make_pair(), + tmp_path / "unused.h5", + model_input_variables=old_evidence["missing_inputs"], + critical_inputs=[], + accepted_gaps={}, + previous_missing_inputs=old_evidence["missing_inputs"], + previous_unusable_critical=[], + previous_input_status=old_evidence["previous_input_status"], + previous_ever_usable_inputs=[] if history == "never_usable" else [INPUT], + tables={}, + ) + evidence["attempt_id"] = uk_qualification.attempt_evidence_id(attempt) + if history == "erased_rehashed": + # Keep the producer's marker while rehashing erased history arrays. + assert INPUT in result.details["ever_usable_inputs"] + evidence["previous_ever_usable_inputs"] = [] + evidence["ever_usable_inputs"] = [] + result = result.model_copy(update={"evidence_sha256": content_sha256(evidence)}) + uk_qualification.write_gate_results(attempt, [result], {"budget_inputs": evidence}) + + receipt = uk_qualification.finalize_attempt( + attempt, + tmp_path / "ledger", + issued_by=ISSUED, + completed_at="2026-10-17T11:00:00Z", + ) + replay = next(gate for gate in receipt.gates if gate.gate_id == "replay") + + if history == "never_usable": + assert receipt.verdict == "needs_review" + assert replay.status == "needs_review" + assert len(replay.details["applied_exceptions"]) == 5 + else: + assert receipt.verdict == "rejected" + assert replay.status == "fail" + assert replay.details["applied_exceptions"] == [] diff --git a/tests/test_replay_exception_hardening_round3.py b/tests/test_replay_exception_hardening_round3.py new file mode 100644 index 00000000..648ce185 --- /dev/null +++ b/tests/test_replay_exception_hardening_round3.py @@ -0,0 +1,247 @@ +"""Trusted evidence boundaries and monotonic lineage for replay exceptions.""" + +from __future__ import annotations + +import copy +import itertools + +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +import policyengine.provenance.qualification as qualification +from policyengine.provenance.qualification import qualification_verdict +from policyengine.provenance.receipt import content_sha256 +from tests.qualification_fixtures import gates +from tests.test_qualification_contracts import _sealed +from tests.test_replay_exceptions import ( + INPUT, + _budget_inputs, + _entry, + _failed_receipt, + _judge, + _registry, + _source_mutant, +) + +STATUS_KEYS = ("in_model", "is_input", "in_data", "usable") +BAD_CANDIDATE_STATUSES = [ + dict(zip(STATUS_KEYS, values, strict=True)) + for values in itertools.product((False, True), repeat=4) + if values != (True, True, False, False) +] +BAD_PREVIOUS_STATUSES = [ + {"in_model": False, "is_input": True, "in_data": False, "usable": False}, + {"in_model": True, "is_input": True, "in_data": False, "usable": True}, + {"in_model": False, "is_input": False, "in_data": True, "usable": True}, +] + + +def _reseal_budget(result, evidence): + return result.model_copy( + update={"evidence_sha256": content_sha256(evidence)} + ), evidence + + +def _assert_rejected(replay): + assert replay.status == "fail", replay.canonical_payload() + assert replay.details["applied_exceptions"] == [] + assert qualification_verdict(gates(replay=replay.status)) == "rejected" + + +@pytest.mark.parametrize("candidate_state", BAD_CANDIDATE_STATUSES) +def test_candidate_status_must_agree_with_required_missing_input(candidate_state): + """Includes all three candidate-status attacks from the independent review.""" + result, evidence = _budget_inputs() + evidence["input_status"][INPUT] = candidate_state + _assert_rejected(_judge(_failed_receipt(), budget=_reseal_budget(result, evidence))) + + +@pytest.mark.parametrize("previous_state", BAD_PREVIOUS_STATUSES) +def test_previous_status_fields_must_be_internally_consistent(previous_state): + """Includes the review's is_input=True/in_model=False predecessor.""" + budget = _budget_inputs(previous_status={INPUT: previous_state}) + _assert_rejected(_judge(_failed_receipt(), budget=budget)) + + +@settings(max_examples=45, deadline=None) +@given( + year=st.integers(2025, 2029), + target=st.sampled_from(["candidate", "previous"]), + index=st.integers(0, len(BAD_CANDIDATE_STATUSES) - 1), +) +def test_contradictory_status_maps_never_qualify(year, target, index): + result, evidence = _budget_inputs() + if target == "candidate": + evidence["input_status"][INPUT] = BAD_CANDIDATE_STATUSES[index] + else: + evidence["previous_input_status"][INPUT] = BAD_PREVIOUS_STATUSES[ + index % len(BAD_PREVIOUS_STATUSES) + ] + replay = _judge( + _failed_receipt(year), + registry=_registry(_entry(years=[year])), + budget=_reseal_budget(result, evidence), + ) + _assert_rejected(replay) + + +IDENTITY_DAMAGE = ( + "different_pair", + "different_attempt", + "missing_pair", + "missing_attempt", + "malformed_pair", +) + + +def _damage_identity(evidence, damage): + if damage == "different_pair": + evidence["pair_id"] = "f" * 64 + elif damage == "different_attempt": + evidence["attempt_id"] += "-another" + elif damage == "missing_pair": + evidence.pop("pair_id") + elif damage == "missing_attempt": + evidence.pop("attempt_id") + elif damage == "malformed_pair": + evidence["pair_id"] = "not-a-pair-id" + else: + raise AssertionError(damage) + + +@pytest.mark.parametrize("damage", IDENTITY_DAMAGE) +def test_budget_evidence_identity_is_checked_after_digest_resealing(damage): + result, evidence = _budget_inputs() + _damage_identity(evidence, damage) + _assert_rejected(_judge(_failed_receipt(), budget=_reseal_budget(result, evidence))) + + +@settings(max_examples=35, deadline=None) +@given(year=st.integers(2025, 2029), damage=st.sampled_from(IDENTITY_DAMAGE)) +def test_mismatched_evidence_identity_never_qualifies(year, damage): + result, evidence = _budget_inputs() + _damage_identity(evidence, damage) + replay = _judge( + _failed_receipt(year), + registry=_registry(_entry(years=[year])), + budget=_reseal_budget(result, evidence), + ) + _assert_rejected(replay) + + +@settings(max_examples=55, deadline=None) +@given( + year=st.integers(2025, 2029), + later_absent_pairs=st.integers(1, 15), + noise=st.sets(st.sampled_from(["another_input", "old_input", "noise"])), +) +def test_ever_usable_input_cannot_reauthorize_after_any_absent_chain( + year, later_absent_pairs, noise +): + """Each later absent pair retains the same monotonic certified history.""" + history = sorted(noise | {INPUT}) + for _ in range(later_absent_pairs): + result, evidence = _budget_inputs() + evidence["previous_ever_usable_inputs"] = history + evidence["ever_usable_inputs"] = history + result = result.model_copy( + update={"details": {**result.details, "ever_usable_inputs": history}} + ) + replay = _judge( + _failed_receipt(year), + registry=_registry(_entry(years=[year])), + budget=_reseal_budget(result, evidence), + ) + _assert_rejected(replay) + assert "a fresh registry decision is required" in str(replay.model_dump()) + + +def test_head_level_invariant_kills_skip_head_coherence_mutant(): + mutant = _source_mutant( + " and row.head_gbp not in (None, 0)\n ]", + ' and row.level != "head" and row.head_gbp not in (None, 0)\n ]', + ) + document = _failed_receipt() + row = copy.deepcopy(document["diff"]["flagged_rows"][0]) + row.update(level="head", key="income_tax", head_gbp=1_000_000_000) + document["diff"]["flagged_rows"].append(row) + for flag in row["flags"]: + document["diff"]["flag_counts"][flag] += 1 + replay = _judge(_sealed(document), evaluator=mutant) + with pytest.raises(AssertionError): + _assert_rejected(replay) + + +def test_history_invariant_kills_predecessor_only_lineage_mutant(monkeypatch): + mutant = _source_mutant( + "if name in ever_usable_inputs:", + 'if state["usable"]:', + function=qualification._replay_lineage_problem, + ) + monkeypatch.setattr(qualification, "_replay_lineage_problem", mutant) + result, evidence = _budget_inputs() + evidence["previous_ever_usable_inputs"] = [INPUT] + evidence["ever_usable_inputs"] = [INPUT] + result = result.model_copy( + update={"details": {**result.details, "ever_usable_inputs": [INPUT]}} + ) + replay = _judge(_failed_receipt(), budget=_reseal_budget(result, evidence)) + with pytest.raises(AssertionError): + _assert_rejected(replay) + + +def test_history_marker_invariant_kills_skip_marker_consistency_mutant(monkeypatch): + result, evidence = _budget_inputs() + # Both rehashed arrays erase a usable input still recorded by the producer. + result = result.model_copy( + update={"details": {**result.details, "ever_usable_inputs": [INPUT]}} + ) + budget = _reseal_budget(result, evidence) + _assert_rejected(_judge(_failed_receipt(), budget=budget)) + mutant = _source_mutant( + "if recorded_ever is None or recorded_ever != _input_name_set(\n" + ' evidence.get("ever_usable_inputs")\n' + " ):", + "if False:", + function=qualification._replay_missing_inputs, + ) + monkeypatch.setattr(qualification, "_replay_missing_inputs", mutant) + replay = _judge(_failed_receipt(), budget=budget) + assert replay.status == "needs_review" + assert replay.details["applied_exceptions"] + with pytest.raises(AssertionError): + _assert_rejected(replay) + + +def test_status_invariant_kills_skip_missing_map_consistency_mutant(monkeypatch): + mutant = _source_mutant( + "or missing\n != {", + "or False and missing\n != {", + function=qualification._replay_missing_inputs, + ) + monkeypatch.setattr(qualification, "_replay_missing_inputs", mutant) + result, evidence = _budget_inputs() + evidence["input_status"][INPUT] = { + "in_model": True, + "is_input": True, + "in_data": True, + "usable": False, + } + replay = _judge(_failed_receipt(), budget=_reseal_budget(result, evidence)) + with pytest.raises(AssertionError): + _assert_rejected(replay) + + +def test_identity_invariant_kills_skip_budget_identity_mutant(monkeypatch): + mutant = _source_mutant( + "or not _replay_budget_identity_matches(evidence, attempt_id)", + "or False", + function=qualification._replay_missing_inputs, + ) + monkeypatch.setattr(qualification, "_replay_missing_inputs", mutant) + result, evidence = _budget_inputs() + _damage_identity(evidence, "different_attempt") + replay = _judge(_failed_receipt(), budget=_reseal_budget(result, evidence)) + with pytest.raises(AssertionError): + _assert_rejected(replay) diff --git a/tests/test_replay_exception_lineage.py b/tests/test_replay_exception_lineage.py new file mode 100644 index 00000000..c97952c0 --- /dev/null +++ b/tests/test_replay_exception_lineage.py @@ -0,0 +1,86 @@ +"""The approved lineage requires a previously absent model input or column.""" + +from __future__ import annotations + +import pytest + +from policyengine.provenance.receipt import content_sha256 +from tests.test_replay_exceptions import ( + INPUT, + _budget_inputs, + _failed_receipt, + _judge, +) + + +@pytest.mark.parametrize( + ("in_model", "in_data"), [(False, False), (True, False), (False, True)] +) +def test_previous_absence_allows_reviewed_exception(in_model, in_data): + budget = _budget_inputs( + previous_status={ + INPUT: {"in_model": in_model, "in_data": in_data, "usable": False} + } + ) + gate = _judge(_failed_receipt(), budget=budget) + assert gate.status == "needs_review" + assert len(gate.details["applied_exceptions"]) == 1 + + +@pytest.mark.parametrize("usable", [False, True]) +def test_previous_supplied_input_requires_fresh_decision_even_if_degenerate(usable): + budget = _budget_inputs( + previous_status={INPUT: {"in_model": True, "in_data": True, "usable": usable}} + ) + gate = _judge(_failed_receipt(), budget=budget) + assert gate.status == "fail" + assert gate.details["applied_exceptions"] == [] + assert "a fresh registry decision is required" in gate.summary + + +@pytest.mark.parametrize( + ("in_model", "in_data"), [(False, False), (True, False), (False, True)] +) +def test_inconsistent_previous_usable_flag_requires_fresh_decision(in_model, in_data): + budget = _budget_inputs( + previous_status={ + INPUT: {"in_model": in_model, "in_data": in_data, "usable": True} + } + ) + gate = _judge(_failed_receipt(), budget=budget) + assert gate.status == "fail" + assert gate.details["applied_exceptions"] == [] + assert "a fresh registry decision is required" in gate.summary + + +@pytest.mark.parametrize( + "previous", + [ + None, + {}, + {INPUT: None}, + {INPUT: {}}, + {INPUT: {"in_model": False, "in_data": False}}, + {INPUT: {"in_model": 0, "in_data": False, "usable": False}}, + {INPUT: {"in_model": False, "in_data": "false", "usable": False}}, + {INPUT: {"in_model": False, "in_data": False, "usable": None}}, + ], +) +def test_unknown_or_malformed_previous_status_cannot_authorize_exception(previous): + result, evidence = _budget_inputs() + evidence["previous_input_status"] = previous + result = result.model_copy(update={"evidence_sha256": content_sha256(evidence)}) + gate = _judge(_failed_receipt(), budget=(result, evidence)) + assert gate.status == "fail" + assert gate.details["applied_exceptions"] == [] + assert "unknown" in gate.summary + + +def test_missing_previous_status_cannot_authorize_exception(): + result, evidence = _budget_inputs() + evidence.pop("previous_input_status") + result = result.model_copy(update={"evidence_sha256": content_sha256(evidence)}) + gate = _judge(_failed_receipt(), budget=(result, evidence)) + assert gate.status == "fail" + assert gate.details["applied_exceptions"] == [] + assert "unknown" in gate.summary diff --git a/tests/test_replay_exception_lineage_round3.py b/tests/test_replay_exception_lineage_round3.py new file mode 100644 index 00000000..8a9316bf --- /dev/null +++ b/tests/test_replay_exception_lineage_round3.py @@ -0,0 +1,501 @@ +"""Certified cumulative history prevents a usable input from being forgotten.""" + +from __future__ import annotations + +import copy +from pathlib import Path + +import pandas as pd +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine.provenance.qualification import ( + PreviousCertificate, + QualificationReceipt, + ReplayQualificationRequest, +) +from policyengine.provenance.receipt import content_sha256 +from policyengine.provenance.uk_candidate import budget_inputs +from tests.qualification_fixtures import gates, golden, make_pair, make_receipt +from tests.test_qualification_contracts import _receipt_with, _sealed +from tests.test_replay_exceptions import INPUT, _failed_receipt, _judge + + +def _coverage(pair, *, supplied=False, formula=False, removed=False, previous=None): + inputs = ["household_weight"] + ([] if formula or removed else [INPUT]) + variables = ["household_weight"] + ([] if removed else [INPUT]) + table = pd.DataFrame({"household_weight": [1.0, 2.0]}) + if supplied: + table[INPUT] = [0, 300_000] + previous_status = previous[1]["input_status"] if previous else None + # Mirror orchestration: retain old variables, including removed/formula + # entries, while adding the candidate's newly introduced input names. + if previous_status is not None: + previous_status = dict(previous_status) + for name in variables: + previous_status.setdefault( + name, + { + "in_model": False, + "is_input": False, + "in_data": False, + "usable": False, + }, + ) + result, evidence = budget_inputs( + pair, + Path("unused.h5"), + model_input_variables=inputs, + model_variables=variables, + critical_inputs=["household_weight"], + accepted_gaps={}, + previous_missing_inputs=previous[1]["missing_inputs"] if previous else None, + previous_unusable_critical=[], + previous_input_status=previous_status, + previous_ever_usable_inputs=( + previous[1]["ever_usable_inputs"] if previous else None + ), + tables={"household": table}, + ) + evidence["attempt_id"] = f"{pair.pair_id}/test-attempt" + result = result.model_copy(update={"evidence_sha256": content_sha256(evidence)}) + return result, evidence + + +def _replay_for_pair(pair, document, coverage): + raw = golden("qualification_request_golden") + candidate = raw["candidate"] + candidate["policyengine_uk"].update( + version=pair.model.version, wheel_sha256=pair.model.wheel_sha256 + ) + candidate["policyengine_core"].update( + version=pair.core.version, wheel_sha256=pair.core.wheel_sha256 + ) + candidate["data"].update( + build_id=pair.data.build_id, + revision=pair.data.dataset_revision, + sha256=pair.data.dataset_sha256, + ) + candidate["expected_release_bundle"].update( + model_version=pair.model.version, data_version=pair.data.release_version + ) + request = ReplayQualificationRequest.model_validate(raw) + document["request_id"] = request.request_id + document["execution"]["packages"]["policyengine-uk"].update( + version=pair.model.version, wheel_sha256=pair.model.wheel_sha256 + ) + document["execution"]["packages"]["policyengine-core"].update( + version=pair.core.version, wheel_sha256=pair.core.wheel_sha256 + ) + document["execution"]["release_bundle"] = candidate["expected_release_bundle"] + document["execution"]["dataset_sha256"] = pair.data.dataset_sha256 + return _judge( + _sealed(document), + budget=coverage, + request=request, + attempt_id=f"{pair.pair_id}/test-attempt", + ) + + +def _qualification(pair, coverage, replay, previous=None): + actual = {"budget_inputs": coverage[0], "replay": replay} + receipt = make_receipt( + pair, + gate_results=[actual.get(gate.gate_id, gate) for gate in gates()], + ) + if previous is not None: + certificate = PreviousCertificate( + bundle_version=previous.harness.policyengine_version, + model_version=previous.pair.model.version, + core_version=previous.pair.core.version, + data_release_version=previous.pair.data.release_version, + dataset_sha256=previous.pair.data.dataset_sha256, + certificate_sha256=previous.receipt_sha256, + qualification_pair_id=previous.pair_id, + model_wheel_sha256=previous.pair.model.wheel_sha256, + core_wheel_sha256=previous.pair.core.wheel_sha256, + ) + receipt = receipt.model_copy(update={"previous_certificate": certificate}) + return QualificationReceipt.model_validate(receipt.canonical_payload()) + + +@pytest.mark.parametrize("intermediate", ["formula", "removed"]) +def test_three_pair_history_cannot_reactivate_expired_exception( + intermediate, +): + """A usable -> B absent/responding -> C absent/inert must be rejected. + + B's reform can respond through an alternate formula or another model input, + so it needs no data-limited exception. Every gate and receipt is well formed. + """ + pair_a = make_pair(model="2.135.0", data="1.59.0", dataset_sha="1" * 64) + pair_b = make_pair(model="2.135.1", data="1.60.0", dataset_sha="2" * 64) + pair_c = make_pair(model="2.135.2", data="1.61.0", dataset_sha="3" * 64) + coverage_a = _coverage(pair_a, supplied=True) + assert coverage_a[1]["input_status"][INPUT]["usable"] is True + # A clean diff has no flagged row for the unchanged, responding measure; + # C's subsequent diff records B's unchanged material £2.5bn base effect. + responding = _receipt_with + receipt_a = _qualification( + pair_a, coverage_a, _replay_for_pair(pair_a, responding(), coverage_a) + ) + assert receipt_a.promotable + + coverage_b = _coverage( + pair_b, + formula=intermediate == "formula", + removed=intermediate == "removed", + previous=coverage_a, + ) + receipt_b = _qualification( + pair_b, + coverage_b, + _replay_for_pair(pair_b, responding(), coverage_b), + previous=receipt_a, + ) + assert coverage_b[0].status == "pass" + assert receipt_b.verdict == "qualified" + assert receipt_b.promotable + assert receipt_b.previous_certificate.qualification_pair_id == receipt_a.pair_id + assert coverage_b[1]["previous_input_status"][INPUT]["usable"] is True + assert INPUT in coverage_b[1]["ever_usable_inputs"] + assert ( + next(g for g in receipt_b.gates if g.gate_id == "replay").details[ + "applied_exceptions" + ] + == [] + ) + + coverage_c = _coverage(pair_c, previous=coverage_b) + replay_c = _replay_for_pair(pair_c, _failed_receipt(), coverage_c) + receipt_c = _qualification(pair_c, coverage_c, replay_c, previous=receipt_b) + assert coverage_c[0].status == "needs_review" + assert receipt_c.verdict == "rejected" + assert receipt_c.previous_certificate.qualification_pair_id == receipt_b.pair_id + assert replay_c.details["applied_exceptions"] == [] + assert "a fresh registry decision is required" in replay_c.summary + + +def test_rehashed_current_history_must_agree_with_its_result_marker(): + pair_a = make_pair(model="2.135.0", data="1.59.0", dataset_sha="1" * 64) + pair_b = make_pair(model="2.135.1", data="1.60.0", dataset_sha="2" * 64) + pair_c = make_pair(model="2.135.2", data="1.61.0", dataset_sha="3" * 64) + coverage_a = _coverage(pair_a, supplied=True) + coverage_b = _coverage(pair_b, removed=True, previous=coverage_a) + result, evidence = _coverage(pair_c, previous=coverage_b) + evidence = copy.deepcopy(evidence) + # Rehashing both history arrays cannot erase the producer's independent + # result marker, even after the immediate predecessor stopped using INPUT. + assert INPUT in result.details["ever_usable_inputs"] + assert evidence["previous_input_status"][INPUT]["usable"] is False + for key in ("previous_ever_usable_inputs", "ever_usable_inputs"): + evidence[key] = [name for name in evidence[key] if name != INPUT] + result = result.model_copy(update={"evidence_sha256": content_sha256(evidence)}) + + replay = _replay_for_pair(pair_c, _failed_receipt(), (result, evidence)) + + assert replay.status == "fail" + assert replay.details["applied_exceptions"] == [] + + +@pytest.mark.parametrize("damage", ["missing", "malformed", "contradictory"]) +def test_current_history_requires_a_valid_consistent_result_marker(damage): + pair = make_pair() + result, evidence = _coverage(pair, previous=_coverage(pair)) + control = _replay_for_pair(pair, _failed_receipt(), (result, evidence)) + assert control.status == "needs_review" + assert len(control.details["applied_exceptions"]) == 1 + details = copy.deepcopy(result.details) + if damage == "missing": + details.pop("ever_usable_inputs") + elif damage == "malformed": + details["ever_usable_inputs"] = [True] + else: + details["ever_usable_inputs"] = [] + result = result.model_copy(update={"details": details}) + + replay = _replay_for_pair(pair, _failed_receipt(), (result, evidence)) + + assert replay.status == "fail" + assert replay.details["applied_exceptions"] == [] + + +def test_legacy_history_seed_uses_recomputed_predecessor_coverage(): + pair = make_pair() + previous_status = { + INPUT: { + "in_model": True, + "is_input": False, + "in_data": True, + "usable": True, + }, + } + _, evidence = budget_inputs( + pair, + Path("unused.h5"), + model_input_variables=[INPUT], + critical_inputs=[], + accepted_gaps={}, + previous_missing_inputs=[], + previous_unusable_critical=[], + previous_input_status=previous_status, + tables={"household": pd.DataFrame()}, + ) + assert evidence["pair_id"] == pair.pair_id + assert evidence["previous_ever_usable_inputs"] == [INPUT] + assert evidence["ever_usable_inputs"] == [INPUT] + assert evidence["ever_usable_inputs_basis"] == "seeded from direct predecessor" + + +def test_formula_backed_supplied_candidate_seeds_cumulative_history(): + _, evidence = _coverage(make_pair(), supplied=True, formula=True) + assert evidence["input_status"][INPUT]["is_input"] is False + assert evidence["input_status"][INPUT]["usable"] is True + assert INPUT in evidence["ever_usable_inputs"] + assert evidence["ever_usable_inputs_basis"] == "candidate inputs" + + +@settings(max_examples=40, deadline=None) +@given( + states=st.lists( + st.sampled_from(["input", "formula", "removed"]), min_size=1, max_size=15 + ) +) +def test_usable_input_history_survives_every_intermediate_model_state(states): + pair = make_pair() + coverage = _coverage(pair, supplied=True) + for state in states: + previous = coverage + coverage = _coverage( + pair, + formula=state == "formula", + removed=state == "removed", + previous=previous, + ) + assert INPUT in coverage[1]["previous_ever_usable_inputs"] + assert INPUT in coverage[1]["ever_usable_inputs"] + assert set(previous[1]["ever_usable_inputs"]).issubset( + coverage[1]["ever_usable_inputs"] + ) + assert coverage[1]["ever_usable_inputs_basis"] == ( + "carried from direct predecessor" + ) + + +@pytest.mark.parametrize("bad_history", [None, {}, [INPUT, INPUT], [True], [""]]) +def test_malformed_certified_cumulative_history_cannot_be_reseeded( + tmp_path, bad_history +): + from policyengine.provenance.uk_candidate import ( + PreviousPairUnavailable, + _previous_gate_evidence, + ) + from tests.test_uk_candidate_gates import ( + _previous_identity, + _write_previous_receipt, + ) + + pair = make_pair() + _, evidence = _coverage(pair) + evidence["ever_usable_inputs"] = bad_history + _write_previous_receipt( + tmp_path, + pair, + { + "supported_years": {"supported_years": [2025, 2026]}, + "budget_inputs": evidence, + }, + ) + with pytest.raises(PreviousPairUnavailable, match="fresh registry decision"): + _previous_gate_evidence(tmp_path, _previous_identity(pair)) + + +def test_cumulative_history_cannot_omit_a_current_usable_variable(tmp_path): + from policyengine.provenance.uk_candidate import ( + PreviousPairUnavailable, + _previous_gate_evidence, + ) + from tests.test_uk_candidate_gates import ( + _previous_identity, + _write_previous_receipt, + ) + + pair = make_pair() + _, evidence = _coverage(pair, supplied=True) + evidence["ever_usable_inputs"] = ["household_weight"] + _write_previous_receipt( + tmp_path, + pair, + { + "supported_years": {"supported_years": [2025, 2026]}, + "budget_inputs": evidence, + }, + ) + with pytest.raises(PreviousPairUnavailable, match="fresh registry decision"): + _previous_gate_evidence(tmp_path, _previous_identity(pair)) + + +@pytest.mark.parametrize("identity", ["pair_id", "attempt_id"]) +def test_certified_cumulative_history_rejects_another_evidence_identity( + tmp_path, identity +): + from policyengine.provenance.uk_candidate import ( + PreviousPairUnavailable, + _previous_gate_evidence, + ) + from tests.test_uk_candidate_gates import ( + _previous_identity, + _write_previous_receipt, + ) + + pair = make_pair() + _, evidence = _coverage(pair) + evidence[identity] = "another-attempt" if identity == "attempt_id" else "0" * 64 + _write_previous_receipt( + tmp_path, + pair, + { + "supported_years": {"supported_years": [2025, 2026]}, + "budget_inputs": evidence, + }, + ) + with pytest.raises(PreviousPairUnavailable, match="another pair or attempt"): + _previous_gate_evidence(tmp_path, _previous_identity(pair)) + + +def test_damaged_cumulative_history_cannot_fall_back_to_a_fresh_seed(tmp_path): + from policyengine.provenance.qualification_ledger import write_json + from policyengine.provenance.uk_candidate import ( + PreviousPairUnavailable, + _previous_gate_evidence, + ) + from tests.test_uk_candidate_gates import ( + _previous_identity, + _write_previous_receipt, + ) + + pair = make_pair() + _, evidence = _coverage(pair, supplied=True) + _, attempt = _write_previous_receipt( + tmp_path, + pair, + { + "supported_years": {"supported_years": [2025, 2026]}, + "budget_inputs": evidence, + }, + ) + write_json(attempt / "gates" / "supported_years.json", {"supported_years": [2025]}) + with pytest.raises(PreviousPairUnavailable, match="fresh registry decision"): + _previous_gate_evidence(tmp_path, _previous_identity(pair)) + + +@pytest.mark.parametrize("gate_name", ["supported_years", "budget_inputs"]) +@pytest.mark.parametrize("damage", ["missing", "invalid_json"]) +def test_unreadable_certified_history_artifacts_cannot_reset_the_chain( + tmp_path, gate_name, damage +): + from policyengine.provenance.uk_candidate import ( + PreviousPairUnavailable, + _previous_gate_evidence, + ) + from tests.test_uk_candidate_gates import ( + _previous_identity, + _write_previous_receipt, + ) + + pair = make_pair() + _, evidence = _coverage(pair, supplied=True) + _, attempt = _write_previous_receipt( + tmp_path, + pair, + { + "supported_years": {"supported_years": [2025, 2026]}, + "budget_inputs": evidence, + }, + ) + path = attempt / "gates" / f"{gate_name}.json" + if damage == "missing": + path.unlink() + else: + path.write_text("not-json") + with pytest.raises(PreviousPairUnavailable, match="fresh registry decision"): + _previous_gate_evidence(tmp_path, _previous_identity(pair)) + + +def test_certified_history_without_a_matching_attempt_cannot_reset_the_chain(tmp_path): + from policyengine.provenance.uk_candidate import ( + PreviousPairUnavailable, + _previous_gate_evidence, + ) + from tests.test_uk_candidate_gates import ( + _previous_identity, + _write_previous_receipt, + ) + + pair = make_pair() + _, evidence = _coverage(pair, supplied=True) + _, attempt = _write_previous_receipt( + tmp_path, + pair, + { + "supported_years": {"supported_years": [2025, 2026]}, + "budget_inputs": evidence, + }, + ) + (attempt / "qualification_receipt.json").unlink() + with pytest.raises(PreviousPairUnavailable, match="no matching attempt"): + _previous_gate_evidence(tmp_path, _previous_identity(pair)) + + +@pytest.mark.parametrize("unavailable", ["no_ledger", "missing_receipt", "bad_receipt"]) +def test_unknown_certified_receipt_cannot_be_treated_as_legacy(tmp_path, unavailable): + from policyengine.provenance.uk_candidate import ( + PreviousPairUnavailable, + _previous_gate_evidence, + ) + from tests.test_uk_candidate_gates import _previous_identity + + pair = make_pair() + ledger_root = None if unavailable == "no_ledger" else tmp_path + if unavailable == "bad_receipt": + path = tmp_path / "uk" / pair.pair_id / "qualification_receipt.json" + path.parent.mkdir(parents=True) + path.write_text("not-json") + with pytest.raises(PreviousPairUnavailable, match="fresh registry decision"): + _previous_gate_evidence(ledger_root, _previous_identity(pair)) + + +def test_true_legacy_certificate_without_a_bound_pair_can_still_be_recomputed(): + from policyengine.provenance.uk_candidate import _previous_gate_evidence + + assert _previous_gate_evidence(None, {"bundle_version": "6.0.0"}) == {} + + +@pytest.mark.parametrize( + "pin", ["model_version", "dataset_sha256", "core_wheel_sha256"] +) +def test_mismatched_certified_pins_cannot_reset_a_recorded_history(tmp_path, pin): + from policyengine.provenance.uk_candidate import ( + PreviousPairUnavailable, + _previous_gate_evidence, + ) + from tests.test_uk_candidate_gates import ( + _previous_identity, + _write_previous_receipt, + ) + + pair = make_pair() + _, evidence = _coverage(pair, supplied=True) + _write_previous_receipt( + tmp_path, + pair, + { + "supported_years": {"supported_years": [2025, 2026]}, + "budget_inputs": evidence, + }, + ) + previous = _previous_identity(pair) + previous[pin] = "999.0.0" if pin == "model_version" else "0" * 64 + with pytest.raises(PreviousPairUnavailable, match="fresh registry decision"): + _previous_gate_evidence(tmp_path, previous) diff --git a/tests/test_replay_exception_policy_review.py b/tests/test_replay_exception_policy_review.py new file mode 100644 index 00000000..82de6211 --- /dev/null +++ b/tests/test_replay_exception_policy_review.py @@ -0,0 +1,149 @@ +"""Independent attacks on the proposed data-limited exception policy. + +These tests preserve the independent review attacks as regression checks. +Synthetic probe evidence and no representative microdata are used. +""" + +from __future__ import annotations + +from pathlib import Path + +import pandas as pd +import pytest + +from policyengine.provenance.qualification import qualification_verdict +from policyengine.provenance.receipt import content_sha256 +from policyengine.provenance.uk_candidate import budget_inputs +from tests.qualification_fixtures import gates, make_pair +from tests.test_qualification_contracts import _base_event, _sealed +from tests.test_replay_exceptions import ( + EVENT, + INPUT, + MEASURE, + TEST_ATTEMPT_ID, + _entry, + _failed_receipt, + _judge, + _registry, +) + + +def _actual_budget(values, *, previously_missing): + table = pd.DataFrame({"household_weight": [1.0, 2.0]}) + if values is not None: + table[INPUT] = values + result, evidence = budget_inputs( + make_pair(), + Path("unused.h5"), + model_input_variables=["household_weight", INPUT], + critical_inputs=["household_weight"], + accepted_gaps={}, + previous_missing_inputs=[INPUT] if previously_missing else [], + previous_input_status={ + INPUT: { + "in_model": True, + "is_input": True, + "in_data": not previously_missing, + "usable": not previously_missing, + } + }, + previous_unusable_critical=[], + tables={"household": table}, + ) + evidence["attempt_id"] = TEST_ATTEMPT_ID + return result.model_copy( + update={"evidence_sha256": content_sha256(evidence)} + ), evidence + + +def test_exception_cannot_reactivate_after_a_previously_supplied_input_is_dropped(): + """A previous usable input requires a fresh registry decision if lost.""" + document = _failed_receipt() + supplied = _actual_budget([0, 100_000], previously_missing=True) + dropped = _actual_budget(None, previously_missing=False) + assert supplied[0].status == "pass" + assert _judge(document, budget=supplied).status == "fail" + assert dropped[0].status == "needs_review" + assert dropped[1]["newly_missing_inputs"] == [INPUT] + replay = _judge(document, budget=dropped) + assert replay.status == "fail" + assert "fresh registry decision" in str(replay.model_dump()).lower() + combined = [ + dropped[0] if gate.gate_id == "budget_inputs" else gate + for gate in gates(replay=replay.status) + ] + assert qualification_verdict(combined) == "rejected" + + +@pytest.mark.parametrize("year", [2025, 2026, 2029]) +def test_different_inert_measure_in_the_registered_year_still_fails(year): + document = _failed_receipt(year) + other = "another_measure_disabled_by_a_model_regression" + document["corpus"]["events"][EVENT]["inert_measure_years"].append( + {"measure_key": other, "year": year} + ) + document["integrity"]["failures"].append( + { + "code": "construction_inert", + "detail": "A different measure also became inert.", + "event": EVENT, + "measure_key": other, + "year": year, + } + ) + replay = _judge(_sealed(document), registry=_registry(_entry(years=[year]))) + assert replay.status == "fail" + assert replay.details["applied_exceptions"] == [] + + +@pytest.mark.parametrize("flag", ["mapping", "coverage"]) +def test_matching_waiver_cannot_hide_an_unmatched_diff_failure(flag): + document = _failed_receipt() + document["diff"]["flagged_rows"].append( + { + "event": EVENT, + "measure_key": "unrelated_lost_result", + "year": 2026, + "level": "measure_year", + "key": "total", + "flags": [flag], + "base_gbp": 2_500_000_000, + "head_gbp": None, + } + ) + document["diff"]["flag_counts"][flag] += 1 + replay = _judge(_sealed(document)) + assert replay.status == "fail" + assert replay.details["applied_exceptions"] + expected = "mapping" if flag == "mapping" else "lost_result" + assert expected in {row["kind"] for row in replay.details["regressions"]} + + +@pytest.mark.parametrize("reverse_registry", [False, True]) +def test_overlapping_registry_entries_cannot_produce_qualified(reverse_registry): + # Partitioning a failure's codes between different reviewed entries must + # preserve the review floor, independently of file ordering. + entries = [ + _entry(id="integrity", codes=["construction_inert"]), + _entry(id="lost-response", codes=["lost_response"]), + _entry(id="zzz-overlap"), + ] + if reverse_registry: + entries.reverse() + replay = _judge(_failed_receipt(), registry=_registry(*entries)) + assert replay.status == "needs_review" + assert qualification_verdict(gates(replay=replay.status)) == "needs_review" + assert {row["id"] for row in replay.details["applied_exceptions"]} == { + "integrity", + "lost-response", + } + + +def test_base_known_gap_cannot_remove_construction_exception_review_floor(): + document = _failed_receipt(rows=[]) + document["diff"]["base_events"] = {EVENT: _base_event(inert=[(MEASURE, 2026)])} + replay = _judge(_sealed(document)) + assert replay.details["known_gaps"] + assert replay.details["applied_exceptions"] + assert replay.status == "needs_review" + assert qualification_verdict(gates(replay=replay.status)) != "qualified" diff --git a/tests/test_replay_exception_probe.py b/tests/test_replay_exception_probe.py new file mode 100644 index 00000000..06ff6b6b --- /dev/null +++ b/tests/test_replay_exception_probe.py @@ -0,0 +1,161 @@ +"""The exact candidate must respond to a positive missing purchase input.""" + +from __future__ import annotations + +from importlib.metadata import version + +import pytest + +import policyengine.provenance.replay_probe as replay_probe +from policyengine.provenance.qualification import ( + ReplayQualificationRequest, + _replay_exceptions, + _replay_probe_passes, +) +from policyengine.provenance.receipt import content_sha256 +from tests.qualification_fixtures import golden + + +@pytest.fixture +def registry(): + return _replay_exceptions()[0] + + +@pytest.fixture(scope="module") +def country(): + model = pytest.importorskip("policyengine_uk") + if not version("policyengine-uk").startswith("2.134."): + pytest.skip("The replay probe real-model tests require pe-uk 2.134.x.") + return model + + +def run(registry): + return replay_probe.run_replay_probes( + attempt_id="probe-attempt", + candidate_model_version=version("policyengine-uk"), + exception_registry=registry, + ) + + +@pytest.mark.real_install +def test_sdlt_positive_input_probe_passes_on_installed_candidate(country, registry): + result, evidence = run(registry) + assert result.status == "pass", evidence + assert result.evidence_sha256 == content_sha256(evidence) + assert evidence["attempt_id"] == "probe-attempt" + assert evidence["candidate_model_version"] == version("policyengine-uk") + (probe,) = evidence["probes"] + assert probe["status"] == "pass" + assert probe["installed_model_version"] == evidence["candidate_model_version"] + assert probe["spec_sha256"] == content_sha256( + registry.exceptions[0].probe.model_dump(mode="json") + ) + assert probe["positive_change_gbp"] == pytest.approx(6000, abs=0.1) + assert ( + probe["positive_reform_gbp"] - probe["positive_baseline_gbp"] + == probe["positive_change_gbp"] + ) + assert probe["zero_change_gbp"] == 0 + assert probe["zero_reform_gbp"] == probe["zero_baseline_gbp"] + assert len(probe["rates"]) == 5 + assert all( + rate["reform"] - rate["baseline"] == pytest.approx(0.02) + for rate in probe["rates"] + ) + request_document = golden("qualification_request_golden") + request_document["candidate"]["policyengine_uk"]["version"] = version( + "policyengine-uk" + ) + request = ReplayQualificationRequest.model_validate(request_document) + assert _replay_probe_passes( + registry.exceptions[0], request, result, evidence, "probe-attempt" + ) + + +@pytest.mark.real_install +def test_probe_fails_when_installed_model_ignores_purchase_input( + country, registry, monkeypatch +): + import numpy as np + from sortedcontainers import SortedDict + + original_init = country.CountryTaxBenefitSystem.__init__ + + def broken_init(self, *args, **kwargs): + original_init(self, *args, **kwargs) + self.variables[ + "sdlt_on_residential_property_transactions" + ].formulas = SortedDict( + { + "0001-01-01": lambda household, period, parameters: np.zeros( + household.count + ) + } + ) + + monkeypatch.setattr(country.CountryTaxBenefitSystem, "__init__", broken_init) + result, evidence = run(registry) + assert result.status == "fail" + (probe,) = evidence["probes"] + assert probe["status"] == "fail" + assert "error" not in probe, probe + assert probe["positive_change_gbp"] == 0 + assert probe["zero_change_gbp"] == 0 + + +@pytest.mark.real_install +def test_wrong_parameter_path_records_failing_probe(country, registry): + payload = registry.canonical_payload() + payload["exceptions"][0]["probe"]["parameter"] = "gov.hmrc.no_such_sdlt_parameter" + registry = type(registry).model_validate(payload) + result, evidence = run(registry) + assert result.status == "fail" + assert evidence["probes"][0]["status"] == "fail" + assert "no_such_sdlt_parameter" in evidence["probes"][0]["error"] + assert result.evidence_sha256 == content_sha256(evidence) + + +def test_probe_for_other_installed_version_records_failure(registry, monkeypatch): + monkeypatch.setattr(replay_probe, "version", lambda package: "0.0.0") + result, evidence = replay_probe.run_replay_probes( + attempt_id="attempt", + candidate_model_version="2.134.7", + exception_registry=registry, + ) + assert result.status == "fail" + assert "does not match candidate" in evidence["probes"][0]["error"] + + +def test_probe_records_the_registry_digest_it_read(registry, monkeypatch): + monkeypatch.setattr( + replay_probe, + "_run_probe", + lambda spec: { + "status": "pass", + "positive_change_gbp": 6000, + "zero_change_gbp": 0, + }, + ) + result, evidence = run(registry) + digest = content_sha256(registry.canonical_payload()) + assert evidence["exception_registry_sha256"] == digest + assert result.details["exception_registry_sha256"] == digest + assert result.evidence_sha256 == content_sha256(evidence) + + +@pytest.mark.parametrize("error", [ImportError("unavailable model"), KeyError("input")]) +def test_probe_calculation_error_records_failure(registry, monkeypatch, error): + monkeypatch.setattr(replay_probe, "version", lambda package: "2.134.7") + + def broken(spec): + raise error + + monkeypatch.setattr(replay_probe, "_run_probe", broken) + result, evidence = replay_probe.run_replay_probes( + attempt_id="attempt", + candidate_model_version="2.134.7", + exception_registry=registry, + ) + assert result.status == "fail" + assert type(error).__name__ in evidence["probes"][0]["error"] + assert result.evidence_sha256 == content_sha256(evidence) diff --git a/tests/test_replay_exception_probe_round3.py b/tests/test_replay_exception_probe_round3.py new file mode 100644 index 00000000..36d09fd8 --- /dev/null +++ b/tests/test_replay_exception_probe_round3.py @@ -0,0 +1,214 @@ +"""Round-two reproductions and round-three probe evidence hardening.""" + +from __future__ import annotations + +import copy + +import pytest +from hypothesis import given +from hypothesis import strategies as st + +from policyengine.provenance import uk_qualification +from policyengine.provenance.qualification import ( + GateResult, + ReplayQualificationRequest, + _replay_probe_passes, + qualification_verdict, +) +from policyengine.provenance.qualification_ledger import write_json +from policyengine.provenance.receipt import content_sha256 +from tests.qualification_fixtures import gates, golden, make_pair +from tests.test_qualification_contracts import _receipt_with +from tests.test_replay_exceptions import ( + TEST_ATTEMPT_ID, + _entry, + _failed_receipt, + _judge, + _probe, + _registry, +) + + +@pytest.mark.parametrize( + "change", + [ + "positive_bool", + "positive_nan", + "positive_inf", + "positive_string", + "zero_bool", + "zero_nan", + "duplicate", + ], +) +def test_incoherent_probe_values_never_authorize_exception(change): + registry = _registry(_entry()) + result, evidence = _probe(registry) + probe = evidence["probes"][0] + if change == "positive_bool": + probe["positive_change_gbp"] = True + elif change == "positive_nan": + probe["positive_change_gbp"] = float("nan") + elif change == "positive_inf": + probe["positive_change_gbp"] = float("inf") + elif change == "positive_string": + probe["positive_change_gbp"] = "6000" + elif change == "zero_bool": + probe["zero_change_gbp"] = False + elif change == "zero_nan": + probe["zero_change_gbp"] = float("nan") + else: + evidence["probes"].append(copy.deepcopy(probe)) + # Non-finite JSON cannot be sealed by the receipt's canonical encoder. + try: + evidence_digest = content_sha256(evidence) + except ValueError: + assert change in {"positive_nan", "positive_inf", "zero_nan"} + return + result = result.model_copy(update={"evidence_sha256": evidence_digest}) + actual = _judge(_failed_receipt(), registry=registry, probe=(result, evidence)) + assert actual.status == "fail" + assert actual.details["applied_exceptions"] == [] + + +@pytest.mark.parametrize( + "status", ["pass", "needs_review", "fail", "error", "unavailable"] +) +def test_probe_status_is_irrelevant_to_a_clean_candidate(status): + registry = _registry(_entry()) + result, evidence = _probe(registry) + result = result.model_copy(update={"status": status}) + clean = _receipt_with() + baseline = _judge(clean, registry=registry, probe=(None, None)) + actual = _judge(clean, registry=registry, probe=(result, evidence)) + assert actual.canonical_payload() == baseline.canonical_payload() + assert actual.status == "pass" + + +@pytest.mark.parametrize("damage", ["failed", "malformed", "wrong_gate", "missing"]) +def test_failed_or_unreadable_probe_is_not_an_unconditional_gate(tmp_path, damage): + uk_qualification.write_gate_results(tmp_path, gates(), {}) + path = tmp_path / "gates/replay_probe.result.json" + if damage == "failed": + write_json( + path, + GateResult( + gate_id="replay_probe", status="fail", summary="failed" + ).canonical_payload(), + ) + elif damage == "malformed": + path.write_text("bad json") + elif damage == "wrong_gate": + write_json( + path, + GateResult( + gate_id="foreign", status="error", summary="failed" + ).canonical_payload(), + ) + loaded = uk_qualification.load_gate_results(tmp_path) + assert sorted(loaded, key=lambda gate: gate.gate_id) == sorted( + gates(), key=lambda gate: gate.gate_id + ) + assert qualification_verdict(loaded) == "qualified" + + +@pytest.mark.parametrize( + "target_pair", + [ + make_pair(data="1.58.1"), + make_pair(dataset_sha="e" * 64), + make_pair(core="3.33.1"), + ], +) +def test_probe_cannot_cross_pairs_with_identical_model_and_attempt_name( + tmp_path, target_pair +): + source = tmp_path / "source/attempts/same-commit" + target = tmp_path / "target/attempts/same-commit" + write_json(source / "candidate.json", {"pair": make_pair().canonical_payload()}) + write_json(target / "candidate.json", {"pair": target_pair.canonical_payload()}) + result, evidence = _probe(attempt_id=uk_qualification.attempt_evidence_id(source)) + uk_qualification.write_gate_results(target, [result], {"replay_probe": evidence}) + assert uk_qualification.attempt_evidence_id( + source + ) != uk_qualification.attempt_evidence_id(target) + assert uk_qualification.load_attempt_replay_probe(target) == (None, None) + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("installed_model_version", "2.134.0"), + ("installed_model_version", None), + ("positive_baseline_gbp", 1), + ("positive_reform_gbp", 1), + ("positive_change_gbp", 6000.5), + ("zero_baseline_gbp", 1), + ("zero_reform_gbp", 1), + ], +) +def test_probe_rejects_contradictory_recorded_version_or_outputs(field, value): + registry = _registry(_entry()) + result, evidence = _probe(registry) + evidence["probes"][0][field] = value + result = result.model_copy(update={"evidence_sha256": content_sha256(evidence)}) + replay = _judge(_failed_receipt(), registry=registry, probe=(result, evidence)) + assert replay.status == "fail" + assert replay.details["applied_exceptions"] == [] + assert qualification_verdict(gates(replay=replay.status)) == "rejected" + + +@pytest.mark.parametrize( + "field", + [ + "positive_baseline_gbp", + "positive_reform_gbp", + "positive_change_gbp", + "zero_baseline_gbp", + "zero_reform_gbp", + "zero_change_gbp", + ], +) +@pytest.mark.parametrize("value", [None, False, "0", float("nan"), float("inf")]) +def test_probe_consumer_fails_closed_on_invalid_recorded_numbers(field, value): + registry = _registry(_entry()) + result, evidence = _probe(registry) + request = ReplayQualificationRequest.model_validate( + golden("qualification_request_golden") + ) + assert _replay_probe_passes( + registry.exceptions[0], request, result, evidence, TEST_ATTEMPT_ID + ) + evidence["probes"][0][field] = value + # Invalid JSON numbers cannot have a digest; the consumer must reject them + # as evidence rather than raising while attempting to rehash the document. + try: + digest = content_sha256(evidence) + except ValueError: + digest = "0" * 64 + result = result.model_copy(update={"evidence_sha256": digest}) + assert not _replay_probe_passes( + registry.exceptions[0], request, result, evidence, TEST_ATTEMPT_ID + ) + + +@given( + field=st.sampled_from( + [ + "positive_baseline_gbp", + "positive_reform_gbp", + "zero_baseline_gbp", + "zero_reform_gbp", + ] + ), + discrepancy=st.integers(min_value=1, max_value=10**9), +) +def test_any_raw_output_discrepancy_cannot_authorize_an_exception(field, discrepancy): + registry = _registry(_entry()) + result, evidence = _probe(registry) + evidence["probes"][0][field] += discrepancy + result = result.model_copy(update={"evidence_sha256": content_sha256(evidence)}) + replay = _judge(_failed_receipt(), registry=registry, probe=(result, evidence)) + assert replay.status == "fail" + assert replay.details["applied_exceptions"] == [] + assert qualification_verdict(gates(replay=replay.status)) != "qualified" diff --git a/tests/test_replay_exceptions.py b/tests/test_replay_exceptions.py new file mode 100644 index 00000000..5ffda372 --- /dev/null +++ b/tests/test_replay_exceptions.py @@ -0,0 +1,1123 @@ +"""Data-limited replay exceptions retain a review floor and expire with inputs. + +The copied real fixtures contain national aggregates and provenance metadata only. +A frozen v1 evaluator provides a differential oracle for unchanged receipt cases. +""" + +from __future__ import annotations + +import copy +import hashlib +import inspect +import json +import os +import subprocess +import sys +from pathlib import Path +from textwrap import dedent + +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st +from pydantic import ValidationError + +import policyengine.provenance.qualification as qualification +from policyengine.provenance.qualification import ( + GateResult, + ReplayDataLimitedException, + ReplayExceptionRegistry, + ReplayQualificationRequest, + ReplayReceipt, + evaluate_replay_receipt, + qualification_verdict, +) +from policyengine.provenance.receipt import content_sha256 +from tests.qualification_fixtures import FIXTURES, gates, golden, make_pair +from tests.test_qualification_contracts import ( + CONTRADICTIONS, + EVENT, + KNOWN_GAP_CASES, + MEASURE, + POLICY_CASES, + _base_event, + _clean_receipt, + _edit, + _receipt_with, + _rejected_receipt, + _sealed, + _status, + _with_requested_wheels, +) + +EXCEPTION_FIXTURES = Path(__file__).parent / "fixtures" / "replay_exceptions" +INPUT = "additional_residential_property_purchased" +TEST_ATTEMPT_ID = f"{make_pair().pair_id}/test-attempt" +RANK = {"pass": 0, "needs_review": 1, "unavailable": 2, "fail": 3, "error": 3} +EXCEPTION_METADATA = { + "exception_registry_sha256", + "applied_exceptions", + "budget_inputs_result_sha256", + "probe_result_sha256", + "probe_evidence_sha256", +} + + +def _json(name): + return json.loads((EXCEPTION_FIXTURES / name).read_text()) + + +def _main_evaluator(): + namespace = dict(vars(qualification)) + metadata = _json("main_policy_metadata.json") + namespace.update( + REPLAY_POLICY_ID=metadata["policy"], + replay_policy_sha256=lambda: metadata["policy_sha256"], + ) + source = (EXCEPTION_FIXTURES / "main_replay_policy.py.txt").read_text() + exec(compile(source, "frozen_main_replay_policy", "exec"), namespace) + return namespace["evaluate_replay_receipt"] + + +MAIN_EVALUATOR = _main_evaluator() + + +def _entry(**overrides): + return { + "id": "test-purchase-input", + "codes": ["construction_inert", "lost_response"], + "event": EVENT, + "measure_key": MEASURE, + "years": [2026], + "requires_missing_inputs": [INPUT], + "cause": "Purchases are absent from the candidate data.", + "issue": "PolicyEngine/policyengine-uk-data#573", + "decided_under": "cos d1344", + "review": "Independent review required before adoption.", + "approved_lineage": "Never supplied usably by any certified data release.", + "publication": "Not computable while the input is unavailable.", + "probe": { + "input_variable": overrides.get("requires_missing_inputs", [INPUT])[0], + "input_entity": "household", + "year": 2026, + "positive_input_gbp": 300_000, + "output_variable": "sdlt_on_residential_property_transactions", + "parameter": "gov.hmrc.stamp_duty.residential.purchase.additional.rate", + "rate_increment": 0.02, + "expected_change_gbp": 6_000, + "absolute_tolerance_gbp": 1, + "region": "LONDON", + }, + **overrides, + } + + +def _registry(*entries): + return ReplayExceptionRegistry.model_validate({"exceptions": list(entries)}) + + +def _budget_inputs( + missing=(INPUT,), + *, + status="needs_review", + newly_missing=None, + previous_status=None, + attempt_id=TEST_ATTEMPT_ID, + request=None, +): + previous = ( + { + name: { + "in_model": False, + "is_input": False, + "in_data": False, + "usable": False, + } + for name in {*missing, INPUT} + } + if previous_status is None + else { + name: {"is_input": state.get("in_model", False), **state} + for name, state in previous_status.items() + } + ) + previous_ever = sorted( + name for name, state in previous.items() if state.get("usable") + ) + states = { + name: { + "in_model": True, + "is_input": True, + "in_data": name not in missing, + "usable": name not in missing, + } + for name in {*missing, INPUT} + } + evidence = { + "pair_id": attempt_id.partition("/")[0], + "attempt_id": attempt_id, + "input_status": states, + "previous_input_status": previous, + "previous_ever_usable_inputs": previous_ever, + "ever_usable_inputs": sorted( + set(previous_ever) + | {name for name, state in states.items() if state["usable"]} + ), + "ever_usable_inputs_basis": "seeded from direct predecessor", + "missing_inputs": sorted(missing), + "newly_missing_inputs": sorted( + missing if newly_missing is None else newly_missing + ), + } + result = GateResult( + gate_id="budget_inputs", + status=status, + summary="Candidate input coverage.", + evidence_sha256=content_sha256(evidence), + details={ + "missing_inputs": len(missing), + "newly_missing_inputs": evidence["newly_missing_inputs"], + "ever_usable_inputs": evidence["ever_usable_inputs"], + }, + ) + return result, evidence + + +def _probe(registry=None, request=None, *, attempt_id=TEST_ATTEMPT_ID): + """Build same-attempt evidence matching every registered probe specification.""" + if registry is None: + registry, _ = qualification._replay_exceptions() + if request is None: + request = ReplayQualificationRequest.model_validate( + golden("qualification_request_golden") + ) + evidence = { + "attempt_id": attempt_id, + "candidate_model_version": request.candidate.policyengine_uk.version, + "probes": [ + { + "exception_id": entry.id, + "spec_sha256": content_sha256(entry.probe.model_dump(mode="json")), + "status": "pass", + "installed_model_version": request.candidate.policyengine_uk.version, + "positive_baseline_gbp": 0, + "positive_reform_gbp": entry.probe.expected_change_gbp, + "zero_baseline_gbp": 0, + "zero_reform_gbp": 0, + "positive_change_gbp": entry.probe.expected_change_gbp, + "zero_change_gbp": 0, + } + for entry in registry.exceptions + ], + } + return ( + GateResult( + gate_id="replay_probe", + status="pass", + summary="Synthetic positive-input and zero-input probes passed.", + evidence_sha256=content_sha256(evidence), + ), + evidence, + ) + + +def _failed_receipt(year=2026, *, extra_failures=(), rows=None): + rows = ( + [{"flags": ["structural", "inertness"], "base_gbp": 2.5e9, "head_gbp": 0}] + if rows is None + else rows + ) + document = _receipt_with( + [{"year": year, **row} for row in rows], + inert=[(MEASURE, year)], + missing=[], + complete=False, + integrity="failed", + sealed=False, + ) + document["status"] = "failed" + document["integrity"]["failures"] = [ + { + "code": "construction_inert", + "detail": "Identical worlds because the input is absent.", + "event": EVENT, + "measure_key": MEASURE, + "year": year, + }, + *extra_failures, + ] + return _sealed(document) + + +def _judge( + document, + *, + registry=None, + budget=None, + probe="default", + request=None, + evaluator=None, + attempt_id=TEST_ATTEMPT_ID, +): + request = request or ReplayQualificationRequest.model_validate( + golden("qualification_request_golden") + ) + result, evidence = budget if budget is not None else _budget_inputs() + evaluator = evaluator or evaluate_replay_receipt + registry = registry if registry is not None else _registry(_entry()) + probe_result, probe_evidence = ( + _probe(registry, request, attempt_id=attempt_id) + if probe == "default" + else probe + ) + return evaluator( + request, + _status(request.request_id, document), + ReplayReceipt.from_document(document), + exception_registry=registry, + budget_inputs_result=result, + budget_inputs_evidence=evidence, + probe_result=probe_result, + probe_evidence=probe_evidence, + attempt_id=attempt_id, + ) + + +def _legacy_result(document, request=None): + request = request or ReplayQualificationRequest.model_validate( + golden("qualification_request_golden") + ) + return MAIN_EVALUATOR( + request, + _status(request.request_id, document), + ReplayReceipt.from_document(document), + ) + + +def _unchanged_payload(gate): + """Policy version and mandatory new provenance fields are the only allowance.""" + payload = gate.model_dump(mode="json") + for key in EXCEPTION_METADATA | {"policy", "policy_sha256"}: + payload["details"].pop(key, None) + return payload + + +def _assert_narrow_mismatch_unchanged( + dimension, year=2026, noise=(), *, evaluator=None +): + entry = _entry(years=[year]) + document = _failed_receipt(year) + budget = _budget_inputs([INPUT, *noise]) + if dimension == "code": + entry["codes"] = ["lost_response"] + elif dimension == "event": + entry["event"] = "another_event" + elif dimension == "measure_key": + entry["measure_key"] = "another_measure" + elif dimension == "year": + entry["years"] = [year + 1] + elif dimension == "missing_input": + budget = _budget_inputs(noise) + elif dimension == "extra_failure": + document["integrity"]["failures"].append( + {"code": "event_outputs_missing", "detail": "Unrelated missing output."} + ) + else: + raise AssertionError(dimension) + document = _sealed(document) + gate = _judge( + document, registry=_registry(entry), budget=budget, evaluator=evaluator + ) + legacy = _legacy_result(document) + assert gate.status == legacy.status == "fail" + assert _unchanged_payload(gate) == _unchanged_payload(legacy) + + +@settings(max_examples=65, deadline=None) +@given( + year=st.integers(2025, 2029), + other_years=st.sets(st.integers(2024, 2035), max_size=5), + missing=st.sets(st.sampled_from([INPUT, "other_input", "persistent_input"])), + distractor_count=st.integers(0, 3), + status=st.sampled_from(["pass", "needs_review", "fail"]), +) +def test_applied_exceptions_always_require_a_named_review( + year, other_years, missing, distractor_count, status +): + """Invariant 1: arbitrary valid registries and input results retain the floor.""" + missing = sorted(missing | {INPUT}) + registry = _registry( + _entry(years=sorted(other_years | {year})), + *[ + _entry(id=f"unrelated-{i}", event=f"unrelated_event_{i}") + for i in range(distractor_count) + ], + ) + gate = _judge( + _failed_receipt(year), + registry=registry, + budget=_budget_inputs(missing, status=status), + ) + assert gate.status == "needs_review", gate.summary + assert qualification_verdict(gates(replay=gate.status)) == "needs_review" + assert len(gate.details["applied_exceptions"]) == 1 + assert gate.details["applied_exceptions"][0]["year"] == year + assert set(gate.details["applied_exceptions"][0]["codes"]) == { + "construction_inert", + "lost_response", + } + items = gate.details["review_items"] + assert any( + item.get("exception_id") == "test-purchase-input" + and item.get("publication") == _entry()["publication"] + and item.get("approved_lineage") == _entry()["approved_lineage"] + and item.get("cause") == _entry()["cause"] + and INPUT in item.get("requires_missing_inputs", []) + for item in items + ) + + +@settings(max_examples=90, deadline=None) +@given( + dimension=st.sampled_from( + ["code", "event", "measure_key", "year", "missing_input", "extra_failure"] + ), + year=st.integers(2025, 2029), + noise=st.sets(st.sampled_from(["irrelevant_input", "other_input"])), +) +def test_unmatched_fields_and_extra_integrity_failures_are_noops( + dimension, year, noise +): + """Invariant 2: each narrow mismatch preserves the old failed verdict.""" + _assert_narrow_mismatch_unchanged(dimension, year, sorted(noise)) + + +def _existing_receipt_cases(): + cases = [] + for name, case in POLICY_CASES.items(): + rows, inert, complete, integrity, *_ = case + cases.append((name, _receipt_with(rows, inert, complete, integrity), None)) + for name, case in KNOWN_GAP_CASES.items(): + inert, missing, unsupported, base_events, *_ = case + cases.append( + ( + f"known-gap: {name}", + _receipt_with( + inert=inert, + missing=missing, + unsupported=unsupported, + base_events=base_events, + complete=not (inert or missing), + integrity="incomplete" if inert or missing else "complete", + ), + None, + ) + ) + for name, edit in CONTRADICTIONS.items(): + cases.append((name, _edit(_clean_receipt(), edit), None)) + cases.extend( + [ + ("rejected", _rejected_receipt(), None), + ("failed", _failed_receipt(), None), + ( + "golden", + _sealed(_with_requested_wheels(golden("replay_receipt_golden"))), + None, + ), + ] + ) + for folder in [FIXTURES / "receipt_6.4.1_attempt1", FIXTURES / "receipt_6.5.0"]: + request = ReplayQualificationRequest.model_validate( + json.loads((folder / "REQUEST.json").read_text()) + ) + cases.append( + ( + folder.name, + json.loads((folder / "REPLAY_RECEIPT.json").read_text()), + request, + ) + ) + return cases + + +@pytest.mark.parametrize( + ("name", "document", "replay_request"), + _existing_receipt_cases(), + ids=[case[0] for case in _existing_receipt_cases()], +) +def test_empty_registry_preserves_existing_replay_receipts( + name, document, replay_request +): + """Invariant 3: all existing decision-table and real-fixture receipt cases.""" + current = _judge(document, request=replay_request, registry=_registry()) + legacy = _legacy_result(document, request=replay_request) + assert _unchanged_payload(current) == _unchanged_payload(legacy), name + + +@settings(max_examples=80, deadline=None) +@given( + year=st.integers(2025, 2029), + included=st.booleans(), + extra_kind=st.sampled_from(["mapping", "lost_response"]), + extra_year=st.integers(2024, 2035), + extra_base=st.sampled_from([1e6, -1e6, 2.5e9]), +) +def test_adding_a_regression_never_improves_a_verdict( + year, included, extra_kind, extra_year, extra_base +): + """Invariant 4 covers both matched and unmatched new regression rows.""" + registry = _registry(_entry(years=[year] if included else [year + 1])) + document = _failed_receipt(year) + before = _judge(document, registry=registry) + row = { + **document["diff"]["flagged_rows"][0], + "flags": ["mapping" if extra_kind == "mapping" else "structural"], + "year": extra_year, + "base_gbp": extra_base, + "head_gbp": 0, + } + document["diff"]["flagged_rows"].append(row) + for flag in row["flags"]: + document["diff"]["flag_counts"][flag] += 1 + after = _judge(_sealed(document), registry=registry) + assert RANK[after.status] >= RANK[before.status] + if extra_kind == "mapping" or extra_year != year: + assert after.status == "fail" + + +@settings(max_examples=45, deadline=None) +@given( + years=st.sets(st.integers(2025, 2029), min_size=1), + missing=st.sets(st.sampled_from([INPUT, "another_input"])), + codes=st.lists( + st.sampled_from(["construction_inert", "lost_response"]), + min_size=1, + unique=True, + ), +) +def test_exception_evaluation_and_recorded_digests_are_deterministic( + years, missing, codes +): + """Invariant 5 compares every result field, including all provenance digests.""" + registry = _registry(_entry(years=sorted(years), codes=codes)) + document = _failed_receipt(min(years)) + budget = _budget_inputs(sorted(missing)) + first = _judge(document, registry=registry, budget=budget) + second = _judge(copy.deepcopy(document), registry=registry, budget=budget) + assert first.model_dump(mode="json") == second.model_dump(mode="json") + assert first.details["exception_registry_sha256"] == content_sha256( + registry.canonical_payload() + ) + if first.details["applied_exceptions"]: + assert first.details["budget_inputs_result_sha256"] == content_sha256( + budget[0].canonical_payload() + ) + else: + assert first.details["budget_inputs_result_sha256"] is None + + +def _real_receipt(): + """Recreate the unavailable REQUEST from execution, retaining original aggregates. + + The archived fixture has no request document. Its bytes and digest remain + unchanged on disk; only a test copy receives this reconstructed request id. + """ + document = _json("replay_receipt_uk2.134.7_failed.json") + raw = golden("qualification_request_golden") + execution = document["execution"] + candidate = raw["candidate"] + candidate["policyengine"].update( + version=execution["packages"]["policyengine"]["version"], + packaged_bundle_manifest_sha256=execution["packaged_bundle_manifest_sha256"], + ) + for name, field in [ + ("policyengine-uk", "policyengine_uk"), + ("policyengine-core", "policyengine_core"), + ]: + candidate[field] = { + key: execution["packages"][name][key] for key in ("version", "wheel_sha256") + } + candidate["data"]["sha256"] = execution["dataset_sha256"] + candidate["expected_release_bundle"] = execution["release_bundle"] + request = ReplayQualificationRequest.model_validate(raw) + document["request_id"] = request.request_id + return request, _sealed(document) + + +def _real_budget(*, attempt_id=TEST_ATTEMPT_ID, request=None): + """Add explicit never-usable lineage to copies of the archived evidence.""" + evidence = _json("uk2.134.7_budget_inputs.json") + evidence.update( + { + "pair_id": attempt_id.partition("/")[0], + "attempt_id": attempt_id, + "input_status": { + name: { + "in_model": True, + "is_input": True, + "in_data": False, + "usable": False, + } + for name in evidence["missing_inputs"] + }, + "previous_input_status": { + INPUT: { + "in_model": False, + "is_input": False, + "in_data": False, + "usable": False, + } + }, + "previous_ever_usable_inputs": [], + "ever_usable_inputs": [], + "ever_usable_inputs_basis": "seeded from direct predecessor", + } + ) + result = GateResult.model_validate(_json("uk2.134.7_budget_inputs.result.json")) + return result.model_copy( + update={ + "evidence_sha256": content_sha256(evidence), + "details": { + **result.details, + "ever_usable_inputs": evidence["ever_usable_inputs"], + }, + } + ), evidence + + +def test_real_21347_failed_receipt_has_exactly_five_reviewed_measure_years(): + """Invariant 6: the production registry handles the real data limitation.""" + request, document = _real_receipt() + budget = _real_budget() + receipt = ReplayReceipt.from_document(document) + probe_result, probe_evidence = _probe(request=request) + gate = evaluate_replay_receipt( + request, + _status(request.request_id, document), + receipt, + budget_inputs_result=budget[0], + budget_inputs_evidence=budget[1], + probe_result=probe_result, + probe_evidence=probe_evidence, + attempt_id=TEST_ATTEMPT_ID, + ) + assert gate.status == "needs_review", gate.summary + applied = gate.details["applied_exceptions"] + assert len(applied) == 5 + assert {item["id"] for item in applied} == {"ab2024-sdlt-hrad-purchase-input"} + assert sorted(item["year"] for item in applied) == list(range(2025, 2030)) + assert all( + set(item["codes"]) == {"construction_inert", "lost_response"} + for item in applied + ) + assert gate.details["regressions"] == [] + assert all(item["publication"] for item in applied) + assert gate.details["probe_result_sha256"] == content_sha256( + probe_result.canonical_payload() + ) + assert gate.details["probe_evidence_sha256"] == content_sha256(probe_evidence) + assert gate.details["budget_inputs_result_sha256"] == content_sha256( + budget[0].canonical_payload() + ) + registry_file = ( + Path(qualification.__file__).parents[1] + / "data/qualification/uk_replay_exceptions.json" + ) + assert ( + gate.details["exception_registry_sha256"] + == hashlib.sha256(registry_file.read_bytes()).hexdigest() + ) + + +@pytest.mark.parametrize("change", ["input_available", "extra_integrity_failure"]) +def test_real_21347_receipt_fails_when_exception_conditions_no_longer_hold(change): + request, document = _real_receipt() + result, evidence = _real_budget() + if change == "input_available": + evidence["missing_inputs"].remove(INPUT) + evidence["newly_missing_inputs"].remove(INPUT) + details = copy.deepcopy(result.details) + details["newly_missing_inputs"].remove(INPUT) + details["missing_inputs"] -= 1 + result = result.model_copy( + update={"details": details, "evidence_sha256": content_sha256(evidence)} + ) + else: + document["integrity"]["failures"].append( + {"code": "event_outputs_missing", "detail": "Synthetic unrelated failure."} + ) + document = _sealed(document) + probe_result, probe_evidence = _probe(request=request) + gate = evaluate_replay_receipt( + request, + _status(request.request_id, document), + ReplayReceipt.from_document(document), + budget_inputs_result=result, + budget_inputs_evidence=evidence, + probe_result=probe_result, + probe_evidence=probe_evidence, + attempt_id=TEST_ATTEMPT_ID, + ) + assert gate.status == "fail" + + +@pytest.mark.parametrize("status", ["error", "unavailable"]) +def test_unavailable_budget_inputs_cannot_authorize_an_exception(status): + gate = _judge(_failed_receipt(), budget=_budget_inputs(status=status)) + assert gate.status == "fail" + + +@pytest.mark.parametrize( + "change", ["wrong_gate", "wrong_digest", "missing_result", "missing_evidence"] +) +def test_unbound_budget_input_evidence_cannot_authorize_an_exception(change): + result, evidence = _budget_inputs() + if change == "wrong_gate": + result = result.model_copy(update={"gate_id": "some_other_gate"}) + elif change == "wrong_digest": + result = result.model_copy(update={"evidence_sha256": "0" * 64}) + elif change == "missing_result": + result = None + else: + evidence = None + assert _judge(_failed_receipt(), budget=(result, evidence)).status == "fail" + + +def test_inputs_missing_in_both_pairs_still_authorize_review(): + gate = _judge(_failed_receipt(), budget=_budget_inputs(newly_missing=[])) + assert gate.status == "needs_review" + + +def test_exception_never_excuses_a_base_gap_with_a_material_value(): + document = _failed_receipt() + document["diff"]["base_events"] = {EVENT: _base_event(inert=[(MEASURE, 2026)])} + assert _judge(_sealed(document)).status == "fail" + + +def test_exception_never_excuses_failures_under_complete_integrity(): + document = _failed_receipt() + document["status"] = "complete" + document["integrity"]["status"] = "complete" + assert _judge(_sealed(document)).status == "fail" + + +@pytest.mark.parametrize("codes", [["event_run_failed"], ["mapping"], ["incomplete"]]) +def test_registry_cannot_expand_the_exception_to_other_failure_kinds(codes): + with pytest.raises(ValidationError): + _registry(_entry(codes=codes)) + + +@pytest.mark.parametrize( + "dropped", ["code", "event", "measure_key", "year", "missing_input"] +) +def test_narrow_match_invariants_kill_five_predicate_mutants(monkeypatch, dropped): + """Execute actual matcher replacements; the invariant assertion kills each. + + Mutants remove the code, event, measure key, year, or missing-input predicate. + The unmodified invariant is exercised above by Hypothesis as well. + """ + + def mutant(self, code, event, measure_key, year, missing_inputs): + predicates = { + "code": code in self.codes, + "event": event == self.event, + "measure_key": measure_key == self.measure_key, + "year": year in self.years, + "missing_input": set(self.requires_missing_inputs) <= missing_inputs, + } + return all(value for key, value in predicates.items() if key != dropped) + + monkeypatch.setattr(ReplayDataLimitedException, "matches", mutant) + with pytest.raises(AssertionError): + _assert_narrow_mismatch_unchanged(dropped) + + +def test_registry_preserves_comment_through_json_roundtrip(): + registry = ReplayExceptionRegistry.model_validate( + {"_comment": "Reviewed like code.", "exceptions": [_entry()]} + ) + assert ( + ReplayExceptionRegistry.model_validate_json(registry.model_dump_json()) + == registry + ) + payload = registry.canonical_payload() + assert payload["_comment"] == "Reviewed like code." + assert ReplayExceptionRegistry.model_validate(payload) == registry + + +@pytest.mark.parametrize("code", ["lost_response", "event_run_failed", "mapping"]) +def test_only_construction_inert_can_be_excepted_as_a_receipt_failure(code): + document = _failed_receipt() + document["integrity"]["failures"][0]["code"] = code + assert _judge(_sealed(document)).status == "fail" + + +def test_a_missing_result_is_not_excused_when_it_is_also_listed_inert(): + document = _failed_receipt() + document["corpus"]["events"][EVENT]["missing_measure_years"] = [ + {"measure_key": MEASURE, "year": 2026} + ] + assert _judge(_sealed(document)).status == "fail" + + +def _source_mutant(before, after, function=evaluate_replay_receipt): + source = dedent(inspect.getsource(function)) + assert source.count(before) == 1, ( + "Mutation site changed; update this explicit mutant." + ) + namespace = dict(vars(qualification)) + exec( + compile(source.replace(before, after), "mutant_replay_policy", "exec"), + namespace, + ) + return namespace[function.__name__] + + +def test_extra_failure_invariant_kills_all_to_any_mutant(): + mutant = _source_mutant("and all(\n", "and any(\n") + with pytest.raises(AssertionError): + _assert_narrow_mismatch_unchanged("extra_failure", evaluator=mutant) + + +def test_review_floor_invariant_kills_automatic_pass_mutant(): + mutant = _source_mutant( + 'if review_items:\n return GateResult(\n gate_id="replay",\n status="needs_review",', + 'if review_items:\n return GateResult(\n gate_id="replay",\n status="pass",', + ) + with pytest.raises(AssertionError): + gate = _judge(_failed_receipt(rows=[]), evaluator=mutant) + assert qualification_verdict(gates(replay=gate.status)) != "qualified" + + +def test_every_existing_replay_evaluation_matches_main_with_an_empty_registry(tmp_path): + """Invariant 3 also covers each generated receipt in the full existing suite. + + A child pytest wraps the suite's evaluator; every invocation is compared + with the frozen main policy, including receipts created by Hypothesis. + """ + plugin = tmp_path / "replay_differential_plugin.py" + plugin.write_text( + """from tests.test_replay_exceptions import ( + MAIN_EVALUATOR, _registry, _unchanged_payload, +) +from policyengine.provenance.qualification import evaluate_replay_receipt + +count = 0 + +def pytest_collection_modifyitems(items): + modules = {item.module for item in items} + for module in modules: + if module.__name__ == "tests.test_qualification_contracts": + module.evaluate_replay_receipt = compare + +def compare(*args, **kwargs): + global count + actual = evaluate_replay_receipt(*args, **kwargs, exception_registry=_registry()) + expected = MAIN_EVALUATOR(*args) + assert _unchanged_payload(actual) == _unchanged_payload(expected) + count += 1 + return actual + +def pytest_terminal_summary(terminalreporter): + assert count > 0 + terminalreporter.write_line(f"Differential replay evaluations: {count}") +""" + ) + environment = dict(os.environ) + environment["POLICYENGINE_SKIP_COUNTRY_IMPORTS"] = "1" + environment["PYTHONPATH"] = os.pathsep.join( + [str(tmp_path), environment.get("PYTHONPATH", "")] + ) + run = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "--noconftest", + "tests/test_qualification_contracts.py", + "-p", + "replay_differential_plugin", + "-q", + ], + cwd=Path(__file__).parents[1], + env=environment, + capture_output=True, + text=True, + timeout=120, + check=False, + ) + assert run.returncode == 0, run.stdout + run.stderr + assert "Differential replay evaluations:" in run.stdout + + +def _assert_every_required_input_missing(required, missing, *, year=2026): + document = _failed_receipt(year) + gate = _judge( + document, + registry=_registry(_entry(years=[year], requires_missing_inputs=required)), + budget=_budget_inputs(sorted(missing)), + ) + if set(required) <= set(missing): + assert gate.status == "needs_review" + assert gate.details["applied_exceptions"] + assert set( + gate.details["applied_exceptions"][0]["requires_missing_inputs"] + ) == set(required) + else: + assert gate.status == "fail" + assert _unchanged_payload(gate) == _unchanged_payload(_legacy_result(document)) + + +@settings(max_examples=80, deadline=None) +@given( + required=st.lists( + st.sampled_from([INPUT, "second_required_input", "third_required_input"]), + min_size=2, + max_size=3, + unique=True, + ), + missing=st.sets( + st.sampled_from( + [INPUT, "second_required_input", "third_required_input", "unrelated_input"] + ) + ), + year=st.integers(2025, 2029), +) +def test_every_required_missing_input_must_be_reported_missing(required, missing, year): + """Invariant 2 requires all names, including multi-input registry entries.""" + _assert_every_required_input_missing(required, missing, year=year) + + +def test_required_input_invariant_kills_subset_to_any_mutant(monkeypatch): + mutant = _source_mutant( + "set(self.requires_missing_inputs) <= missing_inputs", + "bool(set(self.requires_missing_inputs) & missing_inputs)", + function=ReplayDataLimitedException.matches, + ) + monkeypatch.setattr(ReplayDataLimitedException, "matches", mutant) + with pytest.raises(AssertionError): + _assert_every_required_input_missing([INPUT, "second_required_input"], [INPUT]) + + +def _assert_diff_only_exception(*, input_missing=True, base=2.5e9, evaluator=None): + document = _receipt_with( + rows=[{"flags": ["structural"], "base_gbp": base, "head_gbp": 0}], + inert=[], + complete=True, + integrity="complete", + ) + gate = _judge( + document, + registry=_registry(_entry(codes=["lost_response"])), + budget=_budget_inputs([INPUT] if input_missing else []), + evaluator=evaluator, + ) + if input_missing: + assert gate.status == "needs_review" + assert len(gate.details["applied_exceptions"]) == 1 + assert gate.details["applied_exceptions"][0]["codes"] == ["lost_response"] + assert gate.details["regressions"] == [] + else: + assert gate.status == "fail" + assert _unchanged_payload(gate) == _unchanged_payload(_legacy_result(document)) + + +@pytest.mark.parametrize("input_missing", [True, False]) +@pytest.mark.parametrize("base", [1e6, -1e6, 2.5e9]) +def test_diff_only_lost_response_requires_the_missing_input(input_missing, base): + """The £0 total branch works without corpus inertness or integrity failures.""" + _assert_diff_only_exception(input_missing=input_missing, base=base) + + +def test_diff_only_invariant_kills_mutant_removing_zero_total_exception(): + mutant = _source_mutant( + 'if not except_regression(record, "lost_response"):', "if True:" + ) + with pytest.raises(AssertionError): + _assert_diff_only_exception(evaluator=mutant) + + +@settings(max_examples=80, deadline=None) +@given( + value=st.floats(allow_nan=False, allow_infinity=False).filter( + lambda value: value != 0 + ), + level=st.sampled_from(["measure_year", "source_row"]), + base_known_gap=st.booleans(), +) +def test_any_nonzero_candidate_effect_prevents_an_exception( + value, level, base_known_gap +): + document = _failed_receipt() + row = copy.deepcopy(document["diff"]["flagged_rows"][0]) + row.update(level=level, key="total" if level == "measure_year" else "source-1") + row["head_gbp"] = value + if level == "measure_year": + document["diff"]["flagged_rows"][0] = row + else: + document["diff"]["flagged_rows"].append(row) + for flag in row["flags"]: + document["diff"]["flag_counts"][flag] += 1 + if base_known_gap: + document["diff"]["base_events"] = {EVENT: _base_event(inert=[(MEASURE, 2026)])} + gate = _judge(_sealed(document)) + assert gate.status == "fail" + assert gate.details["applied_exceptions"] == [] + assert gate.details["regressions"] + + +@settings(max_examples=40, deadline=None) +@given( + year=st.integers(2025, 2029), + missing=st.sets(st.sampled_from([INPUT, "another_input", "noise"])), +) +def test_a_previously_usable_input_never_authorizes_the_exception(year, missing): + gate = _judge( + _failed_receipt(year), + registry=_registry(_entry(years=[year])), + budget=_budget_inputs( + sorted(missing | {INPUT}), + previous_status={ + INPUT: {"in_model": True, "in_data": True, "usable": True} + }, + ), + ) + assert gate.status == "fail" + assert gate.details["applied_exceptions"] == [] + assert "fresh registry decision" in str(gate.model_dump()).lower() + + +def test_missing_previous_input_lineage_fails_closed(): + gate = _judge(_failed_receipt(), budget=_budget_inputs(previous_status={})) + assert gate.status == "fail" + assert gate.details["applied_exceptions"] == [] + + +def _invalid_probe(change, registry=None, request=None): + result, evidence = _probe(registry or _registry(_entry()), request) + if change == "missing_result": + return None, evidence + if change == "missing_evidence": + return result, None + if change == "wrong_digest": + return result.model_copy(update={"evidence_sha256": "0" * 64}), evidence + if change == "wrong_gate": + return result.model_copy(update={"gate_id": "other_probe"}), evidence + if change == "failed_result": + return result.model_copy(update={"status": "fail"}), evidence + if change == "failed_probe": + evidence["probes"][0]["status"] = "fail" + elif change == "missing_probe": + evidence["probes"] = [] + elif change == "wrong_spec": + evidence["probes"][0]["spec_sha256"] = "0" * 64 + elif change == "wrong_attempt": + evidence["attempt_id"] = "another-attempt" + elif change == "wrong_model": + evidence["candidate_model_version"] = "0.0.0" + elif change == "zero_control_moves": + evidence["probes"][0]["zero_change_gbp"] = 1 + elif change == "positive_control_inert": + evidence["probes"][0]["positive_change_gbp"] = 0 + else: + raise AssertionError(change) + return result.model_copy( + update={"evidence_sha256": content_sha256(evidence)} + ), evidence + + +PROBE_FAILURES = [ + "missing_result", + "missing_evidence", + "wrong_digest", + "wrong_gate", + "failed_result", + "failed_probe", + "missing_probe", + "wrong_spec", + "wrong_attempt", + "wrong_model", + "zero_control_moves", + "positive_control_inert", +] + + +@settings(max_examples=45, deadline=None) +@given(change=st.sampled_from(PROBE_FAILURES), year=st.integers(2025, 2029)) +def test_failing_missing_or_unbound_probe_never_authorizes_an_exception(change, year): + registry = _registry(_entry(years=[year])) + gate = _judge( + _failed_receipt(year), registry=registry, probe=_invalid_probe(change, registry) + ) + assert gate.status == "fail" + assert gate.details["applied_exceptions"] == [] + + +def test_real_21347_receipt_fails_without_a_positive_input_probe(): + request, document = _real_receipt() + registry, _ = qualification._replay_exceptions() + gate = _judge( + document, + registry=registry, + request=request, + budget=_real_budget(), + probe=(None, None), + ) + assert gate.status == "fail" + assert gate.details["applied_exceptions"] == [] + + +def test_nonzero_effect_invariant_kills_skip_coherence_mutant(): + mutant = _source_mutant( + " and row.head_gbp not in (None, 0)\n ]", + " and False\n ]", + ) + document = _failed_receipt( + rows=[ + { + "flags": ["structural", "inertness"], + "base_gbp": 2.5e9, + "head_gbp": 1_000_000_000, + } + ] + ) + gate = _judge(document, evaluator=mutant) + with pytest.raises(AssertionError): + assert gate.status == "fail" + + +def test_previously_usable_invariant_kills_skip_lineage_mutant(): + mutant = _source_mutant("if lineage_problem is not None:", "if False:") + budget = _budget_inputs( + previous_status={ + INPUT: {"in_model": True, "in_data": True, "usable": True}, + } + ) + gate = _judge(_failed_receipt(), budget=budget, evaluator=mutant) + with pytest.raises(AssertionError): + assert gate.status == "fail" + + +def test_failing_probe_invariant_kills_skip_probe_mutant(): + mutant = _source_mutant( + """if not _replay_probe_passes( + entry, request, probe_result, probe_evidence, attempt_id + ):""", + "if False:", + ) + gate = _judge( + _failed_receipt(), evaluator=mutant, probe=_invalid_probe("failed_probe") + ) + with pytest.raises(AssertionError): + assert gate.status == "fail" + + +def test_same_attempt_invariant_kills_accept_another_attempt_mutant(monkeypatch): + mutant = _source_mutant( + ' or evidence.get("attempt_id") != attempt_id\n', + "", + function=qualification._replay_probe_passes, + ) + monkeypatch.setattr(qualification, "_replay_probe_passes", mutant) + gate = _judge(_failed_receipt(), probe=_invalid_probe("wrong_attempt")) + with pytest.raises(AssertionError): + assert gate.status == "fail" diff --git a/tests/test_uk_candidate_gates.py b/tests/test_uk_candidate_gates.py index d5bac838..b6407498 100644 --- a/tests/test_uk_candidate_gates.py +++ b/tests/test_uk_candidate_gates.py @@ -2668,6 +2668,11 @@ def function(): def _write_previous_receipt(root, pair, evidence, *, attempt="a"): from policyengine.provenance.qualification_ledger import write_json + if "ever_usable_inputs" in evidence.get("budget_inputs", {}): + inputs = dict(evidence["budget_inputs"]) + inputs.setdefault("pair_id", pair.pair_id) + inputs.setdefault("attempt_id", f"{pair.pair_id}/{attempt}") + evidence = {**evidence, "budget_inputs": inputs} receipt = make_receipt(pair) receipt = receipt.model_copy( update={ @@ -2676,7 +2681,20 @@ def _write_previous_receipt(root, pair, evidence, *, attempt="a"): update={ "evidence_sha256": gates_module.evidence_sha256( evidence[gate.gate_id] - ) + ), + "details": { + **gate.details, + **( + { + "ever_usable_inputs": evidence[gate.gate_id][ + "ever_usable_inputs" + ] + } + if gate.gate_id == "budget_inputs" + and "ever_usable_inputs" in evidence[gate.gate_id] + else {} + ), + }, } ) if gate.gate_id in evidence @@ -2754,6 +2772,78 @@ def test_previous_evidence_belongs_to_the_receipts_attempt(tmp_path): ) +@pytest.mark.parametrize( + "damage", + [ + "none", + "empty_status", + "missing_counts", + "wrong_model_count", + "wrong_input_count", + "wrong_data_count", + "missing_list", + ], +) +def test_previous_lineage_requires_a_complete_variable_enumeration(tmp_path, damage): + name = "additional_residential_property_purchased" + inputs = { + "model_variables": 1, + "model_input_variables": 1, + "dataset_columns": 0, + "missing_inputs": [name], + "unusable_critical": [], + "input_status": { + name: { + "in_model": True, + "is_input": True, + "in_data": False, + "usable": False, + } + }, + } + if damage == "empty_status": + inputs["input_status"] = {} + elif damage == "missing_counts": + inputs.pop("model_variables") + elif damage == "wrong_model_count": + inputs["model_variables"] = 2 + elif damage == "wrong_input_count": + inputs["model_input_variables"] = 2 + elif damage == "wrong_data_count": + inputs["dataset_columns"] = 1 + elif damage == "missing_list": + inputs["missing_inputs"] = [] + evidence = { + "supported_years": {"supported_years": [2025, 2026]}, + "budget_inputs": inputs, + } + pair = make_pair() + _write_previous_receipt(tmp_path, pair, evidence) + actual = gates_module._previous_gate_evidence(tmp_path, _previous_identity(pair)) + assert actual == (evidence if damage == "none" else {}) + + +def test_formula_backed_supplied_variable_is_recorded_as_previously_usable(tmp_path): + _, evidence = budget_inputs( + make_pair(), + tmp_path / "unused.h5", + model_input_variables=["pure_input"], + model_variables=["pure_input", "formula_backed_variable"], + critical_inputs=[], + accepted_gaps={}, + previous_missing_inputs=None, + previous_unusable_critical=None, + tables={"household": pd.DataFrame({"formula_backed_variable": [0, 300_000]})}, + ) + assert evidence["input_status"]["formula_backed_variable"] == { + "in_model": True, + "is_input": False, + "in_data": True, + "usable": True, + } + assert evidence["model_variables"] == 2 + + @pytest.mark.parametrize( "corruption", ["digest", "shape", "receipt", "missing identity"] ) @@ -2780,12 +2870,18 @@ def test_unverified_previous_evidence_is_recomputed(tmp_path, corruption): else: previous.pop("qualification_pair_id") previous.pop("core_wheel_sha256") - assert gates_module._previous_gate_evidence(tmp_path, previous) == {} + if corruption == "receipt": + with pytest.raises(gates_module.PreviousPairUnavailable): + gates_module._previous_gate_evidence(tmp_path, previous) + else: + assert gates_module._previous_gate_evidence(tmp_path, previous) == {} -@pytest.mark.parametrize("corrupt_baseline", [False, True]) +@pytest.mark.parametrize( + "baseline_kind", ["valid", "corrupt", "legacy", "legacy_history"] +) def test_orchestration_hashes_every_finalized_gate_evidence( - tmp_path, monkeypatch, corrupt_baseline + tmp_path, monkeypatch, baseline_kind ): from policyengine.provenance import dataset_materialization from policyengine.provenance.qualification_ledger import write_json @@ -2807,11 +2903,26 @@ def test_orchestration_hashes_every_finalized_gate_evidence( SimpleNamespace(system=SimpleNamespace(variables={})), ) baseline = { - "budget_inputs": {"missing_inputs": [], "unusable_critical": []}, + "budget_inputs": { + "missing_inputs": [], + "unusable_critical": [], + "input_status": {}, + "model_variables": 0, + "model_input_variables": 0, + "dataset_columns": 0, + "ever_usable_inputs": [], + }, "supported_years": {"supported_years": [2025]}, } + if baseline_kind == "legacy": + baseline["budget_inputs"].pop("input_status") + baseline["budget_inputs"].pop("ever_usable_inputs") + elif baseline_kind == "legacy_history": + baseline["budget_inputs"].pop("ever_usable_inputs") + elif baseline_kind == "corrupt": + baseline["budget_inputs"].pop("ever_usable_inputs") _, attempt = _write_previous_receipt(tmp_path, pair, baseline) - if corrupt_baseline: + if baseline_kind == "corrupt": write_json( attempt / "gates" / "supported_years.json", {"supported_years": [2024]} ) @@ -2823,6 +2934,7 @@ def snapshot(*args, **kwargs): "missing_inputs": [], "unusable_critical": [], "supported_years": [2025], + "input_status": {}, } monkeypatch.setattr(gates_module, "run_inputs_snapshot", snapshot) @@ -2888,10 +3000,10 @@ def snapshot(*args, **kwargs): != gates_module.evidence_sha256(evidence[result.gate_id]) ] assert mismatches == [] - assert snapshots == (["python"] if corrupt_baseline else []) + assert snapshots == (["python"] if baseline_kind != "valid" else []) assert evidence["budget_inputs"]["previous_source"] == ( f"computed with policyengine {HARNESS.policyengine_version}" - if corrupt_baseline + if baseline_kind != "valid" else "ledger" ) @@ -2929,12 +3041,14 @@ def test_inputs_runner_agrees_with_the_gates( "household_weight", "missing_x", ] + model_variables = [*inputs, "rent_person"] snapshot = run_inputs_snapshot( sys.executable, critical=critical, out=directory / "snapshot.json", dataset_path=path, model_input_variables=inputs, + model_variables=model_variables, projected_years=projected, ) tables = dataset_columns(path) @@ -2942,6 +3056,7 @@ def test_inputs_runner_agrees_with_the_gates( make_pair(), path, model_input_variables=inputs, + model_variables=model_variables, critical_inputs=critical, accepted_gaps={}, previous_missing_inputs=None, @@ -2950,6 +3065,7 @@ def test_inputs_runner_agrees_with_the_gates( ) assert snapshot["missing_inputs"] == evidence["missing_inputs"] assert snapshot["unusable_critical"] == evidence["unusable_critical"] + assert snapshot["input_status"] == evidence["input_status"] _, years = supported_years( make_pair(), path, diff --git a/tests/test_uk_qualification_replay_inputs.py b/tests/test_uk_qualification_replay_inputs.py new file mode 100644 index 00000000..c817b445 --- /dev/null +++ b/tests/test_uk_qualification_replay_inputs.py @@ -0,0 +1,619 @@ +"""Replay exceptions use digest-bound input evidence from their own attempt.""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from policyengine.provenance import uk_qualification +from policyengine.provenance.qualification import ( + GateResult, + ReplayQualificationRequest, + ReplayReceipt, + ReplayStatus, +) +from policyengine.provenance.qualification_ledger import read_json, write_json +from policyengine.provenance.receipt import content_sha256 +from tests.qualification_fixtures import HARNESS, ISSUED, gates, golden, make_pair + + +def _input_evidence(attempt): + if not (attempt / "candidate.json").exists(): + write_json( + attempt / "candidate.json", {"pair": make_pair().canonical_payload()} + ) + evidence = { + "pair_id": make_pair().pair_id, + "attempt_id": uk_qualification.attempt_evidence_id(attempt), + "missing_inputs": ["additional_residential_property_purchased"], + "newly_missing_inputs": ["additional_residential_property_purchased"], + } + result = GateResult( + gate_id="budget_inputs", + status="needs_review", + summary="Purchase input is missing", + evidence_sha256=content_sha256(evidence), + details={ + "missing_inputs": len(evidence["missing_inputs"]), + "newly_missing_inputs": evidence["newly_missing_inputs"], + }, + ) + uk_qualification.write_gate_results(attempt, [result], {"budget_inputs": evidence}) + return result, evidence + + +def _replay_documents(): + request = ReplayQualificationRequest.model_validate( + golden("qualification_request_golden") + ) + receipt = ReplayReceipt.from_document(golden("replay_receipt_golden")) + status = ReplayStatus.model_validate( + { + "schema": "policyengine.uk-replay-status/v1", + "request_id": request.request_id, + "status": "complete", + "history": [ + { + "status": "complete", + "at": "2026-10-17T10:00:00Z", + "run_url": "https://github.com/PolicyEngine/policyengine-scorecard/actions/runs/1", + } + ], + "receipt_sha256": receipt.receipt_sha256, + "receipt_commit": "c" * 40, + } + ) + return request, status, receipt + + +def _probe_evidence(attempt, request=None): + from tests.test_replay_exceptions import _probe + + if not (attempt / "candidate.json").exists(): + write_json( + attempt / "candidate.json", {"pair": make_pair().canonical_payload()} + ) + result, evidence = _probe( + request=request, attempt_id=uk_qualification.attempt_evidence_id(attempt) + ) + uk_qualification.write_gate_results(attempt, [result], {"replay_probe": evidence}) + return result, evidence + + +@pytest.mark.parametrize( + "damage", + [ + "none", + "no_result", + "no_evidence", + "changed_evidence", + "another_attempt", + "wrong_gate", + "invalid_json", + ], +) +def test_attempt_probe_evidence_is_digest_bound_and_attempt_bound(tmp_path, damage): + result, evidence = _probe_evidence(tmp_path) + result_path = tmp_path / "gates/replay_probe.result.json" + evidence_path = tmp_path / "gates/replay_probe.json" + if damage == "no_result": + result_path.unlink() + elif damage == "no_evidence": + evidence_path.unlink() + elif damage == "changed_evidence": + write_json(evidence_path, {**evidence, "probes": []}) + elif damage == "another_attempt": + replacement = {**evidence, "attempt_id": "a-different-attempt"} + write_json(evidence_path, replacement) + write_json( + result_path, + { + **result.canonical_payload(), + "evidence_sha256": content_sha256(replacement), + }, + ) + elif damage == "wrong_gate": + write_json(result_path, {**result.canonical_payload(), "gate_id": "other"}) + elif damage == "invalid_json": + evidence_path.write_text("invalid JSON") + actual = uk_qualification.load_attempt_replay_probe(tmp_path) + assert actual == ((result, evidence) if damage == "none" else (None, None)) + + +def test_probe_evidence_never_falls_back_to_another_attempt(tmp_path): + _probe_evidence(tmp_path / "older") + assert uk_qualification.load_attempt_replay_probe(tmp_path / "current") == ( + None, + None, + ) + + +def test_probe_cannot_cross_pairs_with_the_same_attempt_directory_name(tmp_path): + source = tmp_path / "pair_one" / "attempts" / "same-base-commit" + target = tmp_path / "pair_two" / "attempts" / "same-base-commit" + result, evidence = _probe_evidence(source) + write_json( + target / "candidate.json", + {"pair": make_pair(model="2.134.7").canonical_payload()}, + ) + uk_qualification.write_gate_results(target, [result], {"replay_probe": evidence}) + assert uk_qualification.attempt_evidence_id(source) != ( + uk_qualification.attempt_evidence_id(target) + ) + assert uk_qualification.load_attempt_replay_probe(target) == (None, None) + + +def test_read_replay_threads_probe_evidence(tmp_path, monkeypatch): + request, status, receipt = _replay_documents() + result, evidence = _probe_evidence(tmp_path) + calls = [] + + def evaluate(*args, **kwargs): + calls.append(kwargs) + return GateResult(gate_id="replay", status="needs_review", summary="review") + + monkeypatch.setattr(uk_qualification, "evaluate_replay_receipt", evaluate) + uk_qualification.read_replay( + request, + fetch=lambda ref, path: json.dumps( + status.document() if path.endswith("STATUS.json") else receipt.document() + ).encode(), + state_ref="replay-state", + probe_result=result, + probe_evidence=evidence, + attempt_id=uk_qualification.attempt_evidence_id(tmp_path), + ) + assert calls[0]["probe_result"] == result + assert calls[0]["probe_evidence"] == evidence + assert calls[0]["attempt_id"] == uk_qualification.attempt_evidence_id(tmp_path) + + +def test_gates_command_persists_probe_from_installed_candidate(tmp_path, monkeypatch): + from policyengine.provenance import replay_probe, uk_candidate + + spec = importlib.util.spec_from_file_location( + "check_uk_probe_gates", + Path(__file__).resolve().parents[1] / "scripts/check_uk_candidate.py", + ) + cli = importlib.util.module_from_spec(spec) + spec.loader.exec_module(cli) + pair = make_pair() + attempt = tmp_path / "attempt" + result, evidence = _probe_evidence(attempt) + # Remove prewritten files so the command itself must produce both. + for path in (attempt / "gates").iterdir(): + path.unlink() + monkeypatch.setattr( + cli, + "_attempt", + lambda args: (attempt, {"pair": pair.canonical_payload()}, HARNESS), + ) + monkeypatch.setattr(cli, "git", lambda *args: HARNESS.git_commit) + monkeypatch.setattr(cli, "config", lambda: {}) + monkeypatch.setattr( + cli, "resolved_from", lambda payload: SimpleNamespace(pair=pair) + ) + monkeypatch.setattr(uk_candidate, "run_candidate_gates", lambda *a, **k: ([], {})) + calls = [] + + def run(**kwargs): + calls.append(kwargs) + return result, evidence + + monkeypatch.setattr(replay_probe, "run_replay_probes", run) + assert cli.cmd_gates(SimpleNamespace(work_dir=str(tmp_path))) == 0 + assert calls == [ + { + "attempt_id": uk_qualification.attempt_evidence_id(attempt), + "candidate_model_version": pair.model.version, + } + ] + assert uk_qualification.load_attempt_replay_probe(attempt) == (result, evidence) + assert uk_qualification.load_gate_results(attempt) == [] + + +def test_exception_acceptance_requires_publication_designation(): + from tests.qualification_fixtures import make_receipt + + publication = "not computable, in the Budget dashboard and runbook." + replay = GateResult( + gate_id="replay", + status="needs_review", + summary="Data-limited exception", + details={"applied_exceptions": [{"publication": publication}]}, + ) + receipt = make_receipt( + gate_results=[replay if gate.gate_id == "replay" else gate for gate in gates()] + ) + kwargs = { + "reviewer": "reviewer", + "accepted_receipt_sha256": receipt.unreviewed_sha256, + "accepted_at": "2026-10-17T10:00:00Z", + } + with pytest.raises(ValueError, match="publication designation"): + uk_qualification.accept_review(receipt, explanation="Accepted", **kwargs) + reviewed = uk_qualification.accept_review( + receipt, explanation=f"Accepted with publication: {publication}", **kwargs + ) + assert reviewed.promotable + + +@pytest.mark.parametrize( + "damage", + [ + "none", + "no_result", + "no_evidence", + "changed_evidence", + "wrong_gate", + "no_digest", + "different_pair", + "different_attempt", + "missing_identity", + "invalid_json", + ], +) +def test_attempt_input_evidence_is_bound_to_its_result(tmp_path, damage): + result, evidence = _input_evidence(tmp_path) + result_path = tmp_path / "gates/budget_inputs.result.json" + evidence_path = tmp_path / "gates/budget_inputs.json" + if damage == "no_result": + result_path.unlink() + elif damage == "no_evidence": + evidence_path.unlink() + elif damage == "changed_evidence": + write_json(evidence_path, {**evidence, "missing_inputs": []}) + elif damage == "invalid_json": + evidence_path.write_text("invalid JSON") + elif damage in {"different_pair", "different_attempt", "missing_identity"}: + if damage == "different_pair": + evidence["pair_id"] = "f" * 64 + elif damage == "different_attempt": + evidence["attempt_id"] += "-other" + else: + evidence.pop("attempt_id") + write_json(evidence_path, evidence) + write_json( + result_path, + { + **result.canonical_payload(), + "evidence_sha256": content_sha256(evidence), + }, + ) + elif damage in {"wrong_gate", "no_digest"}: + write_json( + result_path, + { + **result.canonical_payload(), + **( + {"gate_id": "another_gate"} + if damage == "wrong_gate" + else {"evidence_sha256": None} + ), + }, + ) + actual = uk_qualification.load_attempt_budget_inputs(tmp_path) + assert actual == ((result, evidence) if damage == "none" else (None, None)) + + +def test_budget_input_persistence_seals_the_exact_pair_and_attempt(tmp_path): + from tests.test_replay_exceptions import _budget_inputs + + write_json(tmp_path / "candidate.json", {"pair": make_pair().canonical_payload()}) + result, evidence = _budget_inputs() + evidence.pop("attempt_id") + result = result.model_copy(update={"evidence_sha256": content_sha256(evidence)}) + uk_qualification.write_gate_results(tmp_path, [result], {"budget_inputs": evidence}) + stored_result, stored_evidence = uk_qualification.load_attempt_budget_inputs( + tmp_path + ) + assert stored_evidence["pair_id"] == make_pair().pair_id + assert stored_evidence["attempt_id"] == uk_qualification.attempt_evidence_id( + tmp_path + ) + assert stored_result.evidence_sha256 == content_sha256(stored_evidence) + assert stored_evidence["ever_usable_inputs"] == evidence["ever_usable_inputs"] + + +@pytest.mark.parametrize("identity", ["pair_id", "attempt_id"]) +def test_budget_input_persistence_cannot_bless_foreign_evidence(tmp_path, identity): + from tests.test_replay_exceptions import _budget_inputs + + write_json(tmp_path / "candidate.json", {"pair": make_pair().canonical_payload()}) + attempt_id = uk_qualification.attempt_evidence_id(tmp_path) + result, evidence = _budget_inputs(attempt_id=attempt_id) + evidence[identity] = "another-identity" + result = result.model_copy(update={"evidence_sha256": content_sha256(evidence)}) + with pytest.raises(ValueError, match="different pair or attempt"): + uk_qualification.write_gate_results( + tmp_path, [result], {"budget_inputs": evidence} + ) + + +def test_input_evidence_does_not_fall_back_to_another_attempt(tmp_path): + _input_evidence(tmp_path / "older_attempt") + assert uk_qualification.load_attempt_budget_inputs(tmp_path / "this_attempt") == ( + None, + None, + ) + + +def test_read_replay_threads_budget_input_evidence(tmp_path, monkeypatch): + request, status, receipt = _replay_documents() + result, evidence = _input_evidence(tmp_path) + expected = GateResult(gate_id="replay", status="needs_review", summary="review") + calls = [] + + def evaluate(*args, **kwargs): + calls.append((args, kwargs)) + return expected + + monkeypatch.setattr(uk_qualification, "evaluate_replay_receipt", evaluate) + actual, _, _ = uk_qualification.read_replay( + request, + fetch=lambda ref, path: json.dumps( + status.document() if path.endswith("STATUS.json") else receipt.document() + ).encode(), + state_ref="replay-state", + budget_inputs_result=result, + budget_inputs_evidence=evidence, + ) + assert actual == expected + assert calls[0][1] == { + "budget_inputs_result": result, + "budget_inputs_evidence": evidence, + "probe_result": None, + "probe_evidence": None, + "attempt_id": None, + } + + +@pytest.mark.parametrize("alter_evidence", [False, True]) +def test_finalize_reevaluates_replay_with_current_attempt_inputs( + tmp_path, monkeypatch, alter_evidence +): + attempt = tmp_path / "attempt" + pair = make_pair() + uk_qualification.stage_attempt( + attempt, + {"pair": pair.canonical_payload(), "previous_certificate": None}, + HARNESS, + ) + uk_qualification.write_gate_results(attempt, gates(), {}) + result, evidence = _input_evidence(attempt) + request, status, replay_receipt = _replay_documents() + write_json(attempt / "replay/request.json", request.document()) + write_json(attempt / "replay/status.json", status.document()) + write_json(attempt / "replay/receipt.json", replay_receipt.document()) + write_json( + attempt / "replay/replay.result.json", + GateResult( + gate_id="replay", status="pass", summary="stale" + ).canonical_payload(), + ) + if alter_evidence: + write_json(attempt / "gates/budget_inputs.json", {"missing_inputs": []}) + calls = [] + + def evaluate(*args, budget_inputs_result, budget_inputs_evidence, **kwargs): + assert kwargs == { + "probe_result": None, + "probe_evidence": None, + "attempt_id": uk_qualification.attempt_evidence_id(attempt), + } + calls.append((budget_inputs_result, budget_inputs_evidence)) + return GateResult( + gate_id="replay", + status="needs_review" if budget_inputs_result else "fail", + summary="Reevaluated with current evidence", + details={ + "budget_inputs_result_sha256": ( + content_sha256(budget_inputs_result.canonical_payload()) + if budget_inputs_result + else None + ) + }, + ) + + monkeypatch.setattr(uk_qualification, "evaluate_replay_receipt", evaluate) + receipt = uk_qualification.finalize_attempt( + attempt, + tmp_path / "ledger", + issued_by=ISSUED, + completed_at="2026-10-17T10:00:00Z", + ) + assert calls == [((None, None) if alter_evidence else (result, evidence))] + assert receipt.verdict == ("rejected" if alter_evidence else "needs_review") + replay_gate = next(gate for gate in receipt.gates if gate.gate_id == "replay") + assert ( + read_json(attempt / "replay/replay.result.json") + == replay_gate.canonical_payload() + ) + + +def test_poll_replay_reads_same_attempt_inputs(tmp_path, monkeypatch): + spec = importlib.util.spec_from_file_location( + "check_uk_replay_inputs", + Path(__file__).resolve().parents[1] / "scripts/check_uk_candidate.py", + ) + cli = importlib.util.module_from_spec(spec) + spec.loader.exec_module(cli) + request, status, receipt = _replay_documents() + result, evidence = _input_evidence(tmp_path) + write_json(tmp_path / "candidate.json", {"pair": make_pair().canonical_payload()}) + write_json(tmp_path / "replay/request.json", request.document()) + monkeypatch.setattr( + cli, + "config", + lambda: {"replay": {"repository": "owner/scorecard", "state_ref": "state"}}, + ) + monkeypatch.setattr(cli, "scorecard_file", lambda _: "fetch") + monkeypatch.setattr(cli, "set_output", lambda **_: None) + calls = [] + + def read(*args, **kwargs): + calls.append(kwargs) + return ( + GateResult(gate_id="replay", status="needs_review", summary="review"), + status, + receipt, + ) + + monkeypatch.setattr(cli, "read_replay", read) + assert ( + cli.cmd_poll_replay( + SimpleNamespace(attempt=str(tmp_path), wait_minutes=0, interval_seconds=1) + ) + == 0 + ) + assert calls == [ + { + "fetch": "fetch", + "state_ref": "state", + "budget_inputs_result": result, + "budget_inputs_evidence": evidence, + "probe_result": None, + "probe_evidence": None, + "attempt_id": uk_qualification.attempt_evidence_id(tmp_path), + } + ] + + +@pytest.mark.parametrize("raw_change", ["request", "status", "receipt", "running"]) +@pytest.mark.parametrize("input_change", ["missing", "changed"]) +def test_finalize_rejects_cached_exception_without_replay_bytes( + tmp_path, raw_change, input_change +): + attempt = tmp_path / "attempt" + pair = make_pair() + uk_qualification.stage_attempt( + attempt, + {"pair": pair.canonical_payload(), "previous_certificate": None}, + HARNESS, + ) + uk_qualification.write_gate_results(attempt, gates(), {}) + _input_evidence(attempt) + request, status, receipt = _replay_documents() + write_json(attempt / "replay/request.json", request.document()) + write_json(attempt / "replay/status.json", status.document()) + write_json(attempt / "replay/receipt.json", receipt.document()) + write_json( + attempt / "replay/replay.result.json", + GateResult( + gate_id="replay", + status="needs_review", + summary="Cached exception", + details={"applied_exceptions": [{"id": "purchase-input", "year": 2026}]}, + ).canonical_payload(), + ) + if raw_change == "running": + write_json( + attempt / "replay/status.json", + { + **status.document(), + "status": "running", + "receipt_sha256": None, + "receipt_commit": None, + }, + ) + else: + (attempt / "replay" / f"{raw_change}.json").unlink() + evidence_path = attempt / "gates/budget_inputs.json" + if input_change == "missing": + evidence_path.unlink() + else: + write_json(evidence_path, {"missing_inputs": []}) + finalized = uk_qualification.finalize_attempt( + attempt, + tmp_path / "ledger", + issued_by=ISSUED, + completed_at="2026-10-17T10:00:00Z", + ) + replay_gate = next(gate for gate in finalized.gates if gate.gate_id == "replay") + assert finalized.verdict == "rejected" + assert replay_gate.status == "fail" + assert replay_gate.details["regressions"] == [{"kind": "unverifiable_exception"}] + assert ( + read_json(attempt / "replay/replay.result.json") + == replay_gate.canonical_payload() + ) + + +@pytest.mark.parametrize("raw_change", ["request", "status", "receipt", "running"]) +def test_finalize_preserves_legacy_cache_without_exceptions(tmp_path, raw_change): + attempt = tmp_path / "attempt" + pair = make_pair() + uk_qualification.stage_attempt( + attempt, + {"pair": pair.canonical_payload(), "previous_certificate": None}, + HARNESS, + ) + uk_qualification.write_gate_results(attempt, gates(), {}) + request, status, receipt = _replay_documents() + write_json(attempt / "replay/request.json", request.document()) + write_json(attempt / "replay/status.json", status.document()) + write_json(attempt / "replay/receipt.json", receipt.document()) + cached = GateResult(gate_id="replay", status="pass", summary="Legacy cached replay") + write_json(attempt / "replay/replay.result.json", cached.canonical_payload()) + if raw_change == "running": + write_json( + attempt / "replay/status.json", + {**status.document(), "status": "running"}, + ) + else: + (attempt / "replay" / f"{raw_change}.json").unlink() + finalized = uk_qualification.finalize_attempt( + attempt, + tmp_path / "ledger", + issued_by=ISSUED, + completed_at="2026-10-17T10:00:00Z", + ) + assert finalized.verdict == "qualified" + assert next(gate for gate in finalized.gates if gate.gate_id == "replay") == cached + + +def test_finalize_real_failed_receipt_uses_same_attempt_input_evidence(tmp_path): + from tests.test_qualification_contracts import _status + from tests.test_replay_exceptions import _real_budget, _real_receipt + + attempt = tmp_path / "attempt" + pair = make_pair() + uk_qualification.stage_attempt( + attempt, + {"pair": pair.canonical_payload(), "previous_certificate": None}, + HARNESS, + ) + uk_qualification.write_gate_results(attempt, gates(), {}) + budget_result, budget_evidence = _real_budget( + attempt_id=uk_qualification.attempt_evidence_id(attempt) + ) + uk_qualification.write_gate_results( + attempt, [budget_result], {"budget_inputs": budget_evidence} + ) + request, receipt = _real_receipt() + _probe_evidence(attempt, request) + write_json(attempt / "replay/request.json", request.document()) + write_json( + attempt / "replay/status.json", _status(request.request_id, receipt).document() + ) + write_json(attempt / "replay/receipt.json", receipt) + finalized = uk_qualification.finalize_attempt( + attempt, + tmp_path / "ledger", + issued_by=ISSUED, + completed_at="2026-10-17T10:00:00Z", + ) + replay_gate = next(gate for gate in finalized.gates if gate.gate_id == "replay") + assert finalized.verdict == "needs_review", replay_gate.summary + assert replay_gate.status == "needs_review" + assert len(replay_gate.details["applied_exceptions"]) == 5 + assert sorted( + item["year"] for item in replay_gate.details["applied_exceptions"] + ) == list(range(2025, 2030)) + assert replay_gate.details["budget_inputs_result_sha256"] == content_sha256( + budget_result.canonical_payload() + )