From d2c0381d9da0b6480ed2ade4d4c4e01e1d0a69aa Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 16 Oct 2026 15:07:13 -0400 Subject: [PATCH 1/4] Apply the small-cell rule and flat findings to the Budget contract's population rows No population row now reports a count of survey records from 1 to 9 (it reads "fewer than 10") or a total over fewer than 10 changed records, and each row states every condition it tests as a top-level true/false finding, so a failed check can be diagnosed from the published findings. Fixes #642. Co-Authored-By: Claude Opus 5.5 --- changelog.d/642.fixed.md | 1 + .../uk/budget_reform_population.py | 116 ++++++-- tests/test_uk_budget_reform_contract.py | 257 ++++++++++++++++++ 3 files changed, 351 insertions(+), 23 deletions(-) create mode 100644 changelog.d/642.fixed.md diff --git a/changelog.d/642.fixed.md b/changelog.d/642.fixed.md new file mode 100644 index 00000000..138601d6 --- /dev/null +++ b/changelog.d/642.fixed.md @@ -0,0 +1 @@ +The UK Budget reform contract's population rows no longer report a count of survey records from 1 to 9 (it reads "fewer than 10"), or a total over fewer than 10 changed records, and they state each condition they test as a flat true/false finding in `detail`. diff --git a/src/policyengine/tax_benefit_models/uk/budget_reform_population.py b/src/policyengine/tax_benefit_models/uk/budget_reform_population.py index 56c9042c..cf9f7a1a 100644 --- a/src/policyengine/tax_benefit_models/uk/budget_reform_population.py +++ b/src/policyengine/tax_benefit_models/uk/budget_reform_population.py @@ -47,10 +47,23 @@ The last two are responsiveness checks: a reform that reaches the parameter but not the award (as when an award is fixed before the reform applies) fails them. + +Detail +------ + +Each row's ``detail`` states every condition the check tests as a flat +true/false finding (``no_award_lower``, ``some_pension_higher``, +``income_tax:2027_matches`` and so on), so a failed check can be diagnosed +from the findings alone. Counts and totals follow them for local reruns, +under the small-cell rule: the report is published, and the subsample's +households are survey records, so no count from 1 to 9 appears (it reads +``"fewer than 10"``), and a total over fewer than 10 changed records is +withheld. """ from __future__ import annotations +import numbers import os import tempfile import traceback @@ -75,6 +88,43 @@ ) RESPONSE_YEAR = 2027 +#: The small-cell rule: a count of survey records from 1 to 9 is never +#: published, nor a total over that many. +SMALL_CELL_LIMIT = 10 +FEWER_THAN_10 = "fewer than 10" +WITHHELD_FEWER_THAN_10 = "withheld: fewer than 10 records" + + +def is_small_cell(count: int) -> bool: + return 0 < count < SMALL_CELL_LIMIT + + +def publishable_detail(detail: Any) -> Any: + """``detail`` with every count from 1 to 9, at any depth, suppressed. + + Booleans are findings, not counts, and stay as they are. + """ + if isinstance(detail, (bool, np.bool_)): + return detail + if isinstance(detail, numbers.Integral): + return FEWER_THAN_10 if is_small_cell(int(detail)) else detail + if isinstance(detail, dict): + return {key: publishable_detail(value) for key, value in detail.items()} + if isinstance(detail, (list, tuple)): + return [publishable_detail(value) for value in detail] + return detail + + +def total_over_records(total: float, records: int) -> float | str: + """A total, withheld when fewer than 10 records moved it.""" + return WITHHELD_FEWER_THAN_10 if is_small_cell(records) else total + + +def matches(names: Sequence[str], different: Sequence[str]) -> dict[str, bool]: + """One ``_matches`` finding per compared output.""" + return {f"{name}_matches": name not in different for name in names} + + POPULATION_BASELINE_ISOLATION = "population_baseline_isolation" POPULATION_QUERY_ORDER = "population_query_order" POPULATION_ROUTES_AGREE = "population_routes_agree" @@ -223,18 +273,22 @@ def uc_allowance_verdict( for state_paid, state_award in ((base_paid, base_award), (paid, award)) ) ) - holds = ( - non_finite == 0 - and scaled - and awards_lower == 0 - and payments_unaccounted == 0 - and payments_below_floor == 0 - and payments_higher > 0 - ) - return holds, { + payments_lower = int(np.sum(paid < base_paid - PENNY)) + findings = { "standard_allowance_scaled_10_percent": scaled, + "no_award_lower": awards_lower == 0, + "payments_are_award_less_deductions": payments_unaccounted == 0, + "no_payment_below_the_payable_floor": payments_below_floor == 0, + "some_payment_higher": payments_higher > 0, + "no_non_finite_units": non_finite == 0, + } + holds = all(findings.values()) + return holds, { + **findings, + # Lawful (deductions can rise more than the award), so not a condition. + "no_payment_lower": payments_lower == 0, "awards_lower": awards_lower, - "payments_lower": int(np.sum(paid < base_paid - PENNY)), + "payments_lower": payments_lower, "payments_higher": payments_higher, "payments_not_award_less_deductions": payments_unaccounted, "payments_below_the_payable_floor": payments_below_floor, @@ -380,7 +434,7 @@ def build(route: str | None, reform: dict | None): POPULATION_BASELINE_ISOLATION, route, "fail" if different else "pass", - detail=different or None, + detail=matches(list(before), different), ) ) after = _arrays(build(None, None)) @@ -390,7 +444,7 @@ def build(route: str | None, reform: dict | None): POPULATION_BASELINE_ISOLATION, "unreformed", "fail" if different else "pass", - detail=different or None, + detail=matches(list(before), different), ) ) different = _same(before, _arrays(build(None, None), order=-1)) @@ -399,7 +453,7 @@ def build(route: str | None, reform: dict | None): POPULATION_QUERY_ORDER, "unreformed", "fail" if different else "pass", - detail=different or None, + detail=matches(list(before), different), ) ) tax = { @@ -411,15 +465,18 @@ def build(route: str | None, reform: dict | None): } first = country[0] for route in country[1:]: - same = all( - np.array_equal(tax[route][year], tax[first][year]) + agree = { + f"income_tax:{year}_matches": bool( + np.array_equal(tax[route][year], tax[first][year]) + ) for year in (2026, 2027) - ) + } rows.append( _row( POPULATION_ROUTES_AGREE, f"{first} = {route}", - "pass" if same else "fail", + "pass" if all(agree.values()) else "fail", + detail=agree, ) ) for route in country: @@ -431,6 +488,8 @@ def build(route: str | None, reform: dict | None): # is on the total, and the counts are reported. total_rises = bool(np.sum(change_2027) > 1e-6) ok = unchanged_2026 and total_rises + lower = int(np.sum(change_2027 < -1e-6)) + higher = int(np.sum(change_2027 > 1e-6)) rows.append( _row( POPULATION_APRIL_RATE_CHANGE, @@ -438,23 +497,29 @@ def build(route: str | None, reform: dict | None): "pass" if ok else "fail", detail={ "2026_unchanged": unchanged_2026, - "2027_total_change": float(np.sum(change_2027)), - "2027_people_with_lower_tax": int(np.sum(change_2027 < -1e-6)), - "2027_people_with_higher_tax": int(np.sum(change_2027 > 1e-6)), + "2027_total_rises": total_rises, + "2027_total_change": total_over_records( + float(np.sum(change_2027)), lower + higher + ), + "2027_people_with_lower_tax": lower, + "2027_people_with_higher_tax": higher, }, ) ) for route in country: simulation = build(route, FINITE) - same = np.array_equal( - np.asarray(simulation.calculate("income_tax", 2028), dtype=float), - before["income_tax:2028"], + same = bool( + np.array_equal( + np.asarray(simulation.calculate("income_tax", 2028), dtype=float), + before["income_tax:2028"], + ) ) rows.append( _row( POPULATION_FINITE_CHANGE_EXPIRES, route, "pass" if same else "fail", + detail={"income_tax:2028_matches": same}, ) ) @@ -506,6 +571,8 @@ def values(simulation, variable): "pass" if scaled and lower == 0 and higher > 0 else "fail", detail={ "basic_and_new_scaled_10_percent": bool(scaled), + "no_pension_lower": lower == 0, + "some_pension_higher": higher > 0, "pensions_lower": lower, "pensions_higher": higher, }, @@ -516,12 +583,15 @@ def values(simulation, variable): _row( *POPULATION_ERROR_DIAGNOSTIC, "error", + reason=type(error).__name__, detail="".join( traceback.format_exception_only(type(error), error) ).strip()[:1000], ) ) for row in rows: + if isinstance(row.get("detail"), dict): + row["detail"] = publishable_detail(row["detail"]) row.setdefault("subsample_households", SUBSAMPLE_HOUSEHOLDS) row.setdefault("seed", seed) return rows diff --git a/tests/test_uk_budget_reform_contract.py b/tests/test_uk_budget_reform_contract.py index 7f13d280..f4bf64a2 100644 --- a/tests/test_uk_budget_reform_contract.py +++ b/tests/test_uk_budget_reform_contract.py @@ -714,6 +714,7 @@ def fail(*args): "check_id": population.POPULATION_ERROR_DIAGNOSTIC[0], "route": population.POPULATION_ERROR_DIAGNOSTIC[1], "status": "error", + "reason": "OSError", "detail": "OSError: forced population failure", "subsample_households": population.SUBSAMPLE_HOUSEHOLDS, "seed": 7, @@ -885,6 +886,7 @@ def test_population_checks_hold_on_a_tiny_dataset(tmp_path): seed=0, ) assert rows and all(row["status"] == "pass" for row in rows), rows + _assert_publishable_findings(rows) from collections import Counter assert Counter((row["check_id"], row["route"]) for row in rows) == Counter( @@ -901,6 +903,7 @@ def test_responsiveness_checks_fail_with_no_award_to_move(tmp_path): rows = run_population_checks( _tiny_dataset_file(tmp_path), ["country.reform_dict"], seed=0 ) + _assert_publishable_findings(rows) status = {row["check_id"]: row["status"] for row in rows} # Earners only: no award to show a reform reaching it. assert status["population_state_pension_plus_10"] == "fail" @@ -1373,3 +1376,257 @@ def test_every_report_row_has_a_reform_identity_field(): assert "reform_identity" in row, (section, row) if row["reform_identity"] is None: assert row["reform_identity_unavailable"], (section, row) + + +# The small-cell rule and flat findings in population rows. + +#: A finding reported for diagnosis that the check does not require. +INFORMATIONAL_FINDINGS = {"no_payment_lower"} + + +def _small_counts(value, path="detail"): + """Where ``value`` holds a count from 1 to 9.""" + import numbers + + import numpy as np + + if isinstance(value, (bool, np.bool_)): + return [] + if isinstance(value, numbers.Integral): + return [path] if 0 < value < 10 else [] + if isinstance(value, dict): + return [ + found + for key, item in value.items() + for found in _small_counts(item, f"{path}.{key}") + ] + if isinstance(value, (list, tuple)): + return [ + found + for index, item in enumerate(value) + for found in _small_counts(item, f"{path}[{index}]") + ] + return [] + + +def _assert_publishable_findings(rows): + """No small count anywhere, and a row passes when all its findings hold.""" + for row in rows: + assert _small_counts(row) == [], row + detail = row.get("detail") + if row["route"] == "-": + continue + assert isinstance(detail, dict), row + findings = { + key: value + for key, value in detail.items() + if isinstance(value, bool) and key not in INFORMATIONAL_FINDINGS + } + assert findings, row + assert (row["status"] == "pass") == all(findings.values()), row + + +def test_publishable_detail_suppresses_counts_from_1_to_9_at_any_depth(): + import numpy as np + + from policyengine.tax_benefit_models.uk.budget_reform_population import ( + FEWER_THAN_10, + publishable_detail, + ) + + detail = { + "none": 0, + "one": 1, + "nine": 9, + "ten": 10, + "negative": -3, + "finding": True, + "other_finding": False, + "total": 3.5, + "numpy_count": np.int64(5), + "nested": {"count": 4, "list": [2, 12, {"deep": 7}]}, + } + assert publishable_detail(detail) == { + "none": 0, + "one": FEWER_THAN_10, + "nine": FEWER_THAN_10, + "ten": 10, + "negative": -3, + "finding": True, + "other_finding": False, + "total": 3.5, + "numpy_count": FEWER_THAN_10, + "nested": { + "count": FEWER_THAN_10, + "list": [FEWER_THAN_10, 12, {"deep": FEWER_THAN_10}], + }, + } + + +def _equal_but_small_counts(original, published): + """``published`` is ``original`` with only its small counts suppressed.""" + import numbers + + from policyengine.tax_benefit_models.uk.budget_reform_population import ( + FEWER_THAN_10, + ) + + if isinstance(original, bool): + return published is original + if isinstance(original, numbers.Integral) and 0 < original < 10: + return published == FEWER_THAN_10 + if isinstance(original, dict): + return published.keys() == original.keys() and all( + _equal_but_small_counts(value, published[key]) + for key, value in original.items() + ) + if isinstance(original, (list, tuple)): + return len(published) == len(original) and all( + _equal_but_small_counts(a, b) for a, b in zip(original, published) + ) + return published == original + + +def test_publication_removes_every_small_count_and_nothing_else(): + pytest.importorskip("hypothesis") + from hypothesis import given, settings + from hypothesis import strategies as st + + from policyengine.tax_benefit_models.uk.budget_reform_population import ( + publishable_detail, + ) + + leaves = st.one_of( + st.booleans(), + st.integers(-20, 1_000), + st.floats(allow_nan=False), + st.text(max_size=5), + st.none(), + ) + details = st.recursive( + leaves, + lambda children: st.one_of( + st.lists(children, max_size=4), + st.dictionaries(st.text(max_size=5), children, max_size=4), + ), + max_leaves=20, + ) + + @settings(max_examples=200, deadline=None, derandomize=True) + @given(details) + def check(detail): + published = publishable_detail(detail) + assert _small_counts(published) == [] + assert _equal_but_small_counts(detail, published) + assert publishable_detail(published) == published + + check() + + +def test_a_total_over_fewer_than_10_changed_records_is_withheld(): + from policyengine.tax_benefit_models.uk.budget_reform_population import ( + WITHHELD_FEWER_THAN_10, + total_over_records, + ) + + assert total_over_records(12.5, 0) == 12.5 + assert total_over_records(12.5, 1) == WITHHELD_FEWER_THAN_10 + assert total_over_records(12.5, 9) == WITHHELD_FEWER_THAN_10 + assert total_over_records(12.5, 10) == 12.5 + + +def test_uc_findings_agree_with_their_counts_and_the_verdict(): + pytest.importorskip("hypothesis") + import numpy as np + from hypothesis import given, settings + from hypothesis import strategies as st + + from policyengine.tax_benefit_models.uk.budget_reform_population import ( + UC_ALLOWANCE_VARIABLES, + ) + + amounts = st.one_of(st.floats(0, 2_000), st.just(float("nan"))) + + @settings(max_examples=200, deadline=None, derandomize=True) + @given(units=st.integers(1, 4), data=st.data()) + def check(units, data): + def draw(): + return { + name: np.array( + data.draw(st.lists(amounts, min_size=units, max_size=units)), + dtype=float, + ) + for name in UC_ALLOWANCE_VARIABLES + } + + holds, detail = _verdict(draw(), draw()) + counts = { + "no_award_lower": "awards_lower", + "payments_are_award_less_deductions": "payments_not_award_less_deductions", + "no_payment_below_the_payable_floor": "payments_below_the_payable_floor", + "no_non_finite_units": "units_with_non_finite_values", + "no_payment_lower": "payments_lower", + } + for finding, count in counts.items(): + assert detail[finding] is (detail[count] == 0), finding + assert detail["some_payment_higher"] is (detail["payments_higher"] > 0) + # The verdict as it was defined from the counts. + assert holds == ( + detail["units_with_non_finite_values"] == 0 + and detail["standard_allowance_scaled_10_percent"] + and detail["awards_lower"] == 0 + and detail["payments_not_award_less_deductions"] == 0 + and detail["payments_below_the_payable_floor"] == 0 + and detail["payments_higher"] > 0 + ) + + check() + + +def test_population_rows_state_each_condition_as_a_flat_finding(stub_population_checks): + population = stub_population_checks + rows = population.run_population_checks("unused.h5", list(contract.COUNTRY_ROUTES)) + _assert_publishable_findings(rows) + outputs = { + f"{variable}:{year}_matches" + for variable in population.OUTPUTS + for year in population.YEARS + } + expected = { + population.POPULATION_BASELINE_ISOLATION: outputs, + population.POPULATION_QUERY_ORDER: outputs, + population.POPULATION_ROUTES_AGREE: { + "income_tax:2026_matches", + "income_tax:2027_matches", + }, + population.POPULATION_APRIL_RATE_CHANGE: {"2026_unchanged", "2027_total_rises"}, + population.POPULATION_FINITE_CHANGE_EXPIRES: {"income_tax:2028_matches"}, + population.POPULATION_UC_STANDARD_ALLOWANCE_PLUS_10: { + "standard_allowance_scaled_10_percent", + "no_award_lower", + "payments_are_award_less_deductions", + "no_payment_below_the_payable_floor", + "some_payment_higher", + "no_non_finite_units", + "no_payment_lower", + }, + population.POPULATION_STATE_PENSION_PLUS_10: { + "basic_and_new_scaled_10_percent", + "no_pension_lower", + "some_pension_higher", + }, + } + for row in rows: + findings = { + key for key, value in row["detail"].items() if isinstance(value, bool) + } + assert findings == expected[row["check_id"]], row + # The stub moves one person's tax: one record, so the count reads + # "fewer than 10" and the total is withheld. + april = next( + row + for row in rows + if row["check_id"] == population.POPULATION_APRIL_RATE_CHANGE + ) + assert april["detail"]["2027_people_with_higher_tax"] == population.FEWER_THAN_10 + assert april["detail"]["2027_total_change"] == population.WITHHELD_FEWER_THAN_10 From ee521042027d47d766eb8a0308891fd3b06cb7ee Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 16 Oct 2026 15:59:37 -0400 Subject: [PATCH 2/4] Withhold totals by every record they sum, fail closed on detail types, and test the findings on random runs The April total is withheld by the records whose tax moved at all, not only those past the counting threshold. publishable_detail raises on a type it cannot check. The informational finding is named no_payment_lower_informational. The docstring says the rule covers counts and totals, not error text or cross-row differences, so the raw report is not for publication; the ledger is the publishing control. The UC differential now draws holding verdicts, from lawful pairs, and a property runs the population checks on random stubbed arrays, lawful responses included, checking every row's findings decide it. Co-Authored-By: Claude Opus 5.5 --- .../uk/budget_reform_population.py | 41 ++-- tests/test_uk_budget_reform_contract.py | 215 ++++++++++++++++-- 2 files changed, 224 insertions(+), 32 deletions(-) diff --git a/src/policyengine/tax_benefit_models/uk/budget_reform_population.py b/src/policyengine/tax_benefit_models/uk/budget_reform_population.py index cf9f7a1a..4e559a5b 100644 --- a/src/policyengine/tax_benefit_models/uk/budget_reform_population.py +++ b/src/policyengine/tax_benefit_models/uk/budget_reform_population.py @@ -54,11 +54,18 @@ Each row's ``detail`` states every condition the check tests as a flat true/false finding (``no_award_lower``, ``some_pension_higher``, ``income_tax:2027_matches`` and so on), so a failed check can be diagnosed -from the findings alone. Counts and totals follow them for local reruns, -under the small-cell rule: the report is published, and the subsample's -households are survey records, so no count from 1 to 9 appears (it reads -``"fewer than 10"``), and a total over fewer than 10 changed records is -withheld. +from the findings alone. A row passes exactly when its findings all hold; +one whose name ends ``_informational`` is reported for diagnosis and is not +a condition. Counts and totals follow the findings for local reruns. + +The subsample's households are survey records, so the counts and totals +also follow the small-cell rule: no count from 1 to 9 (it reads ``"fewer +than 10"``), and no total over fewer than 10 changed records. The rule +covers counts and totals only, not an error's message or the difference +between two rows' figures (two routes' rows share a subsample), so the raw +report is not for publication. The qualification ledger publishes from it +only the fixed row fields and the top-level findings; the rule here keeps +what raw copies (test fixtures, local runs) carry to the same standard. """ from __future__ import annotations @@ -102,21 +109,26 @@ def is_small_cell(count: int) -> bool: def publishable_detail(detail: Any) -> Any: """``detail`` with every count from 1 to 9, at any depth, suppressed. - Booleans are findings, not counts, and stay as they are. + Booleans are findings, not counts, and stay as they are. A detail holds + only booleans, integers, floats, text, None, and lists or text-keyed + mappings of them; anything else (an array, a set) is an error rather + than passed through unchecked. """ if isinstance(detail, (bool, np.bool_)): - return detail + return bool(detail) if isinstance(detail, numbers.Integral): - return FEWER_THAN_10 if is_small_cell(int(detail)) else detail - if isinstance(detail, dict): + return FEWER_THAN_10 if is_small_cell(int(detail)) else int(detail) + if detail is None or isinstance(detail, (float, np.floating, str)): + return detail + if isinstance(detail, dict) and all(isinstance(key, str) for key in detail): return {key: publishable_detail(value) for key, value in detail.items()} if isinstance(detail, (list, tuple)): return [publishable_detail(value) for value in detail] - return detail + raise TypeError(f"A population detail cannot hold {type(detail).__name__}.") def total_over_records(total: float, records: int) -> float | str: - """A total, withheld when fewer than 10 records moved it.""" + """A total, withheld when fewer than 10 records moved it at all.""" return WITHHELD_FEWER_THAN_10 if is_small_cell(records) else total @@ -286,7 +298,7 @@ def uc_allowance_verdict( return holds, { **findings, # Lawful (deductions can rise more than the award), so not a condition. - "no_payment_lower": payments_lower == 0, + "no_payment_lower_informational": payments_lower == 0, "awards_lower": awards_lower, "payments_lower": payments_lower, "payments_higher": payments_higher, @@ -498,8 +510,11 @@ def build(route: str | None, reform: dict | None): detail={ "2026_unchanged": unchanged_2026, "2027_total_rises": total_rises, + # Withheld by the records that moved at all, which + # the total sums, not just those past the threshold. "2027_total_change": total_over_records( - float(np.sum(change_2027)), lower + higher + float(np.sum(change_2027)), + int(np.count_nonzero(change_2027)), ), "2027_people_with_lower_tax": lower, "2027_people_with_higher_tax": higher, diff --git a/tests/test_uk_budget_reform_contract.py b/tests/test_uk_budget_reform_contract.py index f4bf64a2..4f4b60bc 100644 --- a/tests/test_uk_budget_reform_contract.py +++ b/tests/test_uk_budget_reform_contract.py @@ -1380,8 +1380,10 @@ def test_every_report_row_has_a_reform_identity_field(): # The small-cell rule and flat findings in population rows. -#: A finding reported for diagnosis that the check does not require. -INFORMATIONAL_FINDINGS = {"no_payment_lower"} + +def _is_condition(key, value): + """A finding the check requires; ``*_informational`` ones it does not.""" + return isinstance(value, bool) and not key.endswith("_informational") def _small_counts(value, path="detail"): @@ -1418,9 +1420,7 @@ def _assert_publishable_findings(rows): continue assert isinstance(detail, dict), row findings = { - key: value - for key, value in detail.items() - if isinstance(value, bool) and key not in INFORMATIONAL_FINDINGS + key: value for key, value in detail.items() if _is_condition(key, value) } assert findings, row assert (row["status"] == "pass") == all(findings.values()), row @@ -1546,26 +1546,29 @@ def test_uc_findings_agree_with_their_counts_and_the_verdict(): ) amounts = st.one_of(st.floats(0, 2_000), st.just(float("nan"))) + verdicts = set() + # From the lawful pair (one capped unit whose payment falls with its + # deductions, one that gains), so verdicts that hold are drawn too; each + # example changes up to three of its values. @settings(max_examples=200, deadline=None, derandomize=True) - @given(units=st.integers(1, 4), data=st.data()) - def check(units, data): - def draw(): - return { - name: np.array( - data.draw(st.lists(amounts, min_size=units, max_size=units)), - dtype=float, - ) - for name in UC_ALLOWANCE_VARIABLES - } - - holds, detail = _verdict(draw(), draw()) + @given(data=st.data()) + def check(data): + base, reformed = _lawful_pair() + for _ in range(data.draw(st.integers(0, 3))): + state = data.draw(st.sampled_from([base, reformed])) + name = data.draw(st.sampled_from(UC_ALLOWANCE_VARIABLES)) + unit = data.draw(st.integers(0, 1)) + state[name] = np.array(state[name], dtype=float) + state[name][unit] = data.draw(amounts) + holds, detail = _verdict(base, reformed) + verdicts.add(holds) counts = { "no_award_lower": "awards_lower", "payments_are_award_less_deductions": "payments_not_award_less_deductions", "no_payment_below_the_payable_floor": "payments_below_the_payable_floor", "no_non_finite_units": "units_with_non_finite_values", - "no_payment_lower": "payments_lower", + "no_payment_lower_informational": "payments_lower", } for finding, count in counts.items(): assert detail[finding] is (detail[count] == 0), finding @@ -1581,6 +1584,7 @@ def draw(): ) check() + assert verdicts == {True, False} def test_population_rows_state_each_condition_as_a_flat_finding(stub_population_checks): @@ -1608,7 +1612,7 @@ def test_population_rows_state_each_condition_as_a_flat_finding(stub_population_ "no_payment_below_the_payable_floor", "some_payment_higher", "no_non_finite_units", - "no_payment_lower", + "no_payment_lower_informational", }, population.POPULATION_STATE_PENSION_PLUS_10: { "basic_and_new_scaled_10_percent", @@ -1630,3 +1634,176 @@ def test_population_rows_state_each_condition_as_a_flat_finding(stub_population_ ) assert april["detail"]["2027_people_with_higher_tax"] == population.FEWER_THAN_10 assert april["detail"]["2027_total_change"] == population.WITHHELD_FEWER_THAN_10 + + +def test_a_detail_of_another_type_is_an_error_not_passed_through(): + import numpy as np + + from policyengine.tax_benefit_models.uk.budget_reform_population import ( + publishable_detail, + ) + + assert publishable_detail({"finding": np.bool_(True)}) == {"finding": True} + assert type(publishable_detail(np.bool_(False))) is bool + for unchecked in (np.array([3, 12]), {3, 12}, {3: "count key"}): + with pytest.raises(TypeError): + publishable_detail({"value": unchecked}) + + +def test_every_population_row_publishes_the_findings_that_decide_it(monkeypatch): + """On random arrays: no small count, and each row passes when its findings do.""" + pytest.importorskip("hypothesis") + pytest.importorskip("policyengine_uk.utils.scenario") + from types import SimpleNamespace + + import numpy as np + import policyengine_uk + from hypothesis import HealthCheck, given, settings + from hypothesis import strategies as st + from policyengine_core.reforms import Reform + from policyengine_uk.utils import scenario + + from policyengine.tax_benefit_models.uk import ( + budget_reform_population as population, + ) + + routes = list(contract.COUNTRY_ROUTES) + passes = set() + + @settings( + max_examples=60, + deadline=None, + derandomize=True, + suppress_health_check=[HealthCheck.function_scoped_fixture], + ) + @given(data=st.data()) + def check(data): + units = data.draw(st.integers(1, 12)) + amounts = st.floats(0, 1_000, width=32) + if data.draw(st.booleans()): + amounts = st.one_of(amounts, st.just(float("nan"))) + # Lawful responses: the Universal Credit and State Pension + # simulations move their awards as the law does, so those rows can + # pass; otherwise every reformed array is random or unreformed. + lawful = data.draw(st.booleans()) + pensions = ("basic_state_pension", "new_state_pension", "state_pension") + arrays = {} + + def drawn(variable): + values = amounts + if lawful and variable == "uc_standard_allowance": + values = st.floats(1, 1_000, width=32) + return np.array( + data.draw(st.lists(values, min_size=units, max_size=units)), + dtype=float, + ) + + def array(tag, variable, year): + key = (tag, variable, year) + if key in arrays: + return arrays[key] + if tag is None: + if lawful and variable in ("benefit_cap_reduction", "uc_deductions"): + value = np.zeros(units) + elif lawful and variable == "universal_credit_pre_benefit_cap": + value = array(None, "universal_credit", year) + else: + value = drawn(variable) + elif ( + lawful + and "universal_credit.standard_allowance" in tag + and variable in population.UC_ALLOWANCE_VARIABLES + ): + allowance = array(None, "uc_standard_allowance", year) + base = array(None, variable, year) + value = { + "uc_standard_allowance": 1.1 * allowance, + "universal_credit_pre_benefit_cap": base + 0.1 * allowance, + "universal_credit": base + 0.1 * allowance, + }.get(variable, base) + elif lawful and "state_pension" in tag and variable in pensions: + value = 1.1 * array(None, variable, year) + elif data.draw(st.booleans()): + value = array(None, variable, year) + else: + value = drawn(variable) + arrays[key] = value + return value + + class Simulation: + def __init__(self, dataset, reform=None, scenario=None): + self.tag = ( + None + if reform is None and scenario is None + else repr(reform if scenario is None else ("scenario", scenario)) + ) + + @property + def baseline(self): + # A reformed simulation's baseline, often the unreformed one. + baseline = Simulation(dataset=None) + if self.tag is not None: + baseline.tag = repr(("baseline", self.tag)) + return baseline + + def calculate(self, variable, year): + return array(self.tag, variable, year) + + with monkeypatch.context() as patch: + patch.setattr(policyengine_uk, "Microsimulation", Simulation) + patch.setattr( + scenario, "Scenario", lambda **kwargs: repr(sorted(kwargs.items())) + ) + patch.setattr( + Reform, + "from_dict", + staticmethod(lambda reform, **kwargs: ("from_dict", repr(reform))), + ) + patch.setattr( + population, + "subsample", + lambda *args: SimpleNamespace(copy=lambda: None), + ) + patch.setattr( + population, "plus_ten_percent", lambda paths: {"response": paths} + ) + rows = population.run_population_checks("unused.h5", routes) + _assert_publishable_findings(rows) + passes.update((row["check_id"], row["status"]) for row in rows) + + check() + # Every check both passed and failed somewhere in the run. + for check_id in population.COUNTRY_POPULATION_CHECKS: + assert {(check_id, "pass"), (check_id, "fail")} <= passes, check_id + + +def test_a_total_moved_only_by_tiny_changes_in_few_records_is_withheld( + monkeypatch, stub_population_checks +): + """Withheld by the records that moved at all, not just past the threshold.""" + import numpy as np + import policyengine_uk + + population = stub_population_checks + stub = policyengine_uk.Microsimulation + calculate = stub.calculate + + def three_people(self, variable, year): + if variable != "income_tax": + return calculate(self, variable, year) + tax = np.full(3, 1.0) + if year == 2027 and self.reform in (population.APRIL, population.FINITE): + # Each below the 1e-6 threshold the counts use; together above it. + return tax + 4.77e-7 + return tax + + monkeypatch.setattr(stub, "calculate", three_people) + rows = population.run_population_checks("unused.h5", ["country.reform_dict"]) + april = next( + row + for row in rows + if row["check_id"] == population.POPULATION_APRIL_RATE_CHANGE + ) + assert april["detail"]["2027_people_with_higher_tax"] == 0 + assert april["detail"]["2027_total_rises"] is True + assert april["detail"]["2027_total_change"] == population.WITHHELD_FEWER_THAN_10 From ca1345e14ed2c84293155fc78603cb9c3985f36d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 16 Oct 2026 17:43:36 -0400 Subject: [PATCH 3/4] Withhold a total when either count of its records is small, and err a row whose detail cannot be checked A few large changes among negligible ones are still a few records, so the April total is withheld when the changes past the threshold number 1 to 9 as well as when all changes do. A detail publication cannot check makes its row an error instead of raising out of the run. New tests pin routes that disagree only in 2026 and a State Pension that falls. Co-Authored-By: Claude Opus 5.5 --- .../uk/budget_reform_population.py | 30 ++- tests/test_uk_budget_reform_contract.py | 229 ++++++++++++++---- 2 files changed, 207 insertions(+), 52 deletions(-) diff --git a/src/policyengine/tax_benefit_models/uk/budget_reform_population.py b/src/policyengine/tax_benefit_models/uk/budget_reform_population.py index 4e559a5b..83775f80 100644 --- a/src/policyengine/tax_benefit_models/uk/budget_reform_population.py +++ b/src/policyengine/tax_benefit_models/uk/budget_reform_population.py @@ -65,7 +65,9 @@ between two rows' figures (two routes' rows share a subsample), so the raw report is not for publication. The qualification ledger publishes from it only the fixed row fields and the top-level findings; the rule here keeps -what raw copies (test fixtures, local runs) carry to the same standard. +raw copies (test fixtures, local runs) free of small counts too. A detail +that cannot be checked turns its row into an error, so the report does not +hold. """ from __future__ import annotations @@ -118,7 +120,9 @@ def publishable_detail(detail: Any) -> Any: return bool(detail) if isinstance(detail, numbers.Integral): return FEWER_THAN_10 if is_small_cell(int(detail)) else int(detail) - if detail is None or isinstance(detail, (float, np.floating, str)): + if isinstance(detail, np.floating): + return float(detail) + if detail is None or isinstance(detail, (float, str)): return detail if isinstance(detail, dict) and all(isinstance(key, str) for key in detail): return {key: publishable_detail(value) for key, value in detail.items()} @@ -127,9 +131,16 @@ def publishable_detail(detail: Any) -> Any: raise TypeError(f"A population detail cannot hold {type(detail).__name__}.") -def total_over_records(total: float, records: int) -> float | str: - """A total, withheld when fewer than 10 records moved it at all.""" - return WITHHELD_FEWER_THAN_10 if is_small_cell(records) else total +def total_over_records(total: float, *records: int) -> float | str: + """A total, withheld when any count of the records behind it is 1 to 9. + + Pass each count that bounds how few records the total reflects: those + that moved at all, and those that moved by more than a negligible amount + (a few large changes among negligible ones are still a few records). + """ + if any(is_small_cell(count) for count in records): + return WITHHELD_FEWER_THAN_10 + return total def matches(names: Sequence[str], different: Sequence[str]) -> dict[str, bool]: @@ -511,10 +522,11 @@ def build(route: str | None, reform: dict | None): "2026_unchanged": unchanged_2026, "2027_total_rises": total_rises, # Withheld by the records that moved at all, which - # the total sums, not just those past the threshold. + # the total sums, and by those past the threshold. "2027_total_change": total_over_records( float(np.sum(change_2027)), int(np.count_nonzero(change_2027)), + lower + higher, ), "2027_people_with_lower_tax": lower, "2027_people_with_higher_tax": higher, @@ -606,7 +618,11 @@ def values(simulation, variable): ) for row in rows: if isinstance(row.get("detail"), dict): - row["detail"] = publishable_detail(row["detail"]) + try: + row["detail"] = publishable_detail(row["detail"]) + except TypeError as error: + # Fail closed: drop what cannot be checked, and the row errs. + row.update(status="error", reason=type(error).__name__, detail=None) row.setdefault("subsample_households", SUBSAMPLE_HOUSEHOLDS) row.setdefault("seed", seed) return rows diff --git a/tests/test_uk_budget_reform_contract.py b/tests/test_uk_budget_reform_contract.py index 4f4b60bc..d531ee3c 100644 --- a/tests/test_uk_budget_reform_contract.py +++ b/tests/test_uk_budget_reform_contract.py @@ -1533,6 +1533,9 @@ def test_a_total_over_fewer_than_10_changed_records_is_withheld(): assert total_over_records(12.5, 1) == WITHHELD_FEWER_THAN_10 assert total_over_records(12.5, 9) == WITHHELD_FEWER_THAN_10 assert total_over_records(12.5, 10) == 12.5 + # Any small count of the records behind it withholds it. + assert total_over_records(300.0, 10, 3) == WITHHELD_FEWER_THAN_10 + assert total_over_records(300.0, 10, 0) == 300.0 def test_uc_findings_agree_with_their_counts_and_the_verdict(): @@ -1650,24 +1653,73 @@ def test_a_detail_of_another_type_is_an_error_not_passed_through(): publishable_detail({"value": unchecked}) +def _run_population_with(monkeypatch, array, routes=None): + """Run the population checks on arrays from ``array(tag, variable, year)``. + + ``tag`` is None for an unreformed simulation, and otherwise names the + route and reform: the reform dict's repr, or a repr containing + "scenario" or "from_dict", or one starting "('baseline'" for a reformed + simulation's baseline. + """ + from types import SimpleNamespace + + import policyengine_uk + from policyengine_core.reforms import Reform + from policyengine_uk.utils import scenario + + from policyengine.tax_benefit_models.uk import ( + budget_reform_population as population, + ) + + class Simulation: + def __init__(self, dataset, reform=None, scenario=None): + self.tag = ( + None + if reform is None and scenario is None + else repr(reform if scenario is None else ("scenario", scenario)) + ) + + @property + def baseline(self): + baseline = Simulation(dataset=None) + if self.tag is not None: + baseline.tag = repr(("baseline", self.tag)) + return baseline + + def calculate(self, variable, year): + return array(self.tag, variable, year) + + with monkeypatch.context() as patch: + patch.setattr(policyengine_uk, "Microsimulation", Simulation) + patch.setattr( + scenario, "Scenario", lambda **kwargs: repr(sorted(kwargs.items())) + ) + patch.setattr( + Reform, + "from_dict", + staticmethod(lambda reform, **kwargs: ("from_dict", repr(reform))), + ) + patch.setattr( + population, "subsample", lambda *args: SimpleNamespace(copy=lambda: None) + ) + patch.setattr(population, "plus_ten_percent", lambda paths: {"response": paths}) + return population.run_population_checks( + "unused.h5", routes or list(contract.COUNTRY_ROUTES) + ) + + def test_every_population_row_publishes_the_findings_that_decide_it(monkeypatch): """On random arrays: no small count, and each row passes when its findings do.""" pytest.importorskip("hypothesis") pytest.importorskip("policyengine_uk.utils.scenario") - from types import SimpleNamespace - import numpy as np - import policyengine_uk from hypothesis import HealthCheck, given, settings from hypothesis import strategies as st - from policyengine_core.reforms import Reform - from policyengine_uk.utils import scenario from policyengine.tax_benefit_models.uk import ( budget_reform_population as population, ) - routes = list(contract.COUNTRY_ROUTES) passes = set() @settings( @@ -1730,53 +1782,140 @@ def array(tag, variable, year): arrays[key] = value return value - class Simulation: - def __init__(self, dataset, reform=None, scenario=None): - self.tag = ( - None - if reform is None and scenario is None - else repr(reform if scenario is None else ("scenario", scenario)) - ) - - @property - def baseline(self): - # A reformed simulation's baseline, often the unreformed one. - baseline = Simulation(dataset=None) - if self.tag is not None: - baseline.tag = repr(("baseline", self.tag)) - return baseline - - def calculate(self, variable, year): - return array(self.tag, variable, year) - - with monkeypatch.context() as patch: - patch.setattr(policyengine_uk, "Microsimulation", Simulation) - patch.setattr( - scenario, "Scenario", lambda **kwargs: repr(sorted(kwargs.items())) - ) - patch.setattr( - Reform, - "from_dict", - staticmethod(lambda reform, **kwargs: ("from_dict", repr(reform))), - ) - patch.setattr( - population, - "subsample", - lambda *args: SimpleNamespace(copy=lambda: None), - ) - patch.setattr( - population, "plus_ten_percent", lambda paths: {"response": paths} - ) - rows = population.run_population_checks("unused.h5", routes) + rows = _run_population_with(monkeypatch, array) _assert_publishable_findings(rows) passes.update((row["check_id"], row["status"]) for row in rows) check() - # Every check both passed and failed somewhere in the run. + # Every country check both passed and failed somewhere in the run. for check_id in population.COUNTRY_POPULATION_CHECKS: assert {(check_id, "pass"), (check_id, "fail")} <= passes, check_id +def _steady(tag, variable, year): + """Arrays where every reform moves nothing: one unit at 1.""" + import numpy as np + + return np.ones(1) + + +def test_routes_that_differ_only_in_2026_disagree(monkeypatch): + pytest.importorskip("policyengine_uk.utils.scenario") + import numpy as np + + from policyengine.tax_benefit_models.uk import ( + budget_reform_population as population, + ) + + def array(tag, variable, year): + if variable == "income_tax" and tag is not None and "baseline" not in tag: + if year == 2026 and "scenario" in tag: + return np.array([5.0]) + if year == 2027: + return np.array([2.0]) + return _steady(tag, variable, year) + + rows = _run_population_with(monkeypatch, array) + _assert_publishable_findings(rows) + agree = { + row["route"]: row + for row in rows + if row["check_id"] == population.POPULATION_ROUTES_AGREE + } + scenario_row = agree["country.reform_dict = country.scenario_parameter_changes"] + assert scenario_row["status"] == "fail" + assert scenario_row["detail"] == { + "income_tax:2026_matches": False, + "income_tax:2027_matches": True, + } + assert agree["country.reform_dict = country.reform_from_dict"]["status"] == "pass" + + +def test_a_state_pension_that_falls_fails_the_pension_check(monkeypatch): + pytest.importorskip("policyengine_uk.utils.scenario") + import numpy as np + + from policyengine.tax_benefit_models.uk import ( + budget_reform_population as population, + ) + + def array(tag, variable, year): + if tag is not None and "state_pension" in tag: + if variable in ("basic_state_pension", "new_state_pension"): + return np.array([1.1, 1.1]) + if variable == "state_pension": + # One higher, one lower: the rates scaled, yet a pension fell. + return np.array([1.1, 0.9]) + return np.ones(2) + + rows = _run_population_with(monkeypatch, array, ["country.reform_dict"]) + _assert_publishable_findings(rows) + pension = next( + row + for row in rows + if row["check_id"] == population.POPULATION_STATE_PENSION_PLUS_10 + ) + assert pension["status"] == "fail" + assert pension["detail"]["basic_and_new_scaled_10_percent"] is True + assert pension["detail"]["some_pension_higher"] is True + assert pension["detail"]["no_pension_lower"] is False + + +def test_a_few_large_changes_among_tiny_ones_withhold_the_total(monkeypatch): + """Three people's £100 rises with seven negligible changes: three records.""" + pytest.importorskip("policyengine_uk.utils.scenario") + import numpy as np + + from policyengine.tax_benefit_models.uk import ( + budget_reform_population as population, + ) + + def array(tag, variable, year): + tax = np.ones(10) + if ( + variable == "income_tax" + and year == 2027 + and tag is not None + and "baseline" not in tag + ): + return tax + np.array([100.0] * 3 + [4.77e-7] * 7) + return tax + + rows = _run_population_with(monkeypatch, array, ["country.reform_dict"]) + april = next( + row + for row in rows + if row["check_id"] == population.POPULATION_APRIL_RATE_CHANGE + ) + assert april["detail"]["2027_people_with_higher_tax"] == population.FEWER_THAN_10 + assert april["detail"]["2027_total_change"] == population.WITHHELD_FEWER_THAN_10 + + +def test_a_detail_that_cannot_be_checked_turns_its_row_into_an_error(monkeypatch): + pytest.importorskip("policyengine_uk.utils.scenario") + import numpy as np + + from policyengine.tax_benefit_models.uk import ( + budget_reform_population as population, + ) + + monkeypatch.setattr( + population, + "uc_allowance_verdict", + lambda base, reformed: (True, {"counts": np.array([3, 12])}), + ) + rows = _run_population_with(monkeypatch, _steady, ["country.reform_dict"]) + uc = next( + row + for row in rows + if row["check_id"] == population.POPULATION_UC_STANDARD_ALLOWANCE_PLUS_10 + ) + assert uc["status"] == "error" + assert uc["reason"] == "TypeError" + assert uc["detail"] is None + assert all(row["status"] != "error" for row in rows if row is not uc) + + def test_a_total_moved_only_by_tiny_changes_in_few_records_is_withheld( monkeypatch, stub_population_checks ): From b02bc5998edee4fcbe34867917e8588a35c79d7a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 16 Oct 2026 18:13:13 -0400 Subject: [PATCH 4/4] Test the unreformed isolation and query-order verdicts An unreformed rebuild unlike the first fails isolation, and outputs that depend on the order they are asked for fail the query-order check; each verdict forced to pass is now caught. Co-Authored-By: Claude Opus 5.5 --- tests/test_uk_budget_reform_contract.py | 58 +++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/tests/test_uk_budget_reform_contract.py b/tests/test_uk_budget_reform_contract.py index d531ee3c..a0a6b80d 100644 --- a/tests/test_uk_budget_reform_contract.py +++ b/tests/test_uk_budget_reform_contract.py @@ -1946,3 +1946,61 @@ def three_people(self, variable, year): assert april["detail"]["2027_people_with_higher_tax"] == 0 assert april["detail"]["2027_total_rises"] is True assert april["detail"]["2027_total_change"] == population.WITHHELD_FEWER_THAN_10 + + +def test_an_unreformed_rebuild_that_differs_fails_isolation(monkeypatch): + """A later unreformed simulation unlike the first: a reform reached it.""" + pytest.importorskip("policyengine_uk.utils.scenario") + import numpy as np + + from policyengine.tax_benefit_models.uk import ( + budget_reform_population as population, + ) + + asked = [] + + def array(tag, variable, year): + if tag is None and (variable, year) == ("income_tax", 2026): + asked.append(1) + return np.array([1.0 if len(asked) == 1 else 2.0]) + return np.ones(1) + + rows = _run_population_with(monkeypatch, array, ["country.reform_dict"]) + _assert_publishable_findings(rows) + status = {(row["check_id"], row["route"]): row for row in rows} + isolation = status[(population.POPULATION_BASELINE_ISOLATION, "unreformed")] + assert isolation["status"] == "fail" + assert isolation["detail"]["income_tax:2026_matches"] is False + + +def test_outputs_that_depend_on_query_order_fail_the_order_check(monkeypatch): + """2026 income tax computed right after 2027's (reverse order) comes out different.""" + pytest.importorskip("policyengine_uk.utils.scenario") + import numpy as np + + from policyengine.tax_benefit_models.uk import ( + budget_reform_population as population, + ) + + last = [] + + def array(tag, variable, year): + value = np.ones(1) + if tag is None: + if (variable, year) == ("income_tax", 2026) and last[-1:] == [ + ("income_tax", 2027) + ]: + value = np.array([2.0]) + last.append((variable, year)) + return value + + rows = _run_population_with(monkeypatch, array, ["country.reform_dict"]) + _assert_publishable_findings(rows) + status = {(row["check_id"], row["route"]): row for row in rows} + assert ( + status[(population.POPULATION_BASELINE_ISOLATION, "unreformed")]["status"] + == "pass" + ) + order = status[(population.POPULATION_QUERY_ORDER, "unreformed")] + assert order["status"] == "fail" + assert order["detail"]["income_tax:2026_matches"] is False