diff --git a/changelog.d/642.fixed.md b/changelog.d/642.fixed.md new file mode 100644 index 00000000..138601d6 --- /dev/null +++ b/changelog.d/642.fixed.md @@ -0,0 +1 @@ +The UK Budget reform contract's population rows no longer report a count of survey records from 1 to 9 (it reads "fewer than 10"), or a total over fewer than 10 changed records, and they state each condition they test as a flat true/false finding in `detail`. diff --git a/src/policyengine/tax_benefit_models/uk/budget_reform_population.py b/src/policyengine/tax_benefit_models/uk/budget_reform_population.py index 56c9042c..83775f80 100644 --- a/src/policyengine/tax_benefit_models/uk/budget_reform_population.py +++ b/src/policyengine/tax_benefit_models/uk/budget_reform_population.py @@ -47,10 +47,32 @@ The last two are responsiveness checks: a reform that reaches the parameter but not the award (as when an award is fixed before the reform applies) fails them. + +Detail +------ + +Each row's ``detail`` states every condition the check tests as a flat +true/false finding (``no_award_lower``, ``some_pension_higher``, +``income_tax:2027_matches`` and so on), so a failed check can be diagnosed +from the findings alone. A row passes exactly when its findings all hold; +one whose name ends ``_informational`` is reported for diagnosis and is not +a condition. Counts and totals follow the findings for local reruns. + +The subsample's households are survey records, so the counts and totals +also follow the small-cell rule: no count from 1 to 9 (it reads ``"fewer +than 10"``), and no total over fewer than 10 changed records. The rule +covers counts and totals only, not an error's message or the difference +between two rows' figures (two routes' rows share a subsample), so the raw +report is not for publication. The qualification ledger publishes from it +only the fixed row fields and the top-level findings; the rule here keeps +raw copies (test fixtures, local runs) free of small counts too. A detail +that cannot be checked turns its row into an error, so the report does not +hold. """ from __future__ import annotations +import numbers import os import tempfile import traceback @@ -75,6 +97,57 @@ ) RESPONSE_YEAR = 2027 +#: The small-cell rule: a count of survey records from 1 to 9 is never +#: published, nor a total over that many. +SMALL_CELL_LIMIT = 10 +FEWER_THAN_10 = "fewer than 10" +WITHHELD_FEWER_THAN_10 = "withheld: fewer than 10 records" + + +def is_small_cell(count: int) -> bool: + return 0 < count < SMALL_CELL_LIMIT + + +def publishable_detail(detail: Any) -> Any: + """``detail`` with every count from 1 to 9, at any depth, suppressed. + + Booleans are findings, not counts, and stay as they are. A detail holds + only booleans, integers, floats, text, None, and lists or text-keyed + mappings of them; anything else (an array, a set) is an error rather + than passed through unchecked. + """ + if isinstance(detail, (bool, np.bool_)): + return bool(detail) + if isinstance(detail, numbers.Integral): + return FEWER_THAN_10 if is_small_cell(int(detail)) else int(detail) + if isinstance(detail, np.floating): + return float(detail) + if detail is None or isinstance(detail, (float, str)): + return detail + if isinstance(detail, dict) and all(isinstance(key, str) for key in detail): + return {key: publishable_detail(value) for key, value in detail.items()} + if isinstance(detail, (list, tuple)): + return [publishable_detail(value) for value in detail] + raise TypeError(f"A population detail cannot hold {type(detail).__name__}.") + + +def total_over_records(total: float, *records: int) -> float | str: + """A total, withheld when any count of the records behind it is 1 to 9. + + Pass each count that bounds how few records the total reflects: those + that moved at all, and those that moved by more than a negligible amount + (a few large changes among negligible ones are still a few records). + """ + if any(is_small_cell(count) for count in records): + return WITHHELD_FEWER_THAN_10 + return total + + +def matches(names: Sequence[str], different: Sequence[str]) -> dict[str, bool]: + """One ``_matches`` finding per compared output.""" + return {f"{name}_matches": name not in different for name in names} + + POPULATION_BASELINE_ISOLATION = "population_baseline_isolation" POPULATION_QUERY_ORDER = "population_query_order" POPULATION_ROUTES_AGREE = "population_routes_agree" @@ -223,18 +296,22 @@ def uc_allowance_verdict( for state_paid, state_award in ((base_paid, base_award), (paid, award)) ) ) - holds = ( - non_finite == 0 - and scaled - and awards_lower == 0 - and payments_unaccounted == 0 - and payments_below_floor == 0 - and payments_higher > 0 - ) - return holds, { + payments_lower = int(np.sum(paid < base_paid - PENNY)) + findings = { "standard_allowance_scaled_10_percent": scaled, + "no_award_lower": awards_lower == 0, + "payments_are_award_less_deductions": payments_unaccounted == 0, + "no_payment_below_the_payable_floor": payments_below_floor == 0, + "some_payment_higher": payments_higher > 0, + "no_non_finite_units": non_finite == 0, + } + holds = all(findings.values()) + return holds, { + **findings, + # Lawful (deductions can rise more than the award), so not a condition. + "no_payment_lower_informational": payments_lower == 0, "awards_lower": awards_lower, - "payments_lower": int(np.sum(paid < base_paid - PENNY)), + "payments_lower": payments_lower, "payments_higher": payments_higher, "payments_not_award_less_deductions": payments_unaccounted, "payments_below_the_payable_floor": payments_below_floor, @@ -380,7 +457,7 @@ def build(route: str | None, reform: dict | None): POPULATION_BASELINE_ISOLATION, route, "fail" if different else "pass", - detail=different or None, + detail=matches(list(before), different), ) ) after = _arrays(build(None, None)) @@ -390,7 +467,7 @@ def build(route: str | None, reform: dict | None): POPULATION_BASELINE_ISOLATION, "unreformed", "fail" if different else "pass", - detail=different or None, + detail=matches(list(before), different), ) ) different = _same(before, _arrays(build(None, None), order=-1)) @@ -399,7 +476,7 @@ def build(route: str | None, reform: dict | None): POPULATION_QUERY_ORDER, "unreformed", "fail" if different else "pass", - detail=different or None, + detail=matches(list(before), different), ) ) tax = { @@ -411,15 +488,18 @@ def build(route: str | None, reform: dict | None): } first = country[0] for route in country[1:]: - same = all( - np.array_equal(tax[route][year], tax[first][year]) + agree = { + f"income_tax:{year}_matches": bool( + np.array_equal(tax[route][year], tax[first][year]) + ) for year in (2026, 2027) - ) + } rows.append( _row( POPULATION_ROUTES_AGREE, f"{first} = {route}", - "pass" if same else "fail", + "pass" if all(agree.values()) else "fail", + detail=agree, ) ) for route in country: @@ -431,6 +511,8 @@ def build(route: str | None, reform: dict | None): # is on the total, and the counts are reported. total_rises = bool(np.sum(change_2027) > 1e-6) ok = unchanged_2026 and total_rises + lower = int(np.sum(change_2027 < -1e-6)) + higher = int(np.sum(change_2027 > 1e-6)) rows.append( _row( POPULATION_APRIL_RATE_CHANGE, @@ -438,23 +520,33 @@ def build(route: str | None, reform: dict | None): "pass" if ok else "fail", detail={ "2026_unchanged": unchanged_2026, - "2027_total_change": float(np.sum(change_2027)), - "2027_people_with_lower_tax": int(np.sum(change_2027 < -1e-6)), - "2027_people_with_higher_tax": int(np.sum(change_2027 > 1e-6)), + "2027_total_rises": total_rises, + # Withheld by the records that moved at all, which + # the total sums, and by those past the threshold. + "2027_total_change": total_over_records( + float(np.sum(change_2027)), + int(np.count_nonzero(change_2027)), + lower + higher, + ), + "2027_people_with_lower_tax": lower, + "2027_people_with_higher_tax": higher, }, ) ) for route in country: simulation = build(route, FINITE) - same = np.array_equal( - np.asarray(simulation.calculate("income_tax", 2028), dtype=float), - before["income_tax:2028"], + same = bool( + np.array_equal( + np.asarray(simulation.calculate("income_tax", 2028), dtype=float), + before["income_tax:2028"], + ) ) rows.append( _row( POPULATION_FINITE_CHANGE_EXPIRES, route, "pass" if same else "fail", + detail={"income_tax:2028_matches": same}, ) ) @@ -506,6 +598,8 @@ def values(simulation, variable): "pass" if scaled and lower == 0 and higher > 0 else "fail", detail={ "basic_and_new_scaled_10_percent": bool(scaled), + "no_pension_lower": lower == 0, + "some_pension_higher": higher > 0, "pensions_lower": lower, "pensions_higher": higher, }, @@ -516,12 +610,19 @@ def values(simulation, variable): _row( *POPULATION_ERROR_DIAGNOSTIC, "error", + reason=type(error).__name__, detail="".join( traceback.format_exception_only(type(error), error) ).strip()[:1000], ) ) for row in rows: + if isinstance(row.get("detail"), dict): + try: + row["detail"] = publishable_detail(row["detail"]) + except TypeError as error: + # Fail closed: drop what cannot be checked, and the row errs. + row.update(status="error", reason=type(error).__name__, detail=None) row.setdefault("subsample_households", SUBSAMPLE_HOUSEHOLDS) row.setdefault("seed", seed) return rows diff --git a/tests/test_uk_budget_reform_contract.py b/tests/test_uk_budget_reform_contract.py index 7f13d280..a0a6b80d 100644 --- a/tests/test_uk_budget_reform_contract.py +++ b/tests/test_uk_budget_reform_contract.py @@ -714,6 +714,7 @@ def fail(*args): "check_id": population.POPULATION_ERROR_DIAGNOSTIC[0], "route": population.POPULATION_ERROR_DIAGNOSTIC[1], "status": "error", + "reason": "OSError", "detail": "OSError: forced population failure", "subsample_households": population.SUBSAMPLE_HOUSEHOLDS, "seed": 7, @@ -885,6 +886,7 @@ def test_population_checks_hold_on_a_tiny_dataset(tmp_path): seed=0, ) assert rows and all(row["status"] == "pass" for row in rows), rows + _assert_publishable_findings(rows) from collections import Counter assert Counter((row["check_id"], row["route"]) for row in rows) == Counter( @@ -901,6 +903,7 @@ def test_responsiveness_checks_fail_with_no_award_to_move(tmp_path): rows = run_population_checks( _tiny_dataset_file(tmp_path), ["country.reform_dict"], seed=0 ) + _assert_publishable_findings(rows) status = {row["check_id"]: row["status"] for row in rows} # Earners only: no award to show a reform reaching it. assert status["population_state_pension_plus_10"] == "fail" @@ -1373,3 +1376,631 @@ def test_every_report_row_has_a_reform_identity_field(): assert "reform_identity" in row, (section, row) if row["reform_identity"] is None: assert row["reform_identity_unavailable"], (section, row) + + +# The small-cell rule and flat findings in population rows. + + +def _is_condition(key, value): + """A finding the check requires; ``*_informational`` ones it does not.""" + return isinstance(value, bool) and not key.endswith("_informational") + + +def _small_counts(value, path="detail"): + """Where ``value`` holds a count from 1 to 9.""" + import numbers + + import numpy as np + + if isinstance(value, (bool, np.bool_)): + return [] + if isinstance(value, numbers.Integral): + return [path] if 0 < value < 10 else [] + if isinstance(value, dict): + return [ + found + for key, item in value.items() + for found in _small_counts(item, f"{path}.{key}") + ] + if isinstance(value, (list, tuple)): + return [ + found + for index, item in enumerate(value) + for found in _small_counts(item, f"{path}[{index}]") + ] + return [] + + +def _assert_publishable_findings(rows): + """No small count anywhere, and a row passes when all its findings hold.""" + for row in rows: + assert _small_counts(row) == [], row + detail = row.get("detail") + if row["route"] == "-": + continue + assert isinstance(detail, dict), row + findings = { + key: value for key, value in detail.items() if _is_condition(key, value) + } + assert findings, row + assert (row["status"] == "pass") == all(findings.values()), row + + +def test_publishable_detail_suppresses_counts_from_1_to_9_at_any_depth(): + import numpy as np + + from policyengine.tax_benefit_models.uk.budget_reform_population import ( + FEWER_THAN_10, + publishable_detail, + ) + + detail = { + "none": 0, + "one": 1, + "nine": 9, + "ten": 10, + "negative": -3, + "finding": True, + "other_finding": False, + "total": 3.5, + "numpy_count": np.int64(5), + "nested": {"count": 4, "list": [2, 12, {"deep": 7}]}, + } + assert publishable_detail(detail) == { + "none": 0, + "one": FEWER_THAN_10, + "nine": FEWER_THAN_10, + "ten": 10, + "negative": -3, + "finding": True, + "other_finding": False, + "total": 3.5, + "numpy_count": FEWER_THAN_10, + "nested": { + "count": FEWER_THAN_10, + "list": [FEWER_THAN_10, 12, {"deep": FEWER_THAN_10}], + }, + } + + +def _equal_but_small_counts(original, published): + """``published`` is ``original`` with only its small counts suppressed.""" + import numbers + + from policyengine.tax_benefit_models.uk.budget_reform_population import ( + FEWER_THAN_10, + ) + + if isinstance(original, bool): + return published is original + if isinstance(original, numbers.Integral) and 0 < original < 10: + return published == FEWER_THAN_10 + if isinstance(original, dict): + return published.keys() == original.keys() and all( + _equal_but_small_counts(value, published[key]) + for key, value in original.items() + ) + if isinstance(original, (list, tuple)): + return len(published) == len(original) and all( + _equal_but_small_counts(a, b) for a, b in zip(original, published) + ) + return published == original + + +def test_publication_removes_every_small_count_and_nothing_else(): + pytest.importorskip("hypothesis") + from hypothesis import given, settings + from hypothesis import strategies as st + + from policyengine.tax_benefit_models.uk.budget_reform_population import ( + publishable_detail, + ) + + leaves = st.one_of( + st.booleans(), + st.integers(-20, 1_000), + st.floats(allow_nan=False), + st.text(max_size=5), + st.none(), + ) + details = st.recursive( + leaves, + lambda children: st.one_of( + st.lists(children, max_size=4), + st.dictionaries(st.text(max_size=5), children, max_size=4), + ), + max_leaves=20, + ) + + @settings(max_examples=200, deadline=None, derandomize=True) + @given(details) + def check(detail): + published = publishable_detail(detail) + assert _small_counts(published) == [] + assert _equal_but_small_counts(detail, published) + assert publishable_detail(published) == published + + check() + + +def test_a_total_over_fewer_than_10_changed_records_is_withheld(): + from policyengine.tax_benefit_models.uk.budget_reform_population import ( + WITHHELD_FEWER_THAN_10, + total_over_records, + ) + + assert total_over_records(12.5, 0) == 12.5 + assert total_over_records(12.5, 1) == WITHHELD_FEWER_THAN_10 + assert total_over_records(12.5, 9) == WITHHELD_FEWER_THAN_10 + assert total_over_records(12.5, 10) == 12.5 + # Any small count of the records behind it withholds it. + assert total_over_records(300.0, 10, 3) == WITHHELD_FEWER_THAN_10 + assert total_over_records(300.0, 10, 0) == 300.0 + + +def test_uc_findings_agree_with_their_counts_and_the_verdict(): + pytest.importorskip("hypothesis") + import numpy as np + from hypothesis import given, settings + from hypothesis import strategies as st + + from policyengine.tax_benefit_models.uk.budget_reform_population import ( + UC_ALLOWANCE_VARIABLES, + ) + + amounts = st.one_of(st.floats(0, 2_000), st.just(float("nan"))) + verdicts = set() + + # From the lawful pair (one capped unit whose payment falls with its + # deductions, one that gains), so verdicts that hold are drawn too; each + # example changes up to three of its values. + @settings(max_examples=200, deadline=None, derandomize=True) + @given(data=st.data()) + def check(data): + base, reformed = _lawful_pair() + for _ in range(data.draw(st.integers(0, 3))): + state = data.draw(st.sampled_from([base, reformed])) + name = data.draw(st.sampled_from(UC_ALLOWANCE_VARIABLES)) + unit = data.draw(st.integers(0, 1)) + state[name] = np.array(state[name], dtype=float) + state[name][unit] = data.draw(amounts) + holds, detail = _verdict(base, reformed) + verdicts.add(holds) + counts = { + "no_award_lower": "awards_lower", + "payments_are_award_less_deductions": "payments_not_award_less_deductions", + "no_payment_below_the_payable_floor": "payments_below_the_payable_floor", + "no_non_finite_units": "units_with_non_finite_values", + "no_payment_lower_informational": "payments_lower", + } + for finding, count in counts.items(): + assert detail[finding] is (detail[count] == 0), finding + assert detail["some_payment_higher"] is (detail["payments_higher"] > 0) + # The verdict as it was defined from the counts. + assert holds == ( + detail["units_with_non_finite_values"] == 0 + and detail["standard_allowance_scaled_10_percent"] + and detail["awards_lower"] == 0 + and detail["payments_not_award_less_deductions"] == 0 + and detail["payments_below_the_payable_floor"] == 0 + and detail["payments_higher"] > 0 + ) + + check() + assert verdicts == {True, False} + + +def test_population_rows_state_each_condition_as_a_flat_finding(stub_population_checks): + population = stub_population_checks + rows = population.run_population_checks("unused.h5", list(contract.COUNTRY_ROUTES)) + _assert_publishable_findings(rows) + outputs = { + f"{variable}:{year}_matches" + for variable in population.OUTPUTS + for year in population.YEARS + } + expected = { + population.POPULATION_BASELINE_ISOLATION: outputs, + population.POPULATION_QUERY_ORDER: outputs, + population.POPULATION_ROUTES_AGREE: { + "income_tax:2026_matches", + "income_tax:2027_matches", + }, + population.POPULATION_APRIL_RATE_CHANGE: {"2026_unchanged", "2027_total_rises"}, + population.POPULATION_FINITE_CHANGE_EXPIRES: {"income_tax:2028_matches"}, + population.POPULATION_UC_STANDARD_ALLOWANCE_PLUS_10: { + "standard_allowance_scaled_10_percent", + "no_award_lower", + "payments_are_award_less_deductions", + "no_payment_below_the_payable_floor", + "some_payment_higher", + "no_non_finite_units", + "no_payment_lower_informational", + }, + population.POPULATION_STATE_PENSION_PLUS_10: { + "basic_and_new_scaled_10_percent", + "no_pension_lower", + "some_pension_higher", + }, + } + for row in rows: + findings = { + key for key, value in row["detail"].items() if isinstance(value, bool) + } + assert findings == expected[row["check_id"]], row + # The stub moves one person's tax: one record, so the count reads + # "fewer than 10" and the total is withheld. + april = next( + row + for row in rows + if row["check_id"] == population.POPULATION_APRIL_RATE_CHANGE + ) + assert april["detail"]["2027_people_with_higher_tax"] == population.FEWER_THAN_10 + assert april["detail"]["2027_total_change"] == population.WITHHELD_FEWER_THAN_10 + + +def test_a_detail_of_another_type_is_an_error_not_passed_through(): + import numpy as np + + from policyengine.tax_benefit_models.uk.budget_reform_population import ( + publishable_detail, + ) + + assert publishable_detail({"finding": np.bool_(True)}) == {"finding": True} + assert type(publishable_detail(np.bool_(False))) is bool + for unchecked in (np.array([3, 12]), {3, 12}, {3: "count key"}): + with pytest.raises(TypeError): + publishable_detail({"value": unchecked}) + + +def _run_population_with(monkeypatch, array, routes=None): + """Run the population checks on arrays from ``array(tag, variable, year)``. + + ``tag`` is None for an unreformed simulation, and otherwise names the + route and reform: the reform dict's repr, or a repr containing + "scenario" or "from_dict", or one starting "('baseline'" for a reformed + simulation's baseline. + """ + from types import SimpleNamespace + + import policyengine_uk + from policyengine_core.reforms import Reform + from policyengine_uk.utils import scenario + + from policyengine.tax_benefit_models.uk import ( + budget_reform_population as population, + ) + + class Simulation: + def __init__(self, dataset, reform=None, scenario=None): + self.tag = ( + None + if reform is None and scenario is None + else repr(reform if scenario is None else ("scenario", scenario)) + ) + + @property + def baseline(self): + baseline = Simulation(dataset=None) + if self.tag is not None: + baseline.tag = repr(("baseline", self.tag)) + return baseline + + def calculate(self, variable, year): + return array(self.tag, variable, year) + + with monkeypatch.context() as patch: + patch.setattr(policyengine_uk, "Microsimulation", Simulation) + patch.setattr( + scenario, "Scenario", lambda **kwargs: repr(sorted(kwargs.items())) + ) + patch.setattr( + Reform, + "from_dict", + staticmethod(lambda reform, **kwargs: ("from_dict", repr(reform))), + ) + patch.setattr( + population, "subsample", lambda *args: SimpleNamespace(copy=lambda: None) + ) + patch.setattr(population, "plus_ten_percent", lambda paths: {"response": paths}) + return population.run_population_checks( + "unused.h5", routes or list(contract.COUNTRY_ROUTES) + ) + + +def test_every_population_row_publishes_the_findings_that_decide_it(monkeypatch): + """On random arrays: no small count, and each row passes when its findings do.""" + pytest.importorskip("hypothesis") + pytest.importorskip("policyengine_uk.utils.scenario") + import numpy as np + from hypothesis import HealthCheck, given, settings + from hypothesis import strategies as st + + from policyengine.tax_benefit_models.uk import ( + budget_reform_population as population, + ) + + passes = set() + + @settings( + max_examples=60, + deadline=None, + derandomize=True, + suppress_health_check=[HealthCheck.function_scoped_fixture], + ) + @given(data=st.data()) + def check(data): + units = data.draw(st.integers(1, 12)) + amounts = st.floats(0, 1_000, width=32) + if data.draw(st.booleans()): + amounts = st.one_of(amounts, st.just(float("nan"))) + # Lawful responses: the Universal Credit and State Pension + # simulations move their awards as the law does, so those rows can + # pass; otherwise every reformed array is random or unreformed. + lawful = data.draw(st.booleans()) + pensions = ("basic_state_pension", "new_state_pension", "state_pension") + arrays = {} + + def drawn(variable): + values = amounts + if lawful and variable == "uc_standard_allowance": + values = st.floats(1, 1_000, width=32) + return np.array( + data.draw(st.lists(values, min_size=units, max_size=units)), + dtype=float, + ) + + def array(tag, variable, year): + key = (tag, variable, year) + if key in arrays: + return arrays[key] + if tag is None: + if lawful and variable in ("benefit_cap_reduction", "uc_deductions"): + value = np.zeros(units) + elif lawful and variable == "universal_credit_pre_benefit_cap": + value = array(None, "universal_credit", year) + else: + value = drawn(variable) + elif ( + lawful + and "universal_credit.standard_allowance" in tag + and variable in population.UC_ALLOWANCE_VARIABLES + ): + allowance = array(None, "uc_standard_allowance", year) + base = array(None, variable, year) + value = { + "uc_standard_allowance": 1.1 * allowance, + "universal_credit_pre_benefit_cap": base + 0.1 * allowance, + "universal_credit": base + 0.1 * allowance, + }.get(variable, base) + elif lawful and "state_pension" in tag and variable in pensions: + value = 1.1 * array(None, variable, year) + elif data.draw(st.booleans()): + value = array(None, variable, year) + else: + value = drawn(variable) + arrays[key] = value + return value + + rows = _run_population_with(monkeypatch, array) + _assert_publishable_findings(rows) + passes.update((row["check_id"], row["status"]) for row in rows) + + check() + # Every country check both passed and failed somewhere in the run. + for check_id in population.COUNTRY_POPULATION_CHECKS: + assert {(check_id, "pass"), (check_id, "fail")} <= passes, check_id + + +def _steady(tag, variable, year): + """Arrays where every reform moves nothing: one unit at 1.""" + import numpy as np + + return np.ones(1) + + +def test_routes_that_differ_only_in_2026_disagree(monkeypatch): + pytest.importorskip("policyengine_uk.utils.scenario") + import numpy as np + + from policyengine.tax_benefit_models.uk import ( + budget_reform_population as population, + ) + + def array(tag, variable, year): + if variable == "income_tax" and tag is not None and "baseline" not in tag: + if year == 2026 and "scenario" in tag: + return np.array([5.0]) + if year == 2027: + return np.array([2.0]) + return _steady(tag, variable, year) + + rows = _run_population_with(monkeypatch, array) + _assert_publishable_findings(rows) + agree = { + row["route"]: row + for row in rows + if row["check_id"] == population.POPULATION_ROUTES_AGREE + } + scenario_row = agree["country.reform_dict = country.scenario_parameter_changes"] + assert scenario_row["status"] == "fail" + assert scenario_row["detail"] == { + "income_tax:2026_matches": False, + "income_tax:2027_matches": True, + } + assert agree["country.reform_dict = country.reform_from_dict"]["status"] == "pass" + + +def test_a_state_pension_that_falls_fails_the_pension_check(monkeypatch): + pytest.importorskip("policyengine_uk.utils.scenario") + import numpy as np + + from policyengine.tax_benefit_models.uk import ( + budget_reform_population as population, + ) + + def array(tag, variable, year): + if tag is not None and "state_pension" in tag: + if variable in ("basic_state_pension", "new_state_pension"): + return np.array([1.1, 1.1]) + if variable == "state_pension": + # One higher, one lower: the rates scaled, yet a pension fell. + return np.array([1.1, 0.9]) + return np.ones(2) + + rows = _run_population_with(monkeypatch, array, ["country.reform_dict"]) + _assert_publishable_findings(rows) + pension = next( + row + for row in rows + if row["check_id"] == population.POPULATION_STATE_PENSION_PLUS_10 + ) + assert pension["status"] == "fail" + assert pension["detail"]["basic_and_new_scaled_10_percent"] is True + assert pension["detail"]["some_pension_higher"] is True + assert pension["detail"]["no_pension_lower"] is False + + +def test_a_few_large_changes_among_tiny_ones_withhold_the_total(monkeypatch): + """Three people's £100 rises with seven negligible changes: three records.""" + pytest.importorskip("policyengine_uk.utils.scenario") + import numpy as np + + from policyengine.tax_benefit_models.uk import ( + budget_reform_population as population, + ) + + def array(tag, variable, year): + tax = np.ones(10) + if ( + variable == "income_tax" + and year == 2027 + and tag is not None + and "baseline" not in tag + ): + return tax + np.array([100.0] * 3 + [4.77e-7] * 7) + return tax + + rows = _run_population_with(monkeypatch, array, ["country.reform_dict"]) + april = next( + row + for row in rows + if row["check_id"] == population.POPULATION_APRIL_RATE_CHANGE + ) + assert april["detail"]["2027_people_with_higher_tax"] == population.FEWER_THAN_10 + assert april["detail"]["2027_total_change"] == population.WITHHELD_FEWER_THAN_10 + + +def test_a_detail_that_cannot_be_checked_turns_its_row_into_an_error(monkeypatch): + pytest.importorskip("policyengine_uk.utils.scenario") + import numpy as np + + from policyengine.tax_benefit_models.uk import ( + budget_reform_population as population, + ) + + monkeypatch.setattr( + population, + "uc_allowance_verdict", + lambda base, reformed: (True, {"counts": np.array([3, 12])}), + ) + rows = _run_population_with(monkeypatch, _steady, ["country.reform_dict"]) + uc = next( + row + for row in rows + if row["check_id"] == population.POPULATION_UC_STANDARD_ALLOWANCE_PLUS_10 + ) + assert uc["status"] == "error" + assert uc["reason"] == "TypeError" + assert uc["detail"] is None + assert all(row["status"] != "error" for row in rows if row is not uc) + + +def test_a_total_moved_only_by_tiny_changes_in_few_records_is_withheld( + monkeypatch, stub_population_checks +): + """Withheld by the records that moved at all, not just past the threshold.""" + import numpy as np + import policyengine_uk + + population = stub_population_checks + stub = policyengine_uk.Microsimulation + calculate = stub.calculate + + def three_people(self, variable, year): + if variable != "income_tax": + return calculate(self, variable, year) + tax = np.full(3, 1.0) + if year == 2027 and self.reform in (population.APRIL, population.FINITE): + # Each below the 1e-6 threshold the counts use; together above it. + return tax + 4.77e-7 + return tax + + monkeypatch.setattr(stub, "calculate", three_people) + rows = population.run_population_checks("unused.h5", ["country.reform_dict"]) + april = next( + row + for row in rows + if row["check_id"] == population.POPULATION_APRIL_RATE_CHANGE + ) + assert april["detail"]["2027_people_with_higher_tax"] == 0 + assert april["detail"]["2027_total_rises"] is True + assert april["detail"]["2027_total_change"] == population.WITHHELD_FEWER_THAN_10 + + +def test_an_unreformed_rebuild_that_differs_fails_isolation(monkeypatch): + """A later unreformed simulation unlike the first: a reform reached it.""" + pytest.importorskip("policyengine_uk.utils.scenario") + import numpy as np + + from policyengine.tax_benefit_models.uk import ( + budget_reform_population as population, + ) + + asked = [] + + def array(tag, variable, year): + if tag is None and (variable, year) == ("income_tax", 2026): + asked.append(1) + return np.array([1.0 if len(asked) == 1 else 2.0]) + return np.ones(1) + + rows = _run_population_with(monkeypatch, array, ["country.reform_dict"]) + _assert_publishable_findings(rows) + status = {(row["check_id"], row["route"]): row for row in rows} + isolation = status[(population.POPULATION_BASELINE_ISOLATION, "unreformed")] + assert isolation["status"] == "fail" + assert isolation["detail"]["income_tax:2026_matches"] is False + + +def test_outputs_that_depend_on_query_order_fail_the_order_check(monkeypatch): + """2026 income tax computed right after 2027's (reverse order) comes out different.""" + pytest.importorskip("policyengine_uk.utils.scenario") + import numpy as np + + from policyengine.tax_benefit_models.uk import ( + budget_reform_population as population, + ) + + last = [] + + def array(tag, variable, year): + value = np.ones(1) + if tag is None: + if (variable, year) == ("income_tax", 2026) and last[-1:] == [ + ("income_tax", 2027) + ]: + value = np.array([2.0]) + last.append((variable, year)) + return value + + rows = _run_population_with(monkeypatch, array, ["country.reform_dict"]) + _assert_publishable_findings(rows) + status = {(row["check_id"], row["route"]): row for row in rows} + assert ( + status[(population.POPULATION_BASELINE_ISOLATION, "unreformed")]["status"] + == "pass" + ) + order = status[(population.POPULATION_QUERY_ORDER, "unreformed")] + assert order["status"] == "fail" + assert order["detail"]["income_tax:2026_matches"] is False