diff --git a/changelog.d/ruff-complexity-application.changed.md b/changelog.d/ruff-complexity-application.changed.md new file mode 100644 index 000000000..f03191d37 --- /dev/null +++ b/changelog.d/ruff-complexity-application.changed.md @@ -0,0 +1 @@ +Refactored the rest of the application under ruff's complexity rules (C901, PLR): the ASGI shell, Flask migration logging, household and policy routes, v1 and v2 household translation and validation, saved-policy mirrors, catalog extraction and the metadata layers are split into named steps, related arguments travel as small value objects, and magic numbers are named, with characterization tests pinning the behaviour first; responses, error messages, logs and published schemas are unchanged. diff --git a/policyengine_api/asgi_factory.py b/policyengine_api/asgi_factory.py index aa3499585..0976be843 100644 --- a/policyengine_api/asgi_factory.py +++ b/policyengine_api/asgi_factory.py @@ -39,7 +39,7 @@ RouteImplementation, RouteImplementationSettings, ) -from policyengine_api.migration_logging import log_migration_request +from policyengine_api.migration_logging import ServedRequest, log_migration_request from policyengine_api.observability import get_runtime from policyengine_api.observability.identifiers import ( OBSERVABILITY_ID_HEADER, @@ -56,6 +56,7 @@ if TYPE_CHECKING: from collections.abc import AsyncIterator, Awaitable, Callable, Iterable + from contextlib import AbstractAsyncContextManager from policyengine_observability import ObservabilityRuntime from starlette.routing import BaseRoute @@ -103,20 +104,221 @@ def _native_route_template(app: FastAPI, scope: Scope) -> str | None: return _matched_route_template(app.router.routes, scope) -def create_asgi_app( - wsgi_app: Any, # noqa: ANN401 # Flask's WSGI signature and a2wsgi's WSGIApp stubs disagree - *, - route_settings: RouteImplementationSettings | None = None, - dependencies: NativeRouteDependencies | None = None, - shutdown_callback: Callable[[], None] | None = None, - observability_runtime: ObservabilityRuntime | None = None, -) -> ASGIApp: - """Create the Stage 2 FastAPI shell around the existing Flask app.""" - if route_settings is None: - route_settings = RouteImplementationSettings.from_environment() - if dependencies is None: - dependencies = NativeRouteDependencies.defaults() - request_runtime = observability_runtime or get_runtime() +def _bind_request_id(request: Request, request_id: str) -> None: + """Carry ``request_id`` in the request's headers and state.""" + MutableHeaders(scope=request.scope)[REQUEST_ID_HEADER] = request_id + request.state.policyengine_request_id = request_id + + +class _RequestContextTokens: + """Set one ASGI request's context variables, and reset them when it ends.""" + + def __init__(self, request_id: str, incoming_observability_id: str | None) -> None: + """Set the request ID, no observability ID and the incoming one.""" + self._request_id_token = _asgi_request_id.set(request_id) + self._observability_id_token = _asgi_observability_id.set(None) + self._incoming_observability_id_token = _asgi_incoming_observability_id.set( + incoming_observability_id + ) + + def rebind_request_id(self, request_id: str) -> None: + """Replace the request ID in the context.""" + _asgi_request_id.reset(self._request_id_token) + self._request_id_token = _asgi_request_id.set(request_id) + + def reset(self) -> None: + """Restore every context variable to its value before the request.""" + _asgi_request_id.reset(self._request_id_token) + _asgi_observability_id.reset(self._observability_id_token) + _asgi_incoming_observability_id.reset(self._incoming_observability_id_token) + + +class _NativeRouteTelemetry: + """The observability record and migration log of one native-route request. + + Telemetry and migration logging must never fail the request, so every call + to the runtime or the logger swallows its own errors. + """ + + def __init__( + self, + runtime: ObservabilityRuntime, + request: Request, + *, + started_at: float, + initial_route: str, + ) -> None: + """Track ``request``, which started at ``started_at`` on ``initial_route``.""" + self._runtime = runtime + self._request = request + self._started_at = started_at + self._initial_route = initial_route + self._request_id: str | None = None + + def begin(self, request_id: str, context_tokens: _RequestContextTokens) -> str: + """Begin the runtime's request record and return the request ID to use. + + A request ID that the runtime assigns replaces ``request_id`` on the + request and in its context. + """ + try: + runtime_request_id = self._runtime.begin_request( + headers={ + **dict(self._request.headers), + REQUEST_ID_HEADER: request_id, + }, + method=self._request.method, + route=self._initial_route, + ) + if isinstance(runtime_request_id, str) and runtime_request_id: + request_id = runtime_request_id + _bind_request_id(self._request, request_id) + context_tokens.rebind_request_id(request_id) + except Exception: # noqa: BLE001, S110 # telemetry must never fail the request + pass + # Telemetry must never fail the request. + with contextlib.suppress(Exception): + self._runtime.set_context( + request_id=request_id, + ) + self._request_id = request_id + return request_id + + def annotate_response( + self, + response: Response, + observability_id: str | None, + ) -> None: + """Record the response's observability ID and add the runtime's headers.""" + if observability_id is not None: + # Telemetry must never fail the request. + with contextlib.suppress(Exception): + # Route-level spans have completed at this point, so this + # call applies the selected calculation identifier to the + # still-current server request span. + self._runtime.set_context(observability_id=observability_id) + try: + for name, value in self._runtime.response_headers().items(): + response.headers[name] = value + except Exception: # noqa: BLE001, S110 # telemetry must never fail the request + pass + + def log(self, status_code: int) -> None: + """Log the served request in the migration request log.""" + # Migration logging must never fail the request. + with contextlib.suppress(Exception): + log_migration_request( + ServedRequest( + request_id=self._request_id, + method=self._request.method, + path=self._request.url.path, + status_code=status_code, + started_at=self._started_at, + country_id=( + self._request.path_params.get("country_id") + or self._request.query_params.get("country_id") + ), + route_impl=RouteImplementation.FASTAPI_NATIVE, + ) + ) + + def finish( + self, + status_code: int, + error: BaseException | None = None, + ) -> None: + """End the runtime's request record with its resolved route and status.""" + resolved_route = getattr( + self._request.scope.get("route"), "path", self._initial_route + ) + # Telemetry must never fail the request. + with contextlib.suppress(Exception): + self._runtime.update_request_route(resolved_route) + # Telemetry must never fail the request. + with contextlib.suppress(Exception): + self._runtime.update_request_status(status_code) + # Telemetry must never fail the request. + with contextlib.suppress(Exception): + self._runtime.end_request( + status_code=status_code, + error=error, + ) + + +async def _call_route( + request: Request, + call_next: Callable[[Request], Awaitable[Response]], + telemetry: _NativeRouteTelemetry | None, +) -> Response: + """Call the route, recording a native route that raises as a server error.""" + try: + return await call_next(request) + except Exception as error: + if telemetry is not None: + telemetry.log(500) + telemetry.finish(500, error) + raise + + +def _finish_response( + response: Response, + request_id: str, + telemetry: _NativeRouteTelemetry | None, +) -> Response: + """Add the request and observability IDs to a response and record it.""" + response_observability_id = current_observability_id() or ( + normalize_observability_id(response.headers.get(OBSERVABILITY_ID_HEADER)) + ) + if telemetry is not None: + telemetry.annotate_response(response, response_observability_id) + _apply_request_id_header(response, request_id) + if response_observability_id is not None: + _apply_observability_id_header(response, response_observability_id) + elif OBSERVABILITY_ID_HEADER in response.headers: + del response.headers[OBSERVABILITY_ID_HEADER] + if telemetry is not None: + telemetry.log(response.status_code) + telemetry.finish(response.status_code) + return response + + +async def _serve_with_request_context( + app: FastAPI, + request_runtime: ObservabilityRuntime, + request: Request, + call_next: Callable[[Request], Awaitable[Response]], +) -> Response: + """Serve one request with its request ID, context and native-route telemetry.""" + started_at = time.time() + request_id = request.headers.get(REQUEST_ID_HEADER) or generate_request_id() + incoming_observability_id = normalize_observability_id( + request.headers.get(OBSERVABILITY_ID_HEADER) + ) + _bind_request_id(request, request_id) + request.state.policyengine_observability_id = None + context_tokens = _RequestContextTokens(request_id, incoming_observability_id) + native_route_template = _native_route_template(app, request.scope) + telemetry = None + if native_route_template is not None: + telemetry = _NativeRouteTelemetry( + request_runtime, + request, + started_at=started_at, + initial_route=native_route_template or request.url.path, + ) + request_id = telemetry.begin(request_id, context_tokens) + + try: + response = await _call_route(request, call_next, telemetry) + return _finish_response(response, request_id, telemetry) + finally: + context_tokens.reset() + + +def _lifespan_with_shutdown( + shutdown_callback: Callable[[], None] | None, +) -> Callable[[FastAPI], AbstractAsyncContextManager[None]]: + """Build a lifespan that calls ``shutdown_callback``, if any, as the app stops.""" @asynccontextmanager async def lifespan(_app: FastAPI) -> AsyncIterator[None]: @@ -126,204 +328,98 @@ async def lifespan(_app: FastAPI) -> AsyncIterator[None]: if shutdown_callback is not None: shutdown_callback() - app = FastAPI( - title="PolicyEngine API", - version=VERSION, - docs_url=None, - redoc_url=None, - openapi_url=None, - lifespan=lifespan, + return lifespan + + +async def _add_headers_to_unhandled_errors( + request: Request, + _error: Exception, +) -> Response: + """Answer an unhandled error with a plain 500 that keeps the request IDs.""" + response = PlainTextResponse("Internal Server Error", status_code=500) + request_id = getattr( + request.state, + "policyengine_request_id", + request.headers.get(REQUEST_ID_HEADER) or generate_request_id(), ) - # compresslevel 4 instead of starlette's default 9: /us/metadata is ~70MB - # raw, and level-9 compression inside the request costs seconds of CPU on - # Cloud Run because it does not have an nginx sidecar to handle gzip. - # Measured on the real payload: level 9 = 5.5s -> 9.0MB, level 4 = 0.7s -> - # 9.9MB — 7.5x faster for ~10% larger output. - app.add_middleware(GZipMiddleware, minimum_size=1000, compresslevel=4) + _apply_request_id_header(response, request_id) + observability_id = current_observability_id() + if observability_id is not None: + _apply_observability_id_header(response, observability_id) + return response - @app.exception_handler(Exception) - async def add_headers_to_unhandled_errors( - request: Request, - _error: Exception, - ) -> Response: - response = PlainTextResponse("Internal Server Error", status_code=500) - request_id = getattr( - request.state, - "policyengine_request_id", - request.headers.get(REQUEST_ID_HEADER) or generate_request_id(), - ) - _apply_request_id_header(response, request_id) - observability_id = current_observability_id() - if observability_id is not None: - _apply_observability_id_header(response, observability_id) - return response - @app.exception_handler(RequestValidationError) - async def typed_v2_request_validation_error( - request: Request, - error: RequestValidationError, - ) -> Response: - if request.url.path.startswith("/v2/"): - return v2_error_response(422, "Invalid API v2 request") - return await request_validation_exception_handler(request, error) - - @app.exception_handler(V2RequestTooLargeError) - async def oversized_v2_request( - _request: Request, - error: V2RequestTooLargeError, - ) -> Response: - return v2_error_response(413, str(error)) +async def _typed_v2_request_validation_error( + request: Request, + error: RequestValidationError, +) -> Response: + """Answer an invalid v2 request with the typed v2 error, and others as usual.""" + if request.url.path.startswith("/v2/"): + return v2_error_response(422, "Invalid API v2 request") + return await request_validation_exception_handler(request, error) - @app.exception_handler(V2AuthenticationRequiredError) - async def unauthenticated_v2_request( - _request: Request, - error: V2AuthenticationRequiredError, - ) -> Response: - return v2_error_response( - 401, - str(error), - headers={"WWW-Authenticate": error.challenge}, - ) - @app.exception_handler(V2PermissionDeniedError) - async def forbidden_v2_request( - _request: Request, - error: V2PermissionDeniedError, - ) -> Response: - return v2_error_response(403, str(error)) +async def _oversized_v2_request( + _request: Request, + error: V2RequestTooLargeError, +) -> Response: + """Answer an oversized v2 request body with the typed v2 error.""" + return v2_error_response(413, str(error)) - @app.exception_handler(V2AuthenticationUnavailableError) - async def unavailable_v2_authentication( - _request: Request, - error: V2AuthenticationUnavailableError, - ) -> Response: - return v2_error_response(503, str(error)) + +async def _unauthenticated_v2_request( + _request: Request, + error: V2AuthenticationRequiredError, +) -> Response: + """Answer a v2 request without a valid credential with a typed 401 challenge.""" + return v2_error_response( + 401, + str(error), + headers={"WWW-Authenticate": error.challenge}, + ) + + +async def _forbidden_v2_request( + _request: Request, + error: V2PermissionDeniedError, +) -> Response: + """Answer a v2 request the caller may not make with the typed 403 error.""" + return v2_error_response(403, str(error)) + + +async def _unavailable_v2_authentication( + _request: Request, + error: V2AuthenticationUnavailableError, +) -> Response: + """Answer a v2 request whose credential cannot be checked with a typed 503.""" + return v2_error_response(503, str(error)) + + +def _register_request_context_middleware( + app: FastAPI, + request_runtime: ObservabilityRuntime, +) -> None: + """Serve every request through its request context and telemetry.""" @app.middleware("http") async def add_request_context_and_migration_logging( request: Request, call_next: Callable[[Request], Awaitable[Response]], ) -> Response: - started_at = time.time() - request_id = request.headers.get(REQUEST_ID_HEADER) or generate_request_id() - incoming_observability_id = normalize_observability_id( - request.headers.get(OBSERVABILITY_ID_HEADER) - ) - MutableHeaders(scope=request.scope)[REQUEST_ID_HEADER] = request_id - request.state.policyengine_request_id = request_id - request.state.policyengine_observability_id = None - context_token = _asgi_request_id.set(request_id) - observability_context_token = _asgi_observability_id.set(None) - incoming_observability_context_token = _asgi_incoming_observability_id.set( - incoming_observability_id + return await _serve_with_request_context( + app, + request_runtime, + request, + call_next, ) - native_route_template = _native_route_template(app, request.scope) - native_request = native_route_template is not None - initial_route = native_route_template or request.url.path - - if native_request: - try: - runtime_request_id = request_runtime.begin_request( - headers={ - **dict(request.headers), - REQUEST_ID_HEADER: request_id, - }, - method=request.method, - route=initial_route, - ) - if isinstance(runtime_request_id, str) and runtime_request_id: - request_id = runtime_request_id - MutableHeaders(scope=request.scope)[REQUEST_ID_HEADER] = request_id - request.state.policyengine_request_id = request_id - _asgi_request_id.reset(context_token) - context_token = _asgi_request_id.set(request_id) - except Exception: # noqa: BLE001, S110 # telemetry must never fail the request - pass - # Telemetry must never fail the request. - with contextlib.suppress(Exception): - request_runtime.set_context( - request_id=request_id, - ) - - def log_native_route(status_code: int) -> None: - if not native_request: - return - # Migration logging must never fail the request. - with contextlib.suppress(Exception): - log_migration_request( - request_id=request_id, - method=request.method, - path=request.url.path, - status_code=status_code, - started_at=started_at, - country_id=( - request.path_params.get("country_id") - or request.query_params.get("country_id") - ), - route_impl=RouteImplementation.FASTAPI_NATIVE, - ) - - def finish_native_route( - status_code: int, - error: BaseException | None = None, - ) -> None: - if not native_request: - return - resolved_route = getattr(request.scope.get("route"), "path", initial_route) - # Telemetry must never fail the request. - with contextlib.suppress(Exception): - request_runtime.update_request_route(resolved_route) - # Telemetry must never fail the request. - with contextlib.suppress(Exception): - request_runtime.update_request_status(status_code) - # Telemetry must never fail the request. - with contextlib.suppress(Exception): - request_runtime.end_request( - status_code=status_code, - error=error, - ) - try: - try: - response = await call_next(request) - except Exception as error: - log_native_route(500) - finish_native_route(500, error) - raise - response_observability_id = ( - current_observability_id() - or normalize_observability_id( - response.headers.get(OBSERVABILITY_ID_HEADER) - ) - ) - if native_request and response_observability_id is not None: - # Telemetry must never fail the request. - with contextlib.suppress(Exception): - # Route-level spans have completed at this point, so this - # call applies the selected calculation identifier to the - # still-current server request span. - request_runtime.set_context( - observability_id=response_observability_id - ) - if native_request: - try: - for name, value in request_runtime.response_headers().items(): - response.headers[name] = value - except Exception: # noqa: BLE001, S110 # telemetry must never fail the request - pass - _apply_request_id_header(response, request_id) - if response_observability_id is not None: - _apply_observability_id_header(response, response_observability_id) - elif OBSERVABILITY_ID_HEADER in response.headers: - del response.headers[OBSERVABILITY_ID_HEADER] - log_native_route(response.status_code) - finish_native_route(response.status_code) - return response - finally: - _asgi_request_id.reset(context_token) - _asgi_observability_id.reset(observability_context_token) - _asgi_incoming_observability_id.reset(incoming_observability_context_token) +def _include_native_routers( + app: FastAPI, + route_settings: RouteImplementationSettings, + dependencies: NativeRouteDependencies, +) -> None: + """Include the always-native routers and those the settings select.""" app.include_router(build_core_health_router(dependencies)) app.include_router(build_v2_router(dependencies)) if route_settings.health is RouteImplementation.FASTAPI_NATIVE: @@ -333,6 +429,47 @@ def finish_native_route( if route_settings.metadata is RouteImplementation.FASTAPI_NATIVE: app.include_router(build_metadata_router(dependencies)) + +def create_asgi_app( + wsgi_app: Any, # noqa: ANN401 # Flask's WSGI signature and a2wsgi's WSGIApp stubs disagree + *, + route_settings: RouteImplementationSettings | None = None, + dependencies: NativeRouteDependencies | None = None, + shutdown_callback: Callable[[], None] | None = None, + observability_runtime: ObservabilityRuntime | None = None, +) -> ASGIApp: + """Create the Stage 2 FastAPI shell around the existing Flask app.""" + if route_settings is None: + route_settings = RouteImplementationSettings.from_environment() + if dependencies is None: + dependencies = NativeRouteDependencies.defaults() + request_runtime = observability_runtime or get_runtime() + + app = FastAPI( + title="PolicyEngine API", + version=VERSION, + docs_url=None, + redoc_url=None, + openapi_url=None, + lifespan=_lifespan_with_shutdown(shutdown_callback), + ) + # compresslevel 4 instead of starlette's default 9: /us/metadata is ~70MB + # raw, and level-9 compression inside the request costs seconds of CPU on + # Cloud Run because it does not have an nginx sidecar to handle gzip. + # Measured on the real payload: level 9 = 5.5s -> 9.0MB, level 4 = 0.7s -> + # 9.9MB — 7.5x faster for ~10% larger output. + app.add_middleware(GZipMiddleware, minimum_size=1000, compresslevel=4) + app.exception_handler(Exception)(_add_headers_to_unhandled_errors) + app.exception_handler(RequestValidationError)(_typed_v2_request_validation_error) + app.exception_handler(V2RequestTooLargeError)(_oversized_v2_request) + app.exception_handler(V2AuthenticationRequiredError)(_unauthenticated_v2_request) + app.exception_handler(V2PermissionDeniedError)(_forbidden_v2_request) + app.exception_handler(V2AuthenticationUnavailableError)( + _unavailable_v2_authentication + ) + _register_request_context_middleware(app, request_runtime) + _include_native_routers(app, route_settings, dependencies) + app.mount("/", WSGIMiddleware(wsgi_app)) # The public API already permits every web origin. Use the standard ASGI # implementation while preserving the existing reflected-origin response. diff --git a/policyengine_api/constants.py b/policyengine_api/constants.py index a4110cb3c..a1dcb2cad 100644 --- a/policyengine_api/constants.py +++ b/policyengine_api/constants.py @@ -192,6 +192,10 @@ def get_bundle_default_dataset_option(country_id: str) -> dict: "economy_impact": 3, "report_output": 3, } +# Runtime cache versions are stored in legacy VARCHAR(10) columns, and keep at +# least this many digest characters after their scope prefix and schema version. +RUNTIME_CACHE_VERSION_LENGTH = 10 +MINIMUM_RUNTIME_CACHE_DIGEST_LENGTH = 4 def _build_runtime_cache_version( @@ -204,8 +208,8 @@ def _build_runtime_cache_version( """ schema_version = str(RUNTIME_CACHE_SCHEMA_VERSIONS[scope]) prefix = "e" if scope == "economy_impact" else "r" - digest_length = 10 - len(prefix) - len(schema_version) - if digest_length < 4: + digest_length = RUNTIME_CACHE_VERSION_LENGTH - len(prefix) - len(schema_version) + if digest_length < MINIMUM_RUNTIME_CACHE_DIGEST_LENGTH: msg = f"Runtime cache version for {scope} does not fit in VARCHAR(10)" raise ValueError(msg) diff --git a/policyengine_api/country.py b/policyengine_api/country.py index caeb5ff07..8f6ab79de 100644 --- a/policyengine_api/country.py +++ b/policyengine_api/country.py @@ -9,7 +9,7 @@ from datetime import date, datetime from importlib.metadata import version as get_package_version from pathlib import Path -from typing import Any +from typing import Any, NamedTuple import dpath import pandas as pd @@ -87,21 +87,33 @@ def _serialize_axis_result( return entity_values +class _HouseholdCell(NamedTuple): + """One requested household value: an entity's variable in one period.""" + + entity_plural: str + entity_id: str + variable_name: str + period: str + + def _record_calculation_failure( household: dict, calculation_warnings: list[str], + cell: _HouseholdCell, *, - has_axes: bool, - population: Population, - entity_plural: str, - entity_id: str, - variable_name: str, - period: str, + axis_population: Population | None, error: Exception, ) -> None: - if has_axes: + """Record a household value that could not be calculated. + + ``axis_population`` is the cell's population when the household has axes: + the cell then holds one null per axis point and a warning is added. + Otherwise the cell is null and the failure is only logged. + """ + entity_plural, entity_id, variable_name, period = cell + if axis_population is not None: count_entities = len(household[entity_plural]) - axis_point_count = population.count // count_entities + axis_point_count = axis_population.count // count_entities household[entity_plural][entity_id][variable_name][period] = [ None ] * axis_point_count @@ -548,12 +560,8 @@ def calculate( _record_calculation_failure( household, calculation_warnings, - has_axes=has_axes, - population=population, - entity_plural=entity_plural, - entity_id=entity_id, - variable_name=variable_name, - period=period, + _HouseholdCell(entity_plural, entity_id, variable_name, period), + axis_population=population if has_axes else None, error=error, ) diff --git a/policyengine_api/data/congressional_districts.py b/policyengine_api/data/congressional_districts.py index 7f38f0e1d..76f0da35d 100644 --- a/policyengine_api/data/congressional_districts.py +++ b/policyengine_api/data/congressional_districts.py @@ -585,6 +585,11 @@ class CongressionalDistrictMetadataItem(BaseModel): ] +# Numbers whose last two digits are 10 through 20 always take "th" (11th, 12th). +_ALWAYS_TH_REMAINDER_MIN = 10 +_ALWAYS_TH_REMAINDER_MAX = 20 + + def _get_ordinal_suffix(number: int) -> str: """Get the ordinal suffix for a number (st, nd, rd, th). @@ -598,7 +603,7 @@ def _get_ordinal_suffix(number: int) -> str: 22 -> "nd" """ - if 10 <= number % 100 <= 20: + if _ALWAYS_TH_REMAINDER_MIN <= number % 100 <= _ALWAYS_TH_REMAINDER_MAX: # Special case for 11th, 12th, 13th, etc. suffix = "th" else: diff --git a/policyengine_api/data/places.py b/policyengine_api/data/places.py index 6cdc65b19..94c224dbb 100644 --- a/policyengine_api/data/places.py +++ b/policyengine_api/data/places.py @@ -6,6 +6,8 @@ from policyengine_api.data.congressional_districts import get_valid_state_codes +PLACE_FIPS_CODE_LENGTH = 5 + def parse_place_code(place_code: str) -> tuple[str, str]: """Parse a place code into its state abbreviation and FIPS components. @@ -45,6 +47,6 @@ def validate_place_code(place_code: str) -> None: msg = f"Invalid state in place code: '{state_abbrev}'" raise ValueError(msg) - if not place_fips.isdigit() or len(place_fips) != 5: + if not place_fips.isdigit() or len(place_fips) != PLACE_FIPS_CODE_LENGTH: msg = f"Invalid FIPS code in place: '{place_fips}'. Expected 5-digit FIPS code" raise ValueError(msg) diff --git a/policyengine_api/data/v2/catalog/catalog_selection.py b/policyengine_api/data/v2/catalog/catalog_selection.py index b4c027551..d9b50fc76 100644 --- a/policyengine_api/data/v2/catalog/catalog_selection.py +++ b/policyengine_api/data/v2/catalog/catalog_selection.py @@ -14,6 +14,7 @@ ) SUPPORTED_V2_COUNTRY_IDS = frozenset({"us", "uk"}) +MAXIMUM_POLICYENGINE_VERSION_LENGTH = 128 class MetadataCatalogUnavailableError(RuntimeError): @@ -47,8 +48,11 @@ def validate_policyengine_version(value: str) -> str: if not isinstance(value, str) or not value or value != value.strip(): msg = "policyengine_version must be a non-empty canonical version" raise InvalidPolicyEngineVersionError(msg) - if len(value) > 128: - msg = "policyengine_version must be at most 128 characters" + if len(value) > MAXIMUM_POLICYENGINE_VERSION_LENGTH: + msg = ( + "policyengine_version must be at most " + f"{MAXIMUM_POLICYENGINE_VERSION_LENGTH} characters" + ) raise InvalidPolicyEngineVersionError(msg) try: parsed = Version(value) diff --git a/policyengine_api/data/v2/catalog/extraction.py b/policyengine_api/data/v2/catalog/extraction.py index a21e8f7cc..e84e0aa00 100644 --- a/policyengine_api/data/v2/catalog/extraction.py +++ b/policyengine_api/data/v2/catalog/extraction.py @@ -10,7 +10,7 @@ from datetime import UTC, date, datetime, timedelta from enum import Enum from importlib import metadata as importlib_metadata -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, NamedTuple from urllib.parse import urlsplit from uuid import NAMESPACE_URL, UUID, uuid5 @@ -99,6 +99,10 @@ def _optional_text( return value +# The catalog's region filter_value column holds at most this many characters. +MAX_REGION_FILTER_VALUE_LENGTH = 255 + + def _serialize_region_filter_value(value: object, *, region_code: str) -> str: """Keep scalar text or encode flat membership values in the existing column. @@ -124,8 +128,11 @@ def _serialize_region_filter_value(value: object, *, region_code: str) -> str: else: msg = f"region {region_code!r} has an unsupported filter value" raise CatalogExtractionError(msg) - if len(serialized) > 255: - msg = f"region {region_code!r} filter value exceeds 255 characters" + if len(serialized) > MAX_REGION_FILTER_VALUE_LENGTH: + msg = ( + f"region {region_code!r} filter value exceeds " + f"{MAX_REGION_FILTER_VALUE_LENGTH} characters" + ) raise CatalogExtractionError(msg) return serialized @@ -142,32 +149,57 @@ def _type_name(value: object) -> str | None: return name -def normalize_json_value(value: object) -> JSONValue: - """Convert supported public values into strict JSON-compatible values.""" - if value is None or isinstance(value, (str, bool, int)): - return value - if isinstance(value, float): - if math.isnan(value): - msg = "NaN is not JSON-compatible" +def _normalize_json_float(value: float) -> JSONValue: + """Keep a finite float, write an infinity as a string and reject NaN.""" + if math.isnan(value): + msg = "NaN is not JSON-compatible" + raise CatalogExtractionError(msg) + if math.isinf(value): + return "Infinity" if value > 0 else "-Infinity" + return value + + +def _normalize_json_enum(value: Enum) -> JSONValue: + """Normalize an enum member's value.""" + return normalize_json_value(value.value) + + +def _normalize_json_date(value: date) -> JSONValue: + """Write a date or datetime in ISO 8601 form.""" + return value.isoformat() + + +def _normalize_json_object(value: Mapping[object, object]) -> JSONValue: + """Normalize a mapping's values, requiring string keys.""" + normalized: dict[str, JSONValue] = {} + for key, nested in value.items(): + if not isinstance(key, str): + msg = "JSON object keys must be strings" raise CatalogExtractionError(msg) - if math.isinf(value): - return "Infinity" if value > 0 else "-Infinity" - return value - if isinstance(value, Enum): - return normalize_json_value(value.value) - if isinstance(value, (date, datetime)): - return value.isoformat() - if isinstance(value, Mapping): - normalized: dict[str, JSONValue] = {} - for key, nested in value.items(): - if not isinstance(key, str): - msg = "JSON object keys must be strings" - raise CatalogExtractionError(msg) - normalized[key] = normalize_json_value(nested) - return normalized - if isinstance(value, (list, tuple)): - return [normalize_json_value(item) for item in value] + normalized[key] = normalize_json_value(nested) + return normalized + + +def _normalize_json_array(value: list[object] | tuple[object, ...]) -> JSONValue: + """Normalize each item of a list or tuple into a JSON array.""" + return [normalize_json_value(item) for item in value] + +# Checked in order, so an enum or date that is also a container or number is +# normalized as the enum or date it is. +JSON_VALUE_NORMALIZERS: tuple[ + tuple[type | tuple[type, ...], Callable[[Any], JSONValue]], ... +] = ( + (float, _normalize_json_float), + (Enum, _normalize_json_enum), + ((date, datetime), _normalize_json_date), + (Mapping, _normalize_json_object), + ((list, tuple), _normalize_json_array), +) + + +def _normalize_json_scalar(value: object) -> JSONValue: + """Unwrap a NumPy-style scalar through its ``item()``, or reject the value.""" item = getattr(value, "item", None) if callable(item): scalar = item() @@ -177,6 +209,16 @@ def normalize_json_value(value: object) -> JSONValue: raise CatalogExtractionError(msg) +def normalize_json_value(value: object) -> JSONValue: + """Convert supported public values into strict JSON-compatible values.""" + if value is None or isinstance(value, (str, bool, int)): + return value + for value_types, normalize in JSON_VALUE_NORMALIZERS: + if isinstance(value, value_types): + return normalize(value) + return _normalize_json_scalar(value) + + def _aware_datetime(value: object, *, field_name: str) -> datetime: if isinstance(value, date) and not isinstance(value, datetime): value = datetime(value.year, value.month, value.day, tzinfo=UTC) @@ -450,14 +492,20 @@ def _public_named_records( return tuple(normalized) -def _normalize_country( - *, - country_id: str, - source: object, - policyengine_version: str, - expected_country_package_version: str, -) -> CountryCatalog: - source_model = getattr(source, "model", None) +class _RegionDatasetSelection(NamedTuple): + """The logical input datasets a country's regions select. + + Names every selected dataset, each region code's dataset, and how many + regions of each type fall back to the reviewed default dataset. + """ + + dataset_names: set[str] + region_dataset_names: dict[str, str] + fallback_counts: Counter[str] + + +def _verified_model_name(country_id: str, source_model: object) -> str: + """Return the public model's name once it matches the country's model.""" expected_model_name = f"policyengine-{country_id}" model_name = _required_text( getattr(source_model, "id", None), @@ -467,9 +515,22 @@ def _normalize_country( if model_name != expected_model_name: msg = f"{country_id} public model identity does not match {expected_model_name}" raise CatalogExtractionError(msg) - model_id = _identifier("model", model_name) - model_version_id = _identifier("model-version", model_id, policyengine_version) + return model_name + + +def _verified_release_manifest( + country_id: str, + source: object, + *, + policyengine_version: str, + expected_country_package_version: str, +) -> object: + """Return the public model's release manifest once its package and manifest match. + The model package must be the country's at the bundle's version, and the + manifest must name the country and the installed PolicyEngine.py version. + """ + expected_model_name = f"policyengine-{country_id}" model_package = getattr(source, "model_package", None) if getattr(model_package, "name", None) != expected_model_name: msg = ( @@ -490,15 +551,15 @@ def _normalize_country( "PolicyEngine.py version" ) raise CatalogExtractionError(msg) + return release_manifest - variables = tuple( - _normalize_variable(variable, model_version_id=model_version_id) - for variable in _public_named_records( - source, - attribute="variables_by_name", - entity_name="variable", - ) - ) + +def _normalize_parameter_nodes( + source: object, + *, + model_version_id: UUID, +) -> tuple[ParameterNodeRecord, ...]: + """Normalize the public model's named parameter nodes, in name order.""" normalized_nodes: list[ParameterNodeRecord] = [] for node in _public_named_records( source, @@ -531,22 +592,31 @@ def _normalize_country( ), ) ) - parameter_nodes = tuple(normalized_nodes) - parameters = tuple( - _normalize_parameter(parameter, model_version_id=model_version_id) - for parameter in _public_named_records( - source, - attribute="parameters_by_name", - entity_name="parameter", - ) - ) + return tuple(normalized_nodes) + +def _verified_default_dataset( + country_id: str, + release_manifest: object, +) -> tuple[str, object]: + """Return the reviewed default dataset and the manifest's URI for it. + + The manifest's default dataset must be the reviewed one. + """ reviewed_default = REVIEWED_DEFAULT_DATASETS[country_id] manifest_default = getattr(release_manifest, "default_dataset", None) if manifest_default != reviewed_default: msg = f"{country_id} default dataset does not match the reviewed selection" raise CatalogExtractionError(msg) - default_dataset_uri = getattr(release_manifest, "default_dataset_uri", None) + return reviewed_default, getattr(release_manifest, "default_dataset_uri", None) + + +def _verified_source_regions(country_id: str, source: object) -> tuple[Any, ...]: + """Return the country's public regions once their codes are unique and complete. + + The registry must belong to the country, and its codes must be unique and + include the national region. + """ registry = getattr(source, "region_registry", None) if registry is None or getattr(registry, "country_id", None) != country_id: msg = f"{country_id} public region registry is absent" @@ -563,7 +633,21 @@ def _normalize_country( if country_id not in region_codes: msg = f"{country_id} national region is absent" raise CatalogExtractionError(msg) + return source_regions + + +def _select_region_datasets( + country_id: str, + source_regions: tuple[Any, ...], + *, + reviewed_default: str, + default_dataset_uri: object, +) -> _RegionDatasetSelection: + """Choose each region's logical input dataset. + A US subnational region with a dataset path uses that dataset; every other + region uses the reviewed default, and the national region may only name it. + """ selected_dataset_names: set[str] = {reviewed_default} region_dataset_names: dict[str, str] = {} fallback_counts: Counter[str] = Counter() @@ -596,6 +680,133 @@ def _normalize_country( msg = f"{country_id} national region does not use the reviewed dataset" raise CatalogExtractionError(msg) region_dataset_names[code] = dataset_name + return _RegionDatasetSelection( + dataset_names=selected_dataset_names, + region_dataset_names=region_dataset_names, + fallback_counts=fallback_counts, + ) + + +def _region_filter(strategy: object, code: str) -> tuple[str, str, str]: + """Return a filtered region's strategy type, filter field and filter value.""" + strategy_type = _required_text( + getattr(strategy, "strategy_type", None), + field_name=f"region {code} filter strategy", + maximum=64, + ) + filter_field = _required_text( + getattr(strategy, "variable_name", None), + field_name=f"region {code} filter field", + maximum=128, + ) + filter_value = _serialize_region_filter_value( + getattr(strategy, "variable_value", None), region_code=code + ) + additional_filters = getattr(strategy, "additional_filters", {}) + if additional_filters: + msg = f"region {code!r} has unsupported additional filters" + raise CatalogExtractionError(msg) + return strategy_type, filter_field, filter_value + + +def _normalize_region( + source_region: Any, # noqa: ANN401 # a duck-typed PolicyEngine.py region + *, + model_version_id: UUID, + region_dataset_ids: Mapping[str, UUID], +) -> RegionRecord: + """Normalize one public region, with its filter and default dataset.""" + code = str(source_region.code) + strategy = getattr(source_region, "scoping_strategy", None) + requires_filter = bool(getattr(source_region, "requires_filter", False)) + if requires_filter: + strategy_type, filter_field, filter_value = _region_filter(strategy, code) + else: + strategy_type = None + filter_field = None + filter_value = None + + return RegionRecord( + id=_identifier("region", model_version_id, code), + model_version_id=model_version_id, + default_dataset_id=region_dataset_ids[code], + code=code, + label=_required_text( + getattr(source_region, "label", None), + field_name=f"region {code} label", + maximum=255, + ), + region_type=str(source_region.region_type), + requires_filter=requires_filter, + filter_field=filter_field, + filter_value=filter_value, + filter_strategy=strategy_type, + parent_code=_optional_text( + getattr(source_region, "parent_code", None), + field_name=f"region {code} parent_code", + maximum=255, + ), + state_code=_optional_text( + getattr(source_region, "state_code", None), + field_name=f"region {code} state_code", + maximum=16, + ), + state_name=_optional_text( + getattr(source_region, "state_name", None), + field_name=f"region {code} state_name", + maximum=128, + ), + ) + + +def _normalize_country( + *, + country_id: str, + source: object, + policyengine_version: str, + expected_country_package_version: str, +) -> CountryCatalog: + source_model = getattr(source, "model", None) + model_name = _verified_model_name(country_id, source_model) + model_id = _identifier("model", model_name) + model_version_id = _identifier("model-version", model_id, policyengine_version) + release_manifest = _verified_release_manifest( + country_id, + source, + policyengine_version=policyengine_version, + expected_country_package_version=expected_country_package_version, + ) + + variables = tuple( + _normalize_variable(variable, model_version_id=model_version_id) + for variable in _public_named_records( + source, + attribute="variables_by_name", + entity_name="variable", + ) + ) + parameter_nodes = _normalize_parameter_nodes( + source, model_version_id=model_version_id + ) + parameters = tuple( + _normalize_parameter(parameter, model_version_id=model_version_id) + for parameter in _public_named_records( + source, + attribute="parameters_by_name", + entity_name="parameter", + ) + ) + + reviewed_default, default_dataset_uri = _verified_default_dataset( + country_id, release_manifest + ) + source_regions = _verified_source_regions(country_id, source) + dataset_selection = _select_region_datasets( + country_id, + source_regions, + reviewed_default=reviewed_default, + default_dataset_uri=default_dataset_uri, + ) datasets = tuple( DatasetRecord( @@ -605,72 +816,21 @@ def _normalize_country( description=f"PolicyEngine.py logical input dataset {name}", year=_dataset_year(name), ) - for name in sorted(selected_dataset_names) + for name in sorted(dataset_selection.dataset_names) ) dataset_ids = {dataset.name: dataset.id for dataset in datasets} - - regions: list[RegionRecord] = [] - for source_region in source_regions: - code = str(source_region.code) - strategy = getattr(source_region, "scoping_strategy", None) - requires_filter = bool(getattr(source_region, "requires_filter", False)) - if requires_filter: - strategy_type = _required_text( - getattr(strategy, "strategy_type", None), - field_name=f"region {code} filter strategy", - maximum=64, - ) - filter_field = _required_text( - getattr(strategy, "variable_name", None), - field_name=f"region {code} filter field", - maximum=128, - ) - filter_value_source = getattr(strategy, "variable_value", None) - filter_value = _serialize_region_filter_value( - filter_value_source, region_code=code - ) - additional_filters = getattr(strategy, "additional_filters", {}) - if additional_filters: - msg = f"region {code!r} has unsupported additional filters" - raise CatalogExtractionError(msg) - else: - strategy_type = None - filter_field = None - filter_value = None - - regions.append( - RegionRecord( - id=_identifier("region", model_version_id, code), - model_version_id=model_version_id, - default_dataset_id=dataset_ids[region_dataset_names[code]], - code=code, - label=_required_text( - getattr(source_region, "label", None), - field_name=f"region {code} label", - maximum=255, - ), - region_type=str(source_region.region_type), - requires_filter=requires_filter, - filter_field=filter_field, - filter_value=filter_value, - filter_strategy=strategy_type, - parent_code=_optional_text( - getattr(source_region, "parent_code", None), - field_name=f"region {code} parent_code", - maximum=255, - ), - state_code=_optional_text( - getattr(source_region, "state_code", None), - field_name=f"region {code} state_code", - maximum=16, - ), - state_name=_optional_text( - getattr(source_region, "state_name", None), - field_name=f"region {code} state_name", - maximum=128, - ), - ) + region_dataset_ids = { + code: dataset_ids[name] + for code, name in dataset_selection.region_dataset_names.items() + } + regions = [ + _normalize_region( + source_region, + model_version_id=model_version_id, + region_dataset_ids=region_dataset_ids, ) + for source_region in source_regions + ] return CountryCatalog( country_id=country_id, @@ -701,7 +861,7 @@ def _normalize_country( regions=tuple(sorted(regions, key=lambda record: record.code)), fallback_summaries=tuple( FallbackSummary(region_type=region_type, count=count) - for region_type, count in sorted(fallback_counts.items()) + for region_type, count in sorted(dataset_selection.fallback_counts.items()) ), ) diff --git a/policyengine_api/data/v2/catalog/publication_staging.py b/policyengine_api/data/v2/catalog/publication_staging.py index 7f991b25a..76820b27b 100644 --- a/policyengine_api/data/v2/catalog/publication_staging.py +++ b/policyengine_api/data/v2/catalog/publication_staging.py @@ -21,6 +21,7 @@ if TYPE_CHECKING: from collections.abc import Callable, Iterable, Iterator, Sequence + from uuid import UUID COPY_BATCH_SIZE = 10_000 @@ -150,121 +151,135 @@ def _optional_json(value: object) -> Jsonb | None: return None if value is None else Jsonb(value) +def _model_rows(country: CountryCatalog) -> Iterator[tuple[object, ...]]: + """Yield the country's one model and model-version row.""" + yield ( + country.country_id, + country.model.id, + country.model.name, + country.model.description, + country.model_version.id, + country.model_version.version, + country.model_version.description, + country.model_version.current_law_id, + Jsonb(country.model_version.metadata_time_periods), + ) + + +def _variable_rows(country: CountryCatalog) -> Iterator[tuple[object, ...]]: + """Yield one row per variable, in bounded batches.""" + for batch in iter_batches(country.variables, batch_size=COPY_BATCH_SIZE): + for record in batch: + yield ( + country.country_id, + record.id, + record.name, + record.label, + record.entity, + record.description, + record.data_type, + _optional_json(record.possible_values), + Jsonb(record.default_value), + _optional_json(record.adds), + _optional_json(record.subtracts), + ) + + +def _parameter_node_rows(country: CountryCatalog) -> Iterator[tuple[object, ...]]: + """Yield one row per parameter node, in bounded batches.""" + for batch in iter_batches( + country.parameter_nodes, + batch_size=COPY_BATCH_SIZE, + ): + for record in batch: + yield ( + country.country_id, + record.id, + record.name, + record.label, + record.description, + ) + + +def _parameter_rows(country: CountryCatalog) -> Iterator[tuple[object, ...]]: + """Yield one row per parameter, in bounded batches.""" + for batch in iter_batches(country.parameters, batch_size=COPY_BATCH_SIZE): + for record in batch: + yield ( + country.country_id, + record.id, + record.name, + record.label, + record.description, + record.data_type, + record.unit, + ) + + +def _parameter_value_rows(country: CountryCatalog) -> Iterator[tuple[object, ...]]: + """Yield one row per parameter value, keyed by its parameter's name.""" + parameter_names = {parameter.id: parameter.name for parameter in country.parameters} + for batch in country.parameter_value_batches(batch_size=COPY_BATCH_SIZE): + for record in batch: + yield ( + country.country_id, + parameter_names[record.parameter_id], + record.id, + Jsonb(record.value_json), + record.start_date, + record.end_date, + ) + + +def _dataset_rows(country: CountryCatalog) -> Iterator[tuple[object, ...]]: + """Yield one row per logical input dataset, in bounded batches.""" + for batch in iter_batches(country.datasets, batch_size=COPY_BATCH_SIZE): + for record in batch: + yield ( + country.country_id, + record.id, + record.name, + record.description, + record.year, + ) + + +def _region_rows( + country: CountryCatalog, + dataset_names: dict[UUID, str], +) -> Iterator[tuple[object, ...]]: + """Yield one row per region, naming its default dataset.""" + for batch in iter_batches(country.regions, batch_size=COPY_BATCH_SIZE): + for record in batch: + yield ( + country.country_id, + record.id, + record.code, + record.label, + record.region_type, + record.requires_filter, + record.filter_field, + record.filter_value, + record.filter_strategy, + record.parent_code, + record.state_code, + record.state_name, + dataset_names[record.default_dataset_id], + ) + + def _catalog_rows( country: CountryCatalog, ) -> dict[Table, Iterator[tuple[object, ...]]]: dataset_names = {dataset.id: dataset.name for dataset in country.datasets} - - def model_rows() -> Iterator[tuple[object, ...]]: - yield ( - country.country_id, - country.model.id, - country.model.name, - country.model.description, - country.model_version.id, - country.model_version.version, - country.model_version.description, - country.model_version.current_law_id, - Jsonb(country.model_version.metadata_time_periods), - ) - - def variable_rows() -> Iterator[tuple[object, ...]]: - for batch in iter_batches(country.variables, batch_size=COPY_BATCH_SIZE): - for record in batch: - yield ( - country.country_id, - record.id, - record.name, - record.label, - record.entity, - record.description, - record.data_type, - _optional_json(record.possible_values), - Jsonb(record.default_value), - _optional_json(record.adds), - _optional_json(record.subtracts), - ) - - def parameter_node_rows() -> Iterator[tuple[object, ...]]: - for batch in iter_batches( - country.parameter_nodes, - batch_size=COPY_BATCH_SIZE, - ): - for record in batch: - yield ( - country.country_id, - record.id, - record.name, - record.label, - record.description, - ) - - def parameter_rows() -> Iterator[tuple[object, ...]]: - for batch in iter_batches(country.parameters, batch_size=COPY_BATCH_SIZE): - for record in batch: - yield ( - country.country_id, - record.id, - record.name, - record.label, - record.description, - record.data_type, - record.unit, - ) - - def parameter_value_rows() -> Iterator[tuple[object, ...]]: - parameter_names = { - parameter.id: parameter.name for parameter in country.parameters - } - for batch in country.parameter_value_batches(batch_size=COPY_BATCH_SIZE): - for record in batch: - yield ( - country.country_id, - parameter_names[record.parameter_id], - record.id, - Jsonb(record.value_json), - record.start_date, - record.end_date, - ) - - def dataset_rows() -> Iterator[tuple[object, ...]]: - for batch in iter_batches(country.datasets, batch_size=COPY_BATCH_SIZE): - for record in batch: - yield ( - country.country_id, - record.id, - record.name, - record.description, - record.year, - ) - - def region_rows() -> Iterator[tuple[object, ...]]: - for batch in iter_batches(country.regions, batch_size=COPY_BATCH_SIZE): - for record in batch: - yield ( - country.country_id, - record.id, - record.code, - record.label, - record.region_type, - record.requires_filter, - record.filter_field, - record.filter_value, - record.filter_strategy, - record.parent_code, - record.state_code, - record.state_name, - dataset_names[record.default_dataset_id], - ) - return { - STAGE_CATALOG_MODELS: model_rows(), - STAGE_CATALOG_VARIABLES: variable_rows(), - STAGE_CATALOG_PARAMETER_NODES: parameter_node_rows(), - STAGE_CATALOG_PARAMETERS: parameter_rows(), - STAGE_CATALOG_PARAMETER_VALUES: parameter_value_rows(), - STAGE_CATALOG_DATASETS: dataset_rows(), - STAGE_CATALOG_REGIONS: region_rows(), + STAGE_CATALOG_MODELS: _model_rows(country), + STAGE_CATALOG_VARIABLES: _variable_rows(country), + STAGE_CATALOG_PARAMETER_NODES: _parameter_node_rows(country), + STAGE_CATALOG_PARAMETERS: _parameter_rows(country), + STAGE_CATALOG_PARAMETER_VALUES: _parameter_value_rows(country), + STAGE_CATALOG_DATASETS: _dataset_rows(country), + STAGE_CATALOG_REGIONS: _region_rows(country, dataset_names), } diff --git a/policyengine_api/data/v2/report_runs.py b/policyengine_api/data/v2/report_runs.py index 9f11d0b9a..14685f150 100644 --- a/policyengine_api/data/v2/report_runs.py +++ b/policyengine_api/data/v2/report_runs.py @@ -2,6 +2,7 @@ from __future__ import annotations +from dataclasses import dataclass from typing import TYPE_CHECKING from sqlalchemy.exc import IntegrityError @@ -38,6 +39,14 @@ class ReportRunStateError(ValueError): """Raised when a worker transition is invalid for the durable run.""" +@dataclass(frozen=True) +class ReportRunPackageVersions: + """The country package and PolicyEngine.py versions one report run uses.""" + + country_package_version: str + policyengine_version: str + + def _locked_report(session: Session, report_id: UUID) -> Report: # SQLModel's select remains the query surface. Row locking is the bounded # SQLAlchemy capability needed to serialize first-run/type updates. @@ -77,8 +86,7 @@ def create_report_run( session: Session, *, report_id: UUID, - country_package_version: str, - policyengine_version: str, + versions: ReportRunPackageVersions, trigger: ReportRunTrigger, idempotency_key: UUID | None = None, ) -> ReportRun: @@ -87,14 +95,14 @@ def create_report_run( if trigger is ReportRunTrigger.MANUAL and idempotency_key is None: msg = "manual report reruns require an idempotency key" raise ValueError(msg) - if not country_package_version or not policyengine_version: + if not versions.country_package_version or not versions.policyengine_version: msg = "report run package versions must be non-empty" raise ValueError(msg) run = ReportRun( report_id=report_id, - country_package_version=country_package_version, - policyengine_version=policyengine_version, + country_package_version=versions.country_package_version, + policyengine_version=versions.policyengine_version, trigger=trigger, idempotency_key=idempotency_key, ) diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index a23018978..a08c922d1 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -16,7 +16,6 @@ if TYPE_CHECKING: from collections.abc import Callable - from datetime import datetime from uuid import UUID from policyengine_api.json_types import JSONObject @@ -46,6 +45,7 @@ MetadataParameterSummary, MetadataRegion, MetadataVariable, + ParameterValueFilter, ) from policyengine_api.services.v2.policies.types import ( NativePolicyCreation, @@ -62,12 +62,14 @@ ) from policyengine_api.services.v2.user_households.types import ( UserHouseholdCreateFields, + UserHouseholdFilter, UserHouseholdPage, UserHouseholdRead, UserHouseholdUpdateInput, ) from policyengine_api.services.v2.user_policies.types import ( UserPolicyCreateFields, + UserPolicyFilter, UserPolicyPage, UserPolicyRead, UserPolicyUpdateInput, @@ -225,16 +227,14 @@ def list_parameter_values( country_id: str, policyengine_version: str | None = None, *, - parameter_id: UUID | None = None, - current: bool = False, + value_filter: ParameterValueFilter | None = None, offset: int = 0, limit: int = 100, - now: datetime | None = None, ) -> MetadataPageResult[MetadataCanonicalParameterValue]: """Return canonical values, optionally for one parameter. - With ``current``, only values in effect on the UTC day of ``now`` (default: - the present) are returned. + With the filter's ``current``, only values in effect on the UTC day of its + ``now`` (default: the present) are returned. """ ... @@ -410,8 +410,7 @@ def list_user_policies( *, principal: AccountPrincipal, country_id: str, - policy_id: UUID | None = None, - legacy_policy_id: int | None = None, + policy_filter: UserPolicyFilter | None = None, offset: int = 0, limit: int = 100, ) -> UserPolicyPage: @@ -470,8 +469,7 @@ def list_user_households( *, principal: AccountPrincipal, country_id: str, - household_id: UUID | None = None, - legacy_household_id: int | None = None, + household_filter: UserHouseholdFilter | None = None, offset: int = 0, limit: int = 100, ) -> UserHouseholdPage: diff --git a/policyengine_api/fastapi_routes/v2/errors.py b/policyengine_api/fastapi_routes/v2/errors.py index e99ace645..0c1f38525 100644 --- a/policyengine_api/fastapi_routes/v2/errors.py +++ b/policyengine_api/fastapi_routes/v2/errors.py @@ -1,7 +1,7 @@ """Shared error response contract for all native API v2 routes.""" -from collections.abc import Mapping -from typing import Annotated, Literal +from collections.abc import Callable, Mapping, Sequence +from typing import Annotated, Literal, NamedTuple, TypeVar from pydantic import BaseModel, ConfigDict, StringConstraints from starlette.responses import JSONResponse @@ -59,3 +59,38 @@ def v2_error_response( content=error.model_dump(mode="json"), headers=dict(headers) if headers is not None else None, ) + + +OperationT = TypeVar("OperationT") + + +class V2ErrorMapping(NamedTuple): + """The API v2 error response for one family of service errors. + + Without a ``message``, the response carries the error's own message. + """ + + error_types: tuple[type[Exception], ...] + status_code: int + message: str | None = None + + +def run_v2_operation( + operation: Callable[[], OperationT], + error_mappings: Sequence[V2ErrorMapping], + *, + fallback_message: str, +) -> OperationT | JSONResponse: + """Run ``operation``, answering a failure with the first matching v2 error. + + A failure that no mapping matches becomes a 500 with ``fallback_message``, + so no unexpected detail reaches the response. + """ + try: + return operation() + except Exception as error: # noqa: BLE001 # return a secret-safe typed error + for mapping in error_mappings: + if isinstance(error, mapping.error_types): + message = str(error) if mapping.message is None else mapping.message + return v2_error_response(mapping.status_code, message) + return v2_error_response(500, fallback_message) diff --git a/policyengine_api/fastapi_routes/v2/households/routes.py b/policyengine_api/fastapi_routes/v2/households/routes.py index 630e5f708..54e24dbf5 100644 --- a/policyengine_api/fastapi_routes/v2/households/routes.py +++ b/policyengine_api/fastapi_routes/v2/households/routes.py @@ -15,7 +15,9 @@ ) from policyengine_api.fastapi_routes.query_parameters import query_dependency from policyengine_api.fastapi_routes.v2.errors import ( + V2ErrorMapping, V2RequestTooLargeError, + run_v2_operation, v2_error_response, ) from policyengine_api.fastapi_routes.v2.households.request_models import ( @@ -89,24 +91,34 @@ def _service_factory( OperationT = TypeVar("OperationT") +# Checked in order, so the collision is matched before its integrity base class. +HOUSEHOLD_ERROR_MAPPINGS = ( + V2ErrorMapping((HouseholdValidationError,), 400), + V2ErrorMapping((HouseholdNotFoundError,), 404), + V2ErrorMapping( + (HouseholdContentHashCollisionError,), + 409, + "Household content hash conflicts with storage", + ), + V2ErrorMapping( + (HouseholdCreationIntegrityError,), 500, "Stored household integrity failed" + ), + V2ErrorMapping( + (V2ConfigurationError, SQLAlchemyError), + 503, + "V2 household persistence is unavailable", + ), +) + def _household_operation( operation: Callable[[], OperationT], ) -> OperationT | JSONResponse: - try: - return operation() - except HouseholdValidationError as error: - return v2_error_response(400, str(error)) - except HouseholdNotFoundError as error: - return v2_error_response(404, str(error)) - except HouseholdContentHashCollisionError: - return v2_error_response(409, "Household content hash conflicts with storage") - except HouseholdCreationIntegrityError: - return v2_error_response(500, "Stored household integrity failed") - except (V2ConfigurationError, SQLAlchemyError): - return v2_error_response(503, "V2 household persistence is unavailable") - except Exception: # noqa: BLE001 # return a secret-safe typed error - return v2_error_response(500, "V2 household operation failed") + return run_v2_operation( + operation, + HOUSEHOLD_ERROR_MAPPINGS, + fallback_message="V2 household operation failed", + ) def build_v2_household_router( diff --git a/policyengine_api/fastapi_routes/v2/me/routes.py b/policyengine_api/fastapi_routes/v2/me/routes.py index 4b756209a..0919c938c 100644 --- a/policyengine_api/fastapi_routes/v2/me/routes.py +++ b/policyengine_api/fastapi_routes/v2/me/routes.py @@ -62,6 +62,38 @@ def _service_factory( return _default_v2_account_service_factory +def _signed_in_user_dependency( + require_user: Callable[..., AccountPrincipal], +) -> Callable[..., UserPrincipal]: + """Build a dependency that admits only a signed-in principal of ``require_user``.""" + + def require_signed_in_user( + principal: Annotated[AccountPrincipal, Depends(require_user)], + ) -> UserPrincipal: + return require_signed_in_principal(principal) + + return require_signed_in_user + + +def _upgrade_account( + service_factory: Callable[[], V2AccountResourceService], + principal: UserPrincipal, + unauthenticated_user: str, + country_id: str, +) -> MeUpgradeResponse | JSONResponse: + """Move a browser's unauthenticated user into the signed-in account.""" + + def upgrade() -> MeUpgradeResponse: + result = service_factory().upgrade_account( + principal=principal, + unauthenticated_user_id=UUID(unauthenticated_user), + country_id=country_id, + ) + return MeUpgradeResponse(result=MeUpgradeResult.from_upgrade(result)) + + return user_resource_operation(upgrade) + + def build_v2_me_router(dependencies: NativeRouteDependencies) -> APIRouter: """Build account routes without opening a connection.""" require_user = dependencies.require_user @@ -71,11 +103,7 @@ def build_v2_me_router(dependencies: NativeRouteDependencies) -> APIRouter: dependencies=[Depends(require_user)], ) service_factory = _service_factory(dependencies) - - def require_signed_in_user( - principal: Annotated[AccountPrincipal, Depends(require_user)], - ) -> UserPrincipal: - return require_signed_in_principal(principal) + require_signed_in_user = _signed_in_user_dependency(require_user) @router.get( "/me", @@ -216,14 +244,8 @@ def upgrade_me( ], _size: Annotated[None, Depends(enforce_user_write_request_size)], ) -> MeUpgradeResponse | JSONResponse: - def upgrade() -> MeUpgradeResponse: - result = service_factory().upgrade_account( - principal=principal, - unauthenticated_user_id=UUID(unauthenticated_user), - country_id=query.country_id, - ) - return MeUpgradeResponse(result=MeUpgradeResult.from_upgrade(result)) - - return user_resource_operation(upgrade) + return _upgrade_account( + service_factory, principal, unauthenticated_user, query.country_id + ) return router diff --git a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py index 2ed036387..0f4714e5c 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py @@ -2,10 +2,11 @@ from __future__ import annotations +from dataclasses import dataclass from typing import TYPE_CHECKING, Annotated from uuid import UUID -from fastapi import APIRouter, Query +from fastapi import APIRouter, Depends, Query from starlette.responses import JSONResponse from policyengine_api.fastapi_routes.v2.metadata.common import ( @@ -19,6 +20,7 @@ MetadataParameterValueDetailResponse, MetadataParameterValuePageResponse, ) +from policyengine_api.services.v2.metadata.types import ParameterValueFilter if TYPE_CHECKING: from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies @@ -27,6 +29,18 @@ Limit = Annotated[int, Query(ge=1, le=500)] +@dataclass(frozen=True) +class ParameterValueListingQuery: + """The query parameters of the canonical parameter-value listing route.""" + + country_id: str + policyengine_version: str | None = None + parameter_id: UUID | None = None + current: bool = False + offset: Offset = 0 + limit: Limit = 100 + + def build_v2_metadata_parameter_router( dependencies: NativeRouteDependencies, ) -> APIRouter: @@ -111,23 +125,20 @@ def get_parameter( summary="List canonical values from a selected PolicyEngine.py catalog", ) def list_parameter_values( - country_id: str, - policyengine_version: str | None = None, - parameter_id: UUID | None = None, - current: bool = False, # noqa: FBT001, FBT002 # FastAPI query parameter - offset: Offset = 0, - limit: Limit = 100, + query: Annotated[ParameterValueListingQuery, Depends()], ) -> MetadataParameterValuePageResponse | JSONResponse: return read_resource( dependencies, MetadataParameterValuePageResponse, lambda reader: reader.list_parameter_values( - country_id, - policyengine_version, - parameter_id=parameter_id, - current=current, - offset=offset, - limit=limit, + query.country_id, + query.policyengine_version, + value_filter=ParameterValueFilter( + parameter_id=query.parameter_id, + current=query.current, + ), + offset=query.offset, + limit=query.limit, ), ) diff --git a/policyengine_api/fastapi_routes/v2/policies/routes.py b/policyengine_api/fastapi_routes/v2/policies/routes.py index f9b885d6e..88bcc0444 100644 --- a/policyengine_api/fastapi_routes/v2/policies/routes.py +++ b/policyengine_api/fastapi_routes/v2/policies/routes.py @@ -19,7 +19,9 @@ ) from policyengine_api.fastapi_routes.query_parameters import query_dependency from policyengine_api.fastapi_routes.v2.errors import ( + V2ErrorMapping, V2RequestTooLargeError, + run_v2_operation, v2_error_response, ) from policyengine_api.fastapi_routes.v2.policies.request_models import ( @@ -94,24 +96,34 @@ def _service_factory( OperationT = TypeVar("OperationT") +# Checked in order, so the collision is matched before its integrity base class. +POLICY_ERROR_MAPPINGS = ( + V2ErrorMapping((PolicyCatalogValidationError,), 400), + V2ErrorMapping((MetadataCatalogVersionNotFoundError, PolicyNotFoundError), 404), + V2ErrorMapping( + (PolicyContentHashCollisionError,), + 409, + "Policy content hash conflicts with storage", + ), + V2ErrorMapping( + (PolicyCreationIntegrityError,), 500, "Stored policy integrity failed" + ), + V2ErrorMapping( + (V2ConfigurationError, MetadataCatalogUnavailableError, SQLAlchemyError), + 503, + "V2 policy persistence is unavailable", + ), +) + def _policy_operation( operation: Callable[[], OperationT], ) -> OperationT | JSONResponse: - try: - return operation() - except PolicyCatalogValidationError as error: - return v2_error_response(400, str(error)) - except (MetadataCatalogVersionNotFoundError, PolicyNotFoundError) as error: - return v2_error_response(404, str(error)) - except PolicyContentHashCollisionError: - return v2_error_response(409, "Policy content hash conflicts with storage") - except PolicyCreationIntegrityError: - return v2_error_response(500, "Stored policy integrity failed") - except (V2ConfigurationError, MetadataCatalogUnavailableError, SQLAlchemyError): - return v2_error_response(503, "V2 policy persistence is unavailable") - except Exception: # noqa: BLE001 # route must return a secret-safe typed error - return v2_error_response(500, "V2 policy operation failed") + return run_v2_operation( + operation, + POLICY_ERROR_MAPPINGS, + fallback_message="V2 policy operation failed", + ) def build_v2_policy_router( diff --git a/policyengine_api/fastapi_routes/v2/user_households/routes.py b/policyengine_api/fastapi_routes/v2/user_households/routes.py index 04b2c63f7..c642a79d8 100644 --- a/policyengine_api/fastapi_routes/v2/user_households/routes.py +++ b/policyengine_api/fastapi_routes/v2/user_households/routes.py @@ -38,6 +38,7 @@ UserHouseholdUpdateQuery, ) from policyengine_api.services.v2.identity.types import AccountPrincipal +from policyengine_api.services.v2.user_households.types import UserHouseholdFilter UserHouseholdCreateQueryDependency = Annotated[ UserHouseholdCreateQuery, Depends(query_dependency(UserHouseholdCreateQuery)) @@ -65,6 +66,28 @@ def _service_factory( return _default_v2_user_household_service_factory +def _create_user_household( + service_factory: Callable[[], V2UserHouseholdResourceService], + body: UserHouseholdCreateRequest, + query: UserHouseholdCreateQuery, + principal: AccountPrincipal, +) -> UserHouseholdDetailResponse | JSONResponse: + """Create an association once the body and query agree on its country.""" + if body.country_id != query.country_id: + return v2_error_response(400, "Body country_id must match query country_id") + + def create() -> UserHouseholdDetailResponse: + item = service_factory().create_user_household( + principal=principal, + association_input=body, + ) + return UserHouseholdDetailResponse( + result=UserHouseholdDetailResult(item=UserHouseholdItem.from_read(item)) + ) + + return user_resource_operation(create) + + def build_v2_user_household_router( dependencies: NativeRouteDependencies, ) -> APIRouter: @@ -101,19 +124,7 @@ def create_user_household( query: UserHouseholdCreateQueryDependency, principal: Annotated[AccountPrincipal, Depends(require_user)], ) -> UserHouseholdDetailResponse | JSONResponse: - if body.country_id != query.country_id: - return v2_error_response(400, "Body country_id must match query country_id") - - def create() -> UserHouseholdDetailResponse: - item = service_factory().create_user_household( - principal=principal, - association_input=body, - ) - return UserHouseholdDetailResponse( - result=UserHouseholdDetailResult(item=UserHouseholdItem.from_read(item)) - ) - - return user_resource_operation(create) + return _create_user_household(service_factory, body, query, principal) @router.get( "/user-households/{association_id}", @@ -150,8 +161,10 @@ def read() -> UserHouseholdPageResponse: page = service_factory().list_user_households( principal=principal, country_id=query.country_id, - household_id=query.household_id, - legacy_household_id=query.legacy_household_id, + household_filter=UserHouseholdFilter( + household_id=query.household_id, + legacy_household_id=query.legacy_household_id, + ), offset=query.offset, limit=query.limit, ) diff --git a/policyengine_api/fastapi_routes/v2/user_policies/routes.py b/policyengine_api/fastapi_routes/v2/user_policies/routes.py index 4a2f843dc..bb5a06acc 100644 --- a/policyengine_api/fastapi_routes/v2/user_policies/routes.py +++ b/policyengine_api/fastapi_routes/v2/user_policies/routes.py @@ -35,6 +35,7 @@ UserPolicyCollectionQuery, ) from policyengine_api.services.v2.identity.types import AccountPrincipal +from policyengine_api.services.v2.user_policies.types import UserPolicyFilter CountryQueryDependency = Annotated[ CountryQuery, Depends(query_dependency(CountryQuery)) @@ -52,6 +53,31 @@ def _service_factory( return _default_v2_user_policy_service_factory +def _create_user_policy( + service_factory: Callable[[], V2UserPolicyResourceService], + body: UserPolicyCreateRequest, + query: CountryQuery, + principal: AccountPrincipal, +) -> UserPolicyDetailResponse | JSONResponse: + """Create an association once the body and query agree on its country.""" + if body.country_id != query.country_id: + return v2_error_response( + 400, + "Body country_id must match query country_id", + ) + + def create() -> UserPolicyDetailResponse: + item = service_factory().create_user_policy( + principal=principal, + association_input=body, + ) + return UserPolicyDetailResponse( + result=UserPolicyDetailResult(item=UserPolicyItem.from_read(item)) + ) + + return user_resource_operation(create) + + def build_v2_user_policy_router( dependencies: NativeRouteDependencies, ) -> APIRouter: @@ -89,22 +115,7 @@ def create_user_policy( query: CountryQueryDependency, principal: Annotated[AccountPrincipal, Depends(require_user)], ) -> UserPolicyDetailResponse | JSONResponse: - if body.country_id != query.country_id: - return v2_error_response( - 400, - "Body country_id must match query country_id", - ) - - def create() -> UserPolicyDetailResponse: - item = service_factory().create_user_policy( - principal=principal, - association_input=body, - ) - return UserPolicyDetailResponse( - result=UserPolicyDetailResult(item=UserPolicyItem.from_read(item)) - ) - - return user_resource_operation(create) + return _create_user_policy(service_factory, body, query, principal) @router.get( "/user-policies/{association_id}", @@ -141,8 +152,10 @@ def read() -> UserPolicyPageResponse: page = service_factory().list_user_policies( principal=principal, country_id=query.country_id, - policy_id=query.policy_id, - legacy_policy_id=query.legacy_policy_id, + policy_filter=UserPolicyFilter( + policy_id=query.policy_id, + legacy_policy_id=query.legacy_policy_id, + ), offset=query.offset, limit=query.limit, ) diff --git a/policyengine_api/fastapi_routes/v2/user_reports/routes.py b/policyengine_api/fastapi_routes/v2/user_reports/routes.py index d81170bd5..241f19882 100644 --- a/policyengine_api/fastapi_routes/v2/user_reports/routes.py +++ b/policyengine_api/fastapi_routes/v2/user_reports/routes.py @@ -76,6 +76,28 @@ def _service_factory( return _default_v2_user_report_service_factory +def _create_user_report( + service_factory: Callable[[], V2UserReportResourceService], + body: UserReportCreateRequest, + query: CountryQuery, + principal: AccountPrincipal, +) -> UserReportDetailResponse | JSONResponse: + """Create an association once the body and query agree on its country.""" + if body.country_id != query.country_id: + return v2_error_response(400, "Body country_id must match query country_id") + + def create() -> UserReportDetailResponse: + item = service_factory().create_user_report( + principal=principal, + association_input=body, + ) + return UserReportDetailResponse( + result=UserReportDetailResult(item=UserReportItem.from_read(item)) + ) + + return user_resource_operation(create) + + def build_v2_user_report_router( dependencies: NativeRouteDependencies, ) -> APIRouter: @@ -113,19 +135,7 @@ def create_user_report( query: CountryQueryDependency, principal: Annotated[AccountPrincipal, Depends(require_user)], ) -> UserReportDetailResponse | JSONResponse: - if body.country_id != query.country_id: - return v2_error_response(400, "Body country_id must match query country_id") - - def create() -> UserReportDetailResponse: - item = service_factory().create_user_report( - principal=principal, - association_input=body, - ) - return UserReportDetailResponse( - result=UserReportDetailResult(item=UserReportItem.from_read(item)) - ) - - return user_resource_operation(create) + return _create_user_report(service_factory, body, query, principal) @router.get( "/user-reports/{association_id}", diff --git a/policyengine_api/fastapi_routes/v2/user_resources.py b/policyengine_api/fastapi_routes/v2/user_resources.py index 945dc448b..208e4c597 100644 --- a/policyengine_api/fastapi_routes/v2/user_resources.py +++ b/policyengine_api/fastapi_routes/v2/user_resources.py @@ -4,7 +4,7 @@ import logging import traceback -from typing import TYPE_CHECKING, Any, TypeVar +from typing import TYPE_CHECKING, Any, NamedTuple, TypeVar # FastAPI reads the request-size dependency's annotations when routes are # registered, so Request must exist at runtime. @@ -44,7 +44,7 @@ ) if TYPE_CHECKING: - from collections.abc import Awaitable, Callable + from collections.abc import Awaitable, Callable, Mapping from starlette.responses import JSONResponse @@ -167,41 +167,77 @@ def _log_failure(event: str, error: Exception) -> None: pass +class UserResourceFailure(NamedTuple): + """The API v2 error response for one family of user-route service errors. + + Without a ``message``, the response carries the error's own message. A + ``logged_event`` records the failure, without its message, under that name. + """ + + error_types: tuple[type[Exception], ...] + status_code: int + message: str | None = None + headers: Mapping[str, str] | None = None + logged_event: str | None = None + + +# Checked in order, so a subclass is listed before any base it shares a status +# with, and IntegrityError before the SQLAlchemyError it specializes. +USER_RESOURCE_FAILURES: tuple[UserResourceFailure, ...] = ( + UserResourceFailure( + (StandardUserRequiresSignInError,), + 401, + headers={"WWW-Authenticate": BEARER_CHALLENGE}, + ), + UserResourceFailure((AccountEditRequiresSignInError,), 403), + UserResourceFailure((AssociationReferenceConflictError,), 400), + UserResourceFailure( + (LegacyReferenceNotFoundError, AssociationLinkNotFoundError), 422 + ), + UserResourceFailure( + ( + AssociationNotFoundError, + AssociationPolicyNotFoundError, + AssociationHouseholdNotFoundError, + AccountNotFoundError, + ), + 404, + ), + UserResourceFailure((AssociationConflictError,), 409), + UserResourceFailure( + (IntegrityError,), 409, "The record conflicts with stored state" + ), + UserResourceFailure((AccountIdentityDeletionError,), 502), + UserResourceFailure((AccountDeletionUnavailableError,), 503), + # Schema drift and rejected data surface here too, so keep a record. + UserResourceFailure( + (V2ConfigurationError, SQLAlchemyError), + 503, + "V2 user persistence is unavailable", + logged_event="v2_user_persistence_unavailable", + ), +) + + +def _user_failure_response(error: Exception) -> JSONResponse: + """Answer a failure with its first matching typed error, or a logged 500.""" + for failure in USER_RESOURCE_FAILURES: + if isinstance(error, failure.error_types): + if failure.logged_event is not None: + _log_failure(failure.logged_event, error) + message = str(error) if failure.message is None else failure.message + return v2_error_response( + failure.status_code, message, headers=failure.headers + ) + _log_failure("v2_user_operation_failed", error) + return v2_error_response(500, "V2 user operation failed") + + def user_resource_operation( operation: Callable[[], OperationT], ) -> OperationT | JSONResponse: """Run one service call and map typed failures to secret-safe errors.""" try: return operation() - except StandardUserRequiresSignInError as error: - return v2_error_response( - 401, str(error), headers={"WWW-Authenticate": BEARER_CHALLENGE} - ) - except AccountEditRequiresSignInError as error: - return v2_error_response(403, str(error)) - except AssociationReferenceConflictError as error: - return v2_error_response(400, str(error)) - except (LegacyReferenceNotFoundError, AssociationLinkNotFoundError) as error: - return v2_error_response(422, str(error)) - except ( - AssociationNotFoundError, - AssociationPolicyNotFoundError, - AssociationHouseholdNotFoundError, - AccountNotFoundError, - ) as error: - return v2_error_response(404, str(error)) - except AssociationConflictError as error: - return v2_error_response(409, str(error)) - except IntegrityError: - return v2_error_response(409, "The record conflicts with stored state") - except AccountIdentityDeletionError as error: - return v2_error_response(502, str(error)) - except AccountDeletionUnavailableError as error: - return v2_error_response(503, str(error)) - except (V2ConfigurationError, SQLAlchemyError) as error: - # Schema drift and rejected data surface here too, so keep a record. - _log_failure("v2_user_persistence_unavailable", error) - return v2_error_response(503, "V2 user persistence is unavailable") except Exception as error: # noqa: BLE001 # return a secret-safe typed error - _log_failure("v2_user_operation_failed", error) - return v2_error_response(500, "V2 user operation failed") + return _user_failure_response(error) diff --git a/policyengine_api/fastapi_routes/v2/user_simulations/routes.py b/policyengine_api/fastapi_routes/v2/user_simulations/routes.py index 10e6bb6a0..0cac726a4 100644 --- a/policyengine_api/fastapi_routes/v2/user_simulations/routes.py +++ b/policyengine_api/fastapi_routes/v2/user_simulations/routes.py @@ -53,6 +53,28 @@ def _service_factory( return _default_v2_user_simulation_service_factory +def _create_user_simulation( + service_factory: Callable[[], V2UserSimulationResourceService], + body: UserSimulationCreateRequest, + query: CountryQuery, + principal: AccountPrincipal, +) -> UserSimulationDetailResponse | JSONResponse: + """Create an association once the body and query agree on its country.""" + if body.country_id != query.country_id: + return v2_error_response(400, "Body country_id must match query country_id") + + def create() -> UserSimulationDetailResponse: + item = service_factory().create_user_simulation( + principal=principal, + association_input=body, + ) + return UserSimulationDetailResponse( + result=UserSimulationDetailResult(item=UserSimulationItem.from_read(item)) + ) + + return user_resource_operation(create) + + def build_v2_user_simulation_router( dependencies: NativeRouteDependencies, ) -> APIRouter: @@ -89,21 +111,7 @@ def create_user_simulation( query: CountryQueryDependency, principal: Annotated[AccountPrincipal, Depends(require_user)], ) -> UserSimulationDetailResponse | JSONResponse: - if body.country_id != query.country_id: - return v2_error_response(400, "Body country_id must match query country_id") - - def create() -> UserSimulationDetailResponse: - item = service_factory().create_user_simulation( - principal=principal, - association_input=body, - ) - return UserSimulationDetailResponse( - result=UserSimulationDetailResult( - item=UserSimulationItem.from_read(item) - ) - ) - - return user_resource_operation(create) + return _create_user_simulation(service_factory, body, query, principal) @router.get( "/user-simulations/{association_id}", diff --git a/policyengine_api/libs/auth0_management.py b/policyengine_api/libs/auth0_management.py index 224915b97..2985fa800 100644 --- a/policyengine_api/libs/auth0_management.py +++ b/policyengine_api/libs/auth0_management.py @@ -20,6 +20,7 @@ import time from dataclasses import dataclass, field from functools import cache +from http import HTTPStatus from typing import TYPE_CHECKING from urllib.parse import quote @@ -189,7 +190,7 @@ def _access_token(self) -> str: "audience": f"{self._base_url}/api/v2/", }, ) - if response.status_code != 200: + if response.status_code != HTTPStatus.OK: msg = ( f"Management API token request failed: HTTP {response.status_code}" ) @@ -237,7 +238,7 @@ def delete_user(self, subject: str) -> None: """ token = self._access_token() response = self._delete_request(subject, token) - if response.status_code == 401: + if response.status_code == HTTPStatus.UNAUTHORIZED: self._discard_token(token) response = self._delete_request(subject, self._access_token()) if response.status_code in {200, 202, 204, 404}: diff --git a/policyengine_api/libs/gateway_auth.py b/policyengine_api/libs/gateway_auth.py index 110467d06..22ad2f960 100644 --- a/policyengine_api/libs/gateway_auth.py +++ b/policyengine_api/libs/gateway_auth.py @@ -12,7 +12,9 @@ import os import threading import time +from dataclasses import dataclass from functools import cache +from http import HTTPStatus from typing import TYPE_CHECKING import httpx @@ -112,6 +114,20 @@ def _require_all_or_none_gateway_auth_env() -> None: raise GatewayAuthError(msg) +@dataclass(frozen=True) +class GatewayAuthCredentials: + """Auth0 client_credentials settings for the simulation gateway. + + A setting left None is read from its environment variable. + """ + + issuer: str | None = None + audience: str | None = None + client_id: str | None = None + client_secret: str | None = None + client_secret_resource: str | None = None + + class GatewayAuthTokenProvider: """Fetch and cache an Auth0 client_credentials access token.""" @@ -119,38 +135,36 @@ class GatewayAuthTokenProvider: def __init__( self, - issuer: str | None = None, - audience: str | None = None, - client_id: str | None = None, - client_secret: str | None = None, - client_secret_resource: str | None = None, + credentials: GatewayAuthCredentials | None = None, *, http_timeout: float = 10.0, ) -> None: - """Read each Auth0 setting from its argument, else its environment variable.""" + """Read each Auth0 setting from ``credentials``, else its environment.""" + if credentials is None: + credentials = GatewayAuthCredentials() self._issuer = ( - issuer - if issuer is not None + credentials.issuer + if credentials.issuer is not None else os.environ.get(GATEWAY_AUTH_ISSUER_ENV, "") ).rstrip("/") self._audience = ( - audience - if audience is not None + credentials.audience + if credentials.audience is not None else os.environ.get(GATEWAY_AUTH_AUDIENCE_ENV, "") ) self._client_id = ( - client_id - if client_id is not None + credentials.client_id + if credentials.client_id is not None else os.environ.get(GATEWAY_AUTH_CLIENT_ID_ENV, "") ) self._client_secret = ( - client_secret - if client_secret is not None + credentials.client_secret + if credentials.client_secret is not None else os.environ.get(GATEWAY_AUTH_CLIENT_SECRET_ENV, "") ) self._client_secret_resource = ( - client_secret_resource - if client_secret_resource is not None + credentials.client_secret_resource + if credentials.client_secret_resource is not None else os.environ.get(GATEWAY_AUTH_CLIENT_SECRET_RESOURCE_ENV, "") ) self._http_timeout = http_timeout @@ -278,7 +292,7 @@ def auth_flow( request.headers["Authorization"] = f"Bearer {self._token_provider.get_token()}" response = yield request - if response.status_code != 401: + if response.status_code != HTTPStatus.UNAUTHORIZED: return self._token_provider.invalidate() diff --git a/policyengine_api/libs/simulation_entrypoint.py b/policyengine_api/libs/simulation_entrypoint.py index 4c8e9c2b4..cd722894d 100644 --- a/policyengine_api/libs/simulation_entrypoint.py +++ b/policyengine_api/libs/simulation_entrypoint.py @@ -3,6 +3,7 @@ import os import sys from dataclasses import dataclass, field +from http import HTTPStatus from urllib.parse import urlparse import httpx @@ -648,7 +649,7 @@ def health_check(self) -> bool: except Exception: # noqa: BLE001 # a health probe reports any failure as unhealthy return False else: - return response.status_code == 200 + return response.status_code == HTTPStatus.OK # Compatibility aliases preserve existing imports and generated-client consumers. diff --git a/policyengine_api/migration_flags.py b/policyengine_api/migration_flags.py index 747053af3..2b966808b 100644 --- a/policyengine_api/migration_flags.py +++ b/policyengine_api/migration_flags.py @@ -44,6 +44,33 @@ class RouteImplementation(StrEnum): DEFAULT_SIM_ENTRYPOINT = "old_gateway_direct" DEFAULT_SIM_COMPUTE_BACKEND = "old_gateway" +ROUTE_GROUP_BY_PATH = { + "": "home", + "/": "home", + "/health": "health", + "/simulation-gateway-check": "health", + "/liveness-check": "health", + "/readiness-check": "health", + "/specification": "specification", + "/v2/openapi.json": "specification", +} +# Any other path's group is named by one of its leading segments: the resource +# ("/policy"), the resource after a country ("/us/policy") or, under "/v2", the +# resource after the version and a country ("/v2/us/policy"). +ROUTE_GROUP_SEGMENT_DEPTH = 2 +V2_ROUTE_GROUP_SEGMENT_DEPTH = 3 + + +@dataclass(frozen=True) +class ExplicitDatabaseSources: + """Database sources a request names itself, in place of the configured flags. + + Each source is validated against the known sources; None records no source. + """ + + db_write_source: str | None = None + db_read_source: str | None = None + @dataclass(frozen=True) class MigrationContext: @@ -83,32 +110,22 @@ def _read_choice(env_name: str, default: str, valid_values: frozenset[str]) -> s def infer_route_group(path: str) -> str: """Infer a migration route group from a request path.""" - if path in {"/", ""}: - return "home" - if path in { - "/health", - "/simulation-gateway-check", - "/liveness-check", - "/readiness-check", - }: - return "health" - if path in {"/specification", "/v2/openapi.json"}: - return "specification" + path_group = ROUTE_GROUP_BY_PATH.get(path) + if path_group is not None: + return path_group segments = [segment for segment in path.strip("/").split("/") if segment] if not segments: return "home" - first = segments[0] - if first in ROUTE_GROUP_BY_SEGMENT: - return ROUTE_GROUP_BY_SEGMENT[first] - - if len(segments) >= 2 and segments[1] in ROUTE_GROUP_BY_SEGMENT: - return ROUTE_GROUP_BY_SEGMENT[segments[1]] - - if first == "v2" and len(segments) >= 3 and segments[2] in ROUTE_GROUP_BY_SEGMENT: - return ROUTE_GROUP_BY_SEGMENT[segments[2]] - + searched_depth = ( + V2_ROUTE_GROUP_SEGMENT_DEPTH + if segments[0] == "v2" + else ROUTE_GROUP_SEGMENT_DEPTH + ) + for segment in segments[:searched_depth]: + if segment in ROUTE_GROUP_BY_SEGMENT: + return ROUTE_GROUP_BY_SEGMENT[segment] return "unknown" @@ -221,43 +238,47 @@ def get_sim_entrypoint() -> str: ) +def _validated_explicit_sources( + sources: ExplicitDatabaseSources, +) -> tuple[str | None, str | None]: + """Return the explicit write and read sources, or raise ValueError if unknown.""" + db_write_source = sources.db_write_source + if db_write_source is not None and db_write_source not in DB_WRITE_SOURCES: + msg = f"invalid explicit database write source {db_write_source!r}" + raise ValueError(msg) + db_read_source = sources.db_read_source + if db_read_source is not None and db_read_source not in DB_READ_SOURCES: + msg = f"invalid explicit database read source {db_read_source!r}" + raise ValueError(msg) + return db_write_source, db_read_source + + def get_migration_context( route_group: str, *, route_impl: RouteImplementation | None = None, - db_entity: str | None = None, - sim_flow: str | None = None, - use_configured_db_sources: bool = True, - db_write_source: str | None = None, - db_read_source: str | None = None, + explicit_db_sources: ExplicitDatabaseSources | None = None, include_compute: bool = True, ) -> MigrationContext: """Return current migration flag values for a request or route group. + The route group's configuration names its database entity and simulation + flow. ``explicit_db_sources`` replaces the configured database source flags. ``include_compute=False`` describes a persistence-only request in a route group that also computes, so it carries no simulation flow or compute selection. """ route_config = ROUTE_GROUP_CONFIG_BY_NAME.get(route_group) - if db_entity is None and route_config is not None: - db_entity = route_config.db_entity - if not include_compute: - sim_flow = None - elif sim_flow is None and route_config is not None: - sim_flow = route_config.sim_flow - - if use_configured_db_sources: + db_entity = route_config.db_entity if route_config is not None else None + sim_flow = ( + route_config.sim_flow if include_compute and route_config is not None else None + ) + + if explicit_db_sources is None: db_write = get_db_write(db_entity) if db_entity else None db_read = get_db_read(db_entity) if db_entity else None else: - if db_write_source is not None and db_write_source not in DB_WRITE_SOURCES: - msg = f"invalid explicit database write source {db_write_source!r}" - raise ValueError(msg) - if db_read_source is not None and db_read_source not in DB_READ_SOURCES: - msg = f"invalid explicit database read source {db_read_source!r}" - raise ValueError(msg) - db_write = db_write_source - db_read = db_read_source + db_write, db_read = _validated_explicit_sources(explicit_db_sources) return MigrationContext( route_group=route_group, @@ -275,9 +296,7 @@ def get_migration_log_context( route_group: str, *, route_impl: RouteImplementation | None = None, - use_configured_db_sources: bool = True, - db_write_source: str | None = None, - db_read_source: str | None = None, + explicit_db_sources: ExplicitDatabaseSources | None = None, include_compute: bool = True, ) -> dict: """Best-effort logging context; never raises on invalid flag settings.""" @@ -285,9 +304,7 @@ def get_migration_log_context( return get_migration_context( route_group, route_impl=route_impl, - use_configured_db_sources=use_configured_db_sources, - db_write_source=db_write_source, - db_read_source=db_read_source, + explicit_db_sources=explicit_db_sources, include_compute=include_compute, ).to_log_dict() except ValueError as error: diff --git a/policyengine_api/migration_logging.py b/policyengine_api/migration_logging.py index 76056303c..18f5ac7a9 100644 --- a/policyengine_api/migration_logging.py +++ b/policyengine_api/migration_logging.py @@ -4,12 +4,15 @@ import contextlib import time +from dataclasses import dataclass +from http import HTTPStatus from typing import TYPE_CHECKING import flask from policyengine_api.gcp_logging import logger from policyengine_api.migration_flags import ( + ExplicitDatabaseSources, RouteImplementation, get_migration_log_context, infer_route_group, @@ -59,19 +62,35 @@ {"user-policies", "user-households", "user-simulations", "user-reports", "me"} ) V2_RESOURCE_METHODS = frozenset({"GET", "POST", "PATCH", "DELETE"}) +# A v2 resource path has at least the "v2" segment and the resource segment. +V2_RESOURCE_PATH_MINIMUM_SEGMENTS = 2 + + +@dataclass(frozen=True) +class ServedRequest: + """One served API request, as the migration request log records it.""" + + request_id: str | None + method: str + path: str + status_code: int + started_at: float | None + country_id: str | None = None + route_impl: RouteImplementation | None = None def _v2_resource_segment(path: str) -> str | None: + """Return the resource segment of a ``/v2/`` path, or None.""" segments = [segment for segment in path.strip("/").split("/") if segment] - if len(segments) >= 2 and segments[0] == "v2": - return segments[1] - return None + if len(segments) < V2_RESOURCE_PATH_MINIMUM_SEGMENTS or segments[0] != "v2": + return None + return segments[1] def _is_v2_metadata_resource_read(method: str, path: str) -> bool: - return ( - method == "GET" and _v2_resource_segment(path) in V2_METADATA_RESOURCE_SEGMENTS - ) + if method != "GET": + return False + return _v2_resource_segment(path) in V2_METADATA_RESOURCE_SEGMENTS def _is_v2_supabase_resource(method: str, path: str) -> bool: @@ -90,59 +109,18 @@ def register_migration_request_logging( @app.before_request def set_request_migration_context() -> None: - flask.g.request_started_at = time.time() - try: - captured = runtime.capture_context() if runtime is not None else {} - except Exception: # noqa: BLE001 # telemetry must never fail the request - captured = {} - flask.g.request_id = captured.get("request_id") or ( - flask.request.headers.get(REQUEST_ID_HEADER) or generate_request_id() - ) - flask.g.incoming_observability_id = normalize_observability_id( - flask.request.headers.get(OBSERVABILITY_ID_HEADER) - ) - flask.g.observability_id = None - if runtime is not None: - # Telemetry must never fail the request. - with contextlib.suppress(Exception): - runtime.set_context(request_id=flask.g.request_id) + _set_request_migration_context(runtime) @app.after_request def log_request_migration_context(response: flask.Response) -> flask.Response: - request_id = getattr(flask.g, "request_id", None) - if request_id is not None: - response.headers[REQUEST_ID_HEADER] = request_id - observability_id = getattr(flask.g, "observability_id", None) - if observability_id is not None: - response.headers[OBSERVABILITY_ID_HEADER] = observability_id + request_id, observability_id = _apply_request_identifiers(response) try: - country_id = ( - flask.request.view_args.get("country_id") - if flask.request.view_args - else None + _record_request_migration_context( + response, + runtime, + request_id=request_id, + observability_id=observability_id, ) - if runtime is not None: - runtime_context = { - "country_id": country_id, - **_migration_context( - method=flask.request.method, - path=flask.request.path, - route_impl=RouteImplementation.FLASK_FALLBACK, - ), - } - if observability_id is not None: - runtime_context["observability_id"] = observability_id - runtime.set_context(**runtime_context) - else: - log_migration_request( - request_id=request_id, - method=flask.request.method, - path=flask.request.path, - status_code=response.status_code, - started_at=getattr(flask.g, "request_started_at", None), - country_id=country_id, - route_impl=RouteImplementation.FLASK_FALLBACK, - ) except Exception: # A logging failure must not replace the response. with contextlib.suppress(Exception): @@ -150,39 +128,115 @@ def log_request_migration_context(response: flask.Response) -> flask.Response: return response -def log_migration_request( +def _set_request_migration_context(runtime: ObservabilityRuntime | None) -> None: + """Record the request's start time and identifiers on ``flask.g``. + + The request ID comes from the runtime's captured context, else the request + header, else a new ID. + """ + flask.g.request_started_at = time.time() + try: + captured = runtime.capture_context() if runtime is not None else {} + except Exception: # noqa: BLE001 # telemetry must never fail the request + captured = {} + flask.g.request_id = captured.get("request_id") or ( + flask.request.headers.get(REQUEST_ID_HEADER) or generate_request_id() + ) + flask.g.incoming_observability_id = normalize_observability_id( + flask.request.headers.get(OBSERVABILITY_ID_HEADER) + ) + flask.g.observability_id = None + if runtime is not None: + # Telemetry must never fail the request. + with contextlib.suppress(Exception): + runtime.set_context(request_id=flask.g.request_id) + + +def _apply_request_identifiers( + response: flask.Response, +) -> tuple[str | None, str | None]: + """Add the request and observability IDs on ``flask.g`` to the response. + + Return both IDs; either is None when the request has none. + """ + request_id = getattr(flask.g, "request_id", None) + if request_id is not None: + response.headers[REQUEST_ID_HEADER] = request_id + observability_id = getattr(flask.g, "observability_id", None) + if observability_id is not None: + response.headers[OBSERVABILITY_ID_HEADER] = observability_id + return request_id, observability_id + + +def _record_request_migration_context( + response: flask.Response, + runtime: ObservabilityRuntime | None, *, request_id: str | None, - method: str, - path: str, - status_code: int, - started_at: float | None, - country_id: str | None = None, - route_impl: RouteImplementation | None = None, + observability_id: str | None, ) -> None: + """Record a Flask response's migration context. + + With a runtime, the context enriches the runtime's own request record; + without one, it is logged as a migration request record. + """ + country_id = ( + flask.request.view_args.get("country_id") if flask.request.view_args else None + ) + if runtime is not None: + runtime_context = { + "country_id": country_id, + **_migration_context( + method=flask.request.method, + path=flask.request.path, + route_impl=RouteImplementation.FLASK_FALLBACK, + ), + } + if observability_id is not None: + runtime_context["observability_id"] = observability_id + runtime.set_context(**runtime_context) + return + log_migration_request( + ServedRequest( + request_id=request_id, + method=flask.request.method, + path=flask.request.path, + status_code=response.status_code, + started_at=getattr(flask.g, "request_started_at", None), + country_id=country_id, + route_impl=RouteImplementation.FLASK_FALLBACK, + ) + ) + + +def log_migration_request(served_request: ServedRequest) -> None: """Log a migration-aware API request in the shared structured format.""" elapsed_ms = None - if started_at is not None: - elapsed_ms = round((time.time() - started_at) * 1000, 2) + if served_request.started_at is not None: + elapsed_ms = round((time.time() - served_request.started_at) * 1000, 2) migration_context = _migration_context( - method=method, - path=path, - route_impl=route_impl, + method=served_request.method, + path=served_request.path, + route_impl=served_request.route_impl, ) logger.log_struct( { "message": "API request served", - "request_id": request_id, - "method": method, - "path": path, - "status_code": status_code, + "request_id": served_request.request_id, + "method": served_request.method, + "path": served_request.path, + "status_code": served_request.status_code, "latency_ms": elapsed_ms, - "country_id": country_id, + "country_id": served_request.country_id, "migration": migration_context, }, - severity="INFO" if status_code < 500 else "ERROR", + severity=( + "INFO" + if served_request.status_code < HTTPStatus.INTERNAL_SERVER_ERROR + else "ERROR" + ), ) @@ -195,10 +249,8 @@ def _migration_context( route_group = infer_route_group(path) is_v2_metadata_read = _is_v2_metadata_resource_read(method, path) is_v2_supabase_resource = _is_v2_supabase_resource(method, path) - return get_migration_log_context( - route_group, - route_impl=route_impl, - use_configured_db_sources=not (is_v2_metadata_read or is_v2_supabase_resource), + uses_explicit_v2_source = is_v2_metadata_read or is_v2_supabase_resource + explicit_db_sources = ExplicitDatabaseSources( db_write_source=( "supabase" if is_v2_supabase_resource and method != "GET" else None ), @@ -207,5 +259,10 @@ def _migration_context( if is_v2_metadata_read or (is_v2_supabase_resource and method == "GET") else None ), + ) + return get_migration_log_context( + route_group, + route_impl=route_impl, + explicit_db_sources=explicit_db_sources if uses_explicit_v2_source else None, include_compute=_v2_resource_segment(path) not in V2_USER_RESOURCE_SEGMENTS, ) diff --git a/policyengine_api/routes/household_routes.py b/policyengine_api/routes/household_routes.py index 533130375..52159a9e9 100644 --- a/policyengine_api/routes/household_routes.py +++ b/policyengine_api/routes/household_routes.py @@ -6,6 +6,7 @@ import time from collections.abc import Callable from functools import wraps +from http import HTTPStatus from typing import TypeVar from flask import Blueprint, Response, g, request @@ -114,7 +115,7 @@ def _household_persistence_failure( ) message = ( "Household database is temporarily unavailable; please try again later." - if status_code == 503 + if status_code == HTTPStatus.SERVICE_UNAVAILABLE else "Internal database error; please try again later." ) return _make_error_response(message, status_code) @@ -145,6 +146,16 @@ def _spm_error_response(error: Exception) -> Response | None: return None +def _invalid_household_inputs_response(error: InvalidHouseholdInputsError) -> Response: + """Return the 400 that lists every unrecognized calculation input.""" + return _make_error_response( + format_unrecognized_inputs_message(error.invalid_inputs), + 400, + result=None, + errors=[invalid_input.to_dict() for invalid_input in error.invalid_inputs], + ) + + def _calculation_response(calculation: HouseholdCalculationResult) -> dict: result: dict[str, object] = { "status": "ok", @@ -229,15 +240,7 @@ def wrapped( ) ) except InvalidHouseholdInputsError as error: - return _make_error_response( - format_unrecognized_inputs_message(error.invalid_inputs), - 400, - result=None, - errors=[ - invalid_input.to_dict() - for invalid_input in error.invalid_inputs - ], - ) + return _invalid_household_inputs_response(error) except ValueError as error: response = _spm_error_response(error) if response is not None: @@ -245,7 +248,8 @@ def wrapped( raise result = func(country_id, *args, **kwargs) if not g.get("calculation_view_executed", False) and ( - not isinstance(result, Response) or result.status_code < 400 + not isinstance(result, Response) + or result.status_code < HTTPStatus.BAD_REQUEST ): start_observability_id() return result @@ -400,43 +404,75 @@ def get_household_under_policy( int(policy_id), on_accepted=start_observability_id, ) - except HouseholdNotFoundError: + except Exception as error: # noqa: BLE001 # every failure maps to a typed error response + return _stored_calculation_error_response( + error, + household_id=household_id, + policy_id=policy_id, + ) + + return _calculation_response(calculation) + + +def _stored_calculation_error_response( + error: Exception, + *, + household_id: str, + policy_id: str, +) -> Response: + """Map a stored-household calculation failure to its error response. + + Called while handling ``error``; an unexpected failure logs its traceback. + """ + if isinstance(error, HouseholdNotFoundError): return _make_error_response( f"Household #{household_id} not found.", 404, ) - except PolicyNotFoundError: + if isinstance(error, PolicyNotFoundError): return _make_error_response( f"Policy #{policy_id} not found.", 404, ) - except InvalidHouseholdInputsError as error: - return _make_error_response( - format_unrecognized_inputs_message(error.invalid_inputs), - 400, - result=None, - errors=[invalid_input.to_dict() for invalid_input in error.invalid_inputs], - ) - except Exception as error: - response = _spm_error_response(error) - if response is not None: - return response - module_logger.exception( - "Error calculating household #%s under policy #%s", - household_id, - policy_id, - ) - return _make_error_response( - f"Error calculating household #{household_id} under policy " - f"#{policy_id}: {error}", - 500, + if isinstance(error, InvalidHouseholdInputsError): + return _invalid_household_inputs_response(error) + response = _spm_error_response(error) + if response is not None: + return response + module_logger.exception( + "Error calculating household #%s under policy #%s", + household_id, + policy_id, + ) + return _make_error_response( + f"Error calculating household #{household_id} under policy " + f"#{policy_id}: {error}", + 500, + ) + + +def _calculate() -> dict | Response: + g.calculation_view_executed = True + parse_failure = _parse_prepared_household_calculation() + if parse_failure is not None: + return parse_failure + + try: + calculation = household_calculation_service.calculate_prepared_household( + g.prepared_household_calculation, ) + except Exception as error: # noqa: BLE001 # every failure maps to a typed error response + return _calculation_error_response(error) return _calculation_response(calculation) -def _calculate() -> dict | Response: - g.calculation_view_executed = True +def _parse_prepared_household_calculation() -> Response | None: + """Parse the validated request's household, or return its error response. + + An unexpected failure starts the request's observability ID and logs its + traceback. + """ try: g.prepared_household_calculation = ( household_calculation_service.parse_prepared_household( @@ -460,38 +496,30 @@ def _calculate() -> dict | Response: f"Error calculating household under policy: {error}", 500, ) + return None - try: - calculation = household_calculation_service.calculate_prepared_household( - g.prepared_household_calculation, - ) - except InvalidHouseholdInputsError as error: - return _make_error_response( - format_unrecognized_inputs_message(error.invalid_inputs), - 400, - result=None, - errors=[invalid_input.to_dict() for invalid_input in error.invalid_inputs], - ) - except SituationParsingError as error: - response = _spm_error_response(error) - if response is not None: - return response + +def _calculation_error_response(error: Exception) -> Response: + """Map a household calculation failure to its error response. + + Called while handling ``error``; an unexpected failure logs its traceback. + """ + if isinstance(error, InvalidHouseholdInputsError): + return _invalid_household_inputs_response(error) + response = _spm_error_response(error) + if response is not None: + return response + if isinstance(error, SituationParsingError): return _make_error_response( f"Invalid household payload: {error}", 400, result=None, ) - except Exception as error: - response = _spm_error_response(error) - if response is not None: - return response - module_logger.exception("Error calculating household under policy") - return _make_error_response( - f"Error calculating household under policy: {error}", - 500, - ) - - return _calculation_response(calculation) + module_logger.exception("Error calculating household under policy") + return _make_error_response( + f"Error calculating household under policy: {error}", + 500, + ) @household_bp.route("//calculate", methods=["POST"]) diff --git a/policyengine_api/routes/policy_routes.py b/policyengine_api/routes/policy_routes.py index f63e03b3b..d43c0c142 100644 --- a/policyengine_api/routes/policy_routes.py +++ b/policyengine_api/routes/policy_routes.py @@ -3,6 +3,8 @@ import contextlib import json import time +from dataclasses import dataclass +from http import HTTPStatus from flask import Blueprint, Response, request from werkzeug.exceptions import BadRequest, NotFound @@ -24,6 +26,7 @@ ) from policyengine_api.services.policy_service import PolicyService from policyengine_api.services.user_policy_mirroring import ( + UserPolicyMirrorCollaborators, UserPolicyMirrorUnavailableError, mirror_pending_user_policy_events_after_commit, ) @@ -31,6 +34,7 @@ USER_POLICY_MUTABLE_FIELDS, UserPolicyPersistenceError, UserPolicyService, + UserPolicyUpdateResult, ) from policyengine_api.utils.payload_validators import ( validate_country, @@ -41,6 +45,9 @@ policy_service = PolicyService() user_policy_service = UserPolicyService() +# v1 saved-policy IDs are MySQL signed INT values. +MAXIMUM_LEGACY_USER_POLICY_ID = 2_147_483_647 + def _should_mirror_to_v2(country_id: str, write_source: str) -> bool: """Select immediate mirroring only for countries supported by API v2.""" @@ -73,20 +80,26 @@ def _safe_legacy_user_policy_id(value: object) -> int | None: if ( isinstance(value, int) and not isinstance(value, bool) - and 0 <= value <= 2_147_483_647 + and 0 <= value <= MAXIMUM_LEGACY_USER_POLICY_ID ): return value return None +@dataclass(frozen=True) +class _UserPolicyWriteAttempt: + """One saved-policy write, as the record of its failure describes it.""" + + operation: str + country_id: str + configured_write_source: str + started_at: float + legacy_user_policy_id: object | None = None + + def _user_policy_persistence_failure( error: UserPolicyPersistenceError, - *, - operation: str, - country_id: str, - configured_write_source: str, - started_at: float, - legacy_user_policy_id: object | None = None, + attempt: _UserPolicyWriteAttempt, ) -> Response: """Return and record a failure without serializing exception details.""" status_code = 503 if error.retryable else 500 @@ -98,19 +111,19 @@ def _user_policy_persistence_failure( "metric_name": "v1_user_policy_persistence_failures", "metric_value": 1, "resource": "user_policy", - "operation": operation, + "operation": attempt.operation, "database_source": "cloud_sql", - "configured_write_source": configured_write_source, - "country_id": country_id, + "configured_write_source": attempt.configured_write_source, + "country_id": attempt.country_id, "legacy_user_policy_id": _safe_legacy_user_policy_id( - legacy_user_policy_id + attempt.legacy_user_policy_id ), "request_id": current_request_id(), "outcome": "error", "failure_category": error.category.value, "http_status": status_code, "duration_ms": round( - (time.perf_counter() - started_at) * 1000, + (time.perf_counter() - attempt.started_at) * 1000, 3, ), }, @@ -119,7 +132,7 @@ def _user_policy_persistence_failure( message = ( "Policy database is temporarily unavailable; please try again later." - if status_code == 503 + if status_code == HTTPStatus.SERVICE_UNAVAILABLE else "Internal database error; please try again later." ) return _make_error_response( @@ -233,6 +246,47 @@ def set_policy(country_id: str) -> Response: return Response(json.dumps(response_body), status=code, mimetype="application/json") +def _user_policy_write_configuration(country_id: str) -> tuple[str, bool]: + """Return the configured write source and whether a write mirrors to v2. + + Raise ValueError when a source setting the write needs is invalid. + """ + write_source = get_v1_policy_write_source() + mirror_to_v2 = _should_mirror_to_v2(country_id, write_source) + if mirror_to_v2: + get_v1_policy_read_source() + return write_source, mirror_to_v2 + + +def _mirror_user_policy_events( + country_id: str, + user_policy_id: int, + mirror_revision: int | None, +) -> Response | None: + """Mirror a committed saved policy's events to v2 through ``mirror_revision``. + + Return the mirror-unavailable response if no revision was recorded or any + event cannot be mirrored, else None. + """ + if mirror_revision is None: + return _user_policy_mirror_unavailable() + try: + mirror_pending_user_policy_events_after_commit( + country_id, + user_policy_id, + through_revision=mirror_revision, + collaborators=UserPolicyMirrorCollaborators( + event_service=user_policy_service, + reform_snapshot_loader=policy_service.get_policy_snapshot, + ), + ) + except UserPolicyMirrorUnavailableError: + return _user_policy_mirror_unavailable() + except Exception: # noqa: BLE001 # the v1 write is committed; any mirror failure is mirror-unavailable + return _user_policy_mirror_unavailable() + return None + + def _serialize_user_policy(user_policy: UserPolicy) -> dict: return { column.name: getattr(user_policy, column.name) @@ -323,10 +377,7 @@ def set_user_policy(country_id: str) -> Response: } try: - write_source = get_v1_policy_write_source() - mirror_to_v2 = _should_mirror_to_v2(country_id, write_source) - if mirror_to_v2: - get_v1_policy_read_source() + write_source, mirror_to_v2 = _user_policy_write_configuration(country_id) except ValueError: return _policy_configuration_unavailable() @@ -340,27 +391,22 @@ def set_user_policy(country_id: str) -> Response: except UserPolicyPersistenceError as error: return _user_policy_persistence_failure( error, - operation="create", - country_id=country_id, - configured_write_source=write_source, - started_at=persistence_started_at, + _UserPolicyWriteAttempt( + operation="create", + country_id=country_id, + configured_write_source=write_source, + started_at=persistence_started_at, + ), ) if mirror_to_v2: - if creation.mirror_revision is None: - return _user_policy_mirror_unavailable() - try: - mirror_pending_user_policy_events_after_commit( - country_id, - creation.user_policy.id, - through_revision=creation.mirror_revision, - event_service=user_policy_service, - reform_snapshot_loader=policy_service.get_policy_snapshot, - ) - except UserPolicyMirrorUnavailableError: - return _user_policy_mirror_unavailable() - except Exception: # noqa: BLE001 # the v1 write is committed; any mirror failure is mirror-unavailable - return _user_policy_mirror_unavailable() + mirror_failure = _mirror_user_policy_events( + country_id, + creation.user_policy.id, + creation.mirror_revision, + ) + if mirror_failure is not None: + return mirror_failure if not creation.created: return Response( @@ -423,28 +469,12 @@ def update_user_policy(country_id: str) -> Response: ) user_policy_id = payload.pop("id") - unknown_keys = [ - key for key in payload if key not in UPDATE_USER_POLICY_ALLOWED_FIELDS - ] - if unknown_keys: - return _make_error_response( - f"Request body contains unsupported fields: {sorted(unknown_keys)}", - 400, - include_status=False, - ) - - if not payload: - return _make_error_response( - "Request body must include at least one field to update.", - 400, - include_status=False, - ) + field_error = _user_policy_update_field_error(payload) + if field_error is not None: + return field_error try: - write_source = get_v1_policy_write_source() - mirror_to_v2 = _should_mirror_to_v2(country_id, write_source) - if mirror_to_v2: - get_v1_policy_read_source() + write_source, mirror_to_v2 = _user_policy_write_configuration(country_id) except ValueError: return _policy_configuration_unavailable() @@ -459,13 +489,52 @@ def update_user_policy(country_id: str) -> Response: except UserPolicyPersistenceError as error: return _user_policy_persistence_failure( error, - operation="update", - country_id=country_id, - configured_write_source=write_source, - started_at=persistence_started_at, - legacy_user_policy_id=user_policy_id, + _UserPolicyWriteAttempt( + operation="update", + country_id=country_id, + configured_write_source=write_source, + started_at=persistence_started_at, + legacy_user_policy_id=user_policy_id, + ), + ) + + return _committed_update_response( + country_id, + user_policy_id, + update, + mirror_to_v2=mirror_to_v2, + ) + + +def _user_policy_update_field_error(payload: dict) -> Response | None: + """Return the 400 for update fields that are unsupported or absent, else None.""" + unknown_keys = [ + key for key in payload if key not in UPDATE_USER_POLICY_ALLOWED_FIELDS + ] + if unknown_keys: + return _make_error_response( + f"Request body contains unsupported fields: {sorted(unknown_keys)}", + 400, + include_status=False, ) + if not payload: + return _make_error_response( + "Request body must include at least one field to update.", + 400, + include_status=False, + ) + return None + + +def _committed_update_response( + country_id: str, + user_policy_id: object, + update: UserPolicyUpdateResult | None, + *, + mirror_to_v2: bool, +) -> Response: + """Answer a committed saved-policy update, mirroring it to v2 if selected.""" if update is None: return _make_error_response( f"User policy #{user_policy_id} not found.", @@ -474,20 +543,13 @@ def update_user_policy(country_id: str) -> Response: ) if mirror_to_v2: - if update.mirror_revision is None: - return _user_policy_mirror_unavailable() - try: - mirror_pending_user_policy_events_after_commit( - country_id, - update.user_policy.id, - through_revision=update.mirror_revision, - event_service=user_policy_service, - reform_snapshot_loader=policy_service.get_policy_snapshot, - ) - except UserPolicyMirrorUnavailableError: - return _user_policy_mirror_unavailable() - except Exception: # noqa: BLE001 # the v1 write is committed; any mirror failure is mirror-unavailable - return _user_policy_mirror_unavailable() + mirror_failure = _mirror_user_policy_events( + country_id, + update.user_policy.id, + update.mirror_revision, + ) + if mirror_failure is not None: + return mirror_failure return Response( json.dumps( diff --git a/policyengine_api/routes/simulation_routes.py b/policyengine_api/routes/simulation_routes.py index 0c74ab322..91ad8e0a0 100644 --- a/policyengine_api/routes/simulation_routes.py +++ b/policyengine_api/routes/simulation_routes.py @@ -57,6 +57,29 @@ def _require_updated_simulation(simulation: Simulation | None) -> Simulation: return simulation +def _require_simulation_identity( + population_id: object, + population_type: object, + policy_id: object, +) -> None: + """Raise BadRequest for the first missing or invalid simulation identity field.""" + if not population_id: + msg = "population_id is required" + raise BadRequest(msg) + if not population_type: + msg = "population_type is required" + raise BadRequest(msg) + if population_type not in ["household", "geography"]: + msg = "population_type must be 'household' or 'geography'" + raise BadRequest(msg) + if policy_id is None: + msg = "policy_id is required" + raise BadRequest(msg) + if not isinstance(policy_id, int): + msg = "policy_id must be an integer" + raise BadRequest(msg) + + @simulation_bp.route("//simulation", methods=["POST"]) @validate_country def create_simulation(country_id: str) -> Response: @@ -83,22 +106,7 @@ def create_simulation(country_id: str) -> Response: population_type = payload.get("population_type") policy_id = payload.get("policy_id") - # Validate required fields - if not population_id: - msg = "population_id is required" - raise BadRequest(msg) - if not population_type: - msg = "population_type is required" - raise BadRequest(msg) - if population_type not in ["household", "geography"]: - msg = "population_type must be 'household' or 'geography'" - raise BadRequest(msg) - if policy_id is None: - msg = "policy_id is required" - raise BadRequest(msg) - if not isinstance(policy_id, int): - msg = "policy_id must be an integer" - raise BadRequest(msg) + _require_simulation_identity(population_id, population_type, policy_id) try: creation = simulation_service.get_or_create_simulation( diff --git a/policyengine_api/services/household_mirroring.py b/policyengine_api/services/household_mirroring.py index d2df711c3..e82a2d9d9 100644 --- a/policyengine_api/services/household_mirroring.py +++ b/policyengine_api/services/household_mirroring.py @@ -4,6 +4,7 @@ import contextlib import time +from dataclasses import dataclass from typing import TYPE_CHECKING, Protocol from sqlalchemy.exc import SQLAlchemyError @@ -80,17 +81,25 @@ def _failure_category(error: Exception) -> str: return "unexpected" +@dataclass(frozen=True) +class _CopyOutcome: + """How one immediate household copy ended, as its metric record reports it.""" + + outcome: str + actual_write_sources: list[str] + pending_event: bool + result: LegacyHouseholdPersistenceResult | None = None + failure_category: str | None = None + + def _log_copy_operation( *, country_id: str, legacy_household_id: int, started_at: float, - outcome: str, - actual_write_sources: list[str], - pending_event: bool, - result: LegacyHouseholdPersistenceResult | None = None, - failure_category: str | None = None, + copy_outcome: _CopyOutcome, ) -> None: + result = copy_outcome.result # Metric logging must not change the mirror outcome. with contextlib.suppress(Exception): logger.log_struct( @@ -100,24 +109,24 @@ def _log_copy_operation( "metric_value": 1, "configured_write_source": "dual_write", "attempted_write_sources": ["cloud_sql", "supabase"], - "actual_write_sources": actual_write_sources, + "actual_write_sources": copy_outcome.actual_write_sources, "country_id": country_id, "legacy_household_id": legacy_household_id, "destination_household_id": ( str(result.household_id) if result is not None else None ), - "outcome": outcome, - "failure_category": failure_category, + "outcome": copy_outcome.outcome, + "failure_category": copy_outcome.failure_category, "household_created": ( result.household_created if result is not None else None ), "mapping_created": ( result.mapping_created if result is not None else None ), - "pending_event": pending_event, + "pending_event": copy_outcome.pending_event, "duration_ms": round((time.perf_counter() - started_at) * 1000, 3), }, - severity="INFO" if outcome == "ok" else "ERROR", + severity="INFO" if copy_outcome.outcome == "ok" else "ERROR", ) @@ -151,10 +160,12 @@ def after_processed_commit( country_id=event.snapshot.country_id, legacy_household_id=event.snapshot.legacy_household_id, started_at=started_at, - outcome="ok", - actual_write_sources=["cloud_sql", "supabase"], - pending_event=False, - result=result, + copy_outcome=_CopyOutcome( + outcome="ok", + actual_write_sources=["cloud_sql", "supabase"], + pending_event=False, + result=result, + ), ) try: @@ -170,15 +181,17 @@ def after_processed_commit( country_id=country_id, legacy_household_id=legacy_household_id, started_at=started_at, - outcome="error", - actual_write_sources=( - ["cloud_sql", "supabase"] - if destination_result is not None - else ["cloud_sql"] + copy_outcome=_CopyOutcome( + outcome="error", + actual_write_sources=( + ["cloud_sql", "supabase"] + if destination_result is not None + else ["cloud_sql"] + ), + pending_event=True, + result=destination_result, + failure_category=_failure_category(error), ), - pending_event=True, - result=destination_result, - failure_category=_failure_category(error), ) msg = "The committed v1 household event could not be copied to v2" raise HouseholdMirrorUnavailableError(msg) from error diff --git a/policyengine_api/services/policy_mirroring.py b/policyengine_api/services/policy_mirroring.py index ee067ce88..cd28670e3 100644 --- a/policyengine_api/services/policy_mirroring.py +++ b/policyengine_api/services/policy_mirroring.py @@ -4,6 +4,7 @@ import contextlib import time +from dataclasses import dataclass from typing import TYPE_CHECKING, Protocol from sqlalchemy.exc import SQLAlchemyError @@ -87,17 +88,25 @@ def _failure_category(error: Exception) -> str: return "unexpected" +@dataclass(frozen=True) +class _MirrorOutcome: + """How one immediate policy mirror ended, as its metric record reports it.""" + + outcome: str + actual_write_sources: list[str] + destination_policy_id: object | None = None + failure_category: str | None = None + policy_created: bool | None = None + mapping_created: bool | None = None + + def _log_mirror_event( *, snapshot: LegacyPolicySnapshot, started_at: float, - outcome: str, - actual_write_sources: list[str], - destination_policy_id: object | None = None, - failure_category: str | None = None, - policy_created: bool | None = None, - mapping_created: bool | None = None, + mirror_outcome: _MirrorOutcome, ) -> None: + destination_policy_id = mirror_outcome.destination_policy_id # Metric logging must not change the mirror outcome. with contextlib.suppress(Exception): logger.log_struct( @@ -107,7 +116,7 @@ def _log_mirror_event( "metric_value": 1, "configured_write_source": "dual_write", "attempted_write_sources": ["cloud_sql", "supabase"], - "actual_write_sources": actual_write_sources, + "actual_write_sources": mirror_outcome.actual_write_sources, "country_id": snapshot.country_id, "legacy_policy_id": snapshot.legacy_policy_id, "destination_policy_id": ( @@ -115,13 +124,13 @@ def _log_mirror_event( if destination_policy_id is not None else None ), - "outcome": outcome, - "failure_category": failure_category, - "policy_created": policy_created, - "mapping_created": mapping_created, + "outcome": mirror_outcome.outcome, + "failure_category": mirror_outcome.failure_category, + "policy_created": mirror_outcome.policy_created, + "mapping_created": mirror_outcome.mapping_created, "duration_ms": round((time.perf_counter() - started_at) * 1000, 3), }, - severity="INFO" if outcome == "ok" else "ERROR", + severity="INFO" if mirror_outcome.outcome == "ok" else "ERROR", ) @@ -140,9 +149,11 @@ def mirror_policy_after_commit( _log_mirror_event( snapshot=snapshot, started_at=started_at, - outcome="error", - actual_write_sources=["cloud_sql"], - failure_category=_failure_category(error), + mirror_outcome=_MirrorOutcome( + outcome="error", + actual_write_sources=["cloud_sql"], + failure_category=_failure_category(error), + ), ) msg = "The committed v1 policy could not be mirrored to v2" raise PolicyMirrorUnavailableError(msg) from error @@ -150,10 +161,12 @@ def mirror_policy_after_commit( _log_mirror_event( snapshot=snapshot, started_at=started_at, - outcome="ok", - actual_write_sources=["cloud_sql", "supabase"], - destination_policy_id=result.policy_id, - policy_created=result.policy_created, - mapping_created=result.mapping_created, + mirror_outcome=_MirrorOutcome( + outcome="ok", + actual_write_sources=["cloud_sql", "supabase"], + destination_policy_id=result.policy_id, + policy_created=result.policy_created, + mapping_created=result.mapping_created, + ), ) return result diff --git a/policyengine_api/services/policy_service.py b/policyengine_api/services/policy_service.py index 0a8358f72..33c5e829c 100644 --- a/policyengine_api/services/policy_service.py +++ b/policyengine_api/services/policy_service.py @@ -232,26 +232,6 @@ def _set_policy( session.flush() return policy, "Policy created", False - def _create_new_policy( - self, - session: Session, - country_id: str, - policy_json: dict, - policy_hash: str, - label: str | None, - api_version: str, - ) -> Policy: - policy = Policy( - country_id=country_id, - label=label, - policy_json=policy_json, - policy_hash=policy_hash, - api_version=api_version, - ) - session.add(policy) - session.flush() - return policy - def _get_unique_policy_with_label( self, session: Session, diff --git a/policyengine_api/services/report_output_service.py b/policyengine_api/services/report_output_service.py index 29bb28522..1ba5d74c6 100644 --- a/policyengine_api/services/report_output_service.py +++ b/policyengine_api/services/report_output_service.py @@ -12,7 +12,10 @@ from policyengine_api.data.orm import get_v1_session_factory from policyengine_api.data.v1_models import ReportOutput, ReportOutputRun, Simulation from policyengine_api.json_types import JSONObject, JSONValue -from policyengine_api.services.report_run_service import ReportRunService +from policyengine_api.services.report_run_service import ( + NewReportOutputRun, + ReportRunService, +) from policyengine_api.services.report_spec_service import ( ECONOMY_REPORT_KINDS, ReportSpec, @@ -471,14 +474,16 @@ def _ensure_report_output_dual_write_state( self.report_run_service.create_report_output_run( session, report_output_id, - status=report_output.status, - output=report_output.output, - error_message=report_output.error_message, - trigger_type="initial", - report_spec_snapshot=( - report_spec.model_dump() if report_spec else None + NewReportOutputRun( + status=report_output.status, + output=report_output.output, + error_message=report_output.error_message, + trigger_type="initial", + report_spec_snapshot=( + report_spec.model_dump() if report_spec else None + ), + version_manifest=manifest, ), - version_manifest=manifest, ) runs = self._list_runs_descending(session, report_output_id) else: diff --git a/policyengine_api/services/report_run_service.py b/policyengine_api/services/report_run_service.py index be1285dec..bbf17e066 100644 --- a/policyengine_api/services/report_run_service.py +++ b/policyengine_api/services/report_run_service.py @@ -1,6 +1,7 @@ """Run history for v1 report outputs, one numbered row per calculation attempt.""" import uuid +from dataclasses import dataclass from datetime import UTC, datetime from typing import Any @@ -22,6 +23,24 @@ ) +@dataclass(frozen=True) +class NewReportOutputRun: + """The fields of a report-output run to append; omitted fields take these defaults. + + ``version_manifest`` supplies the run's version fields, and ``run_id`` a fixed + ID instead of a new UUID. + """ + + status: str = "pending" + trigger_type: str = "initial" + output: dict[str, Any] | list[Any] | None = None + error_message: str | None = None + source_run_id: str | None = None + report_spec_snapshot: dict[str, Any] | None = None + version_manifest: dict[str, str | None] | None = None + run_id: str | None = None + + class ReportRunService: """Report-run operations performed through a caller-owned ORM Session.""" @@ -37,21 +56,18 @@ def create_report_output_run( self, session: Session, report_output_id: int, - status: str = "pending", - trigger_type: str = "initial", - output: dict[str, Any] | list[Any] | None = None, - error_message: str | None = None, - source_run_id: str | None = None, - report_spec_snapshot: dict[str, Any] | None = None, - version_manifest: dict[str, str | None] | None = None, - run_id: str | None = None, + new_run: NewReportOutputRun | None = None, ) -> ReportOutputRun: """Append the next numbered run to a report output and return it. + ``new_run`` gives the run's fields; without it the run takes the defaults. The parent row is locked while the sequence number is chosen, and timestamps are - set to now according to ``status``. Raise ValueError if the report output does - not exist. The caller owns the transaction. + set to now according to its ``status``. Raise ValueError if the report output + does not exist. The caller owns the transaction. """ + if new_run is None: + new_run = NewReportOutputRun() + status = new_run.status parent = session.scalar( select(ReportOutput) .where(ReportOutput.id == report_output_id) @@ -71,23 +87,23 @@ def create_report_output_run( now = datetime.now(UTC).replace(microsecond=0, tzinfo=None) values = { "status": status, - "output": output, - "error_message": error_message, - "trigger_type": trigger_type, + "output": new_run.output, + "error_message": new_run.error_message, + "trigger_type": new_run.trigger_type, "requested_at": now, "started_at": now if status in {"running", "complete", "error"} else None, "finished_at": now if status in {"complete", "error"} else None, - "source_run_id": source_run_id, - "report_spec_snapshot_json": report_spec_snapshot, + "source_run_id": new_run.source_run_id, + "report_spec_snapshot_json": new_run.report_spec_snapshot, } values.update( { - field: (version_manifest or {}).get(field) + field: (new_run.version_manifest or {}).get(field) for field in REPORT_RUN_VERSION_FIELDS } ) run = ReportOutputRun( - id=run_id or str(uuid.uuid4()), + id=new_run.run_id or str(uuid.uuid4()), report_output_id=report_output_id, run_sequence=sequence, **values, diff --git a/policyengine_api/services/report_spec_service.py b/policyengine_api/services/report_spec_service.py index adfa1e9cf..b553bf868 100644 --- a/policyengine_api/services/report_spec_service.py +++ b/policyengine_api/services/report_spec_service.py @@ -1,6 +1,7 @@ """Explicit specifications of what each v1 report output calculates.""" import json +from dataclasses import dataclass from typing import Any, Literal from pydantic import BaseModel, Field @@ -15,6 +16,18 @@ ECONOMY_REPORT_KINDS = {"economy_single", "economy_comparison"} +@dataclass(frozen=True) +class EconomyReportSettings: + """How an economy report is calculated: its dataset, target and options. + + A household report ignores them. + """ + + dataset: str = "default" + target: Literal["general", "cliff"] = "general" + options: dict[str, Any] | None = None + + class ReportSimulationInput(BaseModel): """The population and policy of one simulation in a household report.""" @@ -160,15 +173,17 @@ def build_report_spec( report_output: ReportOutput, simulation_1: Simulation, simulation_2: Simulation | None = None, - dataset: str = "default", - target: Literal["general", "cliff"] = "general", - options: dict[str, Any] | None = None, + economy_settings: EconomyReportSettings | None = None, ) -> ReportSpec: """Build the spec implied by a report output and its linked simulations. - Raise ValueError if the simulations do not match the report's simulation IDs or - country, or if a comparison's simulations cover different households or regions. + ``economy_settings`` sets an economy report's dataset, target and options; + without it they take their defaults. Raise ValueError if the simulations do + not match the report's simulation IDs or country, or if a comparison's + simulations cover different households or regions. """ + if economy_settings is None: + economy_settings = EconomyReportSettings() self._validate_report_simulation_linkage( report_output, simulation_1, simulation_2 ) @@ -225,9 +240,9 @@ def build_report_spec( reform_policy_id=( simulation_2.policy_id if simulation_2 else simulation_1.policy_id ), - dataset=dataset, - target=target, - options=options or {}, + dataset=economy_settings.dataset, + target=economy_settings.target, + options=economy_settings.options or {}, ) def _validate_report_spec_matches_model( @@ -243,21 +258,13 @@ def _validate_report_spec_matches_model( report_output, simulation_1, simulation_2, - dataset=( - report_spec.dataset - if isinstance(report_spec, EconomyReportSpec) - else "default" - ), - target=( - report_spec.target - if isinstance(report_spec, EconomyReportSpec) - else "general" - ), - options=( - report_spec.options - if isinstance(report_spec, EconomyReportSpec) - else None - ), + EconomyReportSettings( + dataset=report_spec.dataset, + target=report_spec.target, + options=report_spec.options, + ) + if isinstance(report_spec, EconomyReportSpec) + else EconomyReportSettings(), ) if report_spec != expected: msg = "Report spec must match the linked report and simulations" diff --git a/policyengine_api/services/simulation_run_service.py b/policyengine_api/services/simulation_run_service.py index b54a3a890..fa930067d 100644 --- a/policyengine_api/services/simulation_run_service.py +++ b/policyengine_api/services/simulation_run_service.py @@ -1,6 +1,7 @@ """Run history for v1 simulations, one numbered row per calculation attempt.""" import uuid +from dataclasses import dataclass from typing import Any from sqlalchemy import func, select @@ -17,6 +18,26 @@ ) +@dataclass(frozen=True) +class NewSimulationRun: + """The fields of a simulation run to append; omitted fields take these defaults. + + ``version_manifest`` supplies the run's version fields, and ``run_id`` a fixed + ID instead of a new UUID. + """ + + report_output_run_id: str | None = None + input_position: int | None = None + status: str = "pending" + trigger_type: str = "initial" + output: dict[str, Any] | list[Any] | str | None = None + error_message: str | None = None + source_run_id: str | None = None + simulation_spec_snapshot: dict[str, Any] | str | None = None + version_manifest: dict[str, str | None] | None = None + run_id: str | None = None + + class SimulationRunService: """Simulation-run operations performed through a caller-owned ORM session.""" @@ -24,23 +45,17 @@ def create_simulation_run( self, session: Session, simulation_id: int, - report_output_run_id: str | None = None, - input_position: int | None = None, - status: str = "pending", - trigger_type: str = "initial", - output: dict[str, Any] | list[Any] | str | None = None, - error_message: str | None = None, - source_run_id: str | None = None, - simulation_spec_snapshot: dict[str, Any] | str | None = None, - version_manifest: dict[str, str | None] | None = None, - run_id: str | None = None, + new_run: NewSimulationRun | None = None, ) -> SimulationRun: """Append the next numbered run to a simulation and return it. + ``new_run`` gives the run's fields; without it the run takes the defaults. The parent row is locked while the sequence number is chosen; timestamps start unset. Raise ValueError if the simulation does not exist. The caller owns the transaction. """ + if new_run is None: + new_run = NewSimulationRun() parent = session.scalar( select(Simulation).where(Simulation.id == simulation_id).with_for_update() ) @@ -56,26 +71,26 @@ def create_simulation_run( or 0 ) + 1 values = { - "report_output_run_id": report_output_run_id, - "input_position": input_position, - "status": status, - "output": output, - "error_message": error_message, - "trigger_type": trigger_type, + "report_output_run_id": new_run.report_output_run_id, + "input_position": new_run.input_position, + "status": new_run.status, + "output": new_run.output, + "error_message": new_run.error_message, + "trigger_type": new_run.trigger_type, "requested_at": None, "started_at": None, "finished_at": None, - "source_run_id": source_run_id, - "simulation_spec_snapshot_json": simulation_spec_snapshot, + "source_run_id": new_run.source_run_id, + "simulation_spec_snapshot_json": new_run.simulation_spec_snapshot, } values.update( { - field: (version_manifest or {}).get(field) + field: (new_run.version_manifest or {}).get(field) for field in SIMULATION_RUN_VERSION_FIELDS } ) run = SimulationRun( - id=run_id or str(uuid.uuid4()), + id=new_run.run_id or str(uuid.uuid4()), simulation_id=simulation_id, run_sequence=sequence, **values, diff --git a/policyengine_api/services/simulation_service.py b/policyengine_api/services/simulation_service.py index eba3b501c..c6236b496 100644 --- a/policyengine_api/services/simulation_service.py +++ b/policyengine_api/services/simulation_service.py @@ -157,15 +157,14 @@ def _ensure_simulation_dual_write_state( return simulation @staticmethod - def _find_existing_simulation( + def _lock_existing_simulation( session: Session, country_id: str, population_id: str, population_type: str, policy_id: int, - *, - for_update: bool = False, ) -> Simulation | None: + """Return the simulation for this identity, locking it for update, or None.""" exact_identity = Simulation.population_id == population_id statement = ( select(Simulation) @@ -181,9 +180,8 @@ def _find_existing_simulation( Simulation.policy_id == policy_id, ) .order_by(exact_identity.desc(), Simulation.id.desc()) + .with_for_update() ) - if for_update: - statement = statement.with_for_update() return session.scalar(statement) def _create_simulation( @@ -238,13 +236,12 @@ def get_or_create_simulation( # locking read of `simulations` below, which is what they contend on; # the household row was never the exclusion for a non-numeric id. with self._sessions.begin() as session: - simulation = self._find_existing_simulation( + simulation = self._lock_existing_simulation( session, country_id, population_id, population_type, policy_id, - for_update=True, ) created = simulation is None if simulation is None: diff --git a/policyengine_api/services/user_policy_mirroring.py b/policyengine_api/services/user_policy_mirroring.py index 20bd71ff7..8b2243c34 100644 --- a/policyengine_api/services/user_policy_mirroring.py +++ b/policyengine_api/services/user_policy_mirroring.py @@ -4,6 +4,7 @@ import contextlib import time +from dataclasses import dataclass from typing import TYPE_CHECKING, Protocol from sqlalchemy.exc import SQLAlchemyError @@ -101,18 +102,48 @@ def _failure_category(error: Exception) -> str: return "unexpected" +@dataclass(frozen=True) +class UserPolicyMirrorCollaborators: + """The services a saved-policy mirror uses; each one left None takes its default. + + ``event_service`` reads and completes the retained v1 events, + ``reform_snapshot_loader`` loads a saved policy's reform policy, and + ``mirror_factory`` builds the v2 mirror. + """ + + event_service: UserPolicyService | None = None + reform_snapshot_loader: Callable[[str, int], LegacyPolicySnapshot | None] | None = ( + None + ) + mirror_factory: Callable[[], LegacyUserPolicyMirror] | None = None + + +@dataclass(frozen=True) +class _MirroredRevision: + """The saved-policy revision one mirror metric record reports on.""" + + country_id: str + legacy_user_policy_id: int + source_revision: int | None + requested_through_revision: int + started_at: float + + +@dataclass(frozen=True) +class _MirrorOutcome: + """How one immediate saved-policy mirror ended, as its metric record reports it.""" + + outcome: str + actual_write_sources: list[str] + result: LegacyUserPolicyPersistenceResult | None = None + failure_category: str | None = None + + def _log_mirror_operation( - *, - country_id: str, - legacy_user_policy_id: int, - source_revision: int | None, - requested_through_revision: int, - started_at: float, - outcome: str, - actual_write_sources: list[str], - result: LegacyUserPolicyPersistenceResult | None = None, - failure_category: str | None = None, + revision: _MirroredRevision, + mirror_outcome: _MirrorOutcome, ) -> None: + result = mirror_outcome.result # Metric logging must not change the mirror outcome. with contextlib.suppress(Exception): logger.log_struct( @@ -122,19 +153,19 @@ def _log_mirror_operation( "metric_value": 1, "configured_write_source": "dual_write", "attempted_write_sources": ["cloud_sql", "supabase"], - "actual_write_sources": actual_write_sources, - "country_id": country_id, - "legacy_user_policy_id": legacy_user_policy_id, - "source_revision": source_revision, - "requested_through_revision": requested_through_revision, + "actual_write_sources": mirror_outcome.actual_write_sources, + "country_id": revision.country_id, + "legacy_user_policy_id": revision.legacy_user_policy_id, + "source_revision": revision.source_revision, + "requested_through_revision": revision.requested_through_revision, "destination_association_id": ( str(result.association_id) if result is not None else None ), "destination_policy_id": ( str(result.policy_id) if result is not None else None ), - "outcome": outcome, - "failure_category": failure_category, + "outcome": mirror_outcome.outcome, + "failure_category": mirror_outcome.failure_category, "association_created": ( result.association_created if result is not None else None ), @@ -144,9 +175,11 @@ def _log_mirror_operation( "mapping_created": result.mapping_created if result is not None else None, - "duration_ms": round((time.perf_counter() - started_at) * 1000, 3), + "duration_ms": round( + (time.perf_counter() - revision.started_at) * 1000, 3 + ), }, - severity="INFO" if outcome == "ok" else "ERROR", + severity="INFO" if mirror_outcome.outcome == "ok" else "ERROR", ) @@ -186,27 +219,35 @@ def mirror_user_policy_after_commit( ) except Exception as error: _log_mirror_operation( - country_id=snapshot.country_id, - legacy_user_policy_id=snapshot.legacy_user_policy_id, - source_revision=source_revision, - requested_through_revision=source_revision, - started_at=started_at, - outcome="error", - actual_write_sources=["cloud_sql"], - failure_category=_failure_category(error), + _MirroredRevision( + country_id=snapshot.country_id, + legacy_user_policy_id=snapshot.legacy_user_policy_id, + source_revision=source_revision, + requested_through_revision=source_revision, + started_at=started_at, + ), + _MirrorOutcome( + outcome="error", + actual_write_sources=["cloud_sql"], + failure_category=_failure_category(error), + ), ) msg = "The committed v1 saved policy could not be mirrored to v2" raise UserPolicyMirrorUnavailableError(msg) from error _log_mirror_operation( - country_id=snapshot.country_id, - legacy_user_policy_id=snapshot.legacy_user_policy_id, - source_revision=source_revision, - requested_through_revision=source_revision, - started_at=started_at, - outcome="ok", - actual_write_sources=["cloud_sql", "supabase"], - result=result, + _MirroredRevision( + country_id=snapshot.country_id, + legacy_user_policy_id=snapshot.legacy_user_policy_id, + source_revision=source_revision, + requested_through_revision=source_revision, + started_at=started_at, + ), + _MirrorOutcome( + outcome="ok", + actual_write_sources=["cloud_sql", "supabase"], + result=result, + ), ) return result @@ -216,10 +257,7 @@ def mirror_pending_user_policy_events_after_commit( legacy_user_policy_id: int, *, through_revision: int, - event_service: UserPolicyService | None = None, - reform_snapshot_loader: Callable[[str, int], LegacyPolicySnapshot | None] - | None = None, - mirror_factory: Callable[[], LegacyUserPolicyMirror] | None = None, + collaborators: UserPolicyMirrorCollaborators | None = None, ) -> LegacyUserPolicyPersistenceResult: """Copy this saved policy's retained events into v2, up to the request's revision. @@ -227,10 +265,13 @@ def mirror_pending_user_policy_events_after_commit( ``through_revision``. Raise UserPolicyMirrorUnavailableError, after logging the failure, if an event cannot be copied. """ - selected_event_service = event_service or UserPolicyService() + if collaborators is None: + collaborators = UserPolicyMirrorCollaborators() + selected_event_service = collaborators.event_service or UserPolicyService() selected_reform_loader = ( - reform_snapshot_loader or PolicyService().get_policy_snapshot + collaborators.reform_snapshot_loader or PolicyService().get_policy_snapshot ) + mirror_factory = collaborators.mirror_factory request_started_at = time.perf_counter() active_event: PendingUserPolicyMirrorEvent | None = None event_started_at: dict[int, float] = {} @@ -267,14 +308,18 @@ def after_processed_commit( nonlocal active_event _log_mirror_operation( - country_id=event.snapshot.country_id, - legacy_user_policy_id=event.snapshot.legacy_user_policy_id, - source_revision=event.source_revision, - requested_through_revision=through_revision, - started_at=event_started_at.pop(event.event_id), - outcome="ok", - actual_write_sources=["cloud_sql", "supabase"], - result=result, + _MirroredRevision( + country_id=event.snapshot.country_id, + legacy_user_policy_id=event.snapshot.legacy_user_policy_id, + source_revision=event.source_revision, + requested_through_revision=through_revision, + started_at=event_started_at.pop(event.event_id), + ), + _MirrorOutcome( + outcome="ok", + actual_write_sources=["cloud_sql", "supabase"], + result=result, + ), ) destination_results.pop(event.event_id, None) active_event = None @@ -296,25 +341,31 @@ def after_processed_commit( ): failure = error.__cause__ _log_mirror_operation( - country_id=(event.snapshot.country_id if event is not None else country_id), - legacy_user_policy_id=( - event.snapshot.legacy_user_policy_id - if event is not None - else legacy_user_policy_id - ), - source_revision=event.source_revision if event is not None else None, - requested_through_revision=through_revision, - started_at=( - event_started_at.get(event.event_id, request_started_at) - if event is not None - else request_started_at + _MirroredRevision( + country_id=( + event.snapshot.country_id if event is not None else country_id + ), + legacy_user_policy_id=( + event.snapshot.legacy_user_policy_id + if event is not None + else legacy_user_policy_id + ), + source_revision=event.source_revision if event is not None else None, + requested_through_revision=through_revision, + started_at=( + event_started_at.get(event.event_id, request_started_at) + if event is not None + else request_started_at + ), ), - outcome="error", - actual_write_sources=( - ["cloud_sql", "supabase"] if result is not None else ["cloud_sql"] + _MirrorOutcome( + outcome="error", + actual_write_sources=( + ["cloud_sql", "supabase"] if result is not None else ["cloud_sql"] + ), + result=result, + failure_category=_failure_category(failure), ), - result=result, - failure_category=_failure_category(failure), ) msg = "The committed v1 saved policy event could not be mirrored to v2" raise UserPolicyMirrorUnavailableError(msg) from error diff --git a/policyengine_api/services/user_policy_service.py b/policyengine_api/services/user_policy_service.py index 0fa1a4473..648b48043 100644 --- a/policyengine_api/services/user_policy_service.py +++ b/policyengine_api/services/user_policy_service.py @@ -137,6 +137,41 @@ class UserPolicyUpdateResult: mirror_revision: int | None = None +def _decoded_changed_fields(changed_fields: object) -> list[str]: + """Return a mirror event's changed fields once they are sorted, unique and mutable. + + Raise UserPolicyMirrorEventIntegrityError for anything else. + """ + if not isinstance(changed_fields, list) or not all( + isinstance(field, str) for field in changed_fields + ): + msg = "saved-policy mirror event changed fields are invalid" + raise UserPolicyMirrorEventIntegrityError(msg) + if ( + changed_fields != sorted(set(changed_fields)) + or not set(changed_fields) <= USER_POLICY_MUTABLE_FIELDS + ): + msg = "saved-policy mirror event changed fields are unsupported" + raise UserPolicyMirrorEventIntegrityError(msg) + return changed_fields + + +def _require_event_type_matches_changes( + event_type: str, + changed_fields: list[str], +) -> None: + """Require a create event with no changed fields or an update event with some.""" + if event_type not in {"create", "update"}: + msg = "saved-policy mirror event type is unsupported" + raise UserPolicyMirrorEventIntegrityError(msg) + if event_type == "create" and changed_fields: + msg = "saved-policy create mirror event contains changed fields" + raise UserPolicyMirrorEventIntegrityError(msg) + if event_type == "update" and not changed_fields: + msg = "saved-policy update mirror event has no changed fields" + raise UserPolicyMirrorEventIntegrityError(msg) + + class UserPolicyService: """Saved-policy operations with service-owned ORM transactions.""" @@ -230,18 +265,7 @@ def _decode_mirror_event( }: msg = "saved-policy mirror event payload has an invalid shape" raise UserPolicyMirrorEventIntegrityError(msg) - changed_fields = payload["changed_fields"] - if not isinstance(changed_fields, list) or not all( - isinstance(field, str) for field in changed_fields - ): - msg = "saved-policy mirror event changed fields are invalid" - raise UserPolicyMirrorEventIntegrityError(msg) - if ( - changed_fields != sorted(set(changed_fields)) - or not set(changed_fields) <= USER_POLICY_MUTABLE_FIELDS - ): - msg = "saved-policy mirror event changed fields are unsupported" - raise UserPolicyMirrorEventIntegrityError(msg) + changed_fields = _decoded_changed_fields(payload["changed_fields"]) try: snapshot = LegacyUserPolicySnapshot.model_validate(payload["snapshot"]) except (TypeError, ValueError) as error: @@ -257,15 +281,7 @@ def _decode_mirror_event( if fingerprint != event.source_fingerprint_sha256: msg = "saved-policy mirror event fingerprint conflicts with its payload" raise UserPolicyMirrorEventIntegrityError(msg) - if event.event_type not in {"create", "update"}: - msg = "saved-policy mirror event type is unsupported" - raise UserPolicyMirrorEventIntegrityError(msg) - if event.event_type == "create" and changed_fields: - msg = "saved-policy create mirror event contains changed fields" - raise UserPolicyMirrorEventIntegrityError(msg) - if event.event_type == "update" and not changed_fields: - msg = "saved-policy update mirror event has no changed fields" - raise UserPolicyMirrorEventIntegrityError(msg) + _require_event_type_matches_changes(event.event_type, changed_fields) return PendingUserPolicyMirrorEvent( event_id=event.id, source_revision=event.source_revision, diff --git a/policyengine_api/services/v2/comparison_runs/types.py b/policyengine_api/services/v2/comparison_runs/types.py index 27ef6d839..1b130d8a4 100644 --- a/policyengine_api/services/v2/comparison_runs/types.py +++ b/policyengine_api/services/v2/comparison_runs/types.py @@ -75,6 +75,140 @@ class ResultComparisonStatus(StrEnum): FAILED = "failed" +INCOMPLETE_COMPARISON_STATUSES = frozenset( + { + ResultComparisonStatus.NOT_REQUESTED, + ResultComparisonStatus.PENDING, + ResultComparisonStatus.RUNNING, + } +) +COMPLETED_COMPARISON_STATUSES = frozenset( + { + ResultComparisonStatus.MATCHED, + ResultComparisonStatus.DIFFERENT, + } +) +UNSUCCESSFUL_FINAL_REPORT_STATUSES = frozenset( + { + ComparisonRunLifecycleStatus.FAILED, + ComparisonRunLifecycleStatus.SKIPPED, + } +) + + +def _all_supplied(values: tuple[object, ...]) -> bool: + return all(value is not None for value in values) + + +def _any_supplied(values: tuple[object, ...]) -> bool: + return any(value is not None for value in values) + + +def _validate_report_lifecycle(record: ComparisonReportRecord) -> None: + """Require the aggregate output fields to fit the report's lifecycle status.""" + output_fields = ( + record.aggregate_output_uri, + record.aggregate_output_sha256, + record.aggregate_schema_version, + ) + if _any_supplied(output_fields) and not _all_supplied(output_fields): + msg = "aggregate output fields must be supplied together" + raise ValueError(msg) + if record.status is ComparisonRunLifecycleStatus.SUCCEEDED: + _validate_successful_report(record, output_fields) + if ( + record.status in UNSUCCESSFUL_FINAL_REPORT_STATUSES + and record.error_code is None + ): + msg = "a failed or skipped report requires an error_code" + raise ValueError(msg) + + +def _validate_successful_report( + record: ComparisonReportRecord, + output_fields: tuple[object, ...], +) -> None: + """Require successful aggregation, its output and completed_at.""" + if record.aggregation_status is not ComparisonRunAggregationStatus.SUCCEEDED: + msg = "a successful report requires successful aggregation" + raise ValueError(msg) + if not _all_supplied(output_fields): + msg = "a successful report requires an aggregate output" + raise ValueError(msg) + if record.completed_at is None: + msg = "a successful report requires completed_at" + raise ValueError(msg) + + +def _validate_comparison_state(record: ComparisonReportRecord) -> None: + """Require the result comparison's fields to fit its comparison_status.""" + output_fields = ( + record.comparison_output_uri, + record.comparison_output_sha256, + record.comparison_schema_version, + ) + if record.comparison_status in INCOMPLETE_COMPARISON_STATUSES: + _validate_incomplete_comparison(record, output_fields) + elif record.comparison_status in COMPLETED_COMPARISON_STATUSES: + _validate_completed_comparison(record, output_fields) + else: + _validate_failed_comparison(record, output_fields) + + +def _validate_incomplete_comparison( + record: ComparisonReportRecord, + output_fields: tuple[object, ...], +) -> None: + """Reject any output, completion time or error on an unfinished comparison.""" + if _any_supplied(output_fields): + msg = "an incomplete comparison cannot reference an output artifact" + raise ValueError(msg) + if record.comparison_completed_at is not None: + msg = "an incomplete comparison cannot have completed_at" + raise ValueError(msg) + if record.comparison_error_code is not None: + msg = "an incomplete comparison cannot have an error_code" + raise ValueError(msg) + if record.comparison_error_summary is not None: + msg = "an incomplete comparison cannot have an error_summary" + raise ValueError(msg) + + +def _validate_completed_comparison( + record: ComparisonReportRecord, + output_fields: tuple[object, ...], +) -> None: + """Require a matched or different comparison's output and completion time.""" + if not _all_supplied(output_fields): + msg = "a completed comparison requires an output artifact" + raise ValueError(msg) + if record.comparison_completed_at is None: + msg = "a completed comparison requires completed_at" + raise ValueError(msg) + if record.comparison_error_code is not None: + msg = "a completed comparison cannot have an error_code" + raise ValueError(msg) + if record.comparison_error_summary is not None: + msg = "a completed comparison cannot have an error_summary" + raise ValueError(msg) + + +def _validate_failed_comparison( + record: ComparisonReportRecord, + output_fields: tuple[object, ...], +) -> None: + """Require a failed comparison's completion time and error, and no output.""" + if _any_supplied(output_fields): + msg = "a failed comparison cannot reference an output artifact" + raise ValueError(msg) + if record.comparison_completed_at is None: + msg = "a failed comparison requires completed_at" + raise ValueError(msg) + if record.comparison_error_code is None: + msg = "a failed comparison requires an error_code" + raise ValueError(msg) + + class ComparisonReportRecord(StrictContractModel): """Complete producer/consumer contract for one temporary parent row.""" @@ -129,84 +263,8 @@ def validate_state(self) -> ComparisonReportRecord: comparison_status. Retention must be timezone-aware and 1-30 days long. """ _validate_retention(self.created_at, self.retention_expires_at) - output_fields = ( - self.aggregate_output_uri, - self.aggregate_output_sha256, - self.aggregate_schema_version, - ) - if any(value is not None for value in output_fields) and not all( - value is not None for value in output_fields - ): - msg = "aggregate output fields must be supplied together" - raise ValueError(msg) - if self.status is ComparisonRunLifecycleStatus.SUCCEEDED: - if self.aggregation_status is not ComparisonRunAggregationStatus.SUCCEEDED: - msg = "a successful report requires successful aggregation" - raise ValueError(msg) - if not all(value is not None for value in output_fields): - msg = "a successful report requires an aggregate output" - raise ValueError(msg) - if self.completed_at is None: - msg = "a successful report requires completed_at" - raise ValueError(msg) - if ( - self.status - in { - ComparisonRunLifecycleStatus.FAILED, - ComparisonRunLifecycleStatus.SKIPPED, - } - and self.error_code is None - ): - msg = "a failed or skipped report requires an error_code" - raise ValueError(msg) - comparison_output_fields = ( - self.comparison_output_uri, - self.comparison_output_sha256, - self.comparison_schema_version, - ) - if self.comparison_status in { - ResultComparisonStatus.NOT_REQUESTED, - ResultComparisonStatus.PENDING, - ResultComparisonStatus.RUNNING, - }: - if any(value is not None for value in comparison_output_fields): - msg = "an incomplete comparison cannot reference an output artifact" - raise ValueError(msg) - if self.comparison_completed_at is not None: - msg = "an incomplete comparison cannot have completed_at" - raise ValueError(msg) - if self.comparison_error_code is not None: - msg = "an incomplete comparison cannot have an error_code" - raise ValueError(msg) - if self.comparison_error_summary is not None: - msg = "an incomplete comparison cannot have an error_summary" - raise ValueError(msg) - elif self.comparison_status in { - ResultComparisonStatus.MATCHED, - ResultComparisonStatus.DIFFERENT, - }: - if not all(value is not None for value in comparison_output_fields): - msg = "a completed comparison requires an output artifact" - raise ValueError(msg) - if self.comparison_completed_at is None: - msg = "a completed comparison requires completed_at" - raise ValueError(msg) - if self.comparison_error_code is not None: - msg = "a completed comparison cannot have an error_code" - raise ValueError(msg) - if self.comparison_error_summary is not None: - msg = "a completed comparison cannot have an error_summary" - raise ValueError(msg) - else: - if any(value is not None for value in comparison_output_fields): - msg = "a failed comparison cannot reference an output artifact" - raise ValueError(msg) - if self.comparison_completed_at is None: - msg = "a failed comparison requires completed_at" - raise ValueError(msg) - if self.comparison_error_code is None: - msg = "a failed comparison requires an error_code" - raise ValueError(msg) + _validate_report_lifecycle(self) + _validate_comparison_state(self) return self diff --git a/policyengine_api/services/v2/households/database_connectors/creates.py b/policyengine_api/services/v2/households/database_connectors/creates.py index 721a0096e..15df06749 100644 --- a/policyengine_api/services/v2/households/database_connectors/creates.py +++ b/policyengine_api/services/v2/households/database_connectors/creates.py @@ -11,7 +11,10 @@ from policyengine_api.data.v2.models import Household, LegacyHouseholdMapping if TYPE_CHECKING: - from policyengine_api.services.v2.households.types import HouseholdCreationInput + from policyengine_api.services.v2.households.types import ( + HouseholdCreationInput, + LegacyHouseholdSnapshot, + ) def create_household( @@ -48,27 +51,26 @@ def create_household( def create_legacy_household_mapping( session: Session, *, - country_id: str, - legacy_household_id: int, + snapshot: LegacyHouseholdSnapshot, household_id: UUID, - source_api_version: str, fingerprint_version: int, fingerprint_sha256: str, ) -> UUID | None: """Insert the mapping from a v1 household id to its v2 household. - Return the new mapping id, or None when a mapping for this country and v1 id - already exists (for example, a concurrent copy won the race). + The snapshot supplies the v1 country, id and source API version. Return the + new mapping id, or None when a mapping for this country and v1 id already + exists (for example, a concurrent copy won the race). """ mapping_id = uuid4() statement = ( insert(LegacyHouseholdMapping) .values( id=mapping_id, - country_id=country_id, - legacy_household_id=legacy_household_id, + country_id=snapshot.country_id, + legacy_household_id=snapshot.legacy_household_id, household_id=household_id, - source_api_version=source_api_version, + source_api_version=snapshot.api_version, fingerprint_version=fingerprint_version, fingerprint_sha256=fingerprint_sha256, ) diff --git a/policyengine_api/services/v2/households/services.py b/policyengine_api/services/v2/households/services.py index 0ad26681b..d49b4b368 100644 --- a/policyengine_api/services/v2/households/services.py +++ b/policyengine_api/services/v2/households/services.py @@ -188,10 +188,8 @@ def mirror_legacy_household_in_session( ) mapping_id = create_legacy_household_mapping( session, - country_id=snapshot.country_id, - legacy_household_id=snapshot.legacy_household_id, + snapshot=snapshot, household_id=result.household_id, - source_api_version=snapshot.api_version, fingerprint_version=LEGACY_HOUSEHOLD_FINGERPRINT_VERSION, fingerprint_sha256=fingerprint, ) diff --git a/policyengine_api/services/v2/households/transformations.py b/policyengine_api/services/v2/households/transformations.py index 6edf56876..f9db1ba8d 100644 --- a/policyengine_api/services/v2/households/transformations.py +++ b/policyengine_api/services/v2/households/transformations.py @@ -189,6 +189,22 @@ def _derive_default_year(values: list[object]) -> int | None: raise LegacyHouseholdTranslationError(msg) +class _LegacyTranslation: + """What translating one v1 household builds up as its groups are read. + + Holds the people's stable ids, each person's group memberships so far, and + every source value in reading order, from which the default year is derived. + """ + + def __init__(self, people_source: dict[str, Any]) -> None: + """Start a translation of the named people in ``people_source``.""" + self.person_ids = _stable_ids("person", list(people_source)) + self.person_memberships: dict[str, dict[str, str]] = { + name: {} for name in people_source + } + self.all_values: list[object] = [] + + def translate_legacy_household( snapshot: LegacyHouseholdSnapshot, ) -> HouseholdCreationInput: @@ -204,100 +220,158 @@ def translate_legacy_household( msg = "legacy household_json must be an object" raise TypeError(msg) collections = SUPPORTED_ENTITY_COLLECTIONS[snapshot.country_id] + _require_supported_legacy_collections(snapshot.country_id, source, collections) + people_source = source.get("people", {}) + if type(people_source) is not dict: + msg = "legacy people must be an object" + raise LegacyHouseholdTranslationError(msg) + translation = _LegacyTranslation(people_source) + normalized_groups = { + collection: _translate_legacy_groups(source, collection, translation) + for collection in collections + } + people = _translate_legacy_people(people_source, translation) + + document: dict[str, Any] = {"people": people, **normalized_groups} + if "spm" in source: + document["spm"] = source["spm"] + try: + normalized = normalize_household_document(snapshot.country_id, document) + except ValueError as error: + raise LegacyHouseholdTranslationError(str(error)) from error + return HouseholdCreationInput( + country_id=snapshot.country_id, + default_year=_derive_default_year(translation.all_values), + household_data=normalized, + ) + + +def _require_supported_legacy_collections( + country_id: str, + source: dict[str, Any], + collections: tuple[str, ...], +) -> None: + """Reject a v1 document with collections the country does not support.""" allowed_source = { "people", *(LEGACY_COLLECTION_NAMES[item] for item in collections), } - if snapshot.country_id == "us": + if country_id == "us": allowed_source.add("spm") unsupported = sorted(set(source) - allowed_source) if unsupported: msg = f"legacy household contains unsupported collections: {unsupported}" raise LegacyHouseholdTranslationError(msg) - people_source = source.get("people", {}) - if type(people_source) is not dict: - msg = "legacy people must be an object" + + +def _translate_legacy_groups( + source: dict[str, Any], + collection: str, + translation: _LegacyTranslation, +) -> list[dict[str, Any]]: + """Translate one v1 collection's named groups into v2 records, in name order.""" + source_name = LEGACY_COLLECTION_NAMES[collection] + source_groups = source.get(source_name, {}) + if type(source_groups) is not dict: + msg = f"legacy {source_name} must be an object" raise LegacyHouseholdTranslationError(msg) - person_ids = _stable_ids("person", list(people_source)) - all_values: list[object] = [] - normalized_groups: dict[str, list[dict[str, Any]]] = {} - person_memberships: dict[str, dict[str, str]] = {name: {} for name in people_source} - - for collection in collections: - source_name = LEGACY_COLLECTION_NAMES[collection] - source_groups = source.get(source_name, {}) - if type(source_groups) is not dict: - msg = f"legacy {source_name} must be an object" + group_ids = _stable_ids(collection, list(source_groups)) + group_records: list[dict[str, Any]] = [] + for group_name in sorted(source_groups): + group = source_groups[group_name] + if type(group) is not dict: + msg = f"legacy {source_name} records must be objects" raise LegacyHouseholdTranslationError(msg) - group_ids = _stable_ids(collection, list(source_groups)) - group_records: list[dict[str, Any]] = [] - for group_name in sorted(source_groups): - group = source_groups[group_name] - if type(group) is not dict: - msg = f"legacy {source_name} records must be objects" - raise LegacyHouseholdTranslationError(msg) - values: dict[str, Any] = {} - for field, value in group.items(): - if field != "members": - values[field] = value - all_values.append(value) - continue - if type(value) is not list or not all( - isinstance(item, str) and item in person_ids for item in value - ): - msg = f"legacy {source_name} members must reference named people" - raise LegacyHouseholdTranslationError(msg) - for raw_person_name in value: - if not isinstance(raw_person_name, str): - msg = f"legacy {source_name} member names must be strings" - raise TypeError(msg) - person_name = raw_person_name - memberships = person_memberships[person_name] - previous = memberships.get(collection) - if previous is not None and previous != group_ids[group_name]: - msg = ( - f"legacy person {person_name!r} belongs to multiple " - f"{source_name} records" - ) - raise LegacyHouseholdTranslationError(msg) - memberships[collection] = group_ids[group_name] - group_records.append( - { - "id": group_ids[group_name], - "source_name": group_name, - "values": values, - } + values = _translate_legacy_group( + group, + collection=collection, + group_id=group_ids[group_name], + translation=translation, + ) + group_records.append( + { + "id": group_ids[group_name], + "source_name": group_name, + "values": values, + } + ) + return group_records + + +def _translate_legacy_group( + group: dict[str, Any], + *, + collection: str, + group_id: str, + translation: _LegacyTranslation, +) -> dict[str, Any]: + """Return a v1 group's values, recording its members' memberships.""" + values: dict[str, Any] = {} + for field, value in group.items(): + if field != "members": + values[field] = value + translation.all_values.append(value) + continue + _record_legacy_memberships( + value, + collection=collection, + group_id=group_id, + translation=translation, + ) + return values + + +def _record_legacy_memberships( + members: object, + *, + collection: str, + group_id: str, + translation: _LegacyTranslation, +) -> None: + """Record that each named member belongs to ``group_id`` in ``collection``.""" + source_name = LEGACY_COLLECTION_NAMES[collection] + if type(members) is not list or not all( + isinstance(item, str) and item in translation.person_ids for item in members + ): + msg = f"legacy {source_name} members must reference named people" + raise LegacyHouseholdTranslationError(msg) + for raw_person_name in members: + if not isinstance(raw_person_name, str): + msg = f"legacy {source_name} member names must be strings" + raise TypeError(msg) + person_name = raw_person_name + memberships = translation.person_memberships[person_name] + previous = memberships.get(collection) + if previous is not None and previous != group_id: + msg = ( + f"legacy person {person_name!r} belongs to multiple " + f"{source_name} records" ) - normalized_groups[collection] = group_records + raise LegacyHouseholdTranslationError(msg) + memberships[collection] = group_id + +def _translate_legacy_people( + people_source: dict[str, Any], + translation: _LegacyTranslation, +) -> list[dict[str, Any]]: + """Translate the named v1 people into v2 person records, in name order.""" people: list[dict[str, Any]] = [] for person_name in sorted(people_source): person_values = people_source[person_name] if type(person_values) is not dict: msg = "legacy person records must be objects" raise LegacyHouseholdTranslationError(msg) - all_values.extend(person_values.values()) + translation.all_values.extend(person_values.values()) people.append( { - "id": person_ids[person_name], + "id": translation.person_ids[person_name], "source_name": person_name, "values": dict(person_values), - "memberships": person_memberships[person_name], + "memberships": translation.person_memberships[person_name], } ) - - document: dict[str, Any] = {"people": people, **normalized_groups} - if "spm" in source: - document["spm"] = source["spm"] - try: - normalized = normalize_household_document(snapshot.country_id, document) - except ValueError as error: - raise LegacyHouseholdTranslationError(str(error)) from error - return HouseholdCreationInput( - country_id=snapshot.country_id, - default_year=_derive_default_year(all_values), - household_data=normalized, - ) + return people def legacy_household_fingerprint(snapshot: LegacyHouseholdSnapshot) -> str: diff --git a/policyengine_api/services/v2/households/validators.py b/policyengine_api/services/v2/households/validators.py index a25a7c165..4ee58f0dd 100644 --- a/policyengine_api/services/v2/households/validators.py +++ b/policyengine_api/services/v2/households/validators.py @@ -15,10 +15,14 @@ MAXIMUM_HOUSEHOLD_ENTITIES = 1_000 MAXIMUM_HOUSEHOLD_RECORD_VALUES = 500 MAXIMUM_JSON_NESTING = 100 +MAXIMUM_IDENTIFIER_LENGTH = 255 SUPPORTED_ENTITY_COLLECTIONS = { "us": ("household", "family", "tax_unit", "spm_unit", "marital_unit"), "uk": ("household", "benunit"), } +PERSON_RECORD_FIELDS = frozenset({"id", "source_name", "values", "memberships"}) +ENTITY_RECORD_FIELDS = frozenset({"id", "source_name", "values"}) +REQUIRED_RECORD_FIELDS = frozenset({"id", "values"}) class HouseholdValidationError(ValueError): @@ -65,26 +69,47 @@ def require_json_value( if type(value) not in {list, dict}: msg = "value must contain only standards-compliant JSON types" raise ValueError(msg) - identity = id(value) + _require_json_container(value, depth=depth, containers=containers) + return value + + +def _require_json_container( + container: list[Any] | dict[Any, Any], + *, + depth: int, + containers: frozenset[int], +) -> None: + """Check a JSON list's items or object's members, rejecting reference cycles. + + ``containers`` holds the identities of the containers enclosing this one. + """ + identity = id(container) if identity in containers: msg = "JSON values must not contain reference cycles" raise ValueError(msg) nested_containers = containers | {identity} - if type(value) is list: - for item in value: + # The caller admits only exact lists and dicts, so isinstance is exact here. + if isinstance(container, list): + for item in container: require_json_value(item, depth=depth + 1, containers=nested_containers) - return value - for key, item in value.items(): + return + for key, item in container.items(): if type(key) is not str: msg = "JSON object keys must be strings" raise ValueError(msg) require_json_value(item, depth=depth + 1, containers=nested_containers) - return value def _require_identifier(value: object, *, field: str) -> str: - if not isinstance(value, str) or not value.strip() or len(value) > 255: - msg = f"{field} must be a non-empty string of at most 255 characters" + if ( + not isinstance(value, str) + or not value.strip() + or len(value) > MAXIMUM_IDENTIFIER_LENGTH + ): + msg = ( + f"{field} must be a non-empty string of at most " + f"{MAXIMUM_IDENTIFIER_LENGTH} characters" + ) raise HouseholdValidationError(msg) return value @@ -114,6 +139,32 @@ def normalize_household_document( document: dict[str, Any], ) -> dict[str, Any]: """Validate and normalize one country-specific household document.""" + collections = _require_supported_collections(country_id, document) + + normalized: dict[str, Any] = {} + if "spm" in document: + normalized["spm"] = _normalized_spm_selection(document["spm"]) + identifiers: dict[str, set[str]] = {} + for collection in ("people", *collections): + normalized_records, record_ids = _normalized_collection( + collection, document.get(collection, []) + ) + identifiers[collection] = record_ids + normalized[collection] = normalized_records + + _require_complete_memberships(normalized["people"], collections, identifiers) + return normalized + + +def _require_supported_collections( + country_id: str, + document: dict[str, Any], +) -> tuple[str, ...]: + """Return the country's entity collections once the document fits them. + + The document must be an object with people and only the collections (and, + for the US, the SPM selection) the country supports. + """ collections = SUPPORTED_ENTITY_COLLECTIONS.get(country_id) if collections is None: msg = f"unsupported country_id {country_id!r}" @@ -131,67 +182,84 @@ def normalize_household_document( if "people" not in document: msg = "household_data must contain people" raise HouseholdValidationError(msg) + return collections - normalized: dict[str, Any] = {} - if "spm" in document: - if type(document["spm"]) is not dict: - msg = "spm must be an object" - raise HouseholdValidationError(msg) - try: - SPMSelection.model_validate(document["spm"]) - except ValueError as error: - msg = "spm selection is invalid" - raise HouseholdValidationError(msg) from error - # Saved selections are immutable identity, not a request to recertify - # against whichever scientific bundle is currently installed. - normalized["spm"] = dict(document["spm"]) - identifiers: dict[str, set[str]] = {} - for collection in ("people", *collections): - records = document.get(collection, []) - if type(records) is not list: - msg = f"{collection} must be a list" - raise HouseholdValidationError(msg) - if len(records) > MAXIMUM_HOUSEHOLD_ENTITIES: - msg = ( - f"{collection} must contain at most " - f"{MAXIMUM_HOUSEHOLD_ENTITIES} records" - ) + +def _normalized_spm_selection(selection: object) -> dict[str, Any]: + """Return a copy of a structurally valid saved SPM selection.""" + if type(selection) is not dict: + msg = "spm must be an object" + raise HouseholdValidationError(msg) + try: + SPMSelection.model_validate(selection) + except ValueError as error: + msg = "spm selection is invalid" + raise HouseholdValidationError(msg) from error + # Saved selections are immutable identity, not a request to recertify + # against whichever scientific bundle is currently installed. + return dict(selection) + + +def _normalized_collection( + collection: str, + records: object, +) -> tuple[list[dict[str, Any]], set[str]]: + """Validate one collection's records; return copies of them and their ids.""" + if type(records) is not list: + msg = f"{collection} must be a list" + raise HouseholdValidationError(msg) + if len(records) > MAXIMUM_HOUSEHOLD_ENTITIES: + msg = f"{collection} must contain at most {MAXIMUM_HOUSEHOLD_ENTITIES} records" + raise HouseholdValidationError(msg) + record_ids: set[str] = set() + normalized_records: list[dict[str, Any]] = [] + for record in records: + record_id = _require_record_identity(record, collection=collection) + if record_id in record_ids: + msg = f"{collection} contains duplicate id {record_id!r}" raise HouseholdValidationError(msg) - record_ids: set[str] = set() - normalized_records: list[dict[str, Any]] = [] - expected_keys = ( - {"id", "source_name", "values", "memberships"} - if collection == "people" - else {"id", "source_name", "values"} + record_ids.add(record_id) + _require_record_contents(record, collection=collection) + normalized_records.append(dict(record)) + return normalized_records, record_ids + + +def _require_record_identity(record: object, *, collection: str) -> str: + """Return the id of a record that has exactly its collection's fields.""" + if type(record) is not dict: + msg = f"{collection} records must be objects" + raise HouseholdValidationError(msg) + expected_fields = ( + PERSON_RECORD_FIELDS if collection == "people" else ENTITY_RECORD_FIELDS + ) + unknown = sorted(set(record) - expected_fields) + missing = sorted(REQUIRED_RECORD_FIELDS - set(record)) + if unknown or missing: + msg = ( + f"{collection} record fields are invalid: " + f"missing={missing}, unsupported={unknown}" ) - for record in records: - if type(record) is not dict: - msg = f"{collection} records must be objects" - raise HouseholdValidationError(msg) - unknown = sorted(set(record) - expected_keys) - missing = sorted({"id", "values"} - set(record)) - if unknown or missing: - msg = ( - f"{collection} record fields are invalid: " - f"missing={missing}, unsupported={unknown}" - ) - raise HouseholdValidationError(msg) - record_id = _require_identifier(record["id"], field=f"{collection} id") - if record_id in record_ids: - msg = f"{collection} contains duplicate id {record_id!r}" - raise HouseholdValidationError(msg) - record_ids.add(record_id) - _require_source_name(record, collection=collection) - _require_values(record, collection=collection) - if collection == "people" and type(record.get("memberships")) is not dict: - msg = "people record memberships must be an object" - raise HouseholdValidationError(msg) - normalized_records.append(dict(record)) - identifiers[collection] = record_ids - normalized[collection] = normalized_records + raise HouseholdValidationError(msg) + return _require_identifier(record["id"], field=f"{collection} id") + + +def _require_record_contents(record: dict[str, Any], *, collection: str) -> None: + """Validate a record's source name, values and, for people, memberships.""" + _require_source_name(record, collection=collection) + _require_values(record, collection=collection) + if collection == "people" and type(record.get("memberships")) is not dict: + msg = "people record memberships must be an object" + raise HouseholdValidationError(msg) + +def _require_complete_memberships( + people: list[dict[str, Any]], + collections: tuple[str, ...], + identifiers: dict[str, set[str]], +) -> None: + """Require every person to belong to one present record of each collection.""" required_memberships = set(collections) - for person in normalized["people"]: + for person in people: memberships = person["memberships"] if not isinstance(memberships, dict): msg = "people record memberships must be an object" @@ -212,7 +280,6 @@ def normalize_household_document( f"person membership references absent {collection} id {linked_id!r}" ) raise HouseholdValidationError(msg) - return normalized def verify_legacy_household_mapping( diff --git a/policyengine_api/services/v2/identity/database_connectors/creates.py b/policyengine_api/services/v2/identity/database_connectors/creates.py index d7e8a26b7..c52df6712 100644 --- a/policyengine_api/services/v2/identity/database_connectors/creates.py +++ b/policyengine_api/services/v2/identity/database_connectors/creates.py @@ -13,6 +13,8 @@ from datetime import datetime from uuid import UUID + from policyengine_api.services.v2.identity.types import UserPrincipal + def create_provisioned_user(session: Session, *, primary_country: str) -> User: """Insert a standard user for a new sign-in identity and return it with its ID.""" @@ -42,19 +44,17 @@ def create_user_identity( session: Session, *, user_id: UUID, - issuer: str, - subject: str, - email: str | None, + principal: UserPrincipal, seen_at: datetime, ) -> UUID | None: - """Insert one identity, returning its user unless another request won.""" + """Insert the principal's identity; return its user unless another request won.""" return session.execute( insert(UserIdentity) .values( user_id=user_id, - issuer=issuer, - subject=subject, - email=email, + issuer=principal.issuer, + subject=principal.subject, + email=principal.email, last_seen_at=seen_at, ) .on_conflict_do_nothing(constraint="uq_user_identities_issuer_subject") diff --git a/policyengine_api/services/v2/identity/services.py b/policyengine_api/services/v2/identity/services.py index 46d1cf1cd..5c1c63685 100644 --- a/policyengine_api/services/v2/identity/services.py +++ b/policyengine_api/services/v2/identity/services.py @@ -141,9 +141,7 @@ def resolve_in_session( created_user_id = create_user_identity( session, user_id=user.id, - issuer=principal.issuer, - subject=principal.subject, - email=principal.email, + principal=principal, seen_at=seen_at, ) if created_user_id is None: @@ -339,9 +337,7 @@ def convert_unauthenticated_user_in_session( and create_user_identity( session, user_id=unauthenticated_user.id, - issuer=principal.issuer, - subject=principal.subject, - email=principal.email, + principal=principal, seen_at=seen_at, ) is not None diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py b/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py index d29390ef5..cd945768e 100644 --- a/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py @@ -146,23 +146,22 @@ def read_parameter_children( *, model_version_id: UUID, parent_path: str, - dialect: str, offset: int, limit: int, ) -> list[Any]: """Return a page of the direct children of parent_path in a model version's tree. Each row has the child's path, label, type (node or parameter), child count - and, for parameters, the parameter's summary columns. dialect selects SQLite - or PostgreSQL string functions; limit + 1 rows are fetched so the caller can - tell whether another page exists. + and, for parameters, the parameter's summary columns. The session's dialect + selects SQLite or PostgreSQL string functions; limit + 1 rows are fetched so + the caller can tell whether another page exists. """ return read_rows( session, _parameter_children_statement( model_version_id=model_version_id, parent_path=parent_path, - dialect=dialect, + dialect=session.get_bind().dialect.name, offset=offset, limit=limit, ), diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py b/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py index 85fa4dc0d..02062798a 100644 --- a/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py @@ -11,9 +11,10 @@ from policyengine_api.services.v2.metadata.database_connectors.reads import read_rows if TYPE_CHECKING: - from datetime import datetime from uuid import UUID + from policyengine_api.services.v2.metadata.types import ParameterValueSelection + def _escape_like(value: str) -> str: return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") @@ -69,18 +70,20 @@ def read_parameter_values( session: Session, *, model_version_id: UUID, - parameter_id: UUID | None, - selected_day: datetime | None, + selection: ParameterValueSelection, offset: int, limit: int, ) -> list[ParameterValue]: """Return a page of a model version's current-law parameter values. - Values tied to a policy or dynamic are excluded. parameter_id keeps one - parameter's values and selected_day keeps those in effect on that day. Rows - are ordered by parameter name, then newest first, and limit + 1 are fetched - so the caller can tell whether another page exists. + Values tied to a policy or dynamic are excluded. The selection's + parameter_id keeps one parameter's values and its selected_day keeps those + in effect on that day. Rows are ordered by parameter name, then newest + first, and limit + 1 are fetched so the caller can tell whether another + page exists. """ + parameter_id = selection.parameter_id + selected_day = selection.selected_day statement = ( select(ParameterValue) .join(Parameter, col(Parameter.id) == col(ParameterValue.parameter_id)) diff --git a/policyengine_api/services/v2/metadata/database_session.py b/policyengine_api/services/v2/metadata/database_session.py index 207b7d07e..daa27927b 100644 --- a/policyengine_api/services/v2/metadata/database_session.py +++ b/policyengine_api/services/v2/metadata/database_session.py @@ -20,11 +20,6 @@ def session(self) -> Session: """The open request session that every metadata query runs on.""" return self._session - @property - def dialect_name(self) -> str: - """The bound dialect's name; it picks SQLite or PostgreSQL string functions.""" - return self._session.get_bind().dialect.name - def close(self) -> None: """Close the request's session.""" self._session.close() diff --git a/policyengine_api/services/v2/metadata/services.py b/policyengine_api/services/v2/metadata/services.py index 4e448f951..938eb660e 100644 --- a/policyengine_api/services/v2/metadata/services.py +++ b/policyengine_api/services/v2/metadata/services.py @@ -2,7 +2,6 @@ from __future__ import annotations -from datetime import UTC, datetime from typing import TYPE_CHECKING from policyengine_api.data.v2.catalog.catalog_selection import ( @@ -46,7 +45,7 @@ metadata_region, metadata_variable, page_result, - utc_day_start, + parameter_value_selection, ) from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, @@ -61,6 +60,7 @@ MetadataParameterSummary, MetadataRegion, MetadataVariable, + ParameterValueFilter, ) from policyengine_api.services.v2.metadata.validators import ( require_metadata_resource, @@ -321,7 +321,6 @@ def list_parameter_children( self._database_session.session, model_version_id=selected.model_version.id, parent_path=parent_path, - dialect=self._database_session.dialect_name, offset=offset, limit=limit, ) @@ -337,27 +336,25 @@ def list_parameter_values( country_id: str, policyengine_version: str | None = None, *, - parameter_id: UUID | None = None, - current: bool = False, + value_filter: ParameterValueFilter | None = None, offset: int = 0, limit: int = 100, - now: datetime | None = None, ) -> MetadataPageResult[MetadataCanonicalParameterValue]: """Return a page of current-law parameter values, newest first per parameter. - Values belonging to a reform or dynamic are never listed. parameter_id keeps - one parameter's values, and current=True keeps those in effect on the UTC - day of now (today by default). + Values belonging to a reform or dynamic are never listed. The filter's + parameter_id keeps one parameter's values, and current=True keeps those in + effect on the UTC day of now (today by default). No filter lists them all. """ selected = self._select_paginated_catalog( country_id, policyengine_version, offset=offset, limit=limit ) - selected_day = utc_day_start(now or datetime.now(UTC)) if current else None rows = read_parameter_values( self._database_session.session, model_version_id=selected.model_version.id, - parameter_id=parameter_id, - selected_day=selected_day, + selection=parameter_value_selection( + value_filter if value_filter is not None else ParameterValueFilter() + ), offset=offset, limit=limit, ) diff --git a/policyengine_api/services/v2/metadata/transformations.py b/policyengine_api/services/v2/metadata/transformations.py index 588869af4..68ebe21fe 100644 --- a/policyengine_api/services/v2/metadata/transformations.py +++ b/policyengine_api/services/v2/metadata/transformations.py @@ -22,6 +22,7 @@ MetadataRegionType, MetadataTimePeriodOption, MetadataVariable, + ParameterValueSelection, ) if TYPE_CHECKING: @@ -33,6 +34,7 @@ Region, Variable, ) + from policyengine_api.services.v2.metadata.types import ParameterValueFilter ResourceT = TypeVar("ResourceT") @@ -136,6 +138,24 @@ def utc_day_start(selected_time: datetime) -> datetime: ) +def parameter_value_selection( + value_filter: ParameterValueFilter, +) -> ParameterValueSelection: + """Resolve a listing's filter to the parameter and day its read selects. + + A current filter selects the UTC day of its ``now``, or of the present. + """ + selected_day = ( + utc_day_start(value_filter.now or datetime.now(UTC)) + if value_filter.current + else None + ) + return ParameterValueSelection( + parameter_id=value_filter.parameter_id, + selected_day=selected_day, + ) + + def metadata_dataset(dataset: Dataset) -> MetadataDataset: """Return the API representation of a stored logical input dataset.""" return MetadataDataset( diff --git a/policyengine_api/services/v2/metadata/types.py b/policyengine_api/services/v2/metadata/types.py index b74333291..7f41e1575 100644 --- a/policyengine_api/services/v2/metadata/types.py +++ b/policyengine_api/services/v2/metadata/types.py @@ -2,6 +2,7 @@ from __future__ import annotations +from dataclasses import dataclass from datetime import datetime from enum import StrEnum from typing import Generic, Literal, TypeVar @@ -10,6 +11,30 @@ from pydantic import BaseModel, ConfigDict, JsonValue +@dataclass(frozen=True) +class ParameterValueFilter: + """Which current-law parameter values a listing keeps. + + ``parameter_id`` keeps one parameter's values, and ``current`` keeps the + values in effect on the UTC day of ``now`` (today when ``now`` is None). + """ + + parameter_id: UUID | None = None + current: bool = False + now: datetime | None = None + + +@dataclass(frozen=True) +class ParameterValueSelection: + """The parameter and effective day a parameter-value read is narrowed to. + + None for either leaves the read unnarrowed by it. + """ + + parameter_id: UUID | None + selected_day: datetime | None + + class StrictResponseModel(BaseModel): """Base for metadata results that rejects unknown fields and non-finite numbers.""" diff --git a/policyengine_api/services/v2/metadata/validators.py b/policyengine_api/services/v2/metadata/validators.py index 60adf8818..bdc59c8d2 100644 --- a/policyengine_api/services/v2/metadata/validators.py +++ b/policyengine_api/services/v2/metadata/validators.py @@ -13,6 +13,9 @@ from policyengine_api.data.v2.models import Dataset, Region +MAXIMUM_METADATA_PAGE_LIMIT = 500 + + class MetadataResourceNotFoundError(LookupError): """Raised when a selected catalog does not contain a requested resource.""" @@ -29,7 +32,7 @@ def validate_metadata_page(offset: int, limit: int) -> tuple[int, int]: if offset < 0: msg = "offset must be at least 0" raise InvalidMetadataPageError(msg) - if not 1 <= limit <= 500: + if not 1 <= limit <= MAXIMUM_METADATA_PAGE_LIMIT: msg = "limit must be between 1 and 500" raise InvalidMetadataPageError(msg) return offset, limit diff --git a/policyengine_api/services/v2/policies/transformations.py b/policyengine_api/services/v2/policies/transformations.py index e20790ee1..5fceabf51 100644 --- a/policyengine_api/services/v2/policies/transformations.py +++ b/policyengine_api/services/v2/policies/transformations.py @@ -40,6 +40,8 @@ ) POLICY_CANONICALIZATION_VERSION = 1 +# A v1 date-range period key is "start.end": two dates joined by one dot. +LEGACY_DATE_RANGE_PART_COUNT = 2 def _canonical_number(value: float) -> str: @@ -234,7 +236,7 @@ def parse_legacy_period(value: str) -> tuple[datetime, datetime]: raise LegacyPolicyTranslationError(msg) if "." in value: parts = value.split(".") - if len(parts) != 2: + if len(parts) != LEGACY_DATE_RANGE_PART_COUNT: msg = f"legacy period {value!r} must contain one date range" raise LegacyPolicyTranslationError(msg) start_date, end_date = (_utc_midnight(item) for item in parts) diff --git a/policyengine_api/services/v2/policies/validators.py b/policyengine_api/services/v2/policies/validators.py index 974423f55..4302d07da 100644 --- a/policyengine_api/services/v2/policies/validators.py +++ b/policyengine_api/services/v2/policies/validators.py @@ -65,21 +65,35 @@ def require_json_value( if type(value) not in {list, dict}: msg = "value must contain only standards-compliant JSON types" raise ValueError(msg) - identity = id(value) + _require_json_container(value, depth=depth, containers=containers) + return value + + +def _require_json_container( + container: list[Any] | dict[Any, Any], + *, + depth: int, + containers: frozenset[int], +) -> None: + """Check a JSON list's items or object's members, rejecting reference cycles. + + ``containers`` holds the identities of the containers enclosing this one. + """ + identity = id(container) if identity in containers: msg = "JSON values must not contain reference cycles" raise ValueError(msg) nested_containers = containers | {identity} - if type(value) is list: - for item in value: + # The caller admits only exact lists and dicts, so isinstance is exact here. + if isinstance(container, list): + for item in container: require_json_value(item, depth=depth + 1, containers=nested_containers) - return value - for key, item in value.items(): + return + for key, item in container.items(): if type(key) is not str: msg = "JSON object keys must be strings" raise ValueError(msg) require_json_value(item, depth=depth + 1, containers=nested_containers) - return value def normalize_utc(value: datetime) -> datetime: diff --git a/policyengine_api/services/v2/user_households/database_connectors/reads.py b/policyengine_api/services/v2/user_households/database_connectors/reads.py index f2210a306..a95a9d097 100644 --- a/policyengine_api/services/v2/user_households/database_connectors/reads.py +++ b/policyengine_api/services/v2/user_households/database_connectors/reads.py @@ -16,6 +16,10 @@ from collections.abc import Collection from uuid import UUID + from policyengine_api.services.v2.user_households.types import ( + UserHouseholdListing, + ) + def read_household_for_association( session: Session, @@ -46,31 +50,25 @@ def read_owned_user_household_row( def read_user_household_rows( session: Session, - *, - country_id: str, - user_id: UUID, - household_id: UUID | None, - offset: int, - limit: int, - legacy_household_id: int | None = None, + listing: UserHouseholdListing, ) -> list[UserHouseholdAssociation]: """Return up to ``limit + 1`` of a user's associations in creation order. - With ``household_id`` or ``legacy_household_id``, only associations to that - household are returned. The extra row lets the caller report whether another - page exists. + With the listing's ``household_id`` or ``legacy_household_id``, only + associations to that household are returned. The extra row lets the caller + report whether another page exists. """ statement = select(UserHouseholdAssociation).where( - UserHouseholdAssociation.country_id == country_id, - UserHouseholdAssociation.user_id == user_id, + UserHouseholdAssociation.country_id == listing.country_id, + UserHouseholdAssociation.user_id == listing.user_id, ) - if household_id is not None: + if listing.household_id is not None: statement = statement.where( - UserHouseholdAssociation.household_id == household_id + UserHouseholdAssociation.household_id == listing.household_id ) - if legacy_household_id is not None: + if listing.legacy_household_id is not None: statement = statement.where( - UserHouseholdAssociation.legacy_household_id == legacy_household_id + UserHouseholdAssociation.legacy_household_id == listing.legacy_household_id ) return list( session.exec( @@ -78,8 +76,8 @@ def read_user_household_rows( col(UserHouseholdAssociation.created_at), col(UserHouseholdAssociation.id), ) - .offset(offset) - .limit(limit + 1) + .offset(listing.offset) + .limit(listing.limit + 1) ).all() ) diff --git a/policyengine_api/services/v2/user_households/services.py b/policyengine_api/services/v2/user_households/services.py index efe9c874a..a2630e897 100644 --- a/policyengine_api/services/v2/user_households/services.py +++ b/policyengine_api/services/v2/user_households/services.py @@ -39,6 +39,10 @@ user_household_page, user_household_read, ) +from policyengine_api.services.v2.user_households.types import ( + UserHouseholdFilter, + UserHouseholdListing, +) from policyengine_api.services.v2.user_households.validators import ( require_user_household, resolve_association_household_id, @@ -222,16 +226,18 @@ def list_user_households( *, principal: AccountPrincipal, country_id: str, - household_id: UUID | None = None, - legacy_household_id: int | None = None, + household_filter: UserHouseholdFilter | None = None, offset: int = 0, limit: int = 100, ) -> UserHouseholdPage: """Return a page of the principal's associations in this country. - ``household_id`` and ``legacy_household_id`` narrow it to one household. A - signed-out browser that has not saved anything yet gets an empty page. + ``household_filter`` narrows it to one household. A signed-out browser + that has not saved anything yet gets an empty page. """ + selected = ( + household_filter if household_filter is not None else UserHouseholdFilter() + ) with self._database_session.transaction() as session: owner = resolve_principal_user_for_read_in_session( session, principal, country_id=country_id @@ -240,12 +246,14 @@ def list_user_households( return user_household_page([], offset=offset, limit=limit) rows = read_user_household_rows( session, - country_id=country_id, - user_id=owner.user_id, - household_id=household_id, - legacy_household_id=legacy_household_id, - offset=offset, - limit=limit, + UserHouseholdListing( + country_id=country_id, + user_id=owner.user_id, + household_id=selected.household_id, + legacy_household_id=selected.legacy_household_id, + offset=offset, + limit=limit, + ), ) return user_household_page(rows, offset=offset, limit=limit) diff --git a/policyengine_api/services/v2/user_households/types.py b/policyengine_api/services/v2/user_households/types.py index b8b9bfd8d..599c1e35d 100644 --- a/policyengine_api/services/v2/user_households/types.py +++ b/policyengine_api/services/v2/user_households/types.py @@ -104,6 +104,33 @@ class UserHouseholdRead: updated_at: datetime +@dataclass(frozen=True) +class UserHouseholdFilter: + """Which of the caller's saved households a listing keeps. + + ``household_id`` keeps links to one v2 household and ``legacy_household_id`` + links to one v1 household; with neither, every saved household is kept. + """ + + household_id: UUID | None = None + legacy_household_id: int | None = None + + +@dataclass(frozen=True) +class UserHouseholdListing: + """One page of a user's associations to read, optionally for one household. + + ``household_id`` names a v2 household and ``legacy_household_id`` a v1 one. + """ + + country_id: str + user_id: UUID + household_id: UUID | None + offset: int + limit: int + legacy_household_id: int | None = None + + @dataclass(frozen=True) class UserHouseholdPage: """One page of associations in creation order; ``has_more`` says if more follow.""" diff --git a/policyengine_api/services/v2/user_policies/database_connectors/creates.py b/policyengine_api/services/v2/user_policies/database_connectors/creates.py index 4ed880968..e54c77879 100644 --- a/policyengine_api/services/v2/user_policies/database_connectors/creates.py +++ b/policyengine_api/services/v2/user_policies/database_connectors/creates.py @@ -17,7 +17,10 @@ if TYPE_CHECKING: from uuid import UUID - from policyengine_api.services.v2.user_policies.types import UserPolicyCreationInput + from policyengine_api.services.v2.user_policies.types import ( + LegacyUserPolicyRevision, + UserPolicyCreationInput, + ) def create_user_policy( @@ -58,14 +61,11 @@ def create_legacy_user_mapping( def create_legacy_user_policy_mapping( session: Session, *, - country_id: str, - legacy_user_policy_id: int, + revision: LegacyUserPolicyRevision, user_policy_id: UUID, - source_revision: int, fingerprint_version: int, - fingerprint: str, ) -> UUID | None: - """Map a v1 saved policy to its v2 association at ``source_revision``. + """Map a v1 saved policy to its v2 association at the revision's source revision. Return the new mapping id, or None when this v1 saved policy is already mapped in this country. @@ -73,12 +73,12 @@ def create_legacy_user_policy_mapping( return session.execute( insert(LegacyUserPolicyMapping) .values( - country_id=country_id, - legacy_user_policy_id=legacy_user_policy_id, + country_id=revision.country_id, + legacy_user_policy_id=revision.legacy_user_policy_id, user_policy_id=user_policy_id, - last_applied_source_revision=source_revision, + last_applied_source_revision=revision.source_revision, fingerprint_version=fingerprint_version, - fingerprint_sha256=fingerprint, + fingerprint_sha256=revision.fingerprint, ) .on_conflict_do_nothing( constraint="uq_legacy_user_policy_mappings_country_legacy" diff --git a/policyengine_api/services/v2/user_policies/database_connectors/reads.py b/policyengine_api/services/v2/user_policies/database_connectors/reads.py index 908b370a1..dfebc5f21 100644 --- a/policyengine_api/services/v2/user_policies/database_connectors/reads.py +++ b/policyengine_api/services/v2/user_policies/database_connectors/reads.py @@ -19,6 +19,8 @@ from collections.abc import Collection from uuid import UUID + from policyengine_api.services.v2.user_policies.types import UserPolicyListing + def read_user(session: Session, user_id: UUID) -> User | None: """Return the v2 user with this id, or None.""" @@ -94,33 +96,29 @@ def read_owned_user_policy_row( def read_user_policy_rows( session: Session, - *, - country_id: str, - user_id: UUID, - policy_id: UUID | None, - offset: int, - limit: int, - legacy_policy_id: int | None = None, + listing: UserPolicyListing, ) -> list[UserPolicy]: """Return up to ``limit + 1`` of a user's associations in creation order. - With ``policy_id`` or ``legacy_policy_id``, only associations to that policy - are returned. The extra row lets the caller report whether another page - exists. + With the listing's ``policy_id`` or ``legacy_policy_id``, only associations to + that policy are returned. The extra row lets the caller report whether + another page exists. """ statement = select(UserPolicy).where( - UserPolicy.country_id == country_id, - UserPolicy.user_id == user_id, + UserPolicy.country_id == listing.country_id, + UserPolicy.user_id == listing.user_id, ) - if policy_id is not None: - statement = statement.where(UserPolicy.policy_id == policy_id) - if legacy_policy_id is not None: - statement = statement.where(UserPolicy.legacy_policy_id == legacy_policy_id) + if listing.policy_id is not None: + statement = statement.where(UserPolicy.policy_id == listing.policy_id) + if listing.legacy_policy_id is not None: + statement = statement.where( + UserPolicy.legacy_policy_id == listing.legacy_policy_id + ) return list( session.exec( statement.order_by(col(UserPolicy.created_at), col(UserPolicy.id)) - .offset(offset) - .limit(limit + 1) + .offset(listing.offset) + .limit(listing.limit + 1) ).all() ) diff --git a/policyengine_api/services/v2/user_policies/database_connectors/updates.py b/policyengine_api/services/v2/user_policies/database_connectors/updates.py index 151ac8499..328c3d32d 100644 --- a/policyengine_api/services/v2/user_policies/database_connectors/updates.py +++ b/policyengine_api/services/v2/user_policies/database_connectors/updates.py @@ -12,7 +12,10 @@ from sqlmodel import Session from policyengine_api.data.v2.models import LegacyUserPolicyMapping, UserPolicy - from policyengine_api.services.v2.user_policies.types import UserPolicyUpdateInput + from policyengine_api.services.v2.user_policies.types import ( + LegacyUserPolicyRevision, + UserPolicyUpdateInput, + ) def update_user_policy( @@ -43,21 +46,19 @@ def update_legacy_user_policy_state( *, association: UserPolicy, mapping: LegacyUserPolicyMapping, - reform_label: str | None, + revision: LegacyUserPolicyRevision, update_name: bool, - fingerprint: str, - source_revision: int, ) -> None: """Record a newly applied v1 revision on its mapping. With ``update_name``, also set the association's name to the v1 reform label. """ if update_name: - association.name = reform_label + association.name = revision.reform_label association.updated_at = utc_now() session.add(association) - mapping.fingerprint_sha256 = fingerprint - mapping.last_applied_source_revision = source_revision + mapping.fingerprint_sha256 = revision.fingerprint + mapping.last_applied_source_revision = revision.source_revision mapping.updated_at = utc_now() session.add(mapping) session.flush() diff --git a/policyengine_api/services/v2/user_policies/services.py b/policyengine_api/services/v2/user_policies/services.py index 24bfaa795..f9f3ade8c 100644 --- a/policyengine_api/services/v2/user_policies/services.py +++ b/policyengine_api/services/v2/user_policies/services.py @@ -56,9 +56,12 @@ from policyengine_api.services.v2.user_policies.types import ( LegacyUserPolicyMappingAction, LegacyUserPolicyPersistenceResult, + LegacyUserPolicyRevision, LegacyUserPolicySnapshot, UserPolicyCreateFields, UserPolicyCreationInput, + UserPolicyFilter, + UserPolicyListing, UserPolicyPage, UserPolicyRead, UserPolicyUpdateInput, @@ -91,25 +94,11 @@ def read_user_policy_page( session: Session, - *, - country_id: str, - user_id: UUID, - policy_id: UUID | None, - offset: int, - limit: int, - legacy_policy_id: int | None = None, + listing: UserPolicyListing, ) -> UserPolicyPage: """Return a page of a user's associations, optionally for one policy.""" - rows = read_user_policy_rows( - session, - country_id=country_id, - user_id=user_id, - policy_id=policy_id, - legacy_policy_id=legacy_policy_id, - offset=offset, - limit=limit, - ) - return user_policy_page(rows, offset=offset, limit=limit) + rows = read_user_policy_rows(session, listing) + return user_policy_page(rows, offset=listing.offset, limit=listing.limit) def read_owned_user_policy( @@ -213,28 +202,22 @@ def apply_existing_legacy_user_policy_mapping( session: Session, *, mapping: LegacyUserPolicyMapping, - country_id: str, - reform_label: str | None, - fingerprint: str, - user_id: UUID, - policy_id: UUID, - changed_fields: frozenset[str], - source_revision: int, + revision: LegacyUserPolicyRevision, + projection: UserPolicyCreationInput, ) -> LegacyUserPolicyPersistenceResult: """Apply one v1 revision to an already-mapped association. - Stale and replayed revisions change nothing. The next revision renames the + ``projection`` is the association the revision projects to. Stale and + replayed revisions change nothing. The next revision renames the association only if the v1 reform label changed. Raise LegacyUserPolicyIntegrityError on conflicting or skipped revisions. """ + policy_id = require_mirrored_policy_id(projection) action, association = validate_existing_legacy_user_policy_mapping( mapping, association=read_mapped_user_policy(session, mapping), - country_id=country_id, - fingerprint=fingerprint, - user_id=user_id, - policy_id=policy_id, - source_revision=source_revision, + revision=revision, + projection=projection, fingerprint_version=USER_POLICY_FINGERPRINT_VERSION, ) if action in { @@ -251,17 +234,15 @@ def apply_existing_legacy_user_policy_mapping( association_updated = legacy_name_requires_update( association, - reform_label=reform_label, - changed_fields=changed_fields, + reform_label=revision.reform_label, + changed_fields=revision.changed_fields, ) update_legacy_user_policy_state( session, association=association, mapping=mapping, - reform_label=reform_label, + revision=revision, update_name=association_updated, - fingerprint=fingerprint, - source_revision=source_revision, ) return LegacyUserPolicyPersistenceResult( association_id=association.id, @@ -275,13 +256,8 @@ def apply_existing_legacy_user_policy_mapping( def persist_legacy_user_policy_mapping( session: Session, *, - country_id: str, - legacy_user_policy_id: int, - reform_label: str | None, + revision: LegacyUserPolicyRevision, projection: UserPolicyCreationInput, - fingerprint: str, - source_revision: int, - changed_fields: frozenset[str], ) -> LegacyUserPolicyPersistenceResult: """Create the association and mapping for a v1 saved policy, or apply a revision. @@ -293,32 +269,24 @@ def persist_legacy_user_policy_mapping( policy_id = require_mirrored_policy_id(projection) existing = read_legacy_user_policy_mapping( session, - country_id=country_id, - legacy_user_policy_id=legacy_user_policy_id, + country_id=revision.country_id, + legacy_user_policy_id=revision.legacy_user_policy_id, lock=True, ) if existing is not None: return apply_existing_legacy_user_policy_mapping( session, mapping=existing, - country_id=country_id, - reform_label=reform_label, - fingerprint=fingerprint, - user_id=projection.user_id, - policy_id=policy_id, - changed_fields=changed_fields, - source_revision=source_revision, + revision=revision, + projection=projection, ) association = create_user_policy(session, projection) mapping_id = create_legacy_user_policy_mapping( session, - country_id=country_id, - legacy_user_policy_id=legacy_user_policy_id, + revision=revision, user_policy_id=association.id, - source_revision=source_revision, fingerprint_version=USER_POLICY_FINGERPRINT_VERSION, - fingerprint=fingerprint, ) if mapping_id is not None: return LegacyUserPolicyPersistenceResult( @@ -332,8 +300,8 @@ def persist_legacy_user_policy_mapping( delete_user_policy(session, association) concurrent = read_legacy_user_policy_mapping( session, - country_id=country_id, - legacy_user_policy_id=legacy_user_policy_id, + country_id=revision.country_id, + legacy_user_policy_id=revision.legacy_user_policy_id, lock=False, ) if concurrent is None: @@ -342,13 +310,8 @@ def persist_legacy_user_policy_mapping( return apply_existing_legacy_user_policy_mapping( session, mapping=concurrent, - country_id=country_id, - reform_label=reform_label, - fingerprint=fingerprint, - user_id=projection.user_id, - policy_id=policy_id, - changed_fields=changed_fields, - source_revision=source_revision, + revision=revision, + projection=projection, ) @@ -383,13 +346,15 @@ def mirror_legacy_user_policy_in_session( ) return persist_legacy_user_policy_mapping( session, - country_id=snapshot.country_id, - legacy_user_policy_id=snapshot.legacy_user_policy_id, - reform_label=snapshot.reform_label, + revision=LegacyUserPolicyRevision( + country_id=snapshot.country_id, + legacy_user_policy_id=snapshot.legacy_user_policy_id, + reform_label=snapshot.reform_label, + fingerprint=fingerprint_legacy_user_policy(snapshot), + source_revision=source_revision, + changed_fields=changed_fields, + ), projection=projection, - fingerprint=fingerprint_legacy_user_policy(snapshot), - source_revision=source_revision, - changed_fields=changed_fields, ) @@ -496,16 +461,16 @@ def list_user_policies( *, principal: AccountPrincipal, country_id: str, - policy_id: UUID | None = None, - legacy_policy_id: int | None = None, + policy_filter: UserPolicyFilter | None = None, offset: int = 0, limit: int = 100, ) -> UserPolicyPage: """Return a page of the principal's associations in this country. - ``policy_id`` and ``legacy_policy_id`` narrow it to one policy. A signed-out - browser that has not saved anything yet gets an empty page. + ``policy_filter`` narrows it to one policy. A signed-out browser that has + not saved anything yet gets an empty page. """ + selected = policy_filter if policy_filter is not None else UserPolicyFilter() with self._database_session.transaction() as session: owner = resolve_principal_user_for_read_in_session( session, principal, country_id=country_id @@ -514,12 +479,14 @@ def list_user_policies( return user_policy_page([], offset=offset, limit=limit) return read_user_policy_page( session, - country_id=country_id, - user_id=owner.user_id, - policy_id=policy_id, - legacy_policy_id=legacy_policy_id, - offset=offset, - limit=limit, + UserPolicyListing( + country_id=country_id, + user_id=owner.user_id, + policy_id=selected.policy_id, + offset=offset, + limit=limit, + legacy_policy_id=selected.legacy_policy_id, + ), ) def patch_user_policy( diff --git a/policyengine_api/services/v2/user_policies/types.py b/policyengine_api/services/v2/user_policies/types.py index 5edd99555..c24b2d49c 100644 --- a/policyengine_api/services/v2/user_policies/types.py +++ b/policyengine_api/services/v2/user_policies/types.py @@ -163,6 +163,49 @@ class UserPolicyRead: updated_at: datetime +@dataclass(frozen=True) +class UserPolicyFilter: + """Which of the caller's saved policies a listing keeps. + + ``policy_id`` keeps links to one v2 policy and ``legacy_policy_id`` links to + one v1 policy; with neither, every saved policy is kept. + """ + + policy_id: UUID | None = None + legacy_policy_id: int | None = None + + +@dataclass(frozen=True) +class UserPolicyListing: + """One page of a user's associations to read, optionally for one policy. + + ``policy_id`` names a v2 policy and ``legacy_policy_id`` a v1 one. + """ + + country_id: str + user_id: UUID + policy_id: UUID | None + offset: int + limit: int + legacy_policy_id: int | None = None + + +@dataclass(frozen=True) +class LegacyUserPolicyRevision: + """One revision of a v1 saved policy, as it is applied to its v2 association. + + ``fingerprint`` identifies the complete v1 row at ``source_revision``, and + ``changed_fields`` names the v1 fields that revision changed. + """ + + country_id: str + legacy_user_policy_id: int + reform_label: str | None + fingerprint: str + source_revision: int + changed_fields: frozenset[str] + + @dataclass(frozen=True) class UserPolicyPage: """One page of associations in creation order; ``has_more`` says if more follow.""" diff --git a/policyengine_api/services/v2/user_policies/validators.py b/policyengine_api/services/v2/user_policies/validators.py index b760b5f8e..fbeea3e98 100644 --- a/policyengine_api/services/v2/user_policies/validators.py +++ b/policyengine_api/services/v2/user_policies/validators.py @@ -24,6 +24,7 @@ ) from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot from policyengine_api.services.v2.user_policies.types import ( + LegacyUserPolicyRevision, LegacyUserPolicySnapshot, UserPolicyCreationInput, ) @@ -135,27 +136,27 @@ def validate_existing_legacy_user_policy_mapping( mapping: LegacyUserPolicyMapping, *, association: UserPolicy | None, - country_id: str, - fingerprint: str, - user_id: UUID, - policy_id: UUID, - source_revision: int, + revision: LegacyUserPolicyRevision, + projection: UserPolicyCreationInput, fingerprint_version: int, ) -> tuple[LegacyUserPolicyMappingAction, UserPolicy]: """Decide how to apply a v1 revision to an existing mapping. - Return the action with the mapped association. Raise - LegacyUserPolicyIntegrityError if the association is missing or its user, - policy or country changed, if the fingerprint version differs, if a replay - has a different fingerprint, or if a revision was skipped. + ``projection`` is the association the revision projects to, whose user and + policy the mapped association must keep. Return the action with the mapped + association. Raise LegacyUserPolicyIntegrityError if the association is + missing or its user, policy or country changed, if the fingerprint version + differs, if a replay has a different fingerprint, or if a revision was + skipped. """ + source_revision = revision.source_revision if association is None: msg = "legacy user-policy mapping has no association" raise LegacyUserPolicyIntegrityError(msg) if ( - association.policy_id != policy_id - or association.country_id != country_id - or association.user_id != user_id + association.policy_id != projection.policy_id + or association.country_id != revision.country_id + or association.user_id != projection.user_id ): msg = "legacy user-policy mapping conflicts with immutable association fields" raise LegacyUserPolicyIntegrityError(msg) @@ -165,7 +166,7 @@ def validate_existing_legacy_user_policy_mapping( if source_revision < mapping.last_applied_source_revision: return LegacyUserPolicyMappingAction.STALE, association if source_revision == mapping.last_applied_source_revision: - if mapping.fingerprint_sha256 != fingerprint: + if mapping.fingerprint_sha256 != revision.fingerprint: msg = "legacy user-policy revision conflicts with its stored fingerprint" raise LegacyUserPolicyIntegrityError(msg) return LegacyUserPolicyMappingAction.REPLAY, association diff --git a/policyengine_api/services/v2/user_reports/database_connectors/reads.py b/policyengine_api/services/v2/user_reports/database_connectors/reads.py index b583bf5ab..e39a1afe4 100644 --- a/policyengine_api/services/v2/user_reports/database_connectors/reads.py +++ b/policyengine_api/services/v2/user_reports/database_connectors/reads.py @@ -11,6 +11,8 @@ if TYPE_CHECKING: from uuid import UUID + from policyengine_api.services.v2.user_reports.types import UserReportListing + def read_owned_user_report_row( session: Session, @@ -46,25 +48,21 @@ def read_user_report_row( def read_user_report_rows( session: Session, - *, - country_id: str, - user_id: UUID, - legacy_report_id: int | None, - offset: int, - limit: int, + listing: UserReportListing, ) -> list[UserReportAssociation]: """Return up to ``limit + 1`` of a user's associations in creation order. - With ``legacy_report_id``, only associations to that v1 report output are - returned. The extra row lets the caller report whether another page exists. + With the listing's ``legacy_report_id``, only associations to that v1 report + output are returned. The extra row lets the caller report whether another + page exists. """ statement = select(UserReportAssociation).where( - UserReportAssociation.country_id == country_id, - UserReportAssociation.user_id == user_id, + UserReportAssociation.country_id == listing.country_id, + UserReportAssociation.user_id == listing.user_id, ) - if legacy_report_id is not None: + if listing.legacy_report_id is not None: statement = statement.where( - UserReportAssociation.legacy_report_id == legacy_report_id + UserReportAssociation.legacy_report_id == listing.legacy_report_id ) return list( session.exec( @@ -72,8 +70,8 @@ def read_user_report_rows( col(UserReportAssociation.created_at), col(UserReportAssociation.id), ) - .offset(offset) - .limit(limit + 1) + .offset(listing.offset) + .limit(listing.limit + 1) ).all() ) diff --git a/policyengine_api/services/v2/user_reports/services.py b/policyengine_api/services/v2/user_reports/services.py index 0fae168de..8b1961eda 100644 --- a/policyengine_api/services/v2/user_reports/services.py +++ b/policyengine_api/services/v2/user_reports/services.py @@ -49,6 +49,7 @@ user_report_read, user_report_update_values, ) +from policyengine_api.services.v2.user_reports.types import UserReportListing from policyengine_api.services.v2.user_reports.validators import ( require_owned_user_policy, require_user_report, @@ -273,11 +274,13 @@ def list_user_reports( return user_report_page([], offset=offset, limit=limit) rows = read_user_report_rows( session, - country_id=country_id, - user_id=owner.user_id, - legacy_report_id=legacy_report_id, - offset=offset, - limit=limit, + UserReportListing( + country_id=country_id, + user_id=owner.user_id, + legacy_report_id=legacy_report_id, + offset=offset, + limit=limit, + ), ) return user_report_page(rows, offset=offset, limit=limit) diff --git a/policyengine_api/services/v2/user_reports/types.py b/policyengine_api/services/v2/user_reports/types.py index 52d4f27ac..5cc0574e9 100644 --- a/policyengine_api/services/v2/user_reports/types.py +++ b/policyengine_api/services/v2/user_reports/types.py @@ -305,6 +305,17 @@ class SharedUserReportRead: updated_at: datetime +@dataclass(frozen=True) +class UserReportListing: + """One page of a user's associations to read, optionally for one v1 report.""" + + country_id: str + user_id: UUID + legacy_report_id: int | None + offset: int + limit: int + + @dataclass(frozen=True) class UserReportPage: """One page of associations in creation order; ``has_more`` says if more follow.""" diff --git a/policyengine_api/services/v2/user_simulations/database_connectors/reads.py b/policyengine_api/services/v2/user_simulations/database_connectors/reads.py index c33b7c57e..64609e364 100644 --- a/policyengine_api/services/v2/user_simulations/database_connectors/reads.py +++ b/policyengine_api/services/v2/user_simulations/database_connectors/reads.py @@ -11,6 +11,10 @@ if TYPE_CHECKING: from uuid import UUID + from policyengine_api.services.v2.user_simulations.types import ( + UserSimulationListing, + ) + def read_owned_user_simulation_row( session: Session, @@ -31,25 +35,22 @@ def read_owned_user_simulation_row( def read_user_simulation_rows( session: Session, - *, - country_id: str, - user_id: UUID, - legacy_simulation_id: int | None, - offset: int, - limit: int, + listing: UserSimulationListing, ) -> list[UserSimulationAssociation]: """Return up to ``limit + 1`` of a user's associations in creation order. - With ``legacy_simulation_id``, only associations to that v1 simulation are - returned. The extra row lets the caller report whether another page exists. + With the listing's ``legacy_simulation_id``, only associations to that v1 + simulation are returned. The extra row lets the caller report whether + another page exists. """ statement = select(UserSimulationAssociation).where( - UserSimulationAssociation.country_id == country_id, - UserSimulationAssociation.user_id == user_id, + UserSimulationAssociation.country_id == listing.country_id, + UserSimulationAssociation.user_id == listing.user_id, ) - if legacy_simulation_id is not None: + if listing.legacy_simulation_id is not None: statement = statement.where( - UserSimulationAssociation.legacy_simulation_id == legacy_simulation_id + UserSimulationAssociation.legacy_simulation_id + == listing.legacy_simulation_id ) return list( session.exec( @@ -57,8 +58,8 @@ def read_user_simulation_rows( col(UserSimulationAssociation.created_at), col(UserSimulationAssociation.id), ) - .offset(offset) - .limit(limit + 1) + .offset(listing.offset) + .limit(listing.limit + 1) ).all() ) diff --git a/policyengine_api/services/v2/user_simulations/services.py b/policyengine_api/services/v2/user_simulations/services.py index cadac3f5c..8e2682aa5 100644 --- a/policyengine_api/services/v2/user_simulations/services.py +++ b/policyengine_api/services/v2/user_simulations/services.py @@ -38,6 +38,7 @@ user_simulation_page, user_simulation_read, ) +from policyengine_api.services.v2.user_simulations.types import UserSimulationListing from policyengine_api.services.v2.user_simulations.validators import ( require_user_simulation, ) @@ -199,11 +200,13 @@ def list_user_simulations( return user_simulation_page([], offset=offset, limit=limit) rows = read_user_simulation_rows( session, - country_id=country_id, - user_id=owner.user_id, - legacy_simulation_id=legacy_simulation_id, - offset=offset, - limit=limit, + UserSimulationListing( + country_id=country_id, + user_id=owner.user_id, + legacy_simulation_id=legacy_simulation_id, + offset=offset, + limit=limit, + ), ) return user_simulation_page(rows, offset=offset, limit=limit) diff --git a/policyengine_api/services/v2/user_simulations/types.py b/policyengine_api/services/v2/user_simulations/types.py index c34c415cf..d456e0cee 100644 --- a/policyengine_api/services/v2/user_simulations/types.py +++ b/policyengine_api/services/v2/user_simulations/types.py @@ -91,6 +91,17 @@ class UserSimulationRead: updated_at: datetime +@dataclass(frozen=True) +class UserSimulationListing: + """One page of a user's associations to read, optionally for one v1 simulation.""" + + country_id: str + user_id: UUID + legacy_simulation_id: int | None + offset: int + limit: int + + @dataclass(frozen=True) class UserSimulationPage: """One page of associations in creation order; ``has_more`` says if more follow.""" diff --git a/policyengine_api/utils/input_validation.py b/policyengine_api/utils/input_validation.py index af691f508..09df2d87a 100644 --- a/policyengine_api/utils/input_validation.py +++ b/policyengine_api/utils/input_validation.py @@ -102,47 +102,81 @@ def find_unrecognized_household_inputs( if not isinstance(entity_group, dict): continue - relationship_keys = { - role["plural"] - for role in entity.get("roles", {}).values() - if "plural" in role - } - for entity_id, entity_inputs in entity_group.items(): - if not isinstance(entity_inputs, dict): - continue - for variable_name in entity_inputs: - if variable_name in relationship_keys: - continue - variable = variables.get(variable_name) - path = f"household.{entity_plural}.{entity_id}.{variable_name}" - if variable is None: - errors.append( - UnrecognizedInput( - input_type="household_variable", - name=variable_name, - path=path, - ) - ) - continue - expected_entity = entities.get(variable["entity"], {}) - expected_entity_plural = expected_entity.get("plural") - if ( - expected_entity_plural is not None - and expected_entity_plural != entity_plural - ): - errors.append( - UnrecognizedInput( - input_type="household_variable_wrong_entity", - name=variable_name, - path=path, - expected_entity_plural=expected_entity_plural, - actual_entity_plural=entity_plural, - ) - ) + errors.extend( + _find_unrecognized_entity_inputs( + entity_plural, + entity_group, + entity, + variables, + entities, + ) + ) return errors +def _find_unrecognized_entity_inputs( + entity_plural: str, + entity_group: dict, + entity: dict, + variables: dict, + entities: dict, +) -> list[UnrecognizedInput]: + """Find unknown and misplaced variables in one entity group's members. + + Role membership keys and members that are not objects are skipped. + """ + relationship_keys = { + role["plural"] for role in entity.get("roles", {}).values() if "plural" in role + } + errors: list[UnrecognizedInput] = [] + for entity_id, entity_inputs in entity_group.items(): + if not isinstance(entity_inputs, dict): + continue + for variable_name in entity_inputs: + if variable_name in relationship_keys: + continue + error = _unrecognized_variable( + variable_name, + path=f"household.{entity_plural}.{entity_id}.{variable_name}", + entity_plural=entity_plural, + variables=variables, + entities=entities, + ) + if error is not None: + errors.append(error) + return errors + + +def _unrecognized_variable( + variable_name: str, + *, + path: str, + entity_plural: str, + variables: dict, + entities: dict, +) -> UnrecognizedInput | None: + """Report a variable the model lacks or that belongs on another entity, if so.""" + variable = variables.get(variable_name) + if variable is None: + return UnrecognizedInput( + input_type="household_variable", + name=variable_name, + path=path, + ) + expected_entity = entities.get(variable["entity"], {}) + expected_entity_plural = expected_entity.get("plural") + if expected_entity_plural is not None and expected_entity_plural != entity_plural: + return UnrecognizedInput( + input_type="household_variable_wrong_entity", + name=variable_name, + path=path, + expected_entity_plural=expected_entity_plural, + actual_entity_plural=entity_plural, + ) + return None + + def find_unrecognized_policy_inputs( policy: dict, metadata: dict, diff --git a/policyengine_api/utils/json.py b/policyengine_api/utils/json.py index 668803ef8..26d25f70c 100644 --- a/policyengine_api/utils/json.py +++ b/policyengine_api/utils/json.py @@ -41,6 +41,15 @@ def hash_object(o: object) -> str: return base64.b64encode(hasher.digest()).decode() +def _safe_json_number(value: float) -> JSONValue: + """Return a number unchanged, or an infinity as the string JavaScript reads.""" + if value == np.inf: + return "Infinity" + if value == -np.inf: + return "-Infinity" + return value + + def get_safe_json(value: object) -> JSONValue: """Return a JSON-safe copy of a metadata value, writing infinities as strings. @@ -49,11 +58,7 @@ def get_safe_json(value: object) -> JSONValue: # This function is used when constructing metadata, # which will be consumed by JS, hence Infinity, not .inf if isinstance(value, (int, float)): - if value == np.inf: - return "Infinity" - if value == -np.inf: - return "-Infinity" - return value + return _safe_json_number(value) if isinstance(value, str): return value if isinstance(value, dict): diff --git a/tests/fixtures/recording_session.py b/tests/fixtures/recording_session.py new file mode 100644 index 000000000..63323df47 --- /dev/null +++ b/tests/fixtures/recording_session.py @@ -0,0 +1,55 @@ +"""A session double that records every statement and write a connector issues.""" + +from __future__ import annotations + +from collections import deque +from dataclasses import dataclass, field + +from sqlalchemy.dialects import postgresql + + +@dataclass +class _ScriptedResult: + """The scripted value a recorded statement returns, however it is read.""" + + value: object + + def one_or_none(self) -> object: + return self.value + + def scalar_one_or_none(self) -> object: + return self.value + + +@dataclass +class RecordingSession: + """Record statements (compiled for PostgreSQL) and ORM writes, in order. + + Each ``exec`` or ``execute`` returns the next value from ``results``. + """ + + results: deque[object] = field(default_factory=deque) + operations: list[tuple[object, ...]] = field(default_factory=list) + + def _record_statement(self, method: str, statement: object) -> _ScriptedResult: + compiled = statement.compile(dialect=postgresql.dialect()) + self.operations.append((method, str(compiled), dict(compiled.params))) + return _ScriptedResult(self.results.popleft()) + + def exec(self, statement: object) -> _ScriptedResult: + return self._record_statement("exec", statement) + + def execute(self, statement: object) -> _ScriptedResult: + return self._record_statement("execute", statement) + + def add(self, instance: object) -> None: + self.operations.append(("add", instance)) + + def delete(self, instance: object) -> None: + self.operations.append(("delete", instance)) + + def flush(self) -> None: + self.operations.append(("flush",)) + + def refresh(self, instance: object) -> None: + self.operations.append(("refresh", instance)) diff --git a/tests/integration/test_v1_user_policy_dual_write.py b/tests/integration/test_v1_user_policy_dual_write.py index 2bcd63e2a..1e2bc2fe6 100644 --- a/tests/integration/test_v1_user_policy_dual_write.py +++ b/tests/integration/test_v1_user_policy_dual_write.py @@ -31,6 +31,7 @@ ) from policyengine_api.services.policy_service import PolicyService from policyengine_api.services.user_policy_mirroring import ( + UserPolicyMirrorCollaborators, UserPolicyMirrorUnavailableError, mirror_pending_user_policy_events_after_commit, mirror_user_policy_after_commit, @@ -187,9 +188,11 @@ def test_create_update_and_v1_only_change_mirror_one_association( "us", created.user_policy.id, through_revision=created.mirror_revision, - event_service=saved_service, - reform_snapshot_loader=policy_service.get_policy_snapshot, - mirror_factory=lambda: mirror_service, + collaborators=UserPolicyMirrorCollaborators( + event_service=saved_service, + reform_snapshot_loader=policy_service.get_policy_snapshot, + mirror_factory=lambda: mirror_service, + ), ) renamed = saved_service.update_user_policy( @@ -202,9 +205,11 @@ def test_create_update_and_v1_only_change_mirror_one_association( "us", created.user_policy.id, through_revision=renamed.mirror_revision, - event_service=saved_service, - reform_snapshot_loader=policy_service.get_policy_snapshot, - mirror_factory=lambda: mirror_service, + collaborators=UserPolicyMirrorCollaborators( + event_service=saved_service, + reform_snapshot_loader=policy_service.get_policy_snapshot, + mirror_factory=lambda: mirror_service, + ), ) v1_only = saved_service.update_user_policy( "us", @@ -216,9 +221,11 @@ def test_create_update_and_v1_only_change_mirror_one_association( "us", created.user_policy.id, through_revision=v1_only.mirror_revision, - event_service=saved_service, - reform_snapshot_loader=policy_service.get_policy_snapshot, - mirror_factory=lambda: mirror_service, + collaborators=UserPolicyMirrorCollaborators( + event_service=saved_service, + reform_snapshot_loader=policy_service.get_policy_snapshot, + mirror_factory=lambda: mirror_service, + ), ) assert first.association_id == rename_result.association_id @@ -275,9 +282,11 @@ def test_failure_after_cloud_commit_and_identical_create_retry_are_idempotent( "us", created.user_policy.id, through_revision=created.mirror_revision, - event_service=saved_service, - reform_snapshot_loader=policy_service.get_policy_snapshot, - mirror_factory=lambda: mirror_service, + collaborators=UserPolicyMirrorCollaborators( + event_service=saved_service, + reform_snapshot_loader=policy_service.get_policy_snapshot, + mirror_factory=lambda: mirror_service, + ), ) with v1_sessions() as session: @@ -310,9 +319,11 @@ def test_failure_after_cloud_commit_and_identical_create_retry_are_idempotent( "us", retry.user_policy.id, through_revision=retry.mirror_revision, - event_service=saved_service, - reform_snapshot_loader=policy_service.get_policy_snapshot, - mirror_factory=lambda: mirror_service, + collaborators=UserPolicyMirrorCollaborators( + event_service=saved_service, + reform_snapshot_loader=policy_service.get_policy_snapshot, + mirror_factory=lambda: mirror_service, + ), ) assert retry.created is False @@ -374,9 +385,11 @@ def test_destination_commit_replays_when_source_processing_marker_is_missing( "us", created.user_policy.id, through_revision=created.mirror_revision, - event_service=saved_service, - reform_snapshot_loader=policy_service.get_policy_snapshot, - mirror_factory=lambda: mirror_service, + collaborators=UserPolicyMirrorCollaborators( + event_service=saved_service, + reform_snapshot_loader=policy_service.get_policy_snapshot, + mirror_factory=lambda: mirror_service, + ), ) assert replayed.association_id == committed.association_id diff --git a/tests/unit/libs/test_gateway_auth.py b/tests/unit/libs/test_gateway_auth.py index 554d759f5..142361849 100644 --- a/tests/unit/libs/test_gateway_auth.py +++ b/tests/unit/libs/test_gateway_auth.py @@ -15,6 +15,7 @@ GATEWAY_AUTH_CORE_ENV_VARS, GATEWAY_AUTH_ENV_VARS, GATEWAY_AUTH_REQUIRED_ENV, + GatewayAuthCredentials, GatewayAuthError, GatewayAuthTokenProvider, GatewayBearerAuth, @@ -54,30 +55,36 @@ class TestGatewayAuthTokenProvider: class TestConfigured: def test__given_all_kwargs__then_configured_true(self): provider = GatewayAuthTokenProvider( - issuer=ISSUER, - audience=AUDIENCE, - client_id=CLIENT_ID, - client_secret=CLIENT_SECRET, + GatewayAuthCredentials( + issuer=ISSUER, + audience=AUDIENCE, + client_id=CLIENT_ID, + client_secret=CLIENT_SECRET, + ), ) assert provider.configured is True def test__given_missing_client_secret__then_configured_false(self): provider = GatewayAuthTokenProvider( - issuer=ISSUER, - audience=AUDIENCE, - client_id=CLIENT_ID, - client_secret="", + GatewayAuthCredentials( + issuer=ISSUER, + audience=AUDIENCE, + client_id=CLIENT_ID, + client_secret="", + ), ) assert provider.configured is False def test__given_secret_resource__then_configured_true(self): provider = GatewayAuthTokenProvider( - issuer=ISSUER, - audience=AUDIENCE, - client_id=CLIENT_ID, - client_secret_resource=CLIENT_SECRET_RESOURCE, + GatewayAuthCredentials( + issuer=ISSUER, + audience=AUDIENCE, + client_id=CLIENT_ID, + client_secret_resource=CLIENT_SECRET_RESOURCE, + ), ) assert provider.configured is True @@ -85,7 +92,9 @@ def test__given_secret_resource__then_configured_true(self): class TestGetToken: def test__given_unconfigured_provider__then_raises(self): provider = GatewayAuthTokenProvider( - issuer="", audience="", client_id="", client_secret="" + GatewayAuthCredentials( + issuer="", audience="", client_id="", client_secret="" + ) ) with pytest.raises(GatewayAuthError): @@ -93,10 +102,12 @@ def test__given_unconfigured_provider__then_raises(self): def test__given_first_call__then_fetches_and_returns_token(self): provider = GatewayAuthTokenProvider( - issuer=ISSUER, - audience=AUDIENCE, - client_id=CLIENT_ID, - client_secret=CLIENT_SECRET, + GatewayAuthCredentials( + issuer=ISSUER, + audience=AUDIENCE, + client_id=CLIENT_ID, + client_secret=CLIENT_SECRET, + ), ) with patch( @@ -117,10 +128,12 @@ def test__given_first_call__then_fetches_and_returns_token(self): def test__given_secret_manager_resource__then_loads_secret_once(self): provider = GatewayAuthTokenProvider( - issuer=ISSUER, - audience=AUDIENCE, - client_id=CLIENT_ID, - client_secret_resource=CLIENT_SECRET_RESOURCE, + GatewayAuthCredentials( + issuer=ISSUER, + audience=AUDIENCE, + client_id=CLIENT_ID, + client_secret_resource=CLIENT_SECRET_RESOURCE, + ), ) with ( @@ -142,10 +155,12 @@ def test__given_secret_manager_resource__then_loads_secret_once(self): def test__given_secret_manager_failure__then_raises(self): provider = GatewayAuthTokenProvider( - issuer=ISSUER, - audience=AUDIENCE, - client_id=CLIENT_ID, - client_secret_resource=CLIENT_SECRET_RESOURCE, + GatewayAuthCredentials( + issuer=ISSUER, + audience=AUDIENCE, + client_id=CLIENT_ID, + client_secret_resource=CLIENT_SECRET_RESOURCE, + ), ) with ( @@ -162,10 +177,12 @@ def test__given_secret_manager_failure__then_raises(self): def test__given_trailing_slash_issuer__then_no_double_slash(self): provider = GatewayAuthTokenProvider( - issuer=f"{ISSUER}/", - audience=AUDIENCE, - client_id=CLIENT_ID, - client_secret=CLIENT_SECRET, + GatewayAuthCredentials( + issuer=f"{ISSUER}/", + audience=AUDIENCE, + client_id=CLIENT_ID, + client_secret=CLIENT_SECRET, + ), ) with patch( @@ -178,10 +195,12 @@ def test__given_trailing_slash_issuer__then_no_double_slash(self): def test__given_fresh_cached_token__then_second_call_reuses(self): provider = GatewayAuthTokenProvider( - issuer=ISSUER, - audience=AUDIENCE, - client_id=CLIENT_ID, - client_secret=CLIENT_SECRET, + GatewayAuthCredentials( + issuer=ISSUER, + audience=AUDIENCE, + client_id=CLIENT_ID, + client_secret=CLIENT_SECRET, + ), ) with patch( @@ -196,10 +215,12 @@ def test__given_fresh_cached_token__then_second_call_reuses(self): def test__given_expired_token__then_second_call_refreshes(self): provider = GatewayAuthTokenProvider( - issuer=ISSUER, - audience=AUDIENCE, - client_id=CLIENT_ID, - client_secret=CLIENT_SECRET, + GatewayAuthCredentials( + issuer=ISSUER, + audience=AUDIENCE, + client_id=CLIENT_ID, + client_secret=CLIENT_SECRET, + ), ) responses = [ @@ -220,10 +241,12 @@ def test__given_expired_token__then_second_call_refreshes(self): def test__given_network_error__then_raises_gateway_auth_error(self): provider = GatewayAuthTokenProvider( - issuer=ISSUER, - audience=AUDIENCE, - client_id=CLIENT_ID, - client_secret=CLIENT_SECRET, + GatewayAuthCredentials( + issuer=ISSUER, + audience=AUDIENCE, + client_id=CLIENT_ID, + client_secret=CLIENT_SECRET, + ), ) with ( @@ -237,10 +260,12 @@ def test__given_network_error__then_raises_gateway_auth_error(self): def test__given_response_without_access_token__then_raises(self): provider = GatewayAuthTokenProvider( - issuer=ISSUER, - audience=AUDIENCE, - client_id=CLIENT_ID, - client_secret=CLIENT_SECRET, + GatewayAuthCredentials( + issuer=ISSUER, + audience=AUDIENCE, + client_id=CLIENT_ID, + client_secret=CLIENT_SECRET, + ), ) response = MagicMock() response.status_code = 200 @@ -258,10 +283,12 @@ def test__given_response_without_access_token__then_raises(self): def test__given_missing_expires_in__then_raises(self): provider = GatewayAuthTokenProvider( - issuer=ISSUER, - audience=AUDIENCE, - client_id=CLIENT_ID, - client_secret=CLIENT_SECRET, + GatewayAuthCredentials( + issuer=ISSUER, + audience=AUDIENCE, + client_id=CLIENT_ID, + client_secret=CLIENT_SECRET, + ), ) response = MagicMock() response.status_code = 200 @@ -279,10 +306,12 @@ def test__given_missing_expires_in__then_raises(self): def test__given_zero_expires_in__then_clamped_to_refresh_margin(self): provider = GatewayAuthTokenProvider( - issuer=ISSUER, - audience=AUDIENCE, - client_id=CLIENT_ID, - client_secret=CLIENT_SECRET, + GatewayAuthCredentials( + issuer=ISSUER, + audience=AUDIENCE, + client_id=CLIENT_ID, + client_secret=CLIENT_SECRET, + ), ) with patch( @@ -296,10 +325,12 @@ def test__given_zero_expires_in__then_clamped_to_refresh_margin(self): def test__given_concurrent_callers__then_fetches_once(self): provider = GatewayAuthTokenProvider( - issuer=ISSUER, - audience=AUDIENCE, - client_id=CLIENT_ID, - client_secret=CLIENT_SECRET, + GatewayAuthCredentials( + issuer=ISSUER, + audience=AUDIENCE, + client_id=CLIENT_ID, + client_secret=CLIENT_SECRET, + ), ) def slow_response(*_args, **_kwargs): @@ -326,10 +357,12 @@ def slow_response(*_args, **_kwargs): class TestInvalidate: def test__given_invalidated_token__then_next_call_refetches(self): provider = GatewayAuthTokenProvider( - issuer=ISSUER, - audience=AUDIENCE, - client_id=CLIENT_ID, - client_secret=CLIENT_SECRET, + GatewayAuthCredentials( + issuer=ISSUER, + audience=AUDIENCE, + client_id=CLIENT_ID, + client_secret=CLIENT_SECRET, + ), ) responses = [ diff --git a/tests/unit/routes/test_household_route_error_responses.py b/tests/unit/routes/test_household_route_error_responses.py new file mode 100644 index 000000000..5505df6e7 --- /dev/null +++ b/tests/unit/routes/test_household_route_error_responses.py @@ -0,0 +1,257 @@ +"""Pin every error response of the stored-household and calculate routes.""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest +from flask import Flask +from policyengine_core.errors import SituationParsingError + +from policyengine_api.extensions import cache +from policyengine_api.routes import household_routes +from policyengine_api.services.household_calculation_service import ( + HouseholdCalculationResult, + HouseholdNotFoundError, + InvalidHouseholdInputsError, + PolicyNotFoundError, +) +from policyengine_api.spm import SPMValidationError +from policyengine_api.utils.input_validation import UnrecognizedInput + +INVALID_INPUT = UnrecognizedInput( + input_type="policy_parameter", + name="gov.unknown", + path="policy.gov.unknown", +) +INVALID_INPUTS_BODY = { + "status": "error", + "message": f"Unrecognized calculate input(s): {INVALID_INPUT.message}", + "result": None, + "errors": [INVALID_INPUT.to_dict()], +} +SPM_ERROR = SPMValidationError("SPM_SETTINGS_INVALID", "spm is malformed") +SPM_BODY = { + "status": "error", + "message": "spm is malformed", + "result": None, + "errors": [{"code": "SPM_SETTINGS_INVALID", "message": "spm is malformed"}], +} +PARSING_ERROR = SituationParsingError(["people"], "people must be an object") + + +@pytest.fixture +def calculation_service(monkeypatch) -> MagicMock: + service = MagicMock() + monkeypatch.setattr(household_routes, "household_calculation_service", service) + monkeypatch.setattr(household_routes, "get_v1_household_read_source", MagicMock()) + return service + + +@pytest.fixture +def client(): + app = Flask(__name__) + app.config.update(TESTING=True, CACHE_TYPE="NullCache") + cache.init_app(app) + app.register_blueprint(household_routes.household_bp) + return app.test_client() + + +@pytest.mark.parametrize( + ("error", "status_code", "body"), + [ + ( + HouseholdNotFoundError(), + 404, + {"status": "error", "message": "Household #3 not found."}, + ), + ( + PolicyNotFoundError(), + 404, + {"status": "error", "message": "Policy #4 not found."}, + ), + (InvalidHouseholdInputsError([INVALID_INPUT]), 400, INVALID_INPUTS_BODY), + (SPM_ERROR, 400, SPM_BODY), + ( + RuntimeError("engine exploded"), + 500, + { + "status": "error", + "message": "Error calculating household #3 under policy #4: " + "engine exploded", + }, + ), + ], +) +def test_stored_household_calculation_errors_map_to_responses( + client, + calculation_service: MagicMock, + error: Exception, + status_code: int, + body: dict[str, object], +) -> None: + calculation_service.calculate_stored_household.side_effect = error + + with patch.object(household_routes, "module_logger") as module_logger: + response = client.get("/us/household/3/policy/4") + + assert response.status_code == status_code + assert response.get_json() == body + if status_code == 500: + module_logger.exception.assert_called_once_with( + "Error calculating household #%s under policy #%s", "3", "4" + ) + else: + module_logger.exception.assert_not_called() + + +def test_stored_household_calculation_needs_a_valid_read_source( + client, + calculation_service: MagicMock, +) -> None: + household_routes.get_v1_household_read_source.side_effect = ValueError("bad") + + response = client.get("/us/household/3/policy/4") + + assert response.status_code == 503 + assert response.get_json() == { + "status": "error", + "message": "Household persistence configuration is unavailable.", + } + calculation_service.calculate_stored_household.assert_not_called() + + +def test_stored_household_calculation_success_returns_the_result( + client, + calculation_service: MagicMock, +) -> None: + calculation_service.calculate_stored_household.return_value = ( + HouseholdCalculationResult(household={"people": {}}, warnings=("late",)) + ) + + response = client.get("/us/household/3/policy/4") + + assert response.status_code == 200 + assert response.get_json() == { + "status": "ok", + "message": None, + "result": {"people": {}}, + "warnings": ["late"], + } + assert calculation_service.calculate_stored_household.call_args.args == ( + "us", + 3, + 4, + ) + + +@pytest.mark.parametrize( + ("error", "status_code", "body"), + [ + ( + PARSING_ERROR, + 400, + { + "status": "error", + "message": f"Invalid household payload: {PARSING_ERROR}", + "result": None, + }, + ), + (SPM_ERROR, 400, SPM_BODY), + ( + RuntimeError("parser exploded"), + 500, + { + "status": "error", + "message": "Error calculating household under policy: parser exploded", + }, + ), + ], +) +def test_calculation_parse_errors_map_to_responses( + client, + calculation_service: MagicMock, + error: Exception, + status_code: int, + body: dict[str, object], +) -> None: + calculation_service.parse_prepared_household.side_effect = error + + with ( + patch.object(household_routes, "module_logger") as module_logger, + patch.object(household_routes, "start_observability_id") as start_observability, + ): + response = client.post("/us/calculate", json={"household": {}}) + + assert response.status_code == status_code + assert response.get_json() == body + assert start_observability.call_count == int(status_code == 500) + assert module_logger.exception.call_count == int(status_code == 500) + calculation_service.calculate_prepared_household.assert_not_called() + + +@pytest.mark.parametrize( + ("error", "status_code", "body"), + [ + (InvalidHouseholdInputsError([INVALID_INPUT]), 400, INVALID_INPUTS_BODY), + ( + PARSING_ERROR, + 400, + { + "status": "error", + "message": f"Invalid household payload: {PARSING_ERROR}", + "result": None, + }, + ), + (SPM_ERROR, 400, SPM_BODY), + ( + RuntimeError("engine exploded"), + 500, + { + "status": "error", + "message": "Error calculating household under policy: engine exploded", + }, + ), + ], +) +def test_calculation_errors_map_to_responses( + client, + calculation_service: MagicMock, + error: Exception, + status_code: int, + body: dict[str, object], +) -> None: + calculation_service.calculate_prepared_household.side_effect = error + + with ( + patch.object(household_routes, "module_logger") as module_logger, + patch.object(household_routes, "start_observability_id") as start_observability, + ): + response = client.post("/us/calculate-full", json={"household": {}}) + + assert response.status_code == status_code + assert response.get_json() == body + start_observability.assert_not_called() + if status_code == 500: + module_logger.exception.assert_called_once_with( + "Error calculating household under policy" + ) + else: + module_logger.exception.assert_not_called() + + +def test_situation_parsing_error_caused_by_spm_returns_the_spm_error( + client, + calculation_service: MagicMock, +) -> None: + parsing_error = SituationParsingError(["spm"], "spm is malformed") + parsing_error.__cause__ = SPM_ERROR + calculation_service.calculate_prepared_household.side_effect = parsing_error + + response = client.post("/us/calculate", json={"household": {}}) + + assert response.status_code == 400 + assert response.get_json() == { + **SPM_BODY, + "message": "spm is malformed", + } diff --git a/tests/unit/routes/test_migration_context_logging.py b/tests/unit/routes/test_migration_context_logging.py index 73302a996..cc77aa61a 100644 --- a/tests/unit/routes/test_migration_context_logging.py +++ b/tests/unit/routes/test_migration_context_logging.py @@ -11,6 +11,7 @@ RouteImplementationSettings, ) from policyengine_api.migration_logging import ( + ServedRequest, log_migration_request, register_migration_request_logging, ) @@ -384,13 +385,15 @@ def test_v2_metadata_resource_logs_its_actual_supabase_read_source(monkeypatch): with patch("policyengine_api.migration_logging.logger") as mock_logger: log_migration_request( - request_id="request-123", - method="GET", - path="/v2/variables", - status_code=200, - started_at=None, - country_id="us", - route_impl=RouteImplementation.FASTAPI_NATIVE, + ServedRequest( + request_id="request-123", + method="GET", + path="/v2/variables", + status_code=200, + started_at=None, + country_id="us", + route_impl=RouteImplementation.FASTAPI_NATIVE, + ) ) migration_context = mock_logger.log_struct.call_args.args[0]["migration"] @@ -408,22 +411,26 @@ def test_v2_policy_resources_log_actual_supabase_read_and_write_sources( with patch("policyengine_api.migration_logging.logger") as mock_logger: log_migration_request( - request_id="request-read", - method="GET", - path="/v2/policies", - status_code=200, - started_at=None, - country_id="us", - route_impl=RouteImplementation.FASTAPI_NATIVE, + ServedRequest( + request_id="request-read", + method="GET", + path="/v2/policies", + status_code=200, + started_at=None, + country_id="us", + route_impl=RouteImplementation.FASTAPI_NATIVE, + ) ) log_migration_request( - request_id="request-write", - method="PATCH", - path="/v2/user-policies/00000000-0000-0000-0000-000000000001", - status_code=200, - started_at=None, - country_id="us", - route_impl=RouteImplementation.FASTAPI_NATIVE, + ServedRequest( + request_id="request-write", + method="PATCH", + path="/v2/user-policies/00000000-0000-0000-0000-000000000001", + status_code=200, + started_at=None, + country_id="us", + route_impl=RouteImplementation.FASTAPI_NATIVE, + ) ) read_context = mock_logger.log_struct.call_args_list[0].args[0]["migration"] @@ -452,22 +459,26 @@ def test_v2_household_resources_log_actual_supabase_read_and_write_sources( with patch("policyengine_api.migration_logging.logger") as mock_logger: log_migration_request( - request_id="request-read", - method="GET", - path="/v2/households", - status_code=200, - started_at=None, - country_id="us", - route_impl=RouteImplementation.FASTAPI_NATIVE, + ServedRequest( + request_id="request-read", + method="GET", + path="/v2/households", + status_code=200, + started_at=None, + country_id="us", + route_impl=RouteImplementation.FASTAPI_NATIVE, + ) ) log_migration_request( - request_id="request-write", - method="PATCH", - path="/v2/user-households/00000000-0000-0000-0000-000000000001", - status_code=200, - started_at=None, - country_id="us", - route_impl=RouteImplementation.FASTAPI_NATIVE, + ServedRequest( + request_id="request-write", + method="PATCH", + path="/v2/user-households/00000000-0000-0000-0000-000000000001", + status_code=200, + started_at=None, + country_id="us", + route_impl=RouteImplementation.FASTAPI_NATIVE, + ) ) read_context = mock_logger.log_struct.call_args_list[0].args[0]["migration"] @@ -537,13 +548,15 @@ def test_signed_in_user_resources_log_supabase_sources( with patch("policyengine_api.migration_logging.logger") as mock_logger: log_migration_request( - request_id="request", - method=method, - path=path, - status_code=200, - started_at=None, - country_id="us", - route_impl=RouteImplementation.FASTAPI_NATIVE, + ServedRequest( + request_id="request", + method=method, + path=path, + status_code=200, + started_at=None, + country_id="us", + route_impl=RouteImplementation.FASTAPI_NATIVE, + ) ) context = mock_logger.log_struct.call_args.args[0]["migration"] @@ -670,13 +683,15 @@ def test_asgi_shell_does_not_add_retired_backend_header_to_fallback(): def test_compute_routes_in_the_same_groups_keep_their_compute_labels(): with patch("policyengine_api.migration_logging.logger") as mock_logger: log_migration_request( - request_id="request", - method="POST", - path="/us/simulation", - status_code=200, - started_at=None, - country_id="us", - route_impl=RouteImplementation.FLASK_FALLBACK, + ServedRequest( + request_id="request", + method="POST", + path="/us/simulation", + status_code=200, + started_at=None, + country_id="us", + route_impl=RouteImplementation.FLASK_FALLBACK, + ) ) context = mock_logger.log_struct.call_args.args[0]["migration"] diff --git a/tests/unit/routes/test_simulation_creation_validation.py b/tests/unit/routes/test_simulation_creation_validation.py new file mode 100644 index 000000000..4fbc31612 --- /dev/null +++ b/tests/unit/routes/test_simulation_creation_validation.py @@ -0,0 +1,153 @@ +"""Pin how the v1 simulation-creation route validates and reports failures.""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest +from flask import Flask +from werkzeug.exceptions import NotFound + +from policyengine_api.routes import simulation_routes + +VALID_PAYLOAD = { + "population_id": "1", + "population_type": "household", + "policy_id": 2, +} + + +@pytest.fixture +def client(): + app = Flask(__name__) + app.config["TESTING"] = True + app.register_blueprint(simulation_routes.simulation_bp) + return app.test_client() + + +@pytest.mark.parametrize( + ("payload", "message"), + [ + (None, "Payload missing from request"), + ({**VALID_PAYLOAD, "population_id": ""}, "population_id is required"), + ({**VALID_PAYLOAD, "population_type": None}, "population_type is required"), + ( + {**VALID_PAYLOAD, "population_type": "region"}, + "population_type must be 'household' or 'geography'", + ), + ({**VALID_PAYLOAD, "policy_id": None}, "policy_id is required"), + ({**VALID_PAYLOAD, "policy_id": "2"}, "policy_id must be an integer"), + ( + {"population_type": "region", "policy_id": "2"}, + "population_id is required", + ), + ( + {"population_id": "1", "population_type": "region"}, + "population_type must be 'household' or 'geography'", + ), + ], +) +def test_invalid_payloads_are_bad_requests_before_any_write( + client, + payload: object, + message: str, +) -> None: + with patch.object(simulation_routes, "simulation_service") as service: + response = client.post( + "/us/simulation", + data="null" if payload is None else None, + json=payload if payload is not None else None, + content_type="application/json", + ) + + assert response.status_code == 400 + assert message in response.get_data(as_text=True) + assert not service.mock_calls + + +@pytest.mark.parametrize( + ("created", "status_code", "message"), + [ + (True, 201, "Simulation created successfully"), + (False, 200, "Simulation already exists"), + ], +) +def test_us_payload_returns_the_serialized_simulation( + client, + created, + status_code: int, + message: str, +) -> None: + with ( + patch.object(simulation_routes, "simulation_service") as service, + patch.object( + simulation_routes, + "_serialize_v1_simulation", + return_value={"id": 7}, + ), + ): + service.get_or_create_simulation.return_value = SimpleNamespace( + simulation=object(), created=created + ) + response = client.post("/us/simulation", json=VALID_PAYLOAD) + + assert response.status_code == status_code + assert response.get_json() == { + "status": "ok", + "message": message, + "result": {"id": 7}, + } + service.get_or_create_simulation.assert_called_once_with( + country_id="us", + population_id="1", + population_type="household", + policy_id=2, + ) + + +def test_service_http_errors_pass_through_unlogged(client) -> None: + with ( + patch.object(simulation_routes, "simulation_service") as service, + patch.object(simulation_routes, "current_app", new=MagicMock()) as current_app, + ): + service.get_or_create_simulation.side_effect = NotFound("missing policy") + response = client.post("/us/simulation", json=VALID_PAYLOAD) + + assert response.status_code == 404 + assert "missing policy" in response.get_data(as_text=True) + assert not current_app.logger.mock_calls + + +def test_service_value_errors_become_logged_bad_requests(client) -> None: + error = ValueError("unknown household") + with ( + patch.object(simulation_routes, "simulation_service") as service, + patch.object(simulation_routes, "current_app", new=MagicMock()) as current_app, + ): + service.get_or_create_simulation.side_effect = error + response = client.post("/us/simulation", json=VALID_PAYLOAD) + + assert response.status_code == 400 + assert "Failed to create simulation: unknown household" in response.get_data( + as_text=True + ) + current_app.logger.warning.assert_called_once_with( + "Bad request creating simulation for country %s: %s", "us", error + ) + + +def test_unexpected_service_errors_are_logged_and_raised(client) -> None: + error = RuntimeError("database exploded") + with ( + patch.object(simulation_routes, "simulation_service") as service, + patch.object(simulation_routes, "current_app", new=MagicMock()) as current_app, + ): + service.get_or_create_simulation.side_effect = error + with pytest.raises(RuntimeError) as raised: + client.post("/us/simulation", json=VALID_PAYLOAD) + + assert raised.value is error + current_app.logger.exception.assert_called_once_with( + "Unexpected error creating simulation for country %s", "us" + ) diff --git a/tests/unit/routes/test_user_policy_route_outcomes.py b/tests/unit/routes/test_user_policy_route_outcomes.py new file mode 100644 index 000000000..4499df2b4 --- /dev/null +++ b/tests/unit/routes/test_user_policy_route_outcomes.py @@ -0,0 +1,444 @@ +"""Pin every response and failure record of the v1 saved-policy write routes.""" + +from __future__ import annotations + +from typing import NamedTuple +from unittest.mock import MagicMock, patch + +import pytest +from flask import Flask + +from policyengine_api.data.v1_models import UserPolicy +from policyengine_api.routes import policy_routes +from policyengine_api.services.user_policy_mirroring import ( + UserPolicyMirrorCollaborators, + UserPolicyMirrorUnavailableError, +) +from policyengine_api.services.user_policy_service import ( + UserPolicyCreateResult, + UserPolicyPersistenceError, + UserPolicyPersistenceFailureCategory, + UserPolicyUpdateResult, +) + +UNAVAILABLE_CONFIGURATION = { + "status": "error", + "message": "Policy persistence configuration is unavailable.", +} +MIRROR_UNAVAILABLE = { + "message": "V2 saved-policy mirroring is unavailable; retry the same request." +} + + +def _row() -> UserPolicy: + return UserPolicy( + id=10, + country_id="us", + reform_id=2, + reform_label="Reform", + baseline_id=1, + baseline_label="Current law", + user_id="auth0|one", + year="2026", + geography="us", + dataset=None, + number_of_provisions=3, + api_version="1.0.0", + added_date=1, + updated_date=2, + budgetary_impact=None, + type=None, + ) + + +def _body() -> dict[str, object]: + return { + "reform_id": 2, + "baseline_id": 1, + "user_id": "auth0|one", + "year": "2026", + "geography": "us", + "number_of_provisions": 3, + "api_version": "1.0.0", + "added_date": 1, + "updated_date": 2, + } + + +@pytest.fixture +def client(): + app = Flask(__name__) + app.config["TESTING"] = True + app.register_blueprint(policy_routes.policy_bp) + return app.test_client() + + +@pytest.fixture +def write_source(monkeypatch) -> MagicMock: + source = MagicMock(return_value="dual_write") + monkeypatch.setattr(policy_routes, "get_v1_policy_write_source", source) + monkeypatch.setattr(policy_routes, "get_v1_policy_read_source", MagicMock()) + return source + + +@pytest.fixture +def mirror(monkeypatch) -> MagicMock: + mirror_events = MagicMock() + monkeypatch.setattr( + policy_routes, + "mirror_pending_user_policy_events_after_commit", + mirror_events, + ) + return mirror_events + + +def _create(client): + return client.post("/us/user-policy", json=_body()) + + +def _update(client): + return client.put("/us/user-policy", json={"id": 10, "reform_label": "Renamed"}) + + +@pytest.mark.parametrize("send_request", [_create, _update]) +@pytest.mark.parametrize("failing_source", ["write", "read"]) +def test_invalid_persistence_configuration_is_unavailable( + client, + write_source: MagicMock, + mirror: MagicMock, + send_request, + failing_source: str, +) -> None: + if failing_source == "write": + write_source.side_effect = ValueError("invalid") + else: + policy_routes.get_v1_policy_read_source.side_effect = ValueError("invalid") + + with ( + patch.object( + policy_routes.user_policy_service, "create_or_get_user_policy" + ) as create, + patch.object(policy_routes.user_policy_service, "update_user_policy") as update, + ): + response = send_request(client) + + assert response.status_code == 503 + assert response.get_json() == UNAVAILABLE_CONFIGURATION + create.assert_not_called() + update.assert_not_called() + mirror.assert_not_called() + + +class _PersistenceFailure(NamedTuple): + category: UserPolicyPersistenceFailureCategory + status_code: int + message: str + + +class _SavedPolicyWrite(NamedTuple): + send_request: object + service_method: str + operation: str + legacy_id: int | None + + +@pytest.mark.parametrize( + "failure", + [ + _PersistenceFailure( + UserPolicyPersistenceFailureCategory.TIMEOUT, + 503, + "Policy database is temporarily unavailable; please try again later.", + ), + _PersistenceFailure( + UserPolicyPersistenceFailureCategory.INTEGRITY, + 500, + "Internal database error; please try again later.", + ), + ], +) +@pytest.mark.parametrize( + "write", + [ + _SavedPolicyWrite(_create, "create_or_get_user_policy", "create", None), + _SavedPolicyWrite(_update, "update_user_policy", "update", 10), + ], +) +def test_persistence_failures_log_one_record_and_return_a_safe_error( + client, + write_source: MagicMock, + mirror: MagicMock, + failure: _PersistenceFailure, + write: _SavedPolicyWrite, +) -> None: + write_source.return_value = "cloud_sql" + category, status_code, message = failure + + with ( + patch.object( + policy_routes.user_policy_service, + write.service_method, + side_effect=UserPolicyPersistenceError(category), + ), + patch.object(policy_routes, "logger") as logger, + patch.object(policy_routes, "current_request_id", return_value="request-1"), + ): + response = write.send_request(client) + + assert response.status_code == status_code + assert response.get_json() == {"message": message} + record = dict(logger.log_struct.call_args.args[0]) + assert isinstance(record.pop("duration_ms"), float) + expected = { + "message": "V1 saved-policy persistence failed", + "metric_name": "v1_user_policy_persistence_failures", + "metric_value": 1, + "resource": "user_policy", + "operation": write.operation, + "database_source": "cloud_sql", + "configured_write_source": "cloud_sql", + "country_id": "us", + "legacy_user_policy_id": write.legacy_id, + "request_id": "request-1", + "outcome": "error", + "failure_category": category.value, + "http_status": status_code, + } + assert list(record) == list(expected) + assert record == expected + assert logger.log_struct.call_args.kwargs == {"severity": "ERROR"} + mirror.assert_not_called() + + +@pytest.mark.usefixtures("write_source") +@pytest.mark.parametrize("send_request", [_create, _update]) +def test_persistence_failure_survives_a_logging_failure( + client, + mirror: MagicMock, + send_request, +) -> None: + error = UserPolicyPersistenceError(UserPolicyPersistenceFailureCategory.DATABASE) + + with ( + patch.object( + policy_routes.user_policy_service, + "create_or_get_user_policy", + side_effect=error, + ), + patch.object( + policy_routes.user_policy_service, + "update_user_policy", + side_effect=error, + ), + patch.object(policy_routes.logger, "log_struct", side_effect=RuntimeError), + ): + response = send_request(client) + + assert response.status_code == 500 + assert response.get_json() == { + "message": "Internal database error; please try again later." + } + mirror.assert_not_called() + + +@pytest.mark.usefixtures("write_source") +@pytest.mark.parametrize( + "legacy_user_policy_id", [-1, 2_147_483_648, True, "10", 2_147_483_647, 0] +) +def test_failure_record_reports_only_a_safe_legacy_id( + client, + legacy_user_policy_id: object, +) -> None: + error = UserPolicyPersistenceError(UserPolicyPersistenceFailureCategory.DATABASE) + + with ( + patch.object( + policy_routes.user_policy_service, + "update_user_policy", + side_effect=error, + ), + patch.object(policy_routes, "logger") as logger, + ): + client.put( + "/us/user-policy", + json={"id": legacy_user_policy_id, "reform_label": "Renamed"}, + ) + + record = logger.log_struct.call_args.args[0] + expected = ( + legacy_user_policy_id + if legacy_user_policy_id in {2_147_483_647, 0} + and not isinstance(legacy_user_policy_id, bool) + else None + ) + assert record["legacy_user_policy_id"] == expected + + +def _creation(*, created: bool, mirror_revision: int | None): + return UserPolicyCreateResult( + user_policy=_row(), + created=created, + snapshot=None, + mirror_revision=mirror_revision, + ) + + +def _update_result(mirror_revision: int | None): + return UserPolicyUpdateResult( + user_policy=_row(), + snapshot=None, + changed_fields=frozenset({"reform_label"}), + mirror_revision=mirror_revision, + ) + + +@pytest.mark.usefixtures("write_source") +@pytest.mark.parametrize( + "mirror_outcome", + [ + (None, None), + (3, UserPolicyMirrorUnavailableError("database secret")), + (3, RuntimeError("unexpected secret")), + ], +) +@pytest.mark.parametrize("send_request", [_create, _update]) +def test_committed_write_without_a_mirror_is_mirror_unavailable( + client, + mirror: MagicMock, + mirror_outcome: tuple[int | None, Exception | None], + send_request, +) -> None: + mirror_revision, mirror.side_effect = mirror_outcome + + with ( + patch.object( + policy_routes.user_policy_service, + "create_or_get_user_policy", + return_value=_creation(created=True, mirror_revision=mirror_revision), + ), + patch.object( + policy_routes.user_policy_service, + "update_user_policy", + return_value=_update_result(mirror_revision), + ), + ): + response = send_request(client) + + assert response.status_code == 503 + assert response.get_json() == MIRROR_UNAVAILABLE + assert mirror.call_count == int(mirror_revision is not None) + + +@pytest.mark.usefixtures("write_source") +@pytest.mark.parametrize( + ("created", "status_code", "message"), + [ + (True, 201, "Record created successfully"), + ( + False, + 200, + "The reform #2 / baseline #1 pair already exists for user auth0|one", + ), + ], +) +def test_mirrored_creation_returns_the_saved_policy( + client, + mirror: MagicMock, + created, + status_code: int, + message: str, +) -> None: + with patch.object( + policy_routes.user_policy_service, + "create_or_get_user_policy", + return_value=_creation(created=created, mirror_revision=4), + ): + response = _create(client) + + assert response.status_code == status_code + body = response.get_json() + assert body["status"] == "ok" + assert body["message"] == message + if created: + assert body["result"]["id"] == 10 + assert body["result"]["reform_label"] == "Reform" + else: + assert body["result"] == {"id": 10} + assert mirror.call_args.args == ("us", 10) + assert mirror.call_args.kwargs == { + "through_revision": 4, + "collaborators": UserPolicyMirrorCollaborators( + event_service=policy_routes.user_policy_service, + reform_snapshot_loader=policy_routes.policy_service.get_policy_snapshot, + ), + } + + +@pytest.mark.usefixtures("write_source") +def test_mirrored_update_returns_the_updated_id( + client, + mirror: MagicMock, +) -> None: + with patch.object( + policy_routes.user_policy_service, + "update_user_policy", + return_value=_update_result(5), + ) as update: + response = _update(client) + + assert response.status_code == 200 + assert response.get_json() == { + "status": "ok", + "message": "Record updated successfully", + "result": {"id": 10}, + } + assert update.call_args.args == ("us", 10, {"reform_label": "Renamed"}) + assert update.call_args.kwargs == {"record_mirror_event": True} + assert mirror.call_args.args == ("us", 10) + assert mirror.call_args.kwargs == { + "through_revision": 5, + "collaborators": UserPolicyMirrorCollaborators( + event_service=policy_routes.user_policy_service, + reform_snapshot_loader=policy_routes.policy_service.get_policy_snapshot, + ), + } + + +@pytest.mark.parametrize( + ("body", "message"), + [ + ([], "Request body must include an 'id' field."), + ({"reform_label": "x"}, "Request body must include an 'id' field."), + ( + {"id": 10, "user_id": "x", "reform_id": 3}, + "Request body contains unsupported fields: ['reform_id', 'user_id']", + ), + ({"id": 10}, "Request body must include at least one field to update."), + ], +) +def test_invalid_update_requests_are_rejected_before_configuration( + client, + write_source: MagicMock, + body: object, + message: str, +) -> None: + response = client.put("/us/user-policy", json=body) + + assert response.status_code == 400 + assert response.get_json() == {"message": message} + write_source.assert_not_called() + + +@pytest.mark.usefixtures("write_source") +def test_update_of_an_unknown_saved_policy_is_not_found( + client, + mirror: MagicMock, +) -> None: + with patch.object( + policy_routes.user_policy_service, "update_user_policy", return_value=None + ): + response = _update(client) + + assert response.status_code == 404 + assert response.get_json() == {"message": "User policy #10 not found."} + mirror.assert_not_called() diff --git a/tests/unit/services/test_mirror_log_payloads.py b/tests/unit/services/test_mirror_log_payloads.py new file mode 100644 index 000000000..493b600de --- /dev/null +++ b/tests/unit/services/test_mirror_log_payloads.py @@ -0,0 +1,467 @@ +"""Pin the complete metric log record of every immediate v1-to-v2 mirror outcome.""" + +from __future__ import annotations + +from contextlib import nullcontext +from unittest.mock import MagicMock, patch +from uuid import UUID + +import pytest +from sqlalchemy.exc import OperationalError + +from policyengine_api.services.household_mirroring import ( + HouseholdMirrorUnavailableError, + process_household_event_after_commit, +) +from policyengine_api.services.household_service import PendingHouseholdMirrorEvent +from policyengine_api.services.policy_mirroring import ( + PolicyMirrorUnavailableError, + mirror_policy_after_commit, +) +from policyengine_api.services.user_policy_mirroring import ( + UserPolicyMirrorCollaborators, + UserPolicyMirrorUnavailableError, + mirror_pending_user_policy_events_after_commit, + mirror_user_policy_after_commit, +) +from policyengine_api.services.user_policy_service import PendingUserPolicyMirrorEvent +from policyengine_api.services.v2.households.types import ( + LegacyHouseholdPersistenceResult, + LegacyHouseholdSnapshot, +) +from policyengine_api.services.v2.policies.types import ( + LegacyPolicyPersistenceResult, + LegacyPolicySnapshot, +) +from policyengine_api.services.v2.user_policies.types import ( + LegacyUserPolicyPersistenceResult, + LegacyUserPolicySnapshot, +) + +DESTINATION_ID = UUID("00000000-0000-0000-0000-000000000010") +ASSOCIATION_ID = UUID("00000000-0000-0000-0000-000000000060") +DATABASE_ERROR = OperationalError("statement", {}, Exception("database")) + + +def _logged(logger: MagicMock) -> tuple[dict[str, object], str]: + """Return the one logged record without its timing, and its severity.""" + logger.log_struct.assert_called_once() + payload = dict(logger.log_struct.call_args.args[0]) + assert isinstance(payload.pop("duration_ms"), float) + return payload, logger.log_struct.call_args.kwargs["severity"] + + +def _household_snapshot() -> LegacyHouseholdSnapshot: + return LegacyHouseholdSnapshot( + country_id="us", + legacy_household_id=42, + api_version="1.0.0", + household_json={"people": {}}, + source_household_hash="source-hash", + ) + + +def _household_event_service(*, fail_after_destination: bool) -> MagicMock: + service = MagicMock() + + def process(_country_id, _legacy_household_id, **callbacks): + event = PendingHouseholdMirrorEvent( + event_id=7, + snapshot=_household_snapshot(), + source_fingerprint_sha256="f" * 64, + was_processed=False, + ) + result = callbacks["processor"](event) + if fail_after_destination: + raise DATABASE_ERROR + callbacks["after_processed_commit"](event, result) + return result + + service.process_mirror_event.side_effect = process + return service + + +def _household_record(**changes: object) -> dict[str, object]: + record: dict[str, object] = { + "message": "V1 household immediate copy completed", + "metric_name": "v1_household_mirror_operations", + "metric_value": 1, + "configured_write_source": "dual_write", + "attempted_write_sources": ["cloud_sql", "supabase"], + "actual_write_sources": ["cloud_sql", "supabase"], + "country_id": "us", + "legacy_household_id": 42, + "destination_household_id": str(DESTINATION_ID), + "outcome": "ok", + "failure_category": None, + "household_created": True, + "mapping_created": False, + "pending_event": False, + } + record.update(changes) + return record + + +@pytest.mark.parametrize( + ("failing_write", "expected", "severity"), + [ + (None, _household_record(), "INFO"), + ( + "source", + _household_record( + outcome="error", failure_category="database", pending_event=True + ), + "ERROR", + ), + ( + "destination", + _household_record( + actual_write_sources=["cloud_sql"], + destination_household_id=None, + outcome="error", + failure_category="database", + household_created=None, + mapping_created=None, + pending_event=True, + ), + "ERROR", + ), + ], +) +def test_household_copy_logs_one_complete_record( + failing_write: str | None, + expected: dict[str, object], + severity: str, +) -> None: + mirror = MagicMock() + if failing_write == "destination": + mirror.mirror_legacy_household.side_effect = DATABASE_ERROR + else: + mirror.mirror_legacy_household.return_value = LegacyHouseholdPersistenceResult( + household_id=DESTINATION_ID, + household_created=True, + mapping_created=False, + ) + + expected_failure = ( + nullcontext() + if failing_write is None + else pytest.raises(HouseholdMirrorUnavailableError) + ) + with ( + patch("policyengine_api.services.household_mirroring.logger") as logger, + expected_failure as raised, + ): + process_household_event_after_commit( + "us", + 42, + event_service=_household_event_service( + fail_after_destination=failing_write == "source" + ), + mirror_factory=lambda: mirror, + ) + + if failing_write is not None: + assert raised.value.__cause__ is DATABASE_ERROR + + payload, logged_severity = _logged(logger) + assert list(payload) == list(expected) + assert payload == expected + assert logged_severity == severity + + +def _policy_snapshot() -> LegacyPolicySnapshot: + return LegacyPolicySnapshot( + country_id="uk", + legacy_policy_id=5, + api_version="1.0.0", + policy_json={}, + source_policy_hash="legacy-hash", + ) + + +def test_policy_mirror_success_logs_one_complete_record() -> None: + mirror = MagicMock() + mirror.mirror_legacy_policy.return_value = LegacyPolicyPersistenceResult( + policy_id=DESTINATION_ID, + policy_created=False, + mapping_created=True, + ) + + with patch("policyengine_api.services.policy_mirroring.logger") as logger: + mirror_policy_after_commit(_policy_snapshot(), mirror_factory=lambda: mirror) + + payload, severity = _logged(logger) + expected = { + "message": "V1 policy immediate mirror completed", + "metric_name": "v1_policy_mirror_operations", + "metric_value": 1, + "configured_write_source": "dual_write", + "attempted_write_sources": ["cloud_sql", "supabase"], + "actual_write_sources": ["cloud_sql", "supabase"], + "country_id": "uk", + "legacy_policy_id": 5, + "destination_policy_id": str(DESTINATION_ID), + "outcome": "ok", + "failure_category": None, + "policy_created": False, + "mapping_created": True, + } + assert list(payload) == list(expected) + assert payload == expected + assert severity == "INFO" + + +def test_policy_mirror_failure_logs_one_complete_record() -> None: + mirror = MagicMock() + mirror.mirror_legacy_policy.side_effect = DATABASE_ERROR + + with ( + patch("policyengine_api.services.policy_mirroring.logger") as logger, + pytest.raises(PolicyMirrorUnavailableError), + ): + mirror_policy_after_commit(_policy_snapshot(), mirror_factory=lambda: mirror) + + payload, severity = _logged(logger) + expected = { + "message": "V1 policy immediate mirror completed", + "metric_name": "v1_policy_mirror_operations", + "metric_value": 1, + "configured_write_source": "dual_write", + "attempted_write_sources": ["cloud_sql", "supabase"], + "actual_write_sources": ["cloud_sql"], + "country_id": "uk", + "legacy_policy_id": 5, + "destination_policy_id": None, + "outcome": "error", + "failure_category": "database", + "policy_created": None, + "mapping_created": None, + } + assert list(payload) == list(expected) + assert payload == expected + assert severity == "ERROR" + + +def _saved_snapshot(legacy_user_policy_id: int = 10) -> LegacyUserPolicySnapshot: + return LegacyUserPolicySnapshot( + country_id="us", + legacy_user_policy_id=legacy_user_policy_id, + reform_id=2, + baseline_id=1, + user_id="auth0|one", + year="2026", + geography="us", + number_of_provisions=3, + api_version="1.0.0", + added_date=1, + updated_date=2, + ) + + +def _user_policy_result() -> LegacyUserPolicyPersistenceResult: + return LegacyUserPolicyPersistenceResult( + association_id=ASSOCIATION_ID, + policy_id=DESTINATION_ID, + association_created=False, + association_updated=True, + mapping_created=False, + ) + + +def _user_policy_record(**changes: object) -> dict[str, object]: + record: dict[str, object] = { + "message": "V1 saved-policy immediate mirror completed", + "metric_name": "v1_user_policy_mirror_operations", + "metric_value": 1, + "configured_write_source": "dual_write", + "attempted_write_sources": ["cloud_sql", "supabase"], + "actual_write_sources": ["cloud_sql", "supabase"], + "country_id": "us", + "legacy_user_policy_id": 10, + "source_revision": 3, + "requested_through_revision": 3, + "destination_association_id": str(ASSOCIATION_ID), + "destination_policy_id": str(DESTINATION_ID), + "outcome": "ok", + "failure_category": None, + "association_created": False, + "association_updated": True, + "mapping_created": False, + } + record.update(changes) + return record + + +FAILED_USER_POLICY_RECORD = _user_policy_record( + actual_write_sources=["cloud_sql"], + destination_association_id=None, + destination_policy_id=None, + outcome="error", + failure_category="database", + association_created=None, + association_updated=None, + mapping_created=None, +) + + +@pytest.mark.parametrize("fails", [False, True]) +def test_single_saved_policy_mirror_logs_one_complete_record(fails) -> None: + mirror = MagicMock() + if fails: + mirror.mirror_legacy_user_policy.side_effect = DATABASE_ERROR + else: + mirror.mirror_legacy_user_policy.return_value = _user_policy_result() + + expected_failure = ( + pytest.raises(UserPolicyMirrorUnavailableError) if fails else nullcontext() + ) + with ( + patch("policyengine_api.services.user_policy_mirroring.logger") as logger, + expected_failure as raised, + ): + mirror_user_policy_after_commit( + _saved_snapshot(), + _policy_snapshot(), + source_revision=3, + mirror_factory=lambda: mirror, + ) + + if fails: + assert raised.value.__cause__ is DATABASE_ERROR + + payload, severity = _logged(logger) + expected = FAILED_USER_POLICY_RECORD if fails else _user_policy_record() + assert list(payload) == list(expected) + assert payload == expected + assert severity == ("ERROR" if fails else "INFO") + + +def _pending_event(source_revision: int) -> PendingUserPolicyMirrorEvent: + return PendingUserPolicyMirrorEvent( + event_id=source_revision, + source_revision=source_revision, + event_type="update", + snapshot=_saved_snapshot(legacy_user_policy_id=11), + changed_fields=frozenset({"reform_label"}), + source_fingerprint_sha256="f" * 64, + ) + + +def _pending_event_service(*, failure: Exception | None) -> MagicMock: + service = MagicMock() + + def process(_country_id, _legacy_user_policy_id, *, through_revision, **callbacks): + result = None + for revision in (2, through_revision): + event = _pending_event(revision) + result = callbacks["processor"](event) + if failure is not None and revision == through_revision: + raise failure + callbacks["after_processed_commit"](event, result) + return result + + service.process_pending_mirror_events.side_effect = process + return service + + +def test_pending_saved_policy_events_log_one_complete_record_each() -> None: + mirror = MagicMock() + mirror.mirror_legacy_user_policy.return_value = _user_policy_result() + + with patch("policyengine_api.services.user_policy_mirroring.logger") as logger: + result = mirror_pending_user_policy_events_after_commit( + "us", + 11, + through_revision=3, + collaborators=UserPolicyMirrorCollaborators( + event_service=_pending_event_service(failure=None), + reform_snapshot_loader=lambda _country_id, _policy_id: ( + _policy_snapshot() + ), + mirror_factory=lambda: mirror, + ), + ) + + assert result == _user_policy_result() + records = [] + for call in logger.log_struct.call_args_list: + payload = dict(call.args[0]) + assert isinstance(payload.pop("duration_ms"), float) + records.append((payload, call.kwargs["severity"])) + assert records == [ + ( + _user_policy_record(legacy_user_policy_id=11, source_revision=2), + "INFO", + ), + (_user_policy_record(legacy_user_policy_id=11), "INFO"), + ] + assert list(records[0][0]) == list(_user_policy_record()) + + +def test_pending_event_failure_logs_the_active_event_and_unwrapped_cause() -> None: + mirror = MagicMock() + mirror.mirror_legacy_user_policy.return_value = _user_policy_result() + wrapped = UserPolicyMirrorUnavailableError("wrapped") + wrapped.__cause__ = DATABASE_ERROR + + with ( + patch("policyengine_api.services.user_policy_mirroring.logger") as logger, + pytest.raises(UserPolicyMirrorUnavailableError) as raised, + ): + mirror_pending_user_policy_events_after_commit( + "us", + 11, + through_revision=3, + collaborators=UserPolicyMirrorCollaborators( + event_service=_pending_event_service(failure=wrapped), + reform_snapshot_loader=lambda _country_id, _policy_id: ( + _policy_snapshot() + ), + mirror_factory=lambda: mirror, + ), + ) + + assert raised.value.__cause__ is wrapped + payload = dict(logger.log_struct.call_args_list[-1].args[0]) + assert isinstance(payload.pop("duration_ms"), float) + expected = _user_policy_record( + legacy_user_policy_id=11, + outcome="error", + failure_category="database", + ) + assert list(payload) == list(expected) + assert payload == expected + assert logger.log_struct.call_args_list[-1].kwargs["severity"] == "ERROR" + + +def test_pending_event_failure_before_any_event_logs_the_request_identity() -> None: + event_service = MagicMock() + event_service.process_pending_mirror_events.side_effect = DATABASE_ERROR + + with ( + patch("policyengine_api.services.user_policy_mirroring.logger") as logger, + pytest.raises(UserPolicyMirrorUnavailableError), + ): + mirror_pending_user_policy_events_after_commit( + "uk", + 12, + through_revision=4, + collaborators=UserPolicyMirrorCollaborators( + event_service=event_service, + reform_snapshot_loader=lambda _country_id, _policy_id: None, + mirror_factory=MagicMock(), + ), + ) + + payload, severity = _logged(logger) + expected = { + **FAILED_USER_POLICY_RECORD, + "country_id": "uk", + "legacy_user_policy_id": 12, + "source_revision": None, + "requested_through_revision": 4, + } + assert list(payload) == list(_user_policy_record()) + assert payload == expected + assert severity == "ERROR" diff --git a/tests/unit/services/test_report_output_service.py b/tests/unit/services/test_report_output_service.py index 394cc5973..ae0d85d8a 100644 --- a/tests/unit/services/test_report_output_service.py +++ b/tests/unit/services/test_report_output_service.py @@ -5,7 +5,10 @@ from policyengine_api.constants import get_report_output_cache_version from policyengine_api.data.v1_models import ReportOutput, ReportOutputRun from policyengine_api.services.report_output_service import ReportOutputService -from policyengine_api.services.report_run_service import ReportRunService +from policyengine_api.services.report_run_service import ( + NewReportOutputRun, + ReportRunService, +) from policyengine_api.services.simulation_service import SimulationService run_service = ReportRunService() @@ -431,7 +434,7 @@ def _add_rerun(orm_session_factory, report_id): report = session.get(ReportOutput, report_id) successful_run_id = report.latest_successful_run_id rerun = run_service.create_report_output_run( - session, report.id, trigger_type="rerun" + session, report.id, NewReportOutputRun(trigger_type="rerun") ) report.active_run_id = rerun.id return rerun.id, successful_run_id diff --git a/tests/unit/services/test_report_run_service.py b/tests/unit/services/test_report_run_service.py index 565148313..3d866f4ba 100644 --- a/tests/unit/services/test_report_run_service.py +++ b/tests/unit/services/test_report_run_service.py @@ -1,7 +1,10 @@ import pytest from policyengine_api.data.v1_models import ReportOutput, ReportOutputRun -from policyengine_api.services.report_run_service import ReportRunService +from policyengine_api.services.report_run_service import ( + NewReportOutputRun, + ReportRunService, +) service = ReportRunService() @@ -26,12 +29,14 @@ def test_creates_mapped_runs_with_incrementing_sequence_and_python_json(orm_sess first = service.create_report_output_run( orm_session, report.id, - trigger_type="initial", - report_spec_snapshot={"country_id": "us"}, - version_manifest={"report_cache_version": "r123"}, + NewReportOutputRun( + trigger_type="initial", + report_spec_snapshot={"country_id": "us"}, + version_manifest={"report_cache_version": "r123"}, + ), ) second = service.create_report_output_run( - orm_session, report.id, trigger_type="rerun" + orm_session, report.id, NewReportOutputRun(trigger_type="rerun") ) assert isinstance(first, ReportOutputRun) @@ -58,7 +63,9 @@ def test_sets_run_timestamps_from_status( ): report = create_report(orm_session) - run = service.create_report_output_run(orm_session, report.id, status=status) + run = service.create_report_output_run( + orm_session, report.id, NewReportOutputRun(status=status) + ) assert (run.started_at is not None) is has_started assert (run.finished_at is not None) is has_finished @@ -71,8 +78,12 @@ def test_raises_when_parent_report_is_missing(orm_session): def test_gets_lists_and_selects_mapped_runs(orm_session): report = create_report(orm_session) - first = service.create_report_output_run(orm_session, report.id, status="complete") - second = service.create_report_output_run(orm_session, report.id, status="running") + first = service.create_report_output_run( + orm_session, report.id, NewReportOutputRun(status="complete") + ) + second = service.create_report_output_run( + orm_session, report.id, NewReportOutputRun(status="running") + ) report.latest_successful_run_id = first.id report.active_run_id = second.id @@ -87,8 +98,7 @@ def test_select_display_run_falls_back_to_matching_error(orm_session): matching = service.create_report_output_run( orm_session, report.id, - status="error", - error_message="failed", + NewReportOutputRun(status="error", error_message="failed"), ) service.create_report_output_run(orm_session, report.id) report.error_message = "failed" diff --git a/tests/unit/services/test_report_spec_service.py b/tests/unit/services/test_report_spec_service.py index ad0f4044c..f8567cd1a 100644 --- a/tests/unit/services/test_report_spec_service.py +++ b/tests/unit/services/test_report_spec_service.py @@ -2,6 +2,7 @@ from policyengine_api.data.v1_models import ReportOutput, Simulation from policyengine_api.services.report_spec_service import ( + EconomyReportSettings, EconomyReportSpec, HouseholdReportSpec, ReportSpecService, @@ -74,7 +75,10 @@ def test_builds_economy_spec_from_models(orm_session): report = add_report(orm_session, first, second) spec = service.build_report_spec( - report, first, second, dataset="cps", options={"foo": "bar"} + report, + first, + second, + EconomyReportSettings(dataset="cps", options={"foo": "bar"}), ) assert isinstance(spec, EconomyReportSpec) diff --git a/tests/unit/services/test_simulation_run_service.py b/tests/unit/services/test_simulation_run_service.py index f1d606ae0..94aa6f412 100644 --- a/tests/unit/services/test_simulation_run_service.py +++ b/tests/unit/services/test_simulation_run_service.py @@ -1,7 +1,10 @@ import pytest from policyengine_api.data.v1_models import SimulationRun -from policyengine_api.services.simulation_run_service import SimulationRunService +from policyengine_api.services.simulation_run_service import ( + NewSimulationRun, + SimulationRunService, +) from policyengine_api.services.simulation_service import SimulationService run_service = SimulationRunService() @@ -20,13 +23,17 @@ def test_creates_mapped_runs_with_incrementing_sequence(orm_session): first = run_service.create_simulation_run( orm_session, simulation.id, - input_position=1, - trigger_type="rerun", - simulation_spec_snapshot={"population_id": "household-1"}, - version_manifest={"simulation_cache_version": "s123"}, + NewSimulationRun( + input_position=1, + trigger_type="rerun", + simulation_spec_snapshot={"population_id": "household-1"}, + version_manifest={"simulation_cache_version": "s123"}, + ), ) second = run_service.create_simulation_run( - orm_session, simulation.id, input_position=1, trigger_type="rerun" + orm_session, + simulation.id, + NewSimulationRun(input_position=1, trigger_type="rerun"), ) assert isinstance(first, SimulationRun) @@ -57,10 +64,10 @@ def test_gets_and_lists_runs_as_models(orm_session): def test_display_run_prefers_active_run(orm_session): simulation = create_simulation(orm_session) successful = run_service.create_simulation_run( - orm_session, simulation.id, status="complete" + orm_session, simulation.id, NewSimulationRun(status="complete") ) active = run_service.create_simulation_run( - orm_session, simulation.id, status="running" + orm_session, simulation.id, NewSimulationRun(status="running") ) simulation.latest_successful_run_id = successful.id simulation.active_run_id = active.id @@ -71,7 +78,7 @@ def test_display_run_prefers_active_run(orm_session): def test_display_run_falls_back_to_latest_successful_run(orm_session): simulation = create_simulation(orm_session) successful = run_service.create_simulation_run( - orm_session, simulation.id, status="complete" + orm_session, simulation.id, NewSimulationRun(status="complete") ) run_service.create_simulation_run(orm_session, simulation.id) simulation.active_run_id = None diff --git a/tests/unit/services/test_user_policy_mirror_event_decoding.py b/tests/unit/services/test_user_policy_mirror_event_decoding.py new file mode 100644 index 000000000..aa4cc0f2c --- /dev/null +++ b/tests/unit/services/test_user_policy_mirror_event_decoding.py @@ -0,0 +1,153 @@ +"""Pin how a retained saved-policy mirror event is decoded or rejected.""" + +from __future__ import annotations + +import pytest + +from policyengine_api.data.v1_models import UserPolicyMirrorEvent +from policyengine_api.services.user_policy_service import ( + PendingUserPolicyMirrorEvent, + UserPolicyMirrorEventIntegrityError, + UserPolicyService, +) +from policyengine_api.services.v2.user_policies.transformations import ( + fingerprint_legacy_user_policy, +) +from policyengine_api.services.v2.user_policies.types import LegacyUserPolicySnapshot + +SNAPSHOT = LegacyUserPolicySnapshot( + country_id="us", + legacy_user_policy_id=10, + reform_id=2, + baseline_id=1, + user_id="auth0|one", + year="2026", + geography="us", + number_of_provisions=3, + api_version="1.0.0", + added_date=1, + updated_date=2, +) + + +def _event(**changes: object) -> UserPolicyMirrorEvent: + fields: dict[str, object] = { + "id": 7, + "country_id": "us", + "legacy_user_policy_id": 10, + "source_revision": 2, + "event_type": "update", + "payload_schema_version": 1, + "payload_json": { + "snapshot": SNAPSHOT.model_dump(mode="json"), + "changed_fields": ["reform_label", "year"], + }, + "source_fingerprint_sha256": fingerprint_legacy_user_policy(SNAPSHOT), + } + fields.update(changes) + return UserPolicyMirrorEvent(**fields) + + +def _payload(**changes: object) -> dict[str, object]: + payload: dict[str, object] = { + "snapshot": SNAPSHOT.model_dump(mode="json"), + "changed_fields": ["reform_label", "year"], + } + payload.update(changes) + return payload + + +def test_valid_update_event_decodes_to_its_pending_revision() -> None: + decoded = UserPolicyService._decode_mirror_event(_event()) + + assert decoded == PendingUserPolicyMirrorEvent( + event_id=7, + source_revision=2, + event_type="update", + snapshot=SNAPSHOT, + changed_fields=frozenset({"reform_label", "year"}), + source_fingerprint_sha256=fingerprint_legacy_user_policy(SNAPSHOT), + ) + + +def test_valid_create_event_has_no_changed_fields() -> None: + decoded = UserPolicyService._decode_mirror_event( + _event(event_type="create", payload_json=_payload(changed_fields=[])) + ) + + assert decoded.event_type == "create" + assert decoded.changed_fields == frozenset() + + +@pytest.mark.parametrize( + ("changes", "message"), + [ + ( + {"payload_schema_version": 2}, + "saved-policy mirror event payload version is unsupported", + ), + ( + {"payload_json": ["snapshot"]}, + "saved-policy mirror event payload has an invalid shape", + ), + ( + {"payload_json": {"snapshot": {}}}, + "saved-policy mirror event payload has an invalid shape", + ), + ( + {"payload_json": _payload(changed_fields="year")}, + "saved-policy mirror event changed fields are invalid", + ), + ( + {"payload_json": _payload(changed_fields=[1])}, + "saved-policy mirror event changed fields are invalid", + ), + ( + {"payload_json": _payload(changed_fields=["year", "reform_label"])}, + "saved-policy mirror event changed fields are unsupported", + ), + ( + {"payload_json": _payload(changed_fields=["year", "year"])}, + "saved-policy mirror event changed fields are unsupported", + ), + ( + {"payload_json": _payload(changed_fields=["user_id"])}, + "saved-policy mirror event changed fields are unsupported", + ), + ( + {"payload_json": _payload(snapshot={"country_id": "us"})}, + "saved-policy mirror event snapshot is invalid", + ), + ( + {"payload_json": _payload(snapshot=["us"])}, + "saved-policy mirror event snapshot is invalid", + ), + ( + {"country_id": "uk"}, + "saved-policy mirror event source identity conflicts with its payload", + ), + ( + {"legacy_user_policy_id": 11}, + "saved-policy mirror event source identity conflicts with its payload", + ), + ( + {"source_fingerprint_sha256": "f" * 64}, + "saved-policy mirror event fingerprint conflicts with its payload", + ), + ({"event_type": "delete"}, "saved-policy mirror event type is unsupported"), + ( + {"event_type": "create"}, + "saved-policy create mirror event contains changed fields", + ), + ( + {"payload_json": _payload(changed_fields=[])}, + "saved-policy update mirror event has no changed fields", + ), + ], +) +def test_invalid_events_are_integrity_errors_with_specific_messages( + changes: dict[str, object], + message: str, +) -> None: + with pytest.raises(UserPolicyMirrorEventIntegrityError, match=rf"^{message}$"): + UserPolicyService._decode_mirror_event(_event(**changes)) diff --git a/tests/unit/services/test_user_policy_mirroring.py b/tests/unit/services/test_user_policy_mirroring.py index 681e2bb4b..df8e7facf 100644 --- a/tests/unit/services/test_user_policy_mirroring.py +++ b/tests/unit/services/test_user_policy_mirroring.py @@ -13,6 +13,7 @@ from policyengine_api.data.v1_models import UserPolicyMirrorEvent from policyengine_api.services.user_policy_mirroring import ( + UserPolicyMirrorCollaborators, UserPolicyMirrorUnavailableError, mirror_pending_user_policy_events_after_commit, mirror_user_policy_after_commit, @@ -206,9 +207,11 @@ def fail_source_commit(_session) -> None: "us", creation.user_policy.id, through_revision=creation.mirror_revision, - event_service=event_service, - reform_snapshot_loader=lambda _country_id, _policy_id: _reform(), - mirror_factory=lambda: mirror, + collaborators=UserPolicyMirrorCollaborators( + event_service=event_service, + reform_snapshot_loader=lambda _country_id, _policy_id: _reform(), + mirror_factory=lambda: mirror, + ), ) finally: sqlalchemy_event.remove( @@ -249,9 +252,11 @@ def test_event_preparation_failure_logs_error_and_retains_the_event( "us", creation.user_policy.id, through_revision=creation.mirror_revision, - event_service=event_service, - reform_snapshot_loader=lambda _country_id, _policy_id: None, - mirror_factory=mirror_factory, + collaborators=UserPolicyMirrorCollaborators( + event_service=event_service, + reform_snapshot_loader=lambda _country_id, _policy_id: None, + mirror_factory=mirror_factory, + ), ) mirror_factory.assert_not_called() diff --git a/tests/unit/test_asgi_factory.py b/tests/unit/test_asgi_factory.py index 51aaa3d7a..56c037bac 100644 --- a/tests/unit/test_asgi_factory.py +++ b/tests/unit/test_asgi_factory.py @@ -272,7 +272,7 @@ def test_native_readiness_uses_shared_asgi_middleware(): def test_native_route_preserves_request_id_in_context_log_and_response(): observed_context_ids = [] - def capture_request(**_kwargs): + def capture_request(_served_request): observed_context_ids.append(current_request_id()) with patch( @@ -285,14 +285,14 @@ def capture_request(**_kwargs): ) assert response.headers[REQUEST_ID_HEADER] == "request-123" - assert log_request.call_args.kwargs["request_id"] == "request-123" + assert log_request.call_args.args[0].request_id == "request-123" assert observed_context_ids == ["request-123"] def test_native_route_generates_one_request_id_for_context_log_and_response(): observed_context_ids = [] - def capture_request(**_kwargs): + def capture_request(_served_request): observed_context_ids.append(current_request_id()) with ( @@ -308,7 +308,7 @@ def capture_request(**_kwargs): response = TestClient(create_asgi_app(create_test_wsgi_app())).get("/health") assert response.headers[REQUEST_ID_HEADER] == "generated-request-id" - assert log_request.call_args.kwargs["request_id"] == "generated-request-id" + assert log_request.call_args.args[0].request_id == "generated-request-id" assert observed_context_ids == ["generated-request-id"] generate_request_id.assert_called_once_with() @@ -445,7 +445,7 @@ def test_native_route_does_not_accept_x_request_id_as_an_alias(): ) assert response.headers[REQUEST_ID_HEADER] == "generated-request-id" - assert log_request.call_args.kwargs["request_id"] == "generated-request-id" + assert log_request.call_args.args[0].request_id == "generated-request-id" def test_asgi_request_context_is_reset_after_response(): @@ -465,10 +465,10 @@ def test_concurrent_asgi_requests_do_not_leak_request_ids(): observed_context_ids = {} observed_lock = threading.Lock() - def capture_request(**kwargs): + def capture_request(served_request): barrier.wait(timeout=5) with observed_lock: - observed_context_ids[kwargs["request_id"]] = current_request_id() + observed_context_ids[served_request.request_id] = current_request_id() app = create_asgi_app(create_test_wsgi_app()) diff --git a/tests/unit/test_asgi_request_lifecycle.py b/tests/unit/test_asgi_request_lifecycle.py new file mode 100644 index 000000000..3610caded --- /dev/null +++ b/tests/unit/test_asgi_request_lifecycle.py @@ -0,0 +1,197 @@ +"""Pin the order and content of the ASGI shell's per-request telemetry calls.""" + +from __future__ import annotations + +from dataclasses import replace +from unittest.mock import ANY, Mock, call, patch + +from fastapi.testclient import TestClient +from flask import Flask, make_response + +from policyengine_api import asgi_factory +from policyengine_api.asgi_factory import create_asgi_app +from policyengine_api.migration_flags import ( + RouteImplementation, + RouteImplementationSettings, +) +from policyengine_api.migration_logging import ServedRequest +from policyengine_api.observability.identifiers import OBSERVABILITY_ID_HEADER +from policyengine_api.request_context import REQUEST_ID_HEADER + +OBSERVABILITY_ID = "00000000-0000-4000-8000-000000000001" + + +def _flask_app() -> Flask: + app = Flask(__name__) + + @app.get("/invalid-observability-id") + def invalid_observability_id(): + response = make_response("fallback", 200) + response.headers[OBSERVABILITY_ID_HEADER] = "not-an-observability-id" + return response + + return app + + +def _native_health_app(runtime: Mock, readiness_probe=lambda: True): + dependencies = replace( + asgi_factory.NativeRouteDependencies( + readiness_probe=lambda: True, + gateway_client_factory=lambda: None, + metadata_reader_factory=lambda: None, + specification_provider=dict, + ), + readiness_probe=readiness_probe, + ) + return create_asgi_app( + _flask_app(), + route_settings=RouteImplementationSettings( + health=RouteImplementation.FASTAPI_NATIVE, + specification=RouteImplementation.FLASK_FALLBACK, + metadata=RouteImplementation.FLASK_FALLBACK, + ), + dependencies=dependencies, + observability_runtime=runtime, + ) + + +def test_native_request_calls_the_runtime_and_logger_in_a_fixed_order() -> None: + timeline = Mock() + timeline.runtime.begin_request.return_value = "runtime-request-id" + timeline.runtime.response_headers.return_value = {"x-trace": "trace-1"} + + with ( + patch.object(asgi_factory, "log_migration_request", timeline.log), + patch.object( + asgi_factory, "current_observability_id", return_value=OBSERVABILITY_ID + ), + ): + response = TestClient(_native_health_app(timeline.runtime)).get( + "/readiness-check?country_id=uk", + headers={REQUEST_ID_HEADER: "incoming-request-id"}, + ) + + assert response.status_code == 200 + assert response.headers[REQUEST_ID_HEADER] == "runtime-request-id" + assert response.headers[OBSERVABILITY_ID_HEADER] == OBSERVABILITY_ID + assert response.headers["x-trace"] == "trace-1" + begin_request = timeline.runtime.begin_request.call_args + assert begin_request.kwargs["headers"][REQUEST_ID_HEADER] == "incoming-request-id" + assert timeline.mock_calls == [ + call.runtime.begin_request(headers=ANY, method="GET", route="/readiness-check"), + call.runtime.set_context(request_id="runtime-request-id"), + call.runtime.set_context(observability_id=OBSERVABILITY_ID), + call.runtime.response_headers(), + call.log( + ServedRequest( + request_id="runtime-request-id", + method="GET", + path="/readiness-check", + status_code=200, + started_at=ANY, + country_id="uk", + route_impl=RouteImplementation.FASTAPI_NATIVE, + ) + ), + call.runtime.update_request_route("/readiness-check"), + call.runtime.update_request_status(200), + call.runtime.end_request(status_code=200, error=None), + ] + + +def test_native_request_keeps_its_id_when_the_runtime_assigns_none() -> None: + runtime = Mock() + runtime.begin_request.return_value = "" + runtime.response_headers.return_value = {} + + with patch.object(asgi_factory, "log_migration_request") as log_request: + response = TestClient(_native_health_app(runtime)).get( + "/readiness-check", + headers={REQUEST_ID_HEADER: "incoming-request-id"}, + ) + + assert response.headers[REQUEST_ID_HEADER] == "incoming-request-id" + runtime.set_context.assert_called_once_with(request_id="incoming-request-id") + assert log_request.call_args.args[0].request_id == "incoming-request-id" + assert log_request.call_args.args[0].country_id is None + + +def test_failing_native_route_logs_and_finishes_as_a_server_error() -> None: + timeline = Mock() + timeline.runtime.begin_request.return_value = None + route_error = RuntimeError("probe failed") + + def failing_probe() -> bool: + raise route_error + + with patch.object(asgi_factory, "log_migration_request", timeline.log): + response = TestClient( + _native_health_app(timeline.runtime, readiness_probe=failing_probe), + raise_server_exceptions=False, + ).get("/readiness-check", headers={REQUEST_ID_HEADER: "failed-request"}) + + assert response.status_code == 500 + assert response.text == "Internal Server Error" + assert response.headers[REQUEST_ID_HEADER] == "failed-request" + assert timeline.mock_calls == [ + call.runtime.begin_request(headers=ANY, method="GET", route="/readiness-check"), + call.runtime.set_context(request_id="failed-request"), + call.log( + ServedRequest( + request_id="failed-request", + method="GET", + path="/readiness-check", + status_code=500, + started_at=ANY, + country_id=None, + route_impl=RouteImplementation.FASTAPI_NATIVE, + ) + ), + call.runtime.update_request_route("/readiness-check"), + call.runtime.update_request_status(500), + call.runtime.end_request(status_code=500, error=route_error), + ] + + +def test_native_request_survives_a_failing_observability_context_update() -> None: + runtime = Mock() + runtime.begin_request.return_value = None + runtime.response_headers.return_value = {} + runtime.set_context.side_effect = RuntimeError("context unavailable") + + with ( + patch.object(asgi_factory, "log_migration_request", side_effect=OSError), + patch.object( + asgi_factory, "current_observability_id", return_value=OBSERVABILITY_ID + ), + ): + response = TestClient(_native_health_app(runtime)).get("/readiness-check") + + assert response.status_code == 200 + assert response.headers[OBSERVABILITY_ID_HEADER] == OBSERVABILITY_ID + runtime.end_request.assert_called_once_with(status_code=200, error=None) + + +def test_fallback_response_drops_an_invalid_observability_id_header() -> None: + runtime = Mock() + + response = TestClient(_native_health_app(runtime)).get( + "/invalid-observability-id", + headers={REQUEST_ID_HEADER: "fallback-request"}, + ) + + assert response.status_code == 200 + assert response.text == "fallback" + assert OBSERVABILITY_ID_HEADER not in response.headers + assert response.headers[REQUEST_ID_HEADER] == "fallback-request" + assert runtime.mock_calls == [] + + +def test_shutdown_callback_runs_once_when_the_lifespan_ends() -> None: + shutdown = Mock() + app = create_asgi_app(_flask_app(), shutdown_callback=shutdown) + + with TestClient(app): + shutdown.assert_not_called() + + shutdown.assert_called_once_with() diff --git a/tests/unit/test_migration_flags.py b/tests/unit/test_migration_flags.py index bcd90902b..38cf15e6b 100644 --- a/tests/unit/test_migration_flags.py +++ b/tests/unit/test_migration_flags.py @@ -155,8 +155,9 @@ def test_explicit_migration_context_rejects_invalid_database_sources( with pytest.raises(ValueError, match="invalid explicit database"): get_migration_context( "metadata", - use_configured_db_sources=False, - **explicit_sources, + explicit_db_sources=migration_flags.ExplicitDatabaseSources( + **explicit_sources + ), ) diff --git a/tests/unit/test_migration_request_records.py b/tests/unit/test_migration_request_records.py new file mode 100644 index 000000000..9a9b36f68 --- /dev/null +++ b/tests/unit/test_migration_request_records.py @@ -0,0 +1,287 @@ +"""Pin the migration request records, route groups and explicit database sources.""" + +from __future__ import annotations + +from unittest.mock import Mock, patch + +import pytest +from flask import Flask, Response + +from policyengine_api.migration_flags import ( + ExplicitDatabaseSources, + RouteImplementation, + get_migration_context, + get_migration_log_context, + infer_route_group, +) +from policyengine_api.migration_logging import ( + ServedRequest, + log_migration_request, + register_migration_request_logging, +) +from policyengine_api.observability.identifiers import OBSERVABILITY_ID_HEADER +from policyengine_api.request_context import REQUEST_ID_HEADER + +MIGRATION_ENVIRONMENT = ( + "ROUTE_IMPL_HEALTH", + "ROUTE_IMPL_POLICY", + "ROUTE_IMPL_METADATA", + "DB_READ_POLICY", + "DB_WRITE_POLICY", + "DB_READ_METADATA", + "DB_WRITE_METADATA", + "SIM_ENTRYPOINT", +) + + +@pytest.fixture(autouse=True) +def default_migration_flags(monkeypatch) -> None: + for name in MIGRATION_ENVIRONMENT: + monkeypatch.delenv(name, raising=False) + + +@pytest.mark.parametrize( + ("path", "route_group"), + [ + ("", "home"), + ("/", "home"), + ("//", "home"), + ("/health", "health"), + ("/liveness-check", "health"), + ("/specification", "specification"), + ("/v2/openapi.json", "specification"), + ("/health/extra", "health"), + ("/us/policy/1", "policy"), + ("/us/unknown/policy", "unknown"), + ("/v2/policies", "policy"), + ("/v2/us/policy", "policy"), + ("/v3/us/policy", "unknown"), + ("/v2", "unknown"), + ("/v2/us", "unknown"), + ("/policy", "policy"), + ], +) +def test_route_groups_follow_exact_paths_then_leading_segments( + path: str, + route_group: str, +) -> None: + assert infer_route_group(path) == route_group + + +def test_explicit_database_sources_replace_the_configured_sources() -> None: + context = get_migration_context( + "policy", + route_impl=RouteImplementation.FASTAPI_NATIVE, + explicit_db_sources=ExplicitDatabaseSources( + db_write_source="supabase", db_read_source=None + ), + ) + + assert context.to_log_dict() == { + "route_group": "policy", + "route_impl": "fastapi_native", + "db_entity": "policy", + "db_write": "supabase", + "db_read": None, + "sim_flow": None, + "sim_entrypoint": "old_gateway_direct", + "sim_compute": None, + } + + +def test_invalid_explicit_write_source_is_reported_before_the_read_source() -> None: + with pytest.raises( + ValueError, match=r"^invalid explicit database write source 'bad-write'$" + ): + get_migration_context( + "policy", + explicit_db_sources=ExplicitDatabaseSources( + db_write_source="bad-write", db_read_source="bad-read" + ), + ) + + +def test_invalid_flags_become_a_log_context_error_instead_of_raising( + monkeypatch, +) -> None: + monkeypatch.setenv("DB_WRITE_POLICY", "invalid-source") + + assert get_migration_log_context("policy") == { + "route_group": "policy", + "migration_flag_error": ( + "DB_WRITE_POLICY='invalid-source' is invalid; expected one of: " + "cloud_sql, dual_write, supabase" + ), + } + assert get_migration_log_context( + "policy", + explicit_db_sources=ExplicitDatabaseSources(db_write_source="unknown"), + ) == { + "route_group": "policy", + "migration_flag_error": "invalid explicit database write source 'unknown'", + } + + +@pytest.mark.parametrize( + ("status_code", "severity"), [(200, "INFO"), (499, "INFO"), (500, "ERROR")] +) +def test_request_record_has_a_fixed_shape_and_severity( + status_code: int, + severity: str, +) -> None: + with patch("policyengine_api.migration_logging.logger") as logger: + log_migration_request( + ServedRequest( + request_id="request-1", + method="GET", + path="/us/policy/3", + status_code=status_code, + started_at=None, + country_id="us", + route_impl=RouteImplementation.FLASK_FALLBACK, + ) + ) + + assert logger.log_struct.call_args.kwargs == {"severity": severity} + record = logger.log_struct.call_args.args[0] + assert list(record) == [ + "message", + "request_id", + "method", + "path", + "status_code", + "latency_ms", + "country_id", + "migration", + ] + assert record == { + "message": "API request served", + "request_id": "request-1", + "method": "GET", + "path": "/us/policy/3", + "status_code": status_code, + "latency_ms": None, + "country_id": "us", + "migration": { + "route_group": "policy", + "route_impl": "flask_fallback", + "db_entity": "policy", + "db_write": "cloud_sql", + "db_read": "cloud_sql", + "sim_flow": None, + "sim_entrypoint": "old_gateway_direct", + "sim_compute": None, + }, + } + + +def test_request_record_reports_latency_from_its_start_time() -> None: + with ( + patch("policyengine_api.migration_logging.logger") as logger, + patch("policyengine_api.migration_logging.time.time", return_value=12.5), + ): + log_migration_request( + ServedRequest( + request_id=None, + method="POST", + path="/v2/user-policies", + status_code=201, + started_at=10.0, + ) + ) + + record = logger.log_struct.call_args.args[0] + assert record["latency_ms"] == 2500.0 + assert record["country_id"] is None + assert record["migration"]["db_write"] == "supabase" + assert record["migration"]["db_read"] is None + assert record["migration"]["route_impl"] == "flask_fallback" + + +def _flask_app(runtime: Mock | None = None) -> Flask: + app = Flask(__name__) + app.config["TESTING"] = True + + @app.route("//policy/") + def policy(country_id: str, policy_id: int) -> Response: + return Response(f"{country_id}:{policy_id}", status=200) + + @app.route("/plain") + def plain() -> Response: + return Response("plain", status=200) + + register_migration_request_logging(app, runtime=runtime) + return app + + +def test_unparameterized_flask_route_logs_without_a_country() -> None: + with patch("policyengine_api.migration_logging.logger") as logger: + response = _flask_app().test_client().get("/plain") + + assert response.status_code == 200 + record = logger.log_struct.call_args.args[0] + assert record["country_id"] is None + assert record["path"] == "/plain" + assert response.headers[REQUEST_ID_HEADER] == record["request_id"] + + +def test_instrumented_flask_route_sets_the_runtime_context() -> None: + runtime = Mock() + runtime.capture_context.return_value = {"request_id": "captured-id"} + + with patch("policyengine_api.migration_logging.logger") as logger: + response = ( + _flask_app(runtime) + .test_client() + .get("/uk/policy/9", headers={REQUEST_ID_HEADER: "ignored-header"}) + ) + + assert response.headers[REQUEST_ID_HEADER] == "captured-id" + assert OBSERVABILITY_ID_HEADER not in response.headers + logger.log_struct.assert_not_called() + assert runtime.set_context.call_args_list[0].kwargs == {"request_id": "captured-id"} + assert runtime.set_context.call_args_list[1].kwargs == { + "country_id": "uk", + "route_group": "policy", + "route_impl": "flask_fallback", + "db_entity": "policy", + "db_write": "cloud_sql", + "db_read": "cloud_sql", + "sim_flow": None, + "sim_entrypoint": "old_gateway_direct", + "sim_compute": None, + } + + +def test_runtime_capture_failure_falls_back_to_the_request_header() -> None: + runtime = Mock() + runtime.capture_context.side_effect = RuntimeError("capture unavailable") + runtime.set_context.side_effect = RuntimeError("context unavailable") + + response = ( + _flask_app(runtime) + .test_client() + .get("/plain", headers={REQUEST_ID_HEADER: "header-id"}) + ) + + assert response.status_code == 200 + assert response.headers[REQUEST_ID_HEADER] == "header-id" + + +def test_logging_failures_never_replace_the_response() -> None: + app = _flask_app() + + with ( + patch( + "policyengine_api.migration_logging.log_migration_request", + side_effect=RuntimeError("logging unavailable"), + ), + patch.object( + app.logger, "exception", side_effect=RuntimeError("logger unavailable") + ) as log_exception, + ): + response = app.test_client().get("/us/policy/1") + + assert response.status_code == 200 + assert response.get_data(as_text=True) == "us:1" + log_exception.assert_called_once_with("Failed to log migration request context") diff --git a/tests/unit/test_simulation_and_report_routes.py b/tests/unit/test_simulation_and_report_routes.py index 559b260d8..9962d124b 100644 --- a/tests/unit/test_simulation_and_report_routes.py +++ b/tests/unit/test_simulation_and_report_routes.py @@ -10,7 +10,10 @@ from policyengine_api.routes.report_output_routes import report_output_bp from policyengine_api.routes.simulation_routes import simulation_bp from policyengine_api.services.report_output_service import ReportOutputService -from policyengine_api.services.report_run_service import ReportRunService +from policyengine_api.services.report_run_service import ( + NewReportOutputRun, + ReportRunService, +) from policyengine_api.services.simulation_service import SimulationService simulation_service = SimulationService() @@ -243,7 +246,7 @@ def test_report_patch_updates_active_rerun_and_preserves_success( report = session.get(ReportOutput, report_id) successful_id = report.latest_successful_run_id rerun = run_service.create_report_output_run( - session, report_id, trigger_type="rerun" + session, report_id, NewReportOutputRun(trigger_type="rerun") ) report.active_run_id = rerun.id rerun_id = rerun.id @@ -276,7 +279,7 @@ def test_report_patch_complete_promotes_active_rerun(orm_session_factory): with orm_session_factory.begin() as session: report = session.get(ReportOutput, report_id) rerun = run_service.create_report_output_run( - session, report_id, trigger_type="rerun" + session, report_id, NewReportOutputRun(trigger_type="rerun") ) report.active_run_id = rerun.id rerun_id = rerun.id diff --git a/tests/unit/utils/test_input_validation_edges.py b/tests/unit/utils/test_input_validation_edges.py new file mode 100644 index 000000000..2ec6f1f86 --- /dev/null +++ b/tests/unit/utils/test_input_validation_edges.py @@ -0,0 +1,125 @@ +"""Pin every household-input finding, in order, and JSON-safe metadata values.""" + +from __future__ import annotations + +import math + +import numpy as np +import pytest + +from policyengine_api.utils.input_validation import ( + UnrecognizedInput, + find_unrecognized_household_inputs, +) +from policyengine_api.utils.json import get_safe_json + +METADATA = { + "variables": { + "age": {"entity": "person"}, + "snap": {"entity": "spm_unit"}, + "region": {"entity": "region_without_plural"}, + "state_code": {"entity": "unknown_entity"}, + }, + "entities": { + "person": {"plural": "people", "roles": {}}, + "spm_unit": { + "plural": "spm_units", + "roles": {"member": {"plural": "members"}, "head": {}}, + }, + "region_without_plural": {}, + }, +} + + +@pytest.mark.parametrize("household", [None, [], "household"]) +def test_non_object_households_report_nothing(household: object) -> None: + assert find_unrecognized_household_inputs(household, METADATA) == [] + + +def test_household_findings_are_reported_in_document_order() -> None: + household = { + "people": { + "you": { + "age": {"2026": 40}, + "snap": {"2026": None}, + "agee": {"2026": 1}, + "region": {"2026": "x"}, + "state_code": {"2026": "CA"}, + }, + "partner": ["not", "an", "object"], + }, + "spm_units": {"unit": {"members": ["you"], "snapp": {"2026": 0}}}, + "tax_units": {"unit": {}}, + "households": ["not an object"], + "axes": [{"name": "agge"}, [{"name": "age"}, {"name": "snapp"}]], + } + + findings = find_unrecognized_household_inputs(household, METADATA) + + assert findings == [ + UnrecognizedInput( + input_type="household_variable_wrong_entity", + name="snap", + path="household.people.you.snap", + expected_entity_plural="spm_units", + actual_entity_plural="people", + ), + UnrecognizedInput( + input_type="household_variable", + name="agee", + path="household.people.you.agee", + ), + UnrecognizedInput( + input_type="household_variable", + name="snapp", + path="household.spm_units.unit.snapp", + ), + UnrecognizedInput( + input_type="household_entity", + name="tax_units", + path="household.tax_units", + ), + UnrecognizedInput( + input_type="household_entity", + name="households", + path="household.households", + ), + UnrecognizedInput( + input_type="household_axis_variable", + name="agge", + path="household.axes[0].name", + ), + UnrecognizedInput( + input_type="household_axis_variable", + name="snapp", + path="household.axes[1][1].name", + ), + ] + + +def test_known_entity_groups_that_are_not_objects_are_skipped() -> None: + assert find_unrecognized_household_inputs({"people": ["you"]}, METADATA) == [] + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (math.inf, "Infinity"), + (-math.inf, "-Infinity"), + (np.inf, "Infinity"), + (-np.inf, "-Infinity"), + (3, 3), + (2.5, 2.5), + (True, True), + ("text", "text"), + ({"a": [1, -math.inf, None]}, {"a": [1, "-Infinity", None]}), + (None, None), + ((1, 2), None), + (object, None), + ], +) +def test_safe_json_writes_infinities_as_strings( + value: object, + expected: object, +) -> None: + assert get_safe_json(value) == expected diff --git a/tests/unit/v2/test_catalog_extraction_failures.py b/tests/unit/v2/test_catalog_extraction_failures.py new file mode 100644 index 000000000..83c10d91a --- /dev/null +++ b/tests/unit/v2/test_catalog_extraction_failures.py @@ -0,0 +1,198 @@ +"""Pin the catalog extractor's error for each inconsistent public-model shape.""" + +from __future__ import annotations + +import math +from datetime import date +from enum import Enum +from typing import TYPE_CHECKING + +import pytest + +from policyengine_api.data.v2.catalog.extraction import ( + CatalogExtractionError, + extract_catalog, + normalize_json_value, +) +from tests.fixtures.v2_catalog import ( + POLICYENGINE_VERSION, + bundle, + installed_version, + source_models, +) + +if TYPE_CHECKING: + from types import SimpleNamespace + + +def _extract(models: dict[str, SimpleNamespace]) -> None: + extract_catalog( + bundle=bundle(), + policyengine_version=POLICYENGINE_VERSION, + models=models, + installed_version=installed_version, + ) + + +def _break_package_version(models: dict[str, SimpleNamespace]) -> None: + models["us"].model_package.version = "0.0.1" + + +def _break_manifest_country(models: dict[str, SimpleNamespace]) -> None: + models["us"].release_manifest.country_id = "uk" + + +def _break_manifest_version(models: dict[str, SimpleNamespace]) -> None: + models["us"].release_manifest.policyengine_version = "9.9.9" + + +def _remove_region_registry(models: dict[str, SimpleNamespace]) -> None: + models["us"].region_registry = None + + +def _mislabel_region_registry(models: dict[str, SimpleNamespace]) -> None: + models["us"].region_registry.country_id = "uk" + + +def _duplicate_region_codes(models: dict[str, SimpleNamespace]) -> None: + registry = models["us"].region_registry + registry.append(registry[1]) + + +def _remove_national_region(models: dict[str, SimpleNamespace]) -> None: + registry = models["us"].region_registry + del registry[0] + + +def _give_national_region_another_dataset(models: dict[str, SimpleNamespace]) -> None: + models["uk"].region_registry[0].dataset_path = "hf://policyengine/uk/other.h5" + + +def _give_filter_an_unsupported_value(models: dict[str, SimpleNamespace]) -> None: + models["us"].region_registry[1].scoping_strategy.variable_value = {"state": "CA"} + + +def _give_filter_additional_filters(models: dict[str, SimpleNamespace]) -> None: + models["uk"].region_registry[1].scoping_strategy.additional_filters = { + "region": "LONDON" + } + + +@pytest.mark.parametrize( + ("break_models", "message"), + [ + ( + _break_package_version, + "^us public model does not match the PolicyEngine.py manifest$", + ), + ( + _break_manifest_country, + "^us public model has an inconsistent release manifest$", + ), + ( + _break_manifest_version, + "^us public model was not certified by the installed PolicyEngine.py " + "version$", + ), + (_remove_region_registry, "^us public region registry is absent$"), + (_mislabel_region_registry, "^us public region registry is absent$"), + ( + _duplicate_region_codes, + r"^duplicate region natural keys: \['state/ca'\]$", + ), + (_remove_national_region, "^us national region is absent$"), + ( + _give_national_region_another_dataset, + "^uk national region does not use the reviewed dataset$", + ), + ( + _give_filter_an_unsupported_value, + "^region 'state/ca' has an unsupported filter value$", + ), + ( + _give_filter_additional_filters, + "^region 'country/england' has unsupported additional filters$", + ), + ], +) +def test_inconsistent_public_models_fail_with_a_specific_message( + break_models, + message: str, +) -> None: + models = source_models() + break_models(models) + + with pytest.raises(CatalogExtractionError, match=message): + _extract(models) + + +def test_national_region_without_a_dataset_path_uses_the_reviewed_dataset() -> None: + models = source_models() + models["us"].region_registry[0].dataset_path = None + + extract_catalog( + bundle=bundle(), + policyengine_version=POLICYENGINE_VERSION, + models=models, + installed_version=installed_version, + ) + + +class _ScalarWrapper: + """A numpy-like scalar whose item() returns a plain Python value.""" + + def __init__(self, scalar: object) -> None: + self._scalar = scalar + + def item(self) -> object: + return self._scalar + + +class _SelfReturningScalar: + """An object whose item() returns itself, so it never becomes a scalar.""" + + def item(self) -> object: + return self + + +def test_normalize_json_value_rejects_nan_and_keeps_infinities_as_strings() -> None: + with pytest.raises(CatalogExtractionError, match=r"^NaN is not JSON-compatible$"): + normalize_json_value(math.nan) + + assert normalize_json_value(math.inf) == "Infinity" + assert normalize_json_value(-math.inf) == "-Infinity" + + +def test_normalize_json_value_unwraps_scalars_and_rejects_self_returning_items() -> ( + None +): + assert normalize_json_value(_ScalarWrapper(3)) == 3 + assert normalize_json_value((1, [2, (3,)])) == [1, [2, [3]]] + with pytest.raises( + CatalogExtractionError, + match=r"^unsupported JSON value type .*_SelfReturningScalar$", + ): + normalize_json_value(_SelfReturningScalar()) + with pytest.raises( + CatalogExtractionError, match=r"^JSON object keys must be strings$" + ): + normalize_json_value({1: "one"}) + + +class _Colour(Enum): + RED = "red" + + +class _Pair(tuple, Enum): + FIRST = (1, 2) + + +class _Date(date, Enum): + LAUNCH = (2024, 1, 2) + + +def test_normalize_json_value_reads_enums_by_value_before_their_base_type() -> None: + assert normalize_json_value(_Colour.RED) == "red" + assert normalize_json_value(_Pair.FIRST) == [1, 2] + assert normalize_json_value(_Date.LAUNCH) == "2024-01-02" + assert normalize_json_value({"colour": [_Colour.RED]}) == {"colour": ["red"]} diff --git a/tests/unit/v2/test_catalog_staging_rows.py b/tests/unit/v2/test_catalog_staging_rows.py new file mode 100644 index 000000000..f43f2772f --- /dev/null +++ b/tests/unit/v2/test_catalog_staging_rows.py @@ -0,0 +1,123 @@ +"""Pin the rows that catalog publication COPYs into each staging table.""" + +from __future__ import annotations + +import inspect + +from psycopg.types.json import Jsonb + +from policyengine_api.data.v2.catalog.publication_staging import ( + STAGING_TABLES, + _catalog_rows, +) +from tests.fixtures.v2_catalog import normalized_catalog + + +def _plain(row: tuple[object, ...]) -> tuple[object, ...]: + return tuple( + ("jsonb", value.obj) if isinstance(value, Jsonb) else value for value in row + ) + + +def test_catalog_rows_stream_every_staging_table_lazily_in_table_order() -> None: + country = normalized_catalog().country("us") + + rows_by_table = _catalog_rows(country) + + assert list(rows_by_table) == list(STAGING_TABLES) + assert all(inspect.isgenerator(rows) for rows in rows_by_table.values()) + + +def test_catalog_rows_project_every_record_field_in_column_order() -> None: + country = normalized_catalog().country("us") + dataset_names = {dataset.id: dataset.name for dataset in country.datasets} + parameter_names = {parameter.id: parameter.name for parameter in country.parameters} + + rows = { + table.name: [_plain(row) for row in table_rows] + for table, table_rows in _catalog_rows(country).items() + } + + assert rows["stage_catalog_models"] == [ + ( + "us", + country.model.id, + country.model.name, + country.model.description, + country.model_version.id, + country.model_version.version, + country.model_version.description, + country.model_version.current_law_id, + ("jsonb", country.model_version.metadata_time_periods), + ) + ] + assert rows["stage_catalog_variables"] == [ + ( + "us", + record.id, + record.name, + record.label, + record.entity, + record.description, + record.data_type, + None + if record.possible_values is None + else ("jsonb", record.possible_values), + ("jsonb", record.default_value), + None if record.adds is None else ("jsonb", record.adds), + None if record.subtracts is None else ("jsonb", record.subtracts), + ) + for record in country.variables + ] + assert rows["stage_catalog_parameter_nodes"] == [ + ("us", record.id, record.name, record.label, record.description) + for record in country.parameter_nodes + ] + assert rows["stage_catalog_parameters"] == [ + ( + "us", + record.id, + record.name, + record.label, + record.description, + record.data_type, + record.unit, + ) + for record in country.parameters + ] + assert rows["stage_catalog_parameter_values"] == [ + ( + "us", + parameter_names[record.parameter_id], + record.id, + ("jsonb", record.value_json), + record.start_date, + record.end_date, + ) + for parameter in country.parameters + for record in parameter.values + ] + assert rows["stage_catalog_datasets"] == [ + ("us", record.id, record.name, record.description, record.year) + for record in country.datasets + ] + assert rows["stage_catalog_regions"] == [ + ( + "us", + record.id, + record.code, + record.label, + record.region_type, + record.requires_filter, + record.filter_field, + record.filter_value, + record.filter_strategy, + record.parent_code, + record.state_code, + record.state_name, + dataset_names[record.default_dataset_id], + ) + for record in country.regions + ] + assert len(rows["stage_catalog_parameter_values"]) == 2 + assert len(rows["stage_catalog_regions"]) == 3 diff --git a/tests/unit/v2/test_comparison_report_state.py b/tests/unit/v2/test_comparison_report_state.py new file mode 100644 index 000000000..de08057e6 --- /dev/null +++ b/tests/unit/v2/test_comparison_report_state.py @@ -0,0 +1,239 @@ +"""Pin every state rule of a comparison report record and their precedence.""" + +from __future__ import annotations + +import re +from datetime import UTC, datetime, timedelta +from uuid import UUID + +import pytest +from pydantic import ValidationError + +from policyengine_api.services.v2.comparison_runs.types import ComparisonReportRecord + +NOW = datetime(2026, 9, 14, tzinfo=UTC) +DIGEST = "a" * 64 +AGGREGATE_OUTPUT = { + "aggregate_output_uri": "gs://private-stage12/reports/aggregate.json", + "aggregate_output_sha256": DIGEST, + "aggregate_schema_version": 1, +} +COMPARISON_OUTPUT = { + "comparison_output_uri": "gs://private-stage12/reports/comparison.json", + "comparison_output_sha256": DIGEST, + "comparison_schema_version": 1, +} +SUCCEEDED = { + "status": "succeeded", + "aggregation_status": "succeeded", + "completed_at": NOW, + **AGGREGATE_OUTPUT, +} + + +def _report(**changes: object) -> ComparisonReportRecord: + fields: dict[str, object] = { + "evaluation_id": UUID("00000000-0000-4000-8000-000000000001"), + "status": "pending", + "aggregation_status": "not_started", + "environment": "test", + "calculation_flow": "economy", + "originating_request_id": "request-1", + "production_identity": "production-job-1", + "worker_version": "5.2.0", + "modal_application": "policyengine-v2-worker-5-2-0", + "report_coordinator_callable": "coordinate_report", + "version_manifest_sha256": DIGEST, + "policyengine_version": "5.2.0", + "country_package_name": "policyengine-us", + "country_package_version": "1.900.0", + "country": "us", + "dataset_identity": "populace_us_2024", + "dataset_uri": "hf://policyengine/populace-us/data.h5@revision", + "data_package_name": "policyengine-us-data", + "data_package_version": "1.0.0", + "data_artifact_revision": "revision", + "created_at": NOW, + "updated_at": NOW, + "retention_expires_at": NOW + timedelta(days=30), + } + fields.update(changes) + return ComparisonReportRecord.model_validate(fields) + + +@pytest.mark.parametrize( + "changes", + [ + {}, + SUCCEEDED, + {"status": "failed", "error_code": "worker_failed"}, + {"status": "skipped", "error_code": "not_applicable"}, + {"comparison_status": "pending"}, + {"comparison_status": "running"}, + {**SUCCEEDED, "comparison_status": "matched", "comparison_completed_at": NOW} + | COMPARISON_OUTPUT, + {**SUCCEEDED, "comparison_status": "different", "comparison_completed_at": NOW} + | COMPARISON_OUTPUT, + { + "comparison_status": "failed", + "comparison_completed_at": NOW, + "comparison_error_code": "compare_failed", + "comparison_error_summary": "The comparison failed.", + }, + ], +) +def test_consistent_reports_are_accepted(changes: dict[str, object]) -> None: + assert _report(**changes).evaluation_id is not None + + +@pytest.mark.parametrize( + ("changes", "message"), + [ + ( + {"created_at": NOW.replace(tzinfo=None)}, + "comparison retention timestamps must include a timezone", + ), + ( + {"aggregate_output_uri": AGGREGATE_OUTPUT["aggregate_output_uri"]}, + "aggregate output fields must be supplied together", + ), + ( + {**SUCCEEDED, "aggregation_status": "running"}, + "a successful report requires successful aggregation", + ), + ( + {"status": "succeeded", "aggregation_status": "succeeded"}, + "a successful report requires an aggregate output", + ), + ( + {**SUCCEEDED, "completed_at": None}, + "a successful report requires completed_at", + ), + ({"status": "failed"}, "a failed or skipped report requires an error_code"), + ({"status": "skipped"}, "a failed or skipped report requires an error_code"), + ( + {"comparison_output_uri": COMPARISON_OUTPUT["comparison_output_uri"]}, + "an incomplete comparison cannot reference an output artifact", + ), + ( + {"comparison_status": "pending", "comparison_completed_at": NOW}, + "an incomplete comparison cannot have completed_at", + ), + ( + {"comparison_status": "running", "comparison_error_code": "early"}, + "an incomplete comparison cannot have an error_code", + ), + ( + {"comparison_error_summary": "early"}, + "an incomplete comparison cannot have an error_summary", + ), + ( + {"comparison_status": "matched", "comparison_completed_at": NOW}, + "a completed comparison requires an output artifact", + ), + ( + {"comparison_status": "different", **COMPARISON_OUTPUT}, + "a completed comparison requires completed_at", + ), + ( + { + "comparison_status": "matched", + "comparison_completed_at": NOW, + "comparison_error_code": "late", + **COMPARISON_OUTPUT, + }, + "a completed comparison cannot have an error_code", + ), + ( + { + "comparison_status": "matched", + "comparison_completed_at": NOW, + "comparison_error_summary": "late", + **COMPARISON_OUTPUT, + }, + "a completed comparison cannot have an error_summary", + ), + ( + { + "comparison_status": "failed", + "comparison_completed_at": NOW, + "comparison_error_code": "compare_failed", + "comparison_schema_version": 1, + }, + "a failed comparison cannot reference an output artifact", + ), + ( + {"comparison_status": "failed", "comparison_error_code": "compare_failed"}, + "a failed comparison requires completed_at", + ), + ( + {"comparison_status": "failed", "comparison_completed_at": NOW}, + "a failed comparison requires an error_code", + ), + ], +) +def test_each_inconsistent_state_is_rejected_with_its_message( + changes: dict[str, object], + message: str, +) -> None: + with pytest.raises(ValidationError, match=re.escape(message)): + _report(**changes) + + +@pytest.mark.parametrize( + ("changes", "message"), + [ + ( + { + "retention_expires_at": NOW + timedelta(days=31), + "aggregate_output_uri": AGGREGATE_OUTPUT["aggregate_output_uri"], + }, + "comparison retention must not exceed 30 days", + ), + ( + { + "status": "succeeded", + "aggregation_status": "running", + "aggregate_output_uri": AGGREGATE_OUTPUT["aggregate_output_uri"], + }, + "aggregate output fields must be supplied together", + ), + ( + {"status": "succeeded", "aggregation_status": "running"}, + "a successful report requires successful aggregation", + ), + ( + { + "status": "failed", + "comparison_status": "failed", + }, + "a failed or skipped report requires an error_code", + ), + ( + { + "comparison_completed_at": NOW, + "comparison_error_code": "early", + **COMPARISON_OUTPUT, + }, + "an incomplete comparison cannot reference an output artifact", + ), + ( + {"comparison_status": "matched", "comparison_error_code": "late"}, + "a completed comparison requires an output artifact", + ), + ( + {"comparison_status": "failed", **COMPARISON_OUTPUT}, + "a failed comparison cannot reference an output artifact", + ), + ], +) +def test_the_first_violated_rule_is_the_one_reported( + changes: dict[str, object], + message: str, +) -> None: + with pytest.raises(ValidationError) as raised: + _report(**changes) + + errors = raised.value.errors() + assert len(errors) == 1 + assert errors[0]["msg"] == f"Value error, {message}" diff --git a/tests/unit/v2/test_household_document_failures.py b/tests/unit/v2/test_household_document_failures.py new file mode 100644 index 000000000..7749463d2 --- /dev/null +++ b/tests/unit/v2/test_household_document_failures.py @@ -0,0 +1,255 @@ +"""Pin each structural failure of v2 household documents and v1 translation.""" + +from __future__ import annotations + +import re + +import pytest + +from policyengine_api.services.v2.households.transformations import ( + translate_legacy_household, +) +from policyengine_api.services.v2.households.types import LegacyHouseholdSnapshot +from policyengine_api.services.v2.households.validators import ( + HouseholdValidationError, + LegacyHouseholdTranslationError, + normalize_household_document, + require_json_value, +) +from policyengine_api.services.v2.policies.validators import ( + require_json_value as require_policy_json_value, +) + +UK_PERSON = { + "id": "person-1", + "values": {"age": {"2026": 40}}, + "memberships": {"household": "household-1", "benunit": "benunit-1"}, +} + + +def _uk_document(**changes: object) -> dict[str, object]: + document: dict[str, object] = { + "people": [dict(UK_PERSON)], + "household": [{"id": "household-1", "values": {}}], + "benunit": [{"id": "benunit-1", "values": {}}], + } + document.update(changes) + return document + + +@pytest.mark.parametrize( + ("country_id", "document", "message"), + [ + ("fr", _uk_document(), "unsupported country_id 'fr'"), + ("uk", [], "household_data must be an object"), + ( + "uk", + _uk_document(spm={}), + "household_data contains unsupported collections: ['spm']", + ), + ("uk", {"household": [], "benunit": []}, "household_data must contain people"), + ("us", {"people": [], "spm": []}, "spm must be an object"), + ("us", {"people": [], "spm": {"unknown": 1}}, "spm selection is invalid"), + ("uk", _uk_document(household={}), "household must be a list"), + ("uk", _uk_document(people=["person-1"]), "people records must be objects"), + ( + "uk", + _uk_document(household=[{"id": "household-1", "colour": "blue"}]), + "household record fields are invalid: missing=['values'], " + "unsupported=['colour']", + ), + ( + "uk", + _uk_document(household=[{"id": " ", "values": {}}]), + "household id must be a non-empty string of at most 255 characters", + ), + ( + "uk", + _uk_document(household=[{"id": "h" * 256, "values": {}}]), + "household id must be a non-empty string of at most 255 characters", + ), + ( + "uk", + _uk_document( + household=[{"id": "household-1", "source_name": "", "values": {}}] + ), + "household source_name must be a non-empty string of at most 255 " + "characters", + ), + ( + "uk", + _uk_document(household=[{"id": "household-1", "values": []}]), + "household record values must be an object", + ), + ( + "uk", + _uk_document(people=[{**UK_PERSON, "memberships": []}]), + "people record memberships must be an object", + ), + ( + "uk", + _uk_document( + people=[{**UK_PERSON, "memberships": {"household": 1, "benunit": "b"}}] + ), + "person household membership must be a non-empty string of at most 255 " + "characters", + ), + ], +) +def test_invalid_documents_fail_with_their_specific_message( + country_id: str, + document: object, + message: str, +) -> None: + with pytest.raises(HouseholdValidationError, match=rf"^{re.escape(message)}$"): + normalize_household_document(country_id, document) + + +def test_normalized_documents_list_spm_first_then_people_and_entities() -> None: + document = { + "family": [], + "tax_unit": [], + "spm_unit": [], + "marital_unit": [], + "household": [], + "people": [], + "spm": {}, + } + + normalized = normalize_household_document("us", document) + + assert list(normalized) == [ + "spm", + "people", + "household", + "family", + "tax_unit", + "spm_unit", + "marital_unit", + ] + + +@pytest.mark.parametrize( + "require_value", [require_json_value, require_policy_json_value] +) +def test_json_values_reject_deep_nesting_and_reference_cycles(require_value) -> None: + nested: list[object] = [] + deepest = nested + for _ in range(101): + child: list[object] = [] + deepest.append(child) + deepest = child + cyclic: dict[str, object] = {} + cyclic["self"] = cyclic + + with pytest.raises( + ValueError, match=r"^JSON values must not exceed 100 nested containers$" + ): + require_value(nested) + with pytest.raises( + ValueError, match=r"^JSON values must not contain reference cycles$" + ): + require_value(cyclic) + with pytest.raises(ValueError, match=r"^JSON numbers must be finite$"): + require_value([float("inf")]) + with pytest.raises( + ValueError, match=r"^value must contain only standards-compliant JSON types$" + ): + require_value({"value": (1,)}) + with pytest.raises(ValueError, match=r"^JSON object keys must be strings$"): + require_value({1: "one"}) + shared = [1.5] + document = {"first": shared, "second": [shared, None, True, "text", 3]} + assert require_value(document) is document + + +def _snapshot(household_json: object) -> LegacyHouseholdSnapshot: + return LegacyHouseholdSnapshot.model_construct( + country_id="uk", + legacy_household_id=42, + label=None, + api_version="1.0.0", + household_json=household_json, + source_household_hash="legacy/base64+hash=", + ) + + +@pytest.mark.parametrize( + ("household_json", "message"), + [ + ( + {"people": {}, "households": {}, "families": {}}, + "legacy household contains unsupported collections: ['families']", + ), + ({"people": []}, "legacy people must be an object"), + ({"people": {}, "benunits": []}, "legacy benunits must be an object"), + ( + {"people": {}, "households": {"home": []}}, + "legacy households records must be objects", + ), + ( + {"people": {"adult": {}}, "households": {"home": {"members": "adult"}}}, + "legacy households members must reference named people", + ), + ( + { + "people": {"adult": {}}, + "benunits": { + "first": {"members": ["adult"]}, + "second": {"members": ["adult"]}, + }, + }, + "legacy person 'adult' belongs to multiple benunits records", + ), + ({"people": {"adult": []}}, "legacy person records must be objects"), + ( + {"people": {"a" * 256: {}}}, + "people source_name must be a non-empty string of at most 255 characters", + ), + ], +) +def test_untranslatable_v1_documents_fail_with_their_specific_message( + household_json: dict[str, object], + message: str, +) -> None: + with pytest.raises( + LegacyHouseholdTranslationError, match=rf"^{re.escape(message)}$" + ): + translate_legacy_household(_snapshot(household_json)) + + +def test_non_object_v1_document_is_a_type_error() -> None: + with pytest.raises(TypeError, match=r"^legacy household_json must be an object$"): + translate_legacy_household(_snapshot(["people"])) + + +def test_repeated_membership_in_one_group_translates_once() -> None: + translated = translate_legacy_household( + _snapshot( + { + "people": {"adult": {"age": {"2026": 40}}}, + "households": {"home": {"members": ["adult", "adult"]}}, + "benunits": {"unit": {"members": ["adult"], "rent": {"2026": 1}}}, + } + ) + ) + + assert translated.default_year == 2026 + assert translated.household_data == { + "people": [ + { + "id": "person-1", + "source_name": "adult", + "values": {"age": {"2026": 40}}, + "memberships": {"household": "household-1", "benunit": "benunit-1"}, + } + ], + "household": [{"id": "household-1", "source_name": "home", "values": {}}], + "benunit": [ + { + "id": "benunit-1", + "source_name": "unit", + "values": {"rent": {"2026": 1}}, + } + ], + } diff --git a/tests/unit/v2/test_legacy_household_mapping_insert.py b/tests/unit/v2/test_legacy_household_mapping_insert.py new file mode 100644 index 000000000..999a7ec21 --- /dev/null +++ b/tests/unit/v2/test_legacy_household_mapping_insert.py @@ -0,0 +1,122 @@ +"""Pin the statement that maps a copied v1 household to its v2 household.""" + +from __future__ import annotations + +from collections import deque +from types import SimpleNamespace +from uuid import UUID + +import pytest + +from policyengine_api.data.v2.models import LegacyHouseholdMapping +from policyengine_api.services.v2.households import services +from policyengine_api.services.v2.households.database_connectors import creates +from policyengine_api.services.v2.households.transformations import ( + legacy_household_fingerprint, +) +from policyengine_api.services.v2.households.types import LegacyHouseholdSnapshot +from policyengine_api.services.v2.households.validators import ( + LegacyHouseholdMappingIntegrityError, +) +from tests.fixtures.recording_session import RecordingSession + +HOUSEHOLD_ID = UUID("00000000-0000-0000-0000-000000000042") +MAPPING_ID = UUID("00000000-0000-0000-0000-000000000099") +INSERT_MAPPING = ( + "INSERT INTO legacy_household_mappings (id, created_at, updated_at, " + "country_id, legacy_household_id, household_id, source_api_version, " + "fingerprint_version, fingerprint_sha256) VALUES (%(id)s::UUID, " + "%(created_at)s, %(updated_at)s, %(country_id)s, %(legacy_household_id)s, " + "%(household_id)s::UUID, %(source_api_version)s, %(fingerprint_version)s, " + "%(fingerprint_sha256)s) ON CONFLICT ON CONSTRAINT " + "uq_legacy_household_mappings_country_legacy DO NOTHING RETURNING " + "legacy_household_mappings.id" +) + + +def _snapshot() -> LegacyHouseholdSnapshot: + return LegacyHouseholdSnapshot( + country_id="uk", + legacy_household_id=42, + api_version="2.3.4", + household_json={"people": {}}, + source_household_hash="source-hash", + ) + + +@pytest.fixture(autouse=True) +def stored_household(monkeypatch) -> None: + monkeypatch.setattr(creates, "uuid4", lambda: MAPPING_ID) + monkeypatch.setattr( + services, + "create_normalized_household", + lambda _session, _household_input: SimpleNamespace( + household_id=HOUSEHOLD_ID, created=True + ), + ) + + +def _mapping_insert() -> tuple: + return ( + "execute", + INSERT_MAPPING, + { + "id": MAPPING_ID, + "created_at": None, + "updated_at": None, + "country_id": "uk", + "legacy_household_id": 42, + "household_id": HOUSEHOLD_ID, + "source_api_version": "2.3.4", + "fingerprint_version": 1, + "fingerprint_sha256": legacy_household_fingerprint(_snapshot()), + }, + ) + + +def test_first_copy_inserts_its_mapping_with_the_snapshot_provenance() -> None: + session = RecordingSession(results=deque([None, MAPPING_ID])) + + result = services.mirror_legacy_household_in_session(session, _snapshot()) + + assert session.operations[1] == _mapping_insert() + assert len(session.operations) == 2 + assert result == services.LegacyHouseholdPersistenceResult( + household_id=HOUSEHOLD_ID, + household_created=True, + mapping_created=True, + ) + + +def test_lost_mapping_race_verifies_and_reuses_the_stored_mapping() -> None: + winner = LegacyHouseholdMapping( + country_id="uk", + legacy_household_id=42, + household_id=HOUSEHOLD_ID, + source_api_version="2.3.4", + fingerprint_version=1, + fingerprint_sha256=legacy_household_fingerprint(_snapshot()), + ) + session = RecordingSession(results=deque([None, None, winner])) + + result = services.mirror_legacy_household_in_session(session, _snapshot()) + + assert session.operations[1] == _mapping_insert() + assert len(session.operations) == 3 + assert result == services.LegacyHouseholdPersistenceResult( + household_id=HOUSEHOLD_ID, + household_created=False, + mapping_created=False, + ) + + +def test_lost_mapping_race_without_a_stored_winner_is_an_integrity_error() -> None: + session = RecordingSession(results=deque([None, None, None])) + + with pytest.raises( + LegacyHouseholdMappingIntegrityError, + match=r"^legacy household mapping conflict did not resolve to a stored row$", + ): + services.mirror_legacy_household_in_session(session, _snapshot()) + + assert session.operations[1] == _mapping_insert() diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index 9880f265e..1f80d1b3f 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -34,6 +34,7 @@ MetadataDatabaseSession, ) from policyengine_api.services.v2.metadata.services import V2MetadataService +from policyengine_api.services.v2.metadata.types import ParameterValueFilter from policyengine_api.services.v2.metadata.validators import ( InvalidMetadataPageError, MetadataResourceNotFoundError, @@ -329,13 +330,15 @@ def test_parameter_values_are_separate_canonical_resources( all_values = _service(catalog_session).list_parameter_values( "us", - parameter_id=parameter.id, + value_filter=ParameterValueFilter(parameter_id=parameter.id), ) current_value = _service(catalog_session).list_parameter_values( "us", - parameter_id=parameter.id, - current=True, - now=datetime(2026, 6, 1, tzinfo=UTC), + value_filter=ParameterValueFilter( + parameter_id=parameter.id, + current=True, + now=datetime(2026, 6, 1, tzinfo=UTC), + ), ) assert [item.value for item in all_values.items] == [0.2, 0.1] @@ -358,8 +361,7 @@ def test_current_parameter_value_uses_inclusive_effective_dates( ) -> None: result = _service(catalog_session).list_parameter_values( "us", - current=True, - now=selected_time, + value_filter=ParameterValueFilter(current=True, now=selected_time), ) assert [item.value for item in result.items] == [expected_value] @@ -440,7 +442,9 @@ def test_all_resource_families_support_collection_and_detail_reads( model_version = service.list_model_versions("us").items[0] variable = service.list_variables("us").items[0] parameter = service.list_parameters("us").items[0] - value = service.list_parameter_values("us", parameter_id=parameter.id).items[0] + value = service.list_parameter_values( + "us", value_filter=ParameterValueFilter(parameter_id=parameter.id) + ).items[0] dataset = service.list_datasets("us").items[0] region = service.list_regions("us").items[0] diff --git a/tests/unit/v2/test_named_limit_messages.py b/tests/unit/v2/test_named_limit_messages.py new file mode 100644 index 000000000..a55c2a46b --- /dev/null +++ b/tests/unit/v2/test_named_limit_messages.py @@ -0,0 +1,69 @@ +"""Pin the checks and messages behind the limits named in the complexity refactor.""" + +from __future__ import annotations + +import pytest + +from policyengine_api.data.places import validate_place_code +from policyengine_api.data.v2.catalog.catalog_selection import ( + InvalidPolicyEngineVersionError, + validate_policyengine_version, +) +from policyengine_api.services.v2.metadata.validators import ( + InvalidMetadataPageError, + validate_metadata_page, +) +from policyengine_api.services.v2.policies.transformations import ( + parse_legacy_period, +) +from policyengine_api.services.v2.policies.validators import ( + LegacyPolicyTranslationError, +) + + +def test_policyengine_version_is_bounded_to_128_characters() -> None: + longest = "1." + "1" * 126 + assert len(longest) == 128 + + assert validate_policyengine_version(longest) == longest + with pytest.raises( + InvalidPolicyEngineVersionError, + match=r"^policyengine_version must be at most 128 characters$", + ): + validate_policyengine_version(longest + "0") + + +@pytest.mark.parametrize("period", ["2026-01-01.2026-06-30.2026-12-31", "a.b.c"]) +def test_legacy_period_with_more_than_one_date_range_is_rejected(period: str) -> None: + with pytest.raises( + LegacyPolicyTranslationError, + match=rf"^legacy period '{period}' must contain one date range$", + ): + parse_legacy_period(period) + + +@pytest.mark.parametrize( + ("offset", "limit", "message"), + [ + (-1, 1, "offset must be at least 0"), + (0, 0, "limit must be between 1 and 500"), + (0, 501, "limit must be between 1 and 500"), + ], +) +def test_metadata_pages_are_bounded(offset: int, limit: int, message: str) -> None: + assert validate_metadata_page(0, 500) == (0, 500) + with pytest.raises(InvalidMetadataPageError, match=rf"^{message}$"): + validate_metadata_page(offset, limit) + + +@pytest.mark.parametrize("place_fips", ["5700", "570000", "5700A"]) +def test_place_codes_need_a_five_digit_fips_code(place_fips: str) -> None: + validate_place_code("NJ-57000") + with pytest.raises( + ValueError, + match=( + rf"^Invalid FIPS code in place: '{place_fips}'\. " + "Expected 5-digit FIPS code$" + ), + ): + validate_place_code(f"NJ-{place_fips}") diff --git a/tests/unit/v2/test_parameter_value_listing_route.py b/tests/unit/v2/test_parameter_value_listing_route.py new file mode 100644 index 000000000..39ac27b72 --- /dev/null +++ b/tests/unit/v2/test_parameter_value_listing_route.py @@ -0,0 +1,77 @@ +"""Pin how the parameter-value listing route forwards its query to the reader.""" + +from __future__ import annotations + +import pytest + +from policyengine_api.services.v2.metadata.types import ParameterValueFilter + +from .test_metadata_routes import ResourceReader, _client, _resource_results + + +def test_listing_forwards_every_query_parameter() -> None: + results, ids = _resource_results() + reader = ResourceReader(results) + + response = _client(lambda: reader).get( + "/v2/parameter-values", + params={ + "country_id": "uk", + "policyengine_version": "4.19.0", + "parameter_id": str(ids["parameter_id"]), + "current": "true", + "offset": 5, + "limit": 7, + }, + ) + + assert response.status_code == 200 + assert reader.calls == [ + ( + "list_parameter_values", + ("uk", "4.19.0"), + { + "value_filter": ParameterValueFilter( + parameter_id=ids["parameter_id"], current=True + ), + "offset": 5, + "limit": 7, + }, + ) + ] + + +def test_listing_defaults_to_every_value_on_the_first_page() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results) + + response = _client(lambda: reader).get( + "/v2/parameter-values", params={"country_id": "us"} + ) + + assert response.status_code == 200 + assert reader.calls == [ + ( + "list_parameter_values", + ("us", None), + {"value_filter": ParameterValueFilter(), "offset": 0, "limit": 100}, + ) + ] + + +@pytest.mark.parametrize( + "params", + [ + {"country_id": "us", "current": "maybe"}, + {"country_id": "us", "parameter_id": "not-a-uuid"}, + ], +) +def test_invalid_listing_query_is_a_typed_422(params: dict[str, str]) -> None: + results, _ids = _resource_results() + reader = ResourceReader(results) + + response = _client(lambda: reader).get("/v2/parameter-values", params=params) + + assert response.status_code == 422 + assert response.json() == {"status": "error", "message": "Invalid API v2 request"} + assert reader.calls == [] diff --git a/tests/unit/v2/test_report_run_creation.py b/tests/unit/v2/test_report_run_creation.py new file mode 100644 index 000000000..3a1b1b82c --- /dev/null +++ b/tests/unit/v2/test_report_run_creation.py @@ -0,0 +1,163 @@ +"""Pin report-run creation's validation, key-less insert and conflict paths.""" + +from __future__ import annotations + +from contextlib import contextmanager +from uuid import uuid4 + +import pytest +from sqlalchemy.exc import IntegrityError +from sqlalchemy.pool import StaticPool +from sqlmodel import Session, create_engine, select + +from policyengine_api.data.v2.models import ( + V2_METADATA, + Report, + ReportRun, + ReportRunStatus, + ReportRunTrigger, + TaxBenefitModel, +) +from policyengine_api.data.v2.report_runs import ( + ReportNotFoundError, + ReportRunPackageVersions, + create_report_run, +) + + +@pytest.fixture +def engine(): + test_engine = create_engine( + "sqlite://", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + V2_METADATA.create_all(test_engine) + yield test_engine + test_engine.dispose() + + +def _create_report(session: Session) -> Report: + report = Report( + label="Distributional report", + country="us", + tax_benefit_model=TaxBenefitModel(name=f"model-{uuid4()}"), + inputs={}, + ) + session.add(report) + session.flush() + return report + + +def test_unknown_report_is_rejected_before_any_validation(engine) -> None: + with ( + Session(engine) as session, + pytest.raises(ReportNotFoundError, match="does not exist"), + ): + create_report_run( + session, + report_id=uuid4(), + versions=ReportRunPackageVersions( + country_package_version="", + policyengine_version="", + ), + trigger=ReportRunTrigger.MANUAL, + ) + + +def test_manual_run_without_a_key_is_rejected_before_version_checks(engine) -> None: + with Session(engine) as session: + report = _create_report(session) + + with pytest.raises( + ValueError, match=r"^manual report reruns require an idempotency key$" + ): + create_report_run( + session, + report_id=report.id, + versions=ReportRunPackageVersions( + country_package_version="", + policyengine_version="", + ), + trigger=ReportRunTrigger.MANUAL, + ) + + +@pytest.mark.parametrize( + ("country_package_version", "policyengine_version"), + [("", "4.5.6"), ("1.2.3", "")], +) +def test_empty_package_versions_are_rejected( + engine, + country_package_version: str, + policyengine_version: str, +) -> None: + with Session(engine) as session: + report = _create_report(session) + + with pytest.raises( + ValueError, match=r"^report run package versions must be non-empty$" + ): + create_report_run( + session, + report_id=report.id, + versions=ReportRunPackageVersions( + country_package_version=country_package_version, + policyengine_version=policyengine_version, + ), + trigger=ReportRunTrigger.SYSTEM, + ) + + +def test_run_without_a_key_is_inserted_without_a_savepoint(engine) -> None: + with Session(engine) as session: + report = _create_report(session) + + run = create_report_run( + session, + report_id=report.id, + versions=ReportRunPackageVersions( + country_package_version="1.2.3", + policyengine_version="4.5.6", + ), + trigger=ReportRunTrigger.INITIAL, + ) + + stored = session.exec(select(ReportRun)).one() + assert stored is run + assert run.report_id == report.id + assert run.country_package_version == "1.2.3" + assert run.policyengine_version == "4.5.6" + assert run.trigger is ReportRunTrigger.INITIAL + assert run.idempotency_key is None + assert run.status is ReportRunStatus.PENDING + + +def test_conflict_without_a_matching_keyed_run_reraises( + engine, + monkeypatch, +) -> None: + conflict = IntegrityError("insert", {}, Exception("unrelated constraint")) + + @contextmanager + def failing_savepoint(): + raise conflict + yield + + with Session(engine) as session: + report = _create_report(session) + monkeypatch.setattr(session, "begin_nested", failing_savepoint) + + with pytest.raises(IntegrityError) as raised: + create_report_run( + session, + report_id=report.id, + versions=ReportRunPackageVersions( + country_package_version="1.2.3", + policyengine_version="4.5.6", + ), + trigger=ReportRunTrigger.MANUAL, + idempotency_key=uuid4(), + ) + + assert raised.value is conflict diff --git a/tests/unit/v2/test_report_runs.py b/tests/unit/v2/test_report_runs.py index d40ca07b2..4383a8110 100644 --- a/tests/unit/v2/test_report_runs.py +++ b/tests/unit/v2/test_report_runs.py @@ -26,6 +26,7 @@ TaxBenefitModelVersion, ) from policyengine_api.data.v2.report_runs import ( + ReportRunPackageVersions, ReportRunStateError, ReportTypeImmutableError, begin_report_run, @@ -78,8 +79,10 @@ def _create_run( return create_report_run( session, report_id=report.id, - country_package_version=country_version, - policyengine_version=policyengine_version, + versions=ReportRunPackageVersions( + country_package_version=country_version, + policyengine_version=policyengine_version, + ), trigger=ReportRunTrigger.MANUAL, idempotency_key=key if key is not None else uuid4(), ) @@ -142,8 +145,10 @@ def test_manual_rerun_requires_an_idempotency_key(engine) -> None: create_report_run( session, report_id=report.id, - country_package_version="1.2.3", - policyengine_version=DEPLOYED_POLICYENGINE_VERSION, + versions=ReportRunPackageVersions( + country_package_version="1.2.3", + policyengine_version=DEPLOYED_POLICYENGINE_VERSION, + ), trigger=ReportRunTrigger.MANUAL, ) @@ -202,8 +207,10 @@ def request_rerun() -> UUID: run = create_report_run( session, report_id=report_id, - country_package_version="1.2.3", - policyengine_version=DEPLOYED_POLICYENGINE_VERSION, + versions=ReportRunPackageVersions( + country_package_version="1.2.3", + policyengine_version=DEPLOYED_POLICYENGINE_VERSION, + ), trigger=ReportRunTrigger.MANUAL, idempotency_key=request_key, ) diff --git a/tests/unit/v2/test_unauthenticated_user_service.py b/tests/unit/v2/test_unauthenticated_user_service.py index 59fb13a63..5bd4c4cc3 100644 --- a/tests/unit/v2/test_unauthenticated_user_service.py +++ b/tests/unit/v2/test_unauthenticated_user_service.py @@ -593,8 +593,8 @@ def lose_to_a_concurrent_first_request(session, **identity): session.add( UserIdentity( user_id=user.id, - issuer=identity["issuer"], - subject=identity["subject"], + issuer=identity["principal"].issuer, + subject=identity["principal"].subject, ) ) session.flush() diff --git a/tests/unit/v2/test_user_household_routes.py b/tests/unit/v2/test_user_household_routes.py index 341ceb438..b9bf0df75 100644 --- a/tests/unit/v2/test_user_household_routes.py +++ b/tests/unit/v2/test_user_household_routes.py @@ -13,6 +13,7 @@ LegacyReferenceNotFoundError, ) from policyengine_api.services.v2.user_households.types import ( + UserHouseholdFilter, UserHouseholdPage, UserHouseholdRead, ) @@ -156,8 +157,9 @@ def test_list_and_patch_use_the_session_user_and_legacy_fields() -> None: { "principal": PRINCIPAL, "country_id": "us", - "household_id": None, - "legacy_household_id": 123, + "household_filter": UserHouseholdFilter( + household_id=None, legacy_household_id=123 + ), "offset": 0, "limit": 1, }, diff --git a/tests/unit/v2/test_user_policy_legacy.py b/tests/unit/v2/test_user_policy_legacy.py index 6ee841ad2..de7a3eb73 100644 --- a/tests/unit/v2/test_user_policy_legacy.py +++ b/tests/unit/v2/test_user_policy_legacy.py @@ -15,7 +15,11 @@ USER_POLICY_FINGERPRINT_VERSION, fingerprint_legacy_user_policy, ) -from policyengine_api.services.v2.user_policies.types import LegacyUserPolicySnapshot +from policyengine_api.services.v2.user_policies.types import ( + LegacyUserPolicyRevision, + LegacyUserPolicySnapshot, + UserPolicyCreationInput, +) from policyengine_api.services.v2.user_policies.validators import ( LegacyUserPolicyIntegrityError, ) @@ -123,13 +127,17 @@ def _apply_update( result = apply_existing_legacy_user_policy_mapping( session, mapping=mapping, - country_id="us", - reform_label="Legacy rename", - fingerprint=fingerprint, - user_id=USER_ID, - policy_id=POLICY_ID, - changed_fields=changed_fields, - source_revision=source_revision, + revision=LegacyUserPolicyRevision( + country_id="us", + legacy_user_policy_id=10, + reform_label="Legacy rename", + fingerprint=fingerprint, + source_revision=source_revision, + changed_fields=changed_fields, + ), + projection=UserPolicyCreationInput( + country_id="us", user_id=USER_ID, policy_id=POLICY_ID + ), ) return association, mapping, result diff --git a/tests/unit/v2/test_user_policy_legacy_persistence.py b/tests/unit/v2/test_user_policy_legacy_persistence.py new file mode 100644 index 000000000..822ef8865 --- /dev/null +++ b/tests/unit/v2/test_user_policy_legacy_persistence.py @@ -0,0 +1,354 @@ +"""Pin the statements and writes that copy one v1 saved-policy revision into v2.""" + +from __future__ import annotations + +from collections import deque +from types import SimpleNamespace +from uuid import UUID + +import pytest + +from policyengine_api.data.v2.models import LegacyUserPolicyMapping, UserPolicy +from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot +from policyengine_api.services.v2.user_policies import services +from policyengine_api.services.v2.user_policies.transformations import ( + USER_POLICY_FINGERPRINT_VERSION, + fingerprint_legacy_user_policy, +) +from policyengine_api.services.v2.user_policies.types import LegacyUserPolicySnapshot +from policyengine_api.services.v2.user_policies.validators import ( + LegacyUserPolicyIntegrityError, +) +from tests.fixtures.recording_session import RecordingSession + +POLICY_ID = UUID("00000000-0000-0000-0000-000000000010") +USER_ID = UUID("00000000-0000-0000-0000-000000000070") +ASSOCIATION_ID = UUID("00000000-0000-0000-0000-000000000060") +MAPPING_ID = UUID("00000000-0000-0000-0000-000000000099") +MAPPING_COLUMNS = ( + "legacy_user_policy_mappings.id, legacy_user_policy_mappings.created_at, " + "legacy_user_policy_mappings.updated_at, legacy_user_policy_mappings.country_id, " + "legacy_user_policy_mappings.legacy_user_policy_id, " + "legacy_user_policy_mappings.user_policy_id, " + "legacy_user_policy_mappings.last_applied_source_revision, " + "legacy_user_policy_mappings.fingerprint_version, " + "legacy_user_policy_mappings.fingerprint_sha256" +) +SELECT_MAPPING = ( + f"SELECT {MAPPING_COLUMNS} \nFROM legacy_user_policy_mappings \n" + "WHERE legacy_user_policy_mappings.country_id = %(country_id_1)s AND " + "legacy_user_policy_mappings.legacy_user_policy_id = " + "%(legacy_user_policy_id_1)s" +) +SELECT_MAPPING_FOR_UPDATE = f"{SELECT_MAPPING} FOR UPDATE" +MAPPING_LOOKUP = {"country_id_1": "us", "legacy_user_policy_id_1": 10} +SELECT_MAPPED_ASSOCIATION = ( + "SELECT user_policies.id, user_policies.created_at, user_policies.updated_at, " + "user_policies.user_id, user_policies.policy_id, " + "user_policies.legacy_policy_id, user_policies.country_id, " + "user_policies.name, user_policies.description, user_policies.baseline, " + "user_policies.provenance \nFROM user_policies \n" + "WHERE user_policies.id = %(id_1)s::UUID AND " + "user_policies.country_id = %(country_id_1)s" +) +INSERT_MAPPING = ( + "INSERT INTO legacy_user_policy_mappings (id, created_at, updated_at, " + "country_id, legacy_user_policy_id, user_policy_id, " + "last_applied_source_revision, fingerprint_version, fingerprint_sha256) " + "VALUES (%(id)s::UUID, %(created_at)s, %(updated_at)s, %(country_id)s, " + "%(legacy_user_policy_id)s, %(user_policy_id)s::UUID, " + "%(last_applied_source_revision)s, %(fingerprint_version)s, " + "%(fingerprint_sha256)s) ON CONFLICT ON CONSTRAINT " + "uq_legacy_user_policy_mappings_country_legacy DO NOTHING RETURNING " + "legacy_user_policy_mappings.id" +) + + +def _snapshot() -> LegacyUserPolicySnapshot: + return LegacyUserPolicySnapshot( + country_id="us", + legacy_user_policy_id=10, + reform_id=2, + reform_label="Renamed reform", + baseline_id=1, + user_id="auth0|one", + year="2026", + geography="us", + number_of_provisions=3, + api_version="1.0.0", + added_date=1, + updated_date=2, + ) + + +FINGERPRINT = fingerprint_legacy_user_policy(_snapshot()) + + +@pytest.fixture(autouse=True) +def resolved_policy_and_user(monkeypatch) -> None: + monkeypatch.setattr( + services, + "mirror_legacy_policy_in_session", + lambda _session, _reform: SimpleNamespace(policy_id=POLICY_ID), + ) + monkeypatch.setattr( + services, "resolve_legacy_user_id", lambda _session, **_kwargs: USER_ID + ) + + +def _mirror( + session: RecordingSession, + *, + source_revision: int = 1, + changed_fields: frozenset[str] = frozenset(), +): + return services.mirror_legacy_user_policy_in_session( + session, + _snapshot(), + LegacyPolicySnapshot( + country_id="us", + legacy_policy_id=2, + api_version="1.0.0", + policy_json={}, + source_policy_hash="legacy-hash", + ), + source_revision=source_revision, + changed_fields=changed_fields, + ) + + +def _mapping(**changes: object) -> LegacyUserPolicyMapping: + fields: dict[str, object] = { + "country_id": "us", + "legacy_user_policy_id": 10, + "user_policy_id": ASSOCIATION_ID, + "last_applied_source_revision": 1, + "fingerprint_version": USER_POLICY_FINGERPRINT_VERSION, + "fingerprint_sha256": "a" * 64, + } + fields.update(changes) + return LegacyUserPolicyMapping(**fields) + + +def _association(**changes: object) -> UserPolicy: + fields: dict[str, object] = { + "id": ASSOCIATION_ID, + "country_id": "us", + "user_id": USER_ID, + "policy_id": POLICY_ID, + "name": "Native name", + } + fields.update(changes) + return UserPolicy(**fields) + + +def _mapping_insert(user_policy_id: UUID, source_revision: int) -> tuple: + return ( + "execute", + INSERT_MAPPING, + { + "id": None, + "created_at": None, + "updated_at": None, + "country_id": "us", + "legacy_user_policy_id": 10, + "user_policy_id": user_policy_id, + "last_applied_source_revision": source_revision, + "fingerprint_version": USER_POLICY_FINGERPRINT_VERSION, + "fingerprint_sha256": FINGERPRINT, + }, + ) + + +def test_new_revision_creates_the_association_and_its_mapping() -> None: + session = RecordingSession(results=deque([None, MAPPING_ID])) + + result = _mirror(session, source_revision=1) + + created = session.operations[1][1] + assert session.operations == [ + ("exec", SELECT_MAPPING_FOR_UPDATE, MAPPING_LOOKUP), + ("add", created), + ("flush",), + ("refresh", created), + _mapping_insert(created.id, 1), + ] + assert (created.country_id, created.user_id, created.policy_id) == ( + "us", + USER_ID, + POLICY_ID, + ) + assert created.name == "Renamed reform" + assert result == services.LegacyUserPolicyPersistenceResult( + association_id=created.id, + policy_id=POLICY_ID, + association_created=True, + association_updated=False, + mapping_created=True, + ) + + +def test_lost_mapping_race_deletes_its_association_and_applies_the_winner() -> None: + winner = _mapping(last_applied_source_revision=1) + association = _association() + session = RecordingSession(results=deque([None, None, winner, association])) + + result = _mirror(session, source_revision=2, changed_fields=frozenset({"year"})) + + created = session.operations[1][1] + assert session.operations == [ + ("exec", SELECT_MAPPING_FOR_UPDATE, MAPPING_LOOKUP), + ("add", created), + ("flush",), + ("refresh", created), + _mapping_insert(created.id, 2), + ("delete", created), + ("flush",), + ("exec", SELECT_MAPPING, MAPPING_LOOKUP), + ( + "exec", + SELECT_MAPPED_ASSOCIATION, + {"id_1": ASSOCIATION_ID, "country_id_1": "us"}, + ), + ("add", winner), + ("flush",), + ] + assert association.name == "Native name" + assert winner.fingerprint_sha256 == FINGERPRINT + assert winner.last_applied_source_revision == 2 + assert result == services.LegacyUserPolicyPersistenceResult( + association_id=ASSOCIATION_ID, + policy_id=POLICY_ID, + association_created=False, + association_updated=False, + mapping_created=False, + ) + + +def test_lost_mapping_race_without_a_stored_winner_is_an_integrity_error() -> None: + session = RecordingSession(results=deque([None, None, None])) + + with pytest.raises( + LegacyUserPolicyIntegrityError, + match=r"^legacy user-policy mapping conflict did not resolve to a stored row$", + ): + _mirror(session) + + assert [operation[0] for operation in session.operations] == [ + "exec", + "add", + "flush", + "refresh", + "execute", + "delete", + "flush", + "exec", + ] + + +def test_next_revision_with_a_new_reform_label_renames_the_association() -> None: + mapping = _mapping(last_applied_source_revision=1) + association = _association() + session = RecordingSession(results=deque([mapping, association])) + + result = _mirror( + session, source_revision=2, changed_fields=frozenset({"reform_label"}) + ) + + assert session.operations == [ + ("exec", SELECT_MAPPING_FOR_UPDATE, MAPPING_LOOKUP), + ( + "exec", + SELECT_MAPPED_ASSOCIATION, + {"id_1": ASSOCIATION_ID, "country_id_1": "us"}, + ), + ("add", association), + ("add", mapping), + ("flush",), + ] + assert association.name == "Renamed reform" + assert mapping.last_applied_source_revision == 2 + assert mapping.fingerprint_sha256 == FINGERPRINT + assert result.association_updated is True + assert result.association_created is False + assert result.mapping_created is False + + +@pytest.mark.parametrize( + ("stored_revision", "stored_fingerprint"), + [(3, "a" * 64), (2, FINGERPRINT)], +) +def test_stale_or_replayed_revisions_write_nothing( + stored_revision: int, + stored_fingerprint: str, +) -> None: + mapping = _mapping( + last_applied_source_revision=stored_revision, + fingerprint_sha256=stored_fingerprint, + ) + session = RecordingSession(results=deque([mapping, _association()])) + + result = _mirror( + session, source_revision=2, changed_fields=frozenset({"reform_label"}) + ) + + assert [operation[0] for operation in session.operations] == ["exec", "exec"] + assert mapping.last_applied_source_revision == stored_revision + assert result == services.LegacyUserPolicyPersistenceResult( + association_id=ASSOCIATION_ID, + policy_id=POLICY_ID, + association_created=False, + association_updated=False, + mapping_created=False, + ) + + +@pytest.mark.parametrize( + ("mapping_changes", "association", "message"), + [ + ({}, None, "legacy user-policy mapping has no association"), + ( + {}, + _association(policy_id=UUID(int=1)), + "legacy user-policy mapping conflicts with immutable association fields", + ), + ( + {}, + _association(country_id="uk"), + "legacy user-policy mapping conflicts with immutable association fields", + ), + ( + {}, + _association(user_id=UUID(int=1)), + "legacy user-policy mapping conflicts with immutable association fields", + ), + ( + {"fingerprint_version": USER_POLICY_FINGERPRINT_VERSION + 1}, + _association(), + "legacy user-policy fingerprint version is unsupported", + ), + ( + {"last_applied_source_revision": 2}, + _association(), + "legacy user-policy revision conflicts with its stored fingerprint", + ), + ( + {"last_applied_source_revision": 0}, + _association(), + "legacy user-policy revision has an unapplied predecessor", + ), + ], +) +def test_inconsistent_existing_mappings_are_integrity_errors( + mapping_changes: dict[str, object], + association: UserPolicy | None, + message: str, +) -> None: + session = RecordingSession( + results=deque([_mapping(**mapping_changes), association]) + ) + + with pytest.raises(LegacyUserPolicyIntegrityError, match=rf"^{message}$"): + _mirror(session, source_revision=2) + + assert [operation[0] for operation in session.operations] == ["exec", "exec"] diff --git a/tests/unit/v2/test_user_policy_routes.py b/tests/unit/v2/test_user_policy_routes.py index 0130d84dc..10d837038 100644 --- a/tests/unit/v2/test_user_policy_routes.py +++ b/tests/unit/v2/test_user_policy_routes.py @@ -17,6 +17,7 @@ LegacyReferenceNotFoundError, ) from policyengine_api.services.v2.user_policies.types import ( + UserPolicyFilter, UserPolicyPage, UserPolicyRead, ) @@ -203,8 +204,9 @@ def test_detail_and_list_are_scoped_to_the_session_user() -> None: { "principal": PRINCIPAL, "country_id": "us", - "policy_id": POLICY_ID, - "legacy_policy_id": 95762, + "policy_filter": UserPolicyFilter( + policy_id=POLICY_ID, legacy_policy_id=95762 + ), "offset": 2, "limit": 3, }, diff --git a/tests/unit/v2/test_user_policy_service.py b/tests/unit/v2/test_user_policy_service.py index 76db00fa3..70352bb34 100644 --- a/tests/unit/v2/test_user_policy_service.py +++ b/tests/unit/v2/test_user_policy_service.py @@ -31,6 +31,7 @@ from policyengine_api.services.v2.user_policies.types import ( LegacyUserPolicySnapshot, UserPolicyCreateFields, + UserPolicyFilter, UserPolicyUpdateInput, ) from policyengine_api.services.v2.user_policies.validators import ( @@ -194,7 +195,9 @@ def test_other_users_rows_are_not_found_and_lists_are_owner_scoped(store) -> Non page = service.list_user_policies(principal=ALICE, country_id="us", limit=1) filtered = service.list_user_policies( - principal=ALICE, country_id="us", legacy_policy_id=2 + principal=ALICE, + country_id="us", + policy_filter=UserPolicyFilter(legacy_policy_id=2), ) assert [item.name for item in page.items] == ["Alice"] assert page.has_more is True diff --git a/tests/unit/v2/test_user_resource_failures.py b/tests/unit/v2/test_user_resource_failures.py new file mode 100644 index 000000000..420fc26b2 --- /dev/null +++ b/tests/unit/v2/test_user_resource_failures.py @@ -0,0 +1,198 @@ +"""Characterization of the status, message and headers of each user-route failure.""" + +from __future__ import annotations + +import json +import logging +from typing import TYPE_CHECKING, NamedTuple + +import pytest +from sqlalchemy.exc import IntegrityError, OperationalError + +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.fastapi_routes.v2 import user_resources +from policyengine_api.services.v2.associations.validators import ( + AssociationConflictError, + AssociationLinkNotFoundError, + AssociationNotFoundError, + AssociationReferenceConflictError, + MirrorOwnedAssociationError, +) +from policyengine_api.services.v2.identity.validators import ( + StandardUserRequiresSignInError, +) +from policyengine_api.services.v2.legacy_refs.validators import ( + LegacyReferenceNotFoundError, +) +from policyengine_api.services.v2.user_accounts.validators import ( + AccountDeletionUnavailableError, + AccountEditRequiresSignInError, + AccountIdentityDeletionError, + AccountNotFoundError, +) +from policyengine_api.services.v2.user_households.validators import ( + AssociationHouseholdNotFoundError, + UserHouseholdNotFoundError, +) +from policyengine_api.services.v2.user_policies.validators import ( + AssociationPolicyNotFoundError, +) + +if TYPE_CHECKING: + from collections.abc import Iterator + +ERROR_MESSAGE = "typed failure message" + + +class _RecordingHandler(logging.Handler): + def __init__(self) -> None: + super().__init__(level=logging.ERROR) + self.records: list[logging.LogRecord] = [] + + def emit(self, record: logging.LogRecord) -> None: + self.records.append(record) + + +@pytest.fixture +def failure_records( + monkeypatch: pytest.MonkeyPatch, +) -> Iterator[list[logging.LogRecord]]: + # Alembic tests run logging.config.fileConfig, which disables loggers that + # already exist, so re-enable this one and record on it directly. + handler = _RecordingHandler() + monkeypatch.setattr(user_resources.logger, "disabled", False) + user_resources.logger.addHandler(handler) + yield handler.records + user_resources.logger.removeHandler(handler) + + +def _failing(error: Exception): + def operation() -> None: + raise error + + return operation + + +class _FailureCase(NamedTuple): + error: Exception + status_code: int + message: str + challenge: str | None + logged_event: str | None + + +@pytest.mark.parametrize( + "case", + [ + _FailureCase( + StandardUserRequiresSignInError(ERROR_MESSAGE), + 401, + ERROR_MESSAGE, + "Bearer", + None, + ), + _FailureCase( + AccountEditRequiresSignInError(ERROR_MESSAGE), + 403, + ERROR_MESSAGE, + None, + None, + ), + _FailureCase( + AssociationReferenceConflictError(ERROR_MESSAGE), + 400, + ERROR_MESSAGE, + None, + None, + ), + _FailureCase( + LegacyReferenceNotFoundError(ERROR_MESSAGE), 422, ERROR_MESSAGE, None, None + ), + _FailureCase( + AssociationLinkNotFoundError(ERROR_MESSAGE), 422, ERROR_MESSAGE, None, None + ), + _FailureCase( + AssociationNotFoundError(ERROR_MESSAGE), 404, ERROR_MESSAGE, None, None + ), + _FailureCase( + UserHouseholdNotFoundError(ERROR_MESSAGE), 404, ERROR_MESSAGE, None, None + ), + _FailureCase( + AssociationPolicyNotFoundError(ERROR_MESSAGE), + 404, + ERROR_MESSAGE, + None, + None, + ), + _FailureCase( + AssociationHouseholdNotFoundError(ERROR_MESSAGE), + 404, + ERROR_MESSAGE, + None, + None, + ), + _FailureCase( + AccountNotFoundError(ERROR_MESSAGE), 404, ERROR_MESSAGE, None, None + ), + _FailureCase( + AssociationConflictError(ERROR_MESSAGE), 409, ERROR_MESSAGE, None, None + ), + _FailureCase( + MirrorOwnedAssociationError(ERROR_MESSAGE), 409, ERROR_MESSAGE, None, None + ), + _FailureCase( + IntegrityError("INSERT", {}, Exception("duplicate")), + 409, + "The record conflicts with stored state", + None, + None, + ), + _FailureCase( + AccountIdentityDeletionError(ERROR_MESSAGE), 502, ERROR_MESSAGE, None, None + ), + _FailureCase( + AccountDeletionUnavailableError(ERROR_MESSAGE), + 503, + ERROR_MESSAGE, + None, + None, + ), + _FailureCase( + V2ConfigurationError(ERROR_MESSAGE), + 503, + "V2 user persistence is unavailable", + None, + "v2_user_persistence_unavailable", + ), + _FailureCase( + OperationalError("SELECT", {}, Exception("down")), + 503, + "V2 user persistence is unavailable", + None, + "v2_user_persistence_unavailable", + ), + _FailureCase( + RuntimeError(ERROR_MESSAGE), + 500, + "V2 user operation failed", + None, + "v2_user_operation_failed", + ), + ], +) +def test_each_failure_maps_to_its_typed_response( + failure_records: list[logging.LogRecord], case: _FailureCase +) -> None: + response = user_resources.user_resource_operation(_failing(case.error)) + + assert response.status_code == case.status_code + body = json.loads(response.body) + assert body == {"status": "error", "message": case.message} + assert response.headers.get("www-authenticate") == case.challenge + assert [record.getMessage() for record in failure_records] == ( + [] if case.logged_event is None else [case.logged_event] + ) + + +def test_a_successful_operation_returns_its_result() -> None: + assert user_resources.user_resource_operation(lambda: "result") == "result"