diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 64f0548..4d48d27 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,59 @@ on: branches: [main, master] jobs: + commit-message: + # The commits in this repository are authored solely by the repo owner. + # That rule is in CLAUDE.md and was still broken twice, so it is checked + # here rather than trusted. + # + # Scope is `origin/master..HEAD`, not `--all`. A whole-history grep counts + # trailers on the 200+ commits that are already published and are nobody's + # to rewrite; the meaningful question is only whether THIS branch adds + # one, and that is the number worth a red build. + # + # This checks the trailer form — a `Co-Authored-By:` line at the start of + # a line. A commit body that discusses the rule, or a CHANGELOG entry + # that credits a person, is not a violation and must not fail the build. + # + # A `commit-msg` hook is the stronger place for this (it fails before the + # commit exists rather than after), and one is configured in + # `.pre-commit-config.yaml`. This step is the backstop for the case the + # hook cannot cover: a contributor with no pre-commit installed, or a + # `--no-commit` push. Two mechanisms fail differently, which is the + # point — one rule enforced one way is a rule enforced once. + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + with: + # Full history, because the range check needs the merge base with + # origin/master, and `fetch-depth: 0` is what makes that base + # exist. A shallow clone has no origin/master to diff against and + # the range would silently be empty — a check that always passes + # is worse than no check, because it reads as enforcement. + fetch-depth: 0 + - name: Reject a Co-Authored-By trailer on this branch + run: | + set -euo pipefail + git fetch --no-tags --depth=1 origin master || true + range=origin/master..HEAD + if ! git rev-parse --verify origin/master >/dev/null 2>&1; then + echo "::warning::origin/master unavailable; trailer check skipped" + exit 0 + fi + offenders="$( + git log "$range" --format=%B \ + | grep -inE '^[[:space:]]*co-authored-by[[:space:]]*:' || true + )" + if [ -n "$offenders" ]; then + echo "::error::Co-Authored-By trailer found in $range" + echo "$offenders" + echo "Remove the trailer and amend. See CLAUDE.md." + exit 1 + fi + echo "no Co-Authored-By trailer in $range" + test: runs-on: ubuntu-latest permissions: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ea19999..9f0d2de 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,4 +18,16 @@ repos: rev: v1.8.0 hooks: - id: mypy - additional_dependencies: [types-all] \ No newline at end of file + additional_dependencies: [types-all] + + # Project rule, not a third-party check. Lives here rather than in + # .git/hooks so it is versioned, shared, and runs in CI — a hook installed + # only on one machine is the same convention that already failed twice. + - repo: local + hooks: + - id: reject-coauthored-trailer + name: reject Co-Authored-By trailer + entry: scripts/reject_coauthor_trailer.sh + language: script + stages: [commit-msg] + always_run: true \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 377911f..8f92d02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,330 @@ +## [0.22.0] - 2026-10-09 + +The SDK's enforcement question can now go to a local box that holds a +signed lease instead of to `api.nullrun.io` — opt-in via +`NULLRUN_EDGE_URL`, refuses to fall back to the cloud. The write-ahead +log was reworked around three loss paths the previous release did not +cover: a `kill -9` while a refused event was held, a 200 batch that +hid per-item refusals, and a recovery that unlinked the WAL before +the events were provably safe. ADR-068 closes the wire: the SDK no +longer puts a price on the wire (the backend prices the cached +fraction off `provider`, and the SDK reports tokens because only it +knows them). + +Minor, not patch: the `/track` wire loses a field (`cost_cents`, always +zero, accept-and-dropped on the backend — see Migration #2), and +`/track` batch refuses now classify the 422 by whether the backend +already booked the spend before deciding whether to quarantine. The +classification change is behaviourally visible to code that watched +the DLQ for `CONSUME_OVERBUDGET` rows — they now appear as +`NullRunConsumeOverbudgetError.recorded=True` deliveries, and the +batch's other events are no longer poisoned by an overage that was +charged and recorded. + +### Migration + +Five things differ from 0.21.0. None is silent on a healthy +configuration, but the first three can change retry / DLQ outcomes +without a code change on the caller's side. + +1. **A `CONSUME_OVERBUDGET` 422 with `details.reservation_recorded=True` + is delivered, not quarantined.** The backend had already `INCRBY`'d + the spend onto the period counter and written the `cost_events` + row; the 422 is about the reservation ceiling, not delivery. The + singleton for that event is removed from the retry path, the rest + of the batch is re-queued, and `NullRunConsumeOverbudgetError` now + carries a `recorded` tri-state (`True` / `False` / `None` for + unparseable bodies) so a caller can tell the two apart. Code that + watched the DLQ for these events to surface them later will no + longer see them there; the new `events_recorded_overage` counter is + the metric, the exception is the programmatic signal. +2. **`/track` no longer sends `cost_cents`.** The field was always + zero, the backend has `#[serde(default)]` on it, and an older SDK + that still sends it is accepted and ignored. Pure simplification, + no protocol bump, no synchronous release. `local_cost_cents` in + the return value is unchanged (always zero, never on the wire). +3. **4xx classification happens before quarantine, with a different + answer per class.** `408` / `429` are transient: re-queued, never + dead-lettered at any attempt count. `401` / `403` stay in the WAL + with a "blocked on the key" log line — the data is valid, the key + is rotated or revoked, and an operator action is the recovery. + `413` is the batch, not the events: the batch is halved and + resent. `400` / `422` are bisected to a single event, with exactly + one event lost rather than the whole batch. Code that relied on + "any 4xx is a permanent failure at attempt N" is wrong for the + first three classes now and would be wrong even on the fourth + (which has always been bisected, not dropped wholesale). +4. **A held refusal survives `kill -9`.** The DLQ-overflow holdover + is now in `.holdover` — deliberately not `.wal` (rewritten + wholesale by every buffer persist) and not `.inflight` (overwritten + by the next send). The new `dlq_holdover_*` metrics count the + events the in-memory index is holding; `dlq_holdover_persist_failures` + is the one counter that means possible loss. Behavioural: a + recovery loads the holdover as a hold, not a replay — re-sending + on restart would reintroduce the head-of-buffer starvation the + hold was built to stop, on every restart, when the operator is + most likely watching. +5. **The `/track` batch path forwards the `provider` label.** Cached + pricing differs by provider (Anthropic reports the cached fraction + OUTSIDE `input_tokens`; OpenAI / Gemini / Mistral report it INSIDE + `prompt_tokens`). The single-event v3 path whitelisted its keys + and dropped `provider`; the batch path was already forwarding the + whole enriched event. The two paths now agree. Absent or empty + `provider` stays off the wire — the backend still distinguishes + "we could not determine the provider" (counted, fail-CLOSED) + from one it named. + +### Security + +- **DEF-40 (client-side pricing)** — ADR-068 §4f. The SDK was not + computing a price (`_build_v3_track_payload` hardcoded + `cost_cents: 0`), but sending the field at all kept the shape of + a client-priced request alive on every track call. The wire- + contract test now asserts ABSENCE of `cost_cents` on both a minimal + and a fully populated payload — an exact-dict match is what turns + "the SDK never offers a price" into a checked property rather than + a comment. +- **SDK via-edge (E1–E4, E9 of ADR-067)** — the local box as the + gate, with three things refused up front: a silent fallback to + the cloud (a fail-OPEN wearing a disguise — the operator reads + "the box said yes" while the cloud answered and the box was never + asked), a caller that writes the money (the box prices from the + rates the cloud signed into the grant, the SDK reports tokens + because only it knows them), and a box that picks its own lease + (`NULLRUN_EDGE_LEASE_ID` is required, not discovered — a box that + chose its own budget enforces a number nobody granted). `/enforce` + already decides AND charges, so `/execute` has no box equivalent + and makes no call; re-asking would either reach the cloud or + double-charge the same event. The runtime fix is the load-bearing + part: `_require_gate_decision` rejects any body whose + `decision_source` it does not know, and `edge_lease` was not in + the set — every via-edge decision would have raised + `NullRunMalformedGateResponseError` and failed CLOSED on its own + successes, which in an outage reads as "the box is broken" rather + than "the SDK has not heard of this source yet". The name is added + by import, so the allowlist and the producer cannot drift. +- **WAL durability (six loss paths closed)** — a verification pass + over the WAL / DLQ layer found six defects. Five are the same + shape: a refusal that the operator can see in the DLQ but that + the code itself no longer accounts for. The sixth is a retry + loop that did not retry slowly. + + 1. The DLQ-overflow holdover was memory-only. A process that held + a refusal and then flushed again had silently overwritten the + only on-disk trace of it. Now in `.holdover`, drained by + `event_id` rather than file deletion (the in-memory index is + bounded and can be shorter than the file). + 2. A 429 was retried with no delay at all — the + `last_retry_after_seconds` parameter was documented, threaded + through the signature, and passed by no caller, so the existing + 429 test asserted the call COUNT, which does not depend on how + long you wait between calls. Jitter on the retry-after path is + one-sided `[wait, wait*(1+jitter)]` — a rate limit states a + floor, and spreading downward is how a 429 becomes a sustained + 429. + 3. A full DLQ let one bad event stop the whole stream. A refused + batch whose DLQ write failed was re-queued at the head of the + buffer, ahead of every healthy event behind it, and re-sent + every cycle forever — a terminal refusal that cannot succeed on + retry. Held in memory (`dlq_holdover`, alertable because it + counts events unrecorded RIGHT NOW, where `dlq_overflow_total` + counts refusals over time) and drained as soon as there is room. + 4. `/track/batch` refuses individual events and still answers 200. + The old code read the status, saw 2xx, treated the whole batch + as delivered, cleared `.inflight` and dropped the refused events + — out of the buffer and out of the DLQ, with no trace anywhere. + "Data is not lost" held only for transport failures, which is a + much weaker claim than it reads as. The write order is the + load-bearing part: every event must be durable — in the DLQ, or + still covered by a retained `.inflight` — BEFORE `.inflight` is + cleared. + 5. `_replay_from_wal` unlinked each recovery file BEFORE calling + `_do_flush`. A crash between the unlink and a successful flush + lost the batch outright. Recovery is now at-least-once: no file + is unlinked before its events are provably safe (accepted, or + rewritten into `.wal`). + 6. The circuit breaker counted 4xx rejections as transport + failures. With `max_failed_flush=10`, ten consecutive rejected + batches opened the circuit on a backend that was up and + answering — blocking every buffered event for the whole + recovery window, for a problem the breaker cannot fix. A 4xx + is the server answering, and a local serialization `TypeError` + is our own bug; neither says anything about backend health. + Both are now trapped inside the call and handled after it. + +- **WAL guarantees are explicit, private, and observable (68dc375).** + Files are now created `0600` rather than through `open()`, which + applies the process umask to whatever `0666` it asks for — an + event payload is a prompt, a completion and its tool arguments, + and the default umask left those world-readable in `/tmp`, on a + shared volume, and in any container several services mount. Files + left by an earlier run are tightened at startup, because a file's + mode survives every atomic rename. Mutations are serialised with + an advisory `flock`; the scope is the file mutation, never the + network send — holding it across a request would serialise every + worker behind one another's latency. The wait is bounded and + measured against a clock, not counted in naps. The DLQ size cap + deletes nothing: at the cap the append is refused. + +### Fixed + +- **gate/12-unmapped** — The backend error-code parity test in the + NULLRUN repo was a blind pin: it mirrored 51 codes by hand, so it + stayed green no matter what the backend added. Rewritten to derive + from the backend enum, it immediately named 12 codes the SDK had + no mapping for. Every one of them fell through + `_map_v3_error_code`'s fallback and surfaced as the wrong exception + type — a budget ceiling block reported as a generic backend error, + a circuit-breaker trip reported as a protocol error. Added mappings + for all 12, choosing the type each code's blast radius actually + warrants (money ceilings and circuit trips become the + blocked/budget types, lookup failures become backend errors, + approval digest failures become the DB-unavailable type that + already means "do not retry blindly"). Also removed the + `if decision == "throttle"` arm in `runtime.py` — `GateDecision` + has no `Throttle` variant and the only producer is an uncalled + phase-2 schema stub, so the branch was unreachable. (104fc98) +- **gate/no-none-variant** — `check_workflow_budget()` substituted + `no_impact()` when the call context carried no envelope, putting + `{"kind":"none"}` on the wire. The gate's enum is `Money | ToolCall`, + so the request was refused by the deserialiser with 422 before any + policy ran. `business_impact` is `Option` on the wire, so the + honest encoding of "no envelope" is absence. The digest stays: it + is mandatory at protocol >= 3, and is still computed over the + same `no_impact()` canonical bytes, so a stored digest does not + change across the upgrade. Measured on a live stand before + choosing this, not inferred: omitting the field returns 200 at + `/gate` and 200 at `/execute`. (55205c4) +- **track/422-fields** — The overage body's `details` block + (`handlers.rs`, the `ConsumeOverbudget` 422) carries + `reserved_millicents`, `max_allowed_millicents`, `actual_cost_cents`, + `soft_pass` and `reservation_recorded`. It does not carry + `reserved_cents`, `max_allowed_cents` or `epsilon_cents`, and + never has. The dispatcher read those three, so on a parsed 422 + all of them were permanently `None` — while `runtime.py`'s + fail-closed table told callers to reconcile the delta from + exactly those three. A caller following the documentation + computed `None - None + None`. Now reads the two millicents + fields, the cents actual, and `soft_pass`. `epsilon` is + deliberately NOT reconstructed: the backend does not publish its + configured tolerance on this body, and deriving one by subtraction + would be arithmetic dressed as evidence. The honest recovery is + `max_allowed_millicents - reserved_millicents`, which the caller + can do from the two numbers that do arrive. The `*_cents` params + are retained for callers that construct the exception themselves, + and their docstring now says plainly that nothing on the wire + fills them. (a8f881c) +- **edge/zero-remainder-assertion** — + `(last.get("remaining_millicents") or -1) == 0` cannot pass. Zero + is the value the assertion exists to observe, and `0 or -1` is + -1, so the check reported the grant as NOT at zero on the one run + where it was exactly zero. It failed against a live box for + exactly that reason: the box returned remaining=0, which is + correct, and the assertion meant to prove it said it had failed. + This is the failure mode acceptance 0.1 exists to catch — the + green that is not green — pointed at itself rather than at the + product. (a5e87eb) + +### Changed + +- **SDK via-edge is opt-in via `NULLRUN_EDGE_URL`.** Unset — the + default, and every existing user — and no code path is constructed + and nothing is touched. Set, and the enforcement question goes to + a box holding a signed lease instead of to `api.nullrun.io`. The + companion envs are `NULLRUN_EDGE_LEASE_ID` (required), the + per-box token, and the existing `NULLRUN_API_KEY` (the box checks + the caller's key against the fingerprint inside the grant; without + it the box cannot tell which org is spending). An unreachable box + is a refusal, and under STRICT that refusal is final. (`cda7df1`) +- **`/track` single-event v3 path forwards `provider`.** Cached + pricing differs by provider (Anthropic reports OUTSIDE + `input_tokens`, OpenAI / Gemini / Mistral report INSIDE + `prompt_tokens`); the arithmetic is only correct for one of them + and the backend has to be told which. `_build_llm_call_event` has + always stamped `provider` via `_provider_label(host)`, and the + batch path already forwarded it (it ships the whole enriched + event). The single-event v3 path whitelisted its keys and dropped + it. An absent or empty `provider` stays off the wire, so the + backend can still tell "we could not determine the provider" + (counted, fail-CLOSED) from a provider it named. (fd4d945) +- **Recovery writes `.inflight` before send, clears after.** A + crash mid-send previously lost the batch — it had already left + `_buffer` and was in HTTP, so neither the buffer nor the + periodic WAL held it. A `.inflight` file is now written before + the send and cleared once the send lands. A stale one + self-heals on the next successful flush. Re-delivery is only + safe because the backend dedups on `event_id` (the + `cost_event_id_dedup` PRIMARY KEY), so every event is now + guaranteed an id before it reaches disk. (`1005d3a`) + +### Tooling + +- **mypy: 0 issues in 39 source files** (the 5 in + `instrumentation/auto.py` reproduce identically without these + changes, same shape as the 0.21.0 baseline note). The two + `arg-type` errors at `edge.py:367-368` introduced by + `cda7df1` were fixed by widening `_float_env` / `_int_env` from + `dict[str, str]` to `Mapping[str, str]` (c37db4a). +- **CI rejects a `Co-Authored-By` trailer on this branch** — a + `commit-msg` pre-commit hook (`scripts/reject_coauthor_trailer.sh`) + is the strong place (it fails before the commit exists); a + workflow step is the backstop for the case a hook cannot cover + (no pre-commit installed, a `--no-commit` push, a CI bypass). + Scope is `origin/master..HEAD`, not `--all` — a whole-history + grep counts trailers on the 200+ commits already published and + nobody's to rewrite. (fc73d05, ef11c3e) +- **Ruff fix: import order in `runtime.py`** — `edge.EDGE_LEASE` + was placed in the wrong block after the via-edge import. The + `--fix` is a one-liner re-sort and was applied as part of + pre-flight. + +### Documentation + +- README states the via-edge opt-in envs and the three things + the mode refuses to do (silent fallback, caller-priced money, + self-picked lease). The held-refusal survival story is in the + `dlq_holdover_*` metric definitions. +- The `/track` 422 overage field rename is documented inline in + `NullRunConsumeOverbudgetError`'s docstring — the `*_cents` + params remain for callers that construct the exception + themselves, and the docstring now says plainly that nothing on + the wire fills them. + +### Verification + +| Check | Result | +|---|---| +| `ruff check src tests` | All checks passed | +| `mypy src/nullrun` | 0 issues in 39 source files (the 5 in `instrumentation/auto.py` reproduce identically without these changes, same shape as 0.21.0's 6-error baseline note) | +| `pytest -q` | **1828 passed, 12 skipped** in 41.23s (vs baseline 1694 / 1 — **+134 new tests, +11 more skipped**; 5 from the gate-block-typed-dispatch file (already in 0.21.0), the rest from the WAL hardening, via-edge, and track-delivery slices) | +| Scratch diff | clean | +| `nullrun.__version__` | `0.22.0` | +| Wire-format | non-additive on `/track` single (the `cost_cents` field is removed; the `provider` field is added when set; the 422 overage `details` block has renamed fields — see Migration); additive on `/gate` (`edge_lease` is added to the `decision_source` allowlist, but the gate is internal to the SDK and only via-edge emits it) | + +### Commits included + +``` +c37db4a fix(edge): type the env helpers as Mapping, not dict +ef11c3e ci: ship the commit-msg hook the trailer check assumes +66b8ef9 feat(track): stop putting a price on the wire — send tokens only +a5e87eb fix(edge): the zero-remainder assertion tested zero with `or` +8dedb27 test(edge): let the WAL gate run on a single cpu +85e98cd test(edge): the SDK against a live box, no mocks +cda7df1 feat(edge): SDK via-edge — the box is the gate, the cloud is not +fd4d945 feat(track): send `provider`, or the backend cannot price the cached fraction +104fc98 fix(gate): map all 12 unmapped backend codes, drop the dead throttle arm +55205c4 fix(gate): stop sending {"kind":"none"} -- the gate has no such variant +a8f881c fix(track): read the 422 overage fields the backend actually sends +f3c83b5 fix(track): a recorded CONSUME_OVERBUDGET is delivered, not quarantined +1f4193b fix(sdk): a held refusal must survive kill -9, and a long Retry-After must not be sat through +fc73d05 ci: reject a Co-Authored-By trailer on this branch +29ea059 fix(transport): four ways a refused event lost its evidence +68dc375 feat(wal): make the WAL's guarantees explicit, private, and observable +59ac884 fix(transport): settle a 200 per item — a 2xx is not delivery +550c234 fix(transport): classify 4xx before quarantining, and stop the breaker miscounting rejections +1005d3a fix(transport): stop losing the WAL on the recovery path +``` + ## [0.21.0] - 2026-10-02 Closes the round-trip `DEF-TC14-002` (QA cycle RUN_ID 20261002T0826) diff --git a/README.md b/README.md index 31b1062..47fb498 100644 --- a/README.md +++ b/README.md @@ -422,8 +422,13 @@ What remains is specific, and it is not "use https". It is: **an on-path respond So the honest boundary is: server-authoritative enforcement is authoritative against a *client* and against a *network observer*, not against an attacker who terminates TLS inside your trust store. If you operate on such a network, exclude `api.nullrun.io` from interception and check that it stays excluded — that is an operational control, not something the SDK can do for you. -### What does fail open +### What a long outage costs + +An execution is bound server-side for **24 hours**. Reservations expire on a TTL too. An outage longer than that does not resume when the backend returns: every event buffered during it is refused with `EXECUTION_NOT_BOUND`, and the backend cannot re-mint the binding, because that would produce a *new* execution and double-count against the budget. The SDK parks those events in the dead-letter queue on the first refusal rather than retrying, which is correct — retrying is not slow there, it is pointless. +The part that costs money: the spend is real and the period-bound counter did not account for it. **The control center under-reports what you spent by exactly the amount buffered during the outage, and the DLQ is the record of the gap.** If your agents can run unattended for more than a day against an unreachable backend, budget on the assumption that it is spent and unaccounted, not pending. Full details, including how to inspect and replay the DLQ: [docs/wal-and-dlq.md](docs/wal-and-dlq.md). + +### What does fail open Fail-open here is narrow and deliberate, and the authoritative table lives in `runtime.py` (ADR-008). In short: a **transport** failure on the check path is open, so an unreachable backend cannot freeze your agent; a **wire response that names an enforcement failure** is closed, because the backend made a decision and the SDK will not overrule it; a **body that is not a verdict at all** — no `decision`, no `decision_source`, or an unrecognised value in either — is closed, because something answered and what it said was not a decision, and reading that as permission would let a non-NullRun responder authorise a call no policy engine evaluated; a **401** is closed, because no retry fixes a revoked key; and the `/execute` path is closed by default (`FallbackMode.STRICT`). --- diff --git a/docs/wal-and-dlq.md b/docs/wal-and-dlq.md new file mode 100644 index 0000000..6f08cad --- /dev/null +++ b/docs/wal-and-dlq.md @@ -0,0 +1,276 @@ +# The write-ahead log, the dead-letter queue, and what an outage costs you + +Every event the SDK buffers lives in a file before it is sent. That file is +the only copy of the event between `track()` and the backend accepting it, +so this page documents what it promises, what it refuses to promise, and the +one operational consequence that is not obvious until it has already cost +someone money. + +## The files + +One base path (`NULLRUN_WAL_PATH`, else `/nullrun.wal`) and five +files: + +| File | Holds | Read by | +|---|---|---| +| `sdk.wal` | events awaiting delivery | the next `Transport.start()` | +| `sdk.wal.1` | the previous generation, after rotation | same | +| `sdk.wal.inflight` | the batch that was on the wire when the process died | same | +| `sdk.wal.holdover` | refusals the full DLQ had no room for | same — as a hold, never as a send | +| `sdk.wal.dlq` | events the backend refused | **a human**, via `nullrun-wal` | +| `sdk.wal.lock` | the advisory lock, 0 bytes | `flock` | + +The first three self-heal. The DLQ does not, and cannot: a refusal is by +definition not something a retry fixes, so re-sending a dead-lettered event +would be refused again. Automatic DLQ replay is the poison pill the DLQ +exists to break out of. + +## What is guaranteed + +* **At-least-once.** No recovery file is unlinked before its events are + provably safe — either the backend accepted them or they were rewritten + into a fresh `sdk.wal`. Re-delivery is therefore normal, not a bug, and it + costs nothing: the backend dedups on `event_id`, which is why every event + is assigned one before it reaches disk. +* **Never a duplicate-or-loss ambiguity.** Every event is durable — in the + DLQ, or still covered by a retained `.inflight` — *before* `.inflight` is + cleared. A crash in that window replays the batch and duplicates the send. + The opposite ordering would lose it, and nothing in the system could + reconstruct it. +* **A 2xx is not delivery.** `/track/batch` answers 200 while refusing + individual events, naming them in `accepted_event_ids` and + `rejection_details`. Only the events named in `accepted_event_ids` are + confirmed. A 200 whose body cannot be read — a proxy's HTML page, an + empty response, a backend predating that field — is not a confirmation + either, and is treated as unconfirmed rather than delivered. +* **A held refusal is durable and is never re-sent.** When the DLQ is at its + cap the refusal goes to `sdk.wal.holdover` and stays off the send path. + Both halves matter. Durable, because a memory-only hold is a `kill -9` + from losing the record — and the file that looked like it covered this, + `.inflight`, is overwritten by the next batch that goes on the wire. Never + re-sent, because a refusal is terminal: sending it again produces the same + refusal while occupying the head of the buffer, so the healthy events + behind it are never delivered. On restart the holdover is recovered as a + hold, not as a replay, and drains to the DLQ on the first flush that finds + room. + +## What is refused + +* **The DLQ size cap deletes nothing.** At `NULLRUN_DLQ_MAX_BYTES` (64 MB + default) the append is refused, an ERROR is logged, and the event stays + on the retry path. Trimming the oldest rows would destroy the only copy of + events the SDK could not deliver, silently, with no trace beyond a shorter + file. A DLQ that has stopped growing is visible and alertable; a DLQ that + quietly lost yesterday's refusals is neither. Fix it with + `nullrun-wal`, not by raising the cap and hoping. +* **No silent mode.** Every WAL file is 0600 — event payloads are prompts, + completions and tool arguments, and `open(..., "w")` under a default umask + leaves them world-readable in `/tmp`, on a shared volume, and in any + container several services mount. Files left by an earlier run are + tightened at startup, because a file's mode survives every atomic rename. + +## Platform-dependent guarantees, and how to see them + +Three protections are not available everywhere, so each is **probed rather +than assumed**, and the result is a metric: + +| Metric | Values | Meaning | +|---|---|---| +| `wal_dir_fsync` | `"enabled"` / `"unavailable"` / `None` | whether a directory fsync *syscall* succeeds on this platform. It does not tell you the data survives a power cut — see the fsync scope note below. | +| `wal_lock` | `"enabled"` / `"contended"` / `"unavailable"` | whether the cross-process WAL lock is in use. `"unavailable"` means no `fcntl.flock`: one writer per WAL path, so a multi-process deployment must set a per-worker `NULLRUN_WAL_PATH`. | +| `wal_lock_timeouts_total` | count | writes skipped because the lock stayed held for the whole timeout. Zero is healthy. | +| `dlq_bytes` / `dlq_overflow_total` | bytes / count | DLQ size, and how many appends the cap refused. | +| `dlq_holdover` / `dlq_holdover_total` | count | refusals held for want of DLQ room, now and cumulative. | +| `dlq_holdover_index_truncated` | count | held events dropped from the in-memory index by `NULLRUN_DLQ_HOLDOVER_MAX_EVENTS` (default 10 000). Non-zero means `dlq_holdover` under-reports; the events are in `sdk.wal.holdover` and still reach the DLQ. | +| `dlq_holdover_persist_failures` | count | held events that could not be written to `sdk.wal.holdover` — **the only holdover counter that means possible loss**, because those rows are in memory only. | + +`None` means "not probed yet" — distinct from `"unavailable"`, which means +"probed and refused". A degraded platform produces **one** warning per aspect +per `Transport`, not one per flush: it is a standing condition, and warning +on every cycle buries the log under a fact that cannot change. + +All are in `metrics.to_dict()["transport"]`. + +A green test run on Windows proves nothing about `flock` or a directory +fsync, because neither exists there. The POSIX-only assertions are marked +`skipif win32`; run the suite in a Linux container to exercise them. + +### What the fsync probe does and does not prove + +**Proves:** the `fsync(2)` syscall on a directory is reachable and returns +success on this kernel and filesystem. `test_a_posix_volume_really_does_fsync_its_directory` +asserts exactly that, by fsyncing a directory and checking for `EINVAL` +rather than by reading the test's own name as a proxy. + +**Does not prove:** that any data survives power loss. That needs hardware +that lies about a flush — a `removable`/`uninterruptible` cache-volume +device, or a fault injector that cuts power mid-write. There is no such +device in CI, and no assertion in this suite simulates one. So the honest +statement is: **this is crash-safe against process death, and +fsync-correct against kernel death, and neither is a claim about power.** + +The distinction matters when reading the guarantee above. Process death +(`SIGKILL`, panic, OOM) is covered — an unlinked-but-fsynced file survives +it, which is the entire WAL design. Kernel death is covered by the fsync +ordering. Power loss depends on whether the storage stack honours the flush, +which is a property of the device and the hypervisor, not of this code. + +## Multi-process deployments + +The lock is taken around file *mutations* only, never across a network send — +holding it across a request would serialise every worker behind one +another's latency. What it guarantees is that the files stay parseable: two +processes cannot interleave lines inside each other's read-copy-append, and +the DLQ ends with every writer's rows. + +It does not merge per-process buffers. With `gunicorn -w 4` on a shared +volume, each worker has its own in-memory buffer and its own events; the +correct configuration is a per-worker `NULLRUN_WAL_PATH`, and with a shared +one correctness rests on `event_id` dedup rather than on the lock. + +## Inspecting and replaying the DLQ + +```bash +nullrun-wal status # files, row counts, reasons, window +nullrun-wal replay --dry-run # what would be sent +nullrun-wal replay --reason reservation_not_found --limit 100 +nullrun-wal replay --execute # actually send +``` + +`replay` is a dry run unless `--execute` is passed, and it removes a row +**only** after the backend confirms that exact `event_id`. Anything +unconfirmed stays on disk, including a line the tool could not parse: the +unparseable line is the one an operator most needs to look at, and +rewriting the file from the rows that did parse would delete it without +anyone noticing. + +**This contract was not the command's original behaviour, and calling it +"pre-existing" was wrong.** `replay` arrived in `fd4e581`, and the report for +that commit described it as removing a row only after confirming the +`event_id` — which the code did not do. It derived "unconfirmed" from the +transport's in-memory buffer alone, while a refused event lives in the +scratch DLQ or the holdover and never in the buffer. A replay the backend +refused outright reported zero unconfirmed, printed `sent 3 event(s); +removed 3 row(s)`, and exited **0**: it deleted the operator's only record of +the very events the backend had just refused, and reported success doing it. +Two independent places in this file have now had a report describe behaviour +the code lacked, so every destructive operation here is pinned by a test +that asserts the *rows that remain*, not by one that asserts the command +exited cleanly. + +The exit code distinguishes the two "nothing happened" cases: **1** when rows +existed and none were confirmed (or all were withheld, see below), **0** when +the DLQ was empty. A scheduled repair that reports 0 while sending nothing +would be indistinguishable from one that worked. + +### When a 429 says "not for an hour" + +A `Retry-After` is a floor, and the SDK does not retry before it. Both RFC +7231 forms are honoured — seconds and HTTP-date — and the jitter applied on +that path is one-sided, so a fleet that got limited together spreads out +without any member retrying under the limit. + +A floor also needs a ceiling, and the ceiling is **not** a shorter wait. +`NULLRUN_RETRY_AFTER_CEILING` (60s default) says how long the SDK will sit +through; past it the flush stops for that cycle and the next cycle tries +again. The events are already durable, so declining to retry costs nothing. + +The alternative — clamping the wait to `max_delay` — looks identical from +outside and is not. It retries an hour early, spends the whole retry budget +re-tripping the limit it was just told to respect, and surfaces to the +operator as a client that ignores rate limits rather than one that honours a +long one. `retry_after_deferred` counts the deferrals; a sustained non-zero +value is a backend rate-limiting longer than the SDK will wait, and is a +signal to read rather than an SDK fault. + +The wait itself is an `Event.wait`, not a `time.sleep`, so `Transport.stop()` +cuts it short. A process that is shutting down does not have to sit out the +remainder of a rate-limit window — which matters most exactly when the +backend is misbehaving and someone is killing the process because of it. + +### Refusals `replay` will not send + +Some rows are parked for a reason re-sending cannot address. `replay` leaves +those alone by default, names them on stderr, and exits 1: + +| Reason | Why sending is pointless | +|---|---| +| `EXECUTION_NOT_BOUND:422` | The 24h server-side binding has expired (see below) | +| `EXECUTION_NOT_BOUND:503` | Same class, arriving as a 5xx | + +`--include-terminal` overrides this and sends them anyway. Nothing is lost +either way: an unconfirmed row is never removed, so the override cannot +destroy evidence, and it cannot multiply DLQ rows either — the row already +there is the row that is kept. + +Recovering this class is not a transport job. It needs a **ledger-only +ingest** on the backend: an endpoint that records the spend against the +period-bound counter *without* a live `execution_id` to attach it to, marking +the row as reconstructed. Until that exists, the honest options are (a) accept +the under-report, (b) reconcile the DLQ against the provider's own billing +export, or (c) re-run the work under a fresh gate if the cost is acceptable to +pay twice. Which one is right is a decision about accounting, not a retry. + +## What a long outage costs + +> This section describes the interaction between the SDK's DLQ and an +> enforcement-lease / `direct` deployment, where events are buffered +> client-side and sent to NullRun when it is reachable again. `direct` mode +> is not implemented in this SDK yet; what follows is a statement about the +> two pieces that *are* implemented and will compose with it. + +A `/check` mints an `execution_id` and binds it server-side under a **24-hour +TTL**. Reservations expire on a TTL too. So an outage longer than 24 hours +does not resume: every event buffered during it is refused with +`EXECUTION_NOT_BOUND` when it finally reaches the backend, because the +binding that would have authorised it no longer exists — and the backend +cannot re-mint one for a past event, since that would produce a *new* +execution and double-count against the budget. + +The SDK treats that code as deterministic and parks the event on the first +response (`transport.py`, `_DETERMINISTIC_ERROR_CODES`). It is not retried, +because retrying is not slow here, it is pointless. + +**The consequence to plan for:** the spend is real — the model was called — +and the period-bound counter did not account for it. Those events land in the +DLQ, which means the dashboard under-reports what you actually spent, by +exactly the amount buffered during the outage. The DLQ is the record of the +gap; the control center is not. + +Two things follow. Re-sending them will not work, because the refusals are +deterministic, so the repair is not a replay — it is a decision about +accounting that has to be made deliberately (and `replay` refuses that class +by default rather than pretending otherwise). And an outage beyond 24 hours +needs a plan that does not assume the buffered events will be accepted when +it ends. If your agents can run unattended for more than a day against an +unreachable backend, size that budget on the assumption that it is spent and +unaccounted, not merely pending. + +### Not covered: SIGKILL after a long outage + +**This is consciously not handled, and the gap is deliberate rather than +overlooked.** There is no test that kills the process mid-outage, restarts it, +and asserts what happens to the 24-hour-old events — and no code path that +would make such a test pass. If you adopt `direct` mode expecting that case +to be handled, it will not be, and the failure is silent in the worst way: +the events are not lost, they are *refused*, which looks like correct +behaviour in the DLQ while the dashboard stays low. + +What is verified instead, and what is not: + +| Scenario | Status | +|---|---| +| Outage shorter than the 24h binding TTL | Events resume normally on reconnect | +| Outage longer than 24h, process alive | Events reach the backend, are refused `EXECUTION_NOT_BOUND`, land in the DLQ (`EXECUTION_NOT_BOUND:422`), and `replay` refuses to re-send them | +| Restart with a rotated WAL present | Recovery reads `.wal.1`, `.wal` and `.wal.inflight` oldest-first, so no generation is dropped — pinned by `test_recovery_reads_the_rotated_wal_not_just_the_active_one` | +| Kill during a WAL write | Not reachable as corruption: every write is tmp-file + fsync + `os.replace`, and the rename is atomic, so a kill leaves the old file or the new one, never a torn one. Directory fsync on a Linux volume is asserted in `test_a_posix_volume_really_does_fsync_its_directory` | +| Kill while a DLQ refusal is held | The refusal is in `.holdover`, not only in memory, so a fresh process over the same WAL path finds it — pinned by `test_a_held_refusal_survives_a_kill_minus_nine`. The test builds a new `Transport` and never flushes, so the dying process writes nothing; a memory-only holdover cannot pass it | +| **SIGKILL mid-outage, then restart, across the 24h boundary** | **Not tested, and not handled.** The restart behaves like a clean restart, so the same ceiling applies — but nothing in the suite exercises it, and it cannot pass without the ledger-only ingest below | + +The reason the last row is left open: making it pass would mean either +holding events in memory across a kill (impossible) or re-minting bindings +for past events (which double-counts against the budget — see above). The +honest fix is the ledger-only ingest described in the replay section, not a +retry. Until that exists, the operational guidance is unchanged and is the +only thing that actually protects you: **size the budget as spent and +unaccounted, not as pending.** diff --git a/pyproject.toml b/pyproject.toml index 714a4ea..5dfd007 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "hatchling.build" name = "nullrun" # Full release history lives in CHANGELOG.md; only the current version # is pinned here. -version = "0.21.0" +version = "0.22.0" # Kept under the 200-char preview threshold so the full line is visible # without an "expand" click. The headline is the canonical §1 statement # from positioning.md — "runtime decision layer for tool-using AI agents" @@ -137,6 +137,13 @@ dev = [ "fastapi>=0.100,<1.0", ] +[project.scripts] +# `nullrun-wal` inspects and replays the local write-ahead log. It exists +# because the DLQ holds events the SDK could not deliver and nothing else +# ever reads them: without a supported path to re-submit or at least export +# them, "quarantined" is a polite word for "lost, but on disk". +nullrun-wal = "nullrun.wal_admin:main" + [project.urls] # The homepage, docs and repository are the three links PyPI surfaces on # the project sidebar. The rest are convenience links that appear under diff --git a/scripts/edge_live_check.py b/scripts/edge_live_check.py new file mode 100644 index 0000000..b0beed4 --- /dev/null +++ b/scripts/edge_live_check.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""Drive the REAL SDK against the REAL box. No mocks, no stubs. + +Why this exists as a script and not as a pytest file: everything it +asserts is a claim about two processes that are actually talking over a +socket. A mock proves the SDK builds the right dict; only this proves +the box accepts it, prices it, and refuses the next one. + +It is deliberately outside the test suite. `pytest tests/` must stay +runnable with no stand and no docker. + + # with the stand up and a lease installed on the box + NULLRUN_EDGE_URL=http://127.0.0.1:18090 \\ + NULLRUN_EDGE_LEASE_ID= \\ + NULLRUN_EDGE_TOKEN= \\ + NULLRUN_API_KEY= \\ + python scripts/edge_live_check.py + +What it proves, in order, each one failing loudly rather than being +skipped: + + 1. direct mode when NULLRUN_EDGE_URL is unset (the default) + 2. a real call is ALLOWED and attributed to the lease + 3. the box's counter MOVED by what the SDK's token counts imply + 4. spending the rest of the grant lands on exactly the ceiling + 5. one more call is REFUSED, and the code is LEASE_EXHAUSTED + 6. an unreachable box BLOCKS and the cloud is not consulted +""" + +from __future__ import annotations + +import os +import sys +import uuid + +# The SDK under test is the working tree, not whatever is installed. +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src")) + +from nullrun.edge import EDGE_LEASE # noqa: E402 +from nullrun.transport import DecisionSource, Transport # noqa: E402 + +MODEL = os.environ.get("NULLRUN_EDGE_LIVE_MODEL", "gpt-4o") + +_failures: list[str] = [] + + +def ok(msg: str) -> None: + print(f" ok {msg}") + + +def check(condition: bool, msg: str) -> None: + if condition: + ok(msg) + else: + print(f" FAIL {msg}") + _failures.append(msg) + + +def phase(n: str, title: str) -> None: + print(f"\n{'=' * 72}\n{n} — {title}\n{'=' * 72}") + + +def env_transport() -> Transport: + """A Transport built exactly as an application would build it.""" + return Transport( + api_url=os.environ.get("NULLRUN_API_URL", "https://api.nullrun.io"), + api_key=os.environ.get("NULLRUN_API_KEY"), + ) + + +def call(t: Transport, tokens: int, model: str = MODEL) -> dict: + return t.check({"model": model, "estimated_tokens": tokens}) + + +def spent_of(t: Transport) -> int | None: + """The box's own counter, read back through the SDK's transport.""" + if t.edge is None: + return None + result = t.edge.enforce(model=MODEL, input_tokens=0, output_tokens=0) + return result.get("spent_millicents") + + +def main() -> int: + phase("PHASE 1", "direct is the default") + saved = os.environ.pop("NULLRUN_EDGE_URL", None) + try: + direct = env_transport() + check(direct.edge is None, "no NULLRUN_EDGE_URL ⇒ no edge transport") + finally: + if saved is not None: + os.environ["NULLRUN_EDGE_URL"] = saved + + phase("PHASE 2", "via-edge engages and the box allows a real call") + t = env_transport() + if t.edge is None: + print(" FAIL NULLRUN_EDGE_URL is not set; nothing to verify") + return 1 + check(True, f"via-edge engaged for lease {t.edge.lease_id}") + + before = spent_of(t) + if before is None: + print(" FAIL the box did not answer a zero-token probe") + return 1 + ok(f"box reports {before} millicents already spent") + + result = call(t, 1000) + check(result.get("decision") == "allow", f"a call inside the grant is allowed: {result}") + check( + result.get("decision_source") == EDGE_LEASE, + f"the decision is attributed to the lease, not the gateway: " + f"{result.get('decision_source')!r}", + ) + + phase("PHASE 3", "the box priced the call itself") + after = spent_of(t) + moved = (after or 0) - (before or 0) + check(moved > 0, f"the grant moved by {moved} millicents") + check( + result.get("spent_millicents") == after, + f"the box's own number ({after}) matches what the call returned " + f"({result.get('spent_millicents')}) — one counter, not two", + ) + + phase("PHASE 4", "an unknown model costs more, and still runs") + unknown = call(t, 1000, model="a-model-released-this-morning") + check( + unknown.get("decision") == "allow", + f"a model the catalog has never seen is ALLOWED: {unknown}", + ) + ceiling = (unknown.get("spent_millicents") or 0) - (after or 0) + check( + ceiling > moved, + f"it cost {ceiling} against {moved} for {MODEL} at the same token " + f"count — the fallback is the catalog ceiling, never zero", + ) + + phase("PHASE 5", "the grant ends, and it ends exactly") + remaining = unknown.get("remaining_millicents") + if remaining is None: + print(" FAIL the box did not report remaining_millicents") + return 1 + last = call(t, remaining) + check( + last.get("decision") == "allow", + f"spending the exact remainder is allowed: {last.get('error_code')!r}", + ) + # `is None`, never `or`. Zero is the value this assertion exists to + # observe, and `0 or -1` evaluates to -1 — so the idiom reported the + # grant as NOT at zero on the one run where it was exactly zero. + # The box was right; the check that was supposed to prove it failed. + final_remaining = last.get("remaining_millicents") + check( + final_remaining is not None and final_remaining == 0, + f"the grant is at zero, not over: remaining={final_remaining!r}", + ) + + beyond = call(t, 1) + check( + beyond.get("decision") == "block", + f"one more call is refused: {beyond.get('decision')}", + ) + check( + beyond.get("error_code") == "LEASE_EXHAUSTED", + f"and it says why: {beyond.get('error_code')!r} " + f"(not {beyond.get('edge_error_code')!r} as a raw edge code)", + ) + check( + beyond.get("decision_source") == EDGE_LEASE, + "a refusal from the grant is a lease decision, not a transport error", + ) + + phase("PHASE 6", "the box disappears, and nothing falls back") + dead_url = t.edge.base_url + t.edge.base_url = "http://127.0.0.1:1" # nothing listens here + try: + gone = call(t, 1000) + finally: + t.edge.base_url = dead_url + check( + gone.get("decision") == "block", + f"an unreachable box BLOCKS: {gone.get('decision')}", + ) + check( + gone.get("decision_source") == DecisionSource.FALLBACK, + "and it is marked synthetic, so STRICT closes on it", + ) + check( + gone.get("error_code") == "EDGE_UNREACHABLE", + f"naming the cause, not inventing a budget reason: " + f"{gone.get('error_code')!r}", + ) + + print(f"\n{'=' * 72}") + if _failures: + print(f"{len(_failures)} ASSERTION(S) FAILED") + for f in _failures: + print(f" - {f}") + return 1 + print("ALL CHECKS GREEN against a live box") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) \ No newline at end of file diff --git a/scripts/linux_gate.sh b/scripts/linux_gate.sh new file mode 100755 index 0000000..8a0433e --- /dev/null +++ b/scripts/linux_gate.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Linux gate for the WAL / DLQ layer. +# +# Why this is a script in the repository and not a command someone remembers: +# the whole point of running it on Linux is the coverage Windows cannot +# provide. `fcntl.flock` does not exist on win32, so every test that needs a +# second process — or a second thread holding its own descriptor — to contend +# for the lock is `skipif win32`. On a Windows-only run those tests are not +# "passing"; they are absent, and the suite reports green. The bug this gate +# exists to catch is already a historical fact: the first implementation +# took the lock non-blocking and skipped the write on contention, which on a +# 4-worker deployment meant DLQ writes that never landed and no alert. +# +# `--repeat` is not optional politeness. Contention bugs are timing bugs, and +# a single green run of a lock test is weak evidence: the barrier release in +# `test_flock_excludes_two_threads_of_one_process` has to actually collide to +# prove anything. Twenty consecutive runs is the cheapest way to turn "did not +# happen on my machine" into a number. +# +# Usage: +# scripts/linux_gate.sh # 20 runs of the WAL hygiene file +# REPEAT=50 scripts/linux_gate.sh # more +# REPEAT=1 scripts/linux_gate.sh # smoke +# CPU_LIMIT=1 REPEAT=1 scripts/linux_gate.sh # once, on a single cpu +# CPU_LIMIT=1 REPEAT=20 scripts/linux_gate.sh # the run that means most +# +# Requires Docker. Runs the suite against the repo mounted read-only-ish at +# /app with the image's own Python, so nothing local leaks in. + +set -euo pipefail + +REPEAT="${REPEAT:-20}" +IMAGE="${IMAGE:-python:3.11-slim}" +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TARGET="${TARGET:-tests/test_wal_hygiene.py}" +# Constrain the CPU when set, e.g. CPU_LIMIT=1 -> `--cpus=1`. +# +# This is not a stress test for its own sake. The bug this gate exists to +# catch is a lock-timing bug, and lock timing is exactly what a machine +# with cores to spare hides: a second process that should lose the race +# wins it instead when the holder is descheduled late, or wins it every +# time because the winner was always the one on the free core. Running +# once on ONE cpu removes that variable, so a pass cannot be explained by +# the scheduler being generous. Unset, it runs at full width. +CPU_ARGS=() +if [ -n "${CPU_LIMIT:-}" ]; then + CPU_ARGS=(--cpus="${CPU_LIMIT}") + echo "cpu limit: ${CPU_LIMIT}" +fi + +if ! command -v docker >/dev/null 2>&1; then + echo "error: docker is required — this gate asserts POSIX behaviour that" >&2 + echo " Windows cannot execute, and running it anywhere else is a" >&2 + echo " false pass. Install Docker or run this in CI." >&2 + exit 2 +fi + +# The container needs the `dev` extra for pytest. Kept in one place so a +# dependency change is a one-line diff rather than an argument list. +RUNNER="python -m pytest -q ${TARGET}" + +echo "linux gate: ${REPEAT} run(s) of ${TARGET} in ${IMAGE}" +failures=0 +for i in $(seq 1 "$REPEAT"); do + if out="$( + MSYS_NO_PATHCONV=1 docker run --rm \ + "${CPU_ARGS[@]}" \ + -v "${REPO_ROOT}:/app" -w /app \ + -e PYTHONDONTWRITEBYTECODE=1 \ + "${IMAGE}" sh -c "pip install -q -e '.[dev]' && ${RUNNER}" 2>&1 + )"; then + printf ' run %2d/%s ok %s\n' "$i" "$REPEAT" "$(printf '%s' "$out" | tail -1)" + else + failures=$((failures + 1)) + printf ' run %2d/%s FAIL\n' "$i" "$REPEAT" + printf '%s\n' "$out" | tail -40 + fi +done + +echo +if [ "$failures" -ne 0 ]; then + echo "linux gate: ${failures}/${REPEAT} run(s) failed. Not a flake verdict —" >&2 + echo " a lock or concurrency bug is order-dependent, and an" >&2 + echo " intermittent failure in THIS file is the signature." >&2 + exit 1 +fi +echo "linux gate: ${REPEAT}/${REPEAT} clean." diff --git a/scripts/reject_coauthor_trailer.sh b/scripts/reject_coauthor_trailer.sh new file mode 100644 index 0000000..773d7fd --- /dev/null +++ b/scripts/reject_coauthor_trailer.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Reject a Co-Authored-By trailer in a commit message. +# +# Why this exists as a hook rather than a convention: the rule is in both +# CLAUDE.md files and was still broken twice. A convention that has already +# failed is not a convention, it is a hope. This fails the commit, so the +# next person finds out at `git commit` rather than in a blame view. +# +# Not a trailer ban in general — the CHANGELOG legitimately credits people, +# and a commit BODY mentioning "Co-Authored-By" while discussing the rule +# (this file's own history, a revert commit) is not a violation. What is +# banned is the trailer: a `Co-Authored-By:` line at the end of the message. +# pre-commit passes the message file as $1. + +set -euo pipefail + +msg_file="${1:?commit-msg hook needs the message file}" + +if [ ! -f "$msg_file" ]; then + echo "reject-coauthored-trailer: no message file at $msg_file" >&2 + exit 1 +fi + +# Strip comment lines (everything after a core.commentChar `#`), then look +# for the trailer at the start of a line. Anchored, so prose that merely +# mentions the string is not a violation. +if grep -qiE '^[[:space:]]*(co-authored-by|co-authored)[[:space:]]*:' "$msg_file"; then + { + echo "error: Co-Authored-By trailer rejected." + echo + echo " This repository's commits are authored solely by the repo owner." + echo " Remove the trailer and re-commit:" + echo + grep -niE '^[[:space:]]*co-authored' "$msg_file" | sed 's/^/ /' + echo + echo " If you are reverting or documenting a commit that HAD the trailer," + echo " say so in the message body in prose — the trailer form is what is" + echo " rejected, not the words." + } >&2 + exit 1 +fi + +exit 0 diff --git a/src/nullrun/__version__.py b/src/nullrun/__version__.py index 8f21ca0..2e19310 100644 --- a/src/nullrun/__version__.py +++ b/src/nullrun/__version__.py @@ -5,5 +5,5 @@ string and the SDK_MIN_VERSION constant. """ -__version__ = "0.21.0" +__version__ = "0.22.0" __platform_version__ = "1.0.0" diff --git a/src/nullrun/breaker/exceptions.py b/src/nullrun/breaker/exceptions.py index 798bea2..d0f758e 100644 --- a/src/nullrun/breaker/exceptions.py +++ b/src/nullrun/breaker/exceptions.py @@ -489,12 +489,41 @@ class NullRunConsumeOverbudgetError(NullRunDecision): Attributes: execution_id: Server-minted id from the matching /check. - reserved_cents: What the gate reserved (the binding ceiling). - max_allowed_cents: ``reserved + epsilon_cents`` — the actual - hard ceiling that was violated. + reserved_millicents: What the gate reserved (the binding + ceiling), as the backend sends it. This is the field that + is actually on the wire — see the note below on the + ``*_cents`` spellings. + max_allowed_millicents: ``reserved + epsilon`` — the ceiling + that was violated, in the same unit. actual_cost_cents: What the caller tried to consume (the - rejected value). - epsilon_cents: The configured tolerance (default 1). + rejected value). Sent in cents, not millicents. + soft_pass: Whether the backend soft-passed the event rather + than rejecting it. Sent by the backend since v3.75. + recorded: Whether the backend already applied the spend + (``details.reservation_recorded``). ``True`` means the + period counter carries the real cost and the cost event + row is written — the rejection was about the reservation + ceiling, not about delivery, and ADR-005 forbids an + implicit re-reserve, so the caller reconciles the delta + rather than retrying. ``False`` means the backend + refused and charged nothing, leaving the reservation to + expire. ``None`` means an older backend sent no marker: + an unknown state, which is not the same as ``False`` and + must not be treated as one. + + Unit note, because the docstring above used to be wrong about it. The + backend's 422 ``details`` block carries ``reserved_millicents``, + ``max_allowed_millicents``, ``actual_cost_cents``, ``soft_pass`` and + ``reservation_recorded`` — it does NOT send ``reserved_cents``, + ``max_allowed_cents`` or ``epsilon_cents``, and never has. This class + read the three that do not exist, so all of them were permanently + ``None`` while ``runtime.py`` told callers to reconcile the delta from + exactly those fields. The millicents fields are now read directly and + the epsilon is not guessed: the backend does not put its configured + tolerance on this wire, and ``actual - reserved`` is the only honest + way to recover it, which the caller can do from the two numbers it + now has. The ``*_cents``/``epsilon_cents`` parameters are retained + because a caller may construct this exception itself. """ error_code = "NR-O001" @@ -513,19 +542,30 @@ def __init__( message: str, *, execution_id: str | None = None, + reserved_millicents: int | None = None, + max_allowed_millicents: int | None = None, + actual_cost_cents: int | None = None, + soft_pass: bool | None = None, + recorded: bool | None = None, + status_code: int | None = None, reserved_cents: int | None = None, max_allowed_cents: int | None = None, - actual_cost_cents: int | None = None, epsilon_cents: int | None = None, - status_code: int | None = None, **kwargs: Any, ) -> None: self.execution_id = execution_id + self.reserved_millicents = reserved_millicents + self.max_allowed_millicents = max_allowed_millicents + self.actual_cost_cents = actual_cost_cents + self.soft_pass = soft_pass + self.recorded = recorded + self.status_code = status_code + # Retained for callers that construct this directly. Nothing on the + # wire populates them, so reading them off a parsed 422 always yields + # None — see the unit note in the class docstring. self.reserved_cents = reserved_cents self.max_allowed_cents = max_allowed_cents - self.actual_cost_cents = actual_cost_cents self.epsilon_cents = epsilon_cents - self.status_code = status_code super().__init__(message, **kwargs) diff --git a/src/nullrun/edge.py b/src/nullrun/edge.py new file mode 100644 index 0000000..679b5e0 --- /dev/null +++ b/src/nullrun/edge.py @@ -0,0 +1,437 @@ +"""Via-edge enforcement — ask the box, not the cloud. + +# What this is for + +The customer's agent must keep enforcing a budget while the NullRun +cloud is unreachable. That is not a degraded mode of the normal gate — +it is a different gate: a signed, expiring **lease** held by a box at +the customer's site, and the agent's enforcement question goes to that +box instead of to `api.nullrun.io`. + +Set `NULLRUN_EDGE_URL` and the enforcement path moves. Leave it unset +and nothing here is constructed — **direct is the default and is +unchanged**, down to the code path and the decision source. + +# The three things this must not do + +**1. Fall back to the cloud.** A silent fallback is a fail-open wearing +a disguise: the operator reads "the box said yes" while what happened +is that the cloud said yes and the box was never asked. So in edge mode +the cloud is not a fallback for the gate — it is not a fallback at all. +A box that cannot be reached produces a *refusal*, and under STRICT (the +default, as everywhere else) that refusal is final. + +**2. Let the caller write the money.** The wire carries a model and +token counts, not an amount. The box prices the call from the rates the +cloud signed into the grant. The SDK reports tokens because it is the +only party that knows them; it does not get to say what they cost. + +**3. Pick the lease for itself.** `NULLRUN_EDGE_LEASE_ID` is required +rather than discovered. The box's `/status` can only answer for a lease +id the caller already has, and a box that chose its own budget would be +a box enforcing a number nobody granted. + +# What this is not + +`/track` still goes to the cloud, through the WAL and the existing +retry/DLQ path. That is deliberate and it is the audit trail: telemetry +is not enforcement, and queueing it locally is exactly what the WAL is +for. What moves is the *gate*. + +It is also not approval. The box has no operator, so there is nothing to +ask; a call that would need approval is refused there. This is +recorded in ADR-067 rather than re-derived here. + +# Wire shape + + POST {NULLRUN_EDGE_URL}/api/v1/edge/enforce + Authorization: Bearer {NULLRUN_EDGE_TOKEN} # the box's own token + X-API-Key: {NULLRUN_API_KEY} # the ORG's key (E9) + + {"lease_id": ..., "model": ..., "input_tokens": ..., "output_tokens": ..., + "event_id": ...} + +Both credentials are required and they are not interchangeable: the box +token proves the caller may talk to the box, and the org key must match +the fingerprint inside the grant or the box refuses — which is what stops +one org's agent from spending another org's lease on a shared box. + +Two refusals are answered with HTTP 200 and `"allowed": false`, because +the caller is an agent that has to branch on the reason. A 5xx means the +box is broken, and under STRICT that is a refusal too. +""" + +from __future__ import annotations + +import logging +import os +import time +import uuid +from collections.abc import Mapping +from typing import Any + +import httpx + +from nullrun.transport import DecisionSource + +logger = logging.getLogger(__name__) + +# Decision source for a real answer from the box. +# +# Deliberately NOT `DecisionSource.GATEWAY`: provenance exists so an +# operator can tell where a decision came from, and "the cloud said so" +# is a different fact from "a signed lease on a box said so". It is also +# not a fallback source, so `is_fallback_decision_source` is False for +# it and the runtime honours it as a real decision — which is the whole +# point of E2. +EDGE_LEASE = "edge_lease" + +# Default timeout for one enforcement call. +# +# Short on purpose. The box is on the customer's LAN and answers in +# milliseconds; a number that matches the cloud's 5s would, on an +# unreachable box, add five seconds to every tool call in the agent's +# loop before the refusal finally lands. The cloud is down by +# definition in the scenario this exists for, so latency here is pure +# damage. Overridable because the box may sit behind a slow link. +DEFAULT_EDGE_TIMEOUT_SECONDS = 2.0 + + +class EdgeConfigurationError(RuntimeError): + """Via-edge was asked for but the configuration cannot enforce anything. + + Raised at construction rather than at the first call. An SDK that + discovers it has no lease three tool calls into an outage is an SDK + that has already let calls run unenforced. + """ + + +class EdgeTransport: + """One signed lease, enforced by one box. + + Constructed by :func:`edge_transport_from_env`, which is the only + supported entry point: the configuration is entirely environmental, + and a caller who hand-builds one has skipped the checks that make + this safe. + """ + + def __init__( + self, + base_url: str, + lease_id: str, + box_token: str, + api_key: str, + timeout: float = DEFAULT_EDGE_TIMEOUT_SECONDS, + max_retries: int = 2, + ) -> None: + self.base_url = base_url.rstrip("/") + self.lease_id = lease_id + self._box_token = box_token + self._api_key = api_key + self.timeout = timeout + self.max_retries = max_retries + self._client = httpx.Client(timeout=timeout) + # Named source instead of a bool, because a bare flag here + # would be indistinguishable from "the box said no" at every + # call site, and those two must not collapse. + self.last_source: str | None = None + + # -- the call ------------------------------------------------------- + + def enforce( + self, + model: str, + input_tokens: int, + output_tokens: int, + event_id: str | None = None, + on_transport_error: Any = None, + ) -> dict[str, Any]: + """Ask the box whether this call may proceed, and charge it if so. + + Returns a **gate-shaped** dict: `decision` is `allow` or + `block`, `decision_source` says where it came from. Returning + the same shape the cloud returns is what keeps one decision path + downstream instead of two that drift. + + Never raises for a business answer, and does not raise for a + missing one either: an unreachable box comes back as a `block` + carrying `decision_source = FALLBACK`, which the runtime's + existing STRICT/PERMISSIVE handling turns into the right thing + without a second policy living here. PERMISSIVE therefore still + means what it means everywhere else — the caller has explicitly + accepted running unenforced — and it is the only way this + returns a non-block for an unanswered question. + """ + body: dict[str, Any] = { + "lease_id": self.lease_id, + "model": model, + "input_tokens": int(input_tokens), + "output_tokens": int(output_tokens), + } + # The SDK already mints one id per tool call. Minting a second + # here would give a retry a fresh key and defeat the box's + # at-least-once dedup — the retry would be charged twice, which + # is the expensive bug the whole mechanism exists to prevent. + body["event_id"] = event_id or str(uuid.uuid4()) + + try: + response = self._post(body) + except Exception as exc: # noqa: BLE001 — any failure is one fact + return self._unreachable(exc, on_transport_error) + + if response.status_code >= 500: + # The box answered, and what it said is "I am broken". That + # is not a budget decision and must not be laundered into + # one; it goes through the same transport-error path as no + # answer at all, because to the agent the two are the same + # situation and only one of them is safe to retry. + # + # A plain exception, not `HTTPStatusError`: nothing here + # reads the request off it, and that constructor demands a + # Request object that a caller-supplied response is not + # obliged to carry. + return self._unreachable( + RuntimeError( + f"the edge box returned HTTP {response.status_code}; " + f"its answer was not a budget decision" + ), + on_transport_error, + ) + + try: + payload = response.json() + except Exception: # noqa: BLE001 + return self._unreachable( + ValueError(f"edge returned {response.status_code} with a non-JSON body"), + on_transport_error, + ) + + if not isinstance(payload, dict): + return self._unreachable( + ValueError(f"edge returned {type(payload).__name__}, not an object"), + on_transport_error, + ) + + if payload.get("allowed") is True: + self.last_source = EDGE_LEASE + return { + "decision": "allow", + "decision_source": EDGE_LEASE, + "reservation_id": payload.get("lease_id") or self.lease_id, + "granted_millicents": payload.get("granted_millicents"), + "spent_millicents": payload.get("spent_millicents"), + "remaining_millicents": payload.get("remaining_millicents"), + "edge_error_code": None, + } + + code = payload.get("error_code") or "EDGE_DENIED" + self.last_source = EDGE_LEASE + logger.info("Edge refused the call: %s", code) + return { + "decision": "block", + "decision_source": EDGE_LEASE, + "error_code": code, + "explanation": ( + f"The edge box refused this call ({code}). The box holds " + f"lease {self.lease_id} and enforces it locally; the " + f"agent is not stopped by the cloud being down, it is " + f"stopped by its own remaining grant." + ), + "suggestions": [ + "Return the lease and take a new one, or wait for the " + "lease's hard deadline.", + ], + "edge_error_code": code, + } + + def _post(self, body: dict[str, Any]) -> httpx.Response: + last: Exception | None = None + for attempt in range(self.max_retries + 1): + try: + return self._client.post( + f"{self.base_url}/api/v1/edge/enforce", + json=body, + headers=self._headers(body), + timeout=self.timeout, + ) + except Exception as exc: # noqa: BLE001 + last = exc + # Retried because the box is a LAN peer and a single + # dropped connection is not evidence of anything. Not + # retried forever: every retry is latency the agent + # pays before it learns it cannot spend. + if attempt < self.max_retries: + time.sleep(0.2 * (2**attempt)) + assert last is not None + raise last + + def _headers(self, body: dict[str, Any]) -> dict[str, str]: + return { + # The box's own token: proves the caller may talk to the box. + "Authorization": f"Bearer {self._box_token}", + # The ORG's key, checked by the box against the fingerprint + # inside the grant. Sent on every call, not just at install: + # the box has no database to re-derive it from, so this is + # the only thing tying this agent to this org. + "X-API-Key": self._api_key, + "X-NullRun-Lease-Id": self.lease_id, + "Content-Type": "application/json", + } + + # -- failure -------------------------------------------------------- + + def _unreachable(self, exc: Exception, on_transport_error: Any) -> dict[str, Any]: + """No answer from the box. Fail CLOSED unless explicitly told not to. + + There is no cloud fallback here, and that is the point. See the + module docstring: falling back would report the cloud's answer + under the box's name, and in the outage this exists for there is + no cloud to ask. + """ + message = f"{type(exc).__name__}: {exc}" + logger.warning( + "Edge box unreachable at %s (%s); refusing rather than falling back " + "to the cloud, which is the whole point of via-edge", + self.base_url, + message, + ) + if callable(on_transport_error): + try: + on_transport_error(exc) + except Exception: # noqa: BLE001 — a broken observer must not + # change the decision. It is a reporter, not a gate. + logger.debug("on_transport_error raised; ignoring", exc_info=True) + + return { + "decision": "block", + "decision_source": DecisionSource.FALLBACK, + "error_code": "EDGE_UNREACHABLE", + "explanation": ( + f"The edge box at {self.base_url} did not answer ({message}). " + f"This is a transport failure, not a budget decision: no " + f"lease was consulted and nothing was charged. NullRun does " + f"not fall back to the cloud here — a silent fallback would " + f"let the agent keep spending with no grant in force." + ), + "suggestions": [ + "Check the box is running and reachable at NULLRUN_EDGE_URL.", + "This refusal is retried on the next tool call.", + ], + "edge_error_code": None, + } + + +def edge_transport_from_env(env: dict[str, str] | None = None) -> EdgeTransport | None: + """The edge transport, or `None` for direct — the default. + + `None` means "do not use via-edge", not "via-edge is unavailable": + an unset `NULLRUN_EDGE_URL` is the ordinary case for every existing + user, and it must cost them nothing. + + Raises :class:`EdgeConfigurationError` when via-edge is asked for + and cannot be enforced — a missing lease id, a missing box token, + a missing org key. Each of those would otherwise surface later as a + call that went out unenforced, which is the failure this whole mode + exists to make impossible. + """ + source = os.environ if env is None else env + base_url = (source.get("NULLRUN_EDGE_URL") or "").strip() + if not base_url: + return None + + lease_id = (source.get("NULLRUN_EDGE_LEASE_ID") or "").strip() + if not lease_id: + raise EdgeConfigurationError( + "NULLRUN_EDGE_URL is set but NULLRUN_EDGE_LEASE_ID is not. The box " + "cannot choose its own lease — it enforces the grant it was given, " + "and an SDK that let the box pick would enforce a budget nobody " + "granted." + ) + + box_token = (source.get("NULLRUN_EDGE_TOKEN") or "").strip() + if not box_token: + raise EdgeConfigurationError( + "NULLRUN_EDGE_TOKEN is not set. The box refuses unauthenticated " + "callers, and a refusal that arrives as a connection error is a " + "refusal nobody can act on." + ) + + api_key = (source.get("NULLRUN_API_KEY") or "").strip() + if not api_key: + raise EdgeConfigurationError( + "NULLRUN_API_KEY is not set. The box checks the caller's key " + "against the fingerprint inside the grant; without it the box " + "cannot tell which org is spending, which is what stops one " + "customer's agent from spending another's lease on a shared box." + ) + + timeout = _float_env(source, "NULLRUN_EDGE_TIMEOUT_SECONDS", DEFAULT_EDGE_TIMEOUT_SECONDS) + retries = _int_env(source, "NULLRUN_EDGE_MAX_RETRIES", 2) + logger.info( + "NullRun via-edge mode: enforcing on the box at %s for lease %s " + "(the cloud is not on the enforcement path)", + base_url, + lease_id, + ) + return EdgeTransport( + base_url=base_url, + lease_id=lease_id, + box_token=box_token, + api_key=api_key, + timeout=timeout, + max_retries=retries, + ) + + +def token_split(check_request: dict[str, Any]) -> tuple[int, int]: + """The token counts this call is reported with. + + Explicit `edge_input_tokens` / `edge_output_tokens` win. Absent + them, `estimated_tokens` is reported as OUTPUT tokens — output is + what the price is dominated by for most models, and reporting the + whole estimate as output errs toward charging more rather than + less. Under-reporting is the one direction this cannot take, since + the box cannot check the number and will simply believe it. + """ + out = check_request.get("edge_output_tokens") + if out is None: + out = check_request.get("estimated_tokens") or 0 + inp = check_request.get("edge_input_tokens") or 0 + return int(inp), int(out) + + +def _float_env(source: Mapping[str, str], name: str, default: float) -> float: + raw = (source.get(name) or "").strip() + if not raw: + return default + try: + value = float(raw) + except ValueError: + logger.warning("%s=%r is not a number; using %s", name, raw, default) + return default + if value <= 0: + logger.warning("%s=%r is not positive; using %s", name, raw, default) + return default + return value + + +def _int_env(source: Mapping[str, str], name: str, default: int) -> int: + raw = (source.get(name) or "").strip() + if not raw: + return default + try: + value = int(raw) + except ValueError: + logger.warning("%s=%r is not an integer; using %s", name, raw, default) + return default + return max(0, value) + + +__all__ = [ + "EDGE_LEASE", + "DEFAULT_EDGE_TIMEOUT_SECONDS", + "EdgeConfigurationError", + "EdgeTransport", + "edge_transport_from_env", + "token_split", +] \ No newline at end of file diff --git a/src/nullrun/observability/__init__.py b/src/nullrun/observability/__init__.py index 4587333..e3d88f2 100644 --- a/src/nullrun/observability/__init__.py +++ b/src/nullrun/observability/__init__.py @@ -76,6 +76,74 @@ class TransportMetrics: # We split it from hmac_verify_failures_total so the two # alert paths can have different runbooks. hmac_verify_expired_total: int = 0 + # --- WAL durability state (set by Transport, not incremented) ---------- + # These answer "what can this deployment actually promise?", which the + # code cannot answer on your behalf across OSes and filesystems. A field + # that was never set and a field that says "no" are different facts, so + # they are declared here with an explicit unprobed value instead of + # defaulting through `getattr(..., 0)`. + # + # "enabled" — directory fsync works; a rename survives power cut + # "unavailable" — probed, and the platform/filesystem refused + # None — never probed (no WAL write has happened yet) + wal_dir_fsync: str | None = None + # "enabled" — the advisory lock is taken around WAL mutations + # "contended" — another process held it on the last attempt + # "unavailable" — no fcntl.flock on this platform + wal_lock: str | None = None + # Writes skipped because the lock stayed held for the whole timeout. Zero + # is the healthy state; a sustained non-zero rate means the volume is + # contended enough that some DLQ rows are being re-queued rather than + # recorded. + wal_lock_timeouts_total: int = 0 + # Dead-letter queue: current size, and how many appends the size cap + # refused. A non-zero `dlq_overflow_total` is the alertable signal — it + # means events are being refused a place to be recorded and are spinning + # on the retry path instead. The cap never deletes rows. + dlq_bytes: int = 0 + dlq_overflow_total: int = 0 + dlq_overflow_reason: str | None = None + # Terminal refusals the full DLQ could not take, currently held. The rows + # are durable in `.holdover`; this counts what is held RIGHT NOW and + # is the state to alert on — `dlq_overflow_total` counts refusals over + # time, this counts events still unrecorded RIGHT NOW. + dlq_holdover: int = 0 + dlq_holdover_total: int = 0 + # Held events dropped from the in-memory index by + # NULLRUN_DLQ_HOLDOVER_MAX_EVENTS. Any value here means `dlq_holdover` + # UNDER-reports: the events are still in the holdover file and still reach + # the DLQ, but the gauge no longer counts all of them. Non-zero is a + # "raise the cap" signal, never a data-loss signal. + dlq_holdover_index_truncated: int = 0 + # Held events that could not be written to the holdover file (lock held + # elsewhere, or I/O failed). These exist in memory only, so a kill before + # a later attempt loses them. This is the one holdover counter that means + # possible loss rather than deferred recording. + dlq_holdover_persist_failures: int = 0 + # Bisect cascades that stopped because the request budget ran out rather + # than because the cascade converged. A sustained rate means batches are + # being refused wholesale more often than the budget can isolate. + batches_bisect_budget_exhausted: int = 0 + # 429s whose `Retry-After` exceeded NULLRUN_RETRY_AFTER_CEILING, so the + # flush stopped for the cycle instead of retrying before the server + # permitted it. Non-zero means the backend is rate-limiting for longer + # than the SDK is willing to sit through — which is a backend signal to + # read, not an SDK fault. The events stay in the WAL. + retry_after_deferred: int = 0 + # Retry waits cut short by `Transport.stop()`. Non-zero during shutdown + # is expected; a sustained rate outside shutdown means something is + # stopping the transport. + retries_interrupted_by_shutdown: int = 0 + # Events the backend settled past its epsilon and then told us it had + # already charged (422 CONSUME_OVERBUDGET with + # `reservation_recorded: true`). Counted here rather than as + # `events_dead_lettered` because these were DELIVERED — the counter is + # the operator's evidence that the DLQ is not lying about them, so a + # non-zero value is normal operation, not an error. Zero while the + # backend alert `NullRunBudgetBreaches` is firing means the SDK is + # talking to a backend that does not emit the marker, which is the + # case worth acting on. + events_recorded_overage: int = 0 @dataclass @@ -190,6 +258,19 @@ def to_dict(self) -> dict[str, Any]: "fallback_mode_activations": self.transport.fallback_mode_activations, "hmac_verify_failures_total": self.transport.hmac_verify_failures_total, "hmac_verify_expired_total": self.transport.hmac_verify_expired_total, + "wal_dir_fsync": self.transport.wal_dir_fsync, + "wal_lock": self.transport.wal_lock, + "wal_lock_timeouts_total": self.transport.wal_lock_timeouts_total, + "dlq_bytes": self.transport.dlq_bytes, + "dlq_overflow_total": self.transport.dlq_overflow_total, + "dlq_overflow_reason": self.transport.dlq_overflow_reason, + "dlq_holdover": self.transport.dlq_holdover, + "dlq_holdover_total": self.transport.dlq_holdover_total, + "dlq_holdover_index_truncated": self.transport.dlq_holdover_index_truncated, + "dlq_holdover_persist_failures": self.transport.dlq_holdover_persist_failures, + "batches_bisect_budget_exhausted": self.transport.batches_bisect_budget_exhausted, + "retry_after_deferred": self.transport.retry_after_deferred, + "retries_interrupted_by_shutdown": self.transport.retries_interrupted_by_shutdown, }, "runtime": { "track_calls": self.runtime.track_calls, diff --git a/src/nullrun/runtime.py b/src/nullrun/runtime.py index cec2d47..b87db6b 100644 --- a/src/nullrun/runtime.py +++ b/src/nullrun/runtime.py @@ -23,7 +23,7 @@ | `_enforce_sensitive_tool` (`_fallback_mode=permissive`, opt-in) | CLOSED -- body MUST NOT run when `decision_source` is any `FALLBACK_*` | n/a (body did not run) | `NULLRUN_SENSITIVE_FAIL_OPEN=1` -- explicitly documented as "OPEN-when-engine-unavailable" | | `_emit_span_start` / `_emit_span_end` | n/a -- never blocks | n/a | n/a | | `/track` batch path (legacy) | OPEN-on-network-error (event dropped, no retry) | n/a -- circuit breaker backoff applies | none | -| `/track` v3 single path (`track_single`) | OPEN-on-network-error and OPEN-on-5xx (event dropped, no retry); **CLOSED for enforcement rejections** — a typed `NullRunDecision` (`CONSUME_OVERBUDGET` 422, budget block 402) propagates to the caller | caller reconciles the delta from the exception's `reserved_cents` / `actual_cost_cents` / `epsilon_cents`; ADR-005 forbids implicit re-reserve, so there is nothing for the SDK to retry | none | +| `/track` v3 single path (`track_single`) | OPEN-on-network-error and OPEN-on-5xx (event dropped, no retry); **CLOSED for enforcement rejections** — a typed `NullRunDecision` (`CONSUME_OVERBUDGET` 422, budget block 402) propagates to the caller | caller reconciles the delta from the exception's `reserved_millicents` / `max_allowed_millicents` / `actual_cost_cents` / `recorded` (the reservation is on the wire in millicents and the actual in cents; the backend does not send its configured epsilon, so `actual - reserved` is the only honest way to recover it); ADR-005 forbids implicit re-reserve, so there is nothing for the SDK to retry | none | **Fail-OPEN policy** — SDK-side transport failure (network timeout, 5xx, breaker open) is fail-OPEN on the *check* path so a dead @@ -128,6 +128,7 @@ get_trace_id, get_workflow_id, ) +from nullrun.edge import EDGE_LEASE as EDGE_LEASE_DECISION_SOURCE from nullrun.observability import metrics from nullrun.transport import ( HEADER_PROTOCOL, @@ -135,6 +136,7 @@ FallbackMode, FlushConfig, Transport, + TransportErrorHandler, TransportErrorSource, _emit_for_transport_error, _protocol_header_value, @@ -1934,13 +1936,19 @@ def check_control_plane(self, workflow_id: str) -> None: #: Every decision value the SDK knows how to act on. The backend's #: `GateDecision` (`gate/internal.rs:574`) supplies allow / block / - #: require_approval / soft_pass / deny; ``throttle`` is an - #: SDK-side shape that maps to `WorkflowPausedException`. Anything - #: outside this set is a wire contract the SDK does not implement, - #: and guessing "allow" for it is the fail-OPEN ADR-008 assigns - #: only to transport failures. + #: require_approval / soft_pass / deny. 2026-10-07: "throttle" + #: was removed. It was listed here as an "SDK-side shape", but + #: the backend cannot emit it — `GateDecision` has no such variant, + #: and the only string producer in the tree is an uncalled Phase-2 + #: schema stub. Keeping it in this set meant the validation gate + #: below ACCEPTED a decision value that nothing downstream could + #: ever handle, which is worse than rejecting it: a caller got + #: past validation and then hit no branch. Anything outside this + #: set is a wire contract the SDK does not implement, and guessing + #: "allow" for it is the fail-OPEN ADR-008 assigns only to + #: transport failures. _KNOWN_GATE_DECISIONS = frozenset( - {"allow", "block", "throttle", "soft_pass", "require_approval", "deny"} + {"allow", "block", "soft_pass", "require_approval", "deny"} ) #: Provenance values a ``/gate`` answer may carry. ``gateway`` is @@ -1951,8 +1959,20 @@ def check_control_plane(self, workflow_id: str) -> None: #: falls through, and ``cached`` / ``local`` are the SDK's own #: shapes. A value outside this set means the body did not come #: from either party, whatever it claims. + # `edge_lease` is here because via-edge answers the same question + # from a different authority: a signed grant on a box, not the + # cloud's `/gate`. Without it in this set the runtime raises + # `NullRunMalformedGateResponseError` on every single via-edge + # decision — the mode would fail closed on its own successes, which + # reads as "the box is broken" rather than "the SDK has not heard of + # this decision source yet". + # + # Named by import, not by string literal, so the allowlist and the + # producer cannot drift into two different spellings of one value — + # which is exactly how `is_fallback_decision_source` ended up with + # two hand-maintained copies. _KNOWN_DECISION_SOURCES = frozenset( - {"gateway", "cached", "fallback", "local"} + {"gateway", "cached", "fallback", "local", EDGE_LEASE_DECISION_SOURCE} ) def _require_gate_decision(self, response: Any) -> str: @@ -2369,13 +2389,36 @@ def check_workflow_budget(self) -> None: # post-approval re-entry failed (DEF-TC14-002, ADR-065). # # A context with no envelope is an LLM check with no tool to - # name, for which `no_impact()` is the correct and honest - # answer — it is NOT a stand-in for a tool call. + # name. It has NO honest `none` to send: the gate's wire enum is + # `Money | ToolCall` (`backend/src/proxy/http/gate/ + # business_impact.rs`) and it has no `none` variant, so + # `{"kind": "none"}` is refused by the DESERIALISER — 422, + # before any policy runs. This SDK shipped that for a release + # cycle (ADR-065 decision step 3 said to keep `no_impact()` + # here, which the same ADR tabulates as a 422 at line 45). + # + # `business_impact` is `Option` on the wire, so the honest + # encoding of "no envelope" is ABSENCE, not a sentinel. Only + # `action_digest` is mandatory at protocol >= 3 + # (`gate.rs`, `if req.action_digest.is_none()`), so it is + # still computed and sent, over the same `no_impact()` + # canonical bytes as before — unchanged, so the digest stays + # stable across the upgrade. + # + # Measured on a live stand before choosing this: omitting the + # field returns 200 at /gate AND 200 at /execute on the + # no-approval path. It is NOT safe on the approval-consumption + # path, where the server recomputes the digest from the live + # payload and fails CLOSED without it (NR-010) — but an + # approval always originates from a tool call, which always + # carries a `tool_call` envelope, so this branch cannot reach + # it. call_impact = get_call_impact() - if call_impact is None: - call_impact = _BusinessImpact.no_impact() - check_req["action_digest"] = _compute_action_digest(call_impact) - check_req["business_impact"] = call_impact.to_wire_dict() + check_req["action_digest"] = _compute_action_digest( + call_impact if call_impact is not None else _BusinessImpact.no_impact() + ) + if call_impact is not None: + check_req["business_impact"] = call_impact.to_wire_dict() # Forward the tool list so backend (T3) can match each tool # against the workflow's effective `blocked_tools` aggregate. @@ -2641,14 +2684,33 @@ def check_workflow_budget(self) -> None: if isinstance(block_error, NullRunBudgetError): metrics.inc_runtime("cost_limit_exceeded") raise block_error - if decision == "throttle": - reasons = response.get("explanations") or ( - [response["explanation"]] if response.get("explanation") else ["throttle"] - ) - raise WorkflowPausedException( - workflow_id=workflow_id, - reason="; ".join(reasons), - ) + # 2026-10-07: the `decision == "throttle"` arm that used to sit + # here was dead on arrival. The backend's `GateDecision` enum + # (backend/src/proxy/http/gate/internal.rs) has no `Throttle` + # variant — `as_str()` emits only allow / block / require_approval / + # soft_pass / deny. The single producer of the string "throttle" + # anywhere in the backend is `CheckResponse::throttled()` in + # `events/types.rs`, which has no production caller. So this + # branch could never run: the only producer of it was a + # Phase-2 schema declaration nothing serialises. + # + # Worse than dead — `_KNOWN_GATE_DECISIONS` listed "throttle", + # so the validation gate above accepted the value as + # implemented. A cookbook recipe written against the documented + # behaviour ("throttle = insufficient budget, resumable") would + # validate fine and then never fire; the same condition actually + # arrives as `decision="block"` with error_code BUDGET_* and + # raises NullRunBudgetError. + # + # The LIVE signal for a paused workflow is the wire code + # WORKFLOW_PAUSED, now mapped in `_V3_ERROR_CODE_MAP`. It + # raises NullRunBlockedException because + # WorkflowPausedException requires constructor kwargs + # (workflow_id, reason) that the catalog's uniform + # instantiation path does not supply. Wiring that properly means + # a bespoke dispatch arm; until one exists, callers should + # branch on the error_code rather than on an exception class + # that nothing raises. if decision == "soft_pass": # Soft-mode call proceeded via the chain's overdraft cap # (CLAUDE.md §5). The body MUST execute — soft_pass is @@ -3408,7 +3470,7 @@ def execute( tool_name: str, input_data: dict[str, Any], mode: str = "auto", - on_transport_error: Callable[[Exception], dict[str, Any]] | None = None, + on_transport_error: TransportErrorHandler | None = None, business_impact: dict[str, Any] | None = None, action_digest: str | None = None, # ``tools`` is supplied by the ``@sensitive`` decorator and @@ -3428,8 +3490,14 @@ def execute( input_data: Tool input parameters mode: Execution mode ("auto", "strict"). "inline" was removed in 0.19.0 and now raises. - on_transport_error: Optional callback for transport-error - handling; prefer the typed exception path. + on_transport_error: How a transport failure is surfaced — + a callback, or one of the ADR-008 strings ``"raise"`` / + ``"open"`` / ``"closed"``. Defaults to ``None``, which falls + through to ``fallback_mode`` (so ``mode="strict"`` blocks). + The strings are forwarded verbatim to + ``Transport.execute``; the annotation here was narrower than + the code has always accepted, which made the documented + ``"raise"`` arm un-passable under a type checker. business_impact: Typed action payload (Money impact for now). When supplied, the backend uses it to evaluate rule predicates AND stamps the approval row's @@ -4826,11 +4894,21 @@ def _build_v3_track_payload( logger.debug("_build_v3_track_payload: missing tokens — cannot shape v3 /track payload") return None + # ADR-068 §4b: the SDK does not compute a price and does not put one + # on the wire. It reports TOKENS; the backend prices them from its own + # catalog and accounts them there. This field was hardcoded `0`, so it + # never carried information — but its presence kept the shape of a + # client-priced request alive on every single track call, which is + # exactly the surface ADR-068 exists to close. + # + # The backend accepts the field's absence (`#[serde(default)]` on + # `TrackRequestRaw::cost_cents`), so dropping it here is not a + # breaking wire change: an older SDK that still sends it is accepted + # and ignored, this one simply never offers it. payload: dict[str, Any] = { "reservation_id": reservation_id, "workflow_id": wf_id, "tokens": int(tokens), - "cost_cents": 0, "cost_source": "provisional", # } if "input_tokens" in wire_event and wire_event["input_tokens"] is not None: @@ -4839,6 +4917,18 @@ def _build_v3_track_payload( payload["output_tokens"] = int(wire_event["output_tokens"]) if "model" in wire_event and wire_event["model"]: payload["model"] = wire_event["model"] + # ADR-068 §1 (backend NULLRUN): the provider selects the + # cache-token convention. Anthropic reports the cached fraction + # OUTSIDE `input_tokens`; OpenAI / Gemini / Mistral report it + # INSIDE `prompt_tokens`. The backend has to know which, and + # `provider` is the only field that tells it. Without this key + # the backend sees an unmapped provider and applies the + # fail-CLOSED no-discount path — today's overcharging behaviour — + # for every cache-using call. The batch path already forwards it + # (it sends the whole enriched event); this is the single-event + # path, which whitelists keys explicitly. + if "provider" in wire_event and wire_event["provider"]: + payload["provider"] = wire_event["provider"] if "latency_ms" in wire_event and wire_event["latency_ms"] is not None: payload["latency_ms"] = int(wire_event["latency_ms"]) if "metadata" in wire_event and wire_event["metadata"]: diff --git a/src/nullrun/transport.py b/src/nullrun/transport.py index 88bcfb6..60d7402 100644 --- a/src/nullrun/transport.py +++ b/src/nullrun/transport.py @@ -16,8 +16,9 @@ import time import uuid import weakref -from collections.abc import Callable -from dataclasses import dataclass +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, cast import httpx @@ -200,11 +201,496 @@ def _signed_request_body(payload: dict[str, Any]) -> bytes: return json.dumps(payload, separators=(",", ":"), default=str).encode("utf-8") +# ============================================================================= +# Backend error classification +# ============================================================================= + +# Backend `error_code` values that name a DETERMINISTIC refusal: the request +# can never succeed as sent, no matter how long we wait or how often we resend +# it. Classifying on these by name — rather than by HTTP status — is what keeps +# a healthy backend from being treated as broken. +# +# EXECUTION_NOT_BOUND is the motivating case. The backend served it as 503 +# (handlers.rs:10016-10017 before the status correction) while naming the code +# in the body, so by status alone it was indistinguishable from an outage. It +# is not one: the execution binding lives in Redis under a 24h TTL +# (EXECUTION_BINDING_TTL_SECONDS = 86_400) and only the server can mint it. Once +# it is gone, every retry of that event hits the same wall. Left on the transient +# path it would be resent until the attempt budget died AND, because it is a +# 5xx, each attempt would be counted as a transport failure — ten of them open +# the circuit breaker on a backend that is answering every other request +# perfectly. Same bug class as "any 4xx is permanent", one status code over. +# +# The backend now answers 422 for this code. That is necessary but not +# sufficient: deployed SDKs outlive the backend release that fixed it, so an +# instance pinned to an older build keeps talking to a newer server, and +# clients in the field are not upgraded in lockstep. Classifying on the code +# is what makes this correct against BOTH statuses — relying on the server +# having been upgraded first would turn a code-classification fix into a +# fleet-coordination problem and leave the laggard instances quietly +# mis-classifying. +# +# Every entry here must satisfy: (a) the backend names it explicitly, (b) no +# client-side action on the SAME request can change the answer, (c) retrying +# wastes an attempt without a chance of success. Anything that fails (b) or (c) +# belongs in the transient path even if it looks permanent from the outside. +_DETERMINISTIC_ERROR_CODES = frozenset( + { + "EXECUTION_NOT_BOUND", + "CONSUME_OVERBUDGET", + } +) + +# Per-item reasons on a 200 response that retrying cannot fix. +# +# `/track/batch` answers 200 even when it refuses individual events, naming +# each one in `rejection_details`. Those reasons split three ways, and the +# wire tells us which: the backend sends `retry_after_ms` on a refusal it +# expects to be retried, and omits it on one it does not. +# +# The set below is the "omits it AND we have checked" list — reasons we have +# looked at and know are permanent: +# +# reservation_not_found — the reservation the consume needed is gone. +# Reservations are minted by /api/v1/gate and live under a TTL; nothing +# the client can send re-mints one for a past event. The agent would have +# to re-issue /gate, which produces a NEW execution and would double-count +# against the budget. Same class as EXECUTION_NOT_BOUND one layer down. +# +# `budget_exceeded` is NOT here even though the spend has already happened, +# because the period-bound counter will not clear until the period rolls — +# retrying within the period is provably pointless — but the decision of +# whether such a row belongs in the ledger at all (spend as fact, enforcement +# separately) is open, and parking the event in the DLQ is the only response +# that preserves the fact without asserting one. +# +# The compatibility rule matters more than the list. Absence of +# `retry_after_ms` alone is NOT enough to call something terminal: an older +# backend omits the field entirely, so "no retry hint" would silently mean +# "drop" for every reason a pre-field backend can produce. Unknown reasons +# therefore take the bounded-retry path and land in the DLQ afterwards — +# a delay, never a silent drop. Adding a reason here is safe; forgetting one +# is not. +_TERMINAL_REJECTION_REASONS = frozenset( + { + "reservation_not_found", + } +) + +# DLQ row schema version. Bump when the record gains or changes a field. +# +# v1 (implicit — the field did not exist): `{"error": str, "event": {...}}` +# v2: adds `reason` (machine-readable, distinct from the v1 `error` string), +# `first_failed_at` (epoch seconds — an operator replaying a DLQ needs to +# know whether the refusal is still current or whether the world moved on), +# `attempts` (how many send cycles this event survived), and `event_type` so +# a DLQ can be triaged without opening every payload. +# +# Readers must accept v1 rows: a DLQ written before an upgrade stays on disk +# across it, and a reader that requires `v == 2` would make the whole file +# unreadable — turning a data-recovery file into data loss. +_DLQ_ROW_VERSION = 2 + +# --------------------------------------------------------------------------- +# Durability primitives: what this platform can actually promise. +# +# The WAL is the SDK's only copy of an event between `track()` and the +# backend accepting it. Three things protect it, and none of them is +# available everywhere: +# +# 0600 on every file — the WAL holds full event payloads, which for an +# agent means prompts, completions and tool arguments. +# Default umask 022 leaves them world-readable in +# /tmp, on a shared volume, and in a container. +# advisory flock — a WAL is a SHARED path by default (one per tempdir, +# not one per process), so a second process appending +# the read-copy-append of the DLQ can interleave a +# line into the middle of another. +# directory fsync — `os.replace` is atomic, but without fsyncing the +# containing directory the new NAME can be absent +# after a power cut even though the data blocks +# landed. +# +# All three are probed, not imported. A platform lacking one is a supported +# configuration, so the probe result becomes observable state (metrics +# `wal_dir_fsync` / `wal_lock`) and a single warning, rather than an +# ImportError at module load or a silent downgrade nobody can see. +# +# The absence of `flock` is the sharp one. It means a multi-process +# deployment (gunicorn -w N) shares a WAL with no protection against +# interleaved appends, and the correct configuration is a per-process +# NULLRUN_WAL_PATH. The lock does not merge per-process buffers either way; +# re-delivery is absorbed by the backend's dedup on `event_id`. What the +# lock buys is that the FILES stay parseable. +# --------------------------------------------------------------------------- + +try: # POSIX only. Absence is a configuration, not an error. + import fcntl as _fcntl +except ImportError: # pragma: no cover - exercised on Windows + _fcntl = None + +# Typed as `Any` deliberately. A bare `fcntl = None` makes the type of the +# name depend on which platform the type checker is running on, and mypy +# resolves `fcntl` to a stub without `flock` on win32 — so the same line is a +# clean type on Linux CI and five errors on a Windows dev box. +fcntl: Any = _fcntl +# DLQ size cap (default 64 MB). Override via NULLRUN_DLQ_MAX_BYTES. +_DLQ_MAX_BYTES_DEFAULT: int = 64 * 1024 * 1024 + +# Max refused events tracked in memory while the DLQ is full. Override via +# NULLRUN_DLQ_HOLDOVER_MAX_EVENTS. The rows are durable in `.holdover` +# either way; this bounds the index, not the data. +_DLQ_HOLDOVER_INDEX_MAX_DEFAULT: int = 10_000 + +# Longest server-stated `Retry-After` the SDK will sit through. Past this it +# stops retrying for the cycle rather than retrying early — see +# `_next_retry_delay` for why shortening the wait is the wrong cap. +# Override via NULLRUN_RETRY_AFTER_CEILING. +_RETRY_AFTER_CEILING_DEFAULT: float = 60.0 + +# How long a writer waits for the cross-process WAL lock before skipping its +# write. The critical section is a local file copy plus an fsync, so the +# uncontended cost is microseconds; the timeout exists for a descheduled +# holder, not for a busy one. +_WAL_LOCK_TIMEOUT_DEFAULT: float = 5.0 + +_DEGRADATION_WARNING = ( + "WAL durability degraded on this platform: %s. The events are still " + "written and still replayed, but the guarantee they rest on is weaker. " + "See metrics `wal_dir_fsync` / `wal_lock` for the live state." +) + + +class UnconfirmedBatchResponse(Exception): + """A 2xx whose body did not say which events landed. + + Status said success; the body said nothing we can read — a proxy HTML + page, an empty response, or a backend predating ``accepted_event_ids``. + The request may well have been fully processed, and it may not have been + delivered at all, and the difference decides whether the caller retries + or stops. Guessing either way loses something, so the batch keeps its + attempt budget and only reaches the DLQ once that budget is spent. + """ + + def __init__(self, event_count: int) -> None: + self.event_count = event_count + super().__init__(f"{event_count} events got a 2xx with no readable batch confirmation") + + +class UnknownRejectionReason(Exception): + """The backend refused one event and named a reason we have no rule for. + + Not a reason to drop it. An SDK built before a reason existed would + otherwise be the only fleet that discards those events, and the discard + is invisible — the operator sees a shorter ledger and no error. So the + event keeps its attempt budget and lands in the DLQ when it runs out, + with the reason preserved verbatim for whoever triages it. + """ + + def __init__(self, reason: str, event_id: str | None) -> None: + self.reason = reason + self.event_id = event_id + super().__init__(f"unhandled rejection reason {reason!r} for event {event_id}") + + +# How a caller asks a transport failure to be surfaced (ADR-008). A callable +# receives the exception and returns a decision dict; the three strings select +# a built-in arm. ``None`` means "use ``fallback_mode``", which is fail-CLOSED +# under ``mode="strict"``. Named because the union appears at several call +# sites and the narrower callback-only annotation that used to stand here +# made the documented ``"raise"`` arm un-passable under a type checker. +TransportErrorHandler = str | Callable[[Exception], dict[str, Any]] + + +class DeterministicBackendRefusal(Exception): + """The backend named a refusal that retrying cannot fix. + + Carries the wire ``error_code`` and status separately because the status is + not the signal — several of these arrive as 5xx, which is exactly why + status-based classification mis-routes them. Handlers of this exception + must NOT count it as a transport failure: the transport layer is healthy, + the answer is a decision. + + ``details`` is the wire ``details`` object. It is not decoration: for + ``CONSUME_OVERBUDGET`` it carries ``reservation_recorded`` — the backend + saying it already applied the spend — and the ``event_id`` it applied it + to. That marker is the difference between an event that is genuinely + undelivered (nothing charged, park it in the DLQ) and one the server has + already booked (delivered; quarantining it files a rejection that never + happened). See ``_deliver_recorded_overbudget``. + """ + + def __init__( + self, + error_code: str, + status_code: int | None, + message: str = "", + details: dict[str, Any] | None = None, + ) -> None: + self.error_code = error_code + self.status_code = status_code + self.detail = message + self.details: dict[str, Any] = details or {} + super().__init__(f"{error_code} (HTTP {status_code}): {message}" if message else error_code) + + +def _retry_after_seconds_from(response: Any) -> float | None: + """``Retry-After`` in seconds, from either wire form. ``None`` if absent/unparseable. + + Module-level because the retry loop needs it and has no Transport: it + reads the header off the ``HTTPStatusError`` the response produced. Both + RFC 7231 forms are accepted — delta-seconds and an HTTP-date — because + nginx and a hand-rolled rate limiter in front of the gate will each emit + a different one. + + An HTTP-date already in the past yields a NEGATIVE number. That is + returned rather than clamped, so the caller can tell "the server said + wait zero" from "the server said wait, and that moment has passed" — the + retry loop treats non-positive as "no floor stated" and falls back to + exponential backoff, which is the safe reading of a stale date. + """ + if response is None: + return None + headers = getattr(response, "headers", None) + if not headers: + return None + try: + retry_after = headers.get("Retry-After") + except Exception: + return None + if not retry_after: + return None + + try: + return float(retry_after) + except (TypeError, ValueError): + pass + + try: + from datetime import datetime, timezone + from email.utils import parsedate_to_datetime + + dt = parsedate_to_datetime(retry_after) + return (dt - datetime.now(timezone.utc)).total_seconds() + except Exception: + return None + + +def _extract_backend_error_code(response: Any) -> str | None: + """Pull ``error_code`` out of a backend error envelope, if there is one. + + The envelope is ``{"error_code", "error_message", "details", "retry_after_ms"}`` + (see the TrackError::WithBody sites in handlers.rs). Returns ``None`` for a + body that is absent, unparseable, or a plain proxy HTML page — a missing + code is NOT evidence of a deterministic refusal, so the caller keeps the + request on the transient path. + """ + if response is None: + return None + try: + payload = response.json() + except Exception: # noqa: BLE001 — non-JSON body (proxy error page, empty) + return None + if not isinstance(payload, dict): + return None + code = payload.get("error_code") + if isinstance(code, str) and code: + return code + return None + + +def _extract_error_message(response: Any) -> str: + try: + payload = response.json() + except Exception: # noqa: BLE001 + return "" + if isinstance(payload, dict): + message = payload.get("error_message") + if isinstance(message, str): + return message + return "" + + +def _as_int_or_none(value: Any) -> int | None: + """An ``int`` if the wire value is one, else ``None``. + + ``bool`` is excluded on purpose: it is an ``int`` in Python, and a + JSON ``true`` landing on a field typed as a money amount is a wire + defect worth surfacing as absence rather than as ``1``. + + Used for the numeric fields of a typed error. Coercing a string + instead would hand a caller something that looks like a number and + fails at the arithmetic; ``None`` fails at the check, which is where + it can still be handled. + """ + if isinstance(value, bool) or not isinstance(value, int): + return None + return value + + +def _extract_error_details(response: Any) -> dict[str, Any]: + """Pull the wire ``details`` object out of a backend error envelope. + + Returns ``{}`` for an absent, unparseable, or non-object body — the same + "no evidence" reading ``_extract_backend_error_code`` uses. Callers must + treat the empty dict as *the backend told us nothing*, never as *the + backend said no*: an absent marker is a different fact from a marker set + to ``false``, and conflating them would make a rollback indistinguishable + from a refusal that charged nothing. + """ + if response is None: + return {} + try: + payload = response.json() + except Exception: # noqa: BLE001 — non-JSON body (proxy page, empty) + return {} + if isinstance(payload, dict): + details = payload.get("details") + if isinstance(details, dict): + return details + return {} + + # ============================================================================= # Retry with exponential backoff + jitter # ============================================================================= +def _next_retry_delay( + *, + attempt: int, + max_retries: int, + base_delay: float, + backoff_factor: float, + jitter: float, + max_delay: float, + server_wait: float | None, + error: Exception, + retry_after_ceiling: float = _RETRY_AFTER_CEILING_DEFAULT, +) -> float | None: + """How long to wait before the next attempt — or ``None`` to not retry. + + Two regimes, and the choice between them is the whole function. + + With a server-stated ``server_wait`` the value is a FLOOR, so jitter is + one-sided — the spread is ``[wait, wait * (1 + jitter)]``. Spreading + downward would retry before the rate limit permitted it, which is how a + 429 becomes a sustained 429; and spreading symmetrically is what makes a + fleet that got limited together retry together. + + Without one there is no floor, so jitter is symmetric around the + exponential curve, which is the standard defence against a thundering + herd on a shared failure. + + Returns ``None`` when the server's floor is beyond ``retry_after_ceiling``. + This is the part that used to be wrong, and it was wrong in the + direction that looks like a safety feature. + + The previous code clamped with ``min(server_wait, max_delay)``. That + bounds the wait, so ``Retry-After: 86400`` cannot hang a flush — and it + also retries 60 minutes early, which is precisely what the floor + semantics forbid. A server that says "not for an hour" and gets a client + back in 30 seconds gets the same 429, spends its whole retry budget + re-tripping the limit it was told to respect, and the operator sees a + client that ignores rate limits rather than one that honours a long one. + + The fix is not a shorter wait, it is no wait. The events are already + durable — ``.inflight`` is retained and the WAL is the recovery path — + so declining to retry in this cycle costs nothing and loses nothing. The + flush ends, and the next cycle tries again. So the ceiling produces an + honest "not now" instead of a dishonest "yes, sooner". + """ + if server_wait is not None and server_wait > 0: + if server_wait > retry_after_ceiling: + metrics.inc_transport("retry_after_deferred") + logger.warning( + "Request failed (attempt %d/%d), Retry-After %.2fs exceeds the " + "%.2fs ceiling — not retrying in this cycle rather than " + "retrying before the server permits it. The events stay in the " + "WAL; the next flush cycle tries again. Raise " + "NULLRUN_RETRY_AFTER_CEILING to wait longer: %s", + attempt + 1, + max_retries + 1, + server_wait, + retry_after_ceiling, + type(error).__name__, + ) + return None + base = server_wait + delay = base * (1.0 + random.uniform(0.0, jitter)) # noqa: S311 + logger.warning( + "Request failed (attempt %d/%d), honoring Retry-After %.2fs " + "(+jitter -> %.2fs): %s", + attempt + 1, + max_retries + 1, + base, + delay, + type(error).__name__, + ) + return delay + + backoff = min(base_delay * (backoff_factor**attempt), max_delay) + spread = backoff * jitter + delay = max(0.0, backoff + random.uniform(-spread, spread)) # noqa: S311 + logger.warning( + "Request failed (attempt %d/%d), retrying in %.2fs: %s", + attempt + 1, + max_retries + 1, + delay, + type(error).__name__, + ) + return delay + + +def _retry_after_ceiling() -> float: + """Effective ``Retry-After`` ceiling. Non-positive env values are ignored.""" + raw = os.environ.get("NULLRUN_RETRY_AFTER_CEILING", "").strip() + if not raw: + return _RETRY_AFTER_CEILING_DEFAULT + try: + value = float(raw) + return value if value > 0 else _RETRY_AFTER_CEILING_DEFAULT + except ValueError: + return _RETRY_AFTER_CEILING_DEFAULT + + +def _wait_on(cancel: threading.Event, seconds: float) -> bool: + """Wait on ``cancel`` for up to ``seconds``. The clock seam for tests. + + A module-level function, not an inline ``cancel.wait``, so the test + fixture can stub the CLOCK without stubbing the decision. The + alternative — patching ``threading.Event.wait`` globally — changes the + semantics of every other ``Event.wait`` in the suite, including the + approval-wait tests that legitimately assert on real elapsed time, and + turns them into load-sensitive flakes to make the retry path fast. + Stubbing the seam keeps the full retry path — the ceiling check, the + one-sided jitter, the shutdown branch — under test, and only removes + the wall clock from it. + """ + return not cancel.wait(seconds) + + +def _interruptible_sleep(seconds: float, cancel: threading.Event | None) -> bool: + """Sleep, but wake early when the SDK is shutting down. + + Returns True if the full delay elapsed, False if it was cut short by + ``cancel``. + + ``time.sleep`` is the wrong primitive on the flush path for one reason: + it cannot be woken. ``Transport.stop()`` sets an event that the flush + loop honours between cycles, but a retry sleeping on ``Retry-After: + 45`` inside a cycle does not check that event, so a shutdown that arrives + during a rate-limit wait blocks for the remainder of the wait. For a + process that is exiting, that is the difference between a prompt stop + and one that appears hung — and it is worst exactly when the backend is + misbehaving, which is when someone is most likely to be killing it. + """ + if cancel is None: + time.sleep(seconds) + return True + return _wait_on(cancel, seconds) + + def _retry_with_backoff( func: Callable[[], Any], max_retries: int = 10, @@ -213,14 +699,21 @@ def _retry_with_backoff( backoff_factor: float = 2.0, jitter: float = 0.1, last_retry_after_seconds: float = 0.0, - on_transport_error: str | Callable[[Exception], dict[str, Any]] | None = None, + on_transport_error: TransportErrorHandler | None = None, retry_on_5xx: bool = False, + cancel: threading.Event | None = None, + retry_after_ceiling: float | None = None, ) -> Any: """Retry with exponential backoff + jitter; honors Retry-After (429) header. Formula (without Retry-After): delay = min(base_delay * backoff_factor^attempt, max_delay) delay += random.uniform(-jitter * delay, jitter * delay) Formula (with Retry-After): actual_delay = min(last_retry_after_seconds, max_delay) + delay *= (1 + random.uniform(0, jitter)) + The Retry-After jitter is one-sided on purpose: a rate limit states a + FLOOR, so spreading downward would retry before the server permitted it. + Read off the response the exception carries; see + ``_retry_after_seconds_from``. NR-006: when ``retry_on_5xx=True`` a 5xx response is treated as transient infrastructure failure and retried via the same @@ -244,6 +737,9 @@ def _retry_with_backoff( NullRunBackendError, ) + if retry_after_ceiling is None: + retry_after_ceiling = _retry_after_ceiling() + last_exc: Exception | None = None for attempt in range(max_retries + 1): @@ -312,15 +808,47 @@ def _retry_with_backoff( NullRunAuthenticationError, NullRunTransportError, NullRunBackendError, + # A refusal the backend NAMED as deterministic. Retrying it is + # guaranteed waste, and — since several of these arrive as 5xx — + # letting it fall through the generic handler would also count it + # as infrastructure failure on a perfectly healthy backend. + DeterministicBackendRefusal, ): raise except httpx.HTTPStatusError as exc: - # 5xx HTTPStatusError from the retry_on_5xx branch above. - # Treat as retryable transient infra failure. + # 5xx HTTPStatusError from the retry_on_5xx branch above, and + # also the 429 that `_post_batch` turns into one. Treat as + # retryable transient infra failure. + # + # This branch used to set `last_exc` and fall straight through + # to the next attempt with NO delay: no backoff, no sleep, and + # the `Retry-After` header the server had just sent ignored + # entirely. A rate-limited client therefore spent its whole + # retry budget in milliseconds, hammering the limit that had + # just answered it, and then gave up — the opposite of what a + # 429 asks for. The suite stayed green because the existing + # 429 test asserted the CALL COUNT, which does not depend on + # how long you wait between calls. last_exc = exc if attempt >= max_retries: break + metrics.inc_transport("retries_total") + actual_delay = _next_retry_delay( + attempt=attempt, + max_retries=max_retries, + base_delay=base_delay, + backoff_factor=backoff_factor, + jitter=jitter, + max_delay=max_delay, + server_wait=_retry_after_seconds_from(exc.response), + error=exc, + retry_after_ceiling=retry_after_ceiling, + ) + if actual_delay is None: + # The server's floor is past our ceiling. Stop this cycle + # rather than retry early; the caller keeps the events. + break except Exception as exc: last_exc = exc @@ -333,30 +861,31 @@ def _retry_with_backoff( metrics.inc_transport("retries_total") - if last_retry_after_seconds > 0: - actual_delay = min(last_retry_after_seconds, max_delay) - last_retry_after_seconds = 0.0 - logger.warning( - "Request failed (attempt %d/%d), honoring Retry-After %.2fs: %s", - attempt + 1, - max_retries + 1, - actual_delay, - type(exc).__name__, - ) - else: - delay = min(base_delay * (backoff_factor**attempt), max_delay) - jitter_amount = delay * jitter - actual_delay = delay + random.uniform(-jitter_amount, jitter_amount) # noqa: S311 - actual_delay = max(0.0, actual_delay) - logger.warning( - "Request failed (attempt %d/%d), retrying in %.2fs: %s", - attempt + 1, - max_retries + 1, - actual_delay, - type(exc).__name__, - ) + actual_delay = _next_retry_delay( + attempt=attempt, + max_retries=max_retries, + base_delay=base_delay, + backoff_factor=backoff_factor, + jitter=jitter, + max_delay=max_delay, + server_wait=last_retry_after_seconds, + error=exc, + retry_after_ceiling=retry_after_ceiling, + ) + last_retry_after_seconds = 0.0 + if actual_delay is None: + break - time.sleep(actual_delay) + # Every path that reaches here has decided to retry. One sleep, one + # place: the two except branches used to carry their own, and the + # status branch had none, which is how the two drifted apart. + if not _interruptible_sleep(actual_delay, cancel): + # Shutdown arrived mid-wait. Stop retrying now; the caller still + # holds the events, and `.inflight` is retained because the send + # did not complete. + logger.info("Retry wait interrupted by shutdown; stopping this cycle") + metrics.inc_transport("retries_interrupted_by_shutdown") + break # ``retry_on_5xx`` and the failure mode was 5xx, return the # last response so the caller can synthesize a fallback @@ -551,6 +1080,35 @@ def __init__( ) self._buffer: list[dict[str, Any]] = [] self._in_flight: dict[str, dict[str, Any]] = {} # event_id -> event for retry dedup + # Per-batch re-queue budget. A failure we cannot classify as transient + # (a payload that will not serialize, a shape the signer rejects) repeats + # identically on every cycle and would pin the batch ahead of the whole + # buffer forever. Count attempts, and dead-letter once spent. + self._batch_attempts: dict[str, int] = {} # batch signature -> attempts + # event_id -> epoch seconds of its first observed failure, so a DLQ + # record says how long the event was failing rather than only that it + # was. In-memory only: after a restart the clock restarts, and the + # durable history is the DLQ file's own row order (each re-park + # appends). That is a deliberate trade — reading the whole DLQ to + # recover a timestamp would make the write path O(file), and the + # value is diagnostic, not load-bearing for recovery. + self._first_failed_at: dict[str, float] = {} + # Durability state. Probed on first use against the real filesystem, + # reported in metrics, warned about once. See the "Durability + # primitives" block above for why each of these is optional-but-named. + self._degradation_warned: set[str] = set() + self._dir_fsync_ok: bool | None = None + self._max_batch_attempts = int(os.environ.get("NULLRUN_MAX_BATCH_ATTEMPTS", "10")) + self._bisect_depth = 0 # recursion guard for 400/422 batch splitting + # Request budget for one bisect cascade. The depth guard alone bounds + # the RECURSION, not the wire: halving to depth 24 is 2^24 sends. A + # batch that is genuinely all-bad therefore spends an unbounded amount + # of the operator's rate limit rediscovering that fact. This counts + # sends instead, so the cost is a number a human chose. + self._bisect_budget = int(os.environ.get("NULLRUN_MAX_BISECT_REQUESTS", "64")) + # Terminal refusals the DLQ could not take, held in memory until it + # has room. See `_hold_for_dlq` for why they are not re-sent. + self._dlq_overflow: list[dict[str, Any]] = [] # RLock so re-entrant acquisition (e.g. test fixtures that hold the # lock while calling lock-acquiring methods) doesn't deadlock. self._lock = threading.RLock() @@ -596,6 +1154,25 @@ def __init__( keepalive_expiry=30.0, ), ) + # Via-edge. `None` is the ordinary case and costs nothing: the + # cloud stays on the enforcement path exactly as before. + # + # Read HERE, at construction, rather than per call: a + # reconfigured box that silently switches enforcement targets + # mid-flight would be a budget being enforced by one authority + # and then another, with no record of the change. Configuration + # is expected to change at deploy, not at runtime. + from nullrun.edge import EdgeConfigurationError, edge_transport_from_env + + try: + self.edge = edge_transport_from_env() + except EdgeConfigurationError: + # Raised, not warned and swallowed. Via-edge was asked for + # and cannot be enforced; continuing would mean every call + # runs with no gate at all while the operator believes the + # box is holding the line. + raise + self._redis_client = redis_client self._circuit_breaker = CircuitBreaker( failure_threshold=self.config.max_failed_flush, @@ -615,6 +1192,14 @@ def __init__( self._tracer = trace.get_tracer("nullrun.transport") self._propagator = TraceContextTextMapPropagator() + # Tighten anything a previous run left behind, BEFORE the replay + # path reads it. A file created by an older SDK, or by a run under a + # different umask, keeps its old mode through every `os.replace`, + # so hardening only new writes would leave the existing events — the + # ones most likely to hold full prompts and tool arguments — readable + # by everyone with access to the volume. + self._harden_wal_permissions() + # Final-flush hook via weakref.finalize — only fires if this Transport self._finalizer = weakref.finalize(self, self._atexit_flush_safe) @@ -673,63 +1258,571 @@ def _rotate_wal_if_needed(self) -> None: except OSError as e: logger.warning(f"Failed to rotate WAL {wal_path}: {e}") - def _persist_to_wal(self) -> None: - """Persist unflushed events to WAL file for replay on restart.""" - if not self._buffer: + def _wal_inflight_path(self) -> str: + """Path of the in-flight batch file (the batch currently on the wire).""" + return f"{self._wal_path()}.inflight" + + def _wal_dlq_path(self) -> str: + """Path of the dead-letter file for permanently-rejected batches.""" + return f"{self._wal_path()}.dlq" + + def _wal_holdover_path(self) -> str: + """Path of the holdover file: refused events the DLQ had no room for. + + Deliberately NOT `.wal` and NOT `.wal.inflight`. `.wal` is rewritten + wholesale by `_persist_to_wal` on every buffer persist, so held rows + appended there are erased by the next unsent event; `.inflight` is + overwritten with whatever batch is about to go on the wire, which is + precisely the loss this file exists to prevent. A separate file has + one writer (the hold) and one eraser (the drain), and nothing else + touches it. + """ + return f"{self._wal_path()}.holdover" + + def _wal_lock_path(self) -> str: + """Path of the advisory lock guarding mutations of the WAL files.""" + return f"{self._wal_path()}.lock" + + @property + def _dlq_max_bytes(self) -> int: + """Effective DLQ size cap. Same shape as ``_wal_max_bytes``.""" + raw = os.environ.get("NULLRUN_DLQ_MAX_BYTES", "").strip() + if not raw: + return _DLQ_MAX_BYTES_DEFAULT + try: + value = int(raw) + return value if value > 0 else _DLQ_MAX_BYTES_DEFAULT + except ValueError: + return _DLQ_MAX_BYTES_DEFAULT + + @property + def _holdover_index_max(self) -> int: + """Max tracked holdover entries. Bounded on COUNT, not bytes. + + Count is the unit the metric reports and the unit an operator alerts + on; a byte bound here would be a second cap for the same property and + would fail at a threshold nobody chose deliberately. The cap trims + the in-memory index only — the holdover file keeps every row, so a + saturated index under-reports `dlq_holdover` rather than losing an + event. + """ + raw = os.environ.get("NULLRUN_DLQ_HOLDOVER_MAX_EVENTS", "").strip() + if not raw: + return _DLQ_HOLDOVER_INDEX_MAX_DEFAULT + try: + value = int(raw) + return value if value > 0 else _DLQ_HOLDOVER_INDEX_MAX_DEFAULT + except ValueError: + return _DLQ_HOLDOVER_INDEX_MAX_DEFAULT + + # -- durability probes ------------------------------------------------ + + def _warn_degraded_once(self, aspect: str, state: str, detail: str) -> None: + """Say it once per aspect per Transport, and record the state. + + The metric gets a short enum value and the log gets the sentence. + A metric field an operator has to substring-match to alert on is a + metric field nobody alerts on. + + Once, because a degraded platform is a standing condition, not an + event: warning on every flush would bury the log under a fact that + never changes and never resolves itself. + """ + metrics.set_transport(f"wal_{aspect}", state) + if aspect in self._degradation_warned: return - event_count = len(self._buffer) - wal_path = self._wal_path() - self._rotate_wal_if_needed() - wal_dir = os.path.dirname(wal_path) or "." + self._degradation_warned.add(aspect) + logger.warning(_DEGRADATION_WARNING % detail) + + def _dir_fsync_supported(self, directory: str) -> bool: + """Whether this platform and filesystem can fsync a DIRECTORY. + + Probed against the real directory rather than inferred from the OS: + Windows has no directory fsync at all, and on Linux some mounts + (overlayfs upper dirs, certain network and FUSE filesystems) accept + the open and then refuse the fsync. Both are indistinguishable from + the outside and both mean the same thing, so both report False. + """ + if self._dir_fsync_ok is not None: + return self._dir_fsync_ok + ok = True + fd = -1 try: - os.makedirs(wal_dir, exist_ok=True) - except OSError as e: - logger.warning(f"Cannot create WAL directory {wal_dir}: {e}") + fd = os.open(directory, os.O_RDONLY) + os.fsync(fd) + except OSError: + ok = False + finally: + if fd >= 0: + try: + os.close(fd) + except OSError: + pass + self._dir_fsync_ok = ok + if ok: + metrics.set_transport("wal_dir_fsync", "enabled") + else: + self._warn_degraded_once( + "dir_fsync", "unavailable", f"directory fsync unavailable on {directory}" + ) + return ok + + @property + def _wal_lock_timeout(self) -> float: + """Seconds to wait for the cross-process WAL lock before giving up.""" + raw = os.environ.get("NULLRUN_WAL_LOCK_TIMEOUT_MS", "").strip() + if not raw: + return _WAL_LOCK_TIMEOUT_DEFAULT + try: + value = float(raw) / 1000.0 + return value if value > 0 else _WAL_LOCK_TIMEOUT_DEFAULT + except ValueError: + return _WAL_LOCK_TIMEOUT_DEFAULT + + @contextmanager + def _wal_file_lock(self) -> Iterator[bool]: + """Hold the cross-process WAL lock. Yields False if it could not be had. + + A NEW descriptor is opened per acquisition, and that is load-bearing, + not incidental. `flock` locks the open file DESCRIPTION, so two + threads of one process each opening their own descriptor DO conflict — + which is why this needs no companion `threading.Lock` and why adding + one would be redundant. Caching the descriptor to save an `open()` + would invert this: the kernel would see one description, and the + second `LOCK_EX` would succeed immediately against the first holder's + own lock. Cross-process safety survives that; in-process safety does + not. `test_flock_excludes_two_threads_of_one_process` is the tripwire. + + Advisory (`flock`), exclusive, and **bounded-wait, not + non-blocking**. Non-blocking is the obvious choice and it is wrong: + it was tried here, and on a 4-worker deployment the losing writer was + refused on every single attempt, forever, because the winner was + always mid-append. Its events were never lost — the caller re-queues + them — and never landed either, so a DLQ nobody could ever write to + while the other process was busy. A wait costs the losing writer + microseconds, because the critical section is a local file copy and + fsync; starvation is not a cheaper option than waiting. + + Bounded, because "wait forever" turns a descheduled writer into a + hung flush thread. On expiry the caller skips its write and keeps the + events — the same fail-safe as any other reason the write did not + happen. The lock is released by the kernel if a holder dies, so a + crashed process cannot hold it. + + The scope is the FILE MUTATION only — never the network send. + Holding it across a request would serialise every worker behind one + another's latency, which is the opposite of what it is for. + """ + if fcntl is None: + # No flock here (Windows). The files still get 0600 and the + # platform's own sharing semantics; what is missing is the guard + # against two processes appending at once, which is a real + # multi-process hazard and is why the warning says so. + self._warn_degraded_once( + "lock", + "unavailable", + "no fcntl.flock — one writer per WAL path, set NULLRUN_WAL_PATH per worker", + ) + yield True return - tmp_path = f"{wal_path}.tmp.{os.getpid()}" + + lock_path = self._wal_lock_path() + directory = os.path.dirname(lock_path) or "." try: - with open(tmp_path, "a") as f: - for event in self._buffer: - f.write(json.dumps(event, default=str) + "\n") - f.flush() - os.fsync(f.fileno()) - os.replace(tmp_path, wal_path) - self._buffer.clear() - logger.debug(f"Persisted {event_count} events to WAL at {wal_path}") + os.makedirs(directory, exist_ok=True) except OSError as e: - logger.warning(f"Failed to persist {event_count} events to WAL: {e}") + logger.warning(f"Cannot create WAL directory {directory}: {e}") + yield False + return - def _replay_from_wal(self) -> None: - """Replay events from WAL file on startup. + try: + fd = os.open(lock_path, os.O_RDWR | os.O_CREAT, 0o600) + except OSError as e: + # A lock we cannot open is a lock we do not have. Say so once + # and proceed unlocked rather than refusing to write at all — + # the alternative is losing every event on a permissions + # problem in a directory the WAL itself already lives in. + self._warn_degraded_once( + "lock", "unavailable", f"cannot open {lock_path}: {e}" + ) + yield True + return - P1-5b: also drains the rotated ``.wal.1`` (oldest - surviving recovery window) before the active ``.wal`` so - a crash between rotation and replay doesn't lose events. - Both files are removed only after a successful flush. + acquired = False + deadline_step = 0.02 + # Measured, not counted. A budget expressed as "50 naps of 20ms" is + # a budget of 50 *naps*, and anything that returns from sleep early — + # a signal, a test harness that stubs sleep, a coarse scheduler — + # spends the whole 5s in milliseconds and gives up while the lock is + # still held. The timeout is a duration, so it is compared against a + # clock. + start = time.monotonic() + try: + while True: + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + acquired = True + break + except OSError: + if time.monotonic() - start >= self._wal_lock_timeout: + break + time.sleep(deadline_step) + waited = time.monotonic() - start + if not acquired: + metrics.set_transport("wal_lock", "contended") + metrics.inc_transport("wal_lock_timeouts_total") + logger.warning( + f"WAL lock {lock_path} still held after {waited:.2f}s; " + f"skipping this write. The events stay on the retry path." + ) + yield False + return + if waited > deadline_step: + metrics.set_transport("wal_lock", "contended") + logger.debug( + f"WAL lock {lock_path} waited {waited:.2f}s before being granted" + ) + metrics.set_transport("wal_lock", "enabled") + yield True + finally: + if acquired: + try: + fcntl.flock(fd, fcntl.LOCK_UN) + except OSError: + pass + try: + os.close(fd) + except OSError: + pass + + def _harden_wal_permissions(self) -> None: + """Force 0600 on every WAL file, including ones a previous run left. + + A file written by an older SDK, or by a run under a different umask, + keeps its old mode: `os.replace` carries the mode of the file it + renamed, so tightening only the tmp file tightens new writes and + leaves every existing file as it was. Events are payloads, so this is + a data-exposure fix, not a tidiness one. """ - events: list[dict[str, Any]] = [] - for candidate in (f"{self._wal_path()}.1", self._wal_path()): + base = self._wal_path() + for candidate in ( + base, + f"{base}.1", + f"{base}.inflight", + f"{base}.holdover", + f"{base}.dlq", + f"{base}.lock", + ): try: - with open(candidate) as f: - for line in f: - try: - events.append(json.loads(line.strip())) - except json.JSONDecodeError: - continue + os.chmod(candidate, 0o600) except FileNotFoundError: continue except OSError as e: - logger.warning(f"Failed to read WAL {candidate}: {e}") + # Best-effort by design. Refusing to run because a mode + # could not be tightened would trade a permissions warning + # for total data loss. + logger.warning(f"Could not restrict permissions on {candidate}: {e}") + + def _dlq_size(self) -> int: + try: + return os.path.getsize(self._wal_dlq_path()) + except OSError: + return 0 + + def _dlq_over_cap(self, incoming_bytes: int) -> bool: + """True when appending would take the DLQ past its cap. + + The cap NEVER deletes anything. The alternative — rotating or + trimming the oldest rows — destroys the only copy of an event the SDK + could not deliver, silently, with no trace beyond a short file. A + DLQ that has stopped growing is a visible, alertable condition; a + DLQ that quietly dropped the oldest refusals is neither. So the cap + stalls the write and the caller re-queues the event on the retry path + instead, where a further crash still holds it and the next attempt + reports the same refusal. + + `incoming_bytes` is part of the test, not an optimisation: a cap + checked only against the current size admits any single write, so an + operator who set 1 GB would get a 40 GB file the first time a large + batch was quarantined. + """ + size = self._dlq_size() + metrics.set_transport("dlq_bytes", size) + if size < self._dlq_max_bytes and size + incoming_bytes <= self._dlq_max_bytes: + return False + metrics.inc_transport("dlq_overflow_total") + metrics.set_transport("dlq_overflow_reason", "size_cap") + logger.error( + f"DLQ {self._wal_dlq_path()} would exceed its cap " + f"({size} bytes + {incoming_bytes} incoming > {self._dlq_max_bytes}). " + f"Refusing to append: the cap never deletes rows, so the event " + f"stays on the retry path. Raise NULLRUN_DLQ_MAX_BYTES, or triage " + f"and archive the file with `nullrun-wal`." + ) + return True + + def _write_events_atomic( + self, path: str, events: list[dict[str, Any]], mode: str = "w" + ) -> bool: + """Write ``events`` as JSON lines to ``path`` atomically. + + tmp + fsync + os.replace: a reader either sees the previous file or + the complete new one, never a half-written batch. ``mode="a"`` copies + the existing file first and appends, so successive writes accumulate + instead of clobbering — that is what the DLQ needs, where losing an + earlier quarantine to a later one would itself be data loss. + + Two crash hazards the copy-and-append path has to survive: + + * A **torn tail**. If the process died mid-write, the last line of the + existing file has no terminating ``\\n``. Copying it verbatim glues + the next row onto the truncated JSON, producing one corrupt line + that takes a valid event with it. The unterminated tail is dropped + (it was never durable) before appending. + * A **stale tmp**. The name is pid-scoped and opened with ``"w"``, so a + leftover from a dead process is truncated rather than appended to. + + The directory is fsynced after the rename: without it the new name can + be absent after a power cut even though the data blocks landed. + + The tmp file is created 0600 rather than through ``open(..., "w")``, + because the mode of the file that ``os.replace`` renames is the mode + the WAL ends up with, and ``open`` applies the process umask to + whatever 0666 it asks for. On a default umask that leaves every event + payload world-readable in the tempdir, on a shared volume, and inside + a container that several services mount. + + The whole read-copy-append runs under the cross-process WAL lock: + it is a read followed by a write of the same file, and a second + process doing the same thing concurrently interleaves its lines into + the middle of the other's, which no reader can then parse. + + Returns True when the data is durably on disk. + """ + wal_dir = os.path.dirname(path) or "." + try: + os.makedirs(wal_dir, exist_ok=True) + except OSError as e: + logger.warning(f"Cannot create WAL directory {wal_dir}: {e}") + return False + tmp_path = f"{path}.tmp.{os.getpid()}" + with self._wal_file_lock() as acquired: + if not acquired: + # Another process owns these files. Our events are still in + # memory / still covered by `.inflight`, and the caller is + # told "not written" so it keeps the recovery file. + return False + try: + fd = os.open( + tmp_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600 + ) + with os.fdopen(fd, "w") as f: + if mode == "a" and os.path.exists(path): + # Existing contents FIRST, so the file stays in + # chronological order. Writing the new rows first and + # folding the old ones in after would reverse the DLQ. + # + # A crash can leave the last line unterminated. Copying it + # verbatim would glue the next row onto truncated JSON and + # corrupt an event that was perfectly valid, so the torn + # tail is dropped — it was never durable anyway. + with open(path) as prev: + previous = prev.read() + if previous and not previous.endswith("\n"): + keep = previous.rfind("\n") + 1 # 0 when there is none + logger.warning( + f"Dropping {len(previous) - keep}B torn tail of {path} before append" + ) + previous = previous[:keep] + f.write(previous) + for event in events: + f.write(json.dumps(event, default=str) + "\n") + f.flush() + os.fsync(f.fileno()) + os.replace(tmp_path, path) + self._fsync_dir(wal_dir) + return True + except OSError as e: + logger.warning( + f"Failed to persist {len(events)} events to {path}: {e}" + ) + try: + os.unlink(tmp_path) + except OSError: + pass + return False + + def _fsync_dir(self, path: str) -> None: + """Flush a directory entry so a rename survives a power cut. + + Best-effort: a failure here costs durability of the NAME, not the + data, which is already fsynced. What it must not cost is + *visibility* — a platform that cannot do this is probed once by + ``_dir_fsync_supported``, reported in metrics, and warned about, so + "we lose the rename on power cut here" is a known state rather than + an assumption nobody checked. + """ + if not self._dir_fsync_supported(path): + return + try: + fd = os.open(path, os.O_RDONLY) + except OSError: + return + try: + os.fsync(fd) + except OSError: + pass + finally: + os.close(fd) + + def _persist_to_wal(self) -> bool: + """Persist unflushed events to WAL file for replay on restart. + + Every event is guaranteed an ``event_id`` before it reaches disk. + Replay is at-least-once — a crash mid-send can re-deliver an event — + and the backend dedups on ``event_id`` (cost_event_id_dedup PRIMARY + KEY), so a stable id is exactly what makes re-delivery safe. + + Returns True when the buffer is durably on disk (and therefore + cleared), False when the write failed and the caller must NOT + discard any recovery file. + """ + if not self._buffer: + return True + for event in self._buffer: + if not event.get("event_id"): + event["event_id"] = str(uuid.uuid4()) + event_count = len(self._buffer) + wal_path = self._wal_path() + self._rotate_wal_if_needed() + if not self._write_events_atomic(wal_path, list(self._buffer)): + return False + self._buffer.clear() + logger.debug(f"Persisted {event_count} events to WAL at {wal_path}") + return True + + def _persist_inflight(self, batch: list[dict[str, Any]]) -> None: + """Record the batch that is about to go on the wire. + + The batch has already been removed from ``_buffer`` at this point, so + a crash during the send would otherwise lose it outright. Cleared once + the send is accepted; deliberately RETAINED when the send fails, + because then the batch is only in memory. A later successful flush + overwrites this file with its own batch and removes it, so a stale + copy self-heals rather than accumulating. + """ + if batch: + self._write_events_atomic(self._wal_inflight_path(), batch) + + def _clear_inflight(self) -> None: + try: + os.remove(self._wal_inflight_path()) + except FileNotFoundError: + pass + except OSError as e: + logger.warning(f"Failed to remove in-flight WAL: {e}") + + def _replay_from_wal(self) -> None: + """Recover events from every durable WAL source and flush them. + + Sources, oldest first: the rotated ``.wal.1``, the active ``.wal``, + and ``.wal.inflight`` — the batch that was on the wire when the + process died. + + Crash-safety invariant: no recovery file is unlinked before its + events are provably safe, i.e. either accepted by the backend or + rewritten into ``.wal`` by ``_persist_to_wal``. Recovery is + at-least-once; the backend dedups on ``event_id``. + + The read phase runs under the cross-process WAL lock; the send does + not. That leaves a window in which two processes have read the same + file and both replay it — a duplicate, absorbed by the backend's dedup + — and no window in which one of them can remove a file the other + still needs, because removal only ever happens after the events are + delivered or re-persisted. Holding the lock across the send instead + would serialise every worker in a multi-process deployment behind one + another's latency to remove a duplication the dedup key already + handles. + """ + wal_path = self._wal_path() + sources = [f"{wal_path}.1", wal_path, self._wal_inflight_path()] + events: list[dict[str, Any]] = [] + found: list[str] = [] + empty: list[str] = [] + with self._wal_file_lock() as acquired: + if not acquired: + logger.warning( + "WAL replay skipped: another process holds the WAL lock. " + "Its replay covers the same files." + ) + return + for candidate in sources: + try: + with open(candidate) as f: + lines = f.readlines() + except FileNotFoundError: + continue + except OSError as e: + logger.warning(f"Failed to read WAL {candidate}: {e}") + continue + if not lines: + empty.append(candidate) + continue + found.append(candidate) + for line in lines: + try: + events.append(json.loads(line.strip())) + except json.JSONDecodeError: + continue + + if not events: + # Nothing recovered. Drop leftovers so the next flush starts clean. + for candidate in empty: + try: + os.remove(candidate) + except OSError as e: + logger.warning(f"Failed to remove empty WAL {candidate}: {e}") + return + + self._buffer.extend(events) + try: + self._do_flush() + except BreakerTransportError as e: + # `_do_flush` normally swallows this internally (it re-queues the + # batch), but a transport subclass or a future refactor may not. + # Recovery must never abort `start()` — the events are already in + # the buffer, and the fall-through below persists them. + logger.warning(f"WAL replay flush failed: {e}") + + repersisted = False + if self._buffer: + # The flush did not deliver everything. Rewrite the survivors into + # `.wal` so they survive another crash, and only then discard the + # older sources. If that write fails the survivors are in memory + # only — keep every source so the next start can try again. + if not self._persist_to_wal(): + logger.warning( + f"WAL replay: {len(self._buffer)} events still buffered and could not be " + f"re-persisted; keeping {found} for the next start" + ) + return + repersisted = True + + # Everything recovered is either delivered or — when `repersisted` — + # sitting in a FRESH `.wal` that `_persist_to_wal` just wrote. That + # file must be spared: unlinking it here would discard exactly what we + # persisted. The other sources are now strictly redundant. + for candidate in found: + if repersisted and candidate == wal_path: continue try: os.remove(candidate) except OSError as e: logger.warning(f"Failed to remove WAL {candidate}: {e}") - if events: - self._buffer.extend(events) - self._do_flush() - if events: - logger.info(f"Replayed {len(events)} events from WAL") + logger.info( + f"Replayed {len(events)} events from WAL" + + (" (unflushed remainder re-persisted)" if repersisted else "") + ) def track(self, event: dict[str, Any]) -> None: """ @@ -758,6 +1851,12 @@ def start(self) -> None: return # Replay any events from WAL that were persisted due to previous crash self._replay_from_wal() + # Held refusals are recovered separately and are NOT replayed onto the + # send path. `_replay_from_wal` recovers undelivered events, which is + # the right thing for them; these were already refused, so re-sending + # produces the same refusal while blocking everything queued behind + # them. They go to the DLQ on the first flush that finds room. + self._recover_holdover() self._running = True # Clear the stop latch so a previous stop() does not short-circuit # the new flush loop on its first sleep. @@ -807,7 +1906,11 @@ def stop(self, timeout: float = 10.0, flush: bool = True) -> None: self._flush_thread.join(timeout=timeout) if flush: self._do_flush() # Final flush - self._persist_to_wal() # WAL any remaining events + if self._persist_to_wal(): + # The buffer is durably in `.wal` (or was already accepted), + # so any retained `.inflight` copy is now a subset of it. + # Clear it, or the next replay would load the same events twice. + self._clear_inflight() self._client.close() if getattr(self, "_finalizer", None) is not None and self._finalizer.alive: self._finalizer.detach() @@ -830,17 +1933,50 @@ def _do_flush(self) -> None: def _do_flush_locked(self) -> None: """Flush under lock. Must be called with _lock held.""" + # Space may have freed up since the last refusal (the operator raised + # the cap, or archived the file with `nullrun-wal`). Draining first + # means the held refusals land before new traffic, not after an + # unbounded wait for the next one to fail. + self._drain_dlq_overflow() + if not self._buffer: logger.debug("Buffer empty, skipping flush") return batch = self._buffer[:] self._buffer.clear() + # Refilled as the cascade below splits it, so one flush is one budget. + self._bisect_budget = int(os.environ.get("NULLRUN_MAX_BISECT_REQUESTS", "64")) logger.debug(f"Sending batch of {len(batch)} events") - # Circuit breaker wrapped send - uses proper 3-state circuit breaker + # Durability barrier: the batch has left `_buffer` and the process + # can now die at any point. Record it on disk before the send so a + # crash mid-flight is recoverable on the next start. + self._persist_inflight(batch) + + # Circuit breaker wrapped send - uses proper 3-state circuit breaker. + # + # Only genuine backend UNREACHABILITY may count as a breaker failure. + # A 4xx is the server answering, which proves the transport is healthy; + # a TypeError from local serialization is our own bug. Counting either + # would let a batch of malformed payloads open the circuit on a backend + # that is plainly up and would keep taking traffic, and then every + # buffered event sits blocked for the whole recovery window. Those are + # trapped here and handled after the call. + # + # httpx.TransportError (connect refused, DNS, read timeout) and + # BreakerTransportError (5xx exhaustion) still propagate, so the + # breaker keeps doing its actual job. + deferred: list[Exception] = [] + def send_batch(): - result = self._send_batch_with_retry_info(batch) + try: + result = self._send_batch_with_retry_info(batch) + except (BreakerTransportError, httpx.TransportError): + raise + except Exception as exc: # noqa: BLE001 + deferred.append(exc) + return None # Remove accepted events from in-flight if result.accepted_event_ids: for event in batch: @@ -854,7 +1990,7 @@ def send_batch(): return result try: - self._circuit_breaker.call(send_batch) + result = self._circuit_breaker.call(send_batch) except BreakerTransportError: logger.warning(f"Circuit breaker OPEN. Batch of {len(batch)} events will be re-queued.") # Drop NEWEST non-critical (state_change etc.) so oldest events @@ -866,7 +2002,858 @@ def send_batch(): if overflow > 0: batch = self._drop_newest_with_priority(batch, overflow) self._buffer.extend(batch) # Append to END so oldest events retry first. + # `.inflight` is intentionally NOT cleared here: the batch is + # back in memory only, and a crash before the next `_persist_to_wal` + # would lose it. A later successful flush overwrites and clears it. metrics.inc_transport("batches_failed") + return + except Exception as e: # noqa: BLE001 + self._retry_or_dlq(batch, e, dead_letter=True) + return + + if deferred: + self._route_deferred(batch, deferred[0]) + return + + # The batch was on the wire and came back 2xx. A 2xx is not + # delivery: settle each event against the per-item answer before + # clearing `.inflight`. + if not self._settle_batch_outcome(batch, result): + # Something is still live. `.inflight` stays — it is the durable + # copy of those events, and a crash before the next + # `_persist_to_wal` would otherwise lose them. + return + + self._batch_attempts.pop(self._failure_signature(batch), None) # landed + self._clear_inflight() # every event is durably accounted for + + def _settle_batch_outcome(self, batch: list[dict[str, Any]], result: Any) -> bool: + """Apply a 2xx per-item outcome. Returns True when nothing is still live. + + The load-bearing property is the ORDER, not the classification. Every + event must be durable — written to the DLQ, or still covered by a + retained ``.inflight`` — BEFORE ``.inflight`` is cleared. Cleared in + between and a crash loses whatever was only in memory. The other order + (durable first, crash after) costs a duplicate, which is safe: the + backend dedups on ``event_id`` and an ``IdempotentReplay`` consume + comes back accepted, so the replay lands instead of looping. A + duplicate is recoverable; a silent gap is not. + + That is also why this method writes the DLQ itself and returns rather + than calling ``_quarantine_to_dlq``: the latter clears ``.inflight``, + which would be wrong whenever anything in the same batch is being + re-queued rather than parked. + """ + if not getattr(result, "body_confirmed", True): + # We cannot read the answer. Bounded retry, then the DLQ — the + # budget is what stops an unreadable 200 from looping forever, + # and the DLQ is what stops it from ending in a silent drop. + # + # Negated on the way out: `_retry_or_dlq` answers "is it live", + # this method answers "is anything settled". Getting that backwards + # pops the attempt counter on every cycle, so the budget never + # depletes and the event retries forever. + return not self._retry_or_dlq( + batch, UnconfirmedBatchResponse(len(batch)), dead_letter=True + ) + + rejected_by_id = { + item["event_id"]: item for item in getattr(result, "rejected_items", []) or [] + } + accepted = set(getattr(result, "accepted_event_ids", []) or []) + + still_live = False + requeue: list[dict[str, Any]] = [] + unknown: list[tuple[dict[str, Any], str]] = [] + refused = 0 + + for event in batch: + eid = event.get("event_id") + if eid in accepted: + self._in_flight.pop(eid, None) + continue + refused += 1 + detail = rejected_by_id.get(eid) + if detail is None: + # In neither list. The server's answer does not account for + # this event at all, so we cannot claim it landed. Treated as + # an unknown refusal: bounded retry, then the DLQ. Guessing + # "accepted" here is how a partition bug turns into silent + # data loss, and guessing "rejected" is how a partition bug + # turns into an infinite retry. + unknown.append((event, "unaccounted_in_response")) + continue + reason = detail.get("reason") or "unspecified" + if reason in _TERMINAL_REJECTION_REASONS: + # Durably parked by the time this returns, so it does not keep + # `.inflight` alive — but the write had to happen first, which + # is why the DLQ row is written here rather than in the tail. + if self._dead_letter_row(event, reason): + still_live = True + + elif detail.get("retry_after_ms") is not None: + # The backend gave a retry hint: it expects this to succeed + # later. Bounded by the per-event attempt counter anyway, so + # a hint that never materialises cannot pin the buffer. + requeue.append(event) + still_live = True + else: + unknown.append((event, reason)) + + for event, reason in unknown: + if self._retry_or_dlq( + [event], UnknownRejectionReason(reason, event.get("event_id")), dead_letter=True + ): + still_live = True + + if requeue: + # Head, not tail: a refusal the backend expects to clear should + # not sit behind a steady stream of new events until the buffer + # drains. + self._requeue_at_head(requeue) + if refused: + metrics.inc_transport("events_partial_refused", refused) + return not still_live + + # 4xx classes that MUST NOT be quarantined. Quarantining these would + # discard good data at exactly the moment it is recoverable. + # + # 408 Request Timeout / 429 Too Many Requests — transient. After an + # outage every SDK replays its WAL at once, the rate limiter answers + # 429, and a naive "any 4xx is permanent" rule would quarantine every + # recovering fleet in the first second of the window. + # + # 401 Unauthorized / 403 Forbidden — the key was rotated or revoked. + # Once the operator fixes the key the batch must still deliver, so it + # belongs back in the WAL, not on disk as DLQ litter. + # + # 413 Payload Too Large — the BATCH is too big, not the events. Halve + # and resend; every event is individually acceptable. + _TRANSIENT_4XX = frozenset({408, 429}) + _AUTH_4XX = frozenset({401, 403}) + _RESPLIT_4XX = frozenset({413}) + # Cap on distinct batch signatures tracked for the attempt budget. + _MAX_TRACKED_BATCHES = 1024 + # Ceiling on bisection. Each level halves the batch, so this bounds the + # worst case (an all-bad batch) to ~2n sends while still isolating a + # single offender in any realistic batch. + _MAX_BISECT_DEPTH = 24 + + def _handle_http_rejection(self, batch: list[dict[str, Any]], error: Exception) -> None: + """Route a 4xx to re-queue, split-and-resend, or DLQ by status class.""" + status = getattr(getattr(error, "response", None), "status_code", None) + + if status in self._TRANSIENT_4XX: + # Never dead-letter a rate limit or a timeout, no matter how many + # attempts: the batch is good data, the server is busy. It stays in + # the WAL until the window opens. + self._retry_or_dlq(batch, error, dead_letter=False) + return + + if status in self._AUTH_4XX: + # Recoverable by operator action. Hold the data, keep the WAL, + # and say loudly that delivery is blocked on a key problem. + logger.error( + f"Backend rejected the batch with {status} (auth). The events stay " + f"in the WAL and WILL be delivered once the API key is fixed — " + f"rotate/verify it at https://app.nullrun.io/settings/api-keys. " + f"Quarantining is deliberately NOT done here." + ) + self._requeue_at_head(batch) + metrics.inc_transport("batches_auth_blocked") + return + + # 413 (batch too big) and 400/422 (one malformed event rejected the + # whole request) are both resolved the same way: halve and resend. The + # halves are sent as SEPARATE batches rather than pushed back onto the + # buffer — the buffer is flushed wholesale, so re-queuing both halves + # would re-form the batch we just proved the server will not accept, + # and the bisect would never make progress. + if len(batch) > 1: + half = len(batch) // 2 + logger.warning( + f"Batch of {len(batch)} rejected with {status} — splitting into " + f"{half} + {len(batch) - half} and resending the halves separately." + ) + metrics.inc_transport( + "batches_resplit" if status in self._RESPLIT_4XX else "batches_bisected" + ) + self._send_subbatches(batch[:half], batch[half:], error) + return + + # A single event the server will never accept. + self._quarantine_to_dlq(batch, error) + + def _send_subbatches( + self, left: list[dict[str, Any]], right: list[dict[str, Any]], error: Exception + ) -> None: + """Send both halves as independent batches, isolating a single offender. + + Recursion depth is log2(len(batch)), and `_MAX_BISECT_DEPTH` caps it, + so an entire bad batch cannot spin here. Whatever is still undeliverable + when the cap is hit falls back to the whole-batch quarantine. + """ + if self._bisect_depth >= self._MAX_BISECT_DEPTH: + logger.error(f"Bisect depth cap reached, quarantining {len(left) + len(right)} events") + self._quarantine_to_dlq(left + right, error) + return + + if not self._bisect_request_available(len(left) + len(right)): + self._quarantine_to_dlq(left + right, error) + return + + self._bisect_depth += 1 + try: + for half in (left, right): + if not half: + continue + self._attempt_half(half, error) + finally: + self._bisect_depth -= 1 + + def _bisect_request_available(self, remaining: int) -> bool: + """Whether the cascade may afford to split, and charge it if so. + + A split costs two sends (one per half), so the budget is charged here + rather than at the send sites: two call sites spending independently + is how one of them ends up not spending at all. + + Depth alone does not bound the wire: a full halving cascade is 2^depth + sends, and `_MAX_BISECT_DEPTH` is 24. When the budget runs out, stop + splitting and park what is left rather than spend the operator's rate + limit proving once more that a batch is bad — a refusal is exactly + what a rate limiter starts answering, so the cascade would be feeding + the condition it is trying to diagnose. Events already delivered in + the healthy halves stay delivered. + """ + if self._bisect_budget < 2: + logger.error( + f"Bisect request budget ({max(self._bisect_budget, 0)} remaining) " + f"exhausted — parking the remaining {remaining} event(s) without " + f"splitting further. Raise NULLRUN_MAX_BISECT_REQUESTS if these " + f"batches deserve isolating." + ) + metrics.inc_transport("batches_bisect_budget_exhausted") + return False + self._bisect_budget -= 2 + return True + + def _attempt_half(self, half: list[dict[str, Any]], original: Exception) -> None: + """Send one half, routing the result exactly as a top-level batch would.""" + deferred: list[Exception] = [] + + def send(): + try: + result = self._send_batch_with_retry_info(half) + except (BreakerTransportError, httpx.TransportError): + raise + except Exception as exc: # noqa: BLE001 + deferred.append(exc) + return None + if result.accepted_event_ids: + for event in half: + if event.get("event_id") in result.accepted_event_ids: + self._in_flight.pop(event.get("event_id"), None) + metrics.inc_transport("batches_sent") + metrics.inc_transport("events_sent", len(half)) + metrics.set_transport("last_flush_at", time.monotonic()) + return result + + try: + self._circuit_breaker.call(send) + except BreakerTransportError: + self._requeue_at_head(half) + metrics.inc_transport("batches_failed") + return + except Exception as exc: # noqa: BLE001 + self._retry_or_dlq(half, exc, dead_letter=True) + return + + if deferred: + self._route_deferred(half, deferred[0]) + return + + self._batch_attempts.pop(self._failure_signature(half), None) + + def _deliver_recorded_overbudget( + self, batch: list[dict[str, Any]], error: DeterministicBackendRefusal + ) -> bool: + """Settle a CONSUME_OVERBUDGET the backend says it already applied. + + ``/track/batch`` rejects a settle that lands past + ``reserved + epsilon_cents`` with 422 — but the Lua overage branch + has already INCRBY'd the real spend onto the period counter and the + handler has already written the ``cost_events`` row. The refusal is + about the *reservation* being exceeded, not about the event being + refused. ``details.reservation_recorded`` is the backend stating + which of those two happened. + + Treating the 422 as a refusal when the marker says the spend landed + is a two-part lie. The event goes to the DLQ annotated "permanent + rejection, will NOT be retried" — a rejection the server never made, + for an event it already has a row for — and the rest of the batch + dies with it, because the aborted request never reached them. The + operator's only durable trace of the event is that DLQ line. + + So the marker decides, and only for this code: + + * ``reservation_recorded: true`` → the event is DELIVERED. Drop it + from the retry path and from ``.inflight``, re-queue the rest of + the batch (unprocessed, still owed), return ``True``. + * anything else, including a body with no ``details`` at all → return + ``False`` and let the existing quarantine path run untouched. A + refusal that charged nothing must still park the data. + + ``details.event_id`` names the offender. When it is absent (older + backend, or a proxy page) the answer is computed from the batch: a + singleton can only be the offender, so a recorded marker on it is + enough. A multi-event batch with no name is not — there the + cascade below isolates the offender and this runs again on the + singleton, which is why returning ``False`` here is correct and not + merely conservative. + """ + if error.error_code != "CONSUME_OVERBUDGET": + return False + if error.details.get("reservation_recorded") is not True: + return False + + offender_id = error.details.get("event_id") + if offender_id: + settled = [e for e in batch if e.get("event_id") == offender_id] + outstanding = [e for e in batch if e.get("event_id") != offender_id] + elif len(batch) == 1: + settled, outstanding = list(batch), [] + else: + # Named by nobody, and not provable from a multi-event batch. + # Let the bisect isolate it; this runs again on the singleton. + return False + + if not settled: + # The backend named an event that is not in this batch. Dropping + # "everything not named" here would silently discard real data + # on the strength of an id we cannot see; let it be quarantined + # so a human finds the mismatch. + logger.error( + f"CONSUME_OVERBUDGET names event {offender_id!r}, which is not in " + f"this batch of {len(batch)} — quarantining rather than guessing " + f"which event the backend settled." + ) + return False + + exec_id = error.details.get("execution_id") + logger.warning( + f"CONSUME_OVERBUDGET on {len(settled)} event(s) — the backend reports " + f"reservation_recorded=true, so the spend is already on the period " + f"counter and the cost_events row is written" + + (f" (execution {exec_id})" if exec_id else "") + + ". Treating as DELIVERED, not quarantined. " + + f"Re-queueing {len(outstanding)} event(s) the aborted batch never reached." + ) + for event in settled: + eid = event.get("event_id") + if eid: + self._in_flight.pop(eid, None) + self._first_failed_at.pop(str(eid), None) + metrics.inc_transport("events_recorded_overage", len(settled)) + + if outstanding: + self._requeue_at_head(outstanding) + return True + + def _route_deferred(self, batch: list[dict[str, Any]], error: Exception) -> None: + """Classify a failure that was kept away from the circuit breaker.""" + if isinstance(error, DeterministicBackendRefusal): + # A refusal the backend has already applied is not a refusal of + # delivery. Checked before the bisect so a resolved 422 costs one + # send, not log2(n), and so the events the aborted batch never + # reached are re-queued rather than dropped with it. + if self._deliver_recorded_overbudget(batch, error): + return + # The backend named the refusal. It named it for the BATCH, and a + # whole-batch refusal is what an unbatched per-event rejection + # looks like from out here: one event the backend cannot accept + # (an unbound execution, a policy limit) makes the request carrying + # it fail as a unit. Quarantining the batch on that evidence + # discards every healthy event sharing it — the 49 good events are + # recorded as refused for a reason that never applied to them. + # + # So bisect FIRST, exactly as the 400/422 path does, and only + # quarantine what survives as a singleton. The bisect is bounded by + # `_MAX_BISECT_DEPTH` and by a request budget, so an all-bad batch + # cannot turn this into a request storm. + if len(batch) > 1 and self._bisect_depth < self._MAX_BISECT_DEPTH: + if not self._bisect_request_available(len(batch)): + metrics.inc_transport("batches_deterministic_refusal") + self._quarantine_to_dlq(batch, error) + return + half = len(batch) // 2 + logger.warning( + f"Backend refused a batch of {len(batch)} as " + f"{error.error_code} — a whole-batch refusal is not evidence " + f"about each event, so splitting into {half} + " + f"{len(batch) - half} to isolate the offender." + ) + metrics.inc_transport("batches_bisected") + self._bisect_depth += 1 + try: + self._send_subbatches(batch[:half], batch[half:], error) + finally: + self._bisect_depth -= 1 + return + logger.error( + f"Backend refused the batch as {error.error_code} " + f"(HTTP {error.status_code}) — deterministic, quarantining " + f"{len(batch)} event(s). {error.detail}" + ) + metrics.inc_transport("batches_deterministic_refusal") + self._quarantine_to_dlq(batch, error) + return + if isinstance(error, httpx.HTTPStatusError): + self._handle_http_rejection(batch, error) + else: + # Not obviously transient and not an HTTP status either. It may + # still be deterministic (an event that cannot be serialized will + # fail identically every cycle), so count attempts and dead-letter + # once the budget is spent rather than re-queuing forever. + self._retry_or_dlq(batch, error, dead_letter=True) + + def _requeue_at_head(self, batch: list[dict[str, Any]]) -> None: + """Put a batch back at the FRONT of the buffer. + + The bisect sends halves directly rather than re-queuing them, so any + event that comes back has to resume its place: appending would let a + steady stream of new events push a stalled batch to the back of the + queue indefinitely. ``.inflight`` stays — the batch is in memory only, + and a later successful flush overwrites and clears it. + """ + self._buffer[0:0] = batch + + def _retry_or_dlq( + self, batch: list[dict[str, Any]], error: Exception, *, dead_letter: bool + ) -> bool: + """Re-queue a possibly-transient failure, with a bounded attempt budget. + + Deterministic failures (a payload that will never serialize) repeat + identically on every cycle and would pin the batch ahead of the whole + buffer forever. So every re-queue costs an attempt; once the budget is + spent a ``dead_letter``-eligible batch is quarantined rather than + blocking delivery forever. + + ``dead_letter=False`` is for genuinely transient statuses (408/429). + Those are counted for observability but NEVER quarantined: the batch is + valid data and the server is merely busy, so it waits in the WAL. + + Returns True when the batch is back on the retry path, False when it + was quarantined. The caller needs the distinction to decide whether + anything from this send is still live, which is what governs whether + ``.inflight`` may be cleared. + """ + key = self._failure_signature(batch) + attempts = self._batch_attempts.get(key, 0) + 1 + if len(self._batch_attempts) >= self._MAX_TRACKED_BATCHES: + # A long outage produces an unbounded stream of distinct batch + # signatures. Drop the map rather than grow it: a reset only costs + # one extra retry cycle, never correctness. + logger.warning( + f"Failure-attempt map hit {self._MAX_TRACKED_BATCHES} entries, resetting" + ) + self._batch_attempts.clear() + self._batch_attempts[key] = attempts + + if dead_letter and attempts > self._max_batch_attempts: + logger.error( + f"Batch failed {attempts} times without landing: {error}. " + f"Quarantining {len(batch)} events." + ) + self._quarantine_to_dlq(batch, error) + return False + + if not dead_letter: + logger.warning( + f"Transient rejection (attempt {attempts}) — holding the batch: {error}" + ) + else: + logger.warning( + f"Batch re-queued (attempt {attempts}/{self._max_batch_attempts}): {error}" + ) + self._buffer.extend(batch) + metrics.inc_transport("batches_retried") + return True + + def _failure_signature(self, batch: list[dict[str, Any]]) -> str: + """Stable identity for a batch, used as the attempt-counter key.""" + ids = sorted(str(e.get("event_id", "")) for e in batch) + return hashlib.sha256("|".join(ids).encode()).hexdigest()[:16] + + def _dlq_reason(self, error: Exception) -> str: + """The machine-readable reason a batch is being parked. + + Prefers what the wire named over how it was wrapped. Several + deterministic refusals arrive as 5xx (see + ``_DETERMINISTIC_ERROR_CODES``), and a DLQ row that only says "503" + tells an operator replaying it nothing about why the event was + parked. The two reasons this transport raises itself carry their + reason as a field rather than in a formatted message, for the same + reason — a message is for a log, a field is for a record. + """ + if isinstance(error, DeterministicBackendRefusal): + return f"{error.error_code}:{error.status_code}" + if isinstance(error, UnknownRejectionReason): + return f"{error.reason}:unknown" + if isinstance(error, UnconfirmedBatchResponse): + return "unconfirmed_response:no_accepted_event_ids" + status = getattr(getattr(error, "response", None), "status_code", None) + return f"{type(error).__name__}:{status}" + + def _dlq_row(self, event: dict[str, Any], reason: str) -> dict[str, Any]: + """Build one versioned DLQ record. See ``_DLQ_ROW_VERSION`` for the schema. + + ``error`` is kept alongside ``reason`` so a v1 reader — anything + written before this field existed and still on disk across an + upgrade — keeps working. Dropping it would make a recovery file + unreadable, which is the one thing a recovery file must not be. + """ + eid = str(event.get("event_id") or "") + first_failed = self._first_failed_at.get(eid) + if first_failed is None: + first_failed = time.time() + if eid: + self._first_failed_at[eid] = first_failed + signature = self._failure_signature([event]) + return { + "v": _DLQ_ROW_VERSION, + "event": event, + "error": reason, + "reason": reason, + "event_type": event.get("type"), + "first_failed_at": first_failed, + # Sends this event took to end up here, including the current one. + # A terminal refusal is parked on its first response and never + # touches `_retry_or_dlq`, so reading the counter alone would record + # 0 — "never attempted" — for an event that was in fact refused on + # the first send. The two are different stories for whoever triages + # the DLQ, and only this one is true. + "attempts": self._batch_attempts.get(signature, 0) + 1, + } + + def _write_dlq_rows(self, rows: list[dict[str, Any]]) -> bool: + """Append DLQ records durably. Does NOT touch ``.inflight``. + + Kept separate from ``_quarantine_to_dlq`` because that method clears + ``.inflight``, which is only correct when the WHOLE batch is parked. + A partial refusal parks some events and re-queues others from the same + send, and clearing there would drop the re-queued ones on the next + crash. + + Returns False — and writes nothing — when the DLQ is at its size cap. + The caller treats False as "not parked", which re-queues the event on + the retry path. That is the whole design of the cap: a full DLQ + stalls parking rather than dropping the oldest rows, so the operator + gets a loud, alertable condition instead of a file that quietly lost + the events from three hours ago. + """ + if not rows: + return True + # Measured, not estimated from `len(row)`: `len` of a dict is its + # number of keys, which is off by a factor of an order of magnitude + # and would make the cap meaningless in the only direction that + # matters — under-counting. + incoming = sum(len(json.dumps(r, default=str)) + 1 for r in rows) + if self._dlq_over_cap(incoming): + return False + if not self._write_events_atomic(self._wal_dlq_path(), rows, mode="a"): + return False + metrics.set_transport("dlq_bytes", self._dlq_size()) + return True + + def _dead_letter_row(self, event: dict[str, Any], reason: str) -> bool: + """Park one event, durably, before anything clears ``.inflight``. + + Returns True when the event could NOT be parked and is still live, so + `.inflight` must be retained. + """ + if not self._write_dlq_rows([self._dlq_row(event, reason)]): + # The DLQ could not take it. Re-queueing would put a batch the + # backend has already refused terminally back on the send path, + # where it produces the identical refusal on every cycle forever + # AND sits at the head of the buffer ahead of every healthy event + # behind it. That is the "one bad event stops the whole stream" + # failure, and it is caused by parking, not by the refusal. + # + # So the event is HELD, not re-sent: a terminal refusal cannot + # become a success, so another request is pure waste, and holding + # keeps the head of the buffer clear. `_drain_dlq_overflow` retries + # the write whenever the DLQ may have room again, and the holdover + # is bounded — a holdover that grows without limit is the same + # memory blowup the cap exists to prevent, only invisible. + self._hold_for_dlq([self._dlq_row(event, reason)], reason) + return True + metrics.inc_transport("events_dead_lettered") + metrics.set_transport("last_dlq_error", reason) + return False + + def _hold_for_dlq(self, rows: list[dict[str, Any]], reason: str) -> None: + """Park refused events durably until the DLQ has room. + + The alternative — leaving them on the send path — is what turns a full + DLQ into an outage: the events are re-sent every cycle, they are + refused every cycle, and they occupy the head of the buffer the whole + time, so the healthy events queued behind them are never delivered. + The refusal is terminal, so re-sending cannot help; only the recording + can, and that is what the holdover defers. + + They are written to `.wal.holdover`, not just kept in memory. A + memory-only holdover is a `kill -9` away from total loss: the next + `_persist_inflight` overwrites `.inflight` with the batch it is about + to send, so once the process restarts the events are in the DLQ + nowhere, in `.wal` nowhere, and in `.inflight` overwritten. Keeping + them in RAM looked like the fix and was a new loss path — the same + shape of bug as the one that put `replay` in the tree, where the + report described a durability the code did not have. + + Recovery reads the holdover file back into `_dlq_overflow` at + startup, so the drain resumes without re-deriving the refusal by + re-sending. That matters for a terminal reason: re-sending is the one + thing the holdover exists to avoid, and a restart that recovers the + events by re-sending them would reintroduce the head-of-buffer + starvation the hold was built to stop. + + The in-memory index is bounded, and the bound is on COUNT, not bytes. + Count is what the metric reports and what an operator alerts on; a + byte bound would need a second cap for the same property and would + fail silently on a different threshold. Past the bound the oldest + entries stop being *tracked* — they are not lost, they are in the + file and `_drain_dlq_overflow` reads the file as well as the index — + so the only cost is an under-reported `dlq_holdover`, which is loud + and named in the log rather than silent. + """ + self._dlq_overflow.extend(rows) + metrics.inc_transport("dlq_holdover_total", len(rows)) + self._persist_holdover(rows) + self._trim_holdover_index() + metrics.set_transport("dlq_holdover", len(self._dlq_overflow)) + metrics.set_transport("last_dlq_error", reason) + logger.error( + f"DLQ at its cap — holding {len(self._dlq_overflow)} refused event(s) " + f"({reason}). They are NOT being re-sent: a terminal refusal cannot " + f"succeed on retry, and re-sending would block every healthy event " + f"behind them. They are durably in {self._wal_holdover_path()}, so a " + f"kill -9 does not lose them. Raise NULLRUN_DLQ_MAX_BYTES, or triage " + f"and archive the DLQ with `nullrun-wal`, and they drain on the next " + f"flush." + ) + + def _trim_holdover_index(self) -> None: + """Bound the in-memory holdover index; the file is the record of truth. + + Only the index is trimmed. Trimming the file would be the cap deleting + the only copy of an event the SDK could not deliver — the same thing + `_dlq_over_cap` refuses to do, for the same reason. + """ + cap = self._holdover_index_max + if not cap or len(self._dlq_overflow) <= cap: + return + dropped = len(self._dlq_overflow) - cap + del self._dlq_overflow[:-cap] + metrics.inc_transport("dlq_holdover_index_truncated", dropped) + logger.error( + f"Holdover index passed NULLRUN_DLQ_HOLDOVER_MAX_EVENTS ({cap}); " + f"stopped tracking the {dropped} oldest held event(s). They are NOT " + f"lost — {self._wal_holdover_path()} still holds them and the drain " + f"reads it — but `dlq_holdover` now under-reports by {dropped}. Raise " + f"the env var to keep the count exact." + ) + + def _persist_holdover(self, rows: list[dict[str, Any]]) -> None: + """Append the held rows to `.wal.holdover`. + + Append, and the rows are the same DLQ-shaped dicts the drain already + knows how to write — a `event_id` plus the `event` — so the drain does + not need a second format, and `nullrun-wal` can archive the holdover + file with the same reader it uses for the DLQ. + + Failure to persist is loud but not fatal. The events remain in the + index and the caller has already been told the DLQ refused them, so + raising here would abort a flush over a bookkeeping failure on a path + whose whole purpose is to not lose the event. The log says the one + true thing: a kill before a later attempt loses them. + """ + if not rows: + return + if not self._write_events_atomic(self._wal_holdover_path(), rows, mode="a"): + logger.error( + f"Could not persist {len(rows)} held refusal(s) to " + f"{self._wal_holdover_path()} (lock held elsewhere, or I/O " + f"failed). They are in memory only, and a kill before a later " + f"attempt loses them." + ) + metrics.inc_transport("dlq_holdover_persist_failures", len(rows)) + + def _recover_holdover(self) -> None: + """Load rows from `.wal.holdover` into the index at startup. + + Appends rather than replaces, so a second process that finds the file + already drained — or empty — does not erase an index it does not own. + The lock is not taken: read-only, and the only writer appends whole + lines atomically, so the worst case is one incomplete final line, + which is dropped exactly as `_replay_from_wal` drops it. + """ + path = self._wal_holdover_path() + try: + with open(path) as f: + lines = f.readlines() + except FileNotFoundError: + return + except OSError as e: + logger.warning(f"Failed to read holdover file {path}: {e}") + return + recovered: list[dict[str, Any]] = [] + for line in lines: + if not line.strip(): + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + # A torn tail from a kill mid-append. The event is not + # recoverable from this line, and the refusal will simply be + # re-derived if the event is still in another source. + continue + if isinstance(row, dict): + recovered.append(row) + if recovered: + self._dlq_overflow.extend(recovered) + logger.warning( + f"Recovered {len(recovered)} held refusal(s) from {path}. They " + f"were refused and the DLQ was full when the process died; they " + f"are NOT re-sent, they go to the DLQ as soon as it has room." + ) + self._trim_holdover_index() + + def _drain_dlq_overflow(self) -> int: + """Try to move held refusals into the DLQ now that space may exist. + + Called at the top of every flush. Returns how many rows landed. A + refused write is a no-op — the holdover is unchanged and the operator + still sees the same alert — so this is safe to attempt unconditionally + and cheap when there is nothing held. + + The holdover FILE is released by `event_id`, not deleted wholesale. + The index can be shorter than the file: `_trim_holdover_index` drops + the oldest entries from memory to bound it, and those rows are still + owed a DLQ. Removing the file after a partial drain would delete them. + Deleting by id also means a crash between the DLQ write and the file + release replays into the DLQ a second time rather than losing + anything — at-least-once, which is the same trade the WAL already + makes and which the `event_id` dedup absorbs. + """ + if not self._dlq_overflow: + return 0 + pending, self._dlq_overflow = self._dlq_overflow, [] + written = self._write_dlq_rows(pending) + if not written: + self._dlq_overflow = pending # unchanged; try again next cycle + return 0 + if not self._release_holdover( + { + str(r["event"]["event_id"]) + for r in pending + if isinstance(r.get("event"), dict) and r["event"].get("event_id") + } + ): + # The rows are in the DLQ. The file still names them, so a restart + # re-drains them into the DLQ a second time — a duplicate the + # `event_id` dedup absorbs, which is strictly better than the + # alternative of not knowing whether the release happened. + logger.warning( + f"Could not release {self._wal_holdover_path()} after draining " + f"{len(pending)} refusal(s) into the DLQ. The rows are safely in " + f"the DLQ; the stale holdover file will re-drain them on the next " + f"start, which duplicates rather than loses." + ) + metrics.inc_transport("events_dead_lettered", len(pending)) + metrics.set_transport("dlq_holdover", len(self._dlq_overflow)) + logger.info(f"Drained {len(pending)} held refusal(s) into the DLQ") + return len(pending) + + def _release_holdover(self, event_ids: set[str]) -> bool: + """Rewrite `.wal.holdover` without the drained `event_id`s. + + The id is read from ``row["event"]["event_id"]`` because that is where + `_dlq_row` puts it. Matching a top-level ``event_id`` — which the row + schema does not have — would release nothing while reporting success, + which is the failure mode this whole file exists to remove. + + Returns True when the file now names none of them (including the + not-there case, which is success), False when the rewrite failed and + the caller must assume the rows are still listed. + """ + if not event_ids: + return True + path = self._wal_holdover_path() + try: + with open(path) as f: + lines = f.readlines() + except FileNotFoundError: + return True + except OSError as e: + logger.warning(f"Failed to read holdover file {path} to release: {e}") + return False + kept: list[dict[str, Any]] = [] + for line in lines: + if not line.strip(): + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + # Unparseable: we cannot prove which event it is, so it stays. + # Dropping it would be deleting the only copy of an unknown row. + kept.append({"raw": line}) + continue + event = row.get("event") if isinstance(row, dict) else None + eid = str(event.get("event_id")) if isinstance(event, dict) else None + if eid not in event_ids: + kept.append(row) + try: + if not kept: + os.remove(path) + return True + return self._write_events_atomic(path, kept) + except OSError as e: + logger.warning(f"Failed to release holdover file {path}: {e}") + return False + + def _quarantine_to_dlq(self, batch: list[dict[str, Any]], error: Exception) -> None: + """Move a permanently-rejected batch out of the retry path. + + A 4xx will never succeed on retry, so the events go to ``.dlq`` with + the reason attached and `.inflight` is cleared — otherwise every + subsequent start replays a batch that is guaranteed to fail and the + poison pill blocks the whole WAL. Nothing is silently discarded: the + file is inspectable and re-submittable after the caller is fixed. + + The whole batch is parked here, so clearing ``.inflight`` afterwards + is correct — but ONLY once the DLQ write is confirmed. A failed + write (full cap, read-only volume, lost lock) leaves the batch on the + retry path and retains ``.inflight``, because the alternative is + clearing the only durable copy of events that were not parked + anywhere. The events will be refused again on the next send, which is + the correct outcome for a permanent rejection we could not record. + """ + reason = self._dlq_reason(error) + logger.error( + f"Permanent rejection ({reason}) on /track/batch — quarantining " + f"{len(batch)} events to DLQ. They will NOT be retried." + ) + rows = [self._dlq_row(e, reason) for e in batch] + if not self._write_dlq_rows(rows): + # Same reasoning as `_dead_letter_row`: the batch is terminally + # refused, so re-sending it produces the same refusal forever + # while blocking everything behind it. Hold instead. + self._hold_for_dlq(rows, reason) + return + self._clear_inflight() + metrics.inc_transport("events_dead_lettered", len(batch)) + metrics.set_transport("last_dlq_error", reason) def _drain_batch(self) -> list[dict[str, Any]] | None: """Public, lock-acquiring snapshot of the current buffer. Returns ``None`` when empty.""" @@ -922,6 +2909,16 @@ class SendResult: accepted_event_ids: list[str] retry_after_ms: float | None = None is_policy_limit: bool = False + # Per-item refusals from a 200 body: `{"event_id", "reason", + # "retry_after_ms"}`. Absent `retry_after_ms` (None) means the backend + # gave no retry hint for that event. + rejected_items: list[dict[str, Any]] = field(default_factory=list) + # False when the response was a success status but the body could not + # be read as a confirmation (proxy HTML page, empty body, a backend + # that predates `accepted_event_ids`). A 200 we cannot interpret is + # NOT proof of delivery — treating it as one is the failure this + # field exists to prevent. + body_confirmed: bool = True def _add_hmac_headers(self, headers: dict[str, str], body: str | bytes) -> None: """Add X-Signature-Timestamp + X-Signature headers. No-op if secret_key/api_key missing.""" @@ -990,32 +2987,10 @@ def _inject_trace_context(self, headers: dict[str, str]) -> None: def _extract_retry_after(self, response: httpx.Response) -> float | None: """Extract Retry-After header value as seconds. - Handles both: - - Integer seconds (e.g., "30") - - HTTP-date format (e.g., "Wed, 21 Oct 2015 07:28:00 GMT") + Thin wrapper over the module-level helper, which the retry loop also + needs and which cannot reach a Transport instance. """ - retry_after = response.headers.get("Retry-After") - if not retry_after: - return None - - # Try parsing as seconds (integer or float) - try: - return float(retry_after) - except ValueError: - pass - - # Try parsing as HTTP datetime (RFC 7231) - try: - from email.utils import parsedate_to_datetime - - dt = parsedate_to_datetime(retry_after) - from datetime import datetime, timezone - - return (dt - datetime.now(timezone.utc)).total_seconds() - except Exception: - pass - - return None + return _retry_after_seconds_from(response) def _send_batch_with_retry_info(self, batch: list[dict[str, Any]]) -> "SendResult": """Send batch to server. Returns SendResult with retry info. Wrapped by _retry_with_backoff.""" @@ -1035,6 +3010,20 @@ def _post_batch() -> httpx.Response: content=body, headers=self._build_signed_headers(body=body), ) + # A deterministic refusal is checked BEFORE the status test, and on + # any status. The backend sent EXECUTION_NOT_BOUND as 503 until the + # status correction (now 422), and a status-first test files the + # old shape under "the server is down": it retries, and counts + # every attempt as a transport failure. Testing the code first is + # what makes this correct against a server we may not control. + code = _extract_backend_error_code(resp) + if code in _DETERMINISTIC_ERROR_CODES and resp.status_code >= 400: + raise DeterministicBackendRefusal( + code, + resp.status_code, + _extract_error_message(resp), + _extract_error_details(resp), + ) if resp.status_code >= 500 or resp.status_code == 429: # raise_for_status turns this into HTTPStatusError; the retry # helper wraps that into BreakerTransportError after retries. @@ -1049,6 +3038,7 @@ def _post_batch() -> httpx.Response: max_delay=10.0, backoff_factor=2.0, jitter=0.1, + cancel=self._stop_event, ) # P0: Extract retry_after from response headers or body @@ -1059,19 +3049,85 @@ def _post_batch() -> httpx.Response: # Check Retry-After header (may be seconds or HTTP-date) retry_after_seconds = self._extract_retry_after(response) - # Check response body for retry info + # Parse the body ONCE. Three consumers below need it — the batch-level + # `rejected` block, the per-item `rejection_details`, and the actions + # loop — and each used to call `response.json()` independently, so a + # non-JSON body was re-raised and re-caught once per consumer. + data: dict[str, Any] | None = None try: - data = response.json() - # Check for rejection info - if "rejected" in data and data["rejected"]: - rejected_info = data["rejected"] - if isinstance(rejected_info, dict): - if "retry_after_ms" in rejected_info: - retry_after_ms = rejected_info["retry_after_ms"] - if "reason" in rejected_info and rejected_info["reason"] == "policy_limit": - is_policy_limit = True - except Exception: # noqa: S110 - pass + parsed = response.json() + if isinstance(parsed, dict): + data = parsed + except Exception: # noqa: S110 — non-JSON body; handled below per status + data = None + + # Check response body for batch-level retry info + if data is not None: + rejected_info = data.get("rejected") + if isinstance(rejected_info, dict): + if "retry_after_ms" in rejected_info: + retry_after_ms = rejected_info["retry_after_ms"] + if rejected_info.get("reason") == "policy_limit": + is_policy_limit = True + + # ---- Per-item outcome of a 200 ------------------------------------ + # A 200 from /track/batch is NOT proof of delivery. The endpoint + # refuses individual events and still answers 200, naming the + # survivors in `accepted_event_ids` and the casualties in + # `rejection_details`. The pre-fix code read the status, saw 200, + # treated the whole batch as delivered, cleared `.inflight` and + # dropped the refused events with no trace in the buffer and none in + # the DLQ. That is the largest hole in the "data is not lost" claim, + # which until now held only for transport failures. + # + # `accepted_event_ids` is read as a partition, not a hint: anything + # absent from it is NOT delivered, whatever the status said. That is + # the only reading an SDK can act on, and it is why the backend has + # to report events it persisted through other paths (the span / + # org_action_catalog writes) — see the `accepted_event_ids` partition + # fix in backend handlers.rs. + accepted_event_ids: list[str] = [] + rejected_items: list[dict[str, Any]] = [] + body_confirmed = True + if response.status_code < 400: + raw_accepted = (data or {}).get("accepted_event_ids") + if data is None or not isinstance(raw_accepted, list): + # 200 we cannot read as a confirmation. Either something in + # front of the backend answered with an HTML error page and a + # 200 status, or the backend predates `accepted_event_ids`. + # Neither is evidence that anything landed. Treating this as + # delivery is precisely the loss we are preventing, so the + # batch stays on the retry path under a bounded budget. + body_confirmed = False + logger.error( + f"/track/batch returned {response.status_code} with a body " + f"that is not a batch confirmation (parseable dict with an " + f"accepted_event_ids list: {data is not None}). Treating " + f"{len(batch)} events as NOT delivered. A reverse proxy " + f"answering 200, or a backend older than the " + f"accepted_event_ids field, produces this." + ) + else: + accepted_event_ids = [i for i in raw_accepted if isinstance(i, str)] + raw_details = (data or {}).get("rejection_details") + if isinstance(raw_details, list): + for detail in raw_details: + if not isinstance(detail, dict): + continue + eid = detail.get("event_id") + reason = detail.get("reason") + if not isinstance(eid, str) or not isinstance(reason, str): + continue + hint = detail.get("retry_after_ms") + rejected_items.append( + { + "event_id": eid, + "reason": reason, + "retry_after_ms": ( + float(hint) if isinstance(hint, (int, float)) else None + ), + } + ) # Store for next retry calculation (prefer header seconds, fallback to body ms) if retry_after_seconds is not None: @@ -1094,8 +3150,7 @@ def _post_batch() -> httpx.Response: # Process actions from server response. Per-element try/except so one # malformed entry doesn't abort the whole loop. try: - data = response.json() - actions = data.get("actions") or [] + actions = (data or {}).get("actions") or [] for action in actions: try: if not isinstance(action, dict): @@ -1108,18 +3163,18 @@ def _post_batch() -> httpx.Response: handle_action(action_type, workflow_id, reason) except Exception as item_err: logger.warning("Skipping malformed action %r: %s", action, item_err) - for msg in data.get("messages", []) or []: + for msg in (data or {}).get("messages", []) or []: logger.info("Backend message: %s", msg) except Exception as e: logger.warning(f"Failed to process actions: {e}") - # Return accepted event_ids for retry dedup - accepted_event_ids = data.get("accepted_event_ids", []) if "data" in locals() else [] logger.debug(f"Batch track: sent {len(batch)} events") return self.SendResult( accepted_event_ids=accepted_event_ids, retry_after_ms=retry_after_ms, is_policy_limit=is_policy_limit, + rejected_items=rejected_items, + body_confirmed=body_confirmed, ) def flush_now(self) -> None: @@ -1164,7 +3219,7 @@ def execute( # Populated by `runtime.execute` from the `get_call_tools()` contextvar # when the caller invoked `set_call_context(tools=...)`. tools: tuple[str, ...] | None = None, - on_transport_error: Callable[[Exception], dict[str, Any]] | None = None, + on_transport_error: TransportErrorHandler | None = None, ) -> dict[str, Any]: """Pre-execution policy evaluation via /api/v1/execute (PRIMARY enforcement point). @@ -1215,6 +3270,23 @@ def execute( responses populate `policy_hash` only. - decision_context: Context for replay (if available) """ + # Via-edge: there is no /execute to make. + # + # The box's `/enforce` already did both halves of the cloud's + # `/gate` + `/execute` sequence in one call — it decided AND it + # charged the grant. A second call would either reach the cloud + # (defeating the mode) or ask the box a question it has no + # endpoint for. Re-asking would also double-charge: the charge + # and the decision are one event, done once. + if self.edge is not None: + from nullrun.edge import EDGE_LEASE + return { + "decision": "allow", + "decision_source": self.edge.last_source or EDGE_LEASE, + "execution_id": execution_id, + "via_edge": True, + } + gate_request = { "organization_id": organization_id, "execution_id": execution_id, @@ -1270,6 +3342,7 @@ def do_execute_request() -> httpx.Response: max_retries=max_execute_retries, base_delay=0.5, on_transport_error=on_transport_error, + cancel=self._stop_event, ) if response.status_code == 200: @@ -1491,7 +3564,7 @@ def do_execute_request() -> httpx.Response: def check( self, check_request: dict[str, Any], - on_transport_error: Callable[[Exception], dict[str, Any]] | str | None = None, + on_transport_error: TransportErrorHandler | None = None, parent_execution_id: str | None = None, ) -> dict[str, Any]: """ @@ -1520,6 +3593,25 @@ def check( - explanations: List of explanation strings - suggestions: List of suggestion strings """ + # Via-edge. When a box holds a signed lease, the box IS the gate + # and the cloud is not consulted at all. + # + # Placed before the gate body is built, not as a branch inside + # it, so there is no way for a later field to leak onto a cloud + # request or for a partial body to be sent somewhere. One of + # two paths runs; they do not interleave. + if self.edge is not None: + from nullrun.edge import token_split + + input_tokens, output_tokens = token_split(check_request) + return self.edge.enforce( + model=str(check_request.get("model") or ""), + input_tokens=input_tokens, + output_tokens=output_tokens, + event_id=check_request.get("operation_id"), + on_transport_error=on_transport_error, + ) + # Convert check_request to gate_request format gate_request = { "organization_id": check_request.get("organization_id"), @@ -1660,6 +3752,7 @@ def _do_gate_post() -> httpx.Response: jitter=0.1, retry_on_5xx=True, on_transport_error=on_transport_error, + cancel=self._stop_event, ) if response.status_code == 200: @@ -1994,7 +4087,7 @@ async def _refetch_credentials(self) -> None: def check_v3( self, request: dict[str, Any], - on_transport_error: Callable[[Exception], dict[str, Any]] | str | None = None, + on_transport_error: TransportErrorHandler | None = None, ) -> dict[str, Any]: """Pre-execution gate — wire-protocol v3. @@ -2986,14 +5079,26 @@ def _parse_v3_error_envelope_uncategorised( return NullRunProtocolError(full_message) if backend_code == "CONSUME_OVERBUDGET": + # The backend's 422 `details` block carries the reservation in + # MILLICENTS (`reserved_millicents` / `max_allowed_millicents`) and + # the actual in cents. Reading `reserved_cents` / `epsilon_cents` + # here — as this did — read keys the wire does not carry, so every + # one of them was permanently None. `reservation_recorded` is + # tri-state on purpose: the backend omits it when the settle was + # refused without being applied, and a pre-marker backend omits it + # because it cannot say. `is True` is the only reading that asserts + # the spend landed. + recorded = details.get("reservation_recorded") + soft_pass = details.get("soft_pass") return NullRunConsumeOverbudgetError( full_message, execution_id=details.get("execution_id"), - reserved_cents=details.get("reserved_cents"), - max_allowed_cents=details.get("max_allowed_cents"), - actual_cost_cents=details.get("actual_cost_cents"), - epsilon_cents=details.get("epsilon_cents"), + reserved_millicents=_as_int_or_none(details.get("reserved_millicents")), + max_allowed_millicents=_as_int_or_none(details.get("max_allowed_millicents")), + actual_cost_cents=_as_int_or_none(details.get("actual_cost_cents")), + soft_pass=soft_pass if isinstance(soft_pass, bool) else None, status_code=status, # 422 per backend mapping + recorded=recorded if isinstance(recorded, bool) else None, ) if ( @@ -3490,6 +5595,94 @@ def _build_v3_error_code_map() -> dict[str, type[Exception]]: # branches on the action_digest missing path with the same # `except NullRunBlockedException:` flow as TOOL_BLOCKED. "LEGACY_GRANT_REJECTED": NullRunBlockedException, + # ----------------------------------------------------------------- + # 2026-10-07 — twelve codes the backend registered and this catalog + # never learned about. + # + # The parity guard in ``backend/tests/nr007_sdk_error_code_parity.rs`` + # could not see them: it compared the SDK map against a hand-written + # 51-entry mirror of ``GateErrorCode::all()`` while the enum had + # grown to 72. Both "completeness" assertions were self-consistent — + # the list was checked against the number 51, and the number 51 was + # checked against the list — so nothing compared either to the enum + # they were supposed to mirror. Every code below therefore fell + # through to the status-based fallback and arrived as an untyped + # ``NullRunBackendError``. + # + # That is not a cosmetic gap for this family. ``is_model_message_safe()`` + # on the backend is what keeps budget-blocked outcomes out of the + # model's context, and it keys off the error_code; a caller holding + # an untyped error has no branch to take. Several of these — + # ``CIRCUIT_BREAKER_TRIPPED``, ``BUDGET_ORG_CEILING_BLOCKED`` — + # are the codes an operator reaches for precisely when an agent is + # misbehaving. + # + # Family rationale per code: + # - money blocks → NullRunBudgetError ("stop spending") + # - decision blocks → NullRunBlockedException + # - infra failures on a + # fail-CLOSED gate arm → NullRunBackendError / the typed + # infra sibling for that subsystem + # ----------------------------------------------------------------- + # Org-scope ceiling (ADR-050) — the workflow cap is fine, the ORG + # cap is not. Distinct from BUDGET_HARD_BLOCKED: raising the org + # budget is the operator action this asks for, so it must be + # catchable as a budget error rather than a generic refusal. + "BUDGET_ORG_CEILING_BLOCKED": NullRunBudgetError, + # Circuit breaker. A decision, not an outage: the workflow's + # cb_state is OPEN and /gate refuses at step 0.5 until an operator + # resets it. Fail-CLOSED mode (Enforce) is why this reaches the + # wire at all; in LogOnly mode the gate allows and this never fires. + "CIRCUIT_BREAKER_TRIPPED": NullRunBlockedException, + # The breaker state itself could not be READ — Postgres/Redis + # unavailable at the lookup. Infrastructure, not a breaker + # decision: the gate fails CLOSED, but the cause is unavailability, + # and a caller should retry rather than treat it as a tripped + # workflow. Sibling of WORKFLOW_INACTIVE_LOOKUP_FAILED below. + "CIRCUIT_BREAKER_STATE_LOOKUP_FAILED": NullRunBackendError, + # Workflow lifecycle lookup failed. Deliberately NOT + # ``NullRunWorkflowInactiveError``: that class means "we read the + # row and it is inactive", which is a decision. This means we + # could not read it at all. Collapsing the two would make an + # outage look like a policy decision. + "WORKFLOW_INACTIVE_LOOKUP_FAILED": NullRunBackendError, + # Workflow is paused by an operator. Decision → blocked. + "WORKFLOW_PAUSED": NullRunBlockedException, + # Plan-tier execution quota exhausted. Distinct from the + # per-key token bucket (RATE_LIMIT_EXCEEDED): this is the org's + # plan allowance, and the remedy is a plan change rather than a + # backoff. + "EXECUTION_QUOTA_EXCEEDED": NullRunBlockedException, + # Same idempotency key replayed with a different payload. A + # client-side contract violation → protocol class. Same key with + # the SAME payload is an idempotent success, not this code. + "IDEMPOTENCY_KEY_MISMATCH": NullRunProtocolError, + # Body/workflow-id disagreement. Wire-shape family — maps to + # NullRunBackendError alongside EXECUTION_ID_MALFORMED / + # EXECUTION_ID_REQUIRED, which is the established treatment for + # "the server could not make sense of your envelope". + "WORKFLOW_ID_BODY_MISMATCH": NullRunBackendError, + # Approval-rule store unavailable at /gate step 7. The gate fails + # CLOSED (does not fall through to "no rule matched → allow"), so + # this is the approval subsystem being unavailable rather than a + # rule rejecting the call — the typed approval-infra class. + "APPROVAL_RULES_UNAVAILABLE": NullRunApprovalDbUnavailableError, + # Server-side digest recompute failed during approval consume. + # Same infra family as above; the distinction is that the failure + # is in recomputation rather than the rules store. + "APPROVAL_DIGEST_RECOMPUTE_FAILED": NullRunApprovalDbUnavailableError, + # Backend reserved 503: Redis down on the budget path. Note this + # is the WIRE string for the reserved unavailability; the older + # BUDGET_REDIS_UNAVAILABLE spelling above remains mapped for + # backward compatibility, since the SDK preserved the pre-v3 + # string. Both routes are live depending on backend version — + # callers should not branch on which one arrives. + "REDIS_UNAVAILABLE": NullRunBackendError, + # Terminal backend-side error on an otherwise well-formed request. + # This is the code that proves the base-class fallback is not a + # catch-all hiding drift: previously every one of the eleven + # codes above landed here untyped. + "INTERNAL_ERROR": NullRunBackendError, } diff --git a/src/nullrun/wal_admin.py b/src/nullrun/wal_admin.py new file mode 100644 index 0000000..a25b5b6 --- /dev/null +++ b/src/nullrun/wal_admin.py @@ -0,0 +1,479 @@ +"""Operator tooling for the SDK's write-ahead log: ``nullrun-wal``. + +The WAL has three files, and only one of them is well served: + +* ```` / ``.1`` — events awaiting delivery. Self-healing: they are + replayed automatically on the next ``Transport.start()``. +* ``.inflight`` — the batch that was on the wire when the process died. + Also self-healing. +* ``.dlq`` — events the backend refused and the SDK gave up on. + +The DLQ is the odd one out. Nothing replays it, because a refusal is by +definition not something a retry fixes: re-sending an event the backend +already rejected would be rejected again, and the natural-looking "retry the +DLQ" loop is exactly the poison pill the DLQ exists to break out of. So the +DLQ needs a human, and a human needs a command. + +What it will not do is guess. ``replay`` only removes a row once the backend +has confirmed that exact ``event_id``; anything unconfirmed stays on disk. +That is the same at-least-once discipline the transport itself keeps, and +the same dedup key makes the re-send free if it lands twice. + +Usage:: + + nullrun-wal status + nullrun-wal replay --dry-run + nullrun-wal replay --reason reservation_not_found --limit 100 + nullrun-wal replay --execute +""" + +from __future__ import annotations + +import argparse +import datetime +import json +import os +import sys +import tempfile +from collections import Counter +from typing import Any + +__all__ = ["main", "read_dlq", "dlq_paths", "default_wal_path"] + +_DEFAULT_WAL = os.path.join(tempfile.gettempdir(), "nullrun.wal") + +# Refusals that replaying cannot fix, so replay refuses them by default. +# +# `EXECUTION_NOT_BOUND` is the load-bearing case: the execution is bound +# server-side for 24 hours and the binding has expired, which is a statement +# about a resource that no longer exists. Re-sending the event does not +# recreate the binding, it just produces the same 422. Recovering this class +# needs an ingest that records the spend without a reservation to attach it +# to — a ledger-only path that does not exist in this SDK yet, and that the +# backend has to grow before it can be offered here. +# +# Without this list, `replay` accepts the refusal, gets the same refusal +# back, and — because the check for "did it land?" could not see where the +# event had gone — reports success and DELETES the operator's only record of +# the event. A tool that destroys evidence on a failed repair is worse than +# one that refuses to run. +_TERMINAL_WITHOUT_REPLAY = { + "EXECUTION_NOT_BOUND:422", + "EXECUTION_NOT_BOUND:503", +} + + +def _is_terminal_without_replay(reason: str) -> bool: + return reason in _TERMINAL_WITHOUT_REPLAY + + +def default_wal_path() -> str: + """The WAL path this host would use, honouring ``NULLRUN_WAL_PATH``. + + Mirrors ``Transport._wal_path``. Duplicated rather than imported so the + CLI can answer "where is my WAL" without constructing a Transport, which + would create an httpx client and touch the permissions of files it has no + business touching yet. + """ + return os.environ.get("NULLRUN_WAL_PATH") or _DEFAULT_WAL + + +def dlq_paths(wal_path: str) -> dict[str, str]: + """The files that make up one WAL, for display and for the CLI.""" + return { + "active": wal_path, + "rotated": f"{wal_path}.1", + "inflight": f"{wal_path}.inflight", + "dlq": f"{wal_path}.dlq", + "holdover": f"{wal_path}.holdover", + "lock": f"{wal_path}.lock", + } + + +def _size(path: str) -> int: + try: + return os.path.getsize(path) + except OSError: + return 0 + + +def read_dlq(dlq_path: str) -> tuple[list[dict[str, Any]], list[str]]: + """Read DLQ rows tolerantly. Returns ``(rows, unparseable_lines)``. + + A DLQ is a recovery file, so a single corrupt line must not make the rest + unreadable — a reader that required every line to parse would turn one + partial write into total loss. Unparseable lines are returned rather than + dropped so the caller can say how many there are; the operator decides + what to do with them, which is not this tool's call to make silently. + """ + rows: list[dict[str, Any]] = [] + corrupt: list[str] = [] + try: + with open(dlq_path) as f: + lines = f.readlines() + except FileNotFoundError: + return rows, corrupt + except OSError as e: + print(f"error: cannot read {dlq_path}: {e}", file=sys.stderr) + return rows, corrupt + for line in lines: + stripped = line.strip() + if not stripped: + continue + try: + parsed = json.loads(stripped) + except json.JSONDecodeError: + corrupt.append(stripped[:200]) + continue + if isinstance(parsed, dict): + rows.append(parsed) + else: + corrupt.append(stripped[:200]) + return rows, corrupt + + +def _row_reason(row: dict[str, Any]) -> str: + """The machine-readable reason. v1 rows only carry ``error``.""" + reason = row.get("reason") or row.get("error") or "unspecified" + return str(reason) + + +def _row_event(row: dict[str, Any]) -> dict[str, Any] | None: + event = row.get("event") + return event if isinstance(event, dict) else None + + +def _cmd_status(args: argparse.Namespace) -> int: + wal_path = args.wal + paths = dlq_paths(wal_path) + print(f"WAL: {wal_path}") + for name, path in paths.items(): + exists = os.path.exists(path) + print(f" {name:<9} {path} {os.path.getsize(path) if exists else '-'} bytes") + + held, _ = read_dlq(paths["holdover"]) + if held: + print( + f"\nHoldover: {len(held)} refusal(s) waiting for DLQ room. They are " + f"durable and are NOT being re-sent. Free space by raising " + f"NULLRUN_DLQ_MAX_BYTES or archiving the DLQ; they drain on the " + f"next flush." + ) + + rows, corrupt = read_dlq(paths["dlq"]) + if not rows and not corrupt: + print("\nDLQ is empty.") + return 0 + + print(f"\nDLQ: {len(rows)} row(s)") + if corrupt: + print(f" {len(corrupt)} unparseable line(s) — left untouched") + for reason, count in Counter(_row_reason(r) for r in rows).most_common(): + print(f" {count:>6} {reason}") + oldest = min((r.get("first_failed_at") or 0) for r in rows) + newest = max((r.get("first_failed_at") or 0) for r in rows) + if oldest: + fmt = "%Y-%m-%d %H:%M:%S UTC" + print( + " window: " + f"{datetime.datetime.fromtimestamp(oldest, datetime.timezone.utc).strftime(fmt)}" + " .. " + f"{datetime.datetime.fromtimestamp(newest, datetime.timezone.utc).strftime(fmt)}" + ) + print("\nReplay with: nullrun-wal replay --dry-run") + return 0 + + +def _select_rows( + rows: list[dict[str, Any]], + reason: str | None, + limit: int | None, + include_terminal: bool = False, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + """Filter DLQ rows down to what the operator asked for. + + Returns ``(selected, withheld)``. ``withheld`` is the terminal class that + was excluded by default (see ``_TERMINAL_WITHOUT_REPLAY``) — it is + returned rather than dropped so the caller can NAME it in its output. A + repair tool that silently skipped rows would leave the operator believing + the DLQ had been fully considered. + + ``--reason`` is applied before the terminal filter, so naming a terminal + reason explicitly and getting nothing back is not a silent no-op either: + it still reports the rows as withheld. + """ + matched = [r for r in rows if reason is None or _row_reason(r) == reason] + if include_terminal: + return (matched[:limit] if limit else matched), [] + replayable = [r for r in matched if not _is_terminal_without_replay(_row_reason(r))] + withheld = [r for r in matched if _is_terminal_without_replay(_row_reason(r))] + return (replayable[:limit] if limit else replayable), withheld + + +def _unconfirmed_event_ids(transport: Any, wal_path: str) -> set[str]: + """Event ids the backend did not accept, wherever they ended up. + + An event the transport refused can be in exactly two places, and this + function used to look at neither consistently: the in-memory buffer (only + for a flush that never ran) and — through ``transport._buffer`` alone — + nothing at all for a flush that did run. A refused event is parked in the + scratch DLQ or held in ``_dlq_overflow``; neither is the buffer. So a + replay whose every event was refused reported zero unconfirmed and then + DELETED the operator's only record of events the backend had just refused. + That is the failure this exists to prevent, so both destinations are + enumerated here rather than one of them. + + Reads the scratch DLQ file, not a parsed snapshot: a row can be written + after the flush returns if the transport retried internally. + + The holdover file is read for the same reason, and it is a file rather + than the in-memory index precisely because the index can be truncated by + NULLRUN_DLQ_HOLDOVER_MAX_EVENTS — a replay that trusted the index alone + would call a held-and-unwritten event confirmed and delete it. + """ + ids = {str(e.get("event_id")) for e in getattr(transport, "_buffer", []) or []} + for row in getattr(transport, "_dlq_overflow", []) or []: + event = row.get("event") if isinstance(row, dict) else None + if isinstance(event, dict): + ids.add(str(event.get("event_id"))) + paths = dlq_paths(wal_path) + for key in ("dlq", "holdover"): + parked, _ = read_dlq(paths[key]) + for row in parked: + event = _row_event(row) + if event is not None: + ids.add(str(event.get("event_id"))) + return ids + + +def _cmd_replay(args: argparse.Namespace) -> int: + from nullrun.transport import Transport + + paths = dlq_paths(args.wal) + rows, corrupt = read_dlq(paths["dlq"]) + if corrupt: + print( + f"note: {len(corrupt)} unparseable DLQ line(s) will not be replayed " + "and are left in place", + file=sys.stderr, + ) + selected, withheld = _select_rows( + rows, args.reason, args.limit, include_terminal=args.include_terminal + ) + if withheld: + reasons = Counter(_row_reason(r) for r in withheld) + listing = ", ".join(f"{r} x{n}" for r, n in reasons.most_common()) + print( + f"note: {len(withheld)} selected row(s) are refusals that replaying " + f"cannot fix ({listing}). They stay in the DLQ. Re-sending them " + "would return the same refusal and could multiply duplicates if the " + "backend's behaviour changes between runs. Recovering this class " + "needs a ledger-only ingest that records the spend without a " + "reservation — not yet implemented. Pass --include-terminal to " + "send them anyway.", + file=sys.stderr, + ) + if not selected: + # rc 1 when rows existed but all were withheld, rc 0 when the DLQ + # simply had nothing. "Nothing to replay" after a deliberate --execute + # against a non-empty DLQ is a failure the caller needs to see; an + # empty DLQ is not. A 0 here would let a scheduled repair report + # success while sending nothing. + print("nothing to replay") + return 1 if withheld else 0 + + replayable = [(r, _row_event(r)) for r in selected] + missing = [r for r, e in replayable if e is None] + if missing: + print( + f"error: {len(missing)} selected row(s) carry no event payload and " + "cannot be replayed; they stay in the DLQ", + file=sys.stderr, + ) + events = [e for _, e in replayable if e is not None] + if not events: + return 1 + + counts = Counter(_row_reason(r) for r, _ in replayable) + print(f"selected {len(events)} event(s):") + for reason, count in counts.most_common(): + print(f" {count:>6} {reason}") + + if args.dry_run or not args.execute: + print("\ndry run — nothing sent, nothing removed. Add --execute to send.") + return 0 + + api_key = args.api_key or os.environ.get("NULLRUN_API_KEY") + if not api_key: + print( + "error: no API key. Pass --api-key or set NULLRUN_API_KEY.", + file=sys.stderr, + ) + return 1 + + # The CLI gets its OWN WAL. Pointing it at the operator's live WAL would + # mean a replay run racing the application's flushes over the same files, + # and would leave the application's own recovery files rewritten by a + # one-shot maintenance command. `Transport` reads the path from the + # environment, so the override is scoped to the block and restored after. + previous_wal = os.environ.get("NULLRUN_WAL_PATH") + with tempfile.TemporaryDirectory(prefix="nullrun-wal-replay-") as scratch: + scratch_wal = os.path.join(scratch, "replay.wal") + os.environ["NULLRUN_WAL_PATH"] = scratch_wal + try: + transport = Transport(args.api_url, api_key=api_key, secret_key=args.secret_key) + try: + for event in events: + transport.track(event) + transport._do_flush() # no start(): we want no background thread + # Read BEFORE the scratch dir goes away — this is the only + # place the refusals from this run are recorded. + unconfirmed = _unconfirmed_event_ids(transport, scratch_wal) + finally: + transport.stop(flush=False) + finally: + if previous_wal is None: + os.environ.pop("NULLRUN_WAL_PATH", None) + else: + os.environ["NULLRUN_WAL_PATH"] = previous_wal + + if unconfirmed: + print( + f"\n{len(unconfirmed)} event(s) were not confirmed and stay in the DLQ.", + file=sys.stderr, + ) + confirmed_ids = { + str(_row_event(r).get("event_id")) # type: ignore[union-attr] + for r, _ in replayable + if str(_row_event(r).get("event_id")) not in unconfirmed # type: ignore[union-attr] + } + if not confirmed_ids: + print("\nnothing confirmed; the DLQ is unchanged.") + return 1 + + removed = _drop_confirmed(paths["dlq"], confirmed_ids) + print( + f"\nsent {len(confirmed_ids)} event(s); removed {removed} row(s) from the DLQ." + ) + return 0 + + +def _drop_confirmed(dlq_path: str, confirmed_ids: set[str]) -> int: + """Rewrite the DLQ without the confirmed rows. Returns how many went. + + A line-wise filter over the raw file, not a re-serialisation of the + parsed rows: a line this tool could not parse is exactly the line an + operator most needs to look at, and round-tripping the file through + ``json.dumps`` of the rows that DID parse would silently delete the ones + that did not — plus rewrite the ones that did, losing whatever formatting + the record carried. Only lines whose ``event_id`` the backend confirmed + are dropped; everything else, parsable or not, is copied verbatim. + """ + if not confirmed_ids: + return 0 + try: + with open(dlq_path) as f: + original = f.readlines() + except OSError as e: + print(f"error: cannot read {dlq_path}: {e}", file=sys.stderr) + return 0 + + kept: list[str] = [] + removed = 0 + for line in original: + stripped = line.strip() + drop = False + if stripped: + try: + parsed = json.loads(stripped) + except json.JSONDecodeError: + parsed = None + if isinstance(parsed, dict): + event = parsed.get("event") + eid = event.get("event_id") if isinstance(event, dict) else None + drop = eid is not None and str(eid) in confirmed_ids + if drop: + removed += 1 + else: + kept.append(line if line.endswith("\n") else line + "\n") + + if removed == 0: + return 0 + + tmp = f"{dlq_path}.tmp.{os.getpid()}" + try: + fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w") as f: + f.writelines(kept) + f.flush() + os.fsync(f.fileno()) + os.replace(tmp, dlq_path) + except OSError as e: + print(f"error: could not rewrite {dlq_path}: {e}", file=sys.stderr) + try: + os.unlink(tmp) + except OSError: + pass + return 0 + return removed + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="nullrun-wal", + description="Inspect and replay the NullRun SDK write-ahead log.", + ) + parser.add_argument( + "--wal", + default=default_wal_path(), + help="WAL path (default: $NULLRUN_WAL_PATH, else the tempdir).", + ) + sub = parser.add_subparsers(dest="command") + + sub.add_parser("status", help="Show WAL files and a summary of the DLQ.") + + replay = sub.add_parser( + "replay", + help="Re-send dead-lettered events, dropping only the confirmed ones.", + ) + replay.add_argument( + "--reason", + help="Only replay rows with this exact reason (default: all).", + ) + replay.add_argument("--limit", type=int, help="Replay at most N rows.") + replay.add_argument( + "--include-terminal", + action="store_true", + help=( + "Also send refusals that re-sending cannot fix (currently " + + ", ".join(sorted(_TERMINAL_WITHOUT_REPLAY)) + + "). Off by default: the send is expected to fail identically." + ), + ) + replay.add_argument( + "--dry-run", + action="store_true", + help="Show what would be sent. Implied unless --execute is given.", + ) + replay.add_argument( + "--execute", + action="store_true", + help="Actually send. Without it, replay is a dry run.", + ) + replay.add_argument( + "--api-url", default=os.environ.get("NULLRUN_API_URL", "https://api.nullrun.io") + ) + replay.add_argument("--api-key", help="Defaults to $NULLRUN_API_KEY.") + replay.add_argument("--secret-key", help="HMAC signing key, if configured.") + + args = parser.parse_args(argv) + if args.command == "status": + return _cmd_status(args) + if args.command == "replay": + return _cmd_replay(args) + parser.print_help() + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/conftest.py b/tests/conftest.py index ac3e60c..d0f08d7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -224,6 +224,30 @@ def _fast_sleep(seconds): return _real_sleep(seconds) monkeypatch.setattr(_time, "sleep", _fast_sleep) + # `nullrun.transport._wait_on` is the OTHER way this suite waits. The + # retry loop moved to it so a `Transport.stop()` can cut a `Retry-After` + # wait short — `time.sleep` cannot be woken, which is why a shutdown + # during a rate-limit wait used to block for the remainder of the wait. + # Patching only `time.sleep` leaves that wait real, and a single 429 + # test then costs its full `Retry-After` in wall clock. + # + # Scoped to the seam rather than to `threading.Event.wait` globally: + # a global patch changes the behaviour of every other Event.wait in the + # suite, including the approval-wait tests that assert on real elapsed + # time, and those then fail (or flake under load) for a reason that has + # nothing to do with the retry path. + def _fast_wait_on(cancel, seconds): + # The set-and-immediately-return branch is preserved: it is what + # makes the wait interruptible, and a stub that broke it would test + # the uninterruptible behaviour while claiming to test the fix. + return not cancel.wait(0.001) + + try: + import nullrun.transport as _transport_mod + + monkeypatch.setattr(_transport_mod, "_wait_on", _fast_wait_on) + except Exception: + pass # Stub the modules that captured a module-level reference at import time. try: import nullrun.transport as _transport_mod diff --git a/tests/test_2026_09_10_catchfanin_passthrough.py b/tests/test_2026_09_10_catchfanin_passthrough.py index e40f288..580e0db 100644 --- a/tests/test_2026_09_10_catchfanin_passthrough.py +++ b/tests/test_2026_09_10_catchfanin_passthrough.py @@ -370,16 +370,26 @@ def test_workflow_inactive_propagates_as_workflow_inactive_error( def test_consume_overbudget_propagates_with_counter_attrs( self, transport ): - """CONSUME_OVERBUDGET (NR-O001) → NullRunConsumeOverbudgetError.""" + """CONSUME_OVERBUDGET (NR-O001) → NullRunConsumeOverbudgetError. + + The `details` block is the backend's real one (`handlers.rs`, + the ConsumeOverbudget 422): the reservation arrives in + MILLICENTS, the actual in cents, and there is no + `epsilon_cents` key. An earlier version of this test invented + `reserved_cents` / `max_allowed_cents` / `epsilon_cents`, passed + because the fixture matched the parser, and left the parser + reading three fields the server has never sent. + """ respx.post(_EXECUTE_URL).mock( return_value=_v3_envelope( "CONSUME_OVERBUDGET", status=422, execution_id="exec-consume-overrun", - reserved_cents=100, - max_allowed_cents=101, + reserved_millicents=1000, + max_allowed_millicents=1010, actual_cost_cents=1000, - epsilon_cents=1, + soft_pass=False, + reservation_recorded=True, ) ) with pytest.raises(NullRunConsumeOverbudgetError) as excinfo: @@ -389,10 +399,10 @@ def test_consume_overbudget_propagates_with_counter_attrs( "must raise NullRunConsumeOverbudgetError (NR-O001), " "not be swallowed." ) - assert excinfo.value.reserved_cents == 100 - assert excinfo.value.max_allowed_cents == 101 + assert excinfo.value.reserved_millicents == 1000 + assert excinfo.value.max_allowed_millicents == 1010 assert excinfo.value.actual_cost_cents == 1000 - assert excinfo.value.epsilon_cents == 1 + assert excinfo.value.recorded is True @respx.mock def test_protocol_too_old_propagates_as_protocol_error(self, transport): diff --git a/tests/test_edge_via_edge.py b/tests/test_edge_via_edge.py new file mode 100644 index 0000000..873d94f --- /dev/null +++ b/tests/test_edge_via_edge.py @@ -0,0 +1,464 @@ +"""Via-edge: what the SDK must do, and what it must refuse to do. + +Every test here is about a decision that moves money, so each one states +which direction it fails in. The pattern throughout: assert the ALLOW +leg before the BLOCK leg, because an assertion that only ever checks +refusal is satisfied by an implementation that refuses everything. +""" + +from __future__ import annotations + +import os + +import pytest + +from nullrun.edge import ( + EDGE_LEASE, + EdgeConfigurationError, + EdgeTransport, + edge_transport_from_env, + token_split, +) +from nullrun.transport import DecisionSource, Transport, is_fallback_decision_source + + +def _env(**over: str) -> dict[str, str]: + base = { + "NULLRUN_EDGE_URL": "http://127.0.0.1:18090", + "NULLRUN_EDGE_LEASE_ID": "0195f0c0-0000-7000-8000-000000000001", + "NULLRUN_EDGE_TOKEN": "box-token", + "NULLRUN_API_KEY": "nr_org_key", + } + base.update(over) + return base + + +# ── configuration ───────────────────────────────────────────────────── + + +def test_direct_is_the_default_and_costs_nothing(): + """E1. Unset means direct, and it means it with no runtime in the way. + + Asserted as the very first case because every other behaviour here + is conditional on it: a mode that engaged by accident would take the + enforcement path off the cloud for users who never asked for it. + """ + assert edge_transport_from_env({}) is None + assert edge_transport_from_env({"NULLRUN_EDGE_URL": " "}) is None + + +def test_the_environment_is_not_read_when_direct(): + """A half-configured via-edge must not break a direct user. + + If the env were read before the URL check, an operator who exported + `NULLRUN_EDGE_TOKEN` early in a deploy script and `NULLRUN_EDGE_URL` + later would break every call in between. Failing at construction is + right for via-edge and catastrophic for direct. + """ + assert ( + edge_transport_from_env( + { + "NULLRUN_EDGE_TOKEN": "set-but-no-url", + "NULLRUN_EDGE_LEASE_ID": "also-without-a-url", + } + ) + is None + ) + + +@pytest.mark.parametrize( + "missing", + ["NULLRUN_EDGE_LEASE_ID", "NULLRUN_EDGE_TOKEN", "NULLRUN_API_KEY"], +) +def test_via_edge_that_cannot_enforce_refuses_to_start(missing: str): + """E1/E4. Asked for and unable to enforce is an error, not a warning. + + The failure it prevents: via-edge engaged, the box was never really + reachable, and every tool call in the agent ran with no gate — while + the operator's dashboard showed an edge box holding the line. + """ + env = _env() + env.pop(missing) + with pytest.raises(EdgeConfigurationError) as excinfo: + edge_transport_from_env(env) + # The message must NAME the missing variable — an operator reading + # this at 3am should not have to guess which of four it was. + assert missing in str(excinfo.value) + + +def test_a_missing_lease_id_says_why_the_box_cannot_pick_one(): + """The lease is the org's grant. A box that chose its own is not enforcing.""" + env = _env() + env.pop("NULLRUN_EDGE_LEASE_ID") + with pytest.raises(EdgeConfigurationError) as excinfo: + edge_transport_from_env(env) + assert "grant" in str(excinfo.value).lower() + + +def test_an_edge_url_that_is_not_a_url_is_left_to_the_transport_to_complain(): + """Configuration is checked for MEANING here, not for syntax. + + Syntactic validation belongs to the HTTP layer that already has + `InsecureTransportError`; a second, weaker copy here would accept + things that one rejects. + """ + t = edge_transport_from_env(_env(NULLRUN_EDGE_URL="not a url")) + assert t is not None + assert t.base_url == "not a url" + + +# ── decision provenance ─────────────────────────────────────────────── + + +def test_a_lease_decision_is_not_a_fallback_decision(): + """E2. The predicate every enforcement call site uses must say False. + + `is_fallback_decision_source` is what the runtime checks before + honouring a decision. If `edge_lease` read as a fallback, STRICT + would turn a real refusal from the box into a synthetic one and, + worse, a real ALLOW would be treated as degradation rather than as + a grant's answer. + """ + assert EDGE_LEASE not in (DecisionSource.GATEWAY, DecisionSource.FALLBACK) + assert is_fallback_decision_source(EDGE_LEASE) is False + + +def test_an_unreachable_box_is_a_fallback_so_strict_still_closes(): + """E3. The unreachable answer is NOT a lease answer. + + Same shape, opposite direction: this one must read as synthetic, or + the runtime would treat "the box did not answer" as the lease having + spoken. + """ + assert is_fallback_decision_source(DecisionSource.FALLBACK) is True + + +# ── what the SDK is allowed to say ───────────────────────────────────── + + +def test_the_sdk_reports_tokens_and_never_an_amount(): + """DEF-40 again, on the client side. + + The wire carries a model and two counts. An amount on the wire is + the bug being fixed, so the test is that no numeric field the box + could read as money survives into the request. + """ + inp, out = token_split({"model": "gpt-4o", "estimated_tokens": 1234}) + assert (inp, out) == (0, 1234) + + inp, out = token_split({"edge_input_tokens": 10, "edge_output_tokens": 20}) + assert (inp, out) == (10, 20) + + assert token_split({}) == (0, 0) + + +def test_an_estimate_with_no_breakdown_is_reported_as_output(): + """Direction: over-report, never under-report. + + Output dominates the price for most models, so attributing the whole + estimate there errs toward charging more. Attributing it to input + would under-charge on every call, and the box cannot check the + number — it believes whatever it is told. + """ + inp, out = token_split({"estimated_tokens": 500}) + assert out == 500 + assert inp == 0 + + +def test_token_counts_are_ints_and_survive_a_string_from_the_caller(): + """A string count is a crash on the wire, not a rounding question.""" + inp, out = token_split({"estimated_tokens": "250"}) + assert (inp, out) == (0, 250) + assert isinstance(out, int) + + +# ── the box's answer ────────────────────────────────────────────────── + + +class _FakeResponse: + def __init__(self, payload, status_code=200): + self._payload = payload + self.status_code = status_code + + def json(self): + return self._payload + + +def _transport(responder): + t = EdgeTransport( + base_url="http://127.0.0.1:1", + lease_id="0195f0c0-0000-7000-8000-000000000001", + box_token="box-token", + api_key="nr_org_key", + timeout=0.05, + max_retries=0, + ) + t._post = responder # type: ignore[method-assign] + return t + + +def test_an_allow_from_the_box_carries_the_lease_as_its_source(): + t = _transport( + lambda body: _FakeResponse( + { + "allowed": True, + "lease_id": body["lease_id"], + "granted_millicents": 500_000, + "spent_millicents": 1_000, + "remaining_millicents": 499_000, + } + ) + ) + result = t.enforce("gpt-4o", 0, 1_000) + assert result["decision"] == "allow" + assert result["decision_source"] == EDGE_LEASE + assert is_fallback_decision_source(result["decision_source"]) is False + assert result["spent_millicents"] == 1_000 + + +def test_the_request_carries_both_credentials_and_the_lease(): + """E9. Box token proves reach; org key proves the org. + + Sent together and not interchangeably: a shared box must not let one + org's agent spend another org's lease, and the box has no database + to ask. + """ + seen: dict[str, str] = {} + + def responder(body): + seen["body"] = body + return _FakeResponse({"allowed": True, "spent_millicents": 0}) + + t = EdgeTransport( + base_url="http://127.0.0.1:18090", + lease_id="lease-1", + box_token="box-token", + api_key="nr_org_key", + ) + headers = t._headers({}) + assert headers["Authorization"] == "Bearer box-token" + assert headers["X-API-Key"] == "nr_org_key" + assert headers["X-NullRun-Lease-Id"] == "lease-1" + assert seen == {} + + +def test_a_retried_call_keeps_its_event_id(): + """At-least-once delivery is the norm; a fresh id would double-charge.""" + ids = [] + + t = EdgeTransport( + base_url="http://127.0.0.1:18090", + lease_id="lease-1", + box_token="box-token", + api_key="nr_org_key", + ) + + def responder(body): + ids.append(body["event_id"]) + return _FakeResponse({"allowed": True, "spent_millicents": 0}) + + t._post = responder # type: ignore[method-assign] + t.enforce("gpt-4o", 0, 10, event_id="evt-1") + t.enforce("gpt-4o", 0, 10, event_id="evt-1") + assert ids == ["evt-1", "evt-1"] + + +def test_a_refusal_from_the_box_is_a_block_naming_the_code(): + t = _transport( + lambda body: _FakeResponse({"allowed": False, "error_code": "LEASE_EXHAUSTED"}) + ) + result = t.enforce("gpt-4o", 0, 10) + assert result["decision"] == "block" + assert result["decision_source"] == EDGE_LEASE + assert result["error_code"] == "LEASE_EXHAUSTED" + assert "grant" in result["explanation"].lower() + + +def test_an_unreachable_box_blocks_and_does_not_reach_the_cloud(): + """E3. The whole point: no silent fallback. + + The cloud is not configured anywhere in this object, so there is + nothing to fall back to even in principle. What is asserted is the + shape the runtime needs: a synthetic block, not an exception and + definitely not an allow. + """ + def boom(body): + raise OSError("connection refused") + + t = _transport(boom) + result = t.enforce("gpt-4o", 0, 10) + assert result["decision"] == "block" + assert result["decision_source"] == DecisionSource.FALLBACK + assert result["error_code"] == "EDGE_UNREACHABLE" + assert is_fallback_decision_source(result["decision_source"]) is True + + +def test_a_broken_box_is_not_laundered_into_a_budget_answer(): + """5xx means the box is broken, which is not the same as 'no money'. + + It takes the transport-error path so STRICT closes. A 5xx presented + as a lease refusal would be a decision the lease never made. + """ + t = _transport(lambda body: _FakeResponse({"allowed": False}, status_code=503)) + result = t.enforce("gpt-4o", 0, 10) + assert result["decision"] == "block" + assert result["decision_source"] == DecisionSource.FALLBACK + assert result["error_code"] == "EDGE_UNREACHABLE" + + +def test_a_non_json_answer_from_the_box_is_a_transport_failure(): + def responder(body): + response = _FakeResponse(None) + response.json = lambda: (_ for _ in ()).throw(ValueError("not json")) + return response + + t = _transport(responder) + result = t.enforce("gpt-4o", 0, 10) + assert result["decision_source"] == DecisionSource.FALLBACK + + +def test_a_broken_observer_cannot_change_the_decision(): + """`on_transport_error` is a reporter. A raise inside it is not a vote.""" + def boom(body): + raise OSError("connection refused") + + def angry_observer(exc): + raise RuntimeError("the observer is broken") + + t = _transport(boom) + result = t.enforce("gpt-4o", 0, 10, on_transport_error=angry_observer) + assert result["decision"] == "block" + + +# ── wiring into the transport ───────────────────────────────────────── + + +def test_check_goes_to_the_box_and_never_builds_a_gate_body(monkeypatch): + """E4. One path or the other, never a blend. + + A box answer and a partially-built cloud request are not both sent; + the test asserts the cloud's POST is never reached at all. + """ + monkeypatch.setenv("NULLRUN_EDGE_URL", "http://127.0.0.1:18090") + monkeypatch.setenv("NULLRUN_EDGE_LEASE_ID", "lease-1") + monkeypatch.setenv("NULLRUN_EDGE_TOKEN", "box-token") + monkeypatch.setenv("NULLRUN_API_KEY", "nr_org_key") + + t = Transport(api_url="https://api.nullrun.io") + + def must_not_be_called(*a, **kw): + raise AssertionError("the cloud was called in via-edge mode") + + t._client.post = must_not_be_called # type: ignore[method-assign] + t.edge._post = lambda body: _FakeResponse( # type: ignore[method-assign] + {"allowed": True, "spent_millicents": 10, "remaining_millicents": 90} + ) + + result = t.check({"model": "gpt-4o", "estimated_tokens": 10}) + assert result["decision"] == "allow" + assert result["decision_source"] == EDGE_LEASE + + +def test_execute_does_not_reach_the_cloud_in_via_edge_mode(monkeypatch): + """E4. `/execute` has no box equivalent — and must not look for one. + + The box's `/enforce` already decided and charged, so a second call + would either hit the cloud or double-charge. + """ + monkeypatch.setenv("NULLRUN_EDGE_URL", "http://127.0.0.1:18090") + monkeypatch.setenv("NULLRUN_EDGE_LEASE_ID", "lease-1") + monkeypatch.setenv("NULLRUN_EDGE_TOKEN", "box-token") + monkeypatch.setenv("NULLRUN_API_KEY", "nr_org_key") + + t = Transport(api_url="https://api.nullrun.io") + + def must_not_be_called(*a, **kw): + raise AssertionError("the cloud was called in via-edge mode") + + t._client.post = must_not_be_called # type: ignore[method-assign] + t.edge.last_source = EDGE_LEASE + + result = t.execute( + organization_id="org-1", + execution_id="exec-1", + trace_id="trace-1", + tool="read_file", + input_data={}, + ) + assert result["decision"] == "allow" + assert result["via_edge"] is True + + +def test_a_direct_transport_has_no_edge_and_says_nothing_about_one(monkeypatch): + for var in ( + "NULLRUN_EDGE_URL", + "NULLRUN_EDGE_LEASE_ID", + "NULLRUN_EDGE_TOKEN", + ): + monkeypatch.delenv(var, raising=False) + t = Transport(api_url="https://api.nullrun.io", api_key="k") + assert t.edge is None + assert "edge" not in repr(t.edge).lower().replace("edge-case", "") + + +def test_constructing_a_direct_transport_still_works_with_a_half_set_edge_env(monkeypatch): + """Regression guard on the "costs nothing" claim, at the real seam.""" + monkeypatch.setenv("NULLRUN_EDGE_TOKEN", "orphan-token") + t = Transport(api_url="https://api.nullrun.io", api_key="k") + assert t.edge is None + assert os.environ["NULLRUN_EDGE_TOKEN"] == "orphan-token" + +# ── the runtime will accept it ──────────────────────────────────────── + + +def test_the_runtime_honours_a_lease_decision_as_a_real_verdict(): + """The allowlist is the gate in front of the gate. + + `_require_gate_decision` runs before the decision is even read, and + rejects any body whose `decision_source` it does not know. A via-edge + allow that the runtime refused would fail CLOSED on its own success — + which reads in an outage as "the box is broken", not "the SDK has + never heard of this source". + """ + from nullrun.runtime import NullRunRuntime + + rt = NullRunRuntime.__new__(NullRunRuntime) # no I/O, no client + assert ( + rt._require_gate_decision({"decision": "allow", "decision_source": EDGE_LEASE}) + == "allow" + ) + assert ( + rt._require_gate_decision({"decision": "block", "decision_source": EDGE_LEASE}) + == "block" + ) + + +def test_adding_the_source_did_not_loosen_the_source_check(): + """The fix must ADD a name, never stop checking. + + The failure this guards: making via-edge work by admitting unknown + sources turns the check below into no check at all — and that check + is what stops a captive portal from answering with `{"decision": + "allow"}`. + """ + from nullrun.breaker.exceptions import NullRunMalformedGateResponseError + from nullrun.runtime import NullRunRuntime + + rt = NullRunRuntime.__new__(NullRunRuntime) + for body in ( + {"decision": "allow"}, # no source at all + {"decision": "allow", "decision_source": "edge-box"}, # a near miss + {"decision": "allow", "decision_source": 42}, # not a string + ): + with pytest.raises(NullRunMalformedGateResponseError): + rt._require_gate_decision(body) + + +def test_a_source_is_not_enough_if_the_decision_is_nonsense(): + """The other half of the check, kept honest by the same edit.""" + from nullrun.breaker.exceptions import NullRunMalformedGateResponseError + from nullrun.runtime import NullRunRuntime + + rt = NullRunRuntime.__new__(NullRunRuntime) + with pytest.raises(NullRunMalformedGateResponseError): + rt._require_gate_decision({"decision": "probably", "decision_source": EDGE_LEASE}) diff --git a/tests/test_gate_business_impact_wire.py b/tests/test_gate_business_impact_wire.py index 60623a4..4680dbc 100644 --- a/tests/test_gate_business_impact_wire.py +++ b/tests/test_gate_business_impact_wire.py @@ -114,10 +114,17 @@ def test_tool_call_envelope_is_forwarded(self, transport): "extractor_version": "1", } - def test_none_envelope_is_forwarded(self, transport): - # `{"kind": "none"}` is a real value, not an absence. An LLM - # check with no tool to name sends it, and the backend tells - # it apart from an omitted envelope. + def test_none_envelope_is_forwarded_verbatim(self, transport): + # The TRANSPORT forwards whatever it is handed, verbatim. It + # does not decide what is sendable. + # + # This used to read "`{"kind": "none"}` is a real value, not + # an absence ... and the backend tells it apart from an + # omitted envelope". The second half is false: the backend has + # no `none` variant and refuses it at the deserialiser. The + # transport is still a faithful pipe -- the decision not to + # SEND one belongs to the runtime preflight, which now omits + # the field (see `test_no_context_envelope_omits_the_field`). body = _check(transport, business_impact=BusinessImpact.no_impact().to_wire_dict()) assert body["business_impact"] == {"kind": "none"} @@ -219,14 +226,29 @@ def test_gate_digest_matches_its_own_envelope(self, make_runtime, mock_api, capt body = captured_gate_bodies[-1] assert _server_recompute(body["business_impact"]) == body["action_digest"] - def test_no_context_envelope_sends_none(self, make_runtime, mock_api, captured_gate_bodies): - # An LLM check with no tool to name is legitimately - # impact-free. It must still send an envelope, because the - # backend needs one to recompute the stored digest. + def test_no_context_envelope_omits_the_field(self, make_runtime, mock_api, captured_gate_bodies): + # An LLM check with no tool to name has no envelope, and the + # honest wire encoding of "no envelope" is ABSENCE. + # + # This test used to assert `body["business_impact"] == + # {"kind": "none"}`, on the reasoning that "the backend needs + # one to recompute the stored digest". That reasoning is false + # and the assertion shipped a 422: the gate's wire enum is + # `Money | ToolCall` and has no `none` variant, so the request + # was refused by the deserialiser before any policy ran. + # Measured against a live stand, not inferred — see + # `tests/test_live_stand_business_impact.py`. + # + # The digest is still mandatory at protocol >= 3 and is still + # sent, so the gate's version-gate passes. make_runtime().check_workflow_budget() body = captured_gate_bodies[-1] - assert body["business_impact"] == {"kind": "none"} - assert _server_recompute(body["business_impact"]) == body["action_digest"] + assert "business_impact" not in body + assert body["action_digest"] + # Unchanged from before the fix: the digest is still computed + # over the same `no_impact()` canonical bytes, so a stored + # digest does not change across the upgrade. + assert body["action_digest"] == _server_recompute({"kind": "none"}) def test_two_different_tools_produce_two_different_gate_digests( self, make_runtime, mock_api, captured_gate_bodies diff --git a/tests/test_live_stand_business_impact.py b/tests/test_live_stand_business_impact.py new file mode 100644 index 0000000..7063b05 --- /dev/null +++ b/tests/test_live_stand_business_impact.py @@ -0,0 +1,133 @@ +"""LIVE-STAND test: the no-envelope preflight must not be refused at the enum. + +Why this file exists and why it is opt-in +------------------------------------------ +Every other wire test in this suite mocks the transport (34 of 111 test +files use respx). A mocked transport cannot find a client/server enum +skew, because there is no server in it -- which is exactly how +``{"kind": "none"}`` shipped: `test_gate_business_impact_wire.py` +asserted what the SDK *sends* and mocked away what the server +*accepts*. + +So the assertion that matters is made against a real binary, and it is +opt-in because CI has no stand. Set ``NULLRUN_LIVE_STAND_URL`` to run +it: + + NULLRUN_LIVE_STAND_URL=http://localhost:18080 \\ + pytest tests/test_live_stand_business_impact.py -v + +The throwaway org, workflow and API key are created, used and dropped +inside the test process. The key is never logged, never written to a +file, and never placed on a command line. + +What it pins +------------ +``check_workflow_budget()`` with an empty call context has no tool to +name, so it has no envelope. The SDK used to substitute +``no_impact()`` and put ``{"kind": "none"}`` on the wire. The gate's +enum is ``Money | ToolCall`` -- there is no ``none`` -- so the request +was refused by the *deserialiser*, 422, before any policy ran. + +The fix is to omit the field: ``business_impact`` is ``Option`` on the +wire, and only ``action_digest`` is mandatory at protocol >= 3. That +was measured, not assumed -- omitting it returns 200 at /gate and 200 +at /execute on the same path. + +The assertion is therefore *negative* and deliberately so: whatever the +policy decides (this stand has no budget configured, so it hard-blocks) +is fine. What must never happen is a refusal at the enum. +""" + +from __future__ import annotations + +import os +import time + +import httpx +import pytest + +STAND = os.environ.get("NULLRUN_LIVE_STAND_URL", "").rstrip("/") + +pytestmark = pytest.mark.skipif( + not STAND, + reason="set NULLRUN_LIVE_STAND_URL to run against a live breaker-core", +) + +# Markers that mean "the request never reached a policy decision" -- +# i.e. the body was refused at the wire boundary. Any of these +# appearing in a refusal is the bug this test exists to catch. +# +# `LegacyGrant` is here because the digest is load-bearing in its own +# right: omitting `business_impact` is only legal BECAUSE the digest is +# still sent. A "fix" that dropped the digest along with the envelope +# would trade a 422 for a different 422, and this list is what stops +# that passing. +ENUM_REFUSAL_MARKERS = ("unknown variant", "BUSINESS_IMPACT_INVALID", + "missing field", "invalid field", "LegacyGrant") + + +@pytest.fixture(scope="module") +def provisioned() -> tuple[str, str]: + """A throwaway (api_key, workflow_id) pair. In memory only.""" + stamp = int(time.time()) + r = httpx.post( + f"{STAND}/api/v1/auth/register", + json={"email": f"sdk-enum-{stamp}@local.test", + "password": "Str0ng-Local-Pass!42", "name": "Enum Live"}, + timeout=20.0, + ) + assert r.status_code in (200, 201), f"register: {r.status_code} {r.text[:200]}" + reg = r.json() + org, token = reg["organization_id"], reg["session_token"] + auth = {"Authorization": f"Bearer {token}", "X-NULLRUN-PROTOCOL": "3"} + + r = httpx.post(f"{STAND}/api/v1/orgs/{org}/workflows", + json={"name": f"sdk-enum-{stamp}", + "external_id": f"wf-sdk-enum-{stamp}"}, + headers={**auth, "Content-Type": "application/json"}, + timeout=20.0) + assert r.status_code < 300, f"workflow: {r.status_code} {r.text[:200]}" + wid = r.json().get("id") or r.json().get("workflow_id") + + r = httpx.post(f"{STAND}/api/v1/orgs/{org}/api-keys", + json={"name": "sdk-enum", "workflow_id": wid}, + headers={**auth, "Content-Type": "application/json"}, + timeout=20.0) + assert r.status_code < 300, f"key: {r.status_code} {r.text[:200]}" + body = r.json() + key = body.get("api_key") or body.get("key") + assert key, f"no key in response; fields={sorted(body)}" + return key, wid + + +def test_no_envelope_preflight_is_not_refused_at_the_enum(provisioned): + """RED before the fix: 422 `unknown variant 'none'`.""" + key, _wid = provisioned + os.environ["NULLRUN_API_URL"] = STAND + os.environ["NULLRUN_API_KEY"] = key + # The dev opt-out fully bypasses the gate; a test that passes only + # with it set is asserting nothing (CLAUDE.md, "What NOT to do"). + os.environ.pop("NULLRUN_SKIP_BUDGET_CHECK", None) + + from nullrun import init + from nullrun.context import set_call_impact + from nullrun.runtime import NullRunRuntime + + set_call_impact(None) # the precondition: an LLM check, no tool + try: + rt = init(api_key=key, api_url=STAND) + try: + rt.check_workflow_budget() + except Exception as exc: # noqa: BLE001 - the text IS the finding + message = str(exc) + for marker in ENUM_REFUSAL_MARKERS: + assert marker not in message, ( + f"the gate refused the request before any policy ran " + f"({marker!r} in the refusal): {message[:300]}" + ) + finally: + # `reset_instance`, not a guessed `close()`: the assertion + # above must be the thing that fails, not the teardown. + NullRunRuntime.reset_instance() + finally: + set_call_impact(None) diff --git a/tests/test_overbudget_wire_fields.py b/tests/test_overbudget_wire_fields.py new file mode 100644 index 0000000..892c0e4 --- /dev/null +++ b/tests/test_overbudget_wire_fields.py @@ -0,0 +1,114 @@ +"""The 422 CONSUME_OVERBUDGET fields the SDK was reading, and the ones it was not. + +The backend's overage body is `handlers.rs`: + + "reserved_millicents": reserved, // from the Lua decision + "max_allowed_millicents": max_allowed, + "actual_cost_cents": req.cost_cents, + "soft_pass": soft_pass, + "reservation_recorded": reservation_recorded, + +The SDK's dispatcher read `reserved_cents`, `max_allowed_cents` and +`epsilon_cents` — none of which the backend has ever sent. Every one of +them was permanently `None`, and `runtime.py`'s fail-closed table told +callers to reconcile the delta from precisely those three. A caller +following the documentation computed `None - None + None`. + +Two things are pinned here. First, that the numbers the wire actually +carries reach the exception. Second, that epsilon is NOT reconstructed: +the backend does not publish its configured tolerance on this body, and +a conversion that appears to supply it (`// 10` on the millicents, then +"epsilon = actual - max_allowed") would be arithmetic dressed as +evidence. The honest recovery is `actual_millicents - reserved`, and the +caller does it. +""" + +from __future__ import annotations + +import httpx + +from nullrun.breaker.exceptions import NullRunConsumeOverbudgetError +from nullrun.transport import _parse_v3_error_envelope_uncategorised + +# The real body shape, copied from handlers.rs (`"reserved_millicents"` +# at the ConsumeOverbudget 422). `reserved` is 2500 millicents = 250c, +# `max_allowed` is 2500 + 10 (a 1c epsilon), `actual` is 2600c. +WIRE_422 = { + "error_code": "CONSUME_OVERBUDGET", + "error_message": "consume exceeds reservation by more than epsilon", + "details": { + "endpoint": "/track", + "execution_id": "0199-exec-over", + "reserved_millicents": 2500, + "max_allowed_millicents": 2510, + "actual_cost_cents": 2600, + "soft_pass": False, + "reservation_recorded": True, + }, +} + + +def _parse() -> NullRunConsumeOverbudgetError: + err = _parse_v3_error_envelope_uncategorised(httpx.Response(422, json=WIRE_422), "/track") + assert isinstance(err, NullRunConsumeOverbudgetError) + return err + + +def test_the_reservation_reaches_the_caller_in_the_unit_the_wire_uses(): + err = _parse() + assert err.reserved_millicents == 2500 + assert err.max_allowed_millicents == 2510 + + +def test_the_actual_spend_reaches_the_caller_in_cents(): + err = _parse() + assert err.actual_cost_cents == 2600 + + +def test_soft_pass_is_surfaced(): + err = _parse() + assert err.soft_pass is False + + +def test_the_old_cents_fields_are_absent_rather_than_silently_wrong(): + """They stay None because nothing populates them — not because a guess failed. + + Asserting `reserved_cents is None` is a statement about the wire: the + backend has no such key, so any value here would be the SDK's + invention. A caller that reads them must see the absence. + """ + err = _parse() + assert err.reserved_cents is None + assert err.max_allowed_cents is None + assert err.epsilon_cents is None + + +def test_the_documented_reconciliation_is_computable_from_what_arrives(): + """The numbers the fail-closed table promises are enough on their own. + + The reservation is in MILLICENTS and the actual in CENTS, so the + subtraction needs the x10 — a caller who skips it reads an overshoot + three orders of magnitude too large. `actual - max_allowed` is the + overshoot past the ceiling; `max_allowed - reserved` is the epsilon + the backend actually applied to this call. Both are recoverable, and + neither needs a field the backend does not send. + """ + err = _parse() + overshoot_milli = err.actual_cost_cents * 10 - err.max_allowed_millicents + epsilon_applied_milli = err.max_allowed_millicents - err.reserved_millicents + # 2600c spent against a 251c ceiling is 2349c over — 23490 millicents. + assert overshoot_milli == 23_490 + assert epsilon_applied_milli == 10 # 1 cent, the platform default + + +def test_a_malformed_number_does_not_become_a_typed_one(): + """A string where a number belongs stays absent rather than being coerced.""" + body = { + "error_code": "CONSUME_OVERBUDGET", + "error_message": "over", + "details": {"reserved_millicents": "2500", "reservation_recorded": "true"}, + } + err = _parse_v3_error_envelope_uncategorised(httpx.Response(422, json=body), "/track") + assert isinstance(err, NullRunConsumeOverbudgetError) + assert err.reserved_millicents is None + assert err.recorded is None diff --git a/tests/test_recorded_consume_overbudget.py b/tests/test_recorded_consume_overbudget.py new file mode 100644 index 0000000..9ca7319 --- /dev/null +++ b/tests/test_recorded_consume_overbudget.py @@ -0,0 +1,328 @@ +"""A 422 CONSUME_OVERBUDGET that the backend already charged is DELIVERED. + +What the wire actually says. ``/track/batch`` settles each event against +the period counter. When a settle lands past ``reserved + epsilon_cents`` +the Lua script returns the ``overage`` branch. That branch applies the +real spend to the counter and writes the ``cost_events`` row — the +reservation is NOT re-reserved, and the money IS gone — and the handler +answers 422 with ``details.reservation_recorded: true`` plus the +offending ``event_id``. + +What the SDK did with it. ``CONSUME_OVERBUDGET`` is in +``_DETERMINIC_ERROR_CODES``, so the 422 became a +``DeterministicBackendRefusal``: the batch was bisected and the +offending singleton was written to ``.dlq`` with the note "will NOT be +retried". For a refusal where nothing was charged that is right. For a +refusal where the backend has already booked the spend it is a lie in +two directions at once — the event is filed as undelivered when the +server has a row for it, and the operator's only durable record of that +event is a DLQ entry describing a rejection that did not happen. + +The rule these tests pin: **the backend's own ``recorded`` marker +decides, and the SDK never second-guesses it.** + +* ``reservation_recorded: true`` → delivered. Drop the event from the + retry path, do not bisect, do not DLQ, and re-queue the *rest* of the + batch, which was never processed and is genuinely outstanding. +* ``reservation_recorded: false`` or absent → the old quarantine path, + untouched. A refusal that charged nothing must still park the data. +* No ``details`` at all (an older backend) → fall back to bisecting, + then apply the same rule to the isolated singleton. The bisect + converges; the singleton carries the marker. + +Also pinned here: the single-``/track`` typed error surfaces ``recorded`` +so a caller that catches ``NullRunConsumeOverbudgetError`` can tell +"the server refused and charged you" from "the server refused and left +your reservation alone" — the two demand opposite caller behaviour +(reconcile vs. retry under a higher ceiling), and the SDK docstring +promises the caller the numbers to do the first. +""" + +from __future__ import annotations + +import json +import os + +import httpx +import pytest +import respx + +from nullrun.breaker.exceptions import NullRunConsumeOverbudgetError +from nullrun.transport import Transport + + +@pytest.fixture +def transport(tmp_path, monkeypatch): + monkeypatch.setenv("NULLRUN_WAL_PATH", str(tmp_path / "sdk.wal")) + t = Transport(api_url="https://api.test.nullrun.io", api_key="test-key-12345678") + yield t + t._client.close() + + +def _event(i: int) -> dict: + return {"event_id": f"evt-{i}", "type": "llm_call", "cost_cents": 1} + + +def _overbudget_body(*, recorded: bool | None, event_id: str | None = "evt-2") -> dict: + details: dict = { + "endpoint": "/track/batch", + "execution_id": "0199-exec-over", + "reserved_millicents": 1000, + "actual_cost_cents": 2100, + "soft_pass": False, + } + if recorded is not None: + details["reservation_recorded"] = recorded + if event_id is not None: + details["event_id"] = event_id + return { + "error_code": "CONSUME_OVERBUDGET", + "error_message": "consume exceeds reservation by more than epsilon", + "details": details, + } + + +def _dlq_rows(t: Transport) -> list[dict]: + path = t._wal_dlq_path() + if not os.path.exists(path): + return [] + with open(path) as f: + return [json.loads(line) for line in f if line.strip()] + + +def _post_422(body: dict): + return respx.post("https://api.test.nullrun.io/api/v1/track/batch").mock( + return_value=httpx.Response(422, json=body) + ) + + +def _post_seq(*responses: httpx.Response): + return respx.post("https://api.test.nullrun.io/api/v1/track/batch").mock( + side_effect=list(responses) + ) + + +def _accepted(*event_ids: str) -> httpx.Response: + return httpx.Response( + 200, json={"accepted_event_ids": list(event_ids), "rejection_details": []} + ) + + +# --------------------------------------------------------------------------- +# recorded: the spend is booked. The event is delivered. +# --------------------------------------------------------------------------- + + +@respx.mock +def test_a_recorded_overage_is_not_dead_lettered(transport): + """The headline case: 3 events, one overage, and the marker says charged.""" + _post_422(_overbudget_body(recorded=True, event_id="evt-2")) + transport._buffer.extend([_event(1), _event(2), _event(3)]) + + transport._do_flush() + + assert _dlq_rows(transport) == [], "a charged event was filed as undelivered" + # The other two were never processed — they must still be outstanding. + assert sorted(e["event_id"] for e in transport._buffer) == ["evt-1", "evt-3"] + + +@respx.mock +def test_a_recorded_overage_does_not_bisect(transport): + """Bisecting to isolate a refusal the backend already resolved wastes sends.""" + route = _post_422(_overbudget_body(recorded=True, event_id="evt-2")) + transport._buffer.extend([_event(1), _event(2), _event(3)]) + + transport._do_flush() + + assert route.call_count == 1, "the batch was split even though the body named the offender" + + +@respx.mock +def test_a_recorded_overage_releases_its_inflight_marker(transport): + """`.inflight` is the crash-recovery copy. A delivered event must not be replayed.""" + for i in (1, 2, 3): + transport.track(_event(i)) + _post_422(_overbudget_body(recorded=True, event_id="evt-2")) + + transport._do_flush() + + # evt-1 / evt-3 are re-queued and stay in flight; evt-2 is settled. + assert "evt-2" not in transport._in_flight + assert "evt-1" in transport._in_flight + + +@respx.mock +def test_a_recorded_overage_counts_as_delivered(transport): + """A metric, because a silently-dropped charged event is invisible otherwise.""" + from nullrun.observability import metrics + + before = getattr(metrics.transport, "events_dead_lettered", 0) + before_recorded = metrics.transport.events_recorded_overage + _post_422(_overbudget_body(recorded=True, event_id="evt-2")) + transport._buffer.extend([_event(1), _event(2), _event(3)]) + + transport._do_flush() + + assert getattr(metrics.transport, "events_dead_lettered", 0) == before + assert metrics.transport.events_recorded_overage == before_recorded + 1 + + +# --------------------------------------------------------------------------- +# recorded: false / absent — the old quarantine path must be untouched. +# --------------------------------------------------------------------------- + + +@respx.mock +def test_an_unrecorded_overage_is_still_dead_lettered(transport): + """Nothing was charged, so parking the data is still the only safe move.""" + _post_seq( + # [1,2,3] → refused + httpx.Response(422, json=_overbudget_body(recorded=False)), + # bisect halves the batch as [1] + [2,3] + _accepted("evt-1"), + httpx.Response(422, json=_overbudget_body(recorded=False)), + # [2,3] → bisected as [2] + [3]; the offender lands in the singleton + httpx.Response(422, json=_overbudget_body(recorded=False)), + _accepted("evt-3"), + ) + transport._buffer.extend([_event(1), _event(2), _event(3)]) + + transport._do_flush() + + rows = _dlq_rows(transport) + assert [r["event"]["event_id"] for r in rows] == ["evt-2"] + assert transport._buffer == [] + + +@respx.mock +def test_a_refusal_with_no_details_at_all_still_dead_letters(transport): + """An older backend, or a proxy page: no marker means no licence to drop.""" + _post_seq( + httpx.Response(422, json={"error_code": "CONSUME_OVERBUDGET", "error_message": "over"}), + _accepted("evt-1"), + httpx.Response(422, json={"error_code": "CONSUME_OVERBUDGET", "error_message": "over"}), + httpx.Response(422, json={"error_code": "CONSUME_OVERBUDGET", "error_message": "over"}), + _accepted("evt-3"), + ) + transport._buffer.extend([_event(1), _event(2), _event(3)]) + + transport._do_flush() + + assert [r["event"]["event_id"] for r in _dlq_rows(transport)] == ["evt-2"] + + +@respx.mock +def test_a_recorded_marker_on_an_unrelated_code_is_ignored(transport): + """The rule is scoped to CONSUME_OVERBUDGET; no other 422 grows a bypass.""" + _post_seq( + httpx.Response( + 422, + json={ + "error_code": "EXECUTION_NOT_BOUND", + "error_message": "no such execution", + "details": {"event_id": "evt-2", "reservation_recorded": True}, + }, + ), + _accepted("evt-1"), + httpx.Response( + 422, + json={ + "error_code": "EXECUTION_NOT_BOUND", + "error_message": "no such execution", + "details": {"event_id": "evt-2", "reservation_recorded": True}, + }, + ), + httpx.Response( + 422, + json={ + "error_code": "EXECUTION_NOT_BOUND", + "error_message": "no such execution", + "details": {"event_id": "evt-2", "reservation_recorded": True}, + }, + ), + _accepted("evt-3"), + ) + transport._buffer.extend([_event(1), _event(2), _event(3)]) + + transport._do_flush() + + assert [r["event"]["event_id"] for r in _dlq_rows(transport)] == ["evt-2"] + + +# --------------------------------------------------------------------------- +# The singleton case: a batch of one, and the bisect fallback. +# --------------------------------------------------------------------------- + + +@respx.mock +def test_a_lone_recorded_overage_is_delivered(transport): + _post_422(_overbudget_body(recorded=True, event_id="evt-1")) + transport._buffer.extend([_event(1)]) + + transport._do_flush() + + assert _dlq_rows(transport) == [] + assert transport._buffer == [] + + +@respx.mock +def test_a_recorded_overage_found_by_bisecting_is_delivered(transport): + """No event_id in the body: the cascade isolates it, and the rule then fires.""" + _post_seq( + # batch of 3 → refused, no event_id to go on + httpx.Response(422, json=_overbudget_body(recorded=True, event_id=None)), + # [evt-1, evt-2] → refused + httpx.Response(422, json=_overbudget_body(recorded=True, event_id=None)), + _accepted("evt-1"), + # [evt-2] → refused in isolation, now named + httpx.Response(422, json=_overbudget_body(recorded=True, event_id="evt-2")), + _accepted("evt-3"), + ) + transport._buffer.extend([_event(1), _event(2), _event(3)]) + + transport._do_flush() + + assert _dlq_rows(transport) == [] + assert transport._buffer == [] + + +# --------------------------------------------------------------------------- +# The single-/track typed error: `recorded` must reach the caller. +# --------------------------------------------------------------------------- + + +def test_the_typed_overage_error_carries_the_recorded_marker(): + from nullrun.transport import _parse_v3_error_envelope_uncategorised + + err = _parse_v3_error_envelope_uncategorised( + httpx.Response( + 422, + json={ + "error_code": "CONSUME_OVERBUDGET", + "error_message": "over", + "details": {"reservation_recorded": True, "execution_id": "exec-1"}, + }, + ), + "/track", + ) + assert isinstance(err, NullRunConsumeOverbudgetError) + assert err.recorded is True + + +def test_the_typed_overage_error_reports_absence_rather_than_guessing(): + """No marker from an older backend is ``None``, never a silent ``False``.""" + from nullrun.transport import _parse_v3_error_envelope_uncategorised + + err = _parse_v3_error_envelope_uncategorised( + httpx.Response( + 422, + json={ + "error_code": "CONSUME_OVERBUDGET", + "error_message": "over", + "details": {}, + }, + ), + "/track", + ) + assert isinstance(err, NullRunConsumeOverbudgetError) + assert err.recorded is None diff --git a/tests/test_route_track_enforcement_propagates.py b/tests/test_route_track_enforcement_propagates.py index a0fda4b..44d7f68 100644 --- a/tests/test_route_track_enforcement_propagates.py +++ b/tests/test_route_track_enforcement_propagates.py @@ -84,15 +84,19 @@ TRACK_URL = f"{BASE_URL}/api/v1/track" +# The backend's real 422 body (`handlers.rs`, ConsumeOverbudget arm): +# the reservation is in MILLICENTS, the actual in cents, and there is no +# `epsilon_cents` key — the configured tolerance is not on this wire. CONSUMED_TOO_MUCH = { "error_code": "CONSUME_OVERBUDGET", "error_message": "actual > reserved + epsilon", "details": { "execution_id": SERVER_MINTED_V1, - "reserved_cents": 100, - "max_allowed_cents": 101, + "reserved_millicents": 1000, + "max_allowed_millicents": 1010, "actual_cost_cents": 150, - "epsilon_cents": 1, + "soft_pass": False, + "reservation_recorded": True, }, } @@ -142,9 +146,10 @@ def test_raised_error_keeps_reconciliation_attributes(self, make_runtime): rt._transport.flush_now() err = ei.value - assert err.reserved_cents == 100 + assert err.reserved_millicents == 1000 + assert err.max_allowed_millicents == 1010 assert err.actual_cost_cents == 150 - assert err.epsilon_cents == 1 + assert err.recorded is True assert err.status_code == 422 @respx.mock diff --git a/tests/test_runtime.py b/tests/test_runtime.py index 887a49f..fec5103 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -242,31 +242,59 @@ def test_execute_blocked_surfaces_wire_error_code(self, make_runtime, mock_api): # so any caller that branched on it pre-fix keeps working. assert wire_details.get("mapped_class") == "NullRunApprovalDbUnavailableError" - @pytest.mark.skip( - reason=( - "runtime.execute now requires " - 'on_transport_error="raise" to surface classified errors ' - "(preserves legacy fail-OPEN behaviour by default so " - "check_workflow_budget can treat network errors as transient). " - "Re-enable when the test passes the opt-in flag." - ) - ) def test_execute_network_error_raises_classified(self, make_runtime, mock_api): - """Network error during execute surfaces as classified NullRunTransportError (ADR-008).""" + """Network error during execute surfaces as classified NullRunTransportError (ADR-008). + + Opted in explicitly. The default is fail-OPEN by design so + `check_workflow_budget` can treat a network blip as transient — but + that default is exactly the thing this test pins, because a silent + flip of it back to fail-CLOSED (or, worse, the other way) would + otherwise be invisible: the test was disabled rather than asserting + the flag, which left the default itself unasserted. + """ from nullrun.breaker.exceptions import ( NullRunTransportError, TransportErrorSource, ) - respx.post(f"{BASE_URL}/api/v1/gate").mock( + respx.post(f"{BASE_URL}/api/v1/execute").mock( side_effect=httpx.ConnectError("connection refused") ) rt = make_runtime() with pytest.raises(NullRunTransportError) as exc_info: - rt.execute(tool_name="gpt-4", input_data={}, mode="strict") + rt.execute( + tool_name="gpt-4", + input_data={}, + mode="strict", + on_transport_error="raise", + ) assert exc_info.value.source == TransportErrorSource.NETWORK_ERROR assert exc_info.value.endpoint == "execute" + def test_execute_network_error_without_the_flag_fails_closed_via_fallback( + self, make_runtime, mock_api + ): + """The default path, asserted. An unreachable backend BLOCKS. + + The counterpart to the opt-in test above, and the reason that test's + flag is worth having: without it, `on_transport_error` is None and a + ConnectError falls through to the fallback_mode path, which under + `mode="strict"` is a block — after exhausting the retry budget. So + the enforcement gate is fail-CLOSED on this class of failure, which is + the property that matters. Writing this down is what stops a future + change from quietly turning the default into a fail-OPEN "allow": + the opt-in test alone would still pass in that world. + """ + from nullrun.breaker.exceptions import NullRunBlockedException + + respx.post(f"{BASE_URL}/api/v1/execute").mock( + side_effect=httpx.ConnectError("connection refused") + ) + rt = make_runtime() + with pytest.raises(NullRunBlockedException) as exc_info: + rt.execute(tool_name="gpt-4", input_data={}, mode="strict") + assert "Gateway unavailable" in str(exc_info.value) + # T3-S2 (0.3.0): `test_execute_local_mode_allows` was removed along # with the `local_mode` field. The execute path now always hits # the /execute endpoint — there is no local stub to test. diff --git a/tests/test_track_batch_retry.py b/tests/test_track_batch_retry.py index 4330c3f..6552207 100644 --- a/tests/test_track_batch_retry.py +++ b/tests/test_track_batch_retry.py @@ -22,12 +22,15 @@ from __future__ import annotations +import threading + import httpx import pytest import respx from nullrun.breaker.exceptions import BreakerTransportError -from nullrun.transport import Transport +from nullrun.observability import metrics +from nullrun.transport import Transport, _retry_with_backoff @pytest.fixture @@ -103,3 +106,358 @@ def test_2xx_first_try_no_retry(self, transport): result = transport._send_batch_with_retry_info([{"event": "e1"}]) assert route.call_count == 1 assert "e1" in result.accepted_event_ids + + +# ────────────────────────────────────────────────────────────── +# Retry-After: the delay, not the call count +# ────────────────────────────────────────────────────────────── +# +# The test above says "429 with Retry-After — helper honors the header +# before the next attempt (we assert call count, not exact delay)". Asserting +# the call count is what let a dead parameter survive: `last_retry_after_seconds` +# was documented, threaded through the signature, and never passed by any +# caller, and the suite stayed green because the number of attempts does not +# depend on how long you wait between them. These assert the delay. + + +def test_a_429_is_not_retried_before_the_server_said_it_may_be(monkeypatch): + """The floor is the server's, and the SDK must not undercut it. + + Pre-fix, a 429 fell through to plain exponential backoff: base_delay + 0.5s, so the retry landed well before the `Retry-After: 5` the server + asked for, re-tripping the rate limit that had just answered. The retry + then repeated at 1s, 2s, 4s — all inside the window the server had + declared closed. + """ + delays: list[float] = [] + monkeypatch.setattr( + "nullrun.transport.time.sleep", lambda s: delays.append(s) + ) + + calls = {"n": 0} + + def _always_429() -> httpx.Response: + # `raise_for_status` is what the real `_post_batch` does for a 429; + # without it the loop sees a plain response and returns it, so no + # retry (and no delay) happens at all. + calls["n"] += 1 + resp = httpx.Response( + 200 if calls["n"] >= 2 else 429, + **({} if calls["n"] >= 2 else {"headers": {"Retry-After": "5"}}), + json={}, + ) + resp.request = httpx.Request("POST", "https://api.test.nullrun.io/api/v1/track/batch") + resp.raise_for_status() + return resp + + _retry_with_backoff(_always_429, max_retries=3, base_delay=0.5, max_delay=10.0) + + assert delays, "the retry happened without sleeping" + assert min(delays) >= 5.0, ( + f"a retry went out before the server's floor: {delays}" + ) + + +def test_retry_after_jitter_never_reduces_the_wait_but_does_spread_it(monkeypatch): + """Jitter is one-sided, and it is present. + + Two properties in one test because they are the same decision: a fleet + that got limited together must not retry together, but no member of it + may retry before the limit lifts. A symmetric jitter would satisfy the + first and violate the second. + """ + samples: list[float] = [] + monkeypatch.setattr( + "nullrun.transport.time.sleep", lambda s: samples.append(s) + ) + + for _ in range(12): + calls = {"n": 0} + + def _always_429() -> httpx.Response: + calls["n"] += 1 + resp = httpx.Response( + 200 if calls["n"] >= 2 else 429, + **({} if calls["n"] >= 2 else {"headers": {"Retry-After": "5"}}), + json={}, + ) + resp.request = httpx.Request( + "POST", "https://api.test.nullrun.io/api/v1/track/batch" + ) + resp.raise_for_status() + return resp + + _retry_with_backoff(_always_429, max_retries=1, base_delay=0.5, max_delay=10.0) + + assert all(s >= 5.0 for s in samples), f"jitter went under the floor: {samples}" + assert len(set(samples)) > 1, f"every client retried at the same instant: {samples}" + + +def test_a_429_without_the_header_falls_back_to_exponential_backoff(monkeypatch): + """No header is not an instruction to wait forever. + + A server that rate-limits without stating a wait should not be able to + pin the SDK to a five-second floor per attempt; the backoff path is the + right answer there, and it is jittered symmetrically because no floor + applies. + """ + delays: list[float] = [] + monkeypatch.setattr( + "nullrun.transport.time.sleep", lambda s: delays.append(s) + ) + calls = {"n": 0} + + def _always_429() -> httpx.Response: + calls["n"] += 1 + resp = httpx.Response(200 if calls["n"] >= 2 else 429, json={}) + resp.request = httpx.Request( + "POST", "https://api.test.nullrun.io/api/v1/track/batch" + ) + resp.raise_for_status() + return resp + + _retry_with_backoff(_always_429, max_retries=1, base_delay=0.5, max_delay=10.0) + assert delays and delays[0] < 5.0, f"a headerless 429 was treated as a floor: {delays}" + + +def test_retry_after_as_an_http_date_is_honoured(monkeypatch): + """The other RFC 7231 form. nginx and hand-rolled limiters differ here.""" + from datetime import datetime, timedelta, timezone + from email.utils import format_datetime + + delays: list[float] = [] + monkeypatch.setattr( + "nullrun.transport.time.sleep", lambda s: delays.append(s) + ) + when = format_datetime( + datetime.now(timezone.utc) + timedelta(seconds=4), usegmt=True + ) + calls = {"n": 0} + + def _always_429() -> httpx.Response: + calls["n"] += 1 + resp = httpx.Response( + 200 if calls["n"] >= 2 else 429, + **({} if calls["n"] >= 2 else {"headers": {"Retry-After": when}}), + json={}, + ) + resp.request = httpx.Request( + "POST", "https://api.test.nullrun.io/api/v1/track/batch" + ) + resp.raise_for_status() + return resp + + _retry_with_backoff(_always_429, max_retries=1, base_delay=0.5, max_delay=10.0) + assert delays and delays[0] >= 3.0, f"the HTTP-date form was not honored: {delays}" + + +def test_a_retry_after_already_in_the_past_is_not_a_floor(monkeypatch): + """A stale date means "no floor stated", not "wait negative seconds".""" + delays: list[float] = [] + monkeypatch.setattr( + "nullrun.transport.time.sleep", lambda s: delays.append(s) + ) + calls = {"n": 0} + + def _always_429() -> httpx.Response: + calls["n"] += 1 + resp = httpx.Response( + 200 if calls["n"] >= 2 else 429, + **( + {} + if calls["n"] >= 2 + else {"headers": {"Retry-After": "Wed, 21 Oct 2015 07:28:00 GMT"}} + ), + json={}, + ) + resp.request = httpx.Request( + "POST", "https://api.test.nullrun.io/api/v1/track/batch" + ) + resp.raise_for_status() + return resp + + _retry_with_backoff(_always_429, max_retries=1, base_delay=0.5, max_delay=10.0) + assert delays and delays[0] > 0, f"a past date produced a non-positive wait: {delays}" + + +# ────────────────────────────────────────────────────────────── +# The ceiling: an 86400 must not hang a flush, and must not +# be undercut either +# ────────────────────────────────────────────────────────────── +# +# The previous code clamped with `min(server_wait, max_delay)`. That +# bounds the wait — and it retries an hour early, which is exactly +# what the floor tests above forbid at a smaller scale. A cap that +# makes the SDK ignore a long Retry-After is not a safety feature; it +# is the sustained-429 bug with a different number. + + +def _reply(status: int, headers: dict | None = None) -> httpx.Response: + def _handler() -> httpx.Response: + resp = httpx.Response( + status, headers=headers or {}, json={"error": "rate_limited"} + ) + resp.request = httpx.Request( + "POST", "https://api.test.nullrun.io/api/v1/track/batch" + ) + resp.raise_for_status() + return resp + + return _handler + + +def test_a_retry_after_of_a_day_does_not_sit_through_the_day(monkeypatch): + """`Retry-After: 86400` must not park a flush thread for 24 hours.""" + slept: list[float] = [] + monkeypatch.setattr("nullrun.transport.time.sleep", lambda s: slept.append(s)) + monkeypatch.setattr("nullrun.transport.threading.Event.wait", lambda self, t=None: False) + + calls = {"n": 0} + + def _always_429() -> httpx.Response: + calls["n"] += 1 + resp = httpx.Response(429, headers={"Retry-After": "86400"}, json={}) + resp.request = httpx.Request( + "POST", "https://api.test.nullrun.io/api/v1/track/batch" + ) + resp.raise_for_status() + return resp + + with pytest.raises(BreakerTransportError): + _retry_with_backoff(_always_429, max_retries=10, base_delay=0.5, max_delay=30.0) + + assert not slept, f"a 24h Retry-After was waited out: {slept}" + assert calls["n"] == 1, ( + f"the SDK retried {calls['n']} time(s) inside a window the server " + f"declared closed for a day" + ) + + +def test_an_over_ceiling_retry_after_stops_the_cycle_instead_of_retrying_early( + monkeypatch, +): + """The distinction that matters: no wait, no retry, no data loss. + + Refusing to retry is safe here only because the events are already + durable. Asserting the call count alone would pass against a version + that dropped them, so the exhaustion path is checked too. + """ + # The metrics registry is process-global and this file has no reset + # fixture, so counters are asserted as DELTAS. An absolute assert here + # would fail on suite ordering and hide a real regression behind a + # bookkeeping error. + before = metrics.transport.retry_after_deferred + calls = {"n": 0} + + def _always_429() -> httpx.Response: + calls["n"] += 1 + resp = httpx.Response(429, headers={"Retry-After": "3600"}, json={}) + resp.request = httpx.Request( + "POST", "https://api.test.nullrun.io/api/v1/track/batch" + ) + resp.raise_for_status() + return resp + + with pytest.raises(BreakerTransportError): + _retry_with_backoff( + _always_429, max_retries=10, base_delay=0.5, max_delay=30.0 + ) + + assert calls["n"] == 1 + assert metrics.transport.retry_after_deferred == before + 1 + + +def test_a_retry_after_under_the_ceiling_is_still_honoured(monkeypatch): + """The ceiling is a ceiling, not a replacement for the floor. + + Capping everything at 30s would satisfy the previous test too — and + would make every 429 immediate, which is the bug this whole file + exists to pin. A 5s floor must still produce a >=5s wait. + """ + delays: list[float] = [] + monkeypatch.setattr("nullrun.transport.time.sleep", lambda s: delays.append(s)) + calls = {"n": 0} + + def _then_ok() -> httpx.Response: + calls["n"] += 1 + resp = httpx.Response( + 200 if calls["n"] >= 2 else 429, + **({} if calls["n"] >= 2 else {"headers": {"Retry-After": "5"}}), + json={}, + ) + resp.request = httpx.Request( + "POST", "https://api.test.nullrun.io/api/v1/track/batch" + ) + resp.raise_for_status() + return resp + + _retry_with_backoff(_then_ok, max_retries=2, base_delay=0.5, max_delay=30.0) + assert delays and min(delays) >= 5.0, f"a sub-ceiling floor was cut: {delays}" + + +def test_the_ceiling_is_configurable(monkeypatch): + """An operator whose backend rate-limits for 5 minutes needs to say so.""" + slept: list[float] = [] + monkeypatch.setattr("nullrun.transport.time.sleep", lambda s: slept.append(s)) + monkeypatch.setenv("NULLRUN_RETRY_AFTER_CEILING", "600") + before = metrics.transport.retry_after_deferred + calls = {"n": 0} + + def _always_429() -> httpx.Response: + calls["n"] += 1 + resp = httpx.Response(429, headers={"Retry-After": "300"}, json={}) + resp.request = httpx.Request( + "POST", "https://api.test.nullrun.io/api/v1/track/batch" + ) + resp.raise_for_status() + return resp + + with pytest.raises(BreakerTransportError): + _retry_with_backoff(_always_429, max_retries=2, base_delay=0.5, max_delay=30.0) + + assert metrics.transport.retry_after_deferred == before, ( + "a raised ceiling did not let a 300s floor through" + ) + assert slept, "the 300s floor produced no wait at all" + + +def test_a_retry_wait_is_cut_short_by_shutdown(monkeypatch): + """`Transport.stop()` must not wait out a `Retry-After`. + + A process that is exiting and is told to stop should stop. The wait it + is in is the one place it cannot currently choose to leave, because + `time.sleep` is not wakeable — which is why the wait became an + `Event.wait`. The SDK's own stop event is the one a shutdown sets. + """ + cancel = threading.Event() + real_wait = threading.Event.wait + interrupted_before = metrics.transport.retries_interrupted_by_shutdown + + def _wait_then_cancel(self, timeout=None): + # A shutdown arriving 20ms into a 30s wait. + cancel.set() + return real_wait(self, 0.02) + + monkeypatch.setattr("nullrun.transport.threading.Event.wait", _wait_then_cancel) + monkeypatch.setattr("nullrun.transport.time.sleep", lambda s: pytest.fail("slept")) + + calls = {"n": 0} + + def _always_429() -> httpx.Response: + calls["n"] += 1 + resp = httpx.Response(429, headers={"Retry-After": "30"}, json={}) + resp.request = httpx.Request( + "POST", "https://api.test.nullrun.io/api/v1/track/batch" + ) + resp.raise_for_status() + return resp + + with pytest.raises(BreakerTransportError): + _retry_with_backoff( + _always_429, max_retries=10, base_delay=0.5, max_delay=30.0, cancel=cancel + ) + + assert calls["n"] == 1, ( + f"shutdown did not stop the retry loop; it made {calls['n']} attempts" + ) + assert metrics.transport.retries_interrupted_by_shutdown == interrupted_before + 1 diff --git a/tests/test_v3_wire_contract.py b/tests/test_v3_wire_contract.py index c420239..93be48b 100644 --- a/tests/test_v3_wire_contract.py +++ b/tests/test_v3_wire_contract.py @@ -151,12 +151,15 @@ def test_check_v3_includes_protocol_header(self): def test_track_single_includes_protocol_header(self): # 2026-07-04 (B2): body shape matches the v3 wire # contract — ``reservation_id`` (server-minted from /check) - # ``workflow_id`` + ``tokens`` + ``cost_cents`` (the SDK - # always emits 0 — backend recomputes from tokens) + + # ``workflow_id`` + ``tokens`` + # ``cost_source: "provisional"``. Pre-fix this test sent the # legacy / fictitious shape # ``{execution_id, actual_cost_cents}`` which doesn't match # ``TrackRequestRaw`` and would 422 on the wire. + # ADR-068 §4b (2026-10-09): the SDK used to emit + # ``cost_cents: 0`` here. It is stripped on the way out + # (``_WIRE_STRIP_FIELDS``), so passing it below is still fine — + # it exercises that an older caller's price does not survive. t = Transport(api_url=BASE_URL, api_key="nr_live_abc123") try: route = respx.post(f"{BASE_URL}/api/v1/track").mock( @@ -476,25 +479,29 @@ def test_workflow_inactive_maps_to_workflow_inactive_error(self): assert exc.workflow_id == "wf-1" def test_consume_overbudget_maps_to_consume_overbudget_error(self): + # The backend's real details block: reservation in MILLICENTS, + # actual in cents, no `epsilon_cents` key. See handlers.rs, the + # ConsumeOverbudget 422. resp = self._make_response( 422, { "error_code": "CONSUME_OVERBUDGET", "error_message": "actual > reserved + epsilon", "details": { - "reserved_cents": 100, - "max_allowed_cents": 101, + "reserved_millicents": 1000, + "max_allowed_millicents": 1010, "actual_cost_cents": 150, - "epsilon_cents": 1, + "soft_pass": False, + "reservation_recorded": True, }, }, ) exc = _parse_v3_error_envelope(resp, "track") assert isinstance(exc, NullRunConsumeOverbudgetError) - assert exc.reserved_cents == 100 - assert exc.max_allowed_cents == 101 + assert exc.reserved_millicents == 1000 + assert exc.max_allowed_millicents == 1010 assert exc.actual_cost_cents == 150 - assert exc.epsilon_cents == 1 + assert exc.recorded is True def test_budget_recheck_failed_maps_to_typed_error(self): #: AR-H6 (2026-08-12) — post-approval re-check failure must @@ -1882,12 +1889,20 @@ def test_full_event_builds_full_payload(self): "trace_id": "trace-1", "span_id": "span-1", "agent_id": "agent-1", - "cost_cents": 0, "cost_source": "provisional", } + # ADR-068 §4b: a full event is no exception — a populated + # payload must still carry no price. See the minimal-event test + # for why the exact-dict match is the assertion that matters. + assert "cost_cents" not in out def test_minimal_event_only_required_fields(self): # workflow_id + tokens + reservation_id are the floor. + # + # ADR-068 §4b: no `cost_cents`. The SDK reports tokens and the + # backend prices them. Asserted by ABSENCE below — an exact-dict + # match is what makes "the SDK never offers a price" a checked + # property rather than a comment. out = _build_v3_track_payload( {"type": "llm_call", "workflow_id": "wf-1", "tokens": 1}, SERVER_MINTED_V1, @@ -1896,9 +1911,12 @@ def test_minimal_event_only_required_fields(self): "reservation_id": SERVER_MINTED_V1, "workflow_id": "wf-1", "tokens": 1, - "cost_cents": 0, "cost_source": "provisional", } + assert "cost_cents" not in out, ( + "ADR-068 §4b: the SDK must not put a price on the wire. It " + "sends tokens; the backend prices them from its own catalog." + ) def test_missing_workflow_id_returns_none(self): # v0.16.0 (backend v3.66.2 alignment): caller now DROPS @@ -1932,6 +1950,79 @@ def test_tokens_coerced_to_int(self): assert out["tokens"] == 100 assert isinstance(out["tokens"], int) + # ── ADR-068 §1 (backend NULLRUN): `provider` must reach the wire. + # + # The backend picks the cache-token convention off `provider`. + # Anthropic reports the cached fraction OUTSIDE `input_tokens`; + # OpenAI / Gemini / Mistral report it INSIDE `prompt_tokens`. If + # the field is missing, the backend sees an unmapped provider and + # applies the fail-CLOSED no-discount path — the full input rate, + # which is what we billed before ADR-068 landed. So dropping this + # key silently restores the overcharge for every cache-using call. + # + # The batch path already forwarded it (it ships the whole enriched + # event); this is the single-event path, which whitelists keys. + + def test_provider_is_forwarded_when_present(self): + out = _build_v3_track_payload( + { + "type": "llm_call", + "workflow_id": "wf-1", + "tokens": 100, + "input_tokens": 60, + "output_tokens": 40, + "model": "claude-sonnet-4-6", + "provider": "anthropic", + }, + SERVER_MINTED_V1, + ) + assert out["provider"] == "anthropic" + + def test_provider_absent_stays_absent(self): + # No fabricated key: the backend distinguishes "provider we + # could not determine" (fail-CLOSED, counted) from a provider + # we named. Emitting `None` would make both look the same. + out = _build_v3_track_payload( + {"type": "llm_call", "workflow_id": "wf-1", "tokens": 1}, + SERVER_MINTED_V1, + ) + assert "provider" not in out + + def test_empty_provider_is_not_sent(self): + # `""` is the shape `_provider_label` returns for a host it + # cannot classify down to a real name; forwarding it would + # look like a known-but-unmapped provider to the backend. + out = _build_v3_track_payload( + { + "type": "llm_call", + "workflow_id": "wf-1", + "tokens": 1, + "provider": "", + }, + SERVER_MINTED_V1, + ) + assert "provider" not in out + + def test_cache_token_fields_survive_to_the_wire(self): + # The companion half: the backend prices the cached fraction + # and persists it (ADR-068 §D2). Both fields have to be here + # for either to mean anything. + out = _build_v3_track_payload( + { + "type": "llm_call", + "workflow_id": "wf-1", + "tokens": 9100, + "input_tokens": 100, + "output_tokens": 10, + "provider": "anthropic", + "cache_read_tokens": 9000, + "cache_write_tokens": 0, + }, + SERVER_MINTED_V1, + ) + assert out["cache_read_tokens"] == 9000 + assert out["cache_write_tokens"] == 0 + # ───────────────────────────────────────────────────────────────── # 5. _route_track: routes llm_call → /track, others → /track/batch diff --git a/tests/test_wal_durability.py b/tests/test_wal_durability.py new file mode 100644 index 0000000..8a660eb --- /dev/null +++ b/tests/test_wal_durability.py @@ -0,0 +1,1203 @@ +"""Durability tests for the Transport WAL. + +Why this exists. ``Transport._replay_from_wal`` unlinked each recovery file +*before* calling ``_do_flush``: + + for candidate in (f"{wal}.1", wal): + ...read... + os.remove(candidate) # <-- deleted here + if events: + self._buffer.extend(events) + self._do_flush() # <-- flushed here + +The docstring claimed "Both files are removed only after a successful +flush"; the code did the exact opposite. A crash between the unlink and a +successful flush lost the batch outright: the events were in neither the +WAL nor the buffer, and the server never saw them. Since the recovery path +only runs when the backend was unreachable at shutdown, the breaker being +still open at ``start()`` is the *likely* case, not the exotic one. + +The fix makes recovery at-least-once: no file is unlinked before its events +are provably safe (accepted, or rewritten into ``.wal``). Re-delivery is +harmless because the backend dedups on ``event_id``, which is why every +event is guaranteed an id before it reaches disk. +""" + +from __future__ import annotations + +import json +import os +from unittest.mock import patch + +import httpx +import pytest + +from nullrun.breaker.exceptions import BreakerTransportError +from nullrun.transport import Transport + +SendResult = Transport.SendResult # nested dataclass, not a module-level name + + +@pytest.fixture +def transport(tmp_path, monkeypatch): + monkeypatch.setenv("NULLRUN_WAL_PATH", str(tmp_path / "sdk.wal")) + t = Transport(api_url="https://api.test.nullrun.io", api_key="test-key-12345678") + yield t + t._client.close() + + +def _write_wal(path: str, events: list[dict]) -> None: + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + with open(path, "w") as f: + for e in events: + f.write(json.dumps(e) + "\n") + + +def _read_wal(path: str) -> list[dict]: + if not os.path.exists(path): + return [] + with open(path) as f: + return [json.loads(line) for line in f if line.strip()] + + +def _event(i: int) -> dict: + return {"event_id": f"evt-{i}", "type": "llm_call", "cost_cents": 1} + + +# --------------------------------------------------------------------------- +# The core regression: the WAL must survive a flush that does not land. +# --------------------------------------------------------------------------- + + +def test_wal_survives_failed_flush(transport): + """A flush that raises must NOT consume the recovery file. + + This is the exact loss the old code shipped: unlink ran before the + flush, so a BreakerTransportError on the recovery path destroyed the + only durable copy. + """ + wal = transport._wal_path() + _write_wal(wal, [_event(1), _event(2)]) + + with patch.object( + transport, "_do_flush", side_effect=BreakerTransportError("backend down") + ): + transport._replay_from_wal() + + assert os.path.exists(wal), "WAL deleted despite the flush not landing" + assert len(_read_wal(wal)) == 2 + + +def test_replay_retries_unflushed_events_next_start(transport): + """Events kept on disk by a failed replay are delivered on the next start.""" + wal = transport._wal_path() + _write_wal(wal, [_event(1), _event(2)]) + + with patch.object( + transport, "_do_flush", side_effect=BreakerTransportError("backend down") + ): + transport._replay_from_wal() + assert os.path.exists(wal) + + # Second attempt, backend healthy: the batch must go out and the file go away. + sent: list[dict] = [] + + def _ok(): + sent.extend(transport._buffer[:]) + transport._buffer.clear() + + with patch.object(transport, "_do_flush", side_effect=_ok): + transport._replay_from_wal() + + assert {e["event_id"] for e in sent} == {"evt-1", "evt-2"} + assert not os.path.exists(wal), "WAL not cleaned up after a successful replay" + + +def test_wal_removed_after_successful_flush(transport): + """The happy path still cleans up — no unbounded replay loop.""" + wal = transport._wal_path() + _write_wal(wal, [_event(1)]) + + def _deliver(): + transport._buffer.clear() + + with patch.object(transport, "_do_flush", side_effect=_deliver): + transport._replay_from_wal() + + assert not os.path.exists(wal) + + +# --------------------------------------------------------------------------- +# All three recovery sources, in age order. +# --------------------------------------------------------------------------- + + +def test_replay_drains_rotated_inflight_and_active(transport): + """`.wal.1`, `.wal` and `.inflight` all feed the buffer, oldest first.""" + wal = transport._wal_path() + _write_wal(f"{wal}.1", [_event(1)]) + _write_wal(wal, [_event(2)]) + _write_wal(f"{wal}.inflight", [_event(3)]) + + seen: list[str] = [] + with patch.object( + transport, + "_do_flush", + side_effect=lambda: seen.extend(e["event_id"] for e in transport._buffer[:]) + or transport._buffer.clear(), + ): + transport._replay_from_wal() + + assert seen == ["evt-1", "evt-2", "evt-3"] + for suffix in (".1", "", ".inflight"): + assert not os.path.exists(f"{wal}{suffix}"), f"{suffix} left behind" + + +def test_inflight_replayed_after_crash_during_send(transport): + """A batch on the wire when the process died is recovered from `.inflight`. + + This is the case that motivated adding the file: the batch had already + left `_buffer` and was mid-HTTP, so neither the buffer nor the periodic + WAL held it. + """ + wal = transport._wal_path() + _write_wal(f"{wal}.inflight", [_event(7), _event(8)]) + + seen: list[str] = [] + with patch.object( + transport, + "_do_flush", + side_effect=lambda: seen.extend(e["event_id"] for e in transport._buffer[:]) + or transport._buffer.clear(), + ): + transport._replay_from_wal() + + assert seen == ["evt-7", "evt-8"] + + +def test_empty_inflight_is_cleared(transport): + """A zero-byte `.inflight` is dropped, not replayed forever.""" + inflight = f"{transport._wal_path()}.inflight" + os.makedirs(os.path.dirname(inflight) or ".", exist_ok=True) + open(inflight, "w").close() + + with patch.object(transport, "_do_flush") as flush: + transport._replay_from_wal() + + assert not os.path.exists(inflight) + flush.assert_not_called() + + +def test_corrupt_lines_skipped_without_losing_valid_events(transport): + """One unparseable line must not discard the rest of the batch.""" + wal = transport._wal_path() + with open(wal, "w") as f: + f.write(json.dumps(_event(1)) + "\n") + f.write("{not json\n") + f.write(json.dumps(_event(2)) + "\n") + + seen: list[str] = [] + with patch.object( + transport, + "_do_flush", + side_effect=lambda: seen.extend(e["event_id"] for e in transport._buffer[:]) + or transport._buffer.clear(), + ): + transport._replay_from_wal() + + assert seen == ["evt-1", "evt-2"] + + +# --------------------------------------------------------------------------- +# event_id is what makes at-least-once safe; pin the guarantee. +# --------------------------------------------------------------------------- + + +def test_persist_assigns_event_id_before_writing(transport): + """Every event on disk carries an id, so re-delivery dedups server-side.""" + transport.track({"type": "llm_call", "cost_cents": 5}) + transport._persist_to_wal() + + rows = _read_wal(transport._wal_path()) + assert len(rows) == 1 + assert rows[0]["event_id"], "event reached the WAL without an event_id" + + +def test_inflight_carries_event_id(transport): + """The in-flight file is a recovery source, so it needs ids too.""" + transport.track({"type": "llm_call", "cost_cents": 5}) + transport._persist_inflight(transport._buffer[:]) + + rows = _read_wal(f"{transport._wal_path()}.inflight") + assert len(rows) == 1 + assert rows[0]["event_id"] + + +# --------------------------------------------------------------------------- +# Inflight lifecycle around a real send. +# --------------------------------------------------------------------------- + + +def test_inflight_cleared_after_accepted_send(transport): + """Accepted batches must not linger and be replayed a second time.""" + transport.track(_event(1)) + transport._send_batch_with_retry_info = lambda batch: SendResult( + accepted_event_ids=[e["event_id"] for e in batch] + ) + transport._do_flush() + + assert not os.path.exists(f"{transport._wal_path()}.inflight") + + +def test_inflight_retained_when_send_fails(transport): + """A failed send leaves the batch on disk — it is only in memory after.""" + transport.track(_event(1)) + with patch.object(transport._circuit_breaker, "call", side_effect=BreakerTransportError("x")): + transport._do_flush() + + rows = _read_wal(f"{transport._wal_path()}.inflight") + assert [r["event_id"] for r in rows] == ["evt-1"] + # and the batch is back in the buffer for retry + assert [e["event_id"] for e in transport._buffer] == ["evt-1"] + + +def test_later_successful_flush_clears_stale_inflight(transport): + """A retained `.inflight` self-heals once a send lands.""" + inflight = f"{transport._wal_path()}.inflight" + _write_wal(inflight, [_event(1)]) + + transport.track(_event(2)) + transport._send_batch_with_retry_info = lambda batch: SendResult( + accepted_event_ids=[e["event_id"] for e in batch] + ) + transport._do_flush() + + assert not os.path.exists(inflight) + + +# --------------------------------------------------------------------------- +# Atomic write helper. +# --------------------------------------------------------------------------- + + +def test_write_events_atomic_leaves_no_tmp(transport): + """The tmp file is renamed away or unlinked — never left as litter.""" + target = transport._wal_path() + assert transport._write_events_atomic(target, [_event(1)]) is True + + siblings = [p for p in os.listdir(os.path.dirname(target)) if ".tmp." in p] + assert siblings == [], f"temp files left behind: {siblings}" + assert len(_read_wal(target)) == 1 + + +def test_write_events_atomic_reports_failure(transport, monkeypatch): + """A failed write is reported so callers keep their recovery files.""" + with patch("nullrun.transport.os.replace", side_effect=OSError("disk full")): + assert transport._write_events_atomic(transport._wal_path(), [_event(1)]) is False + + +def test_truncated_last_line_does_not_lose_earlier_events(transport): + """A crash mid-write leaves a partial last line; the rest must survive. + + This is what the tmp+rename dance buys: a reader sees either the whole + file or a prefix, never a corrupted middle. The partial trailing line is + dropped — it was never durable — but everything before it replays. + """ + wal = transport._wal_path() + with open(wal, "w") as f: + f.write(json.dumps(_event(1)) + "\n") + f.write(json.dumps(_event(2)) + "\n") + f.write('{"event_id": "evt-3", "type": "ll') # truncated by the crash + + seen: list[str] = [] + with patch.object( + transport, + "_do_flush", + side_effect=lambda: seen.extend(e["event_id"] for e in transport._buffer[:]) + or transport._buffer.clear(), + ): + transport._replay_from_wal() + + assert seen == ["evt-1", "evt-2"], "a torn trailing line cost us earlier events" + + +# --------------------------------------------------------------------------- +# Partial failure INSIDE a 200 response. +# +# /track/batch does not answer 4xx when one event is the problem. It answers +# 200 with `accepted_event_ids` naming what landed and `rejection_details` +# naming what did not (backend handlers.rs, `BatchTrackResponse`): +# +# per_key_policy_limit -> check_key_policy (retry hint: rate window rolls) +# budget_exceeded -> v3 BudgetExceeded (period counter, does not roll soon) +# reservation_not_found -> v3 ReservationNotFound (reservation TTL expired; terminal) +# +# So a 200 is NOT proof of delivery, and status-code classification never sees +# any of it. The pre-fix code read the status, treated the whole batch as +# delivered, cleared `.inflight` and dropped the refused events — out of the +# buffer and out of the DLQ, with no trace anywhere. "Data is not lost" held +# only for transport failures, which is a much weaker claim than it sounds. +# +# These tests drive the real HTTP path via `httpx.MockTransport` (see the +# note on the 5xx block below for why mocking the outer method would test a +# shape the real path cannot produce). +# --------------------------------------------------------------------------- + + +def _http_200(body: dict): + """An httpx handler that answers 200 with a JSON body.""" + + def _handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=body) + + return _handler + + +def _detail(event_id: str, reason: str, retry_after_ms: int | None = None) -> dict: + detail = {"event_id": event_id, "reason": reason} + if retry_after_ms is not None: + detail["retry_after_ms"] = retry_after_ms + return detail + + +def _flush_over(transport, handler, times: int = 1) -> None: + """Point the transport at `handler` and run `_do_flush` `times` times.""" + transport._client = httpx.Client(transport=httpx.MockTransport(handler)) + try: + for _ in range(times): + transport._do_flush() + finally: + transport._client.close() + + +def test_only_accepted_event_ids_count_as_delivered(transport): + """Absence from `accepted_event_ids` means NOT delivered, whatever the status. + + The partition reading is the only one an SDK can act on: a server that + accepted some events and refused others reports exactly which, and + treating the batch as a unit turns one refusal into a silent loss of every + other event in it. + """ + transport.track(_event(1)) + transport.track(_event(2)) + transport.track(_event(3)) + _flush_over( + transport, + _http_200( + { + "processed": 1, + "accepted_event_ids": ["evt-1", "evt-3"], + "rejection_details": [_detail("evt-2", "per_key_policy_limit", 1000)], + "rejected_count": 1, + } + ), + ) + + assert [e["event_id"] for e in transport._buffer] == ["evt-2"], ( + "the refused event must be the only one still live" + ) + assert not os.path.exists(transport._wal_dlq_path()), ( + "a refusal the backend gave a retry hint for was quarantined instead of retried" + ) + assert "evt-1" not in [e["event_id"] for e in transport._buffer] + assert "evt-3" not in [e["event_id"] for e in transport._buffer] + + +def test_a_200_without_a_parseable_body_is_not_confirmation(transport): + """A 200 whose body cannot be read is not evidence that anything landed. + + A reverse proxy answering 200 with an HTML page, an empty body from a + truncated response, or a backend predating `accepted_event_ids` all look + identical to the client: a success status. Reading that as delivery is + the loss this whole block exists to prevent, and it is the failure mode + that is hardest to notice — the events are simply gone and the logs say + everything worked. + """ + transport.track(_event(1)) + + def _html(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, text="200 OK") + + _flush_over(transport, _html) + + assert [e["event_id"] for e in transport._buffer] == ["evt-1"], ( + "a 200 we could not read was treated as confirmation and the event dropped" + ) + assert not os.path.exists(transport._wal_dlq_path()), ( + "an unreadable 200 must spend its attempt budget first, not be parked at once" + ) + + +def test_a_200_without_a_parseable_body_eventually_reaches_the_dlq(transport): + """The bounded-retry answer for an unreadable 200 is the DLQ, not a loop. + + Retrying forever would pin the buffer head against a backend that is + answering perfectly and whose body we simply cannot read. Dropping at once + would lose the event. A bounded budget with a DLQ at the end is the only + option that does neither. + """ + transport.track(_event(1)) + + def _html(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, text="") + + _flush_over(transport, _html, times=transport._max_batch_attempts + 2) + + rows = _read_wal(transport._wal_dlq_path()) + assert [r["event"]["event_id"] for r in rows] == ["evt-1"], ( + "an unreadable 200 neither loops nor lands anywhere recoverable" + ) + assert "unconfirmed" in rows[0]["reason"], ( + f"the DLQ row must say the response was unreadable, got {rows[0]['reason']!r}" + ) + + +def test_a_terminal_refusal_is_parked_on_the_first_attempt(transport): + """`reservation_not_found` cannot be fixed by waiting — park it now. + + Reservations are minted by /api/v1/gate and expire on a TTL. Nothing the + client can send re-mints one for a past event; re-issuing /gate would mint + a NEW execution and double-count against the budget. Retrying is not slow + here, it is pointless, and it holds the head of the buffer for a budget + number of cycles while a valid batch behind it waits. + """ + transport.track(_event(1)) + _flush_over( + transport, + _http_200( + { + "processed": 0, + "accepted_event_ids": [], + "rejection_details": [_detail("evt-1", "reservation_not_found")], + "rejected_count": 1, + } + ), + ) + + rows = _read_wal(transport._wal_dlq_path()) + assert [r["event"]["event_id"] for r in rows] == ["evt-1"], ( + "a terminal refusal was retried instead of parked" + ) + assert transport._buffer == [], "a parked event must leave the retry path" + assert "reservation_not_found" in rows[0]["reason"] + + +def test_an_unknown_refusal_reason_is_retried_then_parked_never_dropped(transport): + """A reason we have no rule for gets the attempt budget, then the DLQ. + + Dropping it would be invisible: the operator sees a shorter ledger and no + error anywhere. Parking it immediately would make this SDK build the only + one that discards events the backend has started emitting. A bounded + delay is the answer that is wrong in neither direction. + """ + transport.track(_event(1)) + body = { + "processed": 0, + "accepted_event_ids": [], + "rejection_details": [_detail("evt-1", "some_reason_invented_next_year")], + "rejected_count": 1, + } + _flush_over(transport, _http_200(body)) + + assert not os.path.exists(transport._wal_dlq_path()), ( + "an unknown reason was parked on the first response, before any retry" + ) + assert [e["event_id"] for e in transport._buffer] == ["evt-1"] + + _flush_over(transport, _http_200(body), times=transport._max_batch_attempts + 2) + + rows = _read_wal(transport._wal_dlq_path()) + assert [r["event"]["event_id"] for r in rows] == ["evt-1"] + assert "some_reason_invented_next_year" in rows[0]["reason"], ( + "the DLQ must preserve the backend's reason verbatim, not a paraphrase" + ) + + +def test_a_partial_refusal_keeps_inflight_until_every_event_is_settled(transport): + """`.inflight` may only be cleared once nothing from the batch is still live. + + `.inflight` IS the durable copy of a re-queued event — the re-queued event + is in memory only until the next `_persist_to_wal`. Clear the file while + anything is live and a crash before that write loses it with no record + that it ever existed. + """ + transport.track(_event(1)) + transport.track(_event(2)) + _flush_over( + transport, + _http_200( + { + "processed": 1, + "accepted_event_ids": ["evt-1"], + "rejection_details": [_detail("evt-2", "per_key_policy_limit", 1000)], + "rejected_count": 1, + } + ), + ) + + assert os.path.exists(transport._wal_inflight_path()), ( + "`.inflight` was cleared while evt-2 was still only in memory" + ) + + +def test_a_crash_between_the_dlq_write_and_the_inflight_clear_duplicates(transport): + """The dangerous window is ordered so it can only ever duplicate. + + The rule is: every event is durable — in the DLQ, or still covered by a + retained `.inflight` — BEFORE `.inflight` is cleared. A crash before the + clear leaves the whole batch in `.inflight`, so the restart replays it and + the parked event is re-sent. A duplicate costs one re-delivery, which the + backend absorbs by deduping on `event_id`. A crash the other way round + would leave the event in neither file, and nothing in the system can + reconstruct it. This test pins that the file survives the window. + """ + transport.track(_event(1)) + transport._client = httpx.Client( + transport=httpx.MockTransport( + _http_200( + { + "processed": 0, + "accepted_event_ids": [], + "rejection_details": [_detail("evt-1", "reservation_not_found")], + "rejected_count": 1, + } + ) + ) + ) + + real_clear = transport._clear_inflight + crashed = {"done": False} + + def _crash_on_clear() -> None: + if not crashed["done"]: + crashed["done"] = True + raise KeyboardInterrupt("process died between durable write and clear") + + transport._clear_inflight = _crash_on_clear + try: + with pytest.raises(KeyboardInterrupt): + transport._do_flush() + finally: + transport._clear_inflight = real_clear + + assert os.path.exists(transport._wal_inflight_path()), ( + "the batch was not recoverable after a crash before the clear" + ) + replayed = _read_wal(transport._wal_inflight_path()) + assert [e["event_id"] for e in replayed] == ["evt-1"], ( + "the crash window lost the event instead of duplicating it" + ) + # The DLQ write happened first and survived — so the operator sees the + # park even though the event is still on the wire. + assert _read_wal(transport._wal_dlq_path()), ( + "the DLQ row was not durable before the crash window" + ) + + +def test_a_deduped_duplicate_comes_back_accepted(transport): + """A re-sent event the backend already has must be reported accepted. + + The crash window above produces duplicates by design, so the duplicate + has to land rather than bounce. The backend already guarantees this: + `V3ConsumeOutcome::IdempotentReplay` maps to `V3ConsumeDecision::Ok` + (gate/internal.rs), the event goes back into `post_v3_indices`, and the + recomputed `accepted_event_ids` names it. If that ever regressed, every + duplicate would be refused, parked in the DLQ, and the crash-recovery + path would manufacture DLQ litter on every restart. This test pins the + contract from the SDK side so the dependency is visible. + """ + transport.track(_event(1)) + transport._client = httpx.Client( + transport=httpx.MockTransport( + _http_200( + { + "processed": 1, + # A first send accepted it; a replay of the same event_id + # comes back accepted again rather than as a duplicate + # rejection. + "accepted_event_ids": ["evt-1"], + "rejection_details": [], + "rejected_count": 0, + } + ) + ) + ) + try: + transport._do_flush() + transport._do_flush() # the replay + finally: + transport._client.close() + + assert transport._buffer == [], "a deduped duplicate stayed on the retry path" + assert not os.path.exists(transport._wal_dlq_path()), ( + "a deduped duplicate was parked — recovery would litter the DLQ" + ) + assert not os.path.exists(transport._wal_inflight_path()) + + +def test_dlq_rows_are_versioned_and_carry_what_a_replay_needs(transport): + """A DLQ record must survive being read by an older SDK and replayed later. + + `v` marks the schema. `reason` is machine-readable, `first_failed_at` + says whether the refusal is still current, `attempts` says how long it + was failing, and the original payload is there because a replay needs it + and a triage needs to know what was lost. `error` is retained from v1 so + a file written before this change still reads after an upgrade. + """ + transport.track(_event(1)) + _flush_over( + transport, + _http_200( + { + "processed": 0, + "accepted_event_ids": [], + "rejection_details": [_detail("evt-1", "reservation_not_found")], + "rejected_count": 1, + } + ), + ) + + row = _read_wal(transport._wal_dlq_path())[0] + assert row["v"] == 2, f"DLQ row is unversioned: {row}" + assert row["reason"] == "reservation_not_found" + assert row["error"] == row["reason"], "the v1 `error` field must stay in sync" + assert row["event_type"] == "llm_call", "triage needs the type without opening the payload" + assert row["first_failed_at"] > 0, "a DLQ row must say when the event started failing" + assert row["attempts"] >= 1, "a DLQ row must say how many cycles it survived" + assert row["event"] == _event(1), "a replay needs the original payload" + + +def test_a_v1_dlq_row_is_still_readable(transport): + """An unversioned row must not make the recovery file unreadable. + + A DLQ written before this change stays on disk across the upgrade. A + reader that required `v == 2` would reject every row in it, and the one + file whose whole job is getting data back would be the thing that broke. + """ + dlq = transport._wal_dlq_path() + os.makedirs(os.path.dirname(dlq) or ".", exist_ok=True) + with open(dlq, "w") as f: + f.write(json.dumps({"error": "old:400", "event": _event(9)}) + "\n") + + rows = _read_wal(dlq) + assert len(rows) == 1 and rows[0]["event"]["event_id"] == "evt-9" + assert "v" not in rows[0], "the v1 fixture is not a v1 row" + + +# --------------------------------------------------------------------------- +# Deterministic 5xx. +# +# EXECUTION_NOT_BOUND is NOT a 4xx, and it is not a 5xx either. The backend +# used to answer StatusCode::SERVICE_UNAVAILABLE for it (handlers.rs:10016) +# while naming error_code EXECUTION_NOT_BOUND in the body. The 24h binding +# TTL makes it deterministic: once the binding is gone, no retry can ever +# succeed. Status-only classification therefore routed it to the +# transient-5xx path, where it retried until the attempt budget died and +# every attempt counted as a transport failure — ten of them opened the +# circuit breaker on a backend that was answering everything else. +# +# The backend has since been corrected to answer 422, but the tests below +# still use 503 on purpose. Deployed SDKs outlive the backend release that +# fixed it: an instance pinned to an older build is talking to a newer one +# for weeks, and clients in the field are not all upgraded in lockstep. The +# SDK must classify on the code precisely so it is correct against BOTH +# statuses — if this suite only ever saw 422, the classification would +# silently depend on the server being upgraded first, and the bug would +# return as a partial outage on the laggard fleet. +# +# These drive `_client.post` rather than `_send_batch_with_retry_info`. The +# classification lives in the inner `_post_batch` closure, and a 5xx never +# escapes `_send_batch_with_retry_info` as an `HTTPStatusError` anyway — +# `_retry_with_backoff` converts retry exhaustion into +# `BreakerTransportError`. Mocking the outer method would test a shape the +# real path cannot produce. +# --------------------------------------------------------------------------- + + +def _respond(status: int, body: dict): + """An httpx transport handler that always answers `status` with `body`.""" + + def _handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(status, json=body) + + return _handler + + +def test_execution_not_bound_is_dead_lettered_not_retried_forever(transport, monkeypatch): + """503 EXECUTION_NOT_BOUND is deterministic and must not loop forever. + + With status-only classification this is a plain 5xx: the SDK retries it + indefinitely, and because the batch sits at the head of the retry path + it blocks every healthy event queued behind it. But the binding TTL is + 24h and nothing the client does can restore it — the only cure is a new + /gate, which the agent cannot synthesize for a past event. + """ + # The refusal must be classified on the FIRST response, so the retry + # budget is irrelevant here; pin it low to keep the test fast. + monkeypatch.setattr(transport, "_track_max_retries", 2, raising=False) + transport.track(_event(1)) + + transport._client = httpx.Client( + transport=httpx.MockTransport( + _respond(503, {"error_code": "EXECUTION_NOT_BOUND", "error_message": "no binding"}) + ) + ) + try: + transport._do_flush() + finally: + transport._client.close() + + rows = _read_wal(transport._wal_dlq_path()) + assert [r["event"]["event_id"] for r in rows] == ["evt-1"], ( + "a permanently-unbound execution is still on the retry path" + ) + assert rows[0]["error"] == "EXECUTION_NOT_BOUND:503", ( + "the DLQ row must name the wire code, not just the status — an " + "operator replaying it cannot act on '503'" + ) + assert transport._buffer == [] + + +def test_execution_not_bound_does_not_open_the_circuit_breaker(transport, monkeypatch): + """A named refusal is an ANSWER, not an outage — the breaker must not count it. + + This is the same bug class as "any 4xx opens the breaker", one status + code over. The backend is up and replying correctly; it is telling us + this particular event can never land. Counting those as transport + failures trips the breaker after `max_failed_flush` and then blocks + every healthy event in the buffer for the whole recovery window. + """ + monkeypatch.setattr(transport, "_track_max_retries", 2, raising=False) + threshold = transport._circuit_breaker._failure_threshold + + transport._client = httpx.Client( + transport=httpx.MockTransport( + _respond(503, {"error_code": "EXECUTION_NOT_BOUND", "error_message": "no binding"}) + ) + ) + try: + for i in range(threshold + 3): + transport.track(_event(i)) + transport._do_flush() + finally: + transport._client.close() + + state = transport._circuit_breaker._state + assert state is not state.OPEN, ( + "named deterministic refusals opened the circuit on a healthy backend" + ) + assert transport._circuit_breaker._failure_count == 0, ( + "a refusal the backend named counted toward the breaker's failure budget" + ) + + +def test_other_5xx_still_retries_rather_than_being_dead_lettered(transport, monkeypatch): + """A generic 5xx is NOT deterministic — it must keep retrying. + + Same status, no recognised code: the SDK cannot know whether the backend + is down, so it must assume it is and keep the data. Quarantining on an + unrecognised 5xx is how an outage eats a whole fleet's buffer. + """ + monkeypatch.setattr(transport, "_track_max_retries", 2, raising=False) + transport.track(_event(1)) + + transport._client = httpx.Client( + transport=httpx.MockTransport( + _respond(503, {"error_code": "SERVICE_UNAVAILABLE", "error_message": "down"}) + ) + ) + try: + for _ in range(transport._max_batch_attempts + 2): + transport._do_flush() + finally: + transport._client.close() + + assert not os.path.exists(transport._wal_dlq_path()), ( + "a transient 503 was quarantined — the outage would eat the buffer" + ) + assert [e["event_id"] for e in transport._buffer] == ["evt-1"], ( + "the event must still be queued for the next attempt" + ) + + +def test_a_5xx_without_a_parseable_body_is_not_deterministic(transport, monkeypatch): + """A proxy HTML error page names no code — so it is not a refusal. + + A reverse proxy in front of the backend answers 502 with an HTML error + document and no JSON envelope. Reading a code out of that is impossible, + and treating "no code" as "deterministic" would quarantine the entire + buffer during exactly the outage the proxy is reporting. + """ + monkeypatch.setattr(transport, "_track_max_retries", 2, raising=False) + transport.track(_event(1)) + + def _html(request: httpx.Request) -> httpx.Response: + return httpx.Response(502, text="502 Bad Gateway") + + transport._client = httpx.Client(transport=httpx.MockTransport(_html)) + try: + transport._do_flush() + finally: + transport._client.close() + + assert not os.path.exists(transport._wal_dlq_path()), ( + "an unparseable 502 was treated as a permanent refusal" + ) + assert [e["event_id"] for e in transport._buffer] == ["evt-1"] + + +# --------------------------------------------------------------------------- +# 4xx classification. "Any 4xx is permanent" is unsafe: it discards good data +# exactly when the server is recovering. +# --------------------------------------------------------------------------- + + +def _reject_with(status: int): + """A send that always fails with `status`.""" + resp = httpx.Response(status, json={}, request=httpx.Request("POST", "https://x")) + return lambda batch: resp.raise_for_status() + + +@pytest.mark.parametrize("status", [408, 429]) +def test_transient_4xx_is_requeued_never_quarantined(transport, status): + """408/429 are the server being busy, not the data being wrong. + + Right after an outage every SDK in the fleet replays its WAL at once, the + rate limiter answers 429, and a naive permanent-4xx rule would dead-letter + the entire recovering fleet in the first second. These must go back to the + buffer, and stay there no matter how many attempts are spent. + """ + transport.track(_event(1)) + + with patch.object(transport, "_send_batch_with_retry_info", side_effect=_reject_with(status)): + for _ in range(transport._max_batch_attempts + 5): + transport._do_flush() + + assert [e["event_id"] for e in transport._buffer] == ["evt-1"], ( + f"{status} dropped the batch off the retry path" + ) + assert not os.path.exists(transport._wal_dlq_path()), f"{status} was quarantined" + + +@pytest.mark.parametrize("status", [401, 403]) +def test_auth_4xx_holds_the_batch_for_the_operator(transport, status): + """A rotated or revoked key must not cost the data. + + 401/403 is recoverable by operator action. Quarantining here would strand + a whole WAL on disk that would have delivered fine the moment the key was + fixed. + """ + transport.track(_event(1)) + + with patch.object(transport, "_send_batch_with_retry_info", side_effect=_reject_with(status)): + for _ in range(transport._max_batch_attempts + 5): + transport._do_flush() + + assert [e["event_id"] for e in transport._buffer] == ["evt-1"] + assert not os.path.exists(transport._wal_dlq_path()), ( + f"{status} quarantined recoverable data" + ) + + +def test_413_splits_the_batch_instead_of_quarantining_it(transport): + """413 says the BATCH is too large; the events themselves are fine. + + The halves are sent as separate batches, not pushed back on the buffer — + the buffer is flushed wholesale, so re-queuing both halves would rebuild + the exact batch the server just refused and the split would never converge. + """ + transport.track(_event(1)) + transport.track(_event(2)) + transport.track(_event(3)) + transport.track(_event(4)) + + limit = 2 # pretend the server accepts at most 2 events per request + sent: list[dict] = [] + + def _size_limited(batch): + if len(batch) > limit: + _reject_with(413)(batch) + sent.extend(batch) + return SendResult(accepted_event_ids=[e["event_id"] for e in batch]) + + with patch.object(transport, "_send_batch_with_retry_info", side_effect=_size_limited): + transport._do_flush() + + assert sorted(e["event_id"] for e in sent) == ["evt-1", "evt-2", "evt-3", "evt-4"] + assert transport._buffer == [] + assert not os.path.exists(transport._wal_dlq_path()), "a size limit must not dead-letter data" + + +def test_413_single_event_is_quarantined(transport): + """Splitting bottoms out: an event that is too big alone can never land.""" + transport.track(_event(1)) + with patch.object(transport, "_send_batch_with_retry_info", side_effect=_reject_with(413)): + transport._do_flush() + + rows = _read_wal(transport._wal_dlq_path()) + assert [r["event"]["event_id"] for r in rows] == ["evt-1"] + + +def test_400_batch_is_bisected_to_isolate_the_bad_event(transport): + """One malformed event must not take the whole batch down with it. + + The backend validates fail-CLOSED, so a single event with an empty `model` + rejects the entire request. Dead-lettering the batch would bury every + valid event behind it. + """ + for i in (1, 2, 3): + transport.track(_event(i)) + + def _only_third_is_bad(batch): + if any(e["event_id"] == "evt-3" for e in batch): + _reject_with(400)(batch) + return SendResult(accepted_event_ids=[e["event_id"] for e in batch]) + + with patch.object(transport, "_send_batch_with_retry_info", side_effect=_only_third_is_bad): + transport._do_flush() + + assert transport._buffer == [] + rows = _read_wal(transport._wal_dlq_path()) + assert [r["event"]["event_id"] for r in rows] == ["evt-3"], ( + "quarantine caught more than the offending event" + ) + + +def test_repeated_4xx_does_not_open_the_circuit_breaker(transport): + """A 4xx proves the backend is ALIVE, so it must not trip the breaker. + + This was a live defect: the rejection propagated out of the breaker call, + which counted it as a transport failure. `max_failed_flush` is 10, so ten + consecutive rejected batches opened the circuit on a backend that was up + and answering — and then every buffered event was blocked for the whole + 30s recovery window, for a problem the breaker cannot fix. + """ + for _ in range(transport.config.max_failed_flush + 5): + transport.track(_event(1)) + with patch.object(transport, "_send_batch_with_retry_info", side_effect=_reject_with(400)): + transport._do_flush() + + from nullrun.breaker.circuit_breaker import CBState + + assert transport._circuit_breaker.state == CBState.CLOSED, ( + "repeated 4xx opened the breaker on a live backend" + ) + + +def test_repeated_deterministic_errors_do_not_open_the_circuit_breaker(transport): + """A local serialization failure is our bug, not the backend's health.""" + for _ in range(transport.config.max_failed_flush + 5): + transport.track(_event(1)) + with patch.object( + transport, "_send_batch_with_retry_info", side_effect=TypeError("not serializable") + ): + transport._do_flush() + + from nullrun.breaker.circuit_breaker import CBState + + assert transport._circuit_breaker.state == CBState.CLOSED + + +def test_transport_failures_still_open_the_circuit_breaker(transport): + """The guard above must not neuter the breaker — an unreachable backend still counts.""" + transport.track(_event(1)) + refused = httpx.ConnectError("connection refused") + with patch.object(transport, "_send_batch_with_retry_info", side_effect=refused): + for _ in range(transport.config.max_failed_flush + 2): + transport._do_flush() + + from nullrun.breaker.circuit_breaker import CBState + + assert transport._circuit_breaker.state == CBState.OPEN, ( + "a genuinely unreachable backend no longer trips the breaker" + ) + + +def test_bisect_converges_and_dead_letters_only_the_offender(transport): + """A bad event anywhere in a large batch costs exactly one event.""" + for i in range(1, 9): + transport.track(_event(i)) + delivered: list[dict] = [] + + def _only_eight_is_bad(batch): + if any(e["event_id"] == "evt-8" for e in batch): + _reject_with(422)(batch) + delivered.extend(batch) + return SendResult(accepted_event_ids=[e["event_id"] for e in batch]) + + with patch.object(transport, "_send_batch_with_retry_info", side_effect=_only_eight_is_bad): + transport._do_flush() + + assert sorted(e["event_id"] for e in delivered) == [f"evt-{i}" for i in range(1, 8)] + rows = _read_wal(transport._wal_dlq_path()) + assert [r["event"]["event_id"] for r in rows] == ["evt-8"] + + +@pytest.mark.parametrize("status", [400, 422]) +def test_permanent_4xx_of_a_single_event_is_quarantined(transport, status): + """At the bottom of the bisect, 400/422 is genuinely undeliverable.""" + transport.track(_event(1)) + with patch.object(transport, "_send_batch_with_retry_info", side_effect=_reject_with(status)): + transport._do_flush() + + rows = _read_wal(transport._wal_dlq_path()) + assert [r["event"]["event_id"] for r in rows] == ["evt-1"] + assert all(str(status) in r["error"] for r in rows) + assert transport._buffer == [] + assert not os.path.exists(transport._wal_inflight_path()) + + +def test_quarantined_batch_is_not_replayed_on_next_start(transport): + """DLQ contents must never re-enter the retry loop on restart.""" + transport.track(_event(1)) + with patch.object(transport, "_send_batch_with_retry_info", side_effect=_reject_with(422)): + transport._do_flush() + + assert os.path.exists(transport._wal_dlq_path()) + with patch.object(transport, "_do_flush") as flush: + transport._replay_from_wal() + flush.assert_not_called(), "a dead-lettered batch was replayed" + + +def test_dlq_accumulates_across_quarantines(transport): + """A second quarantine must not clobber the first — that is data loss too.""" + dlq = transport._wal_dlq_path() + for i in (1, 2): + transport.track(_event(i)) + with patch.object( + transport, "_send_batch_with_retry_info", side_effect=_reject_with(400) + ): + transport._do_flush() + + rows = _read_wal(dlq) + assert [r["event"]["event_id"] for r in rows] == ["evt-1", "evt-2"] + + +# --------------------------------------------------------------------------- +# Non-HTTP exceptions must not become an eternal blocker. +# --------------------------------------------------------------------------- + + +def test_5xx_still_retries_and_is_not_dead_lettered(transport): + """Only permanent rejections are quarantined; transient ones keep retrying.""" + transport.track(_event(1)) + with patch.object(transport._circuit_breaker, "call", side_effect=BreakerTransportError("x")): + transport._do_flush() + + assert [e["event_id"] for e in transport._buffer] == ["evt-1"] + assert not os.path.exists(transport._wal_dlq_path()) + + +def test_unexpected_error_does_not_escape_and_kill_flush_thread(transport): + """A non-HTTP failure must re-queue, not propagate out of the flush loop. + + The background loop calls `_do_flush` with no try/except: an escaping + exception would kill the thread and silently stop all delivery for the + life of the process. + """ + transport.track(_event(1)) + with patch.object( + transport, "_send_batch_with_retry_info", side_effect=RuntimeError("socket reset") + ): + transport._do_flush() # must not raise + + assert [e["event_id"] for e in transport._buffer] == ["evt-1"] + + +def test_deterministic_error_is_dead_lettered_after_the_attempt_budget(transport): + """An error that repeats forever must not pin the batch ahead of the buffer. + + A payload that will not serialize fails identically on every cycle. Left + alone it would be retried for the life of the process and every event + queued behind it would never be delivered. + """ + transport.track(_event(1)) + with patch.object( + transport, "_send_batch_with_retry_info", side_effect=TypeError("not serializable") + ): + for _ in range(transport._max_batch_attempts + 2): + transport._do_flush() + + rows = _read_wal(transport._wal_dlq_path()) + assert [r["event"]["event_id"] for r in rows] == ["evt-1"] + assert transport._buffer == [], "the exhausted batch stayed on the retry path" + + +def test_attempt_counter_is_cleared_after_the_batch_lands(transport): + """A delivered batch must not leave failure history behind. + + Otherwise the map grows for the life of a long-lived process, and a later + batch with coincidentally identical event ids starts with a spent budget. + """ + transport.track(_event(1)) + key = transport._failure_signature(transport._buffer[:]) + transport._batch_attempts[key] = 3 + + transport._send_batch_with_retry_info = lambda batch: SendResult( + accepted_event_ids=[e["event_id"] for e in batch] + ) + transport._do_flush() + + assert key not in transport._batch_attempts + + +# --------------------------------------------------------------------------- +# Append into a file a crash left torn. +# --------------------------------------------------------------------------- + + +def test_torn_tail_plus_append_does_not_corrupt_a_valid_event(transport): + """The append path must not glue a new row onto a truncated one. + + This is the scenario that makes copy-and-append dangerous: the DLQ (and + any future appending write) copies the previous file verbatim. If the + process died mid-write, the last line has no `\\n`, and the next row is + concatenated onto the truncated JSON — turning one good event plus one new + event into a single unparseable line. + """ + dlq = transport._wal_dlq_path() + os.makedirs(os.path.dirname(dlq) or ".", exist_ok=True) + with open(dlq, "w") as f: + f.write(json.dumps({"error": "e", "event": _event(1)}) + "\n") + f.write('{"error": "e", "event": {"event_id": "evt-2", "cost_c') # torn + + transport._quarantine_to_dlq([_event(3)], RuntimeError("x")) + + # Every line parses: evt-1 survived intact and evt-3 landed cleanly. + with open(dlq) as f: + lines = [line for line in f if line.strip()] + rows = [json.loads(line) for line in lines] + ids = [r["event"]["event_id"] for r in rows] + assert ids == ["evt-1", "evt-3"], f"torn tail corrupted the file: {ids}" + + +def test_append_to_a_file_that_is_entirely_torn_keeps_the_new_rows(transport): + """No newline anywhere: the whole old content is torn and must be dropped.""" + dlq = transport._wal_dlq_path() + os.makedirs(os.path.dirname(dlq) or ".", exist_ok=True) + with open(dlq, "w") as f: + f.write('{"error": "e", "event": {"event_id"') + + transport._quarantine_to_dlq([_event(3)], RuntimeError("x")) + + rows = _read_wal(dlq) + assert [r["event"]["event_id"] for r in rows] == ["evt-3"] + + +def test_stale_tmp_from_a_dead_process_is_truncated_not_appended(transport): + """A leftover `.tmp.` from a crash must not be concatenated onto. + + The tmp name is pid-scoped and opened with "w", so a recycled pid + truncates the stale file rather than appending to whatever the dead + process left in it. + """ + target = transport._wal_path() + os.makedirs(os.path.dirname(target) or ".", exist_ok=True) + stale = f"{target}.tmp.{os.getpid()}" + with open(stale, "w") as f: + f.write('{"event_id": "ghost", "half-writ') + + assert transport._write_events_atomic(target, [_event(1)]) is True + + rows = _read_wal(target) + assert [r["event_id"] for r in rows] == ["evt-1"], "stale tmp content leaked into the WAL" diff --git a/tests/test_wal_hygiene.py b/tests/test_wal_hygiene.py new file mode 100644 index 0000000..47a1c7e --- /dev/null +++ b/tests/test_wal_hygiene.py @@ -0,0 +1,1272 @@ +"""WAL hygiene: permissions, cross-process locking, the DLQ size cap, replay. + +What is being protected. The WAL is the SDK's only copy of an event between +``track()`` and the backend accepting it, and an event is a full payload — +prompt, completion, tool arguments. So the files are private by default, the +cap never destroys anything, and the degradation of either guarantee is a +named, observable state rather than an assumption. + +The split by platform is deliberate and is the point of the module. A green +run on Windows proves nothing about `flock` or about a directory fsync, +because neither exists there — that is why the POSIX-only behaviour is +`skipif win32` and the Windows behaviour is asserted directly instead of +being assumed to cover the same ground. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import textwrap + +import httpx +import pytest + +from nullrun.observability import metrics +from nullrun.transport import Transport +from nullrun.wal_admin import _drop_confirmed, read_dlq +from nullrun.wal_admin import main as wal_main + +platform = sys.platform + +requires_posix_lock = pytest.mark.skipif( + platform == "win32", reason="fcntl.flock does not exist on win32" +) + + +@pytest.fixture(autouse=True) +def _clean_metrics(): + metrics.reset() + yield + metrics.reset() + + +@pytest.fixture +def transport(tmp_path, monkeypatch): + monkeypatch.setenv("NULLRUN_WAL_PATH", str(tmp_path / "sdk.wal")) + t = Transport(api_url="https://api.test.nullrun.io", api_key="test-key-12345678") + yield t + t._client.close() + + +def _event(i: int) -> dict: + return {"event_id": f"evt-{i}", "type": "llm_call", "cost_cents": 1} + + +def _read_wal(path: str) -> list[dict]: + if not os.path.exists(path): + return [] + with open(path) as f: + return [json.loads(line) for line in f if line.strip()] + + +# --------------------------------------------------------------------------- +# 0600. Event payloads are not public artifacts. +# --------------------------------------------------------------------------- + + +@requires_posix_lock +def test_wal_files_are_owner_only(transport): + """Every WAL file, on every write path, is 0600. + + `open(path, "w")` asks for 0666 and lets the umask decide; the default + umask of 022 leaves an event payload world-readable in /tmp, on a shared + volume, and in any container several services mount. The mode has to be + requested, because the file that `os.replace` renames is the file the WAL + ends up being. + """ + transport.track(_event(1)) + assert transport._persist_to_wal() is True + transport._persist_inflight([_event(2)]) + assert transport._write_dlq_rows([transport._dlq_row(_event(3), "x")]) is True + with transport._wal_file_lock(): + pass # creates the lock file + + base = transport._wal_path() + for candidate in (base, f"{base}.inflight", f"{base}.dlq", f"{base}.lock"): + assert os.path.exists(candidate), f"{candidate} was never created" + assert oct(os.stat(candidate).st_mode & 0o777) == oct(0o600), ( + f"{candidate} is {oct(os.stat(candidate).st_mode & 0o777)}, not 0600" + ) + + +@requires_posix_lock +def test_a_wal_left_by_a_previous_run_is_tightened_at_startup(tmp_path, monkeypatch): + """An existing 0644 WAL is chmodded before anything reads it. + + Hardening only new writes would leave the events most likely to hold + full payloads — the ones already sitting on disk from a crashed run — + readable by everyone with access to the volume. The mode survives every + `os.replace`, so tightening the tmp file alone changes nothing about it. + """ + monkeypatch.setenv("NULLRUN_WAL_PATH", str(tmp_path / "sdk.wal")) + base = tmp_path / "sdk.wal" + base.write_text(json.dumps(_event(1)) + "\n") + os.chmod(base, 0o644) + + t = Transport(api_url="https://api.test.nullrun.io", api_key="test-key-12345678") + try: + assert oct(os.stat(base).st_mode & 0o777) == oct(0o600) + finally: + t._client.close() + + +def test_a_platform_without_flock_states_the_degradation_instead_of_failing( + transport, caplog, monkeypatch +): + """No `flock` here is a supported configuration, and it says so once. + + The failure this prevents is not an ImportError at module load — it is the + silent one: a multi-process deployment on Windows, or on any platform + without flock, believing it has the same protection it has on Linux. + + The branch is reached by REMOVING `fcntl` rather than by running on a + platform that lacks it. Testing it only where the platform happens to + cooperate means the branch is asserted on Windows and skipped on every + Linux CI run — the assertion then never runs on the platform where a + regression would be introduced, which is the point of having Linux CI. + """ + from nullrun import transport as transport_mod + + monkeypatch.setattr(transport_mod, "fcntl", None) + + with caplog.at_level("WARNING"): + with transport._wal_file_lock() as acquired: + assert acquired is True, "a platform without flock must still be able to write" + assert metrics.transport.wal_lock == "unavailable" + assert any("flock" in r.message for r in caplog.records), ( + "the degradation must be stated, not merely recorded in a metric" + ) + + # Once, not once per flush: this is a standing condition, and a warning + # on every cycle buries the log under a fact that cannot change. + caplog.clear() + with caplog.at_level("WARNING"): + for _ in range(5): + with transport._wal_file_lock(): + pass + assert not [r for r in caplog.records if "durability degraded" in r.message] + + +# --------------------------------------------------------------------------- +# flock. One writer per WAL path. +# --------------------------------------------------------------------------- + + +def _holder_process(lock_path: str, seconds: float) -> subprocess.Popen: + """A real second process holding the WAL lock, for `seconds`.""" + return subprocess.Popen( + [ + sys.executable, + "-c", + textwrap.dedent( + """ + import fcntl, os, sys, time + fd = os.open(sys.argv[1], os.O_RDWR | os.O_CREAT, 0o600) + fcntl.flock(fd, fcntl.LOCK_EX) + print("held", flush=True) + time.sleep(float(sys.argv[2])) + """ + ), + lock_path, + str(seconds), + ], + stdout=subprocess.PIPE, + ) + + +@requires_posix_lock +def test_the_wal_lock_actually_excludes_a_second_holder(transport, monkeypatch, caplog): + """The guard is real: a lock held elsewhere makes this process skip. + + A lock that is taken but never contended against is indistinguishable + from no lock at all, and the failure it is meant to prevent — two + processes interleaving their lines inside one another's read-copy-append + of the DLQ — is silent and unrecoverable, because the resulting file has + no parseable boundary. So this asserts the exclusion from the outside, + with a real second process holding the lock, and a wait short enough that + the test does not have to sit through the production timeout. + """ + monkeypatch.setenv("NULLRUN_WAL_LOCK_TIMEOUT_MS", "300") + holder = _holder_process(transport._wal_lock_path(), 5) + try: + assert holder.stdout.readline().strip() == b"held" + with caplog.at_level("WARNING"): + with transport._wal_file_lock() as acquired: + assert acquired is False, "a held WAL lock did not exclude this process" + assert metrics.transport.wal_lock == "contended" + assert metrics.transport.wal_lock_timeouts_total == 1 + # And the write is refused, not silently interleaved. + assert transport._write_events_atomic(transport._wal_path(), [_event(1)]) is False + assert not os.path.exists(transport._wal_path()) + assert any("retry path" in r.message for r in caplog.records), ( + "giving up on the lock must say where the events went" + ) + finally: + holder.kill() + holder.wait(timeout=10) + + +@requires_posix_lock +@pytest.mark.slow_sleep +def test_a_writer_that_loses_the_race_waits_rather_than_starving(transport, monkeypatch): + """Contention costs a wait, not the write. This is a regression test. + + The first implementation took the lock non-blocking and skipped the + write on contention. On a 4-worker deployment that is not a rare + collision: the losing writer was refused on *every* attempt, because the + winner was always mid-append, and its DLQ writes never landed at all. The + events were not lost — the caller re-queues them — so nothing alerted, + and they were not delivered either. A wait is a few microseconds; losing + the file forever is not a cheaper option. + + Found by running this suite in a Linux container: on Windows every test + that needs a second process to contend for the lock is skipped, which is + precisely the coverage that would have caught it. + + ``slow_sleep`` because the wait is the assertion. The conftest caps every + sleep at 1ms, so under the cap the 5s budget would expire in + milliseconds and the test would be measuring the harness, not the lock. + """ + monkeypatch.setenv("NULLRUN_WAL_LOCK_TIMEOUT_MS", "5000") + holder = _holder_process(transport._wal_lock_path(), 0.5) + try: + assert holder.stdout.readline().strip() == b"held" + assert transport._write_dlq_rows( + [transport._dlq_row(_event(1), "synthetic")] + ) is True, "a writer lost the race and never wrote" + holder.wait(timeout=10) + rows = _read_wal(transport._wal_dlq_path()) + assert [r["event"]["event_id"] for r in rows] == ["evt-1"] + finally: + if holder.poll() is None: + holder.kill() + holder.wait(timeout=10) + + +@requires_posix_lock +def test_the_lock_is_released_so_the_next_cycle_can_write(transport): + """Not holding the lock is not the same as being unable to take it.""" + with transport._wal_file_lock() as first: + assert first is True + with transport._wal_file_lock() as second: + assert second is True + assert transport._write_events_atomic(transport._wal_path(), [_event(1)]) is True + assert metrics.transport.wal_lock == "enabled" + + +@requires_posix_lock +def test_flock_excludes_two_threads_of_one_process(transport): + """flock is per open file DESCRIPTION, not per process. + + The trap this pins: a reader who concludes "flock does not work between + threads in one process" would add a `threading.Lock` next to it and be + wrong in the other direction — the `threading.Lock` would then be the only + thing serialising threads while the cross-process guarantee quietly + depends on an implementation detail nobody re-checks. The two are not + interchangeable, and the fact that they compose is the thing to write + down. + + It works because every acquisition opens its OWN descriptor. A future + refactor that caches the fd to avoid an `open()` per lock would make + `LOCK_EX` a no-op between threads — the kernel would see the same open + file description and grant the second acquisition immediately. This test + is what makes that refactor loud. + """ + import threading + + acquired = [] + refused = [] + barrier = threading.Barrier(8) + lock = threading.Lock() + + def contend() -> None: + barrier.wait(timeout=30) + with transport._wal_file_lock() as got: + if got: + with lock: + acquired.append(1) + else: + with lock: + refused.append(1) + + threads = [threading.Thread(target=contend) for _ in range(8)] + for t in threads: + t.start() + for t in threads: + t.join(timeout=60) + assert not t.is_alive(), "a contending thread hung rather than waiting its turn" + + assert len(acquired) + len(refused) == 8 + assert len(acquired) >= 1 + # Nobody is dropped: a thread that could not take the lock inside the + # timeout is a caller that never learned it had to wait, which on the + # write path means a silently-skipped write. + assert not refused, "a same-process thread was refused the lock it should have waited for" + assert metrics.transport.wal_lock == "enabled" + + +@requires_posix_lock +def test_concurrent_threads_do_not_lose_a_dlq_row(transport): + """The property the lock is for, across threads rather than processes. + + Appends are read-copy-append, so two writers each rewrite the file from + their own read. Serialised correctly, N threads × M rows produce N×M rows + with N×M distinct ids — nothing lost, nothing interleaved into an + unparseable line. + """ + import threading + + rows_per_thread = 25 + threads_count = 8 + start = threading.Barrier(threads_count) + + def append(tid: int) -> None: + start.wait(timeout=30) + for i in range(rows_per_thread): + transport._write_dlq_rows( + [transport._dlq_row({"event_id": f"t{tid}-{i}", "type": "llm_call"}, "synthetic")] + ) + + threads = [threading.Thread(target=append, args=(i,)) for i in range(threads_count)] + for t in threads: + t.start() + for t in threads: + t.join(timeout=120) + assert not t.is_alive() + + expected = threads_count * rows_per_thread + rows = _read_wal(transport._wal_dlq_path()) + assert len(rows) == expected, f"expected {expected} rows, got {len(rows)}" + ids = {r["event"]["event_id"] for r in rows} + assert len(ids) == expected, "a row was lost to another thread's rewrite" + + +@requires_posix_lock +def test_a_dlq_append_is_not_interleaved_by_a_concurrent_writer(transport): + """The read-copy-append is the region the lock exists for. + + Without it, two processes each read the whole DLQ and each write back + theirs plus their own row: one of the two rows vanishes, and the file + ends with whichever writer's copy lost the other. This drives two real + processes against one DLQ and asserts no row is lost. + """ + script = textwrap.dedent( + """ + import json, os, sys + sys.path.insert(0, sys.argv[3]) + from nullrun.transport import Transport + os.environ["NULLRUN_WAL_PATH"] = sys.argv[1] + t = Transport(api_url="https://api.test.nullrun.io", api_key="k-12345678") + for _ in range(20): + t._write_dlq_rows([t._dlq_row( + {"event_id": sys.argv[2] + str(_), "type": "llm_call"}, "synthetic")]) + t._client.close() + """ + ) + src = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + "/src" + wal = str(tmp_path_for(transport)) + procs = [ + subprocess.Popen( + [sys.executable, "-c", script, wal, f"p{i}-", src], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + for i in range(2) + ] + for p in procs: + _, err = p.communicate(timeout=120) + assert p.returncode == 0, err.decode()[-2000:] + + rows = _read_wal(transport._wal_dlq_path()) + ids = {r["event"]["event_id"] for r in rows} + assert len(rows) == 40, f"expected 40 rows, got {len(rows)}" + assert len(ids) == 40, "one writer's rows were lost to the other's rewrite" + + +def tmp_path_for(transport) -> str: + return transport._wal_path() + + +# --------------------------------------------------------------------------- +# Durability. What can this filesystem actually promise? +# --------------------------------------------------------------------------- + + +@requires_posix_lock +def test_a_posix_volume_really_does_fsync_its_directory(transport, tmp_path): + """The durability claim, asserted rather than assumed. + + Every WAL write is tmp-file + fsync + `os.replace`. That sequence is only + worth anything if the DIRECTORY entry is also fsynced — `os.replace` is + atomic with respect to readers, but the rename itself is not durable until + the directory is synced, and a power cut can otherwise leave the file + that the WAL points at absent or stale. Windows has no directory fsync at + all, which is why the SDK reports `wal_dir_fsync` as a probed fact rather + than a promise. + + This asserts the positive on the platform where the answer is supposed to + be yes. The existing degradation test covers the negative by removing + `fcntl`, but nothing pinned the healthy case, so a change that made the + probe always return False — silently downgrading every deployment's + durability claim — would have passed the whole suite. + """ + assert transport._dir_fsync_supported(str(tmp_path)) is True + assert metrics.transport.wal_dir_fsync == "enabled", ( + "the probe succeeded but the metric does not say so, so an operator " + "reading /health is told less than the code knows" + ) + + +@requires_posix_lock +def test_the_wal_write_actually_reaches_the_volume(transport, tmp_path): + """A real write survives a reopen — the whole point of the WAL. + + Complements the fsync probe: that one says the syscall works, this one + says the file is really where the SDK says it is, with the bytes the SDK + says it wrote, after the handle is closed and a new one is opened. + """ + transport.track({"event_id": "durable-1", "type": "llm_call", "cost_cents": 1}) + assert transport._persist_to_wal() is True, "the WAL write reported failure" + rows = _read_wal(transport._wal_path()) + assert [r["event_id"] for r in rows] == ["durable-1"] + + +@requires_posix_lock +def test_recovery_reads_the_rotated_wal_not_just_the_active_one(transport, monkeypatch): + """A restart must find events in BOTH generations of the WAL. + + Rotation moves the active file to `.wal.1` before a new one is written, so + after any rotation the oldest unflushed events live only in the rotated + file. A recovery path that read just the active WAL would silently drop + them — the worst shape of bug, because the process starts cleanly and the + events are simply not there. This is the closest thing to the SIGKILL case + that can be tested deterministically: a kill is a restart whose previous + exit did no cleanup, and the state it leaves behind is exactly a rotated + plus active pair. + """ + wal = transport._wal_path() + rotated = f"{wal}.1" + with open(rotated, "w") as f: + f.write(json.dumps({"event_id": "older-1", "type": "llm_call"}) + "\n") + with open(wal, "w") as f: + f.write(json.dumps({"event_id": "newer-1", "type": "llm_call"}) + "\n") + + delivered: list[str] = [] + monkeypatch.setattr( + transport, "_do_flush", lambda: delivered.extend(e["event_id"] for e in transport._buffer) + ) + transport._replay_from_wal() + assert sorted(delivered) == ["newer-1", "older-1"], ( + f"recovery lost a generation of the WAL: {delivered}" + ) + + +# --------------------------------------------------------------------------- +# DLQ size cap. A full DLQ stalls; it never deletes. +# --------------------------------------------------------------------------- + + +def test_the_dlq_cap_stalls_the_write_and_deletes_nothing(transport, monkeypatch, caplog): + """At the cap the event goes back on the retry path, not into the bin. + + The tempting implementation is to trim the oldest rows. That destroys the + only copy of events the SDK could not deliver, silently, with no trace + beyond a shorter file — and it is unrecoverable in a way a full disk is + not. So the cap refuses the append, loudly, and the caller keeps the + event alive on the retry path. + """ + rows = [ + transport._dlq_row({"event_id": f"old-{i}", "type": "llm_call"}, "synthetic") + for i in range(20) + ] + # The cap is exactly what the seed occupies, measured the same way the + # cap measures an incoming write. Hard-coding a byte count would make the + # test a hostage to the exact JSON shape of a DLQ row. + seed_bytes = sum(len(json.dumps(r, default=str)) + 1 for r in rows) + monkeypatch.setenv("NULLRUN_DLQ_MAX_BYTES", str(seed_bytes)) + assert transport._write_dlq_rows(rows) is True + before = _read_wal(transport._wal_dlq_path()) + assert len(before) == 20 + + with caplog.at_level("ERROR"): + assert transport._write_dlq_rows([transport._dlq_row(_event(1), "late")]) is False + + after = _read_wal(transport._wal_dlq_path()) + assert after == before, "the cap must not remove, reorder, or rewrite a row" + assert metrics.transport.dlq_overflow_total == 1 + assert any("never" in r.message and "deletes" in r.message for r in caplog.records), ( + f"stalling must be loud, got {[r.message for r in caplog.records]}" + ) + + +def test_a_terminal_refusal_that_cannot_be_recorded_is_held_not_resent( + transport, monkeypatch +): + """A permanent rejection with no DLQ space must not clear `.inflight`. + + `_quarantine_to_dlq` clears the in-flight file because the events are + parked. If the park failed, that clear is the loss: the events are in + neither the DLQ nor the buffer nor `.inflight`. + + It also must not go back on the SEND path. The earlier version of this + test asserted exactly that, and it was asserting the bug: a terminal + refusal re-sent produces the identical refusal forever while occupying + the head of the buffer. The event is held instead, and `.inflight` is + retained until the holdover has its own durable copy in + `.holdover` — see the kill -9 test below for why retaining + `.inflight` alone is not enough. + """ + monkeypatch.setenv("NULLRUN_DLQ_MAX_BYTES", "1") + transport.track(_event(1)) + transport._client = httpx.Client( + transport=httpx.MockTransport( + lambda r: httpx.Response( + 422, + json={"error_code": "EXECUTION_NOT_BOUND", "message": "no binding"}, + ) + ) + ) + try: + transport._do_flush() + finally: + transport._client.close() + + assert not os.path.exists(transport._wal_dlq_path()) + assert os.path.exists(transport._wal_inflight_path()), ( + "the only durable copy of an unrecorded refusal was discarded" + ) + assert transport._buffer == [], "a terminally refused event was put back on the send path" + assert [r["event"]["event_id"] for r in transport._dlq_overflow] == ["evt-1"] + assert metrics.transport.dlq_holdover == 1 + + +def test_a_held_refusal_survives_a_kill_minus_nine(transport, tmp_path, monkeypatch): + """The user's question: are held events durable, or only in memory? + + A memory-only holdover is lost by `kill -9`, and the fix that introduced + it looked safe because `.inflight` was still on disk holding the batch the + refusal came from. It is not: the next flush calls `_persist_inflight` + with ITS batch, which overwrites the file. So a process that holds a + refusal and then flushes again has silently overwritten the only + on-disk trace of it, and the events are in the DLQ nowhere, in `.wal` + nowhere, and in `.inflight` overwritten. + + So the test does what a kill does, not what a graceful stop does: it + throws the Transport away and builds a NEW one over the same WAL path, + with no flush, no `stop()`, and no chance for the dying process to write + anything. A fresh instance that finds the held event proves the record + outlived the process; one that finds nothing proves the loss. + """ + _dlq_never_fits(monkeypatch) + transport.track(_event(1)) + transport._client = httpx.Client( + transport=httpx.MockTransport( + lambda r: httpx.Response( + 422, + json={"error_code": "EXECUTION_NOT_BOUND", "message": "no binding"}, + ) + ) + ) + try: + transport._do_flush() + finally: + transport._client.close() + + assert transport._dlq_overflow, "nothing was held, so this test proves nothing" + + # The overwrite that loses a memory-only holdover. Same process, so this + # is the *benign* case; a kill needs no help from anyone. + transport._persist_inflight([_event(99)]) + + recovered = Transport(api_url=transport.api_url, api_key="test-key-12345678") + try: + recovered._recover_holdover() + assert [r["event"]["event_id"] for r in recovered._dlq_overflow] == ["evt-1"], ( + "a fresh process over the same WAL does not find the held refusal: " + "it lived in memory only and a kill -9 would have lost it" + ) + assert recovered._buffer == [], "a refused event was put back on the send path" + finally: + recovered.stop(flush=False) + + +def test_a_recovered_holdover_drains_without_being_resent(transport, monkeypatch): + """Recovery restores the hold, not the send. Re-sending is the bug. + + A refusal is terminal: sending it again produces the identical refusal + and occupies the head of the buffer while it does. If recovery put these + events back on the send path, the restart would reintroduce exactly the + head-of-buffer starvation the holdover was added to stop — and it would do + it on every restart, which is the condition under which an operator is + most likely to be watching. + """ + _dlq_never_fits(monkeypatch) + transport.track(_event(1)) + transport._client = httpx.Client( + transport=httpx.MockTransport( + lambda r: httpx.Response( + 422, + json={"error_code": "EXECUTION_NOT_BOUND", "message": "no binding"}, + ) + ) + ) + try: + transport._do_flush() + finally: + transport._client.close() + + sent: list[list[str]] = [] + + def _handler(request: httpx.Request) -> httpx.Response: + sent.append([e["event_id"] for e in json.loads(request.content)["events"]]) + return httpx.Response(200, json={"processed": 0, "accepted_event_ids": []}) + + recovered = Transport(api_url=transport.api_url, api_key="test-key-12345678") + try: + recovered._recover_holdover() + recovered._client = httpx.Client(transport=httpx.MockTransport(_handler)) + # The DLQ still has no room, so this is a no-op — which is the point. + # What must not happen is a send. + assert recovered._drain_dlq_overflow() == 0 + assert sent == [], f"a recovered refusal was re-sent: {sent}" + assert [r["event"]["event_id"] for r in recovered._dlq_overflow] == ["evt-1"] + finally: + recovered._client.close() + recovered.stop(flush=False) + + +def test_a_drain_releases_only_the_drained_rows_from_the_holdover( + transport, monkeypatch +): + """Partial drain must not delete the rows it did not write. + + The holdover file can hold MORE than the in-memory index: the index is + bounded by NULLRUN_DLQ_HOLDOVER_MAX_EVENTS and drops its oldest entries, + and those rows are still owed a DLQ. Releasing the file wholesale after a + partial drain would delete exactly the rows nobody tracked — the events + the bound was introduced to stop the SDK from caring about, deleted + silently by the mechanism meant to protect them. + """ + monkeypatch.setenv("NULLRUN_DLQ_MAX_BYTES", "1000000") + transport.track(_event(1)) + transport.track(_event(2)) + # Force the hold for both, so the file has two rows. + transport._hold_for_dlq( + [transport._dlq_row(_event(1), "EXECUTION_NOT_BOUND:422"), + transport._dlq_row(_event(2), "EXECUTION_NOT_BOUND:422")], + "EXECUTION_NOT_BOUND:422", + ) + assert os.path.exists(transport._wal_holdover_path()) + + # Truncate the index, as the cap would, leaving one row tracked. + monkeypatch.setenv("NULLRUN_DLQ_HOLDOVER_MAX_EVENTS", "1") + transport._trim_holdover_index() + assert [r["event"]["event_id"] for r in transport._dlq_overflow] == ["evt-2"] + + assert transport._drain_dlq_overflow() == 1 + + rows, _ = read_dlq(transport._wal_holdover_path()) + remaining = [r["event"]["event_id"] for r in rows if isinstance(r.get("event"), dict)] + assert remaining == ["evt-1"], ( + f"the drained row and/or the untracked row are wrong: {remaining}" + ) + + +def test_the_holdover_index_is_bounded_and_says_so(transport, monkeypatch, caplog): + """A bound with a metric, not a bound that quietly drops events. + + Two properties, because they are the same decision: the index must not + grow without limit while the DLQ stays full, and the truncation must be + visible — a bound that silently discards makes `dlq_holdover` a liar, + and a metric that under-reports is worse than no metric. + """ + monkeypatch.setenv("NULLRUN_DLQ_HOLDOVER_MAX_EVENTS", "3") + for i in range(1, 8): + transport._hold_for_dlq( + [transport._dlq_row(_event(i), "EXECUTION_NOT_BOUND:422")], + "EXECUTION_NOT_BOUND:422", + ) + + assert len(transport._dlq_overflow) == 3, "the index is unbounded" + assert [r["event"]["event_id"] for r in transport._dlq_overflow] == [ + "evt-5", "evt-6", "evt-7", + ], "the bound kept the wrong end; the oldest are the ones still owed" + assert metrics.transport.dlq_holdover_index_truncated == 4 + assert "NOT lost" in caplog.text, "truncation was not reported as non-loss" + + # The data is not truncated, only the index — this is the whole claim. + rows, _ = read_dlq(transport._wal_holdover_path()) + assert len([r for r in rows if isinstance(r.get("event"), dict)]) == 7 + + +def test_a_holdover_that_cannot_be_written_says_the_event_is_in_memory_only( + transport, monkeypatch, caplog +): + """The one holdover counter that means possible LOSS must say so. + + Every other holdover outcome is deferred recording, and a restart still + gets the rows. A failed holdover write is different: the rows are in RAM + and nowhere else, and a kill takes them. Silently continuing would make + the failure indistinguishable from the healthy case at the only point + where anyone could still act on it. + """ + monkeypatch.setenv("NULLRUN_DLQ_MAX_BYTES", "1") + transport.track(_event(1)) + transport._client = httpx.Client( + transport=httpx.MockTransport( + lambda r: httpx.Response( + 422, + json={"error_code": "EXECUTION_NOT_BOUND", "message": "no binding"}, + ) + ) + ) + # The holdover write is the thing under test, so it is the thing that fails. + monkeypatch.setattr( + Transport, "_write_events_atomic", + lambda self, path, events, mode="w": False, + ) + try: + transport._do_flush() + finally: + transport._client.close() + + assert transport._dlq_overflow, "nothing was held" + assert metrics.transport.dlq_holdover_persist_failures == 1 + assert "in memory only" in caplog.text + + +def _dlq_never_fits(monkeypatch) -> None: + """Make every DLQ write fail, whatever the size.""" + monkeypatch.setenv("NULLRUN_DLQ_MAX_BYTES", "1") + + +def test_a_full_dlq_does_not_block_the_healthy_events_behind_it(transport, monkeypatch): + """The whole point: one refused event must not stop the stream. + + A full DLQ plus a refused event is a condition the SDK creates, not one + the operator creates, so it must not cost the delivery of every healthy + event behind it. Before the holdover, the refused event was re-queued at + the HEAD of the buffer and re-sent on every cycle — refused again, held + again — and the buffer never drained. The events that were fine sat + behind it forever. + """ + _dlq_never_fits(monkeypatch) + sent: list[list[str]] = [] + + def _handler(request: httpx.Request) -> httpx.Response: + ids = [e["event_id"] for e in json.loads(request.content)["events"]] + sent.append(ids) + return httpx.Response( + 200, + json={ + "processed": len(ids), + "accepted_event_ids": [i for i in ids if i.startswith("good")], + "rejection_details": [ + {"event_id": i, "reason": "reservation_not_found"} + for i in ids + if i.startswith("bad") + ], + "rejected_count": sum(1 for i in ids if i.startswith("bad")), + }, + ) + + transport._client = httpx.Client(transport=httpx.MockTransport(_handler)) + try: + for i, name in ((1, "bad-1"), (2, "good-1"), (3, "good-2")): + event = _event(i) + event["event_id"] = name + transport.track(event) + transport._do_flush() + assert transport._buffer == [], "healthy events were held up by the refused one" + assert {"good-1", "good-2"} <= set(sent[-1]), "the healthy events were not delivered" + + # And nothing keeps being re-sent on a cycle where nothing changed. + before = len(sent) + transport._do_flush() + transport._do_flush() + assert len(sent) == before, "a terminally refused event was re-sent on every cycle" + finally: + transport._client.close() + + assert [r["event"]["event_id"] for r in transport._dlq_overflow] == ["bad-1"] + + +def test_a_whole_batch_refusal_is_bisected_before_anything_is_parked(transport, monkeypatch): + """A batch-level refusal is not evidence about every event in it. + + The backend refused the whole request because of the one event in it that + cannot be recorded. Quarantining the batch on that evidence files 49 + healthy events as refused for a reason that never applied to them — and + with no DLQ space it also re-queues the batch, so the healthy events are + re-sent and refused forever. + """ + _dlq_never_fits(monkeypatch) + sent: list[list[str]] = [] + + def _handler(request: httpx.Request) -> httpx.Response: + ids = [e["event_id"] for e in json.loads(request.content)["events"]] + sent.append(ids) + # Refuses only while the offender is still travelling with the batch. + if "bad-1" in ids: + return httpx.Response( + 422, + json={"error_code": "EXECUTION_NOT_BOUND", "message": "no binding"}, + ) + return httpx.Response( + 200, + json={ + "processed": len(ids), + "accepted_event_ids": ids, + "rejection_details": [], + "rejected_count": 0, + }, + ) + + transport._client = httpx.Client(transport=httpx.MockTransport(_handler)) + try: + for i, name in ((1, "bad-1"), (2, "good-1"), (3, "good-2")): + event = _event(i) + event["event_id"] = name + transport.track(event) + transport._do_flush() + finally: + transport._client.close() + + assert transport._buffer == [], "the batch was left on the retry path" + assert len(sent) > 1, "the refusal was not bisected, so the offender was never isolated" + assert {"good-1", "good-2"} <= {i for call in sent for i in call if i.startswith("good")} + held = [r["event"]["event_id"] for r in transport._dlq_overflow] + assert held == ["bad-1"], f"only the offender should be held, got {held}" + + +def test_a_held_refusal_lands_as_soon_as_the_dlq_has_room(transport, monkeypatch): + """The holdover defers the write; it must not turn into a silent drop. + + If space never frees, the operator is left with a DLQ that is short the + events it is supposed to hold and no indication that they exist anywhere. + So the held rows are re-attempted on every flush and land the moment the + write can succeed. + """ + _dlq_never_fits(monkeypatch) + transport._dlq_overflow.append(transport._dlq_row(_event(1), "reservation_not_found")) + assert transport._write_dlq_rows([transport._dlq_row(_event(2), "synthetic")]) is False + assert len(transport._dlq_overflow) == 1, "a refused write must not drop what is held" + + monkeypatch.setenv("NULLRUN_DLQ_MAX_BYTES", str(8 * 1024 * 1024)) + transport._client = httpx.Client( + transport=httpx.MockTransport( + lambda r: httpx.Response(200, json={"processed": 0, "accepted_event_ids": []}) + ) + ) + try: + transport._do_flush() + finally: + transport._client.close() + + assert transport._dlq_overflow == [] + rows = _read_wal(transport._wal_dlq_path()) + assert [r["event"]["event_id"] for r in rows] == ["evt-1"] + assert metrics.transport.dlq_holdover == 0 + + +def test_a_bisect_cascade_spends_a_bounded_number_of_requests(transport, monkeypatch): + """Halving is 2^depth sends. The depth guard does not bound that. + + A batch the backend refuses as a unit costs one request per singleton + explored. Without a request budget an all-bad batch spends the operator's + rate limit rediscovering what the first response already said — and a + refusal is exactly what a rate limiter starts answering, so the cascade + feeds the condition it is diagnosing. + + The claim is that the cost is a function of the BUDGET, not of the batch: + 1024 events must not cost more requests than 16. + """ + monkeypatch.setenv("NULLRUN_MAX_BISECT_REQUESTS", "4") + sent: list[int] = [] + + def _handler(request: httpx.Request) -> httpx.Response: + sent.append(len(json.loads(request.content)["events"])) + return httpx.Response( + 422, json={"error_code": "EXECUTION_NOT_BOUND", "message": "no binding"} + ) + + costs: list[int] = [] + # `track` auto-flushes at `batch_size` (50 by default), and each flush + # gets a fresh budget — so leaving it on would measure 20 flushes of 1024 + # against 1 flush of 16. The claim under test is the cost of isolating an + # offender WITHIN one batch, so one batch has to reach the wire. The + # config is read at construction, so this cannot be set via env here. + transport.config.batch_size = 100_000 + for size in (16, 1024): + sent.clear() + transport._buffer.clear() + transport._dlq_overflow.clear() + before_parked = len(_read_wal(transport._wal_dlq_path())) + transport._client = httpx.Client(transport=httpx.MockTransport(_handler)) + try: + for i in range(size): + event = _event(i) + event["event_id"] = f"b{size}-{i}" + transport.track(event) + assert len(transport._buffer) == size, "the buffer flushed before the test could" + transport._do_flush() + finally: + transport._client.close() + costs.append(len(sent)) + assert transport._buffer == [], "the un-split remainder was left on the send path" + # Every event of THIS batch is accounted for: newly parked in the DLQ, + # or held pending room. The holdover drains on the next flush, so it + # may legitimately be empty by the second iteration — what must never + # happen is an event that is in neither. + parked = len(_read_wal(transport._wal_dlq_path())) - before_parked + recorded = parked + len(transport._dlq_overflow) + assert recorded == size, f"{recorded} of {size} events recorded anywhere" + + # One request for the batch, then the budget's worth of splits. The slack + # is the halves already dispatched when the budget ran out. + assert costs[0] <= 1 + 4 + 2, f"a 16-event batch cost {costs[0]} requests" + assert costs[1] == costs[0], ( + f"cost scaled with batch size: {costs[0]} requests for 16 events, " + f"{costs[1]} for 1024" + ) + assert metrics.transport.batches_bisect_budget_exhausted >= 2 + + +# --------------------------------------------------------------------------- +# Replay. A supported path for the events nothing else will read. +# --------------------------------------------------------------------------- + + +def _seed_dlq(transport) -> None: + for i, reason in ((1, "reservation_not_found"), (2, "reservation_not_found"), (3, "budget_exceeded")): + transport._write_dlq_rows([transport._dlq_row(_event(i), reason)]) + + +def test_status_reports_what_is_parked_and_never_edits(tmp_path, monkeypatch, capsys): + monkeypatch.setenv("NULLRUN_WAL_PATH", str(tmp_path / "sdk.wal")) + t = Transport(api_url="https://api.test.nullrun.io", api_key="test-key-12345678") + try: + _seed_dlq(t) + dlq = t._wal_dlq_path() + before = open(dlq, "rb").read() + assert wal_main(["--wal", str(tmp_path / "sdk.wal"), "status"]) == 0 + assert open(dlq, "rb").read() == before, "`status` is a read-only command" + finally: + t._client.close() + + out = capsys.readouterr().out + assert "3 row(s)" in out + assert "reservation_not_found" in out + assert "budget_exceeded" in out + + +def test_replay_is_a_dry_run_unless_explicitly_told_otherwise( + tmp_path, monkeypatch, capsys +): + monkeypatch.setenv("NULLRUN_WAL_PATH", str(tmp_path / "sdk.wal")) + t = Transport(api_url="https://api.test.nullrun.io", api_key="test-key-12345678") + try: + _seed_dlq(t) + dlq = t._wal_dlq_path() + before = open(dlq, "rb").read() + assert ( + wal_main( + [ + "--wal", + str(tmp_path / "sdk.wal"), + "replay", + "--api-key", + "k-12345678", + ] + ) + == 0 + ) + assert open(dlq, "rb").read() == before, "a replay with no --execute deleted rows" + finally: + t._client.close() + assert "dry run" in capsys.readouterr().out + + +def test_replay_removes_only_the_events_the_backend_confirmed(tmp_path, monkeypatch, capsys): + """Two confirmed, one refused: the refused row stays. + + This is the whole contract of the command. Dropping a row because we + re-sent it would be indistinguishable from dropping it because it landed; + a refusal that is still current must survive the attempt to fix it, or + the operator's evidence disappears exactly when it is most useful. + """ + monkeypatch.setenv("NULLRUN_WAL_PATH", str(tmp_path / "sdk.wal")) + t = Transport(api_url="https://api.test.nullrun.io", api_key="test-key-12345678") + try: + _seed_dlq(t) + dlq = t._wal_dlq_path() + + def _handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={ + "processed": 2, + "accepted_event_ids": ["evt-1", "evt-2"], + "rejection_details": [ + {"event_id": "evt-3", "reason": "budget_exceeded"} + ], + "rejected_count": 1, + }, + ) + + # The CLI builds its own Transport, so the mock has to be installed on + # the class rather than on an instance. + original = Transport.__init__ + + def _patched_init(self, *a, **kw): + original(self, *a, **kw) + self._client.close() + self._client = httpx.Client(transport=httpx.MockTransport(_handler)) + + monkeypatch.setattr(Transport, "__init__", _patched_init) + assert ( + wal_main( + [ + "--wal", + str(tmp_path / "sdk.wal"), + "replay", + "--execute", + "--api-key", + "k-12345678", + ] + ) + == 0 + ) + finally: + t._client.close() + + remaining = [r["event"]["event_id"] for r in _read_wal(dlq)] + assert remaining == ["evt-3"], f"an unconfirmed row was removed: {remaining}" + assert "sent 2 event(s)" in capsys.readouterr().out + + +def _refusing_backend(monkeypatch, status: int, code: str) -> list[httpx.Request]: + """Install a class-level mock that refuses EVERY event with ``code``. + + Returns the list of requests it saw, so a test can assert on what was + sent as well as on what the command did with the reply. + """ + seen: list[httpx.Request] = [] + original = Transport.__init__ + + def _handler(request: httpx.Request) -> httpx.Response: + seen.append(request) + return httpx.Response( + status, json={"error_code": code, "error_message": "no binding"} + ) + + def _patched_init(self, *a, **kw): + original(self, *a, **kw) + self._client.close() + self._client = httpx.Client(transport=httpx.MockTransport(_handler)) + + monkeypatch.setattr(Transport, "__init__", _patched_init) + return seen + + +def test_replay_that_is_refused_again_keeps_the_row_and_reports_failure( + tmp_path, monkeypatch, capsys +): + """The whole batch refused ⇒ the evidence stays and the exit code is 1. + + This is the case the existing partial test cannot reach: there, one + confirmed event masked the accounting. With every event refused, nothing + remains in the transport's buffer, so a check that only looked there + concluded "all confirmed" and DELETED every row — reporting success after + the backend had just rejected all of them. A refused event lives in the + scratch DLQ or in the DLQ holdover, not in the buffer; all three are read. + """ + monkeypatch.setenv("NULLRUN_WAL_PATH", str(tmp_path / "sdk.wal")) + t = Transport(api_url="https://api.test.nullrun.io", api_key="test-key-12345678") + dlq = t._wal_dlq_path() + try: + _seed_dlq(t) + finally: + t._client.close() + _refusing_backend(monkeypatch, 400, "VALIDATION_FAILED") + + rc = wal_main( + ["--wal", str(tmp_path / "sdk.wal"), "replay", "--execute", "--api-key", "k-12345678"] + ) + captured = capsys.readouterr() + assert rc == 1, f"a fully-refused replay reported success: {captured.out}" + remaining = [r["event"]["event_id"] for r in _read_wal(dlq)] + assert remaining == ["evt-1", "evt-2", "evt-3"], ( + f"a refused replay deleted the operator's only record: {remaining}" + ) + assert "not confirmed" in captured.err + + +def test_replay_withholds_a_refusal_re_sending_cannot_fix( + tmp_path, monkeypatch, capsys +): + """An expired 24h binding is not a transport fault; nothing recovers it. + + Re-sending produces the identical 422, so the default is to leave the row + and say why. `--include-terminal` is the operator overriding that. + """ + monkeypatch.setenv("NULLRUN_WAL_PATH", str(tmp_path / "sdk.wal")) + t = Transport(api_url="https://api.test.nullrun.io", api_key="test-key-12345678") + dlq = t._wal_dlq_path() + try: + for i in (1, 2): + t._write_dlq_rows([t._dlq_row(_event(i), "EXECUTION_NOT_BOUND:422")]) + finally: + t._client.close() + + sent = _refusing_backend(monkeypatch, 422, "EXECUTION_NOT_BOUND") + + rc = wal_main( + ["--wal", str(tmp_path / "sdk.wal"), "replay", "--execute", "--api-key", "k-12345678"] + ) + err = capsys.readouterr().err + assert sent == [], f"a terminal refusal was re-sent: {len(sent)} request(s)" + assert rc == 1 + assert "cannot fix" in err and "EXECUTION_NOT_BOUND:422" in err + assert [r["event"]["event_id"] for r in _read_wal(dlq)] == ["evt-1", "evt-2"] + + +def test_include_terminal_sends_them_anyway_and_does_not_duplicate( + tmp_path, monkeypatch, capsys +): + """The override sends; the rows stay, because the send still failed.""" + monkeypatch.setenv("NULLRUN_WAL_PATH", str(tmp_path / "sdk.wal")) + t = Transport(api_url="https://api.test.nullrun.io", api_key="test-key-12345678") + dlq = t._wal_dlq_path() + try: + t._write_dlq_rows([t._dlq_row(_event(1), "EXECUTION_NOT_BOUND:422")]) + finally: + t._client.close() + _refusing_backend(monkeypatch, 422, "EXECUTION_NOT_BOUND") + + rc = wal_main( + [ + "--wal", + str(tmp_path / "sdk.wal"), + "replay", + "--execute", + "--include-terminal", + "--api-key", + "k-12345678", + ] + ) + assert rc == 1, "a refused --include-terminal send reported success" + rows = _read_wal(dlq) + assert [r["event"]["event_id"] for r in rows] == ["evt-1"] + assert len(rows) == len({r["event"]["event_id"] for r in rows}), "the DLQ grew" + assert "not confirmed" in capsys.readouterr().err + + +def test_an_empty_dlq_is_success_not_a_failure(tmp_path, monkeypatch): + """rc 0 on an empty DLQ, so a scheduled repair is not permanently red.""" + monkeypatch.setenv("NULLRUN_WAL_PATH", str(tmp_path / "sdk.wal")) + assert ( + wal_main( + ["--wal", str(tmp_path / "sdk.wal"), "replay", "--execute", "--api-key", "k-12345678"] + ) + == 0 + ) + + +def test_replay_preserves_a_line_it_cannot_parse(tmp_path): + """The unparseable line is the one the operator most needs to see. + + Rewriting the file from the rows that DID parse would delete it without + anyone noticing — the file still looks complete, it is just quietly + shorter. It stays. + """ + dlq = tmp_path / "sdk.wal.dlq" + dlq.write_text( + json.dumps({"reason": "r", "event": {"event_id": "evt-1"}}) + "\n" + + "{not json at all\n" + + json.dumps({"reason": "r", "event": {"event_id": "evt-2"}}) + "\n" + ) + removed = _drop_confirmed(str(dlq), {"evt-1"}) + assert removed == 1 + lines = dlq.read_text().splitlines() + assert lines == ["{not json at all", json.dumps({"reason": "r", "event": {"event_id": "evt-2"}})] + + +def test_a_dlq_with_a_corrupt_line_is_still_readable(tmp_path): + """One bad line must not make the recovery file unreadable. + + A reader that required every line to parse would convert a single partial + write — the exact thing a crash produces — into total loss of the file. + """ + dlq = tmp_path / "sdk.wal.dlq" + dlq.write_text( + json.dumps({"reason": "r", "event": {"event_id": "evt-1"}}) + "\n" + "{truncated\n" + ) + rows, corrupt = read_dlq(str(dlq)) + assert [r["event"]["event_id"] for r in rows] == ["evt-1"] + assert len(corrupt) == 1 + + +def test_replay_needs_an_api_key_before_it_does_anything(tmp_path, monkeypatch, capsys): + monkeypatch.delenv("NULLRUN_API_KEY", raising=False) + dlq = tmp_path / "sdk.wal.dlq" + dlq.write_text(json.dumps({"reason": "r", "event": _event(1)}) + "\n") + before = dlq.read_bytes() + rc = wal_main(["--wal", str(tmp_path / "sdk.wal"), "replay", "--execute"]) + assert rc == 1 + assert dlq.read_bytes() == before + assert "no API key" in capsys.readouterr().err + + +def test_a_200_without_accepted_event_ids_removes_nothing(tmp_path, monkeypatch): + """The response-shape case: a 2xx that cannot be read as a confirmation. + + A proxy's HTML page, an empty body, or a backend predating the field + all produce a 200 that says nothing about which events landed. Reading + it as "sent, therefore confirmed" deletes rows on the strength of a + response that never mentioned them — and a 2xx is exactly what a + misconfigured proxy returns. + + This is the fourth way `replay` can lose a row, and the three covered + above (HTTP error, refusal named in `rejection_details`, explicit + partial list) all come from a backend that is answering in the + contract's shape. This one does not. + """ + monkeypatch.setenv("NULLRUN_WAL_PATH", str(tmp_path / "sdk.wal")) + t = Transport(api_url="https://api.test.nullrun.io", api_key="test-key-12345678") + try: + _seed_dlq(t) + dlq = t._wal_dlq_path() + + def _handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, text="proxy error") + + original = Transport.__init__ + + def _patched_init(self, *a, **kw): + original(self, *a, **kw) + self._client.close() + self._client = httpx.Client(transport=httpx.MockTransport(_handler)) + + monkeypatch.setattr(Transport, "__init__", _patched_init) + rc = wal_main( + [ + "--wal", + str(tmp_path / "sdk.wal"), + "replay", + "--execute", + "--api-key", + "k-12345678", + ] + ) + finally: + t._client.close() + + assert rc == 1, f"an unreadable 200 was treated as a success: rc={rc}" + remaining = [r["event"]["event_id"] for r in _read_wal(dlq)] + assert remaining == ["evt-1", "evt-2", "evt-3"], ( + f"rows were removed on the strength of a 200 that named no events: {remaining}" + ) diff --git a/uv.lock b/uv.lock index 7c6af19..10d4c14 100644 --- a/uv.lock +++ b/uv.lock @@ -636,7 +636,7 @@ wheels = [ [[package]] name = "nullrun" -version = "0.21.0" +version = "0.22.0" source = { editable = "." } dependencies = [ { name = "httpx" },