From d34706526d149f8a47baa6c6649b8a8a358cf07b Mon Sep 17 00:00:00 2001 From: raymondginger Date: Wed, 28 Oct 2026 13:05:47 +0800 Subject: [PATCH] Rework goal verifier per review: single async gate, real objective+evidence, blocked-after-N, drop unused classes Address @Zongwei9888's five review points on the goal completion verifier: 1. Verifier runs exactly once, asynchronously, before any durable update. GoalRuntimeRouter.request() is now async and awaits the verifier once; there is no 'verified' bypass parameter. A rejection returns data and never calls handler.update_goal, so no completion is recorded. A sync verifier that raises is caught by the same try and fails closed (rejects) instead of escaping as a raw RuntimeError. 2. The claim handed to the verifier carries the real goal objective (no longer hardcoded empty) plus this Turn's existing evidence references, via GoalExtension.completion_request(), reusing the outcome's evidence filter and caps. 3. A rejection returns the verifier's justification to the model as a structured dict; after max_rejections (default 3) the router itself moves the Goal to blocked and freezes the Turn instead of letting the model burn Turns retrying. 4. Real caller: build_agent_session arms RuleBasedVerifier when config.goal.completion_verifier == 'rule' (new GoalConfig, default 'off' preserves existing behavior). E2e test drives UpdateGoalTool through the ToolRegistry with an async verifier that rejects. 5. Drop the unused GoalVerifierRejected and GoalCompletionVerifierPolicy classes and the .verify protocol; goal_verifier.py is stdlib-only, import-inert, and has no references to other products. --- core/agent_runtime/goal_runtime.py | 132 ++++++- core/agent_runtime/goal_verifier.py | 132 +++++++ core/agent_setup.py | 9 + core/application/goal_extension.py | 55 ++- core/config.py | 17 + core/harness/tools/goal.py | 2 +- tests/app_server/test_server.py | 2 +- .../application/test_automation_goal_runs.py | 2 +- tests/application/test_automation_service.py | 2 +- tests/test_automation_cli.py | 2 +- tests/test_automation_foreground.py | 2 +- tests/test_goal_completion_verifier.py | 374 ++++++++++++++++++ tests/test_goal_runtime.py | 7 +- 13 files changed, 726 insertions(+), 12 deletions(-) create mode 100644 core/agent_runtime/goal_verifier.py create mode 100644 tests/test_goal_completion_verifier.py diff --git a/core/agent_runtime/goal_runtime.py b/core/agent_runtime/goal_runtime.py index 286bb51d2..95f38a9ea 100644 --- a/core/agent_runtime/goal_runtime.py +++ b/core/agent_runtime/goal_runtime.py @@ -4,7 +4,14 @@ import threading from dataclasses import dataclass -from typing import Any, Protocol +from typing import TYPE_CHECKING, Any, Protocol + +if TYPE_CHECKING: + from core.agent_runtime.goal_verifier import ( + GoalCompletionRequest, + GoalCompletionVerifier, + GoalVerdict, + ) GOAL_TOOL_NAMES = frozenset({"get_goal", "update_goal"}) @@ -45,20 +52,67 @@ def update_goal( reason: str | None, ) -> dict[str, Any]: ... + def completion_request( + self, + context: GoalRuntimeContext, + *, + reason: str | None, + ) -> GoalCompletionRequest: + """Build the claim a configured verifier judges. + + Only consulted while a completion verifier is configured; carries the + real objective and this Turn's existing evidence references. + """ + ... + class GoalRuntimeRouter: - """Route Goal tools only while one attributed Turn owns the runtime.""" + """Route Goal tools only while one attributed Turn owns the runtime. + + When a completion verifier is configured (see + :meth:`configure_completion_verifier`), a ``complete`` request is judged + exactly once, asynchronously, before any durable update. A rejection is + returned to the model as data — with the verifier's justification — and + the goal is not updated. After ``max_rejections`` rejected claims the + router moves the goal to ``blocked`` itself, so a model that keeps + re-claiming completion cannot burn Turns forever. + """ def __init__(self) -> None: self._lock = threading.Lock() self._handler: GoalRuntimeHandler | None = None self._context: GoalRuntimeContext | None = None self._terminal_requested = False + self._verifier: GoalCompletionVerifier | None = None + self._max_rejections = 3 + # Rejected completion claims per goal, so a model cannot burn Turns + # retrying the same claim; cleared on success or auto-block. + self._rejections: dict[str, int] = {} def configure(self, handler: GoalRuntimeHandler) -> None: with self._lock: self._handler = handler + def configure_completion_verifier( + self, + verifier: GoalCompletionVerifier, + *, + max_rejections: int = 3, + ) -> None: + """Arm the single asynchronous completion gate. + + There is deliberately no way to pass a pre-computed verdict into + :meth:`request`; every ``complete`` claim goes through ``verifier`` + exactly once. + """ + + if max_rejections < 1: + raise ValueError("max_rejections must be positive") + with self._lock: + self._verifier = verifier + self._max_rejections = max_rejections + self._rejections.clear() + def activate(self, context: GoalRuntimeContext) -> None: with self._lock: if self._context is not None and self._context.turn_id != context.turn_id: @@ -79,13 +133,33 @@ def read(self) -> dict[str, Any]: handler, context = self._snapshot() return handler.read_goal(context) - def request( + async def request( self, *, status: str, reason: str | None, ) -> dict[str, Any]: + """Record one terminal decision, verifying a ``complete`` claim once.""" + handler, context = self._snapshot() + with self._lock: + verifier = self._verifier + + if status == "complete" and verifier is not None: + from core.agent_runtime.goal_verifier import GoalVerdict + + try: + verdict = await verifier( + handler.completion_request(context, reason=reason) + ) + except Exception as exc: # noqa: BLE001 - fail closed, report to the model + verdict = GoalVerdict( + confirmed=False, + justification=f"Completion verification failed: {exc}", + ) + if not verdict.confirmed: + return self._reject_completion(handler, context, verdict) + result = handler.update_goal( context, status=status, @@ -95,8 +169,60 @@ def request( if self._context != context: raise GoalRuntimeError("Goal Turn changed while recording the decision") self._terminal_requested = True + self._rejections.pop(context.goal_id, None) return result + def _reject_completion( + self, + handler: GoalRuntimeHandler, + context: GoalRuntimeContext, + verdict: GoalVerdict, + ) -> dict[str, Any]: + """Return a rejection to the model; auto-block after N rejections.""" + + with self._lock: + if self._context != context: + raise GoalRuntimeError("Goal Turn changed while recording the decision") + count = self._rejections.get(context.goal_id, 0) + 1 + self._rejections[context.goal_id] = count + max_rejections = self._max_rejections + exhausted = count >= max_rejections + + rejection: dict[str, Any] = { + "goalId": context.goal_id, + "updated": False, + "verdict": { + "confirmed": False, + "justification": verdict.justification, + }, + "rejections": count, + "maxRejections": max_rejections, + } + if not exhausted: + return rejection + + # The claim is exhausted: move the Goal to blocked (a terminal + # decision the model asked for by never producing evidence) instead + # of letting it burn Turns on the same rejected completion claim. + result = handler.update_goal( + context, + status="blocked", + reason=verdict.justification, + ) + with self._lock: + if self._context != context: + raise GoalRuntimeError("Goal Turn changed while recording the decision") + self._terminal_requested = True + self._rejections.pop(context.goal_id, None) + return { + **result, + "updated": True, + "verdict": rejection["verdict"], + "rejections": count, + "maxRejections": max_rejections, + "autoBlocked": True, + } + def visible_tool_names(self, names: tuple[str, ...]) -> tuple[str, ...]: """Narrow tools dynamically without ever adding a capability.""" diff --git a/core/agent_runtime/goal_verifier.py b/core/agent_runtime/goal_verifier.py new file mode 100644 index 000000000..75fc5d163 --- /dev/null +++ b/core/agent_runtime/goal_verifier.py @@ -0,0 +1,132 @@ +"""Independent verification of a Goal completion claim. + +The verifier answers one question: does this Turn's evidence support the +model's claim that the Goal is complete? It is consulted exactly once per +completion request, asynchronously, inside the goal runtime — there is no +synchronous side door and no caller-supplied "already verified" flag. + +Rejections are data, not exceptions. A rejected claim returns its +justification to the model so the model can either produce the missing +evidence or declare the Goal blocked; a verifier that raises or cannot be +asked fails closed (rejects). After a configured number of rejections the +goal runtime moves the Goal to ``blocked`` rather than letting the model +burn Turns retrying the same claim. +""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import Protocol + + +@dataclass(frozen=True, slots=True) +class GoalEvidenceItem: + """One Turn evidence item, reduced to what a verifier judges.""" + + kind: str + status: str + summary: str + + +@dataclass(frozen=True, slots=True) +class GoalCompletionRequest: + """The claim under review: the real objective plus Turn evidence.""" + + objective: str + reason: str + evidence: tuple[GoalEvidenceItem, ...] + + +@dataclass(frozen=True, slots=True) +class GoalVerdict: + """The verifier's answer. ``justification`` always travels to the model.""" + + confirmed: bool + justification: str + + +class GoalCompletionVerifier(Protocol): + """An async callable that judges one completion claim.""" + + def __call__(self, request: GoalCompletionRequest) -> Awaitable[GoalVerdict]: ... + + +class RuleBasedVerifier: + """Fail-closed structural checks over the objective, reason, and evidence. + + This is the built-in verifier behind the ``rule`` setting. It does not + judge whether the work is *good*; it judges whether the claim is + *supported*: a completion claim with no objective, no stated reason, or + no recorded Turn evidence is rejected with a justification the model can + act on. + """ + + async def __call__(self, request: GoalCompletionRequest) -> GoalVerdict: + if not request.objective.strip(): + return GoalVerdict( + confirmed=False, + justification="The Goal has no objective to verify the completion claim against.", + ) + if not request.reason.strip(): + return GoalVerdict( + confirmed=False, + justification="The completion claim states no reason. Explain what was done before marking the Goal complete.", + ) + if not request.evidence: + return GoalVerdict( + confirmed=False, + justification=( + "The Turn recorded no evidence (tool calls, file changes, or test " + "results) to support completion. Do the work first, then claim completion." + ), + ) + failed = [ + item + for item in request.evidence + if item.status.strip().lower() in {"failed", "error"} + ] + if failed: + return GoalVerdict( + confirmed=False, + justification=( + f"{len(failed)} of {len(request.evidence)} evidence item(s) failed " + f"(latest: {failed[-1].summary!r}). Resolve the failures before completing." + ), + ) + return GoalVerdict( + confirmed=True, + justification=( + f"{len(request.evidence)} Turn evidence item(s) support the claim." + ), + ) + + +def model_backed_verifier( + ask: Callable[[GoalCompletionRequest], Awaitable[str]], +) -> GoalCompletionVerifier: + """Adapt a one-shot text answerer into a fail-closed verifier. + + ``ask`` returns the model's answer; anything other than an affirmative + answer rejects, and an ``ask`` that raises rejects too (fail closed). + """ + + async def verify(request: GoalCompletionRequest) -> GoalVerdict: + try: + answer = (await ask(request)).strip().lower() + except Exception as exc: # noqa: BLE001 - fail closed, report to the model + return GoalVerdict( + confirmed=False, + justification=f"The completion verifier failed: {exc}", + ) + confirmed = answer.startswith("yes") + return GoalVerdict( + confirmed=confirmed, + justification=( + "The model verifier confirmed the evidence supports completion." + if confirmed + else "The model verifier rejected the completion claim." + ), + ) + + return verify diff --git a/core/agent_setup.py b/core/agent_setup.py index f1217d6bf..784f26850 100644 --- a/core/agent_setup.py +++ b/core/agent_setup.py @@ -339,6 +339,15 @@ def build_agent_session( mode_override=permission_mode_override, profile_override=execution_security_profile, ) + if goal_runtime is not None: + goal_cfg = getattr(active_runtime.config, "goal", None) + if goal_cfg is not None and goal_cfg.completion_verifier == "rule": + from core.agent_runtime.goal_verifier import RuleBasedVerifier + + goal_runtime.configure_completion_verifier( + RuleBasedVerifier(), + max_rejections=goal_cfg.max_rejections, + ) engine = build_permission_engine( security_cfg, cwd=workspace, diff --git a/core/application/goal_extension.py b/core/application/goal_extension.py index 2ae0afc65..f81775a8a 100644 --- a/core/application/goal_extension.py +++ b/core/application/goal_extension.py @@ -10,7 +10,7 @@ from enum import StrEnum from importlib.resources import files from pathlib import Path -from typing import Any, Iterator +from typing import TYPE_CHECKING, Any, Iterator from core.agent_runtime.goal_runtime import GoalRuntimeContext, GoalRuntimeHandler from core.application.errors import ( @@ -43,6 +43,9 @@ ) from core.skills.models import MAX_SELECTED_SKILLS, SkillSelection +if TYPE_CHECKING: + from core.agent_runtime.goal_verifier import GoalCompletionRequest + logger = logging.getLogger(__name__) # Without a token budget, only the model's update_goal ends a Goal. A model @@ -642,6 +645,56 @@ def update_goal( "updated": True, } + def completion_request( + self, + context: GoalRuntimeContext, + *, + reason: str | None, + ) -> GoalCompletionRequest: + """Build the verifier's view: the real objective and this Turn's evidence. + + The old branch hardcoded ``objective=""``; here the verifier judges the + actual objective the model is claiming to satisfy, plus the evidence + references this Turn already recorded (same filter as the outcome). + """ + + from core.agent_runtime.goal_verifier import ( + GoalCompletionRequest, + GoalEvidenceItem, + ) + + goal = self.read(context.thread_id) + objective = goal.objective if goal is not None else "" + try: + snapshot = self.turns.read(context.turn_id) + except TurnNotFoundError as exc: + logger.info( + "Goal completion Turn %s is unavailable: %s", + context.turn_id, + exc, + ) + return GoalCompletionRequest( + objective=objective, + reason=reason or "", + evidence=(), + ) + evidence = tuple( + GoalEvidenceItem( + kind=item.kind.value, + status=item.status.value, + summary=(item.summary.strip() or item.kind.value.replace("_", " "))[ + :GOAL_OUTCOME_EVIDENCE_SUMMARY_MAX_CHARS + ], + ) + for item in sorted(snapshot.items, key=lambda candidate: candidate.ordinal) + if item.kind in _EVIDENCE_KINDS + )[:GOAL_OUTCOME_EVIDENCE_MAX_ITEMS] + return GoalCompletionRequest( + objective=objective, + reason=reason or "", + evidence=evidence, + ) + def recover_incomplete(self) -> int: """Replay committed Goal Turns whose ledger settlement may be missing.""" diff --git a/core/config.py b/core/config.py index 0a53023d8..c79ed9915 100644 --- a/core/config.py +++ b/core/config.py @@ -414,6 +414,22 @@ class WorkspaceConfig(_Base): max_input_mb: int = 100 +class GoalConfig(_Base): + """Thread Goal completion verification policy. + + - ``completion_verifier``: ``"off"`` (default) keeps legacy behavior — the + model's ``update_goal(status="complete")`` is recorded directly. ``"rule"`` + arms the built-in fail-closed verifier: every ``complete`` claim is judged + exactly once, asynchronously, against the real objective plus this Turn's + evidence, and the verifier's justification is returned to the model. + - ``max_rejections``: after this many rejected completion claims the Goal is + moved to ``blocked`` (instead of letting the model burn Turns retrying). + """ + + completion_verifier: Literal["off", "rule"] = "off" + max_rejections: int = 3 + + class DictationConfig(_Base): """Voice input for the prompt box, transcribed by a Parakeet-style server. @@ -576,6 +592,7 @@ class DeepCodeConfig(BaseSettings): skills: SkillsConfig = Field(default_factory=SkillsConfig) security: SecurityConfig = Field(default_factory=SecurityConfig) workspace: WorkspaceConfig = Field(default_factory=WorkspaceConfig) + goal: GoalConfig = Field(default_factory=GoalConfig) # Absent means "voice input is off"; present means the user picked an # endpoint for it. A default *instance* could not tell those apart, and a # disabled microphone that still ships an endpoint would be a config that diff --git a/core/harness/tools/goal.py b/core/harness/tools/goal.py index 4c62b0d41..d6ee3b9d3 100644 --- a/core/harness/tools/goal.py +++ b/core/harness/tools/goal.py @@ -85,7 +85,7 @@ async def execute( status: str, reason: str | None = None, ) -> dict[str, Any]: - return self.runtime.request( + return await self.runtime.request( status=status, reason=reason, ) diff --git a/tests/app_server/test_server.py b/tests/app_server/test_server.py index 4254548cb..c40940d85 100644 --- a/tests/app_server/test_server.py +++ b/tests/app_server/test_server.py @@ -1423,7 +1423,7 @@ def load_history(self, _messages) -> None: async def run_stream(self, _op): yield Event("1", TurnStarted()) - self.goal_runtime.request( + await self.goal_runtime.request( status="complete", reason="Scheduled repository work and verification completed.", ) diff --git a/tests/application/test_automation_goal_runs.py b/tests/application/test_automation_goal_runs.py index 48676b539..7a27d4f0e 100644 --- a/tests/application/test_automation_goal_runs.py +++ b/tests/application/test_automation_goal_runs.py @@ -75,7 +75,7 @@ async def run_stream(self, op): while step.gate is not None and not step.gate.is_set(): await asyncio.sleep(0.005) if step.decision is not None: - self.goal_runtime.request( + await self.goal_runtime.request( status=step.decision, reason=f"Agent requested {step.decision} in scripted Turn {index}.", ) diff --git a/tests/application/test_automation_service.py b/tests/application/test_automation_service.py index 85142063c..b632ec065 100644 --- a/tests/application/test_automation_service.py +++ b/tests/application/test_automation_service.py @@ -43,7 +43,7 @@ async def run_stream(self, op): yield Event("1", TurnStarted()) if self.hang: await asyncio.Event().wait() - self.goal_runtime.request( + await self.goal_runtime.request( status="complete", reason="The requested automation and verification completed.", ) diff --git a/tests/test_automation_cli.py b/tests/test_automation_cli.py index 808b8f543..08d3c1a10 100644 --- a/tests/test_automation_cli.py +++ b/tests/test_automation_cli.py @@ -162,7 +162,7 @@ def load_history(self, messages) -> None: async def run_stream(self, op): self.history.append({"role": "user", "content": op.text}) yield Event("start", TurnStarted()) - self.goal_runtime.request( + await self.goal_runtime.request( status="complete", reason="Automation CLI verification completed.", ) diff --git a/tests/test_automation_foreground.py b/tests/test_automation_foreground.py index 980a20191..d1e604326 100644 --- a/tests/test_automation_foreground.py +++ b/tests/test_automation_foreground.py @@ -38,7 +38,7 @@ async def run_stream(self, operation): self._factory.decisions.append(approved) status = "complete" if approved else "blocked" reason = "verified result written" if approved else "approval denied" - self._goal_runtime.request(status=status, reason=reason) + await self._goal_runtime.request(status=status, reason=reason) yield Event("message", AgentMessage(reason)) yield Event("complete", TaskComplete(reason, status)) self.history.append({"role": "assistant", "content": reason}) diff --git a/tests/test_goal_completion_verifier.py b/tests/test_goal_completion_verifier.py new file mode 100644 index 000000000..ac5539dcb --- /dev/null +++ b/tests/test_goal_completion_verifier.py @@ -0,0 +1,374 @@ +"""Goal completion verifier: single async gate, justification, blocked-after-N.""" + +from __future__ import annotations + +import asyncio + +import pytest + +from core.agent_runtime.goal_runtime import ( + GoalRuntimeContext, + GoalRuntimeRouter, +) +from core.agent_runtime.goal_verifier import ( + GoalCompletionRequest, + GoalCompletionVerifier, + GoalEvidenceItem, + GoalVerdict, + RuleBasedVerifier, + model_backed_verifier, +) +from core.agent_runtime.tools.registry import ToolRegistry +from core.harness.tools.goal import UpdateGoalTool + + +class FakeGoalHandler: + """Application-owned durable operations, recorded for assertions.""" + + def __init__(self, objective: str = "Ship the feature.") -> None: + self.requests: list[dict[str, object]] = [] + self.objective = objective + + def read_goal(self, context: GoalRuntimeContext): + return {"goalId": context.goal_id, "objective": self.objective} + + def update_goal( + self, + context: GoalRuntimeContext, + *, + status: str, + reason: str | None, + ): + self.requests.append( + {"turnId": context.turn_id, "status": status, "reason": reason} + ) + return {"goalId": context.goal_id, "status": status} + + def completion_request( + self, + context: GoalRuntimeContext, + *, + reason: str | None, + ) -> GoalCompletionRequest: + return GoalCompletionRequest( + objective=self.objective, + reason=reason or "", + evidence=( + GoalEvidenceItem( + kind="test_result", + status="passed", + summary="Focused tests pass.", + ), + ), + ) + + +class RejectingVerifier: + """Async verifier that always rejects; counts calls to prove single-gate.""" + + def __init__(self, justification: str = "No file change evidence.") -> None: + self.calls = 0 + self.justification = justification + + async def __call__(self, request: GoalCompletionRequest) -> GoalVerdict: + self.calls += 1 + return GoalVerdict(confirmed=False, justification=self.justification) + + +class ConfirmingVerifier: + async def __call__(self, request: GoalCompletionRequest) -> GoalVerdict: + return GoalVerdict(confirmed=True, justification="Evidence supports it.") + + +class RaisingVerifier: + def __call__(self, request: GoalCompletionRequest) -> GoalVerdict: + raise RuntimeError("verifier backend unavailable") + + +def _context() -> GoalRuntimeContext: + return GoalRuntimeContext( + thread_id="session", + goal_id="goal_1", + turn_id="turn_1", + ) + + +@pytest.mark.asyncio +async def test_update_goal_tool_rejection_returns_justification_and_records_nothing() -> ( + None +): + """E2E through UpdateGoalTool: async verifier rejects, model sees why.""" + + runtime = GoalRuntimeRouter() + handler = FakeGoalHandler() + runtime.configure(handler) + verifier = RejectingVerifier(justification="No file change evidence.") + runtime.configure_completion_verifier(verifier) + runtime.activate(_context()) + registry = ToolRegistry() + registry.register(UpdateGoalTool(runtime)) + + result = await registry.execute( + "update_goal", + {"status": "complete", "reason": "The tests pass, so it is done."}, + ) + + # The verifier ran exactly once — no sync re-check, no bypass. + assert verifier.calls == 1 + # The justification travels back to the model. + assert result["updated"] is False + assert result["verdict"]["confirmed"] is False + assert result["verdict"]["justification"] == "No file change evidence." + assert result["rejections"] == 1 + # No durable completion was recorded; the goal stays open. + assert handler.requests == [] + # The terminal gate did not freeze: the model may keep working. + assert runtime.visible_tool_names(("read", "update_goal")) == ( + "read", + "update_goal", + ) + + +@pytest.mark.asyncio +async def test_goal_blocks_after_max_rejections_instead_of_burning_turns() -> None: + """N rejections move the goal to blocked and freeze the turn.""" + + runtime = GoalRuntimeRouter() + handler = FakeGoalHandler() + runtime.configure(handler) + verifier = RejectingVerifier() + runtime.configure_completion_verifier(verifier, max_rejections=2) + runtime.activate(_context()) + registry = ToolRegistry() + registry.register(UpdateGoalTool(runtime)) + + first = await registry.execute( + "update_goal", {"status": "complete", "reason": "claim one"} + ) + # First rejection is returned as data; nothing durable yet. + assert first["updated"] is False + assert first["rejections"] == 1 + assert handler.requests == [] + + second = await registry.execute( + "update_goal", {"status": "complete", "reason": "claim two"} + ) + + assert verifier.calls == 2 + # Second (max) rejection auto-blocks the goal and freezes the turn. + assert second["autoBlocked"] is True + assert second["verdict"]["justification"] + assert handler.requests == [ + { + "turnId": "turn_1", + "status": "blocked", + "reason": "No file change evidence.", + } + ] + assert runtime.visible_tool_names(("read", "update_goal")) == () + + +@pytest.mark.asyncio +async def test_confirming_verifier_records_completion_exactly_once() -> None: + """A confirmed claim records completion; the verifier is not called twice.""" + + runtime = GoalRuntimeRouter() + handler = FakeGoalHandler() + runtime.configure(handler) + calls = {"n": 0} + + async def verifier(request: GoalCompletionRequest) -> GoalVerdict: + calls["n"] += 1 + return GoalVerdict(confirmed=True, justification="ok") + + runtime.configure_completion_verifier(verifier) + runtime.activate(_context()) + + result = await runtime.request( + status="complete", reason="Feature shipped with tests." + ) + + assert calls["n"] == 1 + assert result["status"] == "complete" + assert handler.requests == [ + { + "turnId": "turn_1", + "status": "complete", + "reason": "Feature shipped with tests.", + } + ] + + +@pytest.mark.asyncio +async def test_raising_verifier_fails_closed_and_reports_to_model() -> None: + """A verifier that raises rejects (fail closed) and tells the model why.""" + + runtime = GoalRuntimeRouter() + handler = FakeGoalHandler() + runtime.configure(handler) + runtime.configure_completion_verifier(RaisingVerifier()) + runtime.activate(_context()) + + result = await runtime.request(status="complete", reason="Done.") + + assert result["updated"] is False + assert "verifier backend unavailable" in result["verdict"]["justification"] + assert handler.requests == [] + + +@pytest.mark.asyncio +async def test_blocked_status_is_not_verified() -> None: + """Only complete claims are verified; blocked goes straight through.""" + + runtime = GoalRuntimeRouter() + handler = FakeGoalHandler() + runtime.configure(handler) + verifier = RejectingVerifier() + runtime.configure_completion_verifier(verifier) + runtime.activate(_context()) + + result = await runtime.request(status="blocked", reason="Upstream API is down.") + + assert verifier.calls == 0 + assert result["status"] == "blocked" + assert handler.requests == [ + {"turnId": "turn_1", "status": "blocked", "reason": "Upstream API is down."} + ] + + +def test_rule_based_verifier_requires_objective_reason_and_evidence() -> None: + verifier = RuleBasedVerifier() + + no_objective = asyncio.run( + verifier(GoalCompletionRequest(objective=" ", reason="done", evidence=())) + ) + assert no_objective.confirmed is False + assert "no objective" in no_objective.justification.lower() + + no_reason = asyncio.run( + verifier(GoalCompletionRequest(objective="Ship it.", reason=" ", evidence=())) + ) + assert no_reason.confirmed is False + assert "no reason" in no_reason.justification.lower() + + no_evidence = asyncio.run( + verifier( + GoalCompletionRequest( + objective="Ship it.", reason="Looks done.", evidence=() + ) + ) + ) + assert no_evidence.confirmed is False + assert "no evidence" in no_evidence.justification.lower() + + failed = asyncio.run( + verifier( + GoalCompletionRequest( + objective="Ship it.", + reason="Ran the suite.", + evidence=( + GoalEvidenceItem( + kind="test_result", + status="failed", + summary="2 tests failed.", + ), + ), + ) + ) + ) + assert failed.confirmed is False + assert "failed" in failed.justification.lower() + + good = asyncio.run( + verifier( + GoalCompletionRequest( + objective="Ship it.", + reason="Ran the suite.", + evidence=( + GoalEvidenceItem( + kind="test_result", + status="passed", + summary="All tests pass.", + ), + ), + ) + ) + ) + assert good.confirmed is True + + +@pytest.mark.asyncio +async def test_model_backed_verifier_fail_closed_on_ask_error() -> None: + async def ask(request: GoalCompletionRequest) -> str: + raise ConnectionError("model unreachable") + + verifier = model_backed_verifier(ask) + verdict = await verifier( + GoalCompletionRequest(objective="Ship it.", reason="done", evidence=()) + ) + + assert verdict.confirmed is False + assert "model unreachable" in verdict.justification + + +@pytest.mark.asyncio +async def test_model_backed_verifier_treats_non_yes_as_rejection() -> None: + async def ask(request: GoalCompletionRequest) -> str: + return "no, the tests did not run" + + verifier = model_backed_verifier(ask) + verdict = await verifier( + GoalCompletionRequest(objective="Ship it.", reason="done", evidence=()) + ) + + assert verdict.confirmed is False + + +@pytest.mark.asyncio +async def test_verifier_protocol_accepts_plain_async_callable() -> None: + """A bare async function satisfies the verifier contract (no class needed).""" + + async def verifier(request: GoalCompletionRequest) -> GoalVerdict: + return GoalVerdict(confirmed=True, justification="ok") + + runtime = GoalRuntimeRouter() + handler = FakeGoalHandler() + runtime.configure(handler) + runtime.configure_completion_verifier(verifier) # type: ignore[arg-type] + runtime.activate(_context()) + + result = await runtime.request(status="complete", reason="Shipped.") + + assert result["status"] == "complete" + + +def test_completion_request_carries_real_objective_and_evidence() -> None: + """The request handed to the verifier is built by the handler, not hardcoded.""" + + handler = FakeGoalHandler(objective="Fix the off-by-one in pagination.") + request = handler.completion_request(_context(), reason="Tests pass now.") + + assert request.objective == "Fix the off-by-one in pagination." + assert request.reason == "Tests pass now." + assert request.evidence and request.evidence[0].kind == "test_result" + + +def test_configure_completion_verifier_rejects_non_positive_max() -> None: + runtime = GoalRuntimeRouter() + + with pytest.raises(ValueError): + runtime.configure_completion_verifier(ConfirmingVerifier(), max_rejections=0) + + +def test_verifier_contract_is_async_callable_protocol() -> None: + """RuleBasedVerifier and model_backed_verifier both satisfy the Protocol.""" + + async def _accept(verifier: GoalCompletionVerifier) -> None: + await verifier(GoalCompletionRequest(objective="x", reason="y", evidence=())) + + asyncio.run(_accept(RuleBasedVerifier())) + asyncio.run(_accept(model_backed_verifier(_yes))) + + +async def _yes(_request: GoalCompletionRequest) -> str: + return "yes" diff --git a/tests/test_goal_runtime.py b/tests/test_goal_runtime.py index 843a1ab60..0de7ea361 100644 --- a/tests/test_goal_runtime.py +++ b/tests/test_goal_runtime.py @@ -45,7 +45,10 @@ def _context() -> GoalRuntimeContext: ) -def test_goal_tools_are_hidden_outside_an_attempt_and_freeze_after_request() -> None: +@pytest.mark.asyncio +async def test_goal_tools_are_hidden_outside_an_attempt_and_freeze_after_request() -> ( + None +): runtime = GoalRuntimeRouter() handler = FakeGoalHandler() runtime.configure(handler) @@ -57,7 +60,7 @@ def test_goal_tools_are_hidden_outside_an_attempt_and_freeze_after_request() -> assert runtime.visible_tool_names(registered) == registered assert runtime.closure_prompt() is not None assert runtime.closure_prompt(stop_hook_active=True) is None - result = runtime.request( + result = await runtime.request( status="complete", reason="The latest Goal is complete.", )