diff --git a/tests/ai_explainer/test_output.py b/tests/ai_explainer/test_output.py index e89f6712..c7c088a6 100644 --- a/tests/ai_explainer/test_output.py +++ b/tests/ai_explainer/test_output.py @@ -12,6 +12,8 @@ format_explanation_line, ) from utils.llm.base import LLMError, LLMProvider, LLMUsage +from utils.llm.fallback import FallbackProvider +from utils.llm.report import ReportContext class TestStructuredOutput(unittest.TestCase): @@ -219,7 +221,7 @@ def test_spill_failure_still_returns_alert_line(self, _mock_mkdir: MagicMock, _m class TestTextRefineDetailBudget(unittest.TestCase): def test_immediate_pass_keeps_detail_without_extra_call(self) -> None: - provider = MagicMock() + provider = MagicMock(model_name="test") provider.supports_structured_output = False provider.complete.side_effect = [ "TLDR: original. LOW.\n\nDETAIL:\nOriginal analysis.", @@ -231,7 +233,7 @@ def test_immediate_pass_keeps_detail_without_extra_call(self) -> None: self.assertIn("original. LOW", result.summary) def test_multiple_revisions_regenerate_detail_once(self) -> None: - provider = MagicMock() + provider = MagicMock(model_name="test") provider.supports_structured_output = False provider.complete.side_effect = [ "TLDR: original. LOW.\n\nDETAIL:\nOriginal analysis.", @@ -248,7 +250,7 @@ def test_multiple_revisions_regenerate_detail_once(self) -> None: def test_expansion_failure_keeps_revised_summary_discards_stale_detail(self) -> None: - provider = MagicMock() + provider = MagicMock(model_name="test") provider.supports_structured_output = False provider.complete.side_effect = [ "TLDR: original. LOW.\n\nDETAIL:\nOriginal analysis.", @@ -264,10 +266,11 @@ def test_expansion_failure_keeps_revised_summary_discards_stale_detail(self) -> class _UsageProvider(LLMProvider): """A text provider that reports a fixed usage per call, like a real provider; fails after ``fail_after`` calls.""" - def __init__(self, responses: list[str], fail_after: int | None = None) -> None: + def __init__(self, responses: list[str], fail_after: int | None = None, *, model_name: str = "test") -> None: self.responses = responses self.fail_after = fail_after self.calls = 0 + self._model_name = model_name def complete(self, prompt: str, system_prompt: str = "") -> str: if self.fail_after is not None and self.calls >= self.fail_after: @@ -278,7 +281,7 @@ def complete(self, prompt: str, system_prompt: str = "") -> str: @property def model_name(self) -> str: - return "test" + return self._model_name class TestStageUsage(unittest.TestCase): @@ -300,6 +303,7 @@ def test_summary_critique_rounds_and_detail_are_recorded(self) -> None: ) self.assertEqual(explanation.stages[1].summary, "Pauses the vault and blocks deposits. MEDIUM") self.assertEqual([s.usage.input_tokens for s in explanation.stages if s.usage], [1000, 2000, 3000, 4000]) + self.assertEqual([s.model_name for s in explanation.stages], ["test"] * 4) def test_failed_call_reports_no_usage(self) -> None: provider = _UsageProvider(["TLDR: Pauses the vault. LOW\nDETAIL: d"], fail_after=1) @@ -309,6 +313,76 @@ def test_failed_call_reports_no_usage(self) -> None: ) # The failed call keeps no usage: the provider's last_usage still belongs to the summary call. self.assertIsNone(explanation.stages[1].usage) + self.assertEqual(explanation.stages[1].model_name, "test") + + +class TestGenerationMetadata(unittest.TestCase): + """Published reports identify all answering models and total generation time.""" + + def test_empty_structured_summary_model_is_omitted_but_usage_is_retained(self) -> None: + cli = MagicMock(spec=LLMProvider, model_name="empty-cli", supports_structured_output=True) + cli.complete_structured.return_value = {"summary": "", "risk_tag": "LOW"} + cli.complete.side_effect = LLMError("text unavailable") + cli.last_usage = LLMUsage(2000, 900, None, 100, 0.5) + api = _UsageProvider(["TLDR: Pauses the vault. LOW", "Detailed analysis."], model_name="api") + provider = FallbackProvider([cli, api]) + + explanation = _generate_explanation(provider, "p", report_ctx=ReportContext(chain_id=1)) + + self.assertIn("- **Model:** api\n", explanation.report) + self.assertNotIn("empty-cli", explanation.report) + self.assertEqual(explanation.stages[0].outcome, "empty") + self.assertEqual(explanation.stages[0].model_name, "empty-cli") + self.assertEqual(explanation.stages[0].usage, cli.last_usage) + + @patch("utils.llm.ai_explainer.perf_counter") + def test_generation_time_is_logged_for_successful_and_empty_summaries(self, clock: MagicMock) -> None: + for response in ("TLDR: Pauses the vault. LOW\nDETAIL: Detail.", " "): + with self.subTest(response=response): + clock.return_value = 100.0 + + def complete(prompt: str, system_prompt: str = "") -> str: + clock.return_value = 110.0 + return response + + provider = MagicMock( + spec=LLMProvider, model_name="api", supports_structured_output=False, last_usage=None + ) + provider.complete.side_effect = complete + with self.assertLogs("utils.llm.ai_explainer", level="INFO") as logs: + explanation = _generate_explanation(provider, "p", report_ctx=ReportContext(chain_id=1)) + self.assertIn("generation time=10.0s", "\n".join(logs.output)) + if explanation.summary: + self.assertIn("- **Time:** 10.0s", explanation.report) + + @patch("utils.llm.ai_explainer.upload_to_gist", return_value="https://gist.wavey.info/abc123") + def test_fallback_models_and_total_time_reach_gist(self, gist: MagicMock) -> None: + primary = _UsageProvider( + ["TLDR: Pauses the vault. LOW\nDETAIL: Initial detail."], fail_after=1, model_name="primary" + ) + fallback = _UsageProvider( + ["TLDR: Pauses the vault and blocks deposits. MEDIUM", "PASS", "Final detail."], model_name="fallback" + ) + unused = _UsageProvider([], model_name="unused") + provider = FallbackProvider([primary, fallback, unused]) + + with self.assertLogs("utils.llm.ai_explainer", level="INFO") as logs: + explanation = _generate_explanation(provider, "prompt", refine=True, report_ctx=ReportContext(chain_id=1)) + format_explanation_line(explanation) + + body = gist.call_args.args[0] + self.assertIn("- **Chain:** Mainnet (chain id 1)", body) + self.assertIn("- **Risk:** MEDIUM\n- **Model:** primary / fallback\n- **Time:**", body) + self.assertRegex(body, r"- \*\*Time:\*\* \d+\.\ds") + self.assertNotIn("unused", body) + self.assertIn("Final detail.", body) + self.assertEqual([stage.model_name for stage in explanation.stages], ["primary"] + ["fallback"] * 3) + output = "\n".join(logs.output) + self.assertIn("summary ok model=primary in=1000 cached=900 out=100", output) + self.assertIn("critique-1 revised model=fallback in=1000 cached=900 out=100", output) + self.assertIn("generation time=", output) + self.assertEqual(sum("LLM usage:" in line for line in logs.output), 1) + self.assertNotIn("LLM stage=", output) class TestEmptyCritique(unittest.TestCase): diff --git a/tests/test_llm_cli.py b/tests/test_llm_cli.py index 5dd65b8a..112848fb 100644 --- a/tests/test_llm_cli.py +++ b/tests/test_llm_cli.py @@ -11,7 +11,7 @@ from typing import Any from unittest.mock import MagicMock, patch -from utils.llm.base import LLMError, LLMProvider, LLMUnavailableError +from utils.llm.base import LLMError, LLMProvider, LLMUnavailableError, LLMUsage from utils.llm.cli_provider import ClaudeCLIProvider, CodexCLIProvider, _cli_environment, _run_cli from utils.llm.factory import get_llm_provider, reset_provider from utils.llm.fallback import FallbackProvider @@ -32,8 +32,10 @@ def _events(*items: dict[str, Any]) -> str: CODEX_SUCCESS = _events({"type": "thread.started"}, {"type": "turn.completed"}) -def _claude_result(**fields: Any) -> str: - return _events({"type": "system", "subtype": "init"}, {"type": "result", "is_error": False, **fields}) +def _claude_result(model: str | None = None, **fields: Any) -> str: + return _events( + {"type": "system", "subtype": "init", "model": model}, {"type": "result", "is_error": False, **fields} + ) def _arg_after(args: list[str], flag: str) -> str: @@ -313,6 +315,48 @@ def fake_run(label: str, args: list[str], *rest: Any) -> MagicMock: class TestClaudeCLIProvider(unittest.TestCase): """Tests for ClaudeCLIProvider.""" + @patch("utils.llm.cli_provider._run_cli") + def test_default_model_is_read_from_init_metadata(self, mock_run: MagicMock) -> None: + mock_run.return_value = _completed([], stdout=_claude_result(model="claude-opus-5-5", result="OK")) + provider = ClaudeCLIProvider(binary="claude") + provider.complete("p") + self.assertEqual(provider.model_name, "claude-cli:claude-opus-5-5") + + @patch("utils.llm.cli_provider._run_cli") + def test_answering_model_overrides_init_model_and_configured_alias(self, mock_run: MagicMock) -> None: + mock_run.return_value = _completed( + [], + stdout=_events( + {"type": "system", "subtype": "init", "model": "initial-model"}, + {"type": "assistant", "message": {"model": "first-model", "content": []}}, + {"type": "assistant", "message": {"model": "answering-model", "content": []}}, + {"type": "result", "is_error": False, "result": "OK"}, + ), + ) + provider = ClaudeCLIProvider(binary="claude", model="opus") + provider.complete("p") + self.assertEqual(provider.model_name, "claude-cli:answering-model") + + @patch("utils.llm.cli_provider._run_cli") + def test_missing_or_invalid_metadata_does_not_reuse_previous_model(self, mock_run: MagicMock) -> None: + for configured in ("", "opus"): + for model in (None, " ", ""): + with self.subTest(configured=configured, model=model): + mock_run.side_effect = [ + _completed([], stdout=_claude_result(model="previous-model", result="OK")), + _completed( + [], + stdout=_events( + {"type": "assistant", "message": {"model": model}}, + {"type": "result", "is_error": False, "result": "OK"}, + ), + ), + ] + provider = ClaudeCLIProvider(binary="claude", model=configured) + provider.complete("previous") + provider.complete("current") + self.assertEqual(provider.model_name, f"claude-cli:{configured}" if configured else "claude-cli") + @patch("utils.llm.cli_provider._run_cli") def test_complete_disables_builtins_and_mcp(self, mock_run: MagicMock) -> None: seen_system: list[str] = [] @@ -348,14 +392,22 @@ def test_complete_structured_reads_structured_output(self, mock_run: MagicMock) mock_run.side_effect = lambda label, args, *rest: _completed( args, stdout=_events( - {"type": "assistant", "message": {"content": [{"type": "tool_use", "name": "StructuredOutput"}]}}, + { + "type": "assistant", + "message": { + "model": "claude-opus-5-5", + "content": [{"type": "tool_use", "name": "StructuredOutput"}], + }, + }, {"type": "result", "is_error": False, "result": "", "structured_output": {"summary": "ok"}}, ), ) - result = ClaudeCLIProvider(binary="claude").complete_structured("p", SCHEMA) + provider = ClaudeCLIProvider(binary="claude", model="opus") + result = provider.complete_structured("p", SCHEMA) self.assertEqual(result, {"summary": "ok"}) + self.assertEqual(provider.model_name, "claude-cli:claude-opus-5-5") self.assertEqual(json.loads(_arg_after(mock_run.call_args.args[1], "--json-schema")), SCHEMA) @patch("utils.llm.cli_provider._run_cli") @@ -439,10 +491,42 @@ def model_name(self) -> str: class TestFallbackProvider(unittest.TestCase): """Tests for FallbackProvider.""" + def test_api_fallback_preserves_usage_for_both_completion_formats(self) -> None: + for structured in (False, True): + with self.subTest(structured=structured): + cli = _StubProvider("cli", [LLMUnavailableError("offline")]) + api = _StubProvider("api", [{"summary": "ok"}] if structured else ["ok"]) + api.last_usage = LLMUsage(1000, 900, 25, 100, 0.5) + chain = FallbackProvider([cli, api]) + if structured: + chain.complete_structured("p", SCHEMA) + else: + chain.complete("p") + self.assertEqual(chain.last_usage, api.last_usage) + + def test_usage_and_model_reset_when_provider_changes_or_all_fail(self) -> None: + cli = _StubProvider("cli", [LLMError("try API"), "ok", LLMError("failed")]) + api = _StubProvider("api", ["ok", LLMError("failed")]) + api.last_usage = LLMUsage(1000, 900, None, 100, 0.5) + chain = FallbackProvider([cli, api]) + chain.complete("p") + self.assertIsNotNone(chain.last_usage) + chain.complete("p") + self.assertIsNone(chain.last_usage) + self.assertEqual(chain.model_name, "cli") + + with self.assertRaises(LLMError): + chain.complete("p") + self.assertIsNone(chain.last_usage) + self.assertEqual(chain.model_name, "") + def test_first_success_wins(self) -> None: first, second = _StubProvider("a", ["from a"]), _StubProvider("b", ["from b"]) chain = FallbackProvider([first, second]) - self.assertEqual(chain.complete("p"), "from a") + with self.assertLogs("utils.llm.fallback", level="DEBUG") as logs: + self.assertEqual(chain.complete("p"), "from a") + self.assertTrue(any("LLM attempt: provider=_StubProvider operation=complete" in line for line in logs.output)) + self.assertFalse(any("model=" in line for line in logs.output)) self.assertEqual(second.calls, 0) self.assertEqual(chain.model_name, "a") diff --git a/tests/test_llm_report.py b/tests/test_llm_report.py index 725f66cf..b1f09f0a 100644 --- a/tests/test_llm_report.py +++ b/tests/test_llm_report.py @@ -469,11 +469,30 @@ def test_metadata_header(self) -> None: ) self.assertIn("- **Chain:** Mainnet (chain id 1)", report) self.assertIn("- **Risk:** HIGH", report) + self.assertNotIn("**Model:**", report) + self.assertNotIn("**Time:**", report) def test_contract_unlinked_without_label_address(self) -> None: report = build_report("Summary.", "Analysis.", _add_farms_ctx()) self.assertIn("- **Contract:** Infinifi Shorttimelock\n", report) + def test_model_metadata_is_plain_text(self) -> None: + report = build_report( + "Summary.", + "Analysis.", + _add_farms_ctx(), + model_names=("model_[link](https://example.com)*`\\\n- injected",), + ) + self.assertIn( + r"- **Model:** model\_\[link\](https://example.com)<b>\*\`\\ - injected", + report, + ) + self.assertNotIn("\n- injected", report) + + def test_zero_generation_time_is_shown(self) -> None: + report = build_report("Summary.", "Analysis.", _add_farms_ctx(), elapsed_s=0) + self.assertIn("- **Time:** 0.0s", report) + def test_contract_unlinked_on_chain_without_explorer(self) -> None: ctx = _add_farms_ctx(chain_id=999999, label_address=TIMELOCK) self.assertIn("- **Contract:** Infinifi Shorttimelock\n", build_report("S.", "A.", ctx)) diff --git a/utils/llm/README.md b/utils/llm/README.md index 6453941f..c7663eb2 100644 --- a/utils/llm/README.md +++ b/utils/llm/README.md @@ -414,7 +414,7 @@ Adapters receive the inner calls of executed governance wrappers too. `expand_ex The prompt is split into a **system** prompt (static instructions) and a **user** prompt (per-tx context). -**Measuring cost.** Each provider records the latest call's `LLMUsage`: input tokens (cached ones included), cached input, cache writes, output, and latency. The explainer keeps one `StageUsage` per call on `Explanation.stages`: `summary`, `critique-N` (`pass`, `revised` with the revised TLDR, or `empty` when the critic returned nothing and the draft was kept) and `detail`. It logs one line per alert: `LLM usage (): summary ok in=… cached=… out=… 8.1s | critique-1 pass … | detail ok … | total …`. +**Measuring cost.** API providers record the latest call's `LLMUsage`: input tokens (cached ones included), cached input, cache writes, output, and latency. The fallback chain forwards the answering provider's usage and clears its previous model and usage before every call. The explainer keeps one `StageUsage` per call on `Explanation.stages`: `summary`, `critique-N` (`pass`, `revised` with the revised TLDR, or `empty` when the critic returned nothing and the draft was kept) and `detail`. One per-alert usage line includes each stage's model, outcome and token counts, plus the total generation time, including when the summary is empty. Empty responses still count toward usage, even though their models are omitted from the gist header. Venice applies automatic prefix caching and reports it as `prompt_tokens_details.cached_tokens`. Anthropic reports cache reads and writes separately; the provider folds them into the same fields. @@ -578,6 +578,8 @@ The gist is the artifact a reviewer actually opens, so it carries more than the - **Contract:** Infinifi Shorttimelock — [`0x4B17…7c32`](https://etherscan.io/address/0x4B17…) - **Chain:** Mainnet (chain id 1) - **Risk:** MEDIUM +- **Model:** codex-cli:my-configured-model / claude-cli:claude-opus-5-5 +- **Time:** 10.0s ## Summary Registers a new type-2 farm in FarmRegistry. … @@ -594,6 +596,8 @@ Registers a new type-2 farm in FarmRegistry. … - [`0x79e1…971f`](https://etherscan.io/address/0x79e1…) ``` +The header's **Model** line lists the distinct models that produced a summary, revision, passing critique or detail, in first-use order. Failed and empty responses do not add a model to that line. Model identifiers are escaped as single-line Markdown text. Claude's CLI reports its resolved model in the event stream: the provider reads the last assistant message's `model`, or the initialization event's `model` when no assistant model is available. This resolves aliases such as `opus` without another CLI call. If neither event reports a model, the label uses `LLM_CLAUDE_MODEL` or `claude-cli`. Codex's `exec --json` stream does not expose the model (verified with CLI 0.163.0), so its label uses `LLM_CODEX_MODEL` or `codex-cli`. **Time** measures total LLM generation time, including retries and refinement, and excludes context gathering and gist upload. These fields are omitted for deterministic reports that do not use an LLM. + The report can end with a code-generated `## Reference` table after Call Flow. It lists only the addresses the rest of the report does not already link: the executor, alert contract, call targets, calldata arguments (nested calldata included) and addresses resolved by protocol adapters, minus any that appear in the body above it. The table is omitted when nothing is left. ```markdown @@ -677,7 +681,7 @@ The API provider is used by default. When CLIs are explicitly enabled, they run - A non-zero exit, timeout, missing binary, or Claude `is_error` result (auth, rate limit, no credits) raises `LLMUnavailableError`. `FallbackProvider` then drops that CLI **for the rest of the process**, so a dead CLI isn't retried on every stage of every alert. CLIs are always wrapped in `FallbackProvider`, even a lone one with no API key, so this tracking applies there too. A bad response (empty text, invalid JSON) only falls through for that one call. - Each CLI runs as the leader of its own process group. The whole group is SIGKILLed on timeout and after the CLI exits, so commands the agent spawned can't outlive the call. Output goes to temp files, not pipes, so a backgrounded child holding stdout can't stall a finished call until the timeout. - Codex has no system-prompt flag, so the system prompt is prepended to the stdin prompt in a `` block. -- The log line `AI summary using ` names the provider that actually answered (`codex-cli`, `claude-cli`, or the API model). +- With DEBUG logging enabled, `LLM attempt: provider=… operation=…` logs each fallback attempt using the provider class, because the resolved model is only known after the call. The per-alert INFO usage line identifies each stage's answering model. The gist header lists models with useful responses, so a fallback during critique or detail cannot hide the model that wrote the summary. **Security note.** The prompt contains attacker-influenced text (proposal descriptions, verified source, labels). With no tools, an injection can only change the summary text, not run commands or make requests. The tool restrictions are CLI flags, so a CLI update that changes their meaning would weaken them; event auditing detects unexpected tool calls after the fact and cannot undo one. Both CLIs still run as the monitoring service's Unix user. Keep `LLM_CLI_PROVIDERS=none` until tested in the target service. diff --git a/utils/llm/ai_explainer.py b/utils/llm/ai_explainer.py index 6f2a4aa5..11f8884d 100644 --- a/utils/llm/ai_explainer.py +++ b/utils/llm/ai_explainer.py @@ -11,6 +11,7 @@ from datetime import datetime, timezone from decimal import Decimal from functools import lru_cache +from time import perf_counter from eth_utils import function_signature_to_4byte_selector, to_checksum_address @@ -384,12 +385,15 @@ class StageUsage: back to text), ``critique-N`` or ``detail``. ``outcome`` says what the call did: ``ok``, ``pass`` / ``revised`` for a critique, ``empty`` or ``error``. ``summary`` holds the TLDR a summary or revising critique produced, to compare rounds. + ``model_name`` identifies the answering model, or a direct provider's configured + model on errors. Failed fallback chains leave it empty. """ stage: str outcome: str usage: LLMUsage | None = None summary: str = "" + model_name: str = "" @dataclass(frozen=True) @@ -1876,10 +1880,12 @@ def _explanation_from_json(data: dict) -> Explanation: def _record( stages: list[StageUsage] | None, stage: str, outcome: str, provider: LLMProvider, summary: str = "" ) -> None: - """Append one stage's usage; a failed call reports no usage (``last_usage`` would be stale).""" + """Record a stage; failed direct API calls can retain stale usage, so omit it.""" if stages is not None: usage = provider.last_usage if outcome != "error" else None - stages.append(StageUsage(stage, outcome, usage if isinstance(usage, LLMUsage) else None, summary)) + stages.append( + StageUsage(stage, outcome, usage if isinstance(usage, LLMUsage) else None, summary, provider.model_name) + ) def _usage_line(stages: list[StageUsage]) -> str: @@ -1898,7 +1904,10 @@ def tokens(usage: LLMUsage | None) -> tuple[int, int, int, float]: parts = [] for stage in stages: inp, cached, out, latency = tokens(stage.usage) - parts.append(f"{stage.stage} {stage.outcome} in={inp} cached={cached} out={out} {latency:.1f}s") + parts.append( + f"{stage.stage} {stage.outcome} model={stage.model_name or 'unknown'} " + f"in={inp} cached={cached} out={out} {latency:.1f}s" + ) totals = [tokens(stage.usage) for stage in stages] parts.append( f"total in={sum(t[0] for t in totals)} cached={sum(t[1] for t in totals)} " @@ -2033,12 +2042,14 @@ def _generate_explanation( When ``report_ctx`` is given, the detail is wrapped into the full gist page (metadata header, summary, deterministic call flow, optional protocol - context, analysis). + context, analysis). The header lists the models that answered and the total + generation time, including refinement and fallback attempts. """ + started = perf_counter() stages: list[StageUsage] = [] summary_draft = _generate_summary(provider, prompt, stages) if not summary_draft.summary: - logger.info("LLM usage (%s): %s", provider.model_name, _usage_line(stages)) + logger.info("LLM usage: %s | generation time=%.1fs", _usage_line(stages), perf_counter() - started) return replace(summary_draft, stages=tuple(stages)) had_detail = bool(summary_draft.detail) @@ -2053,13 +2064,23 @@ def _generate_explanation( detail = summary_draft.detail if not detail and (provider.supports_structured_output or had_detail): detail = _expand_detail(provider, prompt, summary_draft.summary, stages) - logger.info("LLM usage (%s): %s", provider.model_name, _usage_line(stages)) + elapsed_s = perf_counter() - started + logger.info("LLM usage: %s | generation time=%.1fs", _usage_line(stages), elapsed_s) report = "" title = "" if report_ctx is not None: prose, risk_tag = _split_risk_tag(summary_draft.summary) - report = build_report(prose, detail, report_ctx, risk_tag) + report = build_report( + prose, + detail, + report_ctx, + risk_tag, + model_names=tuple( + stage.model_name for stage in stages if stage.model_name and stage.outcome in ("ok", "pass", "revised") + ), + elapsed_s=elapsed_s, + ) title = build_title(report_ctx, risk_tag, fallback=DETAIL_REPORT_TITLE) return Explanation(summary=summary_draft.summary, detail=detail, report=report, title=title, stages=tuple(stages)) @@ -2254,7 +2275,7 @@ def explain_transaction( try: explanation = _generate_explanation(provider, prompt, refine=refine, report_ctx=report_ctx) - logger.info("AI summary using %s:\n%s", provider.model_name, explanation.summary) + logger.info("AI summary:\n%s", explanation.summary) if explanation.detail: logger.info("AI detail:\n%s", explanation.detail) return explanation @@ -2639,7 +2660,7 @@ def explain_batch_transaction( try: explanation = _generate_explanation(provider, prompt, refine=refine, report_ctx=report_ctx) - logger.info("Batch AI summary using %s:\n%s", provider.model_name, explanation.summary) + logger.info("Batch AI summary:\n%s", explanation.summary) if explanation.detail: logger.info("Batch AI detail:\n%s", explanation.detail) return explanation diff --git a/utils/llm/cli_provider.py b/utils/llm/cli_provider.py index eab8a875..5edf320e 100644 --- a/utils/llm/cli_provider.py +++ b/utils/llm/cli_provider.py @@ -105,6 +105,25 @@ def _claude_tool_events(events: list[dict[str, Any]]) -> list[str]: return tools +def _claude_model(events: list[dict[str, Any]]) -> str: + """Read the initialized model, updating it from subsequent assistant responses.""" + model = "" + for event in events: + if (event.get("type"), event.get("subtype")) == ("system", "init"): + candidate = event.get("model") + elif event.get("type") == "assistant": + message = event.get("message") + candidate = message.get("model") if isinstance(message, dict) else None + else: + continue + if not isinstance(candidate, str): + continue + candidate = candidate.strip() + if candidate not in ("", ""): + model = candidate + return model + + def _tail(text: str) -> str: """Return the last ``_ERROR_SNIPPET_CHARS`` of ``text``, stripped, for error messages.""" return text.strip()[-_ERROR_SNIPPET_CHARS:] @@ -306,6 +325,7 @@ def __init__(self, binary: str, model: str = "", timeout: int = DEFAULT_CLI_TIME """ self._binary = binary self._model = model + self._reported_model = "" self._timeout = timeout logger.info("Initialized Claude CLI provider: binary=%s model=%s", binary, model or "") @@ -333,6 +353,7 @@ def complete_structured(self, prompt: str, schema: dict[str, Any], system_prompt def _exec(self, prompt: str, system_prompt: str, schema: dict[str, Any] | None) -> dict[str, Any]: """Run one ``claude -p`` session and return its decoded JSON result envelope.""" + self._reported_model = "" with tempfile.TemporaryDirectory(prefix="llm-claude-") as tmp: workdir = Path(tmp) args = [ @@ -381,9 +402,11 @@ def _exec(self, prompt: str, system_prompt: str, schema: dict[str, Any] | None) if envelope.get("is_error"): # Auth, rate-limit and out-of-credit failures land here. raise LLMUnavailableError(f"claude: {_tail(str(envelope.get('result') or envelope.get('subtype')))}") + self._reported_model = _claude_model(events) return envelope @property def model_name(self) -> str: - """Return a label identifying the CLI and model.""" - return f"claude-cli:{self._model}" if self._model else "claude-cli" + """Return the reported model, falling back to the configured model or CLI label.""" + model = self._reported_model or self._model + return f"claude-cli:{model}" if model else "claude-cli" diff --git a/utils/llm/fallback.py b/utils/llm/fallback.py index 4c10447a..9dfa591c 100644 --- a/utils/llm/fallback.py +++ b/utils/llm/fallback.py @@ -30,8 +30,8 @@ def __init__(self, providers: list[LLMProvider]) -> None: raise LLMError("FallbackProvider needs at least one provider") self._providers = list(providers) self._disabled: set[int] = set() - self._last_used: LLMProvider | None = None - logger.info("LLM fallback chain: %s", " -> ".join(p.model_name for p in self._providers)) + self._model_name = " -> ".join(p.model_name for p in self._providers) + logger.info("LLM fallback chain: %s", self._model_name) def complete(self, prompt: str, system_prompt: str = "") -> str: """Return the first successful completion in chain order.""" @@ -60,10 +60,13 @@ def _first_success(self, op: str, call: Callable[[LLMProvider], T], only_structu Raises: LLMError: If every eligible provider failed. """ + self._model_name = "" + self.last_usage = None errors: list[str] = [] for index, provider in enumerate(self._providers): if index in self._disabled or (only_structured and not provider.supports_structured_output): continue + logger.debug("LLM attempt: provider=%s operation=%s", type(provider).__name__, op) try: result = call(provider) except LLMUnavailableError as e: @@ -75,7 +78,8 @@ def _first_success(self, op: str, call: Callable[[LLMProvider], T], only_structu logger.warning("%s %s failed, trying next provider: %s", provider.model_name, op, e) errors.append(f"{provider.model_name}: {e}") continue - self._last_used = provider + self._model_name = provider.model_name + self.last_usage = provider.last_usage return result if not errors: skipped = "disabled after earlier failures" if self._disabled else "no eligible provider" @@ -84,7 +88,5 @@ def _first_success(self, op: str, call: Callable[[LLMProvider], T], only_structu @property def model_name(self) -> str: - """Model of the provider that last answered, or the chain description before any call.""" - if self._last_used is not None: - return self._last_used.model_name - return " -> ".join(p.model_name for p in self._providers) + """Latest call's answering model, empty after failure; chain description before any call.""" + return self._model_name diff --git a/utils/llm/report.py b/utils/llm/report.py index 5b78fcdd..9e3b737c 100644 --- a/utils/llm/report.py +++ b/utils/llm/report.py @@ -15,6 +15,7 @@ from collections.abc import Iterator from dataclasses import dataclass, field from datetime import datetime, timezone +from html import escape from eth_utils import to_checksum_address @@ -569,8 +570,14 @@ def _chain_name(chain_id: int) -> str: return f"Chain {chain_id}" -def _format_metadata(ctx: ReportContext, risk_tag: str) -> str: - """Header bullet list: protocol, contract label, chain, risk.""" +def _model_label(model: str) -> str: + """Escape model identifiers as single-line Markdown text, including HTML.""" + model = escape(" ".join(model.split()), quote=False) + return re.sub(r"([\\`*_\[\]])", r"\\\1", model) + + +def _format_metadata(ctx: ReportContext, risk_tag: str, model_names: tuple[str, ...], elapsed_s: float | None) -> str: + """Header bullet list: protocol, contract label, chain, risk, models and time.""" lines: list[str] = [] if ctx.protocol: lines.append(f"- **Protocol:** {ctx.protocol}") @@ -585,6 +592,10 @@ def _format_metadata(ctx: ReportContext, risk_tag: str) -> str: lines.append(f"- **Chain:** {_chain_name(ctx.chain_id)} (chain id {ctx.chain_id})") if risk_tag: lines.append(f"- **Risk:** {risk_tag}") + if model_names: + lines.append(f"- **Model:** {' / '.join(_model_label(model) for model in dict.fromkeys(model_names))}") + if elapsed_s is not None: + lines.append(f"- **Time:** {elapsed_s:.1f}s") return "\n".join(lines) @@ -611,7 +622,15 @@ def build_title(ctx: ReportContext, risk_tag: str = "", now: datetime | None = N return " - ".join(part for part in parts if part) -def build_report(summary: str, detail: str, ctx: ReportContext, risk_tag: str = "") -> str: +def build_report( + summary: str, + detail: str, + ctx: ReportContext, + risk_tag: str = "", + *, + model_names: tuple[str, ...] = (), + elapsed_s: float | None = None, +) -> str: """Assemble the full markdown gist body. Sections: metadata header, the Telegram-visible summary (so the gist is @@ -623,6 +642,8 @@ def build_report(summary: str, detail: str, ctx: ReportContext, risk_tag: str = detail: The LLM's detailed analysis. ctx: Decoded calls, labels, and alert metadata. risk_tag: LOW / MEDIUM / HIGH / CRITICAL, when known. + model_names: Models that answered during generation, in call order. + elapsed_s: Total generation time including retries, when measured. Returns: Markdown body, or "" when there is nothing worth publishing. @@ -632,7 +653,7 @@ def build_report(summary: str, detail: str, ctx: ReportContext, risk_tag: str = return "" sections: list[str] = [] - metadata = _format_metadata(ctx, risk_tag) + metadata = _format_metadata(ctx, risk_tag, model_names, elapsed_s) if metadata: sections.append(metadata) if summary: