From f531fbcbd5c63798f14e29c1a3b2a800786ea863 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Weberru=C3=9F?= Date: Fri, 16 Oct 2026 21:17:17 +0000 Subject: [PATCH] fix(assistant): require complete factual advice answers Retain failed source qualification and all existing factual oracles. Reinforce the complete explanation contract at the end of trusted advice context; keep decisions, provenance and actual factual answers distinct. Test transport for all affected advice topics and regenerate source-bound knowledge and scenario catalogue. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/assistant/knowledge-topics.generated.ts | 2 +- .../app/apps/api/src/assistant/model-serving.ts | 9 +++++++++ .../app/tests/assistant-editor.test.ts | 17 +++++++++++++++++ .../docs/assistant-knowledge/README.md | 7 +++++++ .../docs/assistant-test-catalogue.json | 2 +- 5 files changed, 35 insertions(+), 2 deletions(-) diff --git a/landing-zone-configurator/app/apps/api/src/assistant/knowledge-topics.generated.ts b/landing-zone-configurator/app/apps/api/src/assistant/knowledge-topics.generated.ts index 657ff37..747b0fa 100644 --- a/landing-zone-configurator/app/apps/api/src/assistant/knowledge-topics.generated.ts +++ b/landing-zone-configurator/app/apps/api/src/assistant/knowledge-topics.generated.ts @@ -1512,7 +1512,7 @@ export const assistantKnowledgeTopics = { { path: "landing-zone-configurator/app/apps/api/src/assistant/model-serving.ts", sha256: - "0454982ec262c9aaa646f4b8d3910ee92c5f19393979fa0f38cd5d36ad1021e9", + "031ca84b966e5063763c16778d71b8eb0befd935fc653c79e0956887be16c2c9", }, { path: "landing-zone-configurator/app/apps/api/src/assistant/knowledge.ts", diff --git a/landing-zone-configurator/app/apps/api/src/assistant/model-serving.ts b/landing-zone-configurator/app/apps/api/src/assistant/model-serving.ts index 65916f4..ca401dc 100644 --- a/landing-zone-configurator/app/apps/api/src/assistant/model-serving.ts +++ b/landing-zone-configurator/app/apps/api/src/assistant/model-serving.ts @@ -2192,6 +2192,15 @@ Reviewed knowledge: ${JSON.stringify({ fields: draftFields, ...topicKnowledge }) .slice(-12) .map((entry) => ({ role: entry.role, content: entry.content })), { role: "user", content: message }, + ...(!draftAllowed && !firewallAdvice + ? [ + { + role: "system" as const, + content: + "ANSWER COMPLETENESS: Put the complete factual answer in explanation itself, in the user's language. An introduction announcing rules or a list is not an answer: actually include the requested fields, supported values, limits and exclusions from the reviewed topic. Contract names, versions, dates and references are provenance, not substitutes for those facts. Explain native defaults, including null meaning no optional service instance, where relevant; do not promote illustrative enabled values to defaults. Questions may ask only genuinely missing decisions after the factual answer; never move the answer's facts into questions. Before finishing, check that explanation stands alone without an absent continuation or a promised list. Do not invent facts, propose changes or claim anything was saved or executed.", + }, + ] + : []), ...(dnsSettingsDraft ? [ { diff --git a/landing-zone-configurator/app/tests/assistant-editor.test.ts b/landing-zone-configurator/app/tests/assistant-editor.test.ts index b1e9b22..03c3890 100644 --- a/landing-zone-configurator/app/tests/assistant-editor.test.ts +++ b/landing-zone-configurator/app/tests/assistant-editor.test.ts @@ -427,6 +427,23 @@ describe("topic knowledge delivery", () => { topic, ...extra, ]); + if (topic !== "firewall-infrastructure") { + expect(request.messages[0]).toMatchObject({ + role: "system", + content: expect.stringContaining("ANSWER COMPLETENESS:"), + }); + expect(request.messages[0]?.content).toContain( + "including null meaning no optional service instance", + ); + expect(request.messages[0]?.content).toContain( + "Questions may ask only genuinely missing decisions after the factual answer", + ); + expect(request.messages.at(-1)).toEqual({ + role: "user", + content: + "Bitte erläutere die Bedeutung und Grenzen dieser Komponente.", + }); + } }, ); const result = await model.converse( diff --git a/landing-zone-configurator/docs/assistant-knowledge/README.md b/landing-zone-configurator/docs/assistant-knowledge/README.md index a3fe5f3..4a354d2 100644 --- a/landing-zone-configurator/docs/assistant-knowledge/README.md +++ b/landing-zone-configurator/docs/assistant-knowledge/README.md @@ -104,6 +104,13 @@ Raw synthetic receipts are retained as CI artifacts. Promotion requires the original build's successful live-qualification job as well as its build job; older builds without that proof cannot bypass qualification. A missing token, unavailable model or failed criterion blocks qualification, not a silent skip. +General advice ends its merged trusted system context with an explicit +answer-completeness instruction: the explanation must contain the actual requested facts, native +defaults, limits and exclusions, not just introduce a missing list or refer to +contract metadata. Follow-up questions cannot replace the factual answer. +The bounded firewall requirements dialogue retains its separate short-answer +contract. Transport tests check this instruction's delivery; unchanged real-model +factual criteria still decide whether the resulting answers qualify. CI needs `LZC_MODEL_SERVING_TOKEN` available to the protected `assistant-tests` environment. Local receipts do not certify that remote environment or authorize deployment. CI qualification jobs share a non-cancelling concurrency group to diff --git a/landing-zone-configurator/docs/assistant-test-catalogue.json b/landing-zone-configurator/docs/assistant-test-catalogue.json index 0f4c593..6b47aed 100644 --- a/landing-zone-configurator/docs/assistant-test-catalogue.json +++ b/landing-zone-configurator/docs/assistant-test-catalogue.json @@ -2,7 +2,7 @@ "schemaVersion": 1, "status": "field-inventory-and-smoke-corpus-not-ai-qualification", "engineRevision": "a256f6896d11134fdc351786f1be5eba4e56b2e2", - "knowledgeSha256": "ee51045fa0d249b535d9c449df1311af336ffd4daf44a1b67b1fc0852ee5c01b", + "knowledgeSha256": "5db344a9009f2d3e3a519ac6dbef9f4f614611101866756d4e2ad6f142667a67", "sourceDigests": { "scripts/assistant-scenarios.ts": "d2baa94ac001f35ed5c990aefbaaa72e754a35b9cb3740ab0dc7b041514d236a", "scripts/assistant-tests.ts": "fb1ba4a3438fbcd6d12c0a87d9bc9beeae8066bb5b691a7e2901635a842bb5df"