From 6b44b1bd35d278aa100bcff213dac2384ecc838b Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Wed, 28 Oct 2026 13:47:07 -0400 Subject: [PATCH 01/13] feat(engine): POST /provider/probe one-token reachability probe with named statuses (#1984) Per-model {model, status: ok|denied|unavailable|invalid|rate-limited|error, latency_ms, reason, credential_fp, checked_at} via Provider.getLanguage; credentials stay in the engine, reasons are redacted, credential_fp is a truncated domain-separated sha256. Classifier beside parseAPICallError. maxOutputTokens 1, 10s timeout, concurrency 4. Part of #1979. --- packages/app-bundle/manifest.json | 16 +- .../packages/opencode/src/provider/error.ts | 52 +++++ .../packages/opencode/src/provider/probe.ts | 160 ++++++++++++++ .../instance/httpapi/groups/provider.ts | 14 ++ .../instance/httpapi/handlers/provider.ts | 6 + .../test/server/httpapi-provider.test.ts | 196 +++++++++++++++++- 6 files changed, 436 insertions(+), 8 deletions(-) create mode 100644 packages/app-bundle/overlay/packages/opencode/src/provider/probe.ts diff --git a/packages/app-bundle/manifest.json b/packages/app-bundle/manifest.json index cf31e78cb..09734cf56 100644 --- a/packages/app-bundle/manifest.json +++ b/packages/app-bundle/manifest.json @@ -357,7 +357,8 @@ "packages/opencode/src/project/instance-context.ts": "65a35792e50e96a50ea14cd9f928045c0e2fc15eb49506f5bbceee38c4af1457", "packages/opencode/src/project/instance-store.ts": "c22d3e2499e63dd8d9395d58189df75444d306774df833e746f22e327edcf2ab", "packages/opencode/src/provider/auth.ts": "b0ab1322c1039e27176bf9e3500603eae27506065ac141a69710220f3361f6b8", - "packages/opencode/src/provider/error.ts": "d72310be26cfae5c2943f9939a6461e848532a5646ff405d628e9a50fc1ebe91", + "packages/opencode/src/provider/error.ts": "42bce3a89795d5d851fad39d388493cbe06dc5f955aaad20c0bf36c01c4d60fe", + "packages/opencode/src/provider/probe.ts": "c10dce199ce375c8ea7a67ba35d3b4f73599bb4b69c59ca0124c61d433a4e931", "packages/opencode/src/provider/provider.ts": "b89bc0ca0a12ea0cfaf5c956a4b731a0397d4c407e787a16cdd66fb7f54ee476", "packages/opencode/src/provider/transform.ts": "70c7492d4a23a30ebc340a5605702b1602960f4b412e18557534e02962c137b8", "packages/opencode/src/question/index.ts": "8317348c2ca27dd9e4f7d18fdc126a4d355c20495c1b86668430ad477a68e9b3", @@ -442,7 +443,7 @@ "packages/opencode/test/server/httpapi-file.test.ts": "d3386129bc81467ecbcc57a3861f53767ee56ac41b4ff6ad58d8b168fb3b3373", "packages/opencode/test/server/httpapi-global-event.test.ts": "8a78dc59edf48eb3cf6d1c09448f7dfe5ceb138e1bd6125949ebc88976c903f0", "packages/opencode/test/server/httpapi-instance-route-auth.test.ts": "4b73a89d2431c02bc3b665d93f9002914e51eb1add979dc1867efbcbc0c36e25", - "packages/opencode/test/server/httpapi-provider.test.ts": "9e8b3e085b478e3f99b689ffff600e10c0b4b50b3cabdbd749f88dae19bf5904", + "packages/opencode/test/server/httpapi-provider.test.ts": "a8ec90f18028d255be101edb5faeb34738a71a478b7183c01fdafc243af92611", "packages/opencode/test/server/httpapi-session.test.ts": "eda6fd763ac4c17891cedaa4ccd49597c03e7227fcfdcf0f94a1b4545ba623ed", "packages/opencode/test/server/httpapi-ui.test.ts": "d029e959cfd9347fff4cf17c5686dd125b0bd3a41afb3408a23b31418f12e3b0", "packages/opencode/test/server/pasqal-secret.test.ts": "54be8c666160dad22cd37bbf041f5bd40aec120271818f3c56c94107d987b9fb", @@ -1005,14 +1006,14 @@ "packages/opencode/src/server/routes/instance/httpapi/groups/experimental.ts": "944a0aee215a917e867fd9482f3c2bd40f49082bd3174c8a7d967ca40e34ec15", "packages/opencode/src/server/routes/instance/httpapi/groups/file.ts": "a96fdc55ddb95ca7c481c86f67e37ed88d4cf2a190a482765ab55ce4409f3a59", "packages/opencode/src/server/routes/instance/httpapi/groups/mcp.ts": "048efa7b74d43c4ad4a75d0f8421c808760c8c0dcd593f0051a87044e40487d6", - "packages/opencode/src/server/routes/instance/httpapi/groups/provider.ts": "34c6f1ab1a085d1b047c220bbff32a36032dc5e924784e9272cda166cf0ec84b", + "packages/opencode/src/server/routes/instance/httpapi/groups/provider.ts": "2d090445392a2c52efb26335297f4d44c70aa511b0524e4cd3a69880ffdc6078", "packages/opencode/src/server/routes/instance/httpapi/groups/session.ts": "1421f8c07948b121c7e2005bd9b0fcd6b2c456565ddd1da32d8e6c35d03fac9c", "packages/opencode/src/server/routes/instance/httpapi/handlers/event.ts": "88c7bae32fa01ab213e95cb4e4891fd3ccb961782e80a416e321c5daf90fb487", "packages/opencode/src/server/routes/instance/httpapi/handlers/experimental.ts": "69ca3dd48d95157fc822e1f76eb42ac873df50f22616b85194619ca658613867", "packages/opencode/src/server/routes/instance/httpapi/handlers/file.ts": "10bcb79b117beebee04184fb48c1ad789e782e2f7ae4b0c68603edc00f03aee9", "packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts": "040554475b6ba22ad666dfeaca2576658854c46cd18efe3b2d68959f9eaaed80", "packages/opencode/src/server/routes/instance/httpapi/handlers/mcp.ts": "7da08d58765cdf2b6ce4648fc70de371402949f5bb7b75696f674d43f2d5365b", - "packages/opencode/src/server/routes/instance/httpapi/handlers/provider.ts": "8baf16cad0992c5351b61b0c0fb27903bb25b17e1b25fc7c5d77f807c24413a6", + "packages/opencode/src/server/routes/instance/httpapi/handlers/provider.ts": "e899320a028e639a845d6c6aef547d448341671a195f5465ee4e8fa847957c52", "packages/opencode/src/server/routes/instance/httpapi/handlers/session.ts": "b37ff4136c4aea85b23083bf441f03e09820e3e4cdd95e4118ca570d8435b94f", "packages/opencode/src/server/routes/instance/httpapi/middleware/file-write-workspace.ts": "665c33633173ee89c6da1d4a0eaacaf4eebe3ccdd04eb36b063515cb4c093e4a", "packages/opencode/src/server/routes/instance/httpapi/middleware/gateway-secret.ts": "ec6133ee5d331920ded81042fbeb05221bec0c4b26f5897b3f28bc267d26aefa", @@ -1031,7 +1032,7 @@ "packages/ui/src/v2/components/text-shimmer-v2.tsx" ], "exceptions": [], - "extracted_at": "2026-10-26T18:34:42.306Z", + "extracted_at": "2026-10-28T17:45:43.657Z", "per_package": { "packages/app": { "A": 257, @@ -1049,7 +1050,7 @@ "D": 0 }, "packages/opencode": { - "A": 101, + "A": 102, "M": 100, "D": 0 }, @@ -1080,7 +1081,7 @@ } }, "counts": { - "overlay_total": 1009, + "overlay_total": 1010, "deletions": 7, "server_coupled": 0 }, @@ -1435,6 +1436,7 @@ "packages/opencode/src/project/instance-store.ts": "M", "packages/opencode/src/provider/auth.ts": "M", "packages/opencode/src/provider/error.ts": "M", + "packages/opencode/src/provider/probe.ts": "A", "packages/opencode/src/provider/provider.ts": "M", "packages/opencode/src/provider/transform.ts": "M", "packages/opencode/src/question/index.ts": "M", diff --git a/packages/app-bundle/overlay/packages/opencode/src/provider/error.ts b/packages/app-bundle/overlay/packages/opencode/src/provider/error.ts index 592590277..0830cc406 100644 --- a/packages/app-bundle/overlay/packages/opencode/src/provider/error.ts +++ b/packages/app-bundle/overlay/packages/opencode/src/provider/error.ts @@ -204,4 +204,56 @@ export function parseAPICallError(input: { providerID: ProviderV2.ID; error: API } } +// #1984: reachability-probe classifier. A one-token probe against a catalog +// model fails in a handful of recognisable ways; each maps to a named status so +// the picker can hide dead models with a reason instead of failing mid-turn. +// Shapes are the ones observed on Bedrock (2026-10-28), but the patterns are +// provider-neutral where possible. +export type ProbeStatus = "ok" | "denied" | "unavailable" | "invalid" | "rate-limited" | "error" + +export type ProbeClassification = { + status: Exclude + message: string +} + +const UNAVAILABLE = + /not available for this account|aws[- ]marketplace|marketplace (subscription|agreement)|model access is denied|access to (this|the) model is not|not subscribed/i +const RATE_LIMITED = /throttl|too many requests|rate[- ]?limit|quota exceeded|tokens per (minute|day)/i +const INVALID = + /validationexception|inference profile|on-demand throughput|invalid model|model identifier is invalid|model[_ ]not[_ ]found|unknown model|no such model|does not exist|is not supported|resourcenotfoundexception/i +const DENIED = /accessdenied|access denied|not authorized|unauthorized|unrecognizedclient|forbidden|permission denied|invalid api key|invalid_api_key|security token/i + +function headerValue(headers: Record | undefined, name: string) { + if (!headers) return undefined + const key = Object.keys(headers).find((k) => k.toLowerCase() === name) + return key ? headers[key] : undefined +} + +export function classifyProbeError(input: { providerID: ProviderV2.ID; error: unknown }): ProbeClassification { + const error = input.error + if (APICallError.isInstance(error)) { + const parsed = parseAPICallError({ providerID: input.providerID, error }) + const status = error.statusCode + const errorType = headerValue(error.responseHeaders, "x-amzn-errortype") ?? "" + const haystack = [parsed.message, errorType, error.responseBody ?? ""].join("\n") + if (UNAVAILABLE.test(haystack)) return { status: "unavailable", message: parsed.message } + if (status === 429 || RATE_LIMITED.test(haystack)) return { status: "rate-limited", message: parsed.message } + if (status === 404 || INVALID.test(haystack)) return { status: "invalid", message: parsed.message } + if (status === 401 || status === 403 || DENIED.test(haystack)) return { status: "denied", message: parsed.message } + return { status: "error", message: parsed.message } + } + if (error instanceof Error) { + if (error.name === "TimeoutError" || error.name === "AbortError" || error instanceof HeaderTimeoutError) + return { status: "error", message: `probe timed out: ${error.message}` } + if (error.name === "AI_NoSuchModelError" || error.name === "ProviderModelNotFoundError") + return { status: "invalid", message: error.message } + const text = `${error.name}: ${error.message}` + if (UNAVAILABLE.test(text)) return { status: "unavailable", message: error.message } + if (RATE_LIMITED.test(text)) return { status: "rate-limited", message: error.message } + if (DENIED.test(text)) return { status: "denied", message: error.message } + return { status: "error", message: error.message } + } + return { status: "error", message: String(error) } +} + export * as ProviderError from "./error" diff --git a/packages/app-bundle/overlay/packages/opencode/src/provider/probe.ts b/packages/app-bundle/overlay/packages/opencode/src/provider/probe.ts new file mode 100644 index 000000000..9a7c5d862 --- /dev/null +++ b/packages/app-bundle/overlay/packages/opencode/src/provider/probe.ts @@ -0,0 +1,160 @@ +// #1984: one-token reachability probe per catalog model. The probe runs +// entirely inside the engine through Provider.getLanguage, so credentials never +// leave the process; the response carries only a named status, latency, a +// redacted reason and a stable non-secret credential fingerprint. +import { Cause, Effect, Schema } from "effect" +import { Hash } from "@opencode-ai/core/util/hash" +import { ProviderV2 } from "@opencode-ai/core/provider" +import { Auth } from "../auth" +import { Env } from "../env" +import { Provider } from "./provider" +import { ProviderError } from "./error" + +export const PROBE_MAX_OUTPUT_TOKENS = 1 +export const PROBE_TIMEOUT_MS = 10_000 +export const PROBE_CONCURRENCY = 4 +export const PROBE_MAX_MODELS = 500 +const REASON_MAX_CHARS = 300 + +export const Status = Schema.Literals(["ok", "denied", "unavailable", "invalid", "rate-limited", "error"]) +export type Status = Schema.Schema.Type + +export const Input = Schema.Struct({ + providerID: ProviderV2.ID, + modelIDs: Schema.Array(Schema.String).check(Schema.isMaxLength(PROBE_MAX_MODELS)), +}).annotate({ identifier: "ProviderProbeInput" }) +export type Input = Schema.Schema.Type + +export const Result = Schema.Struct({ + model: Schema.String, + status: Status, + latency_ms: Schema.Number, + reason: Schema.NullOr(Schema.String), + credential_fp: Schema.NullOr(Schema.String), + checked_at: Schema.Number, +}).annotate({ identifier: "ProviderProbeResult" }) +export type Result = Schema.Schema.Type + +export const Output = Schema.Struct({ + providerID: Schema.String, + results: Schema.Array(Result), +}).annotate({ identifier: "ProviderProbeOutput" }) +export type Output = Schema.Schema.Type + +function strings(input: unknown, out: string[] = []): string[] { + if (typeof input === "string") out.push(input) + else if (Array.isArray(input)) for (const item of input) strings(item, out) + else if (input && typeof input === "object") for (const value of Object.values(input)) strings(value, out) + return out +} + +/** + * Stable, non-secret fingerprint of the credential a provider resolves with: + * the stored auth entry (cf. #1848 authStamp, which stamps the whole + * auth.json), plus the config/env material that selects an identity or + * endpoint. A truncated domain-separated sha256 — it identifies "same + * credential" for the probe cache and cannot be reversed into the secret. + */ +export function credentialFingerprint(input: { + providerID: string + auth: unknown + provider: Pick + env: Record +}) { + const options = input.provider.options ?? {} + const material = { + auth: input.auth ?? null, + key: input.provider.key ?? null, + apiKey: options.apiKey ?? null, + baseURL: options.baseURL ?? null, + profile: options.profile ?? null, + region: options.region ?? null, + env: Object.fromEntries([...input.provider.env].sort().map((name) => [name, input.env[name] ?? null])), + } + return Hash.sha256(`amicode/credential-fp/v1\0${input.providerID}\0${JSON.stringify(material)}`).slice(0, 16) +} + +/** Strip every known secret (and generic bearer/AWS key shapes) from a reason string. */ +export function redact(text: string, secrets: string[]) { + let out = text + for (const secret of [...new Set(secrets)].filter((s) => s.length >= 6).sort((a, b) => b.length - a.length)) + out = out.split(secret).join("[redacted]") + out = out + .replace(/Bearer\s+\S+/gi, "[redacted]") + .replace(/\b(AKIA|ASIA)[A-Z0-9]{12,}\b/g, "[redacted]") + .replace(/\bsk-[A-Za-z0-9_-]{8,}/g, "[redacted]") + return out.length > REASON_MAX_CHARS ? `${out.slice(0, REASON_MAX_CHARS)}…` : out +} + +export const run = Effect.fn("ProviderProbe.run")(function* (input: Input) { + const provider = yield* Provider.Service + const auth = yield* Auth.Service + const env = yield* Env.Service + + const modelIDs = [...new Set(input.modelIDs)] + const info = yield* provider.getProvider(input.providerID) + const now = () => Date.now() + if (!info) { + const checked_at = now() + return { + providerID: input.providerID, + results: modelIDs.map((model) => ({ + model, + status: "error" as const, + latency_ms: 0, + reason: "provider not connected", + credential_fp: null, + checked_at, + })), + } satisfies Output + } + + const entry = yield* auth.get(input.providerID).pipe(Effect.orElseSucceed(() => undefined)) + const envs = yield* env.all() + const credential_fp = credentialFingerprint({ providerID: input.providerID, auth: entry, provider: info, env: envs }) + const secrets = [ + ...strings(entry), + ...strings(info.key), + ...strings(info.options?.apiKey), + ...strings(info.options?.headers), + ...info.env.map((name) => envs[name]).filter((v): v is string => typeof v === "string"), + ] + + const probeOne = Effect.fn("ProviderProbe.one")(function* (modelID: string) { + const started = now() + const done = (status: Status, reason: string | null): Result => ({ + model: modelID, + status, + latency_ms: now() - started, + reason: reason === null ? null : redact(reason, secrets), + credential_fp, + checked_at: now(), + }) + const model = info.models[modelID] + if (!model) return done("invalid", "model not listed by provider") + const outcome = yield* provider.getLanguage(model).pipe( + Effect.flatMap((language) => + Effect.tryPromise({ + try: () => + language.doGenerate({ + prompt: [{ role: "user", content: [{ type: "text", text: "ping" }] }], + maxOutputTokens: PROBE_MAX_OUTPUT_TOKENS, + headers: model.headers, + abortSignal: AbortSignal.timeout(PROBE_TIMEOUT_MS), + }), + catch: (error) => error, + }), + ), + Effect.as(undefined), + Effect.catchCause((cause) => Effect.succeed({ error: Cause.squash(cause) })), + ) + if (!outcome) return done("ok", null) + const classified = ProviderError.classifyProbeError({ providerID: model.providerID, error: outcome.error }) + return done(classified.status, classified.message) + }) + + const results = yield* Effect.forEach(modelIDs, probeOne, { concurrency: PROBE_CONCURRENCY }) + return { providerID: input.providerID, results } satisfies Output +}) + +export * as ProviderProbe from "./probe" diff --git a/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/groups/provider.ts b/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/groups/provider.ts index 7e572b799..3ed314a1e 100644 --- a/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/groups/provider.ts +++ b/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/groups/provider.ts @@ -1,5 +1,6 @@ import { ProviderAuth } from "@/provider/auth" import { Provider } from "@/provider/provider" +import { ProviderProbe } from "@/provider/probe" import { Schema } from "effect" import { HttpApi, HttpApiEndpoint, HttpApiGroup, OpenApi } from "effect/unstable/httpapi" @@ -56,6 +57,19 @@ export const ProviderApi = HttpApi.make("provider") description: "Retrieve available authentication methods for all AI providers.", }), ), + // #1984: one-token reachability probe per model, credentials stay in the engine. + HttpApiEndpoint.post("probe", `${root}/probe`, { + query: WorkspaceRoutingQuery, + payload: ProviderProbe.Input, + success: described(ProviderProbe.Output, "Per-model reachability results"), + }).annotateMerge( + OpenApi.annotations({ + identifier: "provider.probe", + summary: "Probe model reachability", + description: + "Send a one-token request to each listed model of a provider through the engine's own credentials and return a named status (ok, denied, unavailable, invalid, rate-limited, error) per model. No secret material is returned.", + }), + ), HttpApiEndpoint.post("authorize", `${root}/:providerID/oauth/authorize`, { params: { providerID: ProviderV2.ID }, query: WorkspaceRoutingQuery, diff --git a/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/provider.ts b/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/provider.ts index 9c0c7ed89..ecb04efd6 100644 --- a/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/provider.ts +++ b/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/provider.ts @@ -2,6 +2,7 @@ import { ProviderAuth } from "@/provider/auth" import { Config } from "@/config/config" import { ModelsDev } from "@opencode-ai/core/models-dev" import { Provider } from "@/provider/provider" +import { ProviderProbe } from "@/provider/probe" import { mapValues } from "remeda" import { Effect, Schema } from "effect" @@ -65,6 +66,10 @@ export const providerHandlers = HttpApiBuilder.group(InstanceHttpApi, "provider" return yield* svc.methods() }) + const probe = Effect.fn("ProviderHttpApi.probe")(function* (ctx: { payload: ProviderProbe.Input }) { + return yield* ProviderProbe.run(ctx.payload) + }) + const authorize = Effect.fn("ProviderHttpApi.authorize")(function* (ctx: { params: { providerID: ProviderV2.ID } payload: ProviderAuth.AuthorizeInput @@ -110,6 +115,7 @@ export const providerHandlers = HttpApiBuilder.group(InstanceHttpApi, "provider" return handlers .handle("list", list) .handle("auth", auth) + .handle("probe", probe) .handleRaw("authorize", authorizeRaw) .handle("callback", callback) }), diff --git a/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-provider.test.ts b/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-provider.test.ts index 80dff80d8..5b469e615 100644 --- a/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-provider.test.ts +++ b/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-provider.test.ts @@ -1,4 +1,4 @@ -import { describe, expect } from "bun:test" +import { afterAll, describe, expect } from "bun:test" import { LayerNode } from "@opencode-ai/core/effect/layer-node" import { FSUtil } from "@opencode-ai/core/fs-util" import { Effect, Layer } from "effect" @@ -452,3 +452,197 @@ describe("provider HttpApi", () => { { ...projectOptions, init: writeProviderModelsMutationPlugin }, ) }) + +// #1984 — POST /provider/probe: one-token reachability probe with named +// statuses. A fake OpenAI-compatible upstream answers per model id with the +// error shapes Bedrock returns (AccessDenied, Marketplace "not available for +// this account", ValidationException on a profile-only id, throttling), so the +// engine's classifier is exercised end-to-end through Provider.getLanguage. +const PROBE_SECRET = "sk-probe-SECRET-0123456789abcdef" +const probeSeen: { model?: string; authorization?: string | null; maxTokens?: unknown }[] = [] + +function openAiError(status: number, message: string, type: string, headers: Record = {}) { + return new Response(JSON.stringify({ error: { message, type, code: type } }), { + status, + headers: { "content-type": "application/json", ...headers }, + }) +} + +const probeUpstream = Bun.serve({ + port: 0, + async fetch(req) { + const body = (await req.json().catch(() => ({}))) as Record + const model = typeof body.model === "string" ? body.model : undefined + probeSeen.push({ + model, + authorization: req.headers.get("authorization"), + maxTokens: body.max_tokens ?? body.max_completion_tokens, + }) + switch (model) { + case "probe-ok": + return Response.json({ + id: "chatcmpl-probe", + object: "chat.completion", + created: 1, + model, + choices: [{ index: 0, message: { role: "assistant", content: "p" }, finish_reason: "length" }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }) + case "probe-denied": + return openAiError( + 403, + `User: arn:aws:iam::123456789012:user/probe is not authorized to perform: bedrock:InvokeModel (key ${PROBE_SECRET})`, + "AccessDeniedException", + { "x-amzn-errortype": "AccessDeniedException" }, + ) + case "probe-unavailable": + return openAiError(403, "This model version is not available for this account.", "AccessDeniedException") + case "probe-invalid": + return openAiError( + 400, + "Invocation of model ID probe-invalid with on-demand throughput isn't supported. Retry your request with the ID or ARN of an inference profile that contains this model.", + "ValidationException", + ) + case "probe-throttled": + return openAiError(429, "Too many requests, please wait before trying again.", "ThrottlingException") + default: + return openAiError(500, "unexpected", "InternalServerException") + } + }, +}) +afterAll(() => probeUpstream.stop(true)) + +function probeModel(id: string) { + return { + id, + name: id, + attachment: false, + reasoning: false, + temperature: false, + tool_call: true, + release_date: "2025-01-01", + limit: { context: 100_000, output: 10_000 }, + cost: { input: 0, output: 0 }, + options: {}, + } +} + +const probeOptions = { + config: () => ({ + formatter: false as const, + lsp: false as const, + provider: { + "probe-fake": { + name: "Probe Fake", + id: "probe-fake", + env: [], + npm: "@ai-sdk/openai-compatible", + models: Object.fromEntries( + ["probe-ok", "probe-denied", "probe-unavailable", "probe-invalid", "probe-throttled"].map((id) => [ + id, + probeModel(id), + ]), + ), + options: { apiKey: PROBE_SECRET, baseURL: probeUpstream.url.href.replace(/\/$/, "") }, + }, + }, + }), +} + +describe("provider probe HttpApi (#1984)", () => { + it.instance( + "classifies ok, denied, unavailable, invalid and rate-limited models with no secret material", + Effect.gen(function* () { + probeSeen.length = 0 + const directory = (yield* TestInstance).directory + const response = yield* request("/provider/probe", { + method: "POST", + headers: { "x-opencode-directory": directory, "content-type": "application/json" }, + body: JSON.stringify({ + providerID: "probe-fake", + modelIDs: ["probe-ok", "probe-denied", "probe-unavailable", "probe-invalid", "probe-throttled", "nope"], + }), + }) + const text = yield* response.text + expect(response.status).toBe(200) + const body = JSON.parse(text) + expect(body.providerID).toBe("probe-fake") + type ProbeRow = { + model: string + status: string + latency_ms: number + reason: string | null + credential_fp: string | null + } + const byModel: Record = Object.fromEntries( + (body.results as ProbeRow[]).map((r) => [r.model, r]), + ) + expect(byModel["probe-ok"].status).toBe("ok") + expect(byModel["probe-ok"].reason).toBeNull() + expect(byModel["probe-denied"].status).toBe("denied") + expect(byModel["probe-unavailable"].status).toBe("unavailable") + expect(byModel["probe-invalid"].status).toBe("invalid") + expect(byModel["probe-throttled"].status).toBe("rate-limited") + // a model id the provider does not list never reaches the network + expect(byModel["nope"].status).toBe("invalid") + for (const result of Object.values(byModel)) { + expect(typeof result.latency_ms).toBe("number") + expect(result.credential_fp).toMatch(/^[0-9a-f]{16}$/) + } + // one stable fingerprint per credential + expect(new Set(Object.values(byModel).map((r) => r.credential_fp)).size).toBe(1) + + // the probe really went through the configured credential... + expect(probeSeen.length).toBe(5) + expect(probeSeen.every((seen) => seen.authorization === `Bearer ${PROBE_SECRET}`)).toBe(true) + // ...asked for a single output token... + expect(probeSeen.every((seen) => seen.maxTokens === 1)).toBe(true) + // ...and no secret material comes back, even when the upstream echoes it + expect(text).not.toContain(PROBE_SECRET) + expect(text).not.toContain("SECRET") + expect(text).not.toContain("Bearer") + }), + probeOptions, + 30000, + ) + + it.instance( + "keeps the credential fingerprint stable across probes", + Effect.gen(function* () { + const directory = (yield* TestInstance).directory + const probe = () => + Effect.gen(function* () { + const response = yield* request("/provider/probe", { + method: "POST", + headers: { "x-opencode-directory": directory, "content-type": "application/json" }, + body: JSON.stringify({ providerID: "probe-fake", modelIDs: ["probe-ok"] }), + }) + return JSON.parse(yield* response.text).results[0].credential_fp as string + }) + const first = yield* probe() + const second = yield* probe() + expect(first).toBe(second) + expect(first).not.toContain(PROBE_SECRET.slice(0, 8)) + }), + probeOptions, + 30000, + ) + + it.instance( + "reports models of an unconnected provider as errors without probing", + Effect.gen(function* () { + const directory = (yield* TestInstance).directory + const response = yield* request("/provider/probe", { + method: "POST", + headers: { "x-opencode-directory": directory, "content-type": "application/json" }, + body: JSON.stringify({ providerID: "not-connected", modelIDs: ["x"] }), + }) + expect(response.status).toBe(200) + const body = JSON.parse(yield* response.text) + expect(body.results).toHaveLength(1) + expect(body.results[0]).toMatchObject({ model: "x", status: "error", credential_fp: null }) + }), + projectOptions, + 30000, + ) +}) From 8bd9f3a9929ef8084fa86fbf2ac352c37e5cbb86 Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Wed, 28 Oct 2026 13:53:09 -0400 Subject: [PATCH 02/13] =?UTF-8?q?test(service):=20RED=20=E2=80=94=20model-?= =?UTF-8?q?reachability=20cache,=20rate=20limit,=20demote,=20runner=20wiri?= =?UTF-8?q?ng=20(#1985)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...amicode_service_model_reachability.test.ts | 552 ++++++++++++++++++ 1 file changed, 552 insertions(+) create mode 100644 packages/extension/test/amicode_service_model_reachability.test.ts diff --git a/packages/extension/test/amicode_service_model_reachability.test.ts b/packages/extension/test/amicode_service_model_reachability.test.ts new file mode 100644 index 000000000..3d7fb5978 --- /dev/null +++ b/packages/extension/test/amicode_service_model_reachability.test.ts @@ -0,0 +1,552 @@ +// #1985 (#1979 S2): the amicode service's model-reachability cache + +// scheduler + route family. The probe itself is S1's engine route +// (POST /provider/probe, #1984); every test here drives a FAKE probe so the +// cache / rate-limit / invalidation / locking semantics are pinned without an +// engine. The last groups boot the real service (createAmicodeService) and the +// headless hub runner (bootAmicodeServiceRunner over a fake engine) to prove +// the route family is wired on both paths — the hub runner skipped the +// model-routing wiring once; the reachability family must not repeat that. +import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + ModelReachability, + reachabilityCacheFile, + modelReachabilityResponse, + refreshModelReachabilityResponse, + demoteModelReachabilityResponse, + type ProbeOutput, + type ProbeStatus, +} from "../src/amicode_service/model_reachability"; +import { fetchProviderModels, probeProviderModels } from "../src/llm_creds.mjs"; +import { bootAmicodeServiceRunner, type AmicodeServiceRunnerBoot } from "../src/amicode_service_runner"; + +interface Harness { + file: string; + clock: { now: number }; + fp: { current: string | null }; + statuses: Record; + catalog: Record; + calls: Array<{ providerID: string; modelIDs: string[] }>; + make(overrides?: Partial[0]>): ModelReachability; +} + +function harness(file?: string): Harness { + const dir = mkdtempSync(join(tmpdir(), "amc-1985-")); + const h: Harness = { + file: file ?? join(dir, "model-reachability.json"), + clock: { now: 1_800_000_000_000 }, + fp: { current: "aaaaaaaaaaaaaaaa" }, + statuses: {}, + catalog: { anthropic: ["claude-a", "claude-b"], openai: ["gpt-x"] }, + calls: [], + make(overrides = {}) { + return new ModelReachability({ + file: h.file, + now: () => h.clock.now, + listModels: async () => structuredClone(h.catalog), + probe: async (providerID: string, modelIDs: string[]): Promise => { + h.calls.push({ providerID, modelIDs: [...modelIDs] }); + return { + providerID, + results: modelIDs.map((model) => ({ + model, + status: h.statuses[`${providerID}/${model}`] ?? "ok", + latency_ms: 7, + reason: h.statuses[`${providerID}/${model}`] ? "fake reason" : null, + credential_fp: h.fp.current, + checked_at: h.clock.now, + })), + }; + }, + ...overrides, + }); + }, + }; + return h; +} + +const row = (snap: { models: Array<{ provider: string; model: string }> }, provider: string, model: string) => + snap.models.find((m) => m.provider === provider && m.model === model) as + | { provider: string; model: string; status: string; reason: string | null; credential_fp: string | null; source: string | null } + | undefined; + +describe("unknown is unchecked, never ok", () => { + it("a catalog model that was never probed is reported as unchecked", async () => { + const h = harness(); + const snap = await h.make().snapshot(); + expect(snap.catalog_known).toBe(true); + for (const m of snap.models) expect(m.status).toBe("unchecked"); + expect(row(snap, "anthropic", "claude-a")?.status).toBe("unchecked"); + expect(row(snap, "openai", "gpt-x")?.status).toBe("unchecked"); + }); + + it("no catalog and no cache → no rows (nothing is invented), catalog_known false", async () => { + const h = harness(); + const snap = await h.make({ listModels: async () => undefined }).snapshot(); + expect(snap.catalog_known).toBe(false); + expect(snap.models).toEqual([]); + }); +}); + +describe("fake probe results are cached and served", () => { + it("a refresh caches the probe results; a SECOND instance on the same file serves them without probing", async () => { + const h = harness(); + h.statuses["anthropic/claude-b"] = "denied"; + const a = h.make(); + const res = await a.refresh({ provider: "anthropic" }); + expect(res.refreshed).toEqual(["anthropic"]); + expect(h.calls).toEqual([{ providerID: "anthropic", modelIDs: ["claude-a", "claude-b"] }]); + + const b = h.make(); + const snap = await b.snapshot(); + expect(row(snap, "anthropic", "claude-a")).toMatchObject({ status: "ok", credential_fp: "aaaaaaaaaaaaaaaa", source: "probe" }); + expect(row(snap, "anthropic", "claude-b")).toMatchObject({ status: "denied", reason: "fake reason" }); + expect(row(snap, "openai", "gpt-x")?.status).toBe("unchecked"); + expect(h.calls.length).toBe(1); + // persisted on disk, keyed by provider → model, with the fp + const doc = JSON.parse(readFileSync(h.file, "utf8")); + expect(doc.providers.anthropic.credential_fp).toBe("aaaaaaaaaaaaaaaa"); + expect(doc.providers.anthropic.models["claude-a"].credential_fp).toBe("aaaaaaaaaaaaaaaa"); + }); + + it("the GET body serves the snapshot shape", async () => { + const h = harness(); + const mr = h.make(); + await mr.refresh({ provider: "openai" }); + const body = JSON.parse(await modelReachabilityResponse(mr)); + expect(body.ok).toBe(true); + expect(body.error).toBeNull(); + expect(body.catalog_known).toBe(true); + expect(row(body, "openai", "gpt-x")).toMatchObject({ status: "ok", latency_ms: 7, checked_at: h.clock.now }); + const p = body.providers.find((x: { provider: string }) => x.provider === "openai"); + expect(p).toMatchObject({ credential_fp: "aaaaaaaaaaaaaaaa", last_probe_at: h.clock.now, last_error: null }); + }); + + it("a provider dropped from the live catalog is not served from the cache", async () => { + const h = harness(); + const mr = h.make(); + await mr.refresh({ provider: "openai" }); + delete h.catalog.openai; + const snap = await mr.snapshot(); + expect(row(snap, "openai", "gpt-x")).toBeUndefined(); + }); +}); + +describe("credential_fp change invalidates", () => { + it("a probe under a NEW fingerprint drops every row cached under the old one (→ unchecked)", async () => { + const h = harness(); + const mr = h.make({ minProbeIntervalMs: 0 }); + await mr.refresh({ provider: "anthropic" }); + expect(row(await mr.snapshot(), "anthropic", "claude-b")?.status).toBe("ok"); + + // The credential rotates; the next probe only reaches claude-a (the + // catalog shrank to it) — claude-b's old-fp row must NOT stay "ok". + h.fp.current = "bbbbbbbbbbbbbbbb"; + h.catalog.anthropic = ["claude-a"]; + h.clock.now += 1000; + await mr.refresh({ provider: "anthropic" }); + h.catalog.anthropic = ["claude-a", "claude-b"]; + const snap = await mr.snapshot(); + expect(row(snap, "anthropic", "claude-a")).toMatchObject({ status: "ok", credential_fp: "bbbbbbbbbbbbbbbb" }); + expect(row(snap, "anthropic", "claude-b")).toMatchObject({ status: "unchecked", credential_fp: null }); + }); + + it("an entry whose fp disagrees with the provider's current fp is served as unchecked (hand-edited / partial file)", async () => { + const h = harness(); + mkdirSync(join(h.file, ".."), { recursive: true }); + writeFileSync( + h.file, + JSON.stringify({ + version: 1, + providers: { + anthropic: { + credential_fp: "cccccccccccccccc", + last_probe_at: h.clock.now, + last_attempt_at: h.clock.now, + last_error: null, + models: { + "claude-a": { status: "ok", reason: null, latency_ms: 1, checked_at: h.clock.now, credential_fp: "dddddddddddddddd", source: "probe" }, + }, + }, + }, + }), + ); + const snap = await h.make().snapshot(); + expect(row(snap, "anthropic", "claude-a")?.status).toBe("unchecked"); + }); + + it("a disconnected provider (probe says error, fp null) invalidates the old rows", async () => { + const h = harness(); + const mr = h.make({ minProbeIntervalMs: 0 }); + await mr.refresh({ provider: "openai" }); + h.fp.current = null; + h.statuses["openai/gpt-x"] = "error"; + await mr.refresh({ provider: "openai" }); + expect(row(await mr.snapshot(), "openai", "gpt-x")).toMatchObject({ status: "error", credential_fp: null }); + }); +}); + +describe("concurrent writers don't corrupt the cache", () => { + it("two instances on ONE file, interleaving demotes + probe merges, lose no update and leave valid JSON", async () => { + const h = harness(); + h.catalog = { p1: Array.from({ length: 12 }, (_, i) => `m${i}`), p2: Array.from({ length: 12 }, (_, i) => `n${i}`) }; + const a = h.make(); + const b = h.make(); + const ops: Promise[] = []; + for (let i = 0; i < 20; i++) { + ops.push(a.demote({ provider: "q", model: `a${i}`, status: "denied", reason: `a${i}` })); + ops.push(b.demote({ provider: "q", model: `b${i}`, status: "unavailable", reason: `b${i}` })); + } + ops.push(a.refresh({ provider: "p1" })); + ops.push(b.refresh({ provider: "p2" })); + const outs = await Promise.all(ops); + for (const o of outs) expect((o as { ok: boolean }).ok).toBe(true); + + const doc = JSON.parse(readFileSync(h.file, "utf8")); + expect(Object.keys(doc.providers.q.models).length).toBe(40); + expect(Object.keys(doc.providers.p1.models).length).toBe(12); + expect(Object.keys(doc.providers.p2.models).length).toBe(12); + expect(existsSync(`${h.file}.lock`)).toBe(false); + }); + + it("a stale lock (dead owner, past the stale window) is reclaimed instead of wedging the cache", async () => { + const h = harness(); + mkdirSync(`${h.file}.lock`, { recursive: true }); + writeFileSync(join(`${h.file}.lock`, "owner"), JSON.stringify({ pid: 999999, ts: Date.now() - 60_000 })); + const res = await h.make({ staleLockMs: 1000 }).demote({ provider: "openai", model: "gpt-x", status: "denied", reason: null }); + expect(res.ok).toBe(true); + expect(existsSync(`${h.file}.lock`)).toBe(false); + }); +}); + +describe("refresh is rate-limited", () => { + it("a second refresh inside the floor is refused with retry_after_ms and does NOT probe — across instances too", async () => { + const h = harness(); + const a = h.make({ minProbeIntervalMs: 60_000 }); + const first = await a.refresh({ provider: "anthropic" }); + expect(first.refreshed).toEqual(["anthropic"]); + + h.clock.now += 10_000; + const again = await a.refresh({ provider: "anthropic" }); + expect(again.refreshed).toEqual([]); + expect(again.rate_limited).toEqual([{ provider: "anthropic", retry_after_ms: 50_000 }]); + + const other = await h.make({ minProbeIntervalMs: 60_000 }).refresh({ provider: "anthropic" }); + expect(other.rate_limited.map((r) => r.provider)).toEqual(["anthropic"]); + expect(h.calls.length).toBe(1); + + h.clock.now += 50_000; + const later = await a.refresh({ provider: "anthropic" }); + expect(later.refreshed).toEqual(["anthropic"]); + expect(h.calls.length).toBe(2); + }); + + it("concurrent refreshes from two instances probe once (the claim is taken under the lock)", async () => { + const h = harness(); + const [r1, r2] = await Promise.all([h.make().refresh({ provider: "openai" }), h.make().refresh({ provider: "openai" })]); + expect(h.calls.length).toBe(1); + expect([...r1.refreshed, ...r2.refreshed]).toEqual(["openai"]); + }); + + it("a failed probe still counts against the floor (no hammering a dead engine) and records last_error", async () => { + const h = harness(); + let n = 0; + const mr = h.make({ + probe: async () => { + n++; + throw new Error("engine down"); + }, + }); + const res = await mr.refresh({ provider: "openai" }); + expect(res.failed).toEqual([{ provider: "openai", error: "probe_failed" }]); + const again = await mr.refresh({ provider: "openai" }); + expect(again.rate_limited.length).toBe(1); + expect(n).toBe(1); + const snap = await mr.snapshot(); + expect(snap.providers.find((p) => p.provider === "openai")?.last_error).toBe("probe_failed"); + expect(row(snap, "openai", "gpt-x")?.status).toBe("unchecked"); + }); + + it("the scheduler tick probes on connect (never probed) and daily — not on every tick", async () => { + const h = harness(); + const mr = h.make({ minProbeIntervalMs: 60_000, dailyIntervalMs: 86_400_000 }); + await mr.tick(); + expect(h.calls.map((c) => c.providerID).sort()).toEqual(["anthropic", "openai"]); + h.clock.now += 3_600_000; + await mr.tick(); + expect(h.calls.length).toBe(2); + // a provider connects later → probed on the next tick (on-connect) + h.catalog.google = ["gemini"]; + await mr.tick(); + expect(h.calls.map((c) => c.providerID)).toEqual(expect.arrayContaining(["google"])); + expect(h.calls.length).toBe(3); + h.clock.now += 86_400_000; + await mr.tick(); + expect(h.calls.length).toBe(6); + }); + + it("the refresh route body: bad provider → bad_request; unknown catalog → engine_unavailable", async () => { + const h = harness(); + const bad = JSON.parse(await refreshModelReachabilityResponse({ provider: 42 }, h.make())); + expect(bad).toMatchObject({ ok: false }); + expect(bad.error).toMatch(/^bad_request: /); + const unknown = JSON.parse(await refreshModelReachabilityResponse("{}", h.make({ listModels: async () => undefined }))); + expect(unknown.ok).toBe(false); + expect(unknown.error).toMatch(/^engine_unavailable: /); + const notConnected = JSON.parse(await refreshModelReachabilityResponse({ provider: "nope" }, h.make())); + expect(notConnected.error).toMatch(/^bad_request: /); + const ok = JSON.parse(await refreshModelReachabilityResponse({}, h.make())); + expect(ok.ok).toBe(true); + expect(ok.refreshed.sort()).toEqual(["anthropic", "openai"]); + }); +}); + +describe("demote sticks until the next probe", () => { + it("a demotion overrides a cached ok across instances and ticks, and the next probe replaces it", async () => { + const h = harness(); + const a = h.make({ minProbeIntervalMs: 60_000 }); + await a.refresh({ provider: "anthropic" }); + const d = JSON.parse( + await demoteModelReachabilityResponse( + { provider: "anthropic", model: "claude-a", status: "rate-limited", reason: "429 in dispatch" }, + a, + ), + ); + expect(d).toMatchObject({ ok: true, provider: "anthropic", model: "claude-a", status: "rate-limited", error: null }); + + const b = h.make({ minProbeIntervalMs: 60_000 }); + expect(row(await b.snapshot(), "anthropic", "claude-a")).toMatchObject({ + status: "rate-limited", + reason: "429 in dispatch", + source: "demoted", + credential_fp: "aaaaaaaaaaaaaaaa", + }); + await b.tick(); // not due — the demotion stands + expect(row(await b.snapshot(), "anthropic", "claude-a")?.status).toBe("rate-limited"); + + h.clock.now += 60_000; + await b.refresh({ provider: "anthropic" }); + expect(row(await a.snapshot(), "anthropic", "claude-a")).toMatchObject({ status: "ok", source: "probe" }); + }); + + it("the demote body is validated (no ok demotion, no unknown status, strings only) and never echoed", async () => { + const h = harness(); + const mr = h.make(); + for (const body of [ + { provider: "anthropic", model: "claude-a", status: "ok" }, + { provider: "anthropic", model: "claude-a", status: "unchecked" }, + { provider: "anthropic", model: "claude-a", status: "