diff --git a/packages/app-bundle/manifest.json b/packages/app-bundle/manifest.json index cf31e78cb..eb8572477 100644 --- a/packages/app-bundle/manifest.json +++ b/packages/app-bundle/manifest.json @@ -121,6 +121,13 @@ "packages/app/src/components/dialog-release-notes.tsx": "0ea5b049786787643a7f6b54d4ccf84c4f8ffa4e8272426c0d32f60d445b6890", "packages/app/src/components/dialog-select-directory-v2.css": "2e2f18abdca77f9fe85da157c40f288f3b1641ca92845208749e16455fd58637", "packages/app/src/components/dialog-select-file.tsx": "e6278ee35dd27d203e00ad73f70c709c44a6c06f81ffbd931edfaa0c986b9e03", + "packages/app/src/components/dialog-select-model-reachability-source.ts": "ecd6c5e5f1df6d9e71cc87483d87870acb5c42140d580262c9dc76a8bd8453bf", + "packages/app/src/components/dialog-select-model-reachability.test.ts": "03e4a5e72a03f2c7c05deed3eb008a7f832af7e9beb10d2889c9fe92061e3346", + "packages/app/src/components/dialog-select-model-reachability.ts": "479403f9e2c799542e89fa0a28d4271684c48deccf02bd1e7c2caed9482e1602", + "packages/app/src/components/dialog-select-model-shortlist-source.ts": "3de6605f0e950a0af9e01297f492b14940fb1ef2456afb584215ab5e0a1455c1", + "packages/app/src/components/dialog-select-model-shortlist.test.ts": "57aa5604f7d9cba9f0f82103f382c988e9ea26a2cf104d14e89e56f0b1b0bc29", + "packages/app/src/components/dialog-select-model-shortlist.ts": "3412a56925a540c2d0843406f7018f689b990ff4a81402ee2ed35767590cfbcd", + "packages/app/src/components/dialog-select-model.tsx": "3773becba23527b2b31afd55d337eb61bc3fa4dcd5c9dd5a9b85500af841bea2", "packages/app/src/components/engine-state-signal.ts": "2726628b85d4bc58bf6b3bd3045fc4239ad384deaba14fc8f46b818741549a5e", "packages/app/src/components/engine-toggle-utils.test.ts": "bf53ca7a6fac3696f8d9aac032ceee5e35fb3b7e3dfaef3f9afb1fa0ff3135bb", "packages/app/src/components/engine-toggle-utils.ts": "bab8a7ddef9f49d4fc243477b7206feb741afdf5ec154a2d70a9f31c814f8c5d", @@ -129,6 +136,12 @@ "packages/app/src/components/harness-switch.test.ts": "db263ece42911e756fb8b6e88697a6bbbd0d1317decd1e63b027d9fe8c69b405", "packages/app/src/components/harness-switch.ts": "b4cb2b2241d71c477be71285fe6041d4b0d604a26d74c30dc5239694211a1f90", "packages/app/src/components/help-button.tsx": "780d533f776a784660cfed072a131198bc6fcc8e8fe332e8e1a220d516c9684e", + "packages/app/src/components/model-dead-pick-warning.tsx": "51a3b761437e7cf0d33d49cf06ffa7be20c59cbdfb073637a5d7945e67840c14", + "packages/app/src/components/model-dead-pick.test.ts": "b4bd51255fffc12ca92ebeb21893e2d1ee329cc5845c0dfe0a79cb1c82419013", + "packages/app/src/components/model-dead-pick.ts": "eaa584e8072b59a9a6f0b4056c202ffeff4d96c8a0270d962b3d004e2d79f143", + "packages/app/src/components/model-row-meta.test.ts": "51c5e8b9c4ef3f8e2616e63c8cbc16ae055dedc5fec7c24280ed28a3712938ad", + "packages/app/src/components/model-row-meta.ts": "7870b265f4ca9b08505ae6f1b0afb66ca002fe05f67c3b8194417a45f0b170cd", + "packages/app/src/components/model-tooltip.tsx": "c402abe9674806c9dd308589d7acb96dfd9c400299e650a94fcc44a165e920e2", "packages/app/src/components/new-session-machine-picker.ts": "8c7689f79c7610e32fef478320870d1468fa17a3a02463e2cbe058a487fb4769", "packages/app/src/components/pane-bridge.tsx": "756b3306b51c963fac6778716f6c6c4626f0f3bf75c93b7cb4cab58ea0bf3d4e", "packages/app/src/components/profile-popover.tsx": "eff742e90544455f13234a9351bb1b82189663cba2cd3172bdcea1a8a37284e8", @@ -194,7 +207,9 @@ "packages/app/src/context/local-agent.test.ts": "a5a9d60bb4401d409218cc247c1cc06ede8b7878ddf01dc6080328f54eb9cade", "packages/app/src/context/local-agent.ts": "0aab67e695dc3bb45a733ac0df80a0a5e14cfe29b2375b6dc3cd07fbccea33e2", "packages/app/src/context/local.tsx": "3ab8b9fc2db082df4ba485373679f00a95d0ffe3c691a0afee1dc53db7eabd9e", - "packages/app/src/context/models.tsx": "8904525767b36ed5ba93e85a0fddb9bf737a8d066dfff084380ce1a67182f535", + "packages/app/src/context/models-favorites.test.ts": "6336bc4dd46a0492e393110ee7aab6ee58f152982ec657e17ad9f750d22483f8", + "packages/app/src/context/models-favorites.ts": "60d4e6889ca9143432417f624ddee14047f14bcfd1325fca2f491fb19803b1a2", + "packages/app/src/context/models.tsx": "39821854ea95753c583c05f9b911a0898ed8ef7b6da49613a19aa3ee39404707", "packages/app/src/context/platform.tsx": "edd9b1773feb5c084b922a57b6914f6a473eed558b44f9de0eeb1c4d4e2b49da", "packages/app/src/context/question-inbox.test.ts": "d4e2c666194fe6463d1a1da1901f2aef3b6a17eec00394a2b020d7ce698d4c92", "packages/app/src/context/question-inbox.ts": "f42a2490670e44578f88d626bc0db0a0205fa2b5ec7cd719ba0544915f3a6179", @@ -238,7 +253,7 @@ "packages/app/src/i18n/compaction-failure.test.ts": "a4a3765beb238a2c157c07c9c8ede8418b9febb814d8d1e3a5b2d8868cbd80d9", "packages/app/src/i18n/da.ts": "db685cb94f22ace194850efe9bdc02c530777a4583dc9eb9d00b97002b0a0968", "packages/app/src/i18n/de.ts": "c440ad7543321cae1425e0c2fad92ff73963f21672fbf31176831933573bf103", - "packages/app/src/i18n/en.ts": "d073d90198297bcc33ad5da1975a3829250b20a1e18fec35eb65809c35fca973", + "packages/app/src/i18n/en.ts": "068394508cd4af085017cbd9344dd8e7ef6d6a85fa6f4be4251520834eee95fc", "packages/app/src/i18n/es.ts": "8cdec3592cddf0b91d7fd59002c81c4e8973c5bf6573e065df64d48c42e1802f", "packages/app/src/i18n/fr.ts": "b9f646352189054b6918bfbac670579e76418c3148eb7e5afca9d71d97e7f2b7", "packages/app/src/i18n/ja.ts": "27ffc41a0fd8dc4b2e814a45c55fad4726ff1c7a85c884cf6af95f8397f17cbf", @@ -357,7 +372,8 @@ "packages/opencode/src/project/instance-context.ts": "65a35792e50e96a50ea14cd9f928045c0e2fc15eb49506f5bbceee38c4af1457", "packages/opencode/src/project/instance-store.ts": "c22d3e2499e63dd8d9395d58189df75444d306774df833e746f22e327edcf2ab", "packages/opencode/src/provider/auth.ts": "b0ab1322c1039e27176bf9e3500603eae27506065ac141a69710220f3361f6b8", - "packages/opencode/src/provider/error.ts": "d72310be26cfae5c2943f9939a6461e848532a5646ff405d628e9a50fc1ebe91", + "packages/opencode/src/provider/error.ts": "eef84e43f4d0f2779e53a5570498d08d2f339fdc01c24c12046b9107dacef516", + "packages/opencode/src/provider/probe.ts": "6f5b7f3a201a67b66c387411f4c893c7e91ca74a70fdcf7b220f1a2ff511adc7", "packages/opencode/src/provider/provider.ts": "b89bc0ca0a12ea0cfaf5c956a4b731a0397d4c407e787a16cdd66fb7f54ee476", "packages/opencode/src/provider/transform.ts": "70c7492d4a23a30ebc340a5605702b1602960f4b412e18557534e02962c137b8", "packages/opencode/src/question/index.ts": "8317348c2ca27dd9e4f7d18fdc126a4d355c20495c1b86668430ad477a68e9b3", @@ -442,7 +458,7 @@ "packages/opencode/test/server/httpapi-file.test.ts": "d3386129bc81467ecbcc57a3861f53767ee56ac41b4ff6ad58d8b168fb3b3373", "packages/opencode/test/server/httpapi-global-event.test.ts": "8a78dc59edf48eb3cf6d1c09448f7dfe5ceb138e1bd6125949ebc88976c903f0", "packages/opencode/test/server/httpapi-instance-route-auth.test.ts": "4b73a89d2431c02bc3b665d93f9002914e51eb1add979dc1867efbcbc0c36e25", - "packages/opencode/test/server/httpapi-provider.test.ts": "9e8b3e085b478e3f99b689ffff600e10c0b4b50b3cabdbd749f88dae19bf5904", + "packages/opencode/test/server/httpapi-provider.test.ts": "a047ee63f5a1000738a33afc4fee6c19c994ffe6ba12e124e34b83762e11ac47", "packages/opencode/test/server/httpapi-session.test.ts": "eda6fd763ac4c17891cedaa4ccd49597c03e7227fcfdcf0f94a1b4545ba623ed", "packages/opencode/test/server/httpapi-ui.test.ts": "d029e959cfd9347fff4cf17c5686dd125b0bd3a41afb3408a23b31418f12e3b0", "packages/opencode/test/server/pasqal-secret.test.ts": "54be8c666160dad22cd37bbf041f5bd40aec120271818f3c56c94107d987b9fb", @@ -820,7 +836,7 @@ "packages/app/src/pages/new-session/new-session-machine-picker.test.ts": "2f2adf9d738aee98f2989c5453c97eeba0e48522b7863c99d9c0be9a6c11ff3b", "packages/app/src/pages/new-session/new-session-machine-selection.ts": "618df365b74547d997920fa3d14e1f73f3b5d618e54f2b1297b8374535900f7a", "packages/app/src/pages/new-session/new-session-view.test.ts": "eb337485bf03facbc59d9eadac37cfeb04f75bf97814df80d3be4d0ec9455047", - "packages/app/src/pages/new-session/new-session-view.tsx": "8fa065e3d792f99de6fdb6f0de8c6205fab519159342655fe59fc09ea67691b1", + "packages/app/src/pages/new-session/new-session-view.tsx": "9390bb0c6d24d6ad1cbc753b6a57d85488a7355264e37641991a931769a3f482", "packages/app/src/pages/session/amicode-session-relay.integration.test.ts": "4991e775d3a6c63a514072ccfa5aad190c586383b77ef41689c853fb271d3bf4", "packages/app/src/pages/session/fleet-manager-tab.test.tsx": "f3bb956e343ccd82acd8bf38c9ad2413d48b24ec9f484c5dfd3939d9e07139b7", "packages/app/src/pages/session/fleet-manager-tab.tsx": "1e7db225a1f2fa3234e0eb47f314de49185db1f3ebddb5585fd0247184adebaa", @@ -951,8 +967,8 @@ "packages/app/src/pages/session/composer/session-bug-dock.stories.tsx": "a5730bb3699751d63629134924ce8dea021b264722c40f5f6bb98236dc15eda7", "packages/app/src/pages/session/composer/session-bug-dock.tsx": "e7fed3c5ab80527d69a106440833f795bbbafc5deeba3883902ede9fba612605", "packages/app/src/pages/session/composer/session-composer-controls.ts": "7ba9d56c81b1547acbdd47c4f9371ddc37c9a92059f5785f7286fc68e0cb4483", - "packages/app/src/pages/session/composer/session-composer-region-controller.ts": "d0403567856eb07ee93957e6d8a23c0f5c78620d94fa09407c01fc6ad3d9eadc", - "packages/app/src/pages/session/composer/session-composer-region.tsx": "415cd5c3a77b625a3142b828b6458687e1a35b382d3a208d90ff1555cc7098b1", + "packages/app/src/pages/session/composer/session-composer-region-controller.ts": "39b05c06b9e9ed42678d6b35dd2e2acd323e0ede8587a5592923c74f38307aca", + "packages/app/src/pages/session/composer/session-composer-region.tsx": "fcede95d91c72254d46481140adb3ff33d5ec69a54760f5a201fe87d0a2b1e6e", "packages/app/src/pages/session/composer/session-composer-state.ts": "fcffa11d801309cf58f42a174143bf6eb3ea2b14f860ad91a8ffa4e097df5c16", "packages/app/src/pages/session/composer/session-question-dock.helpers.test.ts": "a1fe31fe4dfdf2914bc184030a0427bcbdfae0f3e7b146e1d6ee29ce31d85b64", "packages/app/src/pages/session/composer/session-question-dock.helpers.ts": "b16282d0cf5d39152bd76f4ef8bac32f964f68143a3067765ae0827a5f90deed", @@ -1005,14 +1021,14 @@ "packages/opencode/src/server/routes/instance/httpapi/groups/experimental.ts": "944a0aee215a917e867fd9482f3c2bd40f49082bd3174c8a7d967ca40e34ec15", "packages/opencode/src/server/routes/instance/httpapi/groups/file.ts": "a96fdc55ddb95ca7c481c86f67e37ed88d4cf2a190a482765ab55ce4409f3a59", "packages/opencode/src/server/routes/instance/httpapi/groups/mcp.ts": "048efa7b74d43c4ad4a75d0f8421c808760c8c0dcd593f0051a87044e40487d6", - "packages/opencode/src/server/routes/instance/httpapi/groups/provider.ts": "34c6f1ab1a085d1b047c220bbff32a36032dc5e924784e9272cda166cf0ec84b", + "packages/opencode/src/server/routes/instance/httpapi/groups/provider.ts": "2d090445392a2c52efb26335297f4d44c70aa511b0524e4cd3a69880ffdc6078", "packages/opencode/src/server/routes/instance/httpapi/groups/session.ts": "1421f8c07948b121c7e2005bd9b0fcd6b2c456565ddd1da32d8e6c35d03fac9c", "packages/opencode/src/server/routes/instance/httpapi/handlers/event.ts": "88c7bae32fa01ab213e95cb4e4891fd3ccb961782e80a416e321c5daf90fb487", "packages/opencode/src/server/routes/instance/httpapi/handlers/experimental.ts": "69ca3dd48d95157fc822e1f76eb42ac873df50f22616b85194619ca658613867", "packages/opencode/src/server/routes/instance/httpapi/handlers/file.ts": "10bcb79b117beebee04184fb48c1ad789e782e2f7ae4b0c68603edc00f03aee9", "packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts": "040554475b6ba22ad666dfeaca2576658854c46cd18efe3b2d68959f9eaaed80", "packages/opencode/src/server/routes/instance/httpapi/handlers/mcp.ts": "7da08d58765cdf2b6ce4648fc70de371402949f5bb7b75696f674d43f2d5365b", - "packages/opencode/src/server/routes/instance/httpapi/handlers/provider.ts": "8baf16cad0992c5351b61b0c0fb27903bb25b17e1b25fc7c5d77f807c24413a6", + "packages/opencode/src/server/routes/instance/httpapi/handlers/provider.ts": "e899320a028e639a845d6c6aef547d448341671a195f5465ee4e8fa847957c52", "packages/opencode/src/server/routes/instance/httpapi/handlers/session.ts": "b37ff4136c4aea85b23083bf441f03e09820e3e4cdd95e4118ca570d8435b94f", "packages/opencode/src/server/routes/instance/httpapi/middleware/file-write-workspace.ts": "665c33633173ee89c6da1d4a0eaacaf4eebe3ccdd04eb36b063515cb4c093e4a", "packages/opencode/src/server/routes/instance/httpapi/middleware/gateway-secret.ts": "ec6133ee5d331920ded81042fbeb05221bec0c4b26f5897b3f28bc267d26aefa", @@ -1031,11 +1047,11 @@ "packages/ui/src/v2/components/text-shimmer-v2.tsx" ], "exceptions": [], - "extracted_at": "2026-10-26T18:34:42.306Z", + "extracted_at": "2026-10-28T19:40:57.940Z", "per_package": { "packages/app": { - "A": 257, - "M": 164, + "A": 270, + "M": 166, "D": 0 }, "packages/core": { @@ -1049,7 +1065,7 @@ "D": 0 }, "packages/opencode": { - "A": 101, + "A": 102, "M": 100, "D": 0 }, @@ -1080,7 +1096,7 @@ } }, "counts": { - "overlay_total": 1009, + "overlay_total": 1025, "deletions": 7, "server_coupled": 0 }, @@ -1198,6 +1214,13 @@ "packages/app/src/components/dialog-release-notes.tsx": "M", "packages/app/src/components/dialog-select-directory-v2.css": "M", "packages/app/src/components/dialog-select-file.tsx": "M", + "packages/app/src/components/dialog-select-model-reachability-source.ts": "A", + "packages/app/src/components/dialog-select-model-reachability.test.ts": "A", + "packages/app/src/components/dialog-select-model-reachability.ts": "A", + "packages/app/src/components/dialog-select-model-shortlist-source.ts": "A", + "packages/app/src/components/dialog-select-model-shortlist.test.ts": "A", + "packages/app/src/components/dialog-select-model-shortlist.ts": "A", + "packages/app/src/components/dialog-select-model.tsx": "M", "packages/app/src/components/engine-state-signal.ts": "A", "packages/app/src/components/engine-toggle-utils.test.ts": "A", "packages/app/src/components/engine-toggle-utils.ts": "A", @@ -1206,6 +1229,12 @@ "packages/app/src/components/harness-switch.test.ts": "A", "packages/app/src/components/harness-switch.ts": "A", "packages/app/src/components/help-button.tsx": "M", + "packages/app/src/components/model-dead-pick-warning.tsx": "A", + "packages/app/src/components/model-dead-pick.test.ts": "A", + "packages/app/src/components/model-dead-pick.ts": "A", + "packages/app/src/components/model-row-meta.test.ts": "A", + "packages/app/src/components/model-row-meta.ts": "A", + "packages/app/src/components/model-tooltip.tsx": "M", "packages/app/src/components/new-session-machine-picker.ts": "A", "packages/app/src/components/pane-bridge.tsx": "A", "packages/app/src/components/profile-popover.tsx": "A", @@ -1271,6 +1300,8 @@ "packages/app/src/context/local-agent.test.ts": "M", "packages/app/src/context/local-agent.ts": "M", "packages/app/src/context/local.tsx": "M", + "packages/app/src/context/models-favorites.test.ts": "A", + "packages/app/src/context/models-favorites.ts": "A", "packages/app/src/context/models.tsx": "M", "packages/app/src/context/platform.tsx": "M", "packages/app/src/context/question-inbox.test.ts": "A", @@ -1435,6 +1466,7 @@ "packages/opencode/src/project/instance-store.ts": "M", "packages/opencode/src/provider/auth.ts": "M", "packages/opencode/src/provider/error.ts": "M", + "packages/opencode/src/provider/probe.ts": "A", "packages/opencode/src/provider/provider.ts": "M", "packages/opencode/src/provider/transform.ts": "M", "packages/opencode/src/question/index.ts": "M", diff --git a/packages/app-bundle/overlay/packages/app/src/components/dialog-select-model-reachability-source.ts b/packages/app-bundle/overlay/packages/app/src/components/dialog-select-model-reachability-source.ts new file mode 100644 index 000000000..f29acbb02 --- /dev/null +++ b/packages/app-bundle/overlay/packages/app/src/components/dialog-select-model-reachability-source.ts @@ -0,0 +1,56 @@ +// amicode#1986 (#1979 S3): the picker's reachability data source — fetches the +// S2 snapshot through the overlay's amicode-fetch (per-active-server Basic +// auth) and exposes a Re-check. Every failure (non-amicode host, 404, network, +// ok:false) resolves to `undefined` ⇒ the pure merge passes the list through +// untouched, so the picker degrades to exactly upstream behaviour. +import { createResource, createSignal } from "solid-js" +import { ServerConnection, useServer } from "@/context/server" +import { amicodeGet, amicodePost } from "@/utils/amicode-fetch" +import { + parseReachabilityResponse, + REACHABILITY_REFRESH_ROUTE, + REACHABILITY_ROUTE, + type ReachabilitySnapshot, +} from "./dialog-select-model-reachability" + +export function createModelReachability() { + const server = useServer() + const [snapshot, { refetch }] = createResource( + () => (server.current ? ServerConnection.key(server.current) : undefined), + async (): Promise => { + try { + return parseReachabilityResponse(await amicodeGet(server.current, REACHABILITY_ROUTE)) + } catch { + return undefined + } + }, + ) + const [showUnavailable, setShowUnavailable] = createSignal(false) + const [rechecking, setRechecking] = createSignal(false) + + return { + /** undefined ⇒ passthrough (no data / route unavailable). */ + snapshot: (): ReachabilitySnapshot | undefined => (snapshot.error ? undefined : snapshot.latest), + /** amicode#1989: true until the first fetch settles (a demote waits for it + * so it never overrides a fresher probe it has not seen yet). */ + loading: () => snapshot.loading, + showUnavailable, + setShowUnavailable, + rechecking, + refetch: () => void refetch(), + recheck: async () => { + if (rechecking()) return + setRechecking(true) + try { + await amicodePost(server.current, REACHABILITY_REFRESH_ROUTE, {}) + } catch { + // a rate-limited / failed refresh still re-reads whatever the cache holds + } finally { + setRechecking(false) + void refetch() + } + }, + } +} + +export type ModelReachabilityState = ReturnType diff --git a/packages/app-bundle/overlay/packages/app/src/components/dialog-select-model-reachability.test.ts b/packages/app-bundle/overlay/packages/app/src/components/dialog-select-model-reachability.test.ts new file mode 100644 index 000000000..5b49be886 --- /dev/null +++ b/packages/app-bundle/overlay/packages/app/src/components/dialog-select-model-reachability.test.ts @@ -0,0 +1,166 @@ +// amicode#1986 (#1979 S3): the picker's reachability merge — pure, so the +// visible-set rules are pinned without mounting the SolidJS picker. +import { describe, expect, test } from "bun:test" +import { + parseReachabilityResponse, + partitionByReachability, + reachabilityReason, + type ReachabilitySnapshot, +} from "./dialog-select-model-reachability" + +type Item = { id: string; provider: { id: string } } +const item = (provider: string, id: string): Item => ({ id, provider: { id: provider } }) +const keyOf = (x: Item) => ({ providerID: x.provider.id, modelID: x.id }) +const ids = (xs: Item[]) => xs.map((x) => `${x.provider.id}:${x.id}`) + +const row = (provider: string, model: string, status: string, reason: string | null = null) => ({ + provider, + model, + status, + reason, + latency_ms: status === "unchecked" ? null : 12, + checked_at: status === "unchecked" ? null : 1_700_000_000_000, + credential_fp: "fp1", + source: status === "unchecked" ? null : "probe", +}) +const prov = (provider: string, probed: boolean) => ({ + provider, + credential_fp: probed ? "fp1" : null, + last_probe_at: probed ? 1_700_000_000_000 : null, + last_attempt_at: probed ? 1_700_000_000_000 : null, + last_error: null, + next_probe_at: null, +}) +const response = (models: unknown[], providers: unknown[]) => ({ + ok: true, + error: null, + catalog_known: true, + models, + providers, +}) + +const items = [ + item("bedrock", "claude-ok"), + item("bedrock", "claude-denied"), + item("bedrock", "claude-unavailable"), + item("bedrock", "claude-unchecked"), + item("bedrock", "claude-invalid"), + item("bedrock", "claude-error"), + item("bedrock", "claude-ratelimited"), + item("bedrock", "claude-norow"), +] +const mixed = parseReachabilityResponse( + response( + [ + row("bedrock", "claude-ok", "ok"), + row("bedrock", "claude-denied", "denied", "AccessDeniedException"), + row("bedrock", "claude-unavailable", "unavailable"), + row("bedrock", "claude-unchecked", "unchecked"), + row("bedrock", "claude-invalid", "invalid"), + row("bedrock", "claude-error", "error", "socket hang up"), + row("bedrock", "claude-ratelimited", "rate-limited"), + ], + [prov("bedrock", true)], + ), +) as ReachabilitySnapshot + +describe("parseReachabilityResponse", () => { + test("a well-formed S2 body parses to a snapshot", () => { + expect(mixed).toBeDefined() + expect(mixed.models.length).toBe(7) + }) + + test("route unavailable / malformed / ok:false → undefined (passthrough)", () => { + expect(parseReachabilityResponse(undefined)).toBeUndefined() + expect(parseReachabilityResponse("nope")).toBeUndefined() + expect(parseReachabilityResponse({ ok: false, error: "cache_unavailable: x" })).toBeUndefined() + expect(parseReachabilityResponse({ ok: true, models: "x" })).toBeUndefined() + }) + + test("rows with an unknown status are dropped, not trusted", () => { + const snap = parseReachabilityResponse(response([row("a", "m", "ok"), row("a", "n", "great")], [])) + expect(snap?.models.map((m) => m.model)).toEqual(["m"]) + }) +}) + +describe("partitionByReachability", () => { + test("ok + rate-limited show by default; dead and probed-provider unchecked hide", () => { + const p = partitionByReachability(items, keyOf, mixed) + expect(ids(p.shown)).toEqual(["bedrock:claude-ok", "bedrock:claude-ratelimited"]) + expect(ids(p.hidden)).toEqual([ + "bedrock:claude-denied", + "bedrock:claude-unavailable", + "bedrock:claude-unchecked", + "bedrock:claude-invalid", + "bedrock:claude-error", + "bedrock:claude-norow", + ]) + expect(p.hiddenCount).toBe(6) + }) + + test("the toggle reveals hidden models greyed, with human reasons", () => { + const p = partitionByReachability(items, keyOf, mixed) + expect(ids(p.list(false))).toEqual(ids(p.shown)) + expect(ids(p.list(true))).toEqual([...ids(p.shown), ...ids(p.hidden)]) + const reasons = Object.fromEntries(p.hidden.map((x) => [x.id, reachabilityReason(p.view(x))])) + expect(reasons["claude-denied"]).toEqual({ key: "dialog.model.reachability.reason.denied" }) + expect(reasons["claude-unavailable"]).toEqual({ key: "dialog.model.reachability.reason.unavailable" }) + expect(reasons["claude-invalid"]).toEqual({ key: "dialog.model.reachability.reason.invalid" }) + expect(reasons["claude-error"]).toEqual({ text: "socket hang up" }) + expect(reasons["claude-unchecked"]).toEqual({ key: "dialog.model.reachability.reason.unchecked" }) + expect(reasons["claude-norow"]).toEqual({ key: "dialog.model.reachability.reason.unchecked" }) + for (const x of p.hidden) expect(p.view(x).greyed).toBe(true) + }) + + test("error without a reason falls back to 'Probe failed'", () => { + const snap = parseReachabilityResponse(response([row("a", "m", "error")], [prov("a", true)])) + const p = partitionByReachability([item("a", "m")], keyOf, snap) + expect(reachabilityReason(p.view(p.hidden[0]))).toEqual({ key: "dialog.model.reachability.reason.error" }) + }) + + test("rate-limited is shown with a warning; ok has none", () => { + const p = partitionByReachability(items, keyOf, mixed) + expect(p.view(item("bedrock", "claude-ratelimited")).tag).toBe("rate-limited") + expect(p.view(item("bedrock", "claude-ratelimited")).greyed).toBe(false) + expect(p.view(item("bedrock", "claude-ok")).tag).toBeUndefined() + }) + + test("unchecked models of a never-probed provider show, labelled unchecked", () => { + const snap = parseReachabilityResponse( + response( + [row("bedrock", "claude-ok", "ok"), row("openai", "gpt-a", "unchecked")], + [prov("bedrock", true), prov("openai", false)], + ), + ) + const list = [item("bedrock", "claude-ok"), item("openai", "gpt-a"), item("openai", "gpt-b")] + const p = partitionByReachability(list, keyOf, snap) + expect(ids(p.shown)).toEqual(["bedrock:claude-ok", "openai:gpt-a", "openai:gpt-b"]) + expect(p.hiddenCount).toBe(0) + expect(p.view(item("openai", "gpt-a")).tag).toBe("unchecked") + expect(p.view(item("openai", "gpt-b")).tag).toBe("unchecked") + expect(p.view(item("openai", "gpt-a")).status).toBe("unchecked") + }) + + test("the currently selected model is retained even if dead, and marked", () => { + const p = partitionByReachability(items, keyOf, mixed, { providerID: "bedrock", modelID: "claude-denied" }) + expect(ids(p.shown)).toEqual(["bedrock:claude-ok", "bedrock:claude-denied", "bedrock:claude-ratelimited"]) + expect(p.hiddenCount).toBe(5) + const v = p.view(item("bedrock", "claude-denied")) + expect(v.tag).toBe("selected-dead") + expect(v.greyed).toBe(false) + expect(reachabilityReason(v)).toEqual({ key: "dialog.model.reachability.reason.denied" }) + }) + + test("route unavailable → passthrough: everything shows exactly as today", () => { + const p = partitionByReachability(items, keyOf, undefined, { providerID: "bedrock", modelID: "claude-denied" }) + expect(ids(p.shown)).toEqual(ids(items)) + expect(p.hiddenCount).toBe(0) + expect(ids(p.list(true))).toEqual(ids(items)) + for (const x of items) { + const v = p.view(x) + expect(v.tag).toBeUndefined() + expect(v.greyed).toBe(false) + expect(reachabilityReason(v)).toBeUndefined() + } + }) +}) diff --git a/packages/app-bundle/overlay/packages/app/src/components/dialog-select-model-reachability.ts b/packages/app-bundle/overlay/packages/app/src/components/dialog-select-model-reachability.ts new file mode 100644 index 000000000..a129c3b28 --- /dev/null +++ b/packages/app-bundle/overlay/packages/app/src/components/dialog-select-model-reachability.ts @@ -0,0 +1,216 @@ +// amicode#1986 (#1979 S3): the model picker's reachability merge. PURE — no +// SolidJS, no fetch — so the visible-set rules are unit-tested and S4/S5 can +// reuse them. The fetch lives in dialog-select-model-reachability-source.ts. +// +// Data: GET /amicode/model-reachability (S2, #1985). Rules (the issue's): +// - ok → shown. rate-limited → shown with a warning tag. +// - denied / unavailable / invalid / error → hidden behind the toggle. +// - unchecked (or no row at all) → shown labelled "unchecked" ONLY when the +// provider has no probe data at all; otherwise hidden behind the toggle. +// Unchecked is never presented as working. +// - the currently selected model is always shown, even if dead (tagged). +// - no snapshot (route unavailable / non-amicode host / ok:false) → +// passthrough: the picker behaves exactly as upstream. + +export type ReachabilityStatus = "ok" | "denied" | "unavailable" | "invalid" | "rate-limited" | "error" | "unchecked" + +const STATUSES: readonly ReachabilityStatus[] = [ + "ok", + "denied", + "unavailable", + "invalid", + "rate-limited", + "error", + "unchecked", +] + +export type ReachabilityModel = { + provider: string + model: string + status: ReachabilityStatus + reason: string | null + latency_ms: number | null + checked_at: number | null + credential_fp: string | null + source: "probe" | "demoted" | null +} + +export type ReachabilityProvider = { + provider: string + credential_fp: string | null + last_probe_at: number | null + last_attempt_at: number | null + last_error: string | null + next_probe_at: number | null +} + +export type ReachabilitySnapshot = { + catalog_known: boolean + models: ReachabilityModel[] + providers: ReachabilityProvider[] +} + +export type ModelRef = { providerID: string; modelID: string } + +/** What the picker renders for one model. `tag` drives the inline label: + * "rate-limited" (warning), "unchecked" (never presented as working), + * "selected-dead" (the current model, kept although dead), or undefined. */ +export type ModelReachabilityView = { + status: ReachabilityStatus | undefined + reason: string | null + hidden: boolean + greyed: boolean + tag: "rate-limited" | "unchecked" | "selected-dead" | undefined +} + +export type ReachabilityReasonKey = + | "dialog.model.reachability.reason.denied" + | "dialog.model.reachability.reason.unavailable" + | "dialog.model.reachability.reason.invalid" + | "dialog.model.reachability.reason.rateLimited" + | "dialog.model.reachability.reason.unchecked" + | "dialog.model.reachability.reason.error" + +/** A human reason: an i18n key, or the probe's own (already redacted) text. */ +export type ReachabilityReason = { key: ReachabilityReasonKey } | { text: string } + +export const REACHABILITY_ROUTE = "/amicode/model-reachability" +export const REACHABILITY_REFRESH_ROUTE = "/amicode/model-reachability/refresh" + +const DEAD: ReadonlySet = new Set(["denied", "unavailable", "invalid", "error"]) + +const isObj = (v: unknown): v is Record => typeof v === "object" && v !== null && !Array.isArray(v) +const str = (v: unknown) => (typeof v === "string" ? v : null) +const num = (v: unknown) => (typeof v === "number" && Number.isFinite(v) ? v : null) + +/** Parse the S2 GET body. undefined ⇒ passthrough (no usable data). */ +export function parseReachabilityResponse(raw: unknown): ReachabilitySnapshot | undefined { + if (!isObj(raw) || raw.ok !== true || !Array.isArray(raw.models)) return undefined + const models: ReachabilityModel[] = [] + for (const m of raw.models) { + if (!isObj(m) || typeof m.provider !== "string" || typeof m.model !== "string") continue + if (!STATUSES.includes(m.status as ReachabilityStatus)) continue + models.push({ + provider: m.provider, + model: m.model, + status: m.status as ReachabilityStatus, + reason: str(m.reason), + latency_ms: num(m.latency_ms), + checked_at: num(m.checked_at), + credential_fp: str(m.credential_fp), + source: m.source === "probe" || m.source === "demoted" ? m.source : null, + }) + } + const providers: ReachabilityProvider[] = [] + for (const p of Array.isArray(raw.providers) ? raw.providers : []) { + if (!isObj(p) || typeof p.provider !== "string") continue + providers.push({ + provider: p.provider, + credential_fp: str(p.credential_fp), + last_probe_at: num(p.last_probe_at), + last_attempt_at: num(p.last_attempt_at), + last_error: str(p.last_error), + next_probe_at: num(p.next_probe_at), + }) + } + return { catalog_known: raw.catalog_known === true, models, providers } +} + +const refKey = (ref: ModelRef) => `${ref.providerID}\u0000${ref.modelID}` + +const PASSTHROUGH: ModelReachabilityView = { + status: undefined, + reason: null, + hidden: false, + greyed: false, + tag: undefined, +} + +/** Merge reachability into the picker's (already visibility-filtered) list. + * `shown` / `hidden` preserve input order; `list(showUnavailable)` is what to + * render (shown, then hidden when the toggle is on); `view(item)` annotates. */ +export function partitionByReachability( + items: readonly T[], + keyOf: (item: T) => ModelRef, + snapshot: ReachabilitySnapshot | undefined, + current?: ModelRef, +) { + if (!snapshot) { + const all = [...items] + return { + shown: all, + hidden: [] as T[], + hiddenCount: 0, + list: (_showUnavailable: boolean) => all, + view: (_item: T): ModelReachabilityView => PASSTHROUGH, + } + } + + const rows = new Map() + const probed = new Set() + for (const m of snapshot.models) { + rows.set(refKey({ providerID: m.provider, modelID: m.model }), m) + if (m.status !== "unchecked") probed.add(m.provider) + } + for (const p of snapshot.providers) if (p.last_probe_at !== null) probed.add(p.provider) + const currentKey = current ? refKey(current) : undefined + + const views = new Map() + const viewOf = (ref: ModelRef): ModelReachabilityView => { + const k = refKey(ref) + const cached = views.get(k) + if (cached) return cached + const row = rows.get(k) + const status: ReachabilityStatus = row?.status ?? "unchecked" + const reason = row?.reason ?? null + const dead = DEAD.has(status) || (status === "unchecked" && probed.has(ref.providerID)) + const v: ModelReachabilityView = !dead + ? { + status, + reason, + hidden: false, + greyed: false, + tag: status === "ok" ? undefined : (status as "rate-limited" | "unchecked"), + } + : k === currentKey + ? { status, reason, hidden: false, greyed: false, tag: "selected-dead" } + : { status, reason, hidden: true, greyed: true, tag: undefined } + views.set(k, v) + return v + } + + const shown: T[] = [] + const hidden: T[] = [] + for (const item of items) (viewOf(keyOf(item)).hidden ? hidden : shown).push(item) + const revealed = [...shown, ...hidden] + return { + shown, + hidden, + hiddenCount: hidden.length, + list: (showUnavailable: boolean) => (showUnavailable ? revealed : shown), + view: (item: T) => viewOf(keyOf(item)), + } +} + +export type ReachabilityPartition = ReturnType> + +/** The human reason for a view (undefined for ok / passthrough). */ +export function reachabilityReason(view: ModelReachabilityView): ReachabilityReason | undefined { + switch (view.status) { + case undefined: + case "ok": + return undefined + case "denied": + return { key: "dialog.model.reachability.reason.denied" } + case "unavailable": + return { key: "dialog.model.reachability.reason.unavailable" } + case "invalid": + return { key: "dialog.model.reachability.reason.invalid" } + case "rate-limited": + return { key: "dialog.model.reachability.reason.rateLimited" } + case "unchecked": + return { key: "dialog.model.reachability.reason.unchecked" } + case "error": + return view.reason ? { text: view.reason } : { key: "dialog.model.reachability.reason.error" } + } +} diff --git a/packages/app-bundle/overlay/packages/app/src/components/dialog-select-model-shortlist-source.ts b/packages/app-bundle/overlay/packages/app/src/components/dialog-select-model-shortlist-source.ts new file mode 100644 index 000000000..d9d9c6ca4 --- /dev/null +++ b/packages/app-bundle/overlay/packages/app/src/components/dialog-select-model-shortlist-source.ts @@ -0,0 +1,24 @@ +// amicode#1987 (#1979 S4): the picker's deployment config source — fetches +// GET /amicode/model-picker (model.recommended / model.allowlist) through the +// overlay's amicode-fetch. Every failure (non-amicode host, 404, network, +// ok:false) resolves to `undefined` ⇒ no recommended, no allowlist: the short +// list is favorites + recents only and nothing is filtered. +import { createResource } from "solid-js" +import { ServerConnection, useServer } from "@/context/server" +import { amicodeGet } from "@/utils/amicode-fetch" +import { parseShortlistConfig, SHORTLIST_CONFIG_ROUTE, type ShortlistConfig } from "./dialog-select-model-shortlist" + +export function createModelShortlistConfig() { + const server = useServer() + const [config] = createResource( + () => (server.current ? ServerConnection.key(server.current) : undefined), + async (): Promise => { + try { + return parseShortlistConfig(await amicodeGet(server.current, SHORTLIST_CONFIG_ROUTE)) + } catch { + return undefined + } + }, + ) + return (): ShortlistConfig | undefined => (config.error ? undefined : config.latest) +} diff --git a/packages/app-bundle/overlay/packages/app/src/components/dialog-select-model-shortlist.test.ts b/packages/app-bundle/overlay/packages/app/src/components/dialog-select-model-shortlist.test.ts new file mode 100644 index 000000000..0e07e6e69 --- /dev/null +++ b/packages/app-bundle/overlay/packages/app/src/components/dialog-select-model-shortlist.test.ts @@ -0,0 +1,317 @@ +// amicode#1987 (#1979 S4): the picker's short-list view — pure, so ordering, +// dedupe, the cap, the allowlist hard-filter and search reach are pinned +// without mounting the SolidJS picker. +import { describe, expect, test } from "bun:test" +import { + SHORTLIST_CAP, + buildModelShortlist, + isAllowed, + parseShortlistConfig, + type ShortlistConfig, +} from "./dialog-select-model-shortlist" +import { compareCheapestOk } from "./model-row-meta" + +type Item = { id: string; name: string; provider: { id: string; name: string } } +const item = (provider: string, id: string): Item => ({ id, name: id, provider: { id: provider, name: provider } }) +const keyOf = (x: Item) => ({ providerID: x.provider.id, modelID: x.id }) +const ref = (providerID: string, modelID: string) => ({ providerID, modelID }) +const ids = (xs: readonly Item[]) => xs.map((x) => `${x.provider.id}/${x.id}`) +const searchText = (x: Item) => [x.name, x.id, x.provider.name] + +const base = { + keyOf, + searchText, + hidden: [] as Item[], + showUnavailable: false, + favorites: [] as ReturnType[], + recents: [] as ReturnType[], + config: undefined as ShortlistConfig | undefined, + query: "", +} + +const many = (n: number, provider = "p") => Array.from({ length: n }, (_, i) => item(provider, `m${i}`)) + +describe("parseShortlistConfig", () => { + test("a well-formed body parses recommended + allowlist", () => { + const cfg = parseShortlistConfig({ + ok: true, + model: { + recommended: [ + { model: "anthropic/claude-sonnet", role: "default" }, + { model: "openrouter/meta/llama-3", role: "open-weight" }, + ], + allowlist: ["anthropic/*", "openrouter/meta/llama-3"], + }, + }) + expect(cfg).toEqual({ + recommended: [ + { ref: ref("anthropic", "claude-sonnet"), role: "default" }, + // the model id keeps everything after the FIRST slash + { ref: ref("openrouter", "meta/llama-3"), role: "open-weight" }, + ], + allowlist: ["anthropic/*", "openrouter/meta/llama-3"], + }) + }) + + test("route unavailable / ok:false / malformed → undefined (no config)", () => { + expect(parseShortlistConfig(undefined)).toBeUndefined() + expect(parseShortlistConfig("x")).toBeUndefined() + expect(parseShortlistConfig({ ok: false, error: "x" })).toBeUndefined() + }) + + test("bad entries are dropped, never guessed: unknown role, no slash, non-string", () => { + const cfg = parseShortlistConfig({ + ok: true, + model: { + recommended: [ + { model: "a/x", role: "turbo" }, + { model: "noslash", role: "fast" }, + { model: 3, role: "fast" }, + { model: "a/y" }, + { model: "a/z", role: "strong" }, + ], + allowlist: ["a/*", 7, "", "noslash"], + }, + }) + expect(cfg?.recommended).toEqual([{ ref: ref("a", "z"), role: "strong" }]) + expect(cfg?.allowlist).toEqual(["a/*"]) + }) + + test("no allowlist key → undefined allowlist (no filter); no recommended → none invented", () => { + const cfg = parseShortlistConfig({ ok: true, model: {} }) + expect(cfg).toEqual({ recommended: [], allowlist: undefined }) + }) +}) + +describe("isAllowed", () => { + test("no allowlist (or an empty one) allows everything", () => { + expect(isAllowed(ref("a", "x"), undefined)).toBe(true) + expect(isAllowed(ref("a", "x"), [])).toBe(true) + }) + test("exact provider/id and provider/* wildcard", () => { + const list = ["a/*", "b/keep", "c/ns/deep"] + expect(isAllowed(ref("a", "anything"), list)).toBe(true) + expect(isAllowed(ref("b", "keep"), list)).toBe(true) + expect(isAllowed(ref("b", "drop"), list)).toBe(false) + expect(isAllowed(ref("c", "ns/deep"), list)).toBe(true) + expect(isAllowed(ref("d", "x"), list)).toBe(false) + }) +}) + +describe("buildModelShortlist — default view", () => { + const shown = [item("a", "fav1"), item("a", "rec-fast"), item("b", "rec-default"), item("b", "recent1"), item("c", "rest")] + const config: ShortlistConfig = { + recommended: [ + { ref: ref("a", "rec-fast"), role: "fast" }, + { ref: ref("b", "rec-default"), role: "default" }, + ], + allowlist: undefined, + } + + test("Favorites → Recommended (by role: default, fast, strong, open-weight) → Recent", () => { + const view = buildModelShortlist({ + ...base, + shown, + config, + favorites: [ref("a", "fav1")], + recents: [ref("b", "recent1")], + }) + expect(view.mode).toBe("shortlist") + expect(view.groups.map((g) => g.kind + (g.role ? `:${g.role}` : ""))).toEqual([ + "favorites", + "recommended:default", + "recommended:fast", + "recent", + ]) + expect(ids(view.items)).toEqual(["a/fav1", "b/rec-default", "a/rec-fast", "b/recent1"]) + // everything else is behind search + expect(view.rest).toBe(1) + }) + + test("favorites keep their stored order", () => { + const view = buildModelShortlist({ ...base, shown, favorites: [ref("b", "recent1"), ref("a", "fav1")] }) + expect(ids(view.items)).toEqual(["b/recent1", "a/fav1"]) + }) + + test("dedupe: a model appears once, at its first (highest) section", () => { + const view = buildModelShortlist({ + ...base, + shown, + config: { + recommended: [...config.recommended, { ref: ref("a", "fav1"), role: "strong" }, { ref: ref("a", "rec-fast"), role: "strong" }], + allowlist: undefined, + }, + favorites: [ref("a", "fav1"), ref("a", "fav1")], + recents: [ref("a", "fav1"), ref("b", "rec-default"), ref("b", "recent1")], + }) + expect(ids(view.items)).toEqual(["a/fav1", "b/rec-default", "a/rec-fast", "b/recent1"]) + expect(view.groups.find((g) => g.role === "strong")).toBeUndefined() + }) + + test("refs to models not in the list (disconnected / hidden by visibility) are skipped", () => { + const view = buildModelShortlist({ + ...base, + shown, + favorites: [ref("zz", "gone")], + recents: [ref("a", "nope"), ref("b", "recent1")], + }) + expect(ids(view.items)).toEqual(["b/recent1"]) + expect(view.groups.map((g) => g.kind)).toEqual(["recent"]) + }) + + test(`capped at ${SHORTLIST_CAP} entries across sections, in section order`, () => { + expect(SHORTLIST_CAP).toBe(12) + const all = many(30) + const view = buildModelShortlist({ + ...base, + shown: all, + favorites: all.slice(0, 8).map(keyOf), + config: { recommended: all.slice(8, 14).map((x) => ({ ref: keyOf(x), role: "default" as const })), allowlist: undefined }, + recents: all.slice(14, 19).map(keyOf), + }) + expect(view.items.length).toBe(12) + expect(ids(view.items)).toEqual(ids(all.slice(0, 12))) + expect(view.groups.map((g) => g.kind)).toEqual(["favorites", "recommended"]) + expect(view.rest).toBe(30 - 12) + }) + + test("no favorites, no recommended, no recents → mode all: every shown model (nothing invented)", () => { + const view = buildModelShortlist({ ...base, shown }) + expect(view.mode).toBe("all") + expect(ids(view.items)).toEqual(ids(shown)) + expect(view.groups.map((g) => g.kind)).toEqual(["other"]) + expect(view.rest).toBe(0) + }) + + test("hidden (unreachable) favorites stay out unless show-unavailable is on", () => { + const dead = item("a", "dead-fav") + const off = buildModelShortlist({ ...base, shown, hidden: [dead], favorites: [keyOf(dead)] }) + expect(ids(off.items)).not.toContain("a/dead-fav") + const on = buildModelShortlist({ ...base, shown, hidden: [dead], favorites: [keyOf(dead)], showUnavailable: true }) + expect(ids(on.items)).toEqual(["a/dead-fav"]) + }) + + test("the allowlist hard-filters favorites, recommended and recents", () => { + const view = buildModelShortlist({ + ...base, + shown, + config: { ...config, allowlist: ["b/*"] }, + favorites: [ref("a", "fav1")], + recents: [ref("b", "recent1")], + }) + expect(ids(view.items)).toEqual(["b/rec-default", "b/recent1"]) + expect(view.rest).toBe(0) + }) +}) + +describe("buildModelShortlist — search", () => { + test("search reaches everything shown, beyond the cap, grouped by role with Other last", () => { + const all = [...many(20), item("q", "rec")] + const config: ShortlistConfig = { recommended: [{ ref: ref("q", "rec"), role: "strong" }], allowlist: undefined } + const view = buildModelShortlist({ + ...base, + shown: all, + favorites: all.slice(0, 12).map(keyOf), + config, + query: "m1", + }) + expect(view.mode).toBe("search") + // m1, m10..m19 — including the ones past the 12 cap + expect(ids(view.items)).toEqual(["p/m1", ...Array.from({ length: 10 }, (_, i) => `p/m1${i}`)]) + expect(view.groups.map((g) => g.kind)).toEqual(["other"]) + + const all2 = buildModelShortlist({ ...base, shown: all, config, query: "rec" }) + expect(all2.groups.map((g) => `${g.kind}:${g.role ?? ""}`)).toEqual(["recommended:strong"]) + }) + + test("search results group by role: default, fast, strong, open-weight, then Other", () => { + const shown = [item("p", "x-other"), item("p", "x-ow"), item("p", "x-strong"), item("p", "x-default")] + const view = buildModelShortlist({ + ...base, + shown, + config: { + recommended: [ + { ref: ref("p", "x-ow"), role: "open-weight" }, + { ref: ref("p", "x-strong"), role: "strong" }, + { ref: ref("p", "x-default"), role: "default" }, + ], + allowlist: undefined, + }, + query: "x", + }) + expect(view.groups.map((g) => g.role ?? g.kind)).toEqual(["default", "strong", "open-weight", "other"]) + expect(ids(view.items)).toEqual(["p/x-default", "p/x-strong", "p/x-ow", "p/x-other"]) + }) + + test("hidden models are searchable only with show-unavailable on", () => { + const shown = [item("p", "alive")] + const hidden = [item("p", "alive-but-dead")] + const off = buildModelShortlist({ ...base, shown, hidden, query: "alive" }) + expect(ids(off.items)).toEqual(["p/alive"]) + const on = buildModelShortlist({ ...base, shown, hidden, query: "alive", showUnavailable: true }) + expect(ids(on.items)).toEqual(["p/alive", "p/alive-but-dead"]) + }) + + test("the allowlist hard-filters search too", () => { + const shown = [item("a", "same"), item("b", "same")] + const view = buildModelShortlist({ ...base, shown, config: { recommended: [], allowlist: ["b/same"] }, query: "same" }) + expect(ids(view.items)).toEqual(["b/same"]) + }) + + test("whitespace-only query is the default view", () => { + const shown = [item("a", "x")] + const view = buildModelShortlist({ ...base, shown, favorites: [ref("a", "x")], query: " " }) + expect(view.mode).toBe("shortlist") + }) +}) + +// amicode#1988 (#1979 S5): cheapest-that-works order — search only. +describe("buildModelShortlist — search sort (#1988)", () => { + const price: Record = { "x-cheap": 1, "x-mid": 4, "x-pricey": 9, "x-dead": 0 } + const dead = new Set(["x-dead"]) + const searchSort = compareCheapestOk({ + tierOf: (x) => (dead.has(x.id) ? 2 : 0), + priceOf: (x) => price[x.id], + nameOf: (x) => x.name, + }) + + test("a non-empty query sorts matches cheapest ok first; dead never above ok", () => { + const shown = [item("p", "x-pricey"), item("p", "x-mid"), item("p", "x-cheap")] + const hidden = [item("p", "x-dead")] + const view = buildModelShortlist({ ...base, shown, hidden, showUnavailable: true, query: "x", searchSort }) + expect(ids(view.items)).toEqual(["p/x-cheap", "p/x-mid", "p/x-pricey", "p/x-dead"]) + }) + + test("the sort applies within each role group; groups keep their order", () => { + const shown = [item("p", "x-pricey"), item("p", "x-mid"), item("p", "x-cheap")] + const view = buildModelShortlist({ + ...base, + shown, + config: { + recommended: [ + { ref: ref("p", "x-pricey"), role: "default" }, + { ref: ref("p", "x-mid"), role: "default" }, + ], + allowlist: undefined, + }, + query: "x", + searchSort, + }) + expect(view.groups.map((g) => g.role ?? g.kind)).toEqual(["default", "other"]) + expect(ids(view.items)).toEqual(["p/x-mid", "p/x-pricey", "p/x-cheap"]) + }) + + test("the default (no-query) short list and the all view keep their order", () => { + const shown = [item("p", "x-pricey"), item("p", "x-mid"), item("p", "x-cheap")] + const short = buildModelShortlist({ + ...base, + shown, + favorites: [ref("p", "x-pricey"), ref("p", "x-cheap"), ref("p", "x-mid")], + searchSort, + }) + expect(ids(short.items)).toEqual(["p/x-pricey", "p/x-cheap", "p/x-mid"]) + const all = buildModelShortlist({ ...base, shown, searchSort }) + expect(all.mode).toBe("all") + expect(ids(all.items)).toEqual(["p/x-pricey", "p/x-mid", "p/x-cheap"]) + }) +}) diff --git a/packages/app-bundle/overlay/packages/app/src/components/dialog-select-model-shortlist.ts b/packages/app-bundle/overlay/packages/app/src/components/dialog-select-model-shortlist.ts new file mode 100644 index 000000000..d7609cce6 --- /dev/null +++ b/packages/app-bundle/overlay/packages/app/src/components/dialog-select-model-shortlist.ts @@ -0,0 +1,167 @@ +// amicode#1987 (#1979 S4): the model picker's short-list view. PURE — no +// SolidJS, no fetch — so ordering, dedupe, the cap, the allowlist hard-filter +// and search reach are unit-tested. The config fetch lives in +// dialog-select-model-shortlist-source.ts; favorites persist in +// context/models.tsx (`user[].favorite`, pure seam context/models-favorites.ts). +// +// Rules (the issue's): +// - Default view (empty query): Favorites → Recommended (grouped by role, in +// ROLE_ORDER) → Recent; deduped (a model shows once, at its first section); +// capped at SHORTLIST_CAP entries. Everything else is behind search. +// - Search (non-empty query): across ALL shown models (and the reachability- +// hidden ones only when "show unavailable" is on), uncapped, grouped by the +// recommended role; a model with no recommended role goes under "Other". +// The role ALWAYS comes from config — no heuristic. +// - `model.allowlist` (["provider/id" | "provider/*"]) hard-filters +// everything, search included. Absent or empty ⇒ no filter. +// - No favorites, no recommended, no recents ⇒ mode "all": every candidate, +// exactly as the uncapped list (nothing is invented to fill the short list). +// - Default `model.recommended` with no config: none. +// - amicode#1988 (S5): an optional `searchSort` orders search matches +// (within each role group) — cheapest-that-works, model-row-meta.ts. +import { matchesModelSearch } from "./dialog-select-model-search" +import type { ModelRef } from "./dialog-select-model-reachability" + +export const SHORTLIST_CAP = 12 + +export const ROLE_ORDER = ["default", "fast", "strong", "open-weight"] as const +export type ModelRole = (typeof ROLE_ORDER)[number] + +export type RecommendedEntry = { ref: ModelRef; role: ModelRole } +export type ShortlistConfig = { + recommended: RecommendedEntry[] + /** undefined ⇒ no allowlist (every model allowed). */ + allowlist: string[] | undefined +} + +export const SHORTLIST_CONFIG_ROUTE = "/amicode/model-picker" + +export type ShortlistGroup = { + kind: "favorites" | "recommended" | "recent" | "other" + /** set for kind "recommended" */ + role?: ModelRole + items: T[] +} + +export type ModelShortlist = { + mode: "shortlist" | "search" | "all" + groups: ShortlistGroup[] + /** the groups' items, flattened in display order (keyboard navigation) */ + items: T[] + /** candidates not in the view (reachable through search); 0 outside "shortlist" */ + rest: number +} + +const isObj = (v: unknown): v is Record => typeof v === "object" && v !== null && !Array.isArray(v) + +/** "provider/id" → ref; the id keeps everything after the FIRST slash. */ +export function parseModelRef(value: string): ModelRef | undefined { + const slash = value.indexOf("/") + if (slash <= 0 || slash === value.length - 1) return undefined + return { providerID: value.slice(0, slash), modelID: value.slice(slash + 1) } +} + +/** Parse the GET /amicode/model-picker body. undefined ⇒ no config. */ +export function parseShortlistConfig(raw: unknown): ShortlistConfig | undefined { + if (!isObj(raw) || raw.ok !== true) return undefined + const model = isObj(raw.model) ? raw.model : {} + const recommended = (Array.isArray(model.recommended) ? model.recommended : []).flatMap((entry) => { + if (!isObj(entry) || typeof entry.model !== "string") return [] + if (!ROLE_ORDER.includes(entry.role as ModelRole)) return [] + const ref = parseModelRef(entry.model) + return ref ? [{ ref, role: entry.role as ModelRole }] : [] + }) + const allowlist = Array.isArray(model.allowlist) + ? model.allowlist.filter((x): x is string => typeof x === "string" && parseModelRef(x) !== undefined) + : undefined + return { recommended, allowlist } +} + +/** The allowlist hard filter. Absent or empty ⇒ everything allowed. */ +export function isAllowed(ref: ModelRef, allowlist: readonly string[] | undefined) { + if (!allowlist || allowlist.length === 0) return true + return allowlist.some((pattern) => { + const p = parseModelRef(pattern) + if (!p || p.providerID !== ref.providerID) return false + return p.modelID === "*" || p.modelID === ref.modelID + }) +} + +const refKey = (ref: ModelRef) => `${ref.providerID}\u0000${ref.modelID}` + +export function buildModelShortlist(input: { + /** reachability-shown models (already visibility-filtered) */ + shown: readonly T[] + /** reachability-hidden models */ + hidden: readonly T[] + showUnavailable: boolean + keyOf: (item: T) => ModelRef + /** the strings a search query matches against (name, id, provider name) */ + searchText: (item: T) => string[] + favorites: readonly ModelRef[] + recents: readonly ModelRef[] + config: ShortlistConfig | undefined + query: string + cap?: number + /** amicode#1988 (S5): search-only order (cheapest-that-works); applied to + * the matches before role grouping, so it orders within each group. The + * default short list and the "all" view never use it. */ + searchSort?: (a: T, b: T) => number +}): ModelShortlist { + const allowlist = input.config?.allowlist + const candidates = [...input.shown, ...(input.showUnavailable ? input.hidden : [])].filter((item) => + isAllowed(input.keyOf(item), allowlist), + ) + const byKey = new Map(candidates.map((item) => [refKey(input.keyOf(item)), item] as const)) + const roleOf = new Map() + for (const entry of input.config?.recommended ?? []) { + const k = refKey(entry.ref) + if (!roleOf.has(k)) roleOf.set(k, entry.role) + } + + const query = input.query.trim() + if (query) { + const filtered = candidates.filter((item) => matchesModelSearch(query, input.searchText(item))) + const matched = input.searchSort ? [...filtered].sort(input.searchSort) : filtered + return byRole("search", matched, (item) => roleOf.get(refKey(input.keyOf(item)))) + } + + const cap = input.cap ?? SHORTLIST_CAP + const taken = new Set() + const groups: ShortlistGroup[] = [] + let count = 0 + const take = (refs: readonly ModelRef[]) => + refs.flatMap((ref) => { + const k = refKey(ref) + const item = byKey.get(k) + if (!item || taken.has(k) || count >= cap) return [] + taken.add(k) + count++ + return [item] + }) + const push = (group: ShortlistGroup) => { + if (group.items.length > 0) groups.push(group) + } + + push({ kind: "favorites", items: take(input.favorites) }) + const recommended = input.config?.recommended ?? [] + for (const role of ROLE_ORDER) { + push({ + kind: "recommended", + role, + items: take(recommended.filter((entry) => entry.role === role).map((entry) => entry.ref)), + }) + } + push({ kind: "recent", items: take(input.recents) }) + + if (groups.length === 0) return byRole("all", candidates, (item) => roleOf.get(refKey(input.keyOf(item)))) + return { mode: "shortlist", groups, items: groups.flatMap((g) => g.items), rest: candidates.length - count } +} + +function byRole(mode: "search" | "all", items: readonly T[], role: (item: T) => ModelRole | undefined): ModelShortlist { + const groups: ShortlistGroup[] = [ + ...ROLE_ORDER.map((r): ShortlistGroup => ({ kind: "recommended", role: r, items: items.filter((x) => role(x) === r) })), + { kind: "other" as const, items: items.filter((x) => role(x) === undefined) }, + ].filter((g) => g.items.length > 0) + return { mode, groups, items: groups.flatMap((g) => g.items), rest: 0 } +} diff --git a/packages/app-bundle/overlay/packages/app/src/components/dialog-select-model.tsx b/packages/app-bundle/overlay/packages/app/src/components/dialog-select-model.tsx new file mode 100644 index 000000000..352554052 --- /dev/null +++ b/packages/app-bundle/overlay/packages/app/src/components/dialog-select-model.tsx @@ -0,0 +1,875 @@ +import { Popover as Kobalte } from "@kobalte/core/popover" +import { Component, ComponentProps, createEffect, createMemo, For, JSX, Show } from "solid-js" +import { createStore } from "solid-js/store" +import { useLocal } from "@/context/local" +import { useDialog } from "@opencode-ai/ui/context/dialog" +import { Button } from "@opencode-ai/ui/button" +import { IconButton } from "@opencode-ai/ui/icon-button" +import { ScrollView } from "@opencode-ai/ui/scroll-view" +import { Tag } from "@opencode-ai/ui/tag" +import { Dialog } from "@opencode-ai/ui/dialog" +import { List } from "@opencode-ai/ui/list" +import { Tooltip } from "@opencode-ai/ui/tooltip" +import { Icon } from "@opencode-ai/ui/v2/icon" +import { Tag as TagV2 } from "@opencode-ai/ui/v2/badge-v2" +import { MenuV2 } from "@opencode-ai/ui/v2/menu-v2" +import { TooltipV2 } from "@opencode-ai/ui/v2/tooltip-v2" +import { ModelTooltip } from "./model-tooltip" +import { useLanguage } from "@/context/language" +import { decode64 } from "@/utils/base64" +import { handleDocumentSearchKeydown } from "@/utils/search-keydown" +import { createMenuDismissController } from "@/utils/menu-dismiss-controller" +import { createEventListener } from "@solid-primitives/event-listener" +// amicode#1986 (#1979 S3): reachability merge — hide dead models, "Show unavailable (N)". +import { createModelReachability, type ModelReachabilityState } from "./dialog-select-model-reachability-source" +import { + partitionByReachability, + reachabilityReason, + type ModelReachabilityView, +} from "./dialog-select-model-reachability" +// amicode#1987 (#1979 S4): short-list view — Favorites → Recommended (by role) → Recent, +// capped; everything else behind search; model.allowlist hard-filters all of it. +import { useModels } from "@/context/models" +import { + buildModelShortlist, + isAllowed, + type ModelShortlist, + type ShortlistGroup, +} from "./dialog-select-model-shortlist" +import { createModelShortlistConfig } from "./dialog-select-model-shortlist-source" +// amicode#1988 (#1979 S5): price / context / tools / measured latency inline, +// full detail in the tooltip; cheapest-that-works order while searching. +import { + compareCheapestOk, + effectivePrice, + formatRowInline, + latencyOf, + modelRowMeta, + searchTier, + type ModelRowMeta, +} from "./model-row-meta" + +const isFree = (provider: string, cost: { input: number } | undefined) => + provider === "opencode" && (!cost || cost.input === 0) + +type ModelState = ReturnType["model"] +type ModelItem = ReturnType[number] + +const modelKey = (model: ModelItem) => `${model.provider.id}:${model.id}` +const manageKey = "action:manage" + +// amicode#1986: reachability helpers shared by both pickers. +const modelRef = (model: ModelItem) => ({ providerID: model.provider.id, modelID: model.id }) +const currentRef = (model: ModelState) => { + const value = model.current() + return value ? modelRef(value) : undefined +} + +function ReachabilityBadge(props: { view: ModelReachabilityView; v2?: boolean }) { + const language = useLanguage() + const reason = () => { + const r = reachabilityReason(props.view) + return r ? ("key" in r ? language.t(r.key) : r.text) : undefined + } + const label = () => { + if (props.view.tag === "unchecked") return language.t("dialog.model.reachability.tag.unchecked") + if (props.view.tag === "rate-limited") return language.t("dialog.model.reachability.tag.rateLimited") + if (props.view.tag === "selected-dead") return language.t("dialog.model.reachability.tag.selectedDead") + return undefined + } + return ( + <> + + {(text) => + props.v2 ? ( + + {text()} + + ) : ( + {text()} + ) + } + + + {(text) => ( + + {text()} + + )} + + + ) +} + +function ReachabilityControls(props: { state: ModelReachabilityState; hiddenCount: number; class?: string }) { + const language = useLanguage() + return ( + +
+ 0} fallback={}> + + + +
+ + ) +} + +// amicode#1987: the shared short-list wiring for both pickers. The allowlist +// filters BEFORE the reachability merge so "Show unavailable (N)" counts only +// allowed models; buildModelShortlist re-applies it (the tested hard filter). +function createModelShortlist(input: { model: ModelState; provider: () => string | undefined }) { + const models = useModels() + const language = useLanguage() + const reachability = createModelReachability() + const config = createModelShortlistConfig() + const byName = (a: ModelItem, b: ModelItem) => a.name.localeCompare(b.name) + const partition = createMemo(() => + partitionByReachability( + input.model + .list() + .filter((m) => input.model.visible({ modelID: m.id, providerID: m.provider.id })) + .filter((m) => (input.provider() ? m.provider.id === input.provider() : true)) + .filter((m) => isAllowed(modelRef(m), config()?.allowlist)), + modelRef, + reachability.snapshot(), + currentRef(input.model), + ), + ) + const sorted = createMemo(() => ({ + shown: [...partition().shown].sort(byName), + hidden: [...partition().hidden].sort(byName), + })) + // amicode#1988: search-only — a dead model never sorts above an ok one. + const searchSort = compareCheapestOk({ + tierOf: (item) => searchTier(partition().view(item)), + priceOf: (item) => effectivePrice(item.cost), + nameOf: (item) => item.name, + }) + const labels = () => ({ + free: language.t("model.meta.free"), + cache: language.t("model.meta.cache"), + perMtok: language.t("model.meta.perMtok"), + }) + return { + reachability, + partition, + view: (search: string): ModelShortlist => + buildModelShortlist({ + shown: sorted().shown, + hidden: sorted().hidden, + showUnavailable: reachability.showUnavailable(), + keyOf: modelRef, + searchText: (item) => [item.name, item.id, item.provider.name], + favorites: models.favorite.list(), + recents: models.recent.list(), + config: config(), + query: search, + searchSort, + }), + meta: (item: ModelItem): ModelRowMeta => + modelRowMeta(item, latencyOf(reachability.snapshot(), modelRef(item)), labels()), + isFavorite: (item: ModelItem) => models.favorite.has(modelRef(item)), + toggleFavorite: (item: ModelItem) => models.favorite.toggle(modelRef(item)), + } +} + +type ShortlistLabelKey = + | "dialog.model.shortlist.favorites" + | "dialog.model.shortlist.recent" + | "dialog.model.shortlist.other" + | "dialog.model.shortlist.role.default" + | "dialog.model.shortlist.role.fast" + | "dialog.model.shortlist.role.strong" + | "dialog.model.shortlist.role.openWeight" + +const shortlistLabelKey = (group: ShortlistGroup): ShortlistLabelKey => { + if (group.kind === "favorites") return "dialog.model.shortlist.favorites" + if (group.kind === "recent") return "dialog.model.shortlist.recent" + if (group.kind === "other") return "dialog.model.shortlist.other" + if (group.role === "fast") return "dialog.model.shortlist.role.fast" + if (group.role === "strong") return "dialog.model.shortlist.role.strong" + if (group.role === "open-weight") return "dialog.model.shortlist.role.openWeight" + return "dialog.model.shortlist.role.default" +} + +/** The group heading; empty when the whole (uncapped) list is one "Other" + * group — a lone "Other" header over everything says nothing. */ +const shortlistGroupLabel = ( + view: ModelShortlist, + group: ShortlistGroup, + t: (key: ShortlistLabelKey) => string, +) => (view.mode === "all" && view.groups.length === 1 ? "" : t(shortlistLabelKey(group))) + +function FavoriteStar(props: { active: boolean; onToggle: () => void }) { + const language = useLanguage() + const stop = (event: Event) => { + event.preventDefault() + event.stopPropagation() + } + const label = () => language.t(props.active ? "dialog.model.favorite.remove" : "dialog.model.favorite.add") + return ( + { + stop(event) + props.onToggle() + }} + > + + + ) +} + +function ShortlistMore(props: { count: number; class?: string }) { + const language = useLanguage() + return ( + 0}> +
+ {language.t("dialog.model.shortlist.more", { count: props.count })} +
+ + ) +} + +const ModelList: Component<{ + provider?: string + class?: string + onSelect: () => void + action?: JSX.Element + model?: ModelState +}> = (props) => { + const model = props.model ?? useLocal().model + const language = useLanguage() + + const shortlist = createModelShortlist({ model, provider: () => props.provider }) + const reachability = shortlist.reachability + const partition = shortlist.partition + // amicode#1987: the short-list view owns filtering, order and grouping — + // List's own fuzzy filter is skipped (skipFilter) and its groups follow the + // view's (group label → view order) for the query List last asked about. + const [current, setCurrent] = createStore({ view: shortlist.view("") }) + const groupOf = createMemo(() => { + const view = current.view + const label = new Map() + const order = new Map() + view.groups.forEach((group, index) => { + const text = shortlistGroupLabel(view, group, (key) => language.t(key)) + order.set(text, index) + for (const item of group.items) label.set(modelKey(item), text) + }) + return { label, order } + }) + + return ( + <> + `${x.provider.id}:${x.id}`} + items={(filter) => { + const view = shortlist.view(filter) + setCurrent("view", view) + return view.items + }} + current={model.current()} + skipFilter={() => true} + groupBy={(x) => groupOf().label.get(modelKey(x)) ?? ""} + sortGroupsBy={(a, b) => (groupOf().order.get(a.category) ?? 0) - (groupOf().order.get(b.category) ?? 0)} + itemWrapper={(item, node) => ( + + } + > + {node} + + )} + onSelect={(x) => { + model.set(x ? { modelID: x.id, providerID: x.provider.id } : undefined, { + recent: true, + }) + props.onSelect() + }} + > + {(i) => ( +
+ {i.name} + {i.provider.name} + + {formatRowInline(shortlist.meta(i), { + omitPrice: isFree(i.provider.id, i.cost), + noTools: language.t("model.meta.noTools"), + })} + + + {language.t("model.tag.free")} + + + {language.t("model.tag.latest")} + + + shortlist.toggleFavorite(i)} /> +
+ )} + + + + + ) +} + +type ModelSelectorTriggerProps = Omit, "as" | "ref"> +type ModelSelectorTrigger = (props: ModelSelectorTriggerProps) => JSX.Element +type Dismiss = "escape" | "outside" | "select" | "manage" | "provider" + +export function ModelSelectorPopover(props: { + provider?: string + model?: ModelState + trigger: ModelSelectorTrigger + onClose?: (cause: "escape" | "select") => void +}) { + const [store, setStore] = createStore<{ + open: boolean + dismiss: Dismiss | null + }>({ + open: false, + dismiss: null, + }) + const dialog = useDialog() + const local = useLocal() + const directory = () => decode64(local.slug()) + + const close = (dismiss: Dismiss) => { + setStore("dismiss", dismiss) + setStore("open", false) + } + + const handleManage = () => { + close("manage") + void import("./dialog-manage-models").then((x) => { + dialog.show(() => ) + }) + } + + const handleConnectProvider = () => { + close("provider") + void import("./dialog-connect-provider").then((x) => { + void dialog.show(() => ) + }) + } + const language = useLanguage() + + return ( + { + if (next) setStore("dismiss", null) + setStore("open", next) + }} + modal={false} + placement="top-start" + gutter={4} + > + + + { + close("escape") + event.preventDefault() + event.stopPropagation() + }} + onPointerDownOutside={() => close("outside")} + onFocusOutside={() => close("outside")} + onCloseAutoFocus={(event) => { + const dismiss = store.dismiss + if (dismiss === "outside") event.preventDefault() + if (dismiss === "escape" || dismiss === "select") { + event.preventDefault() + props.onClose?.(dismiss) + } + setStore("dismiss", null) + }} + > + {language.t("dialog.model.select.title")} + close("select")} + class="p-1" + action={ +
+ + + + + + +
+ } + /> + + + + ) +} + +export function ModelSelectorPopoverV2(props: { + provider?: string + model?: ModelState + trigger: ModelSelectorTrigger + onClose?: () => void +}) { + const dialog = useDialog() + const controller = createModelSelectorController({ + model: props.model, + provider: () => props.provider, + onSelect: () => props.onClose?.(), + }) + + return ( + { + void import("./dialog-manage-models").then((module) => { + void dialog.show(() => ) + }) + }} + onClose={() => props.onClose?.()} + /> + ) +} + +function createModelSelectorController(input: { + provider: () => string | undefined + model?: ModelState + onSelect: () => void +}) { + const model = input.model ?? useLocal().model + const language = useLanguage() + const shortlist = createModelShortlist({ model, provider: input.provider }) + const reachability = shortlist.reachability + const partition = shortlist.partition + + return { + view: (search: string) => { + const view = shortlist.view(search) + return { + items: view.items, + rest: view.rest, + groups: view.groups.map((group) => ({ + category: group.role ?? group.kind, + label: shortlistGroupLabel(view, group, (key) => language.t(key)), + items: group.items, + })), + } + }, + favorite: { + has: shortlist.isFavorite, + toggle: shortlist.toggleFavorite, + }, + meta: shortlist.meta, + current: () => { + const value = model.current() + return value ? modelKey(value) : undefined + }, + select: (item: ModelItem) => { + model.set({ modelID: item.id, providerID: item.provider.id }, { recent: true }) + input.onSelect() + }, + reachability: { + state: reachability, + view: (item: ModelItem) => partition().view(item), + hiddenCount: () => partition().hiddenCount, + }, + } +} + +function ModelSelectorPopoverV2View(props: { + trigger: ModelSelectorTrigger + view: (search: string) => { + items: ModelItem[] + rest: number + groups: { category: string; label: string; items: ModelItem[] }[] + } + favorite?: { has: (item: ModelItem) => boolean; toggle: (item: ModelItem) => void } + meta?: (item: ModelItem) => ModelRowMeta + current: () => string | undefined + select: (item: ModelItem) => void + reachability?: { + state: ModelReachabilityState + view: (item: ModelItem) => ModelReachabilityView + hiddenCount: () => number + } + onManage: () => void + onClose: () => void +}) { + const language = useLanguage() + const [store, setStore] = createStore({ open: false, search: "", active: "" }) + let searchRef: HTMLInputElement | undefined + let contentRef: HTMLDivElement | undefined + const dismiss = createMenuDismissController(() => contentRef) + + const view = createMemo(() => props.view(store.search)) + const models = createMemo(() => view().items) + const groups = createMemo(() => view().groups) + const keys = () => [...models().map(modelKey), manageKey] + const initialActive = () => { + const selected = props.current() + const options = keys() + if (selected && options.includes(selected)) return selected + return options[0] ?? "" + } + const activeItem = () => + store.active ? contentRef?.querySelector(`[data-option-key="${CSS.escape(store.active)}"]`) : undefined + const setOpen = (open: boolean) => { + if (open) { + props.reachability?.state.refetch() + dismiss.allowTriggerRestore() + setStore({ open: true, active: initialActive() }) + setTimeout(() => + requestAnimationFrame(() => { + searchRef?.focus() + activeItem()?.scrollIntoView({ block: "nearest" }) + }), + ) + return + } + setStore({ open: false, search: "", active: "" }) + } + const selectModel = (item: ModelItem) => { + dismiss.preventTriggerRestore() + setOpen(false) + dismiss.afterClose(() => props.select(item)) + } + const manage = () => { + dismiss.preventTriggerRestore() + setOpen(false) + dismiss.afterClose(props.onManage) + } + const selectActive = () => { + const item = models().find((item) => modelKey(item) === store.active) + if (item) { + selectModel(item) + return + } + if (store.active === manageKey) manage() + } + const moveActive = (delta: number) => { + const options = keys() + if (options.length === 0) return + const index = options.indexOf(store.active) + const start = index === -1 ? 0 : index + setStore("active", options[(start + delta + options.length) % options.length]) + queueMicrotask(() => activeItem()?.scrollIntoView({ block: "nearest" })) + } + const setSearch = (value: string) => { + const first = props.view(value).items[0] + setStore({ search: value, active: first ? modelKey(first) : manageKey }) + } + + createEffect(() => { + if (!store.open) return + createEventListener( + document, + "keydown", + (event: KeyboardEvent) => handleDocumentSearchKeydown(searchRef, event, store.search, setSearch), + true, + ) + }) + + return ( + + + + (contentRef = element)} + class="w-[284px] overflow-hidden rounded-md border-0 bg-v2-background-bg-layer-01 !p-0 shadow-[var(--v2-elevation-floating)] focus:outline-none" + onPointerDownOutside={dismiss.preventTriggerRestore} + onFocusOutside={dismiss.preventTriggerRestore} + onCloseAutoFocus={dismiss.onCloseAutoFocus} + > +
+
+ + (searchRef = el)} + value={store.search} + placeholder={language.t("dialog.model.search.placeholder")} + class="h-7 min-w-0 flex-1 border-0 bg-transparent text-[13px] font-[440] leading-5 tracking-[-0.04px] text-v2-text-text-base outline-none placeholder:text-v2-text-text-faint" + spellcheck={false} + autocorrect="off" + autocomplete="off" + autocapitalize="off" + onInput={(event) => setSearch(event.currentTarget.value)} + onKeyDown={(event) => { + if (event.key === "Tab") return + event.stopPropagation() + if (event.key === "Escape") { + event.preventDefault() + dismiss.preventTriggerRestore() + setOpen(false) + dismiss.afterClose(props.onClose) + return + } + if (event.altKey || event.metaKey) return + if (event.key === "ArrowDown") { + event.preventDefault() + moveActive(1) + return + } + if (event.key === "ArrowUp") { + event.preventDefault() + moveActive(-1) + return + } + if (event.key === "Enter" && !event.isComposing) { + event.preventDefault() + selectActive() + } + }} + /> + + + +
+
+
+ +
+ 0} + fallback={ +
+ {language.t("dialog.model.empty")} +
+ } + > + + {(group) => ( + + + + {group.label} + + + + + {(item) => ( + + } + > + { + setStore("active", modelKey(item)) + setTimeout(() => searchRef?.focus()) + }} + onSelect={() => selectModel(item)} + > + {item.name} + + {item.provider.name} + + + {(meta) => ( + + {formatRowInline(meta(), { + omitPrice: isFree(item.provider.id, item.cost), + noTools: language.t("model.meta.noTools"), + })} + + )} + + + {language.t("model.tag.free")} + + + {language.t("model.tag.latest")} + + + {(view) => } + + + {(favorite) => ( + favorite().toggle(item)} + /> + )} + + + + )} + + + + )} + + +
+ +
+ + + {(r) => } + +
+ { + setStore("active", manageKey) + setTimeout(() => searchRef?.focus()) + }} + onSelect={manage} + > + + {language.t("dialog.model.manage")} + +
+ + + + ) +} + +export const DialogSelectModel: Component<{ provider?: string; model?: ModelState }> = (props) => { + const dialog = useDialog() + const language = useLanguage() + const local = useLocal() + const directory = () => decode64(local.slug()) + + const provider = () => { + void import("./dialog-connect-provider").then((x) => { + void dialog.show(() => ) + }) + } + + const manage = () => { + void import("./dialog-manage-models").then((x) => { + dialog.show(() => ) + }) + } + + return ( + + {language.t("command.provider.connect")} + + } + > + dialog.close()} /> + + + ) +} diff --git a/packages/app-bundle/overlay/packages/app/src/components/model-dead-pick-warning.tsx b/packages/app-bundle/overlay/packages/app/src/components/model-dead-pick-warning.tsx new file mode 100644 index 000000000..5aef0dbfe --- /dev/null +++ b/packages/app-bundle/overlay/packages/app/src/components/model-dead-pick-warning.tsx @@ -0,0 +1,122 @@ +// amicode#1989 (#1979 S6): the composer's dead-pick warning. Two triggers, +// one surface, never silent: +// - the session's latest assistant turn failed with a reachability-class +// error (denied / unavailable / invalid) → POST the S2 demotion (once per +// model per session) and overlay it locally, so the warning shows even if +// the POST fails or the host serves no reachability route; +// - the selected model is already dead per the S2 snapshot → warn before +// the user sends. +// The warning names the model and the reason, with a one-click switch to the +// recommended equivalent (rules in model-dead-pick.ts). Transient failures +// (rate limits, TLS, timeouts, 5xx) never demote and never warn here. +import { type Accessor, Show, createEffect, createMemo } from "solid-js" +import { useLanguage } from "@/context/language" +import { useLocal } from "@/context/local" +import { useServer } from "@/context/server" +import { useSync } from "@/context/sync" +import { amicodePost } from "@/utils/amicode-fetch" +import { createModelReachability } from "./dialog-select-model-reachability-source" +import { createModelShortlistConfig } from "./dialog-select-model-shortlist-source" +import { + REACHABILITY_DEMOTE_ROUTE, + applyTurnFailure, + createDemoteGate, + deadPickWarning, + demoteRequest, + latestTurnFailure, + recommendedSwitchTargets, + shouldDemote, + type DeadPickCandidate, +} from "./model-dead-pick" +import type { ModelRef } from "./dialog-select-model-reachability" + +export function ModelDeadPickWarning(props: { sessionID?: Accessor }) { + const language = useLanguage() + const local = useLocal() + const server = useServer() + const sync = useSync() + const reachability = createModelReachability() + const config = createModelShortlistConfig() + const gate = createDemoteGate() + + const failure = createMemo(() => { + const id = props.sessionID?.() + if (!id) return undefined + return latestTurnFailure(sync().data.message[id] ?? []) + }) + + createEffect(() => { + const id = props.sessionID?.() + const failed = failure() + if (!id || !failed || reachability.loading()) return + if (!shouldDemote(failed, reachability.snapshot())) return + if (!gate.claim(id, failed.ref)) return + void amicodePost(server.current, REACHABILITY_DEMOTE_ROUTE, demoteRequest(failed)) + .catch(() => { + // the local overlay below still names the failure — never silent + }) + .finally(() => reachability.refetch()) + }) + + const candidates = createMemo(() => + local.model.list().map((m) => ({ + providerID: m.provider.id, + modelID: m.id, + cost: m.cost ? { input: m.cost.input, output: m.cost.output } : undefined, + })), + ) + + const nameOf = (ref: ModelRef) => + local.model.list().find((m) => m.provider.id === ref.providerID && m.id === ref.modelID)?.name ?? ref.modelID + + const warning = createMemo(() => { + const current = local.model.current() + return deadPickWarning({ + current: current ? { providerID: current.provider.id, modelID: current.id } : undefined, + snapshot: applyTurnFailure(reachability.snapshot(), failure()), + candidates: candidates(), + // amicode#1988: the S4 deployment config's recommended list (role order, + // allowlist-filtered); no config ⇒ the cheapest ok same-provider model. + recommended: recommendedSwitchTargets(config()), + }) + }) + + return ( + + {(w) => { + const reason = w.reason + ? "key" in w.reason + ? language.t(w.reason.key) + : w.reason.text + : language.t("composer.model.deadPick.reason.unknown") + return ( + + ) + }} + + ) +} diff --git a/packages/app-bundle/overlay/packages/app/src/components/model-dead-pick.test.ts b/packages/app-bundle/overlay/packages/app/src/components/model-dead-pick.test.ts new file mode 100644 index 000000000..ee2c078dd --- /dev/null +++ b/packages/app-bundle/overlay/packages/app/src/components/model-dead-pick.test.ts @@ -0,0 +1,369 @@ +// amicode#1989 (#1979 S6): fail fast on a dead pick — pure rules, pinned +// without mounting the composer: which real failures demote, which do not, +// and which model the warning offers to switch to. +import { describe, expect, test } from "bun:test" +import type { ReachabilitySnapshot } from "./dialog-select-model-reachability" +import { + applyTurnFailure, + classifyTurnError, + deadPickWarning, + demoteRequest, + latestTurnFailure, + recommendedSwitchTargets, + shouldDemote, + createDemoteGate, + type DeadPickCandidate, +} from "./model-dead-pick" + +const T0 = 1_700_000_000_000 + +const row = ( + provider: string, + model: string, + status: string, + opts: { reason?: string | null; at?: number; source?: "probe" | "demoted" } = {}, +) => ({ + provider, + model, + status: status as ReachabilitySnapshot["models"][number]["status"], + reason: opts.reason ?? null, + latency_ms: 10, + checked_at: opts.at ?? T0, + credential_fp: "fp1", + source: opts.source ?? ("probe" as const), +}) + +const snap = (...models: ReturnType[]): ReachabilitySnapshot => ({ + catalog_known: true, + models, + providers: [], +}) + +const apiError = (statusCode: number | undefined, message: string, extra: Record = {}) => ({ + name: "APIError", + data: { message, statusCode, isRetryable: false, ...extra }, +}) + +describe("classifyTurnError — what a real failure demotes", () => { + test("Bedrock AccessDenied (403) → denied", () => { + const got = classifyTurnError( + apiError(403, "User: arn:aws:iam::123:user/x is not authorized to perform: bedrock:InvokeModel", { + responseHeaders: { "x-amzn-ErrorType": "AccessDeniedException:http://internal.amazon.com/" }, + }), + ) + expect(got?.status).toBe("denied") + expect(got?.reason).toContain("not authorized") + }) + + test("401 → denied; ProviderAuthError → denied", () => { + expect(classifyTurnError(apiError(401, "Invalid API key"))?.status).toBe("denied") + expect( + classifyTurnError({ name: "ProviderAuthError", data: { providerID: "anthropic", message: "bad key" } })?.status, + ).toBe("denied") + }) + + test("marketplace / not available for this account → unavailable", () => { + expect( + classifyTurnError(apiError(400, "Model access is denied due to missing AWS Marketplace subscription"))?.status, + ).toBe("unavailable") + expect(classifyTurnError(apiError(400, "This model is not available for this account."))?.status).toBe( + "unavailable", + ) + }) + + test("ValidationException / 404 / unknown model → invalid", () => { + expect( + classifyTurnError( + apiError(400, "Invocation of model ID x with on-demand throughput isn't supported. Use an inference profile."), + )?.status, + ).toBe("invalid") + expect(classifyTurnError(apiError(404, "Not Found"))?.status).toBe("invalid") + expect(classifyTurnError({ name: "UnknownError", data: { message: "ProviderModelNotFoundError: no such model" } })?.status).toBe( + "invalid", + ) + }) + + test("transient failures do NOT demote", () => { + // rate limit — by status and by text (Bedrock ThrottlingException) + expect(classifyTurnError(apiError(429, "Too many requests"))).toBeNull() + expect(classifyTurnError(apiError(400, "ThrottlingException: Too many tokens, please wait"))).toBeNull() + // TLS / network + expect( + classifyTurnError({ + name: "UnknownError", + data: { + message: + "ERR_TLS_CERT_ALTNAME_INVALID: Hostname/IP does not match certificate's altnames: Host: bedrock.example is not in the cert's altnames", + }, + }), + ).toBeNull() + expect(classifyTurnError({ name: "UnknownError", data: { message: "fetch failed: ECONNRESET" } })).toBeNull() + expect(classifyTurnError(apiError(undefined, "getaddrinfo ENOTFOUND api.example.com"))).toBeNull() + // timeouts + expect(classifyTurnError({ name: "UnknownError", data: { message: "TimeoutError: The operation timed out." } })).toBeNull() + expect(classifyTurnError(apiError(408, "Request Timeout"))).toBeNull() + // 5xx — even with a denial-looking body, a server error is not a verdict + expect(classifyTurnError(apiError(503, "Service Unavailable"))).toBeNull() + expect(classifyTurnError(apiError(500, "Internal error: access denied to upstream cache"))).toBeNull() + }) + + test("non-reachability errors do NOT demote", () => { + expect(classifyTurnError({ name: "MessageAbortedError", data: { message: "aborted" } })).toBeNull() + expect(classifyTurnError({ name: "ContextOverflowError", data: { message: "prompt is too long" } })).toBeNull() + expect(classifyTurnError({ name: "MessageOutputLengthError", data: {} })).toBeNull() + expect(classifyTurnError(apiError(400, "messages: text content blocks must be non-empty"))).toBeNull() + expect(classifyTurnError(undefined)).toBeNull() + expect(classifyTurnError("boom")).toBeNull() + }) +}) + +const assistant = (id: string, providerID: string, modelID: string, error?: unknown, at = T0 + 1000) => ({ + id, + role: "assistant", + providerID, + modelID, + time: { created: at, completed: at + 5 }, + ...(error ? { error } : {}), +}) + +describe("latestTurnFailure — only the session's latest assistant turn counts", () => { + test("a denied latest turn yields the failed model + status", () => { + const got = latestTurnFailure([ + { id: "u1", role: "user", time: { created: T0 } }, + assistant("a1", "amazon-bedrock", "anthropic.claude-x", apiError(403, "AccessDeniedException")), + ]) + expect(got).toMatchObject({ + messageID: "a1", + ref: { providerID: "amazon-bedrock", modelID: "anthropic.claude-x" }, + status: "denied", + at: T0 + 1005, + }) + }) + + test("an earlier failure followed by a good turn is history, not a dead pick", () => { + expect( + latestTurnFailure([ + assistant("a1", "p", "m", apiError(403, "Forbidden")), + assistant("a2", "p", "m", undefined, T0 + 2000), + ]), + ).toBeUndefined() + }) + + test("a transient latest failure yields nothing", () => { + expect(latestTurnFailure([assistant("a1", "p", "m", apiError(429, "rate limit"))])).toBeUndefined() + expect(latestTurnFailure([])).toBeUndefined() + }) +}) + +describe("shouldDemote / applyTurnFailure — a fresher probe wins", () => { + const failure = { + messageID: "a1", + ref: { providerID: "p", modelID: "m" }, + status: "denied" as const, + reason: "AccessDenied", + at: T0 + 1000, + } + + test("demotes when the cached row is ok from an older probe (or absent)", () => { + expect(shouldDemote(failure, snap(row("p", "m", "ok", { at: T0 })))).toBe(true) + expect(shouldDemote(failure, snap())).toBe(true) + expect(shouldDemote(failure, undefined)).toBe(true) + }) + + test("does not demote over a probe newer than the failure, nor re-demote the same status", () => { + expect(shouldDemote(failure, snap(row("p", "m", "ok", { at: T0 + 5000 })))).toBe(false) + expect(shouldDemote(failure, snap(row("p", "m", "denied", { at: T0, source: "demoted" })))).toBe(false) + }) + + test("applyTurnFailure overlays the demotion locally (never silent, even if the POST fails)", () => { + const base = snap(row("p", "m", "ok", { at: T0 }), row("p", "other", "ok")) + const got = applyTurnFailure(base, failure) + expect(got?.models.find((m) => m.model === "m")).toMatchObject({ status: "denied", source: "demoted" }) + expect(got?.models.find((m) => m.model === "other")?.status).toBe("ok") + // no snapshot (passthrough host) still yields a one-row snapshot + expect(applyTurnFailure(undefined, failure)?.models).toHaveLength(1) + // a fresher probe keeps the snapshot as is + const fresh = snap(row("p", "m", "ok", { at: T0 + 5000 })) + expect(applyTurnFailure(fresh, failure)).toBe(fresh) + expect(applyTurnFailure(base, undefined)).toBe(base) + }) + + test("demoteRequest is the S2 route body", () => { + expect(demoteRequest(failure)).toEqual({ provider: "p", model: "m", status: "denied", reason: "AccessDenied" }) + }) + + test("createDemoteGate: once per model per session", () => { + const gate = createDemoteGate() + expect(gate.claim("s1", { providerID: "p", modelID: "m" })).toBe(true) + expect(gate.claim("s1", { providerID: "p", modelID: "m" })).toBe(false) + expect(gate.claim("s2", { providerID: "p", modelID: "m" })).toBe(true) + expect(gate.claim("s1", { providerID: "p", modelID: "m2" })).toBe(true) + }) +}) + +const cand = (providerID: string, modelID: string, input?: number, output?: number): DeadPickCandidate => ({ + providerID, + modelID, + ...(input !== undefined ? { cost: { input, output: output ?? 0 } } : {}), +}) + +describe("deadPickWarning — the named warning and its switch target", () => { + const current = { providerID: "bedrock", modelID: "dead" } + const candidates = [ + cand("bedrock", "dead", 1, 1), + cand("bedrock", "pricey", 15, 75), + cand("bedrock", "cheap", 1, 5), + cand("bedrock", "unknown-cost"), + cand("anthropic", "cheapest", 0.1, 0.1), + cand("anthropic", "rec", 3, 15), + ] + + test("no warning when the current model is ok, unchecked, rate-limited, or unknown", () => { + for (const status of ["ok", "unchecked", "rate-limited"]) + expect( + deadPickWarning({ current, snapshot: snap(row("bedrock", "dead", status)), candidates }), + ).toBeUndefined() + expect(deadPickWarning({ current, snapshot: snap(), candidates })).toBeUndefined() + expect(deadPickWarning({ current, snapshot: undefined, candidates })).toBeUndefined() + expect(deadPickWarning({ current: undefined, snapshot: snap(), candidates })).toBeUndefined() + }) + + test("a dead current model warns with its named reason", () => { + const got = deadPickWarning({ + current, + snapshot: snap(row("bedrock", "dead", "unavailable"), row("bedrock", "cheap", "ok")), + candidates, + }) + expect(got?.model).toEqual(current) + expect(got?.status).toBe("unavailable") + expect(got?.reason).toEqual({ key: "dialog.model.reachability.reason.unavailable" }) + // an error row carries its own (redacted) text + const err = deadPickWarning({ current, snapshot: snap(row("bedrock", "dead", "error", { reason: "boom" })), candidates }) + expect(err?.reason).toEqual({ text: "boom" }) + }) + + test("recommended first: the first ok recommended model wins", () => { + const got = deadPickWarning({ + current, + snapshot: snap( + row("bedrock", "dead", "denied"), + row("bedrock", "cheap", "ok"), + row("anthropic", "rec", "ok"), + row("bedrock", "pricey", "denied"), + ), + candidates, + recommended: [{ providerID: "bedrock", modelID: "pricey" }, current, { providerID: "anthropic", modelID: "rec" }], + }) + expect(got?.switchTo).toEqual({ providerID: "anthropic", modelID: "rec" }) + }) + + test("no (usable) recommended: cheapest-known ok model of the same provider", () => { + const snapshot = snap( + row("bedrock", "dead", "denied"), + row("bedrock", "pricey", "ok"), + row("bedrock", "cheap", "ok"), + row("bedrock", "unknown-cost", "ok"), + row("anthropic", "cheapest", "ok"), + ) + expect(deadPickWarning({ current, snapshot, candidates, recommended: [] })?.switchTo).toEqual({ + providerID: "bedrock", + modelID: "cheap", + }) + // a recommended model that is not ok falls through to the same rule + expect( + deadPickWarning({ current, snapshot, candidates, recommended: [{ providerID: "anthropic", modelID: "rec" }] }) + ?.switchTo, + ).toEqual({ providerID: "bedrock", modelID: "cheap" }) + }) + + test("same provider has only unknown-cost ok models → one of them, still same provider", () => { + const snapshot = snap(row("bedrock", "dead", "denied"), row("bedrock", "unknown-cost", "ok"), row("anthropic", "cheapest", "ok")) + expect(deadPickWarning({ current, snapshot, candidates })?.switchTo).toEqual({ + providerID: "bedrock", + modelID: "unknown-cost", + }) + }) + + test("no ok model on the same provider → cheapest ok model anywhere", () => { + const snapshot = snap( + row("bedrock", "dead", "denied"), + row("bedrock", "cheap", "rate-limited"), + row("anthropic", "rec", "ok"), + row("anthropic", "cheapest", "ok"), + ) + expect(deadPickWarning({ current, snapshot, candidates })?.switchTo).toEqual({ + providerID: "anthropic", + modelID: "cheapest", + }) + }) + + test("ok rows that are not selectable candidates are never offered", () => { + const snapshot = snap(row("bedrock", "dead", "denied"), row("disconnected", "x", "ok")) + expect(deadPickWarning({ current, snapshot, candidates })?.switchTo).toBeUndefined() + }) + + test("no ok models at all → warning without a switch offer", () => { + const got = deadPickWarning({ + current, + snapshot: snap(row("bedrock", "dead", "invalid"), row("bedrock", "cheap", "denied"), row("anthropic", "rec", "unchecked")), + candidates, + recommended: [{ providerID: "anthropic", modelID: "rec" }], + }) + expect(got).toBeDefined() + expect(got?.status).toBe("invalid") + expect(got?.switchTo).toBeUndefined() + }) +}) + +// amicode#1988 (#1979 S5): the S6 follow-up — the S4 deployment config's +// recommended list feeds the switch target. +describe("recommendedSwitchTargets — S4 config → dead-pick recommended (#1988)", () => { + const ref = (providerID: string, modelID: string) => ({ providerID, modelID }) + + test("no config → none (the cheapest-ok fallback rules)", () => { + expect(recommendedSwitchTargets(undefined)).toEqual([]) + expect(recommendedSwitchTargets({ recommended: [], allowlist: undefined })).toEqual([]) + }) + + test("in role order (default, fast, strong, open-weight), config order within a role, deduped", () => { + expect( + recommendedSwitchTargets({ + recommended: [ + { ref: ref("a", "strong"), role: "strong" }, + { ref: ref("a", "fast"), role: "fast" }, + { ref: ref("a", "default-1"), role: "default" }, + { ref: ref("a", "default-2"), role: "default" }, + { ref: ref("a", "fast"), role: "strong" }, + ], + allowlist: undefined, + }), + ).toEqual([ref("a", "default-1"), ref("a", "default-2"), ref("a", "fast"), ref("a", "strong")]) + }) + + test("the allowlist hard-filters the switch targets too", () => { + expect( + recommendedSwitchTargets({ + recommended: [ + { ref: ref("a", "x"), role: "default" }, + { ref: ref("b", "y"), role: "default" }, + ], + allowlist: ["b/*"], + }), + ).toEqual([ref("b", "y")]) + }) + + test("the switch prefers a recommended ok model over a cheaper same-provider one", () => { + const current = ref("bedrock", "dead") + const candidates = [cand("bedrock", "dead", 1, 1), cand("bedrock", "cheap", 1, 5), cand("anthropic", "rec", 3, 15)] + const snapshot = snap(row("bedrock", "dead", "denied"), row("bedrock", "cheap", "ok"), row("anthropic", "rec", "ok")) + const recommended = recommendedSwitchTargets({ + recommended: [{ ref: ref("anthropic", "rec"), role: "default" }], + allowlist: undefined, + }) + expect(deadPickWarning({ current, snapshot, candidates, recommended })?.switchTo).toEqual(ref("anthropic", "rec")) + // without the config the same snapshot falls back to the cheapest same-provider model + expect( + deadPickWarning({ current, snapshot, candidates, recommended: recommendedSwitchTargets(undefined) })?.switchTo, + ).toEqual(ref("bedrock", "cheap")) + }) +}) diff --git a/packages/app-bundle/overlay/packages/app/src/components/model-dead-pick.ts b/packages/app-bundle/overlay/packages/app/src/components/model-dead-pick.ts new file mode 100644 index 000000000..64996c475 --- /dev/null +++ b/packages/app-bundle/overlay/packages/app/src/components/model-dead-pick.ts @@ -0,0 +1,252 @@ +// amicode#1989 (#1979 S6): fail fast on a dead pick. PURE — no SolidJS, no +// fetch — so the demote / no-demote line and the switch-target rules are +// unit-tested. The wiring (POST demote, composer warning) lives in +// model-dead-pick-warning.tsx. +// +// Rules (the issue's): +// - A session's failed assistant turn demotes its model only for a +// reachability-class failure: denied / unavailable / invalid (the S1 +// classifyProbeError classes, overlay packages/opencode/src/provider/error.ts, +// re-derived here from the serialized session error the app receives). +// Rate limits, TLS / network faults, timeouts and 5xx are transient and +// never demote. +// - A demotion never overrides a probe verdict NEWER than the failure (the +// S2 cache: "a demotion sticks until the next probe"). +// - The warning shows whenever the current model is dead per the snapshot +// (probe or demotion), before sending as well as after a failed turn. +// - Switch target: the first ok recommended model; else the cheapest-known ok +// model of the same provider; else the cheapest-known ok model anywhere; +// none ⇒ the warning still shows, without a switch offer. Only selectable +// candidates (connected providers' catalog models) are ever offered. +import { + reachabilityReason, + type ModelRef, + type ReachabilityModel, + type ReachabilityReason, + type ReachabilitySnapshot, + type ReachabilityStatus, +} from "./dialog-select-model-reachability" +import { ROLE_ORDER, isAllowed, type ShortlistConfig } from "./dialog-select-model-shortlist" + +export const REACHABILITY_DEMOTE_ROUTE = "/amicode/model-reachability/demote" + +export type DemoteStatus = "denied" | "unavailable" | "invalid" + +export type TurnFailure = { + messageID: string + ref: ModelRef + status: DemoteStatus + reason: string + /** When the failed turn ended (engine clock, epoch ms). */ + at: number +} + +export type DeadPickCandidate = ModelRef & { cost?: { input?: number; output?: number } } + +export type DeadPickWarning = { + model: ModelRef + status: ReachabilityStatus + reason: ReachabilityReason | undefined + switchTo?: ModelRef +} + +// The S1 patterns (packages/opencode/src/provider/error.ts, #1984) — kept in +// lockstep so a real failure and a probe name the same class. +const UNAVAILABLE = + /not available for this account|aws[- ]marketplace|marketplace (subscription|agreement)|model access is denied|access to (this|the) model is not|not subscribed/i +const RATE_LIMITED = /throttl|too many requests|rate[- ]?limit|quota exceeded|tokens per (minute|day)/i +const INVALID = + /validationexception|inference profile|on-demand throughput|invalid model|model identifier is invalid|model[_ ]not[_ ]found|unknown model|no such model|does not exist|is not supported|resourcenotfoundexception|ProviderModelNotFoundError|AI_NoSuchModelError/i +const DENIED = + /accessdenied|access denied|not authorized|unauthorized|unrecognizedclient|forbidden|permission denied|invalid api key|invalid_api_key|security token/i +// Transient: never a verdict on the model. Checked before every demote class. +const TRANSIENT = + /ERR_TLS|certificate|altnames|self[- ]signed|ECONNRESET|ECONNREFUSED|ENOTFOUND|EAI_AGAIN|ETIMEDOUT|EPIPE|ENETUNREACH|socket hang up|fetch failed|network|timed? ?out|TimeoutError|AbortError|service unavailable|overloaded/i + +const NOT_REACHABILITY = new Set([ + "MessageAbortedError", + "ContextOverflowError", + "MessageOutputLengthError", + "StructuredOutputError", + "ContentFilterError", +]) + +const REASON_MAX = 240 + +const isObj = (v: unknown): v is Record => typeof v === "object" && v !== null && !Array.isArray(v) +const str = (v: unknown) => (typeof v === "string" ? v : "") +const clip = (s: string) => (s.length > REASON_MAX ? `${s.slice(0, REASON_MAX - 1)}…` : s) + +function header(headers: unknown, name: string) { + if (!isObj(headers)) return "" + const key = Object.keys(headers).find((k) => k.toLowerCase() === name) + return key ? str(headers[key]) : "" +} + +/** A session's failed-turn error → the demote class, or null (no demotion). */ +export function classifyTurnError(error: unknown): { status: DemoteStatus; reason: string } | null { + if (!isObj(error) || typeof error.name !== "string") return null + const data = isObj(error.data) ? error.data : {} + const message = str(data.message) + if (NOT_REACHABILITY.has(error.name)) return null + if (error.name === "ProviderAuthError") return { status: "denied", reason: clip(message || "authentication failed") } + + const status = typeof data.statusCode === "number" ? data.statusCode : undefined + if (status !== undefined && (status === 408 || status === 429 || status >= 500)) return null + const haystack = [error.name, message, header(data.responseHeaders, "x-amzn-errortype"), str(data.responseBody)].join( + "\n", + ) + if (TRANSIENT.test(haystack) || RATE_LIMITED.test(haystack)) return null + const reason = clip(message || error.name) + if (UNAVAILABLE.test(haystack)) return { status: "unavailable", reason } + if (status === 404 || INVALID.test(haystack)) return { status: "invalid", reason } + if (status === 401 || status === 403 || DENIED.test(haystack)) return { status: "denied", reason } + return null +} + +/** The session's LATEST assistant turn, when it failed with a demote class. + * An earlier failure followed by a good turn is history, not a dead pick. */ +export function latestTurnFailure(messages: readonly unknown[]): TurnFailure | undefined { + for (let i = messages.length - 1; i >= 0; i--) { + const m = messages[i] + if (!isObj(m) || m.role !== "assistant") continue + if (typeof m.providerID !== "string" || typeof m.modelID !== "string" || typeof m.id !== "string") return undefined + const got = classifyTurnError(m.error) + if (!got) return undefined + const time = isObj(m.time) ? m.time : {} + const at = + typeof time.completed === "number" ? time.completed : typeof time.created === "number" ? time.created : 0 + return { messageID: m.id, ref: { providerID: m.providerID, modelID: m.modelID }, ...got, at } + } + return undefined +} + +const same = (a: ModelRef, b: ModelRef) => a.providerID === b.providerID && a.modelID === b.modelID +const rowOf = (snapshot: ReachabilitySnapshot | undefined, ref: ModelRef) => + snapshot?.models.find((m) => m.provider === ref.providerID && m.model === ref.modelID) +const fresher = (row: ReachabilityModel | undefined, failure: TurnFailure) => + !!row && row.checked_at !== null && row.checked_at > failure.at + +/** POST the demotion? Not over a newer verdict, not twice for the same one. */ +export function shouldDemote(failure: TurnFailure, snapshot: ReachabilitySnapshot | undefined): boolean { + const row = rowOf(snapshot, failure.ref) + if (fresher(row, failure)) return false + if (row && row.source === "demoted" && row.status === failure.status) return false + return true +} + +/** The snapshot with the failure overlaid locally — the warning shows even + * when the demote POST fails or the host serves no reachability route. */ +export function applyTurnFailure( + snapshot: ReachabilitySnapshot | undefined, + failure: TurnFailure | undefined, +): ReachabilitySnapshot | undefined { + if (!failure) return snapshot + const row = rowOf(snapshot, failure.ref) + if (fresher(row, failure)) return snapshot + const demoted: ReachabilityModel = { + provider: failure.ref.providerID, + model: failure.ref.modelID, + status: failure.status, + reason: failure.reason, + latency_ms: null, + checked_at: failure.at, + credential_fp: row?.credential_fp ?? null, + source: "demoted", + } + const base = snapshot ?? { catalog_known: false, models: [], providers: [] } + return { + ...base, + models: row ? base.models.map((m) => (m === row ? demoted : m)) : [...base.models, demoted], + } +} + +/** The S2 demote route body. */ +export function demoteRequest(failure: TurnFailure) { + return { + provider: failure.ref.providerID, + model: failure.ref.modelID, + status: failure.status, + reason: failure.reason, + } +} + +/** Once per model per session. */ +export function createDemoteGate() { + const seen = new Set() + return { + claim(sessionID: string, ref: ModelRef) { + const key = `${sessionID}\u0000${ref.providerID}\u0000${ref.modelID}` + if (seen.has(key)) return false + seen.add(key) + return true + }, + } +} + +const DEAD: ReadonlySet = new Set(["denied", "unavailable", "invalid", "error"]) + +const price = (c: DeadPickCandidate) => + c.cost && (typeof c.cost.input === "number" || typeof c.cost.output === "number") + ? (c.cost.input ?? 0) + (c.cost.output ?? 0) + : undefined + +function cheapest(pool: readonly DeadPickCandidate[]): ModelRef | undefined { + let best: DeadPickCandidate | undefined + let bestPrice: number | undefined + for (const c of pool) { + const p = price(c) + if (!best || (p !== undefined && (bestPrice === undefined || p < bestPrice))) { + best = c + bestPrice = p + } + } + return best ? { providerID: best.providerID, modelID: best.modelID } : undefined +} + +/** The composer's warning for the current model, or undefined when it is not + * known dead (ok / rate-limited / unchecked / no row / no snapshot). */ +export function deadPickWarning(input: { + current: ModelRef | undefined + snapshot: ReachabilitySnapshot | undefined + candidates: readonly DeadPickCandidate[] + recommended?: readonly ModelRef[] +}): DeadPickWarning | undefined { + const current = input.current + if (!current || !input.snapshot) return undefined + const row = rowOf(input.snapshot, current) + if (!row || !DEAD.has(row.status)) return undefined + const reason = reachabilityReason({ + status: row.status, + reason: row.reason, + hidden: false, + greyed: false, + tag: "selected-dead", + }) + const model = { providerID: current.providerID, modelID: current.modelID } + + const ok = input.candidates.filter((c) => !same(c, current) && rowOf(input.snapshot, c)?.status === "ok") + const recommended = (input.recommended ?? []).find((r) => ok.some((c) => same(c, r))) + const switchTo = recommended + ? { providerID: recommended.providerID, modelID: recommended.modelID } + : (cheapest(ok.filter((c) => c.providerID === current.providerID)) ?? cheapest(ok)) + return switchTo ? { model, status: row.status, reason, switchTo } : { model, status: row.status, reason } +} + +/** amicode#1988 (S5, the S6 follow-up): the S4 deployment config's recommended + * list as dead-pick switch targets — role order (default, fast, strong, + * open-weight), config order within a role, deduped, allowlist-filtered. + * No config ⇒ none (deadPickWarning falls back to the cheapest ok model). */ +export function recommendedSwitchTargets(config: ShortlistConfig | undefined): ModelRef[] { + const seen = new Set() + return ROLE_ORDER.flatMap((role) => + (config?.recommended ?? []) + .filter((entry) => entry.role === role && isAllowed(entry.ref, config?.allowlist)) + .flatMap((entry) => { + const key = `${entry.ref.providerID}\u0000${entry.ref.modelID}` + if (seen.has(key)) return [] + seen.add(key) + return [{ providerID: entry.ref.providerID, modelID: entry.ref.modelID }] + }), + ) +} diff --git a/packages/app-bundle/overlay/packages/app/src/components/model-row-meta.test.ts b/packages/app-bundle/overlay/packages/app/src/components/model-row-meta.test.ts new file mode 100644 index 000000000..4ce463cbb --- /dev/null +++ b/packages/app-bundle/overlay/packages/app/src/components/model-row-meta.test.ts @@ -0,0 +1,261 @@ +// amicode#1988 (#1979 S5): the picker row meta — price / context / tools / +// measured latency — and the cheapest-that-works search order. Pure, pinned +// without mounting the SolidJS picker. +import { describe, expect, test } from "bun:test" +import type { ModelReachabilityView, ReachabilitySnapshot } from "./dialog-select-model-reachability" +import { + compareCheapestOk, + effectivePrice, + formatContext, + formatLatency, + formatPrice, + formatPriceCompact, + formatRowInline, + formatTools, + formatUsd, + latencyOf, + modelRowMeta, + searchTier, +} from "./model-row-meta" + +describe("formatUsd — per-Mtok rounding", () => { + test(">= 1: at most 2 decimals, trailing zeros dropped", () => { + expect(formatUsd(3)).toBe("$3") + expect(formatUsd(15)).toBe("$15") + expect(formatUsd(1.25)).toBe("$1.25") + expect(formatUsd(2.5)).toBe("$2.5") + expect(formatUsd(1.256)).toBe("$1.26") + expect(formatUsd(15.004)).toBe("$15") + }) + + test("< 1: up to 3 decimals, trailing zeros dropped", () => { + expect(formatUsd(0.3)).toBe("$0.3") + expect(formatUsd(0.075)).toBe("$0.075") + expect(formatUsd(0.1234)).toBe("$0.123") + expect(formatUsd(0.08)).toBe("$0.08") + }) + + test("tiny non-zero prices never round down to $0", () => { + expect(formatUsd(0.0001)).toBe("<$0.001") + }) + + test("0 is $0; missing / non-finite / negative is —", () => { + expect(formatUsd(0)).toBe("$0") + expect(formatUsd(undefined)).toBe("—") + expect(formatUsd(Number.NaN)).toBe("—") + expect(formatUsd(-1)).toBe("—") + }) +}) + +describe("formatPrice — $in/$out · cache $c per Mtok", () => { + test("full price with cache read", () => { + expect(formatPrice({ input: 3, output: 15, cache_read: 0.3 })).toBe("$3/$15 · cache $0.3 per Mtok") + // the SDK shape (cost.cache.read) reads the same + expect(formatPrice({ input: 3, output: 15, cache: { read: 0.3 } })).toBe("$3/$15 · cache $0.3 per Mtok") + }) + + test("no cache-read price → the cache segment is omitted", () => { + expect(formatPrice({ input: 0.15, output: 0.6 })).toBe("$0.15/$0.6 per Mtok") + }) + + test("all-zero price is free", () => { + expect(formatPrice({ input: 0, output: 0 })).toBe("free") + expect(formatPrice({ input: 0, output: 0, cache_read: 0 })).toBe("free") + expect(formatPriceCompact({ input: 0, output: 0 })).toBe("free") + }) + + test("missing price shows as —", () => { + expect(formatPrice(undefined)).toBe("—") + expect(formatPrice({})).toBe("—") + expect(formatPriceCompact(undefined)).toBe("—") + // one side missing: that side is — + expect(formatPrice({ input: 3 })).toBe("$3/— per Mtok") + }) + + test("compact form is $in/$out", () => { + expect(formatPriceCompact({ input: 3, output: 15, cache_read: 0.3 })).toBe("$3/$15") + }) + + test("labels are injectable (i18n)", () => { + const labels = { free: "gratis", cache: "caché", perMtok: "por Mtok" } + expect(formatPrice({ input: 0, output: 0 }, labels)).toBe("gratis") + expect(formatPrice({ input: 3, output: 15, cache_read: 0.3 }, labels)).toBe("$3/$15 · caché $0.3 por Mtok") + }) +}) + +describe("formatContext — K / M units", () => { + test("thousands → K, millions → M", () => { + expect(formatContext(200_000)).toBe("200K") + expect(formatContext(128_000)).toBe("128K") + expect(formatContext(131_072)).toBe("131K") + expect(formatContext(1_000_000)).toBe("1M") + expect(formatContext(1_048_576)).toBe("1M") + expect(formatContext(1_500_000)).toBe("1.5M") + expect(formatContext(999_999)).toBe("1M") + expect(formatContext(512)).toBe("512") + }) + + test("missing / zero → —", () => { + expect(formatContext(undefined)).toBe("—") + expect(formatContext(0)).toBe("—") + }) +}) + +describe("formatTools", () => { + test("SDK capabilities.toolcall or catalog tool_call; unknown → undefined", () => { + expect(formatTools({ capabilities: { toolcall: true } })).toBe("yes") + expect(formatTools({ capabilities: { toolcall: false } })).toBe("no") + expect(formatTools({ tool_call: true })).toBe("yes") + expect(formatTools({})).toBeUndefined() + }) +}) + +describe("formatLatency — measured, from reachability latency_ms", () => { + test("seconds with one decimal under 10 s, whole seconds above", () => { + expect(formatLatency(900)).toBe("0.9 s") + expect(formatLatency(1234)).toBe("1.2 s") + expect(formatLatency(12_500)).toBe("13 s") + expect(formatLatency(45)).toBe("<0.1 s") + }) + + test("missing → omitted (undefined)", () => { + expect(formatLatency(undefined)).toBeUndefined() + expect(formatLatency(null)).toBeUndefined() + expect(formatLatency(-5)).toBeUndefined() + }) + + test("latencyOf reads the snapshot row", () => { + const snapshot: ReachabilitySnapshot = { + catalog_known: true, + providers: [], + models: [ + { + provider: "a", + model: "x", + status: "ok", + reason: null, + latency_ms: 900, + checked_at: 1, + credential_fp: null, + source: "probe", + }, + ], + } + expect(latencyOf(snapshot, { providerID: "a", modelID: "x" })).toBe(900) + expect(latencyOf(snapshot, { providerID: "a", modelID: "y" })).toBeUndefined() + expect(latencyOf(undefined, { providerID: "a", modelID: "x" })).toBeUndefined() + }) + + test("modelRowMeta bundles it all", () => { + expect( + modelRowMeta( + { cost: { input: 3, output: 15, cache: { read: 0.3 } }, limit: { context: 200_000 }, capabilities: { toolcall: true } }, + 900, + ), + ).toEqual({ + price: "$3/$15 · cache $0.3 per Mtok", + priceCompact: "$3/$15", + context: "200K", + tools: "yes", + latency: "0.9 s", + }) + expect(modelRowMeta({}, undefined).latency).toBeUndefined() + }) +}) + +describe("effectivePrice — cache-read-weighted blend", () => { + test("0.2·input + 0.6·cache_read + 0.2·output", () => { + expect(effectivePrice({ input: 3, output: 15, cache_read: 0.3 })).toBeCloseTo(0.6 + 0.18 + 3) + }) + + test("no cache-read price → cache reads bill as input", () => { + expect(effectivePrice({ input: 1, output: 2 })).toBeCloseTo(0.2 + 0.6 + 0.4) + }) + + test("free is 0; unknown input or output → undefined", () => { + expect(effectivePrice({ input: 0, output: 0 })).toBe(0) + expect(effectivePrice(undefined)).toBeUndefined() + expect(effectivePrice({ input: 1 })).toBeUndefined() + }) +}) + +const view = (over: Partial): ModelReachabilityView => ({ + status: "ok", + reason: null, + hidden: false, + greyed: false, + tag: undefined, + ...over, +}) + +describe("searchTier", () => { + test("ok / no snapshot → 0; shown-but-unproven → 1; dead (hidden or selected-dead) → 2", () => { + expect(searchTier(view({}))).toBe(0) + expect(searchTier(view({ status: undefined }))).toBe(0) + expect(searchTier(view({ status: "rate-limited", tag: "rate-limited" }))).toBe(1) + expect(searchTier(view({ status: "unchecked", tag: "unchecked" }))).toBe(1) + expect(searchTier(view({ status: "denied", hidden: true, greyed: true }))).toBe(2) + expect(searchTier(view({ status: "denied", tag: "selected-dead" }))).toBe(2) + }) +}) + +type Row = { name: string; price: number | undefined; tier: 0 | 1 | 2 } +const sortRows = (rows: Row[]) => + [...rows] + .sort(compareCheapestOk({ tierOf: (r) => r.tier, priceOf: (r) => r.price, nameOf: (r) => r.name })) + .map((r) => r.name) + +describe("compareCheapestOk — cheapest-that-works", () => { + test("cheapest ok model first", () => { + expect( + sortRows([ + { name: "pricey", price: 9, tier: 0 }, + { name: "cheap", price: 1, tier: 0 }, + { name: "mid", price: 4, tier: 0 }, + ]), + ).toEqual(["cheap", "mid", "pricey"]) + }) + + test("a dead model never sorts above an ok one, however cheap", () => { + expect( + sortRows([ + { name: "dead-free", price: 0, tier: 2 }, + { name: "ok-pricey", price: 50, tier: 0 }, + { name: "limited-cheap", price: 0.1, tier: 1 }, + { name: "ok-unknown", price: undefined, tier: 0 }, + ]), + ).toEqual(["ok-pricey", "ok-unknown", "limited-cheap", "dead-free"]) + }) + + test("unknown price sorts after known within a tier; ties by name", () => { + expect( + sortRows([ + { name: "b", price: 2, tier: 0 }, + { name: "z-unknown", price: undefined, tier: 0 }, + { name: "a", price: 2, tier: 0 }, + { name: "a-unknown", price: undefined, tier: 0 }, + ]), + ).toEqual(["a", "b", "a-unknown", "z-unknown"]) + }) +}) + +describe("formatRowInline — the compact meta on each picker row", () => { + const meta = modelRowMeta( + { cost: { input: 3, output: 15, cache: { read: 0.3 } }, limit: { context: 200_000 }, capabilities: { toolcall: true } }, + 900, + ) + + test("$in/$out · context · latency", () => { + expect(formatRowInline(meta)).toBe("$3/$15 · 200K · 0.9 s") + }) + + test("no latency → omitted; missing price stays —", () => { + expect(formatRowInline(modelRowMeta({ limit: { context: 1_000_000 } }, undefined))).toBe("— · 1M") + }) + + test("a model without tool calls says so; the price can be omitted (a Free tag already shows)", () => { + const noTools = modelRowMeta({ cost: { input: 0, output: 0 }, limit: { context: 8000 }, capabilities: { toolcall: false } }, 1500) + expect(formatRowInline(noTools)).toBe("free · 8K · 1.5 s · no tools") + expect(formatRowInline(noTools, { omitPrice: true, noTools: "sin herramientas" })).toBe("8K · 1.5 s · sin herramientas") + }) +}) diff --git a/packages/app-bundle/overlay/packages/app/src/components/model-row-meta.ts b/packages/app-bundle/overlay/packages/app/src/components/model-row-meta.ts new file mode 100644 index 000000000..742edd8b9 --- /dev/null +++ b/packages/app-bundle/overlay/packages/app/src/components/model-row-meta.ts @@ -0,0 +1,184 @@ +// amicode#1988 (#1979 S5): the model picker's row meta — price, context, +// tool support (catalog: overlay packages/core/src/models-dev.ts, surfaced on +// the app's model objects as cost.{input,output,cache.read}, limit.context, +// capabilities.toolcall) and the measured latency (S2 reachability +// latency_ms) — plus the cheapest-that-works search order. PURE — no +// SolidJS — so rounding and ordering are unit-tested. +// +// Rules: +// - Prices are USD per million tokens. >= 1 → at most 2 decimals; < 1 → up +// to 3 decimals (a non-zero price below $0.001 reads "<$0.001", never +// "$0"); trailing zeros dropped. A missing / non-finite price is "—". +// - The whole price is "free" when every known part is 0, "—" when neither +// input nor output is known. The cache segment is omitted when the catalog +// has no cache-read price. +// - Context: "200K" / "1M" / "1.5M"; missing → "—". +// - Latency: "0.9 s" from latency_ms; missing → omitted (undefined). +// - Search order (non-empty query only): reachability tier first — ok (or no +// reachability data) → shown-but-unproven (rate-limited / unchecked) → +// dead (hidden, or the selected-dead current model) — so a dead model never +// sorts above an ok one; then effective price ascending (unknown last); +// ties by name. +// - Effective price blend (per Mtok): 0.2·input + 0.6·cache_read + 0.2·output. +// Agent turns re-send a growing, mostly-cached prompt, so cache reads +// dominate real spend; a model with no cache-read price bills those reads +// at its input price. +import type { ModelRef, ModelReachabilityView, ReachabilitySnapshot } from "./dialog-select-model-reachability" + +export type CatalogCost = { + input?: number + output?: number + /** models.dev catalog shape */ + cache_read?: number + /** SDK model shape */ + cache?: { read?: number } +} + +export type RowMetaModel = { + cost?: CatalogCost + limit?: { context?: number } + capabilities?: { toolcall?: boolean } + tool_call?: boolean +} + +export type PriceLabels = { free: string; cache: string; perMtok: string } + +export const DEFAULT_PRICE_LABELS: PriceLabels = { free: "free", cache: "cache", perMtok: "per Mtok" } + +export const EFFECTIVE_PRICE_BLEND = { input: 0.2, cache_read: 0.6, output: 0.2 } as const + +export type ModelRowMeta = { + price: string + priceCompact: string + context: string + tools: "yes" | "no" | undefined + latency: string | undefined +} + +const MISSING = "—" + +const known = (n: number | undefined | null): n is number => typeof n === "number" && Number.isFinite(n) && n >= 0 + +const trim = (n: number, decimals: number) => String(Number(n.toFixed(decimals))) + +export function formatUsd(n: number | undefined) { + if (!known(n)) return MISSING + if (n === 0) return "$0" + if (n < 0.001) return "<$0.001" + return `$${trim(n, n < 1 ? 3 : 2)}` +} + +const cacheRead = (cost: CatalogCost) => { + const value = cost.cache_read ?? cost.cache?.read + return known(value) ? value : undefined +} + +function priceState(cost: CatalogCost | undefined) { + if (!cost || (!known(cost.input) && !known(cost.output))) return "missing" as const + const cache = cacheRead(cost) + if (cost.input === 0 && cost.output === 0 && (cache === undefined || cache === 0)) return "free" as const + return "priced" as const +} + +export function formatPriceCompact(cost: CatalogCost | undefined, labels: PriceLabels = DEFAULT_PRICE_LABELS) { + const state = priceState(cost) + if (state === "missing" || !cost) return MISSING + if (state === "free") return labels.free + return `${formatUsd(cost.input)}/${formatUsd(cost.output)}` +} + +export function formatPrice(cost: CatalogCost | undefined, labels: PriceLabels = DEFAULT_PRICE_LABELS) { + const state = priceState(cost) + if (state === "missing" || !cost) return MISSING + if (state === "free") return labels.free + const cache = cacheRead(cost) + const segment = cache === undefined ? "" : ` · ${labels.cache} ${formatUsd(cache)}` + return `${formatUsd(cost.input)}/${formatUsd(cost.output)}${segment} ${labels.perMtok}` +} + +export function formatContext(n: number | undefined) { + if (!known(n) || n === 0) return MISSING + if (n < 1000) return String(Math.round(n)) + const thousands = Math.round(n / 1000) + if (thousands < 1000) return `${thousands}K` + return `${trim(n / 1_000_000, 1)}M` +} + +export function formatTools(model: RowMetaModel): "yes" | "no" | undefined { + const value = model.capabilities?.toolcall ?? model.tool_call + if (value === undefined) return undefined + return value ? "yes" : "no" +} + +export function formatLatency(ms: number | undefined | null) { + if (!known(ms)) return undefined + if (ms < 100) return "<0.1 s" + const s = ms / 1000 + return s < 10 ? `${s.toFixed(1)} s` : `${Math.round(s)} s` +} + +export function latencyOf(snapshot: ReachabilitySnapshot | undefined, ref: ModelRef) { + const value = snapshot?.models.find((m) => m.provider === ref.providerID && m.model === ref.modelID)?.latency_ms + return known(value) ? value : undefined +} + +export function modelRowMeta( + model: RowMetaModel, + latencyMs: number | undefined, + labels: PriceLabels = DEFAULT_PRICE_LABELS, +): ModelRowMeta { + return { + price: formatPrice(model.cost, labels), + priceCompact: formatPriceCompact(model.cost, labels), + context: formatContext(model.limit?.context), + tools: formatTools(model), + latency: formatLatency(latencyMs), + } +} + +/** The compact row line: "$in/$out · context · latency", plus a "no tools" + * marker when the catalog says the model cannot call tools. */ +export function formatRowInline(meta: ModelRowMeta, options: { omitPrice?: boolean; noTools?: string } = {}) { + return [ + options.omitPrice ? undefined : meta.priceCompact, + meta.context, + meta.latency, + meta.tools === "no" ? (options.noTools ?? "no tools") : undefined, + ] + .filter((part) => part !== undefined) + .join(" · ") +} + +export function effectivePrice(cost: CatalogCost | undefined) { + if (!cost || !known(cost.input) || !known(cost.output)) return undefined + const cache = cacheRead(cost) ?? cost.input + return ( + EFFECTIVE_PRICE_BLEND.input * cost.input + + EFFECTIVE_PRICE_BLEND.cache_read * cache + + EFFECTIVE_PRICE_BLEND.output * cost.output + ) +} + +/** 0 = ok (or no reachability data), 1 = shown but unproven, 2 = dead. */ +export function searchTier(view: ModelReachabilityView): 0 | 1 | 2 { + if (view.hidden || view.tag === "selected-dead") return 2 + if (view.status === undefined || view.status === "ok") return 0 + return 1 +} + +export function compareCheapestOk(input: { + tierOf: (item: T) => number + priceOf: (item: T) => number | undefined + nameOf: (item: T) => string +}) { + return (a: T, b: T) => { + const tier = input.tierOf(a) - input.tierOf(b) + if (tier !== 0) return tier + const pa = input.priceOf(a) + const pb = input.priceOf(b) + if (pa !== undefined && pb !== undefined && pa !== pb) return pa - pb + if (pa === undefined && pb !== undefined) return 1 + if (pa !== undefined && pb === undefined) return -1 + return input.nameOf(a).localeCompare(input.nameOf(b)) + } +} diff --git a/packages/app-bundle/overlay/packages/app/src/components/model-tooltip.tsx b/packages/app-bundle/overlay/packages/app/src/components/model-tooltip.tsx new file mode 100644 index 000000000..b1fa7ae96 --- /dev/null +++ b/packages/app-bundle/overlay/packages/app/src/components/model-tooltip.tsx @@ -0,0 +1,165 @@ +import { Show, type Component, type JSX } from "solid-js" +import { useLanguage } from "@/context/language" +// amicode#1988 (#1979 S5): full price / tools / measured-latency detail. +import type { ModelRowMeta } from "./model-row-meta" + +type InputKey = "text" | "image" | "audio" | "video" | "pdf" +type InputMap = Record + +type ModelInfo = { + id: string + name: string + provider: { + name: string + } + capabilities?: { + reasoning: boolean + input: InputMap + } + modalities?: { + input: Array + } + reasoning?: boolean + limit: { + context: number + } +} + +function ModelTooltipRow(props: { name: JSX.Element; value: JSX.Element }) { + return ( +
+ {props.name} + {props.value} +
+ ) +} + +export const ModelTooltip: Component<{ + model: ModelInfo + latest?: boolean + free?: boolean + v2?: boolean + meta?: ModelRowMeta +}> = (props) => { + const language = useLanguage() + const sourceName = (model: ModelInfo) => { + const value = `${model.id} ${model.name}`.toLowerCase() + + if (/claude|anthropic/.test(value)) return language.t("model.provider.anthropic") + if (/gpt|o[1-4]|codex|openai/.test(value)) return language.t("model.provider.openai") + if (/gemini|palm|bard|google/.test(value)) return language.t("model.provider.google") + if (/grok|xai/.test(value)) return language.t("model.provider.xai") + if (/llama|meta/.test(value)) return language.t("model.provider.meta") + + return model.provider.name + } + const inputLabel = (value: string) => { + if (value === "text") return language.t("model.input.text") + if (value === "image") return language.t("model.input.image") + if (value === "audio") return language.t("model.input.audio") + if (value === "video") return language.t("model.input.video") + if (value === "pdf") return language.t("model.input.pdf") + return value + } + const title = () => { + const tags: Array = [] + if (props.latest) tags.push(language.t("model.tag.latest")) + if (props.free) tags.push(language.t("model.tag.free")) + const suffix = tags.length ? ` (${tags.join(", ")})` : "" + return `${sourceName(props.model)} ${props.model.name}${suffix}` + } + const name = () => { + const tags: Array = [] + if (props.latest) tags.push(language.t("model.tag.latest")) + if (props.free) tags.push(language.t("model.tag.free")) + const suffix = tags.length ? ` (${tags.join(", ")})` : "" + return `${props.model.name}${suffix}` + } + const inputs = () => { + if (props.model.capabilities) { + const input = props.model.capabilities.input + const order: Array = ["text", "image", "audio", "video", "pdf"] + const entries = order.filter((key) => input[key]).map((key) => inputLabel(key)) + return entries.length ? entries.join(", ") : undefined + } + const raw = props.model.modalities?.input + if (!raw) return + const entries = raw.map((value) => inputLabel(value)) + return entries.length ? entries.join(", ") : undefined + } + const reasoning = () => { + if (props.model.capabilities) + return props.model.capabilities.reasoning + ? language.t("model.tooltip.reasoning.allowed") + : language.t("model.tooltip.reasoning.none") + return props.model.reasoning + ? language.t("model.tooltip.reasoning.allowed") + : language.t("model.tooltip.reasoning.none") + } + const context = () => language.t("model.tooltip.context", { limit: props.model.limit.context.toLocaleString() }) + const contextLimit = () => props.model.limit.context.toLocaleString(language.intl()) + const tools = () => { + const value = props.meta?.tools + if (value === undefined) return undefined + return value === "yes" ? language.t("model.tooltip.tools.yes") : language.t("model.tooltip.tools.no") + } + + if (props.v2) { + return ( +
+ + + + {(value) => } + + + + + {(meta) => } + + + {(value) => } + + + {(value) => } + +
+ ) + } + + return ( +
+
{title()}
+ + {(value) => ( +
+ {language.t("model.tooltip.allows", { inputs: value() })} +
+ )} + +
{reasoning()}
+
{context()}
+ + {(meta) => ( +
+ {language.t("model.tooltip.price.line", { price: meta().price })} +
+ )} + + + {(value) => ( +
+ {language.t("model.tooltip.tools.line", { tools: value() })} +
+ )} + + + {(value) => ( +
+ {language.t("model.tooltip.latency.line", { latency: value() })} +
+ )} + +
+ ) +} diff --git a/packages/app-bundle/overlay/packages/app/src/context/models-favorites.test.ts b/packages/app-bundle/overlay/packages/app/src/context/models-favorites.test.ts new file mode 100644 index 000000000..74df5bf22 --- /dev/null +++ b/packages/app-bundle/overlay/packages/app/src/context/models-favorites.test.ts @@ -0,0 +1,45 @@ +// amicode#1987 (#1979 S4): favorites persistence — the pure seam behind +// models.tsx's `favorite` (the persisted `user[]` entry's `favorite` flag). +import { describe, expect, test } from "bun:test" +import { favoritesOf, isFavorite, toggleFavorite, type UserModelEntry } from "./models-favorites" + +const a = { providerID: "a", modelID: "x" } +const b = { providerID: "b", modelID: "y" } + +describe("models favorites", () => { + test("toggle on adds a favorite (newest first) without inventing a visibility", () => { + const one = toggleFavorite([], a) + expect(one).toEqual([{ ...a, favorite: true }]) + const two = toggleFavorite(one, b) + expect(favoritesOf(two)).toEqual([b, a]) + }) + + test("toggle on an existing visibility entry keeps its visibility", () => { + const users: UserModelEntry[] = [{ ...a, visibility: "hide" }, { ...b, visibility: "show" }] + const next = toggleFavorite(users, b) + expect(next[0]).toEqual({ ...b, visibility: "show", favorite: true }) + expect(next).toContainEqual({ ...a, visibility: "hide" }) + expect(favoritesOf(next)).toEqual([b]) + }) + + test("toggle off clears the flag, keeps the entry's visibility, round-trips", () => { + const users: UserModelEntry[] = [{ ...a, visibility: "show", favorite: true }] + const off = toggleFavorite(users, a) + expect(off).toEqual([{ ...a, visibility: "show", favorite: false }]) + expect(isFavorite(off, a)).toBe(false) + expect(isFavorite(toggleFavorite(off, a), a)).toBe(true) + }) + + test("pre-#1987 persisted entries (no favorite key) read as not-favorite", () => { + const users: UserModelEntry[] = [{ ...a, visibility: "show" }] + expect(favoritesOf(users)).toEqual([]) + expect(isFavorite(users, a)).toBe(false) + }) + + test("the input array is never mutated (store-safe)", () => { + const users: UserModelEntry[] = [{ ...a, visibility: "show" }] + const snapshot = structuredClone(users) + toggleFavorite(users, a) + expect(users).toEqual(snapshot) + }) +}) diff --git a/packages/app-bundle/overlay/packages/app/src/context/models-favorites.ts b/packages/app-bundle/overlay/packages/app/src/context/models-favorites.ts new file mode 100644 index 000000000..e639c0e22 --- /dev/null +++ b/packages/app-bundle/overlay/packages/app/src/context/models-favorites.ts @@ -0,0 +1,32 @@ +// amicode#1987 (#1979 S4): favorites over models.tsx's persisted `user[]` +// entries (the previously-unused `favorite` flag). PURE so the persistence +// rules are unit-tested; models.tsx applies the result with setStore. +// Ported from upstream TUI context/local.tsx toggleFavorite: newest favorite +// first; toggling off keeps the entry (it may carry a visibility choice). +export type ModelKey = { providerID: string; modelID: string } +export type Visibility = "show" | "hide" +/** `visibility` is optional: an entry created by favoriting carries no + * visibility choice (pre-#1987 persisted entries always have one). */ +export type UserModelEntry = ModelKey & { visibility?: Visibility; favorite?: boolean } + +const same = (a: ModelKey, b: ModelKey) => a.providerID === b.providerID && a.modelID === b.modelID + +export function isFavorite(users: readonly UserModelEntry[], model: ModelKey) { + return users.some((x) => same(x, model) && x.favorite === true) +} + +export function favoritesOf(users: readonly UserModelEntry[]): ModelKey[] { + return users.filter((x) => x.favorite === true).map((x) => ({ providerID: x.providerID, modelID: x.modelID })) +} + +export function toggleFavorite(users: readonly UserModelEntry[], model: ModelKey): UserModelEntry[] { + const existing = users.find((x) => same(x, model)) + if (existing?.favorite === true) return users.map((x) => (same(x, model) ? { ...x, favorite: false } : x)) + const entry: UserModelEntry = { + providerID: model.providerID, + modelID: model.modelID, + ...(existing?.visibility ? { visibility: existing.visibility } : {}), + favorite: true, + } + return [entry, ...users.filter((x) => !same(x, model))] +} diff --git a/packages/app-bundle/overlay/packages/app/src/context/models.tsx b/packages/app-bundle/overlay/packages/app/src/context/models.tsx index 28234e9e6..952822446 100644 --- a/packages/app-bundle/overlay/packages/app/src/context/models.tsx +++ b/packages/app-bundle/overlay/packages/app/src/context/models.tsx @@ -5,11 +5,12 @@ import { filter, firstBy, flat, groupBy, mapValues, pipe, uniqueBy, values } fro import { createSimpleContext } from "@opencode-ai/ui/context" import { useProviders } from "@/hooks/use-providers" import { Persist, persisted } from "@/utils/persist" +// amicode#1987 (#1979 S4): favorites — the pure seam over `user[].favorite`. +import { favoritesOf, isFavorite, toggleFavorite, type UserModelEntry, type Visibility } from "./models-favorites" export type ModelKey = { providerID: string; modelID: string } -type Visibility = "show" | "hide" -type User = ModelKey & { visibility: Visibility; favorite?: boolean } +type User = UserModelEntry type Store = { user: User[] recent: ModelKey[] @@ -91,7 +92,7 @@ export const { use: useModels, provider: ModelsProvider } = createSimpleContext( const visibility = createMemo(() => { const map = new Map() - for (const item of store.user) map.set(`${item.providerID}:${item.modelID}`, item.visibility) + for (const item of store.user) if (item.visibility) map.set(`${item.providerID}:${item.modelID}`, item.visibility) return map }) @@ -119,6 +120,8 @@ export const { use: useModels, provider: ModelsProvider } = createSimpleContext( const state = visibility().get(key) if (state === "hide") return false if (state === "show") return true + // amicode#1987: a favorite is visible unless explicitly hidden + if (isFavorite(store.user, model)) return true if (latestSet().has(key)) return true const date = release().get(key) if (!date?.isValid) return true @@ -129,6 +132,12 @@ export const { use: useModels, provider: ModelsProvider } = createSimpleContext( update(model, state ? "show" : "hide") } + // amicode#1987: favorites persist on the same `model.v1` store (user[]). + const favorites = createMemo(() => favoritesOf(store.user)) + const toggleFavoriteModel = (model: ModelKey) => { + setStore("user", toggleFavorite(store.user, model)) + } + const push = (model: ModelKey) => { const uniq = uniqueBy([model, ...store.recent], (x) => `${x.providerID}:${x.modelID}`) if (uniq.length > RECENT_LIMIT) uniq.pop() @@ -173,6 +182,11 @@ export const { use: useModels, provider: ModelsProvider } = createSimpleContext( list: () => recentModels()!, push, }, + favorite: { + list: favorites, + has: (model: ModelKey) => isFavorite(store.user, model), + toggle: toggleFavoriteModel, + }, pin: { get: createMemo(() => store.pinned), set: setPinned, diff --git a/packages/app-bundle/overlay/packages/app/src/i18n/en.ts b/packages/app-bundle/overlay/packages/app/src/i18n/en.ts index ecb0e5d0b..0ab1579c3 100644 --- a/packages/app-bundle/overlay/packages/app/src/i18n/en.ts +++ b/packages/app-bundle/overlay/packages/app/src/i18n/en.ts @@ -125,12 +125,57 @@ export const dict = { "dialog.provider.openrouter.note": "Access all supported models from one provider", "dialog.provider.vercel.note": "Unified access to AI models with smart routing", + // amicode#1987 (#1979 S4): the picker's short-list view. + "dialog.model.shortlist.favorites": "Favorites", + "dialog.model.shortlist.recent": "Recent", + "dialog.model.shortlist.other": "Other", + "dialog.model.shortlist.role.default": "Recommended", + "dialog.model.shortlist.role.fast": "Fast", + "dialog.model.shortlist.role.strong": "Strong", + "dialog.model.shortlist.role.openWeight": "Open-weight", + "dialog.model.shortlist.more": "{{count}} more — search to see all", + "dialog.model.favorite.add": "Add to favorites", + "dialog.model.favorite.remove": "Remove from favorites", + "dialog.model.select.title": "Select model", "dialog.model.search.placeholder": "Search models", "dialog.model.empty": "No model results", "dialog.model.manage": "Manage models", "dialog.model.manage.description": "Customize which models appear in the model selector.", "dialog.model.manage.provider.toggle": "Toggle all {{provider}} models", + // amicode#1986 (#1979 S3): the picker's reachability merge. + "dialog.model.reachability.showUnavailable": "Show unavailable ({{count}})", + "dialog.model.reachability.hideUnavailable": "Hide unavailable", + "dialog.model.reachability.recheck": "Re-check", + "dialog.model.reachability.rechecking": "Checking…", + "dialog.model.reachability.tag.unchecked": "unchecked", + "dialog.model.reachability.tag.rateLimited": "rate-limited", + "dialog.model.reachability.tag.selectedDead": "unavailable", + "dialog.model.reachability.reason.denied": "Not allowed for your credentials", + "dialog.model.reachability.reason.unavailable": "Needs provider/marketplace enablement", + "dialog.model.reachability.reason.invalid": "Not a callable model id", + "dialog.model.reachability.reason.rateLimited": "Rate limited — may be slow to respond", + "dialog.model.reachability.reason.unchecked": "Not checked yet", + "dialog.model.reachability.reason.error": "Probe failed", + // amicode#1989 (#1979 S6): the composer's dead-pick warning. + "composer.model.deadPick.warning": "{{model}} is unavailable: {{reason}}", + "composer.model.deadPick.switch": "Switch to {{model}}", + "composer.model.deadPick.noAlternative": "No working model found — pick another model", + "composer.model.deadPick.reason.unknown": "it failed a real request", + + // amicode#1988 (#1979 S5): picker row meta — price / context / tools / measured latency. + "model.meta.free": "free", + "model.meta.cache": "cache", + "model.meta.perMtok": "per Mtok", + "model.meta.noTools": "no tools", + "model.tooltip.price": "Price", + "model.tooltip.price.line": "Price: {{price}}", + "model.tooltip.tools": "Tools", + "model.tooltip.tools.line": "Tools: {{tools}}", + "model.tooltip.tools.yes": "Yes", + "model.tooltip.tools.no": "No", + "model.tooltip.latency": "Latency", + "model.tooltip.latency.line": "Measured latency: {{latency}}", "dialog.model.unpaid.freeModels.title": "Free models provided by Amicode", "dialog.model.unpaid.addMore.title": "Add more models from popular providers", diff --git a/packages/app-bundle/overlay/packages/app/src/pages/new-session/new-session-view.tsx b/packages/app-bundle/overlay/packages/app/src/pages/new-session/new-session-view.tsx index 0eba52a24..d1899bd7e 100644 --- a/packages/app-bundle/overlay/packages/app/src/pages/new-session/new-session-view.tsx +++ b/packages/app-bundle/overlay/packages/app/src/pages/new-session/new-session-view.tsx @@ -8,6 +8,7 @@ import { createStore } from "solid-js/store" import { Portal } from "solid-js/web" import createPresence from "solid-presence" import { PromptInputV2Composer } from "@/components/prompt-input-v2" +import { ModelDeadPickWarning } from "@/components/model-dead-pick-warning" import { PromptGitStatus, PromptWorkspaceSelector } from "@/components/prompt-workspace-selector" import { PromptProjectAddButton, @@ -69,6 +70,8 @@ export function NewSessionView(props: { + {/* amicode#1989: warn before sending on a model already known dead */} +
diff --git a/packages/app-bundle/overlay/packages/app/src/pages/session/composer/session-composer-region-controller.ts b/packages/app-bundle/overlay/packages/app/src/pages/session/composer/session-composer-region-controller.ts index 74a34a646..50b1f0474 100644 --- a/packages/app-bundle/overlay/packages/app/src/pages/session/composer/session-composer-region-controller.ts +++ b/packages/app-bundle/overlay/packages/app/src/pages/session/composer/session-composer-region-controller.ts @@ -162,6 +162,7 @@ export function createSessionComposerRegionController(input: { return { state: input.state, + sessionID: input.sessionID, centered: input.centered, todo: input.todo, followup: input.followup, diff --git a/packages/app-bundle/overlay/packages/app/src/pages/session/composer/session-composer-region.tsx b/packages/app-bundle/overlay/packages/app/src/pages/session/composer/session-composer-region.tsx index 41a385c07..60d37c997 100644 --- a/packages/app-bundle/overlay/packages/app/src/pages/session/composer/session-composer-region.tsx +++ b/packages/app-bundle/overlay/packages/app/src/pages/session/composer/session-composer-region.tsx @@ -9,6 +9,7 @@ import { SessionTourSpotlight, isTourRequest } from "@/pages/session/composer/se import { SessionFollowupDock } from "@/pages/session/composer/session-followup-dock" import { SessionRevertDock } from "@/pages/session/composer/session-revert-dock" import { SessionTodoDock } from "@/pages/session/composer/session-todo-dock" +import { ModelDeadPickWarning } from "@/components/model-dead-pick-warning" import type { SessionComposerRegionController } from "./session-composer-region-controller" export function SessionComposerRegion(props: { @@ -165,6 +166,8 @@ export function SessionComposerRegion(props: { !controller.state.blocked() || isTourRequest(controller.state.questionRequest()) } > + {/* amicode#1989: named dead-pick warning + one-click switch */} + {props.promptInput} } diff --git a/packages/app-bundle/overlay/packages/opencode/src/provider/error.ts b/packages/app-bundle/overlay/packages/opencode/src/provider/error.ts index 592590277..f978888c2 100644 --- a/packages/app-bundle/overlay/packages/opencode/src/provider/error.ts +++ b/packages/app-bundle/overlay/packages/opencode/src/provider/error.ts @@ -204,4 +204,93 @@ export function parseAPICallError(input: { providerID: ProviderV2.ID; error: API } } +// #1984: reachability-probe classifier. A one-token probe against a catalog +// model fails in a handful of recognisable ways; each maps to a named status so +// the picker can hide dead models with a reason instead of failing mid-turn. +// Shapes are the ones observed on Bedrock (2026-10-28), but the patterns are +// provider-neutral where possible. +export type ProbeStatus = "ok" | "denied" | "unavailable" | "invalid" | "rate-limited" | "error" + +export type ProbeClassification = { + status: Exclude + message: string +} + +const UNAVAILABLE = + /not available for this account|aws[- ]marketplace|marketplace (subscription|agreement)|model access is denied|access to (this|the) model is not|not subscribed/i +const RATE_LIMITED = /throttl|too many requests|rate[- ]?limit|quota exceeded|tokens per (minute|day)/i +const INVALID = + /validationexception|inference profile|on-demand throughput|invalid model|model identifier is invalid|model[_ ]not[_ ]found|unknown model|no such model|does not exist|is not supported|resourcenotfoundexception/i +const DENIED = /accessdenied|access denied|not authorized|unauthorized|unrecognizedclient|forbidden|permission denied|invalid api key|invalid_api_key|security token/i + +// #1994: OpenAI-style models (e.g. on Bedrock) reject the one-token probe with +// "Invalid 'max_output_tokens': integer below minimum value. Expected a value +// >= 16 ..." — sometimes wrapped in "The model returned the following errors:" +// with a ValidationException type. That is a request-shape problem, not a dead +// model: the probe retries at the floor, and the matcher runs before INVALID so +// it can never classify as `invalid`. +const TOKEN_FLOOR = + /max[_ ]?(output|completion)?[_ ]?tokens'?"?\s*:?\s*(integer )?(is )?below (the )?minimum|below minimum value[^\n]*max[_ ]?(output|completion)?[_ ]?tokens/i +const TOKEN_FLOOR_VALUE = /expected a value\s*>=?\s*(\d+)/i +export const PROBE_TOKEN_FLOOR_DEFAULT = 16 +export const PROBE_TOKEN_FLOOR_CAP = 64 + +function probeErrorText(error: unknown) { + if (APICallError.isInstance(error)) return [error.message, error.responseBody ?? ""].join("\n") + if (error instanceof Error) return error.message + return String(error) +} + +/** + * When a probe failure is a max-output-token minimum error, the token count to + * retry with: the floor parsed from the message (default 16), capped at 64. + * `undefined` for any other error. + */ +export function probeTokenFloor(error: unknown): number | undefined { + const text = probeErrorText(error) + if (!TOKEN_FLOOR.test(text)) return undefined + const parsed = Number(TOKEN_FLOOR_VALUE.exec(text)?.[1]) + const floor = Number.isFinite(parsed) && parsed > 0 ? parsed : PROBE_TOKEN_FLOOR_DEFAULT + return Math.min(floor, PROBE_TOKEN_FLOOR_CAP) +} + +function tokenFloorClassification(message: string): ProbeClassification { + return { status: "error", message: `probe rejected by a max_output_tokens floor: ${message}` } +} + +function headerValue(headers: Record | undefined, name: string) { + if (!headers) return undefined + const key = Object.keys(headers).find((k) => k.toLowerCase() === name) + return key ? headers[key] : undefined +} + +export function classifyProbeError(input: { providerID: ProviderV2.ID; error: unknown }): ProbeClassification { + const error = input.error + if (APICallError.isInstance(error)) { + const parsed = parseAPICallError({ providerID: input.providerID, error }) + const status = error.statusCode + const errorType = headerValue(error.responseHeaders, "x-amzn-errortype") ?? "" + const haystack = [parsed.message, errorType, error.responseBody ?? ""].join("\n") + if (UNAVAILABLE.test(haystack)) return { status: "unavailable", message: parsed.message } + if (status === 429 || RATE_LIMITED.test(haystack)) return { status: "rate-limited", message: parsed.message } + if (TOKEN_FLOOR.test(haystack)) return tokenFloorClassification(parsed.message) + if (status === 404 || INVALID.test(haystack)) return { status: "invalid", message: parsed.message } + if (status === 401 || status === 403 || DENIED.test(haystack)) return { status: "denied", message: parsed.message } + return { status: "error", message: parsed.message } + } + if (error instanceof Error) { + if (error.name === "TimeoutError" || error.name === "AbortError" || error instanceof HeaderTimeoutError) + return { status: "error", message: `probe timed out: ${error.message}` } + if (error.name === "AI_NoSuchModelError" || error.name === "ProviderModelNotFoundError") + return { status: "invalid", message: error.message } + const text = `${error.name}: ${error.message}` + if (UNAVAILABLE.test(text)) return { status: "unavailable", message: error.message } + if (RATE_LIMITED.test(text)) return { status: "rate-limited", message: error.message } + if (TOKEN_FLOOR.test(text)) return tokenFloorClassification(error.message) + if (DENIED.test(text)) return { status: "denied", message: error.message } + return { status: "error", message: error.message } + } + return { status: "error", message: String(error) } +} + export * as ProviderError from "./error" diff --git a/packages/app-bundle/overlay/packages/opencode/src/provider/probe.ts b/packages/app-bundle/overlay/packages/opencode/src/provider/probe.ts new file mode 100644 index 000000000..ff0bd8273 --- /dev/null +++ b/packages/app-bundle/overlay/packages/opencode/src/provider/probe.ts @@ -0,0 +1,171 @@ +// #1984: one-token reachability probe per catalog model. The probe runs +// entirely inside the engine through Provider.getLanguage, so credentials never +// leave the process; the response carries only a named status, latency, a +// redacted reason and a stable non-secret credential fingerprint. +import { Cause, Effect, Schema } from "effect" +import { Hash } from "@opencode-ai/core/util/hash" +import { ProviderV2 } from "@opencode-ai/core/provider" +import { Auth } from "../auth" +import { Env } from "../env" +import { Provider } from "./provider" +import { ProviderError } from "./error" + +export const PROBE_MAX_OUTPUT_TOKENS = 1 +export const PROBE_TIMEOUT_MS = 10_000 +export const PROBE_CONCURRENCY = 4 +export const PROBE_MAX_MODELS = 500 +const REASON_MAX_CHARS = 300 + +export const Status = Schema.Literals(["ok", "denied", "unavailable", "invalid", "rate-limited", "error"]) +export type Status = Schema.Schema.Type + +export const Input = Schema.Struct({ + providerID: ProviderV2.ID, + modelIDs: Schema.Array(Schema.String).check(Schema.isMaxLength(PROBE_MAX_MODELS)), +}).annotate({ identifier: "ProviderProbeInput" }) +export type Input = Schema.Schema.Type + +export const Result = Schema.Struct({ + model: Schema.String, + status: Status, + latency_ms: Schema.Number, + reason: Schema.NullOr(Schema.String), + credential_fp: Schema.NullOr(Schema.String), + checked_at: Schema.Number, +}).annotate({ identifier: "ProviderProbeResult" }) +export type Result = Schema.Schema.Type + +export const Output = Schema.Struct({ + providerID: Schema.String, + results: Schema.Array(Result), +}).annotate({ identifier: "ProviderProbeOutput" }) +export type Output = Schema.Schema.Type + +function strings(input: unknown, out: string[] = []): string[] { + if (typeof input === "string") out.push(input) + else if (Array.isArray(input)) for (const item of input) strings(item, out) + else if (input && typeof input === "object") for (const value of Object.values(input)) strings(value, out) + return out +} + +/** + * Stable, non-secret fingerprint of the credential a provider resolves with: + * the stored auth entry (cf. #1848 authStamp, which stamps the whole + * auth.json), plus the config/env material that selects an identity or + * endpoint. A truncated domain-separated sha256 — it identifies "same + * credential" for the probe cache and cannot be reversed into the secret. + */ +export function credentialFingerprint(input: { + providerID: string + auth: unknown + provider: Pick + env: Record +}) { + const options = input.provider.options ?? {} + const material = { + auth: input.auth ?? null, + key: input.provider.key ?? null, + apiKey: options.apiKey ?? null, + baseURL: options.baseURL ?? null, + profile: options.profile ?? null, + region: options.region ?? null, + env: Object.fromEntries([...input.provider.env].sort().map((name) => [name, input.env[name] ?? null])), + } + return Hash.sha256(`amicode/credential-fp/v1\0${input.providerID}\0${JSON.stringify(material)}`).slice(0, 16) +} + +/** Strip every known secret (and generic bearer/AWS key shapes) from a reason string. */ +export function redact(text: string, secrets: string[]) { + let out = text + for (const secret of [...new Set(secrets)].filter((s) => s.length >= 6).sort((a, b) => b.length - a.length)) + out = out.split(secret).join("[redacted]") + out = out + .replace(/Bearer\s+\S+/gi, "[redacted]") + .replace(/\b(AKIA|ASIA)[A-Z0-9]{12,}\b/g, "[redacted]") + .replace(/\bsk-[A-Za-z0-9_-]{8,}/g, "[redacted]") + return out.length > REASON_MAX_CHARS ? `${out.slice(0, REASON_MAX_CHARS)}…` : out +} + +export const run = Effect.fn("ProviderProbe.run")(function* (input: Input) { + const provider = yield* Provider.Service + const auth = yield* Auth.Service + const env = yield* Env.Service + + const modelIDs = [...new Set(input.modelIDs)] + const info = yield* provider.getProvider(input.providerID) + const now = () => Date.now() + if (!info) { + const checked_at = now() + return { + providerID: input.providerID, + results: modelIDs.map((model) => ({ + model, + status: "error" as const, + latency_ms: 0, + reason: "provider not connected", + credential_fp: null, + checked_at, + })), + } satisfies Output + } + + const entry = yield* auth.get(input.providerID).pipe(Effect.orElseSucceed(() => undefined)) + const envs = yield* env.all() + const credential_fp = credentialFingerprint({ providerID: input.providerID, auth: entry, provider: info, env: envs }) + const secrets = [ + ...strings(entry), + ...strings(info.key), + ...strings(info.options?.apiKey), + ...strings(info.options?.headers), + ...info.env.map((name) => envs[name]).filter((v): v is string => typeof v === "string"), + ] + + const probeOne = Effect.fn("ProviderProbe.one")(function* (modelID: string) { + const started = now() + const done = (status: Status, reason: string | null): Result => ({ + model: modelID, + status, + latency_ms: now() - started, + reason: reason === null ? null : redact(reason, secrets), + credential_fp, + checked_at: now(), + }) + const model = info.models[modelID] + if (!model) return done("invalid", "model not listed by provider") + const attempt = (maxOutputTokens: number) => + provider.getLanguage(model).pipe( + Effect.flatMap((language) => + Effect.tryPromise({ + try: () => + language.doGenerate({ + prompt: [{ role: "user", content: [{ type: "text", text: "ping" }] }], + maxOutputTokens, + headers: model.headers, + abortSignal: AbortSignal.timeout(PROBE_TIMEOUT_MS), + }), + catch: (error) => error, + }), + ), + Effect.as(undefined), + Effect.catchCause((cause) => Effect.succeed({ error: Cause.squash(cause) })), + ) + const first = yield* attempt(PROBE_MAX_OUTPUT_TOKENS) + if (!first) return done("ok", null) + // #1994: a max_output_tokens floor is a request-shape rejection, not a dead + // model — retry exactly once at the (parsed, capped) floor and classify that. + const floor = ProviderError.probeTokenFloor(first.error) + const outcome = floor !== undefined && floor > PROBE_MAX_OUTPUT_TOKENS ? yield* attempt(floor) : first + if (!outcome) return done("ok", null) + const classified = ProviderError.classifyProbeError({ providerID: model.providerID, error: outcome.error }) + const retriedIntoFloor = outcome !== first && ProviderError.probeTokenFloor(outcome.error) !== undefined + return done( + classified.status, + retriedIntoFloor ? `retried at ${floor} output tokens; ${classified.message}` : classified.message, + ) + }) + + const results = yield* Effect.forEach(modelIDs, probeOne, { concurrency: PROBE_CONCURRENCY }) + return { providerID: input.providerID, results } satisfies Output +}) + +export * as ProviderProbe from "./probe" diff --git a/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/groups/provider.ts b/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/groups/provider.ts index 7e572b799..3ed314a1e 100644 --- a/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/groups/provider.ts +++ b/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/groups/provider.ts @@ -1,5 +1,6 @@ import { ProviderAuth } from "@/provider/auth" import { Provider } from "@/provider/provider" +import { ProviderProbe } from "@/provider/probe" import { Schema } from "effect" import { HttpApi, HttpApiEndpoint, HttpApiGroup, OpenApi } from "effect/unstable/httpapi" @@ -56,6 +57,19 @@ export const ProviderApi = HttpApi.make("provider") description: "Retrieve available authentication methods for all AI providers.", }), ), + // #1984: one-token reachability probe per model, credentials stay in the engine. + HttpApiEndpoint.post("probe", `${root}/probe`, { + query: WorkspaceRoutingQuery, + payload: ProviderProbe.Input, + success: described(ProviderProbe.Output, "Per-model reachability results"), + }).annotateMerge( + OpenApi.annotations({ + identifier: "provider.probe", + summary: "Probe model reachability", + description: + "Send a one-token request to each listed model of a provider through the engine's own credentials and return a named status (ok, denied, unavailable, invalid, rate-limited, error) per model. No secret material is returned.", + }), + ), HttpApiEndpoint.post("authorize", `${root}/:providerID/oauth/authorize`, { params: { providerID: ProviderV2.ID }, query: WorkspaceRoutingQuery, diff --git a/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/provider.ts b/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/provider.ts index 9c0c7ed89..ecb04efd6 100644 --- a/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/provider.ts +++ b/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/provider.ts @@ -2,6 +2,7 @@ import { ProviderAuth } from "@/provider/auth" import { Config } from "@/config/config" import { ModelsDev } from "@opencode-ai/core/models-dev" import { Provider } from "@/provider/provider" +import { ProviderProbe } from "@/provider/probe" import { mapValues } from "remeda" import { Effect, Schema } from "effect" @@ -65,6 +66,10 @@ export const providerHandlers = HttpApiBuilder.group(InstanceHttpApi, "provider" return yield* svc.methods() }) + const probe = Effect.fn("ProviderHttpApi.probe")(function* (ctx: { payload: ProviderProbe.Input }) { + return yield* ProviderProbe.run(ctx.payload) + }) + const authorize = Effect.fn("ProviderHttpApi.authorize")(function* (ctx: { params: { providerID: ProviderV2.ID } payload: ProviderAuth.AuthorizeInput @@ -110,6 +115,7 @@ export const providerHandlers = HttpApiBuilder.group(InstanceHttpApi, "provider" return handlers .handle("list", list) .handle("auth", auth) + .handle("probe", probe) .handleRaw("authorize", authorizeRaw) .handle("callback", callback) }), diff --git a/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-provider.test.ts b/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-provider.test.ts index 80dff80d8..f731f518b 100644 --- a/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-provider.test.ts +++ b/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-provider.test.ts @@ -1,4 +1,4 @@ -import { describe, expect } from "bun:test" +import { afterAll, describe, expect } from "bun:test" import { LayerNode } from "@opencode-ai/core/effect/layer-node" import { FSUtil } from "@opencode-ai/core/fs-util" import { Effect, Layer } from "effect" @@ -452,3 +452,295 @@ describe("provider HttpApi", () => { { ...projectOptions, init: writeProviderModelsMutationPlugin }, ) }) + +// #1984 — POST /provider/probe: one-token reachability probe with named +// statuses. A fake OpenAI-compatible upstream answers per model id with the +// error shapes Bedrock returns (AccessDenied, Marketplace "not available for +// this account", ValidationException on a profile-only id, throttling), so the +// engine's classifier is exercised end-to-end through Provider.getLanguage. +const PROBE_SECRET = "sk-probe-SECRET-0123456789abcdef" +const probeSeen: { model?: string; authorization?: string | null; maxTokens?: unknown }[] = [] + +function openAiError(status: number, message: string, type: string, headers: Record = {}) { + return new Response(JSON.stringify({ error: { message, type, code: type } }), { + status, + headers: { "content-type": "application/json", ...headers }, + }) +} + +function tokenFloorMessage(floor: number, got: number) { + return `Invalid 'max_output_tokens': integer below minimum value. Expected a value >= ${floor}, but got ${got} instead.` +} + +function probeOk(model: string) { + return Response.json({ + id: "chatcmpl-probe", + object: "chat.completion", + created: 1, + model, + choices: [{ index: 0, message: { role: "assistant", content: "p" }, finish_reason: "length" }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }) +} + +const probeUpstream = Bun.serve({ + port: 0, + async fetch(req) { + const body = (await req.json().catch(() => ({}))) as Record + const model = typeof body.model === "string" ? body.model : undefined + probeSeen.push({ + model, + authorization: req.headers.get("authorization"), + maxTokens: body.max_tokens ?? body.max_completion_tokens, + }) + switch (model) { + case "probe-ok": + return Response.json({ + id: "chatcmpl-probe", + object: "chat.completion", + created: 1, + model, + choices: [{ index: 0, message: { role: "assistant", content: "p" }, finish_reason: "length" }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }) + case "probe-denied": + return openAiError( + 403, + `User: arn:aws:iam::123456789012:user/probe is not authorized to perform: bedrock:InvokeModel (key ${PROBE_SECRET})`, + "AccessDeniedException", + { "x-amzn-errortype": "AccessDeniedException" }, + ) + case "probe-unavailable": + return openAiError(403, "This model version is not available for this account.", "AccessDeniedException") + case "probe-invalid": + return openAiError( + 400, + "Invocation of model ID probe-invalid with on-demand throughput isn't supported. Retry your request with the ID or ARN of an inference profile that contains this model.", + "ValidationException", + ) + case "probe-throttled": + return openAiError(429, "Too many requests, please wait before trying again.", "ThrottlingException") + // #1994: OpenAI-style models on Bedrock enforce a max_output_tokens floor + // (typically 16); the shapes below are the live messages from 2026-10-28. + case "probe-floor": { + const max = Number(body.max_tokens ?? body.max_completion_tokens) + if (max >= 16) return probeOk(model) + return openAiError(400, tokenFloorMessage(16, max), "invalid_request_error") + } + case "probe-floor-wrapped": { + const max = Number(body.max_tokens ?? body.max_completion_tokens) + if (max >= 16) return probeOk(model) + return openAiError( + 400, + `The model returned the following errors: ${tokenFloorMessage(16, max)}`, + "ValidationException", + ) + } + case "probe-floor-throttled": { + const max = Number(body.max_tokens ?? body.max_completion_tokens) + if (max >= 32) return openAiError(429, "Too many requests, please wait before trying again.", "ThrottlingException") + return openAiError(400, tokenFloorMessage(32, max), "invalid_request_error") + } + case "probe-floor-stuck": { + // a floor above the probe's cap: the retry (at the cap) is rejected too + const max = Number(body.max_tokens ?? body.max_completion_tokens) + return openAiError( + 400, + `The model returned the following errors: ${tokenFloorMessage(100, max)}`, + "ValidationException", + ) + } + default: + return openAiError(500, "unexpected", "InternalServerException") + } + }, +}) +afterAll(() => probeUpstream.stop(true)) + +function probeModel(id: string) { + return { + id, + name: id, + attachment: false, + reasoning: false, + temperature: false, + tool_call: true, + release_date: "2025-01-01", + limit: { context: 100_000, output: 10_000 }, + cost: { input: 0, output: 0 }, + options: {}, + } +} + +const probeOptions = { + config: () => ({ + formatter: false as const, + lsp: false as const, + provider: { + "probe-fake": { + name: "Probe Fake", + id: "probe-fake", + env: [], + npm: "@ai-sdk/openai-compatible", + models: Object.fromEntries( + [ + "probe-ok", + "probe-denied", + "probe-unavailable", + "probe-invalid", + "probe-throttled", + "probe-floor", + "probe-floor-wrapped", + "probe-floor-throttled", + "probe-floor-stuck", + ].map((id) => [ + id, + probeModel(id), + ]), + ), + options: { apiKey: PROBE_SECRET, baseURL: probeUpstream.url.href.replace(/\/$/, "") }, + }, + }, + }), +} + +describe("provider probe HttpApi (#1984)", () => { + it.instance( + "classifies ok, denied, unavailable, invalid and rate-limited models with no secret material", + Effect.gen(function* () { + probeSeen.length = 0 + const directory = (yield* TestInstance).directory + const response = yield* request("/provider/probe", { + method: "POST", + headers: { "x-opencode-directory": directory, "content-type": "application/json" }, + body: JSON.stringify({ + providerID: "probe-fake", + modelIDs: ["probe-ok", "probe-denied", "probe-unavailable", "probe-invalid", "probe-throttled", "nope"], + }), + }) + const text = yield* response.text + expect(response.status).toBe(200) + const body = JSON.parse(text) + expect(body.providerID).toBe("probe-fake") + type ProbeRow = { + model: string + status: string + latency_ms: number + reason: string | null + credential_fp: string | null + } + const byModel: Record = Object.fromEntries( + (body.results as ProbeRow[]).map((r) => [r.model, r]), + ) + expect(byModel["probe-ok"].status).toBe("ok") + expect(byModel["probe-ok"].reason).toBeNull() + expect(byModel["probe-denied"].status).toBe("denied") + expect(byModel["probe-unavailable"].status).toBe("unavailable") + expect(byModel["probe-invalid"].status).toBe("invalid") + expect(byModel["probe-throttled"].status).toBe("rate-limited") + // a model id the provider does not list never reaches the network + expect(byModel["nope"].status).toBe("invalid") + for (const result of Object.values(byModel)) { + expect(typeof result.latency_ms).toBe("number") + expect(result.credential_fp).toMatch(/^[0-9a-f]{16}$/) + } + // one stable fingerprint per credential + expect(new Set(Object.values(byModel).map((r) => r.credential_fp)).size).toBe(1) + + // the probe really went through the configured credential... + expect(probeSeen.length).toBe(5) + expect(probeSeen.every((seen) => seen.authorization === `Bearer ${PROBE_SECRET}`)).toBe(true) + // ...asked for a single output token... + expect(probeSeen.every((seen) => seen.maxTokens === 1)).toBe(true) + // ...and no secret material comes back, even when the upstream echoes it + expect(text).not.toContain(PROBE_SECRET) + expect(text).not.toContain("SECRET") + expect(text).not.toContain("Bearer") + }), + probeOptions, + 30000, + ) + + it.instance( + "retries once at a max_output_tokens floor instead of misclassifying the model (#1994)", + Effect.gen(function* () { + probeSeen.length = 0 + const directory = (yield* TestInstance).directory + const response = yield* request("/provider/probe", { + method: "POST", + headers: { "x-opencode-directory": directory, "content-type": "application/json" }, + body: JSON.stringify({ + providerID: "probe-fake", + modelIDs: ["probe-floor", "probe-floor-wrapped", "probe-floor-throttled", "probe-floor-stuck"], + }), + }) + expect(response.status).toBe(200) + const body = JSON.parse(yield* response.text) + type ProbeRow = { model: string; status: string; reason: string | null } + const byModel: Record = Object.fromEntries( + (body.results as ProbeRow[]).map((r) => [r.model, r]), + ) + const sent = (model: string) => probeSeen.filter((seen) => seen.model === model).map((seen) => seen.maxTokens) + + // rejects 1 with the minimum error, accepts 16 → ok after exactly one retry at the parsed floor + expect(byModel["probe-floor"]).toMatchObject({ status: "ok", reason: null }) + expect(sent("probe-floor")).toEqual([1, 16]) + + // the Bedrock "model returned the following errors" wrapper triggers the same retry + expect(byModel["probe-floor-wrapped"]).toMatchObject({ status: "ok", reason: null }) + expect(sent("probe-floor-wrapped")).toEqual([1, 16]) + + // both attempts rejected → the second attempt's classification + expect(byModel["probe-floor-throttled"].status).toBe("rate-limited") + expect(sent("probe-floor-throttled")).toEqual([1, 32]) + + // floor above the cap: one retry at the cap (64); the floor error is never `invalid` + expect(sent("probe-floor-stuck")).toEqual([1, 64]) + expect(byModel["probe-floor-stuck"].status).toBe("error") + expect(byModel["probe-floor-stuck"].reason).toMatch(/max_output_tokens/) + expect(byModel["probe-floor-stuck"].reason).toMatch(/64/) + }), + probeOptions, + 30000, + ) + + it.instance( + "keeps the credential fingerprint stable across probes", + Effect.gen(function* () { + const directory = (yield* TestInstance).directory + const probe = () => + Effect.gen(function* () { + const response = yield* request("/provider/probe", { + method: "POST", + headers: { "x-opencode-directory": directory, "content-type": "application/json" }, + body: JSON.stringify({ providerID: "probe-fake", modelIDs: ["probe-ok"] }), + }) + return JSON.parse(yield* response.text).results[0].credential_fp as string + }) + const first = yield* probe() + const second = yield* probe() + expect(first).toBe(second) + expect(first).not.toContain(PROBE_SECRET.slice(0, 8)) + }), + probeOptions, + 30000, + ) + + it.instance( + "reports models of an unconnected provider as errors without probing", + Effect.gen(function* () { + const directory = (yield* TestInstance).directory + const response = yield* request("/provider/probe", { + method: "POST", + headers: { "x-opencode-directory": directory, "content-type": "application/json" }, + body: JSON.stringify({ providerID: "not-connected", modelIDs: ["x"] }), + }) + expect(response.status).toBe(200) + const body = JSON.parse(yield* response.text) + expect(body.results).toHaveLength(1) + expect(body.results[0]).toMatchObject({ model: "x", status: "error", credential_fp: null }) + }), + projectOptions, + 30000, + ) +}) diff --git a/packages/extension/src/amicode_service/index.ts b/packages/extension/src/amicode_service/index.ts index 740610b82..f0f55048c 100644 --- a/packages/extension/src/amicode_service/index.ts +++ b/packages/extension/src/amicode_service/index.ts @@ -102,6 +102,14 @@ import { resetModelRoutingResponse, type ModelRoutingDeps, } from "./model_routing"; +import { + ModelReachability, + modelReachabilityResponse, + refreshModelReachabilityResponse, + demoteModelReachabilityResponse, + type ModelReachabilityDeps, +} from "./model_reachability"; +import { modelPickerConfigResponse } from "./model_picker_config"; export function registerProfileRoutes(server: AmicodeServiceServer): AmicodeServiceServer { server.add("GET", "/amicode/profile", () => ({ body: profileResponse() })); @@ -426,6 +434,43 @@ export function registerModelRoutingRoutes( return server; } +// Model-reachability routes (#1985, #1979 S2): the per-(provider, model) +// reachability cache over the engine's one-token probe (S1, #1984). GET — +// the rows ("unchecked" for anything the cache cannot vouch for, never ok) + +// per-provider probe metadata; POST /refresh — the on-demand probe, +// rate-limited per provider; POST /demote — a runtime demotion that sticks +// until the next probe. ALWAYS registered (no engine deps → GET serves the +// cache, refresh answers engine_unavailable); the scheduler (on-connect + +// daily, rate-limited, shared across hub shards through the locked cache +// file) is armed only when both engine seams are wired, and stops with the +// service. Fleet clients proxy /amicode/* to the host, so one hub cache +// serves every machine. +export function registerModelReachabilityRoutes( + server: AmicodeServiceServer, + deps: ModelReachabilityDeps = {}, +): AmicodeServiceServer { + const mr = new ModelReachability(deps); + server.add("GET", "/amicode/model-reachability", async () => { + const body = await modelReachabilityResponse(mr); + // the GET is also an on-connect trigger: a newly connected provider is + // probed in the background (the claim + floor still apply) + if (mr.canSchedule) void mr.tick(); + return { body }; + }); + server.add("POST", "/amicode/model-reachability/refresh", async ({ body }) => ({ + body: await refreshModelReachabilityResponse(body, mr), + })); + server.add("POST", "/amicode/model-reachability/demote", async ({ body }) => ({ + body: await demoteModelReachabilityResponse(body, mr), + })); + server.registerCleanup(mr.startScheduler()); + // #1987 (#1979 S4): the picker's deployment config (model.recommended / + // model.allowlist) rides beside the reachability family — see + // model_picker_config.ts for why this is the path (not the engine /config). + server.add("GET", "/amicode/model-picker", () => ({ body: modelPickerConfigResponse() })); + return server; +} + // ── #391: the fleet routes — STAGED SURFACES ──────────────────────────────── // The FLEET-DATA handlers are registered ONLY by the entitlement-staged path // below (createAmicodeService's fleet option → stageFleetDataPlane / @@ -938,6 +983,10 @@ export function createAmicodeService( * credential gate's refresh loop). Absent → the surface renders empty * roles (no cards known) with providers from the stored snapshot. */ modelRouting?: ModelRoutingDeps; + /** #1985: the model-reachability cache's engine seams (the key-free + * catalog + the S1 probe) and tuning. Absent → the routes still answer + * (GET serves the cache; refresh is engine_unavailable; no scheduler). */ + modelReachability?: ModelReachabilityDeps; /** #1771: the service's diagnostic log (the hub runner's stdout → * /tmp/amico-hub.log; the window's Amicode output channel). The fan-in * census and the owner-map refresh outcomes write here. Absent → silent. */ @@ -1551,5 +1600,6 @@ export function createAmicodeService( server.add("POST", PEER_REVOKE_PATH, peerRevokeHandler()); registerPostureRoutes(server); registerModelRoutingRoutes(server, opts.modelRouting); + registerModelReachabilityRoutes(server, opts.modelReachability); return server; } diff --git a/packages/extension/src/amicode_service/model_picker_config.ts b/packages/extension/src/amicode_service/model_picker_config.ts new file mode 100644 index 000000000..a66ac9142 --- /dev/null +++ b/packages/extension/src/amicode_service/model_picker_config.ts @@ -0,0 +1,75 @@ +// model_picker_config.ts (service) — #1987 (#1979 S4): the model picker's +// deployment config, served to the overlay picker. +// +// GET /amicode/model-picker → {ok, error, model: {recommended, allowlist?}} +// +// SOURCE: $AMICODE_OPS_DIR/model-picker.json, shape +// { "model": { "recommended": [{ "model": "provider/id", "role": "default" | "fast" | "strong" | "open-weight" }], +// "allowlist": ["provider/id" | "provider/*"] } } +// +// WHY THIS PATH (not OPENCODE_CONFIG_CONTENT → the engine's /config): the +// engine's config schema types `model` as a string ("provider/model") and its +// decode drops unknown keys (ConfigParse.schema, onExcessProperty: "ignore"), +// so `model.recommended` / `model.allowlist` cannot ride the engine config +// without patching the upstream schema. The amicode service is the overlay's +// existing side channel (amicodeGet, per-server auth), it is booted on both +// paths (extension + headless hub runner) through createAmicodeService, and +// fleet clients proxy /amicode/* to the host — one deployment file serves +// every machine. Registered beside the S2 reachability family. +// +// Same service discipline as the sibling families: tolerant reads, bad +// entries dropped (never guessed), FIXED error strings. A missing file is not +// an error: no recommended, no allowlist — nothing is invented. +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { amicodeOpsDir } from "./connections"; + +export const MODEL_ROLES = ["default", "fast", "strong", "open-weight"] as const; +export type ModelRole = (typeof MODEL_ROLES)[number]; + +export interface ModelPickerConfig { + recommended: Array<{ model: string; role: ModelRole }>; + allowlist?: string[]; +} + +export function modelPickerConfigFile(): string { + return path.join(amicodeOpsDir(), "model-picker.json"); +} + +const isObj = (v: unknown): v is Record => typeof v === "object" && v !== null && !Array.isArray(v); + +/** "provider/id" or "provider/*": a non-empty provider before the first slash + * and a non-empty remainder. */ +const isModelRef = (v: unknown): v is string => { + if (typeof v !== "string") return false; + const slash = v.indexOf("/"); + return slash > 0 && slash < v.length - 1; +}; + +export function readModelPickerConfig(raw: unknown): ModelPickerConfig { + const model = isObj(raw) && isObj(raw.model) ? raw.model : {}; + const recommended = (Array.isArray(model.recommended) ? model.recommended : []).flatMap((e) => + isObj(e) && isModelRef(e.model) && MODEL_ROLES.includes(e.role as ModelRole) + ? [{ model: e.model, role: e.role as ModelRole }] + : [], + ); + if (!Array.isArray(model.allowlist)) return { recommended }; + return { recommended, allowlist: model.allowlist.filter(isModelRef) }; +} + +/** GET /amicode/model-picker. Never throws. */ +export function modelPickerConfigResponse(file: string = modelPickerConfigFile()): string { + let text: string; + try { + text = readFileSync(file, "utf8"); + } catch (e) { + if ((e as NodeJS.ErrnoException).code === "ENOENT") + return JSON.stringify({ ok: true, error: null, model: { recommended: [] } }); + return JSON.stringify({ ok: false, error: "config_unreadable: model-picker.json could not be read" }); + } + try { + return JSON.stringify({ ok: true, error: null, model: readModelPickerConfig(JSON.parse(text)) }); + } catch { + return JSON.stringify({ ok: false, error: "config_invalid: model-picker.json is not valid JSON" }); + } +} diff --git a/packages/extension/src/amicode_service/model_reachability.ts b/packages/extension/src/amicode_service/model_reachability.ts new file mode 100644 index 000000000..25de68425 --- /dev/null +++ b/packages/extension/src/amicode_service/model_reachability.ts @@ -0,0 +1,675 @@ +// model_reachability.ts (service) — #1985 (#1979 S2): the model-reachability +// cache, its rate-limited scheduler, and the route family. +// +// GET /amicode/model-reachability → per-(provider, model) rows + +// per-provider probe metadata +// POST /amicode/model-reachability/refresh → {provider?} — on-demand probe, +// rate-limited per provider +// POST /amicode/model-reachability/demote → {provider, model, status, +// reason} — a runtime demotion +// that sticks until the next probe +// +// THE PROBE is the engine's (S1, #1984: POST /provider/probe — one token per +// model through the engine's own credentials; only a named status, latency, a +// redacted reason and a non-secret credential fingerprint cross). This module +// decides WHEN to call it and remembers what it said. +// +// THE CACHE is one JSON file under AMICODE_OPS_DIR (the routingSnapshotFile +// convention), SHARED by every hub shard on the machine: every write is a +// read-modify-write under a mkdir lock (atomic EEXIST on contention — the +// distill_queue idiom; a dead owner's lock is reclaimed after the stale +// window) and lands by tmp+rename, so readers never see a torn file and two +// writers never lose each other's update. The rate-limit CLAIM is taken under +// the same lock, so N shards probing on the same schedule probe once. +// +// SEMANTICS (the issue's binding rules): +// - Keyed {provider, model, credential_fp}: a probe that returns a NEW +// fingerprint for a provider drops every row cached under the old one, and +// a row whose fp disagrees with its provider's current fp is served as +// "unchecked". A credential change never inherits the old verdicts. +// - Unknown is "unchecked", NEVER ok: a catalog model without a valid cached +// row is reported unchecked; with no catalog and no cache nothing is +// invented. +// - A demotion sticks until the next probe of that model replaces it. +// - Rate limit: every trigger (scheduler, on-connect, on-demand) respects a +// per-provider floor between probe ATTEMPTS (a failed probe counts — never +// hammer a dead engine). The scheduler additionally probes only a provider +// that was never probed (on-connect) or whose last probe is a day old. +// +// Same service discipline as the posture / model-routing families: one success +// shape per route, ok:false + "code: detail" with FIXED strings (nothing the +// caller sent is echoed), tolerant reads that fail safe. +import { randomBytes } from "node:crypto"; +import { existsSync, readFileSync } from "node:fs"; +import { mkdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { fetchProviderModels, probeProviderModels } from "../llm_creds.mjs"; + +export type ProbeStatus = "ok" | "denied" | "unavailable" | "invalid" | "rate-limited" | "error"; +export type ReachabilityStatus = ProbeStatus | "unchecked"; + +const PROBE_STATUSES: readonly ProbeStatus[] = ["ok", "denied", "unavailable", "invalid", "rate-limited", "error"]; +/** A demotion names a FAILURE — "ok" is only ever a probe's word. */ +const DEMOTE_STATUSES: readonly ProbeStatus[] = ["denied", "unavailable", "invalid", "rate-limited", "error"]; + +/** One row of the engine's POST /provider/probe response (#1984). */ +export interface ProbeResult { + model: string; + status: ProbeStatus; + latency_ms: number; + reason: string | null; + credential_fp: string | null; + checked_at: number; +} + +export interface ProbeOutput { + providerID: string; + results: ProbeResult[]; +} + +export interface ModelReachabilityDeps { + /** The cache file. Default: $AMICODE_OPS_DIR/model-reachability.json. */ + file?: string; + /** Clock (epoch ms). */ + now?: () => number; + /** The live provider → model-id catalog (key-free). undefined/throwing = + * unknown — never read as "no providers". */ + listModels?: () => Promise | undefined>; + /** The engine probe (S1). undefined/throwing = the probe failed. */ + probe?: (providerID: string, modelIDs: string[]) => Promise; + /** Floor between probe attempts of one provider, any trigger. Default 5 min. */ + minProbeIntervalMs?: number; + /** The scheduler's re-probe age. Default 24 h. */ + dailyIntervalMs?: number; + /** The scheduler's tick period (the route registration arms it). 0 = no + * scheduler (no timer, no GET-kick). Default 15 min. */ + schedulerIntervalMs?: number; + /** The first tick after registration (the boot on-connect pass). Default 2 s. */ + schedulerInitialDelayMs?: number; + /** A lock older than this whose owner pid is dead is reclaimed. Default 10 s. */ + staleLockMs?: number; + /** Give up acquiring the lock after this long (the write fails, the route + * answers named). Default 10 s. */ + lockTimeoutMs?: number; + log?: (line: string) => void; +} + +interface CachedModel { + status: ProbeStatus; + reason: string | null; + latency_ms: number | null; + checked_at: number; + credential_fp: string | null; + source: "probe" | "demoted"; +} + +interface CachedProvider { + credential_fp: string | null; + last_probe_at: number | null; + last_attempt_at: number | null; + last_error: string | null; + models: Record; +} + +interface CacheDoc { + version: 1; + providers: Record; +} + +export interface ReachabilityRow { + provider: string; + model: string; + status: ReachabilityStatus; + reason: string | null; + latency_ms: number | null; + checked_at: number | null; + credential_fp: string | null; + source: "probe" | "demoted" | null; +} + +export interface ReachabilityProviderMeta { + provider: string; + credential_fp: string | null; + last_probe_at: number | null; + last_attempt_at: number | null; + last_error: string | null; + next_probe_at: number; +} + +export interface ReachabilitySnapshot { + ok: true; + models: ReachabilityRow[]; + providers: ReachabilityProviderMeta[]; + catalog_known: boolean; + error: null; +} + +export interface RefreshResult { + ok: boolean; + refreshed: string[]; + rate_limited: Array<{ provider: string; retry_after_ms: number }>; + failed: Array<{ provider: string; error: string }>; + models: ReachabilityRow[]; + error: string | null; +} + +export type DemoteResult = + | { ok: true; provider: string; model: string; status: ProbeStatus; reason: string | null; error: null } + | { ok: false; error: string }; + +const PROBE_BATCH = 500; // the engine's PROBE_MAX_MODELS +const REASON_MAX = 300; +const ID_MAX = 256; + +/** The cache file (the #864 ops-dir convention: $AMICODE_OPS_DIR → ~/.amico), + * beside model-routing-providers.json. */ +export function reachabilityCacheFile(env: NodeJS.ProcessEnv = process.env): string { + const v = env.AMICODE_OPS_DIR; + const dir = typeof v === "string" && v.trim() !== "" ? v : path.join(os.homedir(), ".amico"); + return path.join(dir, "model-reachability.json"); +} + +const emptyDoc = (): CacheDoc => ({ version: 1, providers: {} }); +const emptyProvider = (): CachedProvider => ({ + credential_fp: null, + last_probe_at: null, + last_attempt_at: null, + last_error: null, + models: {}, +}); + +const isObj = (v: unknown): v is Record => typeof v === "object" && v !== null && !Array.isArray(v); +const numOrNull = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null); +const strOrNull = (v: unknown): string | null => (typeof v === "string" ? v : null); + +/** Tolerant parse: anything malformed degrades to "not cached" (→ unchecked), + * never to a fabricated verdict. */ +function parseDoc(text: string | undefined): CacheDoc { + if (text === undefined) return emptyDoc(); + let raw: unknown; + try { + raw = JSON.parse(text); + } catch { + return emptyDoc(); + } + if (!isObj(raw) || !isObj(raw.providers)) return emptyDoc(); + const doc = emptyDoc(); + for (const [pid, pv] of Object.entries(raw.providers)) { + if (!isObj(pv)) continue; + const p: CachedProvider = { + credential_fp: strOrNull(pv.credential_fp), + last_probe_at: numOrNull(pv.last_probe_at), + last_attempt_at: numOrNull(pv.last_attempt_at), + last_error: strOrNull(pv.last_error), + models: {}, + }; + if (isObj(pv.models)) { + for (const [mid, mv] of Object.entries(pv.models)) { + if (!isObj(mv)) continue; + const status = mv.status; + const checked = numOrNull(mv.checked_at); + if (typeof status !== "string" || !(PROBE_STATUSES as readonly string[]).includes(status) || checked === null) + continue; + p.models[mid] = { + status: status as ProbeStatus, + reason: strOrNull(mv.reason), + latency_ms: numOrNull(mv.latency_ms), + checked_at: checked, + credential_fp: strOrNull(mv.credential_fp), + source: mv.source === "demoted" ? "demoted" : "probe", + }; + } + } + doc.providers[pid] = p; + } + return doc; +} + +/** Keep only well-formed probe rows (the engine is trusted, the wire is not). */ +function validResults(out: ProbeOutput | undefined): ProbeResult[] | undefined { + if (!out || !Array.isArray(out.results)) return undefined; + const rows: ProbeResult[] = []; + for (const r of out.results as unknown[]) { + if (!isObj(r) || typeof r.model !== "string" || r.model === "") continue; + if (typeof r.status !== "string" || !(PROBE_STATUSES as readonly string[]).includes(r.status)) continue; + const fp = r.credential_fp; + if (fp !== null && typeof fp !== "string") continue; + rows.push({ + model: r.model, + status: r.status as ProbeStatus, + latency_ms: numOrNull(r.latency_ms) ?? 0, + reason: typeof r.reason === "string" ? r.reason.slice(0, REASON_MAX) : null, + credential_fp: fp as string | null, + checked_at: numOrNull(r.checked_at) ?? Date.now(), + }); + } + return rows; +} + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +function pidAlive(pid: unknown): boolean { + if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 0) return false; + try { + process.kill(pid, 0); + return true; + } catch (err) { + return (err as NodeJS.ErrnoException).code === "EPERM"; + } +} + +export class ModelReachability { + readonly file: string; + private readonly deps: ModelReachabilityDeps; + private ticking: Promise | undefined; + + constructor(deps: ModelReachabilityDeps = {}) { + this.deps = deps; + this.file = deps.file ?? reachabilityCacheFile(); + } + + private now(): number { + return (this.deps.now ?? Date.now)(); + } + private get floorMs(): number { + return this.deps.minProbeIntervalMs ?? 5 * 60_000; + } + private get dailyMs(): number { + return this.deps.dailyIntervalMs ?? 24 * 60 * 60_000; + } + get schedulerIntervalMs(): number { + return this.deps.schedulerIntervalMs ?? 15 * 60_000; + } + /** The scheduler runs only when it has both engine seams. */ + get canSchedule(): boolean { + return this.schedulerIntervalMs > 0 && this.deps.listModels !== undefined && this.deps.probe !== undefined; + } + + // ── the shared-file lock ──────────────────────────────────────────────── + private readDocSync(): CacheDoc { + if (!existsSync(this.file)) return emptyDoc(); + try { + return parseDoc(readFileSync(this.file, "utf8")); + } catch { + return emptyDoc(); + } + } + + private async acquire(lockDir: string): Promise { + const deadline = Date.now() + (this.deps.lockTimeoutMs ?? 10_000); + const staleMs = this.deps.staleLockMs ?? 10_000; + await mkdir(path.dirname(lockDir), { recursive: true }); + for (;;) { + try { + await mkdir(lockDir); // no recursive: EEXIST on contention is the lock + await writeFile(path.join(lockDir, "owner"), JSON.stringify({ pid: process.pid, ts: Date.now() }) + "\n").catch( + () => undefined, + ); + return; + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err; + } + // Contended: reclaim a dead owner's lock past the stale window (or any + // lock 10× past it — a critical section is milliseconds). + let owner: { pid?: unknown; ts?: unknown } | undefined; + try { + owner = JSON.parse(await readFile(path.join(lockDir, "owner"), "utf8")); + } catch { + owner = undefined; + } + let ts = typeof owner?.ts === "number" ? owner.ts : undefined; + if (ts === undefined) ts = await stat(lockDir).then((s) => s.mtimeMs, () => undefined); + if (ts !== undefined) { + const age = Date.now() - ts; + if ((age > staleMs && !pidAlive(owner?.pid)) || age > staleMs * 10) { + const aside = `${lockDir}.stale-${process.pid}-${randomBytes(4).toString("hex")}`; + try { + await rename(lockDir, aside); // atomic: one reclaimer wins + await rm(aside, { recursive: true, force: true }); + this.deps.log?.(`[model-reachability] reclaimed a stale cache lock`); + } catch { + /* another reclaimer won — retry the claim */ + } + continue; + } + } + if (Date.now() > deadline) throw new Error("model-reachability cache lock timeout"); + await sleep(3 + Math.floor(Math.random() * 12)); + } + } + + /** Read-modify-write under the lock; the write lands by tmp+rename. */ + private async update(fn: (doc: CacheDoc) => T): Promise { + const lockDir = `${this.file}.lock`; + await this.acquire(lockDir); + try { + const text = await readFile(this.file, "utf8").catch(() => undefined); + const doc = parseDoc(text); + const result = fn(doc); + const tmp = path.join(path.dirname(this.file), `.${path.basename(this.file)}.${randomBytes(6).toString("hex")}.tmp`); + try { + await writeFile(tmp, JSON.stringify(doc, null, 2) + "\n"); + await rename(tmp, this.file); + } catch (err) { + await rm(tmp, { force: true }); + throw err; + } + return result; + } finally { + await rm(lockDir, { recursive: true, force: true }); + } + } + + // ── reads ─────────────────────────────────────────────────────────────── + private async catalog(): Promise | undefined> { + if (!this.deps.listModels) return undefined; + try { + const c = await this.deps.listModels(); + if (!isObj(c)) return undefined; + const out: Record = {}; + for (const [pid, models] of Object.entries(c)) { + if (Array.isArray(models)) out[pid] = [...new Set(models.filter((m): m is string => typeof m === "string" && m !== ""))]; + } + return out; + } catch { + return undefined; + } + } + + private rowOf(doc: CacheDoc, provider: string, model: string): ReachabilityRow { + const p = doc.providers[provider]; + const m = p?.models[model]; + // The key includes the fp: a row cached under another credential is not + // a verdict about this one → unchecked. + if (p === undefined || m === undefined || m.credential_fp !== p.credential_fp) { + return { provider, model, status: "unchecked", reason: null, latency_ms: null, checked_at: null, credential_fp: null, source: null }; + } + return { + provider, + model, + status: m.status, + reason: m.reason, + latency_ms: m.latency_ms, + checked_at: m.checked_at, + credential_fp: m.credential_fp, + source: m.source, + }; + } + + private metaOf(doc: CacheDoc, provider: string): ReachabilityProviderMeta { + const p = doc.providers[provider] ?? emptyProvider(); + const floorAt = p.last_attempt_at !== null ? p.last_attempt_at + this.floorMs : 0; + const dueAt = p.last_probe_at !== null ? p.last_probe_at + this.dailyMs : 0; + return { + provider, + credential_fp: p.credential_fp, + last_probe_at: p.last_probe_at, + last_attempt_at: p.last_attempt_at, + last_error: p.last_error, + next_probe_at: Math.max(floorAt, dueAt, this.now()), + }; + } + + private rows(doc: CacheDoc, catalog: Record | undefined, only?: string[]): ReachabilityRow[] { + const out: ReachabilityRow[] = []; + const providers = (catalog ? Object.keys(catalog) : Object.keys(doc.providers)).filter((p) => !only || only.includes(p)).sort(); + for (const provider of providers) { + const models = catalog ? catalog[provider] ?? [] : Object.keys(doc.providers[provider]?.models ?? {}); + for (const model of [...models].sort()) { + const r = this.rowOf(doc, provider, model); + // with no catalog, only serve what the cache can vouch for + if (!catalog && r.status === "unchecked") continue; + out.push(r); + } + } + return out; + } + + /** The GET body. Never throws. */ + async snapshot(): Promise { + const catalog = await this.catalog(); + const doc = this.readDocSync(); + const providers = (catalog ? Object.keys(catalog) : Object.keys(doc.providers)).sort(); + return { + ok: true, + models: this.rows(doc, catalog), + providers: providers.map((p) => this.metaOf(doc, p)), + catalog_known: catalog !== undefined, + error: null, + }; + } + + // ── probes ────────────────────────────────────────────────────────────── + /** Take the rate-limit claim for each provider under the lock: a provider + * inside its floor is refused (with the wait); `due` filters the + * scheduler's triggers. Claimed providers get last_attempt_at = now. */ + private async claim( + providers: string[], + due?: (p: CachedProvider | undefined, now: number) => boolean, + ): Promise<{ claimed: string[]; rate_limited: Array<{ provider: string; retry_after_ms: number }> }> { + const now = this.now(); + return this.update((doc) => { + const claimed: string[] = []; + const rate_limited: Array<{ provider: string; retry_after_ms: number }> = []; + for (const pid of providers) { + const p = doc.providers[pid]; + if (due && !due(p, now)) continue; + const since = p?.last_attempt_at != null ? now - p.last_attempt_at : Infinity; + if (since < this.floorMs) { + rate_limited.push({ provider: pid, retry_after_ms: this.floorMs - since }); + continue; + } + const entry = p ?? emptyProvider(); + entry.last_attempt_at = now; + doc.providers[pid] = entry; + claimed.push(pid); + } + return { claimed, rate_limited }; + }); + } + + /** Probe one claimed provider (batched at the engine's cap) and merge. */ + private async probeAndMerge(provider: string, models: string[]): Promise { + let results: ProbeResult[] | undefined = []; + for (let i = 0; i < models.length && results !== undefined; i += PROBE_BATCH) { + let out: ProbeOutput | undefined; + try { + out = await this.deps.probe?.(provider, models.slice(i, i + PROBE_BATCH)); + } catch { + out = undefined; + } + const batch = validResults(out); + results = batch === undefined ? undefined : [...results, ...batch]; + } + const now = this.now(); + if (results === undefined) { + await this.update((doc) => { + const p = doc.providers[provider] ?? emptyProvider(); + p.last_error = "probe_failed"; + doc.providers[provider] = p; + }); + return false; + } + const rows = results; + await this.update((doc) => { + const p = doc.providers[provider] ?? emptyProvider(); + const fp = rows.length > 0 ? rows[0]!.credential_fp : p.credential_fp; + // The credential changed: nothing cached under the old one survives. + if (fp !== p.credential_fp) p.models = {}; + p.credential_fp = fp; + for (const r of rows) { + p.models[r.model] = { + status: r.status, + reason: r.reason, + latency_ms: r.latency_ms, + checked_at: r.checked_at, + credential_fp: r.credential_fp, + source: "probe", + }; + } + p.last_probe_at = now; + p.last_error = null; + doc.providers[provider] = p; + }); + return true; + } + + /** On-demand refresh (one provider, or every catalog provider). */ + async refresh(opts: { provider?: string } = {}): Promise { + const fail = (error: string): RefreshResult => ({ ok: false, refreshed: [], rate_limited: [], failed: [], models: [], error }); + if (!this.deps.probe) return fail("engine_unavailable: no engine probe is wired"); + const catalog = await this.catalog(); + if (catalog === undefined) return fail("engine_unavailable: the engine's provider catalog could not be read"); + const targets = opts.provider !== undefined ? [opts.provider] : Object.keys(catalog).sort(); + if (opts.provider !== undefined && !(opts.provider in catalog)) + return fail("bad_request: unknown or unconnected provider"); + let claim; + try { + claim = await this.claim(targets); + } catch { + return fail("cache_unavailable: the reachability cache could not be locked"); + } + const refreshed: string[] = []; + const failed: Array<{ provider: string; error: string }> = []; + for (const pid of claim.claimed) { + try { + if (await this.probeAndMerge(pid, catalog[pid] ?? [])) refreshed.push(pid); + else failed.push({ provider: pid, error: "probe_failed" }); + } catch { + failed.push({ provider: pid, error: "cache_unavailable" }); + } + } + return { + ok: true, + refreshed, + rate_limited: claim.rate_limited, + failed, + models: this.rows(this.readDocSync(), catalog, targets), + error: null, + }; + } + + /** The scheduler pass: probe every catalog provider that was never probed + * (on-connect) or whose last probe is a day old — inside the floor, never. + * In-process re-entry joins the running pass. Never throws. */ + tick(): Promise { + if (this.ticking) return this.ticking; + this.ticking = (async () => { + if (!this.deps.probe) return; + const catalog = await this.catalog(); + if (catalog === undefined) return; + const { claimed } = await this.claim(Object.keys(catalog).sort(), (p, now) => + p === undefined || p.last_probe_at === null ? true : now - p.last_probe_at >= this.dailyMs, + ); + for (const pid of claimed) await this.probeAndMerge(pid, catalog[pid] ?? []); + })() + .catch((err) => this.deps.log?.(`[model-reachability] scheduler pass failed: ${err instanceof Error ? err.message : String(err)}`)) + .finally(() => { + this.ticking = undefined; + }); + return this.ticking; + } + + /** Arm the scheduler (first pass after the initial delay, then every + * interval). Returns the stop. Timers are unref'd — never hold a process. */ + startScheduler(): () => void { + if (!this.canSchedule) return () => undefined; + const first = setTimeout(() => void this.tick(), this.deps.schedulerInitialDelayMs ?? 2000); + const every = setInterval(() => void this.tick(), this.schedulerIntervalMs); + first.unref?.(); + every.unref?.(); + return () => { + clearTimeout(first); + clearInterval(every); + }; + } + + /** A runtime demotion (e.g. a dispatch that hit 403/429): sticks until the + * next probe of that model replaces it. */ + async demote(body: { provider?: unknown; model?: unknown; status?: unknown; reason?: unknown }): Promise { + const bad = (): DemoteResult => ({ + ok: false, + error: "bad_request: body must be JSON {provider, model, status: denied|unavailable|invalid|rate-limited|error, reason?: string|null}", + }); + const provider = typeof body.provider === "string" ? body.provider.trim() : ""; + const model = typeof body.model === "string" ? body.model.trim() : ""; + if (provider === "" || model === "" || provider.length > ID_MAX || model.length > ID_MAX) return bad(); + if (typeof body.status !== "string" || !(DEMOTE_STATUSES as readonly string[]).includes(body.status)) return bad(); + if (body.reason !== undefined && body.reason !== null && typeof body.reason !== "string") return bad(); + const status = body.status as ProbeStatus; + const reason = typeof body.reason === "string" ? body.reason.slice(0, REASON_MAX) : null; + const now = this.now(); + try { + await this.update((doc) => { + const p = doc.providers[provider] ?? emptyProvider(); + p.models[model] = { status, reason, latency_ms: null, checked_at: now, credential_fp: p.credential_fp, source: "demoted" }; + doc.providers[provider] = p; + }); + } catch { + return { ok: false, error: "cache_unavailable: the reachability cache could not be locked" }; + } + return { ok: true, provider, model, status, reason, error: null }; + } +} + +// ── the route bodies ────────────────────────────────────────────────────── +const synthesize = (error: string): string => JSON.stringify({ ok: false, error }); + +function parseBody(body: unknown): Record | undefined { + if (typeof body === "string") { + if (body.trim() === "") return {}; + try { + const v = JSON.parse(body); + return isObj(v) ? v : undefined; + } catch { + return undefined; + } + } + return isObj(body) ? body : undefined; +} + +/** GET /amicode/model-reachability. Never throws. */ +export async function modelReachabilityResponse(mr: ModelReachability): Promise { + try { + return JSON.stringify(await mr.snapshot()); + } catch { + return synthesize("cache_unavailable: the reachability cache could not be read"); + } +} + +/** POST /amicode/model-reachability/refresh — {provider?}. */ +export async function refreshModelReachabilityResponse(body: unknown, mr: ModelReachability): Promise { + const parsed = parseBody(body); + if (parsed === undefined || (parsed.provider !== undefined && (typeof parsed.provider !== "string" || parsed.provider.trim() === ""))) + return synthesize("bad_request: body must be JSON {provider?: string}"); + const res = await mr.refresh(typeof parsed.provider === "string" ? { provider: parsed.provider.trim() } : {}); + return res.ok ? JSON.stringify(res) : synthesize(res.error ?? "error"); +} + +/** POST /amicode/model-reachability/demote — {provider, model, status, reason?}. */ +export async function demoteModelReachabilityResponse(body: unknown, mr: ModelReachability): Promise { + const parsed = parseBody(body); + if (parsed === undefined) return JSON.stringify(await mr.demote({})); + return JSON.stringify(await mr.demote(parsed)); +} + +/** The production engine seams over the running engine (late-bound URL — the + * engine restarts under the service's feet): the key-free catalog + * (/config/providers) and the S1 probe (/provider/probe), via the same + * llm_creds client fetchProviderIds uses. */ +export function engineReachabilityDeps(engine: { + getUrl: () => string | undefined; + headers?: Record; +}): Pick { + return { + listModels: async () => { + const url = engine.getUrl(); + return url ? fetchProviderModels(url, { headers: engine.headers }) : undefined; + }, + probe: async (providerID, modelIDs) => { + const url = engine.getUrl(); + return url ? probeProviderModels(url, providerID, modelIDs, { headers: engine.headers }) : undefined; + }, + }; +} diff --git a/packages/extension/src/amicode_service_runner.ts b/packages/extension/src/amicode_service_runner.ts index ad81ec62e..666c3421d 100644 --- a/packages/extension/src/amicode_service_runner.ts +++ b/packages/extension/src/amicode_service_runner.ts @@ -49,6 +49,7 @@ import { buildMachineStableConfig } from "./opencode_config"; import { buildHubFleetOption } from "./hub_fleet_wiring"; import { readFleetTopology } from "./fleet_topology"; import { buildFleetPeerProvider } from "./amicode_service/fleet_peer_provider"; +import { engineReachabilityDeps } from "./amicode_service/model_reachability"; /** The named boot-abort: `reason` is the stable grep-able phrase, `message` * carries the detail (engine output tail for health failures). */ @@ -577,6 +578,15 @@ export async function bootAmicodeServiceRunner(opts: AmicodeServiceRunnerOptions engine: { password: unarmed ? undefined : password, getUrl: () => engineUrl }, shelf: { distRoot: opts.appDistRoot }, ...(hubFleet ? { fleet: hubFleet } : {}), + // #1985 (#1979 S2): the model-reachability cache's engine seams — the hub + // is where the shared cache lives (fleet clients proxy /amicode/* here), + // so the headless path MUST wire it (the model-routing gap is not + // repeated). The engine credential matches the spawn posture: none when + // unarmed, the per-boot mint otherwise. + modelReachability: engineReachabilityDeps({ + getUrl: () => engineUrl, + ...(password !== undefined ? { headers: { Authorization: serverAuthHeader(password) } } : {}), + }), // #1771: the fan-in census + owner-map refresh outcomes → the runner's log. log, }); diff --git a/packages/extension/src/amicode_service_wiring.ts b/packages/extension/src/amicode_service_wiring.ts index d60e86a90..1d3e16905 100644 --- a/packages/extension/src/amicode_service_wiring.ts +++ b/packages/extension/src/amicode_service_wiring.ts @@ -145,6 +145,10 @@ export interface AmicodeServiceWiringOptions { * LIVE-provider getter (the running engine's /config/providers, key-free * ids only) — the credential gate's refresh loop. */ modelRouting?: import("./amicode_service/model_routing").ModelRoutingDeps; + /** #1985 (#1979 S2): the model-reachability cache's engine seams — the + * caller supplies the key-free catalog + the S1 probe over the running + * engine (engineReachabilityDeps, late-bound URL). */ + modelReachability?: import("./amicode_service/model_reachability").ModelReachabilityDeps; /** #1261 (AC7): the version-skew relay-START gate. A client relay sets it * (its pinned version + a host-version probe); the relay REFUSES to boot on * a disagreement beyond tolerance (an actionable message, never a generic @@ -331,6 +335,7 @@ export async function startAmicodeService( shelf: opts.appDistRoot !== undefined ? { distRoot: opts.appDistRoot } : undefined, fleet: fleet ?? opts.fleet, modelRouting: opts.modelRouting, + modelReachability: opts.modelReachability, // #1771: the fan-in census + owner-map refresh outcomes → the Amicode // output channel. log: (line) => log.appendLine(line), diff --git a/packages/extension/src/extension.ts b/packages/extension/src/extension.ts index e3a26aebe..e709d2cff 100644 --- a/packages/extension/src/extension.ts +++ b/packages/extension/src/extension.ts @@ -3,6 +3,7 @@ import * as path from "node:path"; import * as fs from "node:fs"; import { ServerManager } from "./server_manager"; import { fetchProviderSignal, fetchProviderIds } from "./llm_creds.mjs"; +import { engineReachabilityDeps } from "./amicode_service/model_reachability"; import { resolveOpencodeBinary, OpencodeMissingError, unsupportedHostAdvice } from "./opencode_binary"; import { resolveSelectedLaunch, harnessMenu, decideHarnessSwitch } from "./harness"; import { watchHarnessSwitch, writeHarnessOptionsFile } from "./harness_switch"; @@ -1973,6 +1974,13 @@ export async function activate(ctx: vscode.ExtensionContext): Promise { return fetchProviderIds(engineUrl, { headers: serverAuthHeaders }); }, }, + // #1985 (#1979 S2): the model-reachability cache's engine seams — the + // key-free catalog + the one-token probe over the SAME late-bound + // engine URL and credential the routing refresh uses. + modelReachability: engineReachabilityDeps({ + getUrl: () => serverManager?.url?.toString(), + headers: serverAuthHeaders, + }), appDistRoot: resolveAppDistRoot( vscode.workspace.getConfiguration("amicode").get("appBundleDir", ""), ctx.extensionPath, @@ -2340,6 +2348,11 @@ export async function activate(ctx: vscode.ExtensionContext): Promise { return fetchProviderIds(engineUrl, { headers: serverAuthHeaders }); }, }, + // #1985: same reachability seams on the adopted-engine path. + modelReachability: engineReachabilityDeps({ + getUrl: () => opencodeReadyUrl?.toString(), + headers: serverAuthHeaders, + }), appDistRoot: resolveAppDistRoot( vscode.workspace.getConfiguration("amicode").get("appBundleDir", ""), ctx.extensionPath, diff --git a/packages/extension/src/llm_creds.d.mts b/packages/extension/src/llm_creds.d.mts index 546210a2b..46ccbe37a 100644 --- a/packages/extension/src/llm_creds.d.mts +++ b/packages/extension/src/llm_creds.d.mts @@ -33,3 +33,28 @@ export function fetchProviderIds( baseUrl: string, opts?: { fetchImpl?: typeof fetch; timeoutMs?: number; headers?: Record }, ): Promise; + +/** #1985: the live provider → model-id catalog (key-free: provider ids and + * model keys only); undefined when the query fails (unknown, never empty). */ +export function fetchProviderModels( + baseUrl: string, + opts?: { fetchImpl?: typeof fetch; timeoutMs?: number; headers?: Record }, +): Promise | undefined>; + +/** One row of the engine's POST /provider/probe response (#1984). */ +export interface ProviderProbeResultWire { + model: string; + status: "ok" | "denied" | "unavailable" | "invalid" | "rate-limited" | "error"; + latency_ms: number; + reason: string | null; + credential_fp: string | null; + checked_at: number; +} + +/** #1985: POST /provider/probe {providerID, modelIDs}; undefined on failure. */ +export function probeProviderModels( + baseUrl: string, + providerID: string, + modelIDs: string[], + opts?: { fetchImpl?: typeof fetch; timeoutMs?: number; headers?: Record }, +): Promise<{ providerID: string; results: ProviderProbeResultWire[] } | undefined>; diff --git a/packages/extension/src/llm_creds.mjs b/packages/extension/src/llm_creds.mjs index 042720ae7..3566e5066 100644 --- a/packages/extension/src/llm_creds.mjs +++ b/packages/extension/src/llm_creds.mjs @@ -126,3 +126,56 @@ export async function fetchProviderIds(baseUrl, { fetchImpl = fetch, timeoutMs = return undefined; } } + +/** + * The live provider → model-id catalog (#1985, the model-reachability cache's + * probe list): the same /config/providers query, stripped at the parse + * boundary to provider ids and their model KEYS only (never a model's + * options, never a provider key). A failed query yields undefined (UNKNOWN — + * never an empty catalog). + */ +export async function fetchProviderModels(baseUrl, { fetchImpl = fetch, timeoutMs = 4000, headers } = {}) { + const base = String(baseUrl).replace(/\/$/, ""); + try { + const r = await fetchImpl(`${base}/config/providers`, { signal: AbortSignal.timeout(timeoutMs), headers }); + if (!r.ok) return undefined; + const json = await r.json(); + const arr = json && Array.isArray(json.providers) ? json.providers : undefined; + if (!arr) return undefined; + const out = {}; + for (const p of arr) { + if (!p || typeof p.id !== "string" || p.id.length === 0) continue; + const models = p.models && typeof p.models === "object" && !Array.isArray(p.models) ? Object.keys(p.models) : []; + out[p.id] = models.filter((m) => typeof m === "string" && m.length > 0); + } + return out; + } catch { + return undefined; + } +} + +/** + * The engine's one-token reachability probe (#1984, POST /provider/probe): + * the engine sends 1 token per model through its OWN credentials and returns + * a named status per model — no secret crosses. The probe is bounded inside + * the engine (10s per model, concurrency 4), so the client budget scales with + * the batch. Failure (unreachable / non-2xx / malformed) → undefined. + */ +export async function probeProviderModels(baseUrl, providerID, modelIDs, { fetchImpl = fetch, timeoutMs, headers } = {}) { + const base = String(baseUrl).replace(/\/$/, ""); + const budget = timeoutMs ?? Math.ceil(Math.max(1, modelIDs.length) / 4) * 10_000 + 15_000; + try { + const r = await fetchImpl(`${base}/provider/probe`, { + method: "POST", + signal: AbortSignal.timeout(budget), + headers: { ...(headers ?? {}), "content-type": "application/json" }, + body: JSON.stringify({ providerID, modelIDs }), + }); + if (!r.ok) return undefined; + const json = await r.json(); + if (!json || !Array.isArray(json.results)) return undefined; + return json; + } catch { + return undefined; + } +} diff --git a/packages/extension/test/amicode_service_model_picker_config.test.ts b/packages/extension/test/amicode_service_model_picker_config.test.ts new file mode 100644 index 000000000..a9c707900 --- /dev/null +++ b/packages/extension/test/amicode_service_model_picker_config.test.ts @@ -0,0 +1,116 @@ +// #1987 (#1979 S4): the model picker's deployment config — `model.recommended` +// + `model.allowlist` read from $AMICODE_OPS_DIR/model-picker.json and served +// to the overlay picker at GET /amicode/model-picker (same family as the S2 +// reachability routes, so both boot paths — extension + hub runner — wire it). +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { modelPickerConfigFile, modelPickerConfigResponse } from "../src/amicode_service/model_picker_config"; + +const tmp = () => mkdtempSync(join(tmpdir(), "amc-1987-")); + +describe("modelPickerConfigResponse", () => { + it("no file → ok with no recommended and no allowlist (nothing invented)", () => { + const body = JSON.parse(modelPickerConfigResponse(join(tmp(), "missing.json"))); + expect(body).toEqual({ ok: true, error: null, model: { recommended: [] } }); + }); + + it("a well-formed file is served (provider/id refs, roles, allowlist incl. provider/*)", () => { + const file = join(tmp(), "model-picker.json"); + writeFileSync( + file, + JSON.stringify({ + model: { + recommended: [ + { model: "anthropic/claude-sonnet", role: "default" }, + { model: "groq/llama", role: "fast" }, + ], + allowlist: ["anthropic/*", "groq/llama"], + }, + }), + ); + expect(JSON.parse(modelPickerConfigResponse(file))).toEqual({ + ok: true, + error: null, + model: { + recommended: [ + { model: "anthropic/claude-sonnet", role: "default" }, + { model: "groq/llama", role: "fast" }, + ], + allowlist: ["anthropic/*", "groq/llama"], + }, + }); + }); + + it("bad entries are dropped (unknown role, no provider/ prefix, non-string)", () => { + const file = join(tmp(), "model-picker.json"); + writeFileSync( + file, + JSON.stringify({ + model: { + recommended: [{ model: "a/x", role: "turbo" }, { model: "noslash", role: "fast" }, { model: "a/y", role: "strong" }, 5], + allowlist: ["a/*", 3, "", "noslash"], + }, + }), + ); + expect(JSON.parse(modelPickerConfigResponse(file)).model).toEqual({ + recommended: [{ model: "a/y", role: "strong" }], + allowlist: ["a/*"], + }); + }); + + it("malformed JSON → ok:false with a fixed error string", () => { + const file = join(tmp(), "model-picker.json"); + writeFileSync(file, "{ nope"); + expect(JSON.parse(modelPickerConfigResponse(file))).toEqual({ + ok: false, + error: "config_invalid: model-picker.json is not valid JSON", + }); + }); + + describe("default file location", () => { + const prev = process.env.AMICODE_OPS_DIR; + beforeEach(() => { + process.env.AMICODE_OPS_DIR = tmp(); + }); + afterEach(() => { + if (prev === undefined) delete process.env.AMICODE_OPS_DIR; + else process.env.AMICODE_OPS_DIR = prev; + }); + it("lives under $AMICODE_OPS_DIR", () => { + expect(modelPickerConfigFile()).toBe(join(process.env.AMICODE_OPS_DIR!, "model-picker.json")); + }); + }); +}); + +describe("GET /amicode/model-picker on the real service", () => { + const prev = process.env.AMICODE_OPS_DIR; + afterEach(() => { + if (prev === undefined) delete process.env.AMICODE_OPS_DIR; + else process.env.AMICODE_OPS_DIR = prev; + }); + + it("serves the ops-dir file", async () => { + const ops = tmp(); + process.env.AMICODE_OPS_DIR = ops; + writeFileSync( + join(ops, "model-picker.json"), + JSON.stringify({ model: { recommended: [{ model: "a/x", role: "default" }], allowlist: ["a/*"] } }), + ); + const { createAmicodeService } = await import("../src/amicode_service"); + const service = createAmicodeService({}); + const url = await service.start(); + try { + const res = await fetch(new URL("/amicode/model-picker", url), { headers: { Authorization: service.authHeader } }); + expect(res.status).toBe(200); + expect(JSON.parse(await res.text())).toEqual({ + ok: true, + error: null, + model: { recommended: [{ model: "a/x", role: "default" }], allowlist: ["a/*"] }, + }); + } finally { + await service.stop(); + } + }); +}); diff --git a/packages/extension/test/amicode_service_model_reachability.test.ts b/packages/extension/test/amicode_service_model_reachability.test.ts new file mode 100644 index 000000000..3d7fb5978 --- /dev/null +++ b/packages/extension/test/amicode_service_model_reachability.test.ts @@ -0,0 +1,552 @@ +// #1985 (#1979 S2): the amicode service's model-reachability cache + +// scheduler + route family. The probe itself is S1's engine route +// (POST /provider/probe, #1984); every test here drives a FAKE probe so the +// cache / rate-limit / invalidation / locking semantics are pinned without an +// engine. The last groups boot the real service (createAmicodeService) and the +// headless hub runner (bootAmicodeServiceRunner over a fake engine) to prove +// the route family is wired on both paths — the hub runner skipped the +// model-routing wiring once; the reachability family must not repeat that. +import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + ModelReachability, + reachabilityCacheFile, + modelReachabilityResponse, + refreshModelReachabilityResponse, + demoteModelReachabilityResponse, + type ProbeOutput, + type ProbeStatus, +} from "../src/amicode_service/model_reachability"; +import { fetchProviderModels, probeProviderModels } from "../src/llm_creds.mjs"; +import { bootAmicodeServiceRunner, type AmicodeServiceRunnerBoot } from "../src/amicode_service_runner"; + +interface Harness { + file: string; + clock: { now: number }; + fp: { current: string | null }; + statuses: Record; + catalog: Record; + calls: Array<{ providerID: string; modelIDs: string[] }>; + make(overrides?: Partial[0]>): ModelReachability; +} + +function harness(file?: string): Harness { + const dir = mkdtempSync(join(tmpdir(), "amc-1985-")); + const h: Harness = { + file: file ?? join(dir, "model-reachability.json"), + clock: { now: 1_800_000_000_000 }, + fp: { current: "aaaaaaaaaaaaaaaa" }, + statuses: {}, + catalog: { anthropic: ["claude-a", "claude-b"], openai: ["gpt-x"] }, + calls: [], + make(overrides = {}) { + return new ModelReachability({ + file: h.file, + now: () => h.clock.now, + listModels: async () => structuredClone(h.catalog), + probe: async (providerID: string, modelIDs: string[]): Promise => { + h.calls.push({ providerID, modelIDs: [...modelIDs] }); + return { + providerID, + results: modelIDs.map((model) => ({ + model, + status: h.statuses[`${providerID}/${model}`] ?? "ok", + latency_ms: 7, + reason: h.statuses[`${providerID}/${model}`] ? "fake reason" : null, + credential_fp: h.fp.current, + checked_at: h.clock.now, + })), + }; + }, + ...overrides, + }); + }, + }; + return h; +} + +const row = (snap: { models: Array<{ provider: string; model: string }> }, provider: string, model: string) => + snap.models.find((m) => m.provider === provider && m.model === model) as + | { provider: string; model: string; status: string; reason: string | null; credential_fp: string | null; source: string | null } + | undefined; + +describe("unknown is unchecked, never ok", () => { + it("a catalog model that was never probed is reported as unchecked", async () => { + const h = harness(); + const snap = await h.make().snapshot(); + expect(snap.catalog_known).toBe(true); + for (const m of snap.models) expect(m.status).toBe("unchecked"); + expect(row(snap, "anthropic", "claude-a")?.status).toBe("unchecked"); + expect(row(snap, "openai", "gpt-x")?.status).toBe("unchecked"); + }); + + it("no catalog and no cache → no rows (nothing is invented), catalog_known false", async () => { + const h = harness(); + const snap = await h.make({ listModels: async () => undefined }).snapshot(); + expect(snap.catalog_known).toBe(false); + expect(snap.models).toEqual([]); + }); +}); + +describe("fake probe results are cached and served", () => { + it("a refresh caches the probe results; a SECOND instance on the same file serves them without probing", async () => { + const h = harness(); + h.statuses["anthropic/claude-b"] = "denied"; + const a = h.make(); + const res = await a.refresh({ provider: "anthropic" }); + expect(res.refreshed).toEqual(["anthropic"]); + expect(h.calls).toEqual([{ providerID: "anthropic", modelIDs: ["claude-a", "claude-b"] }]); + + const b = h.make(); + const snap = await b.snapshot(); + expect(row(snap, "anthropic", "claude-a")).toMatchObject({ status: "ok", credential_fp: "aaaaaaaaaaaaaaaa", source: "probe" }); + expect(row(snap, "anthropic", "claude-b")).toMatchObject({ status: "denied", reason: "fake reason" }); + expect(row(snap, "openai", "gpt-x")?.status).toBe("unchecked"); + expect(h.calls.length).toBe(1); + // persisted on disk, keyed by provider → model, with the fp + const doc = JSON.parse(readFileSync(h.file, "utf8")); + expect(doc.providers.anthropic.credential_fp).toBe("aaaaaaaaaaaaaaaa"); + expect(doc.providers.anthropic.models["claude-a"].credential_fp).toBe("aaaaaaaaaaaaaaaa"); + }); + + it("the GET body serves the snapshot shape", async () => { + const h = harness(); + const mr = h.make(); + await mr.refresh({ provider: "openai" }); + const body = JSON.parse(await modelReachabilityResponse(mr)); + expect(body.ok).toBe(true); + expect(body.error).toBeNull(); + expect(body.catalog_known).toBe(true); + expect(row(body, "openai", "gpt-x")).toMatchObject({ status: "ok", latency_ms: 7, checked_at: h.clock.now }); + const p = body.providers.find((x: { provider: string }) => x.provider === "openai"); + expect(p).toMatchObject({ credential_fp: "aaaaaaaaaaaaaaaa", last_probe_at: h.clock.now, last_error: null }); + }); + + it("a provider dropped from the live catalog is not served from the cache", async () => { + const h = harness(); + const mr = h.make(); + await mr.refresh({ provider: "openai" }); + delete h.catalog.openai; + const snap = await mr.snapshot(); + expect(row(snap, "openai", "gpt-x")).toBeUndefined(); + }); +}); + +describe("credential_fp change invalidates", () => { + it("a probe under a NEW fingerprint drops every row cached under the old one (→ unchecked)", async () => { + const h = harness(); + const mr = h.make({ minProbeIntervalMs: 0 }); + await mr.refresh({ provider: "anthropic" }); + expect(row(await mr.snapshot(), "anthropic", "claude-b")?.status).toBe("ok"); + + // The credential rotates; the next probe only reaches claude-a (the + // catalog shrank to it) — claude-b's old-fp row must NOT stay "ok". + h.fp.current = "bbbbbbbbbbbbbbbb"; + h.catalog.anthropic = ["claude-a"]; + h.clock.now += 1000; + await mr.refresh({ provider: "anthropic" }); + h.catalog.anthropic = ["claude-a", "claude-b"]; + const snap = await mr.snapshot(); + expect(row(snap, "anthropic", "claude-a")).toMatchObject({ status: "ok", credential_fp: "bbbbbbbbbbbbbbbb" }); + expect(row(snap, "anthropic", "claude-b")).toMatchObject({ status: "unchecked", credential_fp: null }); + }); + + it("an entry whose fp disagrees with the provider's current fp is served as unchecked (hand-edited / partial file)", async () => { + const h = harness(); + mkdirSync(join(h.file, ".."), { recursive: true }); + writeFileSync( + h.file, + JSON.stringify({ + version: 1, + providers: { + anthropic: { + credential_fp: "cccccccccccccccc", + last_probe_at: h.clock.now, + last_attempt_at: h.clock.now, + last_error: null, + models: { + "claude-a": { status: "ok", reason: null, latency_ms: 1, checked_at: h.clock.now, credential_fp: "dddddddddddddddd", source: "probe" }, + }, + }, + }, + }), + ); + const snap = await h.make().snapshot(); + expect(row(snap, "anthropic", "claude-a")?.status).toBe("unchecked"); + }); + + it("a disconnected provider (probe says error, fp null) invalidates the old rows", async () => { + const h = harness(); + const mr = h.make({ minProbeIntervalMs: 0 }); + await mr.refresh({ provider: "openai" }); + h.fp.current = null; + h.statuses["openai/gpt-x"] = "error"; + await mr.refresh({ provider: "openai" }); + expect(row(await mr.snapshot(), "openai", "gpt-x")).toMatchObject({ status: "error", credential_fp: null }); + }); +}); + +describe("concurrent writers don't corrupt the cache", () => { + it("two instances on ONE file, interleaving demotes + probe merges, lose no update and leave valid JSON", async () => { + const h = harness(); + h.catalog = { p1: Array.from({ length: 12 }, (_, i) => `m${i}`), p2: Array.from({ length: 12 }, (_, i) => `n${i}`) }; + const a = h.make(); + const b = h.make(); + const ops: Promise[] = []; + for (let i = 0; i < 20; i++) { + ops.push(a.demote({ provider: "q", model: `a${i}`, status: "denied", reason: `a${i}` })); + ops.push(b.demote({ provider: "q", model: `b${i}`, status: "unavailable", reason: `b${i}` })); + } + ops.push(a.refresh({ provider: "p1" })); + ops.push(b.refresh({ provider: "p2" })); + const outs = await Promise.all(ops); + for (const o of outs) expect((o as { ok: boolean }).ok).toBe(true); + + const doc = JSON.parse(readFileSync(h.file, "utf8")); + expect(Object.keys(doc.providers.q.models).length).toBe(40); + expect(Object.keys(doc.providers.p1.models).length).toBe(12); + expect(Object.keys(doc.providers.p2.models).length).toBe(12); + expect(existsSync(`${h.file}.lock`)).toBe(false); + }); + + it("a stale lock (dead owner, past the stale window) is reclaimed instead of wedging the cache", async () => { + const h = harness(); + mkdirSync(`${h.file}.lock`, { recursive: true }); + writeFileSync(join(`${h.file}.lock`, "owner"), JSON.stringify({ pid: 999999, ts: Date.now() - 60_000 })); + const res = await h.make({ staleLockMs: 1000 }).demote({ provider: "openai", model: "gpt-x", status: "denied", reason: null }); + expect(res.ok).toBe(true); + expect(existsSync(`${h.file}.lock`)).toBe(false); + }); +}); + +describe("refresh is rate-limited", () => { + it("a second refresh inside the floor is refused with retry_after_ms and does NOT probe — across instances too", async () => { + const h = harness(); + const a = h.make({ minProbeIntervalMs: 60_000 }); + const first = await a.refresh({ provider: "anthropic" }); + expect(first.refreshed).toEqual(["anthropic"]); + + h.clock.now += 10_000; + const again = await a.refresh({ provider: "anthropic" }); + expect(again.refreshed).toEqual([]); + expect(again.rate_limited).toEqual([{ provider: "anthropic", retry_after_ms: 50_000 }]); + + const other = await h.make({ minProbeIntervalMs: 60_000 }).refresh({ provider: "anthropic" }); + expect(other.rate_limited.map((r) => r.provider)).toEqual(["anthropic"]); + expect(h.calls.length).toBe(1); + + h.clock.now += 50_000; + const later = await a.refresh({ provider: "anthropic" }); + expect(later.refreshed).toEqual(["anthropic"]); + expect(h.calls.length).toBe(2); + }); + + it("concurrent refreshes from two instances probe once (the claim is taken under the lock)", async () => { + const h = harness(); + const [r1, r2] = await Promise.all([h.make().refresh({ provider: "openai" }), h.make().refresh({ provider: "openai" })]); + expect(h.calls.length).toBe(1); + expect([...r1.refreshed, ...r2.refreshed]).toEqual(["openai"]); + }); + + it("a failed probe still counts against the floor (no hammering a dead engine) and records last_error", async () => { + const h = harness(); + let n = 0; + const mr = h.make({ + probe: async () => { + n++; + throw new Error("engine down"); + }, + }); + const res = await mr.refresh({ provider: "openai" }); + expect(res.failed).toEqual([{ provider: "openai", error: "probe_failed" }]); + const again = await mr.refresh({ provider: "openai" }); + expect(again.rate_limited.length).toBe(1); + expect(n).toBe(1); + const snap = await mr.snapshot(); + expect(snap.providers.find((p) => p.provider === "openai")?.last_error).toBe("probe_failed"); + expect(row(snap, "openai", "gpt-x")?.status).toBe("unchecked"); + }); + + it("the scheduler tick probes on connect (never probed) and daily — not on every tick", async () => { + const h = harness(); + const mr = h.make({ minProbeIntervalMs: 60_000, dailyIntervalMs: 86_400_000 }); + await mr.tick(); + expect(h.calls.map((c) => c.providerID).sort()).toEqual(["anthropic", "openai"]); + h.clock.now += 3_600_000; + await mr.tick(); + expect(h.calls.length).toBe(2); + // a provider connects later → probed on the next tick (on-connect) + h.catalog.google = ["gemini"]; + await mr.tick(); + expect(h.calls.map((c) => c.providerID)).toEqual(expect.arrayContaining(["google"])); + expect(h.calls.length).toBe(3); + h.clock.now += 86_400_000; + await mr.tick(); + expect(h.calls.length).toBe(6); + }); + + it("the refresh route body: bad provider → bad_request; unknown catalog → engine_unavailable", async () => { + const h = harness(); + const bad = JSON.parse(await refreshModelReachabilityResponse({ provider: 42 }, h.make())); + expect(bad).toMatchObject({ ok: false }); + expect(bad.error).toMatch(/^bad_request: /); + const unknown = JSON.parse(await refreshModelReachabilityResponse("{}", h.make({ listModels: async () => undefined }))); + expect(unknown.ok).toBe(false); + expect(unknown.error).toMatch(/^engine_unavailable: /); + const notConnected = JSON.parse(await refreshModelReachabilityResponse({ provider: "nope" }, h.make())); + expect(notConnected.error).toMatch(/^bad_request: /); + const ok = JSON.parse(await refreshModelReachabilityResponse({}, h.make())); + expect(ok.ok).toBe(true); + expect(ok.refreshed.sort()).toEqual(["anthropic", "openai"]); + }); +}); + +describe("demote sticks until the next probe", () => { + it("a demotion overrides a cached ok across instances and ticks, and the next probe replaces it", async () => { + const h = harness(); + const a = h.make({ minProbeIntervalMs: 60_000 }); + await a.refresh({ provider: "anthropic" }); + const d = JSON.parse( + await demoteModelReachabilityResponse( + { provider: "anthropic", model: "claude-a", status: "rate-limited", reason: "429 in dispatch" }, + a, + ), + ); + expect(d).toMatchObject({ ok: true, provider: "anthropic", model: "claude-a", status: "rate-limited", error: null }); + + const b = h.make({ minProbeIntervalMs: 60_000 }); + expect(row(await b.snapshot(), "anthropic", "claude-a")).toMatchObject({ + status: "rate-limited", + reason: "429 in dispatch", + source: "demoted", + credential_fp: "aaaaaaaaaaaaaaaa", + }); + await b.tick(); // not due — the demotion stands + expect(row(await b.snapshot(), "anthropic", "claude-a")?.status).toBe("rate-limited"); + + h.clock.now += 60_000; + await b.refresh({ provider: "anthropic" }); + expect(row(await a.snapshot(), "anthropic", "claude-a")).toMatchObject({ status: "ok", source: "probe" }); + }); + + it("the demote body is validated (no ok demotion, no unknown status, strings only) and never echoed", async () => { + const h = harness(); + const mr = h.make(); + for (const body of [ + { provider: "anthropic", model: "claude-a", status: "ok" }, + { provider: "anthropic", model: "claude-a", status: "unchecked" }, + { provider: "anthropic", model: "claude-a", status: "