From fff1d3b4dd777d17747836f792ef7f734507834b Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Tue, 3 Nov 2026 15:29:29 +0700 Subject: [PATCH 1/7] fix(ai-sandbox-cloudflare): run each command in a subshell so exit cannot end the session --- .changeset/cloudflare-exec-subshell.md | 5 +++++ packages/ai-sandbox-cloudflare/src/handle.ts | 14 ++++++++++-- .../tests/handle.test.ts | 22 ++++++++++++++++++- 3 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 .changeset/cloudflare-exec-subshell.md diff --git a/.changeset/cloudflare-exec-subshell.md b/.changeset/cloudflare-exec-subshell.md new file mode 100644 index 0000000000..84246415e0 --- /dev/null +++ b/.changeset/cloudflare-exec-subshell.md @@ -0,0 +1,5 @@ +--- +'@tanstack/ai-sandbox-cloudflare': patch +--- + +Run each `exec` and `spawn` command in a subshell. Cloudflare runs every command in one shared shell session, so a command that called `exit` ended the session and every later command failed. This broke the `@tanstack/ai-computer-sandbox` driver on Cloudflare. diff --git a/packages/ai-sandbox-cloudflare/src/handle.ts b/packages/ai-sandbox-cloudflare/src/handle.ts index 97e69595c7..6de70cdc68 100644 --- a/packages/ai-sandbox-cloudflare/src/handle.ts +++ b/packages/ai-sandbox-cloudflare/src/handle.ts @@ -59,6 +59,16 @@ function q(value: string): string { const LSTAT_MISSING = '__TANSTACK_LSTAT_MISSING__' +/** + * Cloudflare runs every command in one long-lived shell session, so a bare + * `exit` ends the session and every later command fails. A subshell keeps + * `exit`, `cd`, and variables local to the command, like the other providers. + * The newlines keep a trailing `#` comment or heredoc from eating the `)`. + */ +function subshell(command: string) { + return `(\n${command}\n)` +} + /** Verify a missing path by listing parent entries. `test -e` also fails for inaccessible parents. */ function lstatCommand(path: string): string { return `tanstack_lstat_path=${q(path)}; tanstack_lstat_output=$(stat -c '%f:%s' -- "$tanstack_lstat_path" 2>&1); tanstack_lstat_status=$?; if [ "$tanstack_lstat_status" -eq 0 ]; then printf '%s\n' "$tanstack_lstat_output"; else tanstack_lstat_missing() { tanstack_missing_path=$1; case "$tanstack_missing_path" in /|.) return 1 ;; */*) tanstack_parent=${'$'}{tanstack_missing_path%/*}; tanstack_name=${'$'}{tanstack_missing_path##*/}; [ -n "$tanstack_parent" ] || tanstack_parent=/ ;; *) tanstack_parent=.; tanstack_name=$tanstack_missing_path ;; esac; tanstack_parent_mode=$(stat -L -c '%f' -- "$tanstack_parent" 2>/dev/null); tanstack_parent_status=$?; if [ "$tanstack_parent_status" -ne 0 ]; then tanstack_lstat_missing "$tanstack_parent"; else case "$tanstack_parent_mode" in 4[0-9a-fA-F][0-9a-fA-F][0-9a-fA-F]) case "$tanstack_parent" in /*) tanstack_find_parent=$tanstack_parent ;; *) tanstack_find_parent=./$tanstack_parent ;; esac; tanstack_match=$(find -H "$tanstack_find_parent" -mindepth 1 -maxdepth 1 -exec sh -c 'tanstack_target=$1; shift; for tanstack_candidate do [ "${'$'}{tanstack_candidate##*/}" = "$tanstack_target" ] && { printf 1; exit 0; }; done; exit 0' sh "$tanstack_name" '{}' + 2>/dev/null); tanstack_find_status=$?; [ "$tanstack_find_status" -eq 0 ] && [ -z "$tanstack_match" ] ;; *) return 1 ;; esac; fi; }; if tanstack_lstat_missing "$tanstack_lstat_path"; then printf '%s' '${LSTAT_MISSING}'; else printf '%s\n' "$tanstack_lstat_output" >&2; exit "$tanstack_lstat_status"; fi; fi` @@ -248,7 +258,7 @@ export class CloudflareHandle implements SandboxHandle { command: string, opts?: ProcessOptions, ): Promise { - const result = await this.sandbox.exec(command, { + const result = await this.sandbox.exec(subshell(command), { ...(opts?.cwd ? { cwd: this.abs(opts.cwd) } : { cwd: this.workdir }), ...(opts?.env ? { env: opts.env } : {}), }) @@ -282,7 +292,7 @@ export class CloudflareHandle implements SandboxHandle { // before the command runs. Mid-run cancellation is therefore unavailable // on this provider; a stuck run is bounded by the coordinator's watchdog // and the Durable Object lifecycle instead. `kill()` is a best-effort no-op. - const settled = this.sandbox.exec(command, { + const settled = this.sandbox.exec(subshell(command), { ...(opts?.cwd ? { cwd: this.abs(opts.cwd) } : { cwd: this.workdir }), ...(opts?.env ? { env: opts.env } : {}), stream: true, diff --git a/packages/ai-sandbox-cloudflare/tests/handle.test.ts b/packages/ai-sandbox-cloudflare/tests/handle.test.ts index 089e3d90a4..7501350cdc 100644 --- a/packages/ai-sandbox-cloudflare/tests/handle.test.ts +++ b/packages/ai-sandbox-cloudflare/tests/handle.test.ts @@ -64,7 +64,9 @@ function sandboxFixture(methods: SandboxFixtureMethods): Sandbox { function mockSandbox(): { sandbox: Sandbox; files: Map } { const files = new Map() - const exec = (command: string, opts?: MockExecOpts): Promise => { + const exec = (wrapped: string, opts?: MockExecOpts): Promise => { + // The handle runs each command in a subshell. Match the inner command. + const command = wrapped.replace(/^\(\n([\s\S]*)\n\)$/, '$1') const ok = (stdout = ''): ExecResult => ({ stdout, stderr: '', @@ -358,6 +360,24 @@ describe('CloudflareHandle', () => { expect(r.exitCode).toBe(0) }) + it('runs each command in a subshell, so exit cannot end the shared session', async () => { + const commands: Array = [] + const sandbox = sandboxFixture({ + exec: (command: string) => { + commands.push(command) + return Promise.resolve({ stdout: '', stderr: '', exitCode: 3 }) + }, + setEnvVars: () => Promise.resolve(), + exposePort: (port: number) => + Promise.resolve({ url: `https://${port}.example.workers.dev` }), + destroy: () => Promise.resolve(), + }) + const handle = new CloudflareHandle('sbx-1', sandbox, '/workspace') + await handle.process.exec('exit 3 # done') + await (await handle.process.spawn('exit 3')).wait() + expect(commands).toEqual(['(\nexit 3 # done\n)', '(\nexit 3\n)']) + }) + it('spawn streams output via the queue and resolves wait()', async () => { const handle = createLstatHandle() const proc = await handle.process.spawn('run something') From 305457da37f6165b0e44bb356dbfbd61cde6dfee Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Tue, 3 Nov 2026 16:24:17 +0700 Subject: [PATCH 2/7] feat(ai-computer-sandbox): keep each thread's sandbox and reconnect to it with sandboxIds --- .changeset/computer-sandbox-keep-desktop.md | 5 ++ docs/computer-use/drivers.md | 50 ++++++++++++++++- packages/ai-computer-sandbox/README.md | 19 +++++++ packages/ai-computer-sandbox/src/driver.ts | 55 +++++++++++++++---- .../ai-computer-sandbox/tests/driver.test.ts | 46 +++++++++++++++- 5 files changed, 163 insertions(+), 12 deletions(-) create mode 100644 .changeset/computer-sandbox-keep-desktop.md diff --git a/.changeset/computer-sandbox-keep-desktop.md b/.changeset/computer-sandbox-keep-desktop.md new file mode 100644 index 0000000000..31ef1099dd --- /dev/null +++ b/.changeset/computer-sandbox-keep-desktop.md @@ -0,0 +1,5 @@ +--- +'@tanstack/ai-computer-sandbox': minor +--- + +Add the `sandboxIds` option to `sandboxComputer()`. It keeps the sandbox of each thread and reconnects to it later with `provider.resume`, so a user who comes back to a thread gets the same desktop. A kept sandbox is not destroyed when its thread closes or goes idle. diff --git a/docs/computer-use/drivers.md b/docs/computer-use/drivers.md index 1a93bf5c23..7f4ee94679 100644 --- a/docs/computer-use/drivers.md +++ b/docs/computer-use/drivers.md @@ -130,11 +130,12 @@ const computerUse = createComputerUse({ - `provider`: the sandbox provider. It makes one sandbox for each thread. - `display`: the screen size of each desktop in pixels. The default is 1024 x 768. +- `sandboxIds`: keep the desktop of each thread and reconnect to it later. See [Keep the desktop for later](#keep-the-desktop-for-later). How the sandbox driver behaves: - There is no `navigate` tool, because a desktop has no address. The model opens the browser on the desktop. -- When a thread closes or goes idle, `createComputerUse` destroys its sandbox. +- When a thread closes or goes idle, `createComputerUse` destroys its sandbox, unless you set `sandboxIds`. - One scroll step is 100 pixels. - The driver quotes every key name and all typed text before they go to the shell. @@ -155,6 +156,53 @@ RUN apt-get update && apt-get install -y xvfb xdotool scrot x11-utils If a tool is missing, the driver throws an error with its name. +### Keep the desktop for later + +Give `sandboxComputer()` a `sandboxIds` store. A user who comes back to a thread a day later gets the same desktop, for example with the browser still signed in. + +1. Store one sandbox id for each thread, next to your threads. +2. Set the provider to pause the sandbox, not to kill it. With E2B, use `onTimeout: 'pause'`. + +```ts +import { createComputerUse } from '@tanstack/ai-computer-use' +import { sandboxComputer } from '@tanstack/ai-computer-sandbox' +import { e2bSandbox } from '@tanstack/ai-sandbox-e2b' + +declare const db: { + getSandboxId: (threadId: string) => Promise + setSandboxId: (threadId: string, sandboxId: string) => Promise +} + +const computerUse = createComputerUse({ + driver: sandboxComputer({ + provider: e2bSandbox({ + template: 'desktop', + timeoutMs: 15 * 60_000, + onTimeout: 'pause', + }), + sandboxIds: { + get: (threadId) => db.getSandboxId(threadId), + set: (threadId, sandboxId) => db.setSandboxId(threadId, sandboxId), + }, + }), +}) +``` + +With `sandboxIds`, the driver works like this: + +- The first call of a thread creates a sandbox and stores its id. +- A later call reconnects to that sandbox with `provider.resume`. If the sandbox is gone, the driver creates a new one and stores the new id. +- The driver never destroys a kept sandbox. The provider timeout pauses or removes it. +- If the setup of a reconnected sandbox fails, the driver keeps the sandbox and throws the error. The next call tries again. + +What each provider keeps: + +- E2B with `onTimeout: 'pause'` keeps the memory and the disk. Open apps come back as they were. +- Docker keeps the container. A stopped container starts again with its disk, and the driver starts the display again. + +> [!WARNING] +> A kept desktop holds what the user did in it, like signed-in browser sessions. Delete the sandbox when the user deletes the thread. + Do you already have a desktop sandbox? See [Share one sandbox](#share-one-sandbox). ## Your own computer diff --git a/packages/ai-computer-sandbox/README.md b/packages/ai-computer-sandbox/README.md index dadacba3e3..0550f18baa 100644 --- a/packages/ai-computer-sandbox/README.md +++ b/packages/ai-computer-sandbox/README.md @@ -66,6 +66,25 @@ RUN apt-get update && apt-get install -y xvfb xdotool scrot x11-utils When a tool is missing, the driver throws an error with its name. +## Keep the desktop for later + +Give `sandboxComputer()` a `sandboxIds` store, and set the provider to pause the sandbox. A user who comes back to a thread later gets the same desktop. + +```typescript +const computerUse = createComputerUse({ + driver: sandboxComputer({ + provider: e2bSandbox({ template: 'desktop', onTimeout: 'pause' }), + sandboxIds: { + get: (threadId) => db.getSandboxId(threadId), + set: (threadId, sandboxId) => db.setSandboxId(threadId, sandboxId), + }, + }), +}) +``` + +- A later call of the thread reconnects with `provider.resume`. If the sandbox is gone, the driver creates a new one and stores its id. +- The driver never destroys a kept sandbox. The provider timeout pauses or removes it. + ## Share one sandbox To drive a sandbox that you already have, use `createSandboxComputerDriver`. Every thread shares this sandbox, so it serves one conversation at a time. diff --git a/packages/ai-computer-sandbox/src/driver.ts b/packages/ai-computer-sandbox/src/driver.ts index 1c4dcafbba..aa7cfe71ca 100644 --- a/packages/ai-computer-sandbox/src/driver.ts +++ b/packages/ai-computer-sandbox/src/driver.ts @@ -24,6 +24,16 @@ export interface SandboxComputerOptions { provider: SandboxProvider /** The screen size in pixels. The default is `{ width: 1024, height: 768 }`. */ display?: { width: number; height: number } + /** + * Keep the sandbox of each thread, and reconnect to it later. Store the ids + * where your threads persist. With this option, the driver does not destroy + * a sandbox when its thread closes or goes idle. The provider timeout + * pauses or removes it, like E2B `onTimeout: 'pause'`. + */ + sandboxIds?: { + get: (threadId: string) => Promise + set: (threadId: string, sandboxId: string) => Promise + } } const DEFAULT_DISPLAY = { width: 1024, height: 768 } @@ -234,7 +244,8 @@ export async function createSandboxComputerDriver({ * * Each sandbox comes from `provider.create`. When `createComputerUse` closes * a thread, or the thread goes idle, the sandbox of that thread is - * destroyed. The drivers have `cursorPosition`. They have no `zoom` and no + * destroyed. With `sandboxIds`, the sandbox is kept, and the next call of the + * thread reconnects to it with `provider.resume`. The drivers have `cursorPosition`. They have no `zoom` and no * `navigate`, because a desktop has no address. The model opens the browser * on the desktop. * @@ -248,13 +259,29 @@ export async function createSandboxComputerDriver({ export function sandboxComputer({ provider, display = DEFAULT_DISPLAY, + sandboxIds, }: SandboxComputerOptions) { + /** The sandbox of the thread: its stored sandbox, when it still exists, or a new one. */ + async function open(threadId: string) { + const id = await sandboxIds?.get(threadId) + const resumed = id === undefined ? null : await provider.resume({ id }) + if (resumed) return { sandbox: resumed, isNew: false } + const sandbox = await provider.create({}) + try { + await sandboxIds?.set(threadId, sandbox.id) + } catch (error) { + await destroyAfterFailure(sandbox) + throw error + } + return { sandbox, isNew: true } + } + const factory: ComputerDriverFactory = { display, environment: 'linux', capabilities: { zoom: false, cursorPosition: true, navigate: false }, - async create() { - const sandbox = await provider.create({}) + async create({ threadId }) { + const { sandbox, isNew } = await open(threadId) try { const driver = await createSandboxComputerDriver({ sandbox, display }) const { width, height } = driver.display @@ -263,18 +290,26 @@ export function sandboxComputer({ `The sandbox display is ${width}x${height}, not ${display.width}x${display.height}. Set display to the size of the running display.`, ) } + // A kept sandbox outlives its thread. The provider timeout ends it. + if (sandboxIds) return driver return { ...driver, close: () => sandbox.destroy() } } catch (error) { - // Keep the setup error. A failed destroy does not replace it. - await sandbox.destroy().catch((destroyError: unknown) => { - console.error( - 'Failed to destroy the sandbox after setup failed.', - destroyError, - ) - }) + // A kept sandbox holds the state of the thread, like a signed-in + // browser. Keep it, so the next call of the thread tries it again. + if (isNew) await destroyAfterFailure(sandbox) throw error } }, } return factory } + +/** Destroy a sandbox after a setup error. A failed destroy does not replace that error. */ +async function destroyAfterFailure(sandbox: SandboxHandle) { + await sandbox.destroy().catch((destroyError: unknown) => { + console.error( + 'Failed to destroy the sandbox after setup failed.', + destroyError, + ) + }) +} diff --git a/packages/ai-computer-sandbox/tests/driver.test.ts b/packages/ai-computer-sandbox/tests/driver.test.ts index f656121d49..785773d5f7 100644 --- a/packages/ai-computer-sandbox/tests/driver.test.ts +++ b/packages/ai-computer-sandbox/tests/driver.test.ts @@ -20,6 +20,8 @@ interface FakeOptions { size?: string } +let sandboxCount = 0 + /** A fake sandbox. Each command adds one line to `calls`, prefixed with `exec` or `spawn`. */ function fakeSandbox({ running = true, missing = [], size }: FakeOptions = {}) { const calls: Array = [] @@ -49,6 +51,7 @@ function fakeSandbox({ running = true, missing = [], size }: FakeOptions = {}) { calls.push('destroy') }) const sandbox = { + id: `sbx-${++sandboxCount}`, process: { exec, spawn }, destroy, } as unknown as SandboxHandle @@ -63,7 +66,11 @@ async function setup() { return { driver, calls: fake.calls } } -function fakeProvider(fakes: Array>) { +/** A provider that hands out `fakes` in order, and resumes only the `alive` sandboxes. */ +function fakeProvider( + fakes: Array>, + alive: Array> = [], +) { let next = 0 const provider = { name: 'fake', @@ -72,10 +79,26 @@ function fakeProvider(fakes: Array>) { if (!fake) throw new Error('no more sandboxes') return fake.sandbox }), + resume: vi.fn( + async ({ id }: { id: string }) => + alive.find((fake) => fake.sandbox.id === id)?.sandbox ?? null, + ), } as unknown as SandboxProvider return provider } +/** An in-memory `sandboxIds` store. */ +function memoryIds(initial: Record = {}) { + const ids = new Map(Object.entries(initial)) + return { + ids, + get: async (threadId: string) => ids.get(threadId), + set: async (threadId: string, sandboxId: string) => { + ids.set(threadId, sandboxId) + }, + } +} + describe('createSandboxComputerDriver', () => { it('uses a display that already runs and reports linux', async () => { const { sandbox, calls } = fakeSandbox() @@ -283,4 +306,25 @@ describe('sandboxComputer', () => { ) error.mockRestore() }) + + it('with sandboxIds, resumes the stored sandbox and never destroys it', async () => { + const kept = fakeSandbox() + const provider = fakeProvider([], [kept]) + const sandboxIds = memoryIds({ a: kept.sandbox.id }) + const factory = sandboxComputer({ provider, sandboxIds }) + const driver = await factory.create({ threadId: 'a' }) + expect(provider.create).not.toHaveBeenCalled() + expect(driver.close).toBeUndefined() + expect(kept.destroy).not.toHaveBeenCalled() + }) + + it('with sandboxIds, creates and stores a sandbox when the stored one is gone', async () => { + const created = fakeSandbox() + const provider = fakeProvider([created]) + const sandboxIds = memoryIds({ a: 'gone' }) + const factory = sandboxComputer({ provider, sandboxIds }) + await factory.create({ threadId: 'a' }) + expect(provider.resume).toHaveBeenCalledWith({ id: 'gone' }) + expect(sandboxIds.ids.get('a')).toBe(created.sandbox.id) + }) }) From 7a3d8fd6c0c86c24192455176b80ac677915b9aa Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Tue, 3 Nov 2026 16:24:18 +0700 Subject: [PATCH 3/7] feat(examples): computer use sign-in example where the model never sees the password --- docs/config.json | 16 +- docs/getting-started/overview.md | 1 + docs/tutorials/computer-use-login.md | 490 ++++++++++++++++++ docs/tutorials/computer-use.md | 2 + examples/README.md | 20 + .../react/computer-use-login/.env.example | 3 + examples/react/computer-use-login/.gitignore | 8 + examples/react/computer-use-login/README.md | 62 +++ .../react/computer-use-login/package.json | 37 ++ .../computer-use-login/sandbox/Dockerfile | 18 + .../computer-use-login/sandbox/desktop.sh | 19 + .../react/computer-use-login/sandbox/login.py | 143 +++++ .../computer-use-login/sandbox/policy.json | 6 + .../react/computer-use-login/src/harness.ts | 167 ++++++ .../computer-use-login/src/request-login.ts | 20 + .../computer-use-login/src/routeTree.gen.ts | 104 ++++ .../react/computer-use-login/src/router.tsx | 10 + .../computer-use-login/src/routes/__root.tsx | 41 ++ .../src/routes/api.harness.$.ts | 12 + .../src/routes/api.login.ts | 25 + .../computer-use-login/src/routes/index.tsx | 316 +++++++++++ .../react/computer-use-login/src/styles.css | 10 + .../react/computer-use-login/tsconfig.json | 25 + .../react/computer-use-login/vite.config.ts | 26 + pnpm-lock.yaml | 70 +++ 25 files changed, 1648 insertions(+), 3 deletions(-) create mode 100644 docs/tutorials/computer-use-login.md create mode 100644 examples/react/computer-use-login/.env.example create mode 100644 examples/react/computer-use-login/.gitignore create mode 100644 examples/react/computer-use-login/README.md create mode 100644 examples/react/computer-use-login/package.json create mode 100644 examples/react/computer-use-login/sandbox/Dockerfile create mode 100644 examples/react/computer-use-login/sandbox/desktop.sh create mode 100644 examples/react/computer-use-login/sandbox/login.py create mode 100644 examples/react/computer-use-login/sandbox/policy.json create mode 100644 examples/react/computer-use-login/src/harness.ts create mode 100644 examples/react/computer-use-login/src/request-login.ts create mode 100644 examples/react/computer-use-login/src/routeTree.gen.ts create mode 100644 examples/react/computer-use-login/src/router.tsx create mode 100644 examples/react/computer-use-login/src/routes/__root.tsx create mode 100644 examples/react/computer-use-login/src/routes/api.harness.$.ts create mode 100644 examples/react/computer-use-login/src/routes/api.login.ts create mode 100644 examples/react/computer-use-login/src/routes/index.tsx create mode 100644 examples/react/computer-use-login/src/styles.css create mode 100644 examples/react/computer-use-login/tsconfig.json create mode 100644 examples/react/computer-use-login/vite.config.ts diff --git a/docs/config.json b/docs/config.json index 5ebdf1256b..ad5c530a82 100644 --- a/docs/config.json +++ b/docs/config.json @@ -13,7 +13,7 @@ "label": "Overview", "to": "getting-started/overview", "addedAt": "2026-04-15", - "updatedAt": "2026-10-22" + "updatedAt": "2026-11-03" }, { "label": "Quick Start", @@ -98,7 +98,12 @@ "label": "Computer Use", "to": "tutorials/computer-use", "addedAt": "2026-10-21", - "updatedAt": "2026-10-22" + "updatedAt": "2026-11-03" + }, + { + "label": "Sign In Without the Model", + "to": "tutorials/computer-use-login", + "addedAt": "2026-11-03" }, { "label": "Subscribe to MCP Events", @@ -151,6 +156,11 @@ "to": "framework/react/examples/computer-use", "addedAt": "2026-10-21" }, + { + "label": "Computer Use Login", + "to": "framework/react/examples/computer-use-login", + "addedAt": "2026-11-03" + }, { "label": "MCP Events", "to": "framework/react/examples/mcp-events", @@ -1049,7 +1059,7 @@ "label": "Drivers", "to": "computer-use/drivers", "addedAt": "2026-10-20", - "updatedAt": "2026-10-22" + "updatedAt": "2026-11-03" }, { "label": "Safety Checks", diff --git a/docs/getting-started/overview.md b/docs/getting-started/overview.md index a6a08355d6..5592add865 100644 --- a/docs/getting-started/overview.md +++ b/docs/getting-started/overview.md @@ -90,6 +90,7 @@ Each tutorial builds one small app from start to finish: - [Subagents](../tutorials/subagents): a researcher, an SEO agent, and a writer in one blog-writing chat. - [Build an MCP Server](../tutorials/mcp-server): serve a tool, a resource, and a prompt. - [Computer Use](../tutorials/computer-use): let the model use a web page from screenshots. +- [Sign In Without the Model](../tutorials/computer-use-login): let the model sign in to a site, and keep the password away from it. ## Reference diff --git a/docs/tutorials/computer-use-login.md b/docs/tutorials/computer-use-login.md new file mode 100644 index 0000000000..7080698223 --- /dev/null +++ b/docs/tutorials/computer-use-login.md @@ -0,0 +1,490 @@ +--- +title: Sign In Without the Model +id: computer-use-login +order: 9 +description: "Let a computer use agent sign in to a site, and keep the password away from the model. The user types the password in a card, and the server fills the form in a Docker desktop." +keywords: + - tanstack ai + - tutorial + - computer use + - sign in + - password + - requestLogin + - sandboxComputer + - docker + - harness +--- + +Give the model a `requestLogin` tool with `needsApproval: true`. When a page asks for a password, the run stops and your chat shows a login card. The server fills the form in the browser, so the model never sees the password. + +This tutorial uses React, TanStack Start, the harness, OpenAI, and a Docker desktop with Chromium. It takes about 30 minutes. Start Docker before you begin. + +## 1. Create a Start app + +```bash +npx @tanstack/cli@latest create +``` + +Pick React. Then install the packages: + + + +react: @tanstack/ai @tanstack/ai-react @tanstack/ai-openai @tanstack/ai-harness @tanstack/ai-persistence @tanstack/ai-computer-use @tanstack/ai-computer-sandbox @tanstack/ai-sandbox-docker zod + + + +Put your OpenAI key in `.env.local`: + +```bash +OPENAI_API_KEY=sk-... +``` + +## Client and server + +- The **desktop** is a Docker container with a screen and Chromium. Each chat thread gets its own container. +- The **server** runs the harness. The model clicks and types on the desktop with the `computer` tool. The `requestLogin` tool fills the sign-in form. +- The **client** shows the chat and the login card. It sends the password to its own route, not to the chat. + +The approval of a tool call goes into the harness log. So the card approves the call with only "yes", and the password takes another path. + +## 2. Build the desktop + +Create a `sandbox` folder in your app with four files. + +`sandbox/Dockerfile` installs the screen, the tools of the sandbox computer driver, Chromium, and Python: + +```dockerfile +FROM debian:bookworm-slim + +RUN printf 'Acquire::Retries "10";\nAcquire::ForceIPv4 "true";\n' > /etc/apt/apt.conf.d/80retries \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + xvfb xdotool scrot x11-utils chromium fonts-dejavu-core \ + python3 python3-websockets ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +COPY policy.json /etc/chromium/policies/managed/policy.json +COPY desktop.sh login.py /opt/desktop/ +RUN chmod +x /opt/desktop/desktop.sh +``` + +`sandbox/policy.json` keeps passwords off the screen. The model sees the screen and moves the mouse, so Chromium must not offer to show a saved password: + +```json +{ + "PasswordManagerEnabled": false, + "AutofillAddressEnabled": false, + "AutofillCreditCardEnabled": false, + "URLBlocklist": ["javascript://*"] +} +``` + +`sandbox/desktop.sh` is the main process of the container. It starts the screen and Chromium: + +```bash +#!/bin/sh +rm -f /tmp/.X0-lock +Xvfb :0 -ac -screen 0 1024x768x24 -nolisten tcp & +until DISPLAY=:0 xdpyinfo >/dev/null 2>&1; do sleep 0.2; done + +while true; do + # --no-sandbox: Chromium runs as root in the container, which is the sandbox. + DISPLAY=:0 chromium --no-sandbox --test-type --no-first-run \ + --no-default-browser-check --disable-dev-shm-usage \ + --user-data-dir=/root/chromium-profile \ + --remote-debugging-port=9222 \ + --window-position=0,0 --window-size=1024,768 about:blank + sleep 1 +done +``` + +- `--user-data-dir` keeps the cookies. A restarted container is still signed in. +- `--remote-debugging-port` listens on 127.0.0.1 inside the container only. No port leaves the container. + +`sandbox/login.py` signs in. Copy it from the [example](https://github.com/TanStack/ai/blob/main/examples/react/computer-use-login/sandbox/login.py). It reads the site, the username, and the password from its environment, then talks to Chromium over the debugging port: + +1. It opens the site in the current tab. +2. It stops if the page is not on the origin of the site, so a redirect cannot get the password. +3. It fills the username field and the password field, then submits the form. +4. It clears every password field, and prints the address of the page. + +Build the image: + +```bash +docker build -t computer-use-login-desktop sandbox +``` + +## 3. Share the tool definition + +The server and the browser both import the tool definition. Create `src/request-login.ts`: + +```typescript ignore +import { toolDefinition } from '@tanstack/ai' +import { z } from 'zod' + +export const requestLoginTool = toolDefinition({ + name: 'requestLogin', + description: + 'Ask the user to sign in to a site. The user types the username and password in a form that you never see, and the server fills the sign-in form in the browser. Call it when a page asks for a password. Pass the address of the sign-in page.', + // The run stops until the user answers the login card. + needsApproval: true, + inputSchema: z.object({ + site: z + .url({ protocol: /^https?$/ }) + .describe( + 'The address of the sign-in page, for example https://example.com/login', + ), + }), +}) +``` + +## 4. Set up the harness + +Create `src/harness.ts`. It has the Docker provider, two stores, the tool, and the harness. + +First the provider and the sandbox ids. `sandboxIds` keeps the container of each thread, so the thread finds its signed-in browser again later: + +```typescript ignore +import { mkdir, readFile, writeFile } from 'node:fs/promises' +import { dirname, resolve } from 'node:path' +import { createHarnessHost, defineHarness } from '@tanstack/ai-harness' +import { permissions } from '@tanstack/ai-harness/plugins' +import { createHarnessHandler } from '@tanstack/ai-harness/server' +import { computerUse } from '@tanstack/ai-computer-use/harness' +import { sandboxComputer } from '@tanstack/ai-computer-sandbox' +import { dockerSandbox } from '@tanstack/ai-sandbox-docker' +import { openaiText } from '@tanstack/ai-openai' +import { memoryPersistence } from '@tanstack/ai-persistence' +import { z } from 'zod' +import { requestLoginTool } from './request-login' + +const provider = dockerSandbox({ + image: 'computer-use-login-desktop', + keepAliveCommand: ['/opt/desktop/desktop.sh'], +}) + +// The container id of each thread, in a file. +const SANDBOX_IDS_FILE = resolve('.data/sandbox-ids.json') +const sandboxIdsSchema = z.record(z.string(), z.string()) +let sandboxIdsWrite: Promise = Promise.resolve() + +async function readSandboxIds() { + try { + return sandboxIdsSchema.parse( + JSON.parse(await readFile(SANDBOX_IDS_FILE, 'utf8')), + ) + } catch (error) { + if (error instanceof Error && 'code' in error && error.code === 'ENOENT') { + return {} + } + throw error + } +} + +const sandboxIds = { + get: async (threadId: string) => (await readSandboxIds())[threadId], + set: (threadId: string, sandboxId: string) => { + // One write at a time, so two threads do not lose each other's id. + sandboxIdsWrite = sandboxIdsWrite.then(async () => { + const ids = await readSandboxIds() + await mkdir(dirname(SANDBOX_IDS_FILE), { recursive: true }) + await writeFile( + SANDBOX_IDS_FILE, + JSON.stringify({ ...ids, [threadId]: sandboxId }), + ) + }) + return sandboxIdsWrite + }, +} +``` + +Then the logins. The card sends a login, and the tool takes it once. A login that nobody takes is deleted after 5 minutes: + +```typescript ignore +const LOGIN_TTL_MS = 5 * 60_000 +const logins = new Map< + string, + { + site: string + username: string + password: string + timer: ReturnType + } +>() + +export const loginSchema = z.object({ + threadId: z.string().min(1).max(200), + site: z.url({ protocol: /^https?$/ }).max(2000), + username: z.string().min(1).max(500), + password: z.string().min(1).max(500), +}) + +export function saveLogin({ threadId, ...login }: z.infer) { + clearTimeout(logins.get(threadId)?.timer) + const timer = setTimeout(() => logins.delete(threadId), LOGIN_TTL_MS) + timer.unref() + logins.set(threadId, { ...login, timer }) +} + +function takeLogin(threadId: string, site: string) { + const login = logins.get(threadId) + logins.delete(threadId) + clearTimeout(login?.timer) + if (!login) { + throw new Error( + 'The user gave no username and password. Call requestLogin again.', + ) + } + if (new URL(login.site).origin !== new URL(site).origin) { + throw new Error( + `The user signed in for ${new URL(login.site).origin}, not ${new URL(site).origin}.`, + ) + } + return login +} +``` + +Then the tool. It runs after the user approves the card: + +```typescript ignore +const loginResultSchema = z.object({ + url: z.string(), + hasPasswordField: z.boolean(), +}) + +const requestLogin = requestLoginTool.server(async ({ site }, context) => { + const threadId = context?.threadId + if (!threadId) throw new Error('requestLogin runs only in a chat thread.') + const login = takeLogin(threadId, site) + + // The user can answer the card hours later. `resume` starts a stopped + // container again. + const id = await sandboxIds.get(threadId) + const sandbox = id === undefined ? null : await provider.resume({ id }) + if (!sandbox) { + throw new Error( + 'This thread has no computer. Take a screenshot, then call requestLogin again.', + ) + } + + // The password goes in the environment of one process, not on its command line. + const result = await sandbox.process.exec('python3 /opt/desktop/login.py', { + env: { + LOGIN_SITE: site, + LOGIN_USERNAME: login.username, + LOGIN_PASSWORD: login.password, + }, + }) + if (result.exitCode !== 0) { + throw new Error(`The sign-in failed: ${result.stderr.trim()}`) + } + const page = loginResultSchema.parse(JSON.parse(result.stdout)) + return page.hasPasswordField + ? `The form was sent, but ${page.url} still shows a password field. The sign-in may have failed. Take a screenshot to check.` + : `Signed in. The browser shows ${page.url}. Take a screenshot to see it.` +}) +``` + +- `takeLogin` deletes the login, so the server uses each password once. +- The tool result has no password. The model gets only the address of the page. + +Last, the harness and its HTTP handler: + +```typescript ignore +const computer = computerUse({ + driver: sandboxComputer({ provider, sandboxIds }), + screenshotDelay: 1000, +}) + +const agent = defineHarness({ + name: 'computer-use-login', + adapter: openaiText('gpt-6.1-sol'), + systemPrompts: [ + 'You use a Linux desktop with Chromium on it. When a page asks you to sign in, call requestLogin with the address of the sign-in page. Never ask for a password in the chat, and never type one.', + ], + tools: [requestLogin], + plugins: () => [permissions(), computer], +}) + +const host = createHarnessHost({ persistence: memoryPersistence() }) + +export const handler = createHarnessHandler({ + host, + harness: agent, + openAccess: true, +}) +``` + +- `sandboxComputer` makes one container for each thread. With `sandboxIds`, it keeps the container and reconnects to it later. +- `memoryPersistence()` keeps the chat in memory. A server restart loses the chat, but not the container. For a user who comes back the next day, use a persistence store in your database. See [Chat persistence](../persistence/chat-persistence). +- `openAccess` serves every request with no login. Use it only on your own machine. See [Serve with no login](../harness/access#serve-with-no-login). + +The Docker provider loads `dockerode`, which has native addons. Keep it out of the bundle in `vite.config.ts`: + +```typescript ignore +export default defineConfig({ + optimizeDeps: { exclude: ['dockerode'] }, + ssr: { external: ['dockerode'] }, + build: { rollupOptions: { external: ['dockerode'] } }, + // ...your plugins +}) +``` + +## 5. Add the two routes + +Create `src/routes/api.harness.$.ts`. It serves the harness under `/api/harness`: + +```typescript ignore +import { createFileRoute } from '@tanstack/react-router' +import { handler } from '@/harness' + +export const Route = createFileRoute('/api/harness/$')({ + server: { + handlers: { + GET: ({ request }) => handler(request), + POST: ({ request }) => handler(request), + }, + }, +}) +``` + +Create `src/routes/api.login.ts`. The card sends the login here. The route checks the input, keeps the login in memory, and logs nothing: + +```typescript ignore +import { createFileRoute } from '@tanstack/react-router' +import { loginSchema, saveLogin } from '@/harness' + +export const Route = createFileRoute('/api/login')({ + server: { + handlers: { + POST: async ({ request }) => { + const body: unknown = await request.json().catch(() => undefined) + const login = loginSchema.safeParse(body) + if (!login.success) { + return Response.json( + { error: 'Send a thread id, a site, a username, and a password.' }, + { status: 400 }, + ) + } + saveLogin(login.data) + return new Response(null, { status: 204 }) + }, + }, + }, +}) +``` + +## 6. Show the login card + +In `src/routes/index.tsx`, give `useChat` the tool definition. The tool has no browser code, but `useChat` now types its approval: + +```tsx ignore +import { fetchServerSentEvents, useChat } from '@tanstack/ai-react' +import { requestLoginTool } from '@/request-login' + +const THREAD_ID = 'computer-use-login' + +const { messages, sendMessage, interrupts } = useChat({ + connection: fetchServerSentEvents('/api/harness/run'), + threadId: THREAD_ID, + tools: [requestLoginTool.client()], + // Load the thread from the server after a reload. + persistence: true, +}) +``` + +Add the card. It sends the login to `/api/login` first. Then it approves the tool call: + +```tsx ignore +function LoginCard({ + site, + onAnswer, +}: { + site: string + onAnswer: (approved: boolean) => void +}) { + const [username, setUsername] = useState('') + const [password, setPassword] = useState('') + const [error, setError] = useState() + + const signIn = async (event: React.FormEvent) => { + event.preventDefault() + setError(undefined) + try { + const response = await fetch('/api/login', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ threadId: THREAD_ID, site, username, password }), + }) + if (!response.ok) { + throw new Error(`The server did not take the login (${response.status}).`) + } + setPassword('') + onAnswer(true) + } catch (sendError) { + setError(sendError instanceof Error ? sendError.message : String(sendError)) + } + } + + return ( +
+

Sign in to {new URL(site).origin}

+ setUsername(event.target.value)} + autoComplete="username" + aria-label="Username" + required + /> + setPassword(event.target.value)} + autoComplete="current-password" + aria-label="Password" + required + /> + {error ?

{error}

: null} + + +
+ ) +} +``` + +Show a card for each approval of `requestLogin`, below the messages: + +```tsx ignore +{interrupts.map((interrupt) => + interrupt.kind === 'tool-approval' ? ( + interrupt.resolveInterrupt(approved)} + /> + ) : null, +)} +``` + +The harness keeps each screenshot in its media store. A tool result has a `harness-media:` URL for it. Show it from the `media` path of the handler: + +```tsx ignore +const src = `/api/harness/media?${new URLSearchParams({ threadId: THREAD_ID, id })}` +``` + +The example has the full page, with the screenshots and the card for OpenAI safety checks. + +## 7. Sign in + +Start the app with `npm run dev`. Send this task: + +```text +Open https://the-internet.herokuapp.com/login in Chromium and sign in. +When you are signed in, say what the page shows and stop. +``` + +The model opens the practice page and calls `requestLogin`. Type `tomsmith` and `SuperSecretPassword!` in the card, then click Sign in. The model says that the page shows the Secure Area. Your password is not in the chat, the tool result, or the harness log. + +The full example is on GitHub: [TanStack/ai `examples/react/computer-use-login`](https://github.com/TanStack/ai/tree/main/examples/react/computer-use-login). To host it, see [Run it on a server](https://github.com/TanStack/ai/tree/main/examples/react/computer-use-login#run-it-on-a-server) in its README. diff --git a/docs/tutorials/computer-use.md b/docs/tutorials/computer-use.md index 0d81a5041f..adfd184417 100644 --- a/docs/tutorials/computer-use.md +++ b/docs/tutorials/computer-use.md @@ -308,3 +308,5 @@ A Chromium window opens on tanstack.com. The model clicks through the site, and The full example is on GitHub: [TanStack/ai `examples/react/computer-use`](https://github.com/TanStack/ai/tree/main/examples/react/computer-use). It has the same task in the input box. To use Anthropic, change only the adapter. To run a full desktop in a sandbox, see [Drivers](../computer-use/drivers). + +To let the model sign in to a site without seeing the password, open [Sign In Without the Model](./computer-use-login). diff --git a/examples/README.md b/examples/README.md index 58ce2c069c..79e15f3106 100644 --- a/examples/README.md +++ b/examples/README.md @@ -15,6 +15,7 @@ Choose an example based on your use case: - **Want an agent in the terminal?** → [Harness CLI](#harness-cli) - **Want a signed MCP webhook?** → [MCP Events](#mcp-events) - **Want the model to use a web page?** → [Computer Use](#computer-use) +- **Want the model to sign in without seeing the password?** → [Computer Use Login](#computer-use-login) - **Want a full-stack TypeScript app?** → [TanStack Chat (ts-react-chat)](#tanstack-chat-ts-react-chat) - **Need a live world stream?** → [World generation (ts-react-media)](#world-generation-ts-react-media) - **Need a vanilla JS frontend?** → [Vanilla Chat](#vanilla-chat) @@ -204,6 +205,25 @@ Set `OPENAI_API_KEY` in `.env.local`. Open http://localhost:3100. Click Send. --- +### Computer Use Login + +A TanStack Start app on the harness. The model uses a Docker desktop with Chromium. When a site asks for a password, a card in the chat asks you, and the server fills the form. The model never sees the password. + +**Path:** `examples/react/computer-use-login` + +**Getting Started:** + +```bash +pnpm --filter computer-use-login build:desktop +pnpm --filter computer-use-login dev +``` + +Start Docker first. Set `OPENAI_API_KEY` in `.env.local`. Open http://localhost:3100. Click Send. + +📖 [Full Documentation](react/computer-use-login/README.md) + +--- + ### TanStack Chat (ts-react-chat) A full-featured chat application built with the TanStack ecosystem. diff --git a/examples/react/computer-use-login/.env.example b/examples/react/computer-use-login/.env.example new file mode 100644 index 0000000000..00b69dfa39 --- /dev/null +++ b/examples/react/computer-use-login/.env.example @@ -0,0 +1,3 @@ +# The model reads the screenshots and picks each click. Get a key at +# https://platform.openai.com/api-keys +OPENAI_API_KEY= diff --git a/examples/react/computer-use-login/.gitignore b/examples/react/computer-use-login/.gitignore new file mode 100644 index 0000000000..74d4f47876 --- /dev/null +++ b/examples/react/computer-use-login/.gitignore @@ -0,0 +1,8 @@ +node_modules +dist +.env +.env.local +.nitro +.tanstack +.output +.data diff --git a/examples/react/computer-use-login/README.md b/examples/react/computer-use-login/README.md new file mode 100644 index 0000000000..9c34e2b06d --- /dev/null +++ b/examples/react/computer-use-login/README.md @@ -0,0 +1,62 @@ +# Computer Use Login + +Your agent uses a desktop, and a site asks it for a password. In this app, the model calls `requestLogin`. You type the username and password in a card in the chat. The server fills the form in the browser, and the model never sees the password. + +The desktop is a Docker container with Chromium on it. Each chat thread keeps its own container, so it stays signed in when you come back later. + +## Run it + +1. Start Docker. +2. From the repo root, run `pnpm install`. +3. Build the desktop image one time: `pnpm --filter computer-use-login build:desktop`. +4. Copy `.env.example` to `.env.local`, then set `OPENAI_API_KEY`. Get a key at https://platform.openai.com/api-keys. +5. Run `pnpm --filter computer-use-login dev`. +6. Open http://localhost:3100 and click Send. + +The model opens a practice sign-in page. When the login card shows, type `tomsmith` and `SuperSecretPassword!`, then click Sign in. The model then says what the page shows. + +## How the password stays away from the model + +1. The model calls `requestLogin({ site })`. The tool has `needsApproval: true`, so the run stops, and the login card shows. +2. The card sends the username and password to `/api/login`. That route keeps them in memory for 5 minutes, and logs nothing. +3. The card then approves the tool call. The approval is only "yes". The harness writes approvals to its log, so a password must never be in one. +4. The tool takes the login from memory and deletes it. It runs `sandbox/login.py` in the container, with the password in the environment of that one process, not on its command line. +5. The script opens the site in Chromium and checks its origin. It fills the username and the password field, then submits the form. Then it clears every password field. +6. The model gets "Signed in" and the address of the page. + +The model sees the screen, so Chromium must never show a password. `sandbox/policy.json` turns off the password manager and `javascript:` URLs. Chromium listens for its debugging protocol only inside the container. No port leaves it. + +## Files + +- `src/harness.ts`: the harness, the Docker provider, the `requestLogin` tool, and the two stores. +- `src/request-login.ts`: the tool definition that the server and the browser share. +- `src/routes/api.login.ts`: the route that takes the username and password. +- `src/routes/index.tsx`: the chat and the login card. +- `sandbox/`: the desktop image. `desktop.sh` starts the screen and Chromium. `login.py` fills the form. + +## Run it on a server + +The Docker setup runs on your own machine. To host the app, use a sandbox provider that pauses a sandbox and keeps its memory, like E2B: + +```ts +import { e2bSandbox } from '@tanstack/ai-sandbox-e2b' + +const provider = e2bSandbox({ + template: 'your-desktop-template', + onTimeout: 'pause', +}) +``` + +Build an E2B template from `sandbox/Dockerfile`, and start `desktop.sh` in it. Then move the two stores of `src/harness.ts` to your database: the sandbox id of each thread, and the logins (with a time limit). + +This app keeps the chat in `memoryPersistence()`. When the server restarts, the chat is gone, but the container stays. For a user who comes back the next day, give `createHarnessHost` a persistence store in your database. See [Chat persistence](https://tanstack.com/ai/latest/docs/persistence/chat-persistence). + +Cloudflare containers do not fit this app. Their disk is empty after each restart, so the browser signs out. + +## Not in this app + +- Two-factor codes. +- A live view of the desktop for a captcha. +- A saved Chromium profile in sandbox snapshots. + +CAUTION: The harness route has no login, and the browser can open every address that your server can reach. Run this app only on your own machine. diff --git a/examples/react/computer-use-login/package.json b/examples/react/computer-use-login/package.json new file mode 100644 index 0000000000..188c29a97a --- /dev/null +++ b/examples/react/computer-use-login/package.json @@ -0,0 +1,37 @@ +{ + "name": "computer-use-login", + "private": true, + "type": "module", + "scripts": { + "dev": "vite dev --port 3100", + "build": "vite build", + "build:desktop": "docker build -t computer-use-login-desktop sandbox", + "test:types": "tsc" + }, + "dependencies": { + "@tailwindcss/vite": "^4.1.18", + "@tanstack/ai": "workspace:*", + "@tanstack/ai-computer-sandbox": "workspace:*", + "@tanstack/ai-computer-use": "workspace:*", + "@tanstack/ai-harness": "workspace:*", + "@tanstack/ai-openai": "workspace:*", + "@tanstack/ai-persistence": "workspace:*", + "@tanstack/ai-react": "workspace:*", + "@tanstack/ai-sandbox-docker": "workspace:*", + "@tanstack/react-router": "^1.170.41", + "@tanstack/react-start": "^1.168.60", + "@tanstack/router-plugin": "^1.168.42", + "react": "^19.2.3", + "react-dom": "^19.2.3", + "tailwindcss": "^4.1.18", + "zod": "^4.2.0" + }, + "devDependencies": { + "@types/node": "^24.10.1", + "@types/react": "^19.2.7", + "@types/react-dom": "^19.2.3", + "@vitejs/plugin-react": "^5.2.0", + "typescript": "5.9.3", + "vite": "^8.2.1" + } +} diff --git a/examples/react/computer-use-login/sandbox/Dockerfile b/examples/react/computer-use-login/sandbox/Dockerfile new file mode 100644 index 0000000000..44ded1822d --- /dev/null +++ b/examples/react/computer-use-login/sandbox/Dockerfile @@ -0,0 +1,18 @@ +# The desktop that the model uses: a screen, the tools of the sandbox +# computer driver, Chromium, and the sign-in script. +# Build it from the example folder: docker build -t computer-use-login-desktop sandbox +FROM debian:bookworm-slim + +RUN printf 'Acquire::Retries "10";\nAcquire::ForceIPv4 "true";\n' > /etc/apt/apt.conf.d/80retries \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + xvfb xdotool scrot x11-utils chromium fonts-dejavu-core \ + python3 python3-websockets ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# The model sees the screen and moves the mouse. So Chromium must never show +# a password: no password manager, and no `javascript:` URLs. +# (A DevTools policy would also close the debugging port that login.py uses.) +COPY policy.json /etc/chromium/policies/managed/policy.json +COPY desktop.sh login.py /opt/desktop/ +RUN chmod +x /opt/desktop/desktop.sh diff --git a/examples/react/computer-use-login/sandbox/desktop.sh b/examples/react/computer-use-login/sandbox/desktop.sh new file mode 100644 index 0000000000..2160d62e17 --- /dev/null +++ b/examples/react/computer-use-login/sandbox/desktop.sh @@ -0,0 +1,19 @@ +#!/bin/sh +# The main process of the sandbox. It starts the screen and Chromium. When +# Chromium closes, it starts again. The profile folder keeps the cookies, so a +# restarted container is still signed in. +# Chromium listens for the debugging protocol on 127.0.0.1 inside the +# container only. The sign-in script uses it. No port leaves the container. +rm -f /tmp/.X0-lock +Xvfb :0 -ac -screen 0 1024x768x24 -nolisten tcp & +until DISPLAY=:0 xdpyinfo >/dev/null 2>&1; do sleep 0.2; done + +while true; do + # --no-sandbox: Chromium runs as root in the container, which is the sandbox. + DISPLAY=:0 chromium --no-sandbox --test-type --no-first-run \ + --no-default-browser-check --disable-dev-shm-usage \ + --user-data-dir=/root/chromium-profile \ + --remote-debugging-port=9222 \ + --window-position=0,0 --window-size=1024,768 about:blank + sleep 1 +done diff --git a/examples/react/computer-use-login/sandbox/login.py b/examples/react/computer-use-login/sandbox/login.py new file mode 100644 index 0000000000..64b9701cbe --- /dev/null +++ b/examples/react/computer-use-login/sandbox/login.py @@ -0,0 +1,143 @@ +"""Sign in on the site in the Chromium of this sandbox. + +The server runs this script with LOGIN_SITE, LOGIN_USERNAME, and +LOGIN_PASSWORD in its environment, so the password is never on a command +line. The script talks to Chromium over the debugging protocol on 127.0.0.1: + +1. Open LOGIN_SITE in the current tab. +2. Stop if the page is not on the origin of LOGIN_SITE. +3. Fill the username field and the password field, then submit the form. + +It prints one JSON line: the address of the page after the sign-in, and +whether the page still has a password field. Errors go to stderr. No output +has the password in it. +""" + +import asyncio +import json +import os +import sys +import time +import urllib.request +from urllib.parse import urlsplit + +import websockets + +# Fills the last text field before the password field, then submits the form. +# The native value setter and the input events make frameworks like React +# see the values. +FILL = """(username, password) => { + const visible = (input) => input.offsetParent !== null + const password_ = [...document.querySelectorAll('input[type=password]')].find(visible) + if (!password_) return 'The page has no visible password field.' + if (!password_.form) return 'The password field is not in a form.' + const inputs = [...password_.form.querySelectorAll('input')] + const user = inputs + .slice(0, inputs.indexOf(password_)) + .reverse() + .find((input) => ['text', 'email', 'tel'].includes(input.type) && visible(input)) + if (!user) return 'The form has no username field before the password field.' + const setValue = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value').set + for (const [input, value] of [[user, username], [password_, password]]) { + input.focus() + setValue.call(input, value) + input.dispatchEvent(new Event('input', { bubbles: true })) + input.dispatchEvent(new Event('change', { bubbles: true })) + } + password_.form.requestSubmit() + return 'submitted' +}""" + +TIMEOUT_S = 30 + + +class LoginError(Exception): + pass + + +async def main(): + site = os.environ["LOGIN_SITE"] + username = os.environ["LOGIN_USERNAME"] + password = os.environ["LOGIN_PASSWORD"] + parts = urlsplit(site) + origin = f"{parts.scheme}://{parts.netloc}" + + with urllib.request.urlopen("http://127.0.0.1:9222/json/list") as response: + targets = json.load(response) + page = next((target for target in targets if target["type"] == "page"), None) + if page is None: + raise LoginError("Chromium has no open tab.") + + async with websockets.connect(page["webSocketDebuggerUrl"], max_size=None) as socket: + next_id = 0 + + async def send(method, **params): + nonlocal next_id + next_id += 1 + await socket.send(json.dumps({"id": next_id, "method": method, "params": params})) + while True: + message = json.loads(await socket.recv()) + if message.get("id") == next_id: + if "error" in message: + raise LoginError(f"{method} failed: {message['error']['message']}") + return message["result"] + + async def evaluate(expression): + result = await send( + "Runtime.evaluate", expression=expression, returnByValue=True + ) + # Never print the details: the expression can hold the password. + if "exceptionDetails" in result: + raise LoginError("A script on the page failed.") + return result["result"].get("value") + + async def wait_for(condition): + deadline = time.monotonic() + TIMEOUT_S + while time.monotonic() < deadline: + try: + if await evaluate(condition): + return True + except LoginError: + pass # The page is loading. Try again. + await asyncio.sleep(0.2) + return False + + await send("Page.navigate", url=site) + if not await wait_for("document.readyState === 'complete'"): + raise LoginError(f"{site} did not load in {TIMEOUT_S} s. Nothing was filled.") + # A redirect can leave the site. Then the form belongs to someone else. + page_origin = await evaluate("location.origin") + if page_origin != origin: + raise LoginError(f"The page is on {page_origin}, not {origin}. Nothing was filled.") + + before = await evaluate("location.href") + result = await evaluate(f"({FILL})({json.dumps(username)}, {json.dumps(password)})") + if result != "submitted": + raise LoginError(result) + + # A sign-in usually opens another page. A page that stays put is fine too. + await wait_for( + f"location.href !== {json.dumps(before)} && document.readyState === 'complete'" + ) + # A sign-in that failed can leave the password in the field. Clear it, + # so nothing on the screen can show it. + await evaluate( + "document.querySelectorAll('input[type=password]').forEach((input) => { input.value = '' })" + ) + print( + json.dumps( + { + "url": await evaluate("location.href"), + "hasPasswordField": await evaluate( + "!!document.querySelector('input[type=password]')" + ), + } + ) + ) + + +try: + asyncio.run(main()) +except LoginError as error: + print(error, file=sys.stderr) + sys.exit(1) diff --git a/examples/react/computer-use-login/sandbox/policy.json b/examples/react/computer-use-login/sandbox/policy.json new file mode 100644 index 0000000000..05e27b20b0 --- /dev/null +++ b/examples/react/computer-use-login/sandbox/policy.json @@ -0,0 +1,6 @@ +{ + "PasswordManagerEnabled": false, + "AutofillAddressEnabled": false, + "AutofillCreditCardEnabled": false, + "URLBlocklist": ["javascript://*"] +} diff --git a/examples/react/computer-use-login/src/harness.ts b/examples/react/computer-use-login/src/harness.ts new file mode 100644 index 0000000000..330e481f6b --- /dev/null +++ b/examples/react/computer-use-login/src/harness.ts @@ -0,0 +1,167 @@ +import { mkdir, readFile, writeFile } from 'node:fs/promises' +import { dirname, resolve } from 'node:path' +import { createHarnessHost, defineHarness } from '@tanstack/ai-harness' +import { permissions } from '@tanstack/ai-harness/plugins' +import { createHarnessHandler } from '@tanstack/ai-harness/server' +import { computerUse } from '@tanstack/ai-computer-use/harness' +import { sandboxComputer } from '@tanstack/ai-computer-sandbox' +import { dockerSandbox } from '@tanstack/ai-sandbox-docker' +import { openaiText } from '@tanstack/ai-openai' +import { memoryPersistence } from '@tanstack/ai-persistence' +import { z } from 'zod' +import { requestLoginTool } from './request-login' + +// One Docker container for each chat thread. The image comes from +// `sandbox/Dockerfile`. Its main process starts the screen and Chromium. +const provider = dockerSandbox({ + image: 'computer-use-login-desktop', + keepAliveCommand: ['/opt/desktop/desktop.sh'], +}) + +// The container id of each thread, in a file. A thread keeps its container, +// and so its signed-in browser, also after the server restarts. +// ponytail: one file and one process. Use your database with more servers. +const SANDBOX_IDS_FILE = resolve('.data/sandbox-ids.json') +const sandboxIdsSchema = z.record(z.string(), z.string()) +let sandboxIdsWrite: Promise = Promise.resolve() + +async function readSandboxIds() { + try { + return sandboxIdsSchema.parse( + JSON.parse(await readFile(SANDBOX_IDS_FILE, 'utf8')), + ) + } catch (error) { + if (error instanceof Error && 'code' in error && error.code === 'ENOENT') { + return {} + } + throw error + } +} + +const sandboxIds = { + get: async (threadId: string) => (await readSandboxIds())[threadId], + set: (threadId: string, sandboxId: string) => { + // One write at a time, so two threads do not lose each other's id. + sandboxIdsWrite = sandboxIdsWrite.then(async () => { + const ids = await readSandboxIds() + await mkdir(dirname(SANDBOX_IDS_FILE), { recursive: true }) + await writeFile( + SANDBOX_IDS_FILE, + JSON.stringify({ ...ids, [threadId]: sandboxId }), + ) + }) + return sandboxIdsWrite + }, +} + +// The username and password of each thread, between the login card and the +// sign-in. Only in memory, and never in a log. +// ponytail: one process. With more servers, use a store with a time limit. +const LOGIN_TTL_MS = 5 * 60_000 +const logins = new Map< + string, + { + site: string + username: string + password: string + timer: ReturnType + } +>() + +export const loginSchema = z.object({ + threadId: z.string().min(1).max(200), + site: z.url({ protocol: /^https?$/ }).max(2000), + username: z.string().min(1).max(500), + password: z.string().min(1).max(500), +}) + +/** Keep a username and password for the next sign-in of a thread. */ +export function saveLogin({ threadId, ...login }: z.infer) { + clearTimeout(logins.get(threadId)?.timer) + const timer = setTimeout(() => logins.delete(threadId), LOGIN_TTL_MS) + timer.unref() + logins.set(threadId, { ...login, timer }) +} + +/** Take the saved login of a thread. It is deleted, so it is used once. */ +function takeLogin(threadId: string, site: string) { + const login = logins.get(threadId) + logins.delete(threadId) + clearTimeout(login?.timer) + if (!login) { + throw new Error( + 'The user gave no username and password. Call requestLogin again.', + ) + } + if (new URL(login.site).origin !== new URL(site).origin) { + throw new Error( + `The user signed in for ${new URL(login.site).origin}, not ${new URL(site).origin}.`, + ) + } + return login +} + +const loginResultSchema = z.object({ + url: z.string(), + hasPasswordField: z.boolean(), +}) + +// The model asks for a sign-in, and it never sees the password. +const requestLogin = requestLoginTool.server(async ({ site }, context) => { + const threadId = context?.threadId + if (!threadId) throw new Error('requestLogin runs only in a chat thread.') + const login = takeLogin(threadId, site) + + // The user can answer the card hours later. The container of the thread + // can be stopped by then. `resume` starts it again. + const id = await sandboxIds.get(threadId) + const sandbox = id === undefined ? null : await provider.resume({ id }) + if (!sandbox) { + throw new Error( + 'This thread has no computer. Take a screenshot, then call requestLogin again.', + ) + } + + // The password goes in the environment of one process, not on its command + // line. The script opens the site, checks its origin, then fills the form. + const result = await sandbox.process.exec('python3 /opt/desktop/login.py', { + env: { + LOGIN_SITE: site, + LOGIN_USERNAME: login.username, + LOGIN_PASSWORD: login.password, + }, + }) + if (result.exitCode !== 0) { + throw new Error(`The sign-in failed: ${result.stderr.trim()}`) + } + const page = loginResultSchema.parse(JSON.parse(result.stdout)) + return page.hasPasswordField + ? `The form was sent, but ${page.url} still shows a password field. The sign-in may have failed. Take a screenshot to check.` + : `Signed in. The browser shows ${page.url}. Take a screenshot to see it.` +}) + +// One time, outside `plugins`, so every session shares the containers. +const computer = computerUse({ + driver: sandboxComputer({ provider, sandboxIds }), + // Pages load in a real browser. Give each one time to draw. + screenshotDelay: 1000, +}) + +const agent = defineHarness({ + name: 'computer-use-login', + adapter: openaiText('gpt-6.1-sol'), + systemPrompts: [ + 'You use a Linux desktop with Chromium on it. When a page asks you to sign in, call requestLogin with the address of the sign-in page. Never ask for a password in the chat, and never type one.', + ], + tools: [requestLogin], + plugins: () => [permissions(), computer], +}) + +const host = createHarnessHost({ persistence: memoryPersistence() }) + +// No login: run this app only on your own machine. +export const handler = createHarnessHandler({ + host, + harness: agent, + openAccess: true, +}) diff --git a/examples/react/computer-use-login/src/request-login.ts b/examples/react/computer-use-login/src/request-login.ts new file mode 100644 index 0000000000..8ed6ba9c8a --- /dev/null +++ b/examples/react/computer-use-login/src/request-login.ts @@ -0,0 +1,20 @@ +import { toolDefinition } from '@tanstack/ai' +import { z } from 'zod' + +// The server and the browser both import this definition. The server gives +// it the code (`src/harness.ts`). The browser gets the typed approval card. +export const requestLoginTool = toolDefinition({ + name: 'requestLogin', + description: + 'Ask the user to sign in to a site. The user types the username and password in a form that you never see, and the server fills the sign-in form in the browser. Call it when a page asks for a password. Pass the address of the sign-in page.', + // The run stops until the user answers the login card. The answer is only + // "yes", so the harness log and the persistence keep no secret. + needsApproval: true, + inputSchema: z.object({ + site: z + .url({ protocol: /^https?$/ }) + .describe( + 'The address of the sign-in page, for example https://example.com/login', + ), + }), +}) diff --git a/examples/react/computer-use-login/src/routeTree.gen.ts b/examples/react/computer-use-login/src/routeTree.gen.ts new file mode 100644 index 0000000000..f093a916e8 --- /dev/null +++ b/examples/react/computer-use-login/src/routeTree.gen.ts @@ -0,0 +1,104 @@ +/* eslint-disable */ + +// @ts-nocheck + +// noinspection JSUnusedGlobalSymbols + +// This file was automatically generated by TanStack Router. +// You should NOT make any changes in this file as it will be overwritten. +// Additionally, you should also exclude this file from your linter and/or formatter to prevent it from being checked or modified. + +import { Route as rootRouteImport } from './routes/__root' +import { Route as IndexRouteImport } from './routes/index' +import { Route as ApiLoginRouteImport } from './routes/api.login' +import { Route as ApiHarnessSplatRouteImport } from './routes/api.harness.$' + +const IndexRoute = IndexRouteImport.update({ + id: '/', + path: '/', + getParentRoute: () => rootRouteImport, +} as any) +const ApiLoginRoute = ApiLoginRouteImport.update({ + id: '/api/login', + path: '/api/login', + getParentRoute: () => rootRouteImport, +} as any) +const ApiHarnessSplatRoute = ApiHarnessSplatRouteImport.update({ + id: '/api/harness/$', + path: '/api/harness/$', + getParentRoute: () => rootRouteImport, +} as any) + +export interface FileRoutesByFullPath { + '/': typeof IndexRoute + '/api/login': typeof ApiLoginRoute + '/api/harness/$': typeof ApiHarnessSplatRoute +} +export interface FileRoutesByTo { + '/': typeof IndexRoute + '/api/login': typeof ApiLoginRoute + '/api/harness/$': typeof ApiHarnessSplatRoute +} +export interface FileRoutesById { + __root__: typeof rootRouteImport + '/': typeof IndexRoute + '/api/login': typeof ApiLoginRoute + '/api/harness/$': typeof ApiHarnessSplatRoute +} +export interface FileRouteTypes { + fileRoutesByFullPath: FileRoutesByFullPath + fullPaths: '/' | '/api/login' | '/api/harness/$' + fileRoutesByTo: FileRoutesByTo + to: '/' | '/api/login' | '/api/harness/$' + id: '__root__' | '/' | '/api/login' | '/api/harness/$' + fileRoutesById: FileRoutesById +} +export interface RootRouteChildren { + IndexRoute: typeof IndexRoute + ApiLoginRoute: typeof ApiLoginRoute + ApiHarnessSplatRoute: typeof ApiHarnessSplatRoute +} + +declare module '@tanstack/react-router' { + interface FileRoutesByPath { + '/': { + id: '/' + path: '/' + fullPath: '/' + preLoaderRoute: typeof IndexRouteImport + parentRoute: typeof rootRouteImport + } + '/api/login': { + id: '/api/login' + path: '/api/login' + fullPath: '/api/login' + preLoaderRoute: typeof ApiLoginRouteImport + parentRoute: typeof rootRouteImport + } + '/api/harness/$': { + id: '/api/harness/$' + path: '/api/harness/$' + fullPath: '/api/harness/$' + preLoaderRoute: typeof ApiHarnessSplatRouteImport + parentRoute: typeof rootRouteImport + } + } +} + +const rootRouteChildren: RootRouteChildren = { + IndexRoute: IndexRoute, + ApiLoginRoute: ApiLoginRoute, + ApiHarnessSplatRoute: ApiHarnessSplatRoute, +} +export const routeTree = rootRouteImport + ._addFileChildren(rootRouteChildren) + ._addFileTypes() + +import type { getRouter } from './router.tsx' +import type { createStart } from '@tanstack/react-start' +declare module '@tanstack/react-start' { + interface Register { + ssr: true + router: Awaited> + } +} diff --git a/examples/react/computer-use-login/src/router.tsx b/examples/react/computer-use-login/src/router.tsx new file mode 100644 index 0000000000..a595444648 --- /dev/null +++ b/examples/react/computer-use-login/src/router.tsx @@ -0,0 +1,10 @@ +import { createRouter } from '@tanstack/react-router' +import { routeTree } from './routeTree.gen' + +export const getRouter = () => { + return createRouter({ + routeTree, + scrollRestoration: true, + defaultPreloadStaleTime: 0, + }) +} diff --git a/examples/react/computer-use-login/src/routes/__root.tsx b/examples/react/computer-use-login/src/routes/__root.tsx new file mode 100644 index 0000000000..4fc02422d9 --- /dev/null +++ b/examples/react/computer-use-login/src/routes/__root.tsx @@ -0,0 +1,41 @@ +import { HeadContent, Scripts, createRootRoute } from '@tanstack/react-router' +import appCss from '../styles.css?url' + +export const Route = createRootRoute({ + head: () => ({ + meta: [ + { + charSet: 'utf-8', + }, + { + name: 'viewport', + content: 'width=device-width, initial-scale=1', + }, + { + title: 'Computer Use Login', + }, + ], + links: [ + { + rel: 'stylesheet', + href: appCss, + }, + ], + }), + + shellComponent: RootDocument, +}) + +function RootDocument({ children }: { children: React.ReactNode }) { + return ( + + + + + + {children} + + + + ) +} diff --git a/examples/react/computer-use-login/src/routes/api.harness.$.ts b/examples/react/computer-use-login/src/routes/api.harness.$.ts new file mode 100644 index 0000000000..9072efb42f --- /dev/null +++ b/examples/react/computer-use-login/src/routes/api.harness.$.ts @@ -0,0 +1,12 @@ +import { createFileRoute } from '@tanstack/react-router' +import { handler } from '@/harness' + +// The harness answers every path under /api/harness, like /api/harness/run. +export const Route = createFileRoute('/api/harness/$')({ + server: { + handlers: { + GET: ({ request }) => handler(request), + POST: ({ request }) => handler(request), + }, + }, +}) diff --git a/examples/react/computer-use-login/src/routes/api.login.ts b/examples/react/computer-use-login/src/routes/api.login.ts new file mode 100644 index 0000000000..98e1841b87 --- /dev/null +++ b/examples/react/computer-use-login/src/routes/api.login.ts @@ -0,0 +1,25 @@ +import { createFileRoute } from '@tanstack/react-router' +import { loginSchema, saveLogin } from '@/harness' + +// The login card sends the username and password here, not to the harness. +// The harness logs every answer. This route keeps the login in memory until +// the requestLogin tool uses it, and logs nothing. +export const Route = createFileRoute('/api/login')({ + server: { + handlers: { + POST: async ({ request }) => { + // A body that is not JSON fails the check below too. + const body: unknown = await request.json().catch(() => undefined) + const login = loginSchema.safeParse(body) + if (!login.success) { + return Response.json( + { error: 'Send a thread id, a site, a username, and a password.' }, + { status: 400 }, + ) + } + saveLogin(login.data) + return new Response(null, { status: 204 }) + }, + }, + }, +}) diff --git a/examples/react/computer-use-login/src/routes/index.tsx b/examples/react/computer-use-login/src/routes/index.tsx new file mode 100644 index 0000000000..9d443a52f2 --- /dev/null +++ b/examples/react/computer-use-login/src/routes/index.tsx @@ -0,0 +1,316 @@ +import { useEffect, useRef, useState } from 'react' +import { createFileRoute } from '@tanstack/react-router' +import { fetchServerSentEvents, useChat } from '@tanstack/ai-react' +import { requestLoginTool } from '@/request-login' +import type { UIMessage } from '@tanstack/ai-react' + +const THREAD_ID = 'computer-use-login' + +const TASK = `Open https://the-internet.herokuapp.com/login in Chromium and sign in. +When you are signed in, say what the page shows and stop.` + +type Part = UIMessage['parts'][number] + +const MEDIA_PREFIX = 'harness-media:' + +/** + * The screenshots in one tool result, as `` sources. The harness keeps + * each screenshot in its media store. The message has only its id. + */ +function screenshots(part: Part) { + if (part.type !== 'tool-result' || typeof part.content === 'string') return [] + return part.content.flatMap((content) => { + if (content.type !== 'image') return [] + const { source } = content + if (source.type === 'data') { + return [`data:${source.mimeType};base64,${source.value}`] + } + if (source.value.startsWith(MEDIA_PREFIX)) { + const id = source.value.slice(MEDIA_PREFIX.length) + return [ + `/api/harness/media?${new URLSearchParams({ threadId: THREAD_ID, id })}`, + ] + } + return [source.value] + }) +} + +function Messages({ messages }: { messages: ReadonlyArray }) { + const containerRef = useRef(null) + + useEffect(() => { + if (containerRef.current) { + containerRef.current.scrollTop = containerRef.current.scrollHeight + } + }, [messages]) + + if (!messages.length) { + return ( +
+
+

+ Sign in without the model +

+

+ Click Send. The model opens a sign-in page in a Docker desktop. When + it needs a password, a card asks you. The model never sees what you + type. +

+
+
+ ) + } + + return ( +
+ {messages.map((message) => ( +
+ {message.parts.map((part, index) => { + if (part.type === 'text' && part.content) { + return ( +
+ {part.content} +
+ ) + } + return screenshots(part).map((src) => ( + What the model sees + )) + })} +
+ ))} +
+ ) +} + +/** + * The model asked for a sign-in. The username and password go to + * `/api/login`, not to the chat. The answer to the model is only "yes". + */ +function LoginCard({ + site, + onAnswer, +}: { + site: string + onAnswer: (approved: boolean) => void +}) { + const [username, setUsername] = useState('') + const [password, setPassword] = useState('') + const [error, setError] = useState() + const [isSending, setIsSending] = useState(false) + + const signIn = async (event: React.FormEvent) => { + event.preventDefault() + setIsSending(true) + setError(undefined) + try { + const response = await fetch('/api/login', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ threadId: THREAD_ID, site, username, password }), + }) + if (!response.ok) { + throw new Error( + `The server did not take the login (${response.status}).`, + ) + } + setPassword('') + onAnswer(true) + } catch (sendError) { + setError( + sendError instanceof Error ? sendError.message : String(sendError), + ) + setIsSending(false) + } + } + + return ( +
+

+ Sign in to {new URL(site).origin} +

+

+ The server fills the form in the browser. The model never sees your + password. +

+ setUsername(event.target.value)} + autoComplete="username" + placeholder="Username" + aria-label="Username" + required + className="rounded border border-gray-700 bg-gray-800 px-3 py-2 text-white" + /> + setPassword(event.target.value)} + autoComplete="current-password" + placeholder="Password" + aria-label="Password" + required + className="rounded border border-gray-700 bg-gray-800 px-3 py-2 text-white" + /> + {error ?

{error}

: null} +
+ + +
+
+ ) +} + +/** Another approval, like an OpenAI safety check on a computer action. */ +function ApprovalCard({ + message, + onAnswer, +}: { + message: string | undefined + onAnswer: (approved: boolean) => void +}) { + return ( +
+

{message ?? 'Approve this action?'}

+
+ + +
+
+ ) +} + +function ChatPage() { + const [input, setInput] = useState(TASK) + const { messages, sendMessage, isLoading, error, stop, interrupts } = useChat( + { + connection: fetchServerSentEvents('/api/harness/run'), + threadId: THREAD_ID, + // No browser code: the tool runs on the server. This gives the + // approval of the tool its typed arguments. + tools: [requestLoginTool.client()], + // Load the thread from the server after a reload. + persistence: true, + }, + ) + + const handleSendMessage = () => { + if (!input.trim()) return + sendMessage(input.trim()) + setInput('') + } + + return ( +
+
+ + + {interrupts.map((interrupt) => { + if (interrupt.kind === 'tool-approval') { + return ( + interrupt.resolveInterrupt(approved)} + /> + ) + } + if (interrupt.kind === 'generic') { + return ( + interrupt.resolveInterrupt(approved)} + /> + ) + } + return null + })} + + {error ? ( +
+ {error.message} +
+ ) : null} + +
+
+ {isLoading ? ( +
+ +
+ ) : null} +
+