From cfbc3e8b6d8cf2962ab4d6fec4037a2b2e6506da Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 2 Nov 2026 16:20:55 -0800 Subject: [PATCH 1/3] improvement(mothership): let a code call run for up to fifteen minutes A render or build that needs more than five minutes was cut off at the ceiling and failed, and the model started over in smaller pieces or escaped the call with a background process the paused workbench then stalled. Mothership code calls now run up to 900 seconds. The worker's schema still decides what a model may request, so this is inert until the worker raises its own maximum. --- .../tools/handlers/function-execute-session.test.ts | 7 ++++--- apps/sim/lib/mothership/tools/handlers/function-execute.ts | 6 +++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/apps/sim/lib/mothership/tools/handlers/function-execute-session.test.ts b/apps/sim/lib/mothership/tools/handlers/function-execute-session.test.ts index b67f82bec14..fc3363b05f6 100644 --- a/apps/sim/lib/mothership/tools/handlers/function-execute-session.test.ts +++ b/apps/sim/lib/mothership/tools/handlers/function-execute-session.test.ts @@ -146,7 +146,7 @@ describe('executeFunctionExecute session plumbing', () => { expect(mockExecuteTool.mock.calls[1][1].timeout).toBe(90_000) await executeFunctionExecute({ code: 'x', language: 'python', timeout: 45_000 }, BASE_CONTEXT) - expect(mockExecuteTool.mock.calls[2][1].timeout).toBe(300_000) + expect(mockExecuteTool.mock.calls[2][1].timeout).toBe(900_000) }) }) @@ -187,8 +187,9 @@ describe.each([ { timeout: 7, expected: 7_000 }, { timeout: '90', expected: 90_000 }, { timeout: 300, expected: 300_000 }, - { timeout: 601, expected: 300_000 }, - { timeout: 45_000, expected: 300_000 }, + { timeout: 900, expected: 900_000 }, + { timeout: 901, expected: 900_000 }, + { timeout: 45_000, expected: 900_000 }, { timeout: 0, expected: 60_000 }, { timeout: 'invalid', expected: 60_000 }, ])('maps $timeout seconds to $expected milliseconds once', async ({ timeout, expected }) => { diff --git a/apps/sim/lib/mothership/tools/handlers/function-execute.ts b/apps/sim/lib/mothership/tools/handlers/function-execute.ts index bd385fa99e8..276768fe003 100644 --- a/apps/sim/lib/mothership/tools/handlers/function-execute.ts +++ b/apps/sim/lib/mothership/tools/handlers/function-execute.ts @@ -76,7 +76,11 @@ const logger = createLogger('CopilotFunctionExecute') const MAX_MOUNTED_FILES = 500 /** The call budget includes cold allocation and mounting, before the user's code starts. */ const DEFAULT_TIMEOUT_SECONDS = 60 -const MAX_TIMEOUT_SECONDS = 300 +/** + * The longest call the worker may request. Long renders and builds need it whole: cut off at + * five minutes, they fail and start over. The worker's schema decides what a model may ask for. + */ +const MAX_TIMEOUT_SECONDS = 900 /** An input the run asked for that was not mounted, and why; paths are the model's own. */ export interface ExcludedInput { From c2d073348cfaa7c57ee82c8c5965276b0eb59fc6 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 2 Nov 2026 16:36:08 -0800 Subject: [PATCH 2/3] fix(mothership): renew a running code call's sandbox CLI scope past its expiry --- .../tools/sandbox-resources.test.ts | 47 +++++++++++++++++-- .../lib/mothership/tools/sandbox-resources.ts | 15 ++++++ 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/apps/sim/lib/mothership/tools/sandbox-resources.test.ts b/apps/sim/lib/mothership/tools/sandbox-resources.test.ts index f737879fee2..6d51aea014c 100644 --- a/apps/sim/lib/mothership/tools/sandbox-resources.test.ts +++ b/apps/sim/lib/mothership/tools/sandbox-resources.test.ts @@ -11,19 +11,33 @@ vi.mock('@/lib/core/config/feature-flags', () => featureFlagsMock) const { values, lists, seen, persist, redis } = vi.hoisted(() => { const values = new Map() + const expiresAt = new Map() const lists = new Map() const seen = new Map>() + const live = (key: string) => { + if ((expiresAt.get(key) ?? Number.POSITIVE_INFINITY) <= Date.now()) { + values.delete(key) + expiresAt.delete(key) + } + return values.has(key) + } return { values, lists, seen, persist: vi.fn(async () => {}), redis: { - set: vi.fn(async (key: string, value: string) => { + set: vi.fn(async (key: string, value: string, _mode?: 'EX', ttlSeconds?: number) => { values.set(key, value) + if (ttlSeconds) expiresAt.set(key, Date.now() + ttlSeconds * 1000) + }), + expire: vi.fn(async (key: string, ttlSeconds: number) => { + if (!live(key)) return 0 + expiresAt.set(key, Date.now() + ttlSeconds * 1000) + return 1 }), - get: vi.fn(async (key: string) => values.get(key) ?? null), - exists: vi.fn(async (key: string) => (values.has(key) ? 1 : 0)), + get: vi.fn(async (key: string) => (live(key) ? (values.get(key) ?? null) : null)), + exists: vi.fn(async (key: string) => (live(key) ? 1 : 0)), del: vi.fn(async (key: string) => { values.delete(key) lists.delete(key) @@ -183,6 +197,33 @@ describe('sandbox resource ownership and publication', () => { }) }) + it('keeps the callback scope live for a call that outlasts its expiry, and revokes it after', async () => { + vi.useFakeTimers() + try { + const finish = Promise.withResolvers() + let token = '' + const call = withSandboxResourceScope( + identity, + new AbortController().signal, + undefined, + async () => { + token = + (await sandboxResourceEndpoint('https://sim.test', args, 'secret')).split('/').at(-1) ?? + '' + await finish.promise + } + ) + await vi.advanceTimersByTimeAsync(15 * 60_000) + expect(await readSandboxResourceScope(token, 'secret')).not.toBeNull() + finish.resolve() + await vi.advanceTimersByTimeAsync(1_000) + await call + expect(await readSandboxResourceScope(token, 'secret')).toBeNull() + } finally { + vi.useRealTimers() + } + }) + it('revokes on cancellation and never emits an abandoned callback', async () => { const controller = new AbortController() const emit = vi.fn() diff --git a/apps/sim/lib/mothership/tools/sandbox-resources.ts b/apps/sim/lib/mothership/tools/sandbox-resources.ts index 4fb021da7fb..85f102b29d9 100644 --- a/apps/sim/lib/mothership/tools/sandbox-resources.ts +++ b/apps/sim/lib/mothership/tools/sandbox-resources.ts @@ -16,6 +16,8 @@ import { isProjectsEnabled } from '@/lib/projects/rollout.server' const logger = createLogger('MothershipSandboxResources') const CONTEXT_TTL_SECONDS = 600 +/** A running call renews its scope well inside the TTL; the TTL only bounds a crashed owner. */ +const CONTEXT_RENEW_MS = (CONTEXT_TTL_SECONDS * 1000) / 3 const POLL_MS = 200 const scopeSchema = z .object({ @@ -233,10 +235,23 @@ export async function withSandboxResourceScope( () => {}, () => {} ) + const renew = async () => { + if (!active.token || signal.aborted) return + try { + await redisClient().expire(contextKey(active.token), CONTEXT_TTL_SECONDS) + } catch { + logger.warn('Sandbox resource scope renewal deferred', { toolCallId: identity.toolCallId }) + } + } + let renewedAt = Date.now() try { while (running) { await Promise.race([settled, sleep(POLL_MS)]) await drain() + if (Date.now() - renewedAt >= CONTEXT_RENEW_MS) { + renewedAt = Date.now() + await renew() + } } return await execution } finally { From 90ee0e3aa81527f179cb6e4ab024f9a45bb2cc48 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 2 Nov 2026 16:50:03 -0800 Subject: [PATCH 3/3] fix(mothership): keep a long code call's upload bindings for its whole lifetime --- apps/sim/lib/mothership/tools/sandbox-project-files.ts | 6 ++++-- apps/sim/lib/mothership/tools/sandbox-resources.test.ts | 9 ++++++--- apps/sim/lib/mothership/tools/sandbox-resources.ts | 5 +++++ 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/apps/sim/lib/mothership/tools/sandbox-project-files.ts b/apps/sim/lib/mothership/tools/sandbox-project-files.ts index 61241f4c1bc..a6c0fd83016 100644 --- a/apps/sim/lib/mothership/tools/sandbox-project-files.ts +++ b/apps/sim/lib/mothership/tools/sandbox-project-files.ts @@ -17,6 +17,7 @@ import { } from '@/lib/mothership/tools/sandbox-callback' import { recordSandboxResourceEffects, + SANDBOX_SCOPE_STATE_TTL_SECONDS, type SandboxResourceScope, } from '@/lib/mothership/tools/sandbox-resources' import { chatSandboxSessionKey } from '@/lib/mothership/tools/sandbox-session-key' @@ -66,10 +67,11 @@ export async function proxySandboxProjectFileRequest( uploadBinding: { async record(uploadId) { await redis().eval( - "redis.call('SADD', KEYS[1], ARGV[1]); redis.call('EXPIRE', KEYS[1], 600); return 1", + "redis.call('SADD', KEYS[1], ARGV[1]); redis.call('EXPIRE', KEYS[1], ARGV[2]); return 1", 1, bindingKey, - uploadId + uploadId, + SANDBOX_SCOPE_STATE_TTL_SECONDS ) }, async contains(uploadId) { diff --git a/apps/sim/lib/mothership/tools/sandbox-resources.test.ts b/apps/sim/lib/mothership/tools/sandbox-resources.test.ts index 6d51aea014c..5705bc86847 100644 --- a/apps/sim/lib/mothership/tools/sandbox-resources.test.ts +++ b/apps/sim/lib/mothership/tools/sandbox-resources.test.ts @@ -200,7 +200,10 @@ describe('sandbox resource ownership and publication', () => { it('keeps the callback scope live for a call that outlasts its expiry, and revokes it after', async () => { vi.useFakeTimers() try { - const finish = Promise.withResolvers() + let release = () => {} + const finished = new Promise((resolve) => { + release = resolve + }) let token = '' const call = withSandboxResourceScope( identity, @@ -210,12 +213,12 @@ describe('sandbox resource ownership and publication', () => { token = (await sandboxResourceEndpoint('https://sim.test', args, 'secret')).split('/').at(-1) ?? '' - await finish.promise + await finished } ) await vi.advanceTimersByTimeAsync(15 * 60_000) expect(await readSandboxResourceScope(token, 'secret')).not.toBeNull() - finish.resolve() + release() await vi.advanceTimersByTimeAsync(1_000) await call expect(await readSandboxResourceScope(token, 'secret')).toBeNull() diff --git a/apps/sim/lib/mothership/tools/sandbox-resources.ts b/apps/sim/lib/mothership/tools/sandbox-resources.ts index 85f102b29d9..5590f81255e 100644 --- a/apps/sim/lib/mothership/tools/sandbox-resources.ts +++ b/apps/sim/lib/mothership/tools/sandbox-resources.ts @@ -18,6 +18,11 @@ const logger = createLogger('MothershipSandboxResources') const CONTEXT_TTL_SECONDS = 600 /** A running call renews its scope well inside the TTL; the TTL only bounds a crashed owner. */ const CONTEXT_RENEW_MS = (CONTEXT_TTL_SECONDS * 1000) / 3 +/** + * Outlives the longest code call (15 minutes plus its settle margin). State keyed by a scope token + * is reachable only through that token's live context, so outliving the context widens nothing. + */ +export const SANDBOX_SCOPE_STATE_TTL_SECONDS = 20 * 60 const POLL_MS = 200 const scopeSchema = z .object({