diff --git a/apps/sim/lib/mothership/tools/handlers/function-execute-session.test.ts b/apps/sim/lib/mothership/tools/handlers/function-execute-session.test.ts index b67f82bec14..fc3363b05f6 100644 --- a/apps/sim/lib/mothership/tools/handlers/function-execute-session.test.ts +++ b/apps/sim/lib/mothership/tools/handlers/function-execute-session.test.ts @@ -146,7 +146,7 @@ describe('executeFunctionExecute session plumbing', () => { expect(mockExecuteTool.mock.calls[1][1].timeout).toBe(90_000) await executeFunctionExecute({ code: 'x', language: 'python', timeout: 45_000 }, BASE_CONTEXT) - expect(mockExecuteTool.mock.calls[2][1].timeout).toBe(300_000) + expect(mockExecuteTool.mock.calls[2][1].timeout).toBe(900_000) }) }) @@ -187,8 +187,9 @@ describe.each([ { timeout: 7, expected: 7_000 }, { timeout: '90', expected: 90_000 }, { timeout: 300, expected: 300_000 }, - { timeout: 601, expected: 300_000 }, - { timeout: 45_000, expected: 300_000 }, + { timeout: 900, expected: 900_000 }, + { timeout: 901, expected: 900_000 }, + { timeout: 45_000, expected: 900_000 }, { timeout: 0, expected: 60_000 }, { timeout: 'invalid', expected: 60_000 }, ])('maps $timeout seconds to $expected milliseconds once', async ({ timeout, expected }) => { diff --git a/apps/sim/lib/mothership/tools/handlers/function-execute.ts b/apps/sim/lib/mothership/tools/handlers/function-execute.ts index bd385fa99e8..276768fe003 100644 --- a/apps/sim/lib/mothership/tools/handlers/function-execute.ts +++ b/apps/sim/lib/mothership/tools/handlers/function-execute.ts @@ -76,7 +76,11 @@ const logger = createLogger('CopilotFunctionExecute') const MAX_MOUNTED_FILES = 500 /** The call budget includes cold allocation and mounting, before the user's code starts. */ const DEFAULT_TIMEOUT_SECONDS = 60 -const MAX_TIMEOUT_SECONDS = 300 +/** + * The longest call the worker may request. Long renders and builds need it whole: cut off at + * five minutes, they fail and start over. The worker's schema decides what a model may ask for. + */ +const MAX_TIMEOUT_SECONDS = 900 /** An input the run asked for that was not mounted, and why; paths are the model's own. */ export interface ExcludedInput { diff --git a/apps/sim/lib/mothership/tools/sandbox-project-files.ts b/apps/sim/lib/mothership/tools/sandbox-project-files.ts index 61241f4c1bc..a6c0fd83016 100644 --- a/apps/sim/lib/mothership/tools/sandbox-project-files.ts +++ b/apps/sim/lib/mothership/tools/sandbox-project-files.ts @@ -17,6 +17,7 @@ import { } from '@/lib/mothership/tools/sandbox-callback' import { recordSandboxResourceEffects, + SANDBOX_SCOPE_STATE_TTL_SECONDS, type SandboxResourceScope, } from '@/lib/mothership/tools/sandbox-resources' import { chatSandboxSessionKey } from '@/lib/mothership/tools/sandbox-session-key' @@ -66,10 +67,11 @@ export async function proxySandboxProjectFileRequest( uploadBinding: { async record(uploadId) { await redis().eval( - "redis.call('SADD', KEYS[1], ARGV[1]); redis.call('EXPIRE', KEYS[1], 600); return 1", + "redis.call('SADD', KEYS[1], ARGV[1]); redis.call('EXPIRE', KEYS[1], ARGV[2]); return 1", 1, bindingKey, - uploadId + uploadId, + SANDBOX_SCOPE_STATE_TTL_SECONDS ) }, async contains(uploadId) { diff --git a/apps/sim/lib/mothership/tools/sandbox-resources.test.ts b/apps/sim/lib/mothership/tools/sandbox-resources.test.ts index f737879fee2..5705bc86847 100644 --- a/apps/sim/lib/mothership/tools/sandbox-resources.test.ts +++ b/apps/sim/lib/mothership/tools/sandbox-resources.test.ts @@ -11,19 +11,33 @@ vi.mock('@/lib/core/config/feature-flags', () => featureFlagsMock) const { values, lists, seen, persist, redis } = vi.hoisted(() => { const values = new Map() + const expiresAt = new Map() const lists = new Map() const seen = new Map>() + const live = (key: string) => { + if ((expiresAt.get(key) ?? Number.POSITIVE_INFINITY) <= Date.now()) { + values.delete(key) + expiresAt.delete(key) + } + return values.has(key) + } return { values, lists, seen, persist: vi.fn(async () => {}), redis: { - set: vi.fn(async (key: string, value: string) => { + set: vi.fn(async (key: string, value: string, _mode?: 'EX', ttlSeconds?: number) => { values.set(key, value) + if (ttlSeconds) expiresAt.set(key, Date.now() + ttlSeconds * 1000) + }), + expire: vi.fn(async (key: string, ttlSeconds: number) => { + if (!live(key)) return 0 + expiresAt.set(key, Date.now() + ttlSeconds * 1000) + return 1 }), - get: vi.fn(async (key: string) => values.get(key) ?? null), - exists: vi.fn(async (key: string) => (values.has(key) ? 1 : 0)), + get: vi.fn(async (key: string) => (live(key) ? (values.get(key) ?? null) : null)), + exists: vi.fn(async (key: string) => (live(key) ? 1 : 0)), del: vi.fn(async (key: string) => { values.delete(key) lists.delete(key) @@ -183,6 +197,36 @@ describe('sandbox resource ownership and publication', () => { }) }) + it('keeps the callback scope live for a call that outlasts its expiry, and revokes it after', async () => { + vi.useFakeTimers() + try { + let release = () => {} + const finished = new Promise((resolve) => { + release = resolve + }) + let token = '' + const call = withSandboxResourceScope( + identity, + new AbortController().signal, + undefined, + async () => { + token = + (await sandboxResourceEndpoint('https://sim.test', args, 'secret')).split('/').at(-1) ?? + '' + await finished + } + ) + await vi.advanceTimersByTimeAsync(15 * 60_000) + expect(await readSandboxResourceScope(token, 'secret')).not.toBeNull() + release() + await vi.advanceTimersByTimeAsync(1_000) + await call + expect(await readSandboxResourceScope(token, 'secret')).toBeNull() + } finally { + vi.useRealTimers() + } + }) + it('revokes on cancellation and never emits an abandoned callback', async () => { const controller = new AbortController() const emit = vi.fn() diff --git a/apps/sim/lib/mothership/tools/sandbox-resources.ts b/apps/sim/lib/mothership/tools/sandbox-resources.ts index 4fb021da7fb..5590f81255e 100644 --- a/apps/sim/lib/mothership/tools/sandbox-resources.ts +++ b/apps/sim/lib/mothership/tools/sandbox-resources.ts @@ -16,6 +16,13 @@ import { isProjectsEnabled } from '@/lib/projects/rollout.server' const logger = createLogger('MothershipSandboxResources') const CONTEXT_TTL_SECONDS = 600 +/** A running call renews its scope well inside the TTL; the TTL only bounds a crashed owner. */ +const CONTEXT_RENEW_MS = (CONTEXT_TTL_SECONDS * 1000) / 3 +/** + * Outlives the longest code call (15 minutes plus its settle margin). State keyed by a scope token + * is reachable only through that token's live context, so outliving the context widens nothing. + */ +export const SANDBOX_SCOPE_STATE_TTL_SECONDS = 20 * 60 const POLL_MS = 200 const scopeSchema = z .object({ @@ -233,10 +240,23 @@ export async function withSandboxResourceScope( () => {}, () => {} ) + const renew = async () => { + if (!active.token || signal.aborted) return + try { + await redisClient().expire(contextKey(active.token), CONTEXT_TTL_SECONDS) + } catch { + logger.warn('Sandbox resource scope renewal deferred', { toolCallId: identity.toolCallId }) + } + } + let renewedAt = Date.now() try { while (running) { await Promise.race([settled, sleep(POLL_MS)]) await drain() + if (Date.now() - renewedAt >= CONTEXT_RENEW_MS) { + renewedAt = Date.now() + await renew() + } } return await execution } finally {