From 8a76f7cac939b735f27055791ae5d0b3a77973be Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 19 Oct 2026 13:07:55 -0700 Subject: [PATCH 1/2] improvement(async-jobs): warn when Trigger.dev run acceptance is slow --- .../core/async-jobs/backends/trigger-dev.ts | 5 ++- .../sim/lib/core/async-jobs/scheduled-pass.ts | 16 ++++++---- .../lib/core/async-jobs/trigger-acceptance.ts | 32 +++++++++++++++++++ 3 files changed, 46 insertions(+), 7 deletions(-) create mode 100644 apps/sim/lib/core/async-jobs/trigger-acceptance.ts diff --git a/apps/sim/lib/core/async-jobs/backends/trigger-dev.ts b/apps/sim/lib/core/async-jobs/backends/trigger-dev.ts index a31ea5916b4..7cbb9a343de 100644 --- a/apps/sim/lib/core/async-jobs/backends/trigger-dev.ts +++ b/apps/sim/lib/core/async-jobs/backends/trigger-dev.ts @@ -9,6 +9,7 @@ import { ApiError, runs, type TriggerOptions, tasks } from '@trigger.dev/sdk' import { eq } from 'drizzle-orm' import { getRecordedJobFailureError } from '@/lib/core/async-jobs/recorded-failure' import { resolveTriggerRegion } from '@/lib/core/async-jobs/region' +import { measureTriggerAcceptance } from '@/lib/core/async-jobs/trigger-acceptance' import { AsyncJobEnqueueError, type EnqueueOptions, @@ -412,7 +413,9 @@ export class TriggerDevJobQueue implements JobQueueBackend { let handle: Awaited> try { - handle = await tasks.trigger(taskId, enrichedPayload, triggerOptions) + handle = await measureTriggerAcceptance(taskId, () => + tasks.trigger(taskId, enrichedPayload, triggerOptions) + ) } catch (error) { throw classifyTriggerEnqueueError(error) } diff --git a/apps/sim/lib/core/async-jobs/scheduled-pass.ts b/apps/sim/lib/core/async-jobs/scheduled-pass.ts index 86ac771ea2a..2250946d14f 100644 --- a/apps/sim/lib/core/async-jobs/scheduled-pass.ts +++ b/apps/sim/lib/core/async-jobs/scheduled-pass.ts @@ -1,4 +1,5 @@ import type { TriggerOptions } from '@trigger.dev/sdk' +import { measureTriggerAcceptance } from '@/lib/core/async-jobs/trigger-acceptance' /** * What one cron tick did with a scheduled pass: started none, handed one to Trigger.dev, or @@ -45,12 +46,15 @@ export async function triggerScheduledPass(trigger: ScheduledPassTrigger): Promi import('@/lib/core/async-jobs/region'), ]) const window = Math.floor((trigger.at?.getTime() ?? Date.now()) / trigger.intervalMs) - const handle = await tasks.trigger(trigger.taskId, undefined, { - idempotencyKey: `${trigger.keyPrefix ?? trigger.taskId}:${window}`, - idempotencyKeyTTL: '5m', - ...trigger.options, - region: await resolveTriggerRegion(), - }) + const region = await resolveTriggerRegion() + const handle = await measureTriggerAcceptance(trigger.taskId, () => + tasks.trigger(trigger.taskId, undefined, { + idempotencyKey: `${trigger.keyPrefix ?? trigger.taskId}:${window}`, + idempotencyKeyTTL: '5m', + ...trigger.options, + region, + }) + ) return handle.id } diff --git a/apps/sim/lib/core/async-jobs/trigger-acceptance.ts b/apps/sim/lib/core/async-jobs/trigger-acceptance.ts new file mode 100644 index 00000000000..fcbe2273774 --- /dev/null +++ b/apps/sim/lib/core/async-jobs/trigger-acceptance.ts @@ -0,0 +1,32 @@ +import { createLogger } from '@sim/logger' + +const logger = createLogger('TriggerAcceptance') + +/** + * Trigger.dev normally accepts a run in well under a second. A wait past this means its intake is + * stalling, and every enqueue in the process is waiting on it. + */ +const SLOW_TRIGGER_ACCEPTANCE_MS = 10_000 + +/** + * Awaits a Trigger.dev run submission and warns when acceptance was slow, whether it then + * succeeded or failed. It never bounds the wait: a deadline would turn a late acceptance into one + * whose outcome is unknown. + */ +export async function measureTriggerAcceptance( + taskId: string, + submit: () => Promise +): Promise { + const startedAt = performance.now() + let outcome: 'accepted' | 'failed' = 'failed' + try { + const result = await submit() + outcome = 'accepted' + return result + } finally { + const durationMs = Math.round(performance.now() - startedAt) + if (durationMs >= SLOW_TRIGGER_ACCEPTANCE_MS) { + logger.warn('Slow Trigger.dev run acceptance', { taskId, durationMs, outcome }) + } + } +} From aed46de065c28070d6fb8edc0b9867b1a04de2ac Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 19 Oct 2026 13:18:28 -0700 Subject: [PATCH 2/2] improvement(async-jobs): warn while Trigger.dev acceptance is still pending and emit at WARN in production --- .../lib/core/async-jobs/trigger-acceptance.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/apps/sim/lib/core/async-jobs/trigger-acceptance.ts b/apps/sim/lib/core/async-jobs/trigger-acceptance.ts index fcbe2273774..1949e59316b 100644 --- a/apps/sim/lib/core/async-jobs/trigger-acceptance.ts +++ b/apps/sim/lib/core/async-jobs/trigger-acceptance.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' -const logger = createLogger('TriggerAcceptance') +/** An operational signal, so it is emitted even where the production default level is ERROR. */ +const logger = createLogger('TriggerAcceptance', { logLevel: 'WARN' }) /** * Trigger.dev normally accepts a run in well under a second. A wait past this means its intake is @@ -9,21 +10,29 @@ const logger = createLogger('TriggerAcceptance') const SLOW_TRIGGER_ACCEPTANCE_MS = 10_000 /** - * Awaits a Trigger.dev run submission and warns when acceptance was slow, whether it then - * succeeded or failed. It never bounds the wait: a deadline would turn a late acceptance into one - * whose outcome is unknown. + * Awaits a Trigger.dev run submission and warns when acceptance is slow: once while it is still + * pending, so a stall shows up as it happens, and again when it settles, with its duration and + * outcome. It never bounds the wait: a deadline would turn a late acceptance into one whose + * outcome is unknown. */ export async function measureTriggerAcceptance( taskId: string, submit: () => Promise ): Promise { const startedAt = performance.now() + const pendingWarning = setTimeout(() => { + logger.warn('Trigger.dev run acceptance still pending', { + taskId, + waitedMs: SLOW_TRIGGER_ACCEPTANCE_MS, + }) + }, SLOW_TRIGGER_ACCEPTANCE_MS) let outcome: 'accepted' | 'failed' = 'failed' try { const result = await submit() outcome = 'accepted' return result } finally { + clearTimeout(pendingWarning) const durationMs = Math.round(performance.now() - startedAt) if (durationMs >= SLOW_TRIGGER_ACCEPTANCE_MS) { logger.warn('Slow Trigger.dev run acceptance', { taskId, durationMs, outcome })