improvement(async-jobs): warn when Trigger.dev run acceptance is slow

apps/sim/lib/core/async-jobs/backends/trigger-dev.ts

@@ -9,6 +9,7 @@ import { ApiError, runs, type TriggerOptions, tasks } from '@trigger.dev/sdk'
 import { eq } from 'drizzle-orm'
 import { getRecordedJobFailureError } from '@/lib/core/async-jobs/recorded-failure'
 import { resolveTriggerRegion } from '@/lib/core/async-jobs/region'
+import { measureTriggerAcceptance } from '@/lib/core/async-jobs/trigger-acceptance'
 import {
   AsyncJobEnqueueError,
   type EnqueueOptions,
@@ -412,7 +413,9 @@ export class TriggerDevJobQueue implements JobQueueBackend {
 
     let handle: Awaited<ReturnType<typeof tasks.trigger>>
     try {
-      handle = await tasks.trigger(taskId, enrichedPayload, triggerOptions)
+      handle = await measureTriggerAcceptance(taskId, () =>
+        tasks.trigger(taskId, enrichedPayload, triggerOptions)
+      )
     } catch (error) {
       throw classifyTriggerEnqueueError(error)
     }

apps/sim/lib/core/async-jobs/scheduled-pass.ts

@@ -1,4 +1,5 @@
 import type { TriggerOptions } from '@trigger.dev/sdk'
+import { measureTriggerAcceptance } from '@/lib/core/async-jobs/trigger-acceptance'
 
 /**
  * What one cron tick did with a scheduled pass: started none, handed one to Trigger.dev, or
@@ -45,12 +46,15 @@ export async function triggerScheduledPass(trigger: ScheduledPassTrigger): Promi
     import('@/lib/core/async-jobs/region'),
   ])
   const window = Math.floor((trigger.at?.getTime() ?? Date.now()) / trigger.intervalMs)
-  const handle = await tasks.trigger(trigger.taskId, undefined, {
-    idempotencyKey: `${trigger.keyPrefix ?? trigger.taskId}:${window}`,
-    idempotencyKeyTTL: '5m',
-    ...trigger.options,
-    region: await resolveTriggerRegion(),
-  })
+  const region = await resolveTriggerRegion()
+  const handle = await measureTriggerAcceptance(trigger.taskId, () =>
+    tasks.trigger(trigger.taskId, undefined, {
+      idempotencyKey: `${trigger.keyPrefix ?? trigger.taskId}:${window}`,
+      idempotencyKeyTTL: '5m',
+      ...trigger.options,
+      region,
+    })
+  )
   return handle.id
 }
 

apps/sim/lib/core/async-jobs/trigger-acceptance.ts

@@ -0,0 +1,41 @@
+import { createLogger } from '@sim/logger'
+
+/** An operational signal, so it is emitted even where the production default level is ERROR. */
+const logger = createLogger('TriggerAcceptance', { logLevel: 'WARN' })
+
+/**
+ * Trigger.dev normally accepts a run in well under a second. A wait past this means its intake is
+ * stalling, and every enqueue in the process is waiting on it.
+ */
+const SLOW_TRIGGER_ACCEPTANCE_MS = 10_000
+
+/**
+ * Awaits a Trigger.dev run submission and warns when acceptance is slow: once while it is still
+ * pending, so a stall shows up as it happens, and again when it settles, with its duration and
+ * outcome. It never bounds the wait: a deadline would turn a late acceptance into one whose
+ * outcome is unknown.
+ */
+export async function measureTriggerAcceptance<T>(
+  taskId: string,
+  submit: () => Promise<T>
+): Promise<T> {
+  const startedAt = performance.now()
+  const pendingWarning = setTimeout(() => {
+    logger.warn('Trigger.dev run acceptance still pending', {
+      taskId,
+      waitedMs: SLOW_TRIGGER_ACCEPTANCE_MS,
+    })
+  }, SLOW_TRIGGER_ACCEPTANCE_MS)
+  let outcome: 'accepted' | 'failed' = 'failed'
+  try {
+    const result = await submit()
+    outcome = 'accepted'
+    return result
+  } finally {
+    clearTimeout(pendingWarning)
+    const durationMs = Math.round(performance.now() - startedAt)
+    if (durationMs >= SLOW_TRIGGER_ACCEPTANCE_MS) {
+      logger.warn('Slow Trigger.dev run acceptance', { taskId, durationMs, outcome })
+    }
+  }
+}