Open uptime incidents only after a run of failed checks

packages/core/src/synth/uptime.ts

@@ -6,6 +6,7 @@ import {
   type DailyUptime,
   type HourlyUptime,
   type Incident,
+  type IncidentState,
   type Monitor,
 } from '../uptime.js';
 import { Rng } from './prng.js';
@@ -37,6 +38,7 @@ export const MONITOR_PROFILES: readonly MonitorProfile[] = [
     intervalSec: 60,
     timeoutMs: 10_000,
     degradedAfterMs: 800,
+    failureThreshold: 2,
     baseLatencyMs: 118,
     jitter: 0.22,
     blipRate: 0.00004,
@@ -65,6 +67,7 @@ export const MONITOR_PROFILES: readonly MonitorProfile[] = [
     intervalSec: 60,
     timeoutMs: 10_000,
     degradedAfterMs: 600,
+    failureThreshold: 2,
     baseLatencyMs: 46,
     jitter: 0.3,
     blipRate: 0.00002,
@@ -78,6 +81,7 @@ export const MONITOR_PROFILES: readonly MonitorProfile[] = [
     intervalSec: 60,
     timeoutMs: 10_000,
     degradedAfterMs: 1200,
+    failureThreshold: 2,
     baseLatencyMs: 284,
     jitter: 0.25,
     blipRate: 0.00006,
@@ -112,6 +116,7 @@ export const MONITOR_PROFILES: readonly MonitorProfile[] = [
     intervalSec: 30,
     timeoutMs: 5_000,
     degradedAfterMs: 450,
+    failureThreshold: 2,
     baseLatencyMs: 92,
     jitter: 0.35,
     blipRate: 0.00008,
@@ -135,6 +140,7 @@ export const MONITOR_PROFILES: readonly MonitorProfile[] = [
     intervalSec: 60,
     timeoutMs: 10_000,
     degradedAfterMs: 900,
+    failureThreshold: 2,
     baseLatencyMs: 162,
     jitter: 0.28,
     blipRate: 0.00003,
@@ -156,6 +162,7 @@ export const MONITOR_PROFILES: readonly MonitorProfile[] = [
     intervalSec: 30,
     timeoutMs: 5_000,
     degradedAfterMs: 500,
+    failureThreshold: 2,
     baseLatencyMs: 64,
     jitter: 0.25,
     blipRate: 0.00005,
@@ -192,7 +199,7 @@ export class MonitorSimulator {
   readonly hourly = new Map<number, HourlyUptime & { latencySum: number; ok: number }>();
   readonly recent: CheckResult[] = [];
   readonly incidents: Incident[] = [];
-  private openIncident: Incident | null = null;
+  private incidentState: IncidentState = { open: null, pending: [] };
   private cursor: number;
   private readonly rng: Rng;
   private incidentSeq = 0;
@@ -285,17 +292,18 @@ export class MonitorSimulator {
     }
     if (check.ts >= this.anchor - RAW_RETENTION) this.recent.push(check);
 
-    const { open, changed } = applyCheckToIncident(
-      this.openIncident,
+    const { state, changed } = applyCheckToIncident(
+      this.incidentState,
       check,
+      this.profile.failureThreshold,
       () => `${this.profile.id}-inc-${++this.incidentSeq}`,
     );
     if (changed) {
       const idx = this.incidents.findIndex((i) => i.id === changed.id);
       if (idx >= 0) this.incidents[idx] = changed;
       else this.incidents.push(changed);
     }
-    this.openIncident = open;
+    this.incidentState = state;
   }
 
   private trim(now: number): void {

packages/core/src/uptime.ts

@@ -9,8 +9,13 @@ export interface Monitor {
   timeoutMs: number;
   /** Responses slower than this are "degraded" even when successful. */
   degradedAfterMs: number;
+  /** Failed checks in a row that open an incident; 1 opens one on the first failure. */
+  failureThreshold: number;
 }
 
+/** For monitors created without a `failureThreshold`. */
+export const DEFAULT_FAILURE_THRESHOLD = 2;
+
 export interface CheckResult {
   monitorId: string;
   ts: number;
@@ -146,36 +151,50 @@ export function addToDaily(daily: DailyUptime, check: CheckResult): DailyUptime
   return daily;
 }
 
+export interface IncidentState {
+  /** The incident in progress, `null` while none is open. */
+  open: Incident | null;
+  /** Failed checks in a row that have not opened an incident yet, oldest first. */
+  pending: readonly CheckResult[];
+}
+
 /**
- * Incident state machine: returns what to persist after a check.
- * - failure with no open incident → open one
- * - success with an open incident → resolve it
+ * Incident state machine: returns the next state and what to persist after a check.
+ * - failure with an open incident → extend it
+ * - failure that makes `threshold` in a row → open an incident from the first of them
+ * - any other failure → keep it in `pending`
+ * - success → resolve the open incident, if any, and clear `pending`
  */
 export function applyCheckToIncident(
-  open: Incident | null,
+  state: IncidentState,
   check: CheckResult,
+  threshold: number,
   newId: () => string,
-): { open: Incident | null; changed: Incident | null } {
+): { state: IncidentState; changed: Incident | null } {
+  const { open } = state;
   if (!check.ok) {
     if (open) {
       const next = { ...open, failures: open.failures + 1 };
-      return { open: next, changed: next };
+      return { state: { open: next, pending: [] }, changed: next };
     }
+    const pending = [...state.pending, check];
+    if (pending.length < threshold) return { state: { open: null, pending }, changed: null };
+    const first = pending[0]!;
     const created: Incident = {
       id: newId(),
       monitorId: check.monitorId,
-      startedAt: check.ts,
+      startedAt: first.ts,
       resolvedAt: null,
-      cause: check.error ?? (check.status !== null ? `HTTP ${check.status}` : 'Unknown error'),
-      failures: 1,
+      cause: first.error ?? (first.status !== null ? `HTTP ${first.status}` : 'Unknown error'),
+      failures: pending.length,
     };
-    return { open: created, changed: created };
+    return { state: { open: created, pending: [] }, changed: created };
   }
   if (open) {
     const resolved = { ...open, resolvedAt: check.ts };
-    return { open: null, changed: resolved };
+    return { state: { open: null, pending: [] }, changed: resolved };
   }
-  return { open: null, changed: null };
+  return { state: { open: null, pending: [] }, changed: null };
 }
 
 interface CheckBucket {

packages/core/test/synth.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from 'vitest';
-import { DAY, HOUR, MINUTE, type PageviewEvent } from '../src/index.js';
+import { DAY, HOUR, MINUTE, type CheckResult, type PageviewEvent } from '../src/index.js';
 import {
   MONITOR_PROFILES,
   MonitorSimulator,
@@ -85,4 +85,36 @@ describe('synthetic monitors', () => {
     sim.advance(next + 2 * MINUTE, (c) => emitted.push(c.ts));
     expect(emitted).toEqual([next, next + MINUTE, next + 2 * MINUTE]);
   });
+
+  it('opens an incident only for runs of failureThreshold failed checks, from the first of them', () => {
+    let shortRuns = 0;
+    for (const profile of MONITOR_PROFILES) {
+      expect(profile.failureThreshold).toBe(2);
+      const sim = new MonitorSimulator(profile, ANCHOR, ANCHOR - 90 * DAY);
+      const checks: CheckResult[] = [];
+      sim.advance(ANCHOR, (c) => checks.push(c));
+      const runs: { startedAt: number; resolvedAt: number | null; failures: number }[] = [];
+      let run: (typeof runs)[number] | null = null;
+      for (const c of checks) {
+        if (!c.ok && run) run.failures += 1;
+        else if (!c.ok) {
+          run = { startedAt: c.ts, resolvedAt: null, failures: 1 };
+          runs.push(run);
+        } else if (run) {
+          run.resolvedAt = c.ts;
+          run = null;
+        }
+      }
+      shortRuns += runs.filter((r) => r.failures < 2).length;
+      expect(
+        sim.incidents.map(({ startedAt, resolvedAt, failures }) => ({
+          startedAt,
+          resolvedAt,
+          failures,
+        })),
+      ).toEqual(runs.filter((r) => r.failures >= 2));
+    }
+    // Random single failed checks happen, and none of them opened an incident.
+    expect(shortRuns).toBeGreaterThan(0);
+  });
 });

packages/core/test/uptime.test.ts

@@ -1,6 +1,7 @@
 import { describe, expect, it } from 'vitest';
 import {
   DAY,
+  DEFAULT_FAILURE_THRESHOLD,
   HOUR,
   MINUTE,
   applyCheckToIncident,
@@ -14,6 +15,7 @@ import {
   uptimeRatio,
   type CheckResult,
   type Incident,
+  type IncidentState,
   type Monitor,
 } from '../src/index.js';
 
@@ -25,6 +27,7 @@ const monitor: Monitor = {
   intervalSec: 60,
   timeoutMs: 5_000,
   degradedAfterMs: 500,
+  failureThreshold: 2,
 };
 const NOW = Date.parse('2026-06-10T12:00:00Z');
 const check = (ts: number, ok: boolean, latencyMs = 100): CheckResult => ({
@@ -65,24 +68,101 @@ describe('uptime math', () => {
     expect(currentStatus(monitor, [check(0, true), check(1, false)])).toBe('down');
   });
 
-  it('opens, extends and resolves incidents', () => {
-    let open: Incident | null = null;
-    const changes: Incident[] = [];
-    const seq = [true, false, false, false, true, true];
-    seq.forEach((ok, i) => {
-      const res = applyCheckToIncident(open, check(i * MINUTE, ok), () => 'inc-1');
-      open = res.open;
-      if (res.changed) changes.push(res.changed);
+  /** Feeds checks one minute apart, `false` for a failure; returns the final state and every incident. */
+  function feed(seq: readonly (boolean | CheckResult)[], threshold: number) {
+    let state: IncidentState = { open: null, pending: [] };
+    const incidents = new Map<string, Incident>();
+    seq.forEach((c, i) => {
+      const next = applyCheckToIncident(
+        state,
+        typeof c === 'boolean' ? check(i * MINUTE, c) : c,
+        threshold,
+        () => `inc-${incidents.size + 1}`,
+      );
+      state = next.state;
+      if (next.changed) incidents.set(next.changed.id, next.changed);
     });
-    expect(open).toBeNull();
-    const last = changes.at(-1)!;
-    expect(last).toMatchObject({
-      id: 'inc-1',
+    return { state, incidents: [...incidents.values()] };
+  }
+
+  it('does not open an incident for a single failed check by default', () => {
+    expect(DEFAULT_FAILURE_THRESHOLD).toBe(2);
+    const { state, incidents } = feed([true, false, true, true], DEFAULT_FAILURE_THRESHOLD);
+    expect(incidents).toEqual([]);
+    expect(state).toEqual({ open: null, pending: [] });
+  });
+
+  it('opens an incident at the first of two failed checks in a row and resolves it', () => {
+    const ongoing = feed([true, false, false], 2);
+    expect(ongoing.incidents).toEqual([
+      {
+        id: 'inc-1',
+        monitorId: 'm1',
+        startedAt: MINUTE,
+        resolvedAt: null,
+        cause: 'HTTP 503',
+        failures: 2,
+      },
+    ]);
+    expect(ongoing.state.open).toEqual(ongoing.incidents[0]);
+
+    const resolved = feed([true, false, false, false, true, true], 2);
+    expect(resolved.incidents).toHaveLength(1);
+    expect(resolved.incidents[0]).toMatchObject({
       startedAt: MINUTE,
       resolvedAt: 4 * MINUTE,
       failures: 3,
-      cause: 'HTTP 503',
     });
+    expect(resolved.state).toEqual({ open: null, pending: [] });
+  });
+
+  it('takes the cause from the first failed check of the run', () => {
+    const timeout: CheckResult = {
+      ...check(0, false),
+      status: null,
+      error: 'Timeout after 5s',
+    };
+    const { incidents } = feed([timeout, false], 2);
+    expect(incidents[0]).toMatchObject({ startedAt: 0, cause: 'Timeout after 5s', failures: 2 });
+  });
+
+  it('starts counting again after a successful check', () => {
+    const { state, incidents } = feed([false, true, false, true, false, false, true, false], 3);
+    expect(incidents).toEqual([]);
+    expect(state.pending.map((c) => c.ts)).toEqual([7 * MINUTE]);
+
+    const opened = feed([false, false, true, false, false, false], 3);
+    expect(opened.incidents).toHaveLength(1);
+    expect(opened.incidents[0]).toMatchObject({ startedAt: 3 * MINUTE, failures: 3 });
+  });
+
+  it('continues a run of failed checks passed in as state', () => {
+    const restored: IncidentState = { open: null, pending: [check(0, false)] };
+    const { state, changed } = applyCheckToIncident(restored, check(MINUTE, false), 2, () => 'x');
+    expect(changed).toMatchObject({ id: 'x', startedAt: 0, failures: 2, resolvedAt: null });
+    expect(state).toEqual({ open: changed, pending: [] });
+  });
+
+  it('opens an incident on the first failed check when the threshold is 1', () => {
+    const { incidents } = feed([true, false, true, false, false, false, true, true], 1);
+    expect(incidents).toEqual([
+      {
+        id: 'inc-1',
+        monitorId: 'm1',
+        startedAt: MINUTE,
+        resolvedAt: 2 * MINUTE,
+        cause: 'HTTP 503',
+        failures: 1,
+      },
+      {
+        id: 'inc-2',
+        monitorId: 'm1',
+        startedAt: 3 * MINUTE,
+        resolvedAt: 6 * MINUTE,
+        cause: 'HTTP 503',
+        failures: 3,
+      },
+    ]);
   });
 
   it('buckets checks for sparklines and hourly heatmaps', () => {

server/scripts/seed.ts

@@ -73,6 +73,7 @@ uptime.upsertMonitor({
   intervalSec: 60,
   timeoutMs: 5_000,
   degradedAfterMs: 300,
+  failureThreshold: 2,
 });
 
 console.log(`Done: ${rollups.count().toLocaleString('en')} rollup rows.`);