From 2c572f4648a7f029994599dde5da37d96d0fa9a0 Mon Sep 17 00:00:00 2001
From: sinnercode228 <sinnercode228@users.noreply.github.com>
Date: Fri, 16 Oct 2026 05:54:05 +0000
Subject: [PATCH 1/3] Open uptime incidents only after a configurable run of
 failed checks

---
 packages/core/src/synth/uptime.ts |  16 ++-
 packages/core/src/uptime.ts       |  43 +++++--
 packages/core/test/synth.test.ts  |  34 +++++-
 packages/core/test/uptime.test.ts | 106 ++++++++++++++---
 server/scripts/seed.ts            |   1 +
 server/src/db/database.ts         |   1 +
 server/src/db/uptime-repo.ts      |  30 ++++-
 server/src/routes/admin.ts        |   5 +-
 server/src/uptime/service.ts      |  15 ++-
 server/test/api.test.ts           |   6 +-
 server/test/uptime.test.ts        | 188 +++++++++++++++++++++++++++++-
 web/src/data/demo/engine.ts       |   1 +
 web/test/engine.test.ts           |   7 +-
 13 files changed, 409 insertions(+), 44 deletions(-)

diff --git a/packages/core/src/synth/uptime.ts b/packages/core/src/synth/uptime.ts
index 78dd98e..8425e2e 100644
--- a/packages/core/src/synth/uptime.ts
+++ b/packages/core/src/synth/uptime.ts
@@ -6,6 +6,7 @@ import {
   type DailyUptime,
   type HourlyUptime,
   type Incident,
+  type IncidentState,
   type Monitor,
 } from '../uptime.js';
 import { Rng } from './prng.js';
@@ -37,6 +38,7 @@ export const MONITOR_PROFILES: readonly MonitorProfile[] = [
     intervalSec: 60,
     timeoutMs: 10_000,
     degradedAfterMs: 800,
+    failureThreshold: 2,
     baseLatencyMs: 118,
     jitter: 0.22,
     blipRate: 0.00004,
@@ -65,6 +67,7 @@ export const MONITOR_PROFILES: readonly MonitorProfile[] = [
     intervalSec: 60,
     timeoutMs: 10_000,
     degradedAfterMs: 600,
+    failureThreshold: 2,
     baseLatencyMs: 46,
     jitter: 0.3,
     blipRate: 0.00002,
@@ -78,6 +81,7 @@ export const MONITOR_PROFILES: readonly MonitorProfile[] = [
     intervalSec: 60,
     timeoutMs: 10_000,
     degradedAfterMs: 1200,
+    failureThreshold: 2,
     baseLatencyMs: 284,
     jitter: 0.25,
     blipRate: 0.00006,
@@ -112,6 +116,7 @@ export const MONITOR_PROFILES: readonly MonitorProfile[] = [
     intervalSec: 30,
     timeoutMs: 5_000,
     degradedAfterMs: 450,
+    failureThreshold: 2,
     baseLatencyMs: 92,
     jitter: 0.35,
     blipRate: 0.00008,
@@ -135,6 +140,7 @@ export const MONITOR_PROFILES: readonly MonitorProfile[] = [
     intervalSec: 60,
     timeoutMs: 10_000,
     degradedAfterMs: 900,
+    failureThreshold: 2,
     baseLatencyMs: 162,
     jitter: 0.28,
     blipRate: 0.00003,
@@ -156,6 +162,7 @@ export const MONITOR_PROFILES: readonly MonitorProfile[] = [
     intervalSec: 30,
     timeoutMs: 5_000,
     degradedAfterMs: 500,
+    failureThreshold: 2,
     baseLatencyMs: 64,
     jitter: 0.25,
     blipRate: 0.00005,
@@ -192,7 +199,7 @@ export class MonitorSimulator {
   readonly hourly = new Map<number, HourlyUptime & { latencySum: number; ok: number }>();
   readonly recent: CheckResult[] = [];
   readonly incidents: Incident[] = [];
-  private openIncident: Incident | null = null;
+  private incidentState: IncidentState = { open: null, pending: [] };
   private cursor: number;
   private readonly rng: Rng;
   private incidentSeq = 0;
@@ -285,9 +292,10 @@ export class MonitorSimulator {
     }
     if (check.ts >= this.anchor - RAW_RETENTION) this.recent.push(check);
 
-    const { open, changed } = applyCheckToIncident(
-      this.openIncident,
+    const { state, changed } = applyCheckToIncident(
+      this.incidentState,
       check,
+      this.profile.failureThreshold,
       () => `${this.profile.id}-inc-${++this.incidentSeq}`,
     );
     if (changed) {
@@ -295,7 +303,7 @@ export class MonitorSimulator {
       if (idx >= 0) this.incidents[idx] = changed;
       else this.incidents.push(changed);
     }
-    this.openIncident = open;
+    this.incidentState = state;
   }
 
   private trim(now: number): void {
diff --git a/packages/core/src/uptime.ts b/packages/core/src/uptime.ts
index 65313af..e8c642b 100644
--- a/packages/core/src/uptime.ts
+++ b/packages/core/src/uptime.ts
@@ -9,8 +9,13 @@ export interface Monitor {
   timeoutMs: number;
   /** Responses slower than this are "degraded" even when successful. */
   degradedAfterMs: number;
+  /** Failed checks in a row that open an incident; 1 opens one on the first failure. */
+  failureThreshold: number;
 }
 
+/** For monitors created without a `failureThreshold`. */
+export const DEFAULT_FAILURE_THRESHOLD = 2;
+
 export interface CheckResult {
   monitorId: string;
   ts: number;
@@ -146,36 +151,50 @@ export function addToDaily(daily: DailyUptime, check: CheckResult): DailyUptime
   return daily;
 }
 
+export interface IncidentState {
+  /** The incident in progress, `null` while none is open. */
+  open: Incident | null;
+  /** Failed checks in a row that have not opened an incident yet, oldest first. */
+  pending: readonly CheckResult[];
+}
+
 /**
- * Incident state machine: returns what to persist after a check.
- * - failure with no open incident → open one
- * - success with an open incident → resolve it
+ * Incident state machine: returns the next state and what to persist after a check.
+ * - failure with an open incident → extend it
+ * - failure that makes `threshold` in a row → open an incident from the first of them
+ * - any other failure → keep it in `pending`
+ * - success → resolve the open incident, if any, and clear `pending`
  */
 export function applyCheckToIncident(
-  open: Incident | null,
+  state: IncidentState,
   check: CheckResult,
+  threshold: number,
   newId: () => string,
-): { open: Incident | null; changed: Incident | null } {
+): { state: IncidentState; changed: Incident | null } {
+  const { open } = state;
   if (!check.ok) {
     if (open) {
       const next = { ...open, failures: open.failures + 1 };
-      return { open: next, changed: next };
+      return { state: { open: next, pending: [] }, changed: next };
     }
+    const pending = [...state.pending, check];
+    if (pending.length < threshold) return { state: { open: null, pending }, changed: null };
+    const first = pending[0]!;
     const created: Incident = {
       id: newId(),
       monitorId: check.monitorId,
-      startedAt: check.ts,
+      startedAt: first.ts,
       resolvedAt: null,
-      cause: check.error ?? (check.status !== null ? `HTTP ${check.status}` : 'Unknown error'),
-      failures: 1,
+      cause: first.error ?? (first.status !== null ? `HTTP ${first.status}` : 'Unknown error'),
+      failures: pending.length,
     };
-    return { open: created, changed: created };
+    return { state: { open: created, pending: [] }, changed: created };
   }
   if (open) {
     const resolved = { ...open, resolvedAt: check.ts };
-    return { open: null, changed: resolved };
+    return { state: { open: null, pending: [] }, changed: resolved };
   }
-  return { open: null, changed: null };
+  return { state: { open: null, pending: [] }, changed: null };
 }
 
 interface CheckBucket {
diff --git a/packages/core/test/synth.test.ts b/packages/core/test/synth.test.ts
index bcc32d5..fc1450a 100644
--- a/packages/core/test/synth.test.ts
+++ b/packages/core/test/synth.test.ts
@@ -1,5 +1,5 @@
 import { describe, expect, it } from 'vitest';
-import { DAY, HOUR, MINUTE, type PageviewEvent } from '../src/index.js';
+import { DAY, HOUR, MINUTE, type CheckResult, type PageviewEvent } from '../src/index.js';
 import {
   MONITOR_PROFILES,
   MonitorSimulator,
@@ -85,4 +85,36 @@ describe('synthetic monitors', () => {
     sim.advance(next + 2 * MINUTE, (c) => emitted.push(c.ts));
     expect(emitted).toEqual([next, next + MINUTE, next + 2 * MINUTE]);
   });
+
+  it('opens an incident only for runs of failureThreshold failed checks, from the first of them', () => {
+    let shortRuns = 0;
+    for (const profile of MONITOR_PROFILES) {
+      expect(profile.failureThreshold).toBe(2);
+      const sim = new MonitorSimulator(profile, ANCHOR, ANCHOR - 90 * DAY);
+      const checks: CheckResult[] = [];
+      sim.advance(ANCHOR, (c) => checks.push(c));
+      const runs: { startedAt: number; resolvedAt: number | null; failures: number }[] = [];
+      let run: (typeof runs)[number] | null = null;
+      for (const c of checks) {
+        if (!c.ok && run) run.failures += 1;
+        else if (!c.ok) {
+          run = { startedAt: c.ts, resolvedAt: null, failures: 1 };
+          runs.push(run);
+        } else if (run) {
+          run.resolvedAt = c.ts;
+          run = null;
+        }
+      }
+      shortRuns += runs.filter((r) => r.failures < 2).length;
+      expect(
+        sim.incidents.map(({ startedAt, resolvedAt, failures }) => ({
+          startedAt,
+          resolvedAt,
+          failures,
+        })),
+      ).toEqual(runs.filter((r) => r.failures >= 2));
+    }
+    // Random single failed checks happen, and none of them opened an incident.
+    expect(shortRuns).toBeGreaterThan(0);
+  });
 });
diff --git a/packages/core/test/uptime.test.ts b/packages/core/test/uptime.test.ts
index 26d513c..9af0f53 100644
--- a/packages/core/test/uptime.test.ts
+++ b/packages/core/test/uptime.test.ts
@@ -1,6 +1,7 @@
 import { describe, expect, it } from 'vitest';
 import {
   DAY,
+  DEFAULT_FAILURE_THRESHOLD,
   HOUR,
   MINUTE,
   applyCheckToIncident,
@@ -14,6 +15,7 @@ import {
   uptimeRatio,
   type CheckResult,
   type Incident,
+  type IncidentState,
   type Monitor,
 } from '../src/index.js';
 
@@ -25,6 +27,7 @@ const monitor: Monitor = {
   intervalSec: 60,
   timeoutMs: 5_000,
   degradedAfterMs: 500,
+  failureThreshold: 2,
 };
 const NOW = Date.parse('2026-06-10T12:00:00Z');
 const check = (ts: number, ok: boolean, latencyMs = 100): CheckResult => ({
@@ -65,24 +68,101 @@ describe('uptime math', () => {
     expect(currentStatus(monitor, [check(0, true), check(1, false)])).toBe('down');
   });
 
-  it('opens, extends and resolves incidents', () => {
-    let open: Incident | null = null;
-    const changes: Incident[] = [];
-    const seq = [true, false, false, false, true, true];
-    seq.forEach((ok, i) => {
-      const res = applyCheckToIncident(open, check(i * MINUTE, ok), () => 'inc-1');
-      open = res.open;
-      if (res.changed) changes.push(res.changed);
+  /** Feeds checks one minute apart, `false` for a failure; returns the final state and every incident. */
+  function feed(seq: readonly (boolean | CheckResult)[], threshold: number) {
+    let state: IncidentState = { open: null, pending: [] };
+    const incidents = new Map<string, Incident>();
+    seq.forEach((c, i) => {
+      const next = applyCheckToIncident(
+        state,
+        typeof c === 'boolean' ? check(i * MINUTE, c) : c,
+        threshold,
+        () => `inc-${incidents.size + 1}`,
+      );
+      state = next.state;
+      if (next.changed) incidents.set(next.changed.id, next.changed);
     });
-    expect(open).toBeNull();
-    const last = changes.at(-1)!;
-    expect(last).toMatchObject({
-      id: 'inc-1',
+    return { state, incidents: [...incidents.values()] };
+  }
+
+  it('does not open an incident for a single failed check by default', () => {
+    expect(DEFAULT_FAILURE_THRESHOLD).toBe(2);
+    const { state, incidents } = feed([true, false, true, true], DEFAULT_FAILURE_THRESHOLD);
+    expect(incidents).toEqual([]);
+    expect(state).toEqual({ open: null, pending: [] });
+  });
+
+  it('opens an incident at the first of two failed checks in a row and resolves it', () => {
+    const ongoing = feed([true, false, false], 2);
+    expect(ongoing.incidents).toEqual([
+      {
+        id: 'inc-1',
+        monitorId: 'm1',
+        startedAt: MINUTE,
+        resolvedAt: null,
+        cause: 'HTTP 503',
+        failures: 2,
+      },
+    ]);
+    expect(ongoing.state.open).toEqual(ongoing.incidents[0]);
+
+    const resolved = feed([true, false, false, false, true, true], 2);
+    expect(resolved.incidents).toHaveLength(1);
+    expect(resolved.incidents[0]).toMatchObject({
       startedAt: MINUTE,
       resolvedAt: 4 * MINUTE,
       failures: 3,
-      cause: 'HTTP 503',
     });
+    expect(resolved.state).toEqual({ open: null, pending: [] });
+  });
+
+  it('takes the cause from the first failed check of the run', () => {
+    const timeout: CheckResult = {
+      ...check(0, false),
+      status: null,
+      error: 'Timeout after 5s',
+    };
+    const { incidents } = feed([timeout, false], 2);
+    expect(incidents[0]).toMatchObject({ startedAt: 0, cause: 'Timeout after 5s', failures: 2 });
+  });
+
+  it('starts counting again after a successful check', () => {
+    const { state, incidents } = feed([false, true, false, true, false, false, true, false], 3);
+    expect(incidents).toEqual([]);
+    expect(state.pending.map((c) => c.ts)).toEqual([7 * MINUTE]);
+
+    const opened = feed([false, false, true, false, false, false], 3);
+    expect(opened.incidents).toHaveLength(1);
+    expect(opened.incidents[0]).toMatchObject({ startedAt: 3 * MINUTE, failures: 3 });
+  });
+
+  it('continues a run of failed checks passed in as state', () => {
+    const restored: IncidentState = { open: null, pending: [check(0, false)] };
+    const { state, changed } = applyCheckToIncident(restored, check(MINUTE, false), 2, () => 'x');
+    expect(changed).toMatchObject({ id: 'x', startedAt: 0, failures: 2, resolvedAt: null });
+    expect(state).toEqual({ open: changed, pending: [] });
+  });
+
+  it('opens an incident on the first failed check when the threshold is 1', () => {
+    const { incidents } = feed([true, false, true, false, false, false, true, true], 1);
+    expect(incidents).toEqual([
+      {
+        id: 'inc-1',
+        monitorId: 'm1',
+        startedAt: MINUTE,
+        resolvedAt: 2 * MINUTE,
+        cause: 'HTTP 503',
+        failures: 1,
+      },
+      {
+        id: 'inc-2',
+        monitorId: 'm1',
+        startedAt: 3 * MINUTE,
+        resolvedAt: 6 * MINUTE,
+        cause: 'HTTP 503',
+        failures: 3,
+      },
+    ]);
   });
 
   it('buckets checks for sparklines and hourly heatmaps', () => {
diff --git a/server/scripts/seed.ts b/server/scripts/seed.ts
index 549b7fc..299bc7d 100644
--- a/server/scripts/seed.ts
+++ b/server/scripts/seed.ts
@@ -73,6 +73,7 @@ uptime.upsertMonitor({
   intervalSec: 60,
   timeoutMs: 5_000,
   degradedAfterMs: 300,
+  failureThreshold: 2,
 });
 
 console.log(`Done: ${rollups.count().toLocaleString('en')} rollup rows.`);
diff --git a/server/src/db/database.ts b/server/src/db/database.ts
index ed94fa6..8823d02 100644
--- a/server/src/db/database.ts
+++ b/server/src/db/database.ts
@@ -70,6 +70,7 @@ const MIGRATIONS: readonly string[] = [
      failures     INTEGER NOT NULL
    );
    CREATE INDEX incidents_by_monitor ON incidents (monitor_id, started_at);`,
+  `ALTER TABLE monitors ADD COLUMN failure_threshold INTEGER NOT NULL DEFAULT 2;`,
 ];
 
 export function openDatabase(path: string): Db {
diff --git a/server/src/db/uptime-repo.ts b/server/src/db/uptime-repo.ts
index ce88401..67d8c0c 100644
--- a/server/src/db/uptime-repo.ts
+++ b/server/src/db/uptime-repo.ts
@@ -4,6 +4,7 @@ import {
   type CheckResult,
   type DailyUptime,
   type Incident,
+  type IncidentState,
   type Monitor,
 } from '@pulse/core';
 import { transaction, type Db } from './database.js';
@@ -20,6 +21,7 @@ interface MonitorRow {
   interval_sec: number;
   timeout_ms: number;
   degraded_after_ms: number;
+  failure_threshold: number;
   paused: number;
 }
 
@@ -49,6 +51,7 @@ const toMonitor = (r: MonitorRow): StoredMonitor => ({
   intervalSec: r.interval_sec,
   timeoutMs: r.timeout_ms,
   degradedAfterMs: r.degraded_after_ms,
+  failureThreshold: r.failure_threshold,
   paused: r.paused === 1,
 });
 
@@ -93,11 +96,12 @@ export class UptimeRepository {
   upsertMonitor(m: Monitor, paused = false, now = Date.now()): void {
     this.db
       .prepare(
-        `INSERT INTO monitors (id, name, url, method, interval_sec, timeout_ms, degraded_after_ms, paused, created_at)
-         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+        `INSERT INTO monitors (id, name, url, method, interval_sec, timeout_ms, degraded_after_ms, failure_threshold, paused, created_at)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
          ON CONFLICT(id) DO UPDATE SET name = excluded.name, url = excluded.url, method = excluded.method,
            interval_sec = excluded.interval_sec, timeout_ms = excluded.timeout_ms,
-           degraded_after_ms = excluded.degraded_after_ms, paused = excluded.paused`,
+           degraded_after_ms = excluded.degraded_after_ms, failure_threshold = excluded.failure_threshold,
+           paused = excluded.paused`,
       )
       .run(
         m.id,
@@ -107,6 +111,7 @@ export class UptimeRepository {
         m.intervalSec,
         m.timeoutMs,
         m.degradedAfterMs,
+        m.failureThreshold,
         paused ? 1 : 0,
         now,
       );
@@ -121,6 +126,25 @@ export class UptimeRepository {
     return row ? toIncident(row) : null;
   }
 
+  /**
+   * The incident state the stored checks leave a monitor in: its open incident,
+   * or else the failed checks after its latest successful one.
+   */
+  incidentState(monitorId: string): IncidentState {
+    const open = this.openIncident(monitorId);
+    if (open) return { open, pending: [] };
+    const pending = (
+      this.db
+        .prepare(
+          `SELECT * FROM checks WHERE monitor_id = ?1 AND ts > COALESCE(
+             (SELECT ts FROM checks WHERE monitor_id = ?1 AND ok = 1 ORDER BY ts DESC LIMIT 1), -1)
+           ORDER BY ts`,
+        )
+        .all(monitorId) as unknown as CheckRow[]
+    ).map(toCheck);
+    return { open: null, pending };
+  }
+
   /** Persists one check: raw row, daily rollup and (optionally) the incident it changed. */
   recordCheck(check: CheckResult, incident: Incident | null, timeZone = 'UTC'): void {
     transaction(this.db, () => {
diff --git a/server/src/routes/admin.ts b/server/src/routes/admin.ts
index ec21040..2831cf1 100644
--- a/server/src/routes/admin.ts
+++ b/server/src/routes/admin.ts
@@ -1,4 +1,4 @@
-import { isValidTimeZone } from '@pulse/core';
+import { DEFAULT_FAILURE_THRESHOLD, isValidTimeZone } from '@pulse/core';
 import type { FastifyInstance, FastifyRequest } from 'fastify';
 import { timingSafeEqual } from 'node:crypto';
 import type { AppContext } from '../context.js';
@@ -27,6 +27,7 @@ const monitorBody = {
     intervalSec: { type: 'integer', minimum: 30, maximum: 3600 },
     timeoutMs: { type: 'integer', minimum: 1000, maximum: 30000 },
     degradedAfterMs: { type: 'integer', minimum: 50, maximum: 30000 },
+    failureThreshold: { type: 'integer', minimum: 1, maximum: 10 },
   },
 } as const;
 
@@ -45,6 +46,7 @@ interface MonitorBody {
   intervalSec?: number;
   timeoutMs?: number;
   degradedAfterMs?: number;
+  failureThreshold?: number;
 }
 
 function authorized(req: FastifyRequest, token: string): boolean {
@@ -85,6 +87,7 @@ export function adminRoutes(app: FastifyInstance, ctx: AppContext): void {
         intervalSec: 60,
         timeoutMs: 10_000,
         degradedAfterMs: 1_000,
+        failureThreshold: DEFAULT_FAILURE_THRESHOLD,
         ...req.body,
       };
       ctx.uptimeRepo.upsertMonitor(monitor, false, ctx.now());
diff --git a/server/src/uptime/service.ts b/server/src/uptime/service.ts
index 2d13f7e..c316ea5 100644
--- a/server/src/uptime/service.ts
+++ b/server/src/uptime/service.ts
@@ -1,5 +1,6 @@
 import {
   DAY,
+  DEFAULT_FAILURE_THRESHOLD,
   applyCheckToIncident,
   buildOverview,
   hourlyFromChecks,
@@ -53,10 +54,20 @@ export class UptimeService {
     return check;
   }
 
+  /**
+   * Reads the threshold and the incident state from the database on every check,
+   * so a restart keeps counting failed checks and a threshold posted to the admin
+   * API applies from the next check, without a new timer.
+   */
   record(check: CheckResult): void {
     const { repo } = this.opts;
-    const { changed } = applyCheckToIncident(repo.openIncident(check.monitorId), check, () =>
-      randomUUID(),
+    const threshold =
+      repo.getMonitor(check.monitorId)?.failureThreshold ?? DEFAULT_FAILURE_THRESHOLD;
+    const { changed } = applyCheckToIncident(
+      repo.incidentState(check.monitorId),
+      check,
+      threshold,
+      () => randomUUID(),
     );
     repo.recordCheck(check, changed, this.tz);
     this.opts.hub?.publish({ type: 'check', check });
diff --git a/server/test/api.test.ts b/server/test/api.test.ts
index 6b94a1b..614c4f5 100644
--- a/server/test/api.test.ts
+++ b/server/test/api.test.ts
@@ -239,6 +239,10 @@ describe('admin API', () => {
       payload: { id: 'site', name: 'Site', url: 'https://demo.example/' },
     });
     expect(res.statusCode).toBe(201);
-    expect(t.ctx.uptimeRepo.getMonitor('site')).toMatchObject({ intervalSec: 60, paused: false });
+    expect(t.ctx.uptimeRepo.getMonitor('site')).toMatchObject({
+      intervalSec: 60,
+      failureThreshold: 2,
+      paused: false,
+    });
   });
 });
diff --git a/server/test/uptime.test.ts b/server/test/uptime.test.ts
index 68d257a..ee19418 100644
--- a/server/test/uptime.test.ts
+++ b/server/test/uptime.test.ts
@@ -1,10 +1,14 @@
-import { DAY, MINUTE, type Monitor } from '@pulse/core';
+import { DAY, MINUTE, type CheckResult, type Monitor } from '@pulse/core';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { DatabaseSync } from 'node:sqlite';
 import { afterEach, describe, expect, it, vi } from 'vitest';
 import { openDatabase } from '../src/db/database.js';
 import { UptimeRepository } from '../src/db/uptime-repo.js';
 import { performCheck, type FetchLike } from '../src/uptime/checker.js';
 import { UptimeService } from '../src/uptime/service.js';
-import { createTestApp } from './helpers.js';
+import { NOW, createTestApp } from './helpers.js';
 
 const monitor: Monitor = {
   id: 'api',
@@ -14,12 +18,25 @@ const monitor: Monitor = {
   intervalSec: 60,
   timeoutMs: 50,
   degradedAfterMs: 500,
+  failureThreshold: 2,
 };
 
 const respond =
   (status: number): FetchLike =>
   async () => ({ status });
 
+/** Never answers, so `performCheck` aborts it after `timeoutMs`. */
+const hang: FetchLike = (_url, init) =>
+  new Promise((_, reject) =>
+    init.signal?.addEventListener('abort', () => reject(new Error('aborted'))),
+  );
+
+let dir: string | null = null;
+afterEach(() => {
+  if (dir) rmSync(dir, { recursive: true, force: true });
+  dir = null;
+});
+
 describe('performCheck', () => {
   it('classifies HTTP statuses', async () => {
     expect(await performCheck(monitor, respond(204))).toMatchObject({
@@ -43,10 +60,6 @@ describe('performCheck', () => {
       status: null,
       error: 'ECONNREFUSED',
     });
-    const hang: FetchLike = (_url, init) =>
-      new Promise((_, reject) =>
-        init.signal?.addEventListener('abort', () => reject(new Error('aborted'))),
-      );
     expect(await performCheck(monitor, hang)).toMatchObject({
       ok: false,
       error: 'Timeout after 0.05s',
@@ -88,6 +101,138 @@ describe('UptimeService', () => {
     expect(repo.pruneChecks(now + 8 * DAY)).toBe(7);
   });
 
+  it('opens an incident only after two failed checks in a row, starting at the first', async () => {
+    const repo = new UptimeRepository(openDatabase(':memory:'));
+    repo.upsertMonitor(monitor);
+    const start = Date.parse('2026-06-10T12:00:00Z');
+    let now = start;
+    const ok = respond(200);
+    const down = respond(500);
+    const answers = [ok, hang, ok, down, down, down, ok];
+    let i = 0;
+    const service = new UptimeService({
+      repo,
+      now: () => now,
+      fetch: (u, r) => answers[i++]!(u, r),
+    });
+    const stored = repo.getMonitor('api')!;
+    expect(stored.failureThreshold).toBe(2);
+
+    await service.runCheck(stored);
+    now += MINUTE;
+    expect(await service.runCheck(stored)).toMatchObject({
+      ok: false,
+      error: 'Timeout after 0.05s',
+    });
+    now += MINUTE;
+    await service.runCheck(stored);
+    expect(repo.incidents('api')).toEqual([]);
+
+    for (let n = 3; n < answers.length; n++) {
+      now += MINUTE;
+      await service.runCheck(stored);
+    }
+    expect(repo.incidents('api')).toEqual([
+      {
+        id: expect.any(String),
+        monitorId: 'api',
+        startedAt: start + 3 * MINUTE,
+        resolvedAt: start + 6 * MINUTE,
+        cause: 'HTTP 500',
+        failures: 3,
+      },
+    ]);
+    // The timeout still counts against uptime.
+    expect(repo.checks('api', 0)).toHaveLength(7);
+    expect(repo.daily('api', 0)[0]).toMatchObject({ checks: 7, failures: 4 });
+    expect(service.overview(stored).uptime24h).toBeCloseTo(3 / 7);
+  });
+
+  it('keeps counting failed checks across a restart', () => {
+    dir = mkdtempSync(join(tmpdir(), 'pulse-uptime-'));
+    const path = join(dir, 'pulse.db');
+    const start = Date.parse('2026-06-10T12:00:00Z');
+    const at = (n: number, ok: boolean): CheckResult => ({
+      monitorId: 'api',
+      ts: start + n * MINUTE,
+      ok,
+      status: ok ? 200 : 503,
+      latencyMs: 20,
+      error: ok ? null : 'HTTP 503',
+    });
+    /** Opens the database file the way a fresh process would. */
+    const boot = () => {
+      const db = openDatabase(path);
+      const repo = new UptimeRepository(db);
+      return { db, repo, service: new UptimeService({ repo }) };
+    };
+
+    const first = boot();
+    first.repo.upsertMonitor({ ...monitor, failureThreshold: 3 });
+    for (const [n, ok] of [
+      [0, false],
+      [1, true],
+      [2, false],
+      [3, false],
+    ] as const) {
+      first.service.record(at(n, ok));
+    }
+    expect(first.repo.incidents('api')).toEqual([]);
+    first.db.close();
+
+    const second = boot();
+    second.service.record(at(4, false));
+    expect(second.repo.incidents('api')).toEqual([
+      expect.objectContaining({ startedAt: start + 2 * MINUTE, resolvedAt: null, failures: 3 }),
+    ]);
+    second.db.close();
+
+    const third = boot();
+    third.service.record(at(5, false));
+    third.service.record(at(6, true));
+    expect(third.repo.incidents('api')).toEqual([
+      expect.objectContaining({
+        startedAt: start + 2 * MINUTE,
+        resolvedAt: start + 6 * MINUTE,
+        failures: 4,
+      }),
+    ]);
+    third.db.close();
+  });
+
+  it('takes failureThreshold from the admin API, and 1 opens an incident on the first failure', async () => {
+    const t = await createTestApp();
+    const post = (body: object) =>
+      t.app.inject({
+        method: 'POST',
+        url: '/api/admin/monitors',
+        headers: { authorization: 'Bearer secret' },
+        payload: { id: 'site', name: 'Site', url: 'https://demo.example/', ...body },
+      });
+    for (const failureThreshold of [0, 11, 1.5]) {
+      expect((await post({ failureThreshold })).statusCode).toBe(400);
+    }
+    const res = await post({ failureThreshold: 1 });
+    expect(res.statusCode).toBe(201);
+    expect(res.json()).toMatchObject({ failureThreshold: 1 });
+    expect(t.ctx.uptimeRepo.getMonitor('site')).toMatchObject({ failureThreshold: 1 });
+
+    t.ctx.uptime.record({
+      monitorId: 'site',
+      ts: NOW,
+      ok: false,
+      status: null,
+      latencyMs: 10_000,
+      error: 'Timeout after 10s',
+    });
+    expect(t.ctx.uptimeRepo.incidents('site')).toEqual([
+      expect.objectContaining({ startedAt: NOW, resolvedAt: null, failures: 1 }),
+    ]);
+    const list = (await t.app.inject('/api/monitors')).json<{ monitor: Monitor }[]>();
+    expect(list[0]!.monitor.failureThreshold).toBe(1);
+    await t.app.close();
+  });
+
   it('serves monitors over the API', async () => {
     const t = await createTestApp({}, { fetch: respond(200) });
     t.ctx.uptimeRepo.upsertMonitor(monitor);
@@ -231,3 +376,34 @@ describe('UptimeService', () => {
     });
   });
 });
+
+describe('UptimeRepository', () => {
+  it('gives the monitors of an existing database a failure threshold of 2 on start', () => {
+    dir = mkdtempSync(join(tmpdir(), 'pulse-uptime-'));
+    const path = join(dir, 'pulse.db');
+    // The monitors table as the first version of the schema created it.
+    const old = new DatabaseSync(path);
+    old.exec(`CREATE TABLE monitors (
+       id                 TEXT PRIMARY KEY,
+       name               TEXT NOT NULL,
+       url                TEXT NOT NULL,
+       method             TEXT NOT NULL DEFAULT 'GET',
+       interval_sec       INTEGER NOT NULL,
+       timeout_ms         INTEGER NOT NULL,
+       degraded_after_ms  INTEGER NOT NULL,
+       paused             INTEGER NOT NULL DEFAULT 0,
+       created_at         INTEGER NOT NULL
+     );
+     INSERT INTO monitors (id, name, url, interval_sec, timeout_ms, degraded_after_ms, created_at)
+       VALUES ('api', 'API', 'https://api.example/health', 60, 5000, 500, 0);
+     PRAGMA user_version = 1;`);
+    old.close();
+
+    const db = openDatabase(path);
+    expect(new UptimeRepository(db).getMonitor('api')).toMatchObject({
+      intervalSec: 60,
+      failureThreshold: 2,
+    });
+    db.close();
+  });
+});
diff --git a/web/src/data/demo/engine.ts b/web/src/data/demo/engine.ts
index 9bf6ee6..ed4a44b 100644
--- a/web/src/data/demo/engine.ts
+++ b/web/src/data/demo/engine.ts
@@ -258,5 +258,6 @@ function plainMonitor(p: MonitorProfile): Monitor {
     intervalSec: p.intervalSec,
     timeoutMs: p.timeoutMs,
     degradedAfterMs: p.degradedAfterMs,
+    failureThreshold: p.failureThreshold,
   };
 }
diff --git a/web/test/engine.test.ts b/web/test/engine.test.ts
index 0127385..f3c58e6 100644
--- a/web/test/engine.test.ts
+++ b/web/test/engine.test.ts
@@ -49,7 +49,12 @@ describe('DemoEngine (the in-browser backend used by the Pages demo)', () => {
   it('simulates monitors with a 90-day heatmap and emits checks live', () => {
     const monitors = engine.monitors();
     expect(monitors.length).toBeGreaterThanOrEqual(5);
-    for (const m of monitors) expect(m.daily).toHaveLength(90);
+    for (const m of monitors) {
+      expect(m.daily).toHaveLength(90);
+      // Same rule as the server: a single failed check does not open an incident.
+      expect(m.monitor.failureThreshold).toBe(2);
+      for (const i of m.incidents) expect(i.failures).toBeGreaterThanOrEqual(2);
+    }
     const detail = engine.monitor(monitors[0]!.monitor.id);
     expect(detail.hourly.length).toBeGreaterThan(100);
     clock.now += 3 * MINUTE;

From c5691aa9210ab8bee0c1e6db0d8050e514a04c86 Mon Sep 17 00:00:00 2001
From: sinnercode228 <sinnercode228@users.noreply.github.com>
Date: Fri, 16 Oct 2026 05:54:05 +0000
Subject: [PATCH 2/3] Document the per-monitor failure threshold in the README

---
 README.md | 39 ++++++++++++++++++++++++---------------
 1 file changed, 24 insertions(+), 15 deletions(-)

diff --git a/README.md b/README.md
index 37e839e..74291fa 100644
--- a/README.md
+++ b/README.md
@@ -115,15 +115,16 @@ npx vitest run packages/core/test/query.test.ts server/test/rollup-repo.test.ts
 
 ### Uptime and realtime windows
 
-- Raw checks are kept 7 days ([`uptime-repo.ts:74`](server/src/db/uptime-repo.ts#L74)). The 90-day bar is built from the `uptime_daily` table ([`packages/core/src/uptime.ts:257`](packages/core/src/uptime.ts#L257)), and the 24-hour sparkline has 96 buckets of 15 minutes ([`uptime.ts:218`](packages/core/src/uptime.ts#L218)).
-- A day is operational at 99.99% or more, minor at 99%, partial at 95%, major below that ([`uptime.ts:126-129`](packages/core/src/uptime.ts#L126-L129)). A monitor is degraded when its last successful check took longer than `degradedAfterMs` ([`uptime.ts:136`](packages/core/src/uptime.ts#L136)).
-- A monitor created through the admin API is checked every 60 s with a 10 s timeout and counts as degraded above 1,000 ms; the allowed ranges are 30 to 3,600 s and 1 to 30 s ([`admin.ts:27-28`](server/src/routes/admin.ts#L27-L28), [`:83-89`](server/src/routes/admin.ts#L83-L89)). `performCheck` never throws: network errors and timeouts become failed checks ([`checker.ts`](server/src/uptime/checker.ts)).
-- Each monitor has its own timer at its own interval. The uptime worker re-reads the monitors table every 5 minutes ([`uptime-worker.ts:24`](server/src/uptime-worker.ts#L24)), and `POST /api/admin/monitors` does the same for the inline checker ([`admin.ts:91`](server/src/routes/admin.ts#L91)). Only a new monitor, or one whose URL, method, interval, timeout or degraded threshold changed, is checked at once and gets a new timer; a paused one loses its timer, and the rest keep their schedule ([`service.ts:66-104`](server/src/uptime/service.ts#L66-L104)). So a new monitor reaches the separate worker within 5 minutes, and a monitor set to 3,600 s is checked once an hour. `sync()` leaves the hourly pruning timer alone, so the separate worker deletes raw checks older than 7 days as well ([`uptime.test.ts`](server/test/uptime.test.ts) checks both with fake timers).
+- Raw checks are kept 7 days ([`uptime-repo.ts:77`](server/src/db/uptime-repo.ts#L77)). The 90-day bar is built from the `uptime_daily` table ([`packages/core/src/uptime.ts:276`](packages/core/src/uptime.ts#L276)), and the 24-hour sparkline has 96 buckets of 15 minutes ([`uptime.ts:237`](packages/core/src/uptime.ts#L237)).
+- A day is operational at 99.99% or more, minor at 99%, partial at 95%, major below that ([`uptime.ts:131-134`](packages/core/src/uptime.ts#L131-L134)). A monitor is degraded when its last successful check took longer than `degradedAfterMs` ([`uptime.ts:141`](packages/core/src/uptime.ts#L141)).
+- A monitor created through the admin API is checked every 60 s with a 10 s timeout, counts as degraded above 1,000 ms and opens an incident after 2 failed checks in a row; the allowed ranges are 30 to 3,600 s, 1 to 30 s and 1 to 10 checks ([`admin.ts:27-30`](server/src/routes/admin.ts#L27-L30), [`:85-92`](server/src/routes/admin.ts#L85-L92)). `performCheck` never throws: network errors and timeouts become failed checks ([`checker.ts`](server/src/uptime/checker.ts)).
+- An incident opens when `failureThreshold` checks in a row fail and starts at the first of them, so its duration and failure count include the checks before it opened; the first successful check closes it ([`uptime.ts:168-198`](packages/core/src/uptime.ts#L168-L198)). With a threshold of 1 the first failed check opens it, as before the threshold existed. Every check still goes into the raw table and `uptime_daily`, so a single failure that opens nothing still lowers the uptime percentages. The failed checks that have not opened an incident yet are not kept in memory: before recording a check the service reads them back from the `checks` table, everything after the monitor's latest successful check, together with the monitor's threshold ([`uptime-repo.ts:129-146`](server/src/db/uptime-repo.ts#L129-L146), [`service.ts:57-74`](server/src/uptime/service.ts#L57-L74)). A restart in the middle of a run keeps counting, and a new threshold applies from the next check.
+- Each monitor has its own timer at its own interval. The uptime worker re-reads the monitors table every 5 minutes ([`uptime-worker.ts:24`](server/src/uptime-worker.ts#L24)), and `POST /api/admin/monitors` does the same for the inline checker ([`admin.ts:94`](server/src/routes/admin.ts#L94)). Only a new monitor, or one whose URL, method, interval, timeout or degraded threshold changed, is checked at once and gets a new timer; a paused one loses its timer, and the rest keep their schedule ([`service.ts:77-115`](server/src/uptime/service.ts#L77-L115)). So a new monitor reaches the separate worker within 5 minutes, and a monitor set to 3,600 s is checked once an hour. `sync()` leaves the hourly pruning timer alone, so the separate worker deletes raw checks older than 7 days as well ([`uptime.test.ts`](server/test/uptime.test.ts) checks both with fake timers).
 - "Online now" is the number of distinct visitors in the last 5 minutes ([`active.ts:11`](packages/core/src/active.ts#L11)). The realtime panel shows 30 one-minute bars and the last 50 events ([`service.ts:69-81`](server/src/stats/service.ts#L69-L81), [`pipeline.ts:44`](server/src/ingest/pipeline.ts#L44)). The WebSocket pushes the active count every 10 s ([`live.ts:19-22`](server/src/routes/live.ts#L19-L22)), and the dashboard refetches charts and KPIs every 10 s while the window reaches "now" and the tab is visible ([`App.tsx:16`](web/src/App.tsx#L16), [`:66-72`](web/src/App.tsx#L66-L72)).
 
-### Tests: 118
+### Tests: 128
 
-`npm test` runs 118 Vitest tests in 18 files: 63 in `packages/core` (HLL, time zones and DST, rollups, queries, uptime, generators), 34 in `server` (the API through `fastify.inject` on an in-memory `node:sqlite` database, a WebSocket on a real socket, SQLite vs. in-memory parity, the tracker in `node:vm`, CSV with formula neutralization, the daily salt, uptime scheduling with fake timers) and 21 in `web` under jsdom (demo engine, components, hotkeys, API client, utilities). Vitest reported 1.8 s for the run on my machine before the six tests that came with the bias correction. On 2026-10-15 (Node 22.22.2, 4 CPUs shared with other jobs) six runs each took 6.4 to 7.9 s, with or without them. CI ([`.github/workflows/ci.yml`](.github/workflows/ci.yml)) runs ESLint with `--max-warnings=0`, Prettier, `tsc` in strict mode with `noUncheckedIndexedAccess`, the tests and the build on Node 22 and 24; `npm run check` runs the same steps locally.
+`npm test` runs 128 Vitest tests in 18 files: 69 in `packages/core` (HLL, time zones and DST, rollups, queries, uptime and incidents, generators), 38 in `server` (the API through `fastify.inject` on an in-memory `node:sqlite` database, a WebSocket on a real socket, SQLite vs. in-memory parity, the tracker in `node:vm`, CSV with formula neutralization, the daily salt, uptime scheduling with fake timers, incident counting across a restart, the schema migration) and 21 in `web` under jsdom (demo engine, components, hotkeys, API client, utilities). Vitest reported 1.8 s for the run on my machine before the six tests that came with the bias correction. On 2026-10-15 (Node 22.22.2, 4 CPUs shared with other jobs) six runs each took 6.4 to 7.9 s, with or without them. CI ([`.github/workflows/ci.yml`](.github/workflows/ci.yml)) runs ESLint with `--max-warnings=0`, Prettier, `tsc` in strict mode with `noUncheckedIndexedAccess`, the tests and the build on Node 22 and 24; `npm run check` runs the same steps locally.
 
 ## How it fits together
 
@@ -185,9 +186,9 @@ For windows of a day or longer I first look at the first tenth of the previous p
 
 ![The uptime page: monitors, 90-day bar, incidents](docs/screenshots/uptime.png)
 
-Checks run inside the API (`UPTIME_MODE=inline`, the default) or in a separate [`uptime-worker`](server/src/uptime-worker.ts) process on the same database file. Each check writes a raw row, updates that day's row in `uptime_daily` and advances the incident state, in one transaction ([`uptime-repo.ts:125-146`](server/src/db/uptime-repo.ts#L125-L146)). The built-in `node:sqlite` needs no native addon, and the database is opened in WAL mode with `synchronous = NORMAL` and `busy_timeout = 5000`, so the API and the worker can share one file; the schema migrates through `PRAGMA user_version` ([`server/src/db/database.ts`](server/src/db/database.ts)). The separate worker has no `LiveHub`, so under `docker compose` the uptime page updates only through the 10-second refetch.
+Checks run inside the API (`UPTIME_MODE=inline`, the default) or in a separate [`uptime-worker`](server/src/uptime-worker.ts) process on the same database file. Each check writes a raw row, updates that day's row in `uptime_daily` and advances the incident state, in one transaction ([`uptime-repo.ts:149-170`](server/src/db/uptime-repo.ts#L149-L170)). The built-in `node:sqlite` needs no native addon, and the database is opened in WAL mode with `synchronous = NORMAL` and `busy_timeout = 5000`, so the API and the worker can share one file; the schema migrates through `PRAGMA user_version` ([`server/src/db/database.ts`](server/src/db/database.ts)). The separate worker has no `LiveHub`, so under `docker compose` the uptime page updates only through the 10-second refetch.
 
-In the demo, the "Webhook relay" monitor is scripted to fail with HTTP 500 about a minute after the uptime data is first requested and to recover 6 minutes later ([`packages/core/src/synth/uptime.ts`](packages/core/src/synth/uptime.ts)), so an incident opens and closes on it while the page is open.
+In the demo, the "Webhook relay" monitor is scripted to fail with HTTP 500 about a minute after the uptime data is first requested and to recover 6 minutes later ([`packages/core/src/synth/uptime.ts`](packages/core/src/synth/uptime.ts)), so an incident opens and closes on it while the page is open. The demo monitors run the same incident code with a threshold of 2, so the random single failed checks in their history lower the uptime but open no incident.
 
 ### The demo without a server
 
@@ -247,6 +248,14 @@ curl -X POST http://localhost:8787/api/admin/sites \
 
 Endpoints: `GET /api/sites`, `GET /api/sites/:id/{summary,timeseries,breakdown,realtime,export.csv}` (`from` and `to` in epoch ms, default the last 24 h, clamped to 400 days in [`schemas.ts:57-63`](server/src/routes/schemas.ts#L57-L63); `breakdown` and `export.csv` require `dimension`), `GET /api/monitors`, `GET /api/monitors/:id`, `GET /api/live?site=<id>` (WebSocket), `POST /api/event`, `POST /api/admin/sites`, `POST /api/admin/monitors`, `GET /p.js`, `GET /api/health`.
 
+A monitor is added the same way. Only `id`, `name` and `url` are required. `method` defaults to `GET`; `intervalSec`, `timeoutMs`, `degradedAfterMs` and `failureThreshold` (failed checks in a row before an incident opens, 1 to 10) take the defaults listed under [Uptime and realtime windows](#uptime-and-realtime-windows). Posting the same `id` again replaces every setting, so a field left out goes back to its default. `GET /api/monitors` returns the settings, `failureThreshold` included, with each monitor.
+
+```bash
+curl -X POST http://localhost:8787/api/admin/monitors \
+  -H 'authorization: Bearer dev-token' -H 'content-type: application/json' \
+  -d '{"id":"my-api","name":"My API","url":"https://example.com/health","failureThreshold":3}'
+```
+
 ## Known limitations
 
 Nine, the most serious first. I reproduced 1 with requests to a running server on a seeded database; the rest come from reading the code at the lines given.
@@ -254,8 +263,8 @@ Nine, the most serious first. I reproduced 1 with requests to a running server o
 1. **Visitor identity, country and volume can be forged without a proxy in front.** `TRUST_PROXY` is on unless set to `false` ([`config.ts:28`](server/src/config.ts#L28)), so a client's own `X-Forwarded-For` goes into the visitor hash; the country comes only from headers such as `x-country-code` ([`url.ts:55-70`](packages/core/src/url.ts#L55-L70)); the domain check reads the URL from the request body ([`parse.ts:66`](server/src/ingest/parse.ts#L66)). Checked: a POST with `x-forwarded-for: 8.8.8.8` and `x-country-code: AQ` was accepted, and AQ appeared in the country breakdown. `/api/event` has no rate limit beyond the 8 KB body limit ([`ingest.ts:7`](server/src/routes/ingest.ts#L7)). Unless a reverse proxy overwrites those headers and limits requests, anyone who knows a site id can inflate its numbers.
 2. **Above 512 distinct visitors the count is an estimate, and a single sketch can step where the estimators hand over.** `count()` uses linear counting up to an estimate of 1,800 and the HLL++ bias-corrected raw estimate above it ([`hll.ts:69-86`](packages/core/src/hll.ts#L69-L86), [`hll-bias.ts`](packages/core/src/hll-bias.ts)). That took the 5,000 row above from rms 3.27% to 1.82%; all four rows are now between 1.78% and 2.36%, and single sets of ids still miss by up to 6.93%. Two steps remain. At the 513th hash the exact count becomes an estimate (rms 1.54% at 513 in 50 sets). At 1,800 the average over 100 sketches does not jump, but one sketch switches between two estimates of about the same n and can move by about 3% (3.16% at most in the seeded test). The hash is 32-bit and there is no large-range correction, which would matter only near 2³²/30, about 143 million distinct hashes in one row.
 3. **Minute and hour buckets are aligned to UTC; only day buckets follow the site's time zone** ([`time.ts:66-75`](packages/core/src/time.ts#L66-L75)). The "Today" preset starts at the browser's local midnight, not the site's ([`ranges.ts:24-28`](web/src/lib/ranges.ts#L24-L28)). In a half-hour zone such as Asia/Kolkata that midnight is 18:30 UTC, the hour tier rounds it down to 18:00 ([`query.ts:102`](packages/core/src/query.ts#L102)), and "Today" includes 30 minutes of yesterday.
-4. **Reads have no authentication, and the admin API can only create or overwrite.** Every `GET` is public and `CORS_ORIGIN` defaults to `*` ([`config.ts:27`](server/src/config.ts#L27)); `ADMIN_TOKEN` protects only `POST /api/admin/*`. There is no delete and no pause: the `paused` column ([`database.ts:43`](server/src/db/database.ts#L43)) is set only by the seed script ([`seed.ts:59`](server/scripts/seed.ts#L59)), and the admin route always stores `false` ([`admin.ts:90`](server/src/routes/admin.ts#L90)).
-5. **No alerts, and one failed check is an incident.** A monitor that is down shows up only on the dashboard. The first failed check opens an incident and the first successful one closes it ([`uptime.ts:154-179`](packages/core/src/uptime.ts#L154-L179)), so a single timeout makes an incident of its own.
+4. **Reads have no authentication, and the admin API can only create or overwrite.** Every `GET` is public and `CORS_ORIGIN` defaults to `*` ([`config.ts:27`](server/src/config.ts#L27)); `ADMIN_TOKEN` protects only `POST /api/admin/*`. There is no delete and no pause: the `paused` column ([`database.ts:43`](server/src/db/database.ts#L43)) is set only by the seed script ([`seed.ts:59`](server/scripts/seed.ts#L59)), and the admin route always stores `false` ([`admin.ts:93`](server/src/routes/admin.ts#L93)).
+5. **No alerts, and one successful check ends an incident.** A monitor that is down shows up only on the dashboard: nothing sends an email, a webhook or a chat message when an incident opens or closes. An incident opens after `failureThreshold` failed checks in a row (2 by default) but closes on the first successful check ([`uptime.ts:168-198`](packages/core/src/uptime.ts#L168-L198)), so an endpoint that alternates between failing twice and answering once makes a new incident each time.
 6. **Live state lives in process memory, and the WebSocket is minimal.** "Online now", the recent-events list ([`pipeline.ts:33-34`](server/src/ingest/pipeline.ts#L33-L34)) and the subscriber hub ([`hub.ts`](server/src/live/hub.ts)) reset on restart and do not reach other processes. The socket has no ping and no backpressure ([`live.ts`](server/src/routes/live.ts)); the client reconnects after 1 s × 2ⁿ, up to 30 s, with no jitter ([`ApiDataSource.ts:112`](web/src/data/api/ApiDataSource.ts#L112)), and events that happen while it is disconnected are not resent. The design is one process with one SQLite writer; it does not scale horizontally.
 7. **The SQLite upsert is one `SELECT` plus one `INSERT OR REPLACE` per row** inside a transaction ([`rollup-repo.ts:68-93`](server/src/db/rollup-repo.ts#L68-L93)), because sketches are merged in JavaScript. A flush is about 5,000 rows at most, which is fine at demo volume; for real traffic this is the first thing I would rewrite.
 8. **The dashboard does not go further back than 30 days** ([`ranges.ts:22`](web/src/lib/ranges.ts#L22), used by `shiftRange` and by the custom range in [`RangePicker.tsx:96-115`](web/src/components/RangePicker.tsx#L96-L115)), although the server keeps day rollups indefinitely.
@@ -278,8 +287,8 @@ Pulse — веб-аналитика без cookies и мониторинг до
 - **Трекер: 843 Б минифицированный, 560 Б в gzip, 458 Б в brotli**, бюджет 2 048 Б. Размер считает [`server/tracker/bundle.ts`](server/tracker/bundle.ts); бюджет проверяют и сборка ([`build-tracker.mjs:6-9`](server/scripts/build-tracker.mjs#L6-L9)), и тест ([`tracker.test.ts:47-50`](server/test/tracker.test.ts#L47-L50)), который ещё требует меньше 1 024 Б в gzip. Повторить: `npm run build:tracker -w @pulse/server`.
 - **Посетители: стандартная ошибка 2,3 %, точный счёт до 512.** HyperLogLog с precision 11, 2 048 регистров ([`packages/core/src/hll.ts`](packages/core/src/hll.ts)). Пока хэшей не больше 512, скетч хранит их точным множеством, как sparse-режим в HLL++; на 513-м переходит в плотные регистры. Плотный скетч считается как в HLL++: линейный счёт, пока его оценка не больше 1 800, дальше сырая оценка минус эмпирическое смещение из приложения к статье про HLL++ для precision 11 ([`hll-bias.ts`](packages/core/src/hll-bias.ts)). Формат хранения не менялся, старые строки читаются и сливаются как раньше. Замер на 50 наборах id на размер (2026-10-15, Node 22.22.2): rms 1,78 % при 1 000, 1,82 % при 5 000, 1,99 % при 10 000, 2,36 % при 100 000; худшие случаи от 4,9 до 6,9 %. До коррекции смещения при 5 000 было 3,27 % rms и 7,88 % в худшем случае: там линейный счёт сменялся сырой оценкой без поправки.
 - **Роллапы: минута / час / день, хранение 2 дня / 35 дней / бессрочно** ([`store.ts:8-12`](packages/core/src/store.ts#L8-L12)). Уровень выбирается так, чтобы на графике было не больше примерно 200 точек: окно до 3 ч 1 мин — минуты, до 8 дней 1 ч — часы, дальше дни ([`query.ts:61-71`](packages/core/src/query.ts#L61-L71)). Один просмотр трогает 18 строк буфера (6 срезов × 3 уровня); буфер уходит в SQLite одной транзакцией раз в 2 с или при 5 000 строк, а любой запрос статистики сначала вызывает `flush()`. Проверить: `npx vitest run packages/core/test/query.test.ts server/test/rollup-repo.test.ts`.
-- **Аптайм:** сырые проверки хранятся 7 дней, полоса за 90 дней строится из `uptime_daily` с порогами 99,99 / 99 / 95 %, спарклайн за сутки — 96 корзин по 15 минут ([`packages/core/src/uptime.ts`](packages/core/src/uptime.ts)).
-- **Тесты: 118** в 18 файлах: 63 в `packages/core`, 34 в `server`, 21 в `web`. CI гоняет ESLint, Prettier, `tsc`, тесты и сборку на Node 22 и 24; локально то же делает `npm run check`.
+- **Аптайм:** сырые проверки хранятся 7 дней, полоса за 90 дней строится из `uptime_daily` с порогами 99,99 / 99 / 95 %, спарклайн за сутки — 96 корзин по 15 минут ([`packages/core/src/uptime.ts`](packages/core/src/uptime.ts)). Инцидент открывают 2 неудачные проверки подряд (`failureThreshold` монитора, от 1 до 10), начинается он с первой из них; одиночная неудачная проверка инцидента не открывает, но в процент аптайма входит.
+- **Тесты: 128** в 18 файлах: 69 в `packages/core`, 38 в `server`, 21 в `web`. CI гоняет ESLint, Prettier, `tsc`, тесты и сборку на Node 22 и 24; локально то же делает `npm run check`.
 
 Схема потока данных и разборы (путь одного просмотра, слияние скетчей, часовые пояса, аптайм во втором процессе, демо без сервера) — в английской части.
 
@@ -310,7 +319,7 @@ docker compose restart api           # после seed, иначе API не ув
 
 API читает список сайтов один раз при старте ([`sites-repo.ts:15-17`](server/src/db/sites-repo.ts#L15-L17)), поэтому seed нужно запускать до API или перезапускать API после него.
 
-Сайт для трекера заводится через `POST /api/admin/sites` с `Authorization: Bearer <ADMIN_TOKEN>` (пример с `curl` — в английской части); без `ADMIN_TOKEN` админ-роуты отвечают 401. Сервер берёт переменные только из окружения процесса и `.env` сам не читает.
+Сайт для трекера заводится через `POST /api/admin/sites` с `Authorization: Bearer <ADMIN_TOKEN>` (пример с `curl` — в английской части); без `ADMIN_TOKEN` админ-роуты отвечают 401. Сервер берёт переменные только из окружения процесса и `.env` сам не читает. Монитор заводится так же, через `POST /api/admin/monitors`; поля и значения по умолчанию — в английской части.
 
 ```html
 <script defer src="https://pulse.example/p.js" data-site="my-site"></script>
@@ -323,8 +332,8 @@ API читает список сайтов один раз при старте (
 1. Без обратного прокси цифры можно подделать: `TRUST_PROXY` включён по умолчанию ([`config.ts:28`](server/src/config.ts#L28)), страна берётся только из заголовков вроде `x-country-code`, домен сверяется по URL из тела запроса, rate limit на `/api/event` нет. Проверено: POST с `x-forwarded-for: 8.8.8.8` и `x-country-code: AQ` принят, AQ появилась в разбивке по странам.
 2. Выше 512 посетителей число — оценка HLL++ ([`hll.ts:69-86`](packages/core/src/hll.ts#L69-L86)) с rms от 1,78 до 2,36 % в замерах выше. На 513-м хэше точный счёт сменяется оценкой, а на 1 800 линейный счёт сменяется скорректированной оценкой: в среднем по 100 скетчам скачка нет, но отдельный скетч может сдвинуться примерно на 3 %. Хэш 32-битный, коррекции для больших значений нет; она понадобилась бы около 143 млн разных хэшей в одной строке.
 3. Минутные и часовые бакеты выровнены по UTC, по зоне сайта режутся только дни ([`time.ts:66-75`](packages/core/src/time.ts#L66-L75)). «Сегодня» считается по зоне браузера ([`ranges.ts:24-28`](web/src/lib/ranges.ts#L24-L28)) и в зонах с получасовым сдвигом захватывает 30 минут вчерашнего дня.
-4. Чтение без авторизации, `CORS_ORIGIN` по умолчанию `*`. Админ-API умеет только создавать и перезаписывать, удаления и паузы нет ([`admin.ts:90`](server/src/routes/admin.ts#L90)).
-5. Алертов нет. Инцидент открывает первая неудачная проверка и закрывает первая успешная, так что один таймаут — отдельный инцидент ([`uptime.ts:154-179`](packages/core/src/uptime.ts#L154-L179)).
+4. Чтение без авторизации, `CORS_ORIGIN` по умолчанию `*`. Админ-API умеет только создавать и перезаписывать, удаления и паузы нет ([`admin.ts:93`](server/src/routes/admin.ts#L93)).
+5. Алертов нет: ни письма, ни вебхука, когда инцидент открывается или закрывается. Инцидент открывается после `failureThreshold` неудачных проверок подряд (по умолчанию 2), а закрывается первой успешной ([`uptime.ts:168-198`](packages/core/src/uptime.ts#L168-L198)), поэтому эндпоинт, который дважды падает и раз отвечает, каждый раз открывает новый инцидент.
 6. «Сейчас онлайн», лента последних событий и хаб WebSocket живут в памяти одного процесса. У сокета нет ping и backpressure, клиент переподключается без jitter, пропущенные события не досылаются.
 7. Upsert в SQLite — `SELECT` плюс `INSERT OR REPLACE` на каждую строку, скетчи сливаются в JS ([`rollup-repo.ts:68-93`](server/src/db/rollup-repo.ts#L68-L93)). Для реального трафика это первое, что я бы переписал.
 8. Дашборд не даёт уйти дальше 30 дней назад ([`ranges.ts:22`](web/src/lib/ranges.ts#L22)), хотя дневные роллапы хранятся бессрочно.

From 6d2eef1e563bb0529cb43d7df9b6c5c071a4fade Mon Sep 17 00:00:00 2001
From: sinnercode228 <sinnercode228@users.noreply.github.com>
Date: Fri, 16 Oct 2026 06:38:48 +0000
Subject: [PATCH 3/3] Run the schema migration once when the API and the uptime
 worker open the database together

---
 README.md                  |  6 ++--
 server/src/db/database.ts  | 24 +++++++++----
 server/test/uptime.test.ts | 72 +++++++++++++++++++++++++++++++++++---
 3 files changed, 88 insertions(+), 14 deletions(-)

diff --git a/README.md b/README.md
index 74291fa..e8958bb 100644
--- a/README.md
+++ b/README.md
@@ -122,9 +122,9 @@ npx vitest run packages/core/test/query.test.ts server/test/rollup-repo.test.ts
 - Each monitor has its own timer at its own interval. The uptime worker re-reads the monitors table every 5 minutes ([`uptime-worker.ts:24`](server/src/uptime-worker.ts#L24)), and `POST /api/admin/monitors` does the same for the inline checker ([`admin.ts:94`](server/src/routes/admin.ts#L94)). Only a new monitor, or one whose URL, method, interval, timeout or degraded threshold changed, is checked at once and gets a new timer; a paused one loses its timer, and the rest keep their schedule ([`service.ts:77-115`](server/src/uptime/service.ts#L77-L115)). So a new monitor reaches the separate worker within 5 minutes, and a monitor set to 3,600 s is checked once an hour. `sync()` leaves the hourly pruning timer alone, so the separate worker deletes raw checks older than 7 days as well ([`uptime.test.ts`](server/test/uptime.test.ts) checks both with fake timers).
 - "Online now" is the number of distinct visitors in the last 5 minutes ([`active.ts:11`](packages/core/src/active.ts#L11)). The realtime panel shows 30 one-minute bars and the last 50 events ([`service.ts:69-81`](server/src/stats/service.ts#L69-L81), [`pipeline.ts:44`](server/src/ingest/pipeline.ts#L44)). The WebSocket pushes the active count every 10 s ([`live.ts:19-22`](server/src/routes/live.ts#L19-L22)), and the dashboard refetches charts and KPIs every 10 s while the window reaches "now" and the tab is visible ([`App.tsx:16`](web/src/App.tsx#L16), [`:66-72`](web/src/App.tsx#L66-L72)).
 
-### Tests: 128
+### Tests: 129
 
-`npm test` runs 128 Vitest tests in 18 files: 69 in `packages/core` (HLL, time zones and DST, rollups, queries, uptime and incidents, generators), 38 in `server` (the API through `fastify.inject` on an in-memory `node:sqlite` database, a WebSocket on a real socket, SQLite vs. in-memory parity, the tracker in `node:vm`, CSV with formula neutralization, the daily salt, uptime scheduling with fake timers, incident counting across a restart, the schema migration) and 21 in `web` under jsdom (demo engine, components, hotkeys, API client, utilities). Vitest reported 1.8 s for the run on my machine before the six tests that came with the bias correction. On 2026-10-15 (Node 22.22.2, 4 CPUs shared with other jobs) six runs each took 6.4 to 7.9 s, with or without them. CI ([`.github/workflows/ci.yml`](.github/workflows/ci.yml)) runs ESLint with `--max-warnings=0`, Prettier, `tsc` in strict mode with `noUncheckedIndexedAccess`, the tests and the build on Node 22 and 24; `npm run check` runs the same steps locally.
+`npm test` runs 129 Vitest tests in 18 files: 69 in `packages/core` (HLL, time zones and DST, rollups, queries, uptime and incidents, generators), 39 in `server` (the API through `fastify.inject` on an in-memory `node:sqlite` database, a WebSocket on a real socket, SQLite vs. in-memory parity, the tracker in `node:vm`, CSV with formula neutralization, the daily salt, uptime scheduling with fake timers, incident counting across a restart, the schema migration, also from two processes at once) and 21 in `web` under jsdom (demo engine, components, hotkeys, API client, utilities). Vitest reported 1.8 s for the run on my machine before the six tests that came with the bias correction. On 2026-10-15 (Node 22.22.2, 4 CPUs shared with other jobs) six runs each took 6.4 to 7.9 s, with or without them. CI ([`.github/workflows/ci.yml`](.github/workflows/ci.yml)) runs ESLint with `--max-warnings=0`, Prettier, `tsc` in strict mode with `noUncheckedIndexedAccess`, the tests and the build on Node 22 and 24; `npm run check` runs the same steps locally.
 
 ## How it fits together
 
@@ -288,7 +288,7 @@ Pulse — веб-аналитика без cookies и мониторинг до
 - **Посетители: стандартная ошибка 2,3 %, точный счёт до 512.** HyperLogLog с precision 11, 2 048 регистров ([`packages/core/src/hll.ts`](packages/core/src/hll.ts)). Пока хэшей не больше 512, скетч хранит их точным множеством, как sparse-режим в HLL++; на 513-м переходит в плотные регистры. Плотный скетч считается как в HLL++: линейный счёт, пока его оценка не больше 1 800, дальше сырая оценка минус эмпирическое смещение из приложения к статье про HLL++ для precision 11 ([`hll-bias.ts`](packages/core/src/hll-bias.ts)). Формат хранения не менялся, старые строки читаются и сливаются как раньше. Замер на 50 наборах id на размер (2026-10-15, Node 22.22.2): rms 1,78 % при 1 000, 1,82 % при 5 000, 1,99 % при 10 000, 2,36 % при 100 000; худшие случаи от 4,9 до 6,9 %. До коррекции смещения при 5 000 было 3,27 % rms и 7,88 % в худшем случае: там линейный счёт сменялся сырой оценкой без поправки.
 - **Роллапы: минута / час / день, хранение 2 дня / 35 дней / бессрочно** ([`store.ts:8-12`](packages/core/src/store.ts#L8-L12)). Уровень выбирается так, чтобы на графике было не больше примерно 200 точек: окно до 3 ч 1 мин — минуты, до 8 дней 1 ч — часы, дальше дни ([`query.ts:61-71`](packages/core/src/query.ts#L61-L71)). Один просмотр трогает 18 строк буфера (6 срезов × 3 уровня); буфер уходит в SQLite одной транзакцией раз в 2 с или при 5 000 строк, а любой запрос статистики сначала вызывает `flush()`. Проверить: `npx vitest run packages/core/test/query.test.ts server/test/rollup-repo.test.ts`.
 - **Аптайм:** сырые проверки хранятся 7 дней, полоса за 90 дней строится из `uptime_daily` с порогами 99,99 / 99 / 95 %, спарклайн за сутки — 96 корзин по 15 минут ([`packages/core/src/uptime.ts`](packages/core/src/uptime.ts)). Инцидент открывают 2 неудачные проверки подряд (`failureThreshold` монитора, от 1 до 10), начинается он с первой из них; одиночная неудачная проверка инцидента не открывает, но в процент аптайма входит.
-- **Тесты: 128** в 18 файлах: 69 в `packages/core`, 38 в `server`, 21 в `web`. CI гоняет ESLint, Prettier, `tsc`, тесты и сборку на Node 22 и 24; локально то же делает `npm run check`.
+- **Тесты: 129** в 18 файлах: 69 в `packages/core`, 39 в `server`, 21 в `web`. CI гоняет ESLint, Prettier, `tsc`, тесты и сборку на Node 22 и 24; локально то же делает `npm run check`.
 
 Схема потока данных и разборы (путь одного просмотра, слияние скетчей, часовые пояса, аптайм во втором процессе, демо без сервера) — в английской части.
 
diff --git a/server/src/db/database.ts b/server/src/db/database.ts
index 8823d02..eeda328 100644
--- a/server/src/db/database.ts
+++ b/server/src/db/database.ts
@@ -76,23 +76,33 @@ const MIGRATIONS: readonly string[] = [
 export function openDatabase(path: string): Db {
   if (path !== ':memory:') mkdirSync(dirname(path), { recursive: true });
   const db = new DatabaseSync(path);
-  db.exec('PRAGMA journal_mode = WAL; PRAGMA synchronous = NORMAL; PRAGMA busy_timeout = 5000;');
+  db.exec('PRAGMA busy_timeout = 5000; PRAGMA journal_mode = WAL; PRAGMA synchronous = NORMAL;');
   migrate(db);
   return db;
 }
 
 function migrate(db: Db): void {
-  const { user_version: version } = db.prepare('PRAGMA user_version').get() as {
-    user_version: number;
-  };
-  for (let v = version; v < MIGRATIONS.length; v++) {
-    transaction(db, () => {
+  if (userVersion(db) >= MIGRATIONS.length) return;
+  // The API and the uptime worker can open the file at the same moment. BEGIN IMMEDIATE
+  // takes the write lock before the version is read again, so the second one waits for
+  // the first one's migrations instead of running them a second time.
+  db.exec('BEGIN IMMEDIATE');
+  try {
+    for (let v = userVersion(db); v < MIGRATIONS.length; v++) {
       db.exec(MIGRATIONS[v]!);
       db.exec(`PRAGMA user_version = ${v + 1}`);
-    });
+    }
+    db.exec('COMMIT');
+  } catch (err) {
+    db.exec('ROLLBACK');
+    throw err;
   }
 }
 
+function userVersion(db: Db): number {
+  return (db.prepare('PRAGMA user_version').get() as { user_version: number }).user_version;
+}
+
 export function transaction<T>(db: Db, fn: () => T): T {
   if (db.isTransaction) return fn();
   db.exec('BEGIN');
diff --git a/server/test/uptime.test.ts b/server/test/uptime.test.ts
index ee19418..1704f82 100644
--- a/server/test/uptime.test.ts
+++ b/server/test/uptime.test.ts
@@ -1,4 +1,5 @@
 import { DAY, MINUTE, type CheckResult, type Monitor } from '@pulse/core';
+import { spawn } from 'node:child_process';
 import { mkdtempSync, rmSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
@@ -378,12 +379,13 @@ describe('UptimeService', () => {
 });
 
 describe('UptimeRepository', () => {
-  it('gives the monitors of an existing database a failure threshold of 2 on start', () => {
+  /** A database file with the monitors table as the first version of the schema created it. */
+  function versionOneDatabase(): string {
     dir = mkdtempSync(join(tmpdir(), 'pulse-uptime-'));
     const path = join(dir, 'pulse.db');
-    // The monitors table as the first version of the schema created it.
     const old = new DatabaseSync(path);
-    old.exec(`CREATE TABLE monitors (
+    old.exec(`PRAGMA journal_mode = WAL;
+     CREATE TABLE monitors (
        id                 TEXT PRIMARY KEY,
        name               TEXT NOT NULL,
        url                TEXT NOT NULL,
@@ -398,12 +400,74 @@ describe('UptimeRepository', () => {
        VALUES ('api', 'API', 'https://api.example/health', 60, 5000, 500, 0);
      PRAGMA user_version = 1;`);
     old.close();
+    return path;
+  }
 
-    const db = openDatabase(path);
+  it('gives the monitors of an existing database a failure threshold of 2 on start', () => {
+    const db = openDatabase(versionOneDatabase());
     expect(new UptimeRepository(db).getMonitor('api')).toMatchObject({
       intervalSec: 60,
       failureThreshold: 2,
     });
     db.close();
   });
+
+  it('migrates once when the API and the uptime worker open an old database together', async () => {
+    const path = versionOneDatabase();
+    const results = await Promise.all(openTogether(path, 2));
+    expect(results).toEqual([
+      { code: 0, out: '2', err: '' },
+      { code: 0, out: '2', err: '' },
+    ]);
+    const db = openDatabase(path);
+    expect(new UptimeRepository(db).getMonitor('api')).toMatchObject({ failureThreshold: 2 });
+    db.close();
+  }, 30_000);
 });
+
+const DATABASE_MODULE = new URL('../src/db/database.ts', import.meta.url).href;
+
+/** Waits for a byte on stdin, then opens the database and prints its schema version. */
+const OPEN_ON_SIGNAL = `
+import { readSync } from 'node:fs';
+const { openDatabase } = await import(process.argv[1]);
+process.stdout.write('ready');
+readSync(0, Buffer.alloc(1));
+const db = openDatabase(process.argv[2]);
+const out = String(db.prepare('PRAGMA user_version').get().user_version);
+db.close();
+process.stdout.write(out);
+`;
+
+/** Starts `n` processes and lets them call `openDatabase` on `path` at the same moment. */
+function openTogether(path: string, n: number) {
+  const children = Array.from({ length: n }, () => {
+    const child = spawn(process.execPath, [
+      '--experimental-strip-types',
+      '--no-warnings',
+      '--input-type=module',
+      '-e',
+      OPEN_ON_SIGNAL,
+      DATABASE_MODULE,
+      path,
+    ]);
+    let out = '';
+    let err = '';
+    child.stderr.on('data', (d: Buffer) => (err += d.toString()));
+    const done = new Promise<{ code: number | null; out: string; err: string }>((resolve) =>
+      child.on('close', (code) => resolve({ code, out: out.replace(/^ready/, ''), err })),
+    );
+    const ready = new Promise<void>((resolve) => {
+      child.stdout.on('data', (d: Buffer) => {
+        out += d.toString();
+        if (out.startsWith('ready')) resolve();
+      });
+      void done.then(() => resolve());
+    });
+    return { child, ready, done };
+  });
+  void Promise.all(children.map((c) => c.ready)).then(() => {
+    for (const c of children) c.child.stdin.end('go');
+  });
+  return children.map((c) => c.done);
+}