diff --git a/src/run.ts b/src/run.ts index d9d6bea..7ade731 100755 --- a/src/run.ts +++ b/src/run.ts @@ -125,6 +125,11 @@ runner.on('test:stdout', (data: any) => { }); runner.on('test:fail', (data: any) => { + // node:test semantics: a failing `todo` or `skip` test doesn't fail the run. The marker is `true` + // or the reason string, which can be empty (`{ todo: '' }`). + if (data.todo || data.todo === '' || data.skip || data.skip === '') { + return; + } process.exitCode = 1; if (data.file) { failedFiles.add(resolve(data.file)); diff --git a/test/fixtures/run/empty-reason-todo-fail.ts b/test/fixtures/run/empty-reason-todo-fail.ts new file mode 100644 index 0000000..ca60716 --- /dev/null +++ b/test/fixtures/run/empty-reason-todo-fail.ts @@ -0,0 +1,6 @@ +import { test } from 'node:test'; +import assert from 'node:assert'; + +test('a todo test with an empty reason that fails', { todo: '' }, () => { + assert.strictEqual(1, 2); +}); diff --git a/test/fixtures/run/real-fail.ts b/test/fixtures/run/real-fail.ts new file mode 100644 index 0000000..be2a719 --- /dev/null +++ b/test/fixtures/run/real-fail.ts @@ -0,0 +1,6 @@ +import { test } from 'node:test'; +import assert from 'node:assert'; + +test('a real failure', () => { + assert.strictEqual(1, 2); +}); diff --git a/test/fixtures/run/skip-fail.ts b/test/fixtures/run/skip-fail.ts new file mode 100644 index 0000000..a9a8a19 --- /dev/null +++ b/test/fixtures/run/skip-fail.ts @@ -0,0 +1,9 @@ +import { test } from 'node:test'; +import assert from 'node:assert'; + +// `t.skip()` from inside the body, not `{ skip: true }`: the option skips the body entirely, so the +// test never fails and never reaches the runner's `data.skip` branch. +test('a skipped test that fails', (t) => { + t.skip('skipped for testing'); + assert.strictEqual(1, 2); +}); diff --git a/test/fixtures/run/todo-fail.ts b/test/fixtures/run/todo-fail.ts new file mode 100644 index 0000000..0ce32c5 --- /dev/null +++ b/test/fixtures/run/todo-fail.ts @@ -0,0 +1,6 @@ +import { test } from 'node:test'; +import assert from 'node:assert'; + +test('a todo test that fails', { todo: true }, () => { + assert.strictEqual(1, 2); +}); diff --git a/test/run.test.ts b/test/run.test.ts new file mode 100644 index 0000000..ee18f2b --- /dev/null +++ b/test/run.test.ts @@ -0,0 +1,40 @@ +import { test } from 'node:test'; +import { strictEqual, ok } from 'node:assert'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { fileURLToPath } from 'node:url'; + +const RUN_SCRIPT = fileURLToPath(new URL('../src/run.ts', import.meta.url)); + +function fixture(name: string): string { + return fileURLToPath(new URL(`./fixtures/run/${name}`, import.meta.url)); +} + +const cleanEnv = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith('HARPER_INTEGRATION_TEST_'))); + +async function runCli(globPattern: string, signal: AbortSignal): Promise { + const child = spawn(process.execPath, [RUN_SCRIPT, '--isolation=none', globPattern], { + stdio: 'ignore', + env: cleanEnv, + signal, + }); + const [code, exitSignal] = await once(child, 'exit'); + ok(exitSignal === null, `run.ts was killed by signal ${exitSignal}`); + return code; +} + +test('a suite whose only failure is a todo test exits 0', { timeout: 15_000 }, async (t) => { + strictEqual(await runCli(fixture('todo-fail.ts'), t.signal), 0); +}); + +test('a suite whose only failure is a skipped test exits 0', { timeout: 15_000 }, async (t) => { + strictEqual(await runCli(fixture('skip-fail.ts'), t.signal), 0); +}); + +test('a suite whose only failure is a todo test with an empty reason exits 0', { timeout: 15_000 }, async (t) => { + strictEqual(await runCli(fixture('empty-reason-todo-fail.ts'), t.signal), 0); +}); + +test('a suite with a real failure exits 1 (control)', { timeout: 15_000 }, async (t) => { + strictEqual(await runCli(fixture('real-fail.ts'), t.signal), 1); +});