diff --git a/bin/codecept.js b/bin/codecept.js index ca1d452f6..f5bfbdf75 100755 --- a/bin/codecept.js +++ b/bin/codecept.js @@ -186,6 +186,7 @@ program .option('-p, --plugins ', 'enable plugins, comma-separated') .option('--shuffle', 'Shuffle the order in which test files run') .option('--shard ', 'run only a fraction of tests (e.g., --shard 1/4)') + .option('--last-failed', 'run only tests that failed in the previous run') // mocha options .option('--colors', 'force enabling of colors') @@ -215,6 +216,7 @@ program .option(commandFlags.config.flag, commandFlags.config.description) .option('-g, --grep ', 'only run tests matching ') .option('-i, --invert', 'inverts --grep matches') + .option('--last-failed', 'run only tests that failed in the previous run') .option('-o, --override [value]', 'override current config options') .option('--suites', 'parallel execution of suites not single tests') .option('--serial ', 'run tests matching one by one after parallel workers finish') diff --git a/docs/commands.md b/docs/commands.md index 11ac8d004..dd998286c 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -53,6 +53,14 @@ Run test files in shuffled order npx codeceptjs run --shuffle ``` +Rerun only tests that failed in the previous run + +```sh +npx codeceptjs run --last-failed +``` + +See [Last Failed](#last-failed) for details. + Run single test in debug mode (see more in [debugging](#Debugging) section) ```sh @@ -158,6 +166,9 @@ npx codeceptjs run-workers 3 --by suite # Pool mode with filtering npx codeceptjs run-workers 4 --by pool --grep "@smoke" +# Rerun only tests that failed in the previous run +npx codeceptjs run-workers 3 --last-failed + # Run tests tagged @serial one by one after parallel workers finish npx codeceptjs run-workers 3 --serial "@serial" ``` @@ -174,6 +185,25 @@ The pool mode provides the best load balancing by maintaining tests in a shared See [Parallel Execution](/parallel) documentation for more details. +## Last Failed + +Rerun only the tests that failed in the previous run. Works with `run` and `run-workers`, including all `--by` strategies: + +```sh +npx codeceptjs run --last-failed +npx codeceptjs run-workers 3 --last-failed +``` + +Every `run` and `run-workers` writes a report to `result.json` in the output directory (`output` in config). It lists each executed test with its state. `--last-failed` reads this report and runs only the tests whose state is `failed`: + +- Skipped and pending tests are not considered failed. +- Tests are matched by their title, including the Feature title. A renamed test or Feature is not matched. +- Combined with `--grep`, only failed tests that also match the pattern are run. +- The report always reflects the most recent run, including a `--last-failed` run. Tests that pass in a rerun drop out of the next `--last-failed` set; tests that did not run (filtered out by `--grep`, sharding, and so on) are not included. +- If the report does not exist, the command fails with "No previous run found". If the last run had no failures, it prints "No failed tests in the last run" and exits with code 0. + +`--last-failed` is different from [run-rerun](#run-rerun), which repeats whole runs to detect flaky tests. + ## Run Rerun Run tests multiple times to detect and fix flaky tests. diff --git a/lib/codecept.js b/lib/codecept.js index 40acd95fc..3166c317e 100644 --- a/lib/codecept.js +++ b/lib/codecept.js @@ -24,6 +24,7 @@ import loadTests from './mocha/loadTests.js' import { validateTypeScriptSetup, getTSNodeESMWarning } from './utils/loaderCheck.js' import recorder from './recorder.js' import store from './store.js' +import Result from './result.js' import WebServer from './webServer.js' import storeListener from './listener/store.js' @@ -271,6 +272,21 @@ class Codecept { return testFiles.slice(startIndex, endIndex) } + /** + * Keeps only tests that failed in the previous run when `--last-failed` is set. + * + * @param {Object} mocha Mocha instance with loaded tests + */ + filterLastFailedTests(mocha) { + if (!this.opts.lastFailed) return + const failedTestUids = new Set(Result.loadFailedTests().map(test => test.uid)) + const filterSuite = suite => { + suite.tests = suite.tests.filter(test => failedTestUids.has(test.uid)) + suite.suites.forEach(filterSuite) + } + filterSuite(mocha.suite) + } + /** * Run a specific test or all loaded tests. * @@ -324,6 +340,7 @@ class Codecept { // Worker threads pre-load and filter tests before run(); reloading duplicates Gherkin suites. if (!mocha.suite.total()) { await loadTests(mocha) + this.filterLastFailedTests(mocha) } else { mocha.lazyLoadFiles(true) } @@ -334,6 +351,7 @@ class Codecept { event.emit(event.all.after, this) // Wait for any recorder tasks added by event.all.after handlers await recorder.promise() + if (!this.opts.child && !store.dryRun && container.result().tests.length) container.result().save() // Set exit code based on test failures if (failures) { process.exitCode = 1 diff --git a/lib/command/run-workers.js b/lib/command/run-workers.js index 3c008e7c8..906b7bfca 100644 --- a/lib/command/run-workers.js +++ b/lib/command/run-workers.js @@ -5,7 +5,9 @@ import store from '../store.js' import event from '../event.js' import Workers from '../workers.js' import Codecept from '../codecept.js' +import Config from '../config.js' import { getMachineInfo } from './info.js' +import { getConfig, getTestRoot, createOutputDir, hasLastFailedTests } from './utils.js' export default async function (workerCount, selectedRuns, options) { process.env.profile = options.profile @@ -36,6 +38,12 @@ export default async function (workerCount, selectedRuns, options) { selectedRuns, } + if (options.lastFailed) { + await getConfig(testConfig) + const outputDir = createOutputDir(Config.append(overrideConfigs), getTestRoot(testConfig)) + if (!(await hasLastFailedTests(outputDir))) return + } + const numberOfWorkers = parseInt(workerCount, 10) output.print(`CodeceptJS v${Codecept.version()} ${output.standWithUkraine()}`) diff --git a/lib/command/run.js b/lib/command/run.js index 8a54c10d1..737ef97db 100644 --- a/lib/command/run.js +++ b/lib/command/run.js @@ -1,4 +1,4 @@ -import { getConfig, printError, getTestRoot, createOutputDir, autoExit } from './utils.js' +import { getConfig, printError, getTestRoot, createOutputDir, autoExit, hasLastFailedTests } from './utils.js' import Config from '../config.js' import store from '../store.js' import Codecept from '../codecept.js' @@ -21,7 +21,8 @@ export default async function (test, options) { config = Config.append(JSON.parse(options.override)) } const testRoot = getTestRoot(configFile) - createOutputDir(config, testRoot) + const outputDir = createOutputDir(config, testRoot) + if (options.lastFailed && !(await hasLastFailedTests(outputDir))) return const codecept = new Codecept(config, options) diff --git a/lib/command/utils.js b/lib/command/utils.js index 49b1467a7..1192c1f60 100644 --- a/lib/command/utils.js +++ b/lib/command/utils.js @@ -105,6 +105,21 @@ export const createOutputDir = (config, testRoot) => { output.print(`creating output directory: ${outputDir}`) mkdirp.sync(outputDir) } + return outputDir +} + +export async function hasLastFailedTests(outputDir) { + const { default: Result } = await import('../result.js') + let failedTests + try { + failedTests = Result.loadFailedTests(outputDir) + } catch (err) { + output.error(err.message) + process.exitCode = 1 + return false + } + if (!failedTests.length) output.print('No failed tests in the last run') + return failedTests.length > 0 } export async function autoExit() { diff --git a/lib/command/workers/runTests.js b/lib/command/workers/runTests.js index a0e12ca90..9c9586ec4 100644 --- a/lib/command/workers/runTests.js +++ b/lib/command/workers/runTests.js @@ -287,6 +287,7 @@ async function runPoolTests() { // Load only the assigned test file mocha.files = [testIdentifier] await loadTests(mocha) + codecept.filterLastFailedTests(mocha) filterSuites(mocha.suite.suites, test => !skipTests.includes(test.uid)) if (mocha.suite.total() > 0) { diff --git a/lib/listener/result.js b/lib/listener/result.js index acd74bfaa..8e34c61f2 100644 --- a/lib/listener/result.js +++ b/lib/listener/result.js @@ -9,4 +9,8 @@ export default function () { event.dispatcher.on(event.test.before, test => { container.result().addTest(test) }) + + event.dispatcher.on(event.test.failed, test => { + container.result().addTest(test) + }) } diff --git a/lib/plugin/junitReporter.js b/lib/plugin/junitReporter.js index a3f104e5d..2e84d2076 100644 --- a/lib/plugin/junitReporter.js +++ b/lib/plugin/junitReporter.js @@ -122,7 +122,10 @@ export default function (config = {}) { function buildXml(result, config, hookFailures = []) { const doc = new DOMImplementation().createDocument(null, null, null) - const allTests = result.tests.concat(hookFailures) + const failureKey = (suiteTitle, message) => `${suiteTitle || ''}::${String(message || '').replace(/\s+/g, ' ').trim()}` + const hookFailureKeys = new Set(hookFailures.map(hf => failureKey(hf.parent && hf.parent.title, hf.err && hf.err.message))) + const tests = result.tests.filter(test => !hookFailureKeys.has(failureKey(test.parent && test.parent.title, test.err && test.err.message))) + const allTests = tests.concat(hookFailures) const suites = groupBySuite(allTests) const root = doc.createElement('testsuites') diff --git a/lib/result.js b/lib/result.js index c7ff4f587..8fbe5cc3e 100644 --- a/lib/result.js +++ b/lib/result.js @@ -216,6 +216,22 @@ class Result { fs.writeFileSync(path.join(store.outputDir, fileName), JSON.stringify(this.simplify(), null, 2)) } + /** + * Loads tests that failed in a result saved by a previous run. + * + * @param {string} [outputDir] Directory containing the saved result. Defaults to `output_dir`. + * @param {string} [fileName] Path to the JSON file, relative to `outputDir`. Defaults to "result.json". + * @returns {Array<{uid: string, title: string, state: string}>} Serialized failed tests. + */ + static loadFailedTests(outputDir = store.outputDir, fileName = 'result.json') { + const resultFile = path.join(outputDir, fileName) + if (!fs.existsSync(resultFile)) { + throw new Error(`No previous run found (${resultFile} does not exist), run tests once first`) + } + const { tests = [] } = JSON.parse(fs.readFileSync(resultFile, 'utf8')) + return tests.filter(test => test.state === 'failed') + } + /** * Adds stats to this result. * diff --git a/lib/workers.js b/lib/workers.js index 284645823..25119512d 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -406,6 +406,7 @@ class Workers extends EventEmitter { const mocha = Container.mocha() mocha.files = files await loadTests(mocha) + this.codecept.filterLastFailedTests(mocha) const groups = populateGroups(numberOfWorkers) let groupCounter = 0 @@ -523,6 +524,7 @@ class Workers extends EventEmitter { const mocha = Container.mocha() mocha.files = files await loadTests(mocha) + this.codecept.filterLastFailedTests(mocha) const suites = mocha.suite.suites.map(suite => suite.tests.filter(test => test)) const selected = suites.map(tests => tests.filter(test => matchesGrep(mocha, test))).filter(tests => tests.length) @@ -885,6 +887,7 @@ class Workers extends EventEmitter { // For tests with retries configured, emit all failures + final success // For tests without retries, emit only final state const lastState = states[states.length - 1] + Container.result().addTest(deserializeTest(lastState.data)) // Check if this test had retries by looking for failure followed by success const hasRetryPattern = states.length > 1 && states.some((s, i) => s.isFailed && i < states.length - 1 && !states[i + 1].isFailed) @@ -912,6 +915,7 @@ class Workers extends EventEmitter { this.emit(event.all.result, Container.result()) event.dispatcher.emit(event.workers.result, Container.result()) + if (Container.result().tests.length) Container.result().save() this.emit('end') // internal event } diff --git a/test/data/sandbox/configs/last-failed-hooks/codecept.conf.js b/test/data/sandbox/configs/last-failed-hooks/codecept.conf.js new file mode 100644 index 000000000..ecfd6ff5b --- /dev/null +++ b/test/data/sandbox/configs/last-failed-hooks/codecept.conf.js @@ -0,0 +1,9 @@ +export const config = { + tests: './*_test.js', + output: './output', + helpers: {}, + include: {}, + bootstrap: null, + mocha: {}, + name: 'last-failed-hooks', +} diff --git a/test/data/sandbox/configs/last-failed-hooks/hooks_test.js b/test/data/sandbox/configs/last-failed-hooks/hooks_test.js new file mode 100644 index 000000000..c4773f736 --- /dev/null +++ b/test/data/sandbox/configs/last-failed-hooks/hooks_test.js @@ -0,0 +1,15 @@ +import assert from 'assert' + +Feature('Hooks') + +BeforeSuite(() => { + assert.equal(process.env.LAST_FAILED_HOOKS_OK, '1') +}) + +Scenario('prepares data', () => { + console.log('executed: prepares data') +}) + +Scenario('serves requests @smoke', () => { + console.log('executed: serves requests') +}) diff --git a/test/data/sandbox/configs/last-failed/codecept.conf.js b/test/data/sandbox/configs/last-failed/codecept.conf.js new file mode 100644 index 000000000..5df0d76ac --- /dev/null +++ b/test/data/sandbox/configs/last-failed/codecept.conf.js @@ -0,0 +1,9 @@ +export const config = { + tests: './*_test.js', + output: './output', + helpers: {}, + include: {}, + bootstrap: null, + mocha: {}, + name: 'last-failed', +} diff --git a/test/data/sandbox/configs/last-failed/math_test.js b/test/data/sandbox/configs/last-failed/math_test.js new file mode 100644 index 000000000..d1ba1ae8c --- /dev/null +++ b/test/data/sandbox/configs/last-failed/math_test.js @@ -0,0 +1,22 @@ +import assert from 'assert' + +Feature('Math') + +Scenario('adds numbers', () => { + console.log('executed: adds numbers') + assert.equal(1 + 1, 2) +}) + +Scenario('divides numbers @smoke', () => { + console.log('executed: divides numbers') + assert.equal(4 / 2, 3) +}) + +Scenario('multiplies numbers', () => { + console.log('executed: multiplies numbers') + assert.equal(2 * 2, process.env.LAST_FAILED_FIXED ? 4 : 5) +}) + +xScenario('subtracts numbers', () => { + console.log('executed: subtracts numbers') +}) diff --git a/test/data/sandbox/configs/last-failed/strings_test.js b/test/data/sandbox/configs/last-failed/strings_test.js new file mode 100644 index 000000000..db28988f0 --- /dev/null +++ b/test/data/sandbox/configs/last-failed/strings_test.js @@ -0,0 +1,13 @@ +import assert from 'assert' + +Feature('Strings') + +Scenario('concats strings', () => { + console.log('executed: concats strings') + assert.equal('a' + 'b', 'ab') +}) + +Scenario('uppercases strings @smoke', () => { + console.log('executed: uppercases strings') + assert.equal('a'.toUpperCase(), 'a') +}) diff --git a/test/runner/last_failed_test.js b/test/runner/last_failed_test.js new file mode 100644 index 000000000..d9dd1f386 --- /dev/null +++ b/test/runner/last_failed_test.js @@ -0,0 +1,205 @@ +import { expect } from 'expect' +import path from 'path' +import fs from 'fs' +import { exec } from 'child_process' +import { fileURLToPath } from 'url' + +const __filename = fileURLToPath(import.meta.url) +const __dirname = path.dirname(__filename) + +const runner = path.join(__dirname, '/../../bin/codecept.js') +const codecept_dir = path.join(__dirname, '/../data/sandbox/configs/last-failed') +const config = `--config ${codecept_dir}/codecept.conf.js` +const outputDir = path.join(codecept_dir, 'output') +const resultFile = path.join(outputDir, 'result.json') + +const hooks_dir = path.join(__dirname, '/../data/sandbox/configs/last-failed-hooks') +const hooksConfig = `--config ${hooks_dir}/codecept.conf.js` +const hooksResultFile = path.join(hooks_dir, 'output', 'result.json') + +const run = (command, env = {}) => + new Promise(resolve => { + exec(`${runner} ${command} ${config}`, { env: { ...process.env, LAST_FAILED_FIXED: '', ...env } }, (err, stdout) => { + resolve({ code: err ? err.code : 0, stdout }) + }) + }) + +const runHooks = (command, env = {}) => + new Promise(resolve => { + exec(`${runner} ${command} ${hooksConfig}`, { env: { ...process.env, LAST_FAILED_HOOKS_OK: '', ...env } }, (err, stdout) => { + resolve({ code: err ? err.code : 0, stdout }) + }) + }) + +const reportedTests = () => { + const { tests } = JSON.parse(fs.readFileSync(resultFile, 'utf8')) + return Object.fromEntries(tests.map(test => [test.title, test.state])) +} + +const executedTests = stdout => + stdout + .split('\n') + .filter(line => line.startsWith('executed: ')) + .map(line => line.replace('executed: ', '').trim()) + .sort() + +describe('--last-failed', function () { + this.timeout(60000) + + beforeEach(() => { + fs.rmSync(outputDir, { recursive: true, force: true }) + fs.rmSync(path.join(hooks_dir, 'output'), { recursive: true, force: true }) + }) + + it('should save a report with test states after a run', async () => { + const { code } = await run('run') + expect(code).toEqual(1) + expect(reportedTests()).toEqual({ + 'adds numbers': 'passed', + 'divides numbers @smoke': 'failed', + 'multiplies numbers': 'failed', + 'concats strings': 'passed', + 'uppercases strings @smoke': 'failed', + }) + }) + + it('should rerun only failed tests', async () => { + await run('run') + const { code, stdout } = await run('run --last-failed') + expect(code).toEqual(1) + expect(executedTests(stdout)).toEqual(['divides numbers', 'multiplies numbers', 'uppercases strings']) + expect(stdout).toContain('0 passed, 3 failed') + }) + + it('should drop fixed tests from the next --last-failed run', async () => { + await run('run') + const fixedRun = await run('run --last-failed', { LAST_FAILED_FIXED: '1' }) + expect(fixedRun.stdout).toContain('1 passed, 2 failed') + expect(reportedTests()).toEqual({ + 'divides numbers @smoke': 'failed', + 'multiplies numbers': 'passed', + 'uppercases strings @smoke': 'failed', + }) + + const { stdout } = await run('run --last-failed') + expect(executedTests(stdout)).toEqual(['divides numbers', 'uppercases strings']) + }) + + it('should run intersection with --grep', async () => { + await run('run') + const { stdout } = await run('run --last-failed --grep "@smoke"') + expect(executedTests(stdout)).toEqual(['divides numbers', 'uppercases strings']) + expect(stdout).toContain('0 passed, 2 failed') + }) + + it('should rerun only failed tests in workers', async () => { + const fullRun = await run('run-workers 2') + expect(fullRun.stdout).toContain('2 passed, 3 failed') + expect(reportedTests()).toMatchObject({ + 'adds numbers': 'passed', + 'divides numbers @smoke': 'failed', + 'multiplies numbers': 'failed', + 'concats strings': 'passed', + 'uppercases strings @smoke': 'failed', + }) + + const { code, stdout } = await run('run-workers 2 --last-failed') + expect(code).toEqual(1) + expect(stdout).toContain('Worker 1: 2 tests') + expect(stdout).toContain('Worker 2: 1 test') + expect(stdout).toContain('0 passed, 3 failed') + expect(stdout).not.toContain('adds numbers') + expect(stdout).not.toContain('concats strings') + }) + + it('should run intersection with --grep in workers by suite', async () => { + await run('run') + const { stdout } = await run('run-workers 2 --by suite --last-failed --grep "@smoke"') + expect(stdout).toContain('0 passed, 2 failed') + expect(stdout).not.toContain('multiplies numbers') + }) + + it('should rerun only failed tests in workers by pool', async () => { + await run('run') + const { stdout } = await run('run-workers 2 --by pool --last-failed') + expect(stdout).toContain('0 passed, 3 failed') + expect(reportedTests()).toEqual({ + 'divides numbers @smoke': 'failed', + 'multiplies numbers': 'failed', + 'uppercases strings @smoke': 'failed', + }) + }) + + it('should fail when there is no previous run', async () => { + for (const command of ['run --last-failed', 'run-workers 2 --last-failed']) { + const { code, stdout } = await run(command) + expect(code).toEqual(1) + expect(stdout).toContain('No previous run found') + expect(stdout).toContain('run tests once first') + } + }) + + it('should run nothing when the last run had no failures', async () => { + await run('run --grep "adds numbers"') + for (const command of ['run --last-failed', 'run-workers 2 --last-failed']) { + const { code, stdout } = await run(command) + expect(code).toEqual(0) + expect(stdout).toContain('No failed tests in the last run') + expect(stdout).not.toContain('executed:') + expect(stdout).not.toContain('passed') + } + }) + + it('should not wipe result.json when a run executes no tests', async () => { + await run('run') + expect(reportedTests()['multiplies numbers']).toEqual('failed') + + const { stdout } = await run('run --grep "no such test"') + expect(executedTests(stdout)).toEqual([]) + + expect(reportedTests()['multiplies numbers']).toEqual('failed') + const { stdout: rerun } = await run('run --last-failed') + expect(executedTests(rerun)).toEqual(['divides numbers', 'multiplies numbers', 'uppercases strings']) + }) + + it('should not wipe result.json in workers when a run executes no tests', async () => { + await run('run-workers 2') + expect(reportedTests()['multiplies numbers']).toEqual('failed') + + const { stdout } = await run('run-workers 2 --grep "no such test"') + expect(executedTests(stdout)).toEqual([]) + + expect(reportedTests()['multiplies numbers']).toEqual('failed') + }) + + it('should rerun tests of a suite with a failed BeforeSuite', async () => { + const { code } = await runHooks('run') + expect(code).toEqual(1) + const { tests } = JSON.parse(fs.readFileSync(hooksResultFile, 'utf8')) + expect(Object.fromEntries(tests.map(test => [test.title, test.state]))).toEqual({ + 'prepares data': 'failed', + 'serves requests @smoke': 'failed', + }) + + const { stdout } = await runHooks('run --last-failed', { LAST_FAILED_HOOKS_OK: '1' }) + expect(executedTests(stdout)).toEqual(['prepares data', 'serves requests']) + }) + + it('should rerun tests of a suite with a failed BeforeSuite in workers', async () => { + await runHooks('run-workers 2') + const { tests } = JSON.parse(fs.readFileSync(hooksResultFile, 'utf8')) + expect(Object.fromEntries(tests.map(test => [test.title, test.state]))).toEqual({ + 'prepares data': 'failed', + 'serves requests @smoke': 'failed', + }) + + const { code, stdout } = await runHooks('run-workers 2 --last-failed', { LAST_FAILED_HOOKS_OK: '1' }) + expect(code).toEqual(0) + expect(stdout).toContain('2 passed') + const { tests: rerunTests } = JSON.parse(fs.readFileSync(hooksResultFile, 'utf8')) + expect(Object.fromEntries(rerunTests.map(test => [test.title, test.state]))).toEqual({ + 'prepares data': 'passed', + 'serves requests @smoke': 'passed', + }) + }) +})