feat: --last-failed option for run and run-workers

bin/codecept.js

@@ -186,6 +186,7 @@ program
   .option('-p, --plugins <k=v,k2=v2,...>', 'enable plugins, comma-separated')
   .option('--shuffle', 'Shuffle the order in which test files run')
   .option('--shard <index/total>', 'run only a fraction of tests (e.g., --shard 1/4)')
+  .option('--last-failed', 'run only tests that failed in the previous run')
 
   // mocha options
   .option('--colors', 'force enabling of colors')
@@ -215,6 +216,7 @@ program
   .option(commandFlags.config.flag, commandFlags.config.description)
   .option('-g, --grep <pattern>', 'only run tests matching <pattern>')
   .option('-i, --invert', 'inverts --grep matches')
+  .option('--last-failed', 'run only tests that failed in the previous run')
   .option('-o, --override [value]', 'override current config options')
   .option('--suites', 'parallel execution of suites not single tests')
   .option('--serial <pattern>', 'run tests matching <pattern> one by one after parallel workers finish')

docs/commands.md

@@ -53,6 +53,14 @@ Run test files in shuffled order
 npx codeceptjs run --shuffle
 ```
 
+Rerun only tests that failed in the previous run
+
+```sh
+npx codeceptjs run --last-failed
+```
+
+See [Last Failed](#last-failed) for details.
+
 Run single test in debug mode (see more in [debugging](#Debugging) section)
 
 ```sh
@@ -158,6 +166,9 @@ npx codeceptjs run-workers 3 --by suite
 # Pool mode with filtering
 npx codeceptjs run-workers 4 --by pool --grep "@smoke"
 
+# Rerun only tests that failed in the previous run
+npx codeceptjs run-workers 3 --last-failed
+
 # Run tests tagged @serial one by one after parallel workers finish
 npx codeceptjs run-workers 3 --serial "@serial"
 ```
@@ -174,6 +185,25 @@ The pool mode provides the best load balancing by maintaining tests in a shared
 
 See [Parallel Execution](/parallel) documentation for more details.
 
+## Last Failed
+
+Rerun only the tests that failed in the previous run. Works with `run` and `run-workers`, including all `--by` strategies:
+
+```sh
+npx codeceptjs run --last-failed
+npx codeceptjs run-workers 3 --last-failed
+```
+
+Every `run` and `run-workers` writes a report to `result.json` in the output directory (`output` in config). It lists each executed test with its state. `--last-failed` reads this report and runs only the tests whose state is `failed`:
+
+- Skipped and pending tests are not considered failed.
+- Tests are matched by their title, including the Feature title. A renamed test or Feature is not matched.
+- Combined with `--grep`, only failed tests that also match the pattern are run.
+- The report always reflects the most recent run, including a `--last-failed` run. Tests that pass in a rerun drop out of the next `--last-failed` set; tests that did not run (filtered out by `--grep`, sharding, and so on) are not included.
+- If the report does not exist, the command fails with "No previous run found". If the last run had no failures, it prints "No failed tests in the last run" and exits with code 0.
+
+`--last-failed` is different from [run-rerun](#run-rerun), which repeats whole runs to detect flaky tests.
+
 ## Run Rerun <Badge text="Since 3.3.6" type="warning"/>
 
 Run tests multiple times to detect and fix flaky tests.

lib/codecept.js

@@ -24,6 +24,7 @@ import loadTests from './mocha/loadTests.js'
 import { validateTypeScriptSetup, getTSNodeESMWarning } from './utils/loaderCheck.js'
 import recorder from './recorder.js'
 import store from './store.js'
+import Result from './result.js'
 import WebServer from './webServer.js'
 
 import storeListener from './listener/store.js'
@@ -271,6 +272,21 @@ class Codecept {
     return testFiles.slice(startIndex, endIndex)
   }
 
+  /**
+   * Keeps only tests that failed in the previous run when `--last-failed` is set.
+   *
+   * @param {Object} mocha Mocha instance with loaded tests
+   */
+  filterLastFailedTests(mocha) {
+    if (!this.opts.lastFailed) return
+    const failedTestUids = new Set(Result.loadFailedTests().map(test => test.uid))
+    const filterSuite = suite => {
+      suite.tests = suite.tests.filter(test => failedTestUids.has(test.uid))
+      suite.suites.forEach(filterSuite)
+    }
+    filterSuite(mocha.suite)
+  }
+
   /**
    * Run a specific test or all loaded tests.
    *
@@ -324,6 +340,7 @@ class Codecept {
     // Worker threads pre-load and filter tests before run(); reloading duplicates Gherkin suites.
     if (!mocha.suite.total()) {
       await loadTests(mocha)
+      this.filterLastFailedTests(mocha)
     } else {
       mocha.lazyLoadFiles(true)
     }
@@ -334,6 +351,7 @@ class Codecept {
         event.emit(event.all.after, this)
         // Wait for any recorder tasks added by event.all.after handlers
         await recorder.promise()
+        if (!this.opts.child && !store.dryRun && container.result().tests.length) container.result().save()
         // Set exit code based on test failures
         if (failures) {
           process.exitCode = 1

lib/command/run-workers.js

@@ -5,7 +5,9 @@ import store from '../store.js'
 import event from '../event.js'
 import Workers from '../workers.js'
 import Codecept from '../codecept.js'
+import Config from '../config.js'
 import { getMachineInfo } from './info.js'
+import { getConfig, getTestRoot, createOutputDir, hasLastFailedTests } from './utils.js'
 
 export default async function (workerCount, selectedRuns, options) {
   process.env.profile = options.profile
@@ -36,6 +38,12 @@ export default async function (workerCount, selectedRuns, options) {
     selectedRuns,
   }
 
+  if (options.lastFailed) {
+    await getConfig(testConfig)
+    const outputDir = createOutputDir(Config.append(overrideConfigs), getTestRoot(testConfig))
+    if (!(await hasLastFailedTests(outputDir))) return
+  }
+
   const numberOfWorkers = parseInt(workerCount, 10)
 
   output.print(`CodeceptJS v${Codecept.version()} ${output.standWithUkraine()}`)

lib/command/run.js

@@ -1,4 +1,4 @@
-import { getConfig, printError, getTestRoot, createOutputDir, autoExit } from './utils.js'
+import { getConfig, printError, getTestRoot, createOutputDir, autoExit, hasLastFailedTests } from './utils.js'
 import Config from '../config.js'
 import store from '../store.js'
 import Codecept from '../codecept.js'
@@ -21,7 +21,8 @@ export default async function (test, options) {
     config = Config.append(JSON.parse(options.override))
   }
   const testRoot = getTestRoot(configFile)
-  createOutputDir(config, testRoot)
+  const outputDir = createOutputDir(config, testRoot)
+  if (options.lastFailed && !(await hasLastFailedTests(outputDir))) return
 
   const codecept = new Codecept(config, options)
 

lib/command/utils.js

@@ -105,6 +105,21 @@ export const createOutputDir = (config, testRoot) => {
     output.print(`creating output directory: ${outputDir}`)
     mkdirp.sync(outputDir)
   }
+  return outputDir
+}
+
+export async function hasLastFailedTests(outputDir) {
+  const { default: Result } = await import('../result.js')
+  let failedTests
+  try {
+    failedTests = Result.loadFailedTests(outputDir)
+  } catch (err) {
+    output.error(err.message)
+    process.exitCode = 1
+    return false
+  }
+  if (!failedTests.length) output.print('No failed tests in the last run')
+  return failedTests.length > 0
 }
 
 export async function autoExit() {

lib/command/workers/runTests.js

@@ -287,6 +287,7 @@ async function runPoolTests() {
             // Load only the assigned test file
             mocha.files = [testIdentifier]
             await loadTests(mocha)
+            codecept.filterLastFailedTests(mocha)
             filterSuites(mocha.suite.suites, test => !skipTests.includes(test.uid))
 
             if (mocha.suite.total() > 0) {

lib/listener/result.js

@@ -9,4 +9,8 @@ export default function () {
   event.dispatcher.on(event.test.before, test => {
     container.result().addTest(test)
   })
+
+  event.dispatcher.on(event.test.failed, test => {
+    container.result().addTest(test)
+  })
 }

lib/plugin/junitReporter.js

@@ -122,7 +122,10 @@ export default function (config = {}) {
 
 function buildXml(result, config, hookFailures = []) {
   const doc = new DOMImplementation().createDocument(null, null, null)
-  const allTests = result.tests.concat(hookFailures)
+  const failureKey = (suiteTitle, message) => `${suiteTitle || ''}::${String(message || '').replace(/\s+/g, ' ').trim()}`
+  const hookFailureKeys = new Set(hookFailures.map(hf => failureKey(hf.parent && hf.parent.title, hf.err && hf.err.message)))
+  const tests = result.tests.filter(test => !hookFailureKeys.has(failureKey(test.parent && test.parent.title, test.err && test.err.message)))
+  const allTests = tests.concat(hookFailures)
   const suites = groupBySuite(allTests)
 
   const root = doc.createElement('testsuites')

lib/result.js

@@ -216,6 +216,22 @@ class Result {
     fs.writeFileSync(path.join(store.outputDir, fileName), JSON.stringify(this.simplify(), null, 2))
   }
 
+  /**
+   * Loads tests that failed in a result saved by a previous run.
+   *
+   * @param {string} [outputDir] Directory containing the saved result. Defaults to `output_dir`.
+   * @param {string} [fileName] Path to the JSON file, relative to `outputDir`. Defaults to "result.json".
+   * @returns {Array<{uid: string, title: string, state: string}>} Serialized failed tests.
+   */
+  static loadFailedTests(outputDir = store.outputDir, fileName = 'result.json') {
+    const resultFile = path.join(outputDir, fileName)
+    if (!fs.existsSync(resultFile)) {
+      throw new Error(`No previous run found (${resultFile} does not exist), run tests once first`)
+    }
+    const { tests = [] } = JSON.parse(fs.readFileSync(resultFile, 'utf8'))
+    return tests.filter(test => test.state === 'failed')
+  }
+
   /**
    * Adds stats to this result.
    *

lib/workers.js

@@ -406,6 +406,7 @@ class Workers extends EventEmitter {
     const mocha = Container.mocha()
     mocha.files = files
     await loadTests(mocha)
+    this.codecept.filterLastFailedTests(mocha)
 
     const groups = populateGroups(numberOfWorkers)
     let groupCounter = 0
@@ -523,6 +524,7 @@ class Workers extends EventEmitter {
     const mocha = Container.mocha()
     mocha.files = files
     await loadTests(mocha)
+    this.codecept.filterLastFailedTests(mocha)
 
     const suites = mocha.suite.suites.map(suite => suite.tests.filter(test => test))
     const selected = suites.map(tests => tests.filter(test => matchesGrep(mocha, test))).filter(tests => tests.length)
@@ -885,6 +887,7 @@ class Workers extends EventEmitter {
         // For tests with retries configured, emit all failures + final success
         // For tests without retries, emit only final state
         const lastState = states[states.length - 1]
+        Container.result().addTest(deserializeTest(lastState.data))
 
         // Check if this test had retries by looking for failure followed by success
         const hasRetryPattern = states.length > 1 && states.some((s, i) => s.isFailed && i < states.length - 1 && !states[i + 1].isFailed)
@@ -912,6 +915,7 @@ class Workers extends EventEmitter {
 
     this.emit(event.all.result, Container.result())
     event.dispatcher.emit(event.workers.result, Container.result())
+    if (Container.result().tests.length) Container.result().save()
     this.emit('end') // internal event
   }
 

test/data/sandbox/configs/last-failed-hooks/codecept.conf.js

@@ -0,0 +1,9 @@
+export const config = {
+  tests: './*_test.js',
+  output: './output',
+  helpers: {},
+  include: {},
+  bootstrap: null,
+  mocha: {},
+  name: 'last-failed-hooks',
+}

test/data/sandbox/configs/last-failed-hooks/hooks_test.js

@@ -0,0 +1,15 @@
+import assert from 'assert'
+
+Feature('Hooks')
+
+BeforeSuite(() => {
+  assert.equal(process.env.LAST_FAILED_HOOKS_OK, '1')
+})
+
+Scenario('prepares data', () => {
+  console.log('executed: prepares data')
+})
+
+Scenario('serves requests @smoke', () => {
+  console.log('executed: serves requests')
+})

test/data/sandbox/configs/last-failed/codecept.conf.js

@@ -0,0 +1,9 @@
+export const config = {
+  tests: './*_test.js',
+  output: './output',
+  helpers: {},
+  include: {},
+  bootstrap: null,
+  mocha: {},
+  name: 'last-failed',
+}

test/data/sandbox/configs/last-failed/math_test.js

@@ -0,0 +1,22 @@
+import assert from 'assert'
+
+Feature('Math')
+
+Scenario('adds numbers', () => {
+  console.log('executed: adds numbers')
+  assert.equal(1 + 1, 2)
+})
+
+Scenario('divides numbers @smoke', () => {
+  console.log('executed: divides numbers')
+  assert.equal(4 / 2, 3)
+})
+
+Scenario('multiplies numbers', () => {
+  console.log('executed: multiplies numbers')
+  assert.equal(2 * 2, process.env.LAST_FAILED_FIXED ? 4 : 5)
+})
+
+xScenario('subtracts numbers', () => {
+  console.log('executed: subtracts numbers')
+})

test/data/sandbox/configs/last-failed/strings_test.js

@@ -0,0 +1,13 @@
+import assert from 'assert'
+
+Feature('Strings')
+
+Scenario('concats strings', () => {
+  console.log('executed: concats strings')
+  assert.equal('a' + 'b', 'ab')
+})
+
+Scenario('uppercases strings @smoke', () => {
+  console.log('executed: uppercases strings')
+  assert.equal('a'.toUpperCase(), 'a')
+})