diff --git a/docs/bootstrap.md b/docs/bootstrap.md index b84e7db74..3f08aaede 100644 --- a/docs/bootstrap.md +++ b/docs/bootstrap.md @@ -10,6 +10,8 @@ you can use the `bootstrap` and `teardown` config. Use it to start and stop a we When using the [parallel execution](/parallel) mode, there are two additional hooks available; `bootstrapAll` and `teardownAll`. See [bootstrapAll & teardownAll](#bootstrapall-teardownall) for more information. +> To start and stop the application under test, use the [`webServer`](/configuration#web-server) option instead of writing bootstrap code. + ### Example: Bootstrap & Teardown diff --git a/docs/configuration.md b/docs/configuration.md index 51d8ff097..2725bf048 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -39,6 +39,7 @@ export const config = { - `bootstrap` / `teardown` — run code before / after the whole run; an async function or a path to a JS module. See [Bootstrap](/bootstrap). - `bootstrapAll` / `teardownAll` — run once around a parallel run (before any worker starts / after all finish). See [bootstrapAll / teardownAll](/bootstrap#bootstrapall-teardownall). +- `webServer` — start the application under test before the run and stop it after. See [Web Server](#web-server). **Test runner** @@ -89,6 +90,41 @@ require: ['tsx/esm', 'should', './lib/testSetup'] The config file itself (`codecept.conf.ts`) and helpers are transpiled automatically — only test files need the loader. See [TypeScript](/typescript) for the full setup. +## Web Server + +`webServer` starts your application before tests run and stops it when they finish, so you don't need a `bootstrap` script for it: + +```js +export const config = { + webServer: { + command: 'npm run start', + url: 'http://localhost:3000', + enabled: !process.env.CI, + }, + // ... +} +``` + +- `command` — shell command that starts the app. +- `url` — CodeceptJS sends GET requests here until it gets any HTTP response, then starts the tests. If `url` already responds before `command` is run, the running server is reused. +- `enabled` — set to `false` to skip starting the server, e.g. `enabled: !process.env.CI` when CI provides its own. Default `true`. +- `timeout` — milliseconds to wait for `url`. Default `60000`. +- `cwd` — working directory for `command`, relative to the config file. Default is the config directory. +- `env` — extra environment variables for `command`. + +If the server exits early or doesn't respond within `timeout`, the run fails and prints the last 20 lines of its output. Run with `--debug` to see all of its output. + +To start several services, pass an array. They start in order and stop in reverse order: + +```js +webServer: [ + { command: 'npm run api', url: 'http://localhost:4000/health' }, + { command: 'npm run start', url: 'http://localhost:3000' }, +], +``` + +The server is started as part of bootstrap, right before the `bootstrap` hook (or `bootstrapAll` for `run-workers` and `run-multiple`), and stopped after `teardown` (`teardownAll`). It starts once in the main process; workers and child processes reuse it. The interactive `shell` starts it too, `dry-run` and `check` only with `--bootstrap`. CodeceptJS stops the command and every process it spawned, including when the run is interrupted with Ctrl+C. + ## Dynamic configuration A JS/TS config file is plain code, so you can read environment variables and build the config at runtime: diff --git a/docs/plugins/aiTrace.md b/docs/plugins/aiTrace.md index 04fc78104..4f4e7fe43 100644 --- a/docs/plugins/aiTrace.md +++ b/docs/plugins/aiTrace.md @@ -1,10 +1,3 @@ ---- -permalink: /plugins/aiTrace -editLink: false -sidebar: auto -title: aiTrace ---- - ## aiTrace @@ -46,4 +39,4 @@ Possible config options: ### Parameters -* `config` **any** +* `config` **any** (optional, default `{}`) diff --git a/lib/codecept.js b/lib/codecept.js index 6c10bca08..40acd95fc 100644 --- a/lib/codecept.js +++ b/lib/codecept.js @@ -6,6 +6,7 @@ import { resolve } from 'path' import { fileURLToPath, pathToFileURL } from 'url' import { dirname } from 'path' import { createRequire } from 'module' +import { isMainThread } from 'worker_threads' const __filename = fileURLToPath(import.meta.url) const __dirname = dirname(__filename) @@ -23,6 +24,7 @@ import loadTests from './mocha/loadTests.js' import { validateTypeScriptSetup, getTSNodeESMWarning } from './utils/loaderCheck.js' import recorder from './recorder.js' import store from './store.js' +import WebServer from './webServer.js' import storeListener from './listener/store.js' import stepsListener from './listener/steps.js' @@ -159,6 +161,7 @@ class Codecept { * @returns {Promise} */ async bootstrap() { + if (isMainThread && !this.opts?.child) await WebServer.startAll(this.config.webServer) return runHook(this.config.bootstrap, 'bootstrap') } @@ -168,7 +171,11 @@ class Codecept { * @returns {Promise} */ async teardown() { - return runHook(this.config.teardown, 'teardown') + try { + return await runHook(this.config.teardown, 'teardown') + } finally { + await WebServer.stopAll() + } } /** diff --git a/lib/command/run-multiple.js b/lib/command/run-multiple.js index d3301214d..19bf7a622 100644 --- a/lib/command/run-multiple.js +++ b/lib/command/run-multiple.js @@ -4,6 +4,7 @@ import crypto from 'crypto' import { fileURLToPath } from 'url' import runHook from '../hooks.js' +import WebServer from '../webServer.js' import event from '../event.js' import { createRuns } from './run-multiple/collection.js' import { clearString, replaceValueDeep } from '../utils.js' @@ -67,6 +68,7 @@ export default async function (selectedRuns, options) { fail('No runs provided. Use --all option to run all configured runs') } + await WebServer.startAll(config.webServer) await runHook(config.bootstrapAll, 'bootstrapAll') event.emit(event.multiple.before, null) @@ -110,6 +112,7 @@ export default async function (selectedRuns, options) { return childProcessesPromise.then(async () => { // fire hook await runHook(config.teardownAll, 'teardownAll') + await WebServer.stopAll() event.emit(event.multiple.after, null) }) } diff --git a/lib/command/run-workers.js b/lib/command/run-workers.js index 00c343532..ff83d05d7 100644 --- a/lib/command/run-workers.js +++ b/lib/command/run-workers.js @@ -40,6 +40,7 @@ export default async function (workerCount, selectedRuns, options) { output.print(`CodeceptJS v${Codecept.version()} ${output.standWithUkraine()}`) output.print(`Running tests in ${output.styles.bold(numberOfWorkers)} workers...`) + if (options.verbose || options.debug) store.debugMode = true store.hasWorkers = true store.workerMode = true process.env.RUNS_WITH_WORKERS = 'true' @@ -76,8 +77,6 @@ export default async function (workerCount, selectedRuns, options) { }) try { - if (options.verbose || options.debug) store.debugMode = true - if (options.verbose) { await getMachineInfo() } diff --git a/lib/webServer.js b/lib/webServer.js new file mode 100644 index 000000000..6f4d7d255 --- /dev/null +++ b/lib/webServer.js @@ -0,0 +1,193 @@ +import { spawn, spawnSync } from 'child_process' +import path from 'path' +import output from './output.js' +import store from './store.js' +import { isWindows } from './utils.js' + +const TAIL_SIZE = 20 +const KILL_TIMEOUT = 5000 +const SIGNALS = ['SIGINT', 'SIGTERM'] + +class WebServer { + static running = [] + + static signalHandlers = Object.fromEntries(SIGNALS.map(signal => [signal, () => WebServer.onSignal(signal)])) + + constructor(config) { + const { command, url, enabled = true, timeout = 60000, cwd = '.', env = {} } = config + if (!command) throw new Error('webServer: "command" is required') + if (!url) throw new Error(`webServer: "url" is required for "${command}"`) + this.command = command + this.url = url + this.enabled = enabled + this.timeout = timeout + this.cwd = cwd + this.env = env + this.tail = [] + this.proc = null + } + + static async startAll(config) { + if (!config) return + const configs = Array.isArray(config) ? config : [config] + for (const serverConfig of configs) { + await new WebServer(serverConfig).start() + } + } + + static async stopAll() { + while (WebServer.running.length) { + await WebServer.running[WebServer.running.length - 1].stop() + WebServer.running.pop() + } + WebServer.removeListeners() + } + + static killAllSync() { + for (const server of WebServer.running) server.kill('SIGKILL') + } + + static onSignal(signal) { + WebServer.killAllSync() + WebServer.removeListeners() + if (process.listenerCount(signal) === 0) process.kill(process.pid, signal) + } + + static addListeners() { + if (WebServer.running.length) return + process.on('exit', WebServer.killAllSync) + for (const signal of SIGNALS) process.on(signal, WebServer.signalHandlers[signal]) + } + + static removeListeners() { + process.removeListener('exit', WebServer.killAllSync) + for (const signal of SIGNALS) process.removeListener(signal, WebServer.signalHandlers[signal]) + } + + async isUp() { + try { + const res = await fetch(this.url, { method: 'GET', redirect: 'manual', signal: AbortSignal.timeout(1000) }) + await res.body?.cancel().catch(() => {}) + return true + } catch (err) { + return false + } + } + + async start() { + if (!this.enabled) return + + if (await this.isUp()) { + output.print(`Reusing web server at ${this.url}`) + return + } + + output.print(`Starting web server: ${this.command}`) + + this.proc = spawn(this.command, { + shell: true, + detached: !isWindows(), + cwd: path.resolve(store.codeceptDir || process.cwd(), this.cwd), + env: { ...process.env, ...this.env }, + stdio: ['ignore', 'pipe', 'pipe'], + }) + + this.collect(this.proc.stdout) + this.collect(this.proc.stderr) + + WebServer.addListeners() + WebServer.running.push(this) + + let exited = null + const closed = new Promise(resolve => this.proc.once('close', resolve)) + this.proc.once('error', err => { + exited = `failed to start: ${err.message}` + }) + this.proc.once('exit', (code, signal) => { + exited = `exited with ${signal ? `signal ${signal}` : `code ${code}`}` + }) + + const deadline = Date.now() + this.timeout + let delay = 100 + while (true) { + if (exited) { + await Promise.race([closed, new Promise(resolve => setTimeout(resolve, 500))]) + await WebServer.stopAll() + throw new Error(`webServer "${this.command}" ${exited} before ${this.url} was ready.${this.formatTail()}`) + } + if (await this.isUp()) break + if (Date.now() >= deadline) { + await WebServer.stopAll() + throw new Error(`webServer "${this.command}" did not respond at ${this.url} within ${this.timeout}ms.${this.formatTail()}`) + } + await new Promise(resolve => setTimeout(resolve, delay)) + delay = Math.min(delay * 2, 1000) + } + } + + async stop() { + if (!this.proc) return + this.kill('SIGTERM') + await this.waitForExit() + this.proc.stdout?.destroy() + this.proc.stderr?.destroy() + } + + kill(signal) { + const { proc } = this + if (!proc?.pid) return + try { + if (isWindows()) { + if (proc.exitCode !== null) return + spawnSync('taskkill', ['/pid', String(proc.pid), '/T', '/F']) + } else { + process.kill(-proc.pid, signal) + } + } catch (err) { + try { + proc.kill(signal) + } catch (e) {} + } + } + + waitForExit() { + const { proc } = this + return new Promise(resolve => { + if (proc.exitCode !== null || proc.signalCode !== null) return resolve() + let killTimer + let safetyTimer + const finish = () => { + clearTimeout(killTimer) + clearTimeout(safetyTimer) + resolve() + } + proc.once('exit', finish) + killTimer = setTimeout(() => this.kill('SIGKILL'), KILL_TIMEOUT) + safetyTimer = setTimeout(finish, KILL_TIMEOUT + 500) + }) + } + + collect(stream) { + let rest = '' + stream.on('data', chunk => { + const lines = (rest + chunk.toString()).split(/\r?\n/) + rest = lines.pop() + for (const line of lines) this.addLine(line) + }) + stream.on('end', () => this.addLine(rest)) + } + + addLine(line) { + if (!line.trim()) return + this.tail.push(line) + if (this.tail.length > TAIL_SIZE) this.tail.shift() + if (store.debugMode || output.level() >= 2) output.print(output.styles.debug(`[webServer] ${line}`)) + } + + formatTail() { + if (!this.tail.length) return '\n(no output)' + return `\nLast output:\n${this.tail.join('\n')}` + } +} + +export default WebServer diff --git a/lib/workers.js b/lib/workers.js index 84a96b02d..b11301d38 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -23,6 +23,7 @@ import { deserializeSuite } from './mocha/suite.js' import recorder from './recorder.js' import store from './store.js' import runHook from './hooks.js' +import WebServer from './webServer.js' import WorkerStorage from './workerStorage.js' import { createRuns } from './command/run-multiple/collection.js' @@ -490,12 +491,17 @@ class Workers extends EventEmitter { async bootstrapAll() { await this._ensureInitialized() + await WebServer.startAll(this.codecept.config.webServer) return runHook(this.codecept.config.bootstrapAll, 'bootstrapAll') } async teardownAll() { await this._ensureInitialized() - return runHook(this.codecept.config.teardownAll, 'teardownAll') + try { + return await runHook(this.codecept.config.teardownAll, 'teardownAll') + } finally { + await WebServer.stopAll() + } } async run() { diff --git a/test/data/sandbox/configs/webServer/codecept.conf.js b/test/data/sandbox/configs/webServer/codecept.conf.js new file mode 100644 index 000000000..8c2701377 --- /dev/null +++ b/test/data/sandbox/configs/webServer/codecept.conf.js @@ -0,0 +1,21 @@ +const port = process.env.WEB_SERVER_PORT + +export const config = { + tests: './*_test.js', + output: './output', + helpers: { + FileSystem: {}, + }, + webServer: { + command: process.env.WEB_SERVER_COMMAND || 'node server.js', + url: `http://127.0.0.1:${port}`, + enabled: process.env.WEB_SERVER_ENABLED !== 'false', + timeout: Number(process.env.WEB_SERVER_TIMEOUT || 10000), + }, + multiple: { + default: { + browsers: ['chrome', 'firefox'], + }, + }, + name: 'webServer', +} diff --git a/test/data/sandbox/configs/webServer/server.js b/test/data/sandbox/configs/webServer/server.js new file mode 100644 index 000000000..ceef13efa --- /dev/null +++ b/test/data/sandbox/configs/webServer/server.js @@ -0,0 +1,15 @@ +import http from 'http' +import fs from 'fs' + +const port = Number(process.env.WEB_SERVER_PORT) +const pidFile = process.env.WEB_SERVER_PID_FILE + +const server = http.createServer((req, res) => { + res.statusCode = req.method === 'HEAD' ? 500 : 200 + res.end('web server is up') +}) + +server.listen(port, '127.0.0.1', () => { + if (pidFile) fs.appendFileSync(pidFile, `${process.pid}\n`) + console.log(`listening on ${port}`) +}) diff --git a/test/data/sandbox/configs/webServer/server_test.js b/test/data/sandbox/configs/webServer/server_test.js new file mode 100644 index 000000000..e68ee0d45 --- /dev/null +++ b/test/data/sandbox/configs/webServer/server_test.js @@ -0,0 +1,13 @@ +Feature('Web Server') + +const url = `http://127.0.0.1:${process.env.WEB_SERVER_PORT}` + +Scenario('server is reachable first', async () => { + const body = await (await fetch(url)).text() + if (body !== 'web server is up') throw new Error(`Unexpected response: ${body}`) +}) + +Scenario('server is reachable second', async () => { + const body = await (await fetch(url)).text() + if (body !== 'web server is up') throw new Error(`Unexpected response: ${body}`) +}) diff --git a/test/data/sandbox/configs/webServer/silent.js b/test/data/sandbox/configs/webServer/silent.js new file mode 100644 index 000000000..b4dc6724e --- /dev/null +++ b/test/data/sandbox/configs/webServer/silent.js @@ -0,0 +1,5 @@ +import fs from 'fs' + +if (process.env.WEB_SERVER_PID_FILE) fs.appendFileSync(process.env.WEB_SERVER_PID_FILE, `${process.pid}\n`) +for (let i = 0; i < 30; i++) console.log(`server line ${i}`) +setInterval(() => {}, 1000) diff --git a/test/runner/web_server_test.js b/test/runner/web_server_test.js new file mode 100644 index 000000000..8e2229e03 --- /dev/null +++ b/test/runner/web_server_test.js @@ -0,0 +1,171 @@ +import assert from 'assert' +import path from 'path' +import fs from 'fs' +import os from 'os' +import net from 'net' +import http from 'http' +import { exec } from 'child_process' +import { fileURLToPath } from 'url' +import debugFactory from 'debug' + +const debug = debugFactory('codeceptjs:test') +const __dirname = path.dirname(fileURLToPath(import.meta.url)) + +const runner = path.join(__dirname, '/../../bin/codecept.js') +const codecept_dir = path.join(__dirname, '/../data/sandbox/configs/webServer') +const config = `--config ${codecept_dir}/codecept.conf.js` + +let pidFile + +function run(command, env) { + return new Promise(resolve => { + exec(`${runner} ${command} ${config}`, { env: { ...process.env, WEB_SERVER_PID_FILE: pidFile, ...env } }, (err, stdout, stderr) => { + debug(stdout) + debug(stderr) + resolve({ err, stdout: stdout + stderr }) + }) + }) +} + +function pids() { + if (!fs.existsSync(pidFile)) return [] + return fs.readFileSync(pidFile, 'utf8').split('\n').filter(Boolean).map(Number) +} + +function isPortFree(port) { + return new Promise(resolve => { + const socket = net.connect(port, '127.0.0.1') + socket.once('connect', () => { + socket.destroy() + resolve(false) + }) + socket.once('error', () => resolve(true)) + }) +} + +async function isDead(pid) { + for (let i = 0; i < 20; i++) { + try { + process.kill(pid, 0) + } catch (err) { + return err.code === 'ESRCH' + } + await new Promise(resolve => setTimeout(resolve, 100)) + } + return false +} + +describe('CodeceptJS webServer', function () { + this.timeout(40000) + + beforeEach(() => { + pidFile = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'codecept-webserver-')), 'pids') + }) + + it('should start server before tests and stop it after', async () => { + const port = 18631 + const { err, stdout } = await run('run', { WEB_SERVER_PORT: port }) + assert(!err, stdout) + assert(stdout.includes('Starting web server: node server.js'), stdout) + assert(stdout.includes('2 passed'), stdout) + assert.equal(pids().length, 1) + assert(await isPortFree(port), 'port should be free after run') + assert(await isDead(pids()[0]), 'server process should be killed') + }) + + it('should kill all processes spawned by the command', async function () { + if (process.platform === 'win32') this.skip() + const port = 18640 + const { err, stdout } = await run('run', { WEB_SERVER_PORT: port, WEB_SERVER_COMMAND: 'node silent.js & node server.js' }) + assert(!err, stdout) + assert(stdout.includes('2 passed'), stdout) + assert.equal(pids().length, 2) + for (const pid of pids()) assert(await isDead(pid), `process ${pid} should be killed`) + }) + + it('should reuse existing server', async () => { + const port = 18632 + const server = http.createServer((req, res) => res.end('web server is up')) + await new Promise(resolve => server.listen(port, '127.0.0.1', resolve)) + try { + const { err, stdout } = await run('run', { + WEB_SERVER_PORT: port, + WEB_SERVER_COMMAND: 'node -e "process.exit(1)"', + }) + assert(!err, stdout) + assert(stdout.includes(`Reusing web server at http://127.0.0.1:${port}`), stdout) + assert(!stdout.includes('Starting web server'), stdout) + assert(stdout.includes('2 passed'), stdout) + } finally { + server.close() + } + }) + + it('should not start server when disabled', async () => { + const port = 18633 + const { err, stdout } = await run('run', { WEB_SERVER_PORT: port, WEB_SERVER_ENABLED: 'false' }) + assert(!stdout.includes('Starting web server'), stdout) + assert(!stdout.includes('Reusing web server'), stdout) + assert.equal(pids().length, 0) + }) + + it('should fail on timeout with tail of server output and kill it', async () => { + const port = 18634 + const { err, stdout } = await run('run', { + WEB_SERVER_PORT: port, + WEB_SERVER_COMMAND: 'node silent.js', + WEB_SERVER_TIMEOUT: 1500, + }) + assert(err, stdout) + assert(stdout.includes(`did not respond at http://127.0.0.1:${port} within 1500ms`), stdout) + assert(stdout.includes('server line 29'), stdout) + assert(stdout.includes('server line 10'), stdout) + assert(!stdout.includes('server line 9\n'), stdout) + assert(!stdout.includes('passed'), stdout) + assert.equal(pids().length, 1) + assert(await isDead(pids()[0]), 'server process should be killed') + }) + + it('should fail when server exits early', async () => { + const { err, stdout } = await run('run', { + WEB_SERVER_PORT: 18635, + WEB_SERVER_COMMAND: `node -e "console.error('boom'); process.exit(3)"`, + }) + assert(err, stdout) + assert(stdout.includes('exited with code 3'), stdout) + assert(stdout.includes('boom'), stdout) + }) + + it('should start server once for run-workers', async () => { + const port = 18636 + const { err, stdout } = await run('run-workers 2', { WEB_SERVER_PORT: port }) + assert(!err, stdout) + assert(stdout.includes('2 passed'), stdout) + assert.equal(pids().length, 1) + assert(await isPortFree(port), 'port should be free after run') + }) + + it('should start server once for run-multiple', async () => { + const port = 18637 + const { err, stdout } = await run('run-multiple default', { WEB_SERVER_PORT: port }) + assert(!err, stdout) + assert.equal(stdout.match(/2 passed/g)?.length, 2, stdout) + assert.equal(pids().length, 1) + assert(await isPortFree(port), 'port should be free after run') + }) + + it('should start server once for run-rerun', async () => { + const port = 18638 + const { err, stdout } = await run('run-rerun', { WEB_SERVER_PORT: port }) + assert(!err, stdout) + assert.equal(pids().length, 1) + assert(await isPortFree(port), 'port should be free after run') + }) + + it('should not start server for dry-run', async () => { + const { err, stdout } = await run('dry-run', { WEB_SERVER_PORT: 18639 }) + assert(!err, stdout) + assert(!stdout.includes('Starting web server'), stdout) + assert.equal(pids().length, 0) + }) +}) diff --git a/typings/index.d.ts b/typings/index.d.ts index 3d57f98ba..487491507 100644 --- a/typings/index.d.ts +++ b/typings/index.d.ts @@ -81,6 +81,21 @@ declare namespace CodeceptJS { } } + type WebServerConfig = { + /** Shell command that starts the application, e.g. `npm run start` */ + command: string + /** URL polled with GET until it responds; any HTTP status counts as ready */ + url: string + /** Set to `false` to skip starting the server. Defaults to `true` */ + enabled?: boolean + /** Milliseconds to wait for `url` to respond. Defaults to `60000` */ + timeout?: number + /** Working directory for the command, relative to the config directory */ + cwd?: string + /** Extra environment variables for the command */ + env?: Record + } + type MainConfig = { /** Pattern to locate CodeceptJS tests. * Allows to enter glob pattern or an Array of patterns to match tests / test file names. @@ -359,6 +374,17 @@ declare namespace CodeceptJS { * [Execute JS code after finishing tests in parallel mode](https://codecept.io/bootstrap/#bootstrapall-teardownall) */ teardownAll?: (() => Promise) | boolean | string + /** + * [Start the application under test](https://codecept.io/configuration/#web-server) before tests run and stop it after. + * + * ```js + * webServer: { + * command: 'npm run start', + * url: 'http://localhost:3000', + * } + * ``` + */ + webServer?: WebServerConfig | WebServerConfig[] /** Enable [localized test commands](https://codecept.io/translation/) */ translation?: string