From f8bf148f1b83f81c865bde9c6605c1b8d5901f4f Mon Sep 17 00:00:00 2001 From: DavertMik Date: Fri, 9 Oct 2026 22:49:36 +0300 Subject: [PATCH 1/2] feat(plugins): add sitemap plugin collecting visited pages Records every top-level page visited during a run and writes it as a sitemap XML file. Pages are collected from browser navigation events (Playwright/Puppeteer framenavigated, WebDriver BiDi browsingContext events), so no extra browser calls are added to test steps. Results from run-workers are merged into a single sitemap. Co-Authored-By: Claude Opus 5.5 --- docs/plugins.md | 4 + docs/plugins/sitemap.md | 48 +++++++ lib/plugin/sitemap.js | 207 +++++++++++++++++++++++++++++++ test/unit/plugin/sitemap_test.js | 132 ++++++++++++++++++++ 4 files changed, 391 insertions(+) create mode 100644 docs/plugins/sitemap.md create mode 100644 lib/plugin/sitemap.js create mode 100644 test/unit/plugin/sitemap_test.js diff --git a/docs/plugins.md b/docs/plugins.md index 95bd5956b..530284afb 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -81,6 +81,10 @@ Saves screenshots from the browser at points triggered by `on=`. Saves a screenshot when a test fails. +## [sitemap](/plugins/sitemap) + +Collects all pages visited while tests are running and saves them as a [sitemap][1] XML file. + ## [stepTimeout](/plugins/stepTimeout) Set timeout for test steps globally. diff --git a/docs/plugins/sitemap.md b/docs/plugins/sitemap.md new file mode 100644 index 000000000..0c6f2966f --- /dev/null +++ b/docs/plugins/sitemap.md @@ -0,0 +1,48 @@ +--- +permalink: /plugins/sitemap +editLink: false +sidebar: auto +title: sitemap +--- + + + +## sitemap + +Collects all pages visited while tests are running and saves them as a [sitemap][1] XML file. + +The plugin listens to browser navigation events, so it doesn't slow down test steps. +Page loads and SPA route changes (`history.pushState`) of the top-level page are recorded, iframes are ignored. +Every unique `http(s)` URL is added to the sitemap. URL fragments (`#...`) are dropped. + +Supported helpers: Playwright, Puppeteer, and WebDriver with BiDi protocol enabled (default). +The sitemap is written once all tests finish. + +#### Configuration + +```js +"plugins": { + "sitemap": { + "enabled": true + } + } +``` + +Possible config options: + +* `outputName`: file name for the sitemap. Default: `sitemap.xml`. +* `output`: directory where the sitemap is stored, relative to the project root. Default: the `output` directory. +* `stripQuery`: remove query strings so `/posts?page=1` and `/posts?page=2` become one `/posts` entry. Default: false. + +CLI examples: + + npx codeceptjs run -p sitemap + npx codeceptjs run -p sitemap:outputName=visited.xml + +Works with `run-workers`: URLs from all workers are merged into a single sitemap. + +### Parameters + +* `config` **any** + +[1]: https://www.sitemaps.org/protocol.html diff --git a/lib/plugin/sitemap.js b/lib/plugin/sitemap.js new file mode 100644 index 000000000..aac98326b --- /dev/null +++ b/lib/plugin/sitemap.js @@ -0,0 +1,207 @@ +import fs from 'fs' +import path from 'path' +import { isMainThread, threadId } from 'worker_threads' +import { mkdirp } from 'mkdirp' +import { DOMImplementation, XMLSerializer } from '@xmldom/xmldom' + +import Container from '../container.js' +import event from '../event.js' +import recorder from '../recorder.js' +import store from '../store.js' +import output from '../output.js' + +const defaultConfig = { + outputName: 'sitemap.xml', + output: null, + stripQuery: false, +} + +const SITEMAP_NS = 'http://www.sitemaps.org/schemas/sitemap/0.9' +const PARTIAL_PREFIX = '.sitemap.' + +/** + * Collects all pages visited while tests are running and saves them as a [sitemap](https://www.sitemaps.org/protocol.html) XML file. + * + * The plugin listens to browser navigation events, so it doesn't slow down test steps. + * Page loads and SPA route changes (`history.pushState`) of the top-level page are recorded, iframes are ignored. + * Every unique `http(s)` URL is added to the sitemap. URL fragments (`#...`) are dropped. + * + * Supported helpers: Playwright, Puppeteer, and WebDriver with BiDi protocol enabled (default). + * The sitemap is written once all tests finish. + * + * #### Configuration + * + * ```js + * "plugins": { + * "sitemap": { + * "enabled": true + * } + * } + * ``` + * + * Possible config options: + * + * * `outputName`: file name for the sitemap. Default: `sitemap.xml`. + * * `output`: directory where the sitemap is stored, relative to the project root. Default: the `output` directory. + * * `stripQuery`: remove query strings so `/posts?page=1` and `/posts?page=2` become one `/posts` entry. Default: false. + * + * CLI examples: + * + * ``` + * npx codeceptjs run -p sitemap + * npx codeceptjs run -p sitemap:outputName=visited.xml + * ``` + * + * Works with `run-workers`: URLs from all workers are merged into a single sitemap. + * + * @param {*} config + */ +export default function (config = {}) { + config = Object.assign({}, defaultConfig, config) + if (config.stripQuery === 'true') config.stripQuery = true + + const urls = new Set() + + const outputDir = () => { + if (config.output) return path.resolve(store.codeceptDir || process.cwd(), config.output) + return store.outputDir || process.cwd() + } + + const addUrl = url => { + if (!url) return + let parsed + try { + parsed = new URL(url) + } catch { + return + } + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return + parsed.hash = '' + if (config.stripQuery) parsed.search = '' + urls.add(parsed.toString()) + } + + const writeSitemap = list => { + if (!list.length) return + + const dir = outputDir() + mkdirp.sync(dir) + const file = path.join(dir, config.outputName) + fs.writeFileSync(file, buildXml(list.sort())) + output.plugin('sitemap', `Sitemap with ${list.length} pages saved to ${file}`) + } + + const attached = new WeakSet() + + const onFrameNavigated = frame => { + if (frame.parentFrame()) return + addUrl(frame.url()) + } + + const attachPage = page => { + if (!page || attached.has(page)) return + attached.add(page) + addUrl(page.url()) + page.on('framenavigated', onFrameNavigated) + } + + const attachContext = context => { + if (!context || attached.has(context)) return + attached.add(context) + context.on('page', attachPage) + } + + const subscribeBrowser = async browser => { + attached.add(browser) + const frames = new Set() + const onNavigation = params => { + if (frames.has(params.context)) return + addUrl(params.url) + } + browser.on('browsingContext.contextCreated', params => { + if (params.parent) frames.add(params.context) + }) + browser.on('browsingContext.load', onNavigation) + browser.on('browsingContext.historyUpdated', onNavigation) + try { + await browser.sessionSubscribe({ events: ['browsingContext.contextCreated', 'browsingContext.load'] }) + } catch (err) { + output.plugin('sitemap', `Can't subscribe to navigation events: ${err.message}`) + } + try { + await browser.sessionSubscribe({ events: ['browsingContext.historyUpdated'] }) + } catch {} + } + + event.dispatcher.on(event.test.started, () => { + const helpers = Container.helpers() + if (!helpers.WebDriver) return + const browser = helpers.WebDriver.browser + if (!browser || attached.has(browser)) return + if (!browser.capabilities || !browser.capabilities.webSocketUrl) { + attached.add(browser) + output.plugin('sitemap', 'WebDriver BiDi protocol is disabled, visited pages are not collected') + return + } + recorder.add('sitemap: subscribe to navigation', () => subscribeBrowser(browser), true) + }) + + event.dispatcher.on(event.step.started, () => { + const helpers = Container.helpers() + const helper = helpers.Playwright || helpers.Puppeteer + if (!helper) return + if (helpers.Playwright) attachContext(helper.browserContext) + attachPage(helper.page) + }) + + const partialFiles = dir => { + if (!fs.existsSync(dir)) return [] + return fs + .readdirSync(dir) + .filter(name => name.startsWith(PARTIAL_PREFIX) && name.endsWith('.json')) + .map(name => path.join(dir, name)) + } + + event.dispatcher.on(event.all.result, () => { + if (isMainThread) { + if (store.workerMode) return + writeSitemap([...urls]) + return + } + if (!urls.size) return + const dir = outputDir() + mkdirp.sync(dir) + fs.writeFileSync(path.join(dir, `${PARTIAL_PREFIX}${process.pid}.${threadId}.json`), JSON.stringify([...urls])) + }) + + event.dispatcher.on(event.workers.before, () => { + for (const partial of partialFiles(outputDir())) fs.unlinkSync(partial) + }) + + event.dispatcher.on(event.workers.result, () => { + for (const partial of partialFiles(outputDir())) { + try { + for (const url of JSON.parse(fs.readFileSync(partial, 'utf8'))) urls.add(url) + } catch {} + fs.unlinkSync(partial) + } + writeSitemap([...urls]) + }) +} + +function buildXml(list) { + const doc = new DOMImplementation().createDocument(SITEMAP_NS, 'urlset', null) + const urlset = doc.documentElement + for (const url of list) { + const urlEl = doc.createElementNS(SITEMAP_NS, 'url') + const locEl = doc.createElementNS(SITEMAP_NS, 'loc') + locEl.appendChild(doc.createTextNode(url)) + urlEl.appendChild(doc.createTextNode('\n ')) + urlEl.appendChild(locEl) + urlEl.appendChild(doc.createTextNode('\n ')) + urlset.appendChild(doc.createTextNode('\n ')) + urlset.appendChild(urlEl) + } + urlset.appendChild(doc.createTextNode('\n')) + return `\n${new XMLSerializer().serializeToString(doc)}\n` +} diff --git a/test/unit/plugin/sitemap_test.js b/test/unit/plugin/sitemap_test.js new file mode 100644 index 000000000..47b4c0f92 --- /dev/null +++ b/test/unit/plugin/sitemap_test.js @@ -0,0 +1,132 @@ +import { expect } from 'chai' +import fs from 'fs' +import { EventEmitter } from 'events' +import os from 'os' +import path from 'path' +import sitemap from '../../../lib/plugin/sitemap.js' +import container from '../../../lib/container.js' +import event from '../../../lib/event.js' +import recorder from '../../../lib/recorder.js' +import store from '../../../lib/store.js' + +let page +let outputDir +let originalOutputDir + +const mainFrame = url => ({ url: () => url, parentFrame: () => null }) + +const visit = async url => { + event.dispatcher.emit(event.step.started, {}) + page.emit('framenavigated', mainFrame(url)) +} + +describe('sitemap plugin', () => { + beforeEach(() => { + recorder.reset() + recorder.start() + outputDir = fs.mkdtempSync(path.join(os.tmpdir(), 'sitemap-')) + originalOutputDir = store.outputDir + store.outputDir = outputDir + page = new EventEmitter() + page.url = () => 'about:blank' + container.clear({ + Puppeteer: { + options: {}, + page, + }, + }) + }) + + afterEach(() => { + event.dispatcher.removeAllListeners(event.step.started) + event.dispatcher.removeAllListeners(event.test.started) + event.dispatcher.removeAllListeners(event.all.result) + event.dispatcher.removeAllListeners(event.workers.before) + event.dispatcher.removeAllListeners(event.workers.result) + store.outputDir = originalOutputDir + fs.rmSync(outputDir, { recursive: true, force: true }) + }) + + it('writes unique visited pages to sitemap.xml', async () => { + sitemap({}) + await visit('http://localhost:8000/') + await visit('http://localhost:8000/form/field#top') + await visit('http://localhost:8000/form/field') + await visit('about:blank') + await visit('http://localhost:8000/search?q=a&b=c') + event.dispatcher.emit(event.all.result, {}) + + const xml = fs.readFileSync(path.join(outputDir, 'sitemap.xml'), 'utf8') + expect(xml).to.include('') + expect(xml.match(//g).length).to.equal(3) + expect(xml).to.include('http://localhost:8000/') + expect(xml).to.include('http://localhost:8000/form/field') + expect(xml).to.include('http://localhost:8000/search?q=a&b=c') + expect(xml).not.to.include('about:blank') + }) + + it('ignores iframe navigations and attaches to a page once', async () => { + sitemap({}) + event.dispatcher.emit(event.step.started, {}) + event.dispatcher.emit(event.step.started, {}) + expect(page.listenerCount('framenavigated')).to.equal(1) + page.emit('framenavigated', { url: () => 'http://localhost:8000/frame', parentFrame: () => ({}) }) + page.emit('framenavigated', mainFrame('http://localhost:8000/spa-route')) + event.dispatcher.emit(event.all.result, {}) + + const xml = fs.readFileSync(path.join(outputDir, 'sitemap.xml'), 'utf8') + expect(xml.match(//g).length).to.equal(1) + expect(xml).to.include('http://localhost:8000/spa-route') + }) + + it('collects pages from WebDriver BiDi events', async () => { + const browser = new EventEmitter() + browser.capabilities = { webSocketUrl: 'ws://localhost' } + browser.subscribed = [] + browser.sessionSubscribe = async ({ events }) => browser.subscribed.push(...events) + container.clear({ WebDriver: { options: {}, browser } }) + + sitemap({}) + event.dispatcher.emit(event.test.started, {}) + await recorder.promise() + expect(browser.subscribed).to.include('browsingContext.load') + + browser.emit('browsingContext.contextCreated', { context: 'frame-1', parent: 'top' }) + browser.emit('browsingContext.load', { context: 'top', url: 'http://localhost:8000/' }) + browser.emit('browsingContext.load', { context: 'frame-1', url: 'http://localhost:8000/frame' }) + browser.emit('browsingContext.historyUpdated', { context: 'top', url: 'http://localhost:8000/spa-route' }) + event.dispatcher.emit(event.all.result, {}) + + const xml = fs.readFileSync(path.join(outputDir, 'sitemap.xml'), 'utf8') + expect(xml.match(//g).length).to.equal(2) + expect(xml).not.to.include('/frame') + }) + + it('strips query strings when configured', async () => { + sitemap({ stripQuery: true, outputName: 'pages.xml' }) + await visit('http://localhost:8000/posts?page=1') + await visit('http://localhost:8000/posts?page=2') + event.dispatcher.emit(event.all.result, {}) + + const xml = fs.readFileSync(path.join(outputDir, 'pages.xml'), 'utf8') + expect(xml.match(//g).length).to.equal(1) + expect(xml).to.include('http://localhost:8000/posts') + }) + + it('merges worker partials on workers.result', async () => { + sitemap({}) + fs.writeFileSync(path.join(outputDir, '.sitemap.1.1.json'), JSON.stringify(['http://localhost:8000/a'])) + fs.writeFileSync(path.join(outputDir, '.sitemap.1.2.json'), JSON.stringify(['http://localhost:8000/b', 'http://localhost:8000/a'])) + event.dispatcher.emit(event.workers.result, {}) + + const xml = fs.readFileSync(path.join(outputDir, 'sitemap.xml'), 'utf8') + expect(xml.match(//g).length).to.equal(2) + expect(fs.readdirSync(outputDir)).to.deep.equal(['sitemap.xml']) + }) + + it('does not write a file when no pages were visited', () => { + sitemap({}) + event.dispatcher.emit(event.all.result, {}) + expect(fs.existsSync(path.join(outputDir, 'sitemap.xml'))).to.equal(false) + }) +}) From 13dfeac3ff08b59ecad6a475cdfabc5068388078 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Fri, 9 Oct 2026 23:35:37 +0300 Subject: [PATCH 2/2] refactor(sitemap): attach navigation listeners once per browser Listen to browser context/page creation (Playwright 'context'/'page', Puppeteer 'targetcreated') and record URLs on page 'load' plus main-frame 'framenavigated' for pushState routes, instead of checking the page on every step. Co-Authored-By: Claude Opus 5.5 --- lib/plugin/sitemap.js | 58 +++++++++++++++------- test/unit/plugin/sitemap_test.js | 82 +++++++++++++++++++++++++++----- 2 files changed, 112 insertions(+), 28 deletions(-) diff --git a/lib/plugin/sitemap.js b/lib/plugin/sitemap.js index aac98326b..f56b8a6d4 100644 --- a/lib/plugin/sitemap.js +++ b/lib/plugin/sitemap.js @@ -93,6 +93,8 @@ export default function (config = {}) { const attached = new WeakSet() + const onLoad = page => addUrl(page.url()) + const onFrameNavigated = frame => { if (frame.parentFrame()) return addUrl(frame.url()) @@ -102,16 +104,33 @@ export default function (config = {}) { if (!page || attached.has(page)) return attached.add(page) addUrl(page.url()) + page.on('load', onLoad) page.on('framenavigated', onFrameNavigated) } const attachContext = context => { - if (!context || attached.has(context)) return attached.add(context) context.on('page', attachPage) + context.pages().forEach(attachPage) + } + + const attachPlaywright = browser => { + if (!browser.contexts) return attachContext(browser) + attached.add(browser) + browser.on('context', attachContext) + browser.contexts().forEach(attachContext) + } + + const attachPuppeteer = async browser => { + attached.add(browser) + browser.on('targetcreated', async target => { + if (target.type() !== 'page') return + attachPage(await target.page()) + }) + for (const page of await browser.pages()) attachPage(page) } - const subscribeBrowser = async browser => { + const subscribeBidi = async browser => { attached.add(browser) const frames = new Set() const onNavigation = params => { @@ -135,23 +154,30 @@ export default function (config = {}) { event.dispatcher.on(event.test.started, () => { const helpers = Container.helpers() - if (!helpers.WebDriver) return - const browser = helpers.WebDriver.browser - if (!browser || attached.has(browser)) return - if (!browser.capabilities || !browser.capabilities.webSocketUrl) { - attached.add(browser) - output.plugin('sitemap', 'WebDriver BiDi protocol is disabled, visited pages are not collected') + + if (helpers.Playwright) { + const browser = helpers.Playwright.browser + if (browser && !attached.has(browser)) attachPlaywright(browser) return } - recorder.add('sitemap: subscribe to navigation', () => subscribeBrowser(browser), true) - }) - event.dispatcher.on(event.step.started, () => { - const helpers = Container.helpers() - const helper = helpers.Playwright || helpers.Puppeteer - if (!helper) return - if (helpers.Playwright) attachContext(helper.browserContext) - attachPage(helper.page) + if (helpers.Puppeteer) { + const browser = helpers.Puppeteer.browser + if (!browser || attached.has(browser)) return + recorder.add('sitemap: listen to navigation', () => attachPuppeteer(browser), true) + return + } + + if (helpers.WebDriver) { + const browser = helpers.WebDriver.browser + if (!browser || attached.has(browser)) return + if (!browser.capabilities || !browser.capabilities.webSocketUrl) { + attached.add(browser) + output.plugin('sitemap', 'WebDriver BiDi protocol is disabled, visited pages are not collected') + return + } + recorder.add('sitemap: subscribe to navigation', () => subscribeBidi(browser), true) + } }) const partialFiles = dir => { diff --git a/test/unit/plugin/sitemap_test.js b/test/unit/plugin/sitemap_test.js index 47b4c0f92..1cc82df60 100644 --- a/test/unit/plugin/sitemap_test.js +++ b/test/unit/plugin/sitemap_test.js @@ -13,11 +13,21 @@ let page let outputDir let originalOutputDir -const mainFrame = url => ({ url: () => url, parentFrame: () => null }) +const createPage = () => { + const p = new EventEmitter() + p.currentUrl = 'about:blank' + p.url = () => p.currentUrl + return p +} + +const startTest = async () => { + event.dispatcher.emit(event.test.started, {}) + await recorder.promise() +} const visit = async url => { - event.dispatcher.emit(event.step.started, {}) - page.emit('framenavigated', mainFrame(url)) + page.currentUrl = url + page.emit('load', page) } describe('sitemap plugin', () => { @@ -27,18 +37,18 @@ describe('sitemap plugin', () => { outputDir = fs.mkdtempSync(path.join(os.tmpdir(), 'sitemap-')) originalOutputDir = store.outputDir store.outputDir = outputDir - page = new EventEmitter() - page.url = () => 'about:blank' + page = createPage() + const browser = new EventEmitter() + browser.pages = async () => [page] container.clear({ Puppeteer: { options: {}, - page, + browser, }, }) }) afterEach(() => { - event.dispatcher.removeAllListeners(event.step.started) event.dispatcher.removeAllListeners(event.test.started) event.dispatcher.removeAllListeners(event.all.result) event.dispatcher.removeAllListeners(event.workers.before) @@ -49,6 +59,7 @@ describe('sitemap plugin', () => { it('writes unique visited pages to sitemap.xml', async () => { sitemap({}) + await startTest() await visit('http://localhost:8000/') await visit('http://localhost:8000/form/field#top') await visit('http://localhost:8000/form/field') @@ -65,13 +76,13 @@ describe('sitemap plugin', () => { expect(xml).not.to.include('about:blank') }) - it('ignores iframe navigations and attaches to a page once', async () => { + it('records SPA route changes and ignores iframe navigations', async () => { sitemap({}) - event.dispatcher.emit(event.step.started, {}) - event.dispatcher.emit(event.step.started, {}) - expect(page.listenerCount('framenavigated')).to.equal(1) + await startTest() + await startTest() + expect(page.listenerCount('load')).to.equal(1) page.emit('framenavigated', { url: () => 'http://localhost:8000/frame', parentFrame: () => ({}) }) - page.emit('framenavigated', mainFrame('http://localhost:8000/spa-route')) + page.emit('framenavigated', { url: () => 'http://localhost:8000/spa-route', parentFrame: () => null }) event.dispatcher.emit(event.all.result, {}) const xml = fs.readFileSync(path.join(outputDir, 'sitemap.xml'), 'utf8') @@ -79,6 +90,52 @@ describe('sitemap plugin', () => { expect(xml).to.include('http://localhost:8000/spa-route') }) + it('attaches to Puppeteer pages opened later', async () => { + sitemap({}) + await startTest() + const newTab = createPage() + newTab.currentUrl = 'http://localhost:8000/new-tab' + const browser = container.helpers('Puppeteer').browser + browser.emit('targetcreated', { type: () => 'page', page: async () => newTab }) + browser.emit('targetcreated', { type: () => 'service_worker', page: async () => null }) + await new Promise(resolve => setImmediate(resolve)) + newTab.currentUrl = 'http://localhost:8000/new-tab/next' + newTab.emit('load', newTab) + event.dispatcher.emit(event.all.result, {}) + + const xml = fs.readFileSync(path.join(outputDir, 'sitemap.xml'), 'utf8') + expect(xml).to.include('http://localhost:8000/new-tab') + expect(xml).to.include('http://localhost:8000/new-tab/next') + }) + + it('attaches to Playwright contexts and pages once per browser', async () => { + const context = new EventEmitter() + context.pages = () => [page] + const browser = new EventEmitter() + browser.contexts = () => [context] + container.clear({ Playwright: { options: {}, browser } }) + + sitemap({}) + await startTest() + await startTest() + expect(browser.listenerCount('context')).to.equal(1) + expect(page.listenerCount('load')).to.equal(1) + + await visit('http://localhost:8000/') + const sessionContext = new EventEmitter() + sessionContext.pages = () => [] + browser.emit('context', sessionContext) + const sessionPage = createPage() + sessionContext.emit('page', sessionPage) + sessionPage.currentUrl = 'http://localhost:8000/session' + sessionPage.emit('load', sessionPage) + event.dispatcher.emit(event.all.result, {}) + + const xml = fs.readFileSync(path.join(outputDir, 'sitemap.xml'), 'utf8') + expect(xml.match(//g).length).to.equal(2) + expect(xml).to.include('http://localhost:8000/session') + }) + it('collects pages from WebDriver BiDi events', async () => { const browser = new EventEmitter() browser.capabilities = { webSocketUrl: 'ws://localhost' } @@ -104,6 +161,7 @@ describe('sitemap plugin', () => { it('strips query strings when configured', async () => { sitemap({ stripQuery: true, outputName: 'pages.xml' }) + await startTest() await visit('http://localhost:8000/posts?page=1') await visit('http://localhost:8000/posts?page=2') event.dispatcher.emit(event.all.result, {})