diff --git a/.changeset/wdio-all-skipped-incomplete.md b/.changeset/wdio-all-skipped-incomplete.md new file mode 100644 index 0000000000..650899d646 --- /dev/null +++ b/.changeset/wdio-all-skipped-incomplete.md @@ -0,0 +1,5 @@ +--- +'@fluentui-react-native/storybook-desktop': patch +--- + +Report WDIO and smoke runs in which every selected test was skipped as `incomplete` instead of `passed`, and fail the command so an all-skipped selection cannot satisfy a required test gate. Run reports now include selected/passed/skipped/failed counts. Mixed pass/skip runs are unchanged. diff --git a/packages/agentic/storybook-desktop/README.md b/packages/agentic/storybook-desktop/README.md index 0d337b2f91..bcd0750516 100644 --- a/packages/agentic/storybook-desktop/README.md +++ b/packages/agentic/storybook-desktop/README.md @@ -220,9 +220,11 @@ it is not `process.platform`. Win32 stays distinct even though its WebDriver `platformName` is `windows`. Native feature flags are typed under `browser.capabilities['furn:features']`. `signal` allows cooperative cancellation; `skip(reason)` records an explicit -skip (return from the callback after calling it). Native selectors and -supported WebDriver operations apply; there is no DOM or JavaScript execution -inside the app. +skip (return from the callback after calling it). A run in which every selected +test skipped is reported as `incomplete` in `run.json` and fails the command, +since it provides no coverage; runs with at least one pass are unaffected. +Native selectors and supported WebDriver operations apply; there is no DOM or +JavaScript execution inside the app. The original `wdio: async (context) => { ... }` form remains supported. It is one test, called `default` for filtering. Named collections must be non-empty, diff --git a/packages/agentic/storybook-desktop/src/cli/smokeTests.test.ts b/packages/agentic/storybook-desktop/src/cli/smokeTests.test.ts index 3d6f5c963b..7b9c8a6063 100644 --- a/packages/agentic/storybook-desktop/src/cli/smokeTests.test.ts +++ b/packages/agentic/storybook-desktop/src/cli/smokeTests.test.ts @@ -7,7 +7,7 @@ import type { DesktopStoryRunResult } from '@fluentui-react-native/desktop-drive import { makeDesktopStorybookConfig } from '../config/makeDesktopStorybookConfig.js'; import type { runWdioStoryTests } from '../testing/runWdioTests.js'; -import { WdioStoryTestRunError } from '../testing/runWdioTests.js'; +import { WdioStoryCoverageUnavailableError, WdioStoryTestRunError } from '../testing/runWdioTests.js'; import { formatDesktopStorybookSmokeTestSummary, runDesktopStorybookSmokeTests, @@ -199,13 +199,50 @@ describe('runDesktopStorybookSmokeTests', () => { options.config = makeDesktopStorybookConfig({ projectRoot: root, wdio: { story: inputId } }); const connect = jest.fn(); const runWdio: jest.MockedFunction = jest.fn(async ({ story }) => [ - { storyId: story!, status: 'skipped' as const, durationMs: 0, skipReason: 'Capability unavailable' }, + { storyId: story!, status: 'passed' as const, durationMs: 0 }, ]); const result = await runDesktopStorybookSmokeTests(options, connect, runWdio); expect(connect).not.toHaveBeenCalled(); expect(runWdio).toHaveBeenCalledTimes(1); - expect(result.wdio).toMatchObject([{ storyId: inputId, status: 'skipped' }]); - expect(formatDesktopStorybookSmokeTestSummary(result)).toBe('Ran 1 desktop story tests (0 passed, 1 skipped).'); + expect(result.wdio).toMatchObject([{ storyId: inputId, status: 'passed' }]); + expect(formatDesktopStorybookSmokeTestSummary(result)).toBe('Ran 1 desktop story tests (1 passed, 0 skipped).'); + }); + + test('reports an all-skipped run as incomplete instead of passed', async () => { + options.manifest = { ...manifest, entries: [{ ...story(inputId, true), tests: undefined }] }; + const runWdio: jest.MockedFunction = jest.fn(async ({ story }) => { + throw new WdioStoryCoverageUnavailableError([ + { storyId: story!, status: 'skipped', durationMs: 0, skipReason: 'This test requires physical pointer input.' }, + ]); + }); + await expect(runDesktopStorybookSmokeTests(options, jest.fn(), runWdio)).rejects.toThrow( + 'All 1 selected desktop story tests were skipped', + ); + expect(errorOutput.write).not.toHaveBeenCalled(); + expect(JSON.parse(fs.readFileSync(path.join(root, 'artifacts/windows/desktop-driver/run.json'), 'utf8'))).toMatchObject({ + status: 'incomplete', + counts: { selected: 1, passed: 0, skipped: 1, failed: 0 }, + wdio: [{ storyId: inputId, status: 'skipped', skipReason: 'This test requires physical pointer input.' }], + }); + }); + + test('passes a mixed run when a story has only skipped callbacks', async () => { + options.manifest = { ...manifest, entries: [story(buttonId), { ...story(inputId, true), tests: undefined }] }; + const connect: DesktopStorybookSmokeConnector = async () => ({ + delete: async () => undefined, + listStories: async () => options.manifest, + runStoryTests: async () => planResult(), + }); + const runWdio: jest.MockedFunction = jest.fn(async ({ story }) => { + throw new WdioStoryCoverageUnavailableError([{ storyId: story!, status: 'skipped', durationMs: 0, skipReason: 'Unavailable' }]); + }); + const result = await runDesktopStorybookSmokeTests(options, connect, runWdio); + expect(result.status).toBe('passed'); + expect(formatDesktopStorybookSmokeTestSummary(result)).toBe('Ran 2 desktop story tests (1 passed, 1 skipped).'); + expect(JSON.parse(fs.readFileSync(path.join(root, 'artifacts/windows/desktop-driver/run.json'), 'utf8'))).toMatchObject({ + status: 'passed', + counts: { selected: 2, passed: 1, skipped: 1, failed: 0 }, + }); }); test('fails after persisting failed plan results and closing the session', async () => { diff --git a/packages/agentic/storybook-desktop/src/cli/smokeTests.ts b/packages/agentic/storybook-desktop/src/cli/smokeTests.ts index 3cb23fb0fd..3fa6e8a42b 100644 --- a/packages/agentic/storybook-desktop/src/cli/smokeTests.ts +++ b/packages/agentic/storybook-desktop/src/cli/smokeTests.ts @@ -11,7 +11,14 @@ import type { DesktopWebdriverOptions, DesktopWebdriverSession } from '@fluentui import type { DesktopStorybookConfig } from '../config/makeDesktopStorybookConfig.js'; import type { Platforms } from '../config/platforms.js'; import { resolveWdioOptions } from '../config/wdio.js'; -import { runWdioStoryTests, selectWdioStories, WdioStoryTestRunError, type WdioStoryTestResult } from '../testing/runWdioTests.js'; +import { + countWdioStoryTests, + runWdioStoryTests, + selectWdioStories, + WdioStoryCoverageUnavailableError, + WdioStoryTestRunError, + type WdioStoryTestResult, +} from '../testing/runWdioTests.js'; import { writeDesktopStorybookFailure, type DesktopStorybookErrorOutput } from '../../config/diagnostics.cjs'; import type { DesktopCommandRunner } from './commandRunner.js'; @@ -95,6 +102,11 @@ export async function runDesktopStorybookSmokeTests( )), ); } catch (error) { + // A story whose callbacks all skipped is not a failure on its own; coverage is judged across the whole smoke run. + if (error instanceof WdioStoryCoverageUnavailableError) { + wdio.push(...error.results); + continue; + } writeDesktopStorybookFailure(`smoke story "${storyId}"`, error, options.errorOutput); if (error instanceof WdioStoryTestRunError && error.results.length) { wdio.push(...error.results); @@ -126,6 +138,8 @@ export async function runDesktopStorybookSmokeTests( } failures.push(new Error('Executable smoke tests reported failures.')); } + const counts = countWdioStoryTests([...tests, ...wdio]); + const incomplete = failures.length === 0 && counts.passed === 0; const result: DesktopStorybookSmokeTestResult = { endpoint: platform, finishedAt: new Date().toISOString(), @@ -142,12 +156,17 @@ export async function runDesktopStorybookSmokeTests( try { artifacts.writeMetadata('run', { ...result, + ...(incomplete ? { status: 'incomplete' } : {}), + counts, ...(failures.length ? { errors: failures.map(errorMessage) } : {}), }); } catch (error) { failures.push(error); writeDesktopStorybookFailure('writing the smoke report', error, options.errorOutput); } + if (incomplete && failures.length === 0) { + failures.push(new Error(`All ${counts.selected} selected desktop story tests were skipped; the smoke run provides no coverage.`)); + } if (failures.length === 1) { throw failures[0]; } diff --git a/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts b/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts index 73259b9a66..5071e63dbf 100644 --- a/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts +++ b/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts @@ -48,6 +48,29 @@ export class WdioStoryTestRunError extends Error { } } +/** + * Every selected wdio test was skipped, so the run provides no coverage. This is distinct from a test failure: nothing + * asserted against the product, but the run must not count as passing either. + */ +export class WdioStoryCoverageUnavailableError extends Error { + readonly results: readonly WdioStoryTestResult[]; + + constructor(results: readonly WdioStoryTestResult[]) { + const reasons = [...new Set(results.map(({ skipReason }) => skipReason ?? 'No reason given.'))]; + super(`All ${results.length} selected executable wdio tests were skipped: ${reasons.join('; ')}`); + this.name = 'WdioStoryCoverageUnavailableError'; + this.results = [...results]; + } +} + +export type WdioStoryTestCounts = { selected: number; passed: number; skipped: number; failed: number }; + +export function countWdioStoryTests(results: readonly { status: string }[]): WdioStoryTestCounts { + const passed = results.filter(({ status }) => status === 'passed').length; + const skipped = results.filter(({ status }) => status === 'skipped').length; + return { selected: results.length, passed, skipped, failed: results.length - passed - skipped }; +} + export function selectWdioStories( manifest: DesktopStoryManifest, options: DesktopStorybookWdioOptions, @@ -217,11 +240,14 @@ export async function runWdioStoryTests( ), ); } + const counts = countWdioStoryTests(results); + const incomplete = failures.length === 0 && counts.passed === 0; try { artifacts.writeMetadata('run', { endpoint: options.platform, platformManifestDigest: options.manifest.platformManifestDigest, - status: failures.length ? 'failed' : 'passed', + status: failures.length ? 'failed' : incomplete ? 'incomplete' : 'passed', + counts, ...(infrastructureFailure === undefined ? {} : { error: errorMessage(infrastructureFailure) }), tests: results, }); @@ -236,6 +262,7 @@ export async function runWdioStoryTests( writeDesktopStorybookFailure('cleaning generated wdio files', error, options.errorOutput); } if (failures.length) throw new WdioStoryTestRunError(results, failures); + if (incomplete) throw new WdioStoryCoverageUnavailableError(results); return results; } diff --git a/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs b/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs index aa90e3e4fa..ebaf22f57d 100644 --- a/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs +++ b/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs @@ -10,7 +10,7 @@ async function main() { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-inline-wdio-')); const [ { createDesktopStoryManifest }, - { runWdioStoryTests, WdioStoryTestRunError }, + { runWdioStoryTests, WdioStoryCoverageUnavailableError, WdioStoryTestRunError }, { createDesktopDriverStoryHarness, createFakeStoryWindows }, { makeDesktopStorybookConfig }, ] = await Promise.all([ @@ -180,7 +180,25 @@ async function main() { namedTimeout = error.results.map(({ testName, status }) => ({ testName, status })); } const passed = await run('pass'); - const skipped = await run('skip'); + let skipped; + try { + await run('skip'); + throw new Error('Expected an all-skipped run to be rejected as incomplete.'); + } catch (error) { + if ( + !(error instanceof Error) || + !(error instanceof WdioStoryCoverageUnavailableError) || + !error.message.includes('Explicit capability reason') || + !('results' in error) || + !Array.isArray(error.results) + ) + throw error; + skipped = error.results; + } + const skippedReport = JSON.parse(fs.readFileSync(path.join(root, 'artifacts/windows/wdio/run.json'), 'utf8')); + if (skippedReport.status !== 'incomplete' || skippedReport.counts.skipped !== 1 || skippedReport.counts.passed !== 0) { + throw new Error(`All-skipped run was not persisted as incomplete: ${JSON.stringify(skippedReport)}`); + } const failures = []; for (const story of ['fail', 'timeout', 'spin', 'crash', 'exit-zero', 'stale', 'navigation-failure']) { let rejected = false;