fix(storybook-desktop): report all-skipped WDIO runs as incomplete instead of passed

.changeset/wdio-all-skipped-incomplete.md

@@ -0,0 +1,5 @@
+---
+'@fluentui-react-native/storybook-desktop': patch
+---
+
+Report WDIO and smoke runs in which every selected test was skipped as `incomplete` instead of `passed`, and fail the command so an all-skipped selection cannot satisfy a required test gate. Run reports now include selected/passed/skipped/failed counts. Mixed pass/skip runs are unchanged.

packages/agentic/storybook-desktop/README.md

@@ -220,9 +220,11 @@ it is not `process.platform`. Win32 stays distinct even though its WebDriver
 `platformName` is `windows`. Native feature flags are typed under
 `browser.capabilities['furn:features']`.
 `signal` allows cooperative cancellation; `skip(reason)` records an explicit
-skip (return from the callback after calling it). Native selectors and
-supported WebDriver operations apply; there is no DOM or JavaScript execution
-inside the app.
+skip (return from the callback after calling it). A run in which every selected
+test skipped is reported as `incomplete` in `run.json` and fails the command,
+since it provides no coverage; runs with at least one pass are unaffected.
+Native selectors and supported WebDriver operations apply; there is no DOM or
+JavaScript execution inside the app.
 
 The original `wdio: async (context) => { ... }` form remains supported. It is
 one test, called `default` for filtering. Named collections must be non-empty,

packages/agentic/storybook-desktop/src/cli/smokeTests.test.ts

@@ -7,7 +7,7 @@ import type { DesktopStoryRunResult } from '@fluentui-react-native/desktop-drive
 
 import { makeDesktopStorybookConfig } from '../config/makeDesktopStorybookConfig.js';
 import type { runWdioStoryTests } from '../testing/runWdioTests.js';
-import { WdioStoryTestRunError } from '../testing/runWdioTests.js';
+import { WdioStoryCoverageUnavailableError, WdioStoryTestRunError } from '../testing/runWdioTests.js';
 import {
   formatDesktopStorybookSmokeTestSummary,
   runDesktopStorybookSmokeTests,
@@ -199,13 +199,50 @@ describe('runDesktopStorybookSmokeTests', () => {
     options.config = makeDesktopStorybookConfig({ projectRoot: root, wdio: { story: inputId } });
     const connect = jest.fn();
     const runWdio: jest.MockedFunction<typeof runWdioStoryTests> = jest.fn(async ({ story }) => [
-      { storyId: story!, status: 'skipped' as const, durationMs: 0, skipReason: 'Capability unavailable' },
+      { storyId: story!, status: 'passed' as const, durationMs: 0 },
     ]);
     const result = await runDesktopStorybookSmokeTests(options, connect, runWdio);
     expect(connect).not.toHaveBeenCalled();
     expect(runWdio).toHaveBeenCalledTimes(1);
-    expect(result.wdio).toMatchObject([{ storyId: inputId, status: 'skipped' }]);
-    expect(formatDesktopStorybookSmokeTestSummary(result)).toBe('Ran 1 desktop story tests (0 passed, 1 skipped).');
+    expect(result.wdio).toMatchObject([{ storyId: inputId, status: 'passed' }]);
+    expect(formatDesktopStorybookSmokeTestSummary(result)).toBe('Ran 1 desktop story tests (1 passed, 0 skipped).');
+  });
+
+  test('reports an all-skipped run as incomplete instead of passed', async () => {
+    options.manifest = { ...manifest, entries: [{ ...story(inputId, true), tests: undefined }] };
+    const runWdio: jest.MockedFunction<typeof runWdioStoryTests> = jest.fn(async ({ story }) => {
+      throw new WdioStoryCoverageUnavailableError([
+        { storyId: story!, status: 'skipped', durationMs: 0, skipReason: 'This test requires physical pointer input.' },
+      ]);
+    });
+    await expect(runDesktopStorybookSmokeTests(options, jest.fn(), runWdio)).rejects.toThrow(
+      'All 1 selected desktop story tests were skipped',
+    );
+    expect(errorOutput.write).not.toHaveBeenCalled();
+    expect(JSON.parse(fs.readFileSync(path.join(root, 'artifacts/windows/desktop-driver/run.json'), 'utf8'))).toMatchObject({
+      status: 'incomplete',
+      counts: { selected: 1, passed: 0, skipped: 1, failed: 0 },
+      wdio: [{ storyId: inputId, status: 'skipped', skipReason: 'This test requires physical pointer input.' }],
+    });
+  });
+
+  test('passes a mixed run when a story has only skipped callbacks', async () => {
+    options.manifest = { ...manifest, entries: [story(buttonId), { ...story(inputId, true), tests: undefined }] };
+    const connect: DesktopStorybookSmokeConnector = async () => ({
+      delete: async () => undefined,
+      listStories: async () => options.manifest,
+      runStoryTests: async () => planResult(),
+    });
+    const runWdio: jest.MockedFunction<typeof runWdioStoryTests> = jest.fn(async ({ story }) => {
+      throw new WdioStoryCoverageUnavailableError([{ storyId: story!, status: 'skipped', durationMs: 0, skipReason: 'Unavailable' }]);
+    });
+    const result = await runDesktopStorybookSmokeTests(options, connect, runWdio);
+    expect(result.status).toBe('passed');
+    expect(formatDesktopStorybookSmokeTestSummary(result)).toBe('Ran 2 desktop story tests (1 passed, 1 skipped).');
+    expect(JSON.parse(fs.readFileSync(path.join(root, 'artifacts/windows/desktop-driver/run.json'), 'utf8'))).toMatchObject({
+      status: 'passed',
+      counts: { selected: 2, passed: 1, skipped: 1, failed: 0 },
+    });
   });
 
   test('fails after persisting failed plan results and closing the session', async () => {

packages/agentic/storybook-desktop/src/cli/smokeTests.ts

@@ -11,7 +11,14 @@ import type { DesktopWebdriverOptions, DesktopWebdriverSession } from '@fluentui
 import type { DesktopStorybookConfig } from '../config/makeDesktopStorybookConfig.js';
 import type { Platforms } from '../config/platforms.js';
 import { resolveWdioOptions } from '../config/wdio.js';
-import { runWdioStoryTests, selectWdioStories, WdioStoryTestRunError, type WdioStoryTestResult } from '../testing/runWdioTests.js';
+import {
+  countWdioStoryTests,
+  runWdioStoryTests,
+  selectWdioStories,
+  WdioStoryCoverageUnavailableError,
+  WdioStoryTestRunError,
+  type WdioStoryTestResult,
+} from '../testing/runWdioTests.js';
 import { writeDesktopStorybookFailure, type DesktopStorybookErrorOutput } from '../../config/diagnostics.cjs';
 import type { DesktopCommandRunner } from './commandRunner.js';
 
@@ -95,6 +102,11 @@ export async function runDesktopStorybookSmokeTests(
             )),
           );
         } catch (error) {
+          // A story whose callbacks all skipped is not a failure on its own; coverage is judged across the whole smoke run.
+          if (error instanceof WdioStoryCoverageUnavailableError) {
+            wdio.push(...error.results);
+            continue;
+          }
           writeDesktopStorybookFailure(`smoke story "${storyId}"`, error, options.errorOutput);
           if (error instanceof WdioStoryTestRunError && error.results.length) {
             wdio.push(...error.results);
@@ -126,6 +138,8 @@ export async function runDesktopStorybookSmokeTests(
     }
     failures.push(new Error('Executable smoke tests reported failures.'));
   }
+  const counts = countWdioStoryTests([...tests, ...wdio]);
+  const incomplete = failures.length === 0 && counts.passed === 0;
   const result: DesktopStorybookSmokeTestResult = {
     endpoint: platform,
     finishedAt: new Date().toISOString(),
@@ -142,12 +156,17 @@ export async function runDesktopStorybookSmokeTests(
   try {
     artifacts.writeMetadata('run', {
       ...result,
+      ...(incomplete ? { status: 'incomplete' } : {}),
+      counts,
       ...(failures.length ? { errors: failures.map(errorMessage) } : {}),
     });
   } catch (error) {
     failures.push(error);
     writeDesktopStorybookFailure('writing the smoke report', error, options.errorOutput);
   }
+  if (incomplete && failures.length === 0) {
+    failures.push(new Error(`All ${counts.selected} selected desktop story tests were skipped; the smoke run provides no coverage.`));
+  }
   if (failures.length === 1) {
     throw failures[0];
   }

packages/agentic/storybook-desktop/src/testing/runWdioTests.ts

@@ -48,6 +48,29 @@ export class WdioStoryTestRunError extends Error {
   }
 }
 
+/**
+ * Every selected wdio test was skipped, so the run provides no coverage. This is distinct from a test failure: nothing
+ * asserted against the product, but the run must not count as passing either.
+ */
+export class WdioStoryCoverageUnavailableError extends Error {
+  readonly results: readonly WdioStoryTestResult[];
+
+  constructor(results: readonly WdioStoryTestResult[]) {
+    const reasons = [...new Set(results.map(({ skipReason }) => skipReason ?? 'No reason given.'))];
+    super(`All ${results.length} selected executable wdio tests were skipped: ${reasons.join('; ')}`);
+    this.name = 'WdioStoryCoverageUnavailableError';
+    this.results = [...results];
+  }
+}
+
+export type WdioStoryTestCounts = { selected: number; passed: number; skipped: number; failed: number };
+
+export function countWdioStoryTests(results: readonly { status: string }[]): WdioStoryTestCounts {
+  const passed = results.filter(({ status }) => status === 'passed').length;
+  const skipped = results.filter(({ status }) => status === 'skipped').length;
+  return { selected: results.length, passed, skipped, failed: results.length - passed - skipped };
+}
+
 export function selectWdioStories(
   manifest: DesktopStoryManifest,
   options: DesktopStorybookWdioOptions,
@@ -217,11 +240,14 @@ export async function runWdioStoryTests(
       ),
     );
   }
+  const counts = countWdioStoryTests(results);
+  const incomplete = failures.length === 0 && counts.passed === 0;
   try {
     artifacts.writeMetadata('run', {
       endpoint: options.platform,
       platformManifestDigest: options.manifest.platformManifestDigest,
-      status: failures.length ? 'failed' : 'passed',
+      status: failures.length ? 'failed' : incomplete ? 'incomplete' : 'passed',
+      counts,
       ...(infrastructureFailure === undefined ? {} : { error: errorMessage(infrastructureFailure) }),
       tests: results,
     });
@@ -236,6 +262,7 @@ export async function runWdioStoryTests(
     writeDesktopStorybookFailure('cleaning generated wdio files', error, options.errorOutput);
   }
   if (failures.length) throw new WdioStoryTestRunError(results, failures);
+  if (incomplete) throw new WdioStoryCoverageUnavailableError(results);
   return results;
 }
 

packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs

@@ -10,7 +10,7 @@ async function main() {
   const root = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-inline-wdio-'));
   const [
     { createDesktopStoryManifest },
-    { runWdioStoryTests, WdioStoryTestRunError },
+    { runWdioStoryTests, WdioStoryCoverageUnavailableError, WdioStoryTestRunError },
     { createDesktopDriverStoryHarness, createFakeStoryWindows },
     { makeDesktopStorybookConfig },
   ] = await Promise.all([
@@ -180,7 +180,25 @@ async function main() {
       namedTimeout = error.results.map(({ testName, status }) => ({ testName, status }));
     }
     const passed = await run('pass');
-    const skipped = await run('skip');
+    let skipped;
+    try {
+      await run('skip');
+      throw new Error('Expected an all-skipped run to be rejected as incomplete.');
+    } catch (error) {
+      if (
+        !(error instanceof Error) ||
+        !(error instanceof WdioStoryCoverageUnavailableError) ||
+        !error.message.includes('Explicit capability reason') ||
+        !('results' in error) ||
+        !Array.isArray(error.results)
+      )
+        throw error;
+      skipped = error.results;
+    }
+    const skippedReport = JSON.parse(fs.readFileSync(path.join(root, 'artifacts/windows/wdio/run.json'), 'utf8'));
+    if (skippedReport.status !== 'incomplete' || skippedReport.counts.skipped !== 1 || skippedReport.counts.passed !== 0) {
+      throw new Error(`All-skipped run was not persisted as incomplete: ${JSON.stringify(skippedReport)}`);
+    }
     const failures = [];
     for (const story of ['fail', 'timeout', 'spin', 'crash', 'exit-zero', 'stale', 'navigation-failure']) {
       let rejected = false;