diff --git a/src/runner/client.ts b/src/runner/client.ts index af10657f..2e7363bf 100644 --- a/src/runner/client.ts +++ b/src/runner/client.ts @@ -220,7 +220,7 @@ export async function runConformanceTest( } const checks = scenario.getChecks(); - checks.push(...wireSchemaChecks(resolvedVersion)); + checks.push(...wireSchemaChecks(resolvedVersion, scenario.wireSchemaScope)); if (resultDir) { await fs.writeFile( @@ -382,7 +382,7 @@ export async function runInteractiveMode( console.log('\nShutting down...'); const checks = scenario.getChecks(); - checks.push(...wireSchemaChecks(resolvedVersion)); + checks.push(...wireSchemaChecks(resolvedVersion, scenario.wireSchemaScope)); if (resultDir) { await fs.writeFile( diff --git a/src/runner/server.ts b/src/runner/server.ts index 21e80ec4..9a0d5487 100644 --- a/src/runner/server.ts +++ b/src/runner/server.ts @@ -150,7 +150,9 @@ export async function runServerConformanceTest( scenarioName, timeout ); - checks.push(...wireSchemaChecks(resolvedSpecVersion)); + checks.push( + ...wireSchemaChecks(resolvedSpecVersion, scenario.wireSchemaScope) + ); if (resultDir) { await fs.writeFile( diff --git a/src/types.ts b/src/types.ts index ebe75a27..12e87506 100644 --- a/src/types.ts +++ b/src/types.ts @@ -138,6 +138,17 @@ export interface Scenario { * Use this for scenarios where the client is expected to error (e.g., rejecting invalid auth). */ allowClientError?: boolean; + /** + * Methods whose messages this scenario grades for wire-schema validity. When + * set, `wire-schema-valid` covers only the implementation's messages for these + * methods, and violations elsewhere are reported as an INFO check instead. + * An entry ending in `/` matches a prefix, so an events scenario would list + * `['events/', 'notifications/events/']`. Lifecycle messages (`initialize`, + * `server/discover`, `ping`) are out of scope unless listed. Use it for + * extension scenarios, so a gap outside the extension does not fail them. + * Unset or empty means every message is graded. + */ + wireSchemaScope?: readonly string[]; start(ctx: ScenarioContext): Promise; stop(): Promise; getChecks(): ConformanceCheck[]; @@ -147,6 +158,8 @@ export interface ClientScenario { name: string; description: string; source: ScenarioSource; + /** See `Scenario.wireSchemaScope`. */ + wireSchemaScope?: readonly string[]; run(ctx: RunContext): Promise; } diff --git a/src/validation/wire-schema.test.ts b/src/validation/wire-schema.test.ts index d3510475..3ab0a3f1 100644 --- a/src/validation/wire-schema.test.ts +++ b/src/validation/wire-schema.test.ts @@ -579,3 +579,88 @@ describe('schema errata', () => { ).toBe('integer'); }); }); + +describe('wireSchemaChecks scope', () => { + // A tools/list result missing `tools`: invalid at every version. + const record = (requestMethod: string | undefined) => + validateWireMessage( + DRAFT_PROTOCOL_VERSION, + { jsonrpc: '2.0', id: 1, result: {} }, + { origin: 'implementation', context: 'test response', requestMethod } + ); + const status = (checks: ReturnType, id: string) => + checks.find((c) => c.id === id)?.status; + + it('grades only in-scope methods and reports the rest as INFO', () => + withWireRecorder(async () => { + record('tools/list'); + const checks = wireSchemaChecks(DRAFT_PROTOCOL_VERSION, ['events/']); + expect(status(checks, 'wire-schema-valid')).toBe('SUCCESS'); + expect(status(checks, 'wire-schema-valid-out-of-scope')).toBe('INFO'); + })); + + it('still fails a violation inside the scope, by prefix or exact method', () => + withWireRecorder(async () => { + record('tools/list'); + expect( + status( + wireSchemaChecks(DRAFT_PROTOCOL_VERSION, ['tools/']), + 'wire-schema-valid' + ) + ).toBe('FAILURE'); + record('tools/list'); + expect( + status( + wireSchemaChecks(DRAFT_PROTOCOL_VERSION, ['tools/list']), + 'wire-schema-valid' + ) + ).toBe('FAILURE'); + })); + + it('keeps a violation with no known method in scope', () => + withWireRecorder(async () => { + record(undefined); + const checks = wireSchemaChecks(DRAFT_PROTOCOL_VERSION, ['events/']); + expect(status(checks, 'wire-schema-valid')).toBe('FAILURE'); + expect(status(checks, 'wire-schema-valid-out-of-scope')).toBeUndefined(); + })); + + it('grades every message when no scope is set', () => + withWireRecorder(async () => { + record('tools/list'); + const checks = wireSchemaChecks(DRAFT_PROTOCOL_VERSION); + expect(status(checks, 'wire-schema-valid')).toBe('FAILURE'); + expect(status(checks, 'wire-schema-valid-out-of-scope')).toBeUndefined(); + })); + + it('attributes a request or notification by its own method', () => + withWireRecorder(async () => { + validateWireMessage( + DRAFT_PROTOCOL_VERSION, + { jsonrpc: '2.0', method: 'notifications/progress', params: {} }, + { origin: 'implementation', context: 'test notification' } + ); + const checks = wireSchemaChecks(DRAFT_PROTOCOL_VERSION, ['events/']); + expect(status(checks, 'wire-schema-valid')).toBe('SUCCESS'); + expect(status(checks, 'wire-schema-valid-out-of-scope')).toBe('INFO'); + })); + + it('treats an empty scope as grading everything', () => + withWireRecorder(async () => { + record('tools/list'); + const checks = wireSchemaChecks(DRAFT_PROTOCOL_VERSION, []); + expect(status(checks, 'wire-schema-valid')).toBe('FAILURE'); + })); + + it('leaves harness-origin violations failing whatever the scope', () => + withWireRecorder(async () => { + validateWireMessage( + DRAFT_PROTOCOL_VERSION, + { jsonrpc: '2.0', id: 1, result: {} }, + { origin: 'harness', context: 'test', requestMethod: 'tools/list' } + ); + const checks = wireSchemaChecks(DRAFT_PROTOCOL_VERSION, ['events/']); + expect(status(checks, 'wire-schema-harness-error')).toBe('FAILURE'); + expect(status(checks, 'wire-schema-valid-out-of-scope')).toBeUndefined(); + })); +}); diff --git a/src/validation/wire-schema.ts b/src/validation/wire-schema.ts index 1cdbed09..b3cb3094 100644 --- a/src/validation/wire-schema.ts +++ b/src/validation/wire-schema.ts @@ -32,6 +32,8 @@ export interface WireSchemaViolation { origin: WireOrigin; specVersion: SpecVersion; context: string; + /** The message's own `method`, or for a response the answered request's. */ + method?: string; errors: string[]; message: unknown; } @@ -353,10 +355,12 @@ export function validateWireMessage( state.observed++; const errors = wireSchemaErrors(specVersion, message, info.requestMethod); if (errors.length > 0) { + const own = (message as { method?: unknown } | null)?.method; state.violations.push({ origin: info.origin, specVersion, context: info.context, + method: typeof own === 'string' ? own : info.requestMethod, errors, message }); @@ -397,9 +401,27 @@ export function formatWireViolation(v: WireSchemaViolation): string { ); } +/** Whether a violation falls in a scenario's `wireSchemaScope`. A scope entry + * ending in `/` matches a method prefix; any other entry matches exactly. An + * empty scope grades everything. A violation whose method is unknown (a batch, + * or a response not paired with its request) stays in scope, so scoping never + * hides it. */ +function inScope(v: WireSchemaViolation, scope?: readonly string[]): boolean { + if (!scope?.length || v.method === undefined) return true; + const method = v.method; + return scope.some((s) => + s.endsWith('/') ? method.startsWith(s) : method === s + ); +} + /** Drain the recorder and synthesize the per-scenario conformance checks. Returns `[]` - * when no wire traffic was observed (e.g. a scenario asserting only on raw HTTP). */ -export function wireSchemaChecks(specVersion: SpecVersion): ConformanceCheck[] { + * when no wire traffic was observed (e.g. a scenario asserting only on raw HTTP). + * With a `scope`, `wire-schema-valid` grades only the implementation's messages + * for those methods; violations elsewhere are reported as an INFO check. */ +export function wireSchemaChecks( + specVersion: SpecVersion, + scope?: readonly string[] +): ConformanceCheck[] { const { violations: all, observed: count } = takeWireViolations(); if (count === 0 && all.length === 0) return []; @@ -412,7 +434,12 @@ export function wireSchemaChecks(specVersion: SpecVersion): ConformanceCheck[] { ]; const checks: ConformanceCheck[] = []; - const implViolations = all.filter((v) => v.origin === 'implementation'); + const implViolations = all.filter( + (v) => v.origin === 'implementation' && inScope(v, scope) + ); + const outOfScope = all.filter( + (v) => v.origin === 'implementation' && !inScope(v, scope) + ); checks.push({ id: 'wire-schema-valid', name: 'WireSchemaValid', @@ -423,6 +450,7 @@ export function wireSchemaChecks(specVersion: SpecVersion): ConformanceCheck[] { specReferences, details: { messagesValidated: count, + ...(scope !== undefined && { scope }), violations: implViolations.map(violationDetails) }, errorMessage: @@ -431,6 +459,20 @@ export function wireSchemaChecks(specVersion: SpecVersion): ConformanceCheck[] { : undefined }); + if (outOfScope.length > 0) { + checks.push({ + id: 'wire-schema-valid-out-of-scope', + name: 'WireSchemaValidOutOfScope', + description: + "Implementation messages outside this scenario's area that are invalid per the spec JSON schema. Reported for information; they do not fail the scenario.", + status: 'INFO', + timestamp, + specReferences, + details: { scope, violations: outOfScope.map(violationDetails) }, + errorMessage: outOfScope.map(formatWireViolation).join('\n') + }); + } + const harnessViolations = all.filter((v) => v.origin === 'harness'); if (harnessViolations.length > 0) { checks.push({