diff --git a/README.md b/README.md index 622c55a..05a4ecb 100644 --- a/README.md +++ b/README.md @@ -517,6 +517,48 @@ agility reverse-sync --sourceGuid A --targetGuid B --locales en-us --preflight agility reverse-sync --sourceGuid A --targetGuid B --locales en-us ``` +### Verify + +Compare a synced target instance against its source and report every difference, independent of the sync's own logs and change tracking. + +```bash +agility verify --sourceGuid --targetGuid [--locales en-us,fr-ca] [--channel website] [--models ...] [--pages ...] [--containers ...] [--jsonSummary ] +``` + +Use the same `--sourceGuid` / `--targetGuid` as the forward `sync`. Verify never writes to either instance or to the mapping files. + +#### What it does + +1. Waits until neither instance has a content update in progress (preview and published), up to 10 minutes. +2. Runs a full pull of both instances, then checks again. If an update ran during the pull, it pulls once more. +3. Compares the two instances through the mapping files: + - mapping integrity (unmapped source items, mappings to missing targets, two sources mapped to one target) + - models, containers and templates + - content counts per model and container, list order, and field values (linked content, assets and companion fields resolved through the mappings) + - component order within each page zone, and which zone each component is in + - sitemap parent and sibling order per locale and channel + +#### Results + +- **Failures**: field mismatches, missing items, components in the wrong order or zone, sitemap order errors. +- **Warnings**: content that exists only on the target, publish-state differences. These don't fail the run. +- **Source defects**: differences explained by the source data, such as an item missing a required field or an item in a deleted container. These are listed but don't fail the run. + +The console prints a summary table per locale and the first 20 findings per category. The full report is written to `--jsonSummary`, or to `agility-files/logs/verify-{source}-{target}-{timestamp}.json`. + +| Exit code | Meaning | +| --------- | ----------------------------------------------------------------------------- | +| `0` | Target matches source | +| `1` | Differences found | +| `2` | Could not verify (auth, status check timeout, pull failure, missing mappings) | + +```bash +# Blue/green: sync into the new instance, then prove it before switching traffic +agility sync --sourceGuid A --targetGuid B --locales en-us --preflight +agility sync --sourceGuid A --targetGuid B --locales en-us +agility verify --sourceGuid A --targetGuid B --locales en-us +``` + ## File Structure The CLI organizes downloaded content in a structured format: diff --git a/src/core/auth.ts b/src/core/auth.ts index 5a7f6dc..977bc66 100644 --- a/src/core/auth.ts +++ b/src/core/auth.ts @@ -993,7 +993,7 @@ export class Auth { * Validate command-specific requirements and set up instance access * This should be called by each command after auth.init() */ - async validateCommand(commandType: "pull" | "sync" | "clean" | "interactive" | "push"): Promise { + async validateCommand(commandType: "pull" | "sync" | "clean" | "interactive" | "push" | "verify"): Promise { const missingFields: string[] = []; // Check command-specific requirements @@ -1030,6 +1030,20 @@ export class Auth { if (!state.channel) missingFields.push("channel (use --channel or AGILITY_WEBSITE in .env)"); break; + case "verify": + // Verify (PROD-2552) reads both instances and writes to neither + if (!state.sourceGuid) + missingFields.push("sourceGuid (use --sourceGuid or AGILITY_GUID in .env)"); + if (!state.targetGuid) + missingFields.push("targetGuid (use --targetGuid or AGILITY_TARGET_GUID in .env)"); + + const hasVerifyUserLocales = state.locale && state.locale.length > 0; + const hasVerifyAutoDetectedLocales = state.guidLocaleMap && state.guidLocaleMap.size > 0; + if (!hasVerifyUserLocales && !hasVerifyAutoDetectedLocales) { + missingFields.push("locale (use --locales or AGILITY_LOCALES in .env, or locales will be auto-detected)"); + } + break; + case "clean": // Clean needs minimal validation since it prompts for instance selection break; @@ -1078,6 +1092,11 @@ export class Auth { ); return false; } + } else if (commandType === "verify") { + // Verify only needs read access to both instances; the pull uses the API keys auth.init() + // already resolved for each guid. + await this.validateInstanceAccess(state.sourceGuid, "source"); + await this.validateInstanceAccess(state.targetGuid, "target"); } else if (commandType === "pull" && state.sourceGuid) { // Pull operation - validate source access and get API keys await this.validateInstanceAccess(state.sourceGuid, "instance"); diff --git a/src/core/index.ts b/src/core/index.ts index 74f318a..b633026 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -6,7 +6,7 @@ // Core authentication and state management export { Auth } from "./auth"; export { state, setState, resetState, primeFromEnv, getState, getUIMode, enableReverseSync } from "./state"; -export { systemArgs, workflowArgs, type SystemArgsType } from "./system-args"; +export { systemArgs, workflowArgs, verifyArgs, type SystemArgsType } from "./system-args"; export { normalizeProcessArgs, normalizeArgv } from "./arg-normalizer"; // Main operation services diff --git a/src/core/json-summary.ts b/src/core/json-summary.ts index 0969a6e..77556cd 100644 --- a/src/core/json-summary.ts +++ b/src/core/json-summary.ts @@ -310,13 +310,14 @@ export function buildJsonSummary(input: BuildJsonSummaryInput): JsonSummary { } /** - * Write the summary to disk, creating parent directories as needed. + * Write the summary to disk, creating parent directories as needed. Also used for the + * `agility verify` report (PROD-2552), which has its own schema but the same write contract. * * Never throws. A reporting artifact must not be able to fail a sync that otherwise * succeeded — an unwritable path is worth a warning, not a non-zero exit. Returns whether * the write landed so the caller can say so. */ -export function writeJsonSummary(filePath: string, summary: JsonSummary): { written: boolean; error?: string } { +export function writeJsonSummary(filePath: string, summary: JsonSummary | object): { written: boolean; error?: string } { try { const resolved = path.resolve(filePath); const dir = path.dirname(resolved); diff --git a/src/core/logs.ts b/src/core/logs.ts index d099cce..1103cea 100644 --- a/src/core/logs.ts +++ b/src/core/logs.ts @@ -69,7 +69,7 @@ function safeStringify(obj: any, indent?: number): string { } } -export type OperationType = "pull" | "push" | "sync" | "reverse-sync"; +export type OperationType = "pull" | "push" | "sync" | "reverse-sync" | "verify"; export type EntityType = | "model" @@ -524,7 +524,12 @@ export class Logs { } // Build filename with GUID - if (this.operationType === "push" || this.operationType === "sync" || this.operationType === "reverse-sync") { + if ( + this.operationType === "push" || + this.operationType === "sync" || + this.operationType === "reverse-sync" || + this.operationType === "verify" + ) { const sourceGuid = state.sourceGuid || "unknown"; const targetGuid = state.targetGuid || "unknown"; filename = `${sourceGuid}-${targetGuid}-${this.operationType}-${timestamp}.txt`; diff --git a/src/core/system-args.ts b/src/core/system-args.ts index 288df5e..69aba1a 100644 --- a/src/core/system-args.ts +++ b/src/core/system-args.ts @@ -200,6 +200,56 @@ export const workflowArgs = { }, }; +/** + * Args accepted by the `verify` command (PROD-2552). + * + * A deliberate subset of systemArgs, reusing their parsing and aliases — NOT a spread of the whole + * set: verify never writes to an instance, so the sync-only options (overwrite, autoPublish, + * preflight, fullPull, elements, modelsWithDeps) would advertise behavior it does not have. Verify + * always does a full pull of both instances. + */ +export const verifyArgs = { + token: systemArgs.token, + dev: systemArgs.dev, + headless: systemArgs.headless, + verbose: systemArgs.verbose, + sourceGuid: { + ...systemArgs.sourceGuid, + describe: + "The source instance GUID of the sync being verified. Required; falls back to AGILITY_GUID from .env.", + }, + targetGuid: { + ...systemArgs.targetGuid, + describe: + "The target instance GUID of the sync being verified. Required; falls back to AGILITY_TARGET_GUID from .env.", + }, + locales: systemArgs.locales, + channel: { + ...systemArgs.channel, + describe: "Verify only this channel's sitemap. If not provided, every channel is verified.", + default: undefined, + }, + models: { + ...systemArgs.models, + describe: "Verify only these models (comma-separated reference names), as `sync --models` scopes a sync.", + }, + pages: { + ...systemArgs.pages, + describe: + "Verify only these pages and what they need (comma-separated paths, names or IDs), as `sync --pages` scopes a sync.", + }, + containers: { + ...systemArgs.containers, + describe: + "Verify only these containers and what they depend on (comma-separated reference names, titles or IDs), as `sync --containers` scopes a sync.", + }, + jsonSummary: { + ...systemArgs.jsonSummary, + describe: + "Write the full verify report (every finding, per-locale counts, timings) as JSON to this path. Defaults to agility-files/logs/verify---.json.", + }, +}; + /** * Type helper for command arguments that include system args */ diff --git a/src/core/tests/verify.test.ts b/src/core/tests/verify.test.ts new file mode 100644 index 0000000..84258de --- /dev/null +++ b/src/core/tests/verify.test.ts @@ -0,0 +1,433 @@ +/** + * PROD-2552: `agility verify` — the run around the checks: status guard, pull, exit codes, and + * how the command primes state. The checks themselves are covered in src/lib/verify/tests. + */ +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { changedSnapshots, runVerifyCommand, STATUS_TIMEOUT_MS, Verify, VerifyDeps, VerifyOptions } from "../verify"; +import { resetState, setState, state } from "../state"; +import { EXIT_CANNOT_VERIFY, EXIT_DIFFERENCES, EXIT_PASS, VerifyReport } from "../../lib/verify/verify-report"; +import { emptySide } from "../../lib/verify/tests/fixtures"; +import { SideEntities } from "../../lib/verify/verify-context"; + +const SRC = "src-1"; +const TGT = "tgt-1"; +const LOCALE = "en-us"; + +let root: string; + +function writeJson(file: string, data: any): void { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, JSON.stringify(data)); +} + +/** A cache + mapping tree on disk, so the "can we verify at all" preconditions pass. */ +function seedRoot(): void { + writeJson(path.join(root, SRC, "models", "1.json"), { id: 1, referenceName: "Post", fields: [] }); + writeJson(path.join(root, TGT, "models", "11.json"), { id: 11, referenceName: "Post", fields: [] }); + writeJson(path.join(root, "mappings", `${SRC}-${TGT}`, "models", "mappings.json"), [ + { sourceID: 1, targetID: 11, sourceReferenceName: "Post", targetReferenceName: "Post" }, + ]); + writeJson(path.join(root, "mappings", `${SRC}-${TGT}`, LOCALE, "item", "mappings.json"), [ + { sourceContentID: 5, targetContentID: 55 }, + ]); +} + +const POST = { id: 1, referenceName: "Post", fields: [{ name: "Title", type: "Text" }] }; +const POST_T = { ...POST, id: 11 }; +const item = (contentID: number, title: string) => ({ + contentID, + properties: { state: 2, referenceName: "posts", definitionName: "Post", itemOrder: 0, versionID: 1 }, + fields: { title }, +}); + +function sides(targetTitle: string): Record { + return { + [SRC]: emptySide({ models: [POST], content: [item(5, "Hello")] }), + [TGT]: emptySide({ models: [POST_T], content: [item(55, targetTitle)] }), + }; +} + +function options(overrides: Partial = {}): VerifyOptions { + return { + sourceGuid: SRC, + targetGuid: TGT, + locales: [LOCALE], + models: "", + pages: "", + containers: "", + jsonSummary: path.join(root, "report.json"), + rootPath: root, + ...overrides, + }; +} + +const idle = { inProgress: false, lastContentVersionID: 10, maxChangeDate: "2026-10-01T00:00:00Z" } as any; +const busy = { ...idle, inProgress: true }; +const okPull = async () => ({ + success: true, + results: [ + { guidProcessed: SRC, failed: [] }, + { guidProcessed: TGT, failed: [] }, + ], +}); + +/** Deps with a fake clock that sleep() advances, a no-op pull, and in-memory sides. */ +function deps(overrides: Partial = {}, targetTitle = "Hello"): VerifyDeps & { clock: { t: number } } { + const clock = { t: Date.parse("2026-10-07T00:00:00Z") }; + const data = sides(targetTitle); + return { + clock, + getStatus: async () => idle, + pull: okPull, + sleep: async (ms: number) => { + clock.t += ms; + }, + now: () => clock.t, + log: () => {}, + loadSide: async (guid: string) => data[guid], + statusTimeoutMs: STATUS_TIMEOUT_MS, + statusPollMs: 5000, + print: false, + ...overrides, + }; +} + +beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), "verify-core-")); + resetState(); + setState({ rootPath: root, sourceGuid: SRC, targetGuid: TGT }); + seedRoot(); +}); + +afterEach(() => { + fs.rmSync(root, { recursive: true, force: true }); + jest.restoreAllMocks(); +}); + +describe("exit codes", () => { + it("0 when the target matches", async () => { + const report = await new Verify(options(), deps()).run(); + expect(report.exitCode).toBe(EXIT_PASS); + expect(report.verdict).toBe("pass"); + }); + + it("1 when the target differs", async () => { + const report = await new Verify(options(), deps({}, "Changed")).run(); + expect(report.exitCode).toBe(EXIT_DIFFERENCES); + expect(report.findings).toEqual([expect.objectContaining({ category: "fieldMismatch", path: "title" })]); + }); + + it("writes the JSON report to --jsonSummary", async () => { + await new Verify(options(), deps()).run(); + const written = JSON.parse(fs.readFileSync(path.join(root, "report.json"), "utf8")); + expect(written).toMatchObject({ command: "verify", exitCode: 0, source: SRC, target: TGT }); + expect(written.run.status.before).toHaveLength(4); + }); + + it("writes to agility-files/logs/verify-*.json without --jsonSummary", async () => { + await new Verify(options({ jsonSummary: "" }), deps()).run(); + const logs = fs.readdirSync(path.join(root, "logs")); + expect(logs).toEqual([expect.stringMatching(new RegExp(`^verify-${SRC}-${TGT}-.*\\.json$`))]); + }); + + it("2 when the pair has no mapping files", async () => { + fs.rmSync(path.join(root, "mappings"), { recursive: true }); + const report = await new Verify(options(), deps()).run(); + expect(report.exitCode).toBe(EXIT_CANNOT_VERIFY); + expect(report.error).toMatch(/No mapping files/); + }); + + it("2 when a requested locale has no mappings", async () => { + const report = await new Verify(options({ locales: [LOCALE, "fr-ca"] }), deps()).run(); + expect(report.exitCode).toBe(EXIT_CANNOT_VERIFY); + expect(report.error).toMatch(/fr-ca/); + }); + + it("2 when the pulled cache is unreadable", async () => { + fs.writeFileSync(path.join(root, TGT, "models", "12.json"), "{ truncated"); + const report = await new Verify(options(), deps()).run(); + expect(report.exitCode).toBe(EXIT_CANNOT_VERIFY); + expect(report.error).toMatch(/unreadable/); + }); + + it("2 when scope flags are combined", async () => { + const report = await new Verify(options({ models: "Post", pages: "/home" }), deps()).run(); + expect(report.exitCode).toBe(EXIT_CANNOT_VERIFY); + }); + + it("never writes to the mapping files", async () => { + const dir = path.join(root, "mappings"); + const snapshot = () => JSON.stringify(fs.readdirSync(dir, { recursive: true } as any).sort()); + const before = snapshot(); + // A type folder without its mappings.json — fileOperations.getMappingFile would seed one. + fs.mkdirSync(path.join(dir, `${SRC}-${TGT}`, "containers")); + await new Verify(options(), deps()).run(); + expect(snapshot()).toBe(JSON.stringify([...JSON.parse(before), `${SRC}-${TGT}/containers`].sort())); + }); +}); + +describe("status guard", () => { + it("checks both instances in preview and fetch mode before and after the pull", async () => { + const calls: string[] = []; + await new Verify( + options(), + deps({ + getStatus: async (guid, mode) => { + calls.push(`${guid}:${mode}`); + return idle; + }, + }) + ).run(); + const once = [`${SRC}:preview`, `${SRC}:fetch`, `${TGT}:preview`, `${TGT}:fetch`]; + expect(calls).toEqual([...once, ...once]); + }); + + it("waits for an in-progress update to finish", async () => { + let checks = 0; + const d = deps({ + getStatus: async (guid, mode) => (guid === TGT && mode === "fetch" && ++checks <= 3 ? busy : idle), + }); + const sleep = jest.spyOn(d, "sleep"); + const report = await new Verify(options(), d).run(); + expect(report.exitCode).toBe(EXIT_PASS); + expect(sleep).toHaveBeenCalledTimes(3); + }); + + it("gives up after the timeout with exit 2", async () => { + const pull = jest.fn(okPull); + const report = await new Verify(options(), deps({ getStatus: async () => busy, pull })).run(); + expect(report.exitCode).toBe(EXIT_CANNOT_VERIFY); + expect(report.error).toMatch(/Timed out after 10 minute/); + expect(pull).not.toHaveBeenCalled(); + }); + + it("treats a status error as exit 2 (it is not swallowed)", async () => { + const report = await new Verify( + options(), + deps({ + getStatus: async () => { + throw new Error("503 Service Unavailable"); + }, + }) + ).run(); + expect(report.exitCode).toBe(EXIT_CANNOT_VERIFY); + expect(report.error).toMatch(/503/); + }); + + it("re-pulls once when an update lands during the pull", async () => { + let version = 10; + const pull = jest.fn(async () => { + if (pull.mock.calls.length === 1) version = 11; // an update lands during the first pull only + return okPull(); + }); + const report = await new Verify( + options(), + deps({ getStatus: async () => ({ ...idle, lastContentVersionID: version }), pull }) + ).run(); + expect(pull).toHaveBeenCalledTimes(2); + expect(report.exitCode).toBe(EXIT_PASS); + expect(report.run.pull.attempts).toBe(2); + }); + + it("exits 2 when an update lands during the re-pull too", async () => { + let version = 10; + const pull = jest.fn(async () => { + version++; + return okPull(); + }); + const report = await new Verify( + options(), + deps({ getStatus: async () => ({ ...idle, lastContentVersionID: version }), pull }) + ).run(); + expect(pull).toHaveBeenCalledTimes(2); + expect(report.exitCode).toBe(EXIT_CANNOT_VERIFY); + expect(report.error).toMatch(/update ran during the pull/); + }); + + it("detects a change by maxChangeDate or a new in-progress update", () => { + const base = { + guid: "g", + mode: "fetch", + checkedAt: "", + inProgress: false, + lastContentVersionID: 1, + maxChangeDate: "a", + }; + expect(changedSnapshots([base], [{ ...base }])).toEqual([]); + expect(changedSnapshots([base], [{ ...base, maxChangeDate: "b" }])).toHaveLength(1); + expect(changedSnapshots([base], [{ ...base, inProgress: true }])).toHaveLength(1); + }); +}); + +describe("pull", () => { + it("does a full pull with every element and no scope, then restores state", async () => { + setState({ models: "Post", fullPull: false }); + let during: any; + await new Verify( + options({ models: "Post" }), + deps({ + pull: async () => { + during = { fullPull: state.fullPull, models: state.models, elements: state.elements }; + return okPull(); + }, + }) + ).run(); + expect(during.fullPull).toBe(true); + expect(during.models).toBe(""); + expect(during.elements).toContain("Content"); + expect(state.fullPull).toBe(false); + expect(state.models).toBe("Post"); + }); + + it.each([ + ["throws", async () => Promise.reject(new Error("network down"))], + ["reports failure", async () => ({ success: false, results: [] })], + [ + "has a failed operation", + async () => ({ + success: true, + results: [ + { guidProcessed: SRC, failed: [{ operation: "downloadAllModels", error: "boom" }] }, + { guidProcessed: TGT, failed: [] }, + ], + }), + ], + ["misses an instance", async () => ({ success: true, results: [{ guidProcessed: SRC, failed: [] }] })], + ])("exits 2 when the pull %s", async (_label, pull) => { + const report = await new Verify(options(), deps({ pull: pull as any })).run(); + expect(report.exitCode).toBe(EXIT_CANNOT_VERIFY); + expect(report.error).toMatch(/Pull failed/); + }); +}); + +describe("runVerifyCommand", () => { + let cwd: string; + + beforeEach(() => { + cwd = process.cwd(); + process.chdir(root); + }); + + afterEach(() => process.chdir(cwd)); + + const passingReport = { exitCode: 0 } as VerifyReport; + const okAuth = () => ({ init: async () => true, validateCommand: async () => true }); + + it("primes state from argv and hands the verify options over", async () => { + let received: VerifyOptions | undefined; + const code = await runVerifyCommand( + { + sourceGuid: SRC, + targetGuid: TGT, + locales: "en-us,fr-ca", + models: "Post", + jsonSummary: "out.json", + channel: "Mobile", + }, + { + createAuth: okAuth, + createVerify: (opts) => { + received = opts; + return { run: async () => passingReport }; + }, + log: () => {}, + } + ); + expect(code).toBe(0); + expect(received).toEqual({ + sourceGuid: SRC, + targetGuid: TGT, + locales: ["en-us", "fr-ca"], + channel: "Mobile", + models: "Post", + pages: "", + containers: "", + jsonSummary: "out.json", + rootPath: "agility-files", + }); + }); + + it("leaves the channel unset when --channel is not passed (every channel is verified)", async () => { + let received: VerifyOptions | undefined; + await runVerifyCommand( + { sourceGuid: SRC, targetGuid: TGT, locales: "en-us" }, + { + createAuth: okAuth, + createVerify: (opts) => { + received = opts; + return { run: async () => passingReport }; + }, + log: () => {}, + } + ); + expect(received?.channel).toBeUndefined(); + }); + + it("falls back to AGILITY_GUID / AGILITY_TARGET_GUID from .env", async () => { + fs.writeFileSync(path.join(root, ".env"), `AGILITY_GUID=${SRC}\nAGILITY_TARGET_GUID=${TGT}\n`); + let received: VerifyOptions | undefined; + await runVerifyCommand( + { locales: "en-us" }, + { + createAuth: okAuth, + createVerify: (opts) => { + received = opts; + return { run: async () => passingReport }; + }, + log: () => {}, + } + ); + expect(received).toMatchObject({ sourceGuid: SRC, targetGuid: TGT }); + }); + + it("returns the report's exit code", async () => { + const code = await runVerifyCommand( + { sourceGuid: SRC, targetGuid: TGT, locales: "en-us" }, + { + createAuth: okAuth, + createVerify: () => ({ run: async () => ({ exitCode: 1 }) as VerifyReport }), + log: () => {}, + } + ); + expect(code).toBe(1); + }); + + it.each([ + [ + "an authentication error", + { sourceGuid: SRC, targetGuid: TGT }, + () => ({ + init: async () => { + throw new Error("ECONNRESET"); + }, + validateCommand: async () => true, + }), + ], + ["missing guids", {}, okAuth], + ["identical guids", { sourceGuid: SRC, targetGuid: SRC }, okAuth], + [ + "failed authentication", + { sourceGuid: SRC, targetGuid: TGT }, + () => ({ init: async () => false, validateCommand: async () => true }), + ], + [ + "failed validation", + { sourceGuid: SRC, targetGuid: TGT }, + () => ({ init: async () => true, validateCommand: async () => false }), + ], + ])("exits 2 with a report on %s", async (_label, argv: any, createAuth: any) => { + const createVerify = jest.fn(); + const code = await runVerifyCommand( + { ...argv, jsonSummary: "abort.json" }, + { createAuth, createVerify, log: () => {} } + ); + expect(code).toBe(EXIT_CANNOT_VERIFY); + expect(createVerify).not.toHaveBeenCalled(); + expect(JSON.parse(fs.readFileSync(path.join(root, "abort.json"), "utf8"))).toMatchObject({ + verdict: "error", + exitCode: 2, + }); + }); +}); diff --git a/src/core/verify.ts b/src/core/verify.ts new file mode 100644 index 0000000..b70b265 --- /dev/null +++ b/src/core/verify.ts @@ -0,0 +1,571 @@ +/** + * `agility verify` (PROD-2552, PROD-2549): prove a migrated target matches its source. + * + * A sync reports on its own bookkeeping; a customer (Brightstar) had syncs report success while + * the components on their pages were in the wrong order. Verify is the independent check for a + * blue/green cutover: it pulls both instances fresh, then compares them in a neutral form (see + * lib/verify/canonicalize.ts) that shares no code with the push transforms. + * + * Flow: + * 1. Status guard — wait (bounded, 10 minutes) until neither instance has a Fetch API update in + * progress, in both preview and fetch modes. A status error or timeout is exit 2. + * 2. Full pull of both instances. A pull failure is exit 2. Mapping files are never written. + * 3. Re-check status. If an update started or landed during the pull, pull once more; if it + * happens again, exit 2 ("an update ran during the pull, re-run"). + * 4. Per-locale checks, aggregate, console report + JSON report, exit code. + * + * Exit codes: 0 pass, 1 differences found, 2 could not verify. + */ +import * as fs from "fs"; +import ansiColors from "ansi-colors"; +import { Auth } from "./auth"; +import { Pull } from "./pull"; +import { state, setState, resetState, primeFromEnv, getPageCmsLink, getContentCmsLink } from "./state"; +import { normalizeArgv } from "./arg-normalizer"; +import { FetchApiStatus, FetchApiSyncMode, getFetchApiStatus } from "../lib/shared/get-fetch-api-status"; +import { sleep as defaultSleep } from "../lib/shared/sleep"; +import { ModelFilterOptions } from "../lib/pushers/guid-data-loader"; +import { resolvePageSyncScope } from "../lib/pages/resolve-page-sync-scope"; +import { parsePageSelectors } from "../lib/pages/page-scope"; +import { resolveContainerSyncScope } from "../lib/containers/resolve-container-sync-scope"; +import { parseContainerSelectors } from "../lib/containers/container-scope"; +import { + buildMappingIndex, + createFileMappingReader, + mappingPairDirectory, + MappingIndex, +} from "../lib/verify/mapping-index"; +import { + createVerifyContext, + findUnreadableCacheFiles, + loadSideEntities, + SideEntities, +} from "../lib/verify/verify-context"; +import { checkInstanceIntegrity, checkLocaleIntegrity } from "../lib/verify/check-mapping-integrity"; +import { checkStructure } from "../lib/verify/check-structure"; +import { checkContent } from "../lib/verify/check-content"; +import { checkPages } from "../lib/verify/check-page-zones"; +import { checkSitemap } from "../lib/verify/check-sitemap"; +import { + buildErrorReport, + buildVerifyReport, + defaultReportPath, + emptyRunInfo, + EXIT_CANNOT_VERIFY, + Finding, + FindingCollector, + printVerifyReport, + StatusSnapshot, + VerifyOptionsSummary, + VerifyReport, + VerifyRunInfo, + writeVerifyReport, +} from "../lib/verify/verify-report"; +import * as path from "path"; + +/** How long the status guard waits for an in-progress update before giving up. */ +export const STATUS_TIMEOUT_MS = 10 * 60 * 1000; +/** How often the status guard polls while an update is in progress. */ +export const STATUS_POLL_MS = 5000; + +const STATUS_MODES: FetchApiSyncMode[] = ["preview", "fetch"]; + +/** A condition under which verify cannot produce a verdict — always exit 2. */ +export class VerifyAbort extends Error {} + +export interface VerifyOptions { + sourceGuid: string; + targetGuid: string; + locales: string[]; + /** Restrict sitemap checks to this channel; undefined = every channel. */ + channel?: string; + models: string; + pages: string; + containers: string; + /** Report path; empty = agility-files/logs/verify-{src}-{tgt}-{ts}.json. */ + jsonSummary: string; + rootPath: string; +} + +export interface VerifyDeps { + getStatus: (guid: string, mode: FetchApiSyncMode) => Promise; + /** Full pull of state.sourceGuid and state.targetGuid. */ + pull: () => Promise<{ success: boolean; results: any[] }>; + sleep: (ms: number) => Promise; + now: () => number; + log: (line: string) => void; + /** Load one side of one locale from the pulled cache. */ + loadSide: (guid: string, locale: string, filter?: ModelFilterOptions) => Promise; + statusTimeoutMs: number; + statusPollMs: number; + /** Print the console report (default true). */ + print: boolean; +} + +const defaultDeps: VerifyDeps = { + getStatus: (guid, mode) => getFetchApiStatus(guid, mode, false), + pull: () => new Pull().pullInstances(true), + sleep: (ms) => defaultSleep(ms), + now: () => Date.now(), + log: (line) => console.log(line), + loadSide: (guid, locale, filter) => loadSideEntities(guid, locale, filter), + statusTimeoutMs: STATUS_TIMEOUT_MS, + statusPollMs: STATUS_POLL_MS, + print: true, +}; + +function snapshot(guid: string, mode: string, status: FetchApiStatus, at: number): StatusSnapshot { + return { + guid, + mode, + checkedAt: new Date(at).toISOString(), + inProgress: status?.inProgress === true, + lastContentVersionID: status?.lastContentVersionID ?? null, + maxChangeDate: status?.maxChangeDate ?? null, + }; +} + +/** Snapshots whose content version or change date moved, or that are now in progress. */ +export function changedSnapshots(before: StatusSnapshot[], after: StatusSnapshot[]): StatusSnapshot[] { + return after.filter((a) => { + const b = before.find((x) => x.guid === a.guid && x.mode === a.mode); + return ( + a.inProgress || + !b || + b.lastContentVersionID !== a.lastContentVersionID || + (b.maxChangeDate ?? null) !== (a.maxChangeDate ?? null) + ); + }); +} + +export class Verify { + private readonly options: VerifyOptions; + private readonly deps: VerifyDeps; + + constructor(options: VerifyOptions, deps: Partial = {}) { + this.options = options; + this.deps = { ...defaultDeps, ...deps }; + } + + get optionsSummary(): VerifyOptionsSummary { + return { + locales: this.options.locales.slice(), + channel: this.options.channel ?? "", + models: this.options.models, + pages: this.options.pages, + containers: this.options.containers, + }; + } + + /** Run the whole command. Never throws and never exits the process: the exit code is on the report. */ + async run(): Promise { + const startedAt = new Date(this.deps.now()); + const run = emptyRunInfo(startedAt); + let report: VerifyReport; + + try { + await this.guardedPull(run); + const compareStarted = this.deps.now(); + const collector = await this.compare(); + run.compareElapsedMs = this.deps.now() - compareStarted; + this.finishRun(run, startedAt); + report = buildVerifyReport({ + source: this.options.sourceGuid, + target: this.options.targetGuid, + options: this.optionsSummary, + collector, + run, + }); + } catch (error: any) { + this.finishRun(run, startedAt); + const message = + error instanceof VerifyAbort ? error.message : `Unexpected error while verifying: ${error?.message || error}`; + report = buildErrorReport({ + source: this.options.sourceGuid, + target: this.options.targetGuid, + options: this.optionsSummary, + error: message, + run, + }); + } + + this.emit(report, startedAt); + return report; + } + + private finishRun(run: VerifyRunInfo, startedAt: Date): void { + const finished = this.deps.now(); + run.finishedAt = new Date(finished).toISOString(); + run.elapsedMs = finished - startedAt.getTime(); + } + + private emit(report: VerifyReport, startedAt: Date): void { + const { sourceGuid, targetGuid } = this.options; + if (this.deps.print) { + printVerifyReport(report, { + log: this.deps.log, + linkFor: (f) => cmsLinkFor(f, sourceGuid, targetGuid), + }); + const pull = report.run.pull; + if (pull.attempts > 0) { + this.deps.log( + ansiColors.gray( + `Pull: ${pull.attempts} attempt(s), ${pull.elapsedMs.map((ms) => `${(ms / 1000).toFixed(1)}s`).join(" + ")}; ` + + `compare: ${(report.run.compareElapsedMs / 1000).toFixed(1)}s; total: ${(report.run.elapsedMs / 1000).toFixed(1)}s` + ) + ); + } + } + const reportPath = + this.options.jsonSummary || defaultReportPath(this.options.rootPath, sourceGuid, targetGuid, startedAt); + const written = writeVerifyReport(reportPath, report); + if (this.deps.print) { + this.deps.log( + written.written + ? ansiColors.gray(`Report: ${path.resolve(reportPath)}`) + : ansiColors.yellow(`⚠️ Could not write the verify report to ${reportPath}: ${written.error}`) + ); + } + } + + // ─── status guard + pull ───────────────────────────────────────────────── + + private guids(): string[] { + return [this.options.sourceGuid, this.options.targetGuid]; + } + + private async checkStatus(guid: string, mode: FetchApiSyncMode): Promise { + try { + const status = await this.deps.getStatus(guid, mode); + if (!status || typeof status !== "object") throw new Error("empty response"); + return status; + } catch (error: any) { + throw new VerifyAbort(`Could not check the Fetch API status of ${guid} (${mode}): ${error?.message || error}`); + } + } + + /** Wait until no instance has an update in progress, in either mode. Bounded by statusTimeoutMs. */ + async waitForIdle(): Promise { + const deadline = this.deps.now() + this.deps.statusTimeoutMs; + const snapshots: StatusSnapshot[] = []; + for (const guid of this.guids()) { + for (const mode of STATUS_MODES) { + let status = await this.checkStatus(guid, mode); + let announced = false; + while (status.inProgress) { + if (this.deps.now() >= deadline) { + throw new VerifyAbort( + `Timed out after ${Math.round(this.deps.statusTimeoutMs / 60000)} minute(s) waiting for an in-progress ` + + `Fetch API update on ${guid} (${mode}) to finish` + ); + } + if (!announced) { + this.deps.log( + ansiColors.gray(`⏳ ${guid} (${mode}): an update is in progress, waiting for it to finish...`) + ); + announced = true; + } + await this.deps.sleep(this.deps.statusPollMs); + status = await this.checkStatus(guid, mode); + } + snapshots.push(snapshot(guid, mode, status, this.deps.now())); + } + } + return snapshots; + } + + private async currentStatus(): Promise { + const snapshots: StatusSnapshot[] = []; + for (const guid of this.guids()) { + for (const mode of STATUS_MODES) { + snapshots.push(snapshot(guid, mode, await this.checkStatus(guid, mode), this.deps.now())); + } + } + return snapshots; + } + + async guardedPull(run: VerifyRunInfo): Promise { + for (let attempt = 1; attempt <= 2; attempt++) { + const before = await this.waitForIdle(); + run.status.before = before; + + const pullStarted = this.deps.now(); + await this.pullOnce(); + run.pull.attempts = attempt; + run.pull.elapsedMs.push(this.deps.now() - pullStarted); + + const after = await this.currentStatus(); + run.status.after = after; + + const changed = changedSnapshots(before, after); + if (changed.length === 0) return; + + const which = changed.map((s) => `${s.guid} (${s.mode})`).join(", "); + if (attempt === 2) { + throw new VerifyAbort( + `An update ran during the pull again (${which}); re-run verify once the instances are idle` + ); + } + this.deps.log(ansiColors.yellow(`⚠️ An update ran during the pull (${which}); pulling once more`)); + } + } + + /** + * One full pull of both instances. The scope flags are cleared for the pull's duration: a scoped + * verify still needs complete data (the `--models` filter would otherwise pull models only). + */ + private async pullOnce(): Promise { + const saved = { + fullPull: state.fullPull, + models: state.models, + modelsWithDeps: state.modelsWithDeps, + pages: state.pages, + containers: state.containers, + elements: state.elements, + }; + state.fullPull = true; + state.models = ""; + state.modelsWithDeps = ""; + state.pages = ""; + state.containers = ""; + state.elements = "Models,Galleries,Assets,Containers,Content,Templates,Pages,Sitemaps,UrlRedirections"; + + let result: { success: boolean; results: any[] }; + try { + this.deps.log(ansiColors.cyan(`⬇️ Full pull of ${this.guids().join(" and ")}...`)); + result = await this.deps.pull(); + } catch (error: any) { + throw new VerifyAbort(`Pull failed: ${error?.message || error}`); + } finally { + Object.assign(state, saved); + } + + const results = Array.isArray(result?.results) ? result.results : []; + const pulled = new Set(results.map((r: any) => r?.guidProcessed).filter(Boolean)); + const missing = this.guids().filter((g) => !pulled.has(g)); + const failures = results.flatMap((r: any) => + (r?.failed || []).map((f: any) => `${r.guidProcessed}: ${f.operation} — ${f.error}`) + ); + if (!result?.success || failures.length > 0 || missing.length > 0) { + const detail = [...failures, ...missing.map((g) => `${g}: no pull result`)].join("; "); + throw new VerifyAbort(`Pull failed${detail ? `: ${detail}` : ""}`); + } + } + + // ─── comparison ────────────────────────────────────────────────────────── + + private resolveFilter(): Promise | ModelFilterOptions | undefined { + const { sourceGuid, targetGuid, locales, models, pages, containers } = this.options; + const used = [models && "--models", pages && "--pages", containers && "--containers"].filter(Boolean); + if (used.length > 1) { + throw new VerifyAbort(`${used.join(", ")} cannot be combined — pass one scope at a time`); + } + if (pages) { + // Verify does not care whether a page's ancestors were synced: it reports what is there. + const pageScope = resolvePageSyncScope({ + sourceGuid, + targetGuid, + locales, + selectors: parsePageSelectors(pages), + isAncestorMapped: () => true, + }); + return { pageScope }; + } + if (containers) { + return resolveContainerSyncScope({ + sourceGuid, + targetGuid, + locales, + selectors: parseContainerSelectors(containers), + loadEntities: async (guid, locale) => (await this.deps.loadSide(guid, locale)) as any, + }).then((containerScope) => ({ containerScope })); + } + if (models) { + const names = models + .split(",") + .map((m) => m.trim()) + .filter(Boolean); + return names.length > 0 ? { models: names } : undefined; + } + return undefined; + } + + private loadMappings(): MappingIndex { + const { rootPath, sourceGuid, targetGuid, locales } = this.options; + const pairDir = mappingPairDirectory(rootPath, sourceGuid, targetGuid); + if (!fs.existsSync(pairDir)) { + throw new VerifyAbort( + `No mapping files for ${sourceGuid} → ${targetGuid} (expected ${pairDir}). Run a sync first.` + ); + } + const missingLocales = locales.filter((locale) => !fs.existsSync(path.join(pairDir, locale))); + if (missingLocales.length > 0) { + throw new VerifyAbort( + `No content/page mappings for locale(s) ${missingLocales.join(", ")} in ${pairDir}. Sync those locales first, or pass --locales.` + ); + } + try { + return buildMappingIndex(createFileMappingReader(rootPath, sourceGuid, targetGuid), locales); + } catch (error: any) { + throw new VerifyAbort(`Could not read the mapping files in ${pairDir}: ${error?.message || error}`); + } + } + + async compare(): Promise { + const { sourceGuid, targetGuid, locales, rootPath, channel } = this.options; + if (locales.length === 0) throw new VerifyAbort("No locales to verify"); + + const mappings = this.loadMappings(); + + const unreadable = [ + ...findUnreadableCacheFiles(rootPath, sourceGuid, locales), + ...findUnreadableCacheFiles(rootPath, targetGuid, locales), + ]; + if (unreadable.length > 0) { + throw new VerifyAbort( + `The pulled cache is unreadable (${unreadable.length} problem(s)): ${unreadable.slice(0, 5).join("; ")}` + ); + } + + let filter: ModelFilterOptions | undefined; + try { + filter = await this.resolveFilter(); + } catch (error: any) { + if (error instanceof VerifyAbort) throw error; + throw new VerifyAbort(error?.message || String(error)); + } + + const collector = new FindingCollector(); + for (let i = 0; i < locales.length; i++) { + const locale = locales[i]; + this.deps.log(ansiColors.cyan(`🔎 Comparing ${locale}...`)); + const source = await this.deps.loadSide(sourceGuid, locale, filter); + const target = await this.deps.loadSide(targetGuid, locale); + const ctx = createVerifyContext({ + sourceGuid, + targetGuid, + locale, + mappings, + source, + target, + scoped: !!filter, + channels: channel ? [channel] : undefined, + }); + + if (i === 0) { + // Models, containers, templates, assets and galleries are instance-wide: check them once. + checkStructure(ctx, checkInstanceIntegrity(ctx, collector), collector); + } + const localePairs = checkLocaleIntegrity(ctx, collector); + checkContent(ctx, localePairs.content, collector); + checkPages(ctx, localePairs.pages, collector); + checkSitemap(ctx, collector); + } + return collector; + } +} + +/** A CMS link for a finding: the target entity when there is one, else the source entity. */ +export function cmsLinkFor(f: Finding, sourceGuid: string, targetGuid: string): string | undefined { + if (f.locale === "*") return undefined; + const pick = (): { guid: string; id: number } | undefined => { + if (typeof f.targetID === "number") return { guid: targetGuid, id: f.targetID }; + if (typeof f.sourceID === "number") return { guid: sourceGuid, id: f.sourceID }; + return undefined; + }; + const target = pick(); + if (!target) return undefined; + if (f.entity === "content") return getContentCmsLink(target.guid, f.locale, target.id); + if (f.entity === "page" || f.entity === "zone" || f.entity === "sitemap") { + return getPageCmsLink(target.guid, f.locale, target.id); + } + return undefined; +} + +// ─── command ───────────────────────────────────────────────────────────────── + +export interface VerifyCommandDeps { + createAuth?: () => { init(): Promise; validateCommand(commandType: "verify"): Promise }; + createVerify?: (options: VerifyOptions) => { run(): Promise }; + log?: (line: string) => void; +} + +function optionsFromState(channel: string | undefined): VerifyOptions { + return { + sourceGuid: state.sourceGuid, + targetGuid: state.targetGuid, + locales: (state.guidLocaleMap.get(state.sourceGuid) || state.locale || []).slice(), + channel: channel ? String(channel) : undefined, + models: state.models || "", + pages: state.pages || "", + containers: state.containers || "", + jsonSummary: state.jsonSummary || "", + rootPath: state.rootPath, + }; +} + +/** + * The `verify` command handler: prime state the same way every other command does, authenticate, + * validate read access to both instances, run, and return the exit code (the caller exits). + */ +export async function runVerifyCommand(argv: any, deps: VerifyCommandDeps = {}): Promise { + const log = deps.log ?? ((line: string) => console.log(line)); + resetState(); + argv = normalizeArgv(argv); + + const envPriming = primeFromEnv(); + if (envPriming.hasEnvFile && envPriming.primedValues.length > 0) { + log(ansiColors.cyan(`📄 Found .env file, primed: ${envPriming.primedValues.join(", ")}`)); + } + + const startedAt = new Date(); + // --channel has no default for verify: unset means "every channel". + const explicitChannel: string | undefined = argv.channel ? String(argv.channel) : undefined; + + const abort = (reason: string): number => { + const options = optionsFromState(explicitChannel); + const report = buildErrorReport({ + source: state.sourceGuid, + target: state.targetGuid, + options: new Verify(options).optionsSummary, + error: reason, + run: emptyRunInfo(startedAt), + }); + printVerifyReport(report, { log }); + const reportPath = + options.jsonSummary || + defaultReportPath(state.rootPath, state.sourceGuid || "unknown", state.targetGuid || "unknown", startedAt); + writeVerifyReport(reportPath, report); + return EXIT_CANNOT_VERIFY; + }; + + try { + setState(argv); + } catch (error: any) { + return abort(error?.message || String(error)); + } + + if (!state.sourceGuid || !state.targetGuid) { + return abort("verify requires both --sourceGuid and --targetGuid (or AGILITY_GUID / AGILITY_TARGET_GUID in .env)"); + } + if (state.sourceGuid === state.targetGuid) { + return abort("verify requires two different instances (sourceGuid equals targetGuid)"); + } + + const auth = deps.createAuth ? deps.createAuth() : new Auth(); + try { + if (!(await auth.init())) { + return abort("Authentication failed — could not resolve API keys for the requested instances"); + } + if (!(await auth.validateCommand("verify"))) { + return abort("Command validation failed — no read access to both instances, or a required setting is missing"); + } + } catch (error: any) { + return abort(`Authentication failed: ${error?.message || error}`); + } + + const options = optionsFromState(explicitChannel); + const verify = deps.createVerify ? deps.createVerify(options) : new Verify(options); + const report = await verify.run(); + return report.exitCode; +} diff --git a/src/index.ts b/src/index.ts index 43db5cf..e461834 100644 --- a/src/index.ts +++ b/src/index.ts @@ -28,11 +28,13 @@ import { primeFromEnv, systemArgs, workflowArgs, + verifyArgs, normalizeProcessArgs, normalizeArgv, } from "./core"; import { Pull } from "./core/pull"; import { Push } from "./core/push"; +import { runVerifyCommand } from "./core/verify"; import { WorkflowOperation } from "./lib/workflows"; import { initializeLogger, getLogger, finalizeLogger } from "./core/state"; @@ -57,6 +59,9 @@ yargs.command({ console.log( colors.white(" reverse-sync - Sync the target back to the source, reusing the original mapping files") ); + console.log( + colors.white(" verify - Compare a synced target against its source and report every difference") + ); console.log( colors.white(" workflowOperation - Perform workflow operations (publish, unpublish, approve, decline)") ); @@ -216,6 +221,21 @@ yargs.command({ }, }); +// Verify (PROD-2552): full pull of both instances, then an independent comparison of the target +// against the source. Exit 0 = match, 1 = differences found, 2 = could not verify. +yargs.command({ + command: "verify", + describe: + "Compare a synced target instance against its source (fresh full pull of both) and report every difference. Exit code 0 = match, 1 = differences found, 2 = could not verify. Never writes to either instance or to the mapping files.", + builder: { + ...verifyArgs, + }, + handler: async function (argv) { + const exitCode = await runVerifyCommand(argv); + process.exit(exitCode); + }, +}); + /** * Shared handler for push / sync / reverse-sync. * diff --git a/src/lib/pushers/content-pusher/util/tests/validate-required-fields.test.ts b/src/lib/pushers/content-pusher/util/tests/validate-required-fields.test.ts new file mode 100644 index 0000000..e30e35d --- /dev/null +++ b/src/lib/pushers/content-pusher/util/tests/validate-required-fields.test.ts @@ -0,0 +1,78 @@ +import { validateContentItemAgainstModel } from "../validate-required-fields"; + +const field = (name: string, type: string, required: string = "True") => ({ + name, + type, + settings: { Required: required }, +}); + +describe("validateContentItemAgainstModel", () => { + it("reports nothing without a model or fields", () => { + expect(validateContentItemAgainstModel({ a: 1 }, null)).toEqual({ missing: [], warnings: [] }); + expect(validateContentItemAgainstModel(null, { fields: [field("Title", "Text")] })).toEqual({ + missing: [], + warnings: [], + }); + }); + + it("reports a required scalar field that is absent or null", () => { + const model = { + fields: [ + field("Title", "Text"), + field("Body", "HTML"), + field("Count", "Integer"), + field("When", "Date"), + field("Pick", "DropdownList"), + ], + }; + const result = validateContentItemAgainstModel({ title: "x", body: null }, model); + expect(result.missing).toEqual([ + { name: "Body", type: "HTML" }, + { name: "Count", type: "Integer" }, + { name: "When", type: "Date" }, + { name: "Pick", type: "DropdownList" }, + ]); + }); + + it("covers LongText, Html and Decimal too", () => { + const model = { fields: [field("A", "LongText"), field("B", "Html"), field("C", "Decimal")] }; + expect(validateContentItemAgainstModel({}, model).missing.map((m) => m.name)).toEqual(["A", "B", "C"]); + }); + + it("matches field keys case-insensitively", () => { + expect(validateContentItemAgainstModel({ TITLE: "x" }, { fields: [field("title", "Text")] }).missing).toEqual([]); + }); + + it("reads Required case-insensitively and ignores non-required fields", () => { + const model = { fields: [field("A", "Text", "true"), field("B", "Text", "False"), { name: "C", type: "Text" }] }; + expect(validateContentItemAgainstModel({}, model).missing).toEqual([{ name: "A", type: "Text" }]); + }); + + it("skips Tab fields", () => { + expect(validateContentItemAgainstModel({}, { fields: [field("Tab1", "Tab")] })).toEqual({ + missing: [], + warnings: [], + }); + }); + + it("only warns for attachment, gallery, custom and content fields", () => { + const model = { + fields: [ + field("Img", "ImageAttachment"), + field("File", "FileAttachment"), + field("Gallery", "PhotoGallery"), + field("Custom", "CustomField"), + field("Link", "Content"), + ], + }; + const result = validateContentItemAgainstModel({}, model); + expect(result.missing).toEqual([]); + expect(result.warnings).toHaveLength(5); + }); + + it("only warns for an empty string", () => { + const result = validateContentItemAgainstModel({ title: "" }, { fields: [field("Title", "Text")] }); + expect(result.missing).toEqual([]); + expect(result.warnings).toEqual(['Required Text field "Title" is empty']); + }); +}); diff --git a/src/lib/pushers/content-pusher/util/validate-required-fields.ts b/src/lib/pushers/content-pusher/util/validate-required-fields.ts new file mode 100644 index 0000000..5bba02b --- /dev/null +++ b/src/lib/pushers/content-pusher/util/validate-required-fields.ts @@ -0,0 +1,77 @@ +/** + * Check a content item's fields against its model's required-field settings. + * + * A source item that is missing a value for a required scalar field cannot be saved on the target + * as-is (the Management API rejects it), so any difference between that item and its target copy + * is explained by the source, not by the sync. Callers use the `missing` list to classify such + * items rather than reporting them as sync defects. + * + * Rules: + * - No model, or no fields object → nothing to report. + * - `Tab` fields are layout only and never carry a value. + * - `settings.Required` is compared case-insensitively ("True" / "true"). + * - Field keys are matched case-insensitively (the model says "TextBlob", the payload "textblob"). + * - A required scalar field (Text, LongText, HTML/Html, Integer, Decimal, Date, DropdownList) whose + * key is absent, or whose value is null/undefined, is `missing`. + * - A required ImageAttachment / FileAttachment / PhotoGallery / CustomField / Content field with no + * value is only a warning: those are validated differently server-side, or populated later. + * - A required scalar field holding "" is only a warning: the key is present, so the API accepts it. + */ + +export interface RequiredFieldIssue { + name: string; + type: string; +} + +export interface FieldValidationResult { + missing: RequiredFieldIssue[]; + warnings: string[]; +} + +const REQUIRED_SCALAR_TYPES = new Set(["text", "longtext", "html", "integer", "decimal", "date", "dropdownlist"]); + +const WARNING_ONLY_TYPES = new Set(["imageattachment", "fileattachment", "photogallery", "customfield", "content"]); + +function isRequired(field: any): boolean { + const required = field?.settings?.Required ?? field?.settings?.required; + return String(required ?? "").toLowerCase() === "true"; +} + +export function validateContentItemAgainstModel(fields: any, model: any): FieldValidationResult { + const result: FieldValidationResult = { missing: [], warnings: [] }; + if (!model || !Array.isArray(model.fields) || !fields || typeof fields !== "object") { + return result; + } + + const valuesByLowerKey = new Map(); + Object.keys(fields).forEach((key) => valuesByLowerKey.set(key.toLowerCase(), fields[key])); + + for (const field of model.fields) { + const name: string = field?.name; + const type: string = String(field?.type ?? ""); + const lowerType = type.toLowerCase(); + if (!name || lowerType === "tab" || !isRequired(field)) continue; + + const lowerName = name.toLowerCase(); + const hasKey = valuesByLowerKey.has(lowerName); + const value = hasKey ? valuesByLowerKey.get(lowerName) : undefined; + const isAbsent = !hasKey || value === null || value === undefined; + + if (WARNING_ONLY_TYPES.has(lowerType)) { + if (isAbsent || value === "") { + result.warnings.push(`Required ${type} field "${name}" has no value`); + } + continue; + } + + if (!REQUIRED_SCALAR_TYPES.has(lowerType)) continue; + + if (isAbsent) { + result.missing.push({ name, type }); + } else if (value === "") { + result.warnings.push(`Required ${type} field "${name}" is empty`); + } + } + + return result; +} diff --git a/src/lib/verify/canonicalize.ts b/src/lib/verify/canonicalize.ts new file mode 100644 index 0000000..d811a75 --- /dev/null +++ b/src/lib/verify/canonicalize.ts @@ -0,0 +1,462 @@ +/** + * Neutral comparison form for `agility verify` (PROD-2552). + * + * Verify must not reuse the push transforms: a bug in ContentFieldMapper or translateZoneNames + * would then be invisible, because the check would apply the same bug to the expectation. Instead, + * both sides are projected into one neutral form and compared there: + * + * - IDs become TARGET IDs. A source content ID is translated through the item mapping (or becomes + * an `unmapped:` token that can never match); a target ID is used as-is. + * - Linked-content references, in any of their shapes ({contentid}, the scalar form, sortids, a + * companion value field), become an ordered list of target IDs under "@ids"; a full-list link + * becomes the target container reference name under "@list". + * - Asset attachments compare on url + label only (size/dimensions are re-derived by the target). + * - Asset URLs anywhere in a string are rewritten source→target (exact URL first, then container + * prefix), and every known alias of a target container (edge/origin/custom CDN host) collapses + * onto one canonical prefix, so the same file reached through two hosts compares equal. + * - null, "" and a missing key are equal; CRLF equals LF; keys compare case-insensitively; + * numbers and numeric strings compare equal, as do booleans and "true"/"false". + */ +import { getLinkedContentCompanionFields } from "../content/linked-content-companion-fields"; +import { AssetPrefixPair, stripTrailingSlash } from "./mapping-index"; + +export type Side = "source" | "target"; + +// ─── asset URLs ────────────────────────────────────────────────────────────── + +const URL_TAIL = `(?:/[^\\s"'<>()]*)?`; +const URL_END = `(?=$|[/\\s"'<>()])`; + +function escapeRegExp(text: string): string { + return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function safeDecode(url: string): string { + try { + return decodeURI(url); + } catch { + return url; + } +} + +/** One regex matching any of `prefixes` (longest first) followed by the rest of a URL. */ +function prefixRegex(prefixes: string[]): RegExp | null { + if (prefixes.length === 0) return null; + const alternation = prefixes + .slice() + .sort((a, b) => b.length - a.length) + .map(escapeRegExp) + .join("|"); + return new RegExp(`(?:${alternation})${URL_END}${URL_TAIL}`, "g"); +} + +function containerName(prefix: string): string { + const trimmed = stripTrailingSlash(prefix); + return trimmed.slice(trimmed.lastIndexOf("/") + 1).toLowerCase(); +} + +export interface AssetUrlTranslatorInput { + /** Exact URL source → target (mapping records, plus each mapped asset's edge/origin URLs). */ + exact: Map; + /** Container prefix pairs source → target. */ + prefixPairs: AssetPrefixPair[]; + /** Every container prefix known to belong to the SOURCE instance. */ + sourcePrefixes: string[]; + /** Every container prefix known to belong to the TARGET instance (edge, origin, custom host). */ + targetPrefixes: string[]; +} + +export class AssetUrlTranslator { + private readonly exact: Map; + private readonly prefixPairs: AssetPrefixPair[]; + private readonly sourceRegex: RegExp | null; + private readonly detectRegex: RegExp | null; + private readonly aliasRegex: RegExp | null; + private readonly aliasToCanonical = new Map(); + private readonly aliasesLongestFirst: string[]; + + constructor(input: AssetUrlTranslatorInput) { + this.exact = new Map(); + input.exact.forEach((target, source) => { + this.exact.set(source, target); + const decoded = safeDecode(source); + if (!this.exact.has(decoded)) this.exact.set(decoded, target); + }); + this.prefixPairs = input.prefixPairs + .map((p) => ({ source: stripTrailingSlash(p.source), target: stripTrailingSlash(p.target) })) + .sort((a, b) => b.source.length - a.source.length); + + const targetPrefixes = unique(input.targetPrefixes.map(stripTrailingSlash).filter(Boolean)); + const targetSet = new Set(targetPrefixes); + const sourcePrefixes = unique( + [...input.sourcePrefixes, ...this.prefixPairs.map((p) => p.source)].map(stripTrailingSlash).filter(Boolean) + ).filter((p) => !targetSet.has(p)); + + // Group target aliases by container name; the lexically smallest alias is the canonical one. + const groups = new Map(); + targetPrefixes.forEach((prefix) => { + const name = containerName(prefix); + if (!groups.has(name)) groups.set(name, []); + (groups.get(name) as string[]).push(prefix); + }); + groups.forEach((aliases) => { + const canonical = aliases.slice().sort()[0]; + aliases.forEach((alias) => this.aliasToCanonical.set(alias, canonical)); + }); + this.aliasesLongestFirst = Array.from(this.aliasToCanonical.keys()).sort((a, b) => b.length - a.length); + + // Exact source URLs without a known container prefix (legacy mapping records) still need to + // be found inside rich text, so they join the source alternation. + const exactOnly = Array.from(this.exact.keys()).filter( + (url) => !sourcePrefixes.some((p) => url.startsWith(p + "/")) + ); + this.sourceRegex = prefixRegex([...sourcePrefixes, ...exactOnly]); + this.detectRegex = prefixRegex(sourcePrefixes); + this.aliasRegex = prefixRegex(this.aliasesLongestFirst); + } + + /** Canonical form of a URL that is already a target URL. */ + canonicalTargetUrl(url: string): string { + const decoded = safeDecode(url); + for (const alias of this.aliasesLongestFirst) { + if (decoded === alias || decoded.startsWith(alias + "/")) { + return (this.aliasToCanonical.get(alias) as string) + decoded.slice(alias.length); + } + } + return decoded; + } + + /** Translate a source URL to the target URL it should have become, in canonical form. */ + translateSourceUrl(url: string): string { + const exact = this.exact.get(url) ?? this.exact.get(safeDecode(url)); + if (exact) return this.canonicalTargetUrl(exact); + const decoded = safeDecode(url); + for (const pair of this.prefixPairs) { + if (decoded === pair.source || decoded.startsWith(pair.source + "/")) { + return this.canonicalTargetUrl(pair.target + decoded.slice(pair.source.length)); + } + } + return this.canonicalTargetUrl(decoded); + } + + /** Rewrite every source asset URL inside a string, then canonicalize target aliases. */ + rewriteSourceText(text: string): string { + const rewritten = this.sourceRegex ? text.replace(this.sourceRegex, (url) => this.translateSourceUrl(url)) : text; + return this.canonicalTargetText(rewritten); + } + + canonicalTargetText(text: string): string { + return this.aliasRegex ? text.replace(this.aliasRegex, (url) => this.canonicalTargetUrl(url)) : text; + } + + /** Source-instance asset URLs still present in a (target) string. */ + findSourceUrls(text: string): string[] { + if (!this.detectRegex || typeof text !== "string") return []; + return Array.from(new Set(text.match(this.detectRegex) ?? [])); + } +} + +function unique(values: T[]): T[] { + return Array.from(new Set(values)); +} + +/** + * Build the translator from the mapping index's asset records plus both sides' asset lists. + * + * The pulled content does not always use the host the mapping recorded (a mapping can carry a + * custom CDN host while content uses the edge host), so each mapped asset contributes its edge and + * origin URLs as exact pairs, and its container's edge/origin prefixes as prefix pairs. + */ +export function buildAssetUrlTranslator(input: { + mappingUrls: Map; + mappingPrefixes: AssetPrefixPair[]; + assetIdMap: { target(sourceMediaID: number): number | undefined }; + sourceAssets: any[]; + targetAssets: any[]; +}): AssetUrlTranslator { + const exact = new Map(input.mappingUrls); + const prefixPairs: AssetPrefixPair[] = input.mappingPrefixes.slice(); + const pairKeys = new Set(prefixPairs.map((p) => `${p.source}\u0000${p.target}`)); + const addPair = (source: any, target: any) => { + if (typeof source !== "string" || typeof target !== "string" || !source || !target) return; + const key = `${stripTrailingSlash(source)}\u0000${stripTrailingSlash(target)}`; + if (pairKeys.has(key)) return; + pairKeys.add(key); + prefixPairs.push({ source, target }); + }; + + const targetById = new Map(); + input.targetAssets.forEach((a) => targetById.set(a?.mediaID, a)); + + const sourcePrefixes = new Set(); + const targetPrefixes = new Set(); + input.mappingPrefixes.forEach((p) => { + sourcePrefixes.add(p.source); + targetPrefixes.add(p.target); + }); + input.sourceAssets.forEach((a) => { + if (a?.containerEdgeUrl) sourcePrefixes.add(a.containerEdgeUrl); + if (a?.containerOriginUrl) sourcePrefixes.add(a.containerOriginUrl); + }); + input.targetAssets.forEach((a) => { + if (a?.containerEdgeUrl) targetPrefixes.add(a.containerEdgeUrl); + if (a?.containerOriginUrl) targetPrefixes.add(a.containerOriginUrl); + }); + + input.sourceAssets.forEach((s) => { + const targetID = input.assetIdMap.target(s?.mediaID); + const t = targetID !== undefined ? targetById.get(targetID) : undefined; + if (!t) return; + (["edgeUrl", "originUrl", "url"] as const).forEach((key) => { + if (typeof s[key] === "string" && typeof t[key] === "string" && !exact.has(s[key])) exact.set(s[key], t[key]); + }); + addPair(s.containerEdgeUrl, t.containerEdgeUrl); + addPair(s.containerOriginUrl, t.containerOriginUrl); + }); + + return new AssetUrlTranslator({ + exact, + prefixPairs, + sourcePrefixes: Array.from(sourcePrefixes), + targetPrefixes: Array.from(targetPrefixes), + }); +} + +// ─── values ────────────────────────────────────────────────────────────────── + +export interface CanonicalContext { + side: Side; + /** A content ID as it should appear on the target ("unmapped:" when it has no mapping). */ + contentRef(id: number): string; + /** A container reference name as it should appear on the target (lower-cased). */ + containerRef(referenceName: string): string; + /** A gallery (media grouping) ID as it should appear on the target. */ + galleryRef(id: number): string; + /** + * True when a full-list reference points at a list with no items on this side. Linking an empty + * list is the same as linking nothing: the target creates an empty nested list for a grid field + * the source never filled in. + */ + isEmptyList?(referenceName: string): boolean; + assets: AssetUrlTranslator; +} + +const NUMERIC = /^-?(?:0|[1-9]\d*)(?:\.\d+)?$/; +const ID_LIST = /^\s*\d+(?:\s*,\s*\d+)*\s*,?\s*$/; + +function canonicalNumber(text: string): string { + const n = Number(text); + return Number.isSafeInteger(n) || (Number.isFinite(n) && text.includes(".")) ? String(n) : text; +} + +/** Normalize a scalar. Returns undefined for "empty" (null, undefined, ""). */ +export function canonicalScalar(value: any): string | undefined { + if (value === null || value === undefined) return undefined; + if (typeof value === "boolean") return value ? "true" : "false"; + if (typeof value === "number") return Number.isFinite(value) ? canonicalNumber(String(value)) : String(value); + if (typeof value !== "string") return undefined; + const text = value.replace(/\r\n/g, "\n"); + if (text === "") return undefined; + if (NUMERIC.test(text)) return canonicalNumber(text); + const lowered = text.toLowerCase(); + if (lowered === "true" || lowered === "false") return lowered; + return text; +} + +function lowerKeys(obj: Record): Record { + const out: Record = {}; + Object.keys(obj).forEach((key) => { + const lower = key.toLowerCase(); + // A payload with both "Title" and "title" keeps the first non-empty one. + if (!(lower in out) || out[lower] === null || out[lower] === undefined || out[lower] === "") out[lower] = obj[key]; + }); + return out; +} + +function parseIdList(text: string): number[] { + return text + .split(",") + .map((id) => id.trim()) + .filter((id) => /^\d+$/.test(id)) + .map((id) => parseInt(id, 10)) + .filter((id) => id > 0); +} + +function isAssetObject(value: any): boolean { + return !!value && typeof value === "object" && !Array.isArray(value) && typeof value.url === "string"; +} + +function isLinkedRef(lowered: Record): boolean { + return "contentid" in lowered || "sortids" in lowered || "fulllist" in lowered; +} + +function canonicalLinkedRef(lowered: Record, ctx: CanonicalContext): any { + let ids: string[] = []; + const sortids = lowered.sortids; + if (typeof sortids === "string" && sortids.trim()) { + ids = parseIdList(sortids).map((id) => ctx.contentRef(id)); + } else if (typeof sortids === "number" && sortids > 0) { + ids = [ctx.contentRef(sortids)]; + } else { + const contentid = typeof lowered.contentid === "string" ? parseInt(lowered.contentid, 10) : lowered.contentid; + if (typeof contentid === "number" && contentid > 0) ids = [ctx.contentRef(contentid)]; + } + + const fullList = lowered.fulllist === true || String(lowered.fulllist).toLowerCase() === "true"; + const list = + fullList && typeof lowered.referencename === "string" && lowered.referencename + ? ctx.containerRef(lowered.referencename) + : undefined; + + const out: Record = {}; + if (ids.length > 0) out["@ids"] = ids; + if (list && !(ids.length === 0 && ctx.isEmptyList?.(lowered.referencename))) out["@list"] = list; + return Object.keys(out).length > 0 ? out : undefined; +} + +function canonicalAsset(value: any, ctx: CanonicalContext): any { + const url = + ctx.side === "source" ? ctx.assets.translateSourceUrl(value.url) : ctx.assets.canonicalTargetUrl(value.url); + const out: Record = { "@asset": url }; + const label = canonicalScalar(value.label); + if (label !== undefined) out.label = label; + return out; +} + +function canonicalText(text: string, ctx: CanonicalContext): string | undefined { + const scalar = canonicalScalar(text); + if (scalar === undefined || scalar !== text.replace(/\r\n/g, "\n")) return scalar; + return ctx.side === "source" ? ctx.assets.rewriteSourceText(scalar) : ctx.assets.canonicalTargetText(scalar); +} + +/** + * Canonical form of one value. `fieldType` is the model's field type for a top-level field, used + * to recognize the scalar shapes that are only references because the model says so (a Content + * field stored as "123" or "1,2,3", a PhotoGallery stored as a gallery ID). + */ +export function canonicalValue(value: any, ctx: CanonicalContext, fieldType?: string): any { + if (value === null || value === undefined) return undefined; + const type = (fieldType || "").toLowerCase(); + + if (typeof value === "string" || typeof value === "number") { + const text = String(value); + if (type === "content" && ID_LIST.test(text)) { + const ids = parseIdList(text).map((id) => ctx.contentRef(id)); + return ids.length > 0 ? { "@ids": ids } : undefined; + } + if (type === "photogallery" && /^\d+$/.test(text.trim()) && parseInt(text, 10) > 0) { + return { "@gallery": ctx.galleryRef(parseInt(text, 10)) }; + } + return typeof value === "string" ? canonicalText(value, ctx) : canonicalScalar(value); + } + + if (typeof value === "boolean") return canonicalScalar(value); + + if (Array.isArray(value)) { + const items = value.map((item) => { + const canonical = canonicalValue(item, ctx); + return canonical === undefined ? null : canonical; + }); + return items.length > 0 && items.some((i) => i !== null) ? items : undefined; + } + + if (typeof value === "object") { + if (isAssetObject(value)) return canonicalAsset(value, ctx); + const lowered = lowerKeys(value); + if (isLinkedRef(lowered)) return canonicalLinkedRef(lowered, ctx); + if (type === "photogallery" && typeof lowered.galleryid === "number") { + return { "@gallery": ctx.galleryRef(lowered.galleryid) }; + } + const out: Record = {}; + Object.keys(lowered) + .sort() + .forEach((key) => { + const canonical = canonicalValue(lowered[key], ctx); + if (canonical !== undefined) out[key] = canonical; + }); + return Object.keys(out).length > 0 ? out : undefined; + } + + return undefined; +} + +/** + * Canonical form of a content item's fields object, keyed by lower-cased field name. + * + * `model` is the item's model on ITS OWN side: it names the Content-typed fields and their + * companion value fields (LinkeContentDropdownValueField / SortIDFieldName). + */ +export function canonicalFields(fields: any, model: any, ctx: CanonicalContext): Record { + const out: Record = {}; + if (!fields || typeof fields !== "object") return out; + + const typeByField = new Map(); + (Array.isArray(model?.fields) ? model.fields : []).forEach((f: any) => { + if (f?.name) typeByField.set(String(f.name).toLowerCase(), String(f.type || "")); + }); + getLinkedContentCompanionFields(model).forEach(([main, companion]) => { + if (companion.toLowerCase() !== main.toLowerCase()) typeByField.set(companion.toLowerCase(), "Content"); + }); + + const lowered = lowerKeys(fields); + Object.keys(lowered) + .sort() + .forEach((key) => { + const canonical = canonicalValue(lowered[key], ctx, typeByField.get(key)); + if (canonical !== undefined) out[key] = canonical; + }); + return out; +} + +// ─── comparison ────────────────────────────────────────────────────────────── + +/** JSON with sorted keys, so two equal canonical values stringify identically. */ +export function stableStringify(value: any): string { + if (value === undefined) return "undefined"; + if (value === null || typeof value !== "object") return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`; + return `{${Object.keys(value) + .sort() + .map((k) => `${JSON.stringify(k)}:${stableStringify(value[k])}`) + .join(",")}}`; +} + +export function canonicalEquals(a: any, b: any): boolean { + return stableStringify(a) === stableStringify(b); +} + +export interface FieldDifference { + field: string; + expected: any; + actual: any; +} + +/** Top-level keys whose canonical values differ, in key order. */ +export function diffCanonical(expected: Record, actual: Record): FieldDifference[] { + const keys = Array.from(new Set([...Object.keys(expected), ...Object.keys(actual)])).sort(); + const out: FieldDifference[] = []; + keys.forEach((key) => { + if (!canonicalEquals(expected[key], actual[key])) { + out.push({ field: key, ...describeDifference(expected[key], actual[key]) }); + } + }); + return out; +} + +const SNIPPET_CONTEXT = 80; + +/** + * Report-friendly expected/actual: long strings are cut to a window around the first difference, + * so a one-character change in a 20KB rich-text field is readable. + */ +export function describeDifference(expected: any, actual: any): { expected: any; actual: any } { + if (typeof expected === "string" && typeof actual === "string" && (expected.length > 200 || actual.length > 200)) { + let i = 0; + while (i < expected.length && i < actual.length && expected[i] === actual[i]) i++; + const start = Math.max(0, i - SNIPPET_CONTEXT); + const cut = (s: string) => + `${start > 0 ? "…" : ""}${s.slice(start, i + SNIPPET_CONTEXT)}${i + SNIPPET_CONTEXT < s.length ? "…" : ""}`; + return { expected: cut(expected), actual: cut(actual) }; + } + return { expected, actual }; +} diff --git a/src/lib/verify/check-content.ts b/src/lib/verify/check-content.ts new file mode 100644 index 0000000..4067c79 --- /dev/null +++ b/src/lib/verify/check-content.ts @@ -0,0 +1,266 @@ +/** + * Content checks for one locale (PROD-2552). + * + * Per mapped pair: + * - contentModelMismatch the target item is of a different model (through the model mapping) + * - contentContainerMismatch the target item sits in a different container — skipped for items + * placed on a page, whose container name is auto-generated + * - fieldMismatch one per differing field, compared in canonical form + * - sourceAssetUrlInTarget a target field still points at an asset in the SOURCE instance + * - publishStateDifference (warning) the two items are in different workflow states + * Per container / model (unscoped runs only): + * - countMismatch failure when the target has fewer items, warning when it has more + * Per list container: + * - listOrderMismatch the items' order (itemOrder) differs, with both sequences + * + * Ignored: contentID, versionID, modified, and the item's own state (beyond the warning above). + */ +import { Finding, FindingCollector } from "./verify-report"; +import { ComparablePair, contentSourceDefect } from "./check-mapping-integrity"; +import { sourceCanonicalContext, targetCanonicalContext, VerifyContext } from "./verify-context"; +import { canonicalFields, diffCanonical } from "./canonicalize"; + +function lower(value: any): string { + return typeof value === "string" ? value.toLowerCase() : ""; +} + +/** Every string inside a value, with the dotted path to it. */ +function walkStrings(value: any, path: string, visit: (text: string, path: string) => void): void { + if (typeof value === "string") visit(value, path); + else if (Array.isArray(value)) value.forEach((v, i) => walkStrings(v, `${path}[${i}]`, visit)); + else if (value && typeof value === "object") + Object.keys(value).forEach((k) => walkStrings(value[k], `${path}.${k}`, visit)); +} + +export function checkContent(ctx: VerifyContext, pairs: ComparablePair[], collector: FindingCollector): void { + const { locale } = ctx; + const sourceCtx = sourceCanonicalContext(ctx); + const targetCtx = targetCanonicalContext(ctx); + + pairs.forEach(({ sourceID, targetID, source, target }) => { + const findings: Finding[] = []; + const name = source?.properties?.referenceName; + const base = { entity: "content" as const, locale, sourceID, targetID, name }; + + const sourceModelName = lower(source?.properties?.definitionName); + const targetModelName = lower(target?.properties?.definitionName); + const expectedModel = ctx.mappings.modelNames.target(sourceModelName) ?? sourceModelName; + if (expectedModel !== targetModelName) { + findings.push({ + ...base, + category: "contentModelMismatch", + severity: "failure", + path: "definitionName", + expected: expectedModel, + actual: targetModelName, + }); + } + + if (!ctx.source.pageBoundContentIDs.has(sourceID)) { + const sourceContainer = lower(source?.properties?.referenceName); + const expectedContainer = ctx.mappings.containerNames.target(sourceContainer) ?? sourceContainer; + const actualContainer = lower(target?.properties?.referenceName); + if (expectedContainer !== actualContainer) { + findings.push({ + ...base, + category: "contentContainerMismatch", + severity: "failure", + path: "referenceName", + expected: expectedContainer, + actual: actualContainer, + }); + } + } + + const sourceModel = ctx.source.modelsByRef.get(sourceModelName); + const targetModel = ctx.target.modelsByRef.get(targetModelName); + const expectedFields = canonicalFields(source?.fields, sourceModel, sourceCtx); + const actualFields = canonicalFields(target?.fields, targetModel, targetCtx); + diffCanonical(expectedFields, actualFields).forEach((diff) => { + findings.push({ + ...base, + category: "fieldMismatch", + severity: "failure", + path: diff.field, + expected: diff.expected, + actual: diff.actual, + }); + }); + + const leaked = new Map(); + walkStrings(target?.fields, "fields", (text, path) => { + const urls = ctx.assets.findSourceUrls(text); + if (urls.length > 0) leaked.set(path.replace(/^fields\./, ""), urls); + }); + leaked.forEach((urls, path) => { + findings.push({ + ...base, + category: "sourceAssetUrlInTarget", + severity: "failure", + path, + actual: urls, + message: "Target content references an asset URL in the source instance", + }); + }); + + // A difference explained by the source item itself (a required field it lacks) is listed, + // but does not fail the run. + const defect = findings.length > 0 ? contentSourceDefect(source, ctx.source.modelsByRef) : undefined; + findings.forEach((f) => { + if (defect) { + f.severity = "sourceDefect"; + f.message = f.message ? `${f.message} — ${defect}` : defect; + } + collector.add(f); + }); + collector.count(locale, "content", findings.length === 0 ? "matched" : defect ? "known" : "mismatched"); + + const sourceState = source?.properties?.state; + const targetState = target?.properties?.state; + if (sourceState !== undefined && targetState !== undefined && sourceState !== targetState) { + collector.add({ + ...base, + category: "publishStateDifference", + severity: "warning", + path: "state", + expected: sourceState, + actual: targetState, + }); + } + }); + + if (!ctx.scoped) checkCounts(ctx, collector); + checkListOrder(ctx, pairs, collector); +} + +/** Target items whose every mapped source is absent from the source pull. */ +function orphanedTargetIDs(ctx: VerifyContext): Set { + const out = new Set(); + ctx.target.contentById.forEach((_item, targetID) => { + const sources = ctx.localeMappings.content.sources(targetID); + if (sources.length > 0 && !sources.some((s) => ctx.source.contentById.has(s))) out.add(targetID); + }); + return out; +} + +function countBy(items: Iterable<[number, any]>, key: (item: any) => string, skip: Set): Map { + const out = new Map(); + for (const [id, item] of Array.from(items)) { + if (skip.has(id)) continue; + const k = key(item); + if (!k) continue; + out.set(k, (out.get(k) ?? 0) + 1); + } + return out; +} + +function checkCounts(ctx: VerifyContext, collector: FindingCollector): void { + const { locale } = ctx; + const orphaned = orphanedTargetIDs(ctx); + const none = new Set(); + + const groups: Array<{ + entity: "container" | "model"; + key: (item: any) => string; + translate: (name: string) => string; + }> = [ + { + entity: "container", + key: (item) => lower(item?.properties?.referenceName), + translate: (name) => ctx.mappings.containerNames.target(name) ?? name, + }, + { + entity: "model", + key: (item) => lower(item?.properties?.definitionName), + translate: (name) => ctx.mappings.modelNames.target(name) ?? name, + }, + ]; + + groups.forEach(({ entity, key, translate }) => { + const sourceCounts = countBy(ctx.source.contentById.entries(), key, none); + const targetCounts = countBy(ctx.target.contentById.entries(), key, orphaned); + Array.from(sourceCounts.keys()) + .sort() + .forEach((name) => { + const expected = sourceCounts.get(name) as number; + const targetName = translate(name); + const actual = targetCounts.get(targetName) ?? 0; + if (expected === actual) return; + collector.add({ + category: "countMismatch", + severity: actual < expected ? "failure" : "warning", + entity, + locale, + name, + path: "itemCount", + expected, + actual, + message: `${entity === "container" ? "Container" : "Model"} "${name}" has ${expected} item(s) in the source and ${actual} on the target`, + }); + }); + }); +} + +function itemOrder(item: any): number { + const order = item?.properties?.itemOrder; + return typeof order === "number" ? order : Number.MAX_SAFE_INTEGER; +} + +function checkListOrder(ctx: VerifyContext, pairs: ComparablePair[], collector: FindingCollector): void { + const { locale } = ctx; + const bySourceContainer = new Map(); + pairs.forEach((pair) => { + const name = lower(pair.source?.properties?.referenceName); + if (!name) return; + if (!bySourceContainer.has(name)) bySourceContainer.set(name, []); + (bySourceContainer.get(name) as ComparablePair[]).push(pair); + }); + + Array.from(bySourceContainer.keys()) + .sort() + .forEach((name) => { + const group = bySourceContainer.get(name) as ComparablePair[]; + if (group.length < 2) return; + const expectedContainer = ctx.mappings.containerNames.target(name) ?? name; + // Only items that landed in the matching target container take part; a misplaced item is + // already reported as contentContainerMismatch. + const comparable = group.filter((p) => lower(p.target?.properties?.referenceName) === expectedContainer); + if (comparable.length < 2) return; + + const expected = comparable + .slice() + .sort((a, b) => itemOrder(a.source) - itemOrder(b.source) || a.sourceID - b.sourceID); + // Rank by the source's itemOrder value, so items that tie in the source may come in either + // order on the target. Ties in the target's itemOrder are broken by that rank, so only a + // real disagreement is reported. + const sourceRank = new Map(); + let rank = -1; + let previous: number | undefined; + expected.forEach((p) => { + const order = itemOrder(p.source); + if (order !== previous) rank++; + previous = order; + sourceRank.set(p.targetID, rank); + }); + const rankOf = (p: ComparablePair) => sourceRank.get(p.targetID) as number; + const actual = comparable + .slice() + .sort((a, b) => itemOrder(a.target) - itemOrder(b.target) || rankOf(a) - rankOf(b)); + + if (actual.every((p, i) => i === 0 || rankOf(actual[i - 1]) <= rankOf(p))) return; + const expectedIDs = expected.map((p) => p.targetID); + const actualIDs = actual.map((p) => p.targetID); + + collector.add({ + category: "listOrderMismatch", + severity: "failure", + entity: "container", + locale, + name, + path: "itemOrder", + expected: expectedIDs, + actual: actualIDs, + message: "List items are in a different order on the target (target content IDs shown)", + }); + }); +} diff --git a/src/lib/verify/check-mapping-integrity.ts b/src/lib/verify/check-mapping-integrity.ts new file mode 100644 index 0000000..46e7cf5 --- /dev/null +++ b/src/lib/verify/check-mapping-integrity.ts @@ -0,0 +1,299 @@ +/** + * Mapping integrity (PROD-2552): does every source entity have a live counterpart on the target, + * and does every target entity trace back to a live source? + * + * - unmapped (failure) a source entity with no mapping record + * - danglingMapping (failure) a mapping record whose target entity is not on the target + * - duplicateTarget (failure) two live source entities mapped onto one target entity + * - extraInTarget (warning) a target entity no mapping points at + * - sourceNotInPull (warning) a target entity whose mapped source is absent from the source + * pull (deleted or unpublished there since the sync) + * + * A source entity that cannot be migrated as-is (see isContentSourceDefect) is reported with + * severity sourceDefect instead of failure. Entities that pass are returned as comparable pairs + * for the per-type checks; this check only tallies checked / missing / extra / known. + */ +import { BiMap } from "./mapping-index"; +import { EntityKind, FindingCollector, INSTANCE_LOCALE, Severity } from "./verify-report"; +import { VerifyContext } from "./verify-context"; +import { validateContentItemAgainstModel } from "../pushers/content-pusher/util/validate-required-fields"; + +export interface ComparablePair { + sourceID: number; + targetID: number; + source: any; + target: any; +} + +export interface IntegritySet { + entity: EntityKind; + locale: string; + source: Map; + target: Map; + map: BiMap; + name: (entity: any) => string | undefined; + /** When false (scoped runs), target-side completeness is not checked. */ + checkExtra: boolean; + /** Returns a reason when a source entity's absence on the target is explained by the source. */ + sourceDefect?: (source: any) => string | undefined; +} + +export function checkIntegrity(set: IntegritySet, collector: FindingCollector): ComparablePair[] { + const pairs: ComparablePair[] = []; + const { entity, locale } = set; + + set.source.forEach((source, sourceID) => { + collector.count(locale, entity, "checked"); + const targetID = set.map.target(sourceID); + const defect = targetID !== undefined && set.target.has(targetID) ? undefined : set.sourceDefect?.(source); + const severity: Severity = defect ? "sourceDefect" : "failure"; + + if (targetID === undefined) { + collector.count(locale, entity, defect ? "known" : "missing"); + collector.add({ + category: "unmapped", + severity, + entity, + locale, + sourceID, + targetID: null, + name: set.name(source), + message: defect ? `No mapping — ${defect}` : "Source entity has no mapping to the target", + }); + return; + } + + if (!set.target.has(targetID)) { + collector.count(locale, entity, defect ? "known" : "missing"); + collector.add({ + category: "danglingMapping", + severity, + entity, + locale, + sourceID, + targetID, + name: set.name(source), + message: defect + ? `Mapped target is missing — ${defect}` + : "Mapping points at a target entity that is not on the target (deleted or unpublished)", + }); + return; + } + + pairs.push({ sourceID, targetID, source, target: set.target.get(targetID) }); + }); + + set.map.duplicateTargets().forEach(({ target, sources }) => { + const live = sources.filter((s) => set.source.has(s)); + if (live.length < 2) return; + collector.add({ + category: "duplicateTarget", + severity: "failure", + entity, + locale, + sourceID: live.join(","), + targetID: target, + name: set.target.has(target) ? set.name(set.target.get(target)) : undefined, + expected: "one source per target", + actual: live, + message: `${live.length} source entities are mapped onto the same target entity`, + }); + }); + + if (set.checkExtra) { + set.target.forEach((target, targetID) => { + const sources = set.map.sources(targetID); + if (sources.length === 0) { + collector.count(locale, entity, "extra"); + collector.add({ + category: "extraInTarget", + severity: "warning", + entity, + locale, + sourceID: null, + targetID, + name: set.name(target), + message: "Target entity is not mapped from any source entity", + }); + } else if (!sources.some((s) => set.source.has(s))) { + collector.count(locale, entity, "extra"); + collector.add({ + category: "sourceNotInPull", + severity: "warning", + entity, + locale, + sourceID: sources.join(","), + targetID, + name: set.name(target), + message: "Mapped source entity is not in the source pull (deleted or unpublished in the source)", + }); + } + }); + } + + return pairs; +} + +/** + * Why a source content item cannot be migrated as-is, if it can't: a required scalar field with no + * value, which the target's Management API rejects on save. + */ +export function contentSourceDefect(item: any, sourceModelsByRef: Map): string | undefined { + const model = sourceModelsByRef.get(String(item?.properties?.definitionName || "").toLowerCase()); + const { missing } = validateContentItemAgainstModel(item?.fields, model); + if (missing.length === 0) return undefined; + return `source item is missing required field(s): ${missing.map((m) => `${m.name} (${m.type})`).join(", ")}`; +} + +function idMap(items: T[], key: (item: T) => any): Map { + const out = new Map(); + items.forEach((item) => { + const id = key(item); + if (typeof id === "number" && id > 0 && !out.has(id)) out.set(id, item); + }); + return out; +} + +export interface InstanceIntegrityResult { + models: ComparablePair[]; + containers: ComparablePair[]; + templates: ComparablePair[]; + assets: ComparablePair[]; + galleries: ComparablePair[]; +} + +/** Models, containers, templates, assets and galleries — instance-wide, checked once per run. */ +export function checkInstanceIntegrity(ctx: VerifyContext, collector: FindingCollector): InstanceIntegrityResult { + const s = ctx.source.entities; + const t = ctx.target.entities; + const checkExtra = !ctx.scoped; + const locale = INSTANCE_LOCALE; + return { + models: checkIntegrity( + { + entity: "model", + locale, + source: idMap(s.models, (m: any) => m.id), + target: idMap(t.models, (m: any) => m.id), + map: ctx.mappings.models, + name: (m) => m?.referenceName, + checkExtra, + }, + collector + ), + containers: checkIntegrity( + { + entity: "container", + locale, + source: idMap(s.containers, (c: any) => c.contentViewID), + target: idMap(t.containers, (c: any) => c.contentViewID), + map: ctx.mappings.containers, + name: (c) => c?.referenceName, + checkExtra, + }, + collector + ), + templates: checkIntegrity( + { + entity: "template", + locale, + source: idMap(s.templates, (x: any) => x.pageTemplateID), + target: idMap(t.templates, (x: any) => x.pageTemplateID), + map: ctx.mappings.templates, + name: (x) => x?.pageTemplateName, + checkExtra, + }, + collector + ), + assets: checkIntegrity( + { + entity: "asset", + locale, + source: idMap(s.assets, (a: any) => a.mediaID), + target: idMap(t.assets, (a: any) => a.mediaID), + map: ctx.mappings.assets, + name: (a) => a?.originKey || a?.fileName, + checkExtra, + }, + collector + ), + galleries: checkIntegrity( + { + entity: "gallery", + locale, + source: idMap(s.galleries, (g: any) => g.mediaGroupingID), + target: idMap(t.galleries, (g: any) => g.mediaGroupingID), + map: ctx.mappings.galleries, + name: (g) => g?.name, + checkExtra, + }, + collector + ), + }; +} + +export interface LocaleIntegrityResult { + content: ComparablePair[]; + pages: ComparablePair[]; +} + +/** + * Source items outside any live container (deleted or system container): the sync skips them, so + * they are listed once per container as a source defect instead of one "unmapped" per item. + */ +function reportContentOutsideLiveContainers(ctx: VerifyContext, collector: FindingCollector): void { + const byContainer = new Map(); + ctx.source.outsideLiveContainers.forEach((item) => { + const name = String(item?.properties?.referenceName || "").toLowerCase(); + byContainer.set(name, (byContainer.get(name) ?? 0) + 1); + }); + Array.from(byContainer.keys()) + .sort() + .forEach((name) => { + const count = byContainer.get(name) as number; + collector.count(ctx.locale, "content", "checked", count); + collector.count(ctx.locale, "content", "known", count); + collector.add({ + category: "contentNotInLiveContainer", + severity: "sourceDefect", + entity: "content", + locale: ctx.locale, + name, + actual: count, + message: `${count} source item(s) belong to container "${name}", which is deleted or a system container in the source — not migrated`, + }); + }); +} + +/** Content items and pages for one locale. */ +export function checkLocaleIntegrity(ctx: VerifyContext, collector: FindingCollector): LocaleIntegrityResult { + const checkExtra = !ctx.scoped; + reportContentOutsideLiveContainers(ctx, collector); + return { + content: checkIntegrity( + { + entity: "content", + locale: ctx.locale, + source: ctx.source.contentById, + target: ctx.target.contentById, + map: ctx.localeMappings.content, + name: (c) => c?.properties?.referenceName, + checkExtra, + sourceDefect: (item) => contentSourceDefect(item, ctx.source.modelsByRef), + }, + collector + ), + pages: checkIntegrity( + { + entity: "page", + locale: ctx.locale, + source: ctx.source.pagesById, + target: ctx.target.pagesById, + map: ctx.localeMappings.pages, + name: (p) => p?.name, + checkExtra, + }, + collector + ), + }; +} diff --git a/src/lib/verify/check-page-zones.ts b/src/lib/verify/check-page-zones.ts new file mode 100644 index 0000000..5121bd0 --- /dev/null +++ b/src/lib/verify/check-page-zones.ts @@ -0,0 +1,344 @@ +/** + * Page checks for one locale (PROD-2552) — page fields, and the components in each zone. + * + * The defect this exists for: a sync reported success while the components on a page were in the + * wrong order. So zones are compared component by component, in order, and a zone is never paired + * by position: + * + * source zone → target zone via the template section-ID mapping (PROD-2350), else by an exact + * (case-insensitive) zone name, else `unmappedZone`. Two templates that define their zones in a + * different order still pair correctly; a zone that cannot be paired is reported, not guessed. + * + * Each zone is then compared as an ordered list of (module, target content ID): + * - missingComponent expected in this zone, not on the target page at all + * - extraComponent on the target zone, not expected anywhere on the page + * - componentInWrongZone expected in one zone, found in another + * - componentOrderMismatch the shared components are in a different order (both sequences shown) + * A module whose content item is absent from the source pull cannot be migrated: it is listed as + * a sourceDefect and left out of the expectation. + * + * Page fields compared: title, menuText, name, pageType, template (through the template mapping), + * redirectUrl, visible, seo, scripts, securePage. A different workflow state is a warning + * (publishStateDifference), as for content. + */ +import { Finding, FindingCollector } from "./verify-report"; +import { ComparablePair } from "./check-mapping-integrity"; +import { moduleContentID, sourceCanonicalContext, targetCanonicalContext, VerifyContext } from "./verify-context"; +import { canonicalEquals, canonicalValue, describeDifference } from "./canonicalize"; + +function lower(value: any): string { + return typeof value === "string" ? value.toLowerCase() : ""; +} + +export const COMPARED_PAGE_FIELDS = [ + "title", + "menuText", + "name", + "pageType", + "redirectUrl", + "visible", + "seo", + "scripts", + "securePage", +] as const; + +interface ZoneEntry { + key: string; + label: string; +} + +function sections(template: any): any[] { + return Array.isArray(template?.contentSectionDefinitions) ? template.contentSectionDefinitions : []; +} + +/** The target template a source page should be on, by name (lower-case). */ +export function expectedTemplateName(ctx: VerifyContext, sourceTemplateName: any): string { + const name = lower(sourceTemplateName); + if (!name) return ""; + const byName = ctx.mappings.templateNames.target(name); + if (byName) return byName; + const sourceTemplate = ctx.source.templatesByName.get(name); + const targetID = sourceTemplate ? ctx.mappings.templates.target(sourceTemplate.pageTemplateID) : undefined; + const targetTemplate = targetID !== undefined ? ctx.target.templatesById.get(targetID) : undefined; + return targetTemplate ? lower(targetTemplate.pageTemplateName) : name; +} + +/** + * Pair a source zone with a target zone. Section-ID mapping first, then an exact name; never by + * position. Returns the lower-cased target zone name, or undefined when there is no pairing. + */ +export function resolveTargetZone( + ctx: VerifyContext, + sourceZone: string, + sourceTemplate: any, + targetTemplate: any, + targetZoneNames: string[] +): string | undefined { + const zone = lower(sourceZone); + const sourceSection = sections(sourceTemplate).find((s) => lower(s?.pageItemTemplateReferenceName) === zone); + if (sourceSection?.pageItemTemplateID != null) { + const targetSectionID = ctx.mappings.sections.target(sourceSection.pageItemTemplateID); + if (targetSectionID !== undefined) { + const targetSection = sections(targetTemplate).find((s) => s?.pageItemTemplateID === targetSectionID); + if (targetSection?.pageItemTemplateReferenceName) return lower(targetSection.pageItemTemplateReferenceName); + } + } + const candidates = new Set([ + ...sections(targetTemplate).map((s) => lower(s?.pageItemTemplateReferenceName)), + ...targetZoneNames.map(lower), + ]); + return candidates.has(zone) ? zone : undefined; +} + +export function checkPages(ctx: VerifyContext, pairs: ComparablePair[], collector: FindingCollector): void { + pairs.forEach((pair) => { + checkPageFields(ctx, pair, collector); + checkPageZones(ctx, pair, collector); + }); +} + +function checkPageFields(ctx: VerifyContext, pair: ComparablePair, collector: FindingCollector): void { + const { locale } = ctx; + const { sourceID, targetID, source, target } = pair; + const sourceCtx = sourceCanonicalContext(ctx); + const targetCtx = targetCanonicalContext(ctx); + let differs = false; + + const report = (path: string, expected: any, actual: any) => { + differs = true; + collector.add({ + category: "pageFieldMismatch", + severity: "failure", + entity: "page", + locale, + sourceID, + targetID, + name: source?.name, + path, + ...describeDifference(expected, actual), + }); + }; + + COMPARED_PAGE_FIELDS.forEach((field) => { + const expected = canonicalValue(source?.[field], sourceCtx); + const actual = canonicalValue(target?.[field], targetCtx); + if (!canonicalEquals(expected, actual)) report(field, expected, actual); + }); + + const expectedTemplate = expectedTemplateName(ctx, source?.templateName); + const actualTemplate = lower(target?.templateName); + if (expectedTemplate !== actualTemplate) report("template", expectedTemplate || null, actualTemplate || null); + + collector.count(locale, "page", differs ? "mismatched" : "matched"); + + const sourceState = source?.properties?.state; + const targetState = target?.properties?.state; + if (sourceState !== undefined && targetState !== undefined && sourceState !== targetState) { + collector.add({ + category: "publishStateDifference", + severity: "warning", + entity: "page", + locale, + sourceID, + targetID, + name: source?.name, + path: "state", + expected: sourceState, + actual: targetState, + }); + } +} + +function moduleLabel(moduleName: string, id: string): string { + return `${moduleName}#${id}`; +} + +function checkPageZones(ctx: VerifyContext, pair: ComparablePair, collector: FindingCollector): void { + const { locale } = ctx; + const { sourceID, targetID, source, target } = pair; + const base = { entity: "zone" as const, locale, sourceID, targetID, name: source?.name }; + + const sourceTemplate = ctx.source.templatesByName.get(lower(source?.templateName)); + const targetTemplate = ctx.target.templatesByName.get(lower(target?.templateName)); + const targetZones: Record = {}; + Object.keys(target?.zones || {}).forEach((zone) => { + targetZones[lower(zone)] = Array.isArray(target.zones[zone]) ? target.zones[zone] : []; + }); + + const expectedByZone = new Map(); + const unmappedKeys = new Set(); + const findings: Finding[] = []; + const zoneOf = new Map(); + const add = (zone: string, finding: Finding) => { + findings.push(finding); + zoneOf.set(finding, zone); + }; + + Object.keys(source?.zones || {}).forEach((sourceZone) => { + const modules = Array.isArray(source.zones[sourceZone]) ? source.zones[sourceZone] : []; + const entries: ZoneEntry[] = []; + modules.forEach((module: any) => { + const moduleName = ctx.mappings.modelNames.target(lower(module?.module)) ?? lower(module?.module); + const contentID = moduleContentID(module); + if (contentID !== undefined && !ctx.source.contentById.has(contentID)) { + add(lower(sourceZone), { + ...base, + category: "moduleContentMissingInSource", + severity: "sourceDefect", + path: sourceZone, + expected: moduleLabel(lower(module?.module), `source:${contentID}`), + message: `Module ${module?.module} references source content ${contentID}, which is not in the source pull`, + }); + return; + } + const mapped = contentID === undefined ? undefined : ctx.localeMappings.content.target(contentID); + const id = contentID === undefined ? "none" : mapped !== undefined ? String(mapped) : `unmapped:${contentID}`; + entries.push({ key: `${moduleName}|${id}`, label: moduleLabel(moduleName, id) }); + }); + + const targetZone = resolveTargetZone(ctx, sourceZone, sourceTemplate, targetTemplate, Object.keys(targetZones)); + if (!targetZone) { + if (entries.length === 0) return; + entries.forEach((e) => unmappedKeys.add(e.key)); + add(lower(sourceZone), { + ...base, + category: "unmappedZone", + severity: "failure", + path: sourceZone, + expected: entries.map((e) => e.label), + message: `Source zone "${sourceZone}" has no counterpart on the target template (no section mapping, no zone of that name)`, + }); + return; + } + expectedByZone.set(targetZone, [...(expectedByZone.get(targetZone) ?? []), ...entries]); + }); + + const actualByZone = new Map(); + Object.keys(targetZones).forEach((zone) => { + actualByZone.set( + zone, + targetZones[zone].map((module: any) => { + const moduleName = lower(module?.module); + const contentID = moduleContentID(module); + const id = contentID === undefined ? "none" : String(contentID); + return { key: `${moduleName}|${id}`, label: moduleLabel(moduleName, id) }; + }) + ); + }); + + // Pass 1: match within the paired zone. + const zones = Array.from(new Set([...Array.from(expectedByZone.keys()), ...Array.from(actualByZone.keys())])).sort(); + const leftoverExpected = new Map(); + const leftoverActual = new Map>(); + const matchedInZone = new Map>(); + zones.forEach((zone) => { + const expected = expectedByZone.get(zone) ?? []; + const available = new Map(); + (actualByZone.get(zone) ?? []).forEach((e) => available.set(e.key, (available.get(e.key) ?? 0) + 1)); + const leftovers: ZoneEntry[] = []; + const matched = new Map(); + expected.forEach((e) => { + const count = available.get(e.key) ?? 0; + if (count > 0) { + available.set(e.key, count - 1); + matched.set(e.key, (matched.get(e.key) ?? 0) + 1); + } else { + leftovers.push(e); + } + }); + leftoverExpected.set(zone, leftovers); + leftoverActual.set(zone, available); + matchedInZone.set(zone, matched); + }); + + // Pass 2: an expected component found in a different target zone is in the wrong zone. + zones.forEach((zone) => { + (leftoverExpected.get(zone) ?? []).forEach((e) => { + const otherZone = zones.find((z) => z !== zone && (leftoverActual.get(z)?.get(e.key) ?? 0) > 0); + if (otherZone) { + const available = leftoverActual.get(otherZone) as Map; + available.set(e.key, (available.get(e.key) as number) - 1); + add(zone, { + ...base, + category: "componentInWrongZone", + severity: "failure", + path: zone, + expected: `${e.label} in ${zone}`, + actual: `${e.label} in ${otherZone}`, + }); + } else { + add(zone, { + ...base, + category: "missingComponent", + severity: "failure", + path: zone, + expected: e.label, + actual: (actualByZone.get(zone) ?? []).map((a) => a.label), + }); + } + }); + }); + + // Pass 3: whatever is left on the target was not expected anywhere. + zones.forEach((zone) => { + const available = leftoverActual.get(zone) as Map; + (actualByZone.get(zone) ?? []).forEach((e) => { + const count = available.get(e.key) ?? 0; + if (count <= 0) return; + available.set(e.key, count - 1); + if (unmappedKeys.has(e.key)) return; // explained by the unmappedZone finding + add(zone, { + ...base, + category: "extraComponent", + severity: "failure", + path: zone, + actual: e.label, + expected: (expectedByZone.get(zone) ?? []).map((x) => x.label), + }); + }); + }); + + // Order: compare the components the two sides share in this zone. Each side contributes only as + // many occurrences of a component as were matched, so a duplicate (already an extraComponent) + // does not also read as an order change. + const shared = (entries: ZoneEntry[], matched: Map): string[] => { + const remaining = new Map(matched); + const keys: string[] = []; + entries.forEach((e) => { + const left = remaining.get(e.key) ?? 0; + if (left <= 0) return; + remaining.set(e.key, left - 1); + keys.push(e.key); + }); + return keys; + }; + zones.forEach((zone) => { + const matched = matchedInZone.get(zone) as Map; + const expectedKeys = shared(expectedByZone.get(zone) ?? [], matched); + if (expectedKeys.length < 2) return; + const actualKeys = shared(actualByZone.get(zone) ?? [], matched); + if (expectedKeys.every((k, i) => k === actualKeys[i])) return; + add(zone, { + ...base, + category: "componentOrderMismatch", + severity: "failure", + path: zone, + expected: (expectedByZone.get(zone) ?? []).map((e) => e.label), + actual: (actualByZone.get(zone) ?? []).map((e) => e.label), + message: "Components are in a different order on the target", + }); + }); + + findings.forEach((f) => collector.add(f)); + + const outcomeZones = new Set([...zones, ...Array.from(zoneOf.values())]); + outcomeZones.forEach((zone) => { + const zoneFindings = findings.filter((f) => zoneOf.get(f) === zone); + const hasEntries = (expectedByZone.get(zone)?.length ?? 0) > 0 || (actualByZone.get(zone)?.length ?? 0) > 0; + if (!hasEntries && zoneFindings.length === 0) return; + collector.count(locale, "zone", "checked"); + if (zoneFindings.some((f) => f.severity === "failure")) collector.count(locale, "zone", "mismatched"); + else if (zoneFindings.length > 0) collector.count(locale, "zone", "known"); + else collector.count(locale, "zone", "matched"); + }); +} diff --git a/src/lib/verify/check-sitemap.ts b/src/lib/verify/check-sitemap.ts new file mode 100644 index 0000000..96f4532 --- /dev/null +++ b/src/lib/verify/check-sitemap.ts @@ -0,0 +1,151 @@ +/** + * Sitemap checks for one locale, per channel (PROD-2552). + * + * The source nested sitemap is translated to target page IDs through the page mapping and compared + * with the target's nested sitemap for the same channel: + * - pageMissingFromSitemap a mapped page that exists on the target but is not in its sitemap + * - parentMismatch the page sits under a different parent (root = 0) + * - siblingOrderMismatch one finding per parent whose children are in a different order, + * with both sequences (target page IDs) + * Dynamic-page item entries (nodes carrying a contentID) are generated from content, not pages, + * and are skipped. Pages that are not mapped at all are reported by mapping integrity, not here. + */ +import { FindingCollector } from "./verify-report"; +import { ChannelSitemaps, VerifyContext } from "./verify-context"; +import { SitemapNode } from "../../types/syncAnalysis"; + +interface Placement { + parent: number; + path: string; + name: string; +} + +interface FlatSitemap { + placement: Map; + children: Map; +} + +function isPageNode(node: SitemapNode): boolean { + return typeof node?.pageID === "number" && node.pageID > 0 && !node.contentID; +} + +/** Flatten a nested sitemap into parent links and ordered child lists, keyed by page ID. */ +export function flattenSitemap(nodes: SitemapNode[] | null | undefined): FlatSitemap { + const flat: FlatSitemap = { placement: new Map(), children: new Map() }; + const visit = (list: SitemapNode[] | undefined, parent: number) => { + (list || []).forEach((node) => { + if (!isPageNode(node)) return; + if (!flat.placement.has(node.pageID)) { + flat.placement.set(node.pageID, { parent, path: node.path, name: node.name }); + if (!flat.children.has(parent)) flat.children.set(parent, []); + (flat.children.get(parent) as number[]).push(node.pageID); + } + visit(node.children, node.pageID); + }); + }; + visit(nodes || [], 0); + return flat; +} + +function channelKey(sitemaps: ChannelSitemaps): Map { + const out = new Map(); + Object.keys(sitemaps || {}).forEach((name) => out.set(name.toLowerCase(), name)); + return out; +} + +export function checkSitemap(ctx: VerifyContext, collector: FindingCollector): void { + const { locale } = ctx; + const sourceChannels = channelKey(ctx.source.entities.sitemaps); + const targetChannels = channelKey(ctx.target.entities.sitemaps); + const pageMap = ctx.localeMappings.pages; + // A scoped run compares only the pages the scope loaded on the source side. + const inScope = (sourcePageID: number) => !ctx.scoped || ctx.source.pagesById.has(sourcePageID); + + Array.from(sourceChannels.keys()) + .sort() + .forEach((channel) => { + if (ctx.channels && !ctx.channels.includes(channel)) return; + const source = flattenSitemap(ctx.source.entities.sitemaps[sourceChannels.get(channel) as string]); + const targetName = targetChannels.get(channel); + const target = flattenSitemap(targetName ? ctx.target.entities.sitemaps[targetName] : null); + const where = (path: string) => `${channel}:${path}`; + + const toTarget = (sourcePageID: number): number | undefined => + sourcePageID === 0 ? 0 : pageMap.target(sourcePageID); + + source.placement.forEach((placement, sourcePageID) => { + if (!inScope(sourcePageID)) return; + const targetPageID = pageMap.target(sourcePageID); + // Unmapped or dangling: mapping integrity already reports the page itself. + if (targetPageID === undefined || !ctx.target.pagesById.has(targetPageID)) return; + collector.count(locale, "sitemap", "checked"); + + const actual = target.placement.get(targetPageID); + if (!actual) { + collector.count(locale, "sitemap", "missing"); + collector.add({ + category: "pageMissingFromSitemap", + severity: "failure", + entity: "sitemap", + locale, + sourceID: sourcePageID, + targetID: targetPageID, + name: placement.name, + path: where(placement.path), + message: `Page is on the target but not in its "${channel}" sitemap`, + }); + return; + } + + const expectedParent = toTarget(placement.parent); + if (expectedParent !== actual.parent) { + collector.count(locale, "sitemap", "mismatched"); + collector.add({ + category: "parentMismatch", + severity: "failure", + entity: "sitemap", + locale, + sourceID: sourcePageID, + targetID: targetPageID, + name: placement.name, + path: where(placement.path), + expected: expectedParent === undefined ? `unmapped:${placement.parent}` : expectedParent, + actual: actual.parent, + message: "Page is under a different parent on the target (0 = root)", + }); + return; + } + collector.count(locale, "sitemap", "matched"); + }); + + source.children.forEach((sourceChildren, sourceParent) => { + if (sourceParent !== 0 && !inScope(sourceParent)) return; + const targetParent = toTarget(sourceParent); + if (targetParent === undefined) return; + const actualChildren = target.children.get(targetParent) ?? []; + const actualSet = new Set(actualChildren); + const expected = sourceChildren + .filter(inScope) + .map((id) => pageMap.target(id)) + .filter((id): id is number => id !== undefined && actualSet.has(id)); + if (expected.length < 2) return; + const expectedSet = new Set(expected); + const actual = actualChildren.filter((id) => expectedSet.has(id)); + if (expected.every((id, i) => id === actual[i])) return; + const parentPlacement = source.placement.get(sourceParent); + collector.add({ + category: "siblingOrderMismatch", + severity: "failure", + entity: "sitemap", + locale, + sourceID: sourceParent, + targetID: targetParent, + name: parentPlacement?.name ?? "(root)", + path: where(parentPlacement?.path ?? "/"), + expected, + actual, + message: "Child pages are in a different order on the target (target page IDs shown)", + }); + }); + }); +} diff --git a/src/lib/verify/check-structure.ts b/src/lib/verify/check-structure.ts new file mode 100644 index 0000000..d121d7b --- /dev/null +++ b/src/lib/verify/check-structure.ts @@ -0,0 +1,138 @@ +/** + * Structural checks (PROD-2552): models, containers and templates. + * + * Compared on what defines their shape, never on IDs, dates or who modified them: + * - model: reference name, and the ordered list of fields as (name, type) + * - container: reference name, and the model it holds (through the model mapping) + * - template: the ordered list of zone (section) reference names + * Assets and galleries are only checked for presence (mapping integrity); a matched pair counts + * as matched. + */ +import { FindingCollector, INSTANCE_LOCALE } from "./verify-report"; +import { InstanceIntegrityResult } from "./check-mapping-integrity"; +import { VerifyContext } from "./verify-context"; +import { canonicalEquals } from "./canonicalize"; + +function lower(value: any): string { + return typeof value === "string" ? value.toLowerCase() : ""; +} + +export function modelFieldSignature(model: any): string[] { + return (Array.isArray(model?.fields) ? model.fields : []).map((f: any) => `${lower(f?.name)}:${lower(f?.type)}`); +} + +export function templateSectionSignature(template: any): string[] { + return (Array.isArray(template?.contentSectionDefinitions) ? template.contentSectionDefinitions : []) + .slice() + .sort((a: any, b: any) => (a?.itemOrder ?? 0) - (b?.itemOrder ?? 0)) + .map((s: any) => lower(s?.pageItemTemplateReferenceName)); +} + +export function checkStructure(ctx: VerifyContext, pairs: InstanceIntegrityResult, collector: FindingCollector): void { + const locale = INSTANCE_LOCALE; + + pairs.models.forEach(({ sourceID, targetID, source, target }) => { + let differs = false; + if (lower(source.referenceName) !== lower(target.referenceName)) { + differs = true; + collector.add({ + category: "modelMismatch", + severity: "failure", + entity: "model", + locale, + sourceID, + targetID, + name: source.referenceName, + path: "referenceName", + expected: source.referenceName, + actual: target.referenceName, + }); + } + const expected = modelFieldSignature(source); + const actual = modelFieldSignature(target); + if (!canonicalEquals(expected, actual)) { + differs = true; + collector.add({ + category: "modelMismatch", + severity: "failure", + entity: "model", + locale, + sourceID, + targetID, + name: source.referenceName, + path: "fields", + expected, + actual, + message: "Model fields differ (name:type, in order)", + }); + } + collector.count(locale, "model", differs ? "mismatched" : "matched"); + }); + + pairs.containers.forEach(({ sourceID, targetID, source, target }) => { + let differs = false; + if (lower(source.referenceName) !== lower(target.referenceName)) { + differs = true; + collector.add({ + category: "containerMismatch", + severity: "failure", + entity: "container", + locale, + sourceID, + targetID, + name: source.referenceName, + path: "referenceName", + expected: source.referenceName, + actual: target.referenceName, + }); + } + const expectedModelID = ctx.mappings.models.target(source.contentDefinitionID); + if (expectedModelID === undefined || expectedModelID !== target.contentDefinitionID) { + differs = true; + const sourceModel = ctx.source.modelsById.get(source.contentDefinitionID); + const targetModel = ctx.target.modelsById.get(target.contentDefinitionID); + collector.add({ + category: "containerMismatch", + severity: "failure", + entity: "container", + locale, + sourceID, + targetID, + name: source.referenceName, + path: "model", + expected: + expectedModelID === undefined + ? `unmapped model ${sourceModel?.referenceName ?? source.contentDefinitionID}` + : `${sourceModel?.referenceName ?? "?"} (#${expectedModelID})`, + actual: `${targetModel?.referenceName ?? "?"} (#${target.contentDefinitionID})`, + message: "Container holds a different model on the target", + }); + } + collector.count(locale, "container", differs ? "mismatched" : "matched"); + }); + + pairs.templates.forEach(({ sourceID, targetID, source, target }) => { + const expected = templateSectionSignature(source); + const actual = templateSectionSignature(target); + const differs = !canonicalEquals(expected, actual); + if (differs) { + collector.add({ + category: "templateMismatch", + severity: "failure", + entity: "template", + locale, + sourceID, + targetID, + name: source.pageTemplateName, + path: "sections", + expected, + actual, + message: "Template zones differ (reference name, in order)", + }); + } + collector.count(locale, "template", differs ? "mismatched" : "matched"); + }); + + pairs.assets.forEach(() => collector.count(locale, "asset", "matched")); + pairs.galleries.forEach(() => collector.count(locale, "gallery", "matched")); +} diff --git a/src/lib/verify/mapping-index.ts b/src/lib/verify/mapping-index.ts new file mode 100644 index 0000000..27664ce --- /dev/null +++ b/src/lib/verify/mapping-index.ts @@ -0,0 +1,213 @@ +/** + * Bidirectional, Map-backed indices over every mapping file of a source→target pair (PROD-2552). + * + * The mappers in lib/mappers look records up with Array.find, which is fine for the one lookup a + * pusher makes per item but quadratic for a verify pass that touches every item in both + * directions. These indices are built once per run and answer every lookup in O(1). + * + * Read-only: records come in through a reader function and nothing here can write a mapping file. + */ +import * as fs from "fs"; +import * as path from "path"; +import { fileOperations } from "../../core/fileOperations"; + +/** Mapping folder names as the mappers write them. Content and pages are per-locale. */ +export type MappingType = + | "models" + | "containers" + | "templates" + | "sections" + | "galleries" + | "assets" + | "urlredirections" + | "item" + | "page"; + +export type MappingReader = (type: MappingType, locale?: string) => any[]; + +/** + * A one-to-one-ish relation between source keys and target keys. Duplicate sources keep their + * FIRST target (the same record the mappers' Array.find would return); duplicate targets — two + * sources mapped onto one target — are kept and reported. + */ +export class BiMap { + private readonly s2t = new Map(); + private readonly t2s = new Map(); + + add(source: K | null | undefined, target: K | null | undefined): void { + if (!isUsableKey(source) || !isUsableKey(target)) return; + if (this.s2t.has(source as K)) return; + this.s2t.set(source as K, target as K); + const sources = this.t2s.get(target as K); + if (sources) sources.push(source as K); + else this.t2s.set(target as K, [source as K]); + } + + target(source: K | null | undefined): K | undefined { + return isUsableKey(source) ? this.s2t.get(source as K) : undefined; + } + + sources(target: K | null | undefined): K[] { + return isUsableKey(target) ? (this.t2s.get(target as K) ?? []) : []; + } + + hasSource(source: K): boolean { + return this.s2t.has(source); + } + + hasTarget(target: K): boolean { + return this.t2s.has(target); + } + + /** Targets that more than one source maps onto. */ + duplicateTargets(): Array<{ target: K; sources: K[] }> { + const out: Array<{ target: K; sources: K[] }> = []; + this.t2s.forEach((sources, target) => { + if (sources.length > 1) out.push({ target, sources: sources.slice() }); + }); + return out; + } + + entries(): Array<[K, K]> { + return Array.from(this.s2t.entries()); + } + + get size(): number { + return this.s2t.size; + } +} + +function isUsableKey(key: any): boolean { + if (key === null || key === undefined || key === "") return false; + if (typeof key === "number") return Number.isFinite(key) && key > 0; + return true; +} + +function lower(value: any): string | undefined { + return typeof value === "string" && value.length > 0 ? value.toLowerCase() : undefined; +} + +/** A source container-URL prefix and the target prefix it maps onto. */ +export interface AssetPrefixPair { + source: string; + target: string; +} + +export interface LocaleMappingIndex { + content: BiMap; + pages: BiMap; +} + +export interface MappingIndex { + models: BiMap; + /** Lower-cased reference names. */ + modelNames: BiMap; + containers: BiMap; + /** Lower-cased reference names. */ + containerNames: BiMap; + templates: BiMap; + /** Lower-cased template names. */ + templateNames: BiMap; + /** pageItemTemplateID (zone definition) source → target. */ + sections: BiMap; + galleries: BiMap; + assets: BiMap; + /** Exact asset URL source → target. */ + assetUrls: Map; + /** Container URL prefixes (edge and origin) source → target, from the mapping records. */ + assetPrefixes: AssetPrefixPair[]; + byLocale: Map; +} + +export function buildMappingIndex(read: MappingReader, locales: string[]): MappingIndex { + const index: MappingIndex = { + models: new BiMap(), + modelNames: new BiMap(), + containers: new BiMap(), + containerNames: new BiMap(), + templates: new BiMap(), + templateNames: new BiMap(), + sections: new BiMap(), + galleries: new BiMap(), + assets: new BiMap(), + assetUrls: new Map(), + assetPrefixes: [], + byLocale: new Map(), + }; + + (read("models") || []).forEach((m: any) => { + index.models.add(m.sourceID, m.targetID); + index.modelNames.add(lower(m.sourceReferenceName), lower(m.targetReferenceName)); + }); + + (read("containers") || []).forEach((c: any) => { + index.containers.add(c.sourceContentViewID, c.targetContentViewID); + index.containerNames.add(lower(c.sourceReferenceName), lower(c.targetReferenceName)); + }); + + (read("templates") || []).forEach((t: any) => { + index.templates.add(t.sourcePageTemplateID, t.targetPageTemplateID); + index.templateNames.add(lower(t.sourcePageTemplateName), lower(t.targetPageTemplateName)); + }); + + (read("sections") || []).forEach((s: any) => + index.sections.add(s.sourcePageItemTemplateID, s.targetPageItemTemplateID) + ); + + (read("galleries") || []).forEach((g: any) => index.galleries.add(g.sourceMediaGroupingID, g.targetMediaGroupingID)); + + const prefixKeys = new Set(); + const addPrefix = (source: any, target: any) => { + if (typeof source !== "string" || typeof target !== "string" || !source || !target) return; + const pair = { source: stripTrailingSlash(source), target: stripTrailingSlash(target) }; + const key = `${pair.source}\u0000${pair.target}`; + if (prefixKeys.has(key)) return; + prefixKeys.add(key); + index.assetPrefixes.push(pair); + }; + (read("assets") || []).forEach((a: any) => { + index.assets.add(a.sourceMediaID, a.targetMediaID); + if (typeof a.sourceUrl === "string" && typeof a.targetUrl === "string" && a.sourceUrl && a.targetUrl) { + if (!index.assetUrls.has(a.sourceUrl)) index.assetUrls.set(a.sourceUrl, a.targetUrl); + } + addPrefix(a.sourceContainerEdgeUrl, a.targetContainerEdgeUrl); + addPrefix(a.sourceContainerOriginUrl, a.targetContainerOriginUrl); + }); + + locales.forEach((locale) => { + const localeIndex: LocaleMappingIndex = { content: new BiMap(), pages: new BiMap() }; + (read("item", locale) || []).forEach((r: any) => localeIndex.content.add(r.sourceContentID, r.targetContentID)); + (read("page", locale) || []).forEach((r: any) => localeIndex.pages.add(r.sourcePageID, r.targetPageID)); + index.byLocale.set(locale, localeIndex); + }); + + return index; +} + +export function stripTrailingSlash(url: string): string { + return url.endsWith("/") ? url.slice(0, -1) : url; +} + +/** The on-disk directory holding one pair's mappings. */ +export function mappingPairDirectory(rootPath: string, sourceGuid: string, targetGuid: string): string { + return path.join(rootPath, "mappings", `${sourceGuid}-${targetGuid}`); +} + +/** + * Read mappings through fileOperations.getMappingFile, but only when the mappings.json exists: + * getMappingFile seeds an empty file when the folder exists without one, and verify must never + * write to a mapping directory. + */ +export function createFileMappingReader(rootPath: string, sourceGuid: string, targetGuid: string): MappingReader { + const fileOps = new fileOperations(sourceGuid); + const pairDir = mappingPairDirectory(rootPath, sourceGuid, targetGuid); + return (type: MappingType, locale?: string) => { + const file = path.join(pairDir, locale ?? "", type, "mappings.json"); + if (!fs.existsSync(file)) return []; + const records = fileOps.getMappingFile(type, sourceGuid, targetGuid, locale ?? null); + if (!Array.isArray(records)) { + throw new Error(`Mapping file ${file} does not contain an array`); + } + return records; + }; +} diff --git a/src/lib/verify/tests/canonicalize.test.ts b/src/lib/verify/tests/canonicalize.test.ts new file mode 100644 index 0000000..5982c79 --- /dev/null +++ b/src/lib/verify/tests/canonicalize.test.ts @@ -0,0 +1,227 @@ +import { + AssetUrlTranslator, + buildAssetUrlTranslator, + canonicalFields, + canonicalScalar, + canonicalValue, + CanonicalContext, + diffCanonical, + describeDifference, +} from "../canonicalize"; + +const SRC_EDGE = "https://cdn-usa2.aglty.io/src-container"; +const SRC_CUSTOM = "https://cdn.custom.com/src-container"; +const TGT_EDGE = "https://cdn-usa2.aglty.io/tgt-container"; +const TGT_CUSTOM = "https://cdn.custom.com/tgt-container"; +const TGT_ORIGIN = "https://origin.example.net/tgt-container"; + +function translator(): AssetUrlTranslator { + return buildAssetUrlTranslator({ + mappingUrls: new Map([[`${SRC_CUSTOM}/img/exact.png`, `${TGT_CUSTOM}/img/renamed.png`]]), + mappingPrefixes: [{ source: SRC_CUSTOM, target: TGT_CUSTOM }], + assetIdMap: { target: (id: number) => (id === 1 ? 101 : undefined) }, + sourceAssets: [{ mediaID: 1, containerEdgeUrl: SRC_EDGE, edgeUrl: `${SRC_EDGE}/a.png` }], + targetAssets: [ + { mediaID: 101, containerEdgeUrl: TGT_EDGE, containerOriginUrl: TGT_ORIGIN, edgeUrl: `${TGT_EDGE}/a.png` }, + ], + }); +} + +function contexts(): { source: CanonicalContext; target: CanonicalContext } { + const assets = translator(); + const contentMap = new Map([ + [1, 501], + [2, 502], + [3, 503], + ]); + return { + source: { + side: "source", + contentRef: (id) => (contentMap.has(id) ? String(contentMap.get(id)) : `unmapped:${id}`), + containerRef: (name) => (name.toLowerCase() === "posts" ? "poststarget" : `unmapped:${name.toLowerCase()}`), + galleryRef: (id) => String(id + 1000), + assets, + }, + target: { + side: "target", + contentRef: (id) => String(id), + containerRef: (name) => name.toLowerCase(), + galleryRef: (id) => String(id), + assets, + }, + }; +} + +function same(sourceValue: any, targetValue: any, fieldType?: string): boolean { + const { source, target } = contexts(); + return ( + JSON.stringify(canonicalValue(sourceValue, source, fieldType)) === + JSON.stringify(canonicalValue(targetValue, target, fieldType)) + ); +} + +describe("canonicalScalar", () => { + it("treats null, undefined and empty string as empty", () => { + expect(canonicalScalar(null)).toBeUndefined(); + expect(canonicalScalar(undefined)).toBeUndefined(); + expect(canonicalScalar("")).toBeUndefined(); + }); + + it("normalizes CRLF to LF", () => { + expect(canonicalScalar("a\r\nb")).toBe("a\nb"); + }); + + it("equates numbers and numeric strings", () => { + expect(canonicalScalar("13")).toBe(canonicalScalar(13)); + expect(canonicalScalar("37.80")).toBe(canonicalScalar(37.8)); + expect(canonicalScalar("-2")).toBe("-2"); + }); + + it("keeps leading-zero strings (zip codes, codes) as text", () => { + expect(canonicalScalar("007")).toBe("007"); + }); + + it("equates booleans and their string forms", () => { + expect(canonicalScalar(true)).toBe(canonicalScalar("True")); + expect(canonicalScalar(false)).toBe("false"); + }); +}); + +describe("canonicalValue", () => { + it("compares object keys case-insensitively and drops empty values", () => { + expect(same({ Href: "/x", Target: "", Text: "Go" }, { href: "/x", text: "Go", target: null })).toBe(true); + }); + + it("treats a missing value and an empty one as equal", () => { + expect(same("", undefined)).toBe(true); + expect(same(null, "")).toBe(true); + }); + + describe("linked content", () => { + it("translates a {contentid} selection to the mapped target ID", () => { + expect(same({ contentid: 1, fulllist: false }, { contentid: 501, fulllist: false })).toBe(true); + expect(same({ contentid: 1, fulllist: false }, { contentid: 502, fulllist: false })).toBe(false); + }); + + it("equates the object form with the scalar form on a Content field", () => { + expect(same({ contentid: 1, fulllist: false }, "501", "Content")).toBe(true); + expect(same("1", { contentid: 501 }, "Content")).toBe(true); + }); + + it("keeps sortids order significant", () => { + const source = { referencename: "posts", fulllist: true, sortids: "1,2,3" }; + expect(same(source, { referencename: "PostsTarget", fulllist: true, sortids: "501,502,503" })).toBe(true); + expect(same(source, { referencename: "PostsTarget", fulllist: true, sortids: "502,501,503" })).toBe(false); + }); + + it("maps a full-list reference name through the container mapping", () => { + expect(same({ referencename: "Posts", fulllist: true }, { referencename: "poststarget", fulllist: true })).toBe( + true + ); + expect(same({ referencename: "Other", fulllist: true }, { referencename: "other", fulllist: true })).toBe(false); + }); + + it("marks an unmapped source ID so it can never match", () => { + const { source } = contexts(); + expect(canonicalValue({ contentid: 99 }, source)).toEqual({ "@ids": ["unmapped:99"] }); + }); + + it("reads an empty selection as empty", () => { + expect(same({ fulllist: false }, undefined)).toBe(true); + }); + }); + + describe("asset attachments", () => { + it("compares url and label only", () => { + expect( + same( + { url: `${SRC_EDGE}/a.png`, label: "A", filesize: 10, height: 5, width: 5 }, + { url: `${TGT_EDGE}/a.png`, label: "A", filesize: 11, pixelHeight: "5" } + ) + ).toBe(true); + expect(same({ url: `${SRC_EDGE}/a.png`, label: "A" }, { url: `${TGT_EDGE}/a.png`, label: "B" })).toBe(false); + }); + + it("uses the exact mapping first, then the container prefix", () => { + expect(same({ url: `${SRC_CUSTOM}/img/exact.png` }, { url: `${TGT_CUSTOM}/img/renamed.png` })).toBe(true); + expect(same({ url: `${SRC_CUSTOM}/img/other.png` }, { url: `${TGT_CUSTOM}/img/other.png` })).toBe(true); + }); + + it("treats every alias of the target container as the same host", () => { + expect(same({ url: `${SRC_EDGE}/b.png` }, { url: `${TGT_ORIGIN}/b.png` })).toBe(true); + }); + }); + + describe("rich text", () => { + it("rewrites mapped source asset URLs inside text", () => { + expect( + same( + ` and `, + ` and ` + ) + ).toBe(true); + }); + + it("prefers the longest matching URL", () => { + const t = translator(); + expect(t.rewriteSourceText(`${SRC_CUSTOM}/img/exact.png`)).toBe( + t.canonicalTargetText(`${TGT_CUSTOM}/img/renamed.png`) + ); + }); + + it("does not match a container whose name merely starts with a known one", () => { + const t = translator(); + expect(t.rewriteSourceText(`${SRC_EDGE}-old/a.png`)).toBe(`${SRC_EDGE}-old/a.png`); + }); + + it("finds source-instance asset URLs left in target text", () => { + const t = translator(); + expect(t.findSourceUrls(``)).toEqual([ + `${SRC_EDGE}/left.png`, + ]); + }); + + it("equates CRLF and LF in text", () => { + expect(same("

a

\r\n

b

", "

a

\n

b

")).toBe(true); + }); + }); + + it("maps a PhotoGallery ID through the gallery mapping", () => { + expect(same(3, "1003", "PhotoGallery")).toBe(true); + }); +}); + +describe("canonicalFields", () => { + it("matches field keys case-insensitively", () => { + const { source, target } = contexts(); + expect(canonicalFields({ Title: "a" }, null, source)).toEqual(canonicalFields({ title: "a" }, null, target)); + }); + + it("treats a model's companion value field as a content ID list", () => { + const { source, target } = contexts(); + const model = { + fields: [{ name: "Game", type: "Content", settings: { LinkeContentDropdownValueField: "Game_ValueField" } }], + }; + const s = canonicalFields({ game: { contentid: 1 }, game_valuefield: "1,2" }, model, source); + const t = canonicalFields({ game: { contentid: 501 }, Game_ValueField: "501,502" }, model, target); + expect(s).toEqual(t); + expect(s.game_valuefield).toEqual({ "@ids": ["501", "502"] }); + }); +}); + +describe("diffCanonical / describeDifference", () => { + it("lists only the differing keys", () => { + expect(diffCanonical({ a: "1", b: "x" }, { a: "1", b: "y", c: "z" })).toEqual([ + { field: "b", expected: "x", actual: "y" }, + { field: "c", expected: undefined, actual: "z" }, + ]); + }); + + it("cuts long strings to a window around the first difference", () => { + const prefix = "a".repeat(500); + const { expected, actual } = describeDifference(`${prefix}XYZ`, `${prefix}XQZ`); + expect(expected.length).toBeLessThan(200); + expect(expected).toContain("XYZ"); + expect(actual).toContain("XQZ"); + }); +}); diff --git a/src/lib/verify/tests/check-content.test.ts b/src/lib/verify/tests/check-content.test.ts new file mode 100644 index 0000000..863e9e6 --- /dev/null +++ b/src/lib/verify/tests/check-content.test.ts @@ -0,0 +1,308 @@ +import { checkContent } from "../check-content"; +import { checkLocaleIntegrity } from "../check-mapping-integrity"; +import { FindingCollector } from "../verify-report"; +import { context, item, itemMappings, LOCALE, POST_MODEL_SOURCE, POST_MODEL_TARGET } from "./fixtures"; + +const MAPPINGS = { + models: [{ sourceID: 10, targetID: 110, sourceReferenceName: "Post", targetReferenceName: "Post" }], + containers: [ + { sourceContentViewID: 20, targetContentViewID: 120, sourceReferenceName: "posts", targetReferenceName: "posts" }, + ], +}; +const SOURCE_CONTAINERS = [{ contentViewID: 20, referenceName: "posts", contentDefinitionID: 10 }]; +const TARGET_CONTAINERS = [ + { contentViewID: 120, referenceName: "posts", contentDefinitionID: 110 }, + { contentViewID: 121, referenceName: "other", contentDefinitionID: 110 }, +]; + +function run(source: any[], target: any[], pairs: Array<[number, number]>, opts: { scoped?: boolean } = {}) { + const ctx = context({ + source: { models: [POST_MODEL_SOURCE], containers: SOURCE_CONTAINERS, content: source }, + target: { models: [POST_MODEL_TARGET], containers: TARGET_CONTAINERS, content: target }, + mappings: { ...MAPPINGS, locales: { [LOCALE]: { item: itemMappings(pairs) } } }, + scoped: opts.scoped, + }); + const collector = new FindingCollector(); + const { content } = checkLocaleIntegrity(ctx, collector); + checkContent(ctx, content, collector); + return collector; +} + +const byCategory = (collector: FindingCollector, category: string) => + collector.findings.filter((f) => f.category === category); + +describe("checkContent", () => { + it("ignores contentID, versionID and modified", () => { + const collector = run( + [item(1, { title: "A" }, { versionID: 5, modified: "2020-01-01" })], + [item(101, { title: "A" }, { versionID: 999, modified: "2026-09-09" })], + [[1, 101]] + ); + expect(collector.findings).toEqual([]); + expect(collector.talliesFor(LOCALE).content).toMatchObject({ checked: 1, matched: 1 }); + }); + + it("reports each differing field", () => { + const collector = run( + [item(1, { title: "A", body: "

x

" })], + [item(101, { title: "B", body: "

x

" })], + [[1, 101]] + ); + expect(collector.findings).toEqual([ + expect.objectContaining({ + category: "fieldMismatch", + path: "title", + expected: "A", + actual: "B", + sourceID: 1, + targetID: 101, + }), + ]); + expect(collector.talliesFor(LOCALE).content.mismatched).toBe(1); + }); + + it("translates linked content IDs before comparing", () => { + const collector = run( + [item(1, { title: "A" }), item(2, { title: "B", related: { contentid: 1, fulllist: false } })], + [item(101, { title: "A" }), item(102, { title: "B", related: "101" })], + [ + [1, 101], + [2, 102], + ] + ); + expect(collector.findings).toEqual([]); + }); + + it("warns, without failing, on a publish-state difference", () => { + const collector = run( + [item(1, { title: "A" }, { state: 2 })], + [item(101, { title: "A" }, { state: 1 })], + [[1, 101]] + ); + expect(collector.findings).toEqual([ + expect.objectContaining({ category: "publishStateDifference", severity: "warning", expected: 2, actual: 1 }), + ]); + }); + + it("reports an item in the wrong container", () => { + const collector = run( + [item(1, { title: "A" })], + [item(101, { title: "A" }, { referenceName: "other" })], + [[1, 101]] + ); + expect(byCategory(collector, "contentContainerMismatch")).toHaveLength(1); + }); + + describe("counts", () => { + it("fails when the target container has fewer items", () => { + const collector = run( + [item(1, { title: "A" }), item(2, { title: "B" })], + [item(101, { title: "A" })], + [[1, 101]] + ); + const counts = byCategory(collector, "countMismatch"); + expect(counts.map((f) => [f.entity, f.severity, f.expected, f.actual])).toEqual([ + ["container", "failure", 2, 1], + ["model", "failure", 2, 1], + ]); + expect(byCategory(collector, "unmapped")).toHaveLength(1); + }); + + it("only warns when the target has more", () => { + const collector = run( + [item(1, { title: "A" })], + [item(101, { title: "A" }), item(102, { title: "X" })], + [[1, 101]] + ); + expect(byCategory(collector, "countMismatch").every((f) => f.severity === "warning")).toBe(true); + expect(byCategory(collector, "extraInTarget")).toHaveLength(1); + }); + + it("is skipped on a scoped run", () => { + const collector = run( + [item(1, { title: "A" }), item(2, { title: "B" })], + [item(101, { title: "A" })], + [[1, 101]], + { + scoped: true, + } + ); + expect(byCategory(collector, "countMismatch")).toEqual([]); + }); + }); + + describe("list order", () => { + const source = [ + item(1, { title: "A" }, { itemOrder: 0 }), + item(2, { title: "B" }, { itemOrder: 1 }), + item(3, { title: "C" }, { itemOrder: 2 }), + ]; + const pairs: Array<[number, number]> = [ + [1, 101], + [2, 102], + [3, 103], + ]; + + it("passes when the target keeps the order", () => { + const target = [ + item(101, { title: "A" }, { itemOrder: 5 }), + item(102, { title: "B" }, { itemOrder: 6 }), + item(103, { title: "C" }, { itemOrder: 7 }), + ]; + expect(run(source, target, pairs).findings).toEqual([]); + }); + + it("reports a reordered list with both sequences", () => { + const target = [ + item(101, { title: "A" }, { itemOrder: 2 }), + item(102, { title: "B" }, { itemOrder: 1 }), + item(103, { title: "C" }, { itemOrder: 3 }), + ]; + expect(byCategory(run(source, target, pairs), "listOrderMismatch")).toEqual([ + expect.objectContaining({ + entity: "container", + name: "posts", + expected: [101, 102, 103], + actual: [102, 101, 103], + }), + ]); + }); + + it("lets items that tie in the source come in either order", () => { + const tied = source.map((i) => ({ ...i, properties: { ...i.properties, itemOrder: 0 } })); + const target = [ + item(101, { title: "A" }, { itemOrder: 3 }), + item(102, { title: "B" }, { itemOrder: 2 }), + item(103, { title: "C" }, { itemOrder: 1 }), + ]; + expect(byCategory(run(tied, target, pairs), "listOrderMismatch")).toEqual([]); + }); + }); + + describe("unpublished (state 7) items", () => { + it("drops a state-7 item on either side, as a full pull would", () => { + const collector = run( + [item(1, { title: "A" }), item(2, { title: "Gone" }, { state: 7 })], + [item(101, { title: "A" }), item(102, { title: "Gone" }, { state: 7 })], + [ + [1, 101], + [2, 102], + ] + ); + expect(collector.findings).toEqual([]); + expect(collector.talliesFor(LOCALE).content.checked).toBe(1); + }); + + it("unpublished in the source only: the target copy is a warning, not a failure", () => { + const collector = run( + [item(1, { title: "A" })], + [item(101, { title: "A" }), item(102, { title: "Gone" })], + [ + [1, 101], + [2, 102], + ] + ); + expect(collector.findings).toEqual([ + expect.objectContaining({ category: "sourceNotInPull", severity: "warning", targetID: 102 }), + ]); + }); + + it("unpublished in the target only: the source item is reported missing", () => { + const collector = run( + [item(1, { title: "A" }), item(2, { title: "B" })], + [item(101, { title: "A" }), item(102, { title: "B" }, { state: 7 })], + [ + [1, 101], + [2, 102], + ] + ); + expect(byCategory(collector, "danglingMapping")).toEqual([ + expect.objectContaining({ severity: "failure", sourceID: 2, targetID: 102 }), + ]); + }); + }); + + it("treats a link to an empty list as no link", () => { + const collector = run( + [item(1, { title: "A" })], + [item(101, { title: "A", related: { referencename: "posts_related_auto", fulllist: true } })], + [[1, 101]] + ); + expect(collector.findings).toEqual([]); + }); + + it("leaves out items whose container is deleted in the source, listing them once per container", () => { + const collector = run( + [ + item(1, { title: "A" }), + item(2, { title: "x" }, { referenceName: "old_deleted" }), + item(3, {}, { referenceName: "old_deleted" }), + ], + [item(101, { title: "A" })], + [[1, 101]] + ); + expect(collector.findings).toEqual([ + expect.objectContaining({ + category: "contentNotInLiveContainer", + severity: "sourceDefect", + name: "old_deleted", + actual: 2, + }), + ]); + expect(collector.talliesFor(LOCALE).content).toMatchObject({ checked: 3, matched: 1, known: 2, missing: 0 }); + }); + + it("flags a source asset URL left in target content", () => { + const sourceAsset = { mediaID: 1, containerEdgeUrl: "https://cdn/src-c", edgeUrl: "https://cdn/src-c/a.png" }; + const targetAsset = { mediaID: 2, containerEdgeUrl: "https://cdn/tgt-c", edgeUrl: "https://cdn/tgt-c/a.png" }; + const ctx = context({ + source: { + models: [POST_MODEL_SOURCE], + containers: SOURCE_CONTAINERS, + assets: [sourceAsset], + content: [item(1, { body: '' })], + }, + target: { + models: [POST_MODEL_TARGET], + containers: TARGET_CONTAINERS, + assets: [targetAsset], + content: [item(101, { body: '' })], + }, + mappings: { + ...MAPPINGS, + assets: [{ sourceMediaID: 1, targetMediaID: 2 }], + locales: { [LOCALE]: { item: itemMappings([[1, 101]]) } }, + }, + }); + const collector = new FindingCollector(); + checkContent(ctx, checkLocaleIntegrity(ctx, collector).content, collector); + expect(collector.findings.map((f) => f.category).sort()).toEqual(["fieldMismatch", "sourceAssetUrlInTarget"]); + }); +}); + +describe("source defects", () => { + it("classifies a missing target copy of an item lacking a required field as a source defect", () => { + const collector = run( + [item(1, { title: "A" }), item(2, { body: "no title" })], + [item(101, { title: "A" })], + [[1, 101]] + ); + const unmapped = byCategory(collector, "unmapped"); + expect(unmapped).toEqual([expect.objectContaining({ severity: "sourceDefect", sourceID: 2 })]); + expect(unmapped[0].message).toContain("Title (Text)"); + expect(collector.talliesFor(LOCALE).content).toMatchObject({ known: 1, missing: 0 }); + }); + + it("classifies a field difference on such an item as a source defect", () => { + const collector = run([item(1, { body: "x" })], [item(101, { body: "y" })], [[1, 101]]); + expect(collector.findings).toEqual([ + expect.objectContaining({ category: "fieldMismatch", severity: "sourceDefect" }), + ]); + expect(collector.talliesFor(LOCALE).content.known).toBe(1); + }); + + it("keeps an ordinary difference a failure", () => { + const collector = run([item(1, { title: "A", body: "x" })], [item(101, { title: "A", body: "y" })], [[1, 101]]); + expect(collector.findings).toEqual([expect.objectContaining({ category: "fieldMismatch", severity: "failure" })]); + }); +}); diff --git a/src/lib/verify/tests/check-page-zones.test.ts b/src/lib/verify/tests/check-page-zones.test.ts new file mode 100644 index 0000000..05c7926 --- /dev/null +++ b/src/lib/verify/tests/check-page-zones.test.ts @@ -0,0 +1,231 @@ +import { checkPages } from "../check-page-zones"; +import { checkLocaleIntegrity } from "../check-mapping-integrity"; +import { FindingCollector } from "../verify-report"; +import { context, item, itemMappings, LOCALE, module, page, pageMappings, section } from "./fixtures"; + +const SOURCE_TEMPLATE = { + pageTemplateID: 1, + pageTemplateName: "Standard", + contentSectionDefinitions: [section(11, "Main", 1), section(12, "Sidebar", 2)], +}; +const TARGET_TEMPLATE = { + pageTemplateID: 101, + pageTemplateName: "Standard", + contentSectionDefinitions: [section(111, "Main", 1), section(112, "Sidebar", 2)], +}; + +const MAPPINGS_BASE = { + models: [ + { sourceID: 1, targetID: 101, sourceReferenceName: "Hero", targetReferenceName: "Hero" }, + { sourceID: 2, targetID: 102, sourceReferenceName: "Text", targetReferenceName: "Text" }, + ], + templates: [ + { + sourcePageTemplateID: 1, + targetPageTemplateID: 101, + sourcePageTemplateName: "Standard", + targetPageTemplateName: "Standard", + }, + ], + sections: [ + { sourcePageItemTemplateID: 11, targetPageItemTemplateID: 111 }, + { sourcePageItemTemplateID: 12, targetPageItemTemplateID: 112 }, + ], +}; + +const SOURCE_CONTENT = [item(1, {}), item(2, {}), item(3, {})]; +const TARGET_CONTENT = [item(501, {}), item(502, {}), item(503, {})]; +const CONTENT_MAP = itemMappings([ + [1, 501], + [2, 502], + [3, 503], +]); + +function run(input: { + sourceZones: Record; + targetZones: Record; + sourceTemplate?: any; + targetTemplate?: any; + sections?: any[]; + sourcePage?: Record; + targetPage?: Record; + sourceContent?: any[]; +}) { + const ctx = context({ + source: { + templates: [input.sourceTemplate ?? SOURCE_TEMPLATE], + content: input.sourceContent ?? SOURCE_CONTENT, + pages: [page(10, input.sourceZones, input.sourcePage)], + }, + target: { + templates: [input.targetTemplate ?? TARGET_TEMPLATE], + content: TARGET_CONTENT, + pages: [page(90, input.targetZones, input.targetPage)], + }, + mappings: { + ...MAPPINGS_BASE, + sections: input.sections ?? MAPPINGS_BASE.sections, + locales: { [LOCALE]: { item: CONTENT_MAP, page: pageMappings([[10, 90]]) } }, + }, + }); + const collector = new FindingCollector(); + const pairs = checkLocaleIntegrity(ctx, collector); + checkPages(ctx, pairs.pages, collector); + return { findings: collector.findings, tallies: collector.talliesFor(LOCALE), collector }; +} + +const categories = (findings: any[]) => findings.map((f) => f.category).sort(); + +describe("checkPages — zones", () => { + it("reports nothing for identical pages", () => { + const { findings, tallies } = run({ + sourceZones: { Main: [module("Hero", 1), module("Text", 2)], Sidebar: [module("Text", 3)] }, + targetZones: { Main: [module("Hero", 501), module("Text", 502)], Sidebar: [module("Text", 503)] }, + }); + expect(findings).toEqual([]); + expect(tallies.zone).toEqual({ checked: 2, matched: 2, mismatched: 0, missing: 0, extra: 0, known: 0 }); + expect(tallies.page.matched).toBe(1); + }); + + it("reports swapped components as an order mismatch with both sequences", () => { + const { findings } = run({ + sourceZones: { Main: [module("Hero", 1), module("Text", 2)] }, + targetZones: { Main: [module("Text", 502), module("Hero", 501)] }, + }); + expect(findings).toHaveLength(1); + expect(findings[0]).toMatchObject({ + category: "componentOrderMismatch", + severity: "failure", + entity: "zone", + sourceID: 10, + targetID: 90, + path: "main", + expected: ["hero#501", "text#502"], + actual: ["text#502", "hero#501"], + }); + }); + + it("reports a component placed in another zone", () => { + const { findings } = run({ + sourceZones: { Main: [module("Hero", 1)], Sidebar: [module("Text", 3)] }, + targetZones: { Main: [module("Hero", 501), module("Text", 503)], Sidebar: [] }, + }); + expect(categories(findings)).toEqual(["componentInWrongZone"]); + expect(findings[0]).toMatchObject({ expected: "text#503 in sidebar", actual: "text#503 in main" }); + }); + + it("reports missing and extra components", () => { + const { findings } = run({ + sourceZones: { Main: [module("Hero", 1), module("Text", 2)] }, + targetZones: { Main: [module("Hero", 501), module("Text", 503)] }, + }); + expect(categories(findings)).toEqual(["extraComponent", "missingComponent"]); + }); + + it("pairs a renamed zone through the section-ID mapping", () => { + const targetTemplate = { + ...TARGET_TEMPLATE, + contentSectionDefinitions: [section(111, "MainContent", 1), section(112, "Sidebar", 2)], + }; + const { findings } = run({ + targetTemplate, + sourceZones: { Main: [module("Hero", 1), module("Text", 2)] }, + targetZones: { MainContent: [module("Hero", 501), module("Text", 502)] }, + }); + expect(findings).toEqual([]); + }); + + describe("PROD-2350: zones defined in a different order on the two templates", () => { + const targetTemplate = { + ...TARGET_TEMPLATE, + // Same zones, opposite itemOrder — a positional pairing would cross them over. + contentSectionDefinitions: [section(112, "Sidebar", 1), section(111, "Main", 2)], + }; + + it("pairs each zone by section ID, not position", () => { + const { findings } = run({ + targetTemplate, + sourceZones: { Main: [module("Hero", 1)], Sidebar: [module("Text", 3)] }, + targetZones: { Sidebar: [module("Text", 503)], Main: [module("Hero", 501)] }, + }); + expect(findings).toEqual([]); + }); + + it("catches content a positional pairing would have crossed over", () => { + const { findings } = run({ + targetTemplate, + sourceZones: { Main: [module("Hero", 1)], Sidebar: [module("Text", 3)] }, + targetZones: { Sidebar: [module("Hero", 501)], Main: [module("Text", 503)] }, + }); + expect(categories(findings)).toEqual(["componentInWrongZone", "componentInWrongZone"]); + }); + }); + + it("reports an unmapped zone and never falls back to position", () => { + const sourceTemplate = { + ...SOURCE_TEMPLATE, + contentSectionDefinitions: [section(11, "Main", 1), section(13, "Legacy", 2)], + }; + const targetTemplate = { + ...TARGET_TEMPLATE, + contentSectionDefinitions: [section(111, "Main", 1), section(114, "Other", 2)], + }; + const { findings } = run({ + sourceTemplate, + targetTemplate, + sections: [{ sourcePageItemTemplateID: 11, targetPageItemTemplateID: 111 }], + sourceZones: { Main: [module("Hero", 1)], Legacy: [module("Text", 2)] }, + // The leftover target zone holds what a positional fallback would have put there. + targetZones: { Main: [module("Hero", 501)], Other: [module("Text", 502)] }, + }); + expect(categories(findings)).toEqual(["unmappedZone"]); + expect(findings[0]).toMatchObject({ severity: "failure", path: "Legacy", expected: ["text#502"] }); + }); + + it("lists a module whose content is missing from the source pull as a source defect", () => { + const { findings, tallies } = run({ + sourceContent: [item(1, {})], + sourceZones: { Main: [module("Hero", 1), module("Text", 2)] }, + targetZones: { Main: [module("Hero", 501)] }, + }); + // The target copies of 2 and 3 are reported by mapping integrity (sourceNotInPull warnings). + const zoneFindings = findings.filter((f) => f.entity === "zone"); + expect(zoneFindings).toHaveLength(1); + expect(zoneFindings[0]).toMatchObject({ category: "moduleContentMissingInSource", severity: "sourceDefect" }); + expect(findings.some((f) => f.severity === "failure")).toBe(false); + expect(tallies.zone.known).toBe(1); + }); +}); + +describe("checkPages — page fields", () => { + it("compares the listed fields with normalization", () => { + const { findings } = run({ + sourceZones: {}, + targetZones: {}, + sourcePage: { redirectUrl: "", seo: { metaDescription: "d\r\nx" } }, + targetPage: { redirectUrl: null, seo: { MetaDescription: "d\nx", metaKeywords: null } }, + }); + expect(findings).toEqual([]); + }); + + it("reports a differing title and template", () => { + const { findings } = run({ + sourceZones: {}, + targetZones: {}, + targetPage: { title: "Other", templateName: "Different" }, + }); + expect(findings.map((f) => [f.category, f.path])).toEqual([ + ["pageFieldMismatch", "title"], + ["pageFieldMismatch", "template"], + ]); + }); + + it("compares template names through the template mapping, case-insensitively", () => { + const { findings } = run({ + sourceZones: {}, + targetZones: {}, + sourcePage: { templateName: "STANDARD" }, + }); + expect(findings).toEqual([]); + }); +}); diff --git a/src/lib/verify/tests/check-sitemap.test.ts b/src/lib/verify/tests/check-sitemap.test.ts new file mode 100644 index 0000000..8627534 --- /dev/null +++ b/src/lib/verify/tests/check-sitemap.test.ts @@ -0,0 +1,144 @@ +import { checkSitemap, flattenSitemap } from "../check-sitemap"; +import { FindingCollector } from "../verify-report"; +import { context, LOCALE, page, pageMappings } from "./fixtures"; + +function node(pageID: number, path: string, children?: any[], extra: Record = {}): any { + return { + title: path, + name: path.split("/").pop(), + pageID, + menuText: path, + visible: { menu: true, sitemap: true }, + path, + redirect: null, + isFolder: false, + ...(children ? { children } : {}), + ...extra, + }; +} + +// Source: /home, /about (→ /about/team, /about/jobs), /contact. Target IDs are source + 100. +const SOURCE_TREE = [ + node(1, "/home"), + node(2, "/about", [node(3, "/about/team"), node(4, "/about/jobs")]), + node(5, "/contact"), +]; +const MAP = pageMappings([ + [1, 101], + [2, 102], + [3, 103], + [4, 104], + [5, 105], +]); + +function run( + targetTree: any[] | null, + opts: { channels?: Record; targetChannels?: Record; only?: string[] } = {} +) { + const ids = [1, 2, 3, 4, 5]; + const ctx = context({ + source: { pages: ids.map((id) => page(id, {})), sitemaps: opts.channels ?? { website: SOURCE_TREE } }, + target: { + pages: ids.map((id) => page(id + 100, {})), + sitemaps: opts.targetChannels ?? { website: targetTree as any[] }, + }, + mappings: { locales: { [LOCALE]: { page: MAP } } }, + channels: opts.only, + }); + const collector = new FindingCollector(); + checkSitemap(ctx, collector); + return collector; +} + +const translate = (nodes: any[]): any[] => + nodes.map((n) => node(n.pageID + 100, n.path, n.children ? translate(n.children) : undefined)); + +describe("flattenSitemap", () => { + it("records parents and child order, skipping dynamic-item nodes", () => { + const flat = flattenSitemap([node(1, "/a", [node(2, "/a/b"), node(1, "/a/item", undefined, { contentID: 9 })])]); + expect(flat.placement.get(2)?.parent).toBe(1); + expect(flat.children.get(0)).toEqual([1]); + expect(flat.children.get(1)).toEqual([2]); + }); +}); + +describe("checkSitemap", () => { + it("reports nothing for an identical structure", () => { + const collector = run(translate(SOURCE_TREE)); + expect(collector.findings).toEqual([]); + expect(collector.talliesFor(LOCALE).sitemap).toMatchObject({ checked: 5, matched: 5 }); + }); + + it("reports one siblingOrderMismatch per parent with both sequences", () => { + const target = [ + node(101, "/home"), + node(102, "/about", [node(104, "/about/jobs"), node(103, "/about/team")]), + node(105, "/contact"), + ]; + const findings = run(target).findings; + expect(findings).toHaveLength(1); + expect(findings[0]).toMatchObject({ + category: "siblingOrderMismatch", + sourceID: 2, + targetID: 102, + expected: [103, 104], + actual: [104, 103], + }); + }); + + it("reports a reordered root level", () => { + const target = [ + node(105, "/contact"), + node(101, "/home"), + node(102, "/about", [node(103, "/a/t"), node(104, "/a/j")]), + ]; + const findings = run(target).findings; + expect(findings.map((f) => f.category)).toEqual(["siblingOrderMismatch"]); + expect(findings[0]).toMatchObject({ name: "(root)", expected: [101, 102, 105], actual: [105, 101, 102] }); + }); + + it("reports a reparented page", () => { + const target = [ + node(101, "/home", [node(103, "/home/team")]), + node(102, "/about", [node(104, "/about/jobs")]), + node(105, "/contact"), + ]; + const findings = run(target).findings; + expect(findings.map((f) => f.category)).toEqual(["parentMismatch"]); + expect(findings[0]).toMatchObject({ sourceID: 3, targetID: 103, expected: 102, actual: 101 }); + }); + + it("reports a mapped page missing from the target sitemap", () => { + const target = [node(101, "/home"), node(102, "/about", [node(103, "/about/team")]), node(105, "/contact")]; + const findings = run(target).findings; + expect(findings.map((f) => f.category)).toEqual(["pageMissingFromSitemap"]); + expect(findings[0]).toMatchObject({ sourceID: 4, targetID: 104 }); + }); + + describe("multiple channels", () => { + const channels = { website: SOURCE_TREE, mobile: [node(5, "/contact"), node(1, "/home")] }; + + it("compares each channel against the same channel on the target", () => { + const findings = run(null, { + channels, + targetChannels: { website: translate(SOURCE_TREE), mobile: [node(101, "/home"), node(105, "/contact")] }, + }).findings; + expect(findings).toHaveLength(1); + expect(findings[0]).toMatchObject({ category: "siblingOrderMismatch", path: "mobile:/" }); + }); + + it("reports pages of a channel the target lacks", () => { + const findings = run(null, { channels, targetChannels: { website: translate(SOURCE_TREE) } }).findings; + expect(findings.map((f) => f.category)).toEqual(["pageMissingFromSitemap", "pageMissingFromSitemap"]); + }); + + it("checks only the requested channel when one is given", () => { + const findings = run(null, { + channels, + targetChannels: { website: translate(SOURCE_TREE) }, + only: ["Website"], + }).findings; + expect(findings).toEqual([]); + }); + }); +}); diff --git a/src/lib/verify/tests/check-structure.test.ts b/src/lib/verify/tests/check-structure.test.ts new file mode 100644 index 0000000..c2f0801 --- /dev/null +++ b/src/lib/verify/tests/check-structure.test.ts @@ -0,0 +1,220 @@ +import { checkInstanceIntegrity } from "../check-mapping-integrity"; +import { checkStructure } from "../check-structure"; +import { FindingCollector, INSTANCE_LOCALE } from "../verify-report"; +import { context, section } from "./fixtures"; + +function run(source: any, target: any, mappings: any) { + const ctx = context({ source, target, mappings }); + const collector = new FindingCollector(); + checkStructure(ctx, checkInstanceIntegrity(ctx, collector), collector); + return collector; +} + +const MODEL_MAP = [{ sourceID: 1, targetID: 11, sourceReferenceName: "Post", targetReferenceName: "Post" }]; +const fields = (...pairs: Array<[string, string]>) => + pairs.map(([name, type]) => ({ name, type, lastModifiedDate: "x" })); + +describe("models", () => { + it("compares reference name and ordered (name, type) fields, ignoring IDs and dates", () => { + const collector = run( + { + models: [ + { id: 1, referenceName: "Post", lastModifiedDate: "a", fields: fields(["Title", "Text"], ["Body", "HTML"]) }, + ], + }, + { + models: [ + { id: 11, referenceName: "post", lastModifiedDate: "b", fields: fields(["title", "Text"], ["body", "Html"]) }, + ], + }, + { models: MODEL_MAP } + ); + expect(collector.findings).toEqual([]); + expect(collector.talliesFor(INSTANCE_LOCALE).model).toMatchObject({ checked: 1, matched: 1 }); + }); + + it("reports reordered fields", () => { + const collector = run( + { models: [{ id: 1, referenceName: "Post", fields: fields(["Title", "Text"], ["Body", "HTML"]) }] }, + { models: [{ id: 11, referenceName: "Post", fields: fields(["Body", "HTML"], ["Title", "Text"]) }] }, + { models: MODEL_MAP } + ); + expect(collector.findings).toEqual([ + expect.objectContaining({ category: "modelMismatch", path: "fields", expected: ["title:text", "body:html"] }), + ]); + }); + + it("warns on a target model nothing maps to", () => { + const collector = run( + { models: [{ id: 1, referenceName: "Post", fields: [] }] }, + { + models: [ + { id: 11, referenceName: "Post", fields: [] }, + { id: 12, referenceName: "Extra", fields: [] }, + ], + }, + { models: MODEL_MAP } + ); + expect(collector.findings).toEqual([ + expect.objectContaining({ category: "extraInTarget", severity: "warning", targetID: 12 }), + ]); + }); +}); + +describe("containers", () => { + it("compares the held model through the model mapping", () => { + const mappings = { + models: MODEL_MAP, + containers: [ + { sourceContentViewID: 5, targetContentViewID: 55, sourceReferenceName: "posts", targetReferenceName: "posts" }, + ], + }; + const source = { + models: [{ id: 1, referenceName: "Post", fields: [] }], + containers: [{ contentViewID: 5, referenceName: "posts", contentDefinitionID: 1 }], + }; + const ok = run( + source, + { + models: [{ id: 11, referenceName: "Post", fields: [] }], + containers: [{ contentViewID: 55, referenceName: "posts", contentDefinitionID: 11 }], + }, + mappings + ); + expect(ok.findings).toEqual([]); + + const wrong = run( + source, + { + models: [ + { id: 11, referenceName: "Post", fields: [] }, + { id: 12, referenceName: "Other", fields: [] }, + ], + containers: [{ contentViewID: 55, referenceName: "posts", contentDefinitionID: 12 }], + }, + mappings + ); + expect(wrong.findings.filter((f) => f.category === "containerMismatch")).toEqual([ + expect.objectContaining({ path: "model", actual: "Other (#12)" }), + ]); + }); + + it("leaves out the fixed Agility system containers on both sides", () => { + const collector = run( + { containers: [{ contentViewID: 1, referenceName: "AgilityCSSFiles", contentDefinitionID: 3 }] }, + { containers: [{ contentViewID: 1, referenceName: "AgilityCSSFiles", contentDefinitionID: 3 }] }, + {} + ); + expect(collector.findings).toEqual([]); + }); + + it("leaves out deleted containers", () => { + const collector = run({ containers: [{ contentViewID: 9, referenceName: "gone", isDeleted: true }] }, {}, {}); + expect(collector.findings).toEqual([]); + }); +}); + +describe("templates", () => { + const mappings = { + templates: [ + { + sourcePageTemplateID: 1, + targetPageTemplateID: 2, + sourcePageTemplateName: "Std", + targetPageTemplateName: "Std", + }, + ], + }; + + it("compares section reference names in itemOrder", () => { + const collector = run( + { + templates: [ + { + pageTemplateID: 1, + pageTemplateName: "Std", + contentSectionDefinitions: [section(1, "Main", 1), section(2, "Side", 2)], + }, + ], + }, + { + templates: [ + { + pageTemplateID: 2, + pageTemplateName: "Std", + contentSectionDefinitions: [section(9, "Side", 2), section(8, "main", 1)], + }, + ], + }, + mappings + ); + expect(collector.findings).toEqual([]); + }); + + it("reports a differing zone list", () => { + const collector = run( + { + templates: [ + { + pageTemplateID: 1, + pageTemplateName: "Std", + contentSectionDefinitions: [section(1, "Main", 1), section(2, "Side", 2)], + }, + ], + }, + { + templates: [{ pageTemplateID: 2, pageTemplateName: "Std", contentSectionDefinitions: [section(8, "Main", 1)] }], + }, + mappings + ); + expect(collector.findings).toEqual([ + expect.objectContaining({ category: "templateMismatch", expected: ["main", "side"], actual: ["main"] }), + ]); + }); +}); + +describe("mapping integrity", () => { + it("reports unmapped, dangling and duplicate-target mappings", () => { + const collector = run( + { + models: [ + { id: 1, referenceName: "A", fields: [] }, + { id: 2, referenceName: "B", fields: [] }, + { id: 3, referenceName: "C", fields: [] }, + { id: 4, referenceName: "D", fields: [] }, + ], + }, + { models: [{ id: 11, referenceName: "A", fields: [] }] }, + { + models: [ + { sourceID: 1, targetID: 11 }, + { sourceID: 2, targetID: 11 }, + { sourceID: 3, targetID: 13 }, + ], + } + ); + const summary = collector.findings.map((f) => [f.category, f.sourceID, f.targetID]); + expect(summary).toEqual( + expect.arrayContaining([ + ["danglingMapping", 3, 13], + ["unmapped", 4, null], + ["duplicateTarget", "1,2", 11], + ]) + ); + expect(collector.talliesFor(INSTANCE_LOCALE).model).toMatchObject({ checked: 4, missing: 2 }); + }); + + it("does not report a duplicate whose second source no longer exists", () => { + const collector = run( + { models: [{ id: 1, referenceName: "A", fields: [] }] }, + { models: [{ id: 11, referenceName: "A", fields: [] }] }, + { + models: [ + { sourceID: 1, targetID: 11 }, + { sourceID: 2, targetID: 11 }, + ], + } + ); + expect(collector.findings).toEqual([]); + }); +}); diff --git a/src/lib/verify/tests/fixtures.ts b/src/lib/verify/tests/fixtures.ts new file mode 100644 index 0000000..9b4f572 --- /dev/null +++ b/src/lib/verify/tests/fixtures.ts @@ -0,0 +1,127 @@ +/** + * Small hand-built fixtures for the verify checks. Not a test file (no `.test.ts`), so Jest only + * loads it through the tests that import it. + */ +import { buildMappingIndex, MappingIndex, MappingType } from "../mapping-index"; +import { createVerifyContext, SideEntities, VerifyContext } from "../verify-context"; + +export const LOCALE = "en-us"; +export const SRC = "src-guid"; +export const TGT = "tgt-guid"; + +export function emptySide(overrides: Partial = {}): SideEntities { + return { + models: [], + containers: [], + templates: [], + assets: [], + galleries: [], + content: [], + pages: [], + sitemaps: {}, + ...overrides, + }; +} + +export interface MappingFixture { + models?: any[]; + containers?: any[]; + templates?: any[]; + sections?: any[]; + galleries?: any[]; + assets?: any[]; + /** Per locale: item / page mapping records. */ + locales?: Record; +} + +export function mappingIndex(fixture: MappingFixture, locales: string[] = [LOCALE]): MappingIndex { + return buildMappingIndex((type: MappingType, locale?: string) => { + if (locale) return (fixture.locales?.[locale] as any)?.[type] ?? []; + return (fixture as any)[type] ?? []; + }, locales); +} + +export function context(input: { + source: Partial; + target: Partial; + mappings: MappingFixture; + locale?: string; + scoped?: boolean; + channels?: string[]; +}): VerifyContext { + const locale = input.locale ?? LOCALE; + return createVerifyContext({ + sourceGuid: SRC, + targetGuid: TGT, + locale, + mappings: mappingIndex(input.mappings, [locale]), + source: emptySide(input.source), + target: emptySide(input.target), + scoped: input.scoped, + channels: input.channels, + }); +} + +/** Content mapping records from [sourceID, targetID] pairs. */ +export function itemMappings(pairs: Array<[number, number]>): any[] { + return pairs.map(([s, t]) => ({ sourceGuid: SRC, targetGuid: TGT, sourceContentID: s, targetContentID: t })); +} + +export function pageMappings(pairs: Array<[number, number]>): any[] { + return pairs.map(([s, t]) => ({ sourceGuid: SRC, targetGuid: TGT, sourcePageID: s, targetPageID: t })); +} + +export function item(contentID: number, fields: any, props: Partial> = {}): any { + return { + contentID, + properties: { + state: 2, + modified: "2026-01-01T00:00:00", + versionID: contentID * 10, + referenceName: "posts", + definitionName: "Post", + itemOrder: 0, + ...props, + }, + fields, + }; +} + +export const POST_MODEL_SOURCE = { + id: 10, + referenceName: "Post", + fields: [ + { name: "Title", type: "Text", settings: { Required: "True" } }, + { name: "Body", type: "HTML", settings: { Required: "False" } }, + { name: "Related", type: "Content", settings: { Required: "False" } }, + ], +}; + +export const POST_MODEL_TARGET = { ...POST_MODEL_SOURCE, id: 110 }; + +export function module(name: string, contentID: number): any { + return { title: name, module: name, item: { contentid: contentID, fulllist: false } }; +} + +export function section(id: number, referenceName: string, itemOrder: number): any { + return { pageItemTemplateID: id, pageItemTemplateReferenceName: referenceName, itemOrder }; +} + +export function page(pageID: number, zones: Record, overrides: Record = {}): any { + return { + pageID, + name: "home", + title: "Home", + menuText: "Home", + pageType: "static", + templateName: "Standard", + redirectUrl: "", + securePage: false, + visible: { menu: true, sitemap: true }, + seo: { metaDescription: "", metaKeywords: "" }, + scripts: { excludedFromGlobal: false, top: "", bottom: "" }, + properties: { state: 2, modified: "2026-01-01T00:00:00", versionID: pageID * 10 }, + zones, + ...overrides, + }; +} diff --git a/src/lib/verify/tests/mapping-index.test.ts b/src/lib/verify/tests/mapping-index.test.ts new file mode 100644 index 0000000..9491019 --- /dev/null +++ b/src/lib/verify/tests/mapping-index.test.ts @@ -0,0 +1,152 @@ +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { BiMap, buildMappingIndex, createFileMappingReader } from "../mapping-index"; +import { resetState, setState } from "../../../core/state"; + +describe("BiMap", () => { + it("looks up both directions", () => { + const map = new BiMap(); + map.add(1, 101); + map.add(2, 102); + expect(map.target(1)).toBe(101); + expect(map.sources(102)).toEqual([2]); + expect(map.target(3)).toBeUndefined(); + expect(map.sources(999)).toEqual([]); + }); + + it("keeps the first target for a duplicated source, like the mappers' Array.find", () => { + const map = new BiMap(); + map.add(1, 101); + map.add(1, 201); + expect(map.target(1)).toBe(101); + expect(map.hasTarget(201)).toBe(false); + }); + + it("reports targets that more than one source maps onto", () => { + const map = new BiMap(); + map.add(1, 101); + map.add(2, 101); + map.add(3, 103); + expect(map.duplicateTargets()).toEqual([{ target: 101, sources: [1, 2] }]); + }); + + it("ignores unusable keys (missing, zero, negative)", () => { + const map = new BiMap(); + map.add(undefined, 1); + map.add(1, null); + map.add(2, -1); + map.add(0, 5); + expect(map.size).toBe(0); + }); +}); + +describe("buildMappingIndex", () => { + const records: Record = { + models: [{ sourceID: 1, targetID: 11, sourceReferenceName: "Post", targetReferenceName: "Post" }], + containers: [ + { sourceContentViewID: 5, targetContentViewID: 55, sourceReferenceName: "Posts", targetReferenceName: "PostsT" }, + ], + templates: [ + { + sourcePageTemplateID: 2, + targetPageTemplateID: 22, + sourcePageTemplateName: "Std", + targetPageTemplateName: "Std", + }, + ], + sections: [{ sourcePageItemTemplateID: 7, targetPageItemTemplateID: 77 }], + galleries: [{ sourceMediaGroupingID: 3, targetMediaGroupingID: 33 }], + assets: [ + { + sourceMediaID: 4, + targetMediaID: 44, + sourceUrl: "https://cdn/src/a.png", + targetUrl: "https://cdn/tgt/a.png", + sourceContainerEdgeUrl: "https://cdn/src", + targetContainerEdgeUrl: "https://cdn/tgt", + sourceContainerOriginUrl: "https://origin/src/", + targetContainerOriginUrl: "https://origin/tgt/", + }, + { + sourceMediaID: 6, + targetMediaID: 66, + sourceContainerEdgeUrl: "https://cdn/src", + targetContainerEdgeUrl: "https://cdn/tgt", + }, + ], + }; + const locales: Record> = { + "en-us": { + item: [{ sourceContentID: 100, targetContentID: 900 }], + page: [{ sourcePageID: 8, targetPageID: 88 }], + }, + "fr-ca": { item: [{ sourceContentID: 100, targetContentID: 901 }] }, + }; + + const index = buildMappingIndex( + (type, locale) => (locale ? (locales[locale]?.[type] ?? []) : (records[type] ?? [])), + ["en-us", "fr-ca"] + ); + + it("indexes every guid-level mapping type", () => { + expect(index.models.target(1)).toBe(11); + expect(index.modelNames.target("post")).toBe("post"); + expect(index.containers.target(5)).toBe(55); + expect(index.containerNames.target("posts")).toBe("postst"); + expect(index.templates.target(2)).toBe(22); + expect(index.templateNames.target("std")).toBe("std"); + expect(index.sections.target(7)).toBe(77); + expect(index.galleries.target(3)).toBe(33); + expect(index.assets.target(4)).toBe(44); + }); + + it("collects exact asset URLs and de-duplicated container prefixes without trailing slashes", () => { + expect(index.assetUrls.get("https://cdn/src/a.png")).toBe("https://cdn/tgt/a.png"); + expect(index.assetPrefixes).toEqual([ + { source: "https://cdn/src", target: "https://cdn/tgt" }, + { source: "https://origin/src", target: "https://origin/tgt" }, + ]); + }); + + it("keeps content and page mappings per locale", () => { + expect(index.byLocale.get("en-us")?.content.target(100)).toBe(900); + expect(index.byLocale.get("fr-ca")?.content.target(100)).toBe(901); + expect(index.byLocale.get("en-us")?.pages.sources(88)).toEqual([8]); + expect(index.byLocale.get("fr-ca")?.pages.size).toBe(0); + }); +}); + +describe("createFileMappingReader", () => { + let root: string; + + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), "verify-mappings-")); + resetState(); + setState({ rootPath: root }); + }); + + afterEach(() => fs.rmSync(root, { recursive: true, force: true })); + + it("reads existing mapping files and never seeds a missing one", () => { + const pair = path.join(root, "mappings", "a-b"); + fs.mkdirSync(path.join(pair, "models"), { recursive: true }); + fs.writeFileSync(path.join(pair, "models", "mappings.json"), JSON.stringify([{ sourceID: 1, targetID: 2 }])); + // A type folder without its mappings.json: fileOperations.getMappingFile would write "[]" here. + fs.mkdirSync(path.join(pair, "containers"), { recursive: true }); + + const read = createFileMappingReader(root, "a", "b"); + + expect(read("models")).toEqual([{ sourceID: 1, targetID: 2 }]); + expect(read("containers")).toEqual([]); + expect(fs.existsSync(path.join(pair, "containers", "mappings.json"))).toBe(false); + expect(read("item", "en-us")).toEqual([]); + }); + + it("throws on a malformed mapping file", () => { + const dir = path.join(root, "mappings", "a-b", "models"); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, "mappings.json"), "{ not json"); + expect(() => createFileMappingReader(root, "a", "b")("models")).toThrow(); + }); +}); diff --git a/src/lib/verify/tests/verify-report.test.ts b/src/lib/verify/tests/verify-report.test.ts new file mode 100644 index 0000000..afcb696 --- /dev/null +++ b/src/lib/verify/tests/verify-report.test.ts @@ -0,0 +1,233 @@ +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { + buildErrorReport, + buildVerifyReport, + computeExitCode, + defaultReportPath, + emptyRunInfo, + EXIT_CANNOT_VERIFY, + EXIT_DIFFERENCES, + EXIT_PASS, + FindingCollector, + MAX_DETAILS_PER_CATEGORY, + printVerifyReport, + writeVerifyReport, +} from "../verify-report"; + +const OPTIONS = { locales: ["en-us"], channel: "", models: "", pages: "", containers: "" }; + +function collectorWith(severities: Array<"failure" | "warning" | "sourceDefect">): FindingCollector { + const collector = new FindingCollector(); + severities.forEach((severity, i) => + collector.add({ category: `c${i}`, severity, entity: "content", locale: "en-us", sourceID: i, targetID: 100 + i }) + ); + return collector; +} + +describe("exit codes", () => { + it("is 0 with no findings", () => { + expect(computeExitCode([])).toBe(EXIT_PASS); + }); + + it("is 0 when there are only warnings and source defects", () => { + expect(computeExitCode(collectorWith(["warning", "sourceDefect"]).findings)).toBe(EXIT_PASS); + }); + + it("is 1 with any failure", () => { + expect(computeExitCode(collectorWith(["warning", "failure"]).findings)).toBe(EXIT_DIFFERENCES); + }); + + it("is 2 for a run that could not verify", () => { + const report = buildErrorReport({ source: "a", target: "b", options: OPTIONS, error: "boom", run: emptyRunInfo() }); + expect(report).toMatchObject({ verdict: "error", exitCode: EXIT_CANNOT_VERIFY, error: "boom", findings: [] }); + }); +}); + +describe("buildVerifyReport — golden (without the volatile run block)", () => { + it("serializes findings, per-locale tallies and totals", () => { + const collector = new FindingCollector(); + collector.count("*", "model", "checked", 2); + collector.count("*", "model", "matched", 2); + collector.count("en-us", "page", "checked"); + collector.count("en-us", "page", "mismatched"); + collector.count("en-us", "zone", "checked"); + collector.count("en-us", "zone", "mismatched"); + collector.add({ + category: "componentOrderMismatch", + severity: "failure", + entity: "zone", + locale: "en-us", + sourceID: 10, + targetID: 90, + name: "home", + path: "main", + expected: ["hero#1", "text#2"], + actual: ["text#2", "hero#1"], + }); + collector.add({ + category: "publishStateDifference", + severity: "warning", + entity: "page", + locale: "en-us", + sourceID: 10, + targetID: 90, + name: "home", + path: "state", + expected: 2, + actual: 1, + }); + + const report = buildVerifyReport({ + source: "src", + target: "tgt", + options: OPTIONS, + collector, + run: emptyRunInfo(), + }); + const { run, ...stable } = report; + + expect(run).toEqual(expect.objectContaining({ cliVersion: expect.any(String), startedAt: expect.any(String) })); + expect(JSON.parse(JSON.stringify(stable))).toEqual({ + schemaVersion: 1, + command: "verify", + verdict: "fail", + exitCode: 1, + source: "src", + target: "tgt", + options: { locales: ["en-us"], channel: "", models: "", pages: "", containers: "" }, + totals: { + failures: 1, + warnings: 1, + sourceDefects: 0, + byCategory: { componentOrderMismatch: 1, publishStateDifference: 1 }, + entities: { + model: { checked: 2, matched: 2, mismatched: 0, missing: 0, extra: 0, known: 0 }, + page: { checked: 1, matched: 0, mismatched: 1, missing: 0, extra: 0, known: 0 }, + zone: { checked: 1, matched: 0, mismatched: 1, missing: 0, extra: 0, known: 0 }, + }, + }, + byLocale: { + "*": { + failures: 0, + warnings: 0, + sourceDefects: 0, + entities: { model: { checked: 2, matched: 2, mismatched: 0, missing: 0, extra: 0, known: 0 } }, + }, + "en-us": { + failures: 1, + warnings: 1, + sourceDefects: 0, + entities: { + page: { checked: 1, matched: 0, mismatched: 1, missing: 0, extra: 0, known: 0 }, + zone: { checked: 1, matched: 0, mismatched: 1, missing: 0, extra: 0, known: 0 }, + }, + }, + }, + findings: [ + { + category: "componentOrderMismatch", + severity: "failure", + entity: "zone", + locale: "en-us", + sourceID: 10, + targetID: 90, + name: "home", + path: "main", + expected: ["hero#1", "text#2"], + actual: ["text#2", "hero#1"], + }, + { + category: "publishStateDifference", + severity: "warning", + entity: "page", + locale: "en-us", + sourceID: 10, + targetID: 90, + name: "home", + path: "state", + expected: 2, + actual: 1, + }, + ], + }); + }); + + it("orders findings deterministically regardless of insertion order", () => { + const a = collectorWith(["warning", "failure", "sourceDefect"]); + const b = new FindingCollector(); + a.findings + .slice() + .reverse() + .forEach((f) => b.add(f)); + const run = emptyRunInfo(); + expect(buildVerifyReport({ source: "s", target: "t", options: OPTIONS, collector: b, run }).findings).toEqual( + buildVerifyReport({ source: "s", target: "t", options: OPTIONS, collector: a, run }).findings + ); + }); +}); + +describe("writeVerifyReport / defaultReportPath", () => { + it("writes the report, creating directories, and never throws", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "verify-report-")); + const file = path.join(dir, "nested", "report.json"); + const report = buildErrorReport({ source: "a", target: "b", options: OPTIONS, error: "x", run: emptyRunInfo() }); + expect(writeVerifyReport(file, report)).toEqual({ written: true }); + expect(JSON.parse(fs.readFileSync(file, "utf8")).command).toBe("verify"); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + it("defaults to agility-files/logs/verify-{src}-{tgt}-{timestamp}.json", () => { + expect(defaultReportPath("agility-files", "s", "t", new Date("2026-10-07T12:00:00.000Z"))).toBe( + path.join("agility-files", "logs", "verify-s-t-2026-10-07T12-00-00-000Z.json") + ); + }); +}); + +describe("printVerifyReport", () => { + it("prints a table per locale, capped details with links, and a verdict line", () => { + const collector = new FindingCollector(); + collector.count("en-us", "content", "checked", 30); + for (let i = 0; i < MAX_DETAILS_PER_CATEGORY + 5; i++) { + collector.add({ + category: "fieldMismatch", + severity: "failure", + entity: "content", + locale: "en-us", + sourceID: i, + targetID: 500 + i, + path: "title", + expected: "a", + actual: "b", + }); + } + const lines: string[] = []; + printVerifyReport( + buildVerifyReport({ source: "s", target: "t", options: OPTIONS, collector, run: emptyRunInfo() }), + { + log: (l) => lines.push(l.replace(/\x1b\[[0-9;]*m/g, "")), + linkFor: (f) => `link-${f.targetID}`, + } + ); + const text = lines.join("\n"); + expect(text).toContain("Locale en-us: 25 failure(s)"); + expect(text).toMatch(/entity\s+checked\s+matched\s+mismatched\s+missing\s+extra\s+known/); + expect(text).toContain("FAILURE fieldMismatch (25)"); + expect(lines.filter((l) => l.includes("link-")).length).toBe(MAX_DETAILS_PER_CATEGORY); + expect(text).toContain("…and 5 more"); + expect(text).toContain("VERDICT: FAIL"); + }); + + it("prints the reason for a run that could not verify", () => { + const lines: string[] = []; + printVerifyReport( + buildErrorReport({ source: "s", target: "t", options: OPTIONS, error: "no auth", run: emptyRunInfo() }), + { + log: (l) => lines.push(l), + } + ); + expect(lines.join("\n")).toContain("no auth"); + expect(lines.join("\n")).toContain("exit 2"); + }); +}); diff --git a/src/lib/verify/verify-context.ts b/src/lib/verify/verify-context.ts new file mode 100644 index 0000000..d6742cf --- /dev/null +++ b/src/lib/verify/verify-context.ts @@ -0,0 +1,334 @@ +/** + * Everything one locale's verify checks need, loaded once (PROD-2552). + * + * Reads only: the pulled caches through GuidDataLoader / SitemapHierarchy, and the mapping index + * built by mapping-index.ts. Building the context is split from loading it so tests can hand in + * entities directly. + * + * Unpublished items: a full pull omits state-7 (unpublished) items, on both sides. Anything with + * state 7 that is still on disk (an older incremental pull) is dropped here, on BOTH sides, so the + * two sides are always compared on the same terms — what a fresh pull of each instance returns. + */ +import * as fs from "fs"; +import * as path from "path"; +import { GuidDataLoader, GuidEntities, ModelFilterOptions } from "../pushers/guid-data-loader"; +import { SitemapHierarchy } from "../pushers/page-pusher/sitemap-hierarchy"; +import { SitemapNode } from "../../types/syncAnalysis"; +import { LocaleMappingIndex, MappingIndex } from "./mapping-index"; +import { AssetUrlTranslator, buildAssetUrlTranslator, CanonicalContext } from "./canonicalize"; + +export const UNPUBLISHED_STATE = 7; + +/** + * Containers every instance has and the sync never migrates (container-pusher skips them by name), + * so they are left out on both sides. + */ +export const SYSTEM_CONTAINER_NAMES = new Set([ + "agilitycssfiles", + "agilityjavascriptfiles", + "agilityglobalcodetemplates", + "agilitymodulecodetemplates", + "agilitypagecodetemplates", +]); + +export type ChannelSitemaps = { [channel: string]: SitemapNode[] | null }; + +export interface SideEntities { + models: any[]; + containers: any[]; + templates: any[]; + assets: any[]; + galleries: any[]; + content: any[]; + pages: any[]; + sitemaps: ChannelSitemaps; +} + +export interface SideIndex { + entities: SideEntities; + modelsById: Map; + /** Lower-cased reference name → model. */ + modelsByRef: Map; + containersById: Map; + /** Lower-cased reference name → container. */ + containersByRef: Map; + templatesById: Map; + /** Lower-cased template name → template. */ + templatesByName: Map; + contentById: Map; + pagesById: Map; + /** Content IDs placed as a module on some page zone (their container is auto-generated). */ + pageBoundContentIDs: Set; + /** Lower-cased container reference name → number of (live) items in it, for this locale. */ + contentCountByContainer: Map; + /** + * Content items whose container is not a live container on this side (deleted, or a system + * container). The pull still returns them, but they are not reachable content and the sync skips + * them (no container mapping), so they are left out of every comparison. + */ + outsideLiveContainers: any[]; +} + +export interface VerifyContext { + sourceGuid: string; + targetGuid: string; + locale: string; + mappings: MappingIndex; + localeMappings: LocaleMappingIndex; + source: SideIndex; + target: SideIndex; + assets: AssetUrlTranslator; + /** + * A --models / --pages / --containers scope narrowed the SOURCE side. Checks that compare + * whole sets (extra target entities, per-container counts) are skipped: the target side is not + * narrowed, so they would report everything outside the scope. + */ + scoped: boolean; + /** Restrict sitemap checks to these channels (lower-case); undefined = every channel. */ + channels?: string[]; +} + +export function isPulled(entity: any): boolean { + return entity?.properties?.state !== UNPUBLISHED_STATE; +} + +function lower(value: any): string { + return typeof value === "string" ? value.toLowerCase() : ""; +} + +/** Module entries on a page, across every zone. */ +export function zoneModules(page: any): Array<{ zone: string; module: any }> { + const out: Array<{ zone: string; module: any }> = []; + const zones = page?.zones; + if (!zones || typeof zones !== "object") return out; + Object.keys(zones).forEach((zone) => { + const modules = zones[zone]; + if (Array.isArray(modules)) modules.forEach((module) => out.push({ zone, module })); + }); + return out; +} + +/** The content ID a zone module points at, whatever the key's casing. */ +export function moduleContentID(module: any): number | undefined { + const item = module?.item; + if (!item || typeof item !== "object") return undefined; + const raw = item.contentid ?? item.contentID ?? item.ContentID; + const id = typeof raw === "string" ? parseInt(raw, 10) : raw; + return typeof id === "number" && id > 0 ? id : undefined; +} + +export function buildSideIndex(entities: SideEntities, outsideLiveContainers: any[] = []): SideIndex { + const index: SideIndex = { + entities, + modelsById: new Map(), + modelsByRef: new Map(), + containersById: new Map(), + containersByRef: new Map(), + templatesById: new Map(), + templatesByName: new Map(), + contentById: new Map(), + pagesById: new Map(), + pageBoundContentIDs: new Set(), + contentCountByContainer: new Map(), + outsideLiveContainers, + }; + entities.models.forEach((m) => { + if (typeof m?.id === "number") index.modelsById.set(m.id, m); + if (m?.referenceName && !index.modelsByRef.has(lower(m.referenceName))) + index.modelsByRef.set(lower(m.referenceName), m); + }); + entities.containers.forEach((c) => { + if (typeof c?.contentViewID === "number") index.containersById.set(c.contentViewID, c); + if (c?.referenceName && !index.containersByRef.has(lower(c.referenceName))) + index.containersByRef.set(lower(c.referenceName), c); + }); + entities.templates.forEach((t) => { + if (typeof t?.pageTemplateID === "number") index.templatesById.set(t.pageTemplateID, t); + if (t?.pageTemplateName && !index.templatesByName.has(lower(t.pageTemplateName))) + index.templatesByName.set(lower(t.pageTemplateName), t); + }); + entities.content.forEach((c) => { + if (typeof c?.contentID === "number") index.contentById.set(c.contentID, c); + const container = lower(c?.properties?.referenceName); + if (container) + index.contentCountByContainer.set(container, (index.contentCountByContainer.get(container) ?? 0) + 1); + }); + entities.pages.forEach((p) => { + if (typeof p?.pageID === "number") index.pagesById.set(p.pageID, p); + zoneModules(p).forEach(({ module }) => { + const id = moduleContentID(module); + if (id) index.pageBoundContentIDs.add(id); + }); + }); + return index; +} + +export function createVerifyContext(input: { + sourceGuid: string; + targetGuid: string; + locale: string; + mappings: MappingIndex; + source: SideEntities; + target: SideEntities; + scoped?: boolean; + channels?: string[]; +}): VerifyContext { + const clean = (side: SideEntities): SideIndex => { + const containers = side.containers.filter( + (c) => c?.isDeleted !== true && !SYSTEM_CONTAINER_NAMES.has(lower(c?.referenceName)) + ); + const liveContainers = new Set(containers.map((c) => lower(c?.referenceName))); + const pulled = side.content.filter(isPulled); + // Without a container list (nothing pulled, or a hand-built fixture) every item counts as live. + const isLive = (item: any) => containers.length === 0 || liveContainers.has(lower(item?.properties?.referenceName)); + return buildSideIndex( + { + ...side, + content: pulled.filter(isLive), + pages: side.pages.filter(isPulled), + containers, + templates: side.templates.filter((t) => t?.isDeleted !== true), + assets: side.assets.filter((a) => a?.isDeleted !== true && a?.isFolder !== true), + }, + pulled.filter((item) => !isLive(item)) + ); + }; + const source = clean(input.source); + const target = clean(input.target); + const localeMappings = input.mappings.byLocale.get(input.locale); + if (!localeMappings) { + throw new Error(`No mapping index for locale ${input.locale}`); + } + + return { + sourceGuid: input.sourceGuid, + targetGuid: input.targetGuid, + locale: input.locale, + mappings: input.mappings, + localeMappings, + source, + target, + assets: buildAssetUrlTranslator({ + mappingUrls: input.mappings.assetUrls, + mappingPrefixes: input.mappings.assetPrefixes, + assetIdMap: input.mappings.assets, + sourceAssets: source.entities.assets, + targetAssets: target.entities.assets, + }), + scoped: input.scoped === true, + channels: input.channels?.map((c) => c.toLowerCase()), + }; +} + +/** Canonicalization context for SOURCE values: every ID is translated to its target. */ +export function sourceCanonicalContext(ctx: VerifyContext): CanonicalContext { + return { + side: "source", + contentRef: (id) => { + const target = ctx.localeMappings.content.target(id); + return target !== undefined ? String(target) : `unmapped:${id}`; + }, + containerRef: (name) => { + const key = lower(name); + const mapped = ctx.mappings.containerNames.target(key); + if (mapped) return mapped; + // Auto-named nested lists keep their reference name on the target, so an identical name is + // accepted. A name that is not a live container in the source either (a dangling reference + // the sync copies verbatim) is compared literally. Only a live, unmapped source container + // that the target lacks can never match. + if (ctx.target.containersByRef.has(key) || !ctx.source.containersByRef.has(key)) return key; + return `unmapped:${key}`; + }, + galleryRef: (id) => { + const target = ctx.mappings.galleries.target(id); + return target !== undefined ? String(target) : `unmapped:${id}`; + }, + isEmptyList: (name) => !ctx.source.contentCountByContainer.get(lower(name)), + assets: ctx.assets, + }; +} + +/** Canonicalization context for TARGET values: IDs are already target IDs. */ +export function targetCanonicalContext(ctx: VerifyContext): CanonicalContext { + return { + side: "target", + contentRef: (id) => String(id), + containerRef: (name) => lower(name), + galleryRef: (id) => String(id), + isEmptyList: (name) => !ctx.target.contentCountByContainer.get(lower(name)), + assets: ctx.assets, + }; +} + +// ─── loading ───────────────────────────────────────────────────────────────── + +function toSideEntities(entities: GuidEntities, sitemaps: ChannelSitemaps): SideEntities { + return { + models: entities.models || [], + containers: entities.containers || [], + templates: entities.templates || [], + assets: entities.assets || [], + galleries: entities.galleries || [], + content: entities.content || [], + pages: entities.pages || [], + sitemaps, + }; +} + +/** + * Load one side of one locale from the pulled cache. With `filter`, the existing scope filters + * (--models / --pages / --containers) narrow what is loaded, exactly as a scoped sync would. + */ +export async function loadSideEntities( + guid: string, + locale: string, + filter?: ModelFilterOptions +): Promise { + const loader = new GuidDataLoader(guid); + const entities = filter + ? await loader.loadGuidEntities(locale, filter) + : await loader.loadCompleteGuidEntities(locale); + // Sitemaps are always loaded whole: a scoped run narrows the sitemap check to the scoped pages + // itself (see check-sitemap.ts), and siblings outside the scope are still needed for ordering. + const sitemaps = new SitemapHierarchy().loadAllSitemaps(guid, locale); + return toSideEntities(entities, sitemaps); +} + +/** + * The loaders skip a JSON file they cannot parse with only a console warning, which would make a + * corrupt cache read as "entity missing". Verify refuses to compare on top of that: every file a + * check reads is parsed here first, and any failure is a "could not verify" (exit 2). + */ +export function findUnreadableCacheFiles(rootPath: string, guid: string, locales: string[]): string[] { + const problems: string[] = []; + const base = path.join(rootPath, guid); + if (!fs.existsSync(base)) return [`${base} does not exist (the pull wrote nothing)`]; + + const folders = ["models", "containers", "templates", "galleries", path.join("assets", "json")]; + locales.forEach((locale) => + folders.push(path.join(locale, "item"), path.join(locale, "page"), path.join(locale, "nestedsitemap")) + ); + + if (!fs.existsSync(path.join(base, "models"))) problems.push(`${path.join(base, "models")} is missing`); + + folders.forEach((folder) => { + const dir = path.join(base, folder); + if (!fs.existsSync(dir)) return; + let files: string[]; + try { + files = fs.readdirSync(dir).filter((f) => f.endsWith(".json")); + } catch (error: any) { + problems.push(`${dir}: ${error.message}`); + return; + } + files.forEach((file) => { + const full = path.join(dir, file); + try { + JSON.parse(fs.readFileSync(full, "utf8")); + } catch (error: any) { + problems.push(`${full}: ${error.message}`); + } + }); + }); + return problems; +} diff --git a/src/lib/verify/verify-report.ts b/src/lib/verify/verify-report.ts new file mode 100644 index 0000000..dc06603 --- /dev/null +++ b/src/lib/verify/verify-report.ts @@ -0,0 +1,394 @@ +/** + * Findings, tallies and the report for `agility verify` (PROD-2552). + * + * The report is the product of the command: CI reads the exit code, a person reads the console + * table, and a harness asserts on the JSON. All three are derived from the same findings list, so + * they cannot disagree. + * + * Exit codes: + * 0 — every check passed (warnings and source defects do not fail a run) + * 1 — at least one failure-severity finding + * 2 — the run could not verify at all (auth, status guard, pull, missing mappings, unreadable cache) + */ +import * as path from "path"; +import ansiColors from "ansi-colors"; +import { writeJsonSummary, resolveCliVersion } from "../../core/json-summary"; + +/** Bump when a field is removed or its meaning changes. Additions do not require a bump. */ +export const VERIFY_REPORT_SCHEMA_VERSION = 1; + +export const EXIT_PASS = 0; +export const EXIT_DIFFERENCES = 1; +export const EXIT_CANNOT_VERIFY = 2; + +/** + * - failure: the target differs from the source in a way the migration should have prevented. + * - warning: worth knowing, does not fail the run (extra target items, publish-state drift). + * - sourceDefect: a difference explained by the source item itself (e.g. a missing required + * field the target would reject). Listed, does not fail the run. + */ +export type Severity = "failure" | "warning" | "sourceDefect"; + +export type EntityKind = + | "model" + | "container" + | "template" + | "asset" + | "gallery" + | "content" + | "page" + | "zone" + | "sitemap"; + +export interface Finding { + category: string; + severity: Severity; + entity: EntityKind; + /** Locale for locale-scoped entities; "*" for instance-wide ones (models, containers, ...). */ + locale: string; + sourceID?: number | string | null; + targetID?: number | string | null; + name?: string; + /** A field name, zone name, or sitemap path — what inside the entity differs. */ + path?: string; + expected?: any; + actual?: any; + /** Human sentence for the console. */ + message?: string; +} + +export interface Tally { + checked: number; + matched: number; + mismatched: number; + missing: number; + extra: number; + known: number; +} + +/** Instance-wide entities (models, containers, templates, assets, galleries) are tallied here. */ +export const INSTANCE_LOCALE = "*"; + +export function emptyTally(): Tally { + return { checked: 0, matched: 0, mismatched: 0, missing: 0, extra: 0, known: 0 }; +} + +/** + * Collects findings and per-(locale, entity) tallies as the checks run. Each check decides what an + * entity's outcome is; the collector only counts. + */ +export class FindingCollector { + readonly findings: Finding[] = []; + private readonly tallies = new Map>(); + + add(finding: Finding): void { + this.findings.push(finding); + } + + tally(locale: string, entity: EntityKind): Tally { + let byEntity = this.tallies.get(locale); + if (!byEntity) { + byEntity = new Map(); + this.tallies.set(locale, byEntity); + } + let tally = byEntity.get(entity); + if (!tally) { + tally = emptyTally(); + byEntity.set(entity, tally); + } + return tally; + } + + count(locale: string, entity: EntityKind, outcome: keyof Tally, by: number = 1): void { + this.tally(locale, entity)[outcome] += by; + } + + /** Locales in a stable order: instance-wide first, then alphabetical. */ + locales(): string[] { + return Array.from(this.tallies.keys()).sort((a, b) => + a === INSTANCE_LOCALE ? -1 : b === INSTANCE_LOCALE ? 1 : a.localeCompare(b) + ); + } + + talliesFor(locale: string): Record { + const out: Record = {}; + const byEntity = this.tallies.get(locale); + if (!byEntity) return out; + Array.from(byEntity.keys()) + .sort() + .forEach((entity) => (out[entity] = { ...(byEntity.get(entity) as Tally) })); + return out; + } +} + +export function computeExitCode(findings: Finding[]): number { + return findings.some((f) => f.severity === "failure") ? EXIT_DIFFERENCES : EXIT_PASS; +} + +export interface SeverityCounts { + failures: number; + warnings: number; + sourceDefects: number; +} + +function countSeverities(findings: Finding[]): SeverityCounts { + return { + failures: findings.filter((f) => f.severity === "failure").length, + warnings: findings.filter((f) => f.severity === "warning").length, + sourceDefects: findings.filter((f) => f.severity === "sourceDefect").length, + }; +} + +export interface StatusSnapshot { + guid: string; + mode: string; + checkedAt: string; + inProgress: boolean; + lastContentVersionID: number | null; + maxChangeDate: string | null; +} + +export interface VerifyRunInfo { + cliVersion: string; + startedAt: string; + finishedAt: string; + elapsedMs: number; + pull: { attempts: number; elapsedMs: number[] }; + status: { before: StatusSnapshot[]; after: StatusSnapshot[] }; + compareElapsedMs: number; +} + +export interface VerifyOptionsSummary { + locales: string[]; + channel: string; + models: string; + pages: string; + containers: string; +} + +export interface VerifyReport { + schemaVersion: number; + command: "verify"; + verdict: "pass" | "fail" | "error"; + exitCode: number; + /** Set when exitCode is 2: why the run could not verify. */ + error?: string; + source: string; + target: string; + options: VerifyOptionsSummary; + totals: SeverityCounts & { + byCategory: Record; + entities: Record; + }; + byLocale: Record }>; + findings: Finding[]; + /** Everything that differs between two otherwise-identical runs. Drop this for golden files. */ + run: VerifyRunInfo; +} + +/** Stable ordering, so two runs over the same data produce byte-identical findings. */ +export function sortFindings(findings: Finding[]): Finding[] { + const key = (v: any) => (v === undefined || v === null ? "" : String(v)); + const severityRank: Record = { failure: 0, sourceDefect: 1, warning: 2 }; + return findings.slice().sort((a, b) => { + return ( + key(a.locale).localeCompare(key(b.locale)) || + severityRank[a.severity] - severityRank[b.severity] || + a.category.localeCompare(b.category) || + a.entity.localeCompare(b.entity) || + key(a.sourceID).localeCompare(key(b.sourceID), undefined, { numeric: true }) || + key(a.targetID).localeCompare(key(b.targetID), undefined, { numeric: true }) || + key(a.path).localeCompare(key(b.path)) + ); + }); +} + +export function emptyRunInfo(startedAt: Date = new Date()): VerifyRunInfo { + return { + cliVersion: resolveCliVersion(), + startedAt: startedAt.toISOString(), + finishedAt: startedAt.toISOString(), + elapsedMs: 0, + pull: { attempts: 0, elapsedMs: [] }, + status: { before: [], after: [] }, + compareElapsedMs: 0, + }; +} + +export function buildVerifyReport(input: { + source: string; + target: string; + options: VerifyOptionsSummary; + collector: FindingCollector; + run: VerifyRunInfo; +}): VerifyReport { + const findings = sortFindings(input.collector.findings); + const exitCode = computeExitCode(findings); + + const byCategory: Record = {}; + findings.forEach((f) => (byCategory[f.category] = (byCategory[f.category] || 0) + 1)); + const sortedByCategory: Record = {}; + Object.keys(byCategory) + .sort() + .forEach((k) => (sortedByCategory[k] = byCategory[k])); + + const byLocale: VerifyReport["byLocale"] = {}; + const entityTotals: Record = {}; + input.collector.locales().forEach((locale) => { + const entities = input.collector.talliesFor(locale); + byLocale[locale] = { ...countSeverities(findings.filter((f) => f.locale === locale)), entities }; + Object.keys(entities).forEach((entity) => { + const total = entityTotals[entity] ?? emptyTally(); + (Object.keys(total) as Array).forEach((k) => (total[k] += entities[entity][k])); + entityTotals[entity] = total; + }); + }); + const sortedEntityTotals: Record = {}; + Object.keys(entityTotals) + .sort() + .forEach((k) => (sortedEntityTotals[k] = entityTotals[k])); + + return { + schemaVersion: VERIFY_REPORT_SCHEMA_VERSION, + command: "verify", + verdict: exitCode === EXIT_PASS ? "pass" : "fail", + exitCode, + source: input.source, + target: input.target, + options: input.options, + totals: { ...countSeverities(findings), byCategory: sortedByCategory, entities: sortedEntityTotals }, + byLocale, + findings, + run: input.run, + }; +} + +/** Report for a run that could not verify (exit 2). Same schema, no findings. */ +export function buildErrorReport(input: { + source: string; + target: string; + options: VerifyOptionsSummary; + error: string; + run: VerifyRunInfo; +}): VerifyReport { + return { + schemaVersion: VERIFY_REPORT_SCHEMA_VERSION, + command: "verify", + verdict: "error", + exitCode: EXIT_CANNOT_VERIFY, + error: input.error, + source: input.source, + target: input.target, + options: input.options, + totals: { failures: 0, warnings: 0, sourceDefects: 0, byCategory: {}, entities: {} }, + byLocale: {}, + findings: [], + run: input.run, + }; +} + +export function defaultReportPath(rootPath: string, sourceGuid: string, targetGuid: string, at: Date): string { + const timestamp = at.toISOString().replace(/[:.]/g, "-"); + return path.join(rootPath, "logs", `verify-${sourceGuid}-${targetGuid}-${timestamp}.json`); +} + +/** Never throws — an unwritable report path is a warning, not a different verdict. */ +export function writeVerifyReport(filePath: string, report: VerifyReport): { written: boolean; error?: string } { + return writeJsonSummary(filePath, report); +} + +// ─── console output ────────────────────────────────────────────────────────── + +/** How many findings to print per category before summarizing the rest. */ +export const MAX_DETAILS_PER_CATEGORY = 20; + +const TABLE_COLUMNS: Array = ["checked", "matched", "mismatched", "missing", "extra", "known"]; + +function formatValue(value: any, max: number = 160): string { + if (value === undefined) return "(none)"; + let text: string; + if (Array.isArray(value)) { + text = `[${value.map((v) => (typeof v === "string" ? v : JSON.stringify(v))).join(", ")}]`; + } else if (typeof value === "string") { + text = JSON.stringify(value); + } else { + text = JSON.stringify(value); + } + return text.length > max ? `${text.slice(0, max - 1)}…` : text; +} + +export function renderTallyTable(entities: Record): string[] { + const header = ["entity", ...TABLE_COLUMNS]; + const rows = Object.keys(entities).map((entity) => [ + entity, + ...TABLE_COLUMNS.map((c) => String(entities[entity][c])), + ]); + const widths = header.map((h, i) => Math.max(h.length, ...rows.map((r) => r[i].length))); + const line = (cells: string[]) => + cells.map((c, i) => (i === 0 ? c.padEnd(widths[i]) : c.padStart(widths[i]))).join(" "); + return [line(header), widths.map((w) => "-".repeat(w)).join(" "), ...rows.map(line)]; +} + +export interface PrintOptions { + /** Builds a CMS link for a finding, or returns undefined when there is none. */ + linkFor?: (finding: Finding) => string | undefined; + log?: (line: string) => void; +} + +export function printVerifyReport(report: VerifyReport, options: PrintOptions = {}): void { + const log = options.log ?? ((line: string) => console.log(line)); + + log(ansiColors.cyan(`\n🔎 Verify: ${report.source} → ${report.target}`)); + + if (report.exitCode === EXIT_CANNOT_VERIFY) { + log(ansiColors.red(`\n✗ Could not verify: ${report.error}`)); + log(ansiColors.red(`VERDICT: ERROR (exit ${report.exitCode})`)); + return; + } + + Object.keys(report.byLocale).forEach((locale) => { + const section = report.byLocale[locale]; + const title = locale === INSTANCE_LOCALE ? "Instance-wide" : `Locale ${locale}`; + log( + ansiColors.bold( + `\n${title}: ${section.failures} failure(s), ${section.warnings} warning(s), ${section.sourceDefects} source defect(s)` + ) + ); + renderTallyTable(section.entities).forEach((row) => log(` ${row}`)); + }); + + const byCategory = new Map(); + report.findings.forEach((f) => { + const key = `${f.severity}:${f.category}`; + if (!byCategory.has(key)) byCategory.set(key, []); + (byCategory.get(key) as Finding[]).push(f); + }); + + byCategory.forEach((findings, key) => { + const [severity, category] = key.split(":"); + const color = + severity === "failure" ? ansiColors.red : severity === "warning" ? ansiColors.yellow : ansiColors.magenta; + log(color(`\n${severity.toUpperCase()} ${category} (${findings.length})`)); + findings.slice(0, MAX_DETAILS_PER_CATEGORY).forEach((f) => { + const where = f.locale === INSTANCE_LOCALE ? "" : `[${f.locale}] `; + const ids = `source ${f.sourceID ?? "-"} → target ${f.targetID ?? "-"}`; + const label = [f.name, f.path].filter(Boolean).join(" / "); + log(` • ${where}${f.entity} ${label ? `"${label}" ` : ""}(${ids})`); + if (f.message) log(ansiColors.gray(` ${f.message}`)); + if (f.expected !== undefined || f.actual !== undefined) { + log(ansiColors.gray(` expected: ${formatValue(f.expected)}`)); + log(ansiColors.gray(` actual: ${formatValue(f.actual)}`)); + } + const link = options.linkFor?.(f); + if (link) log(ansiColors.gray(` ${link}`)); + }); + if (findings.length > MAX_DETAILS_PER_CATEGORY) { + log(ansiColors.gray(` …and ${findings.length - MAX_DETAILS_PER_CATEGORY} more (see the JSON report)`)); + } + }); + + const t = report.totals; + const verdictLine = `VERDICT: ${report.verdict.toUpperCase()} — ${t.failures} failure(s), ${t.warnings} warning(s), ${t.sourceDefects} source defect(s) (exit ${report.exitCode})`; + log(""); + log(report.exitCode === EXIT_PASS ? ansiColors.green(`✓ ${verdictLine}`) : ansiColors.red(`✗ ${verdictLine}`)); +} diff --git a/src/tests/sync/verify-after-sync.test.ts b/src/tests/sync/verify-after-sync.test.ts new file mode 100644 index 0000000..cb51692 --- /dev/null +++ b/src/tests/sync/verify-after-sync.test.ts @@ -0,0 +1,171 @@ +/** + * PROD-2552: `agility verify` against the output of a real (hermetic) sync. + * + * The sync half is the real push pipeline from full-instance.test.ts, extended with a page that + * carries two components in one zone. Its target is then projected to disk the way a pull would + * write it, and verify runs over the two caches with the pull replaced by a no-op — everything + * else (loaders, mapping files, checks, report) is the real code. + * + * 1. Right after the sync, verify passes (exit 0). + * 2. Swap the two components in the target page's zone — the Brightstar defect — and verify + * fails (exit 1) with a componentOrderMismatch naming both sequences. + */ +import * as fs from "fs"; +import * as path from "path"; +import { SyncHarness } from "./helpers/run-sync"; +import { installNetworkStubs, InstalledNetworkStubs } from "./helpers/network-stubs"; +import { + ALL_ELEMENTS, + CONTENT_HELLO, + FIXTURE_LOCALE, + fullInstanceFixture, + PAGE_HOME, +} from "./helpers/full-instance-fixture"; +import { Verify } from "core/verify"; +import { EXIT_DIFFERENCES, EXIT_PASS, VerifyReport } from "lib/verify/verify-report"; + +jest.mock("lib/pushers/batch-polling", () => ({ + pollBatchUntilComplete: jest.fn(), + extractContentBatchResults: jest.fn(), + extractPageBatchResults: jest.fn(), +})); +jest.mock("axios", () => ({ post: jest.fn(), get: jest.fn(), default: { post: jest.fn() } })); +jest.mock("lib/pushers/url-redirection-api", () => ({ + MAX_URL_REDIRECTION_BATCH_SIZE: 250, + saveUrlRedirections: jest.fn(), +})); + +const CONTENT_WORLD = { + ...CONTENT_HELLO, + contentID: 501, + properties: { ...CONTENT_HELLO.properties, versionID: 901, itemOrder: 1 }, + fields: { title: "World" }, +}; + +const PAGE_WITH_ZONE = { + ...PAGE_HOME, + zones: { + Main: [ + { module: "Post", item: { contentid: 500, fulllist: false } }, + { module: "Post", item: { contentid: 501, fulllist: false } }, + ], + }, +}; + +function fixture() { + const base = fullInstanceFixture(); + base.source[`${FIXTURE_LOCALE}/item/501.json`] = CONTENT_WORLD; + base.source[`${FIXTURE_LOCALE}/page/800.json`] = PAGE_WITH_ZONE; + return base; +} + +/** The target's nested sitemap, as a pull would write it: the fake does not keep one. */ +function writeTargetSitemap(harness: SyncHarness): void { + const pageMapping = (harness.readMappings("page", FIXTURE_LOCALE) ?? [])[0]; + const file = path.join(harness.rootPath, harness.targetGuid, FIXTURE_LOCALE, "nestedsitemap", "website.json"); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync( + file, + JSON.stringify([ + { + title: "Home", + name: "home", + pageID: pageMapping.targetPageID, + menuText: "Home", + visible: { menu: true, sitemap: true }, + path: "/home", + redirect: null, + isFolder: false, + }, + ]) + ); +} + +function targetPagePath(harness: SyncHarness): string { + const pageMapping = (harness.readMappings("page", FIXTURE_LOCALE) ?? [])[0]; + return path.join(harness.rootPath, harness.targetGuid, FIXTURE_LOCALE, "page", `${pageMapping.targetPageID}.json`); +} + +async function verify(harness: SyncHarness): Promise { + return new Verify( + { + sourceGuid: harness.sourceGuid, + targetGuid: harness.targetGuid, + locales: [FIXTURE_LOCALE], + models: "", + pages: "", + containers: "", + jsonSummary: path.join(harness.rootPath, "verify-report.json"), + rootPath: harness.rootPath, + }, + { + getStatus: async () => ({ inProgress: false, lastContentVersionID: 1, maxChangeDate: "x" }) as any, + // Hermetic: the caches are already on disk, so the pull is a no-op that reports success. + pull: async () => ({ + success: true, + results: [ + { guidProcessed: harness.sourceGuid, failed: [] }, + { guidProcessed: harness.targetGuid, failed: [] }, + ], + }), + print: false, + log: () => {}, + } + ).run(); +} + +describe("verify after a hermetic sync", () => { + let harness: SyncHarness; + let net: InstalledNetworkStubs; + let mappingsBefore: Record; + + beforeEach(async () => { + net = installNetworkStubs(); + jest.spyOn(console, "log").mockImplementation(() => {}); + jest.spyOn(console, "warn").mockImplementation(() => {}); + jest.spyOn(console, "error").mockImplementation(() => {}); + + harness = new SyncHarness({ elements: ALL_ELEMENTS, fixture: fixture() }); + const run = await harness.run(); + expect(run.totals.failed).toBe(0); + harness.fake.projectToDisk(harness.rootPath, harness.targetGuid); + writeTargetSitemap(harness); + mappingsBefore = harness.readAllMappings(); + }); + + afterEach(() => { + harness?.cleanup(); + net.restore(); + jest.restoreAllMocks(); + }); + + it("passes right after the sync", async () => { + const report = await verify(harness); + + expect(report.findings.filter((f) => f.severity === "failure")).toEqual([]); + expect(report.exitCode).toBe(EXIT_PASS); + expect(report.totals.entities.zone).toMatchObject({ checked: 1, matched: 1 }); + expect(report.totals.entities.content).toMatchObject({ checked: 2, matched: 2 }); + expect(net.fetchCalls).toEqual([]); + }); + + it("fails with componentOrderMismatch when the target zone is reordered", async () => { + const file = targetPagePath(harness); + const targetPage = JSON.parse(fs.readFileSync(file, "utf8")); + const zone = Object.keys(targetPage.zones)[0]; + targetPage.zones[zone].reverse(); + fs.writeFileSync(file, JSON.stringify(targetPage)); + + const report = await verify(harness); + + expect(report.exitCode).toBe(EXIT_DIFFERENCES); + const failures = report.findings.filter((f) => f.severity === "failure"); + expect(failures.map((f) => f.category)).toEqual(["componentOrderMismatch"]); + expect(failures[0].actual).toEqual([...(failures[0].expected as string[])].reverse()); + }); + + it("leaves the mapping files untouched", async () => { + await verify(harness); + expect(harness.readAllMappings()).toEqual(mappingsBefore); + }); +});