diff --git a/apps/importer/package.json b/apps/importer/package.json index d57a7221..e5ff5421 100644 --- a/apps/importer/package.json +++ b/apps/importer/package.json @@ -37,7 +37,7 @@ "lint": "eslint . --max-warnings 0", "format": "prettier --write \"src/**/*.ts\"", "backfill:specs": "tsx src/shopee/jobs/backfill-specs.ts", - "test": "tsx src/shopee/__tests__/open-core-intelligence.test.ts && tsx src/shopee/__tests__/entity-matcher.test.ts && tsx src/shopee/__tests__/shopee-to-supabase.test.ts && tsx src/shopee/__tests__/paging.test.ts && tsx src/shopee/__tests__/backfill-review-risk.test.ts && tsx src/shopee/__tests__/item-inventory-report.test.ts && tsx src/shopee/__tests__/arena-audit.test.ts && tsx src/shopee/test-ai-enricher.ts && tsx src/shopee/test-review-analyzer.ts", + "test": "tsx src/shopee/__tests__/open-core-intelligence.test.ts && tsx src/shopee/__tests__/entity-matcher.test.ts && tsx src/shopee/__tests__/item-matcher.test.ts && tsx src/shopee/__tests__/shopee-to-supabase.test.ts && tsx src/shopee/__tests__/paging.test.ts && tsx src/shopee/__tests__/backfill-review-risk.test.ts && tsx src/shopee/__tests__/item-inventory-report.test.ts && tsx src/shopee/__tests__/arena-audit.test.ts && tsx src/shopee/test-ai-enricher.ts && tsx src/shopee/test-review-analyzer.ts", "test:dist": "node src/shopee/__tests__/dist-exports.smoke.mjs", "backfill:review-risk": "tsx src/shopee/jobs/backfill-review-risk.ts" }, diff --git a/apps/importer/src/index.ts b/apps/importer/src/index.ts index a0ddd630..fb832767 100644 --- a/apps/importer/src/index.ts +++ b/apps/importer/src/index.ts @@ -12,3 +12,4 @@ export * from './shopee/ai-review-analyzer.js'; export * from './shopee/shopee-to-supabase.js'; export * from './shopee/jobs/backfill-specs.js'; export * from './shopee/paging.js'; +export * from './shopee/item-model-flag.js'; diff --git a/apps/importer/src/shopee/__tests__/dist-exports.smoke.mjs b/apps/importer/src/shopee/__tests__/dist-exports.smoke.mjs index 34f123cd..a7a985ba 100644 --- a/apps/importer/src/shopee/__tests__/dist-exports.smoke.mjs +++ b/apps/importer/src/shopee/__tests__/dist-exports.smoke.mjs @@ -25,6 +25,8 @@ for (const name of [ 'createImporterLogger', 'sanitizeAiGeneratedText', 'computeRetryDelayMs', + 'resolveItemModelMode', + 'UnkeyableProductTitleError', ]) { assert.equal( typeof crawlers[name], @@ -32,6 +34,12 @@ for (const name of [ `@repo/crawlers is missing export ${name}` ); } +// Item model rollout flag (DOS/Rate#1151): Rate-Services reads the same env name. +assert.equal(crawlers.ITEM_MODEL_ENV, 'RATE_ITEM_MODEL'); +assert.deepEqual([...crawlers.ITEM_MODEL_MODES], ['off', 'item', 'nullable_variant']); +assert.equal(crawlers.resolveItemModelMode(undefined), 'off'); +assert.equal(typeof crawlers.EntityMatcher.prototype.resolveItemAndVariant, 'function'); +assert.equal(crawlers.UNKEYED_MODEL_SEGMENT, 'unkeyed'); assert.equal( typeof crawlers.MAX_SPEC_CACHE_ENTRIES, 'number', diff --git a/apps/importer/src/shopee/__tests__/item-matcher.test.ts b/apps/importer/src/shopee/__tests__/item-matcher.test.ts new file mode 100644 index 00000000..e9804691 --- /dev/null +++ b/apps/importer/src/shopee/__tests__/item-matcher.test.ts @@ -0,0 +1,774 @@ +/** + * Item-first matching (DOS/Rate#1151, design doc 4.5): + * 1. RATE_ITEM_MODEL parsing (default off, typos read as off) + * 2. Normalizer Item fields: item_key/variant_key/variant_known/variant_source, no empty segment + * 3. Mode off: resolveItemAndVariant is today's master matcher, never touches rate.items + * 4. Modes item / nullable_variant: Item tiers, exact variant, size not stated (#1105), hints + * 5. Pipeline: item_id / variant_source / nullable cluster_id writes, archived masterIdHint, + * a manual variant is kept, unkeyable titles are rejected before any write + */ +import assert from 'node:assert/strict'; +import { composeModelKey, splitModelKey } from '@repo/shared-data'; +import { EntityMatcher } from '../entity-matcher.js'; +import { ITEM_MODEL_ENV, resolveItemModelMode } from '../item-model-flag.js'; +import { ProductNormalizer, UnkeyableProductTitleError } from '../product-normalizer.js'; +import { ShopeeIngestionPipeline } from '../shopee-to-supabase.js'; +import type { AiSpecEnricher } from '../ai-spec-enricher.js'; +import type { RateSupabaseClient, ShopeeProduct } from '../types.js'; + +type Row = Record; +type Result = { data: unknown; error: { message: string; code?: string } | null }; + +interface Call { + table: string; + op: 'select' | 'upsert' | 'insert' | 'update' | 'rpc'; + cols?: string; + payload?: Row; + filters: Array<[string, unknown]>; +} + +/** + * Chainable PostgREST stand-in. Every query resolves through `handler`, which sees the + * table, operation, selected columns, payload and eq filters; unhandled calls resolve empty. + */ +function createDb(handler: (call: Call) => Result | undefined) { + const calls: Call[] = []; + const run = (call: Call): Result => handler(call) ?? { data: null, error: null }; + const builder = (call: Call) => { + const b: Record = {}; + for (const name of ['neq', 'gt', 'gte', 'lte', 'lt', 'in', 'order', 'limit', 'is']) { + b[name] = () => b; + } + b.eq = (col: string, val: unknown) => { + call.filters.push([col, val]); + return b; + }; + b.select = (cols: string) => { + if (call.op === 'select') call.cols = cols; + return b; + }; + b.maybeSingle = () => Promise.resolve(run(call)); + b.single = () => Promise.resolve(run(call)); + b.then = (resolve: (r: Result) => unknown, reject?: (e: unknown) => unknown) => + Promise.resolve(run(call)).then(resolve, reject); + return b; + }; + const client = { + from(table: string) { + const start = (op: Call['op'], extra: Partial = {}) => { + const call: Call = { table, op, filters: [], ...extra }; + calls.push(call); + return builder(call); + }; + return { + select: (cols: string) => start('select', { cols }), + upsert: (payload: Row) => start('upsert', { payload }), + insert: (payload: Row) => start('insert', { payload }), + update: (payload: Row) => start('update', { payload }), + }; + }, + rpc(fn: string, args: Row) { + const call: Call = { table: fn, op: 'rpc', payload: args, filters: [] }; + calls.push(call); + return Promise.resolve(run(call)); + }, + } as unknown as RateSupabaseClient; + return { client, calls }; +} + +const filter = (call: Call, col: string) => call.filters.find(([c]) => c === col)?.[1]; + +/** Enricher stub: Tier 3 never calls an AI provider in these tests. */ +const passthroughEnricher = { + enrichCandidate: async (candidate: T) => candidate, +} as unknown as AiSpecEnricher; + +const normalizer = new ProductNormalizer(); +const candidateFor = (title: string) => normalizer.normalize({ title } as ShopeeProduct); + +interface ItemDb { + items?: Array<{ id: string; item_key: string }>; + variants?: Array<{ id: string; item_id: string; variant_key: string; status?: string }>; + matchItem?: Row[]; + matchItemErr?: { message: string }; + /** item_id the M1 trigger gives a newly seeded master; null = migration missing. */ + seededItemId?: string | null; +} + +/** Answers the matcher's reads and writes from an in-memory Item / variant table. */ +function itemDbAnswer(opts: ItemDb, call: Call): Result | undefined { + if (call.table === 'items' && call.op === 'select') { + const [[col, val] = []] = call.filters; + const row = (opts.items ?? []).find(i => (col === 'id' ? i.id : i.item_key) === val); + return { data: row ?? null, error: null }; + } + if (call.table === 'match_item') { + return { data: opts.matchItem ?? [], error: opts.matchItemErr ?? null }; + } + if (call.table === 'match_master_product') return { data: [], error: null }; + if (call.table === 'master_products' && call.op === 'select') { + const itemId = filter(call, 'item_id'); + if (itemId !== undefined) { + const rows = (opts.variants ?? []) + .filter(v => v.item_id === itemId) + .map(v => ({ id: v.id, variant_key: v.variant_key, status: v.status ?? 'published' })); + return { data: rows, error: null }; + } + if (call.cols === 'item_id') { + return { data: { item_id: opts.seededItemId ?? null }, error: null }; + } + return { data: null, error: null }; // Tier 1 model_key / race winner: not found + } + if (call.table === 'master_products' && call.op === 'upsert') { + return { data: [{ id: 'seeded-master' }], error: null }; + } + return undefined; +} + +const createItemDb = (opts: ItemDb = {}) => createDb(call => itemDbAnswer(opts, call)); + +const ZENBOOK_ITEM = { id: 'item-zenbook', item_key: 'asus:zenbook-s-14' }; +const ZENBOOK_1TB = { + id: 'master-zenbook-1tb', + item_id: 'item-zenbook', + variant_key: '1024gb', +}; + +async function run() { + console.log('🧪 Item-first matcher suite (DOS/Rate#1151)'); + + // 1. Flag parsing + assert.equal(ITEM_MODEL_ENV, 'RATE_ITEM_MODEL'); + assert.equal(resolveItemModelMode(undefined), 'off'); + assert.equal(resolveItemModelMode(''), 'off'); + assert.equal(resolveItemModelMode('item'), 'item'); + assert.equal(resolveItemModelMode(' Nullable-Variant '), 'nullable_variant'); + assert.equal(resolveItemModelMode('nullable'), 'off', 'a typo must never enable writes'); + assert.equal(resolveItemModelMode(true), 'off'); + console.log(' ✅ [1/5] RATE_ITEM_MODEL defaults to off'); + + // 2. Normalizer Item fields + const cases: Array<[string, Row]> = [ + [ + 'Điện thoại Apple iPhone 16 Pro Max 256GB Chính hãng VN/A', + { + model_key: 'apple:iphone-16-pro-max:256gb', + item_key: 'apple:iphone-16-pro-max', + variant_key: '256gb', + item_name: 'Apple iPhone 16 Pro Max', + item_slug: 'apple-iphone-16-pro-max', + variant_known: true, + variant_source: 'title', + keyable: true, + }, + ], + [ + 'Laptop ASUS Zenbook S 14 UX5406AA-SU348W', + { + model_key: 'asus:zenbook-s-14:unknown', + item_key: 'asus:zenbook-s-14', + variant_key: 'unknown', + variant_known: false, + variant_source: null, + keyable: true, + }, + ], + [ + 'Tai nghe Bluetooth Apple AirPods Pro 2 (USB-C) Chính hãng VN/A', + { + model_key: 'apple:airpods-pro-2:unknown', + variant_known: true, + variant_source: 'single_variant', + }, + ], + [ + 'Apple Watch Ultra 2 49mm GPS + Cellular', + { variant_key: '64gb', variant_known: true, variant_source: 'single_size' }, + ], + [ + 'Samsung', + { + model_key: 'samsung:unkeyed:unknown', + item_key: 'samsung:unkeyed', + keyable: false, + needs_review: true, + }, + ], + ['', { model_key: 'khac:unkeyed:unknown', keyable: false, brand: 'Khác', slug: 'khac' }], + ]; + for (const [title, expected] of cases) { + const c = candidateFor(title) as unknown as Row; + for (const [field, value] of Object.entries(expected)) { + assert.deepEqual(c[field], value, `${JSON.stringify(title)}: ${field}`); + } + const parts = splitModelKey(c.model_key as string); + assert.ok(parts, `${JSON.stringify(title)}: model_key has no empty segment`); + assert.equal(parts.itemKey, c.item_key); + assert.equal(parts.variantKey, c.variant_key); + assert.equal(composeModelKey(c.item_key as string, c.variant_key as string), c.model_key); + } + console.log(' ✅ [2/5] Normalizer emits item_key/variant_key and never an empty segment'); + + // 3. Mode off: today's matcher, no rate.items access + { + const db = createItemDb({ items: [ZENBOOK_ITEM], variants: [ZENBOOK_1TB] }); + const matcher = new EntityMatcher(db.client, passthroughEnricher, { itemModel: 'off' }); + const result = await matcher.resolveItemAndVariant( + candidateFor('Laptop ASUS Zenbook S 14 UX5406AA-SU348W'), + 'cat-laptops' + ); + assert.deepEqual(result, { + itemId: null, + masterId: 'seeded-master', + isNew: true, + variantSource: null, + }); + assert.ok(!db.calls.some(c => c.table === 'items' || c.table === 'match_item')); + assert.ok(db.calls.some(c => c.table === 'match_master_product')); + const seeded = db.calls.find(c => c.op === 'upsert'); + assert.equal(seeded?.payload?.model_key, 'asus:zenbook-s-14:unknown'); + assert.ok(!('item_id' in (seeded?.payload ?? {}))); + + const envDefault = process.env[ITEM_MODEL_ENV]; + delete process.env[ITEM_MODEL_ENV]; + assert.equal(new EntityMatcher(db.client, passthroughEnricher).itemModel, 'off'); + if (envDefault !== undefined) process.env[ITEM_MODEL_ENV] = envDefault; + + await assert.rejects( + matcher.resolveOrCreateMasterProduct(candidateFor('Samsung')), + UnkeyableProductTitleError + ); + } + console.log(' ✅ [3/5] Mode off keeps the master matcher and never reads rate.items'); + + // 4. Item modes + { + // Known size, exact variant of an exact Item. + const db = createItemDb({ + items: [{ id: 'item-ip16pm', item_key: 'apple:iphone-16-pro-max' }], + variants: [{ id: 'master-ip16pm-256', item_id: 'item-ip16pm', variant_key: '256gb' }], + }); + const matcher = new EntityMatcher(db.client, passthroughEnricher, { itemModel: 'item' }); + const result = await matcher.resolveItemAndVariant( + candidateFor('Điện thoại Apple iPhone 16 Pro Max 256GB Chính hãng VN/A') + ); + assert.deepEqual(result, { + itemId: 'item-ip16pm', + masterId: 'master-ip16pm-256', + isNew: false, + variantSource: 'title', + }); + assert.ok(!db.calls.some(c => c.table === 'match_master_product' || c.op === 'upsert')); + } + { + // Known size missing on a known Item: the variant is created under the Item's key. + const db = createItemDb({ items: [ZENBOOK_ITEM], variants: [ZENBOOK_1TB] }); + const matcher = new EntityMatcher(db.client, passthroughEnricher, { itemModel: 'item' }); + const result = await matcher.resolveItemAndVariant( + candidateFor('Laptop ASUS Zenbook S 14 UX5406AA 16GB 512GB SSD'), + 'cat-laptops' + ); + assert.equal(result.itemId, 'item-zenbook'); + assert.equal(result.masterId, 'seeded-master'); + assert.equal(result.isNew, true); + assert.equal( + db.calls.find(c => c.op === 'upsert')?.payload?.model_key, + 'asus:zenbook-s-14:512gb' + ); + } + { + // #1105: a storage-less title on a multi-size Item never lands on the sized variant. + const title = 'Laptop ASUS Zenbook S 14 UX5406AA-SU348W'; + const nullable = createItemDb({ items: [ZENBOOK_ITEM], variants: [ZENBOOK_1TB] }); + const nullableResult = await new EntityMatcher(nullable.client, passthroughEnricher, { + itemModel: 'nullable_variant', + }).resolveItemAndVariant(candidateFor(title)); + assert.deepEqual(nullableResult, { + itemId: 'item-zenbook', + masterId: null, + isNew: false, + variantSource: null, + }); + assert.ok(!nullable.calls.some(c => c.op === 'upsert' || c.table === 'match_master_product')); + + const itemMode = createItemDb({ items: [ZENBOOK_ITEM], variants: [ZENBOOK_1TB] }); + const itemResult = await new EntityMatcher(itemMode.client, passthroughEnricher, { + itemModel: 'item', + }).resolveItemAndVariant(candidateFor(title)); + assert.equal(itemResult.itemId, 'item-zenbook'); + assert.notEqual(itemResult.masterId, ZENBOOK_1TB.id); + assert.equal( + itemMode.calls.find(c => c.op === 'upsert')?.payload?.model_key, + 'asus:zenbook-s-14:unknown', + 'mode item keeps a variant: the Item’s unknown one' + ); + + const withUnknown = createItemDb({ + items: [ZENBOOK_ITEM], + variants: [ + ZENBOOK_1TB, + { id: 'master-zenbook-unknown', item_id: 'item-zenbook', variant_key: 'unknown' }, + ], + }); + const reuse = await new EntityMatcher(withUnknown.client, passthroughEnricher, { + itemModel: 'item', + }).resolveItemAndVariant(candidateFor(title)); + assert.equal(reuse.masterId, 'master-zenbook-unknown'); + + // An archived sized variant does not make the Item multi-size. + const archived = createItemDb({ + items: [ZENBOOK_ITEM], + variants: [ + { ...ZENBOOK_1TB, status: 'archived' }, + { id: 'master-zenbook-unknown', item_id: 'item-zenbook', variant_key: 'unknown' }, + ], + }); + const single = await new EntityMatcher(archived.client, passthroughEnricher, { + itemModel: 'nullable_variant', + }).resolveItemAndVariant(candidateFor(title)); + assert.equal(single.masterId, 'master-zenbook-unknown'); + } + { + // Single-variant Item (earbuds): its one variant, even in nullable mode. + const db = createItemDb({ + items: [{ id: 'item-app2', item_key: 'apple:airpods-pro-2' }], + variants: [{ id: 'master-app2', item_id: 'item-app2', variant_key: 'unknown' }], + }); + const result = await new EntityMatcher(db.client, passthroughEnricher, { + itemModel: 'nullable_variant', + }).resolveItemAndVariant(candidateFor('Tai nghe Apple AirPods Pro 2 (USB-C) Chính hãng VN/A')); + assert.deepEqual(result, { + itemId: 'item-app2', + masterId: 'master-app2', + isNew: false, + variantSource: 'single_variant', + }); + } + { + // Item Tier 2: fuzzy on Item names, then the variant under the matched Item's key. + const db = createItemDb({ + items: [{ id: 'item-s24u', item_key: 'samsung:galaxy-s24-ultra' }], + variants: [{ id: 'master-s24u-256', item_id: 'item-s24u', variant_key: '256gb' }], + matchItem: [ + { item_id: 'item-s24u', item_key: 'samsung:galaxy-s24-ultra', name: 'x', similarity: 0.82 }, + ], + }); + const candidate = { ...candidateFor('Samsung Galaxy S24 Ultra 5G 12GB/256GB Chính hãng') }; + candidate.model_key = 'samsung:galaxy-s24-ultra-5g:256gb'; + const result = await new EntityMatcher(db.client, passthroughEnricher, { + itemModel: 'item', + }).resolveItemAndVariant(candidate); + assert.equal(result.itemId, 'item-s24u'); + assert.equal(result.masterId, 'master-s24u-256'); + const rpc = db.calls.find(c => c.table === 'match_item'); + assert.equal(rpc?.payload?.search_title, candidate.item_name); + assert.ok(!/256/.test(String(rpc?.payload?.search_title)), 'Item search has no size'); + assert.equal(rpc?.payload?.p_threshold, 0.65); + } + { + // No Item: seed the master (draft when ambiguous); the M1 trigger gives its item_id. + const db = createItemDb({ + matchItem: [{ item_id: 'item-other', item_key: 'apple:iphone-15', similarity: 0.67 }], + seededItemId: 'item-new', + }); + const result = await new EntityMatcher(db.client, passthroughEnricher, { + itemModel: 'item', + }).resolveItemAndVariant(candidateFor('Apple iPhone 15 Plus 128GB Chính hãng VN/A')); + assert.deepEqual(result, { + itemId: 'item-new', + masterId: 'seeded-master', + isNew: true, + variantSource: 'title', + }); + assert.equal(db.calls.find(c => c.op === 'upsert')?.payload?.status, 'draft'); + + const noMigration = createItemDb({ seededItemId: null }); + await assert.rejects( + new EntityMatcher(noMigration.client, passthroughEnricher, { + itemModel: 'item', + }).resolveItemAndVariant(candidateFor('Apple iPhone 15 Plus 128GB')), + /DOS\/Rate#1150/ + ); + await assert.rejects( + new EntityMatcher( + createItemDb({ matchItemErr: { message: 'boom' } }).client, + passthroughEnricher, + { + itemModel: 'item', + } + ).resolveItemAndVariant(candidateFor('Apple iPhone 15 Plus 128GB')), + /Item Tier 2 fuzzy match failed: boom/ + ); + await assert.rejects( + new EntityMatcher(createItemDb().client, passthroughEnricher, { + itemModel: 'item', + }).resolveItemAndVariant(candidateFor('Samsung')), + UnkeyableProductTitleError + ); + } + { + // itemIdHint wins over the key; the variant still comes from the title. + const db = createItemDb({ + items: [ZENBOOK_ITEM], + variants: [ZENBOOK_1TB], + }); + const result = await new EntityMatcher(db.client, passthroughEnricher, { + itemModel: 'nullable_variant', + }).resolveItemAndVariant(candidateFor('ASUS Zenbook S14 OLED'), undefined, undefined, { + itemIdHint: 'item-zenbook', + }); + assert.deepEqual(result, { + itemId: 'item-zenbook', + masterId: null, + isNew: false, + variantSource: null, + }); + assert.ok(!db.calls.some(c => c.table === 'match_item')); + } + { + // A retired (archived) unknown variant is never seeded again or adopted. + const retired = { + items: [ZENBOOK_ITEM], + variants: [ + ZENBOOK_1TB, + { + id: 'master-zenbook-unknown', + item_id: 'item-zenbook', + variant_key: 'unknown', + status: 'archived', + }, + ], + }; + const title = 'Laptop ASUS Zenbook S 14 UX5406AA-SU348W'; + const itemDb = createItemDb(retired); + await assert.rejects( + new EntityMatcher(itemDb.client, passthroughEnricher, { + itemModel: 'item', + }).resolveItemAndVariant(candidateFor(title)), + /archived/ + ); + assert.ok(!itemDb.calls.some(c => c.op === 'upsert')); + const singleRetired = createItemDb({ + items: [ZENBOOK_ITEM], + variants: [{ ...retired.variants[1]! }], + }); + const none = await new EntityMatcher(singleRetired.client, passthroughEnricher, { + itemModel: 'nullable_variant', + }).resolveItemAndVariant(candidateFor(title)); + assert.equal(none.masterId, null); + assert.ok(!singleRetired.calls.some(c => c.op === 'upsert')); + } + { + // An archived sized variant takes no new listing in nullable mode; mode item keeps Tier 1's rule. + const archived1tb = { + items: [ZENBOOK_ITEM], + variants: [{ ...ZENBOOK_1TB, status: 'archived' }], + }; + const title = 'Laptop ASUS Zenbook S 14 UX5406AA 1TB'; + const nullable = await new EntityMatcher( + createItemDb(archived1tb).client, + passthroughEnricher, + { + itemModel: 'nullable_variant', + } + ).resolveItemAndVariant(candidateFor(title)); + assert.equal(nullable.masterId, null); + const itemMode = await new EntityMatcher( + createItemDb(archived1tb).client, + passthroughEnricher, + { + itemModel: 'item', + } + ).resolveItemAndVariant(candidateFor(title)); + assert.equal(itemMode.masterId, ZENBOOK_1TB.id); + } + { + // A fuzzy Item hit never creates a variant under that Item's key. + const db = createItemDb({ + items: [{ id: 'item-ip15p', item_key: 'apple:iphone-15-pro' }], + variants: [{ id: 'master-ip15p-128', item_id: 'item-ip15p', variant_key: '128gb' }], + matchItem: [ + { item_id: 'item-ip15p', item_key: 'apple:iphone-15-pro', name: 'x', similarity: 0.9 }, + ], + seededItemId: 'item-ip15pm', + }); + const result = await new EntityMatcher(db.client, passthroughEnricher, { + itemModel: 'item', + }).resolveItemAndVariant(candidateFor('Apple iPhone 15 Pro Max 256GB Chính hãng VN/A')); + assert.equal( + db.calls.find(c => c.op === 'upsert')?.payload?.model_key, + 'apple:iphone-15-pro-max:256gb' + ); + assert.equal(result.itemId, 'item-ip15pm'); + } + { + // A sizeless line keyed `unknown` reuses a seeded `standard` variant. + const db = createItemDb({ + items: [{ id: 'item-app2', item_key: 'apple:airpods-pro-2' }], + variants: [{ id: 'master-app2-std', item_id: 'item-app2', variant_key: 'standard' }], + }); + const result = await new EntityMatcher(db.client, passthroughEnricher, { + itemModel: 'item', + }).resolveItemAndVariant(candidateFor('Tai nghe Apple AirPods Pro 2 (USB-C) Chính hãng VN/A')); + assert.equal(result.masterId, 'master-app2-std'); + assert.ok(!db.calls.some(c => c.op === 'upsert')); + } + console.log(' ✅ [4/5] Item tiers, exact variants, size not stated (#1105) and hints'); + + // 5. Pipeline writes + const product = (title: string): ShopeeProduct => ({ + platform: 'shopee', + itemId: 'item-1151', + shopId: 'shop-1151', + title, + priceMin: 25_000_000, + priceMax: 25_000_000, + currency: 'VND', + historicalSold: 1, + ratingStar: 5, + ratingCount: 1, + images: [], + models: [], + attributes: [], + shop: { shopId: 'shop-1151', name: 'Shop', isMall: false, ratingStar: 5, followerCount: 1 }, + reviews: [], + url: 'https://shopee.vn/product/shop-1151/item-1151', + }); + const ingest = async ( + mode: 'off' | 'item' | 'nullable_variant', + title: string, + item: ItemDb, + extra: { hint?: Row | null; stored?: Row | null } = {}, + options: { masterIdHint?: string; itemIdHint?: string } = {} + ) => { + const db = createDb(call => { + if (call.table === 'categories') { + return { data: [{ id: 'cat-laptops', slug: 'laptops' }], error: null }; + } + if (call.table === 'merchants') return { data: { id: 'merchant-1' }, error: null }; + if (call.table === 'products' && call.op === 'upsert') { + return { data: { id: 'product-1' }, error: null }; + } + if (call.table === 'products' && call.op === 'select') { + return { data: extra.stored ?? null, error: null }; + } + if (call.table === 'record_price_observation') return { data: 'inserted', error: null }; + if ( + call.table === 'master_products' && + call.op === 'select' && + filter(call, 'id') !== undefined && + call.cols !== 'item_id' + ) { + return { data: extra.hint ?? null, error: null }; + } + return itemDbAnswer(item, call); + }); + const pipeline = new ShopeeIngestionPipeline({ supabaseClient: db.client, itemModel: mode }); + (pipeline as unknown as { matcher: EntityMatcher }).matcher = new EntityMatcher( + db.client, + passthroughEnricher, + { itemModel: mode } + ); + const result = await pipeline.ingestProduct(product(title), undefined, { + pricesOnly: true, + ...options, + }); + const listing = db.calls.find(c => c.table === 'products' && c.op === 'upsert')?.payload; + const rpcs = db.calls.filter(c => c.op === 'rpc').map(c => c.table); + return { result, listing, rpcs, calls: db.calls }; + }; + + const zenbook = { items: [ZENBOOK_ITEM], variants: [ZENBOOK_1TB] }; + const bareTitle = 'Laptop ASUS Zenbook S 14 UX5406AA-SU348W'; + { + const off = await ingest('off', 'Laptop ASUS Zenbook S 14 UX5406AA 1TB', zenbook); + assert.ok(off.listing && !('item_id' in off.listing) && !('variant_source' in off.listing)); + assert.equal(off.result.itemId, null); + assert.ok(!off.rpcs.includes('refresh_item_stats')); + assert.ok(!off.calls.some(c => c.table === 'items')); + } + { + const nullable = await ingest('nullable_variant', bareTitle, zenbook); + assert.equal(nullable.listing?.cluster_id, null); + assert.equal(nullable.listing?.item_id, 'item-zenbook'); + assert.equal(nullable.listing?.variant_source, null); + assert.deepEqual( + { masterId: nullable.result.masterId, itemId: nullable.result.itemId }, + { masterId: null, itemId: 'item-zenbook' } + ); + assert.ok(!nullable.rpcs.includes('refresh_master_market_stats')); + assert.ok(nullable.rpcs.includes('refresh_item_stats')); + + const sized = await ingest('item', 'Laptop ASUS Zenbook S 14 UX5406AA 1TB', zenbook); + assert.equal(sized.listing?.cluster_id, ZENBOOK_1TB.id); + assert.equal(sized.listing?.item_id, 'item-zenbook'); + assert.equal(sized.listing?.variant_source, 'title'); + assert.ok(sized.rpcs.includes('refresh_master_market_stats')); + } + { + // A variant set by hand survives a crawl that states no size. + const kept = await ingest('nullable_variant', bareTitle, zenbook, { + stored: { cluster_id: ZENBOOK_1TB.id, variant_source: 'manual' }, + }); + assert.equal(kept.listing?.cluster_id, ZENBOOK_1TB.id); + assert.ok(!('variant_source' in (kept.listing ?? {}))); + + // Mode item too: the Item's unknown variant never replaces a hand-set one. + const keptItem = await ingest( + 'item', + bareTitle, + { + ...zenbook, + variants: [ + ZENBOOK_1TB, + { id: 'master-zenbook-unknown', item_id: 'item-zenbook', variant_key: 'unknown' }, + ], + }, + { stored: { cluster_id: ZENBOOK_1TB.id, variant_source: 'manual' } } + ); + assert.equal(keptItem.listing?.cluster_id, ZENBOOK_1TB.id); + assert.ok(!('variant_source' in (keptItem.listing ?? {}))); + + // A hand-set variant wins even over a size stated in the title. + const manualWins = await ingest( + 'nullable_variant', + 'Laptop ASUS Zenbook S 14 UX5406AA 1TB', + zenbook, + { + stored: { cluster_id: 'master-zenbook-512', variant_source: 'manual' }, + } + ); + assert.equal(manualWins.listing?.cluster_id, 'master-zenbook-512'); + assert.ok(!('variant_source' in (manualWins.listing ?? {}))); + + // Title evidence stays when a later crawl has none; a legacy link does not. + const evidenceKept = await ingest('nullable_variant', bareTitle, zenbook, { + stored: { cluster_id: ZENBOOK_1TB.id, variant_source: 'title' }, + }); + assert.equal(evidenceKept.listing?.cluster_id, ZENBOOK_1TB.id); + const legacyDropped = await ingest('nullable_variant', bareTitle, zenbook, { + stored: { cluster_id: ZENBOOK_1TB.id, variant_source: 'legacy' }, + }); + assert.equal(legacyDropped.listing?.cluster_id, null); + + // A newer size stated in the title moves an evidence-based variant. + const moved = await ingest('item', 'Laptop ASUS Zenbook S 14 UX5406AA 1TB', zenbook, { + stored: { cluster_id: 'master-zenbook-512', variant_source: 'title' }, + }); + assert.equal(moved.listing?.cluster_id, ZENBOOK_1TB.id); + assert.equal(moved.listing?.variant_source, 'title'); + // ...and the variant it left is refreshed as well as the new one. + const refreshed = moved.calls + .filter(c => c.op === 'rpc' && c.table === 'refresh_master_market_stats') + .map(c => c.payload?.p_master_id); + assert.deepEqual(refreshed, [ZENBOOK_1TB.id, 'master-zenbook-512']); + + const left = await ingest('nullable_variant', bareTitle, zenbook, { + stored: { cluster_id: 'master-zenbook-unknown', variant_source: null }, + }); + assert.equal(left.listing?.cluster_id, null); + assert.deepEqual( + left.calls + .filter(c => c.op === 'rpc' && c.table === 'refresh_master_market_stats') + .map(c => c.payload?.p_master_id), + ['master-zenbook-unknown'] + ); + } + { + // An archived masterIdHint maps to its Item; a live one is kept. + const archived = await ingest( + 'nullable_variant', + bareTitle, + zenbook, + { + hint: { + id: 'master-zenbook-unknown', + specifications: { display: { size: 'x' } }, + status: 'archived', + item_id: 'item-zenbook', + variant_key: 'unknown', + }, + }, + { masterIdHint: 'master-zenbook-unknown' } + ); + assert.equal(archived.listing?.cluster_id, null); + assert.equal(archived.listing?.item_id, 'item-zenbook'); + + const live = await ingest( + 'item', + 'Laptop ASUS Zenbook S 14 UX5406AA 1TB', + zenbook, + { + hint: { + id: ZENBOOK_1TB.id, + specifications: { display: { size: 'x' } }, + status: 'published', + item_id: 'item-zenbook', + variant_key: '1024gb', + }, + }, + { masterIdHint: ZENBOOK_1TB.id } + ); + assert.equal(live.listing?.cluster_id, ZENBOOK_1TB.id); + assert.equal(live.listing?.variant_source, 'title'); + assert.ok(!live.calls.some(c => c.table === 'items'), 'a live hint skips matching'); + + const offHint = await ingest( + 'off', + bareTitle, + zenbook, + { hint: { id: 'master-zenbook-unknown', specifications: { display: { size: 'x' } } } }, + { masterIdHint: 'master-zenbook-unknown', itemIdHint: 'item-zenbook' } + ); + assert.equal(offHint.listing?.cluster_id, 'master-zenbook-unknown'); + assert.equal( + offHint.calls.find(c => c.table === 'master_products' && filter(c, 'id'))?.cols, + 'id, specifications' + ); + } + { + // itemIdHint for a storage-less page. + const hinted = await ingest( + 'nullable_variant', + 'ASUS Zenbook S14 OLED', + zenbook, + {}, + { + itemIdHint: 'item-zenbook', + } + ); + assert.equal(hinted.listing?.item_id, 'item-zenbook'); + assert.equal(hinted.listing?.cluster_id, null); + } + { + // An unkeyable title is rejected before the merchant write. + const db = createDb(call => + call.table === 'categories' ? { data: [], error: null } : undefined + ); + const pipeline = new ShopeeIngestionPipeline({ supabaseClient: db.client, itemModel: 'off' }); + await assert.rejects( + pipeline.ingestProduct(product('Samsung'), undefined, { pricesOnly: true }), + UnkeyableProductTitleError + ); + assert.ok(!db.calls.some(c => c.table === 'merchants')); + + // With a live masterIdHint the title's key is unused: the refresh still works. + const hintedRefresh = await ingest( + 'off', + 'Samsung', + zenbook, + { hint: { id: 'master-legacy', specifications: { display: { size: 'x' } } } }, + { masterIdHint: 'master-legacy' } + ); + assert.equal(hintedRefresh.listing?.cluster_id, 'master-legacy'); + } + console.log( + ' ✅ [5/5] Pipeline writes item_id, nullable cluster_id and keeps hand-set variants' + ); + console.log('🎉 All item-first matcher tests passed!'); +} + +run().catch(err => { + console.error(err); + process.exit(1); +}); diff --git a/apps/importer/src/shopee/entity-matcher.ts b/apps/importer/src/shopee/entity-matcher.ts index e5288704..35afb5f7 100644 --- a/apps/importer/src/shopee/entity-matcher.ts +++ b/apps/importer/src/shopee/entity-matcher.ts @@ -1,5 +1,14 @@ import crypto from 'node:crypto'; import { + UNKNOWN_VARIANT_KEY, + composeModelKey, + isItemKey, + isSizedVariantKey, + splitModelKey, + type VariantSource, +} from '@repo/shared-data'; +import { + UnkeyableProductTitleError, detectCategorySlug, type CanonicalCategorySlug, type NormalizedProductCandidate, @@ -7,6 +16,7 @@ import { import { AiSpecEnricher, mergeProductSpecifications } from './ai-spec-enricher.js'; import { mergeKeySpecs } from './jobs/backfill-specs.js'; import { createImporterLogger } from './logger.js'; +import { resolveItemModelMode, type ItemModelMode } from './item-model-flag.js'; import type { RateSupabaseClient } from './types.js'; const logger = createImporterLogger('EntityMatcher'); @@ -132,14 +142,322 @@ export function toBoundedPositiveNumber(value: unknown, max: number): number | n return Number.isFinite(n) && n > 0 && n <= max ? n : null; } +export interface EntityMatcherOptions { + /** Item model rollout mode; defaults to the `RATE_ITEM_MODEL` env (see item-model-flag.ts). */ + itemModel?: ItemModelMode; +} + +export interface ItemMatchResult { + /** The listing's Item; null only in mode `off`, where Items are not read. */ + itemId: string | null; + /** The listing's variant (`cluster_id`); null = no variant, only in mode `nullable_variant`. */ + masterId: string | null; + isNew: boolean; + /** `products.variant_source`; null when the listing does not tell its variant. */ + variantSource: VariantSource | null; +} + +type MatchedItem = { id: string; itemKey: string }; + export class EntityMatcher { private enricher: AiSpecEnricher; + public readonly itemModel: ItemModelMode; constructor( private supabase: RateSupabaseClient, - enricher?: AiSpecEnricher + enricher?: AiSpecEnricher, + options: EntityMatcherOptions = {} ) { this.enricher = enricher ?? new AiSpecEnricher(); + this.itemModel = options?.itemModel ?? resolveItemModelMode(); + } + + /** + * Item-first matching (DOS/Rate#1151, design doc section 4.5). In mode `off` this is + * exactly `resolveOrCreateMasterProduct`. Otherwise: + * 1. Item by `itemIdHint`, then by exact `item_key`, then fuzzy on Item names + * (`match_item`). Item names carry no storage, so a bare title can no longer + * land on a sized variant (#1105). + * 2. Known size: the variant by exact `(item_id, variant_key)`, created when missing. + * 3. Size not stated: a single-variant Item's one variant. On a multi-size Item, no + * variant in mode `nullable_variant`, the Item's `unknown` variant in mode `item`. + * 4. No Item: seed a master as today; the M1 sync trigger creates its draft Item. + */ + public async resolveItemAndVariant( + candidate: NormalizedProductCandidate, + categoryId?: string, + rawTitle?: string, + options: { itemIdHint?: string | null } = {} + ): Promise { + if (this.itemModel === 'off') { + const { masterId, isNew } = await this.resolveOrCreateMasterProduct( + candidate, + categoryId, + rawTitle + ); + return { itemId: null, masterId, isNew, variantSource: null }; + } + + const input = this.sanitizeCandidateInput(candidate, categoryId, rawTitle); + candidate = input.candidate; + const { safeModelKey, safeCategoryId, safeRawTitle } = input; + const keys = splitModelKey(safeModelKey); + if (!keys) { + throw new Error( + `[EntityMatcher] model_key is not in brand:model:variant shape: ${JSON.stringify(safeModelKey)}` + ); + } + const variantKnown = + typeof candidate.variant_known === 'boolean' + ? candidate.variant_known + : keys.variantKey !== UNKNOWN_VARIANT_KEY || candidate.category_slug === 'earbuds'; + const variantSource: VariantSource | null = variantKnown + ? (candidate.variant_source ?? + (keys.variantKey === UNKNOWN_VARIANT_KEY ? 'single_variant' : 'title')) + : null; + + // 1. Item: caller hint, exact item_key, then fuzzy on Item names. + let item: MatchedItem | null = null; + let ambiguousFuzzy = false; + const safeItemIdHint = sanitizeMatcherEntityId(options?.itemIdHint, 128); + if (safeItemIdHint) { + item = await this.findItem('id', safeItemIdHint); + if (item) { + logger.info('entity_match.item_hint', `Item from caller hint: ${item.itemKey}`, { + itemId: item.id, + }); + } else { + logger.warn('entity_match.item_hint_missing', `itemIdHint ${safeItemIdHint} not found`, { + itemIdHint: safeItemIdHint, + }); + } + } + if (!item) { + item = await this.findItem('item_key', keys.itemKey); + if (item) { + logger.info('entity_match.item_tier1_exact', `Item Tier 1 exact: ${item.itemKey}`, { + itemId: item.id, + }); + } + } + let fuzzyItem = false; + if (!item) { + const fuzzy = await this.fuzzyMatchItem(candidate); + item = fuzzy.item; + fuzzyItem = Boolean(fuzzy.item); + ambiguousFuzzy = fuzzy.ambiguous; + } + + // 4. No Item: seed the master under the candidate's own key. A new Item's one + // variant (sized or `unknown`) is its only variant until a sized one appears. + const seedOwnKey = async (): Promise => { + const seeded = await this.seedMasterProduct( + candidate, + safeModelKey, + safeCategoryId, + safeRawTitle, + ambiguousFuzzy + ); + const itemId = await this.itemIdOfMaster(seeded.masterId); + return { itemId, masterId: seeded.masterId, isNew: seeded.isNew, variantSource }; + }; + if (!item) return seedOwnKey(); + if (!isItemKey(item.itemKey)) { + throw new Error( + `[EntityMatcher] Item ${item.id} has no brand:model key (${JSON.stringify(item.itemKey)}); cannot place a variant under it` + ); + } + + const variants = await this.listItemVariants(item.id); + let targetVariantKey = keys.variantKey; + + if (variantKnown) { + // 2. Known size: exact (item_id, variant_key), as Tier 1 matches an exact model_key. + // A sizeless line keyed `unknown` also takes a live `standard` variant (seed keys). + const exact = + variants.find(v => v.variant_key === keys.variantKey) ?? + (keys.variantKey === UNKNOWN_VARIANT_KEY + ? variants.find(v => v.variant_key === 'standard' && v.status !== 'archived') + : undefined); + // An archived variant takes no new listings once variants may be NULL; mode + // item keeps Tier 1's rule (it links to the existing row). + if (exact?.status === 'archived' && this.itemModel === 'nullable_variant') { + logger.info( + 'entity_match.variant_archived', + `Variant ${exact.variant_key} of ${item.itemKey} is archived: no variant`, + { itemId: item.id, masterId: exact.id } + ); + return { itemId: item.id, masterId: null, isNew: false, variantSource: null }; + } + if (exact) { + logger.info('entity_match.variant_exact', `Variant ${keys.variantKey} of ${item.itemKey}`, { + itemId: item.id, + masterId: exact.id, + }); + return { itemId: item.id, masterId: exact.id, isNew: false, variantSource }; + } + } else { + // 3. Size not stated. Multi-size = a live sized variant (design doc 4.2). + const live = variants.filter(v => v.status !== 'archived'); + const multiSize = live.some(v => isSizedVariantKey(v.variant_key)); + if (multiSize && this.itemModel === 'nullable_variant') { + logger.info( + 'entity_match.variant_unknown', + `Size not stated on multi-size Item ${item.itemKey}: no variant`, + { itemId: item.id } + ); + return { itemId: item.id, masterId: null, isNew: false, variantSource: null }; + } + const unknownVariant = live.find(v => v.variant_key === UNKNOWN_VARIANT_KEY); + const only = multiSize + ? unknownVariant + : (unknownVariant ?? + live.find(v => v.variant_key === 'standard') ?? + (live.length === 1 ? live[0] : undefined)); + if (only) { + logger.info( + 'entity_match.variant_size_not_stated', + `Size not stated: variant ${only.variant_key} of ${item.itemKey}`, + { itemId: item.id, masterId: only.id, multiSize } + ); + return { itemId: item.id, masterId: only.id, isNew: false, variantSource: null }; + } + targetVariantKey = UNKNOWN_VARIANT_KEY; + // A retired `unknown` variant (M3) is never seeded again: the upsert would + // adopt the archived row. + if (variants.some(v => v.variant_key === UNKNOWN_VARIANT_KEY)) { + if (this.itemModel === 'nullable_variant') { + return { itemId: item.id, masterId: null, isNew: false, variantSource: null }; + } + throw new Error( + `[EntityMatcher] ${item.itemKey}:unknown is archived; RATE_ITEM_MODEL=item cannot place a listing with no stated size` + ); + } + } + + // A fuzzy Item hit only links to variants that exist, as Tier 2 only links to + // existing masters: the listing's own name and model may not be the Item's + // ("iPhone 15 Pro Max" ~ "iPhone 15 Pro"), so a new variant gets its own key. + if (fuzzyItem) return seedOwnKey(); + + // Missing variant of a known Item: create it under the Item's own key. + const variantModelKey = composeModelKey(item.itemKey, targetVariantKey); + const seeded = await this.seedMasterProduct( + { ...candidate, model_key: variantModelKey }, + variantModelKey, + safeCategoryId, + safeRawTitle, + ambiguousFuzzy + ); + logger.info( + 'entity_match.variant_created', + `Variant ${targetVariantKey} of ${item.itemKey}: ${seeded.masterId}`, + { itemId: item.id, masterId: seeded.masterId, isNew: seeded.isNew } + ); + return { itemId: item.id, masterId: seeded.masterId, isNew: seeded.isNew, variantSource }; + } + + private async findItem(column: 'id' | 'item_key', value: string): Promise { + const { data, error } = await this.supabase + .from('items') + .select('id, item_key') + .eq(column, value) + .maybeSingle(); + if (error) { + throw new Error(`[EntityMatcher] Item lookup by ${column} failed: ${error.message}`); + } + const row = data as { id?: unknown; item_key?: unknown } | null; + const id = sanitizeMatcherEntityId(row?.id, 128); + return id ? { id, itemKey: typeof row?.item_key === 'string' ? row.item_key : '' } : null; + } + + /** Item Tier 2: pg_trgm on Item names through `rate.match_item` (design doc 4.5). */ + private async fuzzyMatchItem( + candidate: NormalizedProductCandidate + ): Promise<{ item: MatchedItem | null; ambiguous: boolean }> { + const searchTitle = + typeof candidate.item_name === 'string' && candidate.item_name.trim() + ? candidate.item_name + : `${candidate.brand} ${candidate.model}`; + const { data, error } = await this.supabase.rpc('match_item', { + search_title: searchTitle, + p_brand: candidate.brand, + p_model: candidate.model, + p_threshold: ENTITY_MATCH_CANDIDATE_FLOOR, + }); + // As in Tier 2: an RPC failure aborts instead of seeding a duplicate Item. + if (error) { + throw new Error(`[EntityMatcher] Item Tier 2 fuzzy match failed: ${error.message}`); + } + const rows = Array.isArray(data) + ? data.filter(m => { + if (!m || typeof m !== 'object' || Array.isArray(m)) return false; + if (!sanitizeMatcherEntityId(m.item_id, 128)) return false; + const sim = typeof m.similarity === 'boolean' ? Number.NaN : Number(m.similarity); + return Number.isFinite(sim) && sim >= ENTITY_MATCH_CANDIDATE_FLOOR && sim <= 1; + }) + : []; + const top = rows[0]; + if (!top) return { item: null, ambiguous: false }; + const sim = Number(top.similarity); + if (sim < ENTITY_MATCH_ACCEPT_THRESHOLD) { + logger.warn( + 'entity_match.item_tier2_ambiguous', + `Item Tier 2 best similarity ${(sim * 100).toFixed(1)}% < ${(ENTITY_MATCH_ACCEPT_THRESHOLD * 100).toFixed(0)}%: seeding as draft if new.`, + { similarity: sim, modelKey: candidate.model_key } + ); + return { item: null, ambiguous: true }; + } + const item = { + id: sanitizeMatcherEntityId(top.item_id, 128), + itemKey: typeof top.item_key === 'string' ? top.item_key : '', + }; + logger.info( + 'entity_match.item_tier2_fuzzy', + `Item Tier 2: "${searchTitle}" ~= "${top.name}" (similarity: ${(sim * 100).toFixed(1)}%)`, + { itemId: item.id, similarity: sim } + ); + return { item, ambiguous: false }; + } + + private async listItemVariants( + itemId: string + ): Promise> { + const { data, error } = await this.supabase + .from('master_products') + .select('id, variant_key, status') + .eq('item_id', itemId) + .limit(200); + if (error) { + throw new Error(`[EntityMatcher] Variant lookup for Item ${itemId} failed: ${error.message}`); + } + return (Array.isArray(data) ? data : []).flatMap(row => { + const id = sanitizeMatcherEntityId(row?.id, 128); + return id && typeof row.variant_key === 'string' + ? [{ id, variant_key: row.variant_key, status: row.status ?? null }] + : []; + }); + } + + /** The Item the M1 sync trigger linked to a master; fails loudly when M1 is missing. */ + private async itemIdOfMaster(masterId: string): Promise { + const { data, error } = await this.supabase + .from('master_products') + .select('item_id') + .eq('id', masterId) + .maybeSingle(); + if (error) { + throw new Error( + `[EntityMatcher] item_id lookup for master ${masterId} failed: ${error.message}` + ); + } + const itemId = sanitizeMatcherEntityId((data as { item_id?: unknown } | null)?.item_id, 128); + if (!itemId) { + throw new Error( + `[EntityMatcher] master ${masterId} has no item_id: the Item migration (DOS/Rate#1150) is not applied` + ); + } + return itemId; } /** @@ -150,26 +468,9 @@ export class EntityMatcher { categoryId?: string, rawTitle?: string ): Promise<{ masterId: string; isNew: boolean }> { - if (!candidate || typeof candidate !== 'object' || Array.isArray(candidate)) { - throw new Error('[EntityMatcher] candidate must be a non-null object'); - } - const safeModelKey = String(candidate.model_key ?? '') - .replace(/\p{Cc}+/gu, '') - .trim() - .slice(0, 200); - if (!safeModelKey) { - throw new Error('[EntityMatcher] candidate.model_key must be a non-empty string'); - } - const safeCategoryId = sanitizeMatcherEntityId(categoryId, 128) || undefined; - const safeRawTitle = - typeof rawTitle === 'string' - ? rawTitle - .replace(/\p{Cc}+/gu, ' ') - .replace(/\s+/g, ' ') - .trim() - .slice(0, 500) - : undefined; - candidate = { ...candidate, model_key: safeModelKey }; + const input = this.sanitizeCandidateInput(candidate, categoryId, rawTitle); + candidate = input.candidate; + const { safeModelKey, safeCategoryId, safeRawTitle } = input; // Tier 1: Deterministic model_key matching const { data: exactMatch, error: exactErr } = await this.supabase @@ -246,6 +547,63 @@ export class EntityMatcher { ); } + return this.seedMasterProduct( + candidate, + safeModelKey, + safeCategoryId, + safeRawTitle, + ambiguousFuzzy + ); + } + + /** Validates and bounds the matcher input; throws on a missing candidate or key. */ + private sanitizeCandidateInput( + candidate: NormalizedProductCandidate, + categoryId?: string, + rawTitle?: string + ): { + candidate: NormalizedProductCandidate; + safeModelKey: string; + safeCategoryId: string | undefined; + safeRawTitle: string | undefined; + } { + if (!candidate || typeof candidate !== 'object' || Array.isArray(candidate)) { + throw new Error('[EntityMatcher] candidate must be a non-null object'); + } + const safeModelKey = String(candidate.model_key ?? '') + .replace(/\p{Cc}+/gu, '') + .trim() + .slice(0, 200); + if (!safeModelKey) { + throw new Error('[EntityMatcher] candidate.model_key must be a non-empty string'); + } + if (candidate.keyable === false) { + throw new UnkeyableProductTitleError(rawTitle ?? candidate.name); + } + const safeCategoryId = sanitizeMatcherEntityId(categoryId, 128) || undefined; + const safeRawTitle = + typeof rawTitle === 'string' + ? rawTitle + .replace(/\p{Cc}+/gu, ' ') + .replace(/\s+/g, ' ') + .trim() + .slice(0, 500) + : undefined; + candidate = { ...candidate, model_key: safeModelKey }; + return { candidate, safeModelKey, safeCategoryId, safeRawTitle }; + } + + /** + * Tier 3: no existing master matches. Enriches the candidate (AI Spec Enricher, + * unless it is in the local benchmark registry) and seeds a new master. + */ + private async seedMasterProduct( + candidate: NormalizedProductCandidate, + safeModelKey: string, + safeCategoryId: string | undefined, + safeRawTitle: string | undefined, + ambiguousFuzzy: boolean + ): Promise<{ masterId: string; isNew: boolean }> { // Tier 3: No existing master product matches -> Enrich with AI Spec Enricher (if not in local benchmark registry) // and seed a new Master Product. candidate = await this.enricher.enrichCandidate(candidate, safeRawTitle); diff --git a/apps/importer/src/shopee/item-model-flag.ts b/apps/importer/src/shopee/item-model-flag.ts new file mode 100644 index 00000000..37378105 --- /dev/null +++ b/apps/importer/src/shopee/item-model-flag.ts @@ -0,0 +1,27 @@ +/** + * Rollout flag for the Item / variant / listing model (DOS/Rate#1151; design doc + * `Docs/Proposals/2026-10-27-item-variant-listing-model.md`, sections 4.5 and 10). + * + * `RATE_ITEM_MODEL` selects how the importer resolves and writes a listing: + * - `off` (default): today's behaviour. Master-level matching only, nothing + * reads or writes `rate.items`, `item_id` or `variant_source`. Safe on a + * database without the DOS-Me M1+M2 migration (#1150). + * - `item`: item-first matching (`match_item`) and `item_id`/`variant_source` + * writes. Needs M1+M2. Every listing still gets a variant: a listing with + * no stated size on a multi-size Item goes to that Item's `unknown` variant. + * - `nullable_variant`: as `item`, and such a listing is written with + * `cluster_id = NULL`. Turn on only after DOS/Rate-Services#670 part 1 has + * merged, so every consumer treats a NULL variant as "no variant". + * + * Any other value reads as `off`, so a typo never starts writing new columns. + */ +export const ITEM_MODEL_ENV = 'RATE_ITEM_MODEL'; + +export const ITEM_MODEL_MODES = ['off', 'item', 'nullable_variant'] as const; +export type ItemModelMode = (typeof ITEM_MODEL_MODES)[number]; + +export function resolveItemModelMode(raw: unknown = process.env[ITEM_MODEL_ENV]): ItemModelMode { + if (typeof raw !== 'string') return 'off'; + const value = raw.trim().toLowerCase().replace(/-/g, '_'); + return (ITEM_MODEL_MODES as readonly string[]).includes(value) ? (value as ItemModelMode) : 'off'; +} diff --git a/apps/importer/src/shopee/product-normalizer.ts b/apps/importer/src/shopee/product-normalizer.ts index 927ae5b9..a0eb635b 100644 --- a/apps/importer/src/shopee/product-normalizer.ts +++ b/apps/importer/src/shopee/product-normalizer.ts @@ -1,4 +1,10 @@ -import { MAX_VALID_VND_PRICE, type ProductSpecifications, type KeySpecs } from '@repo/shared-data'; +import { + MAX_VALID_VND_PRICE, + UNKNOWN_VARIANT_KEY, + type ProductSpecifications, + type KeySpecs, + type VariantSource, +} from '@repo/shared-data'; import type { ShopeeProduct } from './types.js'; export const MAX_VALID_RAM_GB = 2048; @@ -59,6 +65,31 @@ export interface NormalizedProductCandidate { model: string; variant_name: string; model_key: string; + /** + * Item fields (DOS/Rate#1151). `normalize()` always sets them; they are optional + * so candidates built elsewhere stay valid, and the matcher derives the keys + * from `model_key` when they are missing. + * `item_key` = `brand:model`, the first two `model_key` segments. + */ + item_key?: string; + /** The literal third `model_key` segment: `256gb`, `1024gb`, `unknown`. */ + variant_key?: string; + /** Item name and slug, never with a storage size. */ + item_name?: string; + item_slug?: string; + /** + * The listing's variant is known: a stated or single-size storage, or a line + * with no storage axis (earbuds). False only for "storage not stated". + */ + variant_known?: boolean; + /** How the variant was found; null when it is not known. */ + variant_source?: Extract | null; + /** + * False when the title gives no model segment. The key then carries the + * `unkeyed` placeholder model instead of an empty segment (`samsung::unknown`), + * and the pipeline and matcher reject the candidate (UnkeyableProductTitleError). + */ + keyable?: boolean; name: string; slug: string; storage_gb: number | null; // null = unparseable from the title; identity carries ':unknown' @@ -83,6 +114,24 @@ export interface NormalizedProductCandidate { }; } +/** Model segment of a title that names no model (`keyable: false`). */ +export const UNKEYED_MODEL_SEGMENT = 'unkeyed'; + +/** + * A title with no model to key on (e.g. it would key as `samsung::unknown`). The + * listing has no identity to match, so it is rejected before anything is written + * (DOS/Rate#1151). `normalize()` itself never throws; it marks `keyable: false`. + */ +export class UnkeyableProductTitleError extends Error { + readonly code = 'UNKEYABLE_PRODUCT_TITLE'; + constructor(title: string) { + super( + `Cannot key product title (no model segment): ${JSON.stringify(String(title ?? '').slice(0, 200))}` + ); + this.name = 'UnkeyableProductTitleError'; + } +} + /** * Detect listing physical condition from product title and optional description (Audit C6). * Strips promotional trade-in phrases ("thu cũ đổi mới") before matching so new retail listings @@ -1632,11 +1681,13 @@ export class ProductNormalizer { let storageGb: number | null = null; let storageConfident = false; + let variantSource: NormalizedProductCandidate['variant_source'] = null; const marker = romExplicit ?? romLoose; if (categorySlug === 'earbuds') { // Earbuds have no storage variant and must key as :unknown with null storage (DOS/Rate#643, #657). storageGb = null; storageConfident = true; + variantSource = 'single_variant'; } else if (marker) { const candidateStorage = toValidStorageGb( parseFloat(marker[1].replace(',', '.')) * (marker[2].toUpperCase() === 'TB' ? 1024 : 1) @@ -1660,8 +1711,10 @@ export class ProductNormalizer { } else if (storageCandidates.length === 0 && benchmark?.only_storage_gb != null) { storageGb = benchmark.only_storage_gb; storageConfident = true; + variantSource = 'single_size'; } const storageKnown = storageGb !== null; + if (storageKnown && variantSource === null) variantSource = 'title'; // 4. Extract Colors from models / variations const colors: string[] = []; @@ -1695,7 +1748,13 @@ export class ProductNormalizer { // 5. Construct keys & slugs. Unknown storage keeps identity space explicit // (':unknown') instead of guessing a default (plan v7 §5.2). - const modelKey = storageKnown ? `${baseKey}:${storageGb}gb` : `${baseKey}:unknown`; + // An empty key segment ("samsung::unknown") is never emitted: a title with no + // model keys on a placeholder and is marked unkeyable (DOS/Rate#1151). + const keyable = Boolean(slugify(brand) && slugify(model)); + const itemKey = keyable ? baseKey : `${slugify(brand) || 'khac'}:${UNKEYED_MODEL_SEGMENT}`; + const variantKey = storageKnown ? `${storageGb}gb` : UNKNOWN_VARIANT_KEY; + const modelKey = `${itemKey}:${variantKey}`; + const itemName = `${brand} ${model}`.trim(); const storageDisplay = categorySlug === 'earbuds' ? 'Standard' @@ -2075,10 +2134,17 @@ export class ProductNormalizer { model, variant_name: storageDisplay, model_key: modelKey, + item_key: itemKey, + variant_key: variantKey, + item_name: itemName, + item_slug: slugify(itemName), + variant_known: storageKnown || categorySlug === 'earbuds', + variant_source: variantSource, + keyable, name: canonicalName, slug, storage_gb: storageKnown ? storageGb : null, - needs_review: !storageConfident || condition !== 'new', + needs_review: !keyable || !storageConfident || condition !== 'new', condition, origin, ram_gb: ramGb, diff --git a/apps/importer/src/shopee/shopee-to-supabase.ts b/apps/importer/src/shopee/shopee-to-supabase.ts index 8f21d007..85ddf2b7 100644 --- a/apps/importer/src/shopee/shopee-to-supabase.ts +++ b/apps/importer/src/shopee/shopee-to-supabase.ts @@ -16,8 +16,14 @@ import { type SupportedEcommercePlatform, type ValidatedScrapedProduct, } from './types.js'; -import { ProductNormalizer } from './product-normalizer.js'; -import { EntityMatcher, sanitizeHttpUrl, sanitizeHttpUrlList } from './entity-matcher.js'; +import { ProductNormalizer, UnkeyableProductTitleError } from './product-normalizer.js'; +import { + EntityMatcher, + sanitizeHttpUrl, + sanitizeHttpUrlList, + type ItemMatchResult, +} from './entity-matcher.js'; +import type { ItemModelMode } from './item-model-flag.js'; import { AiSpecEnricher, type AiSpecEnricherConfig } from './ai-spec-enricher.js'; import { AiReviewAnalyzer, type ReviewInputItem } from './ai-review-analyzer.js'; import { createImporterLogger } from './logger.js'; @@ -249,6 +255,8 @@ export interface ShopeeIngestionPipelineConfig extends AiSpecEnricherConfig { * Optional pre-scoped `rate` schema Supabase client for dependency injection / deterministic testing. */ supabaseClient?: RateSupabaseClient; + /** Item model rollout mode; defaults to the `RATE_ITEM_MODEL` env (see item-model-flag.ts). */ + itemModel?: ItemModelMode; } export interface IngestOptions { @@ -257,6 +265,12 @@ export interface IngestOptions { * otherwise the listing goes through entity matching as usual. */ masterIdHint?: string | null; + /** + * Item this listing is known to belong to, for a page that states no size + * (DOS/Rate#1151). Its variant still comes from the title. Ignored while + * `RATE_ITEM_MODEL` is `off`, and when `masterIdHint` resolves. + */ + itemIdHint?: string | null; /** * When true, skips Step 6 review re-analysis so price-only (affiliate) ingests * do not re-score stored reviews without reviewer metadata or pay an extra AI pass. @@ -299,7 +313,9 @@ export class ShopeeIngestionPipeline { this.normalizer = new ProductNormalizer(); this.enricher = new AiSpecEnricher(aiConfig); this.reviewAnalyzer = new AiReviewAnalyzer(aiConfig); - this.matcher = new EntityMatcher(this.supabase, this.enricher); + this.matcher = new EntityMatcher(this.supabase, this.enricher, { + itemModel: aiConfig?.itemModel, + }); } /** @@ -405,7 +421,10 @@ export class ShopeeIngestionPipeline { ): Promise<{ productId: string; merchantId: string; - masterId: string; + /** The listing's variant; null only in `RATE_ITEM_MODEL=nullable_variant` (no variant). */ + masterId: string | null; + /** The listing's Item; null while `RATE_ITEM_MODEL` is `off`. */ + itemId: string | null; platform: SupportedEcommercePlatform; }> { if (!options || typeof options !== 'object' || Array.isArray(options)) options = {}; @@ -431,6 +450,15 @@ export class ShopeeIngestionPipeline { shopId: product.shop.shopId, }); + // Normalize first: a title with no keyable brand or model is rejected + // (UnkeyableProductTitleError) before anything is written. + const candidate = this.normalizer.normalize(product); + // With a masterIdHint the title's key is not used; the matcher still rejects + // the candidate if the hint does not resolve. + if (candidate.keyable === false && !options.masterIdHint) { + throw new UnkeyableProductTitleError(product.title); + } + // 1. Upsert merchant const entityHash = hashEntity(`${platform}:${product.shop.shopId}`); const { data: merchant, error: merchantErr } = await this.supabase @@ -465,8 +493,7 @@ export class ShopeeIngestionPipeline { ); await report?.('merchant'); - // 2. Normalize Specs & Match / Seed Master Product - const candidate = this.normalizer.normalize(product); + // 2. Match / Seed Master Product const resolvedCategoryId = this.categoryIdBySlug.get(candidate.category_slug || 'smartphones') || this.smartphoneCategoryId; @@ -485,22 +512,64 @@ export class ShopeeIngestionPipeline { // A caller that already verified the master (coverage discovery, or a refresh of // a listing that is already clustered) keeps the listing on it instead of // re-matching every crawl, which can split it onto a new draft master. - let hinted: { masterId: string; isNew: boolean } | null = null; + const itemModel = this.matcher.itemModel; + let hinted: ItemMatchResult | null = null; + // undefined = leave the stored variant_source as it is. + let hintedVariantSource: ItemMatchResult['variantSource'] | undefined; + let safeItemIdHint = + typeof options.itemIdHint === 'string' + ? (sanitizeIngestionEntityId(options.itemIdHint, 128) ?? '') + : ''; + if (safeItemIdHint && itemModel === 'off') { + platLogger.warn('ingest.item_hint_ignored', 'itemIdHint ignored: RATE_ITEM_MODEL is off', { + itemIdHint: safeItemIdHint, + }); + safeItemIdHint = ''; + } const safeMasterIdHint = typeof options.masterIdHint === 'string' ? (sanitizeIngestionEntityId(options.masterIdHint, 128) ?? '') : ''; if (safeMasterIdHint) { - const { data: hintRow, error: hintErr } = await this.supabase + const { data: hintData, error: hintErr } = await this.supabase .from('master_products') - .select('id, specifications') + .select( + itemModel === 'off' + ? 'id, specifications' + : 'id, specifications, status, item_id, variant_key' + ) .eq('id', safeMasterIdHint) .maybeSingle(); if (hintErr) { throw new Error(`Failed to verify masterIdHint (${safeMasterIdHint}): ${hintErr.message}`); } - if (hintRow?.id) { - hinted = { masterId: hintRow.id, isNew: false }; + const hintRow = hintData as { + id?: string; + specifications?: Record | null; + } | null; + const hintMeta = hintData as { + status?: unknown; + item_id?: unknown; + variant_key?: unknown; + } | null; + const hintItemId = + itemModel === 'off' ? null : sanitizeIngestionEntityId(hintMeta?.item_id, 128); + if (hintRow?.id && itemModel !== 'off' && hintMeta?.status === 'archived') { + // A retired master (an Item's `unknown` variant after M3) must not take the + // listing back: the hint maps to its Item, and the variant comes from the title. + platLogger.info( + 'ingest.master_hint_archived', + `masterIdHint ${hintRow.id} is archived: matching under its Item ${hintItemId ?? '(none)'}`, + { masterId: hintRow.id, itemId: hintItemId } + ); + if (hintItemId) safeItemIdHint = hintItemId; + } else if (hintRow?.id) { + hinted = { masterId: hintRow.id, itemId: hintItemId, isNew: false, variantSource: null }; + // The title's variant source holds only when it names the hinted variant. + hintedVariantSource = + candidate.variant_known && candidate.variant_key === hintMeta?.variant_key + ? (candidate.variant_source ?? null) + : undefined; const existingSpecs = (hintRow as { specifications?: Record | null }) .specifications; if (existingSpecs !== undefined && !hasPopulatedSpecificationGroups(existingSpecs)) { @@ -517,25 +586,67 @@ export class ShopeeIngestionPipeline { } } } - const { masterId, isNew } = + const resolved = hinted ?? - (await this.matcher.resolveOrCreateMasterProduct( + (await this.matcher.resolveItemAndVariant( candidate, resolvedCategoryId || undefined, - product.title + product.title, + { itemIdHint: safeItemIdHint || null } )); + const { itemId, isNew } = resolved; + let masterId = resolved.masterId; + let variantSourceToWrite = hinted ? hintedVariantSource : resolved.variantSource; + // The variant the listing had before this crawl, so its stats follow a move. + let previousClusterId: string | null = null; + if (itemModel !== 'off' && !hinted) { + // A variant set by hand is evidence, never undone by a later page that just + // does not state the size (design doc 4.4): only listing evidence + // (variant_source set) may move it. + const { data: stored, error: storedErr } = await this.supabase + .from('products') + .select('cluster_id, variant_source') + .eq('platform', platform) + .eq('platform_item_id', product.itemId) + .maybeSingle(); + if (storedErr) { + throw new Error(`Failed to read the stored listing variant: ${storedErr.message}`); + } + const storedClusterId = sanitizeIngestionEntityId(stored?.cluster_id, 128); + previousClusterId = storedClusterId; + // A hand-set variant always stays. A variant backed by listing evidence stays + // when this crawl has none; a `legacy` or NULL-source link (e.g. a #1105 + // fuzzy link) does not. + const storedSource = stored?.variant_source; + const keepStored = + storedSource === 'manual' || + (!resolved.variantSource && + (storedSource === 'title' || storedSource === 'sku' || storedSource === 'single_size')); + if (storedClusterId && keepStored) { + masterId = storedClusterId; + variantSourceToWrite = undefined; + } + } platLogger.info( 'ingest.master_resolved', - `Master Product ${hinted ? 'LINKED (caller hint)' : isNew ? 'CREATED' : 'LINKED'}: ID = ${masterId}`, - { masterId, isNew, hinted: Boolean(hinted) } + `Master Product ${hinted ? 'LINKED (caller hint)' : isNew ? 'CREATED' : 'LINKED'}: ID = ${masterId ?? '(no variant)'}`, + { masterId, itemId, isNew, hinted: Boolean(hinted) } ); await report?.('master'); + // Item columns exist only once the DOS-Me Item migration is applied (DOS/Rate#1150). + const itemColumns: { item_id?: string; variant_source?: string | null } = {}; + if (itemModel !== 'off') { + if (itemId) itemColumns.item_id = itemId; + if (variantSourceToWrite !== undefined) itemColumns.variant_source = variantSourceToWrite; + } + // 3. Upsert product listing into rate.products (linked via cluster_id) const { data: savedProduct, error: productErr } = await this.supabase .from('products') .upsert( { + ...itemColumns, merchant_id: merchantId, cluster_id: masterId, category_id: resolvedCategoryId, @@ -576,15 +687,32 @@ export class ShopeeIngestionPipeline { }); // 3b. Refresh canonical master_products market stats (min_price, max_price, merchant_count) via SQL RPC (Audit C5) - if (typeof this.supabase.rpc === 'function') { + // A variant the listing just left is refreshed too, or it keeps the old price. + const statsMasterIds = [masterId, previousClusterId].filter( + (id, i, all): id is string => Boolean(id) && all.indexOf(id) === i + ); + for (const statsMasterId of typeof this.supabase.rpc === 'function' ? statsMasterIds : []) { const { error: marketStatsErr } = await this.supabase.rpc('refresh_master_market_stats', { - p_master_id: masterId, + p_master_id: statsMasterId, }); if (marketStatsErr) { platLogger.warn( 'ingest.market_stats_rpc_warn', `Could not refresh master market stats via RPC: ${marketStatsErr.message}`, - { masterId } + { masterId: statsMasterId } + ); + } + } + // Then the Item roll-up (DOS-Me M2 `refresh_item_stats`, design doc 4.1). + if (typeof this.supabase.rpc === 'function' && itemModel !== 'off' && itemId) { + const { error: itemStatsErr } = await this.supabase.rpc('refresh_item_stats', { + p_item_id: itemId, + }); + if (itemStatsErr) { + platLogger.warn( + 'ingest.item_stats_rpc_warn', + `Could not refresh Item stats via RPC: ${itemStatsErr.message}`, + { itemId } ); } } @@ -870,26 +998,29 @@ export class ShopeeIngestionPipeline { { productId } ); } else if (analysisInput.length > 0) { - const [{ data: masterRow }, { data: peerRows }] = await Promise.all([ - this.supabase - .from('master_products') - .select('min_price') - .eq('id', masterId) - .maybeSingle(), - (() => { - const q = this.supabase - .from('products') - .select('id, price_current, merchants(is_official)') - .eq('cluster_id', masterId) - .gt('price_current', 0); - const withLte = - typeof (q as unknown as { lte?: (col: string, val: number) => typeof q }).lte === - 'function' - ? q.lte('price_current', MAX_VALID_VND_PRICE) - : q; - return withLte.limit(100); - })(), - ]); + // No variant: no peer floor, since peers and min price are per variant. + const [{ data: masterRow }, { data: peerRows }] = !masterId + ? [{ data: null }, { data: null }] + : await Promise.all([ + this.supabase + .from('master_products') + .select('min_price') + .eq('id', masterId) + .maybeSingle(), + (() => { + const q = this.supabase + .from('products') + .select('id, price_current, merchants(is_official)') + .eq('cluster_id', masterId) + .gt('price_current', 0); + const withLte = + typeof (q as unknown as { lte?: (col: string, val: number) => typeof q }) + .lte === 'function' + ? q.lte('price_current', MAX_VALID_VND_PRICE) + : q; + return withLte.limit(100); + })(), + ]); const rawMasterMinPrice = masterRow?.min_price; const parsedMasterMinPrice = @@ -981,6 +1112,6 @@ export class ShopeeIngestionPipeline { `Product ingestion complete! (Master: ${masterId}, Product: ${productId})`, { masterId, productId, merchantId } ); - return { productId, merchantId, masterId, platform }; + return { productId, merchantId, masterId, itemId, platform }; } }