From 3a42c4345b6f75cf2ffcfc64197417b4d9e0ef2a Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Wed, 28 Oct 2026 00:32:21 +0700 Subject: [PATCH 1/2] feat(importer): read-only Item migration inventory report (#1148) Runs the M0 checks of the Item / variant / listing design (section 7.3) over the anon-readable tables, and runs today's ProductNormalizer on each seed, master name and listing title to find key drift, which SQL cannot. SELECTs only, with the anon key. Co-Authored-By: Claude Opus 5.5 --- apps/importer/package.json | 2 +- .../__tests__/item-inventory-report.test.ts | 103 ++++++ .../src/shopee/jobs/item-inventory-report.ts | 311 ++++++++++++++++++ 3 files changed, 415 insertions(+), 1 deletion(-) create mode 100644 apps/importer/src/shopee/__tests__/item-inventory-report.test.ts create mode 100644 apps/importer/src/shopee/jobs/item-inventory-report.ts diff --git a/apps/importer/package.json b/apps/importer/package.json index 43fd5603..d57a7221 100644 --- a/apps/importer/package.json +++ b/apps/importer/package.json @@ -37,7 +37,7 @@ "lint": "eslint . --max-warnings 0", "format": "prettier --write \"src/**/*.ts\"", "backfill:specs": "tsx src/shopee/jobs/backfill-specs.ts", - "test": "tsx src/shopee/__tests__/open-core-intelligence.test.ts && tsx src/shopee/__tests__/entity-matcher.test.ts && tsx src/shopee/__tests__/shopee-to-supabase.test.ts && tsx src/shopee/__tests__/paging.test.ts && tsx src/shopee/__tests__/backfill-review-risk.test.ts && tsx src/shopee/__tests__/arena-audit.test.ts && tsx src/shopee/test-ai-enricher.ts && tsx src/shopee/test-review-analyzer.ts", + "test": "tsx src/shopee/__tests__/open-core-intelligence.test.ts && tsx src/shopee/__tests__/entity-matcher.test.ts && tsx src/shopee/__tests__/shopee-to-supabase.test.ts && tsx src/shopee/__tests__/paging.test.ts && tsx src/shopee/__tests__/backfill-review-risk.test.ts && tsx src/shopee/__tests__/item-inventory-report.test.ts && tsx src/shopee/__tests__/arena-audit.test.ts && tsx src/shopee/test-ai-enricher.ts && tsx src/shopee/test-review-analyzer.ts", "test:dist": "node src/shopee/__tests__/dist-exports.smoke.mjs", "backfill:review-risk": "tsx src/shopee/jobs/backfill-review-risk.ts" }, diff --git a/apps/importer/src/shopee/__tests__/item-inventory-report.test.ts b/apps/importer/src/shopee/__tests__/item-inventory-report.test.ts new file mode 100644 index 00000000..8c4eaf1a --- /dev/null +++ b/apps/importer/src/shopee/__tests__/item-inventory-report.test.ts @@ -0,0 +1,103 @@ +import assert from 'node:assert/strict'; +import { + buildInventoryReport, + itemNameOf, + itemSlugOf, + type InventoryMaster, +} from '../jobs/item-inventory-report.js'; + +const master = ( + id: string, + model_key: string, + slug: string, + name: string, + category_id = 'phones', + storage_gb: number | null = Number(/:(\d+)gb$/.exec(model_key)?.[1] ?? NaN) || null +): InventoryMaster => ({ + id, + model_key, + slug, + name, + brand: name.split(' ')[0]!, + status: 'published', + category_id, + storage_gb, +}); + +function run() { + assert.equal(itemSlugOf('asus-zenbook-s-14-1tb'), 'asus-zenbook-s-14'); + assert.equal(itemSlugOf('sony-wf-1000xm6'), 'sony-wf-1000xm6'); + assert.equal(itemNameOf('Apple iPhone 16 128GB'), 'Apple iPhone 16'); + + const report = buildInventoryReport({ + categories: [ + { id: 'phones', slug: 'smartphones' }, + { id: 'buds', slug: 'earbuds' }, + ], + masters: [ + master('a', 'apple:iphone-16:128gb', 'apple-iphone-16-128gb', 'Apple iPhone 16 128GB'), + master('b', 'apple:iphone-16:256gb', 'apple-iphone-16-256gb', 'Apple iPhone 16 256GB'), + // Another Item whose variant slug is iPhone 16's Item slug. + master('c', 'apple:iphone-16-x:64gb', 'apple-iphone-16', 'Apple iPhone 16 X 64GB'), + master( + 'd', + 'samsung:galaxy-buds3-pro:64gb', + 'samsung-galaxy-buds3-pro', + 'Samsung Galaxy Buds3 Pro', + 'buds', + 64 + ), + master('e', 'khc:legacy', 'legacy', 'Legacy'), + ], + listings: [ + { + id: 'p1', + cluster_id: 'a', + title: 'Apple iPhone 16 256GB', + platform: 'tgdd', + platform_item_id: '1', + price_min: 1, + price_max: 1, + }, + { + id: 'p2', + cluster_id: null, + title: 'x', + platform: 'tgdd', + platform_item_id: '2', + price_min: 1, + price_max: 2, + }, + ], + seeds: [], + }); + + assert.equal(report.counts.total, 5); + assert.equal(report.counts.khc_keys, 1); + assert.equal(report.counts.bad_shape, 1); + assert.equal(report.counts.multi_variant_items, 1); + assert.equal(report.counts.listings_without_master, 1); + assert.deepEqual(report.legacy_keys, [{ model_key: 'khc:legacy', status: 'published' }]); + assert.deepEqual( + report.sizeless_with_size.map(r => r.model_key), + ['samsung:galaxy-buds3-pro:64gb'] + ); + assert.ok(report.master_drift.some(d => d.normalized_key === 'samsung:galaxy-buds3-pro:unknown')); + assert.deepEqual( + report.listing_drift.map(d => [d.kind, d.normalized_key]), + [['variant', 'apple:iphone-16:256gb']] + ); + assert.deepEqual(report.slug_collisions, [ + { slug: 'apple-iphone-16', kind: 'two_items', items: ['apple:iphone-16', 'apple:iphone-16-x'] }, + { + slug: 'apple-iphone-16', + kind: 'item_slug_is_other_variant', + items: ['apple:iphone-16', 'apple:iphone-16-x'], + }, + ]); + assert.equal(report.multi_sku_listings.length, 1); + + console.log('item-inventory-report tests passed'); +} + +run(); diff --git a/apps/importer/src/shopee/jobs/item-inventory-report.ts b/apps/importer/src/shopee/jobs/item-inventory-report.ts new file mode 100644 index 00000000..c882c66d --- /dev/null +++ b/apps/importer/src/shopee/jobs/item-inventory-report.ts @@ -0,0 +1,311 @@ +/** + * Read-only Item migration inventory (DOS/Rate#1148, design doc + * `Docs/Proposals/2026-10-27-item-variant-listing-model.md` sections 7 and 7.3, phase M0). + * + * Usage: + * pnpm --filter @repo/crawlers exec tsx src/shopee/jobs/item-inventory-report.ts + * + * Reads `rate.categories`, `rate.master_products` and `rate.products` with the public anon + * key (SUPABASE_URL + SUPABASE_ANON_KEY, or the NEXT_PUBLIC_ pair the web uses) and prints + * a JSON report. Besides the SQL checks of section 7.3, it runs today's `ProductNormalizer` + * on every seed, master name and listing title, which SQL cannot do: that is how seed key + * drift shows up. + * + * Safety invariant: SELECTs only. There is no write path and no flag that adds one. + * Tables anon cannot read (`price_alerts`) are not queried; see the issue for the SQL. + */ +import { pathToFileURL } from 'node:url'; +import dotenv from 'dotenv'; +import { FLAGSHIP_DEVICES_TO_SEED, type FlagshipDeviceSeed } from '../flagship-seed-data.js'; +import { createImporterLogger } from '../logger.js'; +import { fetchAllPages, resolveBackfillSupabaseTarget } from '../paging.js'; +import { ProductNormalizer } from '../product-normalizer.js'; +import { scopeSupabaseToRate, type RateSupabaseClient, type ShopeeProduct } from '../types.js'; + +const logger = createImporterLogger('ItemInventoryReport'); + +export interface InventoryMaster { + id: string; + model_key: string; + slug: string; + name: string; + brand: string | null; + status: string | null; + category_id: string | null; + storage_gb: number | null; +} + +export interface InventoryListing { + id: string; + cluster_id: string | null; + title: string | null; + platform: string | null; + platform_item_id: string | null; + price_min: number | string | null; + price_max: number | string | null; +} + +export interface InventoryCategory { + id: string; + slug: string; +} + +/** Section 7.3 shape check: three `:`-separated segments. */ +const KEY_SHAPE = /^[a-z0-9][a-z0-9-]*:[a-z0-9][a-z0-9.-]*:[a-z0-9-]+$/; +/** The suffix `rate.item_slug_from_master` strips (section 7.1). */ +const VARIANT_SLUG_SUFFIX = /-(\d+(gb|tb)|standard|unknown)$/; +/** The trailing size `rate.item_name_from_master` strips (section 7.1). */ +const VARIANT_NAME_SUFFIX = /\s\d+\s?(gb|tb)$/i; +/** Categories with no storage axis (section 4.2). */ +const SIZELESS_CATEGORIES = new Set(['earbuds']); + +export const itemKeyOf = (modelKey: string): string => modelKey.split(':').slice(0, 2).join(':'); +export const variantKeyOf = (modelKey: string): string => modelKey.split(':')[2] ?? ''; +export const itemSlugOf = (slug: string): string => slug.replace(VARIANT_SLUG_SUFFIX, ''); +export const itemNameOf = (name: string): string => name.replace(VARIANT_NAME_SUFFIX, ''); + +/** What the normalizer keys a bare title as, with no marketplace SKUs to help it. */ +export function normalizedKeyOf(normalizer: ProductNormalizer, title: string): string { + const raw: ShopeeProduct = { + itemId: 'inventory', + shopId: 'inventory', + url: 'https://rate.vn', + title, + priceMin: 0, + priceMax: 0, + currency: 'VND', + historicalSold: 0, + ratingStar: 0, + ratingCount: 0, + images: [], + models: [], + attributes: [], + shop: { shopId: 'inventory', name: 'inventory', isMall: false }, + reviews: [], + }; + return normalizer.normalize(raw).model_key; +} + +export function buildInventoryReport(input: { + categories: InventoryCategory[]; + masters: InventoryMaster[]; + listings: InventoryListing[]; + seeds?: FlagshipDeviceSeed[]; + normalizer?: ProductNormalizer; +}) { + const normalizer = input.normalizer ?? new ProductNormalizer(); + const seeds = input.seeds ?? FLAGSHIP_DEVICES_TO_SEED; + const categorySlug = new Map(input.categories.map(c => [c.id, c.slug])); + const masters = input.masters; + const byId = new Map(masters.map(m => [m.id, m])); + const byKey = new Map(masters.map(m => [m.model_key, m])); + const bySlug = new Map(masters.map(m => [m.slug, m])); + + const items = new Map(); + for (const m of masters) { + const list = items.get(itemKeyOf(m.model_key)) ?? []; + list.push(m); + items.set(itemKeyOf(m.model_key), list); + } + + const counts = { + total: masters.length, + published: masters.filter(m => m.status === 'published').length, + draft: masters.filter(m => m.status === 'draft').length, + archived: masters.filter(m => m.status === 'archived').length, + null_status: masters.filter(m => m.status === null).length, + bad_shape: masters.filter(m => !KEY_SHAPE.test(m.model_key)).length, + khc_keys: masters.filter(m => m.model_key.startsWith('khc:')).length, + unknown_masters: masters.filter(m => variantKeyOf(m.model_key) === 'unknown').length, + standard_masters: masters.filter(m => variantKeyOf(m.model_key) === 'standard').length, + items: items.size, + multi_variant_items: [...items.values()].filter(v => v.length > 1).length, + listings: input.listings.length, + listings_without_master: input.listings.filter(p => !p.cluster_id).length, + listings_on_missing_master: input.listings.filter(p => p.cluster_id && !byId.has(p.cluster_id)) + .length, + }; + + const legacyKeys = masters + .filter(m => !KEY_SHAPE.test(m.model_key) || m.model_key.startsWith('khc:')) + .map(m => ({ model_key: m.model_key, status: m.status })); + + const itemConflicts = [...items.entries()] + .filter( + ([, v]) => + new Set(v.map(m => m.brand)).size > 1 || new Set(v.map(m => m.category_id)).size > 1 + ) + .map(([item_key, v]) => ({ item_key, variants: v.map(m => m.model_key) })); + + const sizelessWithSize = masters + .filter( + m => + SIZELESS_CATEGORIES.has(categorySlug.get(m.category_id ?? '') ?? '') && + variantKeyOf(m.model_key) !== 'unknown' + ) + .map(m => ({ model_key: m.model_key, status: m.status, storage_gb: m.storage_gb })); + + const seedDrift = seeds + .map(s => { + const normalized = normalizedKeyOf(normalizer, s.name); + const stored = byKey.get(s.model_key); + return { + seed_key: s.model_key, + normalized_key: normalized, + stored_status: stored?.status ?? null, + normalized_key_stored: byKey.has(normalized), + }; + }) + .filter(d => d.seed_key !== d.normalized_key); + + const masterDrift = masters + .map(m => ({ + model_key: m.model_key, + normalized_key: normalizedKeyOf(normalizer, m.name), + status: m.status, + })) + .filter(d => d.model_key !== d.normalized_key); + + // Listings whose own title keys elsewhere: another Item, or the same Item at another size. + const listingDrift = input.listings.flatMap(p => { + const master = p.cluster_id ? byId.get(p.cluster_id) : undefined; + if (!master || !p.title) return []; + const normalized = normalizedKeyOf(normalizer, p.title); + if (normalized === master.model_key) return []; + return [ + { + kind: itemKeyOf(normalized) === itemKeyOf(master.model_key) ? 'variant' : 'item', + master_key: master.model_key, + normalized_key: normalized, + platform: p.platform, + platform_item_id: p.platform_item_id, + title: p.title.slice(0, 120), + }, + ]; + }); + + // Item slugs from the section 7.1 helper: two Items on one slug, or an Item slug that is + // another Item's variant slug, abort the M2 backfill. + const itemSlugOwners = new Map>(); + for (const m of masters) { + const owners = itemSlugOwners.get(itemSlugOf(m.slug)) ?? new Set(); + owners.add(itemKeyOf(m.model_key)); + itemSlugOwners.set(itemSlugOf(m.slug), owners); + } + const slugOnOtherItem = new Map(); + for (const m of masters) { + const other = bySlug.get(itemSlugOf(m.slug)); + if (other && itemKeyOf(other.model_key) !== itemKeyOf(m.model_key)) { + slugOnOtherItem.set(itemSlugOf(m.slug), [itemKeyOf(m.model_key), itemKeyOf(other.model_key)]); + } + } + const slugCollisions = [ + ...[...itemSlugOwners.entries()] + .filter(([, owners]) => owners.size > 1) + .map(([slug, owners]) => ({ slug, kind: 'two_items', items: [...owners] })), + ...[...slugOnOtherItem.entries()].map(([slug, items]) => ({ + slug, + kind: 'item_slug_is_other_variant', + items, + })), + ]; + + const sizedItemNames = masters + .filter(m => /\d+\s?(gb|tb)\b/i.test(itemNameOf(m.name))) + .map(m => ({ model_key: m.model_key, name: m.name })); + + const storageMismatch = masters + .filter(m => { + const v = variantKeyOf(m.model_key); + const sized = /^(\d+)gb$/.exec(v); + return sized ? Number(sized[1]) !== m.storage_gb : m.storage_gb !== null; + }) + .map(m => ({ model_key: m.model_key, storage_gb: m.storage_gb })); + + const multiSkuListings = input.listings + .filter( + p => + p.price_min !== null && p.price_max !== null && Number(p.price_min) !== Number(p.price_max) + ) + .map(p => ({ + master_key: p.cluster_id ? (byId.get(p.cluster_id)?.model_key ?? null) : null, + platform: p.platform, + platform_item_id: p.platform_item_id, + price_min: Number(p.price_min), + price_max: Number(p.price_max), + })); + + return { + counts, + legacy_keys: legacyKeys, + item_conflicts: itemConflicts, + sizeless_with_size: sizelessWithSize, + seed_drift: seedDrift, + master_drift: masterDrift, + listing_drift: listingDrift, + slug_collisions: slugCollisions, + sized_item_names: sizedItemNames, + storage_mismatch: storageMismatch, + multi_sku_listings: multiSkuListings, + }; +} + +export async function loadInventory(supabase: RateSupabaseClient) { + const categories = await fetchAllPages('categories', (from, to) => + supabase.from('categories').select('id, slug').order('id').range(from, to) + ); + const masters = await fetchAllPages('master_products', (from, to) => + supabase + .from('master_products') + .select('id, model_key, slug, name, brand, status, category_id, storage_gb') + .order('id') + .range(from, to) + ); + const listings = await fetchAllPages('products', (from, to) => + supabase + .from('products') + .select('id, cluster_id, title, platform, platform_item_id, price_min, price_max') + .order('id') + .range(from, to) + ); + return { categories, masters, listings }; +} + +const isDirectExecution = + typeof process !== 'undefined' && + Array.isArray(process.argv) && + Boolean(process.argv[1]) && + import.meta.url === pathToFileURL(process.argv[1]!).href; + +if (isDirectExecution) { + dotenv.config({ quiet: true }); + dotenv.config({ path: '../../.env', quiet: true }); + (async () => { + const { createClient } = await import('@supabase/supabase-js'); + const rawUrl = process.env.SUPABASE_URL || process.env.NEXT_PUBLIC_SUPABASE_URL || ''; + // The anon key on purpose: the report must see what any visitor sees, and nothing more. + const key = ( + process.env.SUPABASE_ANON_KEY || + process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY || + '' + ).trim(); + if (!rawUrl.trim() || !key) { + throw new Error('Missing SUPABASE_URL or SUPABASE_ANON_KEY in environment.'); + } + const { url, host } = resolveBackfillSupabaseTarget(rawUrl); + const supabase = scopeSupabaseToRate( + createClient(url, key, { auth: { persistSession: false } }) + ); + // stderr, so stdout stays one JSON document. + process.stderr.write(`Target: ${host} (read-only) +`); + const report = buildInventoryReport(await loadInventory(supabase)); + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); + })().catch((err: unknown) => { + logger.error( + 'inventory.fatal', + `Item inventory report failed: ${err instanceof Error ? err.message : String(err)}` + ); + process.exit(1); + }); +} From 1ff875c620f0018490c110a68f1b25046f10b28d Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Wed, 28 Oct 2026 00:35:49 +0700 Subject: [PATCH 2/2] fix(importer): inventory report review fixes (#1148) NULL brand or category is not an Item conflict, as in SQL count(DISTINCT). TB and suffixed variant keys count as sized. Slug checks now also flag an Item whose masters strip to different slugs and a slug still sized after stripping. Listings with no stated size get their own drift bucket. Co-Authored-By: Claude Opus 5.5 --- .../__tests__/item-inventory-report.test.ts | 129 +++++++++++++----- .../src/shopee/jobs/item-inventory-report.ts | 65 +++++---- 2 files changed, 135 insertions(+), 59 deletions(-) diff --git a/apps/importer/src/shopee/__tests__/item-inventory-report.test.ts b/apps/importer/src/shopee/__tests__/item-inventory-report.test.ts index 8c4eaf1a..676ac291 100644 --- a/apps/importer/src/shopee/__tests__/item-inventory-report.test.ts +++ b/apps/importer/src/shopee/__tests__/item-inventory-report.test.ts @@ -1,8 +1,11 @@ import assert from 'node:assert/strict'; +import type { FlagshipDeviceSeed } from '../flagship-seed-data.js'; import { buildInventoryReport, + itemKeyOf, itemNameOf, itemSlugOf, + type InventoryListing, type InventoryMaster, } from '../jobs/item-inventory-report.js'; @@ -11,8 +14,7 @@ const master = ( model_key: string, slug: string, name: string, - category_id = 'phones', - storage_gb: number | null = Number(/:(\d+)gb$/.exec(model_key)?.[1] ?? NaN) || null + overrides: Partial = {} ): InventoryMaster => ({ id, model_key, @@ -20,14 +22,34 @@ const master = ( name, brand: name.split(' ')[0]!, status: 'published', - category_id, - storage_gb, + category_id: 'phones', + storage_gb: Number(/:(\d+)gb$/.exec(model_key)?.[1] ?? NaN) || null, + ...overrides, }); +const listing = ( + id: string, + cluster_id: string | null, + title: string, + price_max = 1 +): InventoryListing => ({ + id, + cluster_id, + title, + platform: 'tgdd', + platform_item_id: id, + price_min: 1, + price_max, +}); + +const seed = (model_key: string, name: string) => ({ model_key, name }) as FlagshipDeviceSeed; + function run() { assert.equal(itemSlugOf('asus-zenbook-s-14-1tb'), 'asus-zenbook-s-14'); assert.equal(itemSlugOf('sony-wf-1000xm6'), 'sony-wf-1000xm6'); assert.equal(itemNameOf('Apple iPhone 16 128GB'), 'Apple iPhone 16'); + // As SQL split_part does: a one-segment key keeps its trailing ':'. + assert.equal(itemKeyOf('legacy'), 'legacy:'); const report = buildInventoryReport({ categories: [ @@ -36,7 +58,8 @@ function run() { ], masters: [ master('a', 'apple:iphone-16:128gb', 'apple-iphone-16-128gb', 'Apple iPhone 16 128GB'), - master('b', 'apple:iphone-16:256gb', 'apple-iphone-16-256gb', 'Apple iPhone 16 256GB'), + // Same Item, but its slug strips to another Item slug. + master('b', 'apple:iphone-16:256gb', 'apple-iphone-16-vn-256gb', 'Apple iPhone 16 256GB'), // Another Item whose variant slug is iPhone 16's Item slug. master('c', 'apple:iphone-16-x:64gb', 'apple-iphone-16', 'Apple iPhone 16 X 64GB'), master( @@ -44,57 +67,93 @@ function run() { 'samsung:galaxy-buds3-pro:64gb', 'samsung-galaxy-buds3-pro', 'Samsung Galaxy Buds3 Pro', - 'buds', - 64 + { + category_id: 'buds', + } ), master('e', 'khc:legacy', 'legacy', 'Legacy'), + master('f', 'oddkey', 'odd', 'Odd'), + // A size in TB, and a NULL brand next to a real one: neither is a mismatch or a conflict. + master('g', 'asus:zenbook-s-14:1tb', 'asus-zenbook-s-14-1tb', 'Asus Zenbook S 14 1TB', { + storage_gb: 1024, + }), + master('h', 'asus:zenbook-s-14:512gb-oled', 'asus-zenbook-s-14-512gb', 'Asus Zenbook 512GB', { + brand: null, + storage_gb: 512, + }), + // A real conflict: two categories under one Item, and a stored size that disagrees. + master('i', 'oppo:find-x8:256gb', 'oppo-find-x8-256gb', 'OPPO Find X8 256GB', { + category_id: 'buds', + storage_gb: 128, + }), + master('j', 'oppo:find-x8:512gb', 'oppo-find-x8-128gb-512gb', 'OPPO Find X8 512GB'), ], listings: [ - { - id: 'p1', - cluster_id: 'a', - title: 'Apple iPhone 16 256GB', - platform: 'tgdd', - platform_item_id: '1', - price_min: 1, - price_max: 1, - }, - { - id: 'p2', - cluster_id: null, - title: 'x', - platform: 'tgdd', - platform_item_id: '2', - price_min: 1, - price_max: 2, - }, + listing('p1', 'a', 'Apple iPhone 16 256GB'), + listing('p2', 'a', 'Apple iPhone 16'), + listing('p3', null, 'x', 2), + listing('p4', 'gone', 'x'), + ], + seeds: [ + seed('apple:iphone-16:128gb', 'Apple iPhone 16 128GB'), + seed('samsung:galaxy-buds3-pro:64gb', 'Samsung Galaxy Buds3 Pro'), ], - seeds: [], }); - assert.equal(report.counts.total, 5); + assert.equal(report.counts.total, 10); assert.equal(report.counts.khc_keys, 1); - assert.equal(report.counts.bad_shape, 1); - assert.equal(report.counts.multi_variant_items, 1); + assert.equal(report.counts.bad_shape, 2); assert.equal(report.counts.listings_without_master, 1); - assert.deepEqual(report.legacy_keys, [{ model_key: 'khc:legacy', status: 'published' }]); + assert.equal(report.counts.listings_on_missing_master, 1); + assert.deepEqual( + report.legacy_keys.map(r => r.model_key), + ['khc:legacy', 'oddkey'] + ); + assert.deepEqual(report.item_conflicts, [ + { item_key: 'oppo:find-x8', variants: ['oppo:find-x8:256gb', 'oppo:find-x8:512gb'] }, + ]); assert.deepEqual( report.sizeless_with_size.map(r => r.model_key), - ['samsung:galaxy-buds3-pro:64gb'] + ['samsung:galaxy-buds3-pro:64gb', 'oppo:find-x8:256gb'] ); + assert.deepEqual(report.seed_drift, [ + { + seed_key: 'samsung:galaxy-buds3-pro:64gb', + normalized_key: 'samsung:galaxy-buds3-pro:unknown', + stored_status: 'published', + normalized_key_stored: false, + }, + ]); assert.ok(report.master_drift.some(d => d.normalized_key === 'samsung:galaxy-buds3-pro:unknown')); assert.deepEqual( report.listing_drift.map(d => [d.kind, d.normalized_key]), - [['variant', 'apple:iphone-16:256gb']] + [ + ['variant', 'apple:iphone-16:256gb'], + ['size_not_stated', 'apple:iphone-16:unknown'], + ] ); assert.deepEqual(report.slug_collisions, [ - { slug: 'apple-iphone-16', kind: 'two_items', items: ['apple:iphone-16', 'apple:iphone-16-x'] }, { - slug: 'apple-iphone-16', - kind: 'item_slug_is_other_variant', + kind: 'two_items', + slugs: ['apple-iphone-16'], items: ['apple:iphone-16', 'apple:iphone-16-x'], }, + { + kind: 'item_slug_differs', + slugs: ['apple-iphone-16', 'apple-iphone-16-vn'], + items: ['apple:iphone-16'], + }, + { + kind: 'item_slug_differs', + slugs: ['oppo-find-x8', 'oppo-find-x8-128gb'], + items: ['oppo:find-x8'], + }, + { kind: 'still_sized', slugs: ['oppo-find-x8-128gb'], items: ['oppo:find-x8'] }, ]); + assert.deepEqual( + report.storage_mismatch.map(r => r.model_key), + ['oppo:find-x8:256gb'] + ); assert.equal(report.multi_sku_listings.length, 1); console.log('item-inventory-report tests passed'); diff --git a/apps/importer/src/shopee/jobs/item-inventory-report.ts b/apps/importer/src/shopee/jobs/item-inventory-report.ts index c882c66d..7367ef0a 100644 --- a/apps/importer/src/shopee/jobs/item-inventory-report.ts +++ b/apps/importer/src/shopee/jobs/item-inventory-report.ts @@ -59,7 +59,11 @@ const VARIANT_NAME_SUFFIX = /\s\d+\s?(gb|tb)$/i; /** Categories with no storage axis (section 4.2). */ const SIZELESS_CATEGORIES = new Set(['earbuds']); -export const itemKeyOf = (modelKey: string): string => modelKey.split(':').slice(0, 2).join(':'); +/** `split_part(k, ':', 1) || ':' || split_part(k, ':', 2)`, as the M2 backfill groups. */ +export const itemKeyOf = (modelKey: string): string => { + const [brand = '', model = ''] = modelKey.split(':'); + return `${brand}:${model}`; +}; export const variantKeyOf = (modelKey: string): string => modelKey.split(':')[2] ?? ''; export const itemSlugOf = (slug: string): string => slug.replace(VARIANT_SLUG_SUFFIX, ''); export const itemNameOf = (name: string): string => name.replace(VARIANT_NAME_SUFFIX, ''); @@ -99,7 +103,6 @@ export function buildInventoryReport(input: { const masters = input.masters; const byId = new Map(masters.map(m => [m.id, m])); const byKey = new Map(masters.map(m => [m.model_key, m])); - const bySlug = new Map(masters.map(m => [m.slug, m])); const items = new Map(); for (const m of masters) { @@ -133,7 +136,9 @@ export function buildInventoryReport(input: { const itemConflicts = [...items.entries()] .filter( ([, v]) => - new Set(v.map(m => m.brand)).size > 1 || new Set(v.map(m => m.category_id)).size > 1 + // count(DISTINCT ...) ignores NULLs, so a missing brand or category is not a conflict. + new Set(v.map(m => m.brand).filter(b => b !== null)).size > 1 || + new Set(v.map(m => m.category_id).filter(c => c !== null)).size > 1 ) .map(([item_key, v]) => ({ item_key, variants: v.map(m => m.model_key) })); @@ -166,7 +171,8 @@ export function buildInventoryReport(input: { })) .filter(d => d.model_key !== d.normalized_key); - // Listings whose own title keys elsewhere: another Item, or the same Item at another size. + // Listings whose own title keys elsewhere: another Item, the same Item at another size, or + // the same Item with no size stated (the listings that lose their variant on multi-size Items). const listingDrift = input.listings.flatMap(p => { const master = p.cluster_id ? byId.get(p.cluster_id) : undefined; if (!master || !p.title) return []; @@ -174,7 +180,12 @@ export function buildInventoryReport(input: { if (normalized === master.model_key) return []; return [ { - kind: itemKeyOf(normalized) === itemKeyOf(master.model_key) ? 'variant' : 'item', + kind: + itemKeyOf(normalized) !== itemKeyOf(master.model_key) + ? 'item' + : variantKeyOf(normalized) === 'unknown' + ? 'size_not_stated' + : 'variant', master_key: master.model_key, normalized_key: normalized, platform: p.platform, @@ -184,32 +195,36 @@ export function buildInventoryReport(input: { ]; }); - // Item slugs from the section 7.1 helper: two Items on one slug, or an Item slug that is - // another Item's variant slug, abort the M2 backfill. + // Item slugs from the section 7.1 helper. Two Items on one slug abort the M2 backfill. This + // also covers an Item slug that is another Item's variant slug: that variant slug carries no + // size suffix, so it strips to itself. An Item whose masters strip to different slugs gets + // whichever one the backfill's representative master has, and a slug that still ends in a + // size after stripping fails the post-M2 check of section 7.3. const itemSlugOwners = new Map>(); + const itemSlugsByItem = new Map>(); for (const m of masters) { - const owners = itemSlugOwners.get(itemSlugOf(m.slug)) ?? new Set(); + const slug = itemSlugOf(m.slug); + const owners = itemSlugOwners.get(slug) ?? new Set(); owners.add(itemKeyOf(m.model_key)); - itemSlugOwners.set(itemSlugOf(m.slug), owners); - } - const slugOnOtherItem = new Map(); - for (const m of masters) { - const other = bySlug.get(itemSlugOf(m.slug)); - if (other && itemKeyOf(other.model_key) !== itemKeyOf(m.model_key)) { - slugOnOtherItem.set(itemSlugOf(m.slug), [itemKeyOf(m.model_key), itemKeyOf(other.model_key)]); - } + itemSlugOwners.set(slug, owners); + const slugs = itemSlugsByItem.get(itemKeyOf(m.model_key)) ?? new Set(); + slugs.add(slug); + itemSlugsByItem.set(itemKeyOf(m.model_key), slugs); } const slugCollisions = [ ...[...itemSlugOwners.entries()] .filter(([, owners]) => owners.size > 1) - .map(([slug, owners]) => ({ slug, kind: 'two_items', items: [...owners] })), - ...[...slugOnOtherItem.entries()].map(([slug, items]) => ({ - slug, - kind: 'item_slug_is_other_variant', - items, - })), + .map(([slug, owners]) => ({ kind: 'two_items', slugs: [slug], items: [...owners] })), + ...[...itemSlugsByItem.entries()] + .filter(([, slugs]) => slugs.size > 1) + .map(([item, slugs]) => ({ kind: 'item_slug_differs', slugs: [...slugs], items: [item] })), + ...[...itemSlugOwners.entries()] + .filter(([slug]) => /-\d+(gb|tb)$/.test(slug)) + .map(([slug, owners]) => ({ kind: 'still_sized', slugs: [slug], items: [...owners] })), ]; + // Broader than the section 7.3 check (a size anywhere, not only at the end), so a name such as + // "12GB RAM" is flagged too; those rows are read by hand. const sizedItemNames = masters .filter(m => /\d+\s?(gb|tb)\b/i.test(itemNameOf(m.name))) .map(m => ({ model_key: m.model_key, name: m.name })); @@ -217,8 +232,10 @@ export function buildInventoryReport(input: { const storageMismatch = masters .filter(m => { const v = variantKeyOf(m.model_key); - const sized = /^(\d+)gb$/.exec(v); - return sized ? Number(sized[1]) !== m.storage_gb : m.storage_gb !== null; + // `1tb` and suffixed keys such as `256gb-cellular` (section 4.2) are sized too. + const sized = /^(\d+)(gb|tb)(?:-|$)/.exec(v); + if (!sized) return m.storage_gb !== null; + return Number(sized[1]) * (sized[2] === 'tb' ? 1024 : 1) !== m.storage_gb; }) .map(m => ({ model_key: m.model_key, storage_gb: m.storage_gb }));