diff --git a/apps/importer/package.json b/apps/importer/package.json index 43fd5603..d57a7221 100644 --- a/apps/importer/package.json +++ b/apps/importer/package.json @@ -37,7 +37,7 @@ "lint": "eslint . --max-warnings 0", "format": "prettier --write \"src/**/*.ts\"", "backfill:specs": "tsx src/shopee/jobs/backfill-specs.ts", - "test": "tsx src/shopee/__tests__/open-core-intelligence.test.ts && tsx src/shopee/__tests__/entity-matcher.test.ts && tsx src/shopee/__tests__/shopee-to-supabase.test.ts && tsx src/shopee/__tests__/paging.test.ts && tsx src/shopee/__tests__/backfill-review-risk.test.ts && tsx src/shopee/__tests__/arena-audit.test.ts && tsx src/shopee/test-ai-enricher.ts && tsx src/shopee/test-review-analyzer.ts", + "test": "tsx src/shopee/__tests__/open-core-intelligence.test.ts && tsx src/shopee/__tests__/entity-matcher.test.ts && tsx src/shopee/__tests__/shopee-to-supabase.test.ts && tsx src/shopee/__tests__/paging.test.ts && tsx src/shopee/__tests__/backfill-review-risk.test.ts && tsx src/shopee/__tests__/item-inventory-report.test.ts && tsx src/shopee/__tests__/arena-audit.test.ts && tsx src/shopee/test-ai-enricher.ts && tsx src/shopee/test-review-analyzer.ts", "test:dist": "node src/shopee/__tests__/dist-exports.smoke.mjs", "backfill:review-risk": "tsx src/shopee/jobs/backfill-review-risk.ts" }, diff --git a/apps/importer/src/shopee/__tests__/item-inventory-report.test.ts b/apps/importer/src/shopee/__tests__/item-inventory-report.test.ts new file mode 100644 index 00000000..676ac291 --- /dev/null +++ b/apps/importer/src/shopee/__tests__/item-inventory-report.test.ts @@ -0,0 +1,162 @@ +import assert from 'node:assert/strict'; +import type { FlagshipDeviceSeed } from '../flagship-seed-data.js'; +import { + buildInventoryReport, + itemKeyOf, + itemNameOf, + itemSlugOf, + type InventoryListing, + type InventoryMaster, +} from '../jobs/item-inventory-report.js'; + +const master = ( + id: string, + model_key: string, + slug: string, + name: string, + overrides: Partial = {} +): InventoryMaster => ({ + id, + model_key, + slug, + name, + brand: name.split(' ')[0]!, + status: 'published', + category_id: 'phones', + storage_gb: Number(/:(\d+)gb$/.exec(model_key)?.[1] ?? NaN) || null, + ...overrides, +}); + +const listing = ( + id: string, + cluster_id: string | null, + title: string, + price_max = 1 +): InventoryListing => ({ + id, + cluster_id, + title, + platform: 'tgdd', + platform_item_id: id, + price_min: 1, + price_max, +}); + +const seed = (model_key: string, name: string) => ({ model_key, name }) as FlagshipDeviceSeed; + +function run() { + assert.equal(itemSlugOf('asus-zenbook-s-14-1tb'), 'asus-zenbook-s-14'); + assert.equal(itemSlugOf('sony-wf-1000xm6'), 'sony-wf-1000xm6'); + assert.equal(itemNameOf('Apple iPhone 16 128GB'), 'Apple iPhone 16'); + // As SQL split_part does: a one-segment key keeps its trailing ':'. + assert.equal(itemKeyOf('legacy'), 'legacy:'); + + const report = buildInventoryReport({ + categories: [ + { id: 'phones', slug: 'smartphones' }, + { id: 'buds', slug: 'earbuds' }, + ], + masters: [ + master('a', 'apple:iphone-16:128gb', 'apple-iphone-16-128gb', 'Apple iPhone 16 128GB'), + // Same Item, but its slug strips to another Item slug. + master('b', 'apple:iphone-16:256gb', 'apple-iphone-16-vn-256gb', 'Apple iPhone 16 256GB'), + // Another Item whose variant slug is iPhone 16's Item slug. + master('c', 'apple:iphone-16-x:64gb', 'apple-iphone-16', 'Apple iPhone 16 X 64GB'), + master( + 'd', + 'samsung:galaxy-buds3-pro:64gb', + 'samsung-galaxy-buds3-pro', + 'Samsung Galaxy Buds3 Pro', + { + category_id: 'buds', + } + ), + master('e', 'khc:legacy', 'legacy', 'Legacy'), + master('f', 'oddkey', 'odd', 'Odd'), + // A size in TB, and a NULL brand next to a real one: neither is a mismatch or a conflict. + master('g', 'asus:zenbook-s-14:1tb', 'asus-zenbook-s-14-1tb', 'Asus Zenbook S 14 1TB', { + storage_gb: 1024, + }), + master('h', 'asus:zenbook-s-14:512gb-oled', 'asus-zenbook-s-14-512gb', 'Asus Zenbook 512GB', { + brand: null, + storage_gb: 512, + }), + // A real conflict: two categories under one Item, and a stored size that disagrees. + master('i', 'oppo:find-x8:256gb', 'oppo-find-x8-256gb', 'OPPO Find X8 256GB', { + category_id: 'buds', + storage_gb: 128, + }), + master('j', 'oppo:find-x8:512gb', 'oppo-find-x8-128gb-512gb', 'OPPO Find X8 512GB'), + ], + listings: [ + listing('p1', 'a', 'Apple iPhone 16 256GB'), + listing('p2', 'a', 'Apple iPhone 16'), + listing('p3', null, 'x', 2), + listing('p4', 'gone', 'x'), + ], + seeds: [ + seed('apple:iphone-16:128gb', 'Apple iPhone 16 128GB'), + seed('samsung:galaxy-buds3-pro:64gb', 'Samsung Galaxy Buds3 Pro'), + ], + }); + + assert.equal(report.counts.total, 10); + assert.equal(report.counts.khc_keys, 1); + assert.equal(report.counts.bad_shape, 2); + assert.equal(report.counts.listings_without_master, 1); + assert.equal(report.counts.listings_on_missing_master, 1); + assert.deepEqual( + report.legacy_keys.map(r => r.model_key), + ['khc:legacy', 'oddkey'] + ); + assert.deepEqual(report.item_conflicts, [ + { item_key: 'oppo:find-x8', variants: ['oppo:find-x8:256gb', 'oppo:find-x8:512gb'] }, + ]); + assert.deepEqual( + report.sizeless_with_size.map(r => r.model_key), + ['samsung:galaxy-buds3-pro:64gb', 'oppo:find-x8:256gb'] + ); + assert.deepEqual(report.seed_drift, [ + { + seed_key: 'samsung:galaxy-buds3-pro:64gb', + normalized_key: 'samsung:galaxy-buds3-pro:unknown', + stored_status: 'published', + normalized_key_stored: false, + }, + ]); + assert.ok(report.master_drift.some(d => d.normalized_key === 'samsung:galaxy-buds3-pro:unknown')); + assert.deepEqual( + report.listing_drift.map(d => [d.kind, d.normalized_key]), + [ + ['variant', 'apple:iphone-16:256gb'], + ['size_not_stated', 'apple:iphone-16:unknown'], + ] + ); + assert.deepEqual(report.slug_collisions, [ + { + kind: 'two_items', + slugs: ['apple-iphone-16'], + items: ['apple:iphone-16', 'apple:iphone-16-x'], + }, + { + kind: 'item_slug_differs', + slugs: ['apple-iphone-16', 'apple-iphone-16-vn'], + items: ['apple:iphone-16'], + }, + { + kind: 'item_slug_differs', + slugs: ['oppo-find-x8', 'oppo-find-x8-128gb'], + items: ['oppo:find-x8'], + }, + { kind: 'still_sized', slugs: ['oppo-find-x8-128gb'], items: ['oppo:find-x8'] }, + ]); + assert.deepEqual( + report.storage_mismatch.map(r => r.model_key), + ['oppo:find-x8:256gb'] + ); + assert.equal(report.multi_sku_listings.length, 1); + + console.log('item-inventory-report tests passed'); +} + +run(); diff --git a/apps/importer/src/shopee/jobs/item-inventory-report.ts b/apps/importer/src/shopee/jobs/item-inventory-report.ts new file mode 100644 index 00000000..7367ef0a --- /dev/null +++ b/apps/importer/src/shopee/jobs/item-inventory-report.ts @@ -0,0 +1,328 @@ +/** + * Read-only Item migration inventory (DOS/Rate#1148, design doc + * `Docs/Proposals/2026-10-27-item-variant-listing-model.md` sections 7 and 7.3, phase M0). + * + * Usage: + * pnpm --filter @repo/crawlers exec tsx src/shopee/jobs/item-inventory-report.ts + * + * Reads `rate.categories`, `rate.master_products` and `rate.products` with the public anon + * key (SUPABASE_URL + SUPABASE_ANON_KEY, or the NEXT_PUBLIC_ pair the web uses) and prints + * a JSON report. Besides the SQL checks of section 7.3, it runs today's `ProductNormalizer` + * on every seed, master name and listing title, which SQL cannot do: that is how seed key + * drift shows up. + * + * Safety invariant: SELECTs only. There is no write path and no flag that adds one. + * Tables anon cannot read (`price_alerts`) are not queried; see the issue for the SQL. + */ +import { pathToFileURL } from 'node:url'; +import dotenv from 'dotenv'; +import { FLAGSHIP_DEVICES_TO_SEED, type FlagshipDeviceSeed } from '../flagship-seed-data.js'; +import { createImporterLogger } from '../logger.js'; +import { fetchAllPages, resolveBackfillSupabaseTarget } from '../paging.js'; +import { ProductNormalizer } from '../product-normalizer.js'; +import { scopeSupabaseToRate, type RateSupabaseClient, type ShopeeProduct } from '../types.js'; + +const logger = createImporterLogger('ItemInventoryReport'); + +export interface InventoryMaster { + id: string; + model_key: string; + slug: string; + name: string; + brand: string | null; + status: string | null; + category_id: string | null; + storage_gb: number | null; +} + +export interface InventoryListing { + id: string; + cluster_id: string | null; + title: string | null; + platform: string | null; + platform_item_id: string | null; + price_min: number | string | null; + price_max: number | string | null; +} + +export interface InventoryCategory { + id: string; + slug: string; +} + +/** Section 7.3 shape check: three `:`-separated segments. */ +const KEY_SHAPE = /^[a-z0-9][a-z0-9-]*:[a-z0-9][a-z0-9.-]*:[a-z0-9-]+$/; +/** The suffix `rate.item_slug_from_master` strips (section 7.1). */ +const VARIANT_SLUG_SUFFIX = /-(\d+(gb|tb)|standard|unknown)$/; +/** The trailing size `rate.item_name_from_master` strips (section 7.1). */ +const VARIANT_NAME_SUFFIX = /\s\d+\s?(gb|tb)$/i; +/** Categories with no storage axis (section 4.2). */ +const SIZELESS_CATEGORIES = new Set(['earbuds']); + +/** `split_part(k, ':', 1) || ':' || split_part(k, ':', 2)`, as the M2 backfill groups. */ +export const itemKeyOf = (modelKey: string): string => { + const [brand = '', model = ''] = modelKey.split(':'); + return `${brand}:${model}`; +}; +export const variantKeyOf = (modelKey: string): string => modelKey.split(':')[2] ?? ''; +export const itemSlugOf = (slug: string): string => slug.replace(VARIANT_SLUG_SUFFIX, ''); +export const itemNameOf = (name: string): string => name.replace(VARIANT_NAME_SUFFIX, ''); + +/** What the normalizer keys a bare title as, with no marketplace SKUs to help it. */ +export function normalizedKeyOf(normalizer: ProductNormalizer, title: string): string { + const raw: ShopeeProduct = { + itemId: 'inventory', + shopId: 'inventory', + url: 'https://rate.vn', + title, + priceMin: 0, + priceMax: 0, + currency: 'VND', + historicalSold: 0, + ratingStar: 0, + ratingCount: 0, + images: [], + models: [], + attributes: [], + shop: { shopId: 'inventory', name: 'inventory', isMall: false }, + reviews: [], + }; + return normalizer.normalize(raw).model_key; +} + +export function buildInventoryReport(input: { + categories: InventoryCategory[]; + masters: InventoryMaster[]; + listings: InventoryListing[]; + seeds?: FlagshipDeviceSeed[]; + normalizer?: ProductNormalizer; +}) { + const normalizer = input.normalizer ?? new ProductNormalizer(); + const seeds = input.seeds ?? FLAGSHIP_DEVICES_TO_SEED; + const categorySlug = new Map(input.categories.map(c => [c.id, c.slug])); + const masters = input.masters; + const byId = new Map(masters.map(m => [m.id, m])); + const byKey = new Map(masters.map(m => [m.model_key, m])); + + const items = new Map(); + for (const m of masters) { + const list = items.get(itemKeyOf(m.model_key)) ?? []; + list.push(m); + items.set(itemKeyOf(m.model_key), list); + } + + const counts = { + total: masters.length, + published: masters.filter(m => m.status === 'published').length, + draft: masters.filter(m => m.status === 'draft').length, + archived: masters.filter(m => m.status === 'archived').length, + null_status: masters.filter(m => m.status === null).length, + bad_shape: masters.filter(m => !KEY_SHAPE.test(m.model_key)).length, + khc_keys: masters.filter(m => m.model_key.startsWith('khc:')).length, + unknown_masters: masters.filter(m => variantKeyOf(m.model_key) === 'unknown').length, + standard_masters: masters.filter(m => variantKeyOf(m.model_key) === 'standard').length, + items: items.size, + multi_variant_items: [...items.values()].filter(v => v.length > 1).length, + listings: input.listings.length, + listings_without_master: input.listings.filter(p => !p.cluster_id).length, + listings_on_missing_master: input.listings.filter(p => p.cluster_id && !byId.has(p.cluster_id)) + .length, + }; + + const legacyKeys = masters + .filter(m => !KEY_SHAPE.test(m.model_key) || m.model_key.startsWith('khc:')) + .map(m => ({ model_key: m.model_key, status: m.status })); + + const itemConflicts = [...items.entries()] + .filter( + ([, v]) => + // count(DISTINCT ...) ignores NULLs, so a missing brand or category is not a conflict. + new Set(v.map(m => m.brand).filter(b => b !== null)).size > 1 || + new Set(v.map(m => m.category_id).filter(c => c !== null)).size > 1 + ) + .map(([item_key, v]) => ({ item_key, variants: v.map(m => m.model_key) })); + + const sizelessWithSize = masters + .filter( + m => + SIZELESS_CATEGORIES.has(categorySlug.get(m.category_id ?? '') ?? '') && + variantKeyOf(m.model_key) !== 'unknown' + ) + .map(m => ({ model_key: m.model_key, status: m.status, storage_gb: m.storage_gb })); + + const seedDrift = seeds + .map(s => { + const normalized = normalizedKeyOf(normalizer, s.name); + const stored = byKey.get(s.model_key); + return { + seed_key: s.model_key, + normalized_key: normalized, + stored_status: stored?.status ?? null, + normalized_key_stored: byKey.has(normalized), + }; + }) + .filter(d => d.seed_key !== d.normalized_key); + + const masterDrift = masters + .map(m => ({ + model_key: m.model_key, + normalized_key: normalizedKeyOf(normalizer, m.name), + status: m.status, + })) + .filter(d => d.model_key !== d.normalized_key); + + // Listings whose own title keys elsewhere: another Item, the same Item at another size, or + // the same Item with no size stated (the listings that lose their variant on multi-size Items). + const listingDrift = input.listings.flatMap(p => { + const master = p.cluster_id ? byId.get(p.cluster_id) : undefined; + if (!master || !p.title) return []; + const normalized = normalizedKeyOf(normalizer, p.title); + if (normalized === master.model_key) return []; + return [ + { + kind: + itemKeyOf(normalized) !== itemKeyOf(master.model_key) + ? 'item' + : variantKeyOf(normalized) === 'unknown' + ? 'size_not_stated' + : 'variant', + master_key: master.model_key, + normalized_key: normalized, + platform: p.platform, + platform_item_id: p.platform_item_id, + title: p.title.slice(0, 120), + }, + ]; + }); + + // Item slugs from the section 7.1 helper. Two Items on one slug abort the M2 backfill. This + // also covers an Item slug that is another Item's variant slug: that variant slug carries no + // size suffix, so it strips to itself. An Item whose masters strip to different slugs gets + // whichever one the backfill's representative master has, and a slug that still ends in a + // size after stripping fails the post-M2 check of section 7.3. + const itemSlugOwners = new Map>(); + const itemSlugsByItem = new Map>(); + for (const m of masters) { + const slug = itemSlugOf(m.slug); + const owners = itemSlugOwners.get(slug) ?? new Set(); + owners.add(itemKeyOf(m.model_key)); + itemSlugOwners.set(slug, owners); + const slugs = itemSlugsByItem.get(itemKeyOf(m.model_key)) ?? new Set(); + slugs.add(slug); + itemSlugsByItem.set(itemKeyOf(m.model_key), slugs); + } + const slugCollisions = [ + ...[...itemSlugOwners.entries()] + .filter(([, owners]) => owners.size > 1) + .map(([slug, owners]) => ({ kind: 'two_items', slugs: [slug], items: [...owners] })), + ...[...itemSlugsByItem.entries()] + .filter(([, slugs]) => slugs.size > 1) + .map(([item, slugs]) => ({ kind: 'item_slug_differs', slugs: [...slugs], items: [item] })), + ...[...itemSlugOwners.entries()] + .filter(([slug]) => /-\d+(gb|tb)$/.test(slug)) + .map(([slug, owners]) => ({ kind: 'still_sized', slugs: [slug], items: [...owners] })), + ]; + + // Broader than the section 7.3 check (a size anywhere, not only at the end), so a name such as + // "12GB RAM" is flagged too; those rows are read by hand. + const sizedItemNames = masters + .filter(m => /\d+\s?(gb|tb)\b/i.test(itemNameOf(m.name))) + .map(m => ({ model_key: m.model_key, name: m.name })); + + const storageMismatch = masters + .filter(m => { + const v = variantKeyOf(m.model_key); + // `1tb` and suffixed keys such as `256gb-cellular` (section 4.2) are sized too. + const sized = /^(\d+)(gb|tb)(?:-|$)/.exec(v); + if (!sized) return m.storage_gb !== null; + return Number(sized[1]) * (sized[2] === 'tb' ? 1024 : 1) !== m.storage_gb; + }) + .map(m => ({ model_key: m.model_key, storage_gb: m.storage_gb })); + + const multiSkuListings = input.listings + .filter( + p => + p.price_min !== null && p.price_max !== null && Number(p.price_min) !== Number(p.price_max) + ) + .map(p => ({ + master_key: p.cluster_id ? (byId.get(p.cluster_id)?.model_key ?? null) : null, + platform: p.platform, + platform_item_id: p.platform_item_id, + price_min: Number(p.price_min), + price_max: Number(p.price_max), + })); + + return { + counts, + legacy_keys: legacyKeys, + item_conflicts: itemConflicts, + sizeless_with_size: sizelessWithSize, + seed_drift: seedDrift, + master_drift: masterDrift, + listing_drift: listingDrift, + slug_collisions: slugCollisions, + sized_item_names: sizedItemNames, + storage_mismatch: storageMismatch, + multi_sku_listings: multiSkuListings, + }; +} + +export async function loadInventory(supabase: RateSupabaseClient) { + const categories = await fetchAllPages('categories', (from, to) => + supabase.from('categories').select('id, slug').order('id').range(from, to) + ); + const masters = await fetchAllPages('master_products', (from, to) => + supabase + .from('master_products') + .select('id, model_key, slug, name, brand, status, category_id, storage_gb') + .order('id') + .range(from, to) + ); + const listings = await fetchAllPages('products', (from, to) => + supabase + .from('products') + .select('id, cluster_id, title, platform, platform_item_id, price_min, price_max') + .order('id') + .range(from, to) + ); + return { categories, masters, listings }; +} + +const isDirectExecution = + typeof process !== 'undefined' && + Array.isArray(process.argv) && + Boolean(process.argv[1]) && + import.meta.url === pathToFileURL(process.argv[1]!).href; + +if (isDirectExecution) { + dotenv.config({ quiet: true }); + dotenv.config({ path: '../../.env', quiet: true }); + (async () => { + const { createClient } = await import('@supabase/supabase-js'); + const rawUrl = process.env.SUPABASE_URL || process.env.NEXT_PUBLIC_SUPABASE_URL || ''; + // The anon key on purpose: the report must see what any visitor sees, and nothing more. + const key = ( + process.env.SUPABASE_ANON_KEY || + process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY || + '' + ).trim(); + if (!rawUrl.trim() || !key) { + throw new Error('Missing SUPABASE_URL or SUPABASE_ANON_KEY in environment.'); + } + const { url, host } = resolveBackfillSupabaseTarget(rawUrl); + const supabase = scopeSupabaseToRate( + createClient(url, key, { auth: { persistSession: false } }) + ); + // stderr, so stdout stays one JSON document. + process.stderr.write(`Target: ${host} (read-only) +`); + const report = buildInventoryReport(await loadInventory(supabase)); + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); + })().catch((err: unknown) => { + logger.error( + 'inventory.fatal', + `Item inventory report failed: ${err instanceof Error ? err.message : String(err)}` + ); + process.exit(1); + }); +}