From 4a7686dfba64fa18e4f9321f3992622a7bdeb044 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:14:22 +0700 Subject: [PATCH 1/2] fix(web): batch offers read availability; radar and best offer skip sold-out listings - getMasterProductOffersBatch reads rate.products.availability per chunk and attaches it, like the single-product read (#1170). - The deal radar drops out_of_stock and discontinued offers before the market floor, spread, deals and traps, for every caller (#1170). - selectBestOfferFromListings decides on availability, not stock: crawlers report stock 0 when they read no count (#1171). - The availability read no longer falls back to unknown on a failed read or a missing column; it throws RateDataUnavailableError like the offer reads (#1174). Co-Authored-By: Claude Opus 5.5 --- apps/web/src/lib/rate-api.ts | 87 +++--- .../__tests__/core-rate-architecture.test.ts | 266 +++++++++++++++++- apps/web/src/server/rate/api-handler.ts | 20 +- apps/web/src/server/rate/deals.ts | 5 +- 4 files changed, 317 insertions(+), 61 deletions(-) diff --git a/apps/web/src/lib/rate-api.ts b/apps/web/src/lib/rate-api.ts index 203db13a..f87ef989 100644 --- a/apps/web/src/lib/rate-api.ts +++ b/apps/web/src/lib/rate-api.ts @@ -1317,7 +1317,7 @@ export async function getMasterProductOffers( const rows = viewRows as unknown as RawTrustViewRow[]; const [extras, availability] = await Promise.all([ sellerExtrasForTrustViewRows(rows), - getListingAvailabilityForMaster(cleanMasterId), + getListingAvailability([cleanMasterId]), ]); return rows.map((row) => withOfferAvailability( @@ -1342,7 +1342,7 @@ export async function getMasterProductOffers( .order('id') .range(from, to) ), - getListingAvailabilityForMaster(cleanMasterId), + getListingAvailability([cleanMasterId]), ]); return (rows as unknown as RawOfferRow[]).map((row) => withOfferAvailability(mapOfferRowToMerchantOffer(row), availability) @@ -1350,53 +1350,34 @@ export async function getMasterProductOffers( } /** - * Availability of every listing of a master, by listing id. The trust view - * predates the column (DOS-Me 20261020230000), so it is read from - * rate.products. A failed read, or a schema without the column (the E2E - * stack built from this repo's migration copy, until DOS/Rate#1147), leaves - * availability unknown: the offers still render, only without their sold-out - * label, and the page's prices come from the stored stats either way. + * Availability of every listing under the given masters (at most + * IN_FILTER_CHUNK_SIZE), by listing id. The trust view predates the column + * (DOS-Me 20261020230000), so it is read from rate.products. A failed read + * throws like the offer reads it completes (DOS/Rate#1174). */ -async function getListingAvailabilityForMaster( - masterId: string +async function getListingAvailability( + masterIds: readonly string[] ): Promise> { const supabase = getRateClient(); - const byId = new Map(); - const pageSize = 1000; - try { - for (let from = 0, pages = 0; pages < FETCH_ALL_MAX_PAGES; pages++) { - const { data, error } = (await supabase + const rows = await fetchAllRows<{ id: unknown; availability: unknown }>( + 'listing availability', + (from, to) => + supabase .from('products') .select('id, availability') - .eq('cluster_id', masterId) + .in('cluster_id', [...masterIds]) .order('id') - .range(from, from + pageSize - 1)) as unknown as { - data: { id: string; availability: unknown }[] | null; + .range(from, to) as unknown as PromiseLike<{ + data: { id: unknown; availability: unknown }[] | null; error: { message: string } | null; - }; - if (error) { - log.warn('listing availability unavailable', { - masterId, - message: error.message, - }); - return new Map(); - } - const page = data ?? []; - for (const row of page) { - const availability = toOfferAvailability(row?.availability); - if (availability && typeof row?.id === 'string') { - byId.set(row.id, availability); - } - } - if (page.length < pageSize) break; - from += page.length; + }> + ); + const byId = new Map(); + for (const row of rows) { + const availability = toOfferAvailability(row?.availability); + if (availability && typeof row?.id === 'string') { + byId.set(row.id, availability); } - } catch (err) { - log.warn('listing availability unavailable', { - masterId, - message: err instanceof Error ? err.message : String(err), - }); - return new Map(); } return byId; } @@ -1435,9 +1416,8 @@ export async function getMasterProductOffersBatch( const batchIds = uniqueIds.slice(i, i + chunkSize); // Paged per chunk: 150 masters can easily hold more than 1000 offers. - const viewData = await fetchAllRows( - 'v_products_with_merchant_trust', - (from, to) => + const [viewData, availability] = await Promise.all([ + fetchAllRows('v_products_with_merchant_trust', (from, to) => supabase .from('v_products_with_merchant_trust') .select(TRUST_VIEW_SELECT) @@ -1445,7 +1425,9 @@ export async function getMasterProductOffersBatch( .order('price_current', { ascending: true }) .order('id') .range(from, to) - ).catch(() => null); + ).catch(() => null), + getListingAvailability(batchIds), + ]); if (viewData) { const rows = viewData as unknown as RawTrustViewRow[]; @@ -1455,10 +1437,13 @@ export async function getMasterProductOffersBatch( if (!cid) continue; const list = offersByMaster.get(cid) ?? []; list.push( - mapTrustViewRowToMerchantOffer( - row, - extras.followerCounts.get(row.merchant_id), - extras.metadataDosafeRisks.get(row.merchant_id) + withOfferAvailability( + mapTrustViewRowToMerchantOffer( + row, + extras.followerCounts.get(row.merchant_id), + extras.metadataDosafeRisks.get(row.merchant_id) + ), + availability ) ); offersByMaster.set(cid, list); @@ -1480,7 +1465,9 @@ export async function getMasterProductOffersBatch( const cid = item.cluster_id ? String(item.cluster_id) : ''; if (!cid) continue; const list = offersByMaster.get(cid) ?? []; - list.push(mapOfferRowToMerchantOffer(item)); + list.push( + withOfferAvailability(mapOfferRowToMerchantOffer(item), availability) + ); offersByMaster.set(cid, list); } } diff --git a/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts b/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts index b19de859..b1e47e81 100644 --- a/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts +++ b/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts @@ -5936,7 +5936,11 @@ async function runCoreArchitectureSuite() { { status: 200, headers: { 'content-type': 'application/json' } } ); } - if (url.includes('/rest/v1/products') && url.includes('id=in.')) { + if ( + url.includes('/rest/v1/products') && + url.includes('id=in.') && + !url.includes('cluster_id=in.') + ) { return new Response( JSON.stringify([ { @@ -13408,6 +13412,266 @@ async function runCoreArchitectureSuite() { ); } + // ------------------------------------------------------------------------- + // 126k. Batch offers read availability; radar and best offer skip sold-out + // listings; stock 0 is not sold out (#1170, #1171, #1174) + // ------------------------------------------------------------------------- + { + const { + RateDataUnavailableError, + getMasterProductOffers, + getMasterProductOffersBatch, + } = await import('@/lib/rate-api'); + const { computeCrossPlatformDealRadarFromData } = + await import('@/server/rate/deals'); + const { selectBestOfferFromListings } = + await import('@/server/rate/api-handler'); + + const json1170 = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }); + const viewRow1170 = (id: string, clusterId: string, price: number) => ({ + id, + cluster_id: clusterId, + merchant_id: `m-${id}`, + platform: 'shopee', + title: 't', + price_current: price, + stock: 0, + url: 'https://shopee.vn/p', + threat_risk_score: 5, + threat_source: 'dosafe', + dosafe_risk_score: 5, + review_risk_score: 5, + review_risk_breakdown: { total_reviews: 3 }, + }); + const availabilityReads1170: string[] = []; + const mockOffers1170 = (opts: { + viewFails?: boolean; + availabilityFails?: boolean; + }) => { + availabilityReads1170.length = 0; + globalThis.fetch = (async (input: RequestInfo | URL) => { + const url = new URL( + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : input.url + ); + const firstPage = Number(url.searchParams.get('offset') ?? 0) === 0; + if (url.pathname.endsWith('/v_products_with_merchant_trust')) { + if (opts.viewFails) return json1170({ message: 'view down' }, 500); + return json1170( + firstPage + ? [ + viewRow1170('a-sold', 'master-a', 2_694_000), + viewRow1170('a-in', 'master-a', 3_140_000), + viewRow1170('b-new', 'master-b', 5_000_000), + ] + : [] + ); + } + if (url.pathname.endsWith('/products')) { + const select = url.searchParams.get('select') ?? ''; + if (select === 'id,availability') { + if (firstPage) { + availabilityReads1170.push( + url.searchParams.get('cluster_id') ?? '' + ); + } + if (opts.availabilityFails) { + return json1170( + { message: 'column products.availability does not exist' }, + 400 + ); + } + return json1170( + firstPage + ? [ + { id: 'a-sold', availability: 'out_of_stock' }, + { id: 'a-in', availability: 'in_stock' }, + { id: 'b-new', availability: null }, + ] + : [] + ); + } + // The offer fallback when the trust view fails. + return json1170( + firstPage + ? [ + { + id: 'a-sold', + cluster_id: 'master-a', + platform: 'fptshop', + title: 't', + price_current: 2_694_000, + stock: 0, + url: 'https://fptshop.com.vn/p', + merchants: { id: 'm-1', name: 'FPT Shop' }, + }, + ] + : [] + ); + } + return json1170([]); + }) as typeof fetch; + }; + + // The batch read attaches availability per chunk, like the single read. + mockOffers1170({}); + const batch1170 = await getMasterProductOffersBatch([ + 'master-a', + 'master-b', + ]); + assert.deepEqual(availabilityReads1170, ['in.(master-a,master-b)']); + assert.deepEqual( + (batch1170.get('master-a') ?? []).map((o) => [o.id, o.availability]), + [ + ['a-sold', 'out_of_stock'], + ['a-in', 'in_stock'], + ] + ); + assert.deepEqual( + (batch1170.get('master-b') ?? []).map((o) => [o.id, o.availability]), + [['b-new', null]] + ); + mockOffers1170({ viewFails: true }); + const fallback1170 = await getMasterProductOffersBatch(['master-a']); + assert.deepEqual( + (fallback1170.get('master-a') ?? []).map((o) => [o.id, o.availability]), + [['a-sold', 'out_of_stock']] + ); + + // No missing-column fallback: a failed availability read fails the + // offer read, single and batch, with or without the trust view (#1174). + for (const viewFails of [false, true]) { + mockOffers1170({ viewFails, availabilityFails: true }); + await assert.rejects( + () => getMasterProductOffers('master-a'), + (err: unknown) => err instanceof RateDataUnavailableError + ); + await assert.rejects( + () => getMasterProductOffersBatch(['master-a']), + (err: unknown) => err instanceof RateDataUnavailableError + ); + } + + // The radar never cites a sold-out or discontinued listing (#1170). + const radarMaster1170 = { + id: 'master-a', + slug: 'samsung-galaxy-buds3-pro', + name: 'Samsung Galaxy Buds3 Pro', + brand: 'Samsung', + min_price: 3_140_000, + max_price: 3_690_000, + } as unknown as Parameters< + typeof computeCrossPlatformDealRadarFromData + >[0][number]['master']; + const radarOffer1170 = ( + id: string, + platform: string, + price: number, + availability: string | null + ) => + ({ + id, + platform, + title: `Offer ${id}`, + price_current: price, + price_original: 4_490_000, + stock: 0, + url: `https://example.com/${id}`, + availability, + merchant: { + id: `m-${id}`, + name: `Merchant ${id}`, + platform, + is_official: true, + risk_score: 5, + }, + }) as never; + const radar1170 = computeCrossPlatformDealRadarFromData( + [ + { + master: radarMaster1170, + offers: [ + radarOffer1170('fpt-sold', 'fptshop', 2_694_000, 'out_of_stock'), + radarOffer1170('fpt-gone', 'fptshop', 3_032_000, 'discontinued'), + radarOffer1170('tgdd', 'tgdd', 3_140_000, 'in_stock'), + radarOffer1170('cps', 'cellphones', 3_690_000, null), + ], + }, + ], + { categorySlug: 'all', limit: 10 } + ); + const radarText1170 = JSON.stringify(radar1170); + assert.ok(radar1170.priceSpreadLeaders.length > 0, radarText1170); + assert.ok(!radarText1170.includes('fpt-sold'), radarText1170); + assert.ok(!radarText1170.includes('fpt-gone'), radarText1170); + assert.ok(!radarText1170.includes('2694000'), radarText1170); + assert.equal(radar1170.totalOffersAnalyzed, 2); + + // stock 0 means "no count read", not sold out; availability decides (#1171). + const best1171 = selectBestOfferFromListings([ + { + id: 'fpt-sold', + title: 'Buds3 Pro', + price_current: 2_694_000, + stock: 0, + availability: 'out_of_stock', + url: 'https://fptshop.com.vn/p', + platform: 'fptshop', + }, + { + id: 'fpt-gone', + title: 'Buds3 Pro', + price_current: 3_032_000, + availability: 'discontinued', + url: 'https://fptshop.com.vn/q', + platform: 'fptshop', + }, + { + id: 'tgdd', + title: 'Buds3 Pro', + price_current: 3_140_000, + stock: 0, + availability: 'in_stock', + url: 'https://www.thegioididong.com/p', + platform: 'tgdd', + }, + { + id: 'cps', + title: 'Buds3 Pro', + price_current: 3_690_000, + stock: 0, + url: 'https://cellphones.com.vn/p', + platform: 'cellphones', + }, + ] as unknown as Parameters[0]); + assert.equal(best1171.bestOffer?.id, 'tgdd'); + assert.equal(best1171.bestOffer?.price, 3_140_000); + assert.deepEqual(best1171.offerPlatforms, ['tgdd', 'cellphones']); + // An unknown availability with stock 0 is still buyable. + const unknown1171 = selectBestOfferFromListings([ + { + id: 'only', + title: 'Buds3 Pro', + price_current: 3_140_000, + stock: 0, + url: 'https://www.thegioididong.com/p', + platform: 'tgdd', + }, + ] as unknown as Parameters[0]); + assert.equal(unknown1171.bestOffer?.id, 'only'); + + console.log( + ' ✅ [126k/135] Batch offer availability, sold-out radar and best offer, stock 0 (#1170, #1171, #1174) passed' + ); + } + // ------------------------------------------------------------------------- // 127. Rate Server Index & Extension Entity, Slug, and Category Bounds (#1029) // ------------------------------------------------------------------------- diff --git a/apps/web/src/server/rate/api-handler.ts b/apps/web/src/server/rate/api-handler.ts index ffce902d..f28a6e7a 100644 --- a/apps/web/src/server/rate/api-handler.ts +++ b/apps/web/src/server/rate/api-handler.ts @@ -5,6 +5,10 @@ import { MAX_VALID_VND_PRICE, normalizeOfferOriginalPrice, } from '@/lib/rate-api'; +import { + isOfferUnavailable, + type OfferAvailability, +} from '@/lib/product-market-stats'; export const PUBLIC_API_CORS_HEADERS: Record = { 'Access-Control-Allow-Origin': '*', @@ -243,6 +247,7 @@ export interface WatchlistCandidateOffer { price_current: number; price_original?: unknown; stock?: unknown; + availability?: OfferAvailability | null; url?: unknown; platform?: unknown; merchant?: { @@ -274,19 +279,16 @@ export function withPlausibleMinPrice( export function selectBestOfferFromListings( offers: readonly WatchlistCandidateOffer[] ): { bestOffer: WatchlistBestOffer | null; offerPlatforms: string[] } { - const safeOffers = Array.isArray(offers) ? offers : []; + // A sold-out or discontinued listing is never the best offer. stock is no + // signal: the crawlers report 0 when they read no count (DOS/Rate#1171). + const safeOffers = (Array.isArray(offers) ? offers : []).filter( + (o) => !isOfferUnavailable(o) + ); // A bait-price listing is never the best offer (DOS/Rate#797). const valid = plausibleOffers(safeOffers).filter((o) => { if (!o || typeof o !== 'object') return false; const price = parseFiniteDecimalInput(o.price_current); - const stock = o.stock == null ? 1 : parseFiniteDecimalInput(o.stock); - return ( - Number.isFinite(stock) && - stock > 0 && - Number.isFinite(price) && - price > 0 && - price <= MAX_VALID_VND_PRICE - ); + return Number.isFinite(price) && price > 0 && price <= MAX_VALID_VND_PRICE; }); const normalizeOfferPlatform = (o: WatchlistCandidateOffer): string => String(o.merchant?.platform || o.platform || 'shopee') diff --git a/apps/web/src/server/rate/deals.ts b/apps/web/src/server/rate/deals.ts index 8b6ff6c1..fcd561ad 100644 --- a/apps/web/src/server/rate/deals.ts +++ b/apps/web/src/server/rate/deals.ts @@ -18,6 +18,7 @@ import { getRadarSignal, type RadarSignal, } from '../../lib/deal-radar'; +import { isOfferUnavailable } from '../../lib/product-market-stats'; import { plausibleOffers } from '@repo/shared-data/rules'; import { RATE_CACHE_TAGS, @@ -302,12 +303,14 @@ export function computeCrossPlatformDealRadarFromData( const clusterOffers = Array.isArray(offers) ? offers : []; // Bait-price listings (far below every other store) set no market floor, - // spread, deal or trap (DOS/Rate#797). + // spread, deal or trap (DOS/Rate#797), and neither do sold-out or + // discontinued listings, which keep their last price (DOS/Rate#1170). const validOffers = plausibleOffers( clusterOffers .filter( (o): o is MerchantOffer => Boolean(o && typeof o === 'object' && !Array.isArray(o)) && + !isOfferUnavailable(o) && typeof o.price_current === 'number' && Number.isFinite(o.price_current) && o.price_current > 0 && From 7ccd4ecbc3843efe76dc5af6b393a4f4a9e604c8 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:16:46 +0700 Subject: [PATCH 2/2] test(web): restore fetch after the availability block Co-Authored-By: Claude Opus 5.5 --- .../src/server/rate/__tests__/core-rate-architecture.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts b/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts index b1e47e81..f01f633e 100644 --- a/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts +++ b/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts @@ -13427,6 +13427,7 @@ async function runCoreArchitectureSuite() { const { selectBestOfferFromListings } = await import('@/server/rate/api-handler'); + const origFetch1170 = globalThis.fetch; const json1170 = (body: unknown, status = 200) => new Response(JSON.stringify(body), { status, @@ -13666,6 +13667,7 @@ async function runCoreArchitectureSuite() { }, ] as unknown as Parameters[0]); assert.equal(unknown1171.bestOffer?.id, 'only'); + globalThis.fetch = origFetch1170; console.log( ' ✅ [126k/135] Batch offer availability, sold-out radar and best offer, stock 0 (#1170, #1171, #1174) passed'