diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f27e26a1d..58edbe848 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: run: node --test scripts/e2e/stale-stacks.test.mjs - name: Test the agent release scripts - run: node --test scripts/ops/merge-when-green.test.mjs scripts/ops/board.test.mjs scripts/ops/promote.test.mjs scripts/ops/verify-live.test.mjs + run: node --test scripts/ops/merge-when-green.test.mjs scripts/ops/board.test.mjs scripts/ops/promote.test.mjs scripts/ops/verify-live.test.mjs scripts/ops/incident-report.test.mjs - name: Replay rate migrations on PostgreSQL 17 (Docker) run: node scripts/test-migration-replay.mjs diff --git a/.github/workflows/incident-report.yml b/.github/workflows/incident-report.yml index 05912f130..67add2954 100644 --- a/.github/workflows/incident-report.yml +++ b/.github/workflows/incident-report.yml @@ -2,8 +2,9 @@ name: Incident report # The incident inbox for agents (DOS/Rate#1057). When CI fails on dev or main, # open an `incident` issue (or comment on the open one) with the run, commit -# and failing jobs; when it passes again, close that issue. Agents treat an -# open `incident` issue as their first priority (AGENTS.md). +# and failing jobs; when every failing job passes again, close that issue +# (scripts/ops/incident-report.mjs). Agents treat an open `incident` issue +# as their first priority (AGENTS.md). # workflow_run only fires for this file as it is on the default branch (dev). on: workflow_run: @@ -36,57 +37,20 @@ jobs: BRANCH: ${{ github.event.workflow_run.head_branch }} CONCLUSION: ${{ github.event.workflow_run.conclusion }} RUN_ID: ${{ github.event.workflow_run.id }} + RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} RUN_URL: ${{ github.event.workflow_run.html_url }} SHA: ${{ github.event.workflow_run.head_sha }} COMMIT_TITLE: ${{ github.event.workflow_run.head_commit.message }} steps: - - name: Open, update or close the incident - run: | - set -euo pipefail - # Events can arrive out of order: only the run of the branch's - # current head speaks for the branch, so a stale failure never - # reopens an incident a newer run has closed. - head="$(gh api "repos/${REPO}/commits/${BRANCH}" --jq .sha)" - if [ "$head" != "$SHA" ]; then - echo "Run is for ${SHA:0:8}, ${BRANCH} is at ${head:0:8} now: skipped." - exit 0 - fi - - title="incident: ${WORKFLOW} failing on ${BRANCH}" - # Untrusted text, shown as inline code so its @mentions and #refs - # ping or link nothing (backticks removed so it cannot close the span). - subject="\`$(printf '%s' "${COMMIT_TITLE}" | head -n 1 | cut -c1-120 | tr -d '\`')\`" - gh label create incident -R "$REPO" --color B60205 \ - --description "Production or CI breakage: first priority for any free agent" 2>/dev/null || true - # Read the list first so an API error fails the step (pipefail does - # not cover process substitution) instead of creating a duplicate. - list="$(gh issue list -R "$REPO" --state open --label incident --limit 50 \ - --json number,title --jq '.[] | "\(.number)\t\(.title)"')" - open="" - while IFS=$'\t' read -r num t; do - if [ "$t" = "$title" ]; then open="$num"; break; fi - done <<< "$list" + # The script comes from the default branch (workflow_run runs there), + # never from the run's commit: no untrusted code gets the token. + - name: Check out the incident script + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: scripts/ops + persist-credentials: false - if [ "$CONCLUSION" = "success" ]; then - if [ -n "$open" ]; then - gh issue comment "$open" -R "$REPO" --body "Recovered: ${RUN_URL} passed on \`${SHA:0:8}\` (${subject})." - gh issue close "$open" -R "$REPO" --reason completed - fi - exit 0 - fi - - failing="$(gh run view "$RUN_ID" -R "$REPO" --json jobs \ - --jq '[.jobs[] | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "cancelled") | .name] | join(", ")')" - body="$(printf '%s\n' \ - "**${WORKFLOW}** ${CONCLUSION} on \`${BRANCH}\`." \ - "" \ - "- Run: ${RUN_URL}" \ - "- Commit: \`${SHA:0:8}\` ${subject}" \ - "- Failing jobs: ${failing:-unknown}" \ - "" \ - "Agents: this is the first priority (AGENTS.md). Claim it, find the breaking change, fix forward. The issue closes itself when the workflow passes on \`${BRANCH}\` again.")" - if [ -n "$open" ]; then - gh issue comment "$open" -R "$REPO" --body "$body" - else - gh issue create -R "$REPO" --title "$title" --label incident --body "$body" - fi + # Closes only when every job that failed has concluded `success` again: + # a job skipped on a later push proves nothing (DOS/Rate#1134). + - name: Open, update or close the incident + run: node scripts/ops/incident-report.mjs diff --git a/AGENTS.md b/AGENTS.md index d16251171..3b6d9446f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -72,7 +72,7 @@ gh issue list -R DOS/Rate-Services --state open - **Quy tắc Định danh Agent (Agent Nickname) & Tránh trùng lặp:** - Vì mọi Agent đều chạy dưới tài khoản GitHub của JOY (`@me`), **MỖI AGENT PHẢI TỰ ĐẶT MỘT NICKNAME NGẮN GỌN** theo chuyên môn hoặc phiên làm việc (ví dụ: `Crawl`, `Web`, `Core`, `DB`, `QA`) và gắn cả **Nhãn `agent:`** lẫn **Cột `Agent` trên GitHub Project #9**. - **TUYỆT ĐỐI KHÔNG** tự ý làm các Issue đang có nhãn `status:in-progress`, `status:in-review` hoặc đã có nhãn `agent:` đang chạy (trừ khi JOY chỉ định rõ). - - **Issue có nhãn `incident` là ưu tiên số 1**, trên mọi việc khác: chúng do `.github/workflows/incident-report.yml` tự mở khi CI đỏ trên `dev`/`main` (DOS/Rate-Services có workflow tương tự cho crawler theo lịch), hoặc do Upptime mở trong `DOS/status` khi rate.vn sập. Nhận issue, tìm thay đổi gây lỗi, sửa tiếp (fix forward). Issue tự đóng khi workflow xanh lại. + - **Issue có nhãn `incident` là ưu tiên số 1**, trên mọi việc khác: chúng do `.github/workflows/incident-report.yml` tự mở khi CI đỏ trên `dev`/`main` (DOS/Rate-Services có workflow tương tự cho crawler theo lịch), hoặc do Upptime mở trong `DOS/status` khi rate.vn sập. Nhận issue, tìm thay đổi gây lỗi, sửa tiếp (fix forward). Issue tự đóng khi mọi job đã lỗi chạy xanh lại (job bị bỏ qua, `skipped`, không tính là đã hồi phục). - Sau đó ưu tiên chọn các Issue đang có nhãn `status:ready`. - Nếu yêu cầu của người dùng chưa có Issue tương ứng, **TẠO NGAY ISSUE MỚI BẰNG TIẾNG ANH** và gắn vào GitHub Project #9: ```powershell diff --git a/Docs/Proposals/2026-10-27-item-variant-listing-model.html b/Docs/Proposals/2026-10-27-item-variant-listing-model.html new file mode 100644 index 000000000..5f942de97 --- /dev/null +++ b/Docs/Proposals/2026-10-27-item-variant-listing-model.html @@ -0,0 +1,155 @@ + + + + + + + Mô hình Item, Variant, Listing + + + +
+ Đề xuất, chưa code + Chờ JOY quyết 7 câu +

Mô hình Item, Variant, Listing

+

Ngày 27/10/2026 · Issue DOS/Rate#1129 · Bản kỹ thuật đầy đủ (tiếng Anh): 2026-10-27-item-variant-listing-model.md

+ +
+

Tóm tắt một đoạn

+

Hiện mỗi dung lượng là một "sản phẩm gốc" riêng (iPhone 17 Pro 256GB và 512GB là hai trang, hai thẻ, hai suất trên bảng xếp hạng). Đề xuất: thêm bảng items đứng trên, giữ bảng hiện tại làm phiên bản (variant), mỗi gian hàng (listing) gắn vào Item và gắn vào phiên bản nếu biết rõ dung lượng. Mọi phép so giá vẫn so trong cùng một dung lượng như hôm nay, nên phần rủi ro nhất không phải viết lại. Listing không rõ dung lượng gắn vào Item, không bị đoán bừa sang bản 256GB nữa (giải quyết luôn #1105).

+
+ +

1. Mô hình JOY đặt ra, áp vào dữ liệu

+
Item: Apple iPhone 17 Pro (1 trang, 1 điểm đánh giá, giá "từ ...") + ├─ Variant 256GB (giá thấp nhất riêng, deal riêng, báo giá riêng) + │ ├─ Listing: Shopee shop A + │ └─ Listing: FPT Shop + ├─ Variant 512GB + │ └─ Listing: CellphoneS + └─ (chưa rõ dung lượng) + └─ Listing: Lazada shop B (hiện riêng, không tính vào giá "từ ...")
+

iPad Pro 11 inch và 13 inch, Apple Watch 42mm và 46mm vẫn là hai Item khác nhau (khác màn hình, pin, thông số), giữ đúng như #1113 và #1124. Sản phẩm không có dung lượng (tai nghe, nhiều đồng hồ Garmin) là Item có đúng một phiên bản, vẫn có giá bình thường; nhóm "chưa rõ dung lượng" chỉ xuất hiện ở Item bán nhiều dung lượng.

+ +

2. Hiện trạng (đã khảo sát 4 mảng)

+
+

Database

Bảng master_products mỗi dòng là một dung lượng, có slug, giá, điểm riêng. Listing trỏ về qua cluster_id. Không có tầng nào phía trên.

+

Web

Mỗi dung lượng là một trang, một URL, một mục sitemap. Bản 512GB có thể hiện thành "đối thủ" của chính bản 256GB, và một mẫu có thể chiếm nhiều suất trong top 10.

+

Importer

Khóa brand:model:dung-lượng. Tiêu đề không ghi dung lượng bị so gần đúng rồi gắn nhầm vào bản có dung lượng (#1105). Giá thấp nhất của cả listing (mọi dung lượng) bị ghi cho một bản.

+

Rate Services

Mọi bộ kiểm tra giá, rủi ro gian hàng, seeding đều nhóm theo cluster_id, tức theo dung lượng. Đây là lý do nên giữ nguyên nghĩa của nó.

+
+

Số liệu công khai lúc 27/10/2026: 16 sản phẩm đang hiển thị, chưa mẫu nào có hơn một dung lượng được công bố. Dữ liệu chuyển đổi nhỏ, phần tốn công chủ yếu là code web và Rate Services.

+ +

3. Các phương án

+
+ + + + + + +
A. Bảng Item + bảng Variant mớiB. Thêm Item phía trên bảng cũ (đề xuất)C. Một bảng tự tham chiếuD. Biến bảng cũ thành Item
Logic so giá theo dung lượngViết lạiGiữ nguyênGiữ nguyênViết lại, dễ trộn giá các dung lượng
URL, ID cũCần bảng ánh xạGiữ nguyên, tự chuyển hướngGiữID đổi nghĩa ngầm, nguy hiểm
Độ sạch về lâu dàiTốt nhấtTốt (đổi tên sau bằng view)KémTốt nhưng chuyển đổi rủi ro
Chi phíCao nhấtThấp nhấtTrung bình, phức tạp kéo dàiCao, rủi ro cao
+
+ +

4. Dữ liệu nằm ở tầng nào

+
+ + + + + + +
Dữ liệuItemVariant (dung lượng)Listing
Thông sốGần như toàn bộChỉ dung lượng, RAM, giá niêm yếtDữ liệu thô của sàn
GiáGiá "từ ..." và khoảng giáGiá thấp nhất, số gian hàng, deal, chênh giáGiá hiện tại, lịch sử giá
Đánh giá, điểmMột điểm chung cho mọi dung lượngKhông tách riêngTừng review gốc
Báo giá (alert)Để sauGiữ nguyên theo dung lượng
+
+ +

5. Ảnh hưởng

+
    +
  • Web: mỗi Item một trang /smartphones/apple-iphone-17-pro, có nút chọn dung lượng kèm giá từng bản. URL cũ (...-256gb) tự chuyển sang trang Item với đúng dung lượng đã chọn, không mất SEO. Danh mục, tìm kiếm, bảng xếp hạng, sitemap đếm theo Item. So sánh và đối đầu không còn ghép hai bản của cùng một máy.
    • +
  • API v1 và tiện ích trình duyệt: chỉ thêm trường mới, client cũ vẫn chạy. Tiện ích đã có sẵn nút chọn phiên bản, chỉ cần lấy đủ các bản từ database và bỏ việc mặc định 256GB khi không rõ.
    • +
  • Importer: khóa tách thành item_key và variant_key. So gần đúng chỉ còn ở tầng Item (tên Item không chứa dung lượng nên hết lỗi #1105). Dung lượng chỉ được gán khi có bằng chứng rõ, không đoán, không để AI tự điền.
    • +
  • Rate Services: các bộ kiểm tra giá giữ nguyên, chỉ cần xử lý listing chưa rõ dung lượng, nhận Item trong job, và dùng hàm tách khóa dùng chung.
    • +
+ +

6. Kế hoạch chuyển đổi DOS-Me

+
+ + + + + + + + + +
BướcLàm gìQuay lui
M0Đồng bộ 8 migration còn thiếu vào bản tham chiếu, chạy truy vấn kiểm kê chỉ đọc, sửa các khóa seed lệchKhông cần
M1Thêm bảng items và các cột mới (chỉ thêm, không sửa, không xóa), kèm trigger đồng bộ để code cũ đang chạy vẫn ghi đúng Item. M1 và M2 gộp thành một migration để không có khoảng hởXóa phần vừa thêm
M2Điền dữ liệu Item từ bảng cũ, thêm các hàm truy vấn mớiGỡ liên kết rồi xóa dữ liệu Item trong một transaction; dữ liệu cũ không bị đụng tới
Giai đoạn song songCode ghi cả hai tầng, web bật trang Item bằng cờ, URL cũ chuyển hướng. Tối thiểu một đợt promote và khoảng 30 ngàyTắt cờ
M3Với Item bán nhiều dung lượng: chuyển các listing "chưa rõ dung lượng" về thẳng Item, có bảng sao lưu. Chỉ chạy sau khi Rate Services đã sẵn sàngKhôi phục từ bảng sao lưu
M4Khóa ràng buộc (bắt buộc có Item, listing và dung lượng phải cùng Item)Gỡ ràng buộc
M5, M6Tùy chọn: giá theo từng SKU của sàn, đổi tên cho gọnXóa bảng, view
+
+

Lưu ý: luồng beta của DOS-Me đang bị khóa (ROADMAP 6.4). Có thể chuẩn bị và thử trên preview ngay, nhưng lên prod phải chờ luồng beta mở lại. Mỗi bước M1 đến M4 đều cần JOY duyệt riêng.

+ +

7. Bảy câu cần JOY quyết

+

Mỗi câu có phương án đề xuất. Nếu JOY đồng ý hết, chỉ cần trả lời "theo đề xuất".

+
Câu 1. URL: mỗi Item một URL được Google index, chọn dung lượng bằng ?v=256gb (đề xuất). Hay mỗi dung lượng một URL riêng được index (nhiều trang hơn nhưng nội dung gần trùng)?
+
Câu 2. Listing không rõ dung lượng: hiện thành nhóm riêng "Chưa rõ dung lượng" trên trang Item và không tính vào giá "từ ..." (đề xuất). Hay tính luôn vào giá "từ ..." (trông rẻ hơn nhưng dễ gây hiểu nhầm)?
+
Câu 3. Trục phiên bản: trước mắt chỉ theo dung lượng; kích thước màn hình hay vỏ là Item riêng như hiện nay; Wi-Fi và Cellular tạm gộp, tách sau nếu so giá bị lệch (đề xuất). Hay tách Wi-Fi và Cellular ngay từ đầu?
+
Câu 4. Điểm đánh giá: một điểm chung cho cả Item, gộp review mọi dung lượng (đề xuất)?
+
Câu 5. Báo giá: giữ theo từng dung lượng, tính năng "báo khi bất kỳ dung lượng nào xuống giá" để sau (đề xuất)?
+
Câu 6. #1105: tiếp tục giữ, gộp vào task sửa bộ khớp trong kế hoạch này, không làm bản vá tạm (đề xuất)?
+
Câu 7. API công khai: danh sách /api/v1/rate/products giữ mặc định theo dung lượng trong giai đoạn song song, sau đó mới chuyển sang theo Item kèm thông báo (đề xuất)?
+ +

8. Danh sách việc sau khi JOY duyệt

+

15 việc (PO sẽ tạo issue sau khi JOY duyệt): 3 việc nhỏ chuẩn bị, 2 migration DOS-Me, 2 việc importer, 6 việc web/API/tiện ích/E2E, 1 việc Rate Services, 1 việc tùy chọn. Đường găng: chuẩn bị, rồi M1 và M2, rồi importer và tầng dữ liệu web song song, rồi trang và Rate Services, rồi M3 và M4. Chi tiết kích thước và phụ thuộc ở mục 10 bản tiếng Anh.

+ +

9. Kết quả review

+

Đã qua review độc lập (một subagent mới hoàn toàn, Codex, rồi thêm một vòng kiểm tra lại SQL). Không có lỗi P0. Các lỗi P1 nằm ở SQL nháp và thứ tự chuyển đổi (ví dụ tai nghe suýt mất phiên bản duy nhất, chỉ mục tìm kiếm sai schema trên prod, cách quay lui không an toàn) và đã được sửa hết trong bản này. Phương án B được cả hai bên xác nhận là hợp lý.

+ +

Tài liệu này chỉ là đề xuất. Chưa có code, migration hay thay đổi dữ liệu nào được thực hiện.

+
+ + diff --git a/Docs/Proposals/2026-10-27-item-variant-listing-model.md b/Docs/Proposals/2026-10-27-item-variant-listing-model.md new file mode 100644 index 000000000..1e4c52df5 --- /dev/null +++ b/Docs/Proposals/2026-10-27-item-variant-listing-model.md @@ -0,0 +1,726 @@ +# Design: Item / variant / listing data model + +- **Date:** 2026-10-27 +- **Issue:** [DOS/Rate#1129](https://github.com/DOS/Rate/issues/1129) +- **Status:** PROPOSAL, awaiting JOY's decisions (section 9). Nothing in this document has been applied. The SQL is a draft for review and must go through `DOS-Me/supabase/migrations/` with JOY's approval. +- **Vietnamese summary for JOY:** [`2026-10-27-item-variant-listing-model.html`](./2026-10-27-item-variant-listing-model.html) +- **Decides:** [DOS/Rate#1105](https://github.com/DOS/Rate/issues/1105) (Tier 2 links storage-less titles to a sized master). + +JOY's target model, in JOY's words: one Item per product; the other storage sizes are variants of that Item; an Item has many listings. + +## 0. Summary + +- **Today, the storage variant is the unit everywhere.** `rate.master_products` has one row per `brand:model:storage`, and each has its own slug, page, sitemap entry, aggregates, alerts, rating and leaderboard slot. There is nothing above it. The UI and stats therefore count iPhone 17 Pro 256GB and 512GB as two products, and a sibling size can show up as a "competitor" or as a versus rival. +- **Recommendation: Option B, "add Item above the existing variant table".** + - Add a new table, `rate.items`. + - `master_products` stays the **variant** table, with new `item_id` and `variant_key` columns. + - Listings (`rate.products`) gain `item_id`. + - The existing `products.cluster_id` keeps meaning "this listing's variant". It becomes **NULL when the size is not stated on an Item sold in several sizes**. Sizeless lines such as earbuds keep a single variant. +- **Why this option.** Every price computation that must compare like with like keeps working unchanged, because they all group by `cluster_id`: per-variant min price, spread, deal radar, peer floors, merchant risk, price alerts. Item-level views (pages, directory, search, leaderboard, rating) are added on top. +- **Effect on #1105.** A storage-less listing attaches to the Item with no variant, instead of being fuzzy-linked to a sized master. Fuzzy matching moves to Item names, which carry no storage. Variant assignment becomes exact-only. +- **Migration.** Four required DOS-Me phases (M1 to M4, additive first, constraints last) and two optional ones, with a compatibility window. Sync triggers keep old writers consistent during the window. Old variant slugs keep resolving and redirect to the Item page with the variant preselected. The published catalog is small today: 16 published masters, all with three-part keys, and no published Item has more than one size yet (read from the public API on 2026-10-27). The backfill is cheap. The real cost is code, mostly the web app and Rate-Services. + +## 1. Survey of the current model + +### 1.1 Database (schema `rate`) + +`supabase/migrations/` in this repo is a reference copy. The DOS-Me ledger has eight later `rate` migrations that are not mirrored here: row provenance, audit aggregate RPCs, the `rate_worker` role, listing availability, the iPhone 16e offer move, review vote eligibility, plausible min price, and the product images bucket. `packages/shared-data/src/database.types.ts` is generated from the Rate copy only, so it lacks `origin`, `availability` and `spec_provenance`. Mirroring those is a prerequisite of phase M1 (section 7). + +| Object | Level today | What it assumes | Source | +| :--- | :--- | :--- | :--- | +| `master_products` | storage variant | `UNIQUE(model_key)` and `UNIQUE(slug)`, both including storage. It holds every catalog attribute and aggregate: specs, images, `min_price`/`max_price`, `merchant_count`, `user_rating_weighted`, `authentic_review_count`, status. | `20261005130000_rate_recover_master_products.sql:28-83` | +| `products` (listings) | listing | `cluster_id` FK to one master, with NO ACTION on delete. There is no variant, condition or region column. Raw marketplace SKUs live in `models jsonb`. `UNIQUE(platform, platform_item_id)`. | `20260912120000_init_rate_schema.sql:81-108`, recover `:267-270` | +| `price_history` | listing | Keyed by `product_id`. There is no master-level history. | init `:137-146` | +| `reviews`, `review_flags`, `review_votes`, `review_analysis` | listing | Reviews never reference a master. `reviews.variation` is free text. | init `:151-182`, `20261014011054` | +| `price_alerts` | variant | `master_product_id` FK with CASCADE. The alert compares against the variant's `min_price`. | `20261015010000:10-27` | +| `promotions` | listing | `sku_scope` is the only SKU concept in the schema. | `20261005130001:612-635` | +| `crawl_jobs` | listing / platform | No master reference. Payloads carry master ids as JSON. | `20261005130001:25-73` | +| `match_master_product` | variant | Filter `($4 IS NULL OR mp.storage_gb = $4)`: a NULL storage drops the storage check (the cause of #1105). No status filter. The signature is pinned by the contract guard. | recover `:137-179`, `:283-292` | +| `search_master_products`, `get_category_brand_facets`, `search_catalog_autocomplete` | variant | Return one row per storage, and facet counts are inflated by the number of sizes. | `20261019060000`, `20261018120104:30-53`, `20261018120102:94-187` | +| `refresh_master_market_stats`, `refresh_master_product_rating` | variant | Aggregate over `products WHERE cluster_id = master`. | `20261018120103`, DOS-Me `20261024060000` | +| `submit_community_review` | variant to listing | Takes a master id, then writes the review onto that master's cheapest listing. | `20261018100000:45-271` | +| `upsert_price_alert` | variant | Resolves the master by slug. | `20261018120000:60-188` | +| `listing_price_context` (DOS-Me) | variant | Peer price floors are per `cluster_id`. | DOS-Me `20261018160000` | + +**Notes** + +- The contract guard in `20261005130000` compares the live columns to its 28-column list with a FULL OUTER JOIN (`:216-235`). Any extra column therefore counts as drift. It passes only because it runs at its own position in an ordered replay. Re-running that migration on a database that already has DOS-Me's `spec_provenance` would fail today. So the guard is not what limits this design. The real limits are the frozen `match_master_product` signature (`:283-292`) and the code that reads `master_products`. This design **adds** columns and renames nothing. +- `rate.v_products_with_merchant_trust` (`20261014033000:17-61`, re-created in DOS-Me `20261018153000:320-365`) is the listing view the web reads offers from (`rate-api.ts:1292,1358`). It has a fixed column list. A new `item_id` column must be appended at the end, so that `CREATE OR REPLACE VIEW` still works. +- No trigger maintains `min_price` or `merchant_count`. Callers (importer, worker, auditors) invoke the refresh RPCs explicitly. + +### 1.2 Importer (`apps/importer`) + +- **Key shape.** `ProductNormalizer.normalize()` builds `model_key = slugify(brand):slugify(model + appleSize):{N}gb|unknown` in one place (`product-normalizer.ts:1685`). Name and slug include the storage only when it is known (`:1694-1700`). RAM, condition, region and colour never enter the key. +- **Registry.** `CANONICAL_PHONE_REGISTRY` (`:227`) is already keyed `brand:model` with no storage. **That is the Item key.** +- **Apple sizes.** Since #1113 and #1124, Apple screen and case sizes are folded into the model slug, for example `ipad-pro-m5-11` vs `ipad-pro-m5-13`. They are therefore already separate Items. +- **`:unknown` is overloaded.** It means "storage not stated" (`needs_review: true`) for phones. For earbuds it means "storage not applicable" (`needs_review: false`, `:1623-1626`). +- **`EntityMatcher.resolveOrCreateMasterProduct`** (`entity-matcher.ts:148-406`): + - Tier 1: exact `model_key`. + - Tier 2: `match_master_product` with `p_storage_gb` (NULL when unknown). Accepts at 0.70. + - Tier 3: AI enrich, then insert a master, `draft` when `needs_review`. + - `enrichMasterProductById` can back-fill `storage_gb` on an `:unknown` master without changing its key (`:530`). +- **Pipeline** (`shopee-to-supabase.ts:401-985`): + - Upserts the listing with `cluster_id = masterId`. + - Then calls `refresh_master_market_stats`. + - Price history is per listing. + - **Existing defect:** the listing's `price_min`/`price_max` span every marketplace variation (all storages), yet they are attributed to one sized master (`:546-548`). They feed that master's `min_price`/`max_price` through `refresh_master_market_stats`. + - `masterIdHint` (`:488-519`) lets Rate-Services pin a listing to a master. +- **Seeds** (`flagship-seed-data.ts`) list one sized key per device. Several seed keys never match what the normalizer produces: + - earbuds seeded as `:64gb` or `:standard`, while the normalizer gives `:unknown`; + - `apple:watch-ultra-3-49mm:64gb`, while the normalizer gives `apple:watch-ultra-3:64gb`. + + Tier 1 never hits these, which can create duplicates. +- **Spec enrichment** is per variant: the cache key includes storage and RAM (`ai-spec-enricher.ts:812`), although almost every spec is Item-level. +- **Rating refresh** is per master (`ai-review-analyzer.ts:2404`). Peer price floors group by `cluster_id` (`merchant-review-risk.ts:474-489`, `cluster-peer-floor.ts`). +- **Tests pinning the key format:** + - `open-core-intelligence.test.ts` (~131 key assertions) + - `entity-matcher.test.ts` + - `shopee-to-supabase.test.ts` + - `arena-audit.test.ts` + - `dist-exports.smoke.mjs` (the Rate-Services contract) + +### 1.3 Web, API v1, extension (`apps/web`, `apps/extension`) + +- **One master is one page.** + - Product pages: `/{category}/{slug}` (`app/[locale]/smartphones/[slug]/page.tsx`; other categories wrap it). + - Canonical: the variant slug. + - JSON-LD: a single `Product` with `sku = model_key` and its own AggregateOffer and AggregateRating. There is no `ProductGroup`. + - Variants appear only as text in `variant_name` and `name`. There is no sibling list and no picker. +- **Sibling leaks.** + - `rankRelatedCompetitors` (`lib/product-competitors.ts:63-93`) excludes only the same slug, so other sizes of the same model show as "direct competitors". + - The same happens in versus links, leaderboard ranks, popular battles and the category "compare top 2". +- **Directory, search, facets, leaderboard, sitemap and home stats all count variants.** + - `search_master_products`, `get_category_brand_facets`, `getSitemapRows`, `masterProductsCount`. + - One model can hold several top-10 leaderboard slots. +- **Persisted state keyed by variant slug:** + - price alerts (RPC `p_product_slug`); + - watchlist and compare-tray localStorage (`rate_price_watchlist_v1`, `rate_compare_tray_v1`); + - cache tags `rate:product:`; + - public API v1 (`/products/{slug}`, `/compare?slugs=`, `/versus?pair=`, `/watchlist`, `/reviews?slug=`); + - extension response URLs. +- **Redirects.** None exist today apart from www to apex and the locale prefix (`proxy.ts:63-80`, `next.config.mjs:146-155`). +- **The extension is already variant-aware.** + - `lookupExtensionProduct` (`server/rate/extension.ts:960`) groups sibling masters by brand + model among the top 8 search hits. + - It defaults an unknown storage to 256GB (`:1212`). + - The side panel renders a variant picker (`sidepanel.ts:343-356`). + - **Defect (separate):** the extension's `/watchlist?add=` link has no handler on the web. +- **E2E.** `apps/web/e2e/seed.sql` has two single-size masters. About 40 test URLs pin `iphone-16-pro-max-256gb`. + +### 1.4 DOS/Rate-Services (read-only; general terms only) + +- **How it uses this repo.** It consumes `@repo/crawlers` and `@repo/shared-data` through `vendor/rate`: + - the ingestion pipeline (with `masterIdHint`); + - `ProductNormalizer` (to read `model_key`, `storage_gb`); + - `AiReviewAnalyzer`; + - the arena spec syncer; + - shared rules. + + It does not call `EntityMatcher` directly. +- **Key parsing.** Its discovery and listing-verification code parses `model_key` as three `:`-separated segments: storage-strict matching, "stated storage" rewriting, and the "other variant" mismatch classification (lineage of DOS/Rate-Services#616 and #618). Title rewriting for some marketplaces exists only to steer a listing onto the right per-storage master. +- **Seeding is per master, which means per storage.** This covers coverage gaps, search queries built from the sized name, the market floor and refresh rotation. Job payloads carry `master_id`/`cluster_id`/`target_model_key`. +- **Every auditor groups by `cluster_id`:** price spread (which also writes master `min_price`/`max_price`/`merchant_count`/`key_specs`/`status`), price anomaly, merchant risk, review burst, catalog provenance and wrong-listing cleanup. The price alert evaluator uses `master_product_id`. +- **Its test harness pins about 184 key literals.** These include `:standard`, `:1024gb` and legacy `khc:` keys, so the third segment is not always storage. + +## 2. Requirements derived from the survey + +1. **One page, card, leaderboard slot and sitemap entry per Item.** Siblings must never compete with each other. +2. **Price comparisons stay within one variant:** min price, spread, deal radar, peer floors, bait detection, alerts. A 128GB price must never set the floor for 1TB. +3. **A listing whose variant is unknown is kept and shown, but never feeds a per-variant price.** +4. **Old variant slugs, API calls, alerts, watchlists and compare trays keep working.** +5. **No fuzzy assignment of storage.** A variant is assigned only from explicit evidence: title, marketplace SKU, or a single-size registry line. +6. **Additive, idempotent, reversible DOS-Me migrations.** Do not break the `master_products` contract guard. + +## 3. Options + +| | **A. Items + new variants table** | **B. Items above `master_products` (recommended)** | **C. Self-referencing `master_products` (`parent_id`, `kind`)** | **D. Repurpose `master_products` as Item, new variants table** | +| :--- | :--- | :--- | :--- | :--- | +| New tables | `items`, `item_variants` | `items` | none | `item_variants` | +| What `master_products` becomes | legacy, deprecated | the variant table (unchanged identity) | both levels in one table | the Item table | +| `cluster_id` meaning | must move to `variant_id` | unchanged (variant); NULL = unknown | unchanged | changes to Item; every per-cluster price computation would mix storages | +| Per-variant price logic (DB, importer, Rate-Services auditors) | rewrite | **unchanged** | unchanged | rewrite everywhere | +| Old slugs and ids | need an alias map | variant rows keep their slug and id | keep | ids change meaning silently (dangerous) | +| Contract guard / frozen RPC signature | untouched (but table goes stale) | untouched | needs new columns with mixed semantics; queries must filter `kind` everywhere | violated | +| Clarity of the end state | best | good (the table name is legacy; a view alias can fix it later) | poor | good, but unsafe migration | +| Cost | highest | **lowest** | medium, with lasting complexity | high, high risk | + +**Recommendation: B.** It gets JOY's model (one Item, sizes as variants, many listings) at the lowest risk, because the riskiest logic in the system (per-variant price comparison across three codebases) does not change. Option A is cleaner on paper. But it rewrites every per-cluster query in this repo and in Rate-Services for a naming gain. That gain can still be had later with a `rate.item_variants` view over `master_products` (section 7, M6). + +## 4. Proposed schema (Option B) + +### 4.1 Levels and what lives where + +| Data | Item | Variant (`master_products`) | Listing (`products`) | +| :--- | :--- | :--- | :--- | +| Identity | `item_key` = `brand:model` (Apple screen or case size stays in `model`), `slug`, `name`, `brand`, `model`, `category_id`, `status` | `variant_key` (the literal key segment, e.g. `256gb`, `1024gb`, `standard`, `unknown`), `model_key` = `item_key:variant_key` (unchanged), own `slug` (old URL), `variant_name` label, `sort_order` | `item_id` (required), `cluster_id` = variant (NULL = unknown), `platform`, `platform_item_id`, `url` | +| Specs | `specifications`, `key_specs`, `description`, `images`, `thumbnail`, `spec_provenance`, expert review | overrides only: `storage_gb`, `ram_gb`, `key_specs.storage`, launch price | raw `models`, `attributes`, `title` | +| Prices | `min_price` = min over published variants; `max_price`; `price_variant_count` | `min_price`, `max_price`, `merchant_count` (unchanged logic; known-variant listings only) | `price_current`, `price_history` (unchanged) | +| Ratings and reviews | `user_rating_weighted`, `authentic_review_count`, `average_rating`, `total_reviews`, computed over **all** listings of the Item, including unknown-variant ones | during the compatibility window the columns mirror the Item value; no separate per-size rating | review rows stay on listings; `reviews.variation` text stays | +| Counts | `listing_count`, `unknown_variant_listing_count`, `merchant_count` (distinct over all listings) | `merchant_count` | | +| Alerts | (optional later: "any size" alert) | `price_alerts.master_product_id` unchanged | | +| Deals, spread, peer floors, bait detection | none (rolled up for display only) | per variant, as today | input | + +- **Ratings live on the Item.** Reviews of a 256GB and a 512GB phone describe the same device, and splitting them halves the evidence behind every Bayesian score. A per-size rating has no product meaning for storage. +- **Prices live on the variant.** The Item only shows a "from" price and a range. + +### 4.2 Variant axis + +- **v1: storage only.** `variant_key` is the existing third key segment, **stored literally**: `256gb`, `1024gb`, `standard`, `unknown`. This keeps `model_key = item_key:variant_key` true for every existing row and for what the normalizer emits today, so Tier 1 matching and Rate-Services fixtures keep working. Display labels (`1TB`, "Standard") are derived; they are never keys. +- **When `unknown` is a real variant and when it is not.** + - **Single-variant Item:** the Item has no sized variant, for example earbuds (the normalizer keys every earbud as `:unknown` on purpose, `product-normalizer.ts:1623-1626`), Garmin watches, or a line not yet seen with a size. Here the `unknown` (or `standard`) master **is** the Item's only variant. Its listings keep `cluster_id`, and it has a normal price. + - **Multi-size Item:** the Item has at least one **non-archived** sized variant (`variant_key ~ '^\d+(gb|tb)$'`), and its category has a storage axis (never earbuds). Here `unknown` means "size not stated". The listing gets `cluster_id = NULL`, and the `:unknown` master is retired (M3 and the importer rule in 4.5). +- **Single-size registry lines** (`only_storage_gb`) get that size, as today. +- **Apple screen and case sizes stay part of the Item.** This keeps the #1113/#1124 behaviour: iPad Pro 11 and iPad Pro 13 are different Items. They have different specs, battery and screen, not just a different SKU. +- **Colour stays a listing attribute,** not a variant. +- **Connectivity (Wi-Fi vs Cellular for tablets and watches) is an open question (Q3).** The key format reserves room: `256gb-cellular` is a valid `variant_key`, so adding the axis later is data, not schema. +- **RAM** that comes paired with storage (e.g. 12/256 vs 12/512) goes in the variant's `ram_gb`. RAM-only differences at the same storage are rare in the catalog, so they would also be suffixed (`256gb-16gbram`) only if they occur. + +### 4.3 Draft DDL (for review only; the target is a DOS-Me migration) + +```sql +-- M1: additive +CREATE TABLE IF NOT EXISTS rate.items ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + item_key text NOT NULL, + slug text NOT NULL, + name text NOT NULL, + brand text NOT NULL, + model text NOT NULL, + category_id uuid REFERENCES rate.categories(id) ON DELETE SET NULL, + status text NOT NULL DEFAULT 'draft' + CHECK (status IN ('draft','published','archived')), + entity_type text NOT NULL DEFAULT 'product', + description text, + specifications jsonb NOT NULL DEFAULT '{}'::jsonb, + key_specs jsonb NOT NULL DEFAULT '{}'::jsonb, + spec_provenance jsonb NOT NULL DEFAULT '{}'::jsonb, + images text[] NOT NULL DEFAULT '{}', + thumbnail text, + default_variant_id uuid, -- FK added below; nullable, so no insert-order cycle + min_price numeric(15,2) NOT NULL DEFAULT 0, + max_price numeric(15,2) NOT NULL DEFAULT 0, + price_variant_count integer NOT NULL DEFAULT 0, + merchant_count integer NOT NULL DEFAULT 0, + listing_count integer NOT NULL DEFAULT 0, + unknown_variant_listing_count integer NOT NULL DEFAULT 0, + total_reviews integer NOT NULL DEFAULT 0, + average_rating numeric(3,2) NOT NULL DEFAULT 0, + user_rating_weighted numeric(3,2), + authentic_review_count integer NOT NULL DEFAULT 0, + expert_rating numeric(3,2), + expert_review_summary text, + expert_criteria_scores jsonb NOT NULL DEFAULT '{}'::jsonb, + metadata jsonb NOT NULL DEFAULT '{}'::jsonb, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT items_item_key_key UNIQUE (item_key), + CONSTRAINT items_slug_key UNIQUE (slug) +); +-- No shape CHECK on item_key: legacy keys (e.g. `khc:`) must still get an Item (M2) and are +-- reported by the M0 inventory instead. + +ALTER TABLE rate.master_products ADD COLUMN IF NOT EXISTS item_id uuid; +ALTER TABLE rate.master_products ADD COLUMN IF NOT EXISTS variant_key text; +ALTER TABLE rate.master_products ADD COLUMN IF NOT EXISTS sort_order smallint; +ALTER TABLE rate.products ADD COLUMN IF NOT EXISTS item_id uuid; +ALTER TABLE rate.products ADD COLUMN IF NOT EXISTS variant_source text; + +DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conrelid = 'rate.products'::regclass AND conname = 'products_variant_source_check') THEN + ALTER TABLE rate.products ADD CONSTRAINT products_variant_source_check CHECK + (variant_source IS NULL OR variant_source IN ('title','sku','single_size','single_variant','manual','legacy')); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conrelid = 'rate.master_products'::regclass AND conname = 'fk_master_products_item') THEN + ALTER TABLE rate.master_products ADD CONSTRAINT fk_master_products_item + FOREIGN KEY (item_id) REFERENCES rate.items(id) ON DELETE RESTRICT; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conrelid = 'rate.products'::regclass AND conname = 'fk_products_item') THEN + ALTER TABLE rate.products ADD CONSTRAINT fk_products_item + FOREIGN KEY (item_id) REFERENCES rate.items(id) ON DELETE RESTRICT; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conrelid = 'rate.items'::regclass AND conname = 'fk_items_default_variant') THEN + ALTER TABLE rate.items ADD CONSTRAINT fk_items_default_variant + FOREIGN KEY (default_variant_id) REFERENCES rate.master_products(id) ON DELETE SET NULL; + END IF; +END $$; + +CREATE UNIQUE INDEX IF NOT EXISTS uq_master_products_item_variant + ON rate.master_products (item_id, variant_key) WHERE item_id IS NOT NULL AND variant_key IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_rate_products_item ON rate.products (item_id); +CREATE INDEX IF NOT EXISTS idx_items_status_cat_price ON rate.items (status, category_id, min_price); +CREATE INDEX IF NOT EXISTS idx_items_status_rating ON rate.items (status, user_rating_weighted DESC); + +-- pg_trgm lives in schema `sign` on prod and `extensions` elsewhere (DOS-Me 20261018183000:93-103), +-- so the trigram index is built with the schema looked up at run time, as init :41-57 does. +DO $$ DECLARE v_trgm text; BEGIN + SELECT n.nspname INTO v_trgm FROM pg_extension e JOIN pg_namespace n ON n.oid = e.extnamespace + WHERE e.extname = 'pg_trgm'; + EXECUTE format('CREATE INDEX IF NOT EXISTS idx_items_search_trgm ON rate.items USING gin ' + '((brand || '' '' || name || '' '' || model || '' '' || item_key) %I.gin_trgm_ops)', v_trgm); +END $$; + +ALTER TABLE rate.items ENABLE ROW LEVEL SECURITY; +DROP POLICY IF EXISTS "Public can view published items" ON rate.items; +CREATE POLICY "Public can view published items" ON rate.items FOR SELECT USING (status = 'published'); +DROP POLICY IF EXISTS "Service role write items" ON rate.items; +CREATE POLICY "Service role write items" ON rate.items FOR ALL TO service_role USING (true) WITH CHECK (true); +GRANT SELECT ON rate.items TO anon, authenticated; +GRANT ALL ON rate.items TO service_role; +-- plus: rate_worker SELECT policy and grant on rate.items (pattern of DOS-Me 20261018183000; writes go +-- through the SECURITY DEFINER sync triggers and Item RPCs, EXECUTE granted to rate_worker), the updated_at +-- trigger (pattern of 20261018120105), and `item_id` appended at the END of +-- v_products_with_merchant_trust so CREATE OR REPLACE VIEW keeps working. +``` + +**M1 and M2 ship as one DOS-Me migration** (schema, sync triggers and backfill in one transaction), so there is no gap in which the triggers create Items that the backfill then skips. Sections 4.3 and 7.1 stay separate here only for readability. + +**Compatibility sync (M1, removed in M4).** The old importer and Rate-Services keep writing `cluster_id` without `item_id` until their code changes ship. A listing can also move between masters after the backfill. So M1 installs two BEFORE triggers. This makes the order between the migration and the code deploys irrelevant: + +```sql +-- products: item follows the variant whenever a variant is set +CREATE OR REPLACE FUNCTION rate.products_sync_item() RETURNS trigger +LANGUAGE plpgsql SECURITY DEFINER SET search_path = '' AS $$ +BEGIN + IF NEW.cluster_id IS NOT NULL THEN + SELECT mp.item_id INTO NEW.item_id FROM rate.master_products mp WHERE mp.id = NEW.cluster_id; + END IF; -- cluster_id NULL: keep the item_id the writer supplied (unknown variant) + RETURN NEW; +END $$; +-- master_products: a master inserted without an Item gets one from its key (created as draft) +-- SECURITY DEFINER: writers such as rate_worker are subject to RLS and hold no grant on rate.items; +-- the function only derives an Item from the row being written, so it exposes nothing else. +CREATE OR REPLACE FUNCTION rate.master_products_sync_item() RETURNS trigger +LANGUAGE plpgsql SECURITY DEFINER SET search_path = '' AS $$ +DECLARE v_key text := split_part(NEW.model_key, ':', 1) || ':' || split_part(NEW.model_key, ':', 2); +BEGIN + IF TG_OP = 'INSERT' OR NEW.model_key IS DISTINCT FROM OLD.model_key THEN + NEW.variant_key := NULLIF(split_part(NEW.model_key, ':', 3), ''); -- re-derived on a re-key + INSERT INTO rate.items (item_key, slug, name, brand, model, category_id, status) + VALUES (v_key, rate.item_slug_from_master(NEW.slug, NEW.model_key), + rate.item_name_from_master(NEW.name), NEW.brand, NEW.model, NEW.category_id, + CASE WHEN NEW.status = 'published' THEN 'published' ELSE 'draft' END) + ON CONFLICT (item_key) DO NOTHING; + SELECT i.id INTO NEW.item_id FROM rate.items i WHERE i.item_key = v_key; + END IF; + IF NEW.status = 'published' THEN -- publishing a variant publishes its Item + UPDATE rate.items SET status = 'published' WHERE id = NEW.item_id AND status <> 'published'; + END IF; + RETURN NEW; +END $$; +-- An AFTER UPDATE OF model_key, item_id trigger moves the master's listings to the new item_id in +-- the same statement (UPDATE rate.products SET item_id = NEW.item_id WHERE cluster_id = NEW.id). +-- CREATE TRIGGER trg_products_sync_item BEFORE INSERT OR UPDATE OF cluster_id ON rate.products ... +-- CREATE TRIGGER trg_master_products_sync_item BEFORE INSERT OR UPDATE OF model_key ON rate.master_products ... +-- (each wrapped in DROP TRIGGER IF EXISTS; slug collision in the INSERT falls back to a suffixed slug) +``` + +```sql +-- M4: constraints, after the backfill and reconciliation are verified (every step re-runnable) +DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM rate.master_products WHERE item_id IS NULL) THEN + ALTER TABLE rate.master_products ALTER COLUMN item_id SET NOT NULL; + ELSE + RAISE EXCEPTION 'M4: master_products rows without item_id remain'; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conrelid = 'rate.products'::regclass AND conname = 'products_item_id_not_null') THEN + ALTER TABLE rate.products ADD CONSTRAINT products_item_id_not_null CHECK (item_id IS NOT NULL) NOT VALID; + END IF; +END $$; +ALTER TABLE rate.products VALIDATE CONSTRAINT products_item_id_not_null; -- no-op when already valid + +-- A listing's variant must belong to the listing's Item. +CREATE OR REPLACE FUNCTION rate.products_variant_item_guard() RETURNS trigger +LANGUAGE plpgsql SET search_path = '' AS $$ +BEGIN + IF NEW.cluster_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM rate.master_products mp + WHERE mp.id = NEW.cluster_id AND mp.item_id = NEW.item_id) THEN + RAISE EXCEPTION 'listing % variant % is not a variant of item %', NEW.id, NEW.cluster_id, NEW.item_id + USING ERRCODE = '23514'; + END IF; + RETURN NEW; +END $$; +DROP TRIGGER IF EXISTS trg_products_variant_item_guard ON rate.products; +CREATE TRIGGER trg_products_variant_item_guard + BEFORE INSERT OR UPDATE OF cluster_id, item_id ON rate.products + FOR EACH ROW EXECUTE FUNCTION rate.products_variant_item_guard(); +-- Companion trigger on master_products: changing a variant's item_id is rejected while it still has +-- listings on the old Item (moves are done by an RPC that moves the listings in the same transaction). +``` + +### 4.4 Listing with an unknown variant + +This applies only to multi-size Items (section 4.2). On a single-variant Item, every listing belongs to its one variant. + +- **Stored as:** `products.item_id = `, `products.cluster_id = NULL`, `variant_source = NULL`. +- **Never feeds a variant.** It is excluded from variant `min_price`/`max_price`/`merchant_count`, spread, deal radar and peer floors. These already filter by `cluster_id`, so the exclusion is automatic. Every consumer that groups by `cluster_id` must still treat NULL as "no variant" rather than as one group (section 5). +- **Counted at Item level.** It feeds Item `listing_count`, `unknown_variant_listing_count`, Item `merchant_count` and Item ratings, because its reviews are about the device. +- **Shown** on the Item page in a separate group, "Storage not stated" (VI: "Chưa rõ dung lượng"). Each row gets a note and its own price. It is not used for the "from" price (Q2). +- **Kept fresh.** Refresh jobs for these listings are keyed by listing and Item, not by variant, so they keep being re-crawled (section 5.4). +- **Upgraded later, never guessed.** It moves to a variant when a later crawl gives explicit evidence: a stated title, a selected marketplace SKU, or a single-size line. `variant_source` records which. The AI enricher must not set storage on a listing. +- **Multi-size marketplace listings.** A Shopee listing whose SKUs span several storages is one listing with one `price_current`/`price_min`/`price_max`. + - Phase 1 attaches it to the variant of its cheapest SKU only when that SKU names its storage, and otherwise to the Item with unknown variant. + - This fixes the variant's **min** price. But `price_max` still spans every SKU (`shopee-to-supabase.ts:546-548`) and still feeds the variant's `max_price`. Phase 1 should therefore write the SKU's own price as `price_max` for such a listing, or the variant max must ignore multi-SKU listings. + - The full fix is per-SKU offers, the optional phase M5 (`rate.listing_skus`). + +### 4.5 Keys and matching + +- **Normalizer output**: + - `item_key` = `slugify(brand):slugify(model + appleSize)`; + - `variant_key` = the third key segment in today's format (`{N}gb`, `unknown`; earbuds stay `unknown`); + - `item_name` and `item_slug` (no storage); + - `variant_name`; + - a new `variant_known` flag. + + `model_key` keeps exactly today's shape, so Rate-Services helpers and fixtures keep working. +- **Shared helpers.** `packages/shared-data` gets `composeModelKey` and `splitModelKey`, so Rate-Services stops parsing key strings itself. +- **Matcher, item first:** + 1. Item by exact `item_key`. + 2. If the size is known: the variant by exact `(item_id, variant_key)`. A missing sized variant of a known published Item is created as a variant row. Specs are inherited from the Item; only storage and RAM are overridden. The Item stays published. A draft is needed only when the Item is new. + 3. If the size is unknown: on a single-variant Item, its one variant; on a multi-size Item, no variant (`cluster_id = NULL`). + 4. **Retiring `unknown`.** When an Item gets its first sized variant, its `unknown` master is retired by the same rule as M3: listings get `cluster_id = NULL`, and the master is archived. This is an RPC run in one transaction. + 5. Tier 2 fuzzy runs against **Item** names through a new RPC, `match_item(search_title, p_brand, p_model, p_threshold)`. Item names never contain storage, so the #1105 failure mode (bare title vs sized name) disappears. The RPC calls `similarity()` schema-qualified through the run-time pg_trgm lookup, as `match_master_product` does, and is granted to `rate_worker`. The frozen `match_master_product` stays for the compatibility window. + 6. Tier 3 creates a draft Item, plus a variant only when the key states one. +- **Pipeline hints:** + - `masterIdHint` keeps working (a variant id implies its Item). It must **reject archived masters**: today the hint is only checked for existence (`shopee-to-supabase.ts:488-519`), so a queued job that still carries a retired `:unknown` master id would put the listing back on it. When the hinted master is archived, the hint is mapped to its Item with no variant. + - A new `itemIdHint` is added for storage-less pages. +- **Ratings.** `refresh_master_product_rating` keeps its signature. Its body is replaced so that it resolves the master's Item, computes the rating over every listing of that Item (including unknown-variant listings), writes it on the Item, and mirrors it onto each variant row for old readers. `refresh_review_vote_counts` today resolves the master through `cluster_id` and skips NULL (`20261018120103:170`). It switches to `products.item_id`, so votes on unknown-variant listings also count. + +## 5. Impact + +### 5.1 `apps/web` + +| Area | Change | +| :--- | :--- | +| Data layer (`lib/rate-api.ts`, `server/rate/*`) | New `getItemBySlug`, `getItemVariants`, `getItemOffers` (grouped by variant, plus the unknown group), and `resolveCatalogSlug(slug)` returning `{item, variant}` from either an item slug or an old variant slug. Price history: one series per variant. Reviews and summary: across the Item's listings. | +| Product page | One page per Item: `/{category}/{item-slug}`. Variant selector with a price per variant, preselected by `?v=256gb` or by the old slug. Title and meta use "from X". JSON-LD becomes `ProductGroup` (`variesBy: storage`, `hasVariant` Products with their own sku and AggregateOffer) plus one Item-level AggregateRating. Competitors, leaderboard standing and versus suggestions exclude same-Item siblings. The page also checks that the route category matches the Item (fixes the `/tablets/` canonical leak). | +| URLs and redirects | An old variant slug gets a 308 `permanentRedirect` to `/{category}/{item-slug}?v={variant_key}`. Canonical is the Item URL (Q1). No alias table is needed while variants keep their `slug`. Slug-collision rule: an Item slug may equal only one of its own variants' slugs (for example single-size earbuds), checked by a verification query and by the importer. | +| Directory, search, facets, autocomplete | One card per Item with a "from" price and size chips. The `?storage=` filter means "has a variant of that size". Brand facet and pagination counts become Items. New RPCs: `search_items_v1`, `get_category_brand_facets_v2`, `search_catalog_autocomplete_v2`. | +| Compare and versus | Compare by Item, optionally pinning a variant (`slug@256gb`). Versus URLs become `{itemA}-vs-{itemB}`, and old variant pairs 308 to the Item pair. Same-Item pairs are rejected. Tray localStorage entries migrate on read (resolve the slug, keep the variant). | +| Deals | Deal computation stays per variant. Cards link to the Item with the variant preselected. | +| Leaderboard, stats, sitemap | Rank and count Items. The sitemap emits Item URLs only, with lastmod = max over variants and listings. "Products tracked" counts Items. | +| Watchlist and price alerts | Alerts stay per variant (Q5). Old slugs resolve. The UI asks for a size when the watchlist entry is an Item. | +| Cache | Revalidating a variant slug also busts `rate:product:`. | + +### 5.2 API v1 and extension + +- **Additive only within v1.** + - `GET /products/{slug}` accepts an Item or variant slug. It returns the current `product` (the selected or default variant, so existing clients keep working) plus new `item`, `variants[]`, `selected_variant` and `unknown_variant_offers`. + - `GET /products` gains `group=item|variant`. It defaults to `variant` for the compatibility window. Switching the default is a documented breaking change and needs an announcement (Q7). + - OpenAPI gains Item and Variant component schemas. The doc text also gets corrected: it claims `/products/{slug}` accepts a `model_key`, which it does not. +- **Extension.** + - `lookupExtensionProduct` resolves the Item and returns **all** its variants from the DB, not only the siblings among the top 8 search hits. + - Exact listing lookup by URL or `platform_item_id` today requires `cluster_id NOT NULL` (`extension.ts:1101-1103`, and the same in `inspectProductOrShopUrl`, `rate-api.ts:3706-3728`). It gains an `item_id` fallback, so unknown-variant listings are still recognised. + - It drops the "unknown storage means 256GB" default; unknown shows the Item with no size preselected. + - It returns Item URLs. + - The side-panel picker already exists, so the client change is small. + - The dead `/watchlist?add=` link gets its handler in the same task. + +### 5.3 `apps/importer` + +| File | Change | +| :--- | :--- | +| `product-normalizer.ts` | Emit `item_key`, `variant_key` (the literal third segment, format unchanged), `item_name`, `item_slug`, and a `variant_known` flag, so callers stop inferring it from `:unknown`. `needs_review` stops meaning "storage unknown". | +| `entity-matcher.ts` | Item-first tiers (section 4.5). Variant creation under a published Item. Fuzzy only at Item level. Return `{itemId, variantId \| null}`. **Closes #1105.** | +| `shopee-to-supabase.ts` | Write `products.item_id`, nullable `cluster_id`, `variant_source`. Pick the variant from the selected or cheapest SKU only when it states storage. Refresh variant stats, then Item stats. `itemIdHint`. | +| `ai-spec-enricher.ts` | Cache and enrich per Item. Variant overrides only. Never assign storage to a listing. | +| `ai-review-analyzer.ts`, `merchant-review-risk.ts`, `cluster-peer-floor.ts` | Rating refresh at Item level. Peer floors unchanged (per variant), and they must skip NULL `cluster_id`. | +| `flagship-seed-data.ts`, `arena-spec-syncer.ts`, `jobs/backfill-specs.ts` | Seeds become one Item with a list of variants. Fix the seed key drift (earbuds `:64gb`/`:standard`, `watch-ultra-3-49mm`). Specs backfill per Item. | +| `package.json` exports, `dist-exports.smoke.mjs` | Export the new helpers and types. Keep the old exports for one vendor bump. | +| Tests | Key assertions gain `item_key`/`variant_key`. Matcher tests cover the bare title linking to an Item with no variant (the #1105 case). | + +### 5.4 DOS/Rate-Services (general terms; its own issues in its own repo) + +- **Grouping by `cluster_id` stays correct,** because it remains the variant. Every auditor that groups by it must treat NULL as "no variant": skip it for floors and spread, and count it at Item level if needed. This must ship **before** any writer starts producing NULL `cluster_id` (task 13 before M3 and before the importer flag in task 5). +- **Key-parsing helpers** should switch to the shared `splitModelKey`/`composeModelKey`, so they stop splitting strings themselves. Their about 184 fixture literals stay valid during the compatibility window, because `model_key` keeps its shape. +- **Listing verification** gains an Item-level mode. A page with no stated storage is accepted for the Item with no variant, instead of being rejected as "other variant". +- **Job payloads** carry `item_id` plus an optional variant id. Queued jobs with old variant ids stay valid, because variant ids do not change. The one exception is a retired `:unknown` master id (M3, or the importer rule in 4.5): the pipeline maps a hint to an archived master onto its Item with no variant. +- **Refresh rotation** is per master today. Listings with no variant must be refreshed by listing and Item, or their price and availability go stale. +- **Seeding and coverage** can run per Item, with per-variant coverage computed after the search. +- **Writers of master aggregates** (the spread auditor, cleanup scripts) also call the new Item refresh RPC. +- **Bump the `vendor/rate` pin** after the importer change lands. The local submodule checkout is currently ahead of the pinned commit, which is worth syncing first. + +## 6. #1105 decision + +With this model, a storage-less title resolves to the Item with no variant. #1105 option (a) (fall through to a draft `:unknown` master) would be throwaway work. During the interim, it would also hide those listings from public pages, because the `:unknown` master is a draft. + +**Recommendation:** close #1105 into the matcher task of this design (task 5). Keep today's behaviour until then. Rate-Services already guards discovery and refreshes against the wrong-size link. + +## 7. DOS-Me migration plan + +**Common rules for every phase** + +- Every step is idempotent: guarded by `IF NOT EXISTS` / `conrelid` + `conname` checks, or written as re-runnable `UPDATE ... WHERE `. +- Each phase is its own DOS-Me migration: preview, then beta, then prod, each with JOY's approval. +- Reference copies go into `DOS/Rate/supabase/migrations/`, and `database.types.ts` is regenerated. +- **Dependency:** the DOS-Me beta workflow is locked (ROADMAP 6.4). Until it reopens, these migrations cannot reach prod by the normal path. + +| Phase | What | Reversible by | +| :--- | :--- | :--- | +| **M0** | Mirror the eight missing DOS-Me `rate` migrations into this repo's reference copy and regenerate types (a docs/types PR, no DDL). Run the read-only inventory queries (7.3). If duplicates exist, fix seed-key drift as a separate, reviewed data step: earbuds seeded `:64gb`/`:standard` vs normalized `:unknown`, and `watch-ultra-3-49mm`. Duplicates are **archived, never deleted**: `price_alerts` cascade on delete, so alerts are moved to the surviving master first. | not applicable | +| **M1** | Ships in the same migration as M2. Additive DDL (4.3): `rate.items`, new columns, indexes (pg_trgm schema looked up at run time), RLS/grants, `rate_worker` access, updated_at trigger, `item_id` appended to the trust view, the two compatibility sync triggers, and the slug/name helper functions they call. | One transaction: drop the sync triggers, drop the FKs, drop the columns and `rate.items`. No existing data depends on them. | +| **M2** | Backfill (7.1) and new RPCs: `refresh_item_stats`, `match_item`, `resolve_catalog_slug`, `search_items_v1`, facet and autocomplete v2, plus `submit_community_review` accepting an Item. `refresh_master_product_rating` and `refresh_review_vote_counts` get Item-aware bodies with the same signatures (4.5). Other old RPCs are untouched. | One transaction, with writers paused: `UPDATE ... SET item_id = NULL, variant_key = NULL` on both tables, `UPDATE rate.items SET default_variant_id = NULL`, then `DELETE FROM rate.items`. **Never `TRUNCATE rate.items`:** it fails while the FKs exist, and `TRUNCATE ... CASCADE` would wipe `master_products`, `products` and, through them, price history, reviews and alerts. The old rating bodies are restored from the previous migration. | +| **Compatibility window** | Importer and Rate-Services ship Item-aware code. The sync triggers keep old writers consistent meanwhile. The web ships Item pages behind an env flag, then switches, and old slugs redirect. A reconciliation query (7.3) runs after the last writer changes. Lasts at least one promote cycle and until the sitemap has been re-crawled (about 30 days). | Flip the flag off. | +| **M3** | Retire `unknown` masters on **multi-size Items only** (7.2). Single-variant Items (earbuds, Garmin and other sizeless lines) are untouched. A backup table records each change. Runs only after Rate-Services handles NULL `cluster_id` (task 13). | Restore from the backup table (7.2) | +| **M4** | Constraints (4.3): `item_id NOT NULL` on both tables, the listing/variant/Item guard triggers, `default_variant_id` filled. Drop the compatibility sync triggers. | Drop the constraints and guard triggers, and re-create the sync triggers | +| **M5 (optional)** | `rate.listing_skus` (per-SKU price and stock per listing, with nullable `variant_id`). Per-SKU price history. | Drop table | +| **M6 (optional)** | `rate.item_variants` as a view over `master_products`, for naming clarity. Deprecate the per-variant rating mirror columns. | Drop view | + +### 7.1 M2 backfill (draft) + +```sql +-- Variant key = the literal third key segment (format unchanged: 256gb, 1024gb, standard, unknown). +UPDATE rate.master_products mp +SET variant_key = NULLIF(split_part(mp.model_key, ':', 3), '') +WHERE mp.variant_key IS NULL; + +-- One Item per brand:model, including Items whose masters are all archived (status 'archived'), +-- so that M4's NOT NULL can hold. Representative variant: published first, then most merchants, +-- then oldest. Nullable source columns are COALESCEd because the Item columns are NOT NULL. +INSERT INTO rate.items (item_key, slug, name, brand, model, category_id, status, entity_type, + description, specifications, key_specs, spec_provenance, images, thumbnail, + expert_rating, expert_review_summary, expert_criteria_scores, metadata) +SELECT DISTINCT ON (k.item_key) + k.item_key, + rate.item_slug_from_master(mp.slug, mp.model_key), -- strips '-gb|tb' / '-standard' / '-unknown' + rate.item_name_from_master(mp.name), -- strips a trailing ' GB|TB' + mp.brand, mp.model, mp.category_id, + CASE WHEN bool_or(mp.status = 'published') OVER w THEN 'published' + WHEN bool_or(COALESCE(mp.status, 'draft') <> 'archived') OVER w THEN 'draft' + ELSE 'archived' END, + COALESCE(mp.entity_type, 'product'), mp.description, + COALESCE(mp.specifications, '{}'::jsonb), COALESCE(mp.key_specs, '{}'::jsonb), + COALESCE(mp.spec_provenance, '{}'::jsonb), COALESCE(mp.images, '{}'), mp.thumbnail, + mp.expert_rating, mp.expert_review_summary, COALESCE(mp.expert_criteria_scores, '{}'::jsonb), + COALESCE(mp.metadata, '{}'::jsonb) || jsonb_build_object('backfilled_from_master', mp.id) +FROM rate.master_products mp +CROSS JOIN LATERAL (SELECT split_part(mp.model_key, ':', 1) || ':' || split_part(mp.model_key, ':', 2) AS item_key) k +WINDOW w AS (PARTITION BY k.item_key) +ORDER BY k.item_key, (mp.status = 'published') DESC NULLS LAST, + mp.merchant_count DESC NULLS LAST, mp.created_at ASC NULLS LAST +ON CONFLICT (item_key) DO NOTHING; +-- The helpers use single-backslash regexes ('-(\d+(gb|tb)|standard|unknown)$'), which are correct +-- under standard_conforming_strings = on. Slug collisions are found by the M0 inventory and resolved +-- first; the migration aborts if any remain. + +UPDATE rate.master_products mp SET item_id = i.id +FROM rate.items i +WHERE mp.item_id IS NULL + AND i.item_key = split_part(mp.model_key, ':', 1) || ':' || split_part(mp.model_key, ':', 2); + +-- Re-runnable and self-correcting: also fixes listings whose master changed after a previous run. +UPDATE rate.products p SET item_id = mp.item_id, variant_source = COALESCE(p.variant_source, 'legacy') +FROM rate.master_products mp +WHERE p.cluster_id = mp.id AND p.item_id IS DISTINCT FROM mp.item_id; + +-- Items that already existed (created by the sync triggers, or by a re-run): recompute status from +-- their masters, so a published product is never hidden behind a draft Item. +UPDATE rate.items i SET status = s.status +FROM (SELECT item_id, + CASE WHEN bool_or(status = 'published') THEN 'published' + WHEN bool_or(COALESCE(status, 'draft') <> 'archived') THEN 'draft' + ELSE 'archived' END AS status + FROM rate.master_products WHERE item_id IS NOT NULL GROUP BY item_id) s +WHERE i.id = s.item_id AND i.status IS DISTINCT FROM s.status; + +-- Item-level aggregates and ratings +SELECT rate.refresh_item_stats(id) FROM rate.items; +SELECT rate.refresh_master_product_rating(mp.id) -- new body writes the Item and the mirrors +FROM (SELECT DISTINCT ON (item_id) id, item_id FROM rate.master_products ORDER BY item_id, id) mp; +``` + +- **Specs in the backfill.** Item `specifications` are copied from the representative variant. A follow-up importer job (dry-run by default, like `backfill-specs.ts`) strips variant-only fields such as storage from the Item specs and writes them as variant overrides. +- **Listings with no master** (`products.cluster_id IS NULL` today, if any, for example listings never matched) get no Item from this backfill. The M0 inventory counts them. Before M4 each is either matched to an Item by the importer or archived, because M4 requires `item_id`. +- **Legacy keys** (`khc:`, old hyphen-format keys, anything without three segments) still get an Item, so no master is left behind. The inventory lists them for manual review. + +### 7.2 M3 retire `unknown` on multi-size Items (draft) + +```sql +CREATE TABLE IF NOT EXISTS rate.m3_unknown_variant_backup ( + product_id uuid PRIMARY KEY, + old_cluster_id uuid NOT NULL, + old_variant_source text, + master_old_status text, -- nullable: master_products.status is nullable + backed_up_at timestamptz NOT NULL DEFAULT now()); + +-- Abort first if the 7.3 seed-drift query (sized earbuds) still returns rows. +WITH multi AS ( -- Items with at least one LIVE sized variant + SELECT DISTINCT mp.item_id FROM rate.master_products mp + LEFT JOIN rate.categories c ON c.id = mp.category_id + WHERE mp.variant_key ~ '^\d+(gb|tb)$' + AND mp.status IS DISTINCT FROM 'archived' + AND COALESCE(c.slug, '') <> 'earbuds') -- sizeless categories are never multi-size +INSERT INTO rate.m3_unknown_variant_backup (product_id, old_cluster_id, old_variant_source, master_old_status) +SELECT p.id, p.cluster_id, p.variant_source, mp.status +FROM rate.products p +JOIN rate.master_products mp ON mp.id = p.cluster_id +JOIN multi ON multi.item_id = mp.item_id +WHERE mp.variant_key = 'unknown' +ON CONFLICT (product_id) DO NOTHING; + +UPDATE rate.products p SET cluster_id = NULL, variant_source = NULL +FROM rate.m3_unknown_variant_backup b WHERE p.id = b.product_id AND p.cluster_id = b.old_cluster_id; + +-- Archive the retired masters; any with price alerts are reported and resolved first +-- (alerts are per variant and an unknown master has no variant to point to). +UPDATE rate.master_products mp SET status = 'archived' +WHERE mp.id IN (SELECT DISTINCT old_cluster_id FROM rate.m3_unknown_variant_backup) + AND NOT EXISTS (SELECT 1 FROM rate.price_alerts a WHERE a.master_product_id = mp.id); +-- then refresh_master_market_stats for the affected masters and refresh_item_stats for their Items +``` + +**Rollback of M3.** Restore only rows that are still in the state M3 left them in. Report the rest instead of overwriting a variant the importer has assigned since: + +```sql +UPDATE rate.products p SET cluster_id = b.old_cluster_id, variant_source = b.old_variant_source +FROM rate.m3_unknown_variant_backup b +WHERE p.id = b.product_id AND p.cluster_id IS NULL; +SELECT b.product_id, p.cluster_id AS now_on FROM rate.m3_unknown_variant_backup b +JOIN rate.products p ON p.id = b.product_id WHERE p.cluster_id IS NOT NULL + AND p.cluster_id <> b.old_cluster_id; -- conflicts, resolved by hand +UPDATE rate.master_products mp SET status = b.master_old_status +FROM (SELECT DISTINCT old_cluster_id, master_old_status FROM rate.m3_unknown_variant_backup) b +WHERE mp.id = b.old_cluster_id; +-- then refresh the stats +``` + +**Community reviews.** `submit_community_review` attaches reviews to a listing. Moving a listing to "no variant" keeps the review on that listing and on the same Item. This avoids the orphaning seen in the iPhone 16e offer move (DOS-Me `20261022140000`). + +### 7.3 Verification queries (read-only; run after each phase) + +```sql +-- Inventory (M0) +-- Key shape = the shared-data helpers (splitModelKey). `_` is allowed because slugify keeps it (#1149). +SELECT count(*) FILTER (WHERE model_key !~ '^[a-z0-9_][a-z0-9_-]*:[a-z0-9_][a-z0-9_.-]*:[a-z0-9_-]+$') AS bad_shape, + count(*) FILTER (WHERE model_key LIKE 'khc:%') AS khc_keys, + count(*) FILTER (WHERE split_part(model_key, ':', 3) = 'unknown') AS unknown_masters, + count(*) FILTER (WHERE status IS NULL) AS null_status, + count(*) FILTER (WHERE status = 'published') AS published, count(*) AS total +FROM rate.master_products; +SELECT count(*) AS listings_without_master FROM rate.products WHERE cluster_id IS NULL; +SELECT split_part(model_key, ':', 1) || ':' || split_part(model_key, ':', 2) AS item_key, + array_agg(model_key ORDER BY model_key) AS variants, count(DISTINCT brand) AS brands, + count(DISTINCT category_id) AS categories +FROM rate.master_products +GROUP BY 1 HAVING count(DISTINCT brand) > 1 OR count(DISTINCT category_id) > 1; -- expect 0 rows +-- Earbuds or other sizeless categories carrying a fake size (seed drift), to fix in M0 +SELECT mp.model_key FROM rate.master_products mp JOIN rate.categories c ON c.id = mp.category_id +WHERE c.slug = 'earbuds' AND split_part(mp.model_key, ':', 3) NOT IN ('unknown'); +-- Unknown masters on multi-size Items that carry alerts or community reviews (decide before M3) +SELECT mp.model_key, count(DISTINCT a.id) AS alerts, count(DISTINCT r.id) AS reviews +FROM rate.master_products mp +LEFT JOIN rate.price_alerts a ON a.master_product_id = mp.id +LEFT JOIN rate.products p ON p.cluster_id = mp.id +LEFT JOIN rate.reviews r ON r.product_id = p.id +WHERE split_part(mp.model_key, ':', 3) = 'unknown' +GROUP BY 1 HAVING count(DISTINCT a.id) + count(DISTINCT r.id) > 0; + +-- After M2, and again as the reconciliation after the last writer changes +SELECT count(*) FROM rate.master_products WHERE item_id IS NULL; -- 0 +SELECT count(*) FROM rate.products WHERE item_id IS NULL AND cluster_id IS NOT NULL; -- 0 +SELECT count(*) FROM rate.products p JOIN rate.master_products mp ON mp.id = p.cluster_id +WHERE p.item_id IS DISTINCT FROM mp.item_id; -- 0 +SELECT i.item_key FROM rate.items i +WHERE i.min_price IS DISTINCT FROM (SELECT COALESCE(min(mp.min_price) FILTER (WHERE mp.min_price > 0), 0) + FROM rate.master_products mp + WHERE mp.item_id = i.id AND mp.status = 'published'); -- 0 rows +-- An Item slug may only collide with one of its own variants' slugs +SELECT i.slug FROM rate.items i JOIN rate.master_products mp ON mp.slug = i.slug +WHERE mp.item_id IS DISTINCT FROM i.id; -- 0 rows +-- Item names and slugs must not carry a size +SELECT item_key, slug, name FROM rate.items +WHERE slug ~ '-\d+(gb|tb)$' OR name ~* '\s\d+\s?(gb|tb)$'; -- 0 rows +-- Published variant counts per Item, for the page smoke test +SELECT i.slug, count(mp.id) FROM rate.items i JOIN rate.master_products mp ON mp.item_id = i.id +WHERE i.status = 'published' GROUP BY 1 ORDER BY 2 DESC; + +-- After M3 +SELECT count(*) FROM rate.products p JOIN rate.master_products mp ON mp.id = p.cluster_id +WHERE mp.variant_key = 'unknown' + AND EXISTS (SELECT 1 FROM rate.master_products s WHERE s.item_id = mp.item_id + AND s.variant_key ~ '^\d+(gb|tb)$' AND s.status IS DISTINCT FROM 'archived'); -- 0 +SELECT (SELECT count(*) FROM rate.m3_unknown_variant_backup) AS moved, + (SELECT count(*) FROM rate.products WHERE cluster_id IS NULL AND item_id IS NOT NULL) AS unknown_now; +-- Earbuds and other single-variant Items still priced +SELECT i.item_key FROM rate.items i JOIN rate.categories c ON c.id = i.category_id +WHERE c.slug = 'earbuds' AND i.status = 'published' AND i.min_price = 0; -- review any rows +``` + +## 8. Risks + +| Risk | Mitigation | +| :--- | :--- | +| SEO: every indexed product URL changes. | 308 to the Item page with the variant preselected. The canonical points to the Item. The sitemap is swapped in the same deploy. Keep the redirects permanently, since they are cheap. | +| Public API clients break. | Additive fields only within v1. Item grouping is opt-in (`group=item`) until announced. | +| Old writers (importer, Rate-Services) create masters or move listings without `item_id` during the window. | The M1 sync triggers derive `item_id` from `cluster_id` and create draft Items for new keys. The reconciliation query (7.3) runs after the last writer changes. | +| A writer produces NULL `cluster_id` before Rate-Services handles it. | Task 5 writes NULL `cluster_id` only behind a flag that is turned on after task 13 ships. M3 runs after task 13. | +| Rate-Services and this repo drift during the compatibility window. | `model_key` keeps its shape. Variant ids do not change. Helpers are shared through `@repo/shared-data`. The vendor bump happens only after the importer change is merged. | +| Item spec backfill picks the wrong representative. | Published, then most merchants, then oldest. A dry-run diff job is reviewed before applying. | +| Unknown-variant listings make an Item look cheaper. | They are excluded from the "from" price and shown in their own group (Q2). | +| Beta pipeline locked in DOS-Me. | The schema can be prepared and preview-tested now. Prod waits for the beta workflow (ROADMAP 6.4). | +| A listing's variant and Item disagree. | Guard trigger in M4. Verification query after every phase. | + +## 9. Open questions for JOY + +Each question has a recommendation, which is the default if JOY agrees. + +1. **Q1. URL scheme.** **(Recommended)** One indexable URL per Item, `/{category}/{item-slug}`, with the size selected by `?v=256gb` (not indexed). The alternative is a separate indexable URL per size, `/{category}/{item-slug}/256gb`, with more pages but thin and near-duplicate content. +2. **Q2. Listings with an unknown size.** **(Recommended)** Show them on the Item page in a separate "Storage not stated" group, and keep them out of the "from" price. The alternative is to include them in the "from" price, which is cheaper-looking but can mislead. +3. **Q3. Variant axis.** **(Recommended)** Storage only for now. Screen and case sizes (iPad Pro 11 vs 13, Watch 42 vs 46 mm) stay separate Items, as today. Wi-Fi vs Cellular: start as one variant per storage, and add `256gb-cellular` later if price comparisons prove misleading. The alternative is to make connectivity a variant axis from day one. +4. **Q4. Ratings and reviews.** **(Recommended)** One rating per Item, pooled across sizes. +5. **Q5. Price alerts.** **(Recommended)** Alerts stay per size. An "any size" alert is a later feature. +6. **Q6. #1105.** **(Recommended)** Hold, and fold it into the matcher task below. No interim fix. +7. **Q7. Public API default.** **(Recommended)** Keep the variant-level default for `/api/v1/rate/products` during the compatibility window. Switch to Item grouping with an announcement after the window closes. + +## 10. Task breakdown (filed by the PO after JOY approves) + +Sizes: S is up to a day, M is 2-3 days, L is about a week of agent time, including review. + +| # | Task | Repo | Size | Depends on | +| :--- | :--- | :--- | :--- | :--- | +| 1 | M0: mirror the eight DOS-Me `rate` migrations into the reference copy and regenerate `database.types.ts` | Rate | S | | +| 2 | M0: run the inventory queries (read-only) and write the data-fix list (seed key drift, legacy keys, slug collisions) | Rate (PO runs the queries) | S | | +| 3 | `shared-data`: Item and Variant schemas, `composeModelKey`/`splitModelKey`, `item_key` and `variant_key` rules | Rate | S | | +| 4 | DOS-Me M1 + M2: `rate.items`, columns, RLS/grants, backfill, Item RPCs | DOS-Me (with JOY's approval) + Rate reference copy | L | 1, 2 | +| 5 | Normalizer emits `item_key`/`variant_key`; matcher becomes item-first with `match_item` (**closes #1105**); pipeline writes `item_id` with nullable variant and `variant_source`, `itemIdHint`, archived-hint mapping. Writing NULL `cluster_id` sits behind a flag, turned on after task 13 | Rate | L | 3, 4 (flag on: 13) | +| 6 | Seeds as Item plus variants; spec enrichment and backfill per Item; review rating at Item level | Rate | M | 5 | +| 7 | Web data layer: Item queries, `resolveCatalogSlug`, 308 redirects, cache tags | Rate | M | 4 | +| 8 | Product page per Item: variant selector, unknown-size group, `ProductGroup` JSON-LD, canonical, no sibling competitors | Rate | L | 7 | +| 9 | Directory, search, autocomplete, facets, leaderboard, sitemap and stats per Item | Rate | L | 7 | +| 10 | Compare, versus, tray, watchlist and alerts per Item with size pinning; old-URL redirects | Rate | M | 7 | +| 11 | API v1 additive fields and `group=item`, OpenAPI schemas, extension lookup returning all variants and the `?add=` handler | Rate | M | 7 | +| 12 | E2E seed with one multi-size Item; redirect and canonical assertions | Rate | S | 8 | +| 13 | Rate-Services: shared key helpers, NULL-variant handling in auditors, Item-level listing check, payload `item_id`, seeding per Item, vendor bump | Rate-Services | L | 5 | +| 14 | DOS-Me M3 (unknown collapse) and M4 (constraints) | DOS-Me (with JOY's approval) | M | 5, 13, compatibility window | +| 15 | (optional) M5 `listing_skus`, per-SKU prices | DOS-Me + Rate | L | 14 | + +**Critical path:** 1, 2 and 3, then 4, then 5 and 7 in parallel, then 8 to 11 and 13, then the task 5 flag, then 14. The web tasks (8 to 11) can share one feature flag, so that they ship dark and switch together. + +## 11. Review log + +Reviewed on PR #1136 by a fresh-context subagent and by a Codex adversarial pass (no P0). Fixed in this revision: + +- **Earbuds and other sizeless lines would have lost their only variant in M3.** `unknown` is now a real variant on single-variant Items, and M3 is limited to multi-size Items (4.2, 7.2). +- **`variant_key` now keeps today's literal format** (`1024gb`, `standard`, `unknown`), so `model_key` and Tier 1 matching do not change shape (4.2, 4.5). +- **Trigram index:** the pg_trgm schema is looked up at run time, because prod keeps it in `sign` (4.3). +- **M2 backfill now also covers archived masters,** so M4 can enforce `NOT NULL`. It also COALESCEs nullable source columns, sorts with `NULLS LAST`, and copies the expert and provenance columns (7.1). +- **M2 rollback** uses `DELETE`, never `TRUNCATE` (7). +- **M3 rollback** is conditional and restores `variant_source`. The backup allows a NULL master status (7.2). +- **Constraint steps** are idempotent and the guard trigger is created (4.3). +- **New compatibility sync triggers** keep old writers consistent, and a reconciliation step follows (4.3, 7). +- **`masterIdHint`** to an archived master maps to its Item (4.5). +- **Ratings, including votes,** roll up through `products.item_id` (4.5). +- **Round 2:** + - M3's multi-size rule ignores archived sizes and earbuds; + - the sync trigger handles re-keys and publishes the Item; + - M1 and M2 ship as one migration, with an Item status recompute; + - the trigger functions are `SECURITY DEFINER`, so `rate_worker` needs no write grant on `rate.items`. +- **Other consumers covered:** the trust view (1.1), and the URL lookups in the extension and inspect (5.2). +- **Refresh of unknown-variant listings, and ordering** against Rate-Services (5.4, 8, 10). +- **M0 duplicate merges** archive and move alerts instead of deleting. + +Not applied: one finding said the slug regexes needed different escaping. They use single backslashes, which are correct under `standard_conforming_strings = on`, and a verification query now checks that Item names and slugs carry no size. diff --git a/Docs/Proposals/2026-10-28-extension-data-source.html b/Docs/Proposals/2026-10-28-extension-data-source.html new file mode 100644 index 000000000..11f6125a5 --- /dev/null +++ b/Docs/Proposals/2026-10-28-extension-data-source.html @@ -0,0 +1,306 @@ + + + + + + + Extension làm nguồn dữ liệu + + + +
+ Đề xuất, chưa code + Chờ JOY trả lời 7 câu +

Mở rộng extension thành nguồn dữ liệu chính

+

Ngày 28/10/2026 - Issue DOS/Rate#1175 - Bản tóm tắt tiếng Anh: 2026-10-28-extension-data-source.md - Người soạn: session con của Rate Core PO

+ +
+

Tóm tắt một đoạn

+

Crawler đang đụng tường bot (TikTok tạm dừng ở RS#658, Lazada search bị trả về trang chặn (punish page) ở RS#659, Shopee chỉ chạy được qua một runner đã đăng nhập). Trình duyệt thật của người dùng không gặp những tường này vì chính người dùng mở trang. Đề xuất: phương án B, cho extension gửi một quan sát tối thiểu (giá, tồn kho, mã SKU, URL, thời điểm, sàn, không có dữ liệu cá nhân) khi người dùng đã đồng ý, lưu vào vùng cách ly, và chỉ đưa lên giá công khai sau khi Rate Services xác minh chéo với crawler hoặc với người dùng độc lập khác. Crawler vẫn giữ làm nguồn khởi động và đối chứng. Extension vẫn công khai (người dùng tải về thì giải nén ra đọc được), còn luật chống đầu độc và chấm điểm tin cậy nằm ở DOS/Rate-Services private. Extension không bao giờ tự tải trang người dùng không mở, không né bot, không giải captcha.

+
+ +

1. Hiện trạng apps/extension (đọc từ mã, nhánh dev ngày 28/10)

+ +

1.1. Extension đọc gì

+
    +
  • Chạy content script trên 10 sàn và chuỗi bán lẻ: Shopee, Lazada, Tiki, TikTok (*.tiktok.com), CellphoneS, Thế Giới Di Động, FPT Shop, Hoàng Hà, Di Động Việt, Phong Vũ (manifest.json). Quyền khai báo: storage, sidePanel và host của 10 sàn cộng rate.vn.
    • +
  • Chỉ xử lý trang chi tiết sản phẩm có mã sản phẩm trong URL, bỏ trang chủ, danh mục, tìm kiếm (extractProductSignalFromDom).
    • +
  • Lấy: URL đã cắt tham số theo dõi, mã sản phẩm và mã shop từ URL, tiêu đề, phiên bản đang chọn, giá hiện tại, giá gạch ngang, số sao và số lượt đánh giá. Ưu tiên JSON-LD, sau đó mới tới selector DOM. Chưa đọc tồn kho.
    • +
+ +

1.2. Hai luồng gửi lên server

+
Mỗi lần mở trang sản phẩm (luôn chạy, không cần opt-in): + GET /api/v1/rate/extension/lookup?platform&url&title&price&... + -> chỉ đọc, trả giá lịch sử, nơi bán khác, phán định giá ảo. Không ghi DB. + +Chỉ khi người dùng bật "Đóng góp phát hiện sản phẩm mới (Opt-In)" (mặc định TẮT): + POST /api/v1/rate/extension/ingest { platform, externalItemId, externalShopId, + url, title, selectedVariant, price, originalPrice, ratingStar, + ratingCount, userConsentOptIn: true } + -> Zod + giới hạn 15 lần/phút/IP (bộ nhớ trong của từng instance serverless, + không phải giới hạn toàn cục) + chặn phụ kiện + kiểm host khớp sàn + -> RPC rate.enqueue_crowd_verify_signal (không dùng service role) + -> chèn 1 job 'extension_crowd_verify' vào rate.crawl_jobs + (trần 120 job/phút toàn hệ thống, đây là giới hạn toàn cục duy nhất) + -> worker Rate Services nhận job như một yêu cầu CÀO LẠI URL đó
+ +

1.3. "Cách ly" hiện tại nghĩa là gì

+
    +
  • Tín hiệu từ người dùng không bao giờ ghi thẳng vào giá công khai. Giá người dùng thấy chỉ nằm trong payload của job.
    • +
  • Theo đọc mã Rate Services (mô tả chung), worker không có nhánh xử lý riêng cho loại job này: nó cào lại URL như mọi job khác. Tức là giá người dùng thấy không được dùng làm dữ liệu, chỉ là cái cớ để cào. Khi crawler bị chặn ở sàn đó, tín hiệu cộng đồng hiện không giúp được gì. Đây là lý do chính cần đề xuất này.
    • +
+ +
+

1.4. Những điểm lệch phát hiện khi đọc mã (cần sửa dù JOY chọn phương án nào)

+
    +
  1. Nhãn opt-in trong side panel nói "Chỉ gửi URL sản phẩm công khai đã cắt sạch tham số theo dõi", nhưng payload thật gửi mười trường dữ liệu (cộng cờ đồng ý): nền tảng, mã sản phẩm, mã shop, URL, tiêu đề, phiên bản, giá, giá gốc, số sao, số lượt đánh giá. Lời công bố phải khớp dữ liệu gửi đi; Chrome Web Store coi công bố nổi bật trong giao diện là điều kiện để sự đồng ý có giá trị [6].
    2. +
  2. CHROMEWEBSTORE.md giải trình quyền tabs và host localhost:3000 nhưng manifest.json không có hai quyền này; ngược lại manifest có năm host mà tài liệu không giải trình: TikTok, Hoàng Hà, Di Động Việt, Phong Vũ, www.rate.vn. Mục công bố quyền riêng tư của tài liệu này cũng chỉ kể 6 tên miền và nói extension "chỉ chạy trên trang chi tiết sản phẩm", trong khi content script khớp mọi đường dẫn của 10 sàn (việc lọc trang sản phẩm diễn ra trong mã, sau khi script đã chạy).
    3. +
  3. Host https://*.tiktok.com/* cho content script chạy trên toàn bộ TikTok (cả video, tin nhắn), rộng hơn mức cần cho trang sản phẩm. Match pattern hỗ trợ giới hạn theo đường dẫn, nên có thể thu hẹp.
    4. +
  4. Trang /privacy-policy của rate.vn chưa nhắc tới extension.
    5. +
  5. Luồng lookup mặc định, không cần opt-in, đã gửi URL, tiêu đề, giá, mã sản phẩm, mã shop (và IP theo bản chất HTTP) lên rate.vn ở mọi lần mở trang sản phẩm. Chính sách Chrome cho phép gửi hoạt động duyệt web khi phục vụ tính năng người dùng thấy và được công bố nổi bật [6]; giả định của người soạn là luồng này hợp lệ nếu được công bố rõ, nhưng nó cũng là dữ liệu duyệt web theo Luật 91/2025/QH15, nên cần đưa vào câu hỏi cho ý kiến pháp lý (mục 4.7).
    6. +
+

Chưa xác minh extension đã được phát hành trên Chrome Web Store hay chưa.

+
+ +

2. Honey, Keepa, BeeCost lấy dữ liệu thế nào (chỉ từ nguồn công khai)

+
+ + + + + + + + + + +
Sản phẩmĐiều đã công bố hoặc đã được kiểm chứngBài học cho Rate
Honey (PayPal)Trang "Data and privacy" của Honey nói họ thu thập thông tin sản phẩm chung như giá và tình trạng còn hàng để chạy lịch sử giá và Droplist, chỉ trên các trang mua sắm họ hỗ trợ, và dữ liệu sự kiện không chứa thông tin định danh [1]. Tổ chức datarequests.org cho rằng Honey thu thập lịch sử duyệt web nhiều hơn mức chính sách nói và đã khiếu nại lên cơ quan bảo vệ dữ liệu [2].Mô hình gần nhất với đề xuất: extension đọc giá trên trang người dùng xem. Rủi ro uy tín lớn nhất là làm nhiều hơn điều mình công bố.
KeepaTrang Firefox Add-ons của Keepa nói extension gửi mã ASIN của sản phẩm đang xem để vẽ biểu đồ và không lưu gì về người dùng [3]. Nhà nghiên cứu Wladimir Palant (2021) ghi nhận extension giữ kết nối WebSocket với một mã người dùng cố định, gửi số sao và hạng bán, và nhận lệnh từ server để tải nền các trang Amazon người dùng không mở rồi gửi dữ liệu về [4]. Một thảo luận trên nhóm Chromium Extensions cũng nêu việc này [5]. Keepa còn cung cấp dữ liệu lịch sử giá qua API cho bên ngoài [5b].Ví dụ cần tránh: biến máy người dùng thành crawler phân tán. Đó đúng là logic cào dữ liệu, trái tinh thần chính sách store, và không được đặt trong DOS/Rate. Rate chỉ đọc trang người dùng tự mở.
BeeCost (Việt Nam)Có extension cho Chrome và Cốc Cốc, hiện lịch sử giá trên Shopee, Tiki, Lazada, Sendo [7][8]; nhà phát triển nói có hơn 100 triệu sản phẩm để so sánh [9]. Một bài đánh giá nói khả năng lấy dữ liệu tốt nhất trên Shopee, chưa ổn định trên Lazada và FPT Shop [10]. Không tìm thấy tài liệu công khai nào mô tả cách BeeCost thu thập dữ liệu hay chính sách quyền riêng tư của họ.Chứng minh người dùng Việt Nam chấp nhận extension lịch sử giá, và Cốc Cốc là kênh phân phối đáng kể. Không suy đoán thêm về cách họ làm.
+

Các nguồn [2], [4], [5] là bên thứ ba và có thể đã cũ; chúng được dẫn để thấy rủi ro, không phải để kết luận hành vi hiện tại của các sản phẩm đó.

+ +

3. Vì sao trình duyệt thật né được tường bot

+
    +
  • Trang đã được chính người dùng mở, có cookie, phiên đăng nhập, dấu vân tay trình duyệt và IP dân dụng thật. Extension chỉ đọc DOM đã hiện ra, không phát thêm request nào tới sàn, nên sàn không có gì khác thường để chặn.
    • +
  • Lưu lượng tự nhiên phân tán: mỗi người xem vài trang, không có đỉnh truy cập từ một IP.
    • +
  • Đổi lại: Rate không chọn được trang nào được xem. Dữ liệu dồn vào sản phẩm phổ biến, sản phẩm ít người xem có thể không bao giờ được cập nhật. Vì vậy crawler vẫn cần.
    • +
+ +

4. Những gì thay đổi nếu extension là nguồn chính

+ +

4.1. Payload quan sát tối thiểu (đề xuất ExtensionObservation v1)

+
+ + + + + + + + + + + +
TrườngÝ nghĩaGhi chú
schema_versionPhiên bản payloadBắt buộc, để server từ chối bản cũ.
marketplaceSàn (10 giá trị hiện có)Server kiểm lại từ host của URL.
listing_urlURL chuẩn hóa, không queryĐã cắt tham số theo dõi như hiện nay.
external_item_id, external_shop_idMã listing và mã shop của sànMã của gian hàng, không phải của người dùng.
sku_id, variant_labelMã phiên bản (ví dụ model id của Shopee) và nhãn phiên bản đang chọnKhớp với mô hình Item, Variant, Listing (#1129).
price_vnd, list_price_vndGiá bán hiển thị và giá gạch ngangMục tiêu: chỉ lấy giá công khai chính của trang, bỏ giá sau voucher, xu, giá thành viên, vì chúng phụ thuộc tài khoản. Extractor hiện chưa phân biệt được hai loại giá này; đây là việc của M2 và cần kiểm từng sàn.
stock_state, stock_qtyCòn hàng, hết hàng, sắp hết, không rõ; số lượng nếu trang hiệnTrường mới, extractor hiện chưa đọc.
observed_atThời điểm client thấy giáServer ghi thêm received_at và tin received_at hơn observed_at.
extractorNguồn trích xuất (JSON-LD hay DOM) và phiên bản extensionĐể loại bản extractor hỏng.
submitter_idMã theo lượt cài, do server cấp khi người dùng đồng ý; server tự ghi thời điểm cấpCần cho chống đầu độc. Coi là dữ liệu cá nhân đã bút danh hóa (pseudonymous), lưu riêng, không công khai, người dùng yêu cầu xóa được. Kẻ tấn công vẫn xin được mã mới tùy ý, nên mã này chỉ giới hạn được client trung thực; phòng thủ thật nằm ở 4.4.
+

Không bao giờ gửi: cookie, tên tài khoản sàn, giỏ hàng, đơn hàng, địa chỉ, số điện thoại, từ khóa tìm kiếm, referrer, query string, nội dung tab khác, lịch sử duyệt. Bỏ ratingStar, ratingCount khỏi luồng này trừ khi JOY muốn giữ.

+

IP: không lưu IP nguyên. Để kiểm "người gửi độc lập" ở 4.4, phía nhận chỉ giữ một mã băm có muối của dải mạng (ví dụ /24 hoặc ASN) trong thời gian ngắn (vài tuần), không gắn vào giá công khai. Mã băm này vẫn là dữ liệu cá nhân và phải được công bố ở 4.2.

+ +

4.2. Lời đồng ý và chính sách quyền riêng tư

+
    +
  • Giữ opt-in, mặc định tắt. Thêm màn hình chào khi cài lần đầu, nói rõ gửi những trường nào (kể cả mã người gửi và mã băm dải mạng), để làm gì, và nút "Không, cảm ơn" nổi bật ngang nút "Đồng ý". Theo FAQ của Chrome Web Store, sự đồng ý chỉ có giá trị khi đi kèm công bố nổi bật ngay trong giao diện extension; công bố chỉ nằm trong chính sách quyền riêng tư là không đủ [6].
    • +
  • Có nút xem "những gì tôi đã gửi gần đây", tắt bất kỳ lúc nào, và yêu cầu xóa theo submitter_id. Đây cũng là quyền của chủ thể dữ liệu theo Luật Bảo vệ dữ liệu cá nhân 91/2025/QH15, hiệu lực từ 01/01/2026, và Nghị định 356/2025/NĐ-CP [11][12].
    • +
  • Cập nhật /privacy-policy có mục riêng cho extension, kèm công bố "Limited Use": chỉ dùng cho tính năng giá, không quảng cáo, không bán, không chuyển cho bên thứ ba. FAQ yêu cầu công bố này ở trang chủ hoặc cách một cú nhấp (trang chính sách là được), và thêm một bước xác nhận riêng trong developer dashboard khi nộp [6].
    • +
+ +

4.3. Rủi ro duyệt Chrome Web Store và Edge Add-ons

+
    +
  • Mục đích duy nhất: chính sách yêu cầu dữ liệu thu thập phải cần thiết cho mục đích duy nhất đã công bố và ở mức tối thiểu [6]. Đánh giá của người soạn (không phải lời của chính sách): "kiểm tra giá và lịch sử giá" có thể bao gồm việc gửi quan sát giá, miễn là dữ liệu tối thiểu và người dùng thấy lợi ích; người duyệt store có thể nhìn khác.
    • +
  • Hoạt động duyệt web và nội dung trang là dữ liệu người dùng theo định nghĩa của Chrome; đọc nội dung trang người dùng xem được tính là xử lý dữ liệu người dùng [6]. Vì vậy công bố và đồng ý ở 4.2 là bắt buộc, không phải tùy chọn.
    • +
  • Rủi ro bị từ chối hoặc gỡ: lời công bố lệch payload (mục 1.4), quyền host rộng (TikTok), tài liệu store lệch manifest. Sửa hết trước khi nộp.
    • +
  • Edge Add-ons có quy trình duyệt riêng; session này chưa đọc chi tiết chính sách Edge, cần kiểm khi chuẩn bị nộp. Cốc Cốc cài từ Chrome Web Store.
    • +
+ +

4.4. Chống đầu độc dữ liệu

+

Nguyên tắc: mọi bản gửi đều là không tin cậy. Mã extension công khai và chạy trên máy người dùng, ai cũng sửa được hoặc gọi thẳng API. Không có cách nào chứng thực client.

+
+

Lớp công khai (DOS/Rate, ở ingest)

    +
  • Kiểm định dạng, miền giá trị, host khớp sàn, chặn phụ kiện.
    • +
  • Giới hạn tần suất thô theo IP và theo submitter_id (chỉ chặn được client trung thực hoặc lỗi; kẻ tấn công đổi mã được).
    • +
  • Chỉ lưu vào vùng cách ly, không chấm điểm, không quyết định.
    • +
+

Lớp private (DOS/Rate-Services)

    +
  • Điểm tin cậy người gửi (tuổi của mã tính theo thời điểm server cấp, lịch sử khớp, tài khoản rate.vn nếu có). Mã mới bắt đầu ở mức tin cậy thấp, nên đổi mã liên tục không có lợi.
    • +
  • Đối chiếu với crawler và với người gửi độc lập khác (khác mã người gửi, khác mã băm dải mạng, khác thời điểm).
    • +
  • Phát hiện ngoại lai so với giá gần đây, giới hạn biên độ thay đổi.
    • +
  • Ngưỡng cụ thể, danh sách chặn, trọng số: không công bố.
    • +
+
+

Cách ly tới khi được xác nhận: một quan sát chỉ lên giá công khai khi khớp với crawler trong biên sai số, hoặc khớp với đủ số người gửi độc lập có điểm tin cậy. Một kẻ tạo hàng nghìn lượt cài giả vẫn phải vượt qua tính độc lập và lịch sử tin cậy.

+

Lưu ý về trần hiện tại: trần 120 job/phút toàn hệ thống trong RPC hiện nay hợp với "yêu cầu cào lại" nhưng không hợp với nguồn chính: vừa là nút thắt, vừa cho phép một kẻ tấn công làm đầy trần để chặn người dùng thật. Phương án B cần một bảng cách ly riêng có giới hạn theo người gửi thay vì trần toàn cục.

+ +

4.5. Khởi động lạnh (cold start)

+
    +
  • Extension chỉ hữu ích cho người dùng khi đã có lịch sử giá, mà lịch sử giá lại cần người dùng. Crawler phá vòng lặp này: tiếp tục chạy ở các sàn còn cào được để có dữ liệu ban đầu, và làm mốc đối chiếu.
    • +
  • Giả định để ước lượng (chưa có số đo): 1.000 người dùng opt-in, mỗi người xem khoảng 5 trang sản phẩm điện thoại mỗi ngày, cho khoảng 5.000 quan sát/ngày, dồn vào vài trăm mẫu phổ biến. Đủ để làm tươi giá sản phẩm hot, chưa đủ cho đuôi dài. Cần số cài đặt thật để kiểm chứng.
    • +
  • Kênh tăng trưởng: nút cài trên trang sản phẩm rate.vn, giá trị tự thân của extension (phán định giảm giá ảo), Cốc Cốc.
    • +
+ +

4.6. Độ phủ theo sàn

+
+ + + + + + +
SànCrawler hiện nay (theo PO Rate Services)Extension thêm được gìRủi ro riêng
ShopeeChỉ chạy qua một runner đã đăng nhậpGiá trị cao nhất: dữ liệu thật từ nhiều ngườiClass CSS bị làm rối, đổi thường xuyên (selector hiện có như .pqTWkA); giá phụ thuộc tài khoản nhiều
LazadaSearch bị trả về trang chặn (RS#659)Giá trang chi tiết khi người dùng xemExtension không thay được search discovery
TikiChưa có báo cáo bị chặn (chưa xác minh)Chủ yếu để đối chiếuThấp
TikTok ShopTạm dừng (RS#658)Có thể là nguồn duy nhấtPhải thu hẹp quyền host; trang sản phẩm hay mở trong app chứ không trên web, độ phủ có thể thấp
CellphoneS, TGDĐ, FPT Shop, Hoàng Hà, Di Động Việt, Phong VũChưa có báo cáo bị chặn (chưa xác minh)Đối chiếu, làm tươi giá giữa các lượt càoGiá và tồn kho có thể khác theo tỉnh; đề xuất chỉ lấy giá toàn quốc, bỏ tồn kho theo tỉnh để không phải gửi vị trí
+ +

4.7. Pháp lý và điều khoản sàn

+
    +
  • Khác biệt then chốt so với crawler: không có request nào của Rate tới sàn; người dùng tự xem trang, extension đọc những gì đã hiện trên màn hình của họ và chỉ gửi khi họ đồng ý.
    • +
  • Điều khoản của nhiều sàn cấm thu thập dữ liệu tự động. Việc đọc thụ động trang người dùng tự mở có thuộc phạm vi đó hay không là vùng xám; Honey, Keepa, BeeCost đã hoạt động nhiều năm, nhưng đó không phải là kết luận pháp lý. Rủi ro thực tế: sàn đổi DOM để làm hỏng extractor, hoặc gửi thư yêu cầu.
    • +
  • Giá và tồn kho là dữ kiện. Đây là hiểu biết chung, chưa có luật sư xác nhận cho luật Việt Nam. Đề xuất xin ý kiến pháp lý trước khi phát hành rộng (mốc M4).
    • +
  • Dữ liệu cá nhân: payload thiết kế để không có; riêng submitter_id, IP và mã băm dải mạng được xử lý như dữ liệu cá nhân theo Luật 91/2025/QH15 [11].
    • +
  • Câu hỏi cho ý kiến pháp lý: (a) đọc thụ động trang người dùng tự mở có vi phạm điều khoản sàn không; (b) luồng lookup mặc định (gửi URL, tiêu đề, giá của trang đang xem, không cần opt-in) có cần đồng ý riêng theo Luật 91/2025/QH15 không; (c) thời hạn lưu mã người gửi và mã băm dải mạng.
    • +
+ +

5. Câu hỏi của JOY: vì sao extension công khai còn crawler thì private?

+
+

Trả lời ngắn: extension không phải crawler. Nó chỉ đọc trang người dùng đang xem, khi họ đồng ý, và gửi lên rate.vn. Phần nhạy cảm không nằm ở việc đọc trang, mà nằm ở cách server quyết định tin bản gửi nào. Phần đó private.

+
+
    +
  1. Client buộc phải lộ. Extension được phát cho mọi người; tải file .crx về giải nén là đọc được toàn bộ mã. Giấu repo không giấu được client, chỉ làm mất lợi ích của open-core: người dùng và người duyệt store kiểm được extension làm đúng điều nó nói. Với một extension đọc trang mua sắm, sự minh bạch này là điểm bán hàng (ngược với các tranh cãi của Honey và Keepa ở mục 2).
    2. +
  2. Extension không chứa gì thuộc loại crawler. Không tải trang người dùng không mở, không proxy, không giải captcha, không né phát hiện bot, không gọi API nội bộ của sàn. Selector đọc giá trên trang người dùng xem không phải bí mật: ai mở DevTools cũng thấy.
    3. +
  3. Phía nhận mới là chỗ bị tấn công. Kẻ đầu độc không cần sửa extension, họ gọi thẳng API. Nếu họ đọc được ngưỡng, trọng số, cách đối chiếu với crawler, họ sẽ gửi dữ liệu nằm vừa dưới ngưỡng. Vì vậy chấm điểm tin cậy, đối chiếu crawler, ngưỡng và giới hạn chi tiết nằm ở DOS/Rate-Services.
    4. +
  4. Giữ bí mật là lớp phòng thủ thêm, không phải lớp duy nhất. Thiết kế vẫn phải đứng vững nếu luật bị lộ: cách ly mặc định, cần xác nhận độc lập, crawler làm mốc. Bí mật chỉ làm việc lách tốn công hơn.
    5. +
+ +

Bảng phân chia sở hữu

+
+ + + + + + + + + + +
Thành phầnRepoVì sao
Extension (đọc trang, lời đồng ý, side panel)DOS/Rate (public)Phát cho người dùng, đằng nào cũng đọc được; minh bạch giúp qua duyệt store và tạo niềm tin.
Schema payload ExtensionObservationDOS/Rate (packages/shared-data)Là hợp đồng giữa client và server; client cần nó, Rate Services dùng qua submodule.
Ingest API và kiểm tra cơ bản (định dạng, miền giá trị, host, đồng ý, giới hạn thô)DOS/Rate (apps/web)Chỉ là kiểm hợp lệ, không có luật tin cậy. Không chấm điểm ở đây.
Bảng cách ly (lưu quan sát chưa xác minh)Schema: bản tham chiếu ở DOS/Rate; migration chính thức ở DOS-MeCấu trúc bảng không phải bí mật. Anon chỉ ghi qua RPC hẹp, không đọc được.
Xác minh và chấm điểm tin cậy người gửiDOS/Rate-Services (private)Đây là thứ kẻ tấn công muốn đọc.
Đối chiếu với dữ liệu crawlerDOS/Rate-Services (private)Gắn với crawler, vốn đã private.
Ngưỡng, trọng số, danh sách chặn, giới hạn chi tiếtDOS/Rate-Services (private, cấu hình)Lộ ra là bị gửi vừa dưới ngưỡng.
Đưa quan sát đã xác nhận lên giá công khai (price_history)DOS/Rate-Services (job ghi bằng role worker)Chỉ phía private được ghi giá công khai; web chỉ hiển thị kèm nguồn.
Crawler, proxy, captcha, né botDOS/Rate-Services (private)Như hiện nay, quy tắc cứng của AGENTS.md. Không có phần nào vào extension.
+ +

6. Các phương án

+
+ + + + +
Phương ánMô tảĐượcMất
A. Giữ nguyênTín hiệu cộng đồng chỉ là yêu cầu cào lại; chỉ sửa các điểm lệch ở mục 1.4.Rủi ro thấp nhất, tốn ít công (chỉ phần M1).Không giải quyết tường bot: sàn chặn crawler thì tín hiệu vô dụng.
B. Quan sát thụ động, cách ly, xác minh (khuyến nghị)Extension gửi quan sát tối thiểu khi đã đồng ý; Rate Services xác minh chéo rồi mới công khai; crawler làm mốc.Có dữ liệu ở Shopee, Lazada, TikTok mà không cần né bot; minh bạch; ranh giới repo rõ.Phụ thuộc số người cài; đuôi dài thưa; cần bảng mới (migration DOS-Me), việc ở cả hai repo; rủi ro duyệt store.
C. Crawler phân tán trong extensionServer ra lệnh cho extension tải nền trang người dùng không mở (kiểu Keepa bị báo cáo [4]).Phủ rộng, chủ động.Là logic cào dữ liệu, không được ở DOS/Rate; dùng IP và phiên của người dùng; nguy cơ bị gỡ khỏi store và mất uy tín. Không khuyến nghị.
+

Bổ sung độc lập với cả ba: nguồn chính thức (API affiliate hoặc đối tác của sàn) nếu có, do Rate Services đánh giá riêng.

+ +

7. Khuyến nghị

+
+

Chọn phương án B, triển khai theo mốc, có giai đoạn chạy ẩn (shadow) để đo trước khi bất kỳ quan sát nào lên giá công khai. Sửa ngay các điểm lệch ở mục 1.4 vì chúng là rủi ro duyệt store kể cả khi giữ phương án A. Rate Services sở hữu ingestion phía sau, xác minh và đưa lên giá công khai.

+
+ +

8. Các mốc

+
+ + + + + + +
MốcNội dungRepoĐiều kiện qua mốc
M0JOY quyết định về đề xuất này-JOY trả lời mục 10
M1Sửa lời công bố cho khớp payload, mục extension trong /privacy-policy, CHROMEWEBSTORE.md khớp manifest, thu hẹp host TikTok, màn hình chào opt-inDOS/RateReview fresh-context, kiểm bằng tay trên 10 sàn
M2Schema ExtensionObservation v1, extractor đọc tồn kho và SKU, ingest lưu vào bảng cách ly mới (migration qua DOS-Me, JOY duyệt)DOS/Rate, DOS-MeKhông ghi gì vào giá công khai; test ingest đầy đủ
M3Xác minh và chấm điểm ở chế độ shadow: chấm nhưng không công khai, 2 đến 4 tuầnDOS/Rate-ServicesCác chỉ số ở mục 9 đạt ngưỡng đề xuất
M4Đưa quan sát đã xác nhận lên price_history kèm nguồn; web hiện "nguồn: người dùng, đã đối chiếu"; ý kiến pháp lý; xác nhận Limited Use trong developer dashboard; nộp Chrome Web Store và EdgeCả haiJOY duyệt bước production và bước nộp store
+ +

9. Chỉ số quyết định thành công

+
+ + + + + + + + +
Chỉ sốNgưỡng đề xuất (để JOY chỉnh)
Độ khớp với crawler trên listing cả hai cùng có (shadow)Ít nhất 98% quan sát lệch không quá 1%
Độ khớp giữa người gửi độc lập trên listing crawler không vào được (TikTok, Shopee và Lazada bị chặn), cộng một mẫu kiểm tra bằng tay mỗi tuầnÍt nhất 95% cặp quan sát cùng giờ lệch không quá 1%; mẫu tay khớp từ 95%
Tỷ lệ quan sát đã lên công khai mà sau đó bị chứng minh sai (bị crawler hoặc mẫu kiểm tay bác trong vòng 24 giờ, với giá lệch quá 1%)Dưới 0,5%
Tỷ lệ opt-in trên người dùng hoạt độngĐo trước, đặt ngưỡng sau M2
Phần trăm listing Shopee, Lazada, TikTok trong catalog có giá tươi dưới 24 giờTăng rõ so với chỉ crawler (đo mốc trước M3)
Tuổi trung vị của giá đang hiển thịGiảm so với mốc trước M3
Bị từ chối hoặc gỡ trên store; khiếu nại quyền riêng tư0
+ +

10. Câu hỏi mở cho JOY

+
1. Chọn phương án nào? Khuyến nghị B. A nếu muốn chờ thêm; C không khuyến nghị.
+
2. Mặc định đồng ý: giữ tắt và hỏi rõ ở màn hình chào (khuyến nghị), hay chỉ để công tắc trong side panel như hiện nay?
+
3. Danh tính người gửi: chỉ mã ẩn danh theo lượt cài, hay khuyến khích đăng nhập rate.vn để bản gửi có trọng số cao hơn?
+
4. Listing chỉ một người thấy và crawler không vào được: ẩn tới khi có xác nhận (khuyến nghị), hay hiện với nhãn "chưa xác minh"?
+
5. Ý kiến pháp lý trước khi phát hành rộng: có thuê luật sư không, ngân sách bao nhiêu?
+
6. Phát hành store: dưới tài khoản nhà phát triển nào, và có nộp Edge cùng lúc với Chrome không?
+
7. Giữ số sao và lượt đánh giá trong payload, hay bỏ để payload chỉ còn giá, tồn kho, SKU (khuyến nghị bỏ)?
+ +

Nguồn

+
    +
  1. Honey, "Data and privacy": joinhoney.com/data-and-privacy
    2. +
  2. datarequests.org (2020), về việc Honey thu thập dữ liệu: datarequests.org/blog/honey-data-collection
    3. +
  3. Keepa trên Firefox Add-ons: addons.mozilla.org/en-US/firefox/addon/keepa
    4. +
  4. Wladimir Palant, "Data exfiltration in Keepa Price Tracker" (2021): palant.info
    5. +
  5. Chromium Extensions group, "Keepa extension is deceptively scraping Amazon in the background": groups.google.com.
    [5b] Thư viện Python cho Keepa API: pypi.org/project/keepa
    6. +
  6. Chrome Web Store, User Data FAQ: developer.chrome.com
    7. +
  7. mytour.vn, về tính năng BeeCost: mytour.vn
    8. +
  8. uplus, review extension BeeCost: uplus.substack.com
    9. +
  9. BeeCost trên Ameblo: ameblo.jp/beecost
    10. +
  10. GHN, "BeeCost Shopee là gì": ghn.vn
    11. +
  11. Cổng thông tin Hà Tĩnh, Luật Bảo vệ dữ liệu cá nhân có hiệu lực từ 01/01/2026: hatinh.gov.vn
    12. +
  12. Thư viện Pháp luật, Nghị định 356/2025/NĐ-CP (chưa đối chiếu với Công báo; cần luật sư xác nhận ngày ban hành và hiệu lực): thuvienphapluat.vn
    13. +
+

Mã nguồn đã đọc (DOS/Rate): apps/extension/manifest.json, apps/extension/CHROMEWEBSTORE.md, apps/extension/src/background/service-worker.ts, apps/extension/src/extractors/ecommerce-detector.ts, apps/extension/src/sidepanel/sidepanel.html, apps/web/src/app/api/v1/rate/extension/ingest/route.ts, apps/web/src/server/rate/extension.ts, apps/web/src/app/[locale]/privacy-policy/page.tsx, apps/web/locales/*.json, supabase/migrations/20261018120101_rate_crowd_verify_signal_rpc.sql. DOS/Rate-Services chỉ được đọc để hiểu chung, không trích mã.

+
+ + diff --git a/Docs/Proposals/2026-10-28-extension-data-source.md b/Docs/Proposals/2026-10-28-extension-data-source.md new file mode 100644 index 000000000..c9ab698f3 --- /dev/null +++ b/Docs/Proposals/2026-10-28-extension-data-source.md @@ -0,0 +1,96 @@ +# Proposal: the browser extension as a primary data source + +- **Date:** 2026-10-28 +- **Issue:** [DOS/Rate#1175](https://github.com/DOS/Rate/issues/1175) +- **Status:** PROPOSAL, awaiting JOY's decisions. No product code has changed. +- **Full decision document (Vietnamese):** [`2026-10-28-extension-data-source.html`](./2026-10-28-extension-data-source.html) + +## Summary + +The private crawlers hit bot walls. TikTok is paused (DOS/Rate-Services#658), Lazada search returns a punish page (DOS/Rate-Services#659), and Shopee works only through one logged-in runner. A real user's browser does not hit these walls, because the user opened the page. + +The recommendation is **Option B**. With consent, the extension sends a minimal observation: price, stock, SKU, listing URL, timestamp and marketplace, and no personal data. The observation goes into a quarantine table. It becomes a public price only after DOS/Rate-Services corroborates it against the crawler or against independent submitters. The crawler stays as the seed and the cross-check. + +The extension never fetches pages the user did not open, never evades bot detection and never solves captchas. + +## Current state (read from `dev`) + +- **Lookup:** on every product page, with no opt-in, the extension sends `GET /api/v1/rate/extension/lookup` with the URL, title, price, item id and shop id. This path is read-only. +- **Ingest:** with the opt-in on (off by default), the extension sends `POST /api/v1/rate/extension/ingest`. The route checks the payload with Zod and allows 15 requests per minute per IP. That limiter is in memory per serverless instance, not global. It then calls the `rate.enqueue_crowd_verify_signal` RPC, which enqueues an `extension_crowd_verify` job in `rate.crawl_jobs`. The RPC caps all crowd jobs at 120 per minute; this is the only global limit. +- **Quarantine today:** from a read of the Rate-Services code, the worker has no separate handler for this job type and re-crawls the URL like any other job. The price the user saw is not used as data, so the crowd signal does not help when the crawler is blocked. +- **Mismatches to fix under any option:** + - The opt-in label says only the cleaned URL is sent. The payload actually has ten fields: platform, item id, shop id, URL, title, variant, price, original price, rating and rating count. + - `CHROMEWEBSTORE.md` justifies `tabs` and `localhost`, which the manifest does not request. It omits five hosts that the manifest does request: TikTok, Hoang Ha, Di Dong Viet, Phong Vu and `www.rate.vn`. Its privacy section says the extension "only activates" on product pages of 6 domains, while the content script matches every path on 10 sites. + - The default lookup sends browsing data without opt-in. That is probably allowed for a user-facing feature if it is disclosed, but this is an assumption and belongs in the legal review. + - `*.tiktok.com` is broader than product pages need. + - `/privacy-policy` does not mention the extension. + - The extractor does not read stock. + +## Comparable products (public sources only) + +- **Honey:** Honey says it collects general product data such as price and availability, only on supported stores, to power price history and Droplist ([joinhoney.com](https://www.joinhoney.com/data-and-privacy/)). datarequests.org says Honey collects more browsing data than its policy states ([datarequests.org](https://datarequests.org/blog/honey-data-collection)). +- **Keepa:** Keepa says its add-on sends the ASIN of the product the user is viewing ([AMO](https://addons.mozilla.org/en-US/firefox/addon/keepa/)). In 2021, Wladimir Palant reported that the server instructs the extension to load, in the background, Amazon pages the user never opened ([palant.info](https://palant.info/2021/08/02/data-exfiltration-in-keepa-price-tracker/), [Chromium group](https://groups.google.com/a/chromium.org/g/chromium-extensions/c/il9VqQBc8So)). Rate must not do this. +- **BeeCost:** BeeCost has extensions for Chrome and Coc Coc that show price history for Shopee, Tiki, Lazada and Sendo ([mytour.vn](https://mytour.vn/en/blog/bai-viet/top-5-features-of-beecost-the-ultimate-tool-for-finding-the-best-online-shopping-deals-mytour.html)). The developer claims over 100 million products ([ameblo.jp](https://ameblo.jp/beecost/)). No public document describes how it collects data. + +## Anti-poisoning, in short + +- Every submission is untrusted, because the client is open and the API can be called directly. +- The server issues `submitter_id` when the user consents and records the first-seen time itself. A new id starts with low trust, so rotating ids gains nothing. The per-id limit only bounds honest clients. +- The raw IP is not stored. Rate-Services keeps a short-lived, salted hash of the network range (for example /24 or ASN) to test independence. That hash is personal data and is disclosed. +- Observations stay quarantined until they match the crawler or enough independent, trusted submitters. + +## Why the extension is public while crawling is private + +The extension is not a crawler. It ships to every user and can be unpacked, so a private repo would not hide it. Being open lets users and store reviewers check that it does what it says. + +The sensitive part is how the server decides which submissions to trust. An attacker would call the API directly and would want to know the thresholds. That logic lives in DOS/Rate-Services. Secrecy is defense in depth only: the design must still hold if the rules leak, through default quarantine, independent corroboration and the crawler baseline. + +| Piece | Repo | +| :----------------------------------------------------------------------------- | :------------------------------------------------------- | +| Extension: page reading, consent UI, side panel | DOS/Rate (public) | +| Payload schema `ExtensionObservation` | DOS/Rate, `packages/shared-data` | +| Ingest API and basic validation (format, ranges, host, consent, coarse limits) | DOS/Rate, `apps/web` | +| Quarantine table | Reference copy in DOS/Rate; official migration in DOS-Me | +| Verification and submitter trust scoring | DOS/Rate-Services (private) | +| Crawler cross-check | DOS/Rate-Services (private) | +| Thresholds, weights, blocklists, detailed limits | DOS/Rate-Services (private config) | +| Promotion of observations to public `price_history` | DOS/Rate-Services (worker role) | +| Crawlers, proxy, captcha, bot evasion | DOS/Rate-Services (private); never in the extension | + +## Options + +| Option | Description | Verdict | +| :------------------------------------------------------ | :------------------------------------------------------------ | :------------------------------------------------------------------------------------------------ | +| A. Keep as is | Fix the mismatches only. | Safe, but does not solve the bot walls. | +| B. Passive observation with quarantine and verification | Described above. | **Recommended.** | +| C. Distributed crawler in the extension | The server tells extensions to fetch pages in the background. | Rejected. This is scraper logic, it misuses users' IPs and sessions, and it risks store takedown. | + +An official source (a marketplace affiliate or partner API), where one exists, can complement any option. Rate-Services would assess it separately. + +## Milestones + +| Milestone | Work | Repo | +| :-------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | :---------------- | +| M0 | JOY decides. | - | +| M1 | Fix the disclosure copy, the privacy policy, `CHROMEWEBSTORE.md` drift and the TikTok host scope. Add an onboarding opt-in screen. | DOS/Rate | +| M2 | Add the `ExtensionObservation v1` schema. The extractor reads stock and SKU. Ingest writes to a new quarantine table through a DOS-Me migration, with JOY's approval. Nothing is published. | DOS/Rate, DOS-Me | +| M3 | Run verification in shadow mode for 2 to 4 weeks: score observations but publish nothing. | DOS/Rate-Services | +| M4 | Promote corroborated observations with provenance. Get a legal opinion. Certify Limited Use in the developer dashboard. Submit to the Chrome Web Store and Edge Add-ons. | Both | + +**Proposed success thresholds:** + +- On listings both sources cover, at least 98% of shadow observations within 1% of the crawler. +- On listings the crawler cannot reach (TikTok, blocked Shopee and Lazada), at least 95% of same-hour pairs from independent submitters within 1%, plus a weekly manual spot-check sample. +- Under 0.5% of promoted observations later proven wrong, meaning the crawler or a manual check contradicts them by more than 1% within 24 hours. +- Better price freshness on Shopee, Lazada and TikTok than with the crawler alone. +- Zero store rejections and zero privacy complaints. + +## Open questions for JOY + +1. Which option: A, B or C? +2. Should the opt-in stay a side-panel toggle, or also be asked on an onboarding screen? +3. Should submitters stay anonymous per install, or should a rate.vn sign-in add weight? +4. Should a listing seen by one user only, which the crawler cannot reach, stay hidden or show as unverified? +5. Should we get a legal opinion before wide release, and with what budget? +6. Which store developer account should publish, and should Edge ship together with Chrome? +7. Should ratings stay in the payload? diff --git a/Docs/Resources/RELEASE.md b/Docs/Resources/RELEASE.md index fcfb5c3db..34442a318 100644 --- a/Docs/Resources/RELEASE.md +++ b/Docs/Resources/RELEASE.md @@ -11,7 +11,7 @@ This is the release workflow for `rate.vn`. It follows the same model as DOS/DOS ## Normal release -1. Feature PRs go into `dev` as usual: review, green required checks (`Typecheck & Build Monorepo`, `E2E (Playwright)`, `Secret scan (gitleaks)`, `task-claim`), squash merge. +1. Feature PRs go into `dev` as usual: review, green required checks (`Typecheck & Build Monorepo`, `E2E (Playwright)`, `Secret scan (gitleaks)`, `task-claim`), squash merge. `node scripts/ops/merge-when-green.mjs ` writes the squash commit's message itself: the PR title with `(#)`, then each commit wrapped at 100 characters, with the co-author trailers in a footer. It refuses a PR whose title is not a conventional header or does not fit ` (#<pr>)` in 100 characters: shorten the title. The promotion PR's `commitlint` check lints the head of `dev`, which is usually that squash commit, so it must pass `commitlint.config.ts` (DOS/Rate#1130). 2. Check `dev` on its Vercel Preview deployment. 3. Open the promotion PR from `dev` itself, not from a copy branch: `gh pr create --repo DOS/Rate --base main --head dev --title "chore(release): promote dev to main"` diff --git a/Docs/Screenshots/1132-category-layout/after-1280.jpg b/Docs/Screenshots/1132-category-layout/after-1280.jpg new file mode 100644 index 000000000..7436dcab0 Binary files /dev/null and b/Docs/Screenshots/1132-category-layout/after-1280.jpg differ diff --git a/Docs/Screenshots/1132-category-layout/after-1440-scrolled.jpg b/Docs/Screenshots/1132-category-layout/after-1440-scrolled.jpg new file mode 100644 index 000000000..b287a5779 Binary files /dev/null and b/Docs/Screenshots/1132-category-layout/after-1440-scrolled.jpg differ diff --git a/Docs/Screenshots/1132-category-layout/after-1440.jpg b/Docs/Screenshots/1132-category-layout/after-1440.jpg new file mode 100644 index 000000000..f8e62f646 Binary files /dev/null and b/Docs/Screenshots/1132-category-layout/after-1440.jpg differ diff --git a/Docs/Screenshots/1132-category-layout/after-1920-top.jpg b/Docs/Screenshots/1132-category-layout/after-1920-top.jpg new file mode 100644 index 000000000..59857d5f0 Binary files /dev/null and b/Docs/Screenshots/1132-category-layout/after-1920-top.jpg differ diff --git a/Docs/Screenshots/1132-category-layout/after-1920.jpg b/Docs/Screenshots/1132-category-layout/after-1920.jpg new file mode 100644 index 000000000..d9a45d86c Binary files /dev/null and b/Docs/Screenshots/1132-category-layout/after-1920.jpg differ diff --git a/Docs/Screenshots/1132-category-layout/after-390.jpg b/Docs/Screenshots/1132-category-layout/after-390.jpg new file mode 100644 index 000000000..c083922d0 Binary files /dev/null and b/Docs/Screenshots/1132-category-layout/after-390.jpg differ diff --git a/Docs/Screenshots/1132-category-layout/before-1280.jpg b/Docs/Screenshots/1132-category-layout/before-1280.jpg new file mode 100644 index 000000000..a69702a44 Binary files /dev/null and b/Docs/Screenshots/1132-category-layout/before-1280.jpg differ diff --git a/Docs/Screenshots/1132-category-layout/before-1440-scrolled.jpg b/Docs/Screenshots/1132-category-layout/before-1440-scrolled.jpg new file mode 100644 index 000000000..702e7dc5f Binary files /dev/null and b/Docs/Screenshots/1132-category-layout/before-1440-scrolled.jpg differ diff --git a/Docs/Screenshots/1132-category-layout/before-1440.jpg b/Docs/Screenshots/1132-category-layout/before-1440.jpg new file mode 100644 index 000000000..537af7032 Binary files /dev/null and b/Docs/Screenshots/1132-category-layout/before-1440.jpg differ diff --git a/Docs/Screenshots/1132-category-layout/before-1920-top.jpg b/Docs/Screenshots/1132-category-layout/before-1920-top.jpg new file mode 100644 index 000000000..19c1a2c81 Binary files /dev/null and b/Docs/Screenshots/1132-category-layout/before-1920-top.jpg differ diff --git a/Docs/Screenshots/1132-category-layout/before-1920.jpg b/Docs/Screenshots/1132-category-layout/before-1920.jpg new file mode 100644 index 000000000..d4e732d14 Binary files /dev/null and b/Docs/Screenshots/1132-category-layout/before-1920.jpg differ diff --git a/Docs/Screenshots/1132-category-layout/before-390.jpg b/Docs/Screenshots/1132-category-layout/before-390.jpg new file mode 100644 index 000000000..d5bf7c4bb Binary files /dev/null and b/Docs/Screenshots/1132-category-layout/before-390.jpg differ diff --git a/apps/importer/package.json b/apps/importer/package.json index 43fd56034..e5ff54214 100644 --- a/apps/importer/package.json +++ b/apps/importer/package.json @@ -37,7 +37,7 @@ "lint": "eslint . --max-warnings 0", "format": "prettier --write \"src/**/*.ts\"", "backfill:specs": "tsx src/shopee/jobs/backfill-specs.ts", - "test": "tsx src/shopee/__tests__/open-core-intelligence.test.ts && tsx src/shopee/__tests__/entity-matcher.test.ts && tsx src/shopee/__tests__/shopee-to-supabase.test.ts && tsx src/shopee/__tests__/paging.test.ts && tsx src/shopee/__tests__/backfill-review-risk.test.ts && tsx src/shopee/__tests__/arena-audit.test.ts && tsx src/shopee/test-ai-enricher.ts && tsx src/shopee/test-review-analyzer.ts", + "test": "tsx src/shopee/__tests__/open-core-intelligence.test.ts && tsx src/shopee/__tests__/entity-matcher.test.ts && tsx src/shopee/__tests__/item-matcher.test.ts && tsx src/shopee/__tests__/shopee-to-supabase.test.ts && tsx src/shopee/__tests__/paging.test.ts && tsx src/shopee/__tests__/backfill-review-risk.test.ts && tsx src/shopee/__tests__/item-inventory-report.test.ts && tsx src/shopee/__tests__/arena-audit.test.ts && tsx src/shopee/test-ai-enricher.ts && tsx src/shopee/test-review-analyzer.ts", "test:dist": "node src/shopee/__tests__/dist-exports.smoke.mjs", "backfill:review-risk": "tsx src/shopee/jobs/backfill-review-risk.ts" }, diff --git a/apps/importer/src/index.ts b/apps/importer/src/index.ts index a0ddd6300..fb8327674 100644 --- a/apps/importer/src/index.ts +++ b/apps/importer/src/index.ts @@ -12,3 +12,4 @@ export * from './shopee/ai-review-analyzer.js'; export * from './shopee/shopee-to-supabase.js'; export * from './shopee/jobs/backfill-specs.js'; export * from './shopee/paging.js'; +export * from './shopee/item-model-flag.js'; diff --git a/apps/importer/src/shopee/__tests__/dist-exports.smoke.mjs b/apps/importer/src/shopee/__tests__/dist-exports.smoke.mjs index 34f123cdc..a7a985ba2 100644 --- a/apps/importer/src/shopee/__tests__/dist-exports.smoke.mjs +++ b/apps/importer/src/shopee/__tests__/dist-exports.smoke.mjs @@ -25,6 +25,8 @@ for (const name of [ 'createImporterLogger', 'sanitizeAiGeneratedText', 'computeRetryDelayMs', + 'resolveItemModelMode', + 'UnkeyableProductTitleError', ]) { assert.equal( typeof crawlers[name], @@ -32,6 +34,12 @@ for (const name of [ `@repo/crawlers is missing export ${name}` ); } +// Item model rollout flag (DOS/Rate#1151): Rate-Services reads the same env name. +assert.equal(crawlers.ITEM_MODEL_ENV, 'RATE_ITEM_MODEL'); +assert.deepEqual([...crawlers.ITEM_MODEL_MODES], ['off', 'item', 'nullable_variant']); +assert.equal(crawlers.resolveItemModelMode(undefined), 'off'); +assert.equal(typeof crawlers.EntityMatcher.prototype.resolveItemAndVariant, 'function'); +assert.equal(crawlers.UNKEYED_MODEL_SEGMENT, 'unkeyed'); assert.equal( typeof crawlers.MAX_SPEC_CACHE_ENTRIES, 'number', diff --git a/apps/importer/src/shopee/__tests__/entity-matcher.test.ts b/apps/importer/src/shopee/__tests__/entity-matcher.test.ts index 4521e8800..f2c79b54f 100644 --- a/apps/importer/src/shopee/__tests__/entity-matcher.test.ts +++ b/apps/importer/src/shopee/__tests__/entity-matcher.test.ts @@ -675,6 +675,141 @@ async function runEntityMatcherSuite() { assert.equal(ctrlEntry.platform, 'shopee'); assert.equal(ctrlEntry.event, 'evt name'); console.log(' ✅ [9/9] DOS/Rate#876 EntityMatcher & Logger control-char/boolean bounds passed'); + + // 10. DOS/Rate#1142: masters keep only the spec fields their category has. + const enrichByCategory = async ( + row: Record<string, unknown>, + aiResult: Record<string, unknown> + ): Promise<Record<string, unknown> | undefined> => { + let updated: Record<string, unknown> | undefined; + const db = { + from() { + return { + select: () => ({ + eq: () => ({ maybeSingle: async () => ({ data: row, error: null }) }), + }), + update(payload: Record<string, unknown>) { + updated = payload; + return { eq: async () => ({ error: null }) }; + }, + }; + }, + } as unknown as RateSupabaseClient; + const enricher = { fetchSpecsFromAi: async () => aiResult } as unknown as AiSpecEnricher; + assert.equal( + await new EntityMatcher(db, enricher).enrichMasterProductById(String(row.id)), + true + ); + return updated; + }; + const phoneShapedAi = { + ram_gb: 1, + storage_gb: 64, + screen_size_inch: 1, + battery_mah: 515, + has_5g: true, + specifications: { + display: { size_inch: 1, refresh_rate_hz: 60, ppi: 300 }, + hardware: { ram_gb: 1, storage_gb: 64, chipset: 'BES2700' }, + camera: { main_camera: '3 High-SNR Microphones' }, + cellular: { has_5g: true, bands_5g: 'n78', bands_4g: 'LTE' }, + battery: { capacity_mah: 515 }, + multimedia: { audio_features: 'ANC, LDAC' }, + connectivity: { bluetooth: '5.4', wlan: 'Auto Switch' }, + }, + key_specs: { screen: '1"', camera: 'Mic', ram: '1GB', storage: '64GB', battery: '515 mAh' }, + }; + const budsRow = await enrichByCategory( + { + id: 'buds-1142', + brand: 'Samsung', + model: 'Galaxy Buds3 Pro', + name: 'Samsung Galaxy Buds3 Pro', + ram_gb: 1, + storage_gb: 64, + screen_size_inch: 1, + has_5g: false, + specifications: { category_slug: 'earbuds', display: { size_inch: 1 } }, + key_specs: { ram: '1GB' }, + }, + phoneShapedAi + ); + assert.equal(budsRow?.ram_gb, null, 'earbuds keep no RAM, even one stored before'); + assert.equal(budsRow?.storage_gb, null); + assert.equal(budsRow?.screen_size_inch, null); + assert.equal(budsRow?.has_5g, false); + assert.equal(budsRow?.battery_mah, 515); + const budsSpecs = budsRow?.specifications as Record<string, Record<string, unknown>>; + assert.equal(budsSpecs.display, undefined); + assert.deepEqual(budsSpecs.hardware, { chipset: 'BES2700' }, 'the audio chip stays'); + assert.equal(budsSpecs.camera, undefined); + assert.equal(budsSpecs.cellular, undefined); + assert.equal(budsSpecs.connectivity?.wlan, undefined); + assert.equal(budsSpecs.connectivity?.bluetooth, '5.4'); + assert.equal(budsSpecs.multimedia?.audio_features, 'ANC, LDAC'); + assert.equal(budsSpecs.category_slug, 'earbuds'); + assert.deepEqual(budsRow?.key_specs, { battery: '515 mAh' }); + + const watchRow = await enrichByCategory( + { + id: 'watch-1142', + brand: 'Apple', + model: 'Watch Ultra 2', + name: 'Apple Watch Ultra 2 64GB', + storage_gb: 64, + has_5g: true, + specifications: { category_slug: 'smartwatches' }, + key_specs: {}, + }, + phoneShapedAi + ); + // A watch keeps its stored 5G facts (a few models have RedCap 5G); only the + // page hides them, and the camera it cannot have is dropped. + assert.equal(watchRow?.has_5g, true); + assert.equal(watchRow?.storage_gb, 64); + const watchSpecs = watchRow?.specifications as Record<string, Record<string, unknown>>; + assert.equal(watchSpecs.camera, undefined); + assert.equal(watchSpecs.cellular?.has_5g, true); + assert.equal(watchSpecs.cellular?.bands_5g, 'n78'); + assert.equal(watchSpecs.cellular?.bands_4g, 'LTE'); + assert.equal(watchSpecs.display?.size_inch, 1); + + // A category guessed from the title does not scope the write: a master + // without a stored category keeps its RAM even when its name says "tai nghe". + const guessedRow = await enrichByCategory( + { + id: 'guess-1142', + brand: 'Samsung', + model: 'Galaxy S25', + name: 'Samsung Galaxy S25 tặng tai nghe Galaxy Buds', + ram_gb: 12, + storage_gb: 256, + has_5g: true, + specifications: {}, + key_specs: {}, + }, + phoneShapedAi + ); + assert.equal(guessedRow?.ram_gb, 12); + assert.equal(guessedRow?.storage_gb, 256); + assert.equal(guessedRow?.has_5g, true); + + const budsInsertDb = createMockSupabase(); + await new EntityMatcher(budsInsertDb.client, offlineEnricher).resolveOrCreateMasterProduct({ + ...s24New, + model_key: 'samsung:galaxy-buds3-pro', + category_slug: 'earbuds', + ram_gb: 1, + storage_gb: 64, + screen_size_inch: 1, + has_5g: true, + }); + const budsInsert = budsInsertDb.capturedUpserts[0]?.payload; + assert.equal(budsInsert?.ram_gb, null); + assert.equal(budsInsert?.storage_gb, null); + assert.equal(budsInsert?.screen_size_inch, null); + assert.equal(budsInsert?.has_5g, false); + console.log(' ✅ [10/10] DOS/Rate#1142 category spec templates on master writes passed'); console.log('🎉 All EntityMatcher & Open-Core Audit C6/C8/T15/Low tests passed!'); } diff --git a/apps/importer/src/shopee/__tests__/item-inventory-report.test.ts b/apps/importer/src/shopee/__tests__/item-inventory-report.test.ts new file mode 100644 index 000000000..676ac291e --- /dev/null +++ b/apps/importer/src/shopee/__tests__/item-inventory-report.test.ts @@ -0,0 +1,162 @@ +import assert from 'node:assert/strict'; +import type { FlagshipDeviceSeed } from '../flagship-seed-data.js'; +import { + buildInventoryReport, + itemKeyOf, + itemNameOf, + itemSlugOf, + type InventoryListing, + type InventoryMaster, +} from '../jobs/item-inventory-report.js'; + +const master = ( + id: string, + model_key: string, + slug: string, + name: string, + overrides: Partial<InventoryMaster> = {} +): InventoryMaster => ({ + id, + model_key, + slug, + name, + brand: name.split(' ')[0]!, + status: 'published', + category_id: 'phones', + storage_gb: Number(/:(\d+)gb$/.exec(model_key)?.[1] ?? NaN) || null, + ...overrides, +}); + +const listing = ( + id: string, + cluster_id: string | null, + title: string, + price_max = 1 +): InventoryListing => ({ + id, + cluster_id, + title, + platform: 'tgdd', + platform_item_id: id, + price_min: 1, + price_max, +}); + +const seed = (model_key: string, name: string) => ({ model_key, name }) as FlagshipDeviceSeed; + +function run() { + assert.equal(itemSlugOf('asus-zenbook-s-14-1tb'), 'asus-zenbook-s-14'); + assert.equal(itemSlugOf('sony-wf-1000xm6'), 'sony-wf-1000xm6'); + assert.equal(itemNameOf('Apple iPhone 16 128GB'), 'Apple iPhone 16'); + // As SQL split_part does: a one-segment key keeps its trailing ':'. + assert.equal(itemKeyOf('legacy'), 'legacy:'); + + const report = buildInventoryReport({ + categories: [ + { id: 'phones', slug: 'smartphones' }, + { id: 'buds', slug: 'earbuds' }, + ], + masters: [ + master('a', 'apple:iphone-16:128gb', 'apple-iphone-16-128gb', 'Apple iPhone 16 128GB'), + // Same Item, but its slug strips to another Item slug. + master('b', 'apple:iphone-16:256gb', 'apple-iphone-16-vn-256gb', 'Apple iPhone 16 256GB'), + // Another Item whose variant slug is iPhone 16's Item slug. + master('c', 'apple:iphone-16-x:64gb', 'apple-iphone-16', 'Apple iPhone 16 X 64GB'), + master( + 'd', + 'samsung:galaxy-buds3-pro:64gb', + 'samsung-galaxy-buds3-pro', + 'Samsung Galaxy Buds3 Pro', + { + category_id: 'buds', + } + ), + master('e', 'khc:legacy', 'legacy', 'Legacy'), + master('f', 'oddkey', 'odd', 'Odd'), + // A size in TB, and a NULL brand next to a real one: neither is a mismatch or a conflict. + master('g', 'asus:zenbook-s-14:1tb', 'asus-zenbook-s-14-1tb', 'Asus Zenbook S 14 1TB', { + storage_gb: 1024, + }), + master('h', 'asus:zenbook-s-14:512gb-oled', 'asus-zenbook-s-14-512gb', 'Asus Zenbook 512GB', { + brand: null, + storage_gb: 512, + }), + // A real conflict: two categories under one Item, and a stored size that disagrees. + master('i', 'oppo:find-x8:256gb', 'oppo-find-x8-256gb', 'OPPO Find X8 256GB', { + category_id: 'buds', + storage_gb: 128, + }), + master('j', 'oppo:find-x8:512gb', 'oppo-find-x8-128gb-512gb', 'OPPO Find X8 512GB'), + ], + listings: [ + listing('p1', 'a', 'Apple iPhone 16 256GB'), + listing('p2', 'a', 'Apple iPhone 16'), + listing('p3', null, 'x', 2), + listing('p4', 'gone', 'x'), + ], + seeds: [ + seed('apple:iphone-16:128gb', 'Apple iPhone 16 128GB'), + seed('samsung:galaxy-buds3-pro:64gb', 'Samsung Galaxy Buds3 Pro'), + ], + }); + + assert.equal(report.counts.total, 10); + assert.equal(report.counts.khc_keys, 1); + assert.equal(report.counts.bad_shape, 2); + assert.equal(report.counts.listings_without_master, 1); + assert.equal(report.counts.listings_on_missing_master, 1); + assert.deepEqual( + report.legacy_keys.map(r => r.model_key), + ['khc:legacy', 'oddkey'] + ); + assert.deepEqual(report.item_conflicts, [ + { item_key: 'oppo:find-x8', variants: ['oppo:find-x8:256gb', 'oppo:find-x8:512gb'] }, + ]); + assert.deepEqual( + report.sizeless_with_size.map(r => r.model_key), + ['samsung:galaxy-buds3-pro:64gb', 'oppo:find-x8:256gb'] + ); + assert.deepEqual(report.seed_drift, [ + { + seed_key: 'samsung:galaxy-buds3-pro:64gb', + normalized_key: 'samsung:galaxy-buds3-pro:unknown', + stored_status: 'published', + normalized_key_stored: false, + }, + ]); + assert.ok(report.master_drift.some(d => d.normalized_key === 'samsung:galaxy-buds3-pro:unknown')); + assert.deepEqual( + report.listing_drift.map(d => [d.kind, d.normalized_key]), + [ + ['variant', 'apple:iphone-16:256gb'], + ['size_not_stated', 'apple:iphone-16:unknown'], + ] + ); + assert.deepEqual(report.slug_collisions, [ + { + kind: 'two_items', + slugs: ['apple-iphone-16'], + items: ['apple:iphone-16', 'apple:iphone-16-x'], + }, + { + kind: 'item_slug_differs', + slugs: ['apple-iphone-16', 'apple-iphone-16-vn'], + items: ['apple:iphone-16'], + }, + { + kind: 'item_slug_differs', + slugs: ['oppo-find-x8', 'oppo-find-x8-128gb'], + items: ['oppo:find-x8'], + }, + { kind: 'still_sized', slugs: ['oppo-find-x8-128gb'], items: ['oppo:find-x8'] }, + ]); + assert.deepEqual( + report.storage_mismatch.map(r => r.model_key), + ['oppo:find-x8:256gb'] + ); + assert.equal(report.multi_sku_listings.length, 1); + + console.log('item-inventory-report tests passed'); +} + +run(); diff --git a/apps/importer/src/shopee/__tests__/item-matcher.test.ts b/apps/importer/src/shopee/__tests__/item-matcher.test.ts new file mode 100644 index 000000000..e9804691c --- /dev/null +++ b/apps/importer/src/shopee/__tests__/item-matcher.test.ts @@ -0,0 +1,774 @@ +/** + * Item-first matching (DOS/Rate#1151, design doc 4.5): + * 1. RATE_ITEM_MODEL parsing (default off, typos read as off) + * 2. Normalizer Item fields: item_key/variant_key/variant_known/variant_source, no empty segment + * 3. Mode off: resolveItemAndVariant is today's master matcher, never touches rate.items + * 4. Modes item / nullable_variant: Item tiers, exact variant, size not stated (#1105), hints + * 5. Pipeline: item_id / variant_source / nullable cluster_id writes, archived masterIdHint, + * a manual variant is kept, unkeyable titles are rejected before any write + */ +import assert from 'node:assert/strict'; +import { composeModelKey, splitModelKey } from '@repo/shared-data'; +import { EntityMatcher } from '../entity-matcher.js'; +import { ITEM_MODEL_ENV, resolveItemModelMode } from '../item-model-flag.js'; +import { ProductNormalizer, UnkeyableProductTitleError } from '../product-normalizer.js'; +import { ShopeeIngestionPipeline } from '../shopee-to-supabase.js'; +import type { AiSpecEnricher } from '../ai-spec-enricher.js'; +import type { RateSupabaseClient, ShopeeProduct } from '../types.js'; + +type Row = Record<string, unknown>; +type Result = { data: unknown; error: { message: string; code?: string } | null }; + +interface Call { + table: string; + op: 'select' | 'upsert' | 'insert' | 'update' | 'rpc'; + cols?: string; + payload?: Row; + filters: Array<[string, unknown]>; +} + +/** + * Chainable PostgREST stand-in. Every query resolves through `handler`, which sees the + * table, operation, selected columns, payload and eq filters; unhandled calls resolve empty. + */ +function createDb(handler: (call: Call) => Result | undefined) { + const calls: Call[] = []; + const run = (call: Call): Result => handler(call) ?? { data: null, error: null }; + const builder = (call: Call) => { + const b: Record<string, unknown> = {}; + for (const name of ['neq', 'gt', 'gte', 'lte', 'lt', 'in', 'order', 'limit', 'is']) { + b[name] = () => b; + } + b.eq = (col: string, val: unknown) => { + call.filters.push([col, val]); + return b; + }; + b.select = (cols: string) => { + if (call.op === 'select') call.cols = cols; + return b; + }; + b.maybeSingle = () => Promise.resolve(run(call)); + b.single = () => Promise.resolve(run(call)); + b.then = (resolve: (r: Result) => unknown, reject?: (e: unknown) => unknown) => + Promise.resolve(run(call)).then(resolve, reject); + return b; + }; + const client = { + from(table: string) { + const start = (op: Call['op'], extra: Partial<Call> = {}) => { + const call: Call = { table, op, filters: [], ...extra }; + calls.push(call); + return builder(call); + }; + return { + select: (cols: string) => start('select', { cols }), + upsert: (payload: Row) => start('upsert', { payload }), + insert: (payload: Row) => start('insert', { payload }), + update: (payload: Row) => start('update', { payload }), + }; + }, + rpc(fn: string, args: Row) { + const call: Call = { table: fn, op: 'rpc', payload: args, filters: [] }; + calls.push(call); + return Promise.resolve(run(call)); + }, + } as unknown as RateSupabaseClient; + return { client, calls }; +} + +const filter = (call: Call, col: string) => call.filters.find(([c]) => c === col)?.[1]; + +/** Enricher stub: Tier 3 never calls an AI provider in these tests. */ +const passthroughEnricher = { + enrichCandidate: async <T>(candidate: T) => candidate, +} as unknown as AiSpecEnricher; + +const normalizer = new ProductNormalizer(); +const candidateFor = (title: string) => normalizer.normalize({ title } as ShopeeProduct); + +interface ItemDb { + items?: Array<{ id: string; item_key: string }>; + variants?: Array<{ id: string; item_id: string; variant_key: string; status?: string }>; + matchItem?: Row[]; + matchItemErr?: { message: string }; + /** item_id the M1 trigger gives a newly seeded master; null = migration missing. */ + seededItemId?: string | null; +} + +/** Answers the matcher's reads and writes from an in-memory Item / variant table. */ +function itemDbAnswer(opts: ItemDb, call: Call): Result | undefined { + if (call.table === 'items' && call.op === 'select') { + const [[col, val] = []] = call.filters; + const row = (opts.items ?? []).find(i => (col === 'id' ? i.id : i.item_key) === val); + return { data: row ?? null, error: null }; + } + if (call.table === 'match_item') { + return { data: opts.matchItem ?? [], error: opts.matchItemErr ?? null }; + } + if (call.table === 'match_master_product') return { data: [], error: null }; + if (call.table === 'master_products' && call.op === 'select') { + const itemId = filter(call, 'item_id'); + if (itemId !== undefined) { + const rows = (opts.variants ?? []) + .filter(v => v.item_id === itemId) + .map(v => ({ id: v.id, variant_key: v.variant_key, status: v.status ?? 'published' })); + return { data: rows, error: null }; + } + if (call.cols === 'item_id') { + return { data: { item_id: opts.seededItemId ?? null }, error: null }; + } + return { data: null, error: null }; // Tier 1 model_key / race winner: not found + } + if (call.table === 'master_products' && call.op === 'upsert') { + return { data: [{ id: 'seeded-master' }], error: null }; + } + return undefined; +} + +const createItemDb = (opts: ItemDb = {}) => createDb(call => itemDbAnswer(opts, call)); + +const ZENBOOK_ITEM = { id: 'item-zenbook', item_key: 'asus:zenbook-s-14' }; +const ZENBOOK_1TB = { + id: 'master-zenbook-1tb', + item_id: 'item-zenbook', + variant_key: '1024gb', +}; + +async function run() { + console.log('🧪 Item-first matcher suite (DOS/Rate#1151)'); + + // 1. Flag parsing + assert.equal(ITEM_MODEL_ENV, 'RATE_ITEM_MODEL'); + assert.equal(resolveItemModelMode(undefined), 'off'); + assert.equal(resolveItemModelMode(''), 'off'); + assert.equal(resolveItemModelMode('item'), 'item'); + assert.equal(resolveItemModelMode(' Nullable-Variant '), 'nullable_variant'); + assert.equal(resolveItemModelMode('nullable'), 'off', 'a typo must never enable writes'); + assert.equal(resolveItemModelMode(true), 'off'); + console.log(' ✅ [1/5] RATE_ITEM_MODEL defaults to off'); + + // 2. Normalizer Item fields + const cases: Array<[string, Row]> = [ + [ + 'Điện thoại Apple iPhone 16 Pro Max 256GB Chính hãng VN/A', + { + model_key: 'apple:iphone-16-pro-max:256gb', + item_key: 'apple:iphone-16-pro-max', + variant_key: '256gb', + item_name: 'Apple iPhone 16 Pro Max', + item_slug: 'apple-iphone-16-pro-max', + variant_known: true, + variant_source: 'title', + keyable: true, + }, + ], + [ + 'Laptop ASUS Zenbook S 14 UX5406AA-SU348W', + { + model_key: 'asus:zenbook-s-14:unknown', + item_key: 'asus:zenbook-s-14', + variant_key: 'unknown', + variant_known: false, + variant_source: null, + keyable: true, + }, + ], + [ + 'Tai nghe Bluetooth Apple AirPods Pro 2 (USB-C) Chính hãng VN/A', + { + model_key: 'apple:airpods-pro-2:unknown', + variant_known: true, + variant_source: 'single_variant', + }, + ], + [ + 'Apple Watch Ultra 2 49mm GPS + Cellular', + { variant_key: '64gb', variant_known: true, variant_source: 'single_size' }, + ], + [ + 'Samsung', + { + model_key: 'samsung:unkeyed:unknown', + item_key: 'samsung:unkeyed', + keyable: false, + needs_review: true, + }, + ], + ['', { model_key: 'khac:unkeyed:unknown', keyable: false, brand: 'Khác', slug: 'khac' }], + ]; + for (const [title, expected] of cases) { + const c = candidateFor(title) as unknown as Row; + for (const [field, value] of Object.entries(expected)) { + assert.deepEqual(c[field], value, `${JSON.stringify(title)}: ${field}`); + } + const parts = splitModelKey(c.model_key as string); + assert.ok(parts, `${JSON.stringify(title)}: model_key has no empty segment`); + assert.equal(parts.itemKey, c.item_key); + assert.equal(parts.variantKey, c.variant_key); + assert.equal(composeModelKey(c.item_key as string, c.variant_key as string), c.model_key); + } + console.log(' ✅ [2/5] Normalizer emits item_key/variant_key and never an empty segment'); + + // 3. Mode off: today's matcher, no rate.items access + { + const db = createItemDb({ items: [ZENBOOK_ITEM], variants: [ZENBOOK_1TB] }); + const matcher = new EntityMatcher(db.client, passthroughEnricher, { itemModel: 'off' }); + const result = await matcher.resolveItemAndVariant( + candidateFor('Laptop ASUS Zenbook S 14 UX5406AA-SU348W'), + 'cat-laptops' + ); + assert.deepEqual(result, { + itemId: null, + masterId: 'seeded-master', + isNew: true, + variantSource: null, + }); + assert.ok(!db.calls.some(c => c.table === 'items' || c.table === 'match_item')); + assert.ok(db.calls.some(c => c.table === 'match_master_product')); + const seeded = db.calls.find(c => c.op === 'upsert'); + assert.equal(seeded?.payload?.model_key, 'asus:zenbook-s-14:unknown'); + assert.ok(!('item_id' in (seeded?.payload ?? {}))); + + const envDefault = process.env[ITEM_MODEL_ENV]; + delete process.env[ITEM_MODEL_ENV]; + assert.equal(new EntityMatcher(db.client, passthroughEnricher).itemModel, 'off'); + if (envDefault !== undefined) process.env[ITEM_MODEL_ENV] = envDefault; + + await assert.rejects( + matcher.resolveOrCreateMasterProduct(candidateFor('Samsung')), + UnkeyableProductTitleError + ); + } + console.log(' ✅ [3/5] Mode off keeps the master matcher and never reads rate.items'); + + // 4. Item modes + { + // Known size, exact variant of an exact Item. + const db = createItemDb({ + items: [{ id: 'item-ip16pm', item_key: 'apple:iphone-16-pro-max' }], + variants: [{ id: 'master-ip16pm-256', item_id: 'item-ip16pm', variant_key: '256gb' }], + }); + const matcher = new EntityMatcher(db.client, passthroughEnricher, { itemModel: 'item' }); + const result = await matcher.resolveItemAndVariant( + candidateFor('Điện thoại Apple iPhone 16 Pro Max 256GB Chính hãng VN/A') + ); + assert.deepEqual(result, { + itemId: 'item-ip16pm', + masterId: 'master-ip16pm-256', + isNew: false, + variantSource: 'title', + }); + assert.ok(!db.calls.some(c => c.table === 'match_master_product' || c.op === 'upsert')); + } + { + // Known size missing on a known Item: the variant is created under the Item's key. + const db = createItemDb({ items: [ZENBOOK_ITEM], variants: [ZENBOOK_1TB] }); + const matcher = new EntityMatcher(db.client, passthroughEnricher, { itemModel: 'item' }); + const result = await matcher.resolveItemAndVariant( + candidateFor('Laptop ASUS Zenbook S 14 UX5406AA 16GB 512GB SSD'), + 'cat-laptops' + ); + assert.equal(result.itemId, 'item-zenbook'); + assert.equal(result.masterId, 'seeded-master'); + assert.equal(result.isNew, true); + assert.equal( + db.calls.find(c => c.op === 'upsert')?.payload?.model_key, + 'asus:zenbook-s-14:512gb' + ); + } + { + // #1105: a storage-less title on a multi-size Item never lands on the sized variant. + const title = 'Laptop ASUS Zenbook S 14 UX5406AA-SU348W'; + const nullable = createItemDb({ items: [ZENBOOK_ITEM], variants: [ZENBOOK_1TB] }); + const nullableResult = await new EntityMatcher(nullable.client, passthroughEnricher, { + itemModel: 'nullable_variant', + }).resolveItemAndVariant(candidateFor(title)); + assert.deepEqual(nullableResult, { + itemId: 'item-zenbook', + masterId: null, + isNew: false, + variantSource: null, + }); + assert.ok(!nullable.calls.some(c => c.op === 'upsert' || c.table === 'match_master_product')); + + const itemMode = createItemDb({ items: [ZENBOOK_ITEM], variants: [ZENBOOK_1TB] }); + const itemResult = await new EntityMatcher(itemMode.client, passthroughEnricher, { + itemModel: 'item', + }).resolveItemAndVariant(candidateFor(title)); + assert.equal(itemResult.itemId, 'item-zenbook'); + assert.notEqual(itemResult.masterId, ZENBOOK_1TB.id); + assert.equal( + itemMode.calls.find(c => c.op === 'upsert')?.payload?.model_key, + 'asus:zenbook-s-14:unknown', + 'mode item keeps a variant: the Item’s unknown one' + ); + + const withUnknown = createItemDb({ + items: [ZENBOOK_ITEM], + variants: [ + ZENBOOK_1TB, + { id: 'master-zenbook-unknown', item_id: 'item-zenbook', variant_key: 'unknown' }, + ], + }); + const reuse = await new EntityMatcher(withUnknown.client, passthroughEnricher, { + itemModel: 'item', + }).resolveItemAndVariant(candidateFor(title)); + assert.equal(reuse.masterId, 'master-zenbook-unknown'); + + // An archived sized variant does not make the Item multi-size. + const archived = createItemDb({ + items: [ZENBOOK_ITEM], + variants: [ + { ...ZENBOOK_1TB, status: 'archived' }, + { id: 'master-zenbook-unknown', item_id: 'item-zenbook', variant_key: 'unknown' }, + ], + }); + const single = await new EntityMatcher(archived.client, passthroughEnricher, { + itemModel: 'nullable_variant', + }).resolveItemAndVariant(candidateFor(title)); + assert.equal(single.masterId, 'master-zenbook-unknown'); + } + { + // Single-variant Item (earbuds): its one variant, even in nullable mode. + const db = createItemDb({ + items: [{ id: 'item-app2', item_key: 'apple:airpods-pro-2' }], + variants: [{ id: 'master-app2', item_id: 'item-app2', variant_key: 'unknown' }], + }); + const result = await new EntityMatcher(db.client, passthroughEnricher, { + itemModel: 'nullable_variant', + }).resolveItemAndVariant(candidateFor('Tai nghe Apple AirPods Pro 2 (USB-C) Chính hãng VN/A')); + assert.deepEqual(result, { + itemId: 'item-app2', + masterId: 'master-app2', + isNew: false, + variantSource: 'single_variant', + }); + } + { + // Item Tier 2: fuzzy on Item names, then the variant under the matched Item's key. + const db = createItemDb({ + items: [{ id: 'item-s24u', item_key: 'samsung:galaxy-s24-ultra' }], + variants: [{ id: 'master-s24u-256', item_id: 'item-s24u', variant_key: '256gb' }], + matchItem: [ + { item_id: 'item-s24u', item_key: 'samsung:galaxy-s24-ultra', name: 'x', similarity: 0.82 }, + ], + }); + const candidate = { ...candidateFor('Samsung Galaxy S24 Ultra 5G 12GB/256GB Chính hãng') }; + candidate.model_key = 'samsung:galaxy-s24-ultra-5g:256gb'; + const result = await new EntityMatcher(db.client, passthroughEnricher, { + itemModel: 'item', + }).resolveItemAndVariant(candidate); + assert.equal(result.itemId, 'item-s24u'); + assert.equal(result.masterId, 'master-s24u-256'); + const rpc = db.calls.find(c => c.table === 'match_item'); + assert.equal(rpc?.payload?.search_title, candidate.item_name); + assert.ok(!/256/.test(String(rpc?.payload?.search_title)), 'Item search has no size'); + assert.equal(rpc?.payload?.p_threshold, 0.65); + } + { + // No Item: seed the master (draft when ambiguous); the M1 trigger gives its item_id. + const db = createItemDb({ + matchItem: [{ item_id: 'item-other', item_key: 'apple:iphone-15', similarity: 0.67 }], + seededItemId: 'item-new', + }); + const result = await new EntityMatcher(db.client, passthroughEnricher, { + itemModel: 'item', + }).resolveItemAndVariant(candidateFor('Apple iPhone 15 Plus 128GB Chính hãng VN/A')); + assert.deepEqual(result, { + itemId: 'item-new', + masterId: 'seeded-master', + isNew: true, + variantSource: 'title', + }); + assert.equal(db.calls.find(c => c.op === 'upsert')?.payload?.status, 'draft'); + + const noMigration = createItemDb({ seededItemId: null }); + await assert.rejects( + new EntityMatcher(noMigration.client, passthroughEnricher, { + itemModel: 'item', + }).resolveItemAndVariant(candidateFor('Apple iPhone 15 Plus 128GB')), + /DOS\/Rate#1150/ + ); + await assert.rejects( + new EntityMatcher( + createItemDb({ matchItemErr: { message: 'boom' } }).client, + passthroughEnricher, + { + itemModel: 'item', + } + ).resolveItemAndVariant(candidateFor('Apple iPhone 15 Plus 128GB')), + /Item Tier 2 fuzzy match failed: boom/ + ); + await assert.rejects( + new EntityMatcher(createItemDb().client, passthroughEnricher, { + itemModel: 'item', + }).resolveItemAndVariant(candidateFor('Samsung')), + UnkeyableProductTitleError + ); + } + { + // itemIdHint wins over the key; the variant still comes from the title. + const db = createItemDb({ + items: [ZENBOOK_ITEM], + variants: [ZENBOOK_1TB], + }); + const result = await new EntityMatcher(db.client, passthroughEnricher, { + itemModel: 'nullable_variant', + }).resolveItemAndVariant(candidateFor('ASUS Zenbook S14 OLED'), undefined, undefined, { + itemIdHint: 'item-zenbook', + }); + assert.deepEqual(result, { + itemId: 'item-zenbook', + masterId: null, + isNew: false, + variantSource: null, + }); + assert.ok(!db.calls.some(c => c.table === 'match_item')); + } + { + // A retired (archived) unknown variant is never seeded again or adopted. + const retired = { + items: [ZENBOOK_ITEM], + variants: [ + ZENBOOK_1TB, + { + id: 'master-zenbook-unknown', + item_id: 'item-zenbook', + variant_key: 'unknown', + status: 'archived', + }, + ], + }; + const title = 'Laptop ASUS Zenbook S 14 UX5406AA-SU348W'; + const itemDb = createItemDb(retired); + await assert.rejects( + new EntityMatcher(itemDb.client, passthroughEnricher, { + itemModel: 'item', + }).resolveItemAndVariant(candidateFor(title)), + /archived/ + ); + assert.ok(!itemDb.calls.some(c => c.op === 'upsert')); + const singleRetired = createItemDb({ + items: [ZENBOOK_ITEM], + variants: [{ ...retired.variants[1]! }], + }); + const none = await new EntityMatcher(singleRetired.client, passthroughEnricher, { + itemModel: 'nullable_variant', + }).resolveItemAndVariant(candidateFor(title)); + assert.equal(none.masterId, null); + assert.ok(!singleRetired.calls.some(c => c.op === 'upsert')); + } + { + // An archived sized variant takes no new listing in nullable mode; mode item keeps Tier 1's rule. + const archived1tb = { + items: [ZENBOOK_ITEM], + variants: [{ ...ZENBOOK_1TB, status: 'archived' }], + }; + const title = 'Laptop ASUS Zenbook S 14 UX5406AA 1TB'; + const nullable = await new EntityMatcher( + createItemDb(archived1tb).client, + passthroughEnricher, + { + itemModel: 'nullable_variant', + } + ).resolveItemAndVariant(candidateFor(title)); + assert.equal(nullable.masterId, null); + const itemMode = await new EntityMatcher( + createItemDb(archived1tb).client, + passthroughEnricher, + { + itemModel: 'item', + } + ).resolveItemAndVariant(candidateFor(title)); + assert.equal(itemMode.masterId, ZENBOOK_1TB.id); + } + { + // A fuzzy Item hit never creates a variant under that Item's key. + const db = createItemDb({ + items: [{ id: 'item-ip15p', item_key: 'apple:iphone-15-pro' }], + variants: [{ id: 'master-ip15p-128', item_id: 'item-ip15p', variant_key: '128gb' }], + matchItem: [ + { item_id: 'item-ip15p', item_key: 'apple:iphone-15-pro', name: 'x', similarity: 0.9 }, + ], + seededItemId: 'item-ip15pm', + }); + const result = await new EntityMatcher(db.client, passthroughEnricher, { + itemModel: 'item', + }).resolveItemAndVariant(candidateFor('Apple iPhone 15 Pro Max 256GB Chính hãng VN/A')); + assert.equal( + db.calls.find(c => c.op === 'upsert')?.payload?.model_key, + 'apple:iphone-15-pro-max:256gb' + ); + assert.equal(result.itemId, 'item-ip15pm'); + } + { + // A sizeless line keyed `unknown` reuses a seeded `standard` variant. + const db = createItemDb({ + items: [{ id: 'item-app2', item_key: 'apple:airpods-pro-2' }], + variants: [{ id: 'master-app2-std', item_id: 'item-app2', variant_key: 'standard' }], + }); + const result = await new EntityMatcher(db.client, passthroughEnricher, { + itemModel: 'item', + }).resolveItemAndVariant(candidateFor('Tai nghe Apple AirPods Pro 2 (USB-C) Chính hãng VN/A')); + assert.equal(result.masterId, 'master-app2-std'); + assert.ok(!db.calls.some(c => c.op === 'upsert')); + } + console.log(' ✅ [4/5] Item tiers, exact variants, size not stated (#1105) and hints'); + + // 5. Pipeline writes + const product = (title: string): ShopeeProduct => ({ + platform: 'shopee', + itemId: 'item-1151', + shopId: 'shop-1151', + title, + priceMin: 25_000_000, + priceMax: 25_000_000, + currency: 'VND', + historicalSold: 1, + ratingStar: 5, + ratingCount: 1, + images: [], + models: [], + attributes: [], + shop: { shopId: 'shop-1151', name: 'Shop', isMall: false, ratingStar: 5, followerCount: 1 }, + reviews: [], + url: 'https://shopee.vn/product/shop-1151/item-1151', + }); + const ingest = async ( + mode: 'off' | 'item' | 'nullable_variant', + title: string, + item: ItemDb, + extra: { hint?: Row | null; stored?: Row | null } = {}, + options: { masterIdHint?: string; itemIdHint?: string } = {} + ) => { + const db = createDb(call => { + if (call.table === 'categories') { + return { data: [{ id: 'cat-laptops', slug: 'laptops' }], error: null }; + } + if (call.table === 'merchants') return { data: { id: 'merchant-1' }, error: null }; + if (call.table === 'products' && call.op === 'upsert') { + return { data: { id: 'product-1' }, error: null }; + } + if (call.table === 'products' && call.op === 'select') { + return { data: extra.stored ?? null, error: null }; + } + if (call.table === 'record_price_observation') return { data: 'inserted', error: null }; + if ( + call.table === 'master_products' && + call.op === 'select' && + filter(call, 'id') !== undefined && + call.cols !== 'item_id' + ) { + return { data: extra.hint ?? null, error: null }; + } + return itemDbAnswer(item, call); + }); + const pipeline = new ShopeeIngestionPipeline({ supabaseClient: db.client, itemModel: mode }); + (pipeline as unknown as { matcher: EntityMatcher }).matcher = new EntityMatcher( + db.client, + passthroughEnricher, + { itemModel: mode } + ); + const result = await pipeline.ingestProduct(product(title), undefined, { + pricesOnly: true, + ...options, + }); + const listing = db.calls.find(c => c.table === 'products' && c.op === 'upsert')?.payload; + const rpcs = db.calls.filter(c => c.op === 'rpc').map(c => c.table); + return { result, listing, rpcs, calls: db.calls }; + }; + + const zenbook = { items: [ZENBOOK_ITEM], variants: [ZENBOOK_1TB] }; + const bareTitle = 'Laptop ASUS Zenbook S 14 UX5406AA-SU348W'; + { + const off = await ingest('off', 'Laptop ASUS Zenbook S 14 UX5406AA 1TB', zenbook); + assert.ok(off.listing && !('item_id' in off.listing) && !('variant_source' in off.listing)); + assert.equal(off.result.itemId, null); + assert.ok(!off.rpcs.includes('refresh_item_stats')); + assert.ok(!off.calls.some(c => c.table === 'items')); + } + { + const nullable = await ingest('nullable_variant', bareTitle, zenbook); + assert.equal(nullable.listing?.cluster_id, null); + assert.equal(nullable.listing?.item_id, 'item-zenbook'); + assert.equal(nullable.listing?.variant_source, null); + assert.deepEqual( + { masterId: nullable.result.masterId, itemId: nullable.result.itemId }, + { masterId: null, itemId: 'item-zenbook' } + ); + assert.ok(!nullable.rpcs.includes('refresh_master_market_stats')); + assert.ok(nullable.rpcs.includes('refresh_item_stats')); + + const sized = await ingest('item', 'Laptop ASUS Zenbook S 14 UX5406AA 1TB', zenbook); + assert.equal(sized.listing?.cluster_id, ZENBOOK_1TB.id); + assert.equal(sized.listing?.item_id, 'item-zenbook'); + assert.equal(sized.listing?.variant_source, 'title'); + assert.ok(sized.rpcs.includes('refresh_master_market_stats')); + } + { + // A variant set by hand survives a crawl that states no size. + const kept = await ingest('nullable_variant', bareTitle, zenbook, { + stored: { cluster_id: ZENBOOK_1TB.id, variant_source: 'manual' }, + }); + assert.equal(kept.listing?.cluster_id, ZENBOOK_1TB.id); + assert.ok(!('variant_source' in (kept.listing ?? {}))); + + // Mode item too: the Item's unknown variant never replaces a hand-set one. + const keptItem = await ingest( + 'item', + bareTitle, + { + ...zenbook, + variants: [ + ZENBOOK_1TB, + { id: 'master-zenbook-unknown', item_id: 'item-zenbook', variant_key: 'unknown' }, + ], + }, + { stored: { cluster_id: ZENBOOK_1TB.id, variant_source: 'manual' } } + ); + assert.equal(keptItem.listing?.cluster_id, ZENBOOK_1TB.id); + assert.ok(!('variant_source' in (keptItem.listing ?? {}))); + + // A hand-set variant wins even over a size stated in the title. + const manualWins = await ingest( + 'nullable_variant', + 'Laptop ASUS Zenbook S 14 UX5406AA 1TB', + zenbook, + { + stored: { cluster_id: 'master-zenbook-512', variant_source: 'manual' }, + } + ); + assert.equal(manualWins.listing?.cluster_id, 'master-zenbook-512'); + assert.ok(!('variant_source' in (manualWins.listing ?? {}))); + + // Title evidence stays when a later crawl has none; a legacy link does not. + const evidenceKept = await ingest('nullable_variant', bareTitle, zenbook, { + stored: { cluster_id: ZENBOOK_1TB.id, variant_source: 'title' }, + }); + assert.equal(evidenceKept.listing?.cluster_id, ZENBOOK_1TB.id); + const legacyDropped = await ingest('nullable_variant', bareTitle, zenbook, { + stored: { cluster_id: ZENBOOK_1TB.id, variant_source: 'legacy' }, + }); + assert.equal(legacyDropped.listing?.cluster_id, null); + + // A newer size stated in the title moves an evidence-based variant. + const moved = await ingest('item', 'Laptop ASUS Zenbook S 14 UX5406AA 1TB', zenbook, { + stored: { cluster_id: 'master-zenbook-512', variant_source: 'title' }, + }); + assert.equal(moved.listing?.cluster_id, ZENBOOK_1TB.id); + assert.equal(moved.listing?.variant_source, 'title'); + // ...and the variant it left is refreshed as well as the new one. + const refreshed = moved.calls + .filter(c => c.op === 'rpc' && c.table === 'refresh_master_market_stats') + .map(c => c.payload?.p_master_id); + assert.deepEqual(refreshed, [ZENBOOK_1TB.id, 'master-zenbook-512']); + + const left = await ingest('nullable_variant', bareTitle, zenbook, { + stored: { cluster_id: 'master-zenbook-unknown', variant_source: null }, + }); + assert.equal(left.listing?.cluster_id, null); + assert.deepEqual( + left.calls + .filter(c => c.op === 'rpc' && c.table === 'refresh_master_market_stats') + .map(c => c.payload?.p_master_id), + ['master-zenbook-unknown'] + ); + } + { + // An archived masterIdHint maps to its Item; a live one is kept. + const archived = await ingest( + 'nullable_variant', + bareTitle, + zenbook, + { + hint: { + id: 'master-zenbook-unknown', + specifications: { display: { size: 'x' } }, + status: 'archived', + item_id: 'item-zenbook', + variant_key: 'unknown', + }, + }, + { masterIdHint: 'master-zenbook-unknown' } + ); + assert.equal(archived.listing?.cluster_id, null); + assert.equal(archived.listing?.item_id, 'item-zenbook'); + + const live = await ingest( + 'item', + 'Laptop ASUS Zenbook S 14 UX5406AA 1TB', + zenbook, + { + hint: { + id: ZENBOOK_1TB.id, + specifications: { display: { size: 'x' } }, + status: 'published', + item_id: 'item-zenbook', + variant_key: '1024gb', + }, + }, + { masterIdHint: ZENBOOK_1TB.id } + ); + assert.equal(live.listing?.cluster_id, ZENBOOK_1TB.id); + assert.equal(live.listing?.variant_source, 'title'); + assert.ok(!live.calls.some(c => c.table === 'items'), 'a live hint skips matching'); + + const offHint = await ingest( + 'off', + bareTitle, + zenbook, + { hint: { id: 'master-zenbook-unknown', specifications: { display: { size: 'x' } } } }, + { masterIdHint: 'master-zenbook-unknown', itemIdHint: 'item-zenbook' } + ); + assert.equal(offHint.listing?.cluster_id, 'master-zenbook-unknown'); + assert.equal( + offHint.calls.find(c => c.table === 'master_products' && filter(c, 'id'))?.cols, + 'id, specifications' + ); + } + { + // itemIdHint for a storage-less page. + const hinted = await ingest( + 'nullable_variant', + 'ASUS Zenbook S14 OLED', + zenbook, + {}, + { + itemIdHint: 'item-zenbook', + } + ); + assert.equal(hinted.listing?.item_id, 'item-zenbook'); + assert.equal(hinted.listing?.cluster_id, null); + } + { + // An unkeyable title is rejected before the merchant write. + const db = createDb(call => + call.table === 'categories' ? { data: [], error: null } : undefined + ); + const pipeline = new ShopeeIngestionPipeline({ supabaseClient: db.client, itemModel: 'off' }); + await assert.rejects( + pipeline.ingestProduct(product('Samsung'), undefined, { pricesOnly: true }), + UnkeyableProductTitleError + ); + assert.ok(!db.calls.some(c => c.table === 'merchants')); + + // With a live masterIdHint the title's key is unused: the refresh still works. + const hintedRefresh = await ingest( + 'off', + 'Samsung', + zenbook, + { hint: { id: 'master-legacy', specifications: { display: { size: 'x' } } } }, + { masterIdHint: 'master-legacy' } + ); + assert.equal(hintedRefresh.listing?.cluster_id, 'master-legacy'); + } + console.log( + ' ✅ [5/5] Pipeline writes item_id, nullable cluster_id and keeps hand-set variants' + ); + console.log('🎉 All item-first matcher tests passed!'); +} + +run().catch(err => { + console.error(err); + process.exit(1); +}); diff --git a/apps/importer/src/shopee/__tests__/open-core-intelligence.test.ts b/apps/importer/src/shopee/__tests__/open-core-intelligence.test.ts index b2d633139..0b66cf002 100644 --- a/apps/importer/src/shopee/__tests__/open-core-intelligence.test.ts +++ b/apps/importer/src/shopee/__tests__/open-core-intelligence.test.ts @@ -419,7 +419,7 @@ async function runSuite() { ['Apple Watch Series 10 Sport Loop 45mm cho mặt 46mm', 'apple:watch-series-10:unknown'], ['Apple Watch Series 10 Sport Band 45mm cho mặt 42mm', 'apple:watch-series-10:unknown'], ['Apple Watch SE 3 GPS Strap 40mm for 44mm case', 'apple:watch-se-3:unknown'], - ['Apple Watch Series 10 Bracelet 42mm | 46mm', 'apple:watch-series-10-46mm:unknown'], + ['Apple Watch Series 10 Bracelet 42mm | 46mm', 'apple:watch-series-10:unknown'], // A band is sold by the case it fits: a band's size alone is the case's. [ 'Apple Watch Series 10 GPS Dây Cao Su 46mm Chính Hãng VN/A', @@ -447,14 +447,62 @@ async function runSuite() { 'Apple Watch SE 2024 Viền nhôm dây cao su [40mm/GPS] - Mới', 'apple:watch-se-2024-40mm:unknown', ], - // A decomposed or upper-case "dây" is the same word; "Bandit" is no band. - ['Apple Watch Series 10 da\u0302y vải 45mm, 42mm', 'apple:watch-series-10-42mm:unknown'], - ['APPLE WATCH SERIES 10 DÂY VẢI 45MM, 42MM', 'apple:watch-series-10-42mm:unknown'], + // A decomposed or upper-case "dây" is the same word, so "42mm" after its comma is a + // second band size (DOS/Rate#1137); "Bandit" is no band. + ['Apple Watch Series 10 da\u0302y vải 45mm, 42mm', 'apple:watch-series-10:unknown'], + ['APPLE WATCH SERIES 10 DÂY VẢI 45MM, 42MM', 'apple:watch-series-10:unknown'], ['Apple Watch Series 10 Bandit 46mm cho mặt 42mm', 'apple:watch-series-10-46mm:unknown'], ]; for (const [title, expectedKey] of watchBandSizeKeys1124) { assert.equal(keyOf(title), expectedKey, title); } + // DOS/Rate#1137: a band phrase that ends in a size runs on over the sizes listed after + // it with "," or "|"; only a size outside that list is the case's. + const watchBandListKeys1137: Array<[string, string]> = [ + // The titles of the issue. + [ + 'Apple Watch Series 11 GPS (Sport Band 42mm, 46mm) Case 42mm', + 'apple:watch-series-11-42mm:64gb', + ], + ['Apple Watch Series 11 GPS Nhôm Dây Sport Loop 42mm | 46mm', 'apple:watch-series-11:64gb'], + ['Apple Watch Series 11 Dây Vải 42mm 46mm', 'apple:watch-series-11:64gb'], + // The list runs on over every listed size, and a pipe list keys like a comma list. + ['Apple Watch Series 10 Sport Band 42mm, 46mm, 42mm', 'apple:watch-series-10:unknown'], + ['Apple Watch Series 10 Dây Vải 42mm | 46mm | Chính Hãng', 'apple:watch-series-10:unknown'], + [ + 'Apple Watch Series 10 Sport Band 42mm, 46mm | mặt 46mm', + 'apple:watch-series-10-46mm:unknown', + ], + // A band phrase without a size starts no list: the size after its comma is the case's. + ['Apple Watch Series 10 Dây Cao Su, 46mm', 'apple:watch-series-10-46mm:unknown'], + ['Apple Watch Series 10 Dây Cao Su | 42mm GPS', 'apple:watch-series-10-42mm:unknown'], + ['Apple Watch Series 10 Dây Cao Su S/M, 46mm', 'apple:watch-series-10-46mm:unknown'], + // A listed size keeps its band's fit, a bracket or text after it. + ['Apple Watch Series 10 Sport Band 45mm, 46mm GPS', 'apple:watch-series-10:unknown'], + [ + 'Apple Watch Series 10 GPS Dây Cao Su 42mm, 46mm Chính Hãng VN/A', + 'apple:watch-series-10:unknown', + ], + ['Apple Watch Series 10 Sport Band 42mm S/M, 46mm M/L', 'apple:watch-series-10:unknown'], + ['Apple Watch Series 10 Sport Band 42mm | 46mm (S/M)', 'apple:watch-series-10:unknown'], + ['Apple Watch Series 10 [Sport Band 42mm, 46mm] Mới 100%', 'apple:watch-series-10:unknown'], + ['Apple Watch Series 10 Dây Sport Band 46mm, 42mm.', 'apple:watch-series-10:unknown'], + // A case size before the band phrase still wins. + ['Apple Watch Series 11 GPS 46mm, Dây Cao Su 46mm S/M', 'apple:watch-series-11-46mm:64gb'], + // ")", ";", an en dash or " - " still end the band phrase and its list. + ['Apple Watch Series 10 (Sport Band 45mm) 46mm, 42mm', 'apple:watch-series-10-46mm:unknown'], + // A listed part that names the case ("case", "mặt", "vỏ") is the case's. + [ + 'Apple Watch Series 11 GPS Sport Band 46mm, 42mm Aluminum Case', + 'apple:watch-series-11-42mm:64gb', + ], + ['Apple Watch Series 11 GPS Sport Band 46mm | 42mm Case', 'apple:watch-series-11-42mm:64gb'], + ['Apple Watch Series 10 Dây Vải 45mm, 42mm mặt nhôm', 'apple:watch-series-10-42mm:unknown'], + ['Apple Watch Series 10 Dây Vải 45mm, 46mm vỏ nhôm', 'apple:watch-series-10-46mm:unknown'], + ]; + for (const [title, expectedKey] of watchBandListKeys1137) { + assert.equal(keyOf(title), expectedKey, title); + } // The registry has one entry per line: a sized title keeps the line's benchmark, but // the entry's screen and battery describe only the size it was written for. assert.equal( diff --git a/apps/importer/src/shopee/ai-spec-enricher.ts b/apps/importer/src/shopee/ai-spec-enricher.ts index ed053cb91..a9a7efd59 100644 --- a/apps/importer/src/shopee/ai-spec-enricher.ts +++ b/apps/importer/src/shopee/ai-spec-enricher.ts @@ -15,6 +15,7 @@ import { type ProductSpecifications, type KeySpecs, } from '@repo/shared-data'; +import { applyCategorySpecTemplate } from '@repo/shared-data/rules'; import type { NormalizedProductCandidate } from './product-normalizer.js'; import { computeRetryDelayMs, createImporterLogger, sanitizeAiGeneratedText } from './logger.js'; @@ -619,6 +620,7 @@ RULES: 3. Boolean fields (has_5g, nfc, volte, vowifi, headphone_jack_35mm) MUST be JSON booleans (true or false). 4. Separate cameras clearly following PhoneArena structure: main_camera, second_camera (ultrawide), third_camera (telephoto/macro), fourth_camera (periscope if present), front_camera. 5. Keep descriptive text concise, accurate, and readable for Vietnamese & international users. +6. Omit groups and fields the device category does not have. Earbuds have no display, RAM, storage, camera or cellular network; describe their drivers, codecs, ANC and battery instead. Smartwatches have no camera, and a cellular watch is LTE unless the model really supports 5G. REQUIRED JSON SCHEMA: { @@ -977,15 +979,21 @@ REQUIRED JSON SCHEMA: const keySpecsParsed = KeySpecsSchema.safeParse(keySpecsCandidate); const keySpecs: KeySpecs = keySpecsParsed.success ? keySpecsParsed.data : {}; - const result: AiEnrichmentResponse = { - ram_gb: ramGb, - storage_gb: storageGb, - screen_size_inch: screenSizeInch, - battery_mah: batteryMah, - has_5g: has5g, - specifications, - key_specs: keySpecs, - }; + // The schema is phone-shaped; drop what the category does not have + // (earbuds: no screen, RAM or camera; watches: no 5G) (DOS/Rate#1142). + const result: AiEnrichmentResponse = applyCategorySpecTemplate( + catKey, + { + ram_gb: ramGb, + storage_gb: storageGb, + screen_size_inch: screenSizeInch, + battery_mah: batteryMah, + has_5g: has5g, + specifications, + key_specs: keySpecs, + }, + { forStorage: true } + ); this.setCacheEntry(cacheKey, result); logger.info( diff --git a/apps/importer/src/shopee/entity-matcher.ts b/apps/importer/src/shopee/entity-matcher.ts index e52887048..6c8628b6c 100644 --- a/apps/importer/src/shopee/entity-matcher.ts +++ b/apps/importer/src/shopee/entity-matcher.ts @@ -1,12 +1,23 @@ import crypto from 'node:crypto'; import { + UNKNOWN_VARIANT_KEY, + composeModelKey, + isItemKey, + isSizedVariantKey, + splitModelKey, + type VariantSource, +} from '@repo/shared-data'; +import { + UnkeyableProductTitleError, detectCategorySlug, type CanonicalCategorySlug, type NormalizedProductCandidate, } from './product-normalizer.js'; +import { applyCategorySpecTemplate } from '@repo/shared-data/rules'; import { AiSpecEnricher, mergeProductSpecifications } from './ai-spec-enricher.js'; import { mergeKeySpecs } from './jobs/backfill-specs.js'; import { createImporterLogger } from './logger.js'; +import { resolveItemModelMode, type ItemModelMode } from './item-model-flag.js'; import type { RateSupabaseClient } from './types.js'; const logger = createImporterLogger('EntityMatcher'); @@ -132,14 +143,322 @@ export function toBoundedPositiveNumber(value: unknown, max: number): number | n return Number.isFinite(n) && n > 0 && n <= max ? n : null; } +export interface EntityMatcherOptions { + /** Item model rollout mode; defaults to the `RATE_ITEM_MODEL` env (see item-model-flag.ts). */ + itemModel?: ItemModelMode; +} + +export interface ItemMatchResult { + /** The listing's Item; null only in mode `off`, where Items are not read. */ + itemId: string | null; + /** The listing's variant (`cluster_id`); null = no variant, only in mode `nullable_variant`. */ + masterId: string | null; + isNew: boolean; + /** `products.variant_source`; null when the listing does not tell its variant. */ + variantSource: VariantSource | null; +} + +type MatchedItem = { id: string; itemKey: string }; + export class EntityMatcher { private enricher: AiSpecEnricher; + public readonly itemModel: ItemModelMode; constructor( private supabase: RateSupabaseClient, - enricher?: AiSpecEnricher + enricher?: AiSpecEnricher, + options: EntityMatcherOptions = {} ) { this.enricher = enricher ?? new AiSpecEnricher(); + this.itemModel = options?.itemModel ?? resolveItemModelMode(); + } + + /** + * Item-first matching (DOS/Rate#1151, design doc section 4.5). In mode `off` this is + * exactly `resolveOrCreateMasterProduct`. Otherwise: + * 1. Item by `itemIdHint`, then by exact `item_key`, then fuzzy on Item names + * (`match_item`). Item names carry no storage, so a bare title can no longer + * land on a sized variant (#1105). + * 2. Known size: the variant by exact `(item_id, variant_key)`, created when missing. + * 3. Size not stated: a single-variant Item's one variant. On a multi-size Item, no + * variant in mode `nullable_variant`, the Item's `unknown` variant in mode `item`. + * 4. No Item: seed a master as today; the M1 sync trigger creates its draft Item. + */ + public async resolveItemAndVariant( + candidate: NormalizedProductCandidate, + categoryId?: string, + rawTitle?: string, + options: { itemIdHint?: string | null } = {} + ): Promise<ItemMatchResult> { + if (this.itemModel === 'off') { + const { masterId, isNew } = await this.resolveOrCreateMasterProduct( + candidate, + categoryId, + rawTitle + ); + return { itemId: null, masterId, isNew, variantSource: null }; + } + + const input = this.sanitizeCandidateInput(candidate, categoryId, rawTitle); + candidate = input.candidate; + const { safeModelKey, safeCategoryId, safeRawTitle } = input; + const keys = splitModelKey(safeModelKey); + if (!keys) { + throw new Error( + `[EntityMatcher] model_key is not in brand:model:variant shape: ${JSON.stringify(safeModelKey)}` + ); + } + const variantKnown = + typeof candidate.variant_known === 'boolean' + ? candidate.variant_known + : keys.variantKey !== UNKNOWN_VARIANT_KEY || candidate.category_slug === 'earbuds'; + const variantSource: VariantSource | null = variantKnown + ? (candidate.variant_source ?? + (keys.variantKey === UNKNOWN_VARIANT_KEY ? 'single_variant' : 'title')) + : null; + + // 1. Item: caller hint, exact item_key, then fuzzy on Item names. + let item: MatchedItem | null = null; + let ambiguousFuzzy = false; + const safeItemIdHint = sanitizeMatcherEntityId(options?.itemIdHint, 128); + if (safeItemIdHint) { + item = await this.findItem('id', safeItemIdHint); + if (item) { + logger.info('entity_match.item_hint', `Item from caller hint: ${item.itemKey}`, { + itemId: item.id, + }); + } else { + logger.warn('entity_match.item_hint_missing', `itemIdHint ${safeItemIdHint} not found`, { + itemIdHint: safeItemIdHint, + }); + } + } + if (!item) { + item = await this.findItem('item_key', keys.itemKey); + if (item) { + logger.info('entity_match.item_tier1_exact', `Item Tier 1 exact: ${item.itemKey}`, { + itemId: item.id, + }); + } + } + let fuzzyItem = false; + if (!item) { + const fuzzy = await this.fuzzyMatchItem(candidate); + item = fuzzy.item; + fuzzyItem = Boolean(fuzzy.item); + ambiguousFuzzy = fuzzy.ambiguous; + } + + // 4. No Item: seed the master under the candidate's own key. A new Item's one + // variant (sized or `unknown`) is its only variant until a sized one appears. + const seedOwnKey = async (): Promise<ItemMatchResult> => { + const seeded = await this.seedMasterProduct( + candidate, + safeModelKey, + safeCategoryId, + safeRawTitle, + ambiguousFuzzy + ); + const itemId = await this.itemIdOfMaster(seeded.masterId); + return { itemId, masterId: seeded.masterId, isNew: seeded.isNew, variantSource }; + }; + if (!item) return seedOwnKey(); + if (!isItemKey(item.itemKey)) { + throw new Error( + `[EntityMatcher] Item ${item.id} has no brand:model key (${JSON.stringify(item.itemKey)}); cannot place a variant under it` + ); + } + + const variants = await this.listItemVariants(item.id); + let targetVariantKey = keys.variantKey; + + if (variantKnown) { + // 2. Known size: exact (item_id, variant_key), as Tier 1 matches an exact model_key. + // A sizeless line keyed `unknown` also takes a live `standard` variant (seed keys). + const exact = + variants.find(v => v.variant_key === keys.variantKey) ?? + (keys.variantKey === UNKNOWN_VARIANT_KEY + ? variants.find(v => v.variant_key === 'standard' && v.status !== 'archived') + : undefined); + // An archived variant takes no new listings once variants may be NULL; mode + // item keeps Tier 1's rule (it links to the existing row). + if (exact?.status === 'archived' && this.itemModel === 'nullable_variant') { + logger.info( + 'entity_match.variant_archived', + `Variant ${exact.variant_key} of ${item.itemKey} is archived: no variant`, + { itemId: item.id, masterId: exact.id } + ); + return { itemId: item.id, masterId: null, isNew: false, variantSource: null }; + } + if (exact) { + logger.info('entity_match.variant_exact', `Variant ${keys.variantKey} of ${item.itemKey}`, { + itemId: item.id, + masterId: exact.id, + }); + return { itemId: item.id, masterId: exact.id, isNew: false, variantSource }; + } + } else { + // 3. Size not stated. Multi-size = a live sized variant (design doc 4.2). + const live = variants.filter(v => v.status !== 'archived'); + const multiSize = live.some(v => isSizedVariantKey(v.variant_key)); + if (multiSize && this.itemModel === 'nullable_variant') { + logger.info( + 'entity_match.variant_unknown', + `Size not stated on multi-size Item ${item.itemKey}: no variant`, + { itemId: item.id } + ); + return { itemId: item.id, masterId: null, isNew: false, variantSource: null }; + } + const unknownVariant = live.find(v => v.variant_key === UNKNOWN_VARIANT_KEY); + const only = multiSize + ? unknownVariant + : (unknownVariant ?? + live.find(v => v.variant_key === 'standard') ?? + (live.length === 1 ? live[0] : undefined)); + if (only) { + logger.info( + 'entity_match.variant_size_not_stated', + `Size not stated: variant ${only.variant_key} of ${item.itemKey}`, + { itemId: item.id, masterId: only.id, multiSize } + ); + return { itemId: item.id, masterId: only.id, isNew: false, variantSource: null }; + } + targetVariantKey = UNKNOWN_VARIANT_KEY; + // A retired `unknown` variant (M3) is never seeded again: the upsert would + // adopt the archived row. + if (variants.some(v => v.variant_key === UNKNOWN_VARIANT_KEY)) { + if (this.itemModel === 'nullable_variant') { + return { itemId: item.id, masterId: null, isNew: false, variantSource: null }; + } + throw new Error( + `[EntityMatcher] ${item.itemKey}:unknown is archived; RATE_ITEM_MODEL=item cannot place a listing with no stated size` + ); + } + } + + // A fuzzy Item hit only links to variants that exist, as Tier 2 only links to + // existing masters: the listing's own name and model may not be the Item's + // ("iPhone 15 Pro Max" ~ "iPhone 15 Pro"), so a new variant gets its own key. + if (fuzzyItem) return seedOwnKey(); + + // Missing variant of a known Item: create it under the Item's own key. + const variantModelKey = composeModelKey(item.itemKey, targetVariantKey); + const seeded = await this.seedMasterProduct( + { ...candidate, model_key: variantModelKey }, + variantModelKey, + safeCategoryId, + safeRawTitle, + ambiguousFuzzy + ); + logger.info( + 'entity_match.variant_created', + `Variant ${targetVariantKey} of ${item.itemKey}: ${seeded.masterId}`, + { itemId: item.id, masterId: seeded.masterId, isNew: seeded.isNew } + ); + return { itemId: item.id, masterId: seeded.masterId, isNew: seeded.isNew, variantSource }; + } + + private async findItem(column: 'id' | 'item_key', value: string): Promise<MatchedItem | null> { + const { data, error } = await this.supabase + .from('items') + .select('id, item_key') + .eq(column, value) + .maybeSingle(); + if (error) { + throw new Error(`[EntityMatcher] Item lookup by ${column} failed: ${error.message}`); + } + const row = data as { id?: unknown; item_key?: unknown } | null; + const id = sanitizeMatcherEntityId(row?.id, 128); + return id ? { id, itemKey: typeof row?.item_key === 'string' ? row.item_key : '' } : null; + } + + /** Item Tier 2: pg_trgm on Item names through `rate.match_item` (design doc 4.5). */ + private async fuzzyMatchItem( + candidate: NormalizedProductCandidate + ): Promise<{ item: MatchedItem | null; ambiguous: boolean }> { + const searchTitle = + typeof candidate.item_name === 'string' && candidate.item_name.trim() + ? candidate.item_name + : `${candidate.brand} ${candidate.model}`; + const { data, error } = await this.supabase.rpc('match_item', { + search_title: searchTitle, + p_brand: candidate.brand, + p_model: candidate.model, + p_threshold: ENTITY_MATCH_CANDIDATE_FLOOR, + }); + // As in Tier 2: an RPC failure aborts instead of seeding a duplicate Item. + if (error) { + throw new Error(`[EntityMatcher] Item Tier 2 fuzzy match failed: ${error.message}`); + } + const rows = Array.isArray(data) + ? data.filter(m => { + if (!m || typeof m !== 'object' || Array.isArray(m)) return false; + if (!sanitizeMatcherEntityId(m.item_id, 128)) return false; + const sim = typeof m.similarity === 'boolean' ? Number.NaN : Number(m.similarity); + return Number.isFinite(sim) && sim >= ENTITY_MATCH_CANDIDATE_FLOOR && sim <= 1; + }) + : []; + const top = rows[0]; + if (!top) return { item: null, ambiguous: false }; + const sim = Number(top.similarity); + if (sim < ENTITY_MATCH_ACCEPT_THRESHOLD) { + logger.warn( + 'entity_match.item_tier2_ambiguous', + `Item Tier 2 best similarity ${(sim * 100).toFixed(1)}% < ${(ENTITY_MATCH_ACCEPT_THRESHOLD * 100).toFixed(0)}%: seeding as draft if new.`, + { similarity: sim, modelKey: candidate.model_key } + ); + return { item: null, ambiguous: true }; + } + const item = { + id: sanitizeMatcherEntityId(top.item_id, 128), + itemKey: typeof top.item_key === 'string' ? top.item_key : '', + }; + logger.info( + 'entity_match.item_tier2_fuzzy', + `Item Tier 2: "${searchTitle}" ~= "${top.name}" (similarity: ${(sim * 100).toFixed(1)}%)`, + { itemId: item.id, similarity: sim } + ); + return { item, ambiguous: false }; + } + + private async listItemVariants( + itemId: string + ): Promise<Array<{ id: string; variant_key: string; status: string | null }>> { + const { data, error } = await this.supabase + .from('master_products') + .select('id, variant_key, status') + .eq('item_id', itemId) + .limit(200); + if (error) { + throw new Error(`[EntityMatcher] Variant lookup for Item ${itemId} failed: ${error.message}`); + } + return (Array.isArray(data) ? data : []).flatMap(row => { + const id = sanitizeMatcherEntityId(row?.id, 128); + return id && typeof row.variant_key === 'string' + ? [{ id, variant_key: row.variant_key, status: row.status ?? null }] + : []; + }); + } + + /** The Item the M1 sync trigger linked to a master; fails loudly when M1 is missing. */ + private async itemIdOfMaster(masterId: string): Promise<string> { + const { data, error } = await this.supabase + .from('master_products') + .select('item_id') + .eq('id', masterId) + .maybeSingle(); + if (error) { + throw new Error( + `[EntityMatcher] item_id lookup for master ${masterId} failed: ${error.message}` + ); + } + const itemId = sanitizeMatcherEntityId((data as { item_id?: unknown } | null)?.item_id, 128); + if (!itemId) { + throw new Error( + `[EntityMatcher] master ${masterId} has no item_id: the Item migration (DOS/Rate#1150) is not applied` + ); + } + return itemId; } /** @@ -150,26 +469,9 @@ export class EntityMatcher { categoryId?: string, rawTitle?: string ): Promise<{ masterId: string; isNew: boolean }> { - if (!candidate || typeof candidate !== 'object' || Array.isArray(candidate)) { - throw new Error('[EntityMatcher] candidate must be a non-null object'); - } - const safeModelKey = String(candidate.model_key ?? '') - .replace(/\p{Cc}+/gu, '') - .trim() - .slice(0, 200); - if (!safeModelKey) { - throw new Error('[EntityMatcher] candidate.model_key must be a non-empty string'); - } - const safeCategoryId = sanitizeMatcherEntityId(categoryId, 128) || undefined; - const safeRawTitle = - typeof rawTitle === 'string' - ? rawTitle - .replace(/\p{Cc}+/gu, ' ') - .replace(/\s+/g, ' ') - .trim() - .slice(0, 500) - : undefined; - candidate = { ...candidate, model_key: safeModelKey }; + const input = this.sanitizeCandidateInput(candidate, categoryId, rawTitle); + candidate = input.candidate; + const { safeModelKey, safeCategoryId, safeRawTitle } = input; // Tier 1: Deterministic model_key matching const { data: exactMatch, error: exactErr } = await this.supabase @@ -246,6 +548,63 @@ export class EntityMatcher { ); } + return this.seedMasterProduct( + candidate, + safeModelKey, + safeCategoryId, + safeRawTitle, + ambiguousFuzzy + ); + } + + /** Validates and bounds the matcher input; throws on a missing candidate or key. */ + private sanitizeCandidateInput( + candidate: NormalizedProductCandidate, + categoryId?: string, + rawTitle?: string + ): { + candidate: NormalizedProductCandidate; + safeModelKey: string; + safeCategoryId: string | undefined; + safeRawTitle: string | undefined; + } { + if (!candidate || typeof candidate !== 'object' || Array.isArray(candidate)) { + throw new Error('[EntityMatcher] candidate must be a non-null object'); + } + const safeModelKey = String(candidate.model_key ?? '') + .replace(/\p{Cc}+/gu, '') + .trim() + .slice(0, 200); + if (!safeModelKey) { + throw new Error('[EntityMatcher] candidate.model_key must be a non-empty string'); + } + if (candidate.keyable === false) { + throw new UnkeyableProductTitleError(rawTitle ?? candidate.name); + } + const safeCategoryId = sanitizeMatcherEntityId(categoryId, 128) || undefined; + const safeRawTitle = + typeof rawTitle === 'string' + ? rawTitle + .replace(/\p{Cc}+/gu, ' ') + .replace(/\s+/g, ' ') + .trim() + .slice(0, 500) + : undefined; + candidate = { ...candidate, model_key: safeModelKey }; + return { candidate, safeModelKey, safeCategoryId, safeRawTitle }; + } + + /** + * Tier 3: no existing master matches. Enriches the candidate (AI Spec Enricher, + * unless it is in the local benchmark registry) and seeds a new master. + */ + private async seedMasterProduct( + candidate: NormalizedProductCandidate, + safeModelKey: string, + safeCategoryId: string | undefined, + safeRawTitle: string | undefined, + ambiguousFuzzy: boolean + ): Promise<{ masterId: string; isNew: boolean }> { // Tier 3: No existing master product matches -> Enrich with AI Spec Enricher (if not in local benchmark registry) // and seed a new Master Product. candidate = await this.enricher.enrichCandidate(candidate, safeRawTitle); @@ -298,54 +657,60 @@ export class EntityMatcher { .replace(/[^a-z0-9-]+/gi, '-') .toLowerCase() .slice(0, 300); - const baseInsertRow = { - category_id: safeCategoryId || null, - brand: - String(candidate.brand ?? '') - .replace(/\p{Cc}+/gu, ' ') - .trim() - .slice(0, 100) || 'Unknown', - model: - String(candidate.model ?? '') - .replace(/\p{Cc}+/gu, ' ') - .trim() - .slice(0, 200) || 'Unknown', - variant_name: candidate.variant_name - ? String(candidate.variant_name) + // Only the spec fields the category has: no screen or RAM on earbuds, no + // 5G on a watch (DOS/Rate#1142). + const baseInsertRow = applyCategorySpecTemplate( + candidate.category_slug, + { + category_id: safeCategoryId || null, + brand: + String(candidate.brand ?? '') + .replace(/\p{Cc}+/gu, ' ') + .trim() + .slice(0, 100) || 'Unknown', + model: + String(candidate.model ?? '') .replace(/\p{Cc}+/gu, ' ') .trim() - .slice(0, 200) || null - : null, - model_key: - String(candidate.model_key ?? '') - .replace(/\p{Cc}+/gu, '') - .trim() - .slice(0, 200) || safeModelKey, - name: - String(candidate.name ?? '') - .replace(/\p{Cc}+/gu, ' ') - .trim() - .slice(0, 300) || safeModelKey, - slug: safeSlug, - description: - typeof candidate.description === 'string' - ? candidate.description + .slice(0, 200) || 'Unknown', + variant_name: candidate.variant_name + ? String(candidate.variant_name) .replace(/\p{Cc}+/gu, ' ') .trim() - .slice(0, 10_000) - : '', - ram_gb: toBoundedPositiveInt(candidate.ram_gb, MAX_RAM_GB), - storage_gb: toBoundedPositiveInt(candidate.storage_gb, MAX_STORAGE_GB), - screen_size_inch: toBoundedPositiveNumber(candidate.screen_size_inch, MAX_SCREEN_SIZE_INCH), - battery_mah: toBoundedPositiveInt(candidate.battery_mah, MAX_BATTERY_MAH), - has_5g: isStrictBoolean(candidate.has_5g), - specifications: candidate.specifications, - key_specs: candidate.key_specs, - images: safeImages, - thumbnail: safeImages[0] || null, - colors: safeColors, - status: seedStatus, - }; + .slice(0, 200) || null + : null, + model_key: + String(candidate.model_key ?? '') + .replace(/\p{Cc}+/gu, '') + .trim() + .slice(0, 200) || safeModelKey, + name: + String(candidate.name ?? '') + .replace(/\p{Cc}+/gu, ' ') + .trim() + .slice(0, 300) || safeModelKey, + slug: safeSlug, + description: + typeof candidate.description === 'string' + ? candidate.description + .replace(/\p{Cc}+/gu, ' ') + .trim() + .slice(0, 10_000) + : '', + ram_gb: toBoundedPositiveInt(candidate.ram_gb, MAX_RAM_GB), + storage_gb: toBoundedPositiveInt(candidate.storage_gb, MAX_STORAGE_GB), + screen_size_inch: toBoundedPositiveNumber(candidate.screen_size_inch, MAX_SCREEN_SIZE_INCH), + battery_mah: toBoundedPositiveInt(candidate.battery_mah, MAX_BATTERY_MAH), + has_5g: isStrictBoolean(candidate.has_5g), + specifications: candidate.specifications, + key_specs: candidate.key_specs, + images: safeImages, + thumbnail: safeImages[0] || null, + colors: safeColors, + status: seedStatus, + }, + { forStorage: true } + ); let { data: createdRows, error: createErr } = await this.supabase .from('master_products') @@ -525,19 +890,30 @@ export class EntityMatcher { const { error: updateErr } = await this.supabase .from('master_products') - .update({ - ram_gb: existingRamGb ?? toBoundedPositiveInt(aiResult.ram_gb, MAX_RAM_GB), - storage_gb: existingStorageGb ?? toBoundedPositiveInt(aiResult.storage_gb, MAX_STORAGE_GB), - screen_size_inch: - existingScreenInch ?? - toBoundedPositiveNumber(aiResult.screen_size_inch, MAX_SCREEN_SIZE_INCH), - battery_mah: - existingBatteryMah ?? toBoundedPositiveInt(aiResult.battery_mah, MAX_BATTERY_MAH), - has_5g: isStrictBoolean(row.has_5g) ? true : isStrictBoolean(aiResult.has_5g ?? row.has_5g), - specifications: mergedSpecifications, - key_specs: mergedKeySpecs, - updated_at: new Date().toISOString(), - }) + .update( + // Only a stored category scopes the write: a guess from the title + // could strip a phone's real RAM. + applyCategorySpecTemplate( + existingCategorySlug, + { + ram_gb: existingRamGb ?? toBoundedPositiveInt(aiResult.ram_gb, MAX_RAM_GB), + storage_gb: + existingStorageGb ?? toBoundedPositiveInt(aiResult.storage_gb, MAX_STORAGE_GB), + screen_size_inch: + existingScreenInch ?? + toBoundedPositiveNumber(aiResult.screen_size_inch, MAX_SCREEN_SIZE_INCH), + battery_mah: + existingBatteryMah ?? toBoundedPositiveInt(aiResult.battery_mah, MAX_BATTERY_MAH), + has_5g: isStrictBoolean(row.has_5g) + ? true + : isStrictBoolean(aiResult.has_5g ?? row.has_5g), + specifications: mergedSpecifications, + key_specs: mergedKeySpecs, + updated_at: new Date().toISOString(), + }, + { forStorage: true } + ) + ) .eq('id', safeMasterId); if (updateErr) { diff --git a/apps/importer/src/shopee/item-model-flag.ts b/apps/importer/src/shopee/item-model-flag.ts new file mode 100644 index 000000000..373781053 --- /dev/null +++ b/apps/importer/src/shopee/item-model-flag.ts @@ -0,0 +1,27 @@ +/** + * Rollout flag for the Item / variant / listing model (DOS/Rate#1151; design doc + * `Docs/Proposals/2026-10-27-item-variant-listing-model.md`, sections 4.5 and 10). + * + * `RATE_ITEM_MODEL` selects how the importer resolves and writes a listing: + * - `off` (default): today's behaviour. Master-level matching only, nothing + * reads or writes `rate.items`, `item_id` or `variant_source`. Safe on a + * database without the DOS-Me M1+M2 migration (#1150). + * - `item`: item-first matching (`match_item`) and `item_id`/`variant_source` + * writes. Needs M1+M2. Every listing still gets a variant: a listing with + * no stated size on a multi-size Item goes to that Item's `unknown` variant. + * - `nullable_variant`: as `item`, and such a listing is written with + * `cluster_id = NULL`. Turn on only after DOS/Rate-Services#670 part 1 has + * merged, so every consumer treats a NULL variant as "no variant". + * + * Any other value reads as `off`, so a typo never starts writing new columns. + */ +export const ITEM_MODEL_ENV = 'RATE_ITEM_MODEL'; + +export const ITEM_MODEL_MODES = ['off', 'item', 'nullable_variant'] as const; +export type ItemModelMode = (typeof ITEM_MODEL_MODES)[number]; + +export function resolveItemModelMode(raw: unknown = process.env[ITEM_MODEL_ENV]): ItemModelMode { + if (typeof raw !== 'string') return 'off'; + const value = raw.trim().toLowerCase().replace(/-/g, '_'); + return (ITEM_MODEL_MODES as readonly string[]).includes(value) ? (value as ItemModelMode) : 'off'; +} diff --git a/apps/importer/src/shopee/jobs/item-inventory-report.ts b/apps/importer/src/shopee/jobs/item-inventory-report.ts new file mode 100644 index 000000000..7367ef0ad --- /dev/null +++ b/apps/importer/src/shopee/jobs/item-inventory-report.ts @@ -0,0 +1,328 @@ +/** + * Read-only Item migration inventory (DOS/Rate#1148, design doc + * `Docs/Proposals/2026-10-27-item-variant-listing-model.md` sections 7 and 7.3, phase M0). + * + * Usage: + * pnpm --filter @repo/crawlers exec tsx src/shopee/jobs/item-inventory-report.ts + * + * Reads `rate.categories`, `rate.master_products` and `rate.products` with the public anon + * key (SUPABASE_URL + SUPABASE_ANON_KEY, or the NEXT_PUBLIC_ pair the web uses) and prints + * a JSON report. Besides the SQL checks of section 7.3, it runs today's `ProductNormalizer` + * on every seed, master name and listing title, which SQL cannot do: that is how seed key + * drift shows up. + * + * Safety invariant: SELECTs only. There is no write path and no flag that adds one. + * Tables anon cannot read (`price_alerts`) are not queried; see the issue for the SQL. + */ +import { pathToFileURL } from 'node:url'; +import dotenv from 'dotenv'; +import { FLAGSHIP_DEVICES_TO_SEED, type FlagshipDeviceSeed } from '../flagship-seed-data.js'; +import { createImporterLogger } from '../logger.js'; +import { fetchAllPages, resolveBackfillSupabaseTarget } from '../paging.js'; +import { ProductNormalizer } from '../product-normalizer.js'; +import { scopeSupabaseToRate, type RateSupabaseClient, type ShopeeProduct } from '../types.js'; + +const logger = createImporterLogger('ItemInventoryReport'); + +export interface InventoryMaster { + id: string; + model_key: string; + slug: string; + name: string; + brand: string | null; + status: string | null; + category_id: string | null; + storage_gb: number | null; +} + +export interface InventoryListing { + id: string; + cluster_id: string | null; + title: string | null; + platform: string | null; + platform_item_id: string | null; + price_min: number | string | null; + price_max: number | string | null; +} + +export interface InventoryCategory { + id: string; + slug: string; +} + +/** Section 7.3 shape check: three `:`-separated segments. */ +const KEY_SHAPE = /^[a-z0-9][a-z0-9-]*:[a-z0-9][a-z0-9.-]*:[a-z0-9-]+$/; +/** The suffix `rate.item_slug_from_master` strips (section 7.1). */ +const VARIANT_SLUG_SUFFIX = /-(\d+(gb|tb)|standard|unknown)$/; +/** The trailing size `rate.item_name_from_master` strips (section 7.1). */ +const VARIANT_NAME_SUFFIX = /\s\d+\s?(gb|tb)$/i; +/** Categories with no storage axis (section 4.2). */ +const SIZELESS_CATEGORIES = new Set(['earbuds']); + +/** `split_part(k, ':', 1) || ':' || split_part(k, ':', 2)`, as the M2 backfill groups. */ +export const itemKeyOf = (modelKey: string): string => { + const [brand = '', model = ''] = modelKey.split(':'); + return `${brand}:${model}`; +}; +export const variantKeyOf = (modelKey: string): string => modelKey.split(':')[2] ?? ''; +export const itemSlugOf = (slug: string): string => slug.replace(VARIANT_SLUG_SUFFIX, ''); +export const itemNameOf = (name: string): string => name.replace(VARIANT_NAME_SUFFIX, ''); + +/** What the normalizer keys a bare title as, with no marketplace SKUs to help it. */ +export function normalizedKeyOf(normalizer: ProductNormalizer, title: string): string { + const raw: ShopeeProduct = { + itemId: 'inventory', + shopId: 'inventory', + url: 'https://rate.vn', + title, + priceMin: 0, + priceMax: 0, + currency: 'VND', + historicalSold: 0, + ratingStar: 0, + ratingCount: 0, + images: [], + models: [], + attributes: [], + shop: { shopId: 'inventory', name: 'inventory', isMall: false }, + reviews: [], + }; + return normalizer.normalize(raw).model_key; +} + +export function buildInventoryReport(input: { + categories: InventoryCategory[]; + masters: InventoryMaster[]; + listings: InventoryListing[]; + seeds?: FlagshipDeviceSeed[]; + normalizer?: ProductNormalizer; +}) { + const normalizer = input.normalizer ?? new ProductNormalizer(); + const seeds = input.seeds ?? FLAGSHIP_DEVICES_TO_SEED; + const categorySlug = new Map(input.categories.map(c => [c.id, c.slug])); + const masters = input.masters; + const byId = new Map(masters.map(m => [m.id, m])); + const byKey = new Map(masters.map(m => [m.model_key, m])); + + const items = new Map<string, InventoryMaster[]>(); + for (const m of masters) { + const list = items.get(itemKeyOf(m.model_key)) ?? []; + list.push(m); + items.set(itemKeyOf(m.model_key), list); + } + + const counts = { + total: masters.length, + published: masters.filter(m => m.status === 'published').length, + draft: masters.filter(m => m.status === 'draft').length, + archived: masters.filter(m => m.status === 'archived').length, + null_status: masters.filter(m => m.status === null).length, + bad_shape: masters.filter(m => !KEY_SHAPE.test(m.model_key)).length, + khc_keys: masters.filter(m => m.model_key.startsWith('khc:')).length, + unknown_masters: masters.filter(m => variantKeyOf(m.model_key) === 'unknown').length, + standard_masters: masters.filter(m => variantKeyOf(m.model_key) === 'standard').length, + items: items.size, + multi_variant_items: [...items.values()].filter(v => v.length > 1).length, + listings: input.listings.length, + listings_without_master: input.listings.filter(p => !p.cluster_id).length, + listings_on_missing_master: input.listings.filter(p => p.cluster_id && !byId.has(p.cluster_id)) + .length, + }; + + const legacyKeys = masters + .filter(m => !KEY_SHAPE.test(m.model_key) || m.model_key.startsWith('khc:')) + .map(m => ({ model_key: m.model_key, status: m.status })); + + const itemConflicts = [...items.entries()] + .filter( + ([, v]) => + // count(DISTINCT ...) ignores NULLs, so a missing brand or category is not a conflict. + new Set(v.map(m => m.brand).filter(b => b !== null)).size > 1 || + new Set(v.map(m => m.category_id).filter(c => c !== null)).size > 1 + ) + .map(([item_key, v]) => ({ item_key, variants: v.map(m => m.model_key) })); + + const sizelessWithSize = masters + .filter( + m => + SIZELESS_CATEGORIES.has(categorySlug.get(m.category_id ?? '') ?? '') && + variantKeyOf(m.model_key) !== 'unknown' + ) + .map(m => ({ model_key: m.model_key, status: m.status, storage_gb: m.storage_gb })); + + const seedDrift = seeds + .map(s => { + const normalized = normalizedKeyOf(normalizer, s.name); + const stored = byKey.get(s.model_key); + return { + seed_key: s.model_key, + normalized_key: normalized, + stored_status: stored?.status ?? null, + normalized_key_stored: byKey.has(normalized), + }; + }) + .filter(d => d.seed_key !== d.normalized_key); + + const masterDrift = masters + .map(m => ({ + model_key: m.model_key, + normalized_key: normalizedKeyOf(normalizer, m.name), + status: m.status, + })) + .filter(d => d.model_key !== d.normalized_key); + + // Listings whose own title keys elsewhere: another Item, the same Item at another size, or + // the same Item with no size stated (the listings that lose their variant on multi-size Items). + const listingDrift = input.listings.flatMap(p => { + const master = p.cluster_id ? byId.get(p.cluster_id) : undefined; + if (!master || !p.title) return []; + const normalized = normalizedKeyOf(normalizer, p.title); + if (normalized === master.model_key) return []; + return [ + { + kind: + itemKeyOf(normalized) !== itemKeyOf(master.model_key) + ? 'item' + : variantKeyOf(normalized) === 'unknown' + ? 'size_not_stated' + : 'variant', + master_key: master.model_key, + normalized_key: normalized, + platform: p.platform, + platform_item_id: p.platform_item_id, + title: p.title.slice(0, 120), + }, + ]; + }); + + // Item slugs from the section 7.1 helper. Two Items on one slug abort the M2 backfill. This + // also covers an Item slug that is another Item's variant slug: that variant slug carries no + // size suffix, so it strips to itself. An Item whose masters strip to different slugs gets + // whichever one the backfill's representative master has, and a slug that still ends in a + // size after stripping fails the post-M2 check of section 7.3. + const itemSlugOwners = new Map<string, Set<string>>(); + const itemSlugsByItem = new Map<string, Set<string>>(); + for (const m of masters) { + const slug = itemSlugOf(m.slug); + const owners = itemSlugOwners.get(slug) ?? new Set<string>(); + owners.add(itemKeyOf(m.model_key)); + itemSlugOwners.set(slug, owners); + const slugs = itemSlugsByItem.get(itemKeyOf(m.model_key)) ?? new Set<string>(); + slugs.add(slug); + itemSlugsByItem.set(itemKeyOf(m.model_key), slugs); + } + const slugCollisions = [ + ...[...itemSlugOwners.entries()] + .filter(([, owners]) => owners.size > 1) + .map(([slug, owners]) => ({ kind: 'two_items', slugs: [slug], items: [...owners] })), + ...[...itemSlugsByItem.entries()] + .filter(([, slugs]) => slugs.size > 1) + .map(([item, slugs]) => ({ kind: 'item_slug_differs', slugs: [...slugs], items: [item] })), + ...[...itemSlugOwners.entries()] + .filter(([slug]) => /-\d+(gb|tb)$/.test(slug)) + .map(([slug, owners]) => ({ kind: 'still_sized', slugs: [slug], items: [...owners] })), + ]; + + // Broader than the section 7.3 check (a size anywhere, not only at the end), so a name such as + // "12GB RAM" is flagged too; those rows are read by hand. + const sizedItemNames = masters + .filter(m => /\d+\s?(gb|tb)\b/i.test(itemNameOf(m.name))) + .map(m => ({ model_key: m.model_key, name: m.name })); + + const storageMismatch = masters + .filter(m => { + const v = variantKeyOf(m.model_key); + // `1tb` and suffixed keys such as `256gb-cellular` (section 4.2) are sized too. + const sized = /^(\d+)(gb|tb)(?:-|$)/.exec(v); + if (!sized) return m.storage_gb !== null; + return Number(sized[1]) * (sized[2] === 'tb' ? 1024 : 1) !== m.storage_gb; + }) + .map(m => ({ model_key: m.model_key, storage_gb: m.storage_gb })); + + const multiSkuListings = input.listings + .filter( + p => + p.price_min !== null && p.price_max !== null && Number(p.price_min) !== Number(p.price_max) + ) + .map(p => ({ + master_key: p.cluster_id ? (byId.get(p.cluster_id)?.model_key ?? null) : null, + platform: p.platform, + platform_item_id: p.platform_item_id, + price_min: Number(p.price_min), + price_max: Number(p.price_max), + })); + + return { + counts, + legacy_keys: legacyKeys, + item_conflicts: itemConflicts, + sizeless_with_size: sizelessWithSize, + seed_drift: seedDrift, + master_drift: masterDrift, + listing_drift: listingDrift, + slug_collisions: slugCollisions, + sized_item_names: sizedItemNames, + storage_mismatch: storageMismatch, + multi_sku_listings: multiSkuListings, + }; +} + +export async function loadInventory(supabase: RateSupabaseClient) { + const categories = await fetchAllPages<InventoryCategory>('categories', (from, to) => + supabase.from('categories').select('id, slug').order('id').range(from, to) + ); + const masters = await fetchAllPages<InventoryMaster>('master_products', (from, to) => + supabase + .from('master_products') + .select('id, model_key, slug, name, brand, status, category_id, storage_gb') + .order('id') + .range(from, to) + ); + const listings = await fetchAllPages<InventoryListing>('products', (from, to) => + supabase + .from('products') + .select('id, cluster_id, title, platform, platform_item_id, price_min, price_max') + .order('id') + .range(from, to) + ); + return { categories, masters, listings }; +} + +const isDirectExecution = + typeof process !== 'undefined' && + Array.isArray(process.argv) && + Boolean(process.argv[1]) && + import.meta.url === pathToFileURL(process.argv[1]!).href; + +if (isDirectExecution) { + dotenv.config({ quiet: true }); + dotenv.config({ path: '../../.env', quiet: true }); + (async () => { + const { createClient } = await import('@supabase/supabase-js'); + const rawUrl = process.env.SUPABASE_URL || process.env.NEXT_PUBLIC_SUPABASE_URL || ''; + // The anon key on purpose: the report must see what any visitor sees, and nothing more. + const key = ( + process.env.SUPABASE_ANON_KEY || + process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY || + '' + ).trim(); + if (!rawUrl.trim() || !key) { + throw new Error('Missing SUPABASE_URL or SUPABASE_ANON_KEY in environment.'); + } + const { url, host } = resolveBackfillSupabaseTarget(rawUrl); + const supabase = scopeSupabaseToRate( + createClient(url, key, { auth: { persistSession: false } }) + ); + // stderr, so stdout stays one JSON document. + process.stderr.write(`Target: ${host} (read-only) +`); + const report = buildInventoryReport(await loadInventory(supabase)); + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); + })().catch((err: unknown) => { + logger.error( + 'inventory.fatal', + `Item inventory report failed: ${err instanceof Error ? err.message : String(err)}` + ); + process.exit(1); + }); +} diff --git a/apps/importer/src/shopee/product-normalizer.ts b/apps/importer/src/shopee/product-normalizer.ts index ed689baa8..a0eb635b2 100644 --- a/apps/importer/src/shopee/product-normalizer.ts +++ b/apps/importer/src/shopee/product-normalizer.ts @@ -1,4 +1,10 @@ -import { MAX_VALID_VND_PRICE, type ProductSpecifications, type KeySpecs } from '@repo/shared-data'; +import { + MAX_VALID_VND_PRICE, + UNKNOWN_VARIANT_KEY, + type ProductSpecifications, + type KeySpecs, + type VariantSource, +} from '@repo/shared-data'; import type { ShopeeProduct } from './types.js'; export const MAX_VALID_RAM_GB = 2048; @@ -59,6 +65,31 @@ export interface NormalizedProductCandidate { model: string; variant_name: string; model_key: string; + /** + * Item fields (DOS/Rate#1151). `normalize()` always sets them; they are optional + * so candidates built elsewhere stay valid, and the matcher derives the keys + * from `model_key` when they are missing. + * `item_key` = `brand:model`, the first two `model_key` segments. + */ + item_key?: string; + /** The literal third `model_key` segment: `256gb`, `1024gb`, `unknown`. */ + variant_key?: string; + /** Item name and slug, never with a storage size. */ + item_name?: string; + item_slug?: string; + /** + * The listing's variant is known: a stated or single-size storage, or a line + * with no storage axis (earbuds). False only for "storage not stated". + */ + variant_known?: boolean; + /** How the variant was found; null when it is not known. */ + variant_source?: Extract<VariantSource, 'title' | 'single_size' | 'single_variant'> | null; + /** + * False when the title gives no model segment. The key then carries the + * `unkeyed` placeholder model instead of an empty segment (`samsung::unknown`), + * and the pipeline and matcher reject the candidate (UnkeyableProductTitleError). + */ + keyable?: boolean; name: string; slug: string; storage_gb: number | null; // null = unparseable from the title; identity carries ':unknown' @@ -83,6 +114,24 @@ export interface NormalizedProductCandidate { }; } +/** Model segment of a title that names no model (`keyable: false`). */ +export const UNKEYED_MODEL_SEGMENT = 'unkeyed'; + +/** + * A title with no model to key on (e.g. it would key as `samsung::unknown`). The + * listing has no identity to match, so it is rejected before anything is written + * (DOS/Rate#1151). `normalize()` itself never throws; it marks `keyable: false`. + */ +export class UnkeyableProductTitleError extends Error { + readonly code = 'UNKEYABLE_PRODUCT_TITLE'; + constructor(title: string) { + super( + `Cannot key product title (no model segment): ${JSON.stringify(String(title ?? '').slice(0, 200))}` + ); + this.name = 'UnkeyableProductTitleError'; + } +} + /** * Detect listing physical condition from product title and optional description (Audit C6). * Strips promotional trade-in phrases ("thu cũ đổi mới") before matching so new retail listings @@ -1107,6 +1156,8 @@ const sizePatterns = (size: string): [RegExp, RegExp] => [ const WATCH_BAND_WORD = /(?<!kh[\p{L}\p{M}]{1,3}ng )(?:d(?:\u00e2|a\u0302)y|band|strap|loop|bracelet)(?![\p{L}\d])/iu; const WATCH_SIZE = /(?<!\d)(38|4[0-2]|4[4-6]) ?mm\b/i; +const LIST_WATCH_SIZE = /^ ?(?:38|4[0-2]|4[4-6]) ?mm\b/i; +const WATCH_CASE_WORD = /(?<![\p{L}\d])(?:case|mặt|vỏ)(?![\p{L}\d])/iu; const APPLE_SIZE_PATTERNS: Record<string, [RegExp, RegExp]> = { ipad: sizePatterns(IPAD_SIZE), 'macbook air': sizePatterns(MACBOOK_AIR_SIZE), @@ -1140,11 +1191,22 @@ function readAppleSize(model: string, title: string): string { // next ")", ",", "|", ";", en dash or " - ") is the case's. A band is sold by the case // it fits ("Dây Cao Su 46mm"), so a band's size counts only when the title names no // other: "dây Sport Band 45mm cho mặt 42mm" does not tell the watch's (DOS/Rate#1124). - const caseSize = own - .split(/[),|;\u2013]| - /) - .map(part => part.split(WATCH_BAND_WORD)[0]) - .join(' ') - .match(WATCH_SIZE)?.[1]; + // A band phrase with a size runs on over the parts after it that start with a size + // and follow "," or "|": "Sport Loop 42mm | 46mm" or "Dây 42mm S/M, 46mm M/L" names + // two band sizes, not a case (DOS/Rate#1137). A part that names the case ("46mm, 42mm + // Aluminum Case") ends the list. + const parts = own.split(/([),|;\u2013]| - )/); + const outside: string[] = []; + let bandList = false; + for (let i = 0; i < parts.length; i += 2) { + const part = parts[i]!; + const listed = LIST_WATCH_SIZE.test(part) && !WATCH_CASE_WORD.test(part.normalize('NFC')); + if (bandList && /^[,|]$/.test(parts[i - 1]!) && listed) continue; + const [before, ...band] = part.split(WATCH_BAND_WORD); + outside.push(before!); + bandList = WATCH_SIZE.test(band.join(' ')); + } + const caseSize = outside.join(' ').match(WATCH_SIZE)?.[1]; if (caseSize) return `${caseSize}mm`; const bandSize = own.match(WATCH_SIZE)?.[1]; const sizes = new Set( @@ -1619,11 +1681,13 @@ export class ProductNormalizer { let storageGb: number | null = null; let storageConfident = false; + let variantSource: NormalizedProductCandidate['variant_source'] = null; const marker = romExplicit ?? romLoose; if (categorySlug === 'earbuds') { // Earbuds have no storage variant and must key as :unknown with null storage (DOS/Rate#643, #657). storageGb = null; storageConfident = true; + variantSource = 'single_variant'; } else if (marker) { const candidateStorage = toValidStorageGb( parseFloat(marker[1].replace(',', '.')) * (marker[2].toUpperCase() === 'TB' ? 1024 : 1) @@ -1647,8 +1711,10 @@ export class ProductNormalizer { } else if (storageCandidates.length === 0 && benchmark?.only_storage_gb != null) { storageGb = benchmark.only_storage_gb; storageConfident = true; + variantSource = 'single_size'; } const storageKnown = storageGb !== null; + if (storageKnown && variantSource === null) variantSource = 'title'; // 4. Extract Colors from models / variations const colors: string[] = []; @@ -1682,7 +1748,13 @@ export class ProductNormalizer { // 5. Construct keys & slugs. Unknown storage keeps identity space explicit // (':unknown') instead of guessing a default (plan v7 §5.2). - const modelKey = storageKnown ? `${baseKey}:${storageGb}gb` : `${baseKey}:unknown`; + // An empty key segment ("samsung::unknown") is never emitted: a title with no + // model keys on a placeholder and is marked unkeyable (DOS/Rate#1151). + const keyable = Boolean(slugify(brand) && slugify(model)); + const itemKey = keyable ? baseKey : `${slugify(brand) || 'khac'}:${UNKEYED_MODEL_SEGMENT}`; + const variantKey = storageKnown ? `${storageGb}gb` : UNKNOWN_VARIANT_KEY; + const modelKey = `${itemKey}:${variantKey}`; + const itemName = `${brand} ${model}`.trim(); const storageDisplay = categorySlug === 'earbuds' ? 'Standard' @@ -2062,10 +2134,17 @@ export class ProductNormalizer { model, variant_name: storageDisplay, model_key: modelKey, + item_key: itemKey, + variant_key: variantKey, + item_name: itemName, + item_slug: slugify(itemName), + variant_known: storageKnown || categorySlug === 'earbuds', + variant_source: variantSource, + keyable, name: canonicalName, slug, storage_gb: storageKnown ? storageGb : null, - needs_review: !storageConfident || condition !== 'new', + needs_review: !keyable || !storageConfident || condition !== 'new', condition, origin, ram_gb: ramGb, diff --git a/apps/importer/src/shopee/shopee-to-supabase.ts b/apps/importer/src/shopee/shopee-to-supabase.ts index 8f21d0076..85ddf2b7b 100644 --- a/apps/importer/src/shopee/shopee-to-supabase.ts +++ b/apps/importer/src/shopee/shopee-to-supabase.ts @@ -16,8 +16,14 @@ import { type SupportedEcommercePlatform, type ValidatedScrapedProduct, } from './types.js'; -import { ProductNormalizer } from './product-normalizer.js'; -import { EntityMatcher, sanitizeHttpUrl, sanitizeHttpUrlList } from './entity-matcher.js'; +import { ProductNormalizer, UnkeyableProductTitleError } from './product-normalizer.js'; +import { + EntityMatcher, + sanitizeHttpUrl, + sanitizeHttpUrlList, + type ItemMatchResult, +} from './entity-matcher.js'; +import type { ItemModelMode } from './item-model-flag.js'; import { AiSpecEnricher, type AiSpecEnricherConfig } from './ai-spec-enricher.js'; import { AiReviewAnalyzer, type ReviewInputItem } from './ai-review-analyzer.js'; import { createImporterLogger } from './logger.js'; @@ -249,6 +255,8 @@ export interface ShopeeIngestionPipelineConfig extends AiSpecEnricherConfig { * Optional pre-scoped `rate` schema Supabase client for dependency injection / deterministic testing. */ supabaseClient?: RateSupabaseClient; + /** Item model rollout mode; defaults to the `RATE_ITEM_MODEL` env (see item-model-flag.ts). */ + itemModel?: ItemModelMode; } export interface IngestOptions { @@ -257,6 +265,12 @@ export interface IngestOptions { * otherwise the listing goes through entity matching as usual. */ masterIdHint?: string | null; + /** + * Item this listing is known to belong to, for a page that states no size + * (DOS/Rate#1151). Its variant still comes from the title. Ignored while + * `RATE_ITEM_MODEL` is `off`, and when `masterIdHint` resolves. + */ + itemIdHint?: string | null; /** * When true, skips Step 6 review re-analysis so price-only (affiliate) ingests * do not re-score stored reviews without reviewer metadata or pay an extra AI pass. @@ -299,7 +313,9 @@ export class ShopeeIngestionPipeline { this.normalizer = new ProductNormalizer(); this.enricher = new AiSpecEnricher(aiConfig); this.reviewAnalyzer = new AiReviewAnalyzer(aiConfig); - this.matcher = new EntityMatcher(this.supabase, this.enricher); + this.matcher = new EntityMatcher(this.supabase, this.enricher, { + itemModel: aiConfig?.itemModel, + }); } /** @@ -405,7 +421,10 @@ export class ShopeeIngestionPipeline { ): Promise<{ productId: string; merchantId: string; - masterId: string; + /** The listing's variant; null only in `RATE_ITEM_MODEL=nullable_variant` (no variant). */ + masterId: string | null; + /** The listing's Item; null while `RATE_ITEM_MODEL` is `off`. */ + itemId: string | null; platform: SupportedEcommercePlatform; }> { if (!options || typeof options !== 'object' || Array.isArray(options)) options = {}; @@ -431,6 +450,15 @@ export class ShopeeIngestionPipeline { shopId: product.shop.shopId, }); + // Normalize first: a title with no keyable brand or model is rejected + // (UnkeyableProductTitleError) before anything is written. + const candidate = this.normalizer.normalize(product); + // With a masterIdHint the title's key is not used; the matcher still rejects + // the candidate if the hint does not resolve. + if (candidate.keyable === false && !options.masterIdHint) { + throw new UnkeyableProductTitleError(product.title); + } + // 1. Upsert merchant const entityHash = hashEntity(`${platform}:${product.shop.shopId}`); const { data: merchant, error: merchantErr } = await this.supabase @@ -465,8 +493,7 @@ export class ShopeeIngestionPipeline { ); await report?.('merchant'); - // 2. Normalize Specs & Match / Seed Master Product - const candidate = this.normalizer.normalize(product); + // 2. Match / Seed Master Product const resolvedCategoryId = this.categoryIdBySlug.get(candidate.category_slug || 'smartphones') || this.smartphoneCategoryId; @@ -485,22 +512,64 @@ export class ShopeeIngestionPipeline { // A caller that already verified the master (coverage discovery, or a refresh of // a listing that is already clustered) keeps the listing on it instead of // re-matching every crawl, which can split it onto a new draft master. - let hinted: { masterId: string; isNew: boolean } | null = null; + const itemModel = this.matcher.itemModel; + let hinted: ItemMatchResult | null = null; + // undefined = leave the stored variant_source as it is. + let hintedVariantSource: ItemMatchResult['variantSource'] | undefined; + let safeItemIdHint = + typeof options.itemIdHint === 'string' + ? (sanitizeIngestionEntityId(options.itemIdHint, 128) ?? '') + : ''; + if (safeItemIdHint && itemModel === 'off') { + platLogger.warn('ingest.item_hint_ignored', 'itemIdHint ignored: RATE_ITEM_MODEL is off', { + itemIdHint: safeItemIdHint, + }); + safeItemIdHint = ''; + } const safeMasterIdHint = typeof options.masterIdHint === 'string' ? (sanitizeIngestionEntityId(options.masterIdHint, 128) ?? '') : ''; if (safeMasterIdHint) { - const { data: hintRow, error: hintErr } = await this.supabase + const { data: hintData, error: hintErr } = await this.supabase .from('master_products') - .select('id, specifications') + .select( + itemModel === 'off' + ? 'id, specifications' + : 'id, specifications, status, item_id, variant_key' + ) .eq('id', safeMasterIdHint) .maybeSingle(); if (hintErr) { throw new Error(`Failed to verify masterIdHint (${safeMasterIdHint}): ${hintErr.message}`); } - if (hintRow?.id) { - hinted = { masterId: hintRow.id, isNew: false }; + const hintRow = hintData as { + id?: string; + specifications?: Record<string, unknown> | null; + } | null; + const hintMeta = hintData as { + status?: unknown; + item_id?: unknown; + variant_key?: unknown; + } | null; + const hintItemId = + itemModel === 'off' ? null : sanitizeIngestionEntityId(hintMeta?.item_id, 128); + if (hintRow?.id && itemModel !== 'off' && hintMeta?.status === 'archived') { + // A retired master (an Item's `unknown` variant after M3) must not take the + // listing back: the hint maps to its Item, and the variant comes from the title. + platLogger.info( + 'ingest.master_hint_archived', + `masterIdHint ${hintRow.id} is archived: matching under its Item ${hintItemId ?? '(none)'}`, + { masterId: hintRow.id, itemId: hintItemId } + ); + if (hintItemId) safeItemIdHint = hintItemId; + } else if (hintRow?.id) { + hinted = { masterId: hintRow.id, itemId: hintItemId, isNew: false, variantSource: null }; + // The title's variant source holds only when it names the hinted variant. + hintedVariantSource = + candidate.variant_known && candidate.variant_key === hintMeta?.variant_key + ? (candidate.variant_source ?? null) + : undefined; const existingSpecs = (hintRow as { specifications?: Record<string, unknown> | null }) .specifications; if (existingSpecs !== undefined && !hasPopulatedSpecificationGroups(existingSpecs)) { @@ -517,25 +586,67 @@ export class ShopeeIngestionPipeline { } } } - const { masterId, isNew } = + const resolved = hinted ?? - (await this.matcher.resolveOrCreateMasterProduct( + (await this.matcher.resolveItemAndVariant( candidate, resolvedCategoryId || undefined, - product.title + product.title, + { itemIdHint: safeItemIdHint || null } )); + const { itemId, isNew } = resolved; + let masterId = resolved.masterId; + let variantSourceToWrite = hinted ? hintedVariantSource : resolved.variantSource; + // The variant the listing had before this crawl, so its stats follow a move. + let previousClusterId: string | null = null; + if (itemModel !== 'off' && !hinted) { + // A variant set by hand is evidence, never undone by a later page that just + // does not state the size (design doc 4.4): only listing evidence + // (variant_source set) may move it. + const { data: stored, error: storedErr } = await this.supabase + .from('products') + .select('cluster_id, variant_source') + .eq('platform', platform) + .eq('platform_item_id', product.itemId) + .maybeSingle(); + if (storedErr) { + throw new Error(`Failed to read the stored listing variant: ${storedErr.message}`); + } + const storedClusterId = sanitizeIngestionEntityId(stored?.cluster_id, 128); + previousClusterId = storedClusterId; + // A hand-set variant always stays. A variant backed by listing evidence stays + // when this crawl has none; a `legacy` or NULL-source link (e.g. a #1105 + // fuzzy link) does not. + const storedSource = stored?.variant_source; + const keepStored = + storedSource === 'manual' || + (!resolved.variantSource && + (storedSource === 'title' || storedSource === 'sku' || storedSource === 'single_size')); + if (storedClusterId && keepStored) { + masterId = storedClusterId; + variantSourceToWrite = undefined; + } + } platLogger.info( 'ingest.master_resolved', - `Master Product ${hinted ? 'LINKED (caller hint)' : isNew ? 'CREATED' : 'LINKED'}: ID = ${masterId}`, - { masterId, isNew, hinted: Boolean(hinted) } + `Master Product ${hinted ? 'LINKED (caller hint)' : isNew ? 'CREATED' : 'LINKED'}: ID = ${masterId ?? '(no variant)'}`, + { masterId, itemId, isNew, hinted: Boolean(hinted) } ); await report?.('master'); + // Item columns exist only once the DOS-Me Item migration is applied (DOS/Rate#1150). + const itemColumns: { item_id?: string; variant_source?: string | null } = {}; + if (itemModel !== 'off') { + if (itemId) itemColumns.item_id = itemId; + if (variantSourceToWrite !== undefined) itemColumns.variant_source = variantSourceToWrite; + } + // 3. Upsert product listing into rate.products (linked via cluster_id) const { data: savedProduct, error: productErr } = await this.supabase .from('products') .upsert( { + ...itemColumns, merchant_id: merchantId, cluster_id: masterId, category_id: resolvedCategoryId, @@ -576,15 +687,32 @@ export class ShopeeIngestionPipeline { }); // 3b. Refresh canonical master_products market stats (min_price, max_price, merchant_count) via SQL RPC (Audit C5) - if (typeof this.supabase.rpc === 'function') { + // A variant the listing just left is refreshed too, or it keeps the old price. + const statsMasterIds = [masterId, previousClusterId].filter( + (id, i, all): id is string => Boolean(id) && all.indexOf(id) === i + ); + for (const statsMasterId of typeof this.supabase.rpc === 'function' ? statsMasterIds : []) { const { error: marketStatsErr } = await this.supabase.rpc('refresh_master_market_stats', { - p_master_id: masterId, + p_master_id: statsMasterId, }); if (marketStatsErr) { platLogger.warn( 'ingest.market_stats_rpc_warn', `Could not refresh master market stats via RPC: ${marketStatsErr.message}`, - { masterId } + { masterId: statsMasterId } + ); + } + } + // Then the Item roll-up (DOS-Me M2 `refresh_item_stats`, design doc 4.1). + if (typeof this.supabase.rpc === 'function' && itemModel !== 'off' && itemId) { + const { error: itemStatsErr } = await this.supabase.rpc('refresh_item_stats', { + p_item_id: itemId, + }); + if (itemStatsErr) { + platLogger.warn( + 'ingest.item_stats_rpc_warn', + `Could not refresh Item stats via RPC: ${itemStatsErr.message}`, + { itemId } ); } } @@ -870,26 +998,29 @@ export class ShopeeIngestionPipeline { { productId } ); } else if (analysisInput.length > 0) { - const [{ data: masterRow }, { data: peerRows }] = await Promise.all([ - this.supabase - .from('master_products') - .select('min_price') - .eq('id', masterId) - .maybeSingle(), - (() => { - const q = this.supabase - .from('products') - .select('id, price_current, merchants(is_official)') - .eq('cluster_id', masterId) - .gt('price_current', 0); - const withLte = - typeof (q as unknown as { lte?: (col: string, val: number) => typeof q }).lte === - 'function' - ? q.lte('price_current', MAX_VALID_VND_PRICE) - : q; - return withLte.limit(100); - })(), - ]); + // No variant: no peer floor, since peers and min price are per variant. + const [{ data: masterRow }, { data: peerRows }] = !masterId + ? [{ data: null }, { data: null }] + : await Promise.all([ + this.supabase + .from('master_products') + .select('min_price') + .eq('id', masterId) + .maybeSingle(), + (() => { + const q = this.supabase + .from('products') + .select('id, price_current, merchants(is_official)') + .eq('cluster_id', masterId) + .gt('price_current', 0); + const withLte = + typeof (q as unknown as { lte?: (col: string, val: number) => typeof q }) + .lte === 'function' + ? q.lte('price_current', MAX_VALID_VND_PRICE) + : q; + return withLte.limit(100); + })(), + ]); const rawMasterMinPrice = masterRow?.min_price; const parsedMasterMinPrice = @@ -981,6 +1112,6 @@ export class ShopeeIngestionPipeline { `Product ingestion complete! (Master: ${masterId}, Product: ${productId})`, { masterId, productId, merchantId } ); - return { productId, merchantId, masterId, platform }; + return { productId, merchantId, masterId, itemId, platform }; } } diff --git a/apps/web/e2e/bait-price.spec.ts b/apps/web/e2e/bait-price.spec.ts index f6d4a6aad..0e33db37e 100644 --- a/apps/web/e2e/bait-price.spec.ts +++ b/apps/web/e2e/bait-price.spec.ts @@ -10,7 +10,8 @@ const PRODUCT = '/smartphones/e2e-bait-price-phone'; test('a bait-price listing does not set the lowest price', async ({ page }) => { expect((await page.goto(PRODUCT))?.status()).toBe(200); - // Title and hero price take the cheapest plausible offer (3,030,000). + // Title and hero price read the stored min_price, which leaves the bait + // listing out (3,030,000; DOS/Rate#802, #1141). await expect(page).toHaveTitle(/3\.030\.000/); await expect(page).not.toHaveTitle(/1\.250\.000/); await expect(page.locator('.rate-pdv-price-line')).toContainText('3.030.000'); diff --git a/apps/web/e2e/category-spec-template.spec.ts b/apps/web/e2e/category-spec-template.spec.ts new file mode 100644 index 000000000..bf811d4ea --- /dev/null +++ b/apps/web/e2e/category-spec-template.spec.ts @@ -0,0 +1,61 @@ +import { expect, test } from '@playwright/test'; + +// Each category shows its own spec template: an earbud stored with +// phone-shaped specs shows no screen, RAM, ROM or camera, and a watch never +// shows 5G (DOS/Rate#1142). The seeded bud carries every one of those values. +const BUD = '/earbuds/e2e-bud-002'; + +test('earbud page shows audio specs and no phone fields', async ({ page }) => { + await page.setViewportSize({ width: 1280, height: 900 }); + await page.goto(BUD); + const specs = page.locator('#specs-section'); + const headings = specs.locator('h3'); + await expect(headings.first()).toHaveText(/^1\. Âm thanh/); + await expect(headings).toHaveCount(7); + await expect(specs).toContainText('E2E 10.5mm driver'); + await expect(specs).toContainText('E2E ANC'); + for (const text of [ + 'Màn hình', + 'Bộ nhớ RAM', + 'Bộ nhớ trong', + 'Camera', + '60 Hz', + '300 ppi', + 'E2E Mic Array', + 'E2E Audio Engine', + '5G', + ]) { + await expect(specs).not.toContainText(text); + } + const main = page.locator('main'); + await expect(main).not.toContainText('1GB'); + await expect(main).not.toContainText('64GB'); + await expect(main).not.toContainText('1 inch'); + await expect(main).not.toContainText('E2E Mic Array'); +}); + +test('earbud card carries no RAM or storage chip', async ({ page }) => { + await page.goto('/earbuds?q=E2E%20Bud%20002'); + const card = page + .locator('a[href="/earbuds/e2e-bud-002"]') + .first() + .locator('xpath=ancestor::article[1]'); + await expect(card).toBeVisible(); + await expect(card).not.toContainText('GB'); + await expect(card).not.toContainText('5G'); +}); + +test('a watch page never shows 5G', async ({ page }) => { + // The seeded phone is 5G; read through the smartwatch template, the flag + // and the 5G group label are gone while the rest of the page still renders. + await page.goto('/smartwatches/iphone-16-pro-max-256gb'); + const meta = page.locator('.rate-pdv-meta'); + await expect(meta).toContainText('Apple'); + await expect(meta).not.toContainText('5G'); + const specs = page.locator('#specs-section'); + await expect(specs).toContainText('Mạng Di động (Cellular)'); + // The bands row stays, as 4G only. + await expect(specs).toContainText('Băng tần 4G'); + await expect(specs).not.toContainText('5G'); + await expect(specs).not.toContainText('Camera'); +}); diff --git a/apps/web/e2e/merchant-risk-badge.spec.ts b/apps/web/e2e/merchant-risk-badge.spec.ts new file mode 100644 index 000000000..c66b88db1 --- /dev/null +++ b/apps/web/e2e/merchant-risk-badge.spec.ts @@ -0,0 +1,32 @@ +import { expect, test } from '@playwright/test'; + +// The merchant page used to show a red "scam risk" badge and "risk alert" for +// any DOSafe score above 0, so every store looked like a scam while the home +// cards called the same 4/100 "no scam detected" (DOS/Rate#1140). +const LOW = '/merchants/11111111-1111-4111-8111-111111111140'; +const HIGH = '/merchants/11111111-1111-4111-8111-111111111160'; + +test('a 4/100 DOSafe store shows no scam warning', async ({ page }) => { + expect((await page.goto(LOW))?.status()).toBe(200); + const badge = page.getByTestId('merchant-scam-badge'); + await expect(badge).toHaveText('Rủi ro lừa đảo: 4/100'); + await expect(badge).toHaveAttribute('data-tone', 'good'); + await expect(badge).not.toHaveClass(/bg-danger/); + const status = page.getByTestId('merchant-scam-status'); + await expect(status).not.toHaveText('Có cảnh báo rủi ro'); + await expect(status).toHaveClass(/text-success/); + // No reviews analysed: no "100%" authentic ratio. + const summary = page.getByTestId('merchant-review-summary'); + await expect(summary).toHaveText('Chưa có đánh giá để phân tích.'); + await expect(summary).not.toContainText('%'); +}); + +test('a 60/100 DOSafe store shows the scam warning', async ({ page }) => { + expect((await page.goto(HIGH))?.status()).toBe(200); + const badge = page.getByTestId('merchant-scam-badge'); + await expect(badge).toHaveAttribute('data-tone', 'bad'); + await expect(badge).toHaveClass(/bg-danger/); + const status = page.getByTestId('merchant-scam-status'); + await expect(status).toHaveText('Có cảnh báo rủi ro'); + await expect(status).toHaveClass(/text-danger/); +}); diff --git a/apps/web/e2e/product-detail-locale.spec.ts b/apps/web/e2e/product-detail-locale.spec.ts index 8adf4e2cc..13334a376 100644 --- a/apps/web/e2e/product-detail-locale.spec.ts +++ b/apps/web/e2e/product-detail-locale.spec.ts @@ -3,22 +3,23 @@ import { expect, test } from '@playwright/test'; // The rating confidence tier and the deal radar note follow the active locale // instead of the Vietnamese strings rate-api and deals.ts build // (DOS/Rate#574). The seeded iPhone is cheapest on Shopee, below the median of -// its two offers, so it carries a genuine-deal note. +// its two offers, so it carries a genuine-deal note that names that count +// instead of a fixed "10 platforms" (DOS/Rate#1145). const PRODUCT = '/smartphones/iphone-16-pro-max-256gb'; const COPY = { vi: { radar: - /^Rẻ hơn trung vị 10 sàn 1\.500\.000 ₫ \(-4\.9%\) tại Apple Flagship Store \(SHOPEE\)\.$/, + /^Rẻ hơn giá trung vị của 2 tin bán 1\.500\.000 ₫ \(-4\.9%\) tại Apple Flagship Store \(SHOPEE\)\.$/, tier: /^(Độ tin cậy|Ít mẫu kiểm chứng|Chưa đủ mẫu kiểm chứng)/, }, en: { radar: - /^1\.500\.000 ₫ below the 10-platform median \(-4\.9%\) at Apple Flagship Store \(SHOPEE\)\.$/, + /^1\.500\.000 ₫ below the median of 2 listings \(-4\.9%\) at Apple Flagship Store \(SHOPEE\)\.$/, tier: /^(High|Medium|Low) confidence|^Few verified samples|^Not enough verified samples/, }, cs: { radar: - /^O 1\.500\.000 ₫ levnější než medián 10 platforem \(-4\.9 %\) u Apple Flagship Store \(SHOPEE\)\.$/, + /^O 1\.500\.000 ₫ levnější než medián cen 2 nabídek \(-4\.9 %\) u Apple Flagship Store \(SHOPEE\)\.$/, tier: /důvěryhodnost|^Málo ověřených vzorků|^Nedostatek ověřených vzorků/, }, }; @@ -39,7 +40,11 @@ for (const [locale, copy] of Object.entries(COPY)) { ).toHaveText(copy.tier); if (locale !== 'vi') { const text = await page.locator('body').innerText(); - for (const vi of ['Rẻ hơn trung vị', 'Độ tin cậy', 'Ít mẫu kiểm chứng']) + for (const vi of [ + 'Rẻ hơn giá trung vị', + 'Độ tin cậy', + 'Ít mẫu kiểm chứng', + ]) expect(text).not.toContain(vi); } }); diff --git a/apps/web/e2e/seed.sql b/apps/web/e2e/seed.sql index 9ca76bee0..fb51c50ca 100644 --- a/apps/web/e2e/seed.sql +++ b/apps/web/e2e/seed.sql @@ -14,6 +14,14 @@ VALUES -- Its listing query fails on purpose (FAULT_MARKERS in scripts/e2e/run.mjs). ('eeeeeeee-eeee-4eee-8eee-eeeeeeeefa17', 'shopee', 'shop_fault', 'Faulty Listings Store', 'faulty-listings-store', false); +-- DOSafe scores on both sides of the shared risk cut-off, with no reviews: a +-- 4/100 store is not a scam warning and 0 reviews is not "100% authentic"; +-- a 60/100 store is (DOS/Rate#1140). +INSERT INTO rate.merchants (id, platform, platform_shop_id, name, slug, is_official, risk_score) +VALUES + ('11111111-1111-4111-8111-111111111140', 'shopee', 'shop_low_risk', 'Low Risk Store', 'low-risk-store', true, 4), + ('11111111-1111-4111-8111-111111111160', 'shopee', 'shop_high_risk', 'High Risk Store', 'high-risk-store', false, 60); + INSERT INTO rate.master_products (id, category_id, brand, model, model_key, name, slug, ram_gb, storage_gb, screen_size_inch, battery_mah, has_5g, min_price, max_price, merchant_count, status, metadata) @@ -164,6 +172,22 @@ FROM generate_series(1, 201) AS g; UPDATE rate.master_products SET user_rating_weighted = 4 WHERE slug LIKE 'e2e-bud-%'; +-- One bud stored with phone-shaped specs, as the AI enricher wrote them for +-- real earbuds: a 1 inch screen, 1GB RAM, 64GB ROM, microphones as cameras and +-- a 5G flag. Its page and card must show none of them (DOS/Rate#1142). +UPDATE rate.master_products +SET ram_gb = 1, storage_gb = 64, screen_size_inch = 1, has_5g = true, + specifications = '{"category_slug": "earbuds", + "display": {"size_inch": 1, "refresh_rate_hz": 60, "ppi": 300}, + "hardware": {"ram_gb": 1, "storage_gb": 64, "gpu": "E2E Audio Engine"}, + "camera": {"main_camera": "E2E Mic Array"}, + "cellular": {"has_5g": true}, + "battery": {"capacity_mah": 515}, + "multimedia": {"loudspeaker": "E2E 10.5mm driver", "audio_features": "E2E ANC"}, + "connectivity": {"bluetooth": "5.4"}}', + key_specs = '{"screen": "1 inch", "camera": "E2E Mic Array", "ram": "1GB", "storage": "64GB", "battery": "515 mAh"}' +WHERE slug = 'e2e-bud-002'; + -- A phone whose six 5-star reviews are mostly star-only, one of them a -- sponsored post: the product page shows the star-distribution warning and a -- disclosure badge, which must follow the active locale (DOS/Rate#566). Its @@ -253,7 +277,9 @@ VALUES -- A phone whose cheapest listing is a bait price, 1,250,000: under 40% of -- the others' lower median (3,140,000) and above the 1,200,000 accessory -- floor (see bait-price.spec.ts). It must not set the "from" price and is labelled --- (DOS/Rate#797). Old created_at keeps it last in default orderings. Four +-- (DOS/Rate#797). Its stored min_price is 3,030,000, what +-- rate.refresh_master_market_stats writes since DOS-Me 20261024060000 +-- (DOS/Rate#802): every page reads the stored stats (DOS/Rate#1141). Old created_at keeps it last in default orderings. Four -- short-named stores of its own keep the other merchants' cards unchanged. INSERT INTO rate.merchants (id, platform, platform_shop_id, name, slug, is_official) VALUES @@ -264,7 +290,7 @@ VALUES INSERT INTO rate.master_products (id, category_id, brand, model, model_key, name, slug, min_price, max_price, merchant_count, status, metadata, created_at) VALUES ('22222222-2222-4222-8222-222222220797', 'c0000000-0000-4000-8000-000000000001', 'E2E', 'Bait Price Phone', - 'e2e_bait_price_phone', 'E2E Bait Price Phone', 'e2e-bait-price-phone', 1250000, 3690000, 4, 'published', + 'e2e_bait_price_phone', 'E2E Bait Price Phone', 'e2e-bait-price-phone', 3030000, 3690000, 4, 'published', '{"category_slug": "smartphones"}', timestamptz '2026-01-01 00:00:00+00'); INSERT INTO rate.products (id, merchant_id, platform, platform_item_id, title, category_id, price_current, url, cluster_id) VALUES diff --git a/apps/web/locales/cs.json b/apps/web/locales/cs.json index 02ebf6409..2da9e16dc 100644 --- a/apps/web/locales/cs.json +++ b/apps/web/locales/cs.json @@ -676,9 +676,9 @@ "radarMdHeader": "### 🔥 Cenová minima a falešné slevy v 10 obchodech - Rate.vn", "radarMdMeta": "- **Kategorie:** {category} | **Analyzované klastry:** {clusters} ({offers} nabídek)", "radarMdGenuineTitle": "#### 🟢 Skutečná minima napříč obchody", - "radarMdGenuineTableHead": "| Zařízení | Obchod / prodejce | Aktuální cena | Medián 10 obchodů | Skutečná úspora | Skóre JReviews |", + "radarMdGenuineTableHead": "| Zařízení | Obchod / prodejce | Aktuální cena | Medián nabídek | Skutečná úspora | Skóre JReviews |", "radarMdFakeTitle": "#### 🚨 Uměle navýšené původní ceny", - "radarMdFakeTableHead": "| Zařízení | Obchod / prodejce | Prodejní cena | Deklarovaná původní | Medián 10 obchodů | Umělé navýšení |", + "radarMdFakeTableHead": "| Zařízení | Obchod / prodejce | Prodejní cena | Deklarovaná původní | Medián nabídek | Umělé navýšení |", "radarMdSpreadTitle": "#### 📊 Největší cenové rozdíly mezi obchody", "radarMdSpreadTableHead": "| Zařízení | Nejlevnější platforma | Nejdražší platforma | Nejnižší cena | Nejvyšší cena | Absolutní rozdíl |", "noScore": "N/A", @@ -858,7 +858,7 @@ "filteredSpamCount": "{count, plural, one {# falešná nebo spamová recenze odfiltrována} few {# falešné nebo spamové recenze odfiltrovány} other {# falešných nebo spamových recenzí odfiltrováno}}", "noAbnormalReviews": "Žádné podezřelé vzorce recenzí", "priceRangeLabel": "Referenční cenové rozpětí:", - "radarGenuineBadge": "{count} minim napříč obchody (-{pct}%)", + "radarGenuineBadge": "{count} nabídek pod mediánem (-{pct}%)", "radarFakeBadge": "{count} varování na falešnou slevu", "radarZeroFakeBadge": "Žádná varování před falešnou slevou", "viewTrustProfileCta": "Zobrazit profil prodejce →", @@ -888,6 +888,8 @@ "scamStatusLabel": "Stav varování před podvody:", "scamStatusClean": "✓ Žádné záznamy o podvodech", "scamStatusAlert": "Detekováno bezpečnostní riziko", + "scamStatusCaution": "Některé rizikové signály", + "layer2NoReviews": "Zatím žádné recenze k analýze.", "entityVerificationLabel": "Ověření subjektu na tržišti:", "entityOfficial": "✓ Oficiální značkový obchod (Official Mall)", "piiProtectionLabel": "Ochrana osobních údajů (PII):", @@ -902,7 +904,7 @@ "statSabotage": "Sabotáž/Bombing", "profileRadarBadge": "Cenová minima a falešné slevy", "profileRadarAnalyzed": "Automaticky porovnáno napříč {count} nabídkami", - "profileRadarGenuineCount": "{count} minim napříč obchody", + "profileRadarGenuineCount": "{count} nabídek pod mediánem", "profileRadarFakeCount": "{count} varování na falešnou slevu", "profileRadarZeroFake": "0 varování na falešnou slevu", "openRadar10PlatformsBtn": "Slevy z 10 obchodů →", @@ -964,6 +966,8 @@ "metaTitle": "{name} - Od {minPrice} ₫ | Rate.vn", "metaTitleShort": "{name} - Od {minPrice} ₫", "metaDesc": "Porovnejte ceny {name} z {merchantCount} obchodů, zkontrolujte historii cen a přečtěte si ověřené recenze na Rate.vn.", + "metaDescNoStock": "{name} ({brand}) teď žádný obchod nemá skladem. Podívejte se na historii cen, odhalte falešné slevy a čtěte skutečné recenze (falešné odfiltrovány) na Rate.vn.", + "ogDescNoStock": "Teď jej žádný obchod nemá skladem. Podívejte se na 10 skupin parametrů podle GSMArena/PhoneArena a graf historie cen.", "ogTitle": "{name} - Parametry a skutečná historie cen | Rate.vn", "ogDesc": "Nejlepší cena od {minPrice} ₫. Prohlédněte si 10 standardizovaných skupin parametrů GSMArena/PhoneArena a graf historie cen.", "breadcrumbHome": "Domů", @@ -991,7 +995,7 @@ "readAndSubmitReviewsBtn": "Číst a psát recenze ({count})", "tenSpecGroupsBtn": "Parametry", "headToHeadCompareBtn": "Přímé porovnání", - "quickSpecDriver": "Měnič / Displej", + "quickSpecDriver": "Měnič", "quickSpecScreen": "Displej", "quickSpecChipset": "Procesor (Čipset)", "quickSpecAudioAnc": "Zvuk / ANC", @@ -1045,6 +1049,9 @@ "offersOfficialMallBadge": "Oficiální (Mall)", "offersImplausiblyLowBadge": "Podezřele nízká cena", "offersImplausiblyLowTitle": "Mnohem levnější než všechny ostatní obchody: může jít o padělek, příslušenství nebo návnadu. Nezapočítává se do nejnižší ceny.", + "offersUnavailableBadge": "Není skladem", + "offersUnavailableTitle": "Obchod jej hlásí jako vyprodaný nebo ukončený. Cena je naposledy zjištěná a nezapočítává se do nejnižší ceny ani do počtu obchodů.", + "priceNoneInStock": "Žádný obchod jej nemá skladem", "offersMerchantTrustTitle": "Zobrazit profil důvěryhodnosti obchodníka", "offersInspectBtnTitle": "Podrobně zkontrolovat tuto nabídku na stránce Ověřit odkaz", "offersInspectBtn": "Zkontrolovat", @@ -1055,7 +1062,7 @@ "bayesianTierLow": "Nízká důvěryhodnost", "bayesianTierSmoothed": "Málo ověřených vzorků, skóre upraveno", "bayesianTierPrior": "Nedostatek ověřených vzorků", - "radarExplanationGenuineDeal": "O {savings} ₫ levnější než medián 10 platforem (-{pct} %) u <value>{merchant}</value> ({platform}).", + "radarExplanationGenuineDeal": "O {savings} ₫ levnější než medián cen {count} nabídek (-{pct} %) u <value>{merchant}</value> ({platform}).", "radarExplanationFakeSale": "Obchod uvádí přeškrtnutou původní cenu {original} ₫ (+{markupPct} % nad tržním mediánem), aby vyvolal dojem slevy {claimedPct} %, ale skutečná cena je o {aboveFloor} ₫ vyšší než tržní minimum.", "offersMdSpreadLine": "- **Cenové rozpětí ({count} platforem):** Ušetřete **{spreadVnd} ₫ ({spreadPct} %)** na **{cheapestPlatform} ({cheapestMerchant})** namísto **{priciestPlatform}**\n", "shareAuditPriceSources": "{price} ₫ ({count} zkontrolovaných zdrojů)", @@ -1176,7 +1183,7 @@ "scoreDeltaLabel": "Rozdíl {delta} bodu", "view1v1ScoreBtn": "Zobrazit souboj 1v1", "radarHomeBadge": "Cenové signály z 10 obchodů", - "radarHomeAnalyzed": "Automatické porovnání {offers} nabídek napříč {clusters} produktovými klastry", + "radarHomeAnalyzed": "Automatické porovnání {offers} nabídek od {merchants} prodejců napříč {clusters} produktovými klastry", "radarHomeDealHubBtn": "Centrum slev", "radarHomeInspectBtn": "Ověřit odkaz na akci", "radarGenuineTitle": "Skutečná minima ({count})", @@ -1661,7 +1668,7 @@ "leaderboardBadge": "Žebříček", "sortByPrefix": "Řazení: {label}", "radarBannerBadge": "Cenová minima a falešné slevy ({title})", - "radarAnalyzedSub": "Zkontrolováno {count} nabídek v 10 obchodech", + "radarAnalyzedSub": "Zkontrolováno {count} nabídek", "viewAllRadarBtn": "Zobrazit všechny nabídky z 10 obchodů", "vsMedianBadge": "-{pct}% oproti mediánu", "fakeMarkupBadge": "Nadsazeno +{pct}%", diff --git a/apps/web/locales/en.json b/apps/web/locales/en.json index 8ae090dd4..6cf633002 100644 --- a/apps/web/locales/en.json +++ b/apps/web/locales/en.json @@ -676,9 +676,9 @@ "radarMdHeader": "### 🔥 Cross-store price lows and fake-sale warnings (10 stores) - Rate.vn", "radarMdMeta": "- **Category:** {category} | **Clusters analyzed:** {clusters} ({offers} listings)", "radarMdGenuineTitle": "#### 🟢 Real cross-store lows", - "radarMdGenuineTableHead": "| Device | Store / seller | Current price | 10-store median | Real savings | JReviews score |", + "radarMdGenuineTableHead": "| Device | Store / seller | Current price | Listing median | Real savings | JReviews score |", "radarMdFakeTitle": "#### 🚨 Inflated list prices", - "radarMdFakeTableHead": "| Device | Store / seller | Selling price | Claimed list price | 10-store median | Markup |", + "radarMdFakeTableHead": "| Device | Store / seller | Selling price | Claimed list price | Listing median | Markup |", "radarMdSpreadTitle": "#### 📊 Largest price gaps between stores", "radarMdSpreadTableHead": "| Device | Cheapest store | Priciest store | Lowest price | Highest price | Gap |", "noScore": "N/A", @@ -858,7 +858,7 @@ "filteredSpamCount": "{count, plural, one {# fake or spam review filtered} other {# fake or spam reviews filtered}}", "noAbnormalReviews": "No abnormal review patterns detected", "priceRangeLabel": "Reference price range:", - "radarGenuineBadge": "{count} cross-store lows (-{pct}%)", + "radarGenuineBadge": "{count} below-median deals (-{pct}%)", "radarFakeBadge": "{count} fake-sale markup alerts", "radarZeroFakeBadge": "No fake-sale alerts found", "viewTrustProfileCta": "View seller profile →", @@ -888,6 +888,8 @@ "scamStatusLabel": "Scam alert status:", "scamStatusClean": "✓ Zero scam reports / alerts", "scamStatusAlert": "Active risk alert detected", + "scamStatusCaution": "Some risk signals to watch", + "layer2NoReviews": "No reviews to analyse yet.", "entityVerificationLabel": "Marketplace entity status:", "entityOfficial": "✓ Official Mall (brand store)", "piiProtectionLabel": "PII data privacy:", @@ -902,7 +904,7 @@ "statSabotage": "Sabotage/Bomb", "profileRadarBadge": "Price lows and fake-sale signals", "profileRadarAnalyzed": "Automatically cross-checked across {count} multi-platform listings", - "profileRadarGenuineCount": "{count} cross-store lows", + "profileRadarGenuineCount": "{count} below-median listings", "profileRadarFakeCount": "{count} fake-sale markup alerts", "profileRadarZeroFake": "0 fake-sale markup alerts", "openRadar10PlatformsBtn": "Deals across 10 stores →", @@ -964,6 +966,8 @@ "metaTitle": "{name} - From {minPrice} ₫ | Rate.vn", "metaTitleShort": "{name} - From {minPrice} ₫", "metaDesc": "Compare {name} prices across {merchantCount} merchants, spot fake discounts in the price history and read filtered buyer reviews on Rate.vn.", + "metaDescNoStock": "No store has the {name} ({brand}) in stock right now. See its price history to spot fake discounts and read real reviews (fake reviews filtered) on Rate.vn.", + "ogDescNoStock": "No store has it in stock right now. See the 10 GSMArena/PhoneArena spec groups and the price history chart.", "ogTitle": "{name} - Specs and true price history | Rate.vn", "ogDesc": "Best price from {minPrice} ₫. See the 10 standardized GSMArena/PhoneArena spec groups and the price history chart.", "breadcrumbHome": "Home", @@ -991,7 +995,7 @@ "readAndSubmitReviewsBtn": "Read and write reviews ({count})", "tenSpecGroupsBtn": "Specifications", "headToHeadCompareBtn": "Compare head to head", - "quickSpecDriver": "Driver / Display", + "quickSpecDriver": "Driver", "quickSpecScreen": "Display", "quickSpecChipset": "Processor (Chipset)", "quickSpecAudioAnc": "Audio / ANC", @@ -1045,6 +1049,9 @@ "offersOfficialMallBadge": "Official Mall", "offersImplausiblyLowBadge": "Suspiciously low price", "offersImplausiblyLowTitle": "Far cheaper than every other store: possibly a counterfeit, an accessory or a bait listing. Not counted as the lowest price.", + "offersUnavailableBadge": "Out of stock", + "offersUnavailableTitle": "The store reports it sold out or discontinued. The price is the last one seen and is not counted in the lowest price or the number of stores.", + "priceNoneInStock": "No store has it in stock", "offersMerchantTrustTitle": "View merchant trust profile", "offersInspectBtnTitle": "Check this listing in detail on the link check page", "offersInspectBtn": "Check", @@ -1055,7 +1062,7 @@ "bayesianTierLow": "Low confidence", "bayesianTierSmoothed": "Few verified samples, score adjusted", "bayesianTierPrior": "Not enough verified samples", - "radarExplanationGenuineDeal": "{savings} ₫ below the 10-platform median (-{pct}%) at <value>{merchant}</value> ({platform}).", + "radarExplanationGenuineDeal": "{savings} ₫ below the median of {count} listings (-{pct}%) at <value>{merchant}</value> ({platform}).", "radarExplanationFakeSale": "The store shows a struck-through list price of {original} ₫ (+{markupPct}% above the market median) to suggest a {claimedPct}% discount, but the actual price is {aboveFloor} ₫ above the market low.", "offersMdSpreadLine": "- **Price gap between stores ({count} stores):** Save **{spreadVnd} ₫ ({spreadPct}%)** at **{cheapestPlatform} ({cheapestMerchant})** instead of **{priciestPlatform}**\n", "shareAuditPriceSources": "{price} ₫ ({count} checked sources)", @@ -1176,7 +1183,7 @@ "scoreDeltaLabel": "{delta} pts apart", "view1v1ScoreBtn": "See the 1v1 breakdown", "radarHomeBadge": "Price signals across 10 stores", - "radarHomeAnalyzed": "Automatically cross-checking {offers} listings across {clusters} product clusters", + "radarHomeAnalyzed": "Automatically cross-checking {offers} listings from {merchants} merchants across {clusters} product clusters", "radarHomeDealHubBtn": "Deal hub", "radarHomeInspectBtn": "Check a sale link", "radarGenuineTitle": "Real lows ({count})", @@ -1661,7 +1668,7 @@ "leaderboardBadge": "Leaderboard", "sortByPrefix": "Sort: {label}", "radarBannerBadge": "Price lows and fake-sale warnings ({title})", - "radarAnalyzedSub": "Checked {count} store listings across 10 stores", + "radarAnalyzedSub": "Checked {count} store listings", "viewAllRadarBtn": "See all deals across 10 stores", "vsMedianBadge": "-{pct}% vs median", "fakeMarkupBadge": "Inflated +{pct}%", diff --git a/apps/web/locales/vi.json b/apps/web/locales/vi.json index da11c176b..4893124ca 100644 --- a/apps/web/locales/vi.json +++ b/apps/web/locales/vi.json @@ -674,11 +674,11 @@ "personaGaming": "Hiệu năng và chơi game", "personaEndurance": "Pin trâu và sạc nhanh", "radarMdHeader": "### 🔥 Giá đáy giữa các sàn và cảnh báo giá gốc bị nâng khống (10 sàn) - Rate.vn", - "radarMdMeta": "- **Ngành hàng:** {category} | **Cụm sản phẩm phân tích:** {clusters} ({offers} gian hàng)", + "radarMdMeta": "- **Ngành hàng:** {category} | **Cụm sản phẩm phân tích:** {clusters} ({offers} tin bán)", "radarMdGenuineTitle": "#### 🟢 Giá đáy thật giữa các sàn", - "radarMdGenuineTableHead": "| Thiết bị | Sàn / gian hàng | Giá bán hiện tại | Giá trung vị 10 sàn | Tiết kiệm thật | Điểm JReviews |", + "radarMdGenuineTableHead": "| Thiết bị | Sàn / gian hàng | Giá bán hiện tại | Giá trung vị các tin bán | Tiết kiệm thật | Điểm JReviews |", "radarMdFakeTitle": "#### 🚨 Giá gốc bị nâng khống", - "radarMdFakeTableHead": "| Thiết bị | Sàn / gian hàng | Giá bán thực tế | Giá gốc ghi trên tin | Giá trung vị 10 sàn | Mức nâng khống |", + "radarMdFakeTableHead": "| Thiết bị | Sàn / gian hàng | Giá bán thực tế | Giá gốc ghi trên tin | Giá trung vị các tin bán | Mức nâng khống |", "radarMdSpreadTitle": "#### 📊 Chênh lệch giá lớn nhất giữa các sàn", "radarMdSpreadTableHead": "| Thiết bị | Sàn rẻ nhất | Sàn đắt nhất | Giá thấp nhất | Giá cao nhất | Chênh lệch |", "noScore": "Chưa có", @@ -699,7 +699,7 @@ "maxBudgetAria": "Ngân sách tối đa", "searchPlaceholder": "Tìm nhanh theo tên máy hoặc thương hiệu (Apple, Samsung, Xiaomi, Sony...)", "radarBannerBadge": "Deal giá đáy và cảnh báo giảm giá ảo trên 10 sàn", - "radarScannedSummary": "Đã quét {offers} gian hàng trên {clusters} cụm sản phẩm", + "radarScannedSummary": "Đã quét {offers} tin bán trên {clusters} cụm sản phẩm", "copiedRadarMdBtn": "✓ Đã chép tín hiệu giá", "copyRadarMdBtn": "Sao chép tín hiệu giá (Markdown)", "exportRadarCsvBtn": "Xuất tín hiệu giá (CSV)", @@ -858,7 +858,7 @@ "filteredSpamCount": "{count} đánh giá ảo hoặc spam đã lọc", "noAbnormalReviews": "Chưa phát hiện đánh giá bất thường", "priceRangeLabel": "Biên độ giá tham chiếu:", - "radarGenuineBadge": "{count} giá đáy giữa các sàn (-{pct}%)", + "radarGenuineBadge": "{count} tin bán dưới giá trung vị (-{pct}%)", "radarFakeBadge": "{count} Cảnh báo kê khống giá gốc", "radarZeroFakeBadge": "Chưa phát hiện giảm giá ảo", "viewTrustProfileCta": "Xem hồ sơ gian hàng →", @@ -888,6 +888,8 @@ "scamStatusLabel": "Trạng thái cảnh báo lừa đảo:", "scamStatusClean": "✓ Không ghi nhận phốt / cảnh báo", "scamStatusAlert": "Có cảnh báo rủi ro", + "scamStatusCaution": "Có tín hiệu cần lưu ý", + "layer2NoReviews": "Chưa có đánh giá để phân tích.", "entityVerificationLabel": "Xác thực pháp nhân sàn:", "entityOfficial": "✓ Gian hàng chính hãng (Official Mall)", "piiProtectionLabel": "Bảo mật dữ liệu PII:", @@ -901,8 +903,8 @@ "statSeedingFake": "Đánh giá ảo", "statSabotage": "Phá hoại/Bomb", "profileRadarBadge": "Tín hiệu giá đáy và giảm giá ảo", - "profileRadarAnalyzed": "Đối chiếu tự động trên {count} gian hàng đa sàn", - "profileRadarGenuineCount": "{count} giá đáy giữa các sàn", + "profileRadarAnalyzed": "Đối chiếu tự động trên {count} tin bán đa sàn", + "profileRadarGenuineCount": "{count} tin bán dưới giá trung vị", "profileRadarFakeCount": "{count} cảnh báo kê khống giá gốc", "profileRadarZeroFake": "0 cảnh báo nâng khống giá gốc", "openRadar10PlatformsBtn": "Xem deal trên 10 sàn →", @@ -964,6 +966,8 @@ "metaTitle": "{name} - Giá từ {minPrice} ₫ | Rate.vn", "metaTitleShort": "{name} - Giá từ {minPrice} ₫", "metaDesc": "So sánh giá {name} từ {merchantCount} gian hàng, xem lịch sử giá để nhận ra giảm giá ảo và đọc đánh giá thật (đã lọc đánh giá ảo) trên Rate.vn.", + "metaDescNoStock": "{name} ({brand}) hiện chưa có nơi bán còn hàng. Xem lịch sử giá để nhận ra giảm giá ảo và đọc đánh giá thật (đã lọc đánh giá ảo) trên Rate.vn.", + "ogDescNoStock": "Chưa có nơi bán còn hàng. Xem 10 nhóm thông số kỹ thuật chuẩn GSMArena/PhoneArena và biểu đồ lịch sử giá.", "ogTitle": "{name} - So sánh giá và kiểm tra khuyến mãi | Rate.vn", "ogDesc": "Giá tốt nhất từ {minPrice} ₫. Xem 10 nhóm thông số kỹ thuật chuẩn GSMArena/PhoneArena và biểu đồ lịch sử giá.", "breadcrumbHome": "Trang chủ", @@ -991,7 +995,7 @@ "readAndSubmitReviewsBtn": "Đọc và viết đánh giá ({count})", "tenSpecGroupsBtn": "Thông số kỹ thuật", "headToHeadCompareBtn": "So sánh trực tiếp", - "quickSpecDriver": "Màng loa / Hiển thị", + "quickSpecDriver": "Màng loa", "quickSpecScreen": "Màn hình", "quickSpecChipset": "Vi xử lý (Chipset)", "quickSpecAudioAnc": "Âm thanh / ANC", @@ -1045,6 +1049,9 @@ "offersOfficialMallBadge": "Chính hãng (Mall)", "offersImplausiblyLowBadge": "Giá thấp bất thường", "offersImplausiblyLowTitle": "Rẻ hơn nhiều so với mọi nơi bán khác: có thể là hàng giả, phụ kiện hoặc tin câu khách. Không tính vào giá thấp nhất.", + "offersUnavailableBadge": "Hết hàng", + "offersUnavailableTitle": "Nơi bán báo hết hàng hoặc ngừng kinh doanh. Giá là giá ghi nhận lần cuối, không tính vào giá thấp nhất và số nơi bán.", + "priceNoneInStock": "Chưa có nơi bán còn hàng", "offersMerchantTrustTitle": "Xem hồ sơ tín nhiệm gian hàng", "offersInspectBtnTitle": "Kiểm tra chi tiết tin bán này ở trang Kiểm tra link", "offersInspectBtn": "Kiểm tra", @@ -1055,7 +1062,7 @@ "bayesianTierLow": "Độ tin cậy thấp", "bayesianTierSmoothed": "Ít mẫu kiểm chứng, điểm đã được điều chỉnh", "bayesianTierPrior": "Chưa đủ mẫu kiểm chứng", - "radarExplanationGenuineDeal": "Rẻ hơn trung vị 10 sàn {savings} ₫ (-{pct}%) tại <value>{merchant}</value> ({platform}).", + "radarExplanationGenuineDeal": "Rẻ hơn giá trung vị của {count} tin bán {savings} ₫ (-{pct}%) tại <value>{merchant}</value> ({platform}).", "radarExplanationFakeSale": "Gian hàng treo giá gốc gạch ngang {original} ₫ (cao hơn trung vị thị trường +{markupPct}%) để tạo cảm giác giảm {claimedPct}%, nhưng giá thực bán cao hơn đáy thị trường {aboveFloor} ₫.", "offersMdSpreadLine": "- **Chênh lệch giá giữa các sàn ({count} sàn):** Tiết kiệm **{spreadVnd} ₫ ({spreadPct}%)** khi mua tại **{cheapestPlatform} ({cheapestMerchant})** thay vì **{priciestPlatform}**\n", "shareAuditPriceSources": "{price} ₫ ({count} nguồn đã kiểm tra)", @@ -1176,7 +1183,7 @@ "scoreDeltaLabel": "Chênh {delta} điểm", "view1v1ScoreBtn": "Xem chấm điểm 1v1", "radarHomeBadge": "Tín hiệu giá trên 10 sàn", - "radarHomeAnalyzed": "Đối chiếu tự động {offers} gian hàng trên {clusters} cụm sản phẩm", + "radarHomeAnalyzed": "Đối chiếu tự động {offers} tin bán của {merchants} gian hàng trên {clusters} cụm sản phẩm", "radarHomeDealHubBtn": "Trung tâm săn deal", "radarHomeInspectBtn": "Kiểm tra link khuyến mãi", "radarGenuineTitle": "Giá tốt thật ({count})", @@ -1661,7 +1668,7 @@ "leaderboardBadge": "Bảng xếp hạng", "sortByPrefix": "Sắp xếp: {label}", "radarBannerBadge": "Deal giá đáy và cảnh báo giảm giá ảo ({title})", - "radarAnalyzedSub": "Đã kiểm tra {count} tin bán trên 10 sàn", + "radarAnalyzedSub": "Đã kiểm tra {count} tin bán", "viewAllRadarBtn": "Xem tất cả deal trên 10 sàn", "vsMedianBadge": "-{pct}% so trung vị", "fakeMarkupBadge": "Kê khống +{pct}%", diff --git a/apps/web/package.json b/apps/web/package.json index 5bc16df08..b0ff8e7c4 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -8,7 +8,7 @@ "lint": "eslint . --max-warnings 0", "preview": "next build && next start", "start": "next start", - "test": "pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/server/rate/__tests__/core-rate-architecture.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/app/api/v1/rate/__tests__/write-paths.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/logger.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/review-display.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/concurrency.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/catalog-listing.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/discount-verdict-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-detail-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-competitors.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/analyzer-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sold-count.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/content-lang.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/image-url.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/client-messages.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/seo.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/offer-plausibility.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sitemap-lastmod.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/leaderboard-badge-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/focus-trap.test.ts", + "test": "pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/server/rate/__tests__/core-rate-architecture.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/app/api/v1/rate/__tests__/write-paths.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/logger.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/review-display.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/concurrency.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/catalog-listing.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/discount-verdict-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-detail-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-competitors.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/analyzer-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sold-count.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/content-lang.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/image-url.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/client-messages.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/seo.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/offer-plausibility.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-market-stats.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sitemap-lastmod.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/leaderboard-badge-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/focus-trap.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/display-truth.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/spec-template.test.ts", "typecheck": "tsc --noEmit" }, "browserslist": [ diff --git a/apps/web/src/app/[locale]/components/list-sidebar-one.tsx b/apps/web/src/app/[locale]/components/list-sidebar-one.tsx index 0a01f634d..fe1fd9f2a 100644 --- a/apps/web/src/app/[locale]/components/list-sidebar-one.tsx +++ b/apps/web/src/app/[locale]/components/list-sidebar-one.tsx @@ -323,7 +323,7 @@ export default function ListSidebarOne({ return ( <> - <div className="searchingSidebar pe-xl-5"> + <div className="searchingSidebar"> <div className="offcanvas offcanvas-start largeshow" data-bs-scroll="true" @@ -349,14 +349,16 @@ export default function ListSidebarOne({ <form className="search-inner" onSubmit={handleApplyForm}> {/* 1. Keyword Search Box */} <div className="filter-search-box mb-4"> - <div className="filtersearch-title d-flex justify-content-between align-items-center"> + {/* Wraps in the 280px sidebar: the badge drops under the + title instead of squeezing it onto three lines (#1132). */} + <div className="filtersearch-title d-flex justify-content-between align-items-center column-gap-2 flex-wrap"> <h3 id={keywordHeaderId} - className="lh-base text-uppercase fw-medium mb-2 text-sm" + className="lh-base text-uppercase fw-medium text-nowrap mb-2 text-sm" > {t('sidebar.keywordChipsetHeader')} </h3> - <span className="badge bg-light text-muted small border"> + <span className="badge bg-light text-muted small mb-2 border"> {schema.directoryName} </span> </div> diff --git a/apps/web/src/app/[locale]/leaderboard/page.tsx b/apps/web/src/app/[locale]/leaderboard/page.tsx index 553ffe2be..f038fea75 100644 --- a/apps/web/src/app/[locale]/leaderboard/page.tsx +++ b/apps/web/src/app/[locale]/leaderboard/page.tsx @@ -8,6 +8,7 @@ import { } from 'react-icons/bs'; import { Link } from '@/lib/navigation'; import { + getCachedFilteredReviewCountsByMaster, getCachedMasterProducts, getCrossPlatformDealRadar, } from '@/server/rate'; @@ -83,23 +84,29 @@ export default async function LeaderboardPage({ params }: PageProps) { // The 100 best-rated models of each category: one global top 100 could // leave a whole category out of its own leaderboard tab (DOS/Rate#473). - const [productsByCategory, dealRadar] = await Promise.all([ - Promise.all( - [...VALID_CATEGORIES].map((categorySlug) => - getCachedMasterProducts({ - categorySlug, - limit: 100, - sort: 'rating_desc', - }) - ) - ), - // No fallback: the page is cached, so a null radar would be served for - // the whole revalidate window (DOS/Rate#491). - getCrossPlatformDealRadar({ - categorySlug: 'all', - limit: 36, - }), - ]); + const [productsByCategory, dealRadar, filteredReviewCounts] = + await Promise.all([ + Promise.all( + [...VALID_CATEGORIES].map((categorySlug) => + getCachedMasterProducts({ + categorySlug, + limit: 100, + sort: 'rating_desc', + }) + ) + ), + // No fallback: the page is cached, so a null radar would be served for + // the whole revalidate window (DOS/Rate#491). + getCrossPlatformDealRadar({ + categorySlug: 'all', + limit: 36, + }), + // Real counts of excluded stored reviews (DOS/Rate#1139). On a read + // failure no product claims a blocked count, which is still true. + getCachedFilteredReviewCountsByMaster().catch( + () => ({}) as Record<string, number> + ), + ]); // Back in one best-rated-first list (same key as search_master_products' // rating_desc), so the top 15 in the ItemList below stay platform-wide. const ratingOf = (p: (typeof productsByCategory)[number][number]) => @@ -345,6 +352,7 @@ export default async function LeaderboardPage({ params }: PageProps) { <LeaderboardExplorer products={products} dealRadar={dealRadar} + filteredReviewCounts={filteredReviewCounts} locale={resolvedLocale} /> </section> diff --git a/apps/web/src/app/[locale]/merchants/[id]/page.tsx b/apps/web/src/app/[locale]/merchants/[id]/page.tsx index ab014465b..0dd4a3692 100644 --- a/apps/web/src/app/[locale]/merchants/[id]/page.tsx +++ b/apps/web/src/app/[locale]/merchants/[id]/page.tsx @@ -31,6 +31,7 @@ import { BsStarFill, } from 'react-icons/bs'; import { isPendingReview } from '@/lib/review-display'; +import { merchantRiskTone } from '@/lib/merchant-risk-tone'; import ContentText from '@/components/elementary/ContentText'; import { contentLang } from '@/lib/content-lang'; import ClientMessages from '@/components/providers/ClientMessages'; @@ -145,10 +146,13 @@ export default async function MerchantProfilePage({ : 'bg-danger text-white'; const brk = merchant.review_risk_breakdown; + // No ratio without analysed reviews: 0 of 0 is not "100% authentic". const authenticPct = brk.total_reviews > 0 ? Math.round((brk.authentic_count / brk.total_reviews) * 1000) / 10 - : 100; + : null; + // Same cut-offs as the home merchant cards (DOS/Rate#1140). + const scamTone = merchantRiskTone(merchant.dosafe_risk_score); const canonicalUrl = `https://rate.vn/merchants/${merchant.id}`; @@ -396,10 +400,14 @@ export default async function MerchantProfilePage({ </div> <span className={`badge rounded-pill fs-6 px-3 py-2 ${ - merchant.dosafe_risk_score === 0 + scamTone === 'good' ? 'bg-success-subtle text-success' - : 'bg-danger text-white' + : scamTone === 'warn' + ? 'bg-warning-subtle text-warning-emphasis' + : 'bg-danger text-white' }`} + data-testid="merchant-scam-badge" + data-tone={scamTone} > {t('scamRiskBadge', { score: merchant.dosafe_risk_score })} </span> @@ -410,10 +418,21 @@ export default async function MerchantProfilePage({ <div className="rounded-3 bg-light mt-auto p-3"> <div className="d-flex align-items-center justify-content-between small mb-2"> <span className="text-muted">{t('scamStatusLabel')}</span> - <span className="fw-semibold text-success"> - {merchant.dosafe_risk_score === 0 + <span + className={`fw-semibold ${ + scamTone === 'good' + ? 'text-success' + : scamTone === 'warn' + ? 'text-warning-emphasis' + : 'text-danger' + }`} + data-testid="merchant-scam-status" + > + {scamTone === 'good' ? t('scamStatusClean') - : t('scamStatusAlert')} + : scamTone === 'warn' + ? t('scamStatusCaution') + : t('scamStatusAlert')} </span> </div> <div className="d-flex align-items-center justify-content-between small mb-2"> @@ -466,9 +485,19 @@ export default async function MerchantProfilePage({ </span> </div> - <p className="text-muted small mb-3"> - {t('layer2DescPrefix')} <strong>{brk.total_reviews}</strong>{' '} - {t('layer2DescMid')} <strong>{authenticPct}%</strong>). + <p + className="text-muted small mb-3" + data-testid="merchant-review-summary" + > + {authenticPct == null ? ( + t('layer2NoReviews') + ) : ( + <> + {t('layer2DescPrefix')}{' '} + <strong>{brk.total_reviews}</strong> {t('layer2DescMid')}{' '} + <strong>{authenticPct}%</strong>). + </> + )} </p> <div className="row g-2 mt-auto"> diff --git a/apps/web/src/app/[locale]/page.tsx b/apps/web/src/app/[locale]/page.tsx index 3a5c5cddc..26f429d7b 100644 --- a/apps/web/src/app/[locale]/page.tsx +++ b/apps/web/src/app/[locale]/page.tsx @@ -53,7 +53,12 @@ import ContentText from '@/components/elementary/ContentText'; import { contentLang } from '@/lib/content-lang'; import ClientMessages from '@/components/providers/ClientMessages'; import { siteJsonLd } from '@/lib/seo'; -import { productImageUrl } from '@/lib/rate-domain-core'; +import { + getMasterProductCategorySlug, + productImageUrl, +} from '@/lib/rate-domain-core'; +import { applyCategorySpecTemplate } from '@repo/shared-data/rules'; +import { merchantRiskTone } from '@/lib/merchant-risk-tone'; // Cached after the first request, refreshed every 5 minutes or when the // worker revalidates its cache tags (DOS/Rate#491). @@ -465,7 +470,13 @@ export default async function Home({ </div> <div className="row g-4"> - {masterProducts.slice(0, 6).map((item) => { + {masterProducts.slice(0, 6).map((storedItem) => { + // An earbud card carries no RAM, storage or screen chip + // (DOS/Rate#1142). + const item = applyCategorySpecTemplate( + getMasterProductCategorySlug(storedItem, catSlugById), + storedItem + ); const thumb = productImageUrl(item); const keySpecs = item.key_specs || {}; const productHref = getMasterProductHref(item, catSlugById); @@ -627,6 +638,7 @@ export default async function Home({ <h3 className="rate-home-radar-title"> {tHome('radarHomeAnalyzed', { offers: dealRadar.totalOffersAnalyzed, + merchants: dealRadar.totalMerchantsAnalyzed, clusters: dealRadar.totalClustersAnalyzed, })} </h3> @@ -842,8 +854,7 @@ export default async function Home({ {merchants .slice(0, HOME_MERCHANT_PREVIEW_COUNT) .map((merchant) => { - const tone = (risk: number) => - risk < 20 ? 'is-good' : risk < 50 ? 'is-warn' : 'is-bad'; + const tone = (risk: number) => `is-${merchantRiskTone(risk)}`; const overallTone = tone(merchant.risk_score); const trustLabel = overallTone === 'is-good' @@ -901,7 +912,8 @@ export default async function Home({ <dd className={tone(merchant.dosafe_risk_score)}> {merchant.dosafe_risk_score}/100 </dd> - {merchant.dosafe_risk_score < 20 && ( + {merchantRiskTone(merchant.dosafe_risk_score) === + 'good' && ( <dd className="rate-merchant-note"> {tHome('noScamDetected')} </dd> diff --git a/apps/web/src/app/[locale]/smartphones/[slug]/page.tsx b/apps/web/src/app/[locale]/smartphones/[slug]/page.tsx index b7e7fcea4..e12b4ecb4 100644 --- a/apps/web/src/app/[locale]/smartphones/[slug]/page.tsx +++ b/apps/web/src/app/[locale]/smartphones/[slug]/page.tsx @@ -1,4 +1,5 @@ import type { Metadata } from 'next'; +import { Fragment, type ReactNode } from 'react'; import { getTranslations, setRequestLocale } from 'next-intl/server'; import { Link } from '@/lib/navigation'; import { notFound } from 'next/navigation'; @@ -54,7 +55,12 @@ import PageShell from '../../components/page-shell'; import type { KeySpecs, ProductSpecifications } from '@repo/shared-data'; import { normalizeRateLocale } from '@/lib/rate-i18n'; -import { getSpecUiDictionary, localizeSpecValue } from '@/lib/spec-localizer'; +import { + getSpecUiDictionary, + localizeSpecValue, + specGroupHeading, + specSectionBadge, +} from '@/lib/spec-localizer'; import { contentLang } from '@/lib/content-lang'; import ContentText from '@/components/elementary/ContentText'; import { @@ -67,7 +73,16 @@ import ProductValueCalculator from '@/components/leaderboard/ProductValueCalcula import ProductOffersExplorer from '@/components/leaderboard/ProductOffersExplorer'; import ProductSpecsExportToolbar from '@/components/leaderboard/ProductSpecsExportToolbar'; import ClientMessages from '@/components/providers/ClientMessages'; -import { plausibleMinPrice } from '@repo/shared-data/rules'; +import { + applyCategorySpecTemplate, + getCategorySpecTemplate, + specFieldApplies, + type SpecGroupKey, +} from '@repo/shared-data/rules'; +import { + isOfferUnavailable, + productMarketStats, +} from '@/lib/product-market-stats'; import { firstImageUrl, usableImageUrls, @@ -98,29 +113,33 @@ export async function generateCategoryProductMetadata( title: t('notFoundTitle'), }; } - // A bait-price listing does not set the "from" price (DOS/Rate#797). - const offers = await getCachedMasterProductOffers(product.id); - const minPrice = plausibleMinPrice( - Number(product.min_price || 0), - offers - ).toLocaleString('vi-VN'); + // The stored stats, like every other page shows (DOS/Rate#1141). + const marketStats = productMarketStats(product); + const hasPrice = marketStats.minPrice > 0; + const minPrice = marketStats.minPrice.toLocaleString('vi-VN'); const canonicalUrl = `https://rate.vn${categoryBasePath}/${product.slug}`; return { // The product name first; the site name, then the price give way to fit - // a search result (DOS/Rate#782). + // a search result (DOS/Rate#782). No "from 0 ₫" without a price. title: fitTitle([ - t('metaTitle', { name: product.name, minPrice }), - t('metaTitleShort', { name: product.name, minPrice }), + ...(hasPrice + ? [ + t('metaTitle', { name: product.name, minPrice }), + t('metaTitleShort', { name: product.name, minPrice }), + ] + : []), `${product.name} | Rate.vn`, product.name, ]), description: fitDescription( - t('metaDesc', { - name: product.name, - brand: product.brand, - merchantCount: product.merchant_count || 0, - }) + hasPrice + ? t('metaDesc', { + name: product.name, + brand: product.brand, + merchantCount: marketStats.sellerCount, + }) + : t('metaDescNoStock', { name: product.name, brand: product.brand }) ), alternates: { canonical: canonicalUrl, @@ -128,7 +147,7 @@ export async function generateCategoryProductMetadata( openGraph: { ...defaultOpenGraph(lang), title: t('ogTitle', { name: product.name }), - description: t('ogDesc', { minPrice }), + description: hasPrice ? t('ogDesc', { minPrice }) : t('ogDescNoStock'), url: canonicalUrl, images: [firstImageUrl(product.images) ?? SHARE_IMAGE], }, @@ -154,17 +173,21 @@ export async function ProductDetailView({ }) { const { locale, slug } = await params; setRequestLocale(locale); - const [t, tNav, product] = await Promise.all([ + const [t, tNav, storedProduct] = await Promise.all([ getTranslations({ locale, namespace: 'productDetail' }), getTranslations({ locale, namespace: 'rate.nav' }), getCachedMasterProductBySlug(slug), ]); - if (!product) { + if (!storedProduct) { notFound(); } const categorySlug = categoryBasePath.replace(/^\//, '') || 'smartphones'; + // Each category shows only the spec groups and fields it has: earbuds have + // no screen, RAM or camera, watches no 5G (DOS/Rate#1142). + const specTemplate = getCategorySpecTemplate(categorySlug); + const product = applyCategorySpecTemplate(categorySlug, storedProduct); const localizedCategoryLabel = categorySlug === 'smartphones' || categorySlug === 'laptops' || @@ -173,8 +196,8 @@ export async function ProductDetailView({ categorySlug === 'earbuds' ? tNav(categorySlug) : categoryLabel; - const [offers, reviews, categoryPeers, categoryDealRadar] = await Promise.all( - [ + const [offers, reviews, storedCategoryPeers, categoryDealRadar] = + await Promise.all([ getCachedMasterProductOffers(product.id), getCachedMasterProductReviews(product.id), // No fallbacks: this page is cached, so an empty fallback would be @@ -185,14 +208,18 @@ export async function ProductDetailView({ categorySlug: categorySlug as CanonicalCategorySlug, limit: 50, }), - ] + ]); + const categoryPeers = storedCategoryPeers.map((peer) => + applyCategorySpecTemplate(categorySlug, peer) ); const [reviewSummary, priceHistory] = await Promise.all([ getCachedMasterProductReviewSummary(product.id, reviews), getCachedMasterProductPriceHistory(product.id, offers), ]); - const radarSignal = computeProductRadarSignal(product, offers); + // A sold-out listing's last price is no deal (DOS/Rate#1141). + const inStockOffers = offers.filter((o) => !isOfferUnavailable(o)); + const radarSignal = computeProductRadarSignal(product, inStockOffers); const radarNote = radarExplanation(radarSignal); const peerRadarIndex = buildRadarIndex(categoryDealRadar); @@ -200,12 +227,11 @@ export async function ProductDetailView({ product, categoryPeers ); - // The lowest plausible offer: a bait-price listing far below every other - // store does not set the "from" price (DOS/Rate#797). - const currentMinPrice = plausibleMinPrice( - Number(product.min_price || 0), - offers - ); + // "From" price, range and seller count: the stored stats every other page + // shows (in-stock listings, no bait prices), not the offer list below, + // which also holds sold-out listings at their last price (DOS/Rate#1141). + const marketStats = productMarketStats(product); + const currentMinPrice = marketStats.minPrice; const relatedCompetitors = rankRelatedCompetitors( categoryPeers, product.slug, @@ -215,14 +241,8 @@ export async function ProductDetailView({ const specs = product.specifications as ProductSpecifications; const keySpecs = product.key_specs as KeySpecs; const minPriceFormatted = currentMinPrice.toLocaleString('vi-VN'); - // Same meaning as the catalogue card's seller count (distinct merchants with - // a price), not the number of listings. - const sellerCount = new Set( - offers - .filter((o) => Number(o.price_current) > 0) - .map((o) => o.merchant?.id || o.id) - ).size; - const maxPriceFormatted = Number(product.max_price).toLocaleString('vi-VN'); + const sellerCount = marketStats.sellerCount; + const maxPriceFormatted = marketStats.maxPrice.toLocaleString('vi-VN'); const authenticCount = product.authentic_review_count && product.authentic_review_count > 0 @@ -278,19 +298,32 @@ export async function ProductDetailView({ }, sku: product.model_key, url: canonicalUrl, - description: t('metaDesc', { - name: product.name, - brand: product.brand, - merchantCount: product.merchant_count || 0, - }), + description: + currentMinPrice > 0 + ? t('metaDesc', { + name: product.name, + brand: product.brand, + merchantCount: sellerCount, + }) + : t('metaDescNoStock', { + name: product.name, + brand: product.brand, + }), image: usableImageUrls(product.images).map(absoluteUrl), - offers: { - '@type': 'AggregateOffer', - priceCurrency: 'VND', - lowPrice: currentMinPrice, - highPrice: Number(product.max_price || product.min_price || 0), - offerCount: offers.length || product.merchant_count || 0, - }, + // Only with a buyable price: a lowPrice of 0 is not an offer. + ...(currentMinPrice > 0 + ? { + offers: { + '@type': 'AggregateOffer', + priceCurrency: 'VND', + lowPrice: currentMinPrice, + highPrice: marketStats.maxPrice, + // Sellers, like "N nơi bán": one store's several listings of + // the model are one offer to a shopper. + offerCount: sellerCount, + }, + } + : {}), ...(displayUserRating > 0 && authenticCount > 0 ? { aggregateRating: { @@ -752,46 +785,51 @@ export async function ProductDetailView({ </div> </div> </div> - <div className="col-sm-4 col-6"> - <div className="bg-light rounded-3 h-100 border p-3"> - <div className="text-muted small"> - {categorySlug === 'earbuds' && - !keySpecs?.camera && - !specs?.camera?.main_camera - ? t('quickSpecAudioAnc') - : t('quickSpecMainCamera')} - </div> - <div className="fw-semibold small"> - <ContentText - text={String( - keySpecs?.camera || - specs?.camera?.main_camera || - specs?.multimedia?.audio_features || - '-' - )} - locale={locale} - /> + {(specTemplate.groups.includes('camera') || + categorySlug === 'earbuds') && ( + <div className="col-sm-4 col-6"> + <div className="bg-light rounded-3 h-100 border p-3"> + <div className="text-muted small"> + {categorySlug === 'earbuds' && + !keySpecs?.camera && + !specs?.camera?.main_camera + ? t('quickSpecAudioAnc') + : t('quickSpecMainCamera')} + </div> + <div className="fw-semibold small"> + <ContentText + text={String( + keySpecs?.camera || + specs?.camera?.main_camera || + specs?.multimedia?.audio_features || + '-' + )} + locale={locale} + /> + </div> </div> </div> - </div> - <div className="col-sm-4 col-6"> - <div className="bg-light rounded-3 h-100 border p-3"> - <div className="text-muted small"> - {t('quickSpecMemory')} - </div> - <div className="fw-semibold small"> - {(() => { - const ram = product.ram_gb ?? specs?.hardware?.ram_gb; - const rom = - product.storage_gb ?? specs?.hardware?.storage_gb; - if (ram && rom) return `${ram}GB / ${rom}GB`; - if (ram) return `${ram}GB RAM`; - if (rom) return `${rom}GB`; - return keySpecs?.ram || keySpecs?.storage || '-'; - })()} + )} + {(specTemplate.ram || specTemplate.storage) && ( + <div className="col-sm-4 col-6"> + <div className="bg-light rounded-3 h-100 border p-3"> + <div className="text-muted small"> + {t('quickSpecMemory')} + </div> + <div className="fw-semibold small"> + {(() => { + const ram = product.ram_gb ?? specs?.hardware?.ram_gb; + const rom = + product.storage_gb ?? specs?.hardware?.storage_gb; + if (ram && rom) return `${ram}GB / ${rom}GB`; + if (ram) return `${ram}GB RAM`; + if (rom) return `${rom}GB`; + return keySpecs?.ram || keySpecs?.storage || '-'; + })()} + </div> </div> </div> - </div> + )} <div className="col-sm-4 col-6"> <div className="bg-light rounded-3 h-100 border p-3"> <div className="text-muted small"> @@ -834,15 +872,18 @@ export async function ProductDetailView({ <div className="d-flex flex-column flex-sm-row justify-content-between align-items-start align-items-sm-center gap-2"> <div> <div className="text-muted small"> - {t('priceReferenceLabel', { count: offers.length })} + {t('priceReferenceLabel', { count: sellerCount })} </div> <div className="fw-bold text-dark rate-tabular fs-5"> - {minPriceFormatted} ₫ - {currentMinPrice !== Number(product.max_price || 0) && ( - <span className="small fw-normal text-muted ms-2"> - ~ {maxPriceFormatted} ₫ - </span> - )} + {currentMinPrice > 0 + ? `${minPriceFormatted} ₫` + : t('priceNoneInStock')} + {currentMinPrice > 0 && + currentMinPrice !== marketStats.maxPrice && ( + <span className="small fw-normal text-muted ms-2"> + ~ {maxPriceFormatted} ₫ + </span> + )} {priceHistory.lowestEverPrice != null && ( <span className="badge bg-success-subtle text-success fw-normal ms-2"> {t('allTimeLowBadge', { @@ -879,7 +920,7 @@ export async function ProductDetailView({ expertScore10={expertRating10} authenticReviews={authenticCount} minPriceVnd={currentMinPrice} - offerCount={offers.length} + offerCount={sellerCount} keySpecLine={keySpecLine} radarSignal={radarSignal} /> @@ -919,7 +960,7 @@ export async function ProductDetailView({ /> </div> - {/* SECTION #2: 10 Field Groups Specifications (Full 10/10 Canonical Schema per DESIGN.md v3.0) */} + {/* SECTION #2: Spec groups, per category template (DOS/Rate#1142) */} {(() => { const specUi = getSpecUiDictionary(locale); const loc = (v: string | number | boolean | null | undefined) => ( @@ -928,6 +969,577 @@ export async function ProductDetailView({ locale={locale} /> ); + const groupHeading = (group: SpecGroupKey) => + specGroupHeading( + specUi, + group, + specTemplate.groups.indexOf(group) + 1, + specTemplate.has5g + ); + const shows = (group: SpecGroupKey, field: string) => + specFieldApplies(specTemplate, group, field); + const groupNodes: Record<SpecGroupKey, ReactNode> = { + display: ( + <div className="col-md-6 col-12"> + <div className="rounded-4 h-100 border bg-white p-4"> + <h3 className="h6 fw-bold text-primary mb-3"> + {groupHeading('display')} + </h3> + <dl className="row small mb-0"> + <dt className="text-muted col-5"> + {specUi.fields.screenSize} + </dt> + <dd className="fw-medium col-7"> + {specs?.display?.size_inch + ? `${specs.display.size_inch} inch` + : product.screen_size_inch + ? `${product.screen_size_inch} inch` + : '-'} + </dd> + <dt className="text-muted col-5"> + {specUi.fields.panelType} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.display?.type || keySpecs?.screen)} + </dd> + <dt className="text-muted col-5"> + {specUi.fields.resolution} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.display?.resolution)} + </dd> + <dt className="text-muted col-5"> + {specUi.fields.refreshRate} + </dt> + <dd className="fw-medium col-7"> + {specs?.display?.refresh_rate_hz + ? `${specs.display.refresh_rate_hz} Hz` + : '-'} + </dd> + <dt className="text-muted col-5">{specUi.fields.ppi}</dt> + <dd className="fw-medium col-7"> + {specs?.display?.ppi ? `${specs.display.ppi} ppi` : '-'} + </dd> + <dt className="text-muted col-5"> + {specUi.fields.protection} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.display?.protection)} + </dd> + <dt className="text-muted col-5"> + {specUi.fields.displayFeatures} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.display?.features)} + </dd> + </dl> + </div> + </div> + ), + hardware: ( + <div className="col-md-6 col-12"> + <div className="rounded-4 h-100 border bg-white p-4"> + <h3 className="h6 fw-bold text-primary mb-3"> + {groupHeading('hardware')} + </h3> + <dl className="row small mb-0"> + <dt className="text-muted col-5"> + {specUi.fields.chipset} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.hardware?.chipset || keySpecs?.chip)} + </dd> + {shows('hardware', 'cpu') && ( + <> + <dt className="text-muted col-5"> + {specUi.fields.cpu} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.hardware?.cpu)} + </dd> + </> + )} + {shows('hardware', 'gpu') && ( + <> + <dt className="text-muted col-5"> + {specUi.fields.gpu} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.hardware?.gpu)} + </dd> + </> + )} + {shows('hardware', 'ram_gb') && ( + <> + <dt className="text-muted col-5"> + {specUi.fields.ram} + </dt> + <dd className="fw-medium col-7"> + {specs?.hardware?.ram_gb + ? `${specs.hardware.ram_gb} GB` + : product.ram_gb + ? `${product.ram_gb} GB` + : '-'} + </dd> + </> + )} + {shows('hardware', 'storage_gb') && ( + <> + <dt className="text-muted col-5"> + {specUi.fields.storage} + </dt> + <dd className="fw-medium col-7"> + {specs?.hardware?.storage_gb + ? `${specs.hardware.storage_gb} GB` + : product.storage_gb + ? `${product.storage_gb} GB` + : '-'} + </dd> + </> + )} + {shows('hardware', 'storage_expansion') && ( + <> + <dt className="text-muted col-5"> + {specUi.fields.storageExpansion} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.hardware?.storage_expansion)} + </dd> + </> + )} + {shows('hardware', 'os') && ( + <> + <dt className="text-muted col-5"> + {specUi.fields.os} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.hardware?.os)} + </dd> + </> + )} + </dl> + </div> + </div> + ), + camera: ( + <div className="col-md-6 col-12"> + <div className="rounded-4 h-100 border bg-white p-4"> + <h3 className="h6 fw-bold text-primary mb-3"> + {groupHeading('camera')} + </h3> + <dl className="row small mb-0"> + <dt className="text-muted col-5"> + {specUi.fields.rearSetup} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.camera?.rear_setup)} + </dd> + <dt className="text-muted col-5"> + {specUi.fields.mainCamera} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.camera?.main_camera || keySpecs?.camera)} + </dd> + <dt className="text-muted col-5"> + {specUi.fields.secondCamera} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.camera?.second_camera)} + </dd> + <dt className="text-muted col-5"> + {specUi.fields.thirdCamera} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.camera?.third_camera)} + </dd> + {specs?.camera?.fourth_camera && ( + <> + <dt className="text-muted col-5"> + {specUi.fields.fourthCamera} + </dt> + <dd className="fw-medium col-7"> + {loc(specs.camera.fourth_camera)} + </dd> + </> + )} + <dt className="text-muted col-5"> + {specUi.fields.frontCamera} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.camera?.front_camera)} + </dd> + <dt className="text-muted col-5"> + {specUi.fields.videoAndFlash} + </dt> + <dd className="fw-medium col-7"> + {loc( + [specs?.camera?.video_recording, specs?.camera?.flash] + .filter(Boolean) + .join(' | ') + )} + </dd> + </dl> + </div> + </div> + ), + battery: ( + <div className="col-md-6 col-12"> + <div className="rounded-4 h-100 border bg-white p-4"> + <h3 className="h6 fw-bold text-primary mb-3"> + {groupHeading('battery')} + </h3> + <dl className="row small mb-0"> + <dt className="text-muted col-5"> + {specUi.fields.batteryCapacity} + </dt> + <dd className="fw-medium col-7"> + {specs?.battery?.capacity_mah + ? `${specs.battery.capacity_mah} mAh` + : product.battery_mah + ? `${product.battery_mah} mAh` + : loc(keySpecs?.battery)} + </dd> + <dt className="text-muted col-5"> + {specUi.fields.batteryType} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.battery?.type)} + </dd> + <dt className="text-muted col-5"> + {specUi.fields.chargingTech} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.battery?.charging)} + </dd> + <dt className="text-muted col-5"> + {specUi.fields.fastCharging} + </dt> + <dd className="fw-medium col-7"> + {specs?.battery?.fast_charging_w + ? `${specs.battery.fast_charging_w} W` + : '-'} + </dd> + <dt className="text-muted col-5"> + {specUi.fields.reverseCharging} + </dt> + <dd className="fw-medium col-7"> + {typeof specs?.battery?.reverse_charging === 'string' + ? loc(specs.battery.reverse_charging) + : specs?.battery?.reverse_charging === true + ? specUi.supported + : specs?.battery?.reverse_charging === false + ? specUi.no + : '-'} + </dd> + </dl> + </div> + </div> + ), + design: ( + <div className="col-md-6 col-12"> + <div className="rounded-4 h-100 border bg-white p-4"> + <h3 className="h6 fw-bold text-primary mb-3"> + {groupHeading('design')} + </h3> + <dl className="row small mb-0"> + <dt className="text-muted col-5"> + {specUi.fields.dimensions} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.design?.dimensions)} + </dd> + <dt className="text-muted col-5"> + {specUi.fields.weight} + </dt> + <dd className="fw-medium col-7"> + {specs?.design?.weight_g + ? `${specs.design.weight_g} g` + : '-'} + </dd> + <dt className="text-muted col-5"> + {specUi.fields.materialsAndBuild} + </dt> + <dd className="fw-medium col-7"> + {loc( + [specs?.design?.materials, specs?.design?.build] + .filter(Boolean) + .join(' | ') + )} + </dd> + <dt className="text-muted col-5"> + {specUi.fields.resistance} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.design?.resistance)} + </dd> + {shows('design', 'biometrics') && ( + <> + <dt className="text-muted col-5"> + {specUi.fields.biometrics} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.design?.biometrics)} + </dd> + </> + )} + <dt className="text-muted col-5"> + {specUi.fields.physicalKeys} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.design?.keys)} + </dd> + </dl> + </div> + </div> + ), + cellular: ( + <div className="col-md-6 col-12"> + <div className="rounded-4 h-100 border bg-white p-4"> + <h3 className="h6 fw-bold text-primary mb-3"> + {groupHeading('cellular')} + </h3> + <dl className="row small mb-0"> + {shows('cellular', 'has_5g') && ( + <> + <dt className="text-muted col-5"> + {specUi.fields.support5g} + </dt> + <dd className="fw-medium col-7"> + {(specs?.cellular?.has_5g ?? product.has_5g) + ? specUi.has5gYes + : specUi.no} + </dd> + </> + )} + <dt className="text-muted col-5"> + {specUi.fields.networkTech} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.cellular?.technology)} + </dd> + <dt className="text-muted col-5"> + {specUi.fields.simType} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.cellular?.sim_type)} + </dd> + <dt className="text-muted col-5"> + {specUi.fields.dataSpeed} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.cellular?.data_speed)} + </dd> + <dt className="text-muted col-5"> + {shows('cellular', 'bands_5g') + ? specUi.fields.bands5g4g + : specUi.fields.bands4g} + </dt> + <dd className="fw-medium col-7"> + {loc( + [ + shows('cellular', 'bands_5g') + ? specs?.cellular?.bands_5g + : null, + specs?.cellular?.bands_4g, + ] + .filter(Boolean) + .join(' | ') + )} + </dd> + <dt className="text-muted col-5"> + {specUi.fields.volteVowifi} + </dt> + <dd className="fw-medium col-7"> + {specs?.cellular?.volte || specs?.cellular?.vowifi + ? [ + specs?.cellular?.volte ? 'VoLTE' : null, + specs?.cellular?.vowifi ? 'VoWiFi' : null, + ] + .filter(Boolean) + .join(' + ') + : '-'} + </dd> + </dl> + </div> + </div> + ), + connectivity: ( + <div className="col-md-6 col-12"> + <div className="rounded-4 h-100 border bg-white p-4"> + <h3 className="h6 fw-bold text-primary mb-3"> + {groupHeading('connectivity')} + </h3> + <dl className="row small mb-0"> + {shows('connectivity', 'wlan') && ( + <> + <dt className="text-muted col-5"> + {specUi.fields.wlan} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.connectivity?.wlan)} + </dd> + </> + )} + <dt className="text-muted col-5"> + {specUi.fields.bluetooth} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.connectivity?.bluetooth)} + </dd> + <dt className="text-muted col-5"> + {specUi.fields.usbPort} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.connectivity?.usb)} + </dd> + {shows('connectivity', 'nfc') && ( + <> + <dt className="text-muted col-5"> + {specUi.fields.nfc} + </dt> + <dd className="fw-medium col-7"> + {specs?.connectivity?.nfc === true + ? specUi.yes + : specs?.connectivity?.nfc === false + ? specUi.no + : '-'} + </dd> + </> + )} + {shows('connectivity', 'location') && ( + <> + <dt className="text-muted col-5"> + {specUi.fields.locationGps} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.connectivity?.location)} + </dd> + </> + )} + <dt className="text-muted col-5"> + {specUi.fields.sensorsAndOther} + </dt> + <dd className="fw-medium col-7"> + {loc( + [ + specs?.connectivity?.sensors, + specs?.connectivity?.other, + ] + .filter(Boolean) + .join(' | ') + )} + </dd> + </dl> + </div> + </div> + ), + multimedia: ( + <div className="col-md-6 col-12"> + <div className="rounded-4 h-100 border bg-white p-4"> + <h3 className="h6 fw-bold text-primary mb-3"> + {groupHeading('multimedia')} + </h3> + <dl className="row small mb-0"> + <dt className="text-muted col-5"> + {specUi.fields.loudspeaker} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.multimedia?.loudspeaker)} + </dd> + {shows('multimedia', 'headphone_jack_35mm') && ( + <> + <dt className="text-muted col-5"> + {specUi.fields.jack35mm} + </dt> + <dd className="fw-medium col-7"> + {specs?.multimedia?.headphone_jack_35mm === true + ? specUi.yes + : specs?.multimedia?.headphone_jack_35mm === false + ? specUi.no + : '-'} + </dd> + </> + )} + <dt className="text-muted col-5"> + {specUi.fields.audioFeatures} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.multimedia?.audio_features)} + </dd> + </dl> + </div> + </div> + ), + availability: ( + <div className="col-md-6 col-12"> + <div className="rounded-4 h-100 border bg-white p-4"> + <h3 className="h6 fw-bold text-primary mb-3"> + {groupHeading('availability')} + </h3> + <dl className="row small mb-0"> + <dt className="text-muted col-5"> + {specUi.fields.announcedDate} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.availability?.announced)} + </dd> + <dt className="text-muted col-5"> + {specUi.fields.commercialStatus} + </dt> + <dd className="fw-medium col-7"> + {specs?.availability?.status + ? loc(specs.availability.status) + : specUi.availableStatus} + </dd> + <dt className="text-muted col-5"> + {specUi.fields.msrpPrice} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.availability?.prices_msrp)} + </dd> + </dl> + </div> + </div> + ), + regulatory: ( + <div className="col-md-6 col-12"> + <div className="rounded-4 h-100 border bg-white p-4"> + <h3 className="h6 fw-bold text-primary mb-3"> + {groupHeading('regulatory')} + </h3> + <dl className="row small mb-0"> + <dt className="text-muted col-5"> + {specUi.fields.certifications} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.regulatory?.certifications)} + </dd> + <dt className="text-muted col-5"> + {specUi.fields.fccApproval} + </dt> + <dd className="fw-medium col-7"> + {loc(specs?.regulatory?.fcc_approval)} + </dd> + <dt className="text-muted col-5"> + {specUi.fields.sarLevel} + </dt> + <dd className="fw-medium col-7"> + {[ + specs?.regulatory?.sar_head + ? `${specUi.fields.sarHeadPrefix}: ${specs.regulatory.sar_head}` + : null, + specs?.regulatory?.sar_body + ? `${specUi.fields.sarBodyPrefix}: ${specs.regulatory.sar_body}` + : null, + ] + .filter(Boolean) + .join(' | ') || '-'} + </dd> + </dl> + </div> + </div> + ), + }; return ( <div id="specs-section" @@ -953,7 +1565,7 @@ export async function ProductDetailView({ radarSignal={radarSignal} /> <span className="badge bg-dark-subtle text-dark rounded-pill px-3 py-2"> - {specUi.sectionBadge} + {specSectionBadge(specUi, specTemplate.groups.length)} </span> </div> </div> @@ -964,518 +1576,9 @@ export async function ProductDetailView({ showLessLabel={t('specsShowLess')} openOnHash="#specs-section" > - {/* 1. Display */} - <div className="col-md-6 col-12"> - <div className="rounded-4 h-100 border bg-white p-4"> - <h3 className="h6 fw-bold text-primary mb-3"> - {specUi.groups.display} - </h3> - <dl className="row small mb-0"> - <dt className="text-muted col-5"> - {specUi.fields.screenSize} - </dt> - <dd className="fw-medium col-7"> - {specs?.display?.size_inch - ? `${specs.display.size_inch} inch` - : product.screen_size_inch - ? `${product.screen_size_inch} inch` - : '-'} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.panelType} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.display?.type || keySpecs?.screen)} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.resolution} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.display?.resolution)} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.refreshRate} - </dt> - <dd className="fw-medium col-7"> - {specs?.display?.refresh_rate_hz - ? `${specs.display.refresh_rate_hz} Hz` - : '-'} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.ppi} - </dt> - <dd className="fw-medium col-7"> - {specs?.display?.ppi - ? `${specs.display.ppi} ppi` - : '-'} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.protection} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.display?.protection)} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.displayFeatures} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.display?.features)} - </dd> - </dl> - </div> - </div> - - {/* 2. Hardware */} - <div className="col-md-6 col-12"> - <div className="rounded-4 h-100 border bg-white p-4"> - <h3 className="h6 fw-bold text-primary mb-3"> - {specUi.groups.hardware} - </h3> - <dl className="row small mb-0"> - <dt className="text-muted col-5"> - {specUi.fields.chipset} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.hardware?.chipset || keySpecs?.chip)} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.cpu} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.hardware?.cpu)} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.gpu} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.hardware?.gpu)} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.ram} - </dt> - <dd className="fw-medium col-7"> - {specs?.hardware?.ram_gb - ? `${specs.hardware.ram_gb} GB` - : product.ram_gb - ? `${product.ram_gb} GB` - : '-'} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.storage} - </dt> - <dd className="fw-medium col-7"> - {specs?.hardware?.storage_gb - ? `${specs.hardware.storage_gb} GB` - : product.storage_gb - ? `${product.storage_gb} GB` - : '-'} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.storageExpansion} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.hardware?.storage_expansion)} - </dd> - <dt className="text-muted col-5">{specUi.fields.os}</dt> - <dd className="fw-medium col-7"> - {loc(specs?.hardware?.os)} - </dd> - </dl> - </div> - </div> - - {/* 3. Camera (PhoneArena Split Architecture) */} - <div className="col-md-6 col-12"> - <div className="rounded-4 h-100 border bg-white p-4"> - <h3 className="h6 fw-bold text-primary mb-3"> - {specUi.groups.camera} - </h3> - <dl className="row small mb-0"> - <dt className="text-muted col-5"> - {specUi.fields.rearSetup} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.camera?.rear_setup)} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.mainCamera} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.camera?.main_camera || keySpecs?.camera)} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.secondCamera} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.camera?.second_camera)} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.thirdCamera} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.camera?.third_camera)} - </dd> - {specs?.camera?.fourth_camera && ( - <> - <dt className="text-muted col-5"> - {specUi.fields.fourthCamera} - </dt> - <dd className="fw-medium col-7"> - {loc(specs.camera.fourth_camera)} - </dd> - </> - )} - <dt className="text-muted col-5"> - {specUi.fields.frontCamera} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.camera?.front_camera)} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.videoAndFlash} - </dt> - <dd className="fw-medium col-7"> - {loc( - [ - specs?.camera?.video_recording, - specs?.camera?.flash, - ] - .filter(Boolean) - .join(' | ') - )} - </dd> - </dl> - </div> - </div> - - {/* 4. Battery */} - <div className="col-md-6 col-12"> - <div className="rounded-4 h-100 border bg-white p-4"> - <h3 className="h6 fw-bold text-primary mb-3"> - {specUi.groups.battery} - </h3> - <dl className="row small mb-0"> - <dt className="text-muted col-5"> - {specUi.fields.batteryCapacity} - </dt> - <dd className="fw-medium col-7"> - {specs?.battery?.capacity_mah - ? `${specs.battery.capacity_mah} mAh` - : product.battery_mah - ? `${product.battery_mah} mAh` - : loc(keySpecs?.battery)} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.batteryType} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.battery?.type)} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.chargingTech} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.battery?.charging)} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.fastCharging} - </dt> - <dd className="fw-medium col-7"> - {specs?.battery?.fast_charging_w - ? `${specs.battery.fast_charging_w} W` - : '-'} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.reverseCharging} - </dt> - <dd className="fw-medium col-7"> - {typeof specs?.battery?.reverse_charging === 'string' - ? loc(specs.battery.reverse_charging) - : specs?.battery?.reverse_charging === true - ? specUi.supported - : specs?.battery?.reverse_charging === false - ? specUi.no - : '-'} - </dd> - </dl> - </div> - </div> - - {/* 5. Design */} - <div className="col-md-6 col-12"> - <div className="rounded-4 h-100 border bg-white p-4"> - <h3 className="h6 fw-bold text-primary mb-3"> - {specUi.groups.design} - </h3> - <dl className="row small mb-0"> - <dt className="text-muted col-5"> - {specUi.fields.dimensions} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.design?.dimensions)} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.weight} - </dt> - <dd className="fw-medium col-7"> - {specs?.design?.weight_g - ? `${specs.design.weight_g} g` - : '-'} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.materialsAndBuild} - </dt> - <dd className="fw-medium col-7"> - {loc( - [specs?.design?.materials, specs?.design?.build] - .filter(Boolean) - .join(' | ') - )} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.resistance} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.design?.resistance)} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.biometrics} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.design?.biometrics)} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.physicalKeys} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.design?.keys)} - </dd> - </dl> - </div> - </div> - - {/* 6. Cellular */} - <div className="col-md-6 col-12"> - <div className="rounded-4 h-100 border bg-white p-4"> - <h3 className="h6 fw-bold text-primary mb-3"> - {specUi.groups.cellular} - </h3> - <dl className="row small mb-0"> - <dt className="text-muted col-5"> - {specUi.fields.support5g} - </dt> - <dd className="fw-medium col-7"> - {(specs?.cellular?.has_5g ?? product.has_5g) - ? specUi.has5gYes - : specUi.no} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.networkTech} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.cellular?.technology)} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.simType} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.cellular?.sim_type)} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.dataSpeed} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.cellular?.data_speed)} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.bands5g4g} - </dt> - <dd className="fw-medium col-7"> - {loc( - [ - specs?.cellular?.bands_5g, - specs?.cellular?.bands_4g, - ] - .filter(Boolean) - .join(' | ') - )} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.volteVowifi} - </dt> - <dd className="fw-medium col-7"> - {specs?.cellular?.volte || specs?.cellular?.vowifi - ? [ - specs?.cellular?.volte ? 'VoLTE' : null, - specs?.cellular?.vowifi ? 'VoWiFi' : null, - ] - .filter(Boolean) - .join(' + ') - : '-'} - </dd> - </dl> - </div> - </div> - - {/* 7. Connectivity */} - <div className="col-md-6 col-12"> - <div className="rounded-4 h-100 border bg-white p-4"> - <h3 className="h6 fw-bold text-primary mb-3"> - {specUi.groups.connectivity} - </h3> - <dl className="row small mb-0"> - <dt className="text-muted col-5"> - {specUi.fields.wlan} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.connectivity?.wlan)} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.bluetooth} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.connectivity?.bluetooth)} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.usbPort} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.connectivity?.usb)} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.nfc} - </dt> - <dd className="fw-medium col-7"> - {specs?.connectivity?.nfc === true - ? specUi.yes - : specs?.connectivity?.nfc === false - ? specUi.no - : '-'} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.locationGps} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.connectivity?.location)} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.sensorsAndOther} - </dt> - <dd className="fw-medium col-7"> - {loc( - [ - specs?.connectivity?.sensors, - specs?.connectivity?.other, - ] - .filter(Boolean) - .join(' | ') - )} - </dd> - </dl> - </div> - </div> - - {/* 8. Multimedia */} - <div className="col-md-6 col-12"> - <div className="rounded-4 h-100 border bg-white p-4"> - <h3 className="h6 fw-bold text-primary mb-3"> - {specUi.groups.multimedia} - </h3> - <dl className="row small mb-0"> - <dt className="text-muted col-5"> - {specUi.fields.loudspeaker} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.multimedia?.loudspeaker)} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.jack35mm} - </dt> - <dd className="fw-medium col-7"> - {specs?.multimedia?.headphone_jack_35mm === true - ? specUi.yes - : specs?.multimedia?.headphone_jack_35mm === false - ? specUi.no - : '-'} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.audioFeatures} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.multimedia?.audio_features)} - </dd> - </dl> - </div> - </div> - - {/* 9. Availability */} - <div className="col-md-6 col-12"> - <div className="rounded-4 h-100 border bg-white p-4"> - <h3 className="h6 fw-bold text-primary mb-3"> - {specUi.groups.availability} - </h3> - <dl className="row small mb-0"> - <dt className="text-muted col-5"> - {specUi.fields.announcedDate} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.availability?.announced)} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.commercialStatus} - </dt> - <dd className="fw-medium col-7"> - {specs?.availability?.status - ? loc(specs.availability.status) - : specUi.availableStatus} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.msrpPrice} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.availability?.prices_msrp)} - </dd> - </dl> - </div> - </div> - - {/* 10. Regulatory */} - <div className="col-md-6 col-12"> - <div className="rounded-4 h-100 border bg-white p-4"> - <h3 className="h6 fw-bold text-primary mb-3"> - {specUi.groups.regulatory} - </h3> - <dl className="row small mb-0"> - <dt className="text-muted col-5"> - {specUi.fields.certifications} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.regulatory?.certifications)} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.fccApproval} - </dt> - <dd className="fw-medium col-7"> - {loc(specs?.regulatory?.fcc_approval)} - </dd> - <dt className="text-muted col-5"> - {specUi.fields.sarLevel} - </dt> - <dd className="fw-medium col-7"> - {[ - specs?.regulatory?.sar_head - ? `${specUi.fields.sarHeadPrefix}: ${specs.regulatory.sar_head}` - : null, - specs?.regulatory?.sar_body - ? `${specUi.fields.sarBodyPrefix}: ${specs.regulatory.sar_body}` - : null, - ] - .filter(Boolean) - .join(' | ') || '-'} - </dd> - </dl> - </div> - </div> + {specTemplate.groups.map((group) => ( + <Fragment key={group}>{groupNodes[group]}</Fragment> + ))} </PhoneCollapsible> </div> ); @@ -1484,7 +1587,11 @@ export async function ProductDetailView({ {/* SECTION #3: Time-Series Price History Chart, Fake Discount Detector & Neutral Merchant Audit */} <PriceHistorySection productName={product.name} - history={priceHistory} + // Its "current lowest" is the page's "from" price (DOS/Rate#1141). + history={{ + ...priceHistory, + currentMinPrice: currentMinPrice > 0 ? currentMinPrice : null, + }} radarSignal={radarSignal} /> @@ -1533,9 +1640,7 @@ export async function ProductDetailView({ brand={product.brand} categorySlug={categorySlug} minPriceVnd={currentMinPrice} - maxPriceVnd={Number( - product.max_price || product.min_price || 0 - )} + maxPriceVnd={marketStats.maxPrice} userScore5={lbScored.userScore5} expertScore10={lbScored.expertScore10} valueCriterionScore5={lbScored.criteria.value} diff --git a/apps/web/src/app/[locale]/smartphones/page.tsx b/apps/web/src/app/[locale]/smartphones/page.tsx index 21cf48790..80cd949a6 100644 --- a/apps/web/src/app/[locale]/smartphones/page.tsx +++ b/apps/web/src/app/[locale]/smartphones/page.tsx @@ -41,6 +41,7 @@ import { BsExclamationTriangle, } from 'react-icons/bs'; import type { KeySpecs, ProductSpecifications } from '@repo/shared-data'; +import { applyCategorySpecTemplate } from '@repo/shared-data/rules'; import ClientMessages from '@/components/providers/ClientMessages'; import BootstrapClient from '@/app/BootstrapClient'; import { productImageUrl } from '@/lib/rate-domain-core'; @@ -403,7 +404,7 @@ export async function CategoryCatalogContent({ borderBottom: '1px solid rgba(255, 255, 255, 0.08)', }} > - <div className="position-relative z-1 container"> + <div className="position-relative z-1 rate-catalog-wide"> <div className="row justify-content-center align-items-center"> <div className="col-xl-12 col-lg-12 col-md-12 col-sm-12"> <div className="position-relative mt-2 text-start"> @@ -433,9 +434,9 @@ export async function CategoryCatalogContent({ {/* Top Hero Search Bar matching grid-layout-01 `.heroSearch.style-01` */} <div className="position-relative d-flex flex-column mt-n3 z-2"> - <div className="container"> + <div className="rate-catalog-wide"> <div className="row align-items-center justify-content-center"> - <div className="col-xl-11 col-lg-12 col-md-12 col-12"> + <div className="col-12"> <form method="GET" action={basePath} @@ -561,12 +562,14 @@ export async function CategoryCatalogContent({ </div> </div> - {/* Main Content Section matching grid-layout-01 */} - <section> - <div className="container"> - <div className="row g-4"> + {/* Full-width catalog (DOS/Rate#1132): a fixed-width filter sidebar + that sticks under the navbar, and results that take the rest of the + width up to the .rate-catalog-wide cap. */} + <section className="rate-catalog-section"> + <div className="rate-catalog-wide"> + <div className="rate-catalog-layout"> {/* Left Sidebar: Dynamic 10-Field-Groups Filter */} - <div className="col-xl-4 col-lg-4 col-md-12 col-sm-12"> + <div className="rate-catalog-sidebar"> <Suspense fallback={ <div className="bg-light rounded-4 text-muted small p-4"> @@ -583,7 +586,7 @@ export async function CategoryCatalogContent({ </div> {/* Right Listing Grid */} - <div className="col-xl-8 col-lg-8 col-md-12 col-sm-12"> + <div className="rate-catalog-results"> {/* Active Filter Chips matching `.alert.tag-alert` */} {activeChips.length > 0 && ( <div className="row align-items-start mb-4"> @@ -723,173 +726,6 @@ export async function CategoryCatalogContent({ dealRadar={categoryDealRadar} /> - {categoryDealRadar && - (categoryDealRadar.genuineDeals.length > 0 || - categoryDealRadar.fakeSaleTraps.length > 0) && ( - <div - className="rate-card rate-home-radar mb-4" - data-testid="category-deal-radar" - > - <div className="rate-radar-spread-head"> - <div> - <span className="rate-eyebrow mb-2"> - {t('page.radarBannerBadge', { - title: schema.categoryTitle, - })} - </span> - <div className="rate-radar-item-meta"> - {t('page.radarAnalyzedSub', { - count: categoryDealRadar.totalOffersAnalyzed, - })} - </div> - </div> - <Link href="/deals"> - {t('page.viewAllRadarBtn')} - <BsArrowRight aria-hidden="true" /> - </Link> - </div> - - {categoryDealRadar.genuineDeals.length > 0 && ( - <h3 className="rate-radar-list-title is-good mt-3"> - <BsGraphDownArrow aria-hidden="true" /> - {t('page.radarDealsTitle')} - </h3> - )} - <ul className="rate-radar-list"> - {categoryDealRadar.genuineDeals.slice(0, 2).map((g) => ( - <li key={g.id}> - <div className="rate-radar-item-main"> - <Link - href={g.productHref} - className="text-truncate" - > - {g.productName} - </Link> - <span className="rate-radar-item-meta text-truncate"> - <span className="rate-radar-platform"> - {g.platform} - </span> - <ContentText - text={g.merchantName} - locale={locale} - /> - </span> - </div> - <div className="rate-radar-item-side"> - <span className="rate-radar-price"> - {g.priceCurrent.toLocaleString('vi-VN')} ₫ - </span> - <span className="rate-radar-tag is-good"> - {t('page.vsMedianBadge', { - pct: g.savingsVsMedianPct, - })} - </span> - </div> - </li> - ))} - </ul> - {categoryDealRadar.fakeSaleTraps.length > 0 && ( - <h3 className="rate-radar-list-title is-bad mt-3"> - <BsExclamationTriangle aria-hidden="true" /> - {t('page.radarTrapsTitle')} - </h3> - )} - <ul className="rate-radar-list"> - {categoryDealRadar.fakeSaleTraps - .slice(0, 2) - .map((trap) => ( - <li key={trap.id}> - <div className="rate-radar-item-main"> - <Link - href={trap.productHref} - className="text-truncate" - > - {trap.productName} - </Link> - <span className="rate-radar-item-meta text-truncate"> - <span className="rate-radar-platform"> - {trap.platform} - </span> - <ContentText - text={trap.merchantName} - locale={locale} - /> - </span> - </div> - <div className="rate-radar-item-side"> - <span className="rate-radar-price"> - {trap.priceCurrent.toLocaleString('vi-VN')} ₫ - </span> - <span className="rate-radar-tag is-bad"> - {t('page.fakeMarkupBadge', { - pct: trap.fakeMarkupPct, - })} - </span> - </div> - </li> - ))} - </ul> - - {Array.isArray(categoryDealRadar.priceSpreadLeaders) && - categoryDealRadar.priceSpreadLeaders.length > 0 && ( - <div - className="rate-radar-spread" - data-testid="category-price-spread-leaders" - > - <div className="rate-radar-spread-head"> - <h3 className="rate-radar-list-title"> - <BsArrowLeftRight aria-hidden="true" /> - {t('page.spreadHeader', { - title: schema.categoryTitle, - })} - </h3> - <span className="rate-radar-item-meta mb-3"> - {t('page.spreadSub')} - </span> - </div> - <div className="row g-2"> - {categoryDealRadar.priceSpreadLeaders - .slice(0, 2) - .map((sp) => ( - <div key={sp.productSlug} className="col-12"> - <div className="rate-radar-spread-item"> - <div className="rate-radar-item-main"> - <Link - href={sp.productHref} - className="text-truncate" - > - {sp.productName} - </Link> - <span className="rate-radar-item-meta"> - <span className="rate-radar-platform"> - {sp.cheapestPlatform} - </span> - vs{' '} - <span className="rate-radar-platform"> - {sp.priciestPlatform} - </span> - </span> - </div> - <div className="rate-radar-item-side"> - <span className="rate-radar-tag is-warn"> - {t('page.spreadDiffBadge', { - pct: sp.spreadPct, - amount: - sp.spreadVnd.toLocaleString( - 'vi-VN' - ), - })} - </span> - </div> - </div> - </div> - ))} - </div> - </div> - )} - </div> - )} - {/* Product Cards Grid (DESIGN.md v3.0 Rating & Review-First .rate-card) */} {products.length === 0 ? ( <div className="card rounded-4 bg-light border-0 p-5 text-center shadow-sm"> @@ -934,8 +770,13 @@ export async function CategoryCatalogContent({ </div> </div> ) : ( - <div className="row align-items-stretch justify-content-start g-xl-4 g-3"> - {products.map((item) => { + <div className="rate-catalog-grid"> + {products.map((storedItem) => { + // Only the specs the category has (DOS/Rate#1142). + const item = applyCategorySpecTemplate( + categorySlug, + storedItem + ); const keySpecs = (item.key_specs || {}) as KeySpecs; const specs = (item.specifications || {}) as ProductSpecifications; @@ -1004,10 +845,7 @@ export async function CategoryCatalogContent({ } return ( - <div - className="col-xl-6 col-lg-6 col-md-6 col-sm-12 col-12" - key={item.id} - > + <div key={item.id}> <ProductCard slug={item.slug} href={`${basePath}/${item.slug}`} @@ -1072,6 +910,174 @@ export async function CategoryCatalogContent({ )} </nav> )} + {/* Deal radar sits after the results so the product list + starts right under the toolbar (DOS/Rate#1132). */} + {categoryDealRadar && + (categoryDealRadar.genuineDeals.length > 0 || + categoryDealRadar.fakeSaleTraps.length > 0) && ( + <div + className="rate-card rate-home-radar mt-4" + data-testid="category-deal-radar" + > + <div className="rate-radar-spread-head"> + <div> + <span className="rate-eyebrow mb-2"> + {t('page.radarBannerBadge', { + title: schema.categoryTitle, + })} + </span> + <div className="rate-radar-item-meta"> + {t('page.radarAnalyzedSub', { + count: categoryDealRadar.totalOffersAnalyzed, + })} + </div> + </div> + <Link href="/deals"> + {t('page.viewAllRadarBtn')} + <BsArrowRight aria-hidden="true" /> + </Link> + </div> + + {categoryDealRadar.genuineDeals.length > 0 && ( + <h3 className="rate-radar-list-title is-good mt-3"> + <BsGraphDownArrow aria-hidden="true" /> + {t('page.radarDealsTitle')} + </h3> + )} + <ul className="rate-radar-list"> + {categoryDealRadar.genuineDeals.slice(0, 2).map((g) => ( + <li key={g.id}> + <div className="rate-radar-item-main"> + <Link + href={g.productHref} + className="text-truncate" + > + {g.productName} + </Link> + <span className="rate-radar-item-meta text-truncate"> + <span className="rate-radar-platform"> + {g.platform} + </span> + <ContentText + text={g.merchantName} + locale={locale} + /> + </span> + </div> + <div className="rate-radar-item-side"> + <span className="rate-radar-price"> + {g.priceCurrent.toLocaleString('vi-VN')} ₫ + </span> + <span className="rate-radar-tag is-good"> + {t('page.vsMedianBadge', { + pct: g.savingsVsMedianPct, + })} + </span> + </div> + </li> + ))} + </ul> + {categoryDealRadar.fakeSaleTraps.length > 0 && ( + <h3 className="rate-radar-list-title is-bad mt-3"> + <BsExclamationTriangle aria-hidden="true" /> + {t('page.radarTrapsTitle')} + </h3> + )} + <ul className="rate-radar-list"> + {categoryDealRadar.fakeSaleTraps + .slice(0, 2) + .map((trap) => ( + <li key={trap.id}> + <div className="rate-radar-item-main"> + <Link + href={trap.productHref} + className="text-truncate" + > + {trap.productName} + </Link> + <span className="rate-radar-item-meta text-truncate"> + <span className="rate-radar-platform"> + {trap.platform} + </span> + <ContentText + text={trap.merchantName} + locale={locale} + /> + </span> + </div> + <div className="rate-radar-item-side"> + <span className="rate-radar-price"> + {trap.priceCurrent.toLocaleString('vi-VN')} ₫ + </span> + <span className="rate-radar-tag is-bad"> + {t('page.fakeMarkupBadge', { + pct: trap.fakeMarkupPct, + })} + </span> + </div> + </li> + ))} + </ul> + + {Array.isArray(categoryDealRadar.priceSpreadLeaders) && + categoryDealRadar.priceSpreadLeaders.length > 0 && ( + <div + className="rate-radar-spread" + data-testid="category-price-spread-leaders" + > + <div className="rate-radar-spread-head"> + <h3 className="rate-radar-list-title"> + <BsArrowLeftRight aria-hidden="true" /> + {t('page.spreadHeader', { + title: schema.categoryTitle, + })} + </h3> + <span className="rate-radar-item-meta mb-3"> + {t('page.spreadSub')} + </span> + </div> + <div className="row g-2"> + {categoryDealRadar.priceSpreadLeaders + .slice(0, 2) + .map((sp) => ( + <div key={sp.productSlug} className="col-12"> + <div className="rate-radar-spread-item"> + <div className="rate-radar-item-main"> + <Link + href={sp.productHref} + className="text-truncate" + > + {sp.productName} + </Link> + <span className="rate-radar-item-meta"> + <span className="rate-radar-platform"> + {sp.cheapestPlatform} + </span> + vs{' '} + <span className="rate-radar-platform"> + {sp.priciestPlatform} + </span> + </span> + </div> + <div className="rate-radar-item-side"> + <span className="rate-radar-tag is-warn"> + {t('page.spreadDiffBadge', { + pct: sp.spreadPct, + amount: + sp.spreadVnd.toLocaleString( + 'vi-VN' + ), + })} + </span> + </div> + </div> + </div> + ))} + </div> + </div> + )} + </div> + )} </div> </div> </div> diff --git a/apps/web/src/app/api/v1/rate/leaderboard/route.ts b/apps/web/src/app/api/v1/rate/leaderboard/route.ts index f167ba98d..42fb90e2f 100644 --- a/apps/web/src/app/api/v1/rate/leaderboard/route.ts +++ b/apps/web/src/app/api/v1/rate/leaderboard/route.ts @@ -1,6 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { computeProductRadarSignal, + getCachedFilteredReviewCountsByMaster, getCachedMasterProducts, getMasterProductOffersBatch, } from '@/server/rate'; @@ -54,11 +55,18 @@ export async function GET(req: NextRequest) { // Brand, price segment, query and the criterion ranking all run on this // list, so it must be the whole category, not its 100 best-rated models // (DOS/Rate#473). 500 is getMasterProducts' cap. - const allProducts = await getCachedMasterProducts({ - categorySlug: category, - limit: 500, - sort: 'rating_desc', - }); + const [allProducts, filteredReviewCounts] = await Promise.all([ + getCachedMasterProducts({ + categorySlug: category, + limit: 500, + sort: 'rating_desc', + }), + // A failed count read shows no blocked count (still true) instead of + // failing the whole ranking (DOS/Rate#1139). + getCachedFilteredReviewCountsByMaster().catch( + () => ({}) as Record<string, number> + ), + ]); const ranked = rankMasterProductsForLeaderboard(allProducts, { category, @@ -66,6 +74,7 @@ export async function GET(req: NextRequest) { priceSegment, brand, query, + filteredReviewCounts, }); const pageRanked = ranked.slice(0, limit); diff --git a/apps/web/src/app/api/v1/rate/products/[slug]/route.ts b/apps/web/src/app/api/v1/rate/products/[slug]/route.ts index 92181f89f..37dea9fe0 100644 --- a/apps/web/src/app/api/v1/rate/products/[slug]/route.ts +++ b/apps/web/src/app/api/v1/rate/products/[slug]/route.ts @@ -7,6 +7,7 @@ import { getCachedMasterProductReviewSummary, } from '@/server/rate'; import { computeProductRadarSignal } from '@/server/rate/deals'; +import { isOfferUnavailable } from '@/lib/product-market-stats'; import { PUBLIC_API_CORS_HEADERS, jsonApiError, @@ -64,7 +65,11 @@ export async function GET(_request: NextRequest, context: RouteContext) { getCachedMasterProductReviewSummary(product.id, reviews), getCachedMasterProductPriceHistory(product.id, offers), ]); - const radarSignal = computeProductRadarSignal(product, offers); + // A sold-out listing's last price is no deal (DOS/Rate#1141). + const radarSignal = computeProductRadarSignal( + product, + offers.filter((o) => !isOfferUnavailable(o)) + ); return NextResponse.json( { diff --git a/apps/web/src/app/api/v1/rate/versus/route.ts b/apps/web/src/app/api/v1/rate/versus/route.ts index 3d70254f2..939efab2d 100644 --- a/apps/web/src/app/api/v1/rate/versus/route.ts +++ b/apps/web/src/app/api/v1/rate/versus/route.ts @@ -2,6 +2,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { computeProductRadarSignal, getCachedCompareMasterProducts, + getCachedFilteredReviewCountsByMaster, getMasterProductCategorySlug, getMasterProductOffersBatch, } from '@/server/rate'; @@ -191,8 +192,22 @@ export async function GET(req: NextRequest) { const prodB = compared[1]; const category = getMasterProductCategorySlug(prodA); - const scoreA = scoreMasterProductForLeaderboard(prodA, 'combined'); - const scoreB = scoreMasterProductForLeaderboard(prodB, 'combined'); + // filtered_noise_review_count counts stored reviews only (DOS/Rate#1139). + // On a read failure no product claims a blocked count, which is still true. + const filteredReviewCounts = + await getCachedFilteredReviewCountsByMaster().catch( + () => ({}) as Record<string, number> + ); + const scoreA = scoreMasterProductForLeaderboard( + prodA, + 'combined', + filteredReviewCounts + ); + const scoreB = scoreMasterProductForLeaderboard( + prodB, + 'combined', + filteredReviewCounts + ); const requestedPersonaId = ( parseBoundedStringParam(searchParams.get('persona'), 32) || 'balanced' diff --git a/apps/web/src/app/api/v1/rate/watchlist/route.ts b/apps/web/src/app/api/v1/rate/watchlist/route.ts index 3d3c865a4..0eedeb661 100644 --- a/apps/web/src/app/api/v1/rate/watchlist/route.ts +++ b/apps/web/src/app/api/v1/rate/watchlist/route.ts @@ -14,7 +14,6 @@ import { parseWatchlistSlugsParam, resolveWatchlistMasterPriceBounds, selectBestOfferFromListings, - withPlausibleMinPrice, } from '@/server/rate/api-handler'; import { authenticateRateRequest, @@ -130,8 +129,10 @@ export async function GET(request: NextRequest) { const { bestOffer, offerPlatforms } = selectBestOfferFromListings(offers); const radarSignal = computeProductRadarSignal(master, offers); const { currentMinPrice, currentMaxPrice } = + // The stored stats every page shows: in stock, no bait prices + // (DOS/Rate#802, #1141). resolveWatchlistMasterPriceBounds( - withPlausibleMinPrice(master, offers), + master, bestOffer?.price, MAX_VALID_VND_PRICE ); @@ -247,7 +248,7 @@ export async function POST(request: NextRequest) { const { bestOffer, offerPlatforms } = selectBestOfferFromListings(offers); const radarSignal = computeProductRadarSignal(masterProduct, offers); const { currentMinPrice } = resolveWatchlistMasterPriceBounds( - withPlausibleMinPrice(masterProduct, offers), + masterProduct, bestOffer?.price, MAX_VALID_VND_PRICE ); diff --git a/apps/web/src/app/style/scss/_catalog.scss b/apps/web/src/app/style/scss/_catalog.scss index 119c37b05..91ecb8338 100644 --- a/apps/web/src/app/style/scss/_catalog.scss +++ b/apps/web/src/app/style/scss/_catalog.scss @@ -130,3 +130,87 @@ .rate-export-menu-list { max-width: calc(100vw - 1rem); } + +// Full-width catalog layout (DOS/Rate#1132). The hero, search bar and +// results share one wide frame instead of the 1320px .container: a fixed +// filter sidebar on the left and a results grid that adds columns as the +// screen widens, capped at 1800px so rows stay readable on large monitors. +.rate-catalog-wide { + width: 100%; + max-width: 1800px; + margin-inline: auto; + padding-inline: 1rem; + + @media (min-width: 992px) { + padding-inline: 1.5rem; + } +} + +// The template's 80px section padding left a tall gap under the search bar; +// start the results closer to it. +.rate-catalog-section { + padding: 2rem 0 4rem; +} + +.rate-catalog-layout { + display: grid; + grid-template-columns: minmax(0, 1fr); + gap: 1.5rem; + + @media (min-width: 992px) { + grid-template-columns: 280px minmax(0, 1fr); + align-items: start; + } + + @media (min-width: 1600px) { + grid-template-columns: 300px minmax(0, 1fr); + } +} + +// From 992px the filter panel is inline (style.scss .offcanvas.largeshow). +// Keep it under the sticky navbar while the results scroll, and let it +// scroll on its own when it is taller than the window. top clears the +// ~73px navbar, as #main-content's 5rem scroll-margin-top does. Scroll +// chaining stays on, so at the sidebar's end the wheel moves the page. +// Below 992px it is the off-canvas drawer, which must not get a sticky or +// scrolling parent. +@media (min-width: 992px) { + .rate-catalog-sidebar { + position: sticky; + top: 5.5rem; + max-height: calc(100vh - 6.5rem); + overflow-y: auto; + scrollbar-width: thin; + } +} + +// Results: as many 290px+ cards as fit (1 on phones, 2 on tablets, 3 at +// 1280-1440px, 4 at 1920px). min() keeps a single column inside 320px. +.rate-catalog-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(min(290px, 100%), 1fr)); + gap: 1rem; + + @media (min-width: 1200px) { + gap: 1.25rem; + } + + // Near the 290px minimum, longer copy (cs) must not overflow the card: + // the radar badge wraps (Bootstrap's .badge nowrap otherwise wins over + // .rate-product-card-radar in _custom.scss), and the second button drops + // to its own row instead of breaking its label over two lines. + .rate-product-card-radar { + max-width: 100%; + white-space: normal; + text-align: left; + } + + .rate-product-card-cta { + flex-wrap: wrap; + + .btn { + flex: 1 1 auto; + white-space: nowrap; + } + } +} diff --git a/apps/web/src/components/inspect/DeepInspectWorkspace.tsx b/apps/web/src/components/inspect/DeepInspectWorkspace.tsx index e6617ff01..6ac6f976a 100644 --- a/apps/web/src/components/inspect/DeepInspectWorkspace.tsx +++ b/apps/web/src/components/inspect/DeepInspectWorkspace.tsx @@ -56,6 +56,7 @@ import { implausiblyLowOfferIds, plausibleOffers, } from '@repo/shared-data/rules'; +import { isOfferUnavailable } from '@/lib/product-market-stats'; import { productImageUrl } from '@/lib/rate-domain-core'; const INSPECT_CONTROL_CHARS_RE = /\p{Cc}+/gu; @@ -184,8 +185,13 @@ export default function DeepInspectWorkspace({ }; const master = result?.masterProduct ?? null; + // A sold-out listing is never the fallback match (DOS/Rate#1177). const matchedOffer = - result?.matchedOffer ?? plausibleOffers(result?.allOffers ?? [])[0] ?? null; + result?.matchedOffer ?? + plausibleOffers( + (result?.allOffers ?? []).filter((o) => !isOfferUnavailable(o)) + )[0] ?? + null; const discount = result?.discountAnalysis ?? null; // Judged as the product page's offers table judges the row (#797). const implausibleOfferIds = implausiblyLowOfferIds(result?.allOffers ?? []); diff --git a/apps/web/src/components/leaderboard/LeaderboardExplorer.tsx b/apps/web/src/components/leaderboard/LeaderboardExplorer.tsx index 8859e2762..5b0cf660c 100644 --- a/apps/web/src/components/leaderboard/LeaderboardExplorer.tsx +++ b/apps/web/src/components/leaderboard/LeaderboardExplorer.tsx @@ -18,6 +18,7 @@ import { normalizeLeaderboardCriterion, scoreMasterProductForLeaderboard, type CanonicalCategorySlug, + type FilteredReviewCountsByMaster, type LeaderboardCriterionKey, type LeaderboardPriceSegment, type MasterProduct, @@ -112,6 +113,8 @@ interface LeaderboardExplorerProps { readonly products: readonly MasterProduct[]; readonly dealRadar?: CrossPlatformDealRadarResponse | null; readonly locale?: string; + /** Stored reviews kept out of the rating, per master id (DOS/Rate#1139). */ + readonly filteredReviewCounts?: FilteredReviewCountsByMaster; } interface RankedProductEntry { @@ -173,9 +176,14 @@ function buildCategoryTabs(t: TranslateFn): ReadonlyArray<{ function computeProductScores( item: MasterProduct, - criterion: LeaderboardCriterionKey + criterion: LeaderboardCriterionKey, + filteredReviewCounts?: FilteredReviewCountsByMaster ): RankedProductEntry { - const scored = scoreMasterProductForLeaderboard(item, criterion); + const scored = scoreMasterProductForLeaderboard( + item, + criterion, + filteredReviewCounts + ); return { product: scored.product, categorySlug: scored.categorySlug, @@ -218,6 +226,7 @@ function TabFromUrl({ export default function LeaderboardExplorer({ products, dealRadar, + filteredReviewCounts, }: LeaderboardExplorerProps) { const t = useTranslations('leaderboard'); const locale = useLocale(); @@ -464,7 +473,11 @@ export default function LeaderboardExplorer({ const scored = filtered .map((item) => { - const base = computeProductScores(item, activeCriterion); + const base = computeProductScores( + item, + activeCriterion, + filteredReviewCounts + ); if (activePersona !== 'none' && totalWeight > 0) { const weighted5 = Number( ( @@ -524,6 +537,7 @@ export default function LeaderboardExplorer({ activeCriterion, activePersona, customWeights, + filteredReviewCounts, ]); const top3Slugs = useMemo( diff --git a/apps/web/src/components/leaderboard/ProductOffersExplorer.tsx b/apps/web/src/components/leaderboard/ProductOffersExplorer.tsx index 73bbd9c4a..81aada4cc 100644 --- a/apps/web/src/components/leaderboard/ProductOffersExplorer.tsx +++ b/apps/web/src/components/leaderboard/ProductOffersExplorer.tsx @@ -25,6 +25,7 @@ import { } from '@/lib/deal-radar'; import { escapeCsvCell } from '@/lib/csv-safe'; import { implausiblyLowOfferIds } from '@repo/shared-data/rules'; +import { isOfferUnavailable } from '@/lib/product-market-stats'; import { contentLang } from '@/lib/content-lang'; import ContentText from '@/components/elementary/ContentText'; @@ -123,7 +124,8 @@ export function computeOffersPriceSpreadAudit( Number.isFinite(p) && p > 0 && p <= MAX_VALID_VND_PRICE && - !implausibleOfferIds.has(o.id) + !implausibleOfferIds.has(o.id) && + !isOfferUnavailable(o) ); }); if (valid.length < 2) return null; @@ -183,9 +185,11 @@ export default function ProductOffersExplorer({ }, [safeOffers]); // Bait-price listings far below every other store (DOS/Rate#797): labelled, - // listed after the plausible offers and left out of the price spread. + // listed after the plausible offers and left out of the price spread. Judged + // among the in-stock listings, as the stored min_price is (DOS/Rate#1141). const implausibleOfferIds = useMemo( - () => implausiblyLowOfferIds(safeOffers), + () => + implausiblyLowOfferIds(safeOffers.filter((o) => !isOfferUnavailable(o))), [safeOffers] ); @@ -212,6 +216,10 @@ export default function ProductOffersExplorer({ }); return [...base].sort((a, b) => { + // Sold-out listings keep their last price: listed last (DOS/Rate#1141). + const unavailableA = isOfferUnavailable(a) ? 1 : 0; + const unavailableB = isOfferUnavailable(b) ? 1 : 0; + if (unavailableA !== unavailableB) return unavailableA - unavailableB; const implausibleA = implausibleOfferIds.has(a.id) ? 1 : 0; const implausibleB = implausibleOfferIds.has(b.id) ? 1 : 0; if (implausibleA !== implausibleB) return implausibleA - implausibleB; @@ -710,6 +718,15 @@ export default function ProductOffersExplorer({ {t('offersImplausiblyLowBadge')} </span> )} + {isOfferUnavailable(offer) && ( + <span + className="badge bg-secondary-subtle text-secondary small" + data-testid="offer-unavailable" + title={t('offersUnavailableTitle')} + > + {t('offersUnavailableBadge')} + </span> + )} </div> </div> </td> diff --git a/apps/web/src/components/reviews/JReviewsSection.tsx b/apps/web/src/components/reviews/JReviewsSection.tsx index 1a7c40a0f..c8b714e30 100644 --- a/apps/web/src/components/reviews/JReviewsSection.tsx +++ b/apps/web/src/components/reviews/JReviewsSection.tsx @@ -16,6 +16,7 @@ import { BsStarFill, } from 'react-icons/bs'; import { contentLang } from '@/lib/content-lang'; +import { computeJReviewsUserScores } from '@/lib/jreviews-user-score'; import { escapeCsvCell } from '@/lib/csv-safe'; import { computeCombinedDualScore, @@ -270,40 +271,12 @@ export default function JReviewsSection({ (r) => r.reviewer_type === 'expert' && r.helpfulness_weight > 0 ); - let computedUser: number | null = - userRatingWeighted && userRatingWeighted > 0 ? userRatingWeighted : null; - let userWeightSum = 0; - let userWeightedStars = 0; - if (validUserReviews.length > 0) { - userWeightSum = validUserReviews.reduce( - (acc, r) => acc + r.helpfulness_weight, - 0 - ); - userWeightedStars = validUserReviews.reduce( - (acc, r) => acc + r.rating_star * r.helpfulness_weight, - 0 - ); - if (userWeightSum > 0) { - computedUser = Number((userWeightedStars / userWeightSum).toFixed(2)); - } - } else if (computedUser == null && averageRating > 0) { - computedUser = Number(Number(averageRating).toFixed(2)); - } - - const priorMean = 4.2; - const priorStrength = 3; - const bayesianUserScore = - userWeightSum > 0 - ? Number( - ( - (priorStrength * priorMean + userWeightedStars) / - (priorStrength + userWeightSum) - ).toFixed(2) - ) - : computedUser; - const bayesianConfidence = Number( - (userWeightSum / (userWeightSum + priorStrength)).toFixed(2) - ); + const { + user: computedUser, + bayesianUser: bayesianUserScore, + bayesianConfidence, + userWeightSum, + } = computeJReviewsUserScores(reviews, userRatingWeighted, averageRating); const confidenceTier: 'insufficient_data' | 'low' | 'medium' | 'high' = bayesianConfidence < 0.3 ? 'insufficient_data' @@ -336,8 +309,10 @@ export default function JReviewsSection({ } } + // The buyer score the header shows, so a 100% buyer composite equals it + // (DOS/Rate#1144). const dualScore = computeCombinedDualScore({ - userScore: computedUser, + userScore: bayesianUserScore, expertScore: computedExpert, userConfidence: bayesianConfidence, fallbackAverage: averageRating, diff --git a/apps/web/src/lib/__tests__/client-messages.test.ts b/apps/web/src/lib/__tests__/client-messages.test.ts index 872885bf2..939dafa79 100644 --- a/apps/web/src/lib/__tests__/client-messages.test.ts +++ b/apps/web/src/lib/__tests__/client-messages.test.ts @@ -118,6 +118,7 @@ function main() { categoryFilter: 'all', totalClustersAnalyzed: 1, totalOffersAnalyzed: 2, + totalMerchantsAnalyzed: 2, genuineDeals: [ { id: 'gd-ctrl', @@ -142,6 +143,7 @@ function main() { marketMaxPrice: 28_000_000, savingsVsMedianVnd: 2_000_000, savingsVsMedianPct: 7.7, + marketOfferCount: 3, claimedDiscountPct: 14.3, fakeMarkupPct: 0, jreviewsScore10: 9.0, diff --git a/apps/web/src/lib/__tests__/display-truth.test.ts b/apps/web/src/lib/__tests__/display-truth.test.ts new file mode 100644 index 000000000..6533e2b87 --- /dev/null +++ b/apps/web/src/lib/__tests__/display-truth.test.ts @@ -0,0 +1,226 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { computeCrossPlatformDealRadarFromData } from '../../server/rate/deals'; +import { merchantRiskTone } from '../merchant-risk-tone'; +import { radarExplanation } from '../product-detail-copy'; +import { buildRadarIndex } from '../deal-radar'; +import { computeJReviewsUserScores } from '../jreviews-user-score'; +import { + computeBayesianRatingSummary, + computeCombinedDualScore, +} from '../rate-domain-core'; + +// Display defects from the live audit #1131: numbers and warnings the site +// showed that were not true. + +type Messages = Record<string, Record<string, string>>; +const readLocale = (locale: string): Messages => + JSON.parse( + readFileSync( + join(__dirname, '..', '..', '..', 'locales', `${locale}.json`), + 'utf8' + ) + ); +const LOCALES = ['vi', 'en', 'cs'] as const; + +// DOS/Rate#1140: one risk cut-off for the home cards and the merchant page. +// Every merchant scored 4, 11 or 14 and the detail page called them all scams. +for (const score of [0, 4, 11, 14, 19]) { + assert.equal(merchantRiskTone(score), 'good', `DOSafe ${score} is clean`); +} +for (const score of [20, 35, 49]) { + assert.equal(merchantRiskTone(score), 'warn', `DOSafe ${score} is caution`); +} +for (const score of [50, 60, 100]) { + assert.equal(merchantRiskTone(score), 'bad', `DOSafe ${score} is an alert`); +} +for (const unknown of [null, undefined, Number.NaN, 'abc']) { + assert.equal( + merchantRiskTone(unknown), + 'warn', + 'an unknown score is neither clean nor an alert' + ); +} +for (const locale of LOCALES) { + const merchants = readLocale(locale).merchants!; + for (const key of [ + 'scamStatusClean', + 'scamStatusCaution', + 'scamStatusAlert', + 'layer2NoReviews', + ]) { + assert.ok(merchants[key], `${locale}: missing merchants.${key}`); + } + assert.ok( + !/\d+\s*%/.test(merchants.layer2NoReviews!), + `${locale}: the no-reviews text must not show a percentage` + ); +} +const merchantPage = readFileSync( + join( + __dirname, + '..', + '..', + 'app', + '[locale]', + 'merchants', + '[id]', + 'page.tsx' + ), + 'utf8' +); +assert.ok( + merchantPage.includes('merchantRiskTone(merchant.dosafe_risk_score)') && + !merchantPage.includes('dosafe_risk_score === 0'), + 'the merchant page must use the shared risk cut-off, not "any non-zero score"' +); + +// DOS/Rate#1145: the median sentence names the real number of offers, and the +// home radar calls offers listings and merchants merchants. +const master = { + id: 'mp-aw-ultra-2', + slug: 'apple-watch-ultra-2-64gb', + name: 'Apple Watch Ultra 2 64GB', + brand: 'Apple', + category_id: 'smartwatches', + metadata: { category_slug: 'smartwatches' }, +}; +const offer = (i: number, merchantId: string, price: number) => ({ + id: `offer-${i}`, + platform: i === 0 ? 'cellphones' : 'shopee', + price_current: price, + price_original: null, + url: `https://example.com/${i}`, + merchant: { + id: merchantId, + name: `Store ${merchantId}`, + platform: i === 0 ? 'cellphones' : 'shopee', + risk_score: 4, + is_official: true, + }, +}); +const radar = computeCrossPlatformDealRadarFromData( + [ + { + master: master as never, + offers: [ + offer(0, 'm-a', 16_490_000), + offer(1, 'm-b', 23_490_000), + ] as never, + }, + ], + { categorySlug: 'all', limit: 10 } +); +assert.equal(radar.totalOffersAnalyzed, 2); +assert.equal(radar.totalMerchantsAnalyzed, 2); +const deal = radar.genuineDeals[0]; +assert.ok(deal, 'the cheaper of two offers is still below their median'); +assert.equal(deal.marketOfferCount, 2); +assert.ok( + !/10 sàn/i.test(deal.explanationVi) && !/10 Sàn/.test(deal.verdictBadgeVi), + `no "10 sàn" claim from 2 offers: ${deal.explanationVi} / ${deal.verdictBadgeVi}` +); +assert.match(deal.explanationVi, /trung vị của 2 tin bán/); + +const signal = buildRadarIndex(radar).get(master.slug)!; +const note = radarExplanation(signal); +assert.equal(note?.key, 'radarExplanationGenuineDeal'); +assert.equal(note!.values.count, '2'); +// Without a known offer count there is no median sentence at all. +assert.equal( + radarExplanation({ + ...signal, + genuineDeal: { + ...signal.genuineDeal!, + marketOfferCount: undefined as never, + }, + }), + null +); +for (const locale of LOCALES) { + const merchants = readLocale(locale).merchants!; + for (const key of ['radarGenuineBadge', 'profileRadarGenuineCount']) { + assert.ok( + !/giá đáy|cross-store low|minim napříč/i.test(merchants[key]!), + `${locale}: merchants.${key} counts every genuine deal, not floors` + ); + } +} +for (const locale of LOCALES) { + const messages = readLocale(locale); + const sentence = messages.productDetail!.radarExplanationGenuineDeal!; + assert.ok( + sentence.includes('{count}'), + `${locale}: median sentence needs {count}` + ); + assert.ok( + !/10[ -](sàn|platform|platforem)/i.test(sentence), + `${locale}: median sentence must not claim 10 platforms` + ); + const home = messages.homePage!.radarHomeAnalyzed!; + assert.ok( + home.includes('{offers}') && home.includes('{merchants}'), + `${locale}: the home radar line needs both offers and merchants` + ); +} +assert.match( + readLocale('vi').homePage!.radarHomeAnalyzed!, + /\{offers\} tin bán của \{merchants\} gian hàng/ +); + +// One merchant listing twice counts once. +const sameMerchant = computeCrossPlatformDealRadarFromData( + [ + { + master: master as never, + offers: [ + offer(0, 'm-a', 16_490_000), + offer(1, 'm-a', 23_490_000), + offer(2, 'm-b', 20_000_000), + ] as never, + }, + ], + { categorySlug: 'all', limit: 10 } +); +assert.equal(sameMerchant.totalOffersAnalyzed, 3); +assert.equal(sameMerchant.totalMerchantsAnalyzed, 2); + +// DOS/Rate#1144: with no expert score, the composite equals the buyer score +// the header shows. 9 rating-only 5-star reviews (weight 0.25 each) are 4.54 +// in the header and were 5.0 in the composite. +const ratingOnly = Array.from({ length: 9 }, () => ({ + reviewer_type: 'user', + include_in_rating: true, + helpfulness_weight: 0.25, + rating_star: 5, +})); +const header = computeBayesianRatingSummary(ratingOnly, 5).bayesianRating; +const live = computeJReviewsUserScores(ratingOnly, 5, 5); +assert.equal(header, 4.54); +assert.equal(live.bayesianUser, header); +assert.equal( + live.user, + 5, + 'the plain weighted mean stays shown as TB trọng số' +); +const composite = computeCombinedDualScore({ + userScore: live.bayesianUser, + expertScore: null, + userConfidence: live.bayesianConfidence, + fallbackAverage: 5, +}); +assert.equal(composite.blendMode, 'user_only'); +assert.equal(composite.overall, header); +assert.equal(composite.overall.toFixed(1), header.toFixed(1)); +const reviewsSection = readFileSync( + join(__dirname, '..', '..', 'components', 'reviews', 'JReviewsSection.tsx'), + 'utf8' +); +assert.match( + reviewsSection, + /computeCombinedDualScore\(\{\s*userScore: bayesianUserScore,/, + 'the reviews section composite must read the Bayesian buyer score' +); + +console.log('display-truth: ok'); diff --git a/apps/web/src/lib/__tests__/product-market-stats.test.ts b/apps/web/src/lib/__tests__/product-market-stats.test.ts new file mode 100644 index 000000000..7b4eb1e59 --- /dev/null +++ b/apps/web/src/lib/__tests__/product-market-stats.test.ts @@ -0,0 +1,194 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + isOfferUnavailable, + productMarketStats, + toOfferAvailability, +} from '../product-market-stats'; + +const here = dirname(fileURLToPath(import.meta.url)); +const read = (rel: string) => readFileSync(join(here, rel), 'utf8'); + +async function main() { + console.warn('🧪 Product market stats suite (DOS/Rate#1141)'); + + // The live Galaxy Buds3 Pro case: both FPT Shop listings are sold out, so + // the stored stats count TGDD and CellphoneS only. The product page used to + // show 2,694,000 from 3 sellers; every other page showed 3,140,000 from 2. + const buds = { + min_price: 3140000, + max_price: 3690000, + merchant_count: 2, + }; + assert.deepEqual(productMarketStats(buds), { + minPrice: 3140000, + maxPrice: 3690000, + sellerCount: 2, + }); + // PostgREST returns numeric columns as strings. + assert.deepEqual( + productMarketStats({ + min_price: '16490000', + max_price: '16490000', + merchant_count: 1, + }), + { minPrice: 16490000, maxPrice: 16490000, sellerCount: 1 } + ); + // No in-stock listing: no price and no sellers, never "from 0 at 3 stores". + assert.deepEqual( + productMarketStats({ min_price: null, max_price: null, merchant_count: 3 }), + { minPrice: 0, maxPrice: 0, sellerCount: 0 } + ); + assert.deepEqual(productMarketStats(null), { + minPrice: 0, + maxPrice: 0, + sellerCount: 0, + }); + assert.equal( + productMarketStats({ min_price: 500, max_price: 500, merchant_count: null }) + .sellerCount, + 1, + 'A stored price has at least one seller' + ); + // A max below the min (a partial write) never shows a reversed range. + assert.equal( + productMarketStats({ min_price: 500, max_price: 100, merchant_count: 1 }) + .maxPrice, + 500 + ); + assert.equal( + productMarketStats({ min_price: 2e9, max_price: 2e9, merchant_count: 1 }) + .minPrice, + 0, + 'A price above the VND ceiling is not a price' + ); + + assert.equal(isOfferUnavailable({ availability: 'out_of_stock' }), true); + assert.equal(isOfferUnavailable({ availability: 'discontinued' }), true); + assert.equal(isOfferUnavailable({ availability: 'in_stock' }), false); + assert.equal( + isOfferUnavailable({ availability: null }), + false, + 'An unread availability counts as in stock, as the column default does' + ); + assert.equal(isOfferUnavailable({}), false); + assert.equal(toOfferAvailability('out_of_stock'), 'out_of_stock'); + assert.equal(toOfferAvailability('OUT_OF_STOCK'), null); + assert.equal(toOfferAvailability(undefined), null); + + // The offer list labels sold-out listings and leaves them out of the price + // spread, so its cheapest offer matches the stored "from" price. + const { computeOffersPriceSpreadAudit } = + await import('../../components/leaderboard/ProductOffersExplorer'); + const offer = ( + id: string, + platform: string, + price: number, + availability: 'in_stock' | 'out_of_stock' | null + ) => + ({ + id, + platform, + title: id, + price_current: price, + price_original: null, + stock: 0, + rating_star: 0, + rating_count: 0, + url: '', + availability, + merchant: { + id: `m-${platform}`, + name: platform, + platform, + is_official: true, + rating_star: null, + follower_count: 0, + location: null, + risk_score: 0, + dosafe_risk_score: null, + review_risk_score: null, + review_risk_breakdown: null, + }, + }) as const; + const budsOffers = [ + offer('fpt-1', 'fptshop', 2694000, 'out_of_stock'), + offer('fpt-2', 'fptshop', 3032000, 'out_of_stock'), + offer('tgdd', 'tgdd', 3140000, 'in_stock'), + offer('cps', 'cellphones', 3690000, null), + ]; + const spread = computeOffersPriceSpreadAudit(budsOffers); + assert.ok(spread); + assert.equal(spread.cheapest.price_current, buds.min_price); + // Here only: the stored max also counts a listing's variant top price. + assert.equal(spread.priciest.price_current, buds.max_price); + + // Every surface reads the stored stats. The product page, its metadata and + // its JSON-LD go through productMarketStats and never recompute the "from" + // price or the seller count from the offer list. + const page = read('../../app/[locale]/smartphones/[slug]/page.tsx'); + assert.match( + page, + /productMarketStats,\n\} from '@\/lib\/product-market-stats'/ + ); + assert.doesNotMatch( + page, + /plausibleMinPrice\(/, + 'The product page must not derive its "from" price from the offers' + ); + assert.doesNotMatch( + page, + /product\.(merchant_count|max_price)\b/, + 'Seller count and range come from productMarketStats only' + ); + assert.doesNotMatch( + page, + /new Set\(\s*offers/, + 'The seller count is not counted from the offer list' + ); + const metadata = page.slice( + page.indexOf('export async function generateCategoryProductMetadata'), + page.indexOf('export async function generateMetadata') + ); + assert.match(metadata, /productMarketStats\(product\)/); + assert.match(metadata, /merchantCount: marketStats\.sellerCount/); + assert.doesNotMatch(metadata, /getCachedMasterProductOffers/); + assert.match(page, /lowPrice: currentMinPrice/); + assert.match(page, /offerCount: sellerCount/); + + assert.match( + page, + /computeProductRadarSignal\(product, inStockOffers\)/, + 'The radar note never quotes a sold-out listing' + ); + assert.match(page, /currentMinPrice: currentMinPrice > 0 \? currentMinPrice/); + + const productApi = read('../../app/api/v1/rate/products/[slug]/route.ts'); + assert.match( + productApi, + /offers\.filter\(\(o\) => !isOfferUnavailable\(o\)\)/ + ); + const watchlist = read('../../app/api/v1/rate/watchlist/route.ts'); + assert.doesNotMatch( + watchlist, + /withPlausibleMinPrice/, + 'Watchlist prices and alerts use the stored min_price' + ); + + // The catalogue, compare and the API read the same stored columns. + const api = read('../rate-api.ts'); + assert.match( + api, + /'id, slug, name, model, brand, category_id, metadata, specifications, min_price, max_price, merchant_count'/, + 'compareMasterProducts reads the stored stats' + ); + + console.warn('✅ Product market stats suite passed'); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/apps/web/src/lib/__tests__/spec-template.test.ts b/apps/web/src/lib/__tests__/spec-template.test.ts new file mode 100644 index 000000000..3bc2a9727 --- /dev/null +++ b/apps/web/src/lib/__tests__/spec-template.test.ts @@ -0,0 +1,120 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { + applyCategorySpecTemplate, + getCategorySpecTemplate, + type SpecGroupKey, +} from '@repo/shared-data/rules'; +import { + getSpecUiDictionary, + specGroupHeading, + specSectionBadge, +} from '../spec-localizer'; + +const readSource = (path: string) => + readFileSync(new URL(path, import.meta.url), 'utf8'); + +// The headings a category's spec section shows, in order. +const headings = (category: string, locale: string) => { + const ui = getSpecUiDictionary(locale); + const template = getCategorySpecTemplate(category); + return template.groups.map((group: SpecGroupKey, i: number) => + specGroupHeading(ui, group, i + 1, template.has5g) + ); +}; + +function main() { + console.warn('🧪 Category spec template suite (DOS/Rate#1142)'); + + // Phones keep the ten numbered groups, worded as before. + const phone = headings('smartphones', 'vi'); + assert.equal(phone.length, 10); + assert.equal(phone[0], '1. Màn hình (Display)'); + assert.equal(phone[5], '6. Mạng Di động & 5G (Cellular)'); + assert.equal(phone[9], '10. Pháp lý, Chứng nhận & SAR (Regulatory)'); + assert.deepEqual(headings('tablets', 'en'), headings('smartphones', 'en')); + assert.equal( + specSectionBadge(getSpecUiDictionary('vi'), 10), + '10 nhóm thông số' + ); + + // Earbuds open on audio and never show a screen, hardware, camera or + // cellular group, in any locale. + for (const locale of ['vi', 'en', 'cs']) { + const ui = getSpecUiDictionary(locale); + const buds = headings('earbuds', locale); + assert.equal(buds.length, 7); + assert.ok(buds[0]!.startsWith('1. ')); + assert.ok(buds[0]!.endsWith(ui.groups.multimedia.replace(/^\d+\.\s*/, ''))); + for (const group of ['display', 'camera', 'cellular'] as const) { + const label = ui.groups[group].replace(/^\d+\.\s*/, ''); + assert.ok( + !buds.some((h: string) => h.endsWith(label)), + `earbuds (${locale}) must not show ${group}` + ); + } + assert.equal(specSectionBadge(ui, buds.length).startsWith('7 '), true); + } + assert.deepEqual(headings('earbuds', 'en'), [ + '1. Audio & multimedia', + '2. Battery & charging', + '3. Connectivity & sensors', + '4. Hardware & performance', + '5. Design & durability', + '6. Availability & MSRP', + '7. Regulatory, certifications & SAR', + ]); + + // Smartwatches: no camera group and no "5G" anywhere in the headings. + for (const locale of ['vi', 'en', 'cs']) { + const watch = headings('smartwatches', locale); + assert.equal(watch.length, 9); + assert.ok( + watch.every((h: string) => !h.includes('5G')), + watch.join(' | ') + ); + } + assert.equal(headings('smartwatches', 'vi')[4], '5. Mạng Di động (Cellular)'); + + // The card chips read only what the template leaves: an earbud has no + // "1GB RAM · 64GB" and a cellular watch no "5G". + const buds = applyCategorySpecTemplate('earbuds', { + ram_gb: 1, + storage_gb: 1, + has_5g: false, + key_specs: { screen: '1"', ram: '1GB', chip: 'H2' }, + }); + assert.equal(buds.ram_gb, null); + assert.equal(buds.storage_gb, null); + assert.deepEqual(buds.key_specs, { chip: 'H2' }); + const ultra2 = applyCategorySpecTemplate('smartwatches', { + storage_gb: 64, + has_5g: true, + }); + assert.equal(ultra2.has_5g, false); + assert.equal(ultra2.storage_gb, 64); + + // The detail page, the category cards and the home cards all apply it. + const detail = readSource('../../app/[locale]/smartphones/[slug]/page.tsx'); + assert.match( + detail, + /const product = applyCategorySpecTemplate\(categorySlug, storedProduct\)/ + ); + assert.match(detail, /applyCategorySpecTemplate\(categorySlug, peer\)/); + assert.match(detail, /specTemplate\.groups\.map\(/); + assert.doesNotMatch(detail, /specUi\.groups\.\w+\}/); + const catalog = readSource('../../app/[locale]/smartphones/page.tsx'); + assert.match( + catalog, + /applyCategorySpecTemplate\(\s*categorySlug,\s*storedItem/ + ); + const home = readSource('../../app/[locale]/page.tsx'); + assert.match( + home, + /applyCategorySpecTemplate\(\s*getMasterProductCategorySlug/ + ); + + console.warn(' ✅ Category spec templates passed'); +} + +main(); diff --git a/apps/web/src/lib/jreviews-user-score.ts b/apps/web/src/lib/jreviews-user-score.ts new file mode 100644 index 000000000..1ea4ab413 --- /dev/null +++ b/apps/web/src/lib/jreviews-user-score.ts @@ -0,0 +1,73 @@ +/** The user-review fields the buyer score reads. */ +export interface JReviewsUserScoreReview { + reviewer_type?: string | null; + include_in_rating: boolean; + helpfulness_weight: number; + rating_star: number; +} + +export interface JReviewsUserScores { + /** Plain weighted mean of the counted buyer reviews ("TB trọng số"). */ + user: number | null; + /** Bayesian buyer score: the number the product header shows. */ + bayesianUser: number | null; + bayesianConfidence: number; + userWeightSum: number; +} + +const PRIOR_MEAN = 4.2; +const PRIOR_STRENGTH = 3; + +/** + * Buyer scores of the reviews section. The composite score must start from + * bayesianUser, the buyer score the header shows, not the raw mean: 9 + * rating-only 5-star reviews are 4.54 in the header and were 5.0 in the + * composite (DOS/Rate#1144). + */ +export function computeJReviewsUserScores( + reviews: readonly JReviewsUserScoreReview[], + userRatingWeighted: number | null | undefined, + averageRating: number +): JReviewsUserScores { + const validUserReviews = (Array.isArray(reviews) ? reviews : []).filter( + (r) => + r.reviewer_type !== 'expert' && + r.include_in_rating && + r.helpfulness_weight > 0 + ); + + let user: number | null = + userRatingWeighted && userRatingWeighted > 0 ? userRatingWeighted : null; + let userWeightSum = 0; + let userWeightedStars = 0; + if (validUserReviews.length > 0) { + userWeightSum = validUserReviews.reduce( + (acc, r) => acc + r.helpfulness_weight, + 0 + ); + userWeightedStars = validUserReviews.reduce( + (acc, r) => acc + r.rating_star * r.helpfulness_weight, + 0 + ); + if (userWeightSum > 0) { + user = Number((userWeightedStars / userWeightSum).toFixed(2)); + } + } else if (user == null && averageRating > 0) { + user = Number(Number(averageRating).toFixed(2)); + } + + const bayesianUser = + userWeightSum > 0 + ? Number( + ( + (PRIOR_STRENGTH * PRIOR_MEAN + userWeightedStars) / + (PRIOR_STRENGTH + userWeightSum) + ).toFixed(2) + ) + : user; + const bayesianConfidence = Number( + (userWeightSum / (userWeightSum + PRIOR_STRENGTH)).toFixed(2) + ); + + return { user, bayesianUser, bayesianConfidence, userWeightSum }; +} diff --git a/apps/web/src/lib/leaderboard-engine.ts b/apps/web/src/lib/leaderboard-engine.ts index 8ed6325b4..58c0c4492 100644 --- a/apps/web/src/lib/leaderboard-engine.ts +++ b/apps/web/src/lib/leaderboard-engine.ts @@ -385,9 +385,13 @@ export function computeJReviewsCriteriaScores( }; } +/** Filtered-review counts per master id, as read from stored reviews. */ +export type FilteredReviewCountsByMaster = Readonly<Record<string, number>>; + export function scoreMasterProductForLeaderboard( item: MasterProduct, - criterion: LeaderboardCriterionKey = 'combined' + criterion: LeaderboardCriterionKey = 'combined', + filteredReviewCounts?: FilteredReviewCountsByMaster | null ): Omit<ScoredLeaderboardItem, 'rank'> { const safeItem = ( item && typeof item === 'object' && !Array.isArray(item) ? item : {} @@ -479,9 +483,12 @@ export function scoreMasterProductForLeaderboard( Number.isFinite(rawAuthentic) && rawAuthentic >= 0 ? Math.min(Math.floor(rawAuthentic), MAX_LEADERBOARD_COUNT_CEILING) : safeTotal; - const filteredNoiseReviewCount = Math.max( - 0, - safeTotal - authenticReviewCount + // Only reviews Rate.vn stored and the analyser excluded. total_reviews is + // the marketplaces' own rating count, so total minus authentic is not a + // number of blocked reviews (DOS/Rate#1139). + const filteredNoiseReviewCount = readFilteredReviewCount( + filteredReviewCounts, + safeItem.id ); const rawMinPrice = toSafePositiveNumber(safeItem.min_price); @@ -524,6 +531,20 @@ export function scoreMasterProductForLeaderboard( }; } +function readFilteredReviewCount( + counts: FilteredReviewCountsByMaster | null | undefined, + masterId: unknown +): number { + if (!counts || typeof counts !== 'object' || typeof masterId !== 'string') { + return 0; + } + if (!Object.prototype.hasOwnProperty.call(counts, masterId)) return 0; + const raw = Number(counts[masterId]); + return Number.isFinite(raw) && raw > 0 + ? Math.min(Math.floor(raw), MAX_LEADERBOARD_COUNT_CEILING) + : 0; +} + export function rankMasterProductsForLeaderboard( products: MasterProduct[], options?: { @@ -533,6 +554,7 @@ export function rankMasterProductsForLeaderboard( brand?: string; query?: string; limit?: number; + filteredReviewCounts?: FilteredReviewCountsByMaster | null; } ): ScoredLeaderboardItem[] { const safeOptions = @@ -568,7 +590,13 @@ export function rankMasterProductsForLeaderboard( : []; const filtered = safeProducts - .map((item) => scoreMasterProductForLeaderboard(item, criterion)) + .map((item) => + scoreMasterProductForLeaderboard( + item, + criterion, + safeOptions?.filteredReviewCounts + ) + ) .filter((entry) => { if (category !== 'all' && entry.categorySlug !== category) { return false; diff --git a/apps/web/src/lib/merchant-risk-tone.ts b/apps/web/src/lib/merchant-risk-tone.ts new file mode 100644 index 000000000..18885540b --- /dev/null +++ b/apps/web/src/lib/merchant-risk-tone.ts @@ -0,0 +1,17 @@ +/** + * One cut-off for every merchant risk score shown on the site, so the home + * merchant cards and the merchant detail page agree (DOS/Rate#1140): below 20 + * is clean ("no scam detected"), 20-49 needs caution, 50+ is a risk alert. + */ +export type MerchantRiskTone = 'good' | 'warn' | 'bad'; + +export const MERCHANT_RISK_CLEAN_BELOW = 20; +export const MERCHANT_RISK_ALERT_FROM = 50; + +export function merchantRiskTone(score: unknown): MerchantRiskTone { + const n = typeof score === 'number' ? score : Number(score); + // An unknown score is neither clean nor a scam alert. + if (score == null || !Number.isFinite(n)) return 'warn'; + if (n < MERCHANT_RISK_CLEAN_BELOW) return 'good'; + return n < MERCHANT_RISK_ALERT_FROM ? 'warn' : 'bad'; +} diff --git a/apps/web/src/lib/product-detail-copy.ts b/apps/web/src/lib/product-detail-copy.ts index 8456094fc..c27fef1d8 100644 --- a/apps/web/src/lib/product-detail-copy.ts +++ b/apps/web/src/lib/product-detail-copy.ts @@ -55,6 +55,8 @@ export type RadarExplanation = pct: string; merchant: string; platform: string; + /** Offers behind the median, never a fixed "10 platforms" (#1145). */ + count: string; }; } | { @@ -82,6 +84,14 @@ const safePctString = (value: unknown, max = 10_000): string => { return String(Math.min(num, max)); }; +const offerCountString = (value: unknown): string | null => { + const num = parseDecimalDetailNumber(value); + // A median needs at least two offers; deals.ts never makes one from fewer. + return Number.isFinite(num) && num >= 2 + ? String(Math.min(Math.floor(num), 10_000)) + : null; +}; + const safeLabelText = (value: unknown, maxLen: number): string => { if (typeof value !== 'string') return ''; return value.replace(COPY_CONTROL_CHAR_RE, '').trim().slice(0, maxLen); @@ -97,6 +107,9 @@ export function radarExplanation( if (!signal || typeof signal !== 'object') return null; if (signal.verdict === 'genuine_deal' && signal.genuineDeal) { const deal = signal.genuineDeal; + // No offer count, no sentence: never print a number we do not know. + const count = offerCountString(deal.marketOfferCount); + if (count == null) return null; return { key: 'radarExplanationGenuineDeal', values: { @@ -104,6 +117,7 @@ export function radarExplanation( pct: safePctString(deal.savingsVsMedianPct), merchant: safeLabelText(deal.merchantName, 200), platform: safeLabelText(deal.platform, 64).toUpperCase(), + count, }, }; } diff --git a/apps/web/src/lib/product-market-stats.ts b/apps/web/src/lib/product-market-stats.ts new file mode 100644 index 000000000..ea27a41df --- /dev/null +++ b/apps/web/src/lib/product-market-stats.ts @@ -0,0 +1,80 @@ +/** + * One source of truth for a product's "from" price, price range and seller + * count (DOS/Rate#1141): the stored master stats. The database writes them in + * rate.refresh_master_market_stats from crawled, in-stock listings, without + * bait prices (DOS/Rate#802), and the catalogue cards, compare, versus, the + * catalogue search and sort and the public API all read them. The product + * page, its metadata and its JSON-LD read them through this helper too. + * + * The product page's offer list is not that source: it also holds sold-out + * and discontinued listings at their last price, so computing from it showed + * a lower price and more sellers than every other page. + */ +import { MAX_VALID_VND_PRICE } from './rate-domain-core'; + +export interface ProductMarketStats { + /** Lowest price of an in-stock listing, 0 when none is known. */ + minPrice: number; + /** Highest price of an in-stock listing, never below minPrice. */ + maxPrice: number; + /** Distinct merchants with a priced in-stock listing. */ + sellerCount: number; +} + +/** Listing states rate.products.availability can hold. */ +export type OfferAvailability = 'in_stock' | 'out_of_stock' | 'discontinued'; + +function toStoredPrice(raw: unknown): number { + const n = + typeof raw === 'number' + ? raw + : typeof raw === 'string' && raw.trim() !== '' + ? Number(raw.trim()) + : NaN; + return Number.isFinite(n) && n > 0 && n <= MAX_VALID_VND_PRICE ? n : 0; +} + +export function productMarketStats( + product: { + min_price?: unknown; + max_price?: unknown; + merchant_count?: unknown; + } | null +): ProductMarketStats { + const minPrice = toStoredPrice(product?.min_price); + const storedMax = toStoredPrice(product?.max_price); + const rawCount = Number(product?.merchant_count); + // A stored price has at least one seller, even with a stale or null count. + const sellerCount = + minPrice > 0 + ? Number.isFinite(rawCount) && rawCount >= 1 + ? Math.min(Math.floor(rawCount), 100_000) + : 1 + : 0; + return { + minPrice, + maxPrice: minPrice > 0 ? Math.max(minPrice, storedMax) : 0, + sellerCount, + }; +} + +/** + * Whether the retailer last reported the listing as not buyable. Such a + * listing keeps its last price but is not counted in the stored stats. An + * unknown availability (not read) counts as in stock, as the database's + * default does. + */ +export function isOfferUnavailable(offer: { + availability?: OfferAvailability | null; +}): boolean { + return ( + offer?.availability === 'out_of_stock' || + offer?.availability === 'discontinued' + ); +} + +export function toOfferAvailability(raw: unknown): OfferAvailability | null { + return raw === 'in_stock' || raw === 'out_of_stock' || raw === 'discontinued' + ? raw + : null; +} diff --git a/apps/web/src/lib/rate-api.ts b/apps/web/src/lib/rate-api.ts index f0439b5aa..308fd2756 100644 --- a/apps/web/src/lib/rate-api.ts +++ b/apps/web/src/lib/rate-api.ts @@ -2,7 +2,10 @@ import { createClient } from '@supabase/supabase-js'; import type { Database, MasterProduct } from '@repo/shared-data'; import { createLogger } from '@/lib/logger'; import { isPendingReview } from '@/lib/review-display'; -import { plausibleOffers } from '@repo/shared-data/rules'; +import { + implausiblyLowOfferIds, + plausibleOffers, +} from '@repo/shared-data/rules'; import { type ReviewFlagCategory, type ReviewItem, @@ -27,6 +30,11 @@ import { extractProductKeywordFromUrl, firstImageUrl, } from './rate-domain-core'; +import { + isOfferUnavailable, + type OfferAvailability, + toOfferAvailability, +} from './product-market-stats'; export { computeWilsonLowerBound, evaluateReviewTrustAssessment, @@ -160,6 +168,12 @@ export interface MerchantOffer { rating_star: number; rating_count: number; url: string; + /** + * rate.products.availability: a sold-out or discontinued listing keeps its + * last price but is not counted in the master's stats (DOS/Rate#1141). + * `null` when it was not read. + */ + availability?: OfferAvailability | null; merchant: { id: string; name: string; @@ -1301,27 +1315,78 @@ export async function getMasterProductOffers( ).catch(() => null); if (viewRows) { const rows = viewRows as unknown as RawTrustViewRow[]; - const extras = await sellerExtrasForTrustViewRows(rows); + const [extras, availability] = await Promise.all([ + sellerExtrasForTrustViewRows(rows), + getListingAvailability([cleanMasterId]), + ]); return rows.map((row) => - mapTrustViewRowToMerchantOffer( - row, - extras.followerCounts.get(row.merchant_id), - extras.metadataDosafeRisks.get(row.merchant_id) + withOfferAvailability( + mapTrustViewRowToMerchantOffer( + row, + extras.followerCounts.get(row.merchant_id), + extras.metadataDosafeRisks.get(row.merchant_id) + ), + availability ) ); } // Fallback with explicit column list matching column-level SELECT grants on rate.merchants - const rows = await fetchAllRows('product offers', (from, to) => - supabase - .from('products') - .select(OFFER_FALLBACK_SELECT) - .eq('cluster_id', cleanMasterId) - .order('price_current', { ascending: true }) - .order('id') - .range(from, to) + const [rows, availability] = await Promise.all([ + fetchAllRows('product offers', (from, to) => + supabase + .from('products') + .select(OFFER_FALLBACK_SELECT) + .eq('cluster_id', cleanMasterId) + .order('price_current', { ascending: true }) + .order('id') + .range(from, to) + ), + getListingAvailability([cleanMasterId]), + ]); + return (rows as unknown as RawOfferRow[]).map((row) => + withOfferAvailability(mapOfferRowToMerchantOffer(row), availability) ); - return (rows as unknown as RawOfferRow[]).map(mapOfferRowToMerchantOffer); +} + +/** + * Availability of every listing under the given masters (at most + * IN_FILTER_CHUNK_SIZE), by listing id. The trust view predates the column + * (DOS-Me 20261020230000), so it is read from rate.products. A failed read + * throws like the offer reads it completes (DOS/Rate#1174). + */ +async function getListingAvailability( + masterIds: readonly string[] +): Promise<Map<string, OfferAvailability>> { + const supabase = getRateClient(); + const rows = await fetchAllRows<{ id: unknown; availability: unknown }>( + 'listing availability', + (from, to) => + supabase + .from('products') + .select('id, availability') + .in('cluster_id', [...masterIds]) + .order('id') + .range(from, to) as unknown as PromiseLike<{ + data: { id: unknown; availability: unknown }[] | null; + error: { message: string } | null; + }> + ); + const byId = new Map<string, OfferAvailability>(); + for (const row of rows) { + const availability = toOfferAvailability(row?.availability); + if (availability && typeof row?.id === 'string') { + byId.set(row.id, availability); + } + } + return byId; +} + +function withOfferAvailability( + offer: MerchantOffer, + availability: ReadonlyMap<string, OfferAvailability> +): MerchantOffer { + return { ...offer, availability: availability.get(offer.id) ?? null }; } /** @@ -1351,9 +1416,8 @@ export async function getMasterProductOffersBatch( const batchIds = uniqueIds.slice(i, i + chunkSize); // Paged per chunk: 150 masters can easily hold more than 1000 offers. - const viewData = await fetchAllRows( - 'v_products_with_merchant_trust', - (from, to) => + const [viewData, availability] = await Promise.all([ + fetchAllRows('v_products_with_merchant_trust', (from, to) => supabase .from('v_products_with_merchant_trust') .select(TRUST_VIEW_SELECT) @@ -1361,7 +1425,9 @@ export async function getMasterProductOffersBatch( .order('price_current', { ascending: true }) .order('id') .range(from, to) - ).catch(() => null); + ).catch(() => null), + getListingAvailability(batchIds), + ]); if (viewData) { const rows = viewData as unknown as RawTrustViewRow[]; @@ -1371,10 +1437,13 @@ export async function getMasterProductOffersBatch( if (!cid) continue; const list = offersByMaster.get(cid) ?? []; list.push( - mapTrustViewRowToMerchantOffer( - row, - extras.followerCounts.get(row.merchant_id), - extras.metadataDosafeRisks.get(row.merchant_id) + withOfferAvailability( + mapTrustViewRowToMerchantOffer( + row, + extras.followerCounts.get(row.merchant_id), + extras.metadataDosafeRisks.get(row.merchant_id) + ), + availability ) ); offersByMaster.set(cid, list); @@ -1396,7 +1465,9 @@ export async function getMasterProductOffersBatch( const cid = item.cluster_id ? String(item.cluster_id) : ''; if (!cid) continue; const list = offersByMaster.get(cid) ?? []; - list.push(mapOfferRowToMerchantOffer(item)); + list.push( + withOfferAvailability(mapOfferRowToMerchantOffer(item), availability) + ); offersByMaster.set(cid, list); } } @@ -1871,7 +1942,12 @@ export async function getMasterProductPriceHistory( : cleanMasterId ? await getMasterProductOffers(cleanMasterId) : []; - const offers = plausibleOffers(rawOffers); + // Bait prices are judged among the in-stock listings, as the stored + // min_price is; a sold-out listing keeps its chart (DOS/Rate#1141). + const baitIds = implausiblyLowOfferIds( + rawOffers.filter((o) => !isOfferUnavailable(o)) + ); + const offers = rawOffers.filter((o) => !baitIds.has(o.id)); if (offers.length === 0) { return { points: [], @@ -2168,8 +2244,14 @@ export async function getMasterProductPriceHistory( } } + // The lowest in-stock price, like the stored min_price: a sold-out listing + // keeps its last price (DOS/Rate#1141). Unknown availability counts. + const inStockOfferPrices = offers + .filter((o) => !isOfferUnavailable(o)) + .map((o) => o.price_current) + .filter((p) => Number.isFinite(p) && p > 0 && p <= MAX_VALID_VND_PRICE); const currentMinPrice = - validOfferPrices.length > 0 ? Math.min(...validOfferPrices) : null; + inStockOfferPrices.length > 0 ? Math.min(...inStockOfferPrices) : null; return { points, @@ -3357,6 +3439,89 @@ export const EMPTY_RATE_PLATFORM_STATS: RatePlatformStats = { * Fetch live platform statistics, category product counts, and recent verified/flagged * reviews from the `rate` schema to power the Homepage with 100% real database data. */ +/** + * Stored reviews the analyser kept out of the rating, per master product id + * (DOS/Rate#1139). Same rule as the platform's filteredSeedingCount below: + * include_in_rating false, or NULL with a flagged category or a fake verdict, + * and never a review still pending analysis. Masters with none are absent. + * The leaderboard must not derive this from master_products.total_reviews, + * which is the marketplaces' own rating count, not reviews Rate.vn read. + */ +export async function getFilteredReviewCountsByMaster(): Promise< + Record<string, number> +> { + const supabase = getRateClient(); + const [explicitRows, derivedRows] = await Promise.all([ + fetchAllRows<{ review_id: string }>('filtered review flags', (from, to) => + supabase + .from('review_flags') + .select('review_id') + .eq('include_in_rating', false) + .or('verdict.is.null,verdict.neq.unanalyzed') + .order('review_id') + .range(from, to) + ), + fetchAllRows<{ review_id: string }>('filtered review flags', (from, to) => + supabase + .from('review_flags') + .select('review_id') + .is('include_in_rating', null) + .or( + 'and(flag_category.neq.authentic,or(verdict.is.null,verdict.neq.unanalyzed)),verdict.eq.fake,and(flag_category.is.null,verdict.eq.suspicious)' + ) + .order('review_id') + .range(from, to) + ), + ]); + const reviewIds = Array.from( + new Set( + [...explicitRows, ...derivedRows] + .map((r) => String(r.review_id ?? '')) + .filter(Boolean) + ) + ); + if (reviewIds.length === 0) return {}; + + const reviewRows = await fetchAllRowsInChunks<{ + id: string; + product_id: string | null; + }>('filtered reviews', reviewIds, (chunk, from, to) => + supabase + .from('reviews') + .select('id, product_id') + .in('id', chunk) + .order('id') + .range(from, to) + ); + const productIds = Array.from( + new Set(reviewRows.map((r) => String(r.product_id ?? '')).filter(Boolean)) + ); + if (productIds.length === 0) return {}; + + const productRows = await fetchAllRowsInChunks<{ + id: string; + cluster_id: string | null; + }>('filtered review products', productIds, (chunk, from, to) => + supabase + .from('products') + .select('id, cluster_id') + .in('id', chunk) + .order('id') + .range(from, to) + ); + const masterByProduct = new Map( + productRows.map((p) => [String(p.id), String(p.cluster_id ?? '')]) + ); + + const counts: Record<string, number> = {}; + for (const r of reviewRows) { + const masterId = masterByProduct.get(String(r.product_id ?? '')); + if (!masterId) continue; + counts[masterId] = (counts[masterId] ?? 0) + 1; + } + return counts; +} + export async function getRatePlatformStats(): Promise<RatePlatformStats> { const supabase = getRateClient(); @@ -3749,10 +3914,11 @@ export async function inspectProductOrShopUrl( getMasterProductPriceHistory(masterProduct.id, offers), getMasterProductReviewSummary(masterProduct.id, reviews), ]); - // No bait-price listing as the fallback match (DOS/Rate#797). + // No bait-price or sold-out listing as the fallback match + // (DOS/Rate#797, #1177). const matchedOffer = offers.find((o) => o.id === matchedProd.id) || - plausibleOffers(offers)[0] || + plausibleOffers(offers.filter((o) => !isOfferUnavailable(o)))[0] || null; const discountAnalysis = matchedOffer ? priceHistory.byOffer[matchedOffer.id] || null @@ -3858,8 +4024,11 @@ export async function inspectProductOrShopUrl( ); if (byShopMerchant) return byShopMerchant; } - // The cheapest plausible offer, never a bait-price listing (DOS/Rate#797). - return plausibleOffers(offers)[0] || null; + // The cheapest plausible in-stock offer, never a bait-price or sold-out + // listing (DOS/Rate#797, #1177). + return ( + plausibleOffers(offers.filter((o) => !isOfferUnavailable(o)))[0] || null + ); }; // 3. Internal slug or exact master product slug match (including slugified URL keyword) diff --git a/apps/web/src/lib/rate-domain.ts b/apps/web/src/lib/rate-domain.ts index 106f21c5c..80d2b2d5d 100644 --- a/apps/web/src/lib/rate-domain.ts +++ b/apps/web/src/lib/rate-domain.ts @@ -70,6 +70,7 @@ export { normalizeLeaderboardPriceSegment, rankMasterProductsForLeaderboard, scoreMasterProductForLeaderboard, + type FilteredReviewCountsByMaster, type Flagship1v1BattlePair, type LeaderboardCriteriaBreakdown, type LeaderboardCriterionKey, diff --git a/apps/web/src/lib/spec-localizer.ts b/apps/web/src/lib/spec-localizer.ts index e64b76d71..ed27b2983 100644 --- a/apps/web/src/lib/spec-localizer.ts +++ b/apps/web/src/lib/spec-localizer.ts @@ -1,3 +1,4 @@ +import type { SpecGroupKey } from '@repo/shared-data/rules'; import { type RateLocale, normalizeRateLocale } from './rate-i18n'; export interface SpecUiDictionary { @@ -9,6 +10,8 @@ export interface SpecUiDictionary { supported: string; availableStatus: string; has5gYes: string; + /** The cellular group heading of a category without 5G (smartwatches). */ + cellularNo5gGroup: string; groups: { display: string; hardware: string; @@ -59,6 +62,7 @@ export interface SpecUiDictionary { simType: string; dataSpeed: string; bands5g4g: string; + bands4g: string; volteVowifi: string; wlan: string; bluetooth: string; @@ -90,6 +94,7 @@ const SPEC_UI_DICTS: Record<RateLocale, SpecUiDictionary> = { supported: 'Có hỗ trợ', availableStatus: 'Đang kinh doanh', has5gYes: 'Có (5G Sub-6 / mmWave)', + cellularNo5gGroup: '6. Mạng Di động (Cellular)', groups: { display: '1. Màn hình (Display)', hardware: '2. Phần cứng & Hiệu năng (Hardware)', @@ -140,6 +145,7 @@ const SPEC_UI_DICTS: Record<RateLocale, SpecUiDictionary> = { simType: 'Chuẩn SIM', dataSpeed: 'Tốc độ dữ liệu', bands5g4g: 'Băng tần 5G / 4G', + bands4g: 'Băng tần 4G', volteVowifi: 'VoLTE / VoWiFi', wlan: 'Wi-Fi (WLAN)', bluetooth: 'Bluetooth', @@ -169,6 +175,7 @@ const SPEC_UI_DICTS: Record<RateLocale, SpecUiDictionary> = { supported: 'Supported', availableStatus: 'Available', has5gYes: 'Yes (5G Sub-6 / mmWave)', + cellularNo5gGroup: '6. Cellular network', groups: { display: '1. Display & panel', hardware: '2. Hardware & performance', @@ -219,6 +226,7 @@ const SPEC_UI_DICTS: Record<RateLocale, SpecUiDictionary> = { simType: 'SIM configuration', dataSpeed: 'Data speed', bands5g4g: '5G / 4G bands', + bands4g: '4G bands', volteVowifi: 'VoLTE / VoWiFi', wlan: 'Wi-Fi (WLAN)', bluetooth: 'Bluetooth', @@ -248,6 +256,7 @@ const SPEC_UI_DICTS: Record<RateLocale, SpecUiDictionary> = { supported: 'Podporováno', availableStatus: 'V prodeji', has5gYes: 'Ano (5G Sub-6 / mmWave)', + cellularNo5gGroup: '6. Mobilní sítě', groups: { display: '1. Displej a obrazovka', hardware: '2. Hardware a výkon', @@ -298,6 +307,7 @@ const SPEC_UI_DICTS: Record<RateLocale, SpecUiDictionary> = { simType: 'Typ SIM karty', dataSpeed: 'Rychlost dat', bands5g4g: 'Pásma 5G / 4G', + bands4g: 'Pásma 4G', volteVowifi: 'VoLTE / VoWiFi', wlan: 'Wi-Fi (WLAN)', bluetooth: 'Bluetooth', @@ -324,6 +334,26 @@ export function getSpecUiDictionary(locale?: string): SpecUiDictionary { return SPEC_UI_DICTS[normalizeRateLocale(locale)]; } +/** + * A group heading numbered by its place in the category's template, so an + * earbud page that starts with audio reads "1. Audio" (DOS/Rate#1142). + */ +export function specGroupHeading( + ui: SpecUiDictionary, + group: SpecGroupKey, + position: number, + has5g = true +): string { + const label = + group === 'cellular' && !has5g ? ui.cellularNo5gGroup : ui.groups[group]; + return `${position}. ${label.replace(/^\d+\.\s*/, '')}`; +} + +/** The "10 spec groups" badge with the category's own group count. */ +export function specSectionBadge(ui: SpecUiDictionary, count: number): string { + return ui.sectionBadge.replace(/^\d+/, String(count)); +} + const MONTH_VI_MAP: Record<string, string> = { january: 'Tháng 01', february: 'Tháng 02', diff --git a/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts b/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts index 4261ecd1a..1d600c03d 100644 --- a/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts +++ b/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts @@ -5345,6 +5345,7 @@ async function runCoreArchitectureSuite() { marketMaxPrice: 28_990_000, savingsVsMedianVnd: 1_500_000, savingsVsMedianPct: 5.6, + marketOfferCount: 3, claimedDiscountPct: 12, fakeMarkupPct: 0, jreviewsScore10: 9.2, @@ -5373,6 +5374,7 @@ async function runCoreArchitectureSuite() { marketMaxPrice: 28_990_000, savingsVsMedianVnd: 3_100_000, savingsVsMedianPct: 11.5, + marketOfferCount: 3, claimedDiscountPct: 17.6, fakeMarkupPct: 0, jreviewsScore10: 9.2, @@ -5403,6 +5405,7 @@ async function runCoreArchitectureSuite() { marketMaxPrice: 27_000_000, savingsVsMedianVnd: -500_000, savingsVsMedianPct: -2, + marketOfferCount: 3, claimedDiscountPct: 18.8, fakeMarkupPct: 25.5, jreviewsScore10: 9.1, @@ -5431,6 +5434,7 @@ async function runCoreArchitectureSuite() { marketMaxPrice: 27_000_000, savingsVsMedianVnd: -1_000_000, savingsVsMedianPct: -3.9, + marketOfferCount: 3, claimedDiscountPct: 33.7, fakeMarkupPct: 56.8, jreviewsScore10: 9.1, @@ -5486,6 +5490,7 @@ async function runCoreArchitectureSuite() { categoryFilter: 'all', totalClustersAnalyzed: 2, totalOffersAnalyzed: 4, + totalMerchantsAnalyzed: 3, generatedAt: new Date().toISOString(), }); @@ -5527,6 +5532,7 @@ async function runCoreArchitectureSuite() { categoryFilter: 'all', totalClustersAnalyzed: 1, totalOffersAnalyzed: 1, + totalMerchantsAnalyzed: 1, generatedAt: new Date().toISOString(), }, { id: 'non-existent-id', name: ' ' } @@ -5930,7 +5936,11 @@ async function runCoreArchitectureSuite() { { status: 200, headers: { 'content-type': 'application/json' } } ); } - if (url.includes('/rest/v1/products') && url.includes('id=in.')) { + if ( + url.includes('/rest/v1/products') && + url.includes('id=in.') && + !url.includes('cluster_id=in.') + ) { return new Response( JSON.stringify([ { @@ -9666,10 +9676,54 @@ async function runCoreArchitectureSuite() { 0, 'scoreMasterProductForLeaderboard must preserve explicit authentic_review_count: 0' ); + // total_reviews is the marketplaces' rating count, not reviews Rate.vn + // stored and filtered, so it never becomes a blocked count (#1139). assert.equal( scored898.filteredNoiseReviewCount, - 25, - 'scoreMasterProductForLeaderboard must count all 25 reviews as filtered noise when authentic_review_count is 0' + 0, + 'scoreMasterProductForLeaderboard must not derive a blocked count from total_reviews (#1139)' + ); + const iphone15 = { + id: 'mp-iphone-15', + slug: 'apple-iphone-15-128gb', + name: 'Apple iPhone 15 128GB', + brand: 'Apple', + user_rating_weighted: 4.6, + total_reviews: 6833, + authentic_review_count: 15, + } as unknown as import('@repo/shared-data').MasterProduct; + const lb1139 = lbMod.scoreMasterProductForLeaderboard(iphone15, 'combined'); + assert.equal(lb1139.authenticReviewCount, 15); + assert.equal( + lb1139.filteredNoiseReviewCount, + 0, + 'iPhone 15 (6833 marketplace ratings, 15 stored authentic) must not claim 6818 blocked reviews (#1139)' + ); + assert.equal( + lbMod.scoreMasterProductForLeaderboard(iphone15, 'combined', { + 'mp-iphone-15': 3, + 'mp-other': 99, + }).filteredNoiseReviewCount, + 3, + 'the blocked count must be the stored filtered reviews of that product only (#1139)' + ); + for (const bad of [-4, Number.NaN, '7' as unknown as number]) { + assert.equal( + lbMod.scoreMasterProductForLeaderboard(iphone15, 'combined', { + 'mp-iphone-15': bad, + }).filteredNoiseReviewCount, + bad === ('7' as unknown as number) ? 7 : 0, + 'filtered review counts must be non-negative integers (#1139)' + ); + } + const ranked1139 = lbMod.rankMasterProductsForLeaderboard([iphone15], { + category: 'all', + filteredReviewCounts: { 'mp-iphone-15': 2 }, + }); + assert.equal( + ranked1139[0]?.filteredNoiseReviewCount, + 2, + 'rankMasterProductsForLeaderboard must pass filteredReviewCounts through (#1139)' ); assert.ok( Number.isFinite(scored898.activeScore5) && @@ -9791,6 +9845,7 @@ async function runCoreArchitectureSuite() { genuineDeal: { savingsVsMedianVnd: true as unknown as number, savingsVsMedianPct: true as unknown as number, + marketOfferCount: 3, merchantName: 'Shop\u0000A', platform: 'shopee\u0000', } as unknown as import('../../../lib/deal-radar').CrossPlatformDealRadarItem, @@ -13357,6 +13412,472 @@ async function runCoreArchitectureSuite() { ); } + // ------------------------------------------------------------------------- + // 126k. Batch offers read availability; radar and best offer skip sold-out + // listings; stock 0 is not sold out (#1170, #1171, #1174) + // ------------------------------------------------------------------------- + { + const { + RateDataUnavailableError, + getMasterProductOffers, + getMasterProductOffersBatch, + } = await import('@/lib/rate-api'); + const { computeCrossPlatformDealRadarFromData } = + await import('@/server/rate/deals'); + const { selectBestOfferFromListings } = + await import('@/server/rate/api-handler'); + + const origFetch1170 = globalThis.fetch; + const json1170 = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }); + const viewRow1170 = (id: string, clusterId: string, price: number) => ({ + id, + cluster_id: clusterId, + merchant_id: `m-${id}`, + platform: 'shopee', + title: 't', + price_current: price, + stock: 0, + url: 'https://shopee.vn/p', + threat_risk_score: 5, + threat_source: 'dosafe', + dosafe_risk_score: 5, + review_risk_score: 5, + review_risk_breakdown: { total_reviews: 3 }, + }); + const availabilityReads1170: string[] = []; + const mockOffers1170 = (opts: { + viewFails?: boolean; + availabilityFails?: boolean; + }) => { + availabilityReads1170.length = 0; + globalThis.fetch = (async (input: RequestInfo | URL) => { + const url = new URL( + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : input.url + ); + const firstPage = Number(url.searchParams.get('offset') ?? 0) === 0; + if (url.pathname.endsWith('/v_products_with_merchant_trust')) { + if (opts.viewFails) return json1170({ message: 'view down' }, 500); + return json1170( + firstPage + ? [ + viewRow1170('a-sold', 'master-a', 2_694_000), + viewRow1170('a-in', 'master-a', 3_140_000), + viewRow1170('b-new', 'master-b', 5_000_000), + ] + : [] + ); + } + if (url.pathname.endsWith('/products')) { + const select = url.searchParams.get('select') ?? ''; + if (select === 'id,availability') { + if (firstPage) { + availabilityReads1170.push( + url.searchParams.get('cluster_id') ?? '' + ); + } + if (opts.availabilityFails) { + return json1170( + { message: 'column products.availability does not exist' }, + 400 + ); + } + return json1170( + firstPage + ? [ + { id: 'a-sold', availability: 'out_of_stock' }, + { id: 'a-in', availability: 'in_stock' }, + { id: 'b-new', availability: null }, + ] + : [] + ); + } + // The offer fallback when the trust view fails. + return json1170( + firstPage + ? [ + { + id: 'a-sold', + cluster_id: 'master-a', + platform: 'fptshop', + title: 't', + price_current: 2_694_000, + stock: 0, + url: 'https://fptshop.com.vn/p', + merchants: { id: 'm-1', name: 'FPT Shop' }, + }, + ] + : [] + ); + } + return json1170([]); + }) as typeof fetch; + }; + + // The batch read attaches availability per chunk, like the single read. + mockOffers1170({}); + const batch1170 = await getMasterProductOffersBatch([ + 'master-a', + 'master-b', + ]); + assert.deepEqual(availabilityReads1170, ['in.(master-a,master-b)']); + assert.deepEqual( + (batch1170.get('master-a') ?? []).map((o) => [o.id, o.availability]), + [ + ['a-sold', 'out_of_stock'], + ['a-in', 'in_stock'], + ] + ); + assert.deepEqual( + (batch1170.get('master-b') ?? []).map((o) => [o.id, o.availability]), + [['b-new', null]] + ); + mockOffers1170({ viewFails: true }); + const fallback1170 = await getMasterProductOffersBatch(['master-a']); + assert.deepEqual( + (fallback1170.get('master-a') ?? []).map((o) => [o.id, o.availability]), + [['a-sold', 'out_of_stock']] + ); + + // No missing-column fallback: a failed availability read fails the + // offer read, single and batch, with or without the trust view (#1174). + for (const viewFails of [false, true]) { + mockOffers1170({ viewFails, availabilityFails: true }); + await assert.rejects( + () => getMasterProductOffers('master-a'), + (err: unknown) => err instanceof RateDataUnavailableError + ); + await assert.rejects( + () => getMasterProductOffersBatch(['master-a']), + (err: unknown) => err instanceof RateDataUnavailableError + ); + } + + // The radar never cites a sold-out or discontinued listing (#1170). + const radarMaster1170 = { + id: 'master-a', + slug: 'samsung-galaxy-buds3-pro', + name: 'Samsung Galaxy Buds3 Pro', + brand: 'Samsung', + min_price: 3_140_000, + max_price: 3_690_000, + } as unknown as Parameters< + typeof computeCrossPlatformDealRadarFromData + >[0][number]['master']; + const radarOffer1170 = ( + id: string, + platform: string, + price: number, + availability: string | null + ) => + ({ + id, + platform, + title: `Offer ${id}`, + price_current: price, + price_original: 4_490_000, + stock: 0, + url: `https://example.com/${id}`, + availability, + merchant: { + id: `m-${id}`, + name: `Merchant ${id}`, + platform, + is_official: true, + risk_score: 5, + }, + }) as never; + const radar1170 = computeCrossPlatformDealRadarFromData( + [ + { + master: radarMaster1170, + offers: [ + radarOffer1170('fpt-sold', 'fptshop', 2_694_000, 'out_of_stock'), + radarOffer1170('fpt-gone', 'fptshop', 3_032_000, 'discontinued'), + radarOffer1170('tgdd', 'tgdd', 3_140_000, 'in_stock'), + radarOffer1170('cps', 'cellphones', 3_690_000, null), + ], + }, + ], + { categorySlug: 'all', limit: 10 } + ); + const radarText1170 = JSON.stringify(radar1170); + assert.ok(radar1170.priceSpreadLeaders.length > 0, radarText1170); + assert.ok(!radarText1170.includes('fpt-sold'), radarText1170); + assert.ok(!radarText1170.includes('fpt-gone'), radarText1170); + assert.ok(!radarText1170.includes('2694000'), radarText1170); + assert.equal(radar1170.totalOffersAnalyzed, 2); + + // stock 0 means "no count read", not sold out; availability decides (#1171). + const best1171 = selectBestOfferFromListings([ + { + id: 'fpt-sold', + title: 'Buds3 Pro', + price_current: 2_694_000, + stock: 0, + availability: 'out_of_stock', + url: 'https://fptshop.com.vn/p', + platform: 'fptshop', + }, + { + id: 'fpt-gone', + title: 'Buds3 Pro', + price_current: 3_032_000, + availability: 'discontinued', + url: 'https://fptshop.com.vn/q', + platform: 'fptshop', + }, + { + id: 'tgdd', + title: 'Buds3 Pro', + price_current: 3_140_000, + stock: 0, + availability: 'in_stock', + url: 'https://www.thegioididong.com/p', + platform: 'tgdd', + }, + { + id: 'cps', + title: 'Buds3 Pro', + price_current: 3_690_000, + stock: 0, + url: 'https://cellphones.com.vn/p', + platform: 'cellphones', + }, + ] as unknown as Parameters<typeof selectBestOfferFromListings>[0]); + assert.equal(best1171.bestOffer?.id, 'tgdd'); + assert.equal(best1171.bestOffer?.price, 3_140_000); + assert.deepEqual(best1171.offerPlatforms, ['tgdd', 'cellphones']); + // An unknown availability with stock 0 is still buyable. + const unknown1171 = selectBestOfferFromListings([ + { + id: 'only', + title: 'Buds3 Pro', + price_current: 3_140_000, + stock: 0, + url: 'https://www.thegioididong.com/p', + platform: 'tgdd', + }, + ] as unknown as Parameters<typeof selectBestOfferFromListings>[0]); + assert.equal(unknown1171.bestOffer?.id, 'only'); + globalThis.fetch = origFetch1170; + + console.log( + ' ✅ [126k/135] Batch offer availability, sold-out radar and best offer, stock 0 (#1170, #1171, #1174) passed' + ); + } + + // ------------------------------------------------------------------------- + // 126l. Extension verdict, cheaper-elsewhere and fallback match skip + // sold-out listings (#1177) + // ------------------------------------------------------------------------- + { + const { evaluateExtensionPriceVerdict, lookupExtensionProduct } = + await import('@/server/rate/extension'); + + const offer1177 = ( + id: string, + price: number, + availability: string | null + ) => + ({ + id, + platform: 'shopee', + url: `https://shopee.vn/i.1177.${id}`, + price_current: price, + price_original: null, + availability, + merchant: { + id: `m-${id}`, + name: `Shop ${id}`, + risk_score: 5, + is_official: true, + }, + }) as never; + const offers1177 = [ + offer1177('fpt-sold', 2_694_000, 'out_of_stock'), + offer1177('fpt-gone', 3_032_000, 'discontinued'), + offer1177('tgdd', 3_140_000, 'in_stock'), + offer1177('cps', 3_690_000, null), + ]; + const verdict1177 = evaluateExtensionPriceVerdict({ + isAccessory: false, + accessoryReason: null, + isAmbiguousVariant: false, + observedPrice: 3_140_000, + observedOriginalPrice: null, + offers: offers1177, + observedOfferId: 'tgdd', + priceHistory: null, + }); + assert.equal(verdict1177.marketMinPrice, 3_140_000); + assert.equal(verdict1177.marketMaxPrice, 3_690_000); + assert.equal(verdict1177.cheaperOfferCount, 0); + assert.equal(verdict1177.maxSavingsAmount, 0); + // A sold-out listing's own page is judged against the in-stock market. + const ownSold1177 = evaluateExtensionPriceVerdict({ + isAccessory: false, + accessoryReason: null, + isAmbiguousVariant: false, + observedPrice: 3_032_000, + observedOriginalPrice: null, + offers: offers1177, + observedOfferId: 'fpt-gone', + priceHistory: null, + }); + assert.equal(ownSold1177.marketMinPrice, 3_140_000); + assert.equal(ownSold1177.cheaperOfferCount, 0); + assert.notEqual(ownSold1177.badgeLabel, 'Giá thấp bất thường'); + // From the dearest in-stock page, only the in-stock store is cheaper. + const soldPage1177 = evaluateExtensionPriceVerdict({ + isAccessory: false, + accessoryReason: null, + isAmbiguousVariant: false, + observedPrice: 3_690_000, + observedOriginalPrice: null, + offers: offers1177, + observedOfferId: 'cps', + priceHistory: null, + }); + assert.equal(soldPage1177.cheaperOfferCount, 1); + assert.equal(soldPage1177.maxSavingsAmount, 550_000); + + const origFetch1177 = globalThis.fetch; + const itemIds1177: Record<string, number> = { + 'fpt-sold': 117711, + 'fpt-gone': 117712, + tgdd: 117713, + cps: 117714, + }; + const viewRow1177 = (id: string, price: number) => ({ + id, + cluster_id: 'master-1177', + merchant_id: `m-${id}`, + platform: 'shopee', + title: 'Samsung Galaxy Buds3 Pro', + price_current: price, + stock: 0, + url: `https://shopee.vn/product-i.1177.${itemIds1177[id]}`, + threat_risk_score: 5, + threat_source: 'dosafe', + dosafe_risk_score: 5, + review_risk_score: 5, + review_risk_breakdown: { total_reviews: 3 }, + }); + const json1177 = (body: unknown) => + new Response(JSON.stringify(body), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + try { + globalThis.fetch = (async (input: RequestInfo | URL) => { + const url = new URL( + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : input.url + ); + const firstPage = Number(url.searchParams.get('offset') ?? 0) === 0; + const select = url.searchParams.get('select') ?? ''; + if ( + url.pathname.endsWith('/products') && + url.searchParams.has('platform_item_id') + ) { + return json1177([ + { cluster_id: 'master-1177', platform_item_id: '1177_117714' }, + ]); + } + if ( + url.pathname.endsWith('/master_products') && + url.searchParams.get('id') === 'eq.master-1177' + ) { + return json1177({ + id: 'master-1177', + slug: 'samsung-galaxy-buds3-pro', + name: 'Samsung Galaxy Buds3 Pro', + brand: 'Samsung', + model: 'Galaxy Buds3 Pro', + images: [], + key_specs: {}, + specifications: {}, + min_price: 3_140_000, + max_price: 3_690_000, + category_id: 'cat-earbuds', + }); + } + if (url.pathname.endsWith('/v_products_with_merchant_trust')) { + return json1177( + firstPage + ? [ + viewRow1177('fpt-sold', 2_694_000), + viewRow1177('fpt-gone', 3_032_000), + viewRow1177('tgdd', 3_140_000), + viewRow1177('cps', 3_690_000), + ] + : [] + ); + } + if ( + url.pathname.endsWith('/products') && + select === 'id,availability' + ) { + return json1177( + firstPage + ? [ + { id: 'fpt-sold', availability: 'out_of_stock' }, + { id: 'fpt-gone', availability: 'discontinued' }, + { id: 'tgdd', availability: 'in_stock' }, + { id: 'cps', availability: 'in_stock' }, + ] + : [] + ); + } + return json1177([]); + }) as typeof fetch; + + // On the CellphoneS listing: the sold-out FPT listings are not cheaper. + const onCps1177 = await lookupExtensionProduct({ + platform: 'shopee', + externalId: '1177_117714', + mode: 'pill', + }); + assert.equal(onCps1177.masterProduct?.id, 'master-1177'); + assert.equal(onCps1177.matchedOffer?.id, 'cps'); + assert.deepEqual( + onCps1177.topCheaperOffers.map((o) => o.id), + ['tgdd'] + ); + assert.equal(onCps1177.priceAssessment.marketMinPrice, 3_140_000); + + // No listing matches: the fallback is the cheapest in-stock offer. + const unmatched1177 = await lookupExtensionProduct({ + platform: 'shopee', + externalId: '1177_117799', + mode: 'pill', + }); + assert.equal(unmatched1177.matchedOffer?.id, 'tgdd'); + assert.ok( + unmatched1177.topCheaperOffers.every( + (o) => o.id !== 'fpt-sold' && o.id !== 'fpt-gone' + ) + ); + } finally { + globalThis.fetch = origFetch1177; + } + + console.log( + ' ✅ [126l/135] Extension verdict, cheaper-elsewhere and fallback match skip sold-out listings (#1177) passed' + ); + } + // ------------------------------------------------------------------------- // 127. Rate Server Index & Extension Entity, Slug, and Category Bounds (#1029) // ------------------------------------------------------------------------- diff --git a/apps/web/src/server/rate/api-handler.ts b/apps/web/src/server/rate/api-handler.ts index ffce902dd..f28a6e7a3 100644 --- a/apps/web/src/server/rate/api-handler.ts +++ b/apps/web/src/server/rate/api-handler.ts @@ -5,6 +5,10 @@ import { MAX_VALID_VND_PRICE, normalizeOfferOriginalPrice, } from '@/lib/rate-api'; +import { + isOfferUnavailable, + type OfferAvailability, +} from '@/lib/product-market-stats'; export const PUBLIC_API_CORS_HEADERS: Record<string, string> = { 'Access-Control-Allow-Origin': '*', @@ -243,6 +247,7 @@ export interface WatchlistCandidateOffer { price_current: number; price_original?: unknown; stock?: unknown; + availability?: OfferAvailability | null; url?: unknown; platform?: unknown; merchant?: { @@ -274,19 +279,16 @@ export function withPlausibleMinPrice<T extends { min_price?: unknown }>( export function selectBestOfferFromListings( offers: readonly WatchlistCandidateOffer[] ): { bestOffer: WatchlistBestOffer | null; offerPlatforms: string[] } { - const safeOffers = Array.isArray(offers) ? offers : []; + // A sold-out or discontinued listing is never the best offer. stock is no + // signal: the crawlers report 0 when they read no count (DOS/Rate#1171). + const safeOffers = (Array.isArray(offers) ? offers : []).filter( + (o) => !isOfferUnavailable(o) + ); // A bait-price listing is never the best offer (DOS/Rate#797). const valid = plausibleOffers(safeOffers).filter((o) => { if (!o || typeof o !== 'object') return false; const price = parseFiniteDecimalInput(o.price_current); - const stock = o.stock == null ? 1 : parseFiniteDecimalInput(o.stock); - return ( - Number.isFinite(stock) && - stock > 0 && - Number.isFinite(price) && - price > 0 && - price <= MAX_VALID_VND_PRICE - ); + return Number.isFinite(price) && price > 0 && price <= MAX_VALID_VND_PRICE; }); const normalizeOfferPlatform = (o: WatchlistCandidateOffer): string => String(o.merchant?.platform || o.platform || 'shopee') diff --git a/apps/web/src/server/rate/deals.ts b/apps/web/src/server/rate/deals.ts index 7ea837283..fcd561ad5 100644 --- a/apps/web/src/server/rate/deals.ts +++ b/apps/web/src/server/rate/deals.ts @@ -18,6 +18,7 @@ import { getRadarSignal, type RadarSignal, } from '../../lib/deal-radar'; +import { isOfferUnavailable } from '../../lib/product-market-stats'; import { plausibleOffers } from '@repo/shared-data/rules'; import { RATE_CACHE_TAGS, @@ -51,6 +52,8 @@ export interface CrossPlatformDealRadarItem { marketMaxPrice: number; savingsVsMedianVnd: number; savingsVsMedianPct: number; + /** Offers behind marketMedianPrice / marketMinPrice (DOS/Rate#1145). */ + marketOfferCount: number; claimedDiscountPct: number; fakeMarkupPct: number; jreviewsScore10: number | null; @@ -85,7 +88,10 @@ export interface CrossPlatformDealRadarResponse { categoryFilter: CanonicalCategorySlug | 'all'; platformFilter?: string; totalClustersAnalyzed: number; + /** Offers (listings), not merchants: one merchant can list several. */ totalOffersAnalyzed: number; + /** Distinct merchants behind totalOffersAnalyzed (DOS/Rate#1145). */ + totalMerchantsAnalyzed: number; genuineDeals: CrossPlatformDealRadarItem[]; fakeSaleTraps: CrossPlatformDealRadarItem[]; priceSpreadLeaders: CrossPlatformPriceSpreadItem[]; @@ -250,6 +256,7 @@ export function computeCrossPlatformDealRadarFromData( const priceSpreadLeaders: CrossPlatformPriceSpreadItem[] = []; let totalClustersAnalyzed = 0; let totalOffersAnalyzed = 0; + const analyzedMerchantKeys = new Set<string>(); const boundedClusters = Array.isArray(clusters) ? clusters.slice(0, MAX_DEAL_RADAR_CLUSTERS) @@ -296,12 +303,14 @@ export function computeCrossPlatformDealRadarFromData( const clusterOffers = Array.isArray(offers) ? offers : []; // Bait-price listings (far below every other store) set no market floor, - // spread, deal or trap (DOS/Rate#797). + // spread, deal or trap (DOS/Rate#797), and neither do sold-out or + // discontinued listings, which keep their last price (DOS/Rate#1170). const validOffers = plausibleOffers( clusterOffers .filter( (o): o is MerchantOffer => Boolean(o && typeof o === 'object' && !Array.isArray(o)) && + !isOfferUnavailable(o) && typeof o.price_current === 'number' && Number.isFinite(o.price_current) && o.price_current > 0 && @@ -393,6 +402,17 @@ export function computeCrossPlatformDealRadarFromData( totalClustersAnalyzed++; totalOffersAnalyzed += validOffers.length; + for (const o of validOffers) { + // Without an id, a seller is its platform and name; without a name + // either (the name fell back to the platform), only the offer is known. + const named = o.merchant.name !== o.platform.toUpperCase(); + analyzedMerchantKeys.add( + o.merchant.id || + (named + ? `${o.platform.toLowerCase()}:${o.merchant.name.toLowerCase()}` + : `offer:${o.id}`) + ); + } const prices = validOffers.map((o) => o.price_current); const marketMinPrice = Math.min(...prices); @@ -493,7 +513,7 @@ export function computeCrossPlatformDealRadarFromData( verdict, verdictBadgeVi: isAllTimeLow ? '🔥 Đáy Giá Đa Sàn' - : '✅ Giá Tốt Nhất 10 Sàn', + : '✅ Giá Tốt Nhất Hiện Có', productSlug: safeMasterSlug, productName: master.name, brand: master.brand, @@ -513,10 +533,11 @@ export function computeCrossPlatformDealRadarFromData( marketMaxPrice, savingsVsMedianVnd, savingsVsMedianPct, + marketOfferCount: validOffers.length, claimedDiscountPct, fakeMarkupPct: 0, jreviewsScore10, - explanationVi: `Rẻ hơn trung vị 10 sàn ${savingsVsMedianVnd.toLocaleString('vi-VN')} ₫ (-${savingsVsMedianPct}%) tại ${bestOffer.merchant.name} (${bestOffer.platform.toUpperCase()}).`, + explanationVi: `Rẻ hơn giá trung vị của ${validOffers.length} tin bán ${savingsVsMedianVnd.toLocaleString('vi-VN')} ₫ (-${savingsVsMedianPct}%) tại ${bestOffer.merchant.name} (${bestOffer.platform.toUpperCase()}).`, }); } @@ -573,6 +594,7 @@ export function computeCrossPlatformDealRadarFromData( 100 ).toFixed(1) ), + marketOfferCount: validOffers.length, claimedDiscountPct, fakeMarkupPct, jreviewsScore10, @@ -617,6 +639,7 @@ export function computeCrossPlatformDealRadarFromData( platformFilter, totalClustersAnalyzed, totalOffersAnalyzed, + totalMerchantsAnalyzed: analyzedMerchantKeys.size, genuineDeals: genuineDeals.slice(0, limit), fakeSaleTraps: fakeSaleTraps.slice(0, limit), priceSpreadLeaders: priceSpreadLeaders.slice(0, limit), @@ -674,7 +697,8 @@ const _getCrossPlatformDealRadarByKey = cache( }); }, [ - 'rate-cross-platform-deal-radar', + // v2: the response gained totalMerchantsAnalyzed and marketOfferCount. + 'rate-cross-platform-deal-radar-v2', categorySlug, platformFilter, String(limit), diff --git a/apps/web/src/server/rate/extension.ts b/apps/web/src/server/rate/extension.ts index 5186a6ec7..0d9e0e1ec 100644 --- a/apps/web/src/server/rate/extension.ts +++ b/apps/web/src/server/rate/extension.ts @@ -28,6 +28,7 @@ import { type MerchantSummary, type ReviewAnalysisSummary, } from '../../lib/rate-api'; +import { isOfferUnavailable } from '../../lib/product-market-stats'; import { computeProductStandingInCategory, type ProductLeaderboardBadgeInfo, @@ -606,7 +607,11 @@ export function evaluateExtensionPriceVerdict(params: { observedOfferId, priceHistory, } = safeParams; - const safeOffers = Array.isArray(offers) ? offers : []; + // Sold-out and discontinued listings keep their last price; they set no + // market reference and are no cheaper option (DOS/Rate#1177). + const safeOffers = (Array.isArray(offers) ? offers : []).filter( + (o) => !isOfferUnavailable(o) + ); if (isAccessory) { return { @@ -1223,15 +1228,17 @@ export async function lookupExtensionProduct( // Fetch offers, price history, and reviews for matched master product const offers = await getMasterProductOffers(matchedMaster.id); - // Without a URL match, fall back to the cheapest plausible offer, never a - // bait-price listing (DOS/Rate#797). + // Without a URL match, fall back to the cheapest plausible in-stock offer, + // never a bait-price or sold-out listing (DOS/Rate#797, #1177). + const inStockOffers = offers.filter((o) => !isOfferUnavailable(o)); const urlMatchedOffer = offers.find( (o) => (canonicalUrl && o.url.includes(canonicalUrl)) || doesOfferUrlMatchCandidateIds(o.url, candidateItemIds) ) || null; - const matchedOffer = urlMatchedOffer || plausibleOffers(offers)[0] || null; + const matchedOffer = + urlMatchedOffer || plausibleOffers(inStockOffers)[0] || null; if ( matchedOffer && @@ -1305,10 +1312,11 @@ export async function lookupExtensionProduct( : null; const isAmbiguous = matchStatus === 'ambiguous_variant'; - // The market reference and "cheaper elsewhere" leave bait-price listings - // out; the matched offer above still comes from every offer, so a shopper - // on a bait listing sees it judged against the real market (DOS/Rate#797). - const marketOffers = plausibleOffers(offers); + // The market reference and "cheaper elsewhere" leave bait-price and + // sold-out listings out; the URL-matched offer above still comes from every + // offer, so a shopper on such a listing sees it judged against the real + // market (DOS/Rate#797, #1177). + const marketOffers = plausibleOffers(inStockOffers); const priceAssessment = evaluateExtensionPriceVerdict({ isAccessory: false, accessoryReason: null, diff --git a/apps/web/src/server/rate/index.ts b/apps/web/src/server/rate/index.ts index bb1bce96b..fe2728516 100644 --- a/apps/web/src/server/rate/index.ts +++ b/apps/web/src/server/rate/index.ts @@ -16,6 +16,7 @@ import { getMerchantProfileById as _getMerchantProfileById, getMerchantsList as _getMerchantsList, getRatePlatformStats as _getRatePlatformStats, + getFilteredReviewCountsByMaster as _getFilteredReviewCountsByMaster, getSitemapRows as _getSitemapRows, inspectProductOrShopUrl as _inspectProductOrShopUrl, isCanonicalCategorySlug, @@ -500,6 +501,18 @@ export const getCachedRatePlatformStats = cache(() => })() ); +/** Stored reviews kept out of the rating, per master id (DOS/Rate#1139). */ +export const getCachedFilteredReviewCountsByMaster = cache(() => + withRateDataCache( + _getFilteredReviewCountsByMaster, + ['rate-filtered-review-counts-by-master'], + { + revalidate: 600, + tags: [RATE_CACHE_TAGS.REVIEWS], + } + )() +); + export const getCachedInspectProductOrShopUrl = cache( inspectProductOrShopUrlWithRadar ); diff --git a/packages/shared-data/src/__tests__/shared-data.test.ts b/packages/shared-data/src/__tests__/shared-data.test.ts index 09c94fc63..869241bf4 100644 --- a/packages/shared-data/src/__tests__/shared-data.test.ts +++ b/packages/shared-data/src/__tests__/shared-data.test.ts @@ -823,30 +823,404 @@ function testPriceRules(): void { } } +function testCategorySpecTemplates(): void { + const phoneShaped = { + ram_gb: 1, + storage_gb: 64, + screen_size_inch: 1, + has_5g: true, + specifications: { + category_slug: 'x', + display: { size_inch: 1, refresh_rate_hz: 60, ppi: 300 }, + hardware: { ram_gb: 1, storage_gb: 64, gpu: 'g', chipset: 'c' }, + camera: { main_camera: 'mic' }, + cellular: { has_5g: true, bands_5g: 'n78', bands_4g: 'LTE' }, + design: { biometrics: 'b', weight_g: 5 }, + connectivity: { bluetooth: '5.4', wlan: 'w', nfc: false }, + multimedia: { audio_features: 'ANC', headphone_jack_35mm: false }, + battery: { capacity_mah: 500 }, + }, + key_specs: { + screen: 's', + chip: 'c', + camera: 'm', + battery: 'b', + storage: '64GB', + ram: '1GB', + }, + }; + + // Phones and tablets, and any unknown slug, keep everything untouched. + for (const slug of [ + 'smartphones', + 'tablets', + 'laptops', + undefined, + null, + 'nope', + ]) { + assert.equal( + rules.applyCategorySpecTemplate(slug, phoneShaped), + phoneShaped + ); + assert.equal(rules.getCategorySpecTemplate(slug).groups.length, 10); + } + + // Earbuds: audio groups only, no screen, RAM, ROM, camera or cellular. + const buds = rules.applyCategorySpecTemplate(' Earbuds ', phoneShaped); + const budsSpecs = buds.specifications as unknown as Record< + string, + Record<string, unknown> + >; + assert.deepEqual(rules.getCategorySpecTemplate('earbuds').groups, [ + 'multimedia', + 'battery', + 'connectivity', + 'hardware', + 'design', + 'availability', + 'regulatory', + ]); + assert.equal(buds.ram_gb, null); + assert.equal(buds.storage_gb, null); + assert.equal(buds.screen_size_inch, null); + assert.equal(buds.has_5g, false); + for (const group of ['display', 'camera', 'cellular']) { + assert.equal(budsSpecs[group], undefined, `earbuds drop ${group}`); + } + assert.equal(budsSpecs.category_slug, 'x', 'non-group keys are kept'); + assert.deepEqual(budsSpecs.connectivity, { bluetooth: '5.4' }); + assert.deepEqual(budsSpecs.multimedia, { audio_features: 'ANC' }); + assert.deepEqual(budsSpecs.design, { weight_g: 5 }); + // The audio chip is the one hardware fact earbuds keep. + assert.deepEqual(budsSpecs.hardware, { chipset: 'c' }); + const budsStored = rules.applyCategorySpecTemplate('earbuds', phoneShaped, { + forStorage: true, + }); + assert.deepEqual(budsStored, buds, 'earbuds store what they show'); + assert.deepEqual(buds.key_specs, { chip: 'c', battery: 'b' }); + + // Smartwatches: the page shows no camera and no 5G; screen, storage and + // LTE stay. + const watch = rules.applyCategorySpecTemplate('smartwatches', phoneShaped); + const watchSpecs = watch.specifications as unknown as Record< + string, + Record<string, unknown> + >; + assert.equal(watch.has_5g, false); + assert.equal(watch.storage_gb, 64); + assert.equal(watch.screen_size_inch, 1); + assert.equal(watchSpecs.camera, undefined); + assert.deepEqual(watchSpecs.cellular, { bands_4g: 'LTE' }); + assert.deepEqual(watchSpecs.hardware, { + ram_gb: 1, + storage_gb: 64, + chipset: 'c', + }); + assert.deepEqual(watch.key_specs, { + screen: 's', + chip: 'c', + battery: 'b', + storage: '64GB', + ram: '1GB', + }); + // Stored, a watch keeps its 5G facts: a few models (RedCap) do have 5G. + const watchStored = rules.applyCategorySpecTemplate( + 'smartwatches', + phoneShaped, + { forStorage: true } + ); + const watchStoredSpecs = watchStored.specifications as unknown as Record< + string, + Record<string, unknown> + >; + assert.equal(watchStored.has_5g, true); + assert.deepEqual( + watchStoredSpecs.cellular, + phoneShaped.specifications.cellular + ); + assert.equal(watchStoredSpecs.hardware?.gpu, 'g'); + assert.equal(watchStoredSpecs.camera, undefined); + + // Idempotent, never mutates the input, and leaves absent columns absent. + assert.deepEqual(rules.applyCategorySpecTemplate('earbuds', buds), buds); + assert.equal(phoneShaped.ram_gb, 1); + assert.equal(phoneShaped.specifications.display.size_inch, 1); + assert.equal( + 'ram_gb' in rules.applyCategorySpecTemplate('earbuds', { has_5g: true }), + false + ); + + const watchTemplate = rules.getCategorySpecTemplate('smartwatches'); + assert.equal( + rules.specFieldApplies(watchTemplate, 'cellular', 'has_5g'), + false + ); + assert.equal( + rules.specFieldApplies(watchTemplate, 'cellular', 'technology'), + true + ); + assert.equal( + rules.specFieldApplies(watchTemplate, 'camera', 'main_camera'), + false + ); +} + +function assertRoundTrip(modelKey: string): void { + const parts = sharedDataRoot.splitModelKey(modelKey); + assert.ok(parts, `splitModelKey must parse ${modelKey}`); + const segments = modelKey.split(':'); + assert.equal(parts.itemKey, `${segments[0]}:${segments[1]}`, modelKey); + assert.equal(parts.variantKey, segments[2], modelKey); + assert.equal( + sharedDataRoot.composeModelKey(parts.itemKey, parts.variantKey), + modelKey + ); + assert.equal(sharedDataRoot.isItemKey(parts.itemKey), true, modelKey); + assert.equal(sharedDataRoot.isVariantKey(parts.variantKey), true, modelKey); +} + +function testItemVariantKeys(): void { + const { + splitModelKey, + composeModelKey, + isModelKey, + isItemKey, + isVariantKey, + isSizedVariantKey, + UNKNOWN_VARIANT_KEY, + ItemSchema, + VariantSchema, + VariantSourceSchema, + ItemKeySchema, + VariantKeySchema, + ModelKeySchema, + } = sharedDataRoot; + + // Exported from the root entry and from the zod-free rules entry. + assert.equal(rules.splitModelKey, splitModelKey); + assert.equal(rules.composeModelKey, composeModelKey); + + // Every key form the normalizer emits today (design doc 4.2). + const forms: Array<[string, string, string]> = [ + ['apple:iphone-17-pro:256gb', 'apple:iphone-17-pro', '256gb'], + ['apple:iphone-17-pro-max:1024gb', 'apple:iphone-17-pro-max', '1024gb'], + // Apple screen / case size stays in the Item (#1113, #1124). + ['apple:ipad-pro-m5-11:256gb', 'apple:ipad-pro-m5-11', '256gb'], + ['apple:ipad-pro-m5-13:1024gb', 'apple:ipad-pro-m5-13', '1024gb'], + ['apple:watch-series-11-46mm:64gb', 'apple:watch-series-11-46mm', '64gb'], + // Storage not stated, and earbuds / sizeless watches (single variant). + ['apple:iphone-18-pro:unknown', 'apple:iphone-18-pro', 'unknown'], + ['apple:airpods-pro-2:unknown', 'apple:airpods-pro-2', 'unknown'], + ['samsung:galaxy-buds3-pro:unknown', 'samsung:galaxy-buds3-pro', 'unknown'], + ['garmin:fenix-8:unknown', 'garmin:fenix-8', 'unknown'], + // Literal legacy / future variant keys (design doc 4.2). + [ + 'samsung:galaxy-buds-3-pro:standard', + 'samsung:galaxy-buds-3-pro', + 'standard', + ], + [ + 'apple:ipad-air-m3-11:256gb-cellular', + 'apple:ipad-air-m3-11', + '256gb-cellular', + ], + [ + 'samsung:galaxy-s26-ultra:256gb-16gbram', + 'samsung:galaxy-s26-ultra', + '256gb-16gbram', + ], + // A dot is allowed in the model segment (design doc 7.3), although + // today's slugify strips it. + ['xiaomi:redmi-note-14.5:128gb', 'xiaomi:redmi-note-14.5', '128gb'], + // slugify keeps `_` (it strips [^\w-]), including at the start. + ['asus:vivobook-x1504_va:512gb', 'asus:vivobook-x1504_va', '512gb'], + ['samsung:_galaxy-s25:256gb', 'samsung:_galaxy-s25', '256gb'], + ]; + for (const [modelKey, itemKey, variantKey] of forms) { + assert.deepEqual(splitModelKey(modelKey), { itemKey, variantKey }); + assert.equal(composeModelKey(itemKey, variantKey), modelKey); + assertRoundTrip(modelKey); + } + + // Every three-part key literal in the importer's normalizer fixtures. + const fixtureDir = new URL( + '../../../../apps/importer/src/shopee/__tests__/', + import.meta.url + ); + const fixtureKeys = new Set<string>(); + for (const file of readdirSync(fixtureDir)) { + if (!/\.(ts|mjs)$/.test(file)) continue; + const source = readFileSync(new URL(file, fixtureDir), 'utf8'); + for (const m of source.matchAll( + // Wider than the key shape (upper case), so a fixture key the helpers + // would reject fails here instead of being skipped. The brand must + // start with a letter or `_`, which leaves out ISO timestamps. + /['"`]([A-Za-z_][\w-]*:[\w.-]+:[\w.-]+)['"`]/g + )) { + fixtureKeys.add(m[1] ?? ''); + } + } + assert.ok( + fixtureKeys.size >= 100, + `expected the importer fixtures to hold 100+ keys, found ${fixtureKeys.size}` + ); + for (const modelKey of fixtureKeys) assertRoundTrip(modelKey); + + // Not in the brand:model:variant shape: not parsed, not composed. + for (const bad of [ + '', + 'apple_iphone_16', + 'other:key', + 'khc:legacy', + 'a:b:c:d', + 'Apple:iphone-17:256gb', + 'apple::256gb', + ':iphone-17:256gb', + 'apple:iphone-17:', + 'apple:-iphone:256gb', + 'apple:iphone 17:256gb', + // Empty model segment, as the normalizer emits for a title with no model. + 'samsung::unknown', + 'khac::256gb', + 'apple:iphone-17:256GB', + `apple:${'x'.repeat(200)}:256gb`, + ]) { + assert.equal(splitModelKey(bad), null, bad); + assert.equal(isModelKey(bad), false, bad); + assert.equal(ModelKeySchema.safeParse(bad).success, false, bad); + } + assert.equal(splitModelKey(undefined as unknown as string), null); + assert.throws(() => composeModelKey('apple', '256gb'), RangeError); + assert.throws( + () => composeModelKey('apple:iphone-17:256gb', '256gb'), + RangeError + ); + assert.throws(() => composeModelKey('apple:iphone-17', ''), RangeError); + assert.throws(() => composeModelKey('apple:iphone-17', '256:gb'), RangeError); + assert.throws( + () => composeModelKey(`apple:${'x'.repeat(190)}`, '1024gb'), + RangeError + ); + assert.equal(isItemKey('apple:ipad-pro-m5-11'), true); + assert.equal(isItemKey('apple:ipad-pro-m5-11:256gb'), false); + assert.equal(isVariantKey('256gb-cellular'), true); + assert.equal(isVariantKey('256gb:x'), false); + assert.equal(ItemKeySchema.safeParse('apple:ipad-pro-m5-11').success, true); + assert.equal(ItemKeySchema.safeParse('apple').success, false); + assert.equal(VariantKeySchema.safeParse('unknown').success, true); + assert.equal(VariantKeySchema.safeParse('').success, false); + + // Only a stated storage size makes an Item multi-size (design doc 4.2). + for (const sized of ['8gb', '64gb', '256gb', '1024gb', '1tb']) { + assert.equal(isSizedVariantKey(sized), true, sized); + } + for (const other of [ + UNKNOWN_VARIANT_KEY, + 'standard', + '256gb-cellular', + 'gb', + '', + ]) { + assert.equal(isSizedVariantKey(other), false, other); + } + assert.equal(UNKNOWN_VARIANT_KEY, 'unknown'); + + const id = '018f3b2c-1a2b-4c3d-8e4f-5a6b7c8d9e0f'; + const item = ItemSchema.parse({ + id, + item_key: 'apple:ipad-pro-m5-11', + slug: 'apple-ipad-pro-m5-11', + name: 'Apple iPad Pro M5 11', + brand: 'Apple', + model: 'iPad Pro M5 11', + }); + assert.equal(item.status, 'draft'); + assert.equal(item.min_price, 0); + assert.equal(item.unknown_variant_listing_count, 0); + assert.deepEqual(item.specifications, {}); + // Legacy keys still get an Item (design doc 4.3): no shape check on rows. + assert.equal( + ItemSchema.safeParse({ ...item, item_key: 'khc:legacy-thing' }).success, + true + ); + assert.equal( + ItemSchema.safeParse({ ...item, status: 'hidden' }).success, + false + ); + assert.equal( + ItemSchema.safeParse({ ...item, average_rating: 6 }).success, + false + ); + + const variant = { + id, + brand: 'Apple', + model: 'iPad Pro M5 11', + model_key: 'apple:ipad-pro-m5-11:256gb', + name: 'Apple iPad Pro M5 11 256GB', + slug: 'apple-ipad-pro-m5-11-256gb', + item_id: id, + variant_key: '256gb', + sort_order: 1, + }; + assert.equal(VariantSchema.parse(variant).variant_key, '256gb'); + // Before the backfill both links may be missing. + assert.equal( + VariantSchema.safeParse({ ...variant, item_id: null, variant_key: null }) + .success, + true + ); + // model_key = item_key:variant_key must hold. + assert.equal( + VariantSchema.safeParse({ ...variant, variant_key: '512gb' }).success, + false + ); + assert.equal( + VariantSchema.safeParse({ ...variant, sort_order: 40000 }).success, + false + ); + + assert.deepEqual(VariantSourceSchema.options, [ + 'title', + 'sku', + 'single_size', + 'single_variant', + 'manual', + 'legacy', + ]); +} + function main(): void { console.log('🧪 Running @repo/shared-data unit tests...'); testVersionAndZeroStrapiExports(); console.log( - ' ✅ [1/7] Version & Zero-Strapi root export isolation verified' + ' ✅ [1/8] Version & Zero-Strapi root export isolation verified' ); testRouteConstants(); - console.log(' ✅ [2/7] Route & REST API v1 constants verified'); + console.log(' ✅ [2/8] Route & REST API v1 constants verified'); testTenFieldGroupsSpecificationsSchema(); console.log( - ' ✅ [3/7] 10-Field-Groups ProductSpecificationsSchema verified' + ' ✅ [3/8] 10-Field-Groups ProductSpecificationsSchema verified' ); testMasterProductAndScrapedProductSchemas(); console.log( - ' ✅ [4/7] MasterProduct, ScrapedProduct & common schemas verified' + ' ✅ [4/8] MasterProduct, ScrapedProduct & common schemas verified' ); testSupabaseClientGuards(); - console.log(' ✅ [5/7] Supabase client fail-closed env guards verified'); + console.log(' ✅ [5/8] Supabase client fail-closed env guards verified'); testSeedingRiskPenalty(); - console.log(' ✅ [6/7] Shared seeding risk penalty verified'); + console.log(' ✅ [6/8] Shared seeding risk penalty verified'); testPriceRules(); console.log( - ' ✅ [7/7] Shared price rules (@repo/shared-data/rules) verified' + ' ✅ [7/8] Shared price rules (@repo/shared-data/rules) verified' + ); + testItemVariantKeys(); + console.log( + ' ✅ [8/8] Item / variant schemas and model_key helpers verified' ); + testCategorySpecTemplates(); + console.log(' ✅ [9/9] Category spec templates (DOS/Rate#1142) verified'); console.log('🎉 All @repo/shared-data tests passed!'); } diff --git a/packages/shared-data/src/database.types.ts b/packages/shared-data/src/database.types.ts index fd61c82c0..ec7c430bd 100644 --- a/packages/shared-data/src/database.types.ts +++ b/packages/shared-data/src/database.types.ts @@ -189,6 +189,7 @@ export type Database = { ram_gb: number | null; screen_size_inch: number | null; slug: string; + spec_provenance: Json; specifications: Json; status: string | null; storage_gb: number | null; @@ -225,6 +226,7 @@ export type Database = { ram_gb?: number | null; screen_size_inch?: number | null; slug: string; + spec_provenance?: Json; specifications?: Json; status?: string | null; storage_gb?: number | null; @@ -261,6 +263,7 @@ export type Database = { ram_gb?: number | null; screen_size_inch?: number | null; slug?: string; + spec_provenance?: Json; specifications?: Json; status?: string | null; storage_gb?: number | null; @@ -448,8 +451,10 @@ export type Database = { }; price_history: { Row: { + crawl_job_id: string | null; id: number; observation_id: string | null; + origin: string; price: number; price_original: number | null; product_id: string; @@ -458,8 +463,10 @@ export type Database = { source: string | null; }; Insert: { + crawl_job_id?: string | null; id?: number; observation_id?: string | null; + origin?: string; price: number; price_original?: number | null; product_id: string; @@ -468,8 +475,10 @@ export type Database = { source?: string | null; }; Update: { + crawl_job_id?: string | null; id?: number; observation_id?: string | null; + origin?: string; price?: number; price_original?: number | null; product_id?: string; @@ -501,6 +510,8 @@ export type Database = { products: { Row: { attributes: Json | null; + availability: string; + availability_checked_at: string | null; brand: string | null; category_id: string | null; cluster_id: string | null; @@ -511,9 +522,12 @@ export type Database = { historical_sold_display: string | null; id: string; images: string[] | null; + last_crawl_job_id: string | null; + last_crawled_at: string | null; merchant_id: string; metadata: Json | null; models: Json | null; + origin: string; platform: string; platform_item_id: string; price_current: number; @@ -529,6 +543,8 @@ export type Database = { }; Insert: { attributes?: Json | null; + availability?: string; + availability_checked_at?: string | null; brand?: string | null; category_id?: string | null; cluster_id?: string | null; @@ -539,9 +555,12 @@ export type Database = { historical_sold_display?: string | null; id?: string; images?: string[] | null; + last_crawl_job_id?: string | null; + last_crawled_at?: string | null; merchant_id: string; metadata?: Json | null; models?: Json | null; + origin?: string; platform: string; platform_item_id: string; price_current: number; @@ -557,6 +576,8 @@ export type Database = { }; Update: { attributes?: Json | null; + availability?: string; + availability_checked_at?: string | null; brand?: string | null; category_id?: string | null; cluster_id?: string | null; @@ -567,9 +588,12 @@ export type Database = { historical_sold_display?: string | null; id?: string; images?: string[] | null; + last_crawl_job_id?: string | null; + last_crawled_at?: string | null; merchant_id?: string; metadata?: Json | null; models?: Json | null; + origin?: string; platform?: string; platform_item_id?: string; price_current?: number; @@ -833,10 +857,13 @@ export type Database = { created_at: string | null; criteria_ratings: Json | null; downvotes: number | null; + first_seen_job_id: string | null; helpfulness_weight: number | null; id: string; images: string[] | null; is_verified_buyer: boolean | null; + observed_at: string; + origin: string; platform_created_at: string | null; platform_review_id: string | null; product_id: string; @@ -853,10 +880,13 @@ export type Database = { created_at?: string | null; criteria_ratings?: Json | null; downvotes?: number | null; + first_seen_job_id?: string | null; helpfulness_weight?: number | null; id?: string; images?: string[] | null; is_verified_buyer?: boolean | null; + observed_at?: string; + origin?: string; platform_created_at?: string | null; platform_review_id?: string | null; product_id: string; @@ -873,10 +903,13 @@ export type Database = { created_at?: string | null; criteria_ratings?: Json | null; downvotes?: number | null; + first_seen_job_id?: string | null; helpfulness_weight?: number | null; id?: string; images?: string[] | null; is_verified_buyer?: boolean | null; + observed_at?: string; + origin?: string; platform_created_at?: string | null; platform_review_id?: string | null; product_id?: string; @@ -998,6 +1031,7 @@ export type Database = { }; Returns: number; }; + current_user_can_vote_reviews: { Args: never; Returns: boolean }; dequeue_crawl_job: { Args: { p_lease?: string; p_platform: string; p_worker_id: string }; Returns: { @@ -1079,6 +1113,35 @@ export type Database = { Args: { p_claim_token: string; p_extend?: string; p_id: string }; Returns: boolean; }; + listing_price_context: { + Args: { + p_after?: string; + p_history_days?: number; + p_limit?: number; + p_merchant_ids?: string[]; + }; + Returns: { + cluster_id: string; + cluster_official_min: number; + history_avg: number; + history_count: number; + history_max: number; + history_median: number; + history_min: number; + merchant_id: string; + merchant_is_official: boolean; + peer_reference_min: number; + platform: string; + price_current: number; + price_original: number; + product_id: string; + title: string; + }[]; + }; + mark_listing_unavailable: { + Args: { p_availability: string; p_job_id: string }; + Returns: string; + }; match_master_product: { Args: { p_brand?: string; @@ -1096,6 +1159,17 @@ export type Database = { similarity: number; }[]; }; + merchant_review_counts: { + Args: { p_merchant_ids: string[] }; + Returns: { + authentic_count: number; + coin_spam_count: number; + merchant_id: string; + pending_count: number; + seeding_count: number; + total_reviews: number; + }[]; + }; purge_completed_crawl_jobs: { Args: { p_retention_days?: number }; Returns: number; @@ -1203,6 +1277,7 @@ export type Database = { ram_gb: number | null; screen_size_inch: number | null; slug: string; + spec_provenance: Json; specifications: Json; status: string | null; storage_gb: number | null; @@ -1220,6 +1295,10 @@ export type Database = { }; }; signals_valid: { Args: { p: Json }; Returns: boolean }; + stamp_crawl_provenance: { + Args: { p_job_id: string; p_product_id: string }; + Returns: Json; + }; start_crawl_run: { Args: { p_params?: Json; p_source: string }; Returns: string; diff --git a/packages/shared-data/src/index.ts b/packages/shared-data/src/index.ts index a1a9c407a..fd477b1ac 100644 --- a/packages/shared-data/src/index.ts +++ b/packages/shared-data/src/index.ts @@ -15,6 +15,9 @@ export * from './clients/index.js'; // Export constants export * from './constants/index.js'; +// Item / variant key helpers: model_key = item_key:variant_key (DOS/Rate#1149) +export * from './rules/model-key.js'; + // `rate` schema types generated from supabase/migrations (pnpm gen:db-types) export type { Database, Json } from './database.types.js'; diff --git a/packages/shared-data/src/rules/index.ts b/packages/shared-data/src/rules/index.ts index 3523dc373..5c111d6f3 100644 --- a/packages/shared-data/src/rules/index.ts +++ b/packages/shared-data/src/rules/index.ts @@ -1,9 +1,12 @@ /** - * Business rules about prices, shared by rate.vn, the importer and the - * crawlers (DOS/Rate#1089). Plain functions and constants only: no zod and + * Business rules about prices (DOS/Rate#1089) and catalog keys + * (DOS/Rate#1149), shared by rate.vn, the importer and the crawlers. + * Plain functions and constants only: no zod and * no Supabase client, so client components can import * `@repo/shared-data/rules` without pulling either into the bundle. */ export { MAX_VALID_VND_PRICE } from './price-limits.js'; export * from './offer-plausibility.js'; export * from './listing-price-floor.js'; +export * from './spec-template.js'; +export * from './model-key.js'; diff --git a/packages/shared-data/src/rules/model-key.ts b/packages/shared-data/src/rules/model-key.ts new file mode 100644 index 000000000..5830bfa18 --- /dev/null +++ b/packages/shared-data/src/rules/model-key.ts @@ -0,0 +1,113 @@ +/** + * Item / variant key rules (DOS/Rate#1149; design doc + * `Docs/Proposals/2026-10-27-item-variant-listing-model.md`, section 4). + * + * `model_key` = `item_key:variant_key`, the shape the normalizer emits today: + * - `item_key` = `brand:model`. An Apple screen or case size stays in the + * model segment, so `apple:ipad-pro-m5-11` and `apple:ipad-pro-m5-13` are + * different Items. + * - `variant_key` = the literal third segment: `256gb`, `1024gb`, `standard`, + * `unknown` (earbuds), and later suffixed forms such as `256gb-cellular`. + * Display labels (`1TB`, "Standard") are derived from it, never keys. + * + * Segment shapes follow the design doc's inventory check (section 7.3). They + * allow `_`, which the normalizer's slugify keeps (`x1504_va`). A key outside + * them (legacy keys without three segments, upper case, empty segments such + * as `samsung::unknown`) is not parsed: `splitModelKey` returns null for it. + */ + +const BRAND_SEGMENT = '[a-z0-9_][a-z0-9_-]*'; +const MODEL_SEGMENT = '[a-z0-9_][a-z0-9_.-]*'; +const VARIANT_SEGMENT = '[a-z0-9_-]+'; + +export const ITEM_KEY_PATTERN = new RegExp( + `^${BRAND_SEGMENT}:${MODEL_SEGMENT}$` +); +export const VARIANT_KEY_PATTERN = new RegExp(`^${VARIANT_SEGMENT}$`); +export const MODEL_KEY_PATTERN = new RegExp( + `^(${BRAND_SEGMENT}:${MODEL_SEGMENT}):(${VARIANT_SEGMENT})$` +); + +/** + * A variant that states a storage size. An Item with a non-archived sized + * variant is multi-size, and there `unknown` means "size not stated". + */ +export const SIZED_VARIANT_KEY_PATTERN = /^\d+(gb|tb)$/; + +/** + * Storage not stated (multi-size Item) or not applicable (single-variant + * Item, e.g. earbuds). + */ +const UNKNOWN = 'unknown'; +export const UNKNOWN_VARIANT_KEY = UNKNOWN; + +/** Same 200-character cap as `master_products.model_key`. */ +export const MAX_MODEL_KEY_LENGTH = 200; + +const isString = (value: unknown): value is string => typeof value === 'string'; + +export interface ModelKeyParts { + itemKey: string; + variantKey: string; +} + +export function isItemKey(value: unknown): value is string { + return ( + isString(value) && + value.length <= MAX_MODEL_KEY_LENGTH && + ITEM_KEY_PATTERN.test(value) + ); +} + +export function isVariantKey(value: unknown): value is string { + return ( + isString(value) && + value.length <= MAX_MODEL_KEY_LENGTH && + VARIANT_KEY_PATTERN.test(value) + ); +} + +export function isModelKey(value: unknown): value is string { + return ( + isString(value) && + value.length <= MAX_MODEL_KEY_LENGTH && + MODEL_KEY_PATTERN.test(value) + ); +} + +export function isSizedVariantKey(value: unknown): value is string { + return isString(value) && SIZED_VARIANT_KEY_PATTERN.test(value); +} + +/** + * Splits a `model_key` into its Item and variant keys, or returns null when + * the key is not in the `brand:model:variant` shape. + * `composeModelKey(itemKey, variantKey)` gives the same key back. + */ +export function splitModelKey(modelKey: string): ModelKeyParts | null { + if (!isModelKey(modelKey)) return null; + const match = MODEL_KEY_PATTERN.exec(modelKey); + if (!match || match[1] === undefined || match[2] === undefined) return null; + return { itemKey: match[1], variantKey: match[2] }; +} + +/** + * Builds a `model_key` from an Item key and a variant key. Throws a + * RangeError when either part, or the result, is not a valid key, so a + * malformed key never reaches a writer. + */ +export function composeModelKey(itemKey: string, variantKey: string): string { + if (!isItemKey(itemKey)) { + throw new RangeError(`Invalid item_key: ${JSON.stringify(itemKey)}`); + } + if (!isVariantKey(variantKey)) { + throw new RangeError(`Invalid variant_key: ${JSON.stringify(variantKey)}`); + } + const modelKey = `${itemKey}:${variantKey}`; + if (!isModelKey(modelKey)) { + throw new RangeError( + `model_key longer than ${MAX_MODEL_KEY_LENGTH} characters: ${JSON.stringify(modelKey)}` + ); + } + return modelKey; +} diff --git a/packages/shared-data/src/rules/spec-template.ts b/packages/shared-data/src/rules/spec-template.ts new file mode 100644 index 000000000..9c1b3a76b --- /dev/null +++ b/packages/shared-data/src/rules/spec-template.ts @@ -0,0 +1,245 @@ +/** + * Which spec groups and fields apply to each category (DOS/Rate#1142). + * + * The spec schema is one phone-shaped schema for every device, and the AI + * enricher fills it for earbuds and watches too: earbuds came out with a + * "1 inch" screen, 1GB RAM and microphones as cameras, and a cellular watch + * with 5G. rate.vn renders each category with its own template, and the + * importer drops the fields a category does not have before it writes a + * master. Plain data and functions only, like the rest of `rules`. + */ + +export type SpecGroupKey = + | 'display' + | 'hardware' + | 'camera' + | 'battery' + | 'design' + | 'cellular' + | 'connectivity' + | 'multimedia' + | 'availability' + | 'regulatory'; + +export type KeySpecKey = + 'screen' | 'chip' | 'camera' | 'battery' | 'storage' | 'ram'; + +export interface CategorySpecTemplate { + /** Groups the category has, in display order. */ + groups: readonly SpecGroupKey[]; + /** Fields of a listed group that the category does not have. */ + hiddenFields: Partial<Record<SpecGroupKey, readonly string[]>>; + /** + * Fields the page does not show although a model may really have them, so + * the importer keeps them: a few watches do carry 5G (RedCap). + */ + displayHiddenFields: Partial<Record<SpecGroupKey, readonly string[]>>; + /** `key_specs` entries that apply. */ + keySpecs: readonly KeySpecKey[]; + /** Whether the top-level `ram_gb` / `storage_gb` / `screen_size_inch` apply. */ + ram: boolean; + storage: boolean; + screen: boolean; + /** + * Whether the page shows 5G. Watches sell as LTE / GPS + Cellular and the + * enricher flags them 5G wrongly; the stored flag is cleared only for a + * category without a cellular group. + */ + has5g: boolean; +} + +const ALL_GROUPS: readonly SpecGroupKey[] = [ + 'display', + 'hardware', + 'camera', + 'battery', + 'design', + 'cellular', + 'connectivity', + 'multimedia', + 'availability', + 'regulatory', +]; + +const ALL_KEY_SPECS: readonly KeySpecKey[] = [ + 'screen', + 'chip', + 'camera', + 'battery', + 'storage', + 'ram', +]; + +const PHONE_TEMPLATE: CategorySpecTemplate = { + groups: ALL_GROUPS, + hiddenFields: {}, + displayHiddenFields: {}, + keySpecs: ALL_KEY_SPECS, + ram: true, + storage: true, + screen: true, + has5g: true, +}; + +const SPEC_TEMPLATES: Readonly<Record<string, CategorySpecTemplate>> = { + smartphones: PHONE_TEMPLATE, + tablets: PHONE_TEMPLATE, + // Laptops keep the full template: some ship with 5G modems. + laptops: PHONE_TEMPLATE, + smartwatches: { + groups: [ + 'display', + 'hardware', + 'battery', + 'design', + 'cellular', + 'connectivity', + 'multimedia', + 'availability', + 'regulatory', + ], + hiddenFields: {}, + displayHiddenFields: { + hardware: ['gpu', 'storage_expansion'], + cellular: ['has_5g', 'bands_5g'], + }, + keySpecs: ['screen', 'chip', 'battery', 'storage', 'ram'], + ram: true, + storage: true, + screen: true, + has5g: false, + }, + earbuds: { + groups: [ + 'multimedia', + 'battery', + 'connectivity', + 'hardware', + 'design', + 'availability', + 'regulatory', + ], + hiddenFields: { + // The audio chip (Apple H2, BES2700) is the one hardware fact earbuds have. + hardware: [ + 'cpu', + 'gpu', + 'ram_gb', + 'storage_gb', + 'storage_expansion', + 'os', + ], + design: ['biometrics'], + connectivity: ['wlan', 'location', 'nfc'], + multimedia: ['headphone_jack_35mm'], + }, + displayHiddenFields: {}, + keySpecs: ['chip', 'battery'], + ram: false, + storage: false, + screen: false, + has5g: false, + }, +}; + +const normalizeSlug = (categorySlug: unknown): string => + typeof categorySlug === 'string' ? categorySlug.trim().toLowerCase() : ''; + +/** The template for a category slug; unknown slugs keep the full phone template. */ +export function getCategorySpecTemplate( + categorySlug: string | null | undefined +): CategorySpecTemplate { + return SPEC_TEMPLATES[normalizeSlug(categorySlug)] ?? PHONE_TEMPLATE; +} + +const hiddenIn = ( + template: CategorySpecTemplate, + group: SpecGroupKey, + forStorage: boolean +): readonly string[] => [ + ...(template.hiddenFields[group] ?? []), + ...(forStorage ? [] : (template.displayHiddenFields[group] ?? [])), +]; + +/** Whether the page shows a field of a group for the category. */ +export function specFieldApplies( + template: CategorySpecTemplate, + group: SpecGroupKey, + field: string +): boolean { + return ( + template.groups.includes(group) && + !hiddenIn(template, group, false).includes(field) + ); +} + +export interface CategoryScopedSpecs { + specifications?: unknown; + key_specs?: unknown; + ram_gb?: number | null; + storage_gb?: number | null; + screen_size_inch?: number | null; + has_5g?: boolean | null; +} + +const isRecord = (v: unknown): v is Record<string, unknown> => + Boolean(v) && typeof v === 'object' && !Array.isArray(v); + +/** + * A copy of a product with the spec groups, fields and flat columns its + * category does not have removed. Fields outside the ten groups (such as + * `category_slug`) are kept. Values the category has are never changed, so + * applying it twice gives the same result. It only narrows: removed flat + * columns become `null` (`has_5g` becomes `false`) and groups lose keys, so + * callers should type those fields as nullable, as the row types do. + * + * `forStorage` (the importer) removes only what the category cannot have and + * keeps what the page merely hides, such as a watch's real 5G support. + */ +export function applyCategorySpecTemplate<T extends CategoryScopedSpecs>( + categorySlug: string | null | undefined, + product: T, + { forStorage = false }: { forStorage?: boolean } = {} +): T { + const template = getCategorySpecTemplate(categorySlug); + if (template === PHONE_TEMPLATE) return product; + const out: T = { ...product }; + + if (isRecord(product.specifications)) { + const specs: Record<string, unknown> = {}; + for (const [group, value] of Object.entries(product.specifications)) { + if (!(ALL_GROUPS as readonly string[]).includes(group)) { + specs[group] = value; + continue; + } + if (!template.groups.includes(group as SpecGroupKey)) continue; + const hidden = hiddenIn(template, group as SpecGroupKey, forStorage); + if (hidden.length > 0 && isRecord(value)) { + specs[group] = Object.fromEntries( + Object.entries(value).filter(([field]) => !hidden.includes(field)) + ); + } else { + specs[group] = value; + } + } + out.specifications = specs; + } + + if (isRecord(product.key_specs)) { + out.key_specs = Object.fromEntries( + Object.entries(product.key_specs).filter(([key]) => + (template.keySpecs as readonly string[]).includes(key) + ) + ); + } + + if (!template.ram && 'ram_gb' in product) out.ram_gb = null; + if (!template.storage && 'storage_gb' in product) out.storage_gb = null; + if (!template.screen && 'screen_size_inch' in product) + out.screen_size_inch = null; + const shows5g = forStorage + ? template.groups.includes('cellular') + : template.has5g; + if (!shows5g && 'has_5g' in product) out.has_5g = false; + return out; +} diff --git a/packages/shared-data/src/schemas/index.ts b/packages/shared-data/src/schemas/index.ts index 37b17a032..db4fdd3b3 100644 --- a/packages/shared-data/src/schemas/index.ts +++ b/packages/shared-data/src/schemas/index.ts @@ -8,3 +8,4 @@ export * from './listing.schema.js'; export * from './review.schema.js'; export * from './common.schema.js'; export * from './specs.schema.js'; +export * from './item.schema.js'; diff --git a/packages/shared-data/src/schemas/item.schema.ts b/packages/shared-data/src/schemas/item.schema.ts new file mode 100644 index 000000000..38b7cb23e --- /dev/null +++ b/packages/shared-data/src/schemas/item.schema.ts @@ -0,0 +1,121 @@ +import { z } from 'zod'; +import { MAX_VALID_VND_PRICE } from '../rules/price-limits.js'; +import { + ITEM_KEY_PATTERN, + MAX_MODEL_KEY_LENGTH, + MODEL_KEY_PATTERN, + VARIANT_KEY_PATTERN, + splitModelKey, +} from '../rules/model-key.js'; +import { HttpUrlSchema, StatusSchema } from './common.schema.js'; +import { + KeySpecsSchema, + MasterProductSchema, + ProductSpecificationsSchema, +} from './specs.schema.js'; + +/** + * Item / variant / listing model (DOS/Rate#1149; design doc + * `Docs/Proposals/2026-10-27-item-variant-listing-model.md`, section 4). + * One Item per product (`rate.items`); each storage size is a variant + * (`rate.master_products`); listings (`rate.products`) point at the Item and, + * when the size is known, at the variant (`cluster_id`). + */ + +/** Strict key shapes, for keys a writer builds. See `rules/model-key.ts`. */ +export const ItemKeySchema = z + .string() + .max(MAX_MODEL_KEY_LENGTH) + .regex(ITEM_KEY_PATTERN, { message: 'Invalid item_key' }); +export const VariantKeySchema = z + .string() + .max(MAX_MODEL_KEY_LENGTH) + .regex(VARIANT_KEY_PATTERN, { message: 'Invalid variant_key' }); +export const ModelKeySchema = z + .string() + .max(MAX_MODEL_KEY_LENGTH) + .regex(MODEL_KEY_PATTERN, { message: 'Invalid model_key' }); + +/** How a listing's variant was assigned (`products.variant_source`). */ +export const VariantSourceSchema = z.enum([ + 'title', + 'sku', + 'single_size', + 'single_variant', + 'manual', + 'legacy', +]); + +/** + * A `rate.items` row. `item_key` has no shape check here, as in the table: + * legacy keys still get an Item (design doc 4.3). + */ +export const ItemSchema = z.object({ + id: z.string().uuid(), + item_key: z.string().min(1).max(MAX_MODEL_KEY_LENGTH), + slug: z.string().min(1).max(200), + name: z.string().min(1).max(300), + brand: z.string().min(1).max(120), + model: z.string().min(1).max(200), + category_id: z.string().uuid().nullable().optional(), + status: StatusSchema.default('draft'), + entity_type: z.enum(['product', 'service']).default('product'), + description: z.string().max(50_000).nullable().optional(), + specifications: ProductSpecificationsSchema.default({}), + key_specs: KeySpecsSchema.default({}), + spec_provenance: z.record(z.string(), z.unknown()).default({}), + images: z.array(HttpUrlSchema).max(100).default([]), + thumbnail: HttpUrlSchema.nullable().optional(), + default_variant_id: z.string().uuid().nullable().optional(), + min_price: z.number().nonnegative().max(MAX_VALID_VND_PRICE).default(0), + max_price: z.number().nonnegative().max(MAX_VALID_VND_PRICE).default(0), + price_variant_count: z.number().int().nonnegative().default(0), + merchant_count: z.number().int().nonnegative().default(0), + listing_count: z.number().int().nonnegative().default(0), + unknown_variant_listing_count: z.number().int().nonnegative().default(0), + total_reviews: z.number().int().nonnegative().default(0), + average_rating: z.number().min(0).max(5).default(0), + user_rating_weighted: z.number().min(0).max(5).nullable().optional(), + authentic_review_count: z.number().int().nonnegative().default(0), + expert_rating: z.number().min(0).max(5).nullable().optional(), + expert_review_summary: z.string().max(10_000).nullable().optional(), + expert_criteria_scores: z.record(z.string(), z.unknown()).default({}), + metadata: z.record(z.string(), z.unknown()).default({}), + created_at: z.string().max(64).optional(), + updated_at: z.string().max(64).optional(), +}); + +/** + * A variant: a `rate.master_products` row with its Item link. `item_id` and + * `variant_key` stay nullable until the backfill and the M4 constraints. + * When the `model_key` is well formed, `variant_key` must be its third + * segment, so `model_key = item_key:variant_key` holds. A legacy key outside + * that shape is not checked; the DB derives its `variant_key` with + * `split_part(model_key, ':', 3)` (design doc 4.3, 7.1). + */ +export const VariantSchema = MasterProductSchema.extend({ + item_id: z.string().uuid().nullable().optional(), + variant_key: z + .string() + .min(1) + .max(MAX_MODEL_KEY_LENGTH) + .nullable() + .optional(), + sort_order: z.number().int().min(-32768).max(32767).nullable().optional(), +}).superRefine((variant, ctx) => { + if (variant.variant_key == null) return; + const parts = splitModelKey(variant.model_key); + if (parts && parts.variantKey !== variant.variant_key) { + ctx.addIssue({ + code: 'custom', + path: ['variant_key'], + message: `variant_key must be the model_key's third segment (${parts.variantKey})`, + }); + } +}); + +export type ItemKey = z.infer<typeof ItemKeySchema>; +export type VariantKey = z.infer<typeof VariantKeySchema>; +export type VariantSource = z.infer<typeof VariantSourceSchema>; +export type Item = z.infer<typeof ItemSchema>; +export type Variant = z.infer<typeof VariantSchema>; diff --git a/scripts/e2e/run.mjs b/scripts/e2e/run.mjs index 7ecc95869..eb34c890f 100644 --- a/scripts/e2e/run.mjs +++ b/scripts/e2e/run.mjs @@ -329,7 +329,8 @@ try { name: dbName, runArgs: ['--network', network, '--network-alias', 'db'], }); - db.sql(`CREATE ROLE authenticator LOGIN NOINHERIT PASSWORD '${AUTHENTICATOR_PASSWORD}'; + // startRateDb created authenticator (NOLOGIN); PostgREST logs in as it. + db.sql(`ALTER ROLE authenticator LOGIN PASSWORD '${AUTHENTICATOR_PASSWORD}'; GRANT anon, authenticated, service_role TO authenticator;`); db.sql( readFileSync(path.join(repoRoot, 'apps', 'web', 'e2e', 'seed.sql'), 'utf8') diff --git a/scripts/lib/rate-db.mjs b/scripts/lib/rate-db.mjs index f2e2febef..01fd11c7f 100644 --- a/scripts/lib/rate-db.mjs +++ b/scripts/lib/rate-db.mjs @@ -106,12 +106,16 @@ export function startRateDb({ sleep(250); } - // Supabase-shaped fixture: API roles, auth.uid() from request.jwt.claims, and - // the prod extension placement (uuid-ossp in `extensions`, pg_trgm in `sign`, - // neither on the default search_path and neither usable by anon). + // Supabase-shaped fixture: API roles (authenticator is PostgREST's login + // role, the only member of rate_worker), auth.uid() from request.jwt.claims, + // and the prod extension placement (uuid-ossp in `extensions`, pg_trgm in + // `sign`, neither on the default search_path and neither usable by anon). sql(`CREATE ROLE anon; CREATE ROLE authenticated; CREATE ROLE service_role; + CREATE ROLE authenticator NOINHERIT; CREATE SCHEMA auth; - CREATE TABLE auth.users (id UUID PRIMARY KEY, email TEXT, email_confirmed_at TIMESTAMPTZ); + CREATE TABLE auth.users (id UUID PRIMARY KEY, email TEXT, email_confirmed_at TIMESTAMPTZ, + phone_confirmed_at TIMESTAMPTZ, is_anonymous BOOLEAN NOT NULL DEFAULT false, + created_at TIMESTAMPTZ DEFAULT now()); CREATE FUNCTION auth.uid() RETURNS UUID LANGUAGE sql STABLE AS $$ SELECT NULLIF(NULLIF(current_setting('request.jwt.claims', true), '')::jsonb ->> 'sub', '')::uuid $$; diff --git a/scripts/ops/incident-report.mjs b/scripts/ops/incident-report.mjs new file mode 100644 index 000000000..06520d32d --- /dev/null +++ b/scripts/ops/incident-report.mjs @@ -0,0 +1,290 @@ +#!/usr/bin/env node +// Opens, updates or closes the CI incident issue for one completed push run +// of the Verify workflow (DOS/Rate#1057, #1134). Run by +// .github/workflows/incident-report.yml with the run in environment variables: +// +// REPO WORKFLOW BRANCH CONCLUSION RUN_ID RUN_ATTEMPT RUN_URL SHA COMMIT_TITLE +// +// The incident issue keeps a hidden marker with the jobs still waiting to +// recover. A job leaves it only when it concludes `success` in a later run: +// a job skipped on a push (path filter, `if:`) proved nothing. The issue +// closes when the marker is empty. An incident without a readable marker +// (opened before #1134, or with no failing job names) closes only on a run +// in which every job succeeded. + +import { + DEFAULT_REPO, + encodeRef, + fail, + gh, + ghJson, + isMain, +} from './lib.mjs'; + +/** Job conclusions that open or extend an incident. */ +const FAILED = new Set(['failure', 'timed_out', 'cancelled']); +const MARKER = /<!-- incident-pending: (.*?) -->/g; + +/** + * The job names still waiting to recover, or null when unknown: then every + * job must succeed before the incident closes. + */ +export function readPending(body) { + // The script appends its marker last: an earlier one may be quoted from a + // commit title. + const match = [...(body ?? '').matchAll(MARKER)].at(-1); + if (!match) return null; + try { + const names = JSON.parse(match[1]); + if (!Array.isArray(names)) return null; + if (!names.every((n) => typeof n === 'string')) return null; + return names; + } catch { + return null; + } +} + +/** The body with its marker set to `pending` (null keeps it unknown). */ +export function writePending(body, pending) { + const rest = (body ?? '').replaceAll(MARKER, '').trimEnd(); + if (pending === null) return rest; + // `--` cannot appear inside an HTML comment: - keeps the JSON equal. + const json = JSON.stringify(pending).replaceAll('-', '\\u002d'); + return `${rest}\n\n<!-- incident-pending: ${json} -->`; +} + +export const failingJobs = (jobs) => + jobs.filter((j) => FAILED.has(j.conclusion)).map((j) => j.name); + +/** Events can arrive out of order: only the branch head's run counts. */ +export const isCurrentHead = (sha, headSha) => sha === headSha; + +const describe = (jobs, names) => + names + .map((name) => { + const job = jobs.find((j) => j.name === name); + return `${name} (${job?.conclusion || 'not run'})`; + }) + .join(', '); + +/** + * What to do with one run. `open` is the open incident `{ number, body }` or + * null, `jobs` the run's `{ name, conclusion }` list. Returns `{ action }`, + * one of none, create, update (a failure on an open incident), close or + * unverified (passed, but a pending job did not succeed), plus `pending` (the + * new marker), `failing` and `unverified` job names. + */ +export function decide({ conclusion, jobs, open }) { + // Matrix jobs can share a name: it succeeded only if each of them did. + const succeeded = new Set( + jobs + .filter((j) => j.conclusion === 'success') + .map((j) => j.name) + .filter((name) => + jobs.every((j) => j.name !== name || j.conclusion === 'success') + ) + ); + const before = open ? readPending(open.body) : []; + + if (conclusion === 'success') { + if (!open) return { action: 'none' }; + // A pending job the run does not list at all was renamed or removed from + // the workflow: it can never pass again. A skipped job is still listed. + const listed = new Set(jobs.map((j) => j.name)); + const unverified = + before === null + ? jobs.filter((j) => j.conclusion !== 'success').map((j) => j.name) + : before.filter( + (name) => + !succeeded.has(name) && (jobs.length === 0 || listed.has(name)) + ); + // Unknown pending jobs and a run that lists none: nothing proves recovery. + if (before === null && jobs.length === 0) { + return { action: 'unverified', pending: null, unverified: [] }; + } + if (unverified.length === 0) return { action: 'close' }; + return { + action: 'unverified', + pending: before === null ? null : unverified, + unverified, + }; + } + + const failing = failingJobs(jobs); + // A new incident with no failing job names stays unknown; an open one + // keeps the jobs it already waits for. + let pending = open ? before : failing.length > 0 ? [] : null; + if (pending !== null) { + const still = pending.filter((name) => !succeeded.has(name)); + pending = [...new Set([...still, ...failing])]; + } + return { action: open ? 'update' : 'create', pending, failing }; +} + +/** Untrusted text as inline code, so its @mentions and #refs do nothing. */ +export const codeSpan = (text) => + `\`${String(text ?? '') + .split('\n')[0] + .slice(0, 120) + .replaceAll('`', '')}\``; + +export function failureBody(run, failing) { + return [ + `**${run.workflow}** ${run.conclusion} on \`${run.branch}\`.`, + '', + `- Run: ${run.url}`, + `- Commit: \`${run.sha.slice(0, 8)}\` ${codeSpan(run.commitTitle)}`, + `- Failing jobs: ${failing.join(', ') || 'unknown'}`, + '', + `Agents: this is the first priority (AGENTS.md). Claim it, find the breaking change, fix forward. The issue closes itself when every failing job passes on \`${run.branch}\` again.`, + ].join('\n'); +} + +export function recoveredBody(run) { + return `Recovered: ${run.url} passed on \`${run.sha.slice(0, 8)}\` (${codeSpan(run.commitTitle)}).`; +} + +export function unverifiedBody(run, jobs, decision) { + const head = `${run.url} passed on \`${run.sha.slice(0, 8)}\` (${codeSpan(run.commitTitle)}), but it does not prove recovery`; + if (decision.pending === null) { + const which = decision.unverified.length + ? `: ${describe(jobs, decision.unverified)} did not succeed` + : ': the run listed no jobs'; + return `${head}${which}. This incident does not record which jobs failed, so it stays open until a run in which every job succeeds.`; + } + return `${head}: ${describe(jobs, decision.unverified)} did not succeed in it. The incident stays open until each of them concludes \`success\`.`; +} + +const incidentTitle = (run) => + `incident: ${run.workflow} failing on ${run.branch}`; + +function main() { + const env = process.env; + const repo = env.REPO || DEFAULT_REPO; + const run = { + workflow: env.WORKFLOW, + branch: env.BRANCH, + conclusion: env.CONCLUSION, + id: env.RUN_ID, + attempt: env.RUN_ATTEMPT, + url: env.RUN_URL, + sha: env.SHA, + commitTitle: env.COMMIT_TITLE, + }; + for (const key of ['workflow', 'branch', 'conclusion', 'id', 'sha']) { + if (!run[key]) fail(`incident-report: ${key} is missing`); + } + + const head = ghJson([ + 'api', + `repos/${repo}/commits/${encodeRef(run.branch)}`, + ]).sha; + if (!isCurrentHead(run.sha, head)) { + console.log( + `Run is for ${run.sha.slice(0, 8)}, ${run.branch} is at ${head.slice(0, 8)} now: skipped.` + ); + return; + } + + gh( + [ + 'label', + 'create', + 'incident', + '-R', + repo, + '--color', + 'B60205', + '--description', + 'Production or CI breakage: first priority for any free agent', + ], + { allowFailure: true } + ); + // An API error throws here instead of creating a duplicate incident. + const issues = ghJson([ + 'issue', + 'list', + '-R', + repo, + '--state', + 'open', + '--label', + 'incident', + '--limit', + '50', + '--json', + 'number,title,body', + ]); + const open = + issues.find((issue) => issue.title === incidentTitle(run)) ?? null; + // The jobs of the attempt this event reports, not of a later re-run. + const attempt = run.attempt ? ['--attempt', run.attempt] : []; + const jobs = ghJson([ + 'run', + 'view', + run.id, + '-R', + repo, + ...attempt, + '--json', + 'jobs', + ]).jobs.map((j) => ({ name: j.name, conclusion: j.conclusion })); + + const decision = decide({ conclusion: run.conclusion, jobs, open }); + const comment = (body) => + gh(['issue', 'comment', String(open.number), '-R', repo, '--body', body]); + const setPending = (pending) => { + const body = writePending(open.body, pending); + if (body === (open.body ?? '').trimEnd()) return; + gh([ + 'api', + '-X', + 'PATCH', + `repos/${repo}/issues/${open.number}`, + '-f', + `body=${body}`, + ]); + }; + + switch (decision.action) { + case 'none': + break; + case 'close': + comment(recoveredBody(run)); + gh([ + 'issue', + 'close', + String(open.number), + '-R', + repo, + '--reason', + 'completed', + ]); + break; + case 'unverified': + setPending(decision.pending); + comment(unverifiedBody(run, jobs, decision)); + break; + case 'update': + setPending(decision.pending); + comment(failureBody(run, decision.failing)); + break; + case 'create': + gh([ + 'issue', + 'create', + '-R', + repo, + '--title', + incidentTitle(run), + '--label', + 'incident', + '--body', + writePending(failureBody(run, decision.failing), decision.pending), + ]); + break; + } + console.log(`incident-report: ${decision.action}`); +} + +if (isMain(import.meta.url)) main(); diff --git a/scripts/ops/incident-report.test.mjs b/scripts/ops/incident-report.test.mjs new file mode 100644 index 000000000..850a61d50 --- /dev/null +++ b/scripts/ops/incident-report.test.mjs @@ -0,0 +1,239 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { + codeSpan, + decide, + failureBody, + isCurrentHead, + readPending, + unverifiedBody, + writePending, +} from './incident-report.mjs'; + +const BUILD = 'Typecheck & Build Monorepo'; +const E2E = 'E2E (Playwright)'; +const job = (name, conclusion) => ({ name, conclusion }); +const incident = (pending, number = 7) => ({ + number, + body: writePending('**Verify** failure on `dev`.', pending), +}); +const run = { + workflow: 'Verify', + branch: 'dev', + conclusion: 'success', + url: 'https://github.com/DOS/Rate/actions/runs/1', + sha: 'abcdef1234567890', + commitTitle: 'fix: thing @someone #12', +}; + +test('the marker round-trips through the issue body', () => { + const body = writePending('Body text.', [BUILD, E2E]); + assert.deepEqual(readPending(body), [BUILD, E2E]); + assert.ok(body.startsWith('Body text.')); + const again = writePending(body, [E2E]); + assert.deepEqual(readPending(again), [E2E]); + assert.equal(again.match(/incident-pending/g).length, 1); +}); + +test('a job name with dashes cannot end the HTML comment early', () => { + const body = writePending('', ['a --> b']); + assert.ok(!body.slice(0, -4).includes('-->')); + assert.deepEqual(readPending(body), ['a --> b']); +}); + +test('a marker quoted from a commit title cannot replace the real one', () => { + const fake = '<!-- incident-pending: [] -->'; + const body = writePending(`Commit ${fake} ${fake}`, [E2E]); + assert.deepEqual(readPending(body), [E2E]); + assert.equal(body.match(/incident-pending/g).length, 1); + assert.equal( + decide({ + conclusion: 'success', + jobs: [job(BUILD, 'success'), job(E2E, 'skipped')], + open: { number: 1, body: `Commit ${fake} + +${writePending('', [E2E])}` }, + }).action, + 'unverified' + ); +}); + +test('a matrix name recovers only when every job of that name succeeded', () => { + assert.equal( + decide({ + conclusion: 'success', + jobs: [job('Test', 'success'), job('Test', 'skipped')], + open: incident(['Test']), + }).action, + 'unverified' + ); +}); + +test('a missing or broken marker reads as unknown', () => { + assert.equal(readPending('An incident opened before #1134.'), null); + assert.equal(readPending('<!-- incident-pending: [oops -->'), null); + assert.equal(readPending('<!-- incident-pending: {"a":1} -->'), null); + assert.equal(readPending(undefined), null); + assert.equal(writePending('x\n\n<!-- incident-pending: [] -->', null), 'x'); +}); + +test('a failure with no open incident creates one pending its failing jobs', () => { + assert.deepEqual( + decide({ + conclusion: 'failure', + jobs: [job(BUILD, 'success'), job(E2E, 'failure')], + open: null, + }), + { action: 'create', pending: [E2E], failing: [E2E] } + ); +}); + +test('timed out and cancelled jobs count as failing', () => { + const d = decide({ + conclusion: 'cancelled', + jobs: [job(BUILD, 'timed_out'), job(E2E, 'cancelled')], + open: null, + }); + assert.deepEqual(d.pending, [BUILD, E2E]); +}); + +test('closes when every pending job concluded success', () => { + assert.deepEqual( + decide({ + conclusion: 'success', + jobs: [job(BUILD, 'success'), job(E2E, 'success')], + open: incident([E2E]), + }), + { action: 'close' } + ); +}); + +test('a pending job skipped on a passing push keeps the incident open', () => { + const jobs = [job(BUILD, 'success'), job(E2E, 'skipped')]; + const d = decide({ conclusion: 'success', jobs, open: incident([E2E]) }); + assert.deepEqual(d, { + action: 'unverified', + pending: [E2E], + unverified: [E2E], + }); + assert.match(unverifiedBody(run, jobs, d), /E2E \(Playwright\) \(skipped\)/); +}); + +test('a skipped job that never failed does not block closing', () => { + assert.equal( + decide({ + conclusion: 'success', + jobs: [job(BUILD, 'success'), job(E2E, 'skipped')], + open: incident([BUILD]), + }).action, + 'close' + ); +}); + +test('a pending job no longer in the workflow does not block closing', () => { + const jobs = [job(BUILD, 'success'), job('E2E (Chromium)', 'success')]; + assert.equal( + decide({ conclusion: 'success', jobs, open: incident([E2E]) }).action, + 'close' + ); + assert.equal( + decide({ conclusion: 'success', jobs: [], open: incident([E2E]) }).action, + 'unverified' + ); +}); + +test('recovery is tracked across runs: each job may pass in a different one', () => { + const first = decide({ + conclusion: 'success', + jobs: [job(BUILD, 'success'), job(E2E, 'skipped')], + open: incident([BUILD, E2E]), + }); + assert.deepEqual(first.pending, [E2E]); + const second = decide({ + conclusion: 'success', + jobs: [job(BUILD, 'skipped'), job(E2E, 'success')], + open: incident(first.pending), + }); + assert.equal(second.action, 'close'); +}); + +test('a new failure adds to the pending jobs and drops the recovered ones', () => { + assert.deepEqual( + decide({ + conclusion: 'failure', + jobs: [job(BUILD, 'failure'), job(E2E, 'success')], + open: incident([E2E]), + }), + { action: 'update', pending: [BUILD], failing: [BUILD] } + ); + assert.deepEqual( + decide({ + conclusion: 'failure', + jobs: [job(BUILD, 'failure'), job(E2E, 'skipped')], + open: incident([E2E]), + }).pending, + [E2E, BUILD] + ); +}); + +test('an incident without a marker closes only when every job succeeded', () => { + const legacy = { number: 3, body: 'Opened before #1134.' }; + assert.equal( + decide({ + conclusion: 'success', + jobs: [job(BUILD, 'success'), job(E2E, 'success')], + open: legacy, + }).action, + 'close' + ); + const jobs = [job(BUILD, 'success'), job(E2E, 'skipped')]; + const d = decide({ conclusion: 'success', jobs, open: legacy }); + assert.deepEqual(d, { + action: 'unverified', + pending: null, + unverified: [E2E], + }); + assert.match(unverifiedBody(run, jobs, d), /every job succeeds/); + assert.equal( + decide({ conclusion: 'success', jobs: [], open: legacy }).action, + 'unverified' + ); +}); + +test('a failure without failing job names keeps the jobs already pending', () => { + assert.deepEqual( + decide({ conclusion: 'cancelled', jobs: [], open: incident([E2E]) }), + { action: 'update', pending: [E2E], failing: [] } + ); + assert.equal( + decide({ conclusion: 'failure', jobs: [], open: null }).pending, + null + ); +}); + +test('a passing run with no open incident does nothing', () => { + assert.deepEqual( + decide({ conclusion: 'success', jobs: [job(BUILD, 'success')], open: null }), + { action: 'none' } + ); +}); + +test('a superseded run is not the branch head', () => { + assert.equal(isCurrentHead('aaa', 'bbb'), false); + assert.equal(isCurrentHead('aaa', 'aaa'), true); +}); + +test('the commit title is shown as one inert line of inline code', () => { + assert.equal(codeSpan('fix: `x` @a #1\nsecond line'), '`fix: x @a #1`'); + assert.equal(codeSpan('a'.repeat(200)).length, 122); + assert.equal(codeSpan(undefined), '``'); +}); + +test('the failure body keeps its run, commit and failing jobs', () => { + const body = failureBody({ ...run, conclusion: 'failure' }, [E2E]); + assert.match(body, /\*\*Verify\*\* failure on `dev`\./); + assert.match(body, /- Commit: `abcdef12` `fix: thing @someone #12`/); + assert.match(body, /- Failing jobs: E2E \(Playwright\)/); + assert.match(failureBody(run, []), /- Failing jobs: unknown/); +}); diff --git a/scripts/ops/lib.mjs b/scripts/ops/lib.mjs index 297cd4b39..e7c6f4ff1 100644 --- a/scripts/ops/lib.mjs +++ b/scripts/ops/lib.mjs @@ -7,9 +7,10 @@ import { pathToFileURL } from 'node:url'; export const DEFAULT_REPO = 'DOS/Rate'; -export function gh(args, { allowFailure = false } = {}) { +export function gh(args, { allowFailure = false, input } = {}) { const r = spawnSync('gh', args, { encoding: 'utf8', + input, maxBuffer: 64 * 1024 * 1024, }); if (r.error) throw r.error; diff --git a/scripts/ops/merge-when-green.mjs b/scripts/ops/merge-when-green.mjs index e1f258229..93426e0b9 100644 --- a/scripts/ops/merge-when-green.mjs +++ b/scripts/ops/merge-when-green.mjs @@ -131,6 +131,110 @@ export function shouldDeleteBranch({ method, headRef, headRepo, baseRepo }) { ); } +/** + * commitlint's limit (commitlint.config.ts) for the header and for every body + * and footer line. The commitlint check of a dev to main promotion lints the + * head of dev, which is usually a squash commit made here (DOS/Rate#1130). + */ +export const MAX_LINE = 100; +// commitlint exempts any line holding a URL from the line limit. +const URL_LINE = /\bhttps?:\/\/\S+/; +const TRAILER = /^(?:co-authored-by|signed-off-by):\s*\S/i; +const LIST_MARKER = /^\s*(?:[-*+]\s+|\d+[.)]\s+)?/; + +/** + * One line as lines of at most `width` characters: words wrap under the text + * of a list item, and a word longer than a whole line is split. + */ +export function wrapLine(line, width = MAX_LINE) { + if (!(width >= 2)) throw new RangeError(`cannot wrap to width ${width}`); + if (line.length <= width || URL_LINE.test(line)) return [line]; + const marker = line.match(LIST_MARKER)[0]; + const first = marker.length <= width / 2 ? marker : ''; + const hang = ' '.repeat(first.length); + const lines = []; + let current = first; + let empty = true; + for (let word of line.slice(first.length).split(/\s+/).filter(Boolean)) { + for (;;) { + const sep = empty ? '' : ' '; + if (current.length + sep.length + word.length <= width) { + current += sep + word; + empty = false; + break; + } + if (empty) { + // Not even an empty line holds the word: split it. + // Cut between characters, never inside a surrogate pair. + const room = width - current.length; + let piece = ''; + for (const char of word) { + if (piece.length + char.length > room) break; + piece += char; + } + if (piece === '') piece = word.slice(0, Math.max(room, 1)); + lines.push(current + piece); + word = word.slice(piece.length); + current = hang; + continue; + } + lines.push(current); + current = hang; + empty = true; + } + } + if (!empty) lines.push(current); + return lines; +} + +// `type(scope)!: subject`, the header shape commitlint's parser reads. +const CONVENTIONAL_HEADER = /^[a-z]+(?:\([^()\r\n]*\))?!?: \S/; + +/** + * The squash commit's header: the PR title with its number, which promote.mjs + * reads back. Null when the title is not a conventional header or the header + * is over the limit: cutting it could leave an empty or start-case subject. + */ +export function squashTitle(title, number) { + const text = title.trim().replace(/\s+/g, ' '); + const header = `${text} (#${number})`; + return CONVENTIONAL_HEADER.test(text) && header.length <= MAX_LINE + ? header + : null; +} + +/** + * The squash commit's body, like GitHub's default (each commit as a list + * item), with every line wrapped to the limit and the commits' co-author and + * sign-off trailers gathered once each into a footer after a blank line. + */ +export function squashBody(messages) { + const trailers = new Map(); + const items = []; + for (const message of messages) { + const lines = []; + for (const line of message.replace(/\r\n?/g, '\n').split('\n')) { + if (TRAILER.test(line.trim())) { + const trailer = line.trim().replace(/:\s*/, ': '); + const key = trailer.toLowerCase(); + if (!trailers.has(key)) trailers.set(key, trailer); + } else { + lines.push(line.trimEnd()); + } + } + while (lines.length > 0 && lines.at(-1) === '') lines.pop(); + while (lines.length > 0 && lines[0] === '') lines.shift(); + if (lines.length === 0) continue; + lines[0] = `* ${lines[0]}`; + items.push(lines.flatMap((line) => wrapLine(line)).join('\n')); + } + // A wrapped trailer no longer credits anyone: leave out one too long. + const footer = [...trailers.values()].filter( + (line) => line.length <= MAX_LINE + ); + return [items.join('\n\n'), footer.join('\n')].filter(Boolean).join('\n\n'); +} + /** * What to do with the PR now: 'merge', 'wait', or a final 'done' / 'fail' * with a one-line message. @@ -210,6 +314,7 @@ export async function mergeWhenGreen({ let blockedPolls = 0; let apiErrors = 0; let mergeRefusals = 0; + let commitListErrors = 0; let branchUpdates = 0; let lastUpdatedSha = null; for (;;) { @@ -249,6 +354,12 @@ export async function mergeWhenGreen({ message: `PR ${number}: cannot --ignore required check(s) ${conflict.join(', ')}`, }; } + if (method === 'squash' && !squashTitle(pr.title, number)) { + return { + ok: false, + message: `PR ${number}: its title must be a conventional header that fits "<title> (#${number})" in ${MAX_LINE} characters, so the squash commit passes commitlint`, + }; + } } const checks = evaluateChecks(head.runs, head.statuses, { ignore: [...IGNORED_CHECKS, ...ignore], @@ -293,20 +404,58 @@ export async function mergeWhenGreen({ continue; } if (next.action === 'merge') { + // GitHub's default squash message keeps the commits' long lines and + // mixes their trailers into the body: send one commitlint accepts. + const squash = {}; + if (method === 'squash') { + let messages; + try { + messages = ghPaginated( + `repos/${repo}/pulls/${number}/commits?per_page=100`, + '.[] | {message: .commit.message}' + ).map((c) => c.message); + commitListErrors = 0; + } catch (err) { + if (++commitListErrors >= MAX_API_ERRORS) { + return { + ok: false, + message: `PR ${number}: cannot list its commits: ${err.message}`, + }; + } + await sleep(intervalMs); + continue; + } + // The commits API stops at 250: never write a body that drops some. + if (messages.length < pr.commits) { + return { + ok: false, + message: `PR ${number}: only ${messages.length} of its ${pr.commits} commits could be listed for the squash message`, + }; + } + squash.commit_title = squashTitle(pr.title, number); + squash.commit_message = squashBody(messages); + } + // The fields go as JSON on stdin: a long squash message is over the + // Windows command line limit. const merged = gh( [ 'api', '-X', 'PUT', `repos/${repo}/pulls/${number}/merge`, - '-f', - `merge_method=${method}`, - '-f', - `sha=${pr.head.sha}`, + '--input', + '-', '--jq', '.sha', ], - { allowFailure: true } + { + allowFailure: true, + input: JSON.stringify({ + merge_method: method, + sha: pr.head.sha, + ...squash, + }), + } ); if (merged.status !== 0) { // A check re-queued after the last read (for example by a PR edit) diff --git a/scripts/ops/merge-when-green.test.mjs b/scripts/ops/merge-when-green.test.mjs index 3329c13a9..2000e1f1c 100644 --- a/scripts/ops/merge-when-green.test.mjs +++ b/scripts/ops/merge-when-green.test.mjs @@ -1,4 +1,6 @@ import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; import test from 'node:test'; import { @@ -7,9 +9,13 @@ import { evaluateChecks, ignoredRequired, latestRuns, + MAX_LINE, mergeMethodFor, requiredChecksFromRules, shouldDeleteBranch, + squashBody, + squashTitle, + wrapLine, } from './merge-when-green.mjs'; import { encodeRef } from './lib.mjs'; @@ -291,3 +297,104 @@ test('gives a green but blocked PR a few polls before reporting it', () => { assert.equal(last.action, 'fail'); assert.match(last.message, /required review or check/); }); + +// The commits of DOS/Rate#1127, whose default squash failed commitlint on the +// promotion PR DOS/Rate#1128 (DOS/Rate#1130). +const CO_AUTHOR = 'Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>'; +const LONG_COMMITS = [ + `fix(shared-data): earbuds regex reads glued brands and drops over-ear titles that say tai nghe (#1125) + +Any brand glued to Buds, FreeBuds, AirPods or WF1000XM now reads as earbuds +(PixelBudsPro2, HuaweiFreeBudsPro4, SonyWF1000XM5), as do AirPodsPro2 and +two glued tiers (Buds3ProFE). + +${CO_AUTHOR}`, + `fix(shared-data): over-ear WH with a space or underscore, glued Redmi/Samsung/Enco/Tune Buds (#1125) + +Review follow-up on PR #1127, with a reference line that is far longer than one hundred characters in total. +See https://github.com/DOS/Rate/pull/1127#issuecomment-1234567890 for the review thread and the whole discussion. +A path with no spaces: ${'packages/shared-data/src/'.repeat(6)}earbuds.ts + +${CO_AUTHOR}`, +]; +const LONG_TITLE = + 'fix(shared-data): earbuds regex reads glued brands, drops over-ear tai nghe titles (#1125)'; + +const repoRoot = fileURLToPath(new URL('../../', import.meta.url)); +/** Lints a message with the repository's commitlint.config.ts. */ +function commitlint(message) { + const r = spawnSync( + process.execPath, + ['node_modules/@commitlint/cli/cli.js'], + { cwd: repoRoot, input: message, encoding: 'utf8' } + ); + assert.equal(r.error, undefined); + return { ok: r.status === 0, output: r.stdout + r.stderr }; +} + +test('wraps a long line under its list marker and splits a word longer than a line', () => { + const words = Array.from({ length: 40 }, (_, i) => `word${i}`).join(' '); + const wrapped = wrapLine(`* ${words}`); + assert.ok(wrapped.length > 1); + assert.ok(wrapped.every((line) => line.length <= MAX_LINE)); + assert.ok(wrapped.slice(1).every((line) => line.startsWith(' w'))); + assert.equal(wrapped.join(' ').replace(/\s+/g, ' '), `* ${words}`); + const word = 'x'.repeat(250); + assert.equal(wrapLine(word).join(''), word); + assert.ok(wrapLine(word).every((line) => line.length <= MAX_LINE)); + assert.deepEqual(wrapLine('short'), ['short']); + const url = `see https://example.com/${'a'.repeat(120)}`; + assert.deepEqual(wrapLine(url), [url]); +}); + +test('splits a long word between characters, never inside an emoji', () => { + const wrapped = wrapLine(`a${'😀'.repeat(50)}`); + assert.ok(wrapped.every((line) => line.length <= MAX_LINE)); + assert.ok(wrapped.every((line) => !/[\uD800-\uDBFF]$/.test(line))); + assert.equal(wrapped.join(''), `a${'😀'.repeat(50)}`); + assert.throws(() => wrapLine('x y', 0), RangeError); +}); + +test('keeps the PR number in the squash title, or refuses a title that cannot carry it', () => { + assert.equal( + squashTitle(' fix(web): a thing ', 7), + 'fix(web): a thing (#7)' + ); + assert.equal(squashTitle(LONG_TITLE, 1127), `${LONG_TITLE} (#1127)`); + // Cutting these would leave an empty subject or a start-case one. + assert.equal(squashTitle(`${LONG_TITLE} and more`, 1127), null); + assert.equal(squashTitle(`fix: ${'a'.repeat(89)}`, 1133), null); + // Not conventional: commitlint ignores `v1.2.3` alone, not with `(#n)`. + assert.equal(squashTitle('v1.2.3', 1133), null); + assert.equal(squashTitle('Update stuff', 1133), null); + assert.equal( + squashTitle('feat(web)!: drop old route', 3), + 'feat(web)!: drop old route (#3)' + ); +}); + +test('gathers the trailers once each into a footer after the commits', () => { + const body = squashBody(LONG_COMMITS); + assert.ok(body.startsWith('* fix(shared-data): earbuds regex')); + assert.ok(body.endsWith(`\n\n${CO_AUTHOR}`)); + assert.equal(body.split(CO_AUTHOR).length, 2); + assert.ok(body.includes('\n\n* fix(shared-data): over-ear WH')); + assert.equal(squashBody(['fix: a\r\n\r\nbody\r\n']), '* fix: a\n\nbody'); + // A trailer is never wrapped: one too long for commitlint is left out. + const long = `Co-authored-by: ${'a'.repeat(90)} <a@b.com>`; + assert.equal( + squashBody([`fix: a\n\n${long}\n${CO_AUTHOR}`]), + `* fix: a\n\n${CO_AUTHOR}` + ); +}); + +test('the squash message of PR bodies with lines over 100 characters passes commitlint', () => { + const header = squashTitle(LONG_TITLE, 1127); + const message = `${header}\n\n${squashBody(LONG_COMMITS)}\n`; + // GitHub's default squash of the same commits fails, so the test has teeth. + const github = `${header}\n\n${LONG_COMMITS.map((m) => `* ${m}`).join('\n\n')}\n`; + assert.equal(commitlint(github).ok, false); + const result = commitlint(message); + assert.ok(result.ok, result.output); + assert.doesNotMatch(result.output, /warning/); +}); diff --git a/supabase/migrations/20261018153000_rate_row_provenance.sql b/supabase/migrations/20261018153000_rate_row_provenance.sql new file mode 100644 index 000000000..430866151 --- /dev/null +++ b/supabase/migrations/20261018153000_rate_row_provenance.sql @@ -0,0 +1,799 @@ +-- ============================================================================= +-- Migration: 20261018153000_rate_row_provenance.sql +-- Purpose (Rate audit K2): record where every listing, review and price point +-- came from, so real and fabricated data can be told apart in SQL instead of +-- by ID-prefix heuristics and hard-coded seed lists. +-- +-- 1. origin on rate.products, rate.reviews and rate.price_history +-- ('crawl' | 'community' | 'seed' | 'synthetic' | 'fixture' | 'backfill'). +-- The crawler is the only bulk writer, so new rows default to 'crawl'; +-- community reviews are labelled by a trigger. +-- 2. Crawl lineage: products.last_crawled_at / last_crawl_job_id, +-- price_history.crawl_job_id, reviews.first_seen_job_id + observed_at, +-- stamped by rate.stamp_crawl_provenance() from the queue worker. +-- No FK to crawl_jobs: terminal jobs are purged after the retention +-- window (purge_completed_crawl_jobs) and the id must outlive them. +-- 3. master_products.spec_provenance: per-source record of where the +-- specifications came from (filled from key_specs.spec_source for now). +-- 4. Public reads (RLS + the merchant-trust view) and the price / rating +-- aggregates only use crawled listings and crawled or community reviews. +-- 5. Backfill from the existing conventions: syn-* listings, syn-rv-* / +-- bf-rv-* reviews, rate_comm_* community reviews, *_backfill prices, +-- localhost and the scraper suite's fixture URLs. Re-running only +-- touches rows still 'crawl'. Hand-written seed rows need no label: the +-- owner-approved purge (Rate-Services 983d136, 9068be4) deleted every +-- seed row that no finished crawl job also matched, so the seed ids left +-- were verified by a crawl. +-- 6. New rows inherit a non-crawl parent listing's origin, syn-* ids are +-- always labelled synthetic, and community reviews and price dedupe only +-- look at crawled listings / price points. +-- ============================================================================= + +-- Fail fast instead of queueing behind long transactions on the shared database. +SET lock_timeout = '5s'; + +-- 1. Columns --------------------------------------------------------------- + +ALTER TABLE rate.products + ADD COLUMN IF NOT EXISTS origin TEXT NOT NULL DEFAULT 'crawl', + ADD COLUMN IF NOT EXISTS last_crawled_at TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS last_crawl_job_id UUID; + +ALTER TABLE rate.reviews + ADD COLUMN IF NOT EXISTS origin TEXT NOT NULL DEFAULT 'crawl', + -- Existing reviews get -infinity (unknown), so the stamp window below can + -- never claim them; only rows written after this migration get now(). + ADD COLUMN IF NOT EXISTS observed_at TIMESTAMPTZ NOT NULL DEFAULT '-infinity', + ADD COLUMN IF NOT EXISTS first_seen_job_id UUID; + +ALTER TABLE rate.reviews ALTER COLUMN observed_at SET DEFAULT now(); + +ALTER TABLE rate.price_history + ADD COLUMN IF NOT EXISTS origin TEXT NOT NULL DEFAULT 'crawl', + ADD COLUMN IF NOT EXISTS crawl_job_id UUID; + +ALTER TABLE rate.master_products + ADD COLUMN IF NOT EXISTS spec_provenance JSONB NOT NULL DEFAULT '{}'::jsonb; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conrelid = 'rate.products'::regclass AND conname = 'products_origin_check') THEN + ALTER TABLE rate.products ADD CONSTRAINT products_origin_check + CHECK (origin IN ('crawl', 'seed', 'synthetic', 'fixture', 'backfill')); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conrelid = 'rate.reviews'::regclass AND conname = 'reviews_origin_check') THEN + ALTER TABLE rate.reviews ADD CONSTRAINT reviews_origin_check + CHECK (origin IN ('crawl', 'community', 'seed', 'synthetic', 'fixture', 'backfill')); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conrelid = 'rate.price_history'::regclass AND conname = 'price_history_origin_check') THEN + ALTER TABLE rate.price_history ADD CONSTRAINT price_history_origin_check + CHECK (origin IN ('crawl', 'seed', 'synthetic', 'fixture', 'backfill')); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conrelid = 'rate.master_products'::regclass AND conname = 'master_products_spec_provenance_object') THEN + ALTER TABLE rate.master_products ADD CONSTRAINT master_products_spec_provenance_object + CHECK (jsonb_typeof(spec_provenance) = 'object'); + END IF; +END $$; + +COMMENT ON COLUMN rate.products.origin IS 'Where the listing came from. Only crawl rows are public.'; +COMMENT ON COLUMN rate.products.last_crawl_job_id IS 'rate.crawl_jobs id of the last crawl that refreshed this listing. No FK: jobs are purged after retention.'; +COMMENT ON COLUMN rate.reviews.origin IS 'Where the review came from. crawl and community rows are public.'; +COMMENT ON COLUMN rate.reviews.observed_at IS 'When Rate first stored the review; -infinity for reviews stored before 20261018153000 (platform_created_at is the platform''s own date).'; +COMMENT ON COLUMN rate.reviews.first_seen_job_id IS 'rate.crawl_jobs id of the crawl that first stored this review. No FK: jobs are purged after retention.'; +COMMENT ON COLUMN rate.price_history.crawl_job_id IS 'rate.crawl_jobs id of the crawl that observed this price. No FK: jobs are purged after retention.'; +COMMENT ON COLUMN rate.master_products.spec_provenance IS 'Per-source provenance of specifications, e.g. {"source": "verified_benchmark"}.'; + +CREATE INDEX IF NOT EXISTS idx_rate_products_origin_not_crawl + ON rate.products (origin) WHERE origin <> 'crawl'; +CREATE INDEX IF NOT EXISTS idx_rate_reviews_origin_not_crawl + ON rate.reviews (origin) WHERE origin <> 'crawl'; +CREATE INDEX IF NOT EXISTS idx_rate_price_history_crawl_job + ON rate.price_history (crawl_job_id) WHERE crawl_job_id IS NOT NULL; + +-- 2. Community reviews label themselves ------------------------------------ + +CREATE OR REPLACE FUNCTION rate.reviews_set_origin() +RETURNS trigger +LANGUAGE plpgsql +SET search_path = '' +AS $$ +DECLARE + v_parent TEXT; +BEGIN + -- rate.submit_community_review is the only writer of rate_comm_* ids. + IF NEW.platform_review_id LIKE 'rate\_comm\_%' THEN + NEW.origin := 'community'; + ELSIF NEW.origin = 'community' THEN + RAISE EXCEPTION 'origin community is reserved for rate.submit_community_review' + USING ERRCODE = '22023'; + ELSIF NEW.platform_review_id LIKE 'syn-rv-%' OR NEW.platform_review_id LIKE 'bf-rv-%' THEN + -- The synthetic / backfilled review id conventions are never public. + NEW.origin := 'synthetic'; + ELSIF NEW.origin = 'crawl' THEN + -- A review on a fabricated listing is fabricated too. + SELECT p.origin INTO v_parent FROM rate.products p WHERE p.id = NEW.product_id; + IF v_parent IS NOT NULL AND v_parent <> 'crawl' THEN + NEW.origin := v_parent; + END IF; + END IF; + RETURN NEW; +END; +$$; + +DROP TRIGGER IF EXISTS trg_reviews_set_origin ON rate.reviews; +CREATE TRIGGER trg_reviews_set_origin + BEFORE INSERT OR UPDATE OF platform_review_id, origin ON rate.reviews + FOR EACH ROW EXECUTE FUNCTION rate.reviews_set_origin(); + +REVOKE ALL ON FUNCTION rate.reviews_set_origin() FROM PUBLIC, anon, authenticated; + +-- syn-* listing ids are the synthetic-offer convention: never public. +CREATE OR REPLACE FUNCTION rate.products_set_origin() +RETURNS trigger +LANGUAGE plpgsql +SET search_path = '' +AS $$ +BEGIN + IF NEW.platform_item_id LIKE 'syn-%' THEN + NEW.origin := 'synthetic'; + END IF; + RETURN NEW; +END; +$$; + +DROP TRIGGER IF EXISTS trg_products_set_origin ON rate.products; +CREATE TRIGGER trg_products_set_origin + BEFORE INSERT OR UPDATE OF platform_item_id, origin ON rate.products + FOR EACH ROW EXECUTE FUNCTION rate.products_set_origin(); + +REVOKE ALL ON FUNCTION rate.products_set_origin() FROM PUBLIC, anon, authenticated; + +CREATE OR REPLACE FUNCTION rate.price_history_inherit_origin() +RETURNS trigger +LANGUAGE plpgsql +SET search_path = '' +AS $$ +DECLARE + v_parent TEXT; +BEGIN + IF NEW.origin = 'crawl' THEN + SELECT p.origin INTO v_parent FROM rate.products p WHERE p.id = NEW.product_id; + IF v_parent IS NOT NULL AND v_parent <> 'crawl' THEN + NEW.origin := v_parent; + ELSIF NEW.source LIKE '%\_backfill' THEN + -- *_backfill sources are reconstructed, not observed. + NEW.origin := 'backfill'; + END IF; + END IF; + RETURN NEW; +END; +$$; + +DROP TRIGGER IF EXISTS trg_price_history_inherit_origin ON rate.price_history; +CREATE TRIGGER trg_price_history_inherit_origin + BEFORE INSERT ON rate.price_history + FOR EACH ROW EXECUTE FUNCTION rate.price_history_inherit_origin(); + +REVOKE ALL ON FUNCTION rate.price_history_inherit_origin() FROM PUBLIC, anon, authenticated; + +-- 3. Backfill (idempotent: only rows still on the 'crawl' default move) ----- + +UPDATE rate.products +SET origin = 'synthetic' +WHERE origin = 'crawl' AND platform_item_id LIKE 'syn-%'; + +UPDATE rate.products +SET origin = 'fixture' +WHERE origin = 'crawl' + AND ( + url ~* '^https?://(localhost|127\.0\.0\.1|\[::1\])(:[0-9]+)?(/|$)' + -- TEST_FIXTURE_LISTING_URLS in Rate-Services src/scrapers/fabricated-seed-ids.ts + OR url IN ( + 'https://tiki.vn/dien-thoai-samsung-galaxy-s24-ultra-5g-p274819201.html', + 'https://www.lazada.vn/products/samsung-galaxy-s24-ultra-i284910293.html' + ) + ); + +UPDATE rate.reviews +SET origin = 'community' +WHERE origin = 'crawl' AND platform_review_id LIKE 'rate\_comm\_%'; + +UPDATE rate.reviews +SET origin = 'synthetic' +WHERE origin = 'crawl' + AND (platform_review_id LIKE 'syn-rv-%' OR platform_review_id LIKE 'bf-rv-%'); + +UPDATE rate.reviews r +SET origin = p.origin +FROM rate.products p +WHERE r.product_id = p.id + AND r.origin = 'crawl' + AND p.origin IN ('synthetic', 'fixture'); + +UPDATE rate.price_history +SET origin = 'backfill' +WHERE origin = 'crawl' AND source LIKE '%\_backfill'; + +UPDATE rate.price_history h +SET origin = p.origin +FROM rate.products p +WHERE h.product_id = p.id + AND h.origin = 'crawl' + AND p.origin IN ('synthetic', 'fixture'); + +UPDATE rate.master_products +SET spec_provenance = jsonb_build_object('source', key_specs ->> 'spec_source') +WHERE spec_provenance = '{}'::jsonb + AND key_specs IS NOT NULL + AND jsonb_typeof(key_specs) = 'object' + AND COALESCE(key_specs ->> 'spec_source', '') <> ''; + +-- 4. Crawl lineage stamp, called by the queue worker before it acks a job --- + +CREATE OR REPLACE FUNCTION rate.stamp_crawl_provenance(p_job_id UUID, p_product_id UUID) +RETURNS JSONB +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + v_prices INTEGER; + v_reviews INTEGER; +BEGIN + IF p_job_id IS NULL OR p_product_id IS NULL THEN + RAISE EXCEPTION 'job id and product id are required' USING ERRCODE = '22023'; + END IF; + + -- Only a job that is currently being processed may stamp rows. + IF NOT EXISTS ( + SELECT 1 FROM rate.crawl_jobs WHERE id = p_job_id AND status = 'running' + ) THEN + RAISE EXCEPTION 'crawl job % is not running', p_job_id USING ERRCODE = 'P0002'; + END IF; + + -- The job must be for this listing: same platform, and the same item id or + -- URL when the job carries them (URL-only discovery jobs have no item id). + UPDATE rate.products p + SET last_crawled_at = now(), + last_crawl_job_id = p_job_id + FROM rate.crawl_jobs j + WHERE p.id = p_product_id + AND p.origin = 'crawl' + AND j.id = p_job_id + AND j.platform = p.platform + AND ( + j.platform_item_id = p.platform_item_id + OR j.target_url = p.url + OR j.platform_item_id IS NULL + ); + + IF NOT FOUND THEN + RAISE EXCEPTION 'crawled listing % not found for job %', p_product_id, p_job_id USING ERRCODE = 'P0002'; + END IF; + + -- Rows this job just wrote: the dedupe index keeps one live job per + -- listing, and one job runs for minutes, so the 1h window cannot reach a + -- previous crawl's rows (those are already stamped or predate this column). + UPDATE rate.price_history + SET crawl_job_id = p_job_id + WHERE product_id = p_product_id + AND crawl_job_id IS NULL + AND origin = 'crawl' + AND recorded_at > now() - interval '1 hour'; + GET DIAGNOSTICS v_prices = ROW_COUNT; + + UPDATE rate.reviews + SET first_seen_job_id = p_job_id + WHERE product_id = p_product_id + AND first_seen_job_id IS NULL + AND origin = 'crawl' + AND observed_at > now() - interval '1 hour'; + GET DIAGNOSTICS v_reviews = ROW_COUNT; + + RETURN jsonb_build_object('price_points', v_prices, 'reviews', v_reviews); +END; +$$; + +REVOKE ALL ON FUNCTION rate.stamp_crawl_provenance(UUID, UUID) FROM PUBLIC, anon, authenticated; +GRANT EXECUTE ON FUNCTION rate.stamp_crawl_provenance(UUID, UUID) TO service_role; + +-- 5. Public reads only see crawled data ------------------------------------- + +DROP POLICY IF EXISTS "Public can view products" ON rate.products; +CREATE POLICY "Public can view products" ON rate.products + FOR SELECT USING (origin = 'crawl'); + +DROP POLICY IF EXISTS "Public can view price_history" ON rate.price_history; +CREATE POLICY "Public can view price_history" ON rate.price_history + FOR SELECT USING (origin = 'crawl'); + +DROP POLICY IF EXISTS "Public can view reviews" ON rate.reviews; +CREATE POLICY "Public can view reviews" ON rate.reviews + FOR SELECT USING (origin IN ('crawl', 'community')); + +-- The subquery runs under the caller's rate.reviews policy above. +DROP POLICY IF EXISTS "Public can view review_flags" ON rate.review_flags; +CREATE POLICY "Public can view review_flags" ON rate.review_flags + FOR SELECT USING (EXISTS (SELECT 1 FROM rate.reviews r WHERE r.id = review_flags.review_id)); + +-- Owner-rights view (see 20261018120102 T4), so RLS does not apply: filter here. +CREATE OR REPLACE VIEW rate.v_products_with_merchant_trust AS +SELECT + p.id, + p.merchant_id, + p.platform, + p.platform_item_id, + p.title, + p.description, + p.category_id, + p.brand, + p.price_current, + p.price_original, + p.price_min, + p.price_max, + p.currency, + p.historical_sold, + p.stock, + p.rating_star, + p.rating_count, + p.images, + p.models, + p.attributes, + p.url, + p.cluster_id, + p.metadata, + p.created_at, + p.updated_at, + m.name AS merchant_name, + m.is_official AS merchant_is_official, + m.location AS merchant_location, + GREATEST( + COALESCE(t.risk_score::integer, 0), + COALESCE(m.risk_score, 0), + COALESCE(m.review_risk_score, 0) + ) AS threat_risk_score, + t.category AS threat_category, + t.source AS threat_source, + COALESCE(t.risk_score::integer, m.risk_score, 0) AS dosafe_risk_score, + COALESCE(m.review_risk_score, 0) AS review_risk_score, + COALESCE(m.review_risk_breakdown, '{}'::jsonb) AS review_risk_breakdown +FROM rate.products p +JOIN rate.merchants m ON p.merchant_id = m.id +LEFT JOIN dosafe.threat_intel t ON t.entity_hash = m.entity_hash +WHERE p.origin = 'crawl'; + +GRANT SELECT ON rate.v_products_with_merchant_trust TO anon, authenticated, service_role; + +-- 6. Aggregates only count crawled listings and crawled/community reviews ---- + +CREATE OR REPLACE FUNCTION rate.refresh_master_market_stats(p_master_id UUID) +RETURNS void +LANGUAGE sql +SECURITY DEFINER +SET search_path = '' +AS $$ + UPDATE rate.master_products m + SET min_price = s.min_price, + max_price = s.max_price, + merchant_count = COALESCE(s.merchant_count, 0), + updated_at = now() + FROM ( + SELECT + MIN(p.price_current) FILTER (WHERE p.price_current > 0) AS min_price, + MAX(COALESCE(NULLIF(p.price_max, 0), p.price_current)) FILTER (WHERE p.price_current > 0) AS max_price, + COUNT(DISTINCT p.merchant_id) FILTER (WHERE p.price_current > 0)::INTEGER AS merchant_count + FROM rate.products p + WHERE p.cluster_id = p_master_id + AND p.origin = 'crawl' + ) s + WHERE m.id = p_master_id; +$$; + +REVOKE ALL ON FUNCTION rate.refresh_master_market_stats(UUID) FROM PUBLIC, anon, authenticated; +GRANT EXECUTE ON FUNCTION rate.refresh_master_market_stats(UUID) TO service_role; + +CREATE OR REPLACE FUNCTION rate.refresh_master_product_rating(p_master_id UUID) +RETURNS void +LANGUAGE sql +SECURITY DEFINER +SET search_path = '' +AS $$ + UPDATE rate.master_products m + SET user_rating_weighted = COALESCE(s.weighted, 0), + authentic_review_count = COALESCE(s.n, 0), + updated_at = now() + FROM ( + SELECT + round( + (sum(r.rating_star * r.helpfulness_weight) FILTER (WHERE r.reviewer_type = 'user')) + / NULLIF(sum(r.helpfulness_weight) FILTER (WHERE r.reviewer_type = 'user'), 0), + 2 + ) AS weighted, + (count(*) FILTER (WHERE r.reviewer_type = 'user'))::INTEGER AS n + FROM rate.products p + JOIN rate.reviews r ON r.product_id = p.id + JOIN rate.review_flags f ON f.review_id = r.id + WHERE p.cluster_id = p_master_id + AND p.origin = 'crawl' + AND r.origin IN ('crawl', 'community') + -- Reviews excluded from the rating (unanalyzed community reviews, + -- flagged ones) never count, whatever their weight (20261014033000). + AND COALESCE(f.include_in_rating, true) + AND r.helpfulness_weight > 0 + ) s + WHERE m.id = p_master_id; +$$; + +REVOKE EXECUTE ON FUNCTION rate.refresh_master_product_rating(UUID) FROM PUBLIC, anon, authenticated; +GRANT EXECUTE ON FUNCTION rate.refresh_master_product_rating(UUID) TO service_role; + +-- 7. Writers that pick or compare rows only use crawled ones ---------------- +-- Bodies copied from 20261018100000 and 20261018120106; only the origin +-- filter is new. + +CREATE OR REPLACE FUNCTION rate.submit_community_review( + p_master_product_id UUID, + p_author_name TEXT, + p_rating_star INTEGER, + p_comment TEXT, + p_variation TEXT DEFAULT NULL, + p_reviewer_type TEXT DEFAULT 'user', + p_criteria_ratings JSONB DEFAULT '{}'::jsonb +) +RETURNS JSONB +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = rate, public, pg_temp +AS $$ +DECLARE + v_uid UUID := auth.uid(); + v_claims JSONB := COALESCE(NULLIF(current_setting('request.jwt.claims', true), '')::jsonb, '{}'::jsonb); + v_claim_role TEXT := COALESCE(v_claims ->> 'role', ''); + v_is_service BOOLEAN; + v_product_id UUID; + v_review_id UUID; + v_created_at TIMESTAMPTZ; + v_clean_author TEXT; + v_clean_comment TEXT; + v_clean_variation TEXT; + v_clean_type TEXT; + v_clean_criteria JSONB; + v_comment_len INTEGER; + v_recent INTEGER; + v_verdict TEXT := 'unanalyzed'; + v_flag_category TEXT := NULL; + v_reason TEXT := 'Đánh giá cộng đồng đang chờ phân tích, chưa được tính vào điểm trung bình.'; +BEGIN + v_is_service := v_claim_role = 'service_role'; + + -- Fail closed: anonymous callers are rejected even if EXECUTE is ever + -- re-granted by mistake. + IF v_uid IS NULL AND NOT v_is_service THEN + RAISE EXCEPTION 'Bạn cần đăng nhập để gửi đánh giá' USING ERRCODE = '42501'; + END IF; + + -- Supabase anonymous sign-ins mint role=authenticated users at will, which + -- would make the per-user limits below meaningless. + IF NOT v_is_service AND COALESCE(v_claims ->> 'is_anonymous', 'false') = 'true' THEN + RAISE EXCEPTION 'Bạn cần đăng nhập bằng tài khoản thật để gửi đánh giá' USING ERRCODE = '42501'; + END IF; + + IF p_master_product_id IS NULL THEN + RAISE EXCEPTION 'master_product_id is required' USING ERRCODE = '22023'; + END IF; + + v_clean_author := left(btrim(COALESCE(p_author_name, '')), 80); + IF char_length(v_clean_author) < 2 THEN + v_clean_author := 'Người dùng Rate.vn'; + END IF; + + v_clean_comment := btrim(COALESCE(p_comment, '')); + v_comment_len := char_length(v_clean_comment); + IF v_comment_len < 15 OR v_comment_len > 3000 THEN + RAISE EXCEPTION 'Nội dung đánh giá phải từ 15 đến 3000 ký tự' USING ERRCODE = '22023'; + END IF; + + IF p_rating_star IS NULL OR p_rating_star < 1 OR p_rating_star > 5 THEN + RAISE EXCEPTION 'Số sao đánh giá phải từ 1 đến 5' USING ERRCODE = '22023'; + END IF; + + -- Only service_role may submit 'expert' reviews; everyone else is 'user'. + v_clean_type := CASE + WHEN p_reviewer_type = 'expert' AND v_is_service THEN 'expert' + ELSE 'user' + END; + + v_clean_criteria := CASE + WHEN p_criteria_ratings IS NOT NULL AND jsonb_typeof(p_criteria_ratings) = 'object' + THEN p_criteria_ratings + ELSE '{}'::jsonb + END; + + v_clean_variation := NULLIF(left(btrim(COALESCE(p_variation, '')), 120), ''); + + -- Only a crawled listing: a review on a fabricated one is hidden (20261018153000). + SELECT id INTO v_product_id + FROM rate.products + WHERE cluster_id = p_master_product_id + AND origin = 'crawl' + ORDER BY price_current ASC NULLS LAST, created_at ASC + LIMIT 1; + + IF v_product_id IS NULL THEN + RAISE EXCEPTION 'Không tìm thấy listing liên kết cho sản phẩm chuẩn hóa này' USING ERRCODE = 'P0002'; + END IF; + + IF v_uid IS NOT NULL THEN + -- Serialize submissions per user so the checks below cannot be raced. + PERFORM pg_advisory_xact_lock( + hashtextextended('rate.submit_community_review:' || v_uid::text, 0) + ); + + IF EXISTS ( + SELECT 1 + FROM rate.review_authors a + JOIN rate.reviews r ON r.id = a.review_id + JOIN rate.products p ON p.id = r.product_id + WHERE a.author_user_id = v_uid + AND p.cluster_id = p_master_product_id + ) THEN + RAISE EXCEPTION 'Bạn đã gửi đánh giá cho sản phẩm này' USING ERRCODE = '23505'; + END IF; + + SELECT count(*) INTO v_recent + FROM rate.review_authors + WHERE author_user_id = v_uid + AND created_at > now() - interval '24 hours'; + + IF v_recent >= 5 THEN + RAISE EXCEPTION 'Bạn đã gửi quá 5 đánh giá trong 24 giờ, vui lòng thử lại sau' USING ERRCODE = '54000'; + END IF; + END IF; + + -- Keyword rules only decide exclusion. Nothing is ever included at submit + -- time; inclusion is the analyzer's decision. + IF v_clean_comment ~* '(nhận xu|kiếm xu|lấy xu|comment lấy xu|hình ảnh chỉ mang tính chất|video chỉ mang tính chất)' THEN + v_verdict := 'suspicious'; + v_flag_category := 'low_value_coin_spam'; + v_reason := 'Phát hiện cụm từ bình luận cày xu mẫu, loại khỏi điểm trung bình.'; + ELSIF v_clean_comment ~* '(inbox zalo|liên hệ zalo|nhắn zalo|http://|https://|shopee\.ee|s\.shopee)' THEN + v_verdict := 'suspicious'; + v_flag_category := 'seeding_fake'; + v_reason := 'Phát hiện đường dẫn quảng cáo hoặc điều hướng ngoài luồng, loại khỏi điểm trung bình.'; + ELSIF p_rating_star <= 2 AND v_clean_comment ~* '(shop lừa đảo sang shop|mua bên shop khác tốt hơn|đừng mua shop này qua shop)' THEN + v_verdict := 'suspicious'; + v_flag_category := 'competitor_sabotage'; + v_reason := 'Nghi vấn điều hướng sang đối thủ, loại khỏi điểm trung bình.'; + ELSIF v_comment_len < 30 THEN + v_verdict := 'suspicious'; + v_flag_category := 'low_value_coin_spam'; + v_reason := 'Nội dung quá ngắn để kiểm chứng trải nghiệm, loại khỏi điểm trung bình.'; + END IF; + + INSERT INTO rate.reviews ( + product_id, + platform_review_id, + author_name, + rating_star, + comment, + variation, + is_verified_buyer, + reviewer_type, + criteria_ratings, + upvotes, + downvotes, + report_count, + helpfulness_weight + ) + VALUES ( + v_product_id, + 'rate_comm_' || replace(gen_random_uuid()::text, '-', ''), + v_clean_author, + p_rating_star::smallint, + v_clean_comment, + v_clean_variation, + NULL, + v_clean_type, + v_clean_criteria, + 0, + 0, + 0, + 0.00 + ) + RETURNING id, created_at INTO v_review_id, v_created_at; + + IF v_uid IS NOT NULL THEN + INSERT INTO rate.review_authors (review_id, author_user_id) + VALUES (v_review_id, v_uid); + END IF; + + INSERT INTO rate.review_flags ( + review_id, + verdict, + fake_probability, + flag_category, + include_in_rating, + reason, + signals, + detector, + detector_version, + input_hash + ) + VALUES ( + v_review_id, + v_verdict, + NULL, + v_flag_category, + false, + v_reason, + '{}'::jsonb, + 'rate-community-guard', + 'v2', + rate.review_input_hash(v_clean_author, p_rating_star::smallint, v_clean_comment) + ); + + -- No rating refresh: an excluded review cannot change the weighted rating. + + RETURN jsonb_build_object( + 'id', v_review_id, + 'product_id', v_product_id, + 'author_name', v_clean_author, + 'rating_star', p_rating_star, + 'comment', v_clean_comment, + 'variation', v_clean_variation, + 'reviewer_type', v_clean_type, + 'criteria_ratings', v_clean_criteria, + 'verdict', v_verdict, + 'flag_category', v_flag_category, + 'fake_probability', NULL, + 'include_in_rating', false, + 'helpfulness_weight', 0, + 'reason', v_reason, + 'created_at', v_created_at + ); +END; +$$; + +REVOKE ALL ON FUNCTION rate.submit_community_review(UUID, TEXT, INTEGER, TEXT, TEXT, TEXT, JSONB) + FROM PUBLIC, anon; +GRANT EXECUTE ON FUNCTION rate.submit_community_review(UUID, TEXT, INTEGER, TEXT, TEXT, TEXT, JSONB) + TO authenticated, service_role; + +CREATE OR REPLACE FUNCTION rate.record_price_observation( + p_product_id UUID, + p_price NUMERIC, + p_price_original NUMERIC DEFAULT NULL, + p_source TEXT DEFAULT 'shopee_crawler', + p_observation_id TEXT DEFAULT NULL, + p_promotion_id UUID DEFAULT NULL +) +RETURNS TEXT +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + v_prev_price NUMERIC; + v_prev_original NUMERIC; + v_prev_recorded TIMESTAMPTZ; + v_origin TEXT; + v_inserted INTEGER := 0; +BEGIN + IF p_product_id IS NULL OR p_price IS NULL OR p_price < 0 THEN + RAISE EXCEPTION 'Invalid product_id or negative price in record_price_observation' + USING ERRCODE = '22023'; + END IF; + + SELECT p.origin INTO v_origin FROM rate.products p WHERE p.id = p_product_id; + v_origin := COALESCE(v_origin, 'crawl'); + + SELECT ph.price, ph.price_original, ph.recorded_at + INTO v_prev_price, v_prev_original, v_prev_recorded + FROM rate.price_history ph + WHERE ph.product_id = p_product_id + -- Dedupe against points of the listing's own origin (20261018153000): + -- public (crawled) points for a crawled listing, so hidden backfill or + -- synthetic points never suppress a real observation. + AND ph.origin = v_origin + ORDER BY ph.recorded_at DESC + LIMIT 1; + + IF v_prev_recorded IS NOT NULL + AND v_prev_price = p_price + AND COALESCE(v_prev_original, 0) = COALESCE(p_price_original, 0) + AND v_prev_recorded > pg_catalog.now() - INTERVAL '7 days' + THEN + RETURN 'unchanged_delta'; + END IF; + + IF p_observation_id IS NOT NULL THEN + INSERT INTO rate.price_history ( + product_id, + observation_id, + price, + price_original, + recorded_at, + source, + promotion_id + ) + VALUES ( + p_product_id, + p_observation_id, + p_price, + p_price_original, + pg_catalog.now(), + COALESCE(NULLIF(pg_catalog.btrim(p_source), ''), 'shopee_crawler'), + p_promotion_id + ) + ON CONFLICT (observation_id) WHERE observation_id IS NOT NULL DO NOTHING; + ELSE + INSERT INTO rate.price_history ( + product_id, + price, + price_original, + recorded_at, + source, + promotion_id + ) + VALUES ( + p_product_id, + p_price, + p_price_original, + pg_catalog.now(), + COALESCE(NULLIF(pg_catalog.btrim(p_source), ''), 'shopee_crawler'), + p_promotion_id + ); + END IF; + + GET DIAGNOSTICS v_inserted = ROW_COUNT; + IF v_inserted = 1 THEN + RETURN 'inserted'; + END IF; + RETURN 'deduped_conflict'; +END; +$$; + +REVOKE ALL ON FUNCTION rate.record_price_observation(UUID, NUMERIC, NUMERIC, TEXT, TEXT, UUID) FROM PUBLIC, anon, authenticated; +GRANT EXECUTE ON FUNCTION rate.record_price_observation(UUID, NUMERIC, NUMERIC, TEXT, TEXT, UUID) TO service_role; + +-- Community reviews already attached to a fabricated listing move to the +-- master's cheapest crawled listing, when it has one. +UPDATE rate.reviews r +SET product_id = target.id +FROM rate.products bad +JOIN LATERAL ( + SELECT c.id + FROM rate.products c + WHERE c.cluster_id = bad.cluster_id + AND c.origin = 'crawl' + ORDER BY c.price_current ASC NULLS LAST, c.created_at ASC + LIMIT 1 +) target ON true +WHERE r.product_id = bad.id + AND r.origin = 'community' + AND bad.origin <> 'crawl' + AND bad.cluster_id IS NOT NULL; + +-- 8. Recompute the cached prices and ratings of every master that had a +-- fabricated listing or review, so public pages stop showing them. +DO $$ +DECLARE + v_master UUID; +BEGIN + -- Both branches use the partial origin indexes created above. + FOR v_master IN + SELECT p.cluster_id + FROM rate.products p + WHERE p.origin <> 'crawl' AND p.cluster_id IS NOT NULL + UNION + SELECT p.cluster_id + FROM rate.reviews r + JOIN rate.products p ON p.id = r.product_id + WHERE r.origin <> 'crawl' AND r.origin <> 'community' AND p.cluster_id IS NOT NULL + LOOP + PERFORM rate.refresh_master_market_stats(v_master); + PERFORM rate.refresh_master_product_rating(v_master); + END LOOP; +END $$; + +RESET lock_timeout; diff --git a/supabase/migrations/20261018160000_rate_audit_aggregate_rpcs.sql b/supabase/migrations/20261018160000_rate_audit_aggregate_rpcs.sql new file mode 100644 index 000000000..ffde9ff0d --- /dev/null +++ b/supabase/migrations/20261018160000_rate_audit_aggregate_rpcs.sql @@ -0,0 +1,207 @@ +-- ============================================================================= +-- Migration: 20261018160000_rate_audit_aggregate_rpcs.sql +-- Purpose (Rate audit C2): compute the Rate-Services auditors' aggregates in +-- SQL instead of pulling capped pages of products, reviews and price history +-- into Node. PostgREST clamps every read to max_rows, so the auditors used +-- arbitrary subsets of the catalog and then wrote public risk scores. +-- +-- rate.listing_price_context - one row per priced, crawled listing with its +-- peer price floors and price-history statistics; keyset paged by id. +-- rate.merchant_review_counts - review classes per merchant, the same +-- classification as MerchantRiskAuditor; reviews that were never +-- analyzed (no review_flags row, or verdict 'unanalyzed') are pending, +-- not authentic. +-- +-- Both are STABLE, SECURITY INVOKER and only executable by service_role (the +-- auditors' role), and only use crawled listings (origin, 20261018153000). +-- listing_price_context is plpgsql so each branch gets an index-backed plan +-- (keyset on the products primary key, or the merchant_id index). +-- ============================================================================= + +CREATE OR REPLACE FUNCTION rate.listing_price_context( + p_after UUID DEFAULT NULL, + p_limit INTEGER DEFAULT 500, + p_history_days INTEGER DEFAULT 60, + p_merchant_ids UUID[] DEFAULT NULL +) +RETURNS TABLE ( + product_id UUID, + cluster_id UUID, + merchant_id UUID, + merchant_is_official BOOLEAN, + platform TEXT, + title TEXT, + price_current NUMERIC, + price_original NUMERIC, + -- Minimum price among the OTHER priced listings of the cluster, official + -- merchants first, any merchant when the cluster has no other official + -- listing (MerchantRiskAuditor.computePeerReferenceMinPrice). + peer_reference_min NUMERIC, + -- Minimum official-merchant price of the cluster, this listing included + -- (PriceAnomalyAuditor.computeOfficialMinByCluster). + cluster_official_min NUMERIC, + history_count INTEGER, + history_avg NUMERIC, + history_median NUMERIC, + history_min NUMERIC, + history_max NUMERIC +) +LANGUAGE plpgsql +STABLE +SET search_path = '' +AS $$ +DECLARE + -- Index-friendly predicates: no "param IS NULL OR ..." (which forces a + -- scan in a generic plan); the nil uuid sorts before every real id. + v_after UUID := COALESCE(p_after, '00000000-0000-0000-0000-000000000000'::uuid); + v_limit INTEGER := LEAST(GREATEST(COALESCE(p_limit, 500), 1), 1000); + v_since TIMESTAMPTZ := pg_catalog.now() - pg_catalog.make_interval(days => GREATEST(COALESCE(p_history_days, 60), 1)); + v_page UUID[]; +BEGIN + IF p_merchant_ids IS NULL THEN + SELECT pg_catalog.array_agg(x.id ORDER BY x.id) INTO v_page + FROM ( + SELECT p.id + FROM rate.products p + WHERE p.id > v_after + AND p.origin = 'crawl' + AND p.price_current > 0 + ORDER BY p.id + LIMIT v_limit + ) x; + ELSE + SELECT pg_catalog.array_agg(x.id ORDER BY x.id) INTO v_page + FROM ( + SELECT p.id + FROM rate.products p + WHERE p.merchant_id = ANY (p_merchant_ids) + AND p.id > v_after + AND p.origin = 'crawl' + AND p.price_current > 0 + ORDER BY p.id + LIMIT v_limit + ) x; + END IF; + + IF v_page IS NULL THEN + RETURN; + END IF; + + RETURN QUERY + SELECT + pg.id, + pg.cluster_id, + pg.merchant_id, + COALESCE(m.is_official, false), + pg.platform, + pg.title, + pg.price_current, + pg.price_original, + peers.peer_reference_min, + peers.cluster_official_min, + COALESCE(h.n, 0)::INTEGER, + h.avg_price, + h.median_price::NUMERIC, + h.min_price, + h.max_price + FROM rate.products pg + JOIN rate.merchants m ON m.id = pg.merchant_id + LEFT JOIN LATERAL ( + SELECT + COALESCE( + MIN(o.price_current) FILTER (WHERE o.id <> pg.id AND om.is_official), + MIN(o.price_current) FILTER (WHERE o.id <> pg.id) + ) AS peer_reference_min, + MIN(o.price_current) FILTER (WHERE om.is_official) AS cluster_official_min + FROM rate.products o + JOIN rate.merchants om ON om.id = o.merchant_id + WHERE pg.cluster_id IS NOT NULL + AND o.cluster_id = pg.cluster_id + AND o.origin = 'crawl' + AND o.price_current > 0 + ) peers ON true + LEFT JOIN LATERAL ( + SELECT + pg_catalog.count(*) AS n, + pg_catalog.avg(ph.price) AS avg_price, + percentile_cont(0.5) WITHIN GROUP (ORDER BY ph.price) AS median_price, + pg_catalog.min(ph.price) AS min_price, + pg_catalog.max(ph.price) AS max_price + FROM rate.price_history ph + WHERE ph.product_id = pg.id + AND ph.origin = 'crawl' + AND ph.price > 0 + AND ph.recorded_at >= v_since + ) h ON true + WHERE pg.id = ANY (v_page) + ORDER BY pg.id; +END; +$$; + +REVOKE ALL ON FUNCTION rate.listing_price_context(UUID, INTEGER, INTEGER, UUID[]) FROM PUBLIC, anon, authenticated; +GRANT EXECUTE ON FUNCTION rate.listing_price_context(UUID, INTEGER, INTEGER, UUID[]) TO service_role; + +CREATE OR REPLACE FUNCTION rate.merchant_review_counts(p_merchant_ids UUID[]) +RETURNS TABLE ( + merchant_id UUID, + total_reviews INTEGER, + authentic_count INTEGER, + seeding_count INTEGER, + coin_spam_count INTEGER, + pending_count INTEGER +) +LANGUAGE sql +STABLE +SET search_path = '' +AS $$ + WITH classified AS ( + SELECT + p.merchant_id, + f.review_id IS NULL AS unflagged, + f.verdict, + NULLIF(f.flag_category, '') AS flag_category, + lower(COALESCE(NULLIF(f.flag_category, ''), f.verdict, '')) AS cat, + COALESCE(f.fake_probability, 0) AS fake_probability + FROM rate.products p + JOIN rate.reviews r ON r.product_id = p.id + LEFT JOIN rate.review_flags f ON f.review_id = r.id + WHERE p.merchant_id = ANY (p_merchant_ids) + AND p.origin = 'crawl' + AND r.origin IN ('crawl', 'community') + ), + labelled AS ( + SELECT + merchant_id, + CASE + -- Not analyzed yet (no verdict row, or the community 'unanalyzed' + -- placeholder): neither authentic nor seeding. + -- A reserved re-analysis also resets verdict to 'unanalyzed' but keeps + -- the previous flag_category: a review already flagged keeps counting + -- as flagged; only one with no classification yet is pending. + WHEN unflagged OR (verdict = 'unanalyzed' AND COALESCE(flag_category, 'authentic') = 'authentic') THEN 'pending' + WHEN cat IN ('seeding_fake', 'fake') THEN 'seeding' + WHEN cat = 'low_value_coin_spam' THEN 'coin_spam' + WHEN cat IN ('authentic', 'competitor_sabotage', 'review_bombing') THEN 'authentic' + WHEN fake_probability >= 0.65 THEN 'seeding' + ELSE 'authentic' + END AS label + FROM classified + ) + SELECT + merchant_id, + (count(*) FILTER (WHERE label <> 'pending'))::INTEGER, + (count(*) FILTER (WHERE label = 'authentic'))::INTEGER, + (count(*) FILTER (WHERE label = 'seeding'))::INTEGER, + (count(*) FILTER (WHERE label = 'coin_spam'))::INTEGER, + (count(*) FILTER (WHERE label = 'pending'))::INTEGER + FROM labelled + GROUP BY merchant_id; +$$; + +REVOKE ALL ON FUNCTION rate.merchant_review_counts(UUID[]) FROM PUBLIC, anon, authenticated; +GRANT EXECUTE ON FUNCTION rate.merchant_review_counts(UUID[]) TO service_role; + +-- The peer-floor lateral scans one cluster at a time. +CREATE INDEX IF NOT EXISTS idx_rate_products_cluster_crawl_priced + ON rate.products (cluster_id, price_current) + WHERE origin = 'crawl' AND price_current > 0; diff --git a/supabase/migrations/20261018183000_rate_worker_role.sql b/supabase/migrations/20261018183000_rate_worker_role.sql new file mode 100644 index 000000000..47f0ad561 --- /dev/null +++ b/supabase/migrations/20261018183000_rate_worker_role.sql @@ -0,0 +1,225 @@ +-- ============================================================================= +-- Migration: 20261018183000_rate_worker_role.sql +-- Purpose (Rate audit K3): a least-privilege role for the Rate-Services crawl +-- worker, so the process that drives Chromium on untrusted e-commerce pages no +-- longer holds the shared project's service_role key. +-- +-- * rate_worker: NOLOGIN NOINHERIT, no BYPASSRLS. PostgREST switches to it +-- when a Supabase Auth user whose auth.users.role is 'rate_worker' signs +-- in (the access token's role claim), so authenticator is its only member. +-- * Privileges cover only the worker cycle (ingest, queue, auditors, price +-- alerts) in schema rate: SELECT/INSERT/UPDATE on the tables it writes, +-- SELECT on what it reads, EXECUTE on the RPCs it calls. No DELETE, no +-- TRUNCATE, no direct crawl_jobs/crawl_runs access (RPCs only), nothing +-- outside rate apart from USAGE on pg_trgm's schema for match_master_product. +-- * PII and other products' fields stay out of reach: merchants phone / +-- bank columns are not readable and DOSafe's risk_score is not writable; +-- price_alerts only lets the worker mark delivery; community reviews and +-- their verdicts are read-only to it. +-- * Catalog maintenance that deletes or rewrites data (purge-fabricated, +-- remove-wrong-listings, reanalyze-reviews, cancel_crawl_jobs) stays +-- service_role-only behind the prod-ops gate. The worker keeps +-- purge_completed_crawl_jobs, which only trims terminal queue rows past +-- the retention window (at least 3 days, 5000 rows per call). +-- +-- New rate tables or RPCs the worker needs must grant rate_worker explicitly: +-- there are no default privileges for this role. +-- Idempotent: guarded CREATE ROLE with a drift check, GRANT/REVOKE, DROP POLICY +-- IF EXISTS before CREATE POLICY. +-- ============================================================================= + +BEGIN; + +-- Fail fast instead of queueing behind long transactions on the shared database. +SET LOCAL lock_timeout = '5s'; +-- The creating role must not become a member of rate_worker. +SET LOCAL createrole_self_grant = ''; + +DO $$ +DECLARE + r RECORD; +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'rate_worker') THEN + CREATE ROLE rate_worker NOLOGIN NOINHERIT; + END IF; + + SELECT rolcanlogin, rolsuper, rolbypassrls, rolcreaterole, rolcreatedb, rolinherit + INTO r + FROM pg_roles WHERE rolname = 'rate_worker'; + IF r.rolcanlogin OR r.rolsuper OR r.rolbypassrls OR r.rolcreaterole OR r.rolcreatedb OR r.rolinherit THEN + RAISE EXCEPTION 'rate_worker drift: role must be NOLOGIN NOINHERIT without SUPERUSER, BYPASSRLS, CREATEROLE or CREATEDB'; + END IF; + + -- rate_worker must not inherit anything either. + IF EXISTS ( + SELECT 1 FROM pg_auth_members am + JOIN pg_roles member ON member.oid = am.member + WHERE member.rolname = 'rate_worker' + ) THEN + RAISE EXCEPTION 'rate_worker drift: rate_worker must not be a member of any role'; + END IF; +END $$; + +GRANT rate_worker TO authenticator; + +DO $$ +BEGIN + -- Only authenticator may switch to rate_worker (SET, no INHERIT). The only + -- other membership allowed is postgres's ADMIN-only grant that PostgreSQL + -- gives the creating role (no INHERIT, no SET), as for dosai_pricing_worker + -- (20261002110500). + IF EXISTS ( + SELECT 1 FROM pg_auth_members am + JOIN pg_roles granted ON granted.oid = am.roleid + JOIN pg_roles member ON member.oid = am.member + WHERE granted.rolname = 'rate_worker' + AND NOT ( + (member.rolname = 'authenticator' AND NOT am.inherit_option AND NOT am.admin_option) + OR (member.rolname = 'postgres' AND am.admin_option AND NOT am.inherit_option AND NOT am.set_option) + ) + ) THEN + RAISE EXCEPTION 'rate_worker drift: unexpected member of rate_worker (only authenticator, plus postgres as ADMIN only)'; + END IF; +END $$; + +-- Schema access ------------------------------------------------------------ + +GRANT USAGE ON SCHEMA rate TO rate_worker; + +DO $$ +DECLARE + v_trgm_schema TEXT; +BEGIN + -- match_master_product (SECURITY INVOKER) calls pg_trgm's similarity(); + -- production keeps the extension in schema "sign", other environments in + -- "extensions". On production "sign" also holds Crove Sign's tables, which + -- are granted only to sign_app / service_role / postgres, so USAGE alone + -- reaches nothing there beyond what PUBLIC can already execute. + SELECT n.nspname INTO v_trgm_schema + FROM pg_extension e JOIN pg_namespace n ON n.oid = e.extnamespace + WHERE e.extname = 'pg_trgm'; + IF v_trgm_schema IS NOT NULL AND v_trgm_schema <> 'rate' THEN + EXECUTE format('GRANT USAGE ON SCHEMA %I TO rate_worker', v_trgm_schema); + END IF; +END $$; + +-- Tables ------------------------------------------------------------------- + +REVOKE ALL ON ALL TABLES IN SCHEMA rate FROM rate_worker; + +GRANT SELECT ON rate.categories TO rate_worker; +GRANT SELECT, INSERT, UPDATE ON + rate.products, + rate.master_products, + rate.price_history, + rate.reviews, + rate.review_flags, + rate.review_analysis +TO rate_worker; + +-- merchants: no phone / bank_account / bank_name, and DOSafe's risk_score is +-- readable but never writable. SELECT on every upserted column is required: +-- ON CONFLICT DO UPDATE SET col = EXCLUDED.col reads it. +GRANT SELECT ( + id, platform, platform_shop_id, name, slug, url, is_official, rating_star, + follower_count, location, entity_hash, risk_score, metadata, created_at, + updated_at, review_risk_score, review_risk_breakdown, review_risk_updated_at, + archived_at +) ON rate.merchants TO rate_worker; +-- INSERT and UPDATE cover the same columns, so any upsert payload the worker +-- can update it can also insert. +GRANT INSERT ( + platform, platform_shop_id, name, url, is_official, rating_star, + follower_count, location, entity_hash, updated_at, review_risk_score, + review_risk_breakdown, review_risk_updated_at, metadata +) ON rate.merchants TO rate_worker; +-- PostgREST's merge-duplicates upsert SETs every payload column, including the +-- conflict key (platform, platform_shop_id), and PostgreSQL checks those UPDATE +-- privileges at plan time even when the row is new. +GRANT UPDATE ( + platform, platform_shop_id, name, url, is_official, rating_star, + follower_count, location, entity_hash, updated_at, review_risk_score, + review_risk_breakdown, review_risk_updated_at, metadata +) ON rate.merchants TO rate_worker; + +-- price_alerts: the evaluator reads alerts (subscriber_email for delivery) +-- and only marks delivery; it cannot redirect, retarget or disable an alert. +GRANT SELECT ON rate.price_alerts TO rate_worker; +GRANT UPDATE (last_triggered_at, updated_at) ON rate.price_alerts TO rate_worker; +-- price_history.id is BIGSERIAL (the no-RPC ingest fallback inserts directly). +GRANT USAGE ON SEQUENCE rate.price_history_id_seq TO rate_worker; + +-- RLS: the write policies on these tables are TO service_role; give the worker +-- its own. Table grants above, not the policies, limit it to SELECT/INSERT/ +-- UPDATE (no DELETE). +DO $$ +DECLARE + t TEXT; +BEGIN + FOREACH t IN ARRAY ARRAY[ + 'categories', 'merchants', 'products', 'master_products', 'price_history', + 'review_analysis', 'price_alerts' + ] LOOP + EXECUTE format('DROP POLICY IF EXISTS "Rate worker access %s" ON rate.%I', t, t); + EXECUTE format( + 'CREATE POLICY "Rate worker access %s" ON rate.%I FOR ALL TO rate_worker USING (true) WITH CHECK (true)', + t, t + ); + END LOOP; +END $$; + +-- reviews / review_flags: the worker writes crawled reviews and their +-- verdicts only. Community reviews (and their verdicts) are read-only to it. +DROP POLICY IF EXISTS "Rate worker access reviews" ON rate.reviews; +DROP POLICY IF EXISTS "Rate worker read reviews" ON rate.reviews; +DROP POLICY IF EXISTS "Rate worker insert reviews" ON rate.reviews; +DROP POLICY IF EXISTS "Rate worker update reviews" ON rate.reviews; +CREATE POLICY "Rate worker read reviews" ON rate.reviews + FOR SELECT TO rate_worker USING (true); +CREATE POLICY "Rate worker insert reviews" ON rate.reviews + FOR INSERT TO rate_worker WITH CHECK (origin <> 'community'); +CREATE POLICY "Rate worker update reviews" ON rate.reviews + FOR UPDATE TO rate_worker USING (origin <> 'community') WITH CHECK (origin <> 'community'); + +DROP POLICY IF EXISTS "Rate worker access review_flags" ON rate.review_flags; +DROP POLICY IF EXISTS "Rate worker read review_flags" ON rate.review_flags; +DROP POLICY IF EXISTS "Rate worker insert review_flags" ON rate.review_flags; +DROP POLICY IF EXISTS "Rate worker update review_flags" ON rate.review_flags; +CREATE POLICY "Rate worker read review_flags" ON rate.review_flags + FOR SELECT TO rate_worker USING (true); +CREATE POLICY "Rate worker insert review_flags" ON rate.review_flags + FOR INSERT TO rate_worker + WITH CHECK (EXISTS (SELECT 1 FROM rate.reviews r WHERE r.id = review_flags.review_id AND r.origin <> 'community')); +CREATE POLICY "Rate worker update review_flags" ON rate.review_flags + FOR UPDATE TO rate_worker + USING (EXISTS (SELECT 1 FROM rate.reviews r WHERE r.id = review_flags.review_id AND r.origin <> 'community')) + WITH CHECK (EXISTS (SELECT 1 FROM rate.reviews r WHERE r.id = review_flags.review_id AND r.origin <> 'community')); + +-- RPCs --------------------------------------------------------------------- + +REVOKE ALL ON ALL FUNCTIONS IN SCHEMA rate FROM rate_worker; + +-- Queue and run bookkeeping (SECURITY DEFINER; crawl_jobs has no table grant). +GRANT EXECUTE ON FUNCTION rate.start_crawl_run(TEXT, JSONB) TO rate_worker; +GRANT EXECUTE ON FUNCTION rate.finish_crawl_run(UUID, JSONB) TO rate_worker; +GRANT EXECUTE ON FUNCTION rate.enqueue_crawl_job(TEXT, TEXT, JSONB, TEXT, TEXT, TEXT, INTEGER, UUID) TO rate_worker; +GRANT EXECUTE ON FUNCTION rate.dequeue_crawl_job(TEXT, TEXT, INTERVAL) TO rate_worker; +GRANT EXECUTE ON FUNCTION rate.heartbeat_crawl_job(UUID, UUID, INTERVAL) TO rate_worker; +GRANT EXECUTE ON FUNCTION rate.checkpoint_crawl_job(UUID, UUID, JSONB) TO rate_worker; +GRANT EXECUTE ON FUNCTION rate.ack_crawl_job(UUID, UUID, JSONB) TO rate_worker; +GRANT EXECUTE ON FUNCTION rate.fail_crawl_job(UUID, UUID, TEXT) TO rate_worker; +GRANT EXECUTE ON FUNCTION rate.release_crawl_job(UUID, UUID, TEXT) TO rate_worker; +GRANT EXECUTE ON FUNCTION rate.purge_completed_crawl_jobs(INTEGER) TO rate_worker; +GRANT EXECUTE ON FUNCTION rate.stamp_crawl_provenance(UUID, UUID) TO rate_worker; + +-- Ingest and aggregates. +GRANT EXECUTE ON FUNCTION rate.record_price_observation(UUID, NUMERIC, NUMERIC, TEXT, TEXT, UUID) TO rate_worker; +GRANT EXECUTE ON FUNCTION rate.refresh_master_market_stats(UUID) TO rate_worker; +GRANT EXECUTE ON FUNCTION rate.refresh_master_product_rating(UUID) TO rate_worker; +GRANT EXECUTE ON FUNCTION rate.match_master_product(TEXT, TEXT, TEXT, NUMERIC, REAL) TO rate_worker; + +-- Auditor aggregates (SECURITY INVOKER: run with the table grants above). +GRANT EXECUTE ON FUNCTION rate.listing_price_context(UUID, INTEGER, INTEGER, UUID[]) TO rate_worker; +GRANT EXECUTE ON FUNCTION rate.merchant_review_counts(UUID[]) TO rate_worker; + +COMMIT; diff --git a/supabase/migrations/20261020230000_rate_listing_availability.sql b/supabase/migrations/20261020230000_rate_listing_availability.sql new file mode 100644 index 000000000..fe61ffe1a --- /dev/null +++ b/supabase/migrations/20261020230000_rate_listing_availability.sql @@ -0,0 +1,378 @@ +-- Rate: listing availability (DOS/Rate-Services#123) +-- +-- Purpose: a retailer page can load fine and state that the item is sold out +-- or discontinued (schema.org OutOfStock / Discontinued, TGDD flight data with +-- no buyable price). The Rate-Services worker completes such crawl jobs as +-- `listing_unavailable` without ingesting anything, so until now the listing +-- kept its last in-stock price and kept setting the master's min_price. +-- +-- 1. rate.products.availability ('in_stock' | 'out_of_stock' | 'discontinued') +-- and availability_checked_at. Existing listings were ingested with a price, +-- so they default to 'in_stock'. +-- 2. rate.mark_listing_unavailable(job, availability): called by the worker for +-- a running job whose page reports the item unavailable. Marks the crawled +-- listing of that job (no-op returning NULL for a discovery job that has no +-- listing yet) and refreshes its master's market stats. +-- 3. rate.stamp_crawl_provenance marks the listing 'in_stock' again: it only +-- runs after a successful ingest with a price. The ingest refreshed the +-- master's stats while the listing was still unavailable, so a restock +-- refreshes them again. +-- 4. rate.refresh_master_market_stats only counts in-stock listings. +-- 5. rate.listing_price_context ignores sold-out peers in its price floors. +-- +-- rate.v_products_with_merchant_trust is left unchanged: appending columns to it +-- would make re-running 20261018153000 fail ("cannot drop columns from view"). +-- Public reads see the new columns on rate.products under its existing policy. +-- +-- Idempotent: ADD COLUMN IF NOT EXISTS, guarded constraint, CREATE OR REPLACE. +-- Touches the `rate` schema only. + +-- 1. Columns ----------------------------------------------------------------- +-- A constant default is a catalog-only change on PG 11+, but ADD COLUMN still +-- takes ACCESS EXCLUSIVE: give up fast instead of queueing behind a long +-- transaction and blocking every read. The CHECK is added NOT VALID and then +-- validated; within the migration's transaction the scan still runs under the +-- ADD COLUMN lock, which is fine for a table this size. + +SET lock_timeout = '5s'; + +ALTER TABLE rate.products ADD COLUMN IF NOT EXISTS availability TEXT NOT NULL DEFAULT 'in_stock'; +ALTER TABLE rate.products ADD COLUMN IF NOT EXISTS availability_checked_at TIMESTAMPTZ; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'rate.products'::regclass AND conname = 'products_availability_check' + ) THEN + ALTER TABLE rate.products + ADD CONSTRAINT products_availability_check + CHECK (availability IN ('in_stock', 'out_of_stock', 'discontinued')) NOT VALID; + END IF; +END $$; + +RESET lock_timeout; + +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'rate.products'::regclass + AND conname = 'products_availability_check' + AND NOT convalidated + ) THEN + ALTER TABLE rate.products VALIDATE CONSTRAINT products_availability_check; + END IF; +END $$; + +COMMENT ON COLUMN rate.products.availability IS + 'Stock state the retailer page last reported: in_stock (priced crawl), out_of_stock or discontinued (set by rate.mark_listing_unavailable).'; +COMMENT ON COLUMN rate.products.availability_checked_at IS + 'When availability was last confirmed by a crawl job; NULL for listings not crawled since the column was added.'; + +-- 2. Worker RPC ---------------------------------------------------------------- + +CREATE OR REPLACE FUNCTION rate.mark_listing_unavailable(p_job_id UUID, p_availability TEXT) +RETURNS UUID +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + v_product_id UUID; + v_cluster_id UUID; +BEGIN + IF p_job_id IS NULL OR p_availability IS NULL OR p_availability NOT IN ('out_of_stock', 'discontinued') THEN + RAISE EXCEPTION 'job id and availability out_of_stock|discontinued are required' USING ERRCODE = '22023'; + END IF; + + -- Only a job that is currently being processed may report availability. + IF NOT EXISTS ( + SELECT 1 FROM rate.crawl_jobs WHERE id = p_job_id AND status = 'running' + ) THEN + RAISE EXCEPTION 'crawl job % is not running', p_job_id USING ERRCODE = 'P0002'; + END IF; + + -- The job's own listing: same platform, and the same item id, or the same + -- URL when the job carries no item id (URL-only discovery jobs). One row: + -- (platform, platform_item_id) is unique, and URL matches prefer the newest. + SELECT p.id, p.cluster_id + INTO v_product_id, v_cluster_id + FROM rate.products p + JOIN rate.crawl_jobs j ON j.id = p_job_id + WHERE p.origin = 'crawl' + AND p.platform = j.platform + AND ( + (j.platform_item_id IS NOT NULL AND p.platform_item_id = j.platform_item_id) + OR (j.platform_item_id IS NULL AND j.target_url IS NOT NULL AND p.url = j.target_url) + ) + ORDER BY p.updated_at DESC NULLS LAST, p.id + LIMIT 1; + + -- A discovery job for a listing that was never ingested has nothing to mark. + IF v_product_id IS NULL THEN + RETURN NULL; + END IF; + + UPDATE rate.products + SET availability = p_availability, + availability_checked_at = now(), + last_crawled_at = now(), + last_crawl_job_id = p_job_id + WHERE id = v_product_id; + + IF v_cluster_id IS NOT NULL THEN + PERFORM rate.refresh_master_market_stats(v_cluster_id); + END IF; + + RETURN v_product_id; +END; +$$; + +REVOKE ALL ON FUNCTION rate.mark_listing_unavailable(UUID, TEXT) FROM PUBLIC, anon, authenticated; +GRANT EXECUTE ON FUNCTION rate.mark_listing_unavailable(UUID, TEXT) TO service_role; + +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'rate_worker') THEN + GRANT EXECUTE ON FUNCTION rate.mark_listing_unavailable(UUID, TEXT) TO rate_worker; + END IF; +END $$; + +-- 3. A priced crawl puts the listing back in stock -------------------------- +-- Body as in 20261018153000, plus the availability columns in the listing update +-- and a stats refresh when that update is a restock. + +CREATE OR REPLACE FUNCTION rate.stamp_crawl_provenance(p_job_id UUID, p_product_id UUID) +RETURNS JSONB +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + v_prices INTEGER; + v_reviews INTEGER; + v_prev_availability TEXT; + v_cluster_id UUID; +BEGIN + IF p_job_id IS NULL OR p_product_id IS NULL THEN + RAISE EXCEPTION 'job id and product id are required' USING ERRCODE = '22023'; + END IF; + + -- Only a job that is currently being processed may stamp rows. + IF NOT EXISTS ( + SELECT 1 FROM rate.crawl_jobs WHERE id = p_job_id AND status = 'running' + ) THEN + RAISE EXCEPTION 'crawl job % is not running', p_job_id USING ERRCODE = 'P0002'; + END IF; + + -- Locked so a concurrent mark_listing_unavailable cannot slip between this + -- read and the update below and leave the restock refresh skipped. + SELECT p.availability, p.cluster_id + INTO v_prev_availability, v_cluster_id + FROM rate.products p + WHERE p.id = p_product_id + FOR UPDATE; + + -- The job must be for this listing: same platform, and the same item id or + -- URL when the job carries them (URL-only discovery jobs have no item id). + -- The job ingested a price, so the listing is in stock. + UPDATE rate.products p + SET last_crawled_at = now(), + last_crawl_job_id = p_job_id, + availability = 'in_stock', + availability_checked_at = now() + FROM rate.crawl_jobs j + WHERE p.id = p_product_id + AND p.origin = 'crawl' + AND j.id = p_job_id + AND j.platform = p.platform + AND ( + j.platform_item_id = p.platform_item_id + OR j.target_url = p.url + OR j.platform_item_id IS NULL + ); + + IF NOT FOUND THEN + RAISE EXCEPTION 'crawled listing % not found for job %', p_product_id, p_job_id USING ERRCODE = 'P0002'; + END IF; + + -- The ingest refreshed the master's stats before this stamp, while the + -- listing was still out of stock and so not counted. + IF v_prev_availability IS DISTINCT FROM 'in_stock' AND v_cluster_id IS NOT NULL THEN + PERFORM rate.refresh_master_market_stats(v_cluster_id); + END IF; + + -- Rows this job just wrote: the dedupe index keeps one live job per + -- listing, and one job runs for minutes, so the 1h window cannot reach a + -- previous crawl's rows (those are already stamped or predate this column). + UPDATE rate.price_history + SET crawl_job_id = p_job_id + WHERE product_id = p_product_id + AND crawl_job_id IS NULL + AND origin = 'crawl' + AND recorded_at > now() - interval '1 hour'; + GET DIAGNOSTICS v_prices = ROW_COUNT; + + UPDATE rate.reviews + SET first_seen_job_id = p_job_id + WHERE product_id = p_product_id + AND first_seen_job_id IS NULL + AND origin = 'crawl' + AND observed_at > now() - interval '1 hour'; + GET DIAGNOSTICS v_reviews = ROW_COUNT; + + RETURN jsonb_build_object('price_points', v_prices, 'reviews', v_reviews); +END; +$$; + +-- 4. Market stats count in-stock listings only ------------------------------- +-- Body as in 20261018153000, plus the availability filter. + +CREATE OR REPLACE FUNCTION rate.refresh_master_market_stats(p_master_id UUID) +RETURNS void +LANGUAGE sql +SECURITY DEFINER +SET search_path = '' +AS $$ + UPDATE rate.master_products m + SET min_price = s.min_price, + max_price = s.max_price, + merchant_count = COALESCE(s.merchant_count, 0), + updated_at = now() + FROM ( + SELECT + MIN(p.price_current) FILTER (WHERE p.price_current > 0) AS min_price, + MAX(COALESCE(NULLIF(p.price_max, 0), p.price_current)) FILTER (WHERE p.price_current > 0) AS max_price, + COUNT(DISTINCT p.merchant_id) FILTER (WHERE p.price_current > 0)::INTEGER AS merchant_count + FROM rate.products p + WHERE p.cluster_id = p_master_id + AND p.origin = 'crawl' + AND p.availability = 'in_stock' + ) s + WHERE m.id = p_master_id; +$$; + +-- 5. Sold-out peers no longer set the auditors' price floors ---------------- +-- Body as in 20261018160000, plus the availability filter on peers. + +CREATE OR REPLACE FUNCTION rate.listing_price_context( + p_after UUID DEFAULT NULL, + p_limit INTEGER DEFAULT 500, + p_history_days INTEGER DEFAULT 60, + p_merchant_ids UUID[] DEFAULT NULL +) +RETURNS TABLE ( + product_id UUID, + cluster_id UUID, + merchant_id UUID, + merchant_is_official BOOLEAN, + platform TEXT, + title TEXT, + price_current NUMERIC, + price_original NUMERIC, + -- Minimum price among the OTHER priced, in-stock listings of the cluster, + -- official merchants first, any merchant when the cluster has no other + -- official listing (MerchantRiskAuditor.computePeerReferenceMinPrice). + peer_reference_min NUMERIC, + -- Minimum official-merchant price of the cluster's in-stock listings, this + -- listing included while it is in stock (PriceAnomalyAuditor.computeOfficialMinByCluster). + cluster_official_min NUMERIC, + history_count INTEGER, + history_avg NUMERIC, + history_median NUMERIC, + history_min NUMERIC, + history_max NUMERIC +) +LANGUAGE plpgsql +STABLE +SET search_path = '' +AS $$ +DECLARE + -- Index-friendly predicates: no "param IS NULL OR ..." (which forces a + -- scan in a generic plan); the nil uuid sorts before every real id. + v_after UUID := COALESCE(p_after, '00000000-0000-0000-0000-000000000000'::uuid); + v_limit INTEGER := LEAST(GREATEST(COALESCE(p_limit, 500), 1), 1000); + v_since TIMESTAMPTZ := pg_catalog.now() - pg_catalog.make_interval(days => GREATEST(COALESCE(p_history_days, 60), 1)); + v_page UUID[]; +BEGIN + IF p_merchant_ids IS NULL THEN + SELECT pg_catalog.array_agg(x.id ORDER BY x.id) INTO v_page + FROM ( + SELECT p.id + FROM rate.products p + WHERE p.id > v_after + AND p.origin = 'crawl' + AND p.price_current > 0 + ORDER BY p.id + LIMIT v_limit + ) x; + ELSE + SELECT pg_catalog.array_agg(x.id ORDER BY x.id) INTO v_page + FROM ( + SELECT p.id + FROM rate.products p + WHERE p.merchant_id = ANY (p_merchant_ids) + AND p.id > v_after + AND p.origin = 'crawl' + AND p.price_current > 0 + ORDER BY p.id + LIMIT v_limit + ) x; + END IF; + + IF v_page IS NULL THEN + RETURN; + END IF; + + RETURN QUERY + SELECT + pg.id, + pg.cluster_id, + pg.merchant_id, + COALESCE(m.is_official, false), + pg.platform, + pg.title, + pg.price_current, + pg.price_original, + peers.peer_reference_min, + peers.cluster_official_min, + COALESCE(h.n, 0)::INTEGER, + h.avg_price, + h.median_price::NUMERIC, + h.min_price, + h.max_price + FROM rate.products pg + JOIN rate.merchants m ON m.id = pg.merchant_id + LEFT JOIN LATERAL ( + SELECT + COALESCE( + MIN(o.price_current) FILTER (WHERE o.id <> pg.id AND om.is_official), + MIN(o.price_current) FILTER (WHERE o.id <> pg.id) + ) AS peer_reference_min, + MIN(o.price_current) FILTER (WHERE om.is_official) AS cluster_official_min + FROM rate.products o + JOIN rate.merchants om ON om.id = o.merchant_id + WHERE pg.cluster_id IS NOT NULL + AND o.cluster_id = pg.cluster_id + AND o.origin = 'crawl' + AND o.price_current > 0 + AND o.availability = 'in_stock' + ) peers ON true + LEFT JOIN LATERAL ( + SELECT + pg_catalog.count(*) AS n, + pg_catalog.avg(ph.price) AS avg_price, + percentile_cont(0.5) WITHIN GROUP (ORDER BY ph.price) AS median_price, + pg_catalog.min(ph.price) AS min_price, + pg_catalog.max(ph.price) AS max_price + FROM rate.price_history ph + WHERE ph.product_id = pg.id + AND ph.origin = 'crawl' + AND ph.price > 0 + AND ph.recorded_at >= v_since + ) h ON true + WHERE pg.id = ANY (v_page) + ORDER BY pg.id; +END; +$$; diff --git a/supabase/migrations/20261022140000_rate_move_iphone_16e_offer.sql b/supabase/migrations/20261022140000_rate_move_iphone_16e_offer.sql new file mode 100644 index 000000000..48108e3dd --- /dev/null +++ b/supabase/migrations/20261022140000_rate_move_iphone_16e_offer.sql @@ -0,0 +1,174 @@ +-- Rate (DOS/Rate#594): a CellphoneS "iPhone 16e 128GB" offer was clustered +-- into the "Apple iPhone 16 128GB" master product (before the 16e parse fix, +-- DOS/Rate#352). It pulled the iPhone 16 price floor to 13.69M and showed a +-- false -43% deal on rate.vn/deals. JOY approved moving it (2026-10-22). +-- +-- This moves that one offer to an "Apple iPhone 16e 128GB" master product, +-- creating it as the importer would (model_key apple:iphone-16e:128gb), then +-- refreshes price and rating stats of both masters. The offer's crawled +-- reviews, price history and review analysis stay keyed by the product id and +-- move with it. Community reviews stored on it were written for the iPhone 16 +-- master and stay there, on its cheapest other crawled listing. +-- +-- Data only, rate schema only. Idempotent: on a database without the offer +-- (Preview, Beta) it changes nothing; once moved, a re-run only refreshes the +-- stats. Fails closed if the offer or the iPhone 16 master is not what it +-- expects. + +SELECT pg_catalog.set_config('lock_timeout', '5s', true); + +DO $$ +DECLARE + c_offer CONSTANT UUID := 'b6f84fe3-8a9f-4775-abac-5400cf69efef'; + c_iphone_16 CONSTANT UUID := '3afb2a7a-d791-4763-8ef6-842ee01bfd48'; + c_model_key CONSTANT TEXT := 'apple:iphone-16e:128gb'; + c_slug CONSTANT TEXT := 'apple-iphone-16e-128gb'; + v_offer RECORD; + v_slug TEXT := c_slug; + v_target UUID; + v_target_status TEXT; + v_rehome UUID; + v_rehomed INTEGER; +BEGIN + SELECT id, cluster_id, category_id, title, images + INTO v_offer + FROM rate.products + WHERE id = c_offer + FOR UPDATE; + + IF NOT FOUND THEN + RAISE NOTICE 'DOS/Rate#594: offer % not present; nothing to move', c_offer; + RETURN; + END IF; + + IF v_offer.title !~* 'iphone\s*16\s*e([^a-z0-9]|$)' THEN + RAISE EXCEPTION 'DOS/Rate#594: offer % is not an iPhone 16e listing (title %)', + c_offer, v_offer.title; + END IF; + + IF NOT EXISTS ( + SELECT 1 FROM rate.master_products + WHERE id = c_iphone_16 AND model_key = 'apple:iphone-16:128gb' + ) THEN + RAISE EXCEPTION 'DOS/Rate#594: master % is not apple:iphone-16:128gb', c_iphone_16; + END IF; + + -- The importer's slug fallback when the slug is taken by another model_key. + IF EXISTS ( + SELECT 1 FROM rate.master_products + WHERE slug = c_slug AND model_key <> c_model_key + ) THEN + v_slug := c_slug || '-' || left(encode(sha256(convert_to(c_model_key, 'UTF8')), 'hex'), 8); + END IF; + + -- Specs come only from the importer's benchmark registry entry for + -- apple:iphone-16e (apps/importer/src/shopee/product-normalizer.ts), so the + -- master is published as the importer seeds benchmark-backed products. + INSERT INTO rate.master_products ( + category_id, brand, model, variant_name, model_key, name, slug, description, + ram_gb, storage_gb, screen_size_inch, battery_mah, has_5g, + specifications, key_specs, images, thumbnail, colors, status + ) + VALUES ( + v_offer.category_id, + 'Apple', + 'iPhone 16e', + '128GB', + c_model_key, + 'Apple iPhone 16e 128GB', + v_slug, + '', + 8, + 128, + 6.1, + 4005, + true, + jsonb_build_object( + 'category_slug', 'smartphones', + 'availability', jsonb_build_object('announced', 'February 2025'), + 'design', jsonb_build_object( + 'dimensions', '146.7 x 71.5 x 7.8 mm', + 'weight_g', 167, + 'resistance', 'IP68 (6m up to 30 mins)' + ), + 'display', jsonb_build_object( + 'size_inch', 6.1, + 'type', 'Super Retina XDR OLED, HDR10, Dolby Vision, 1200 nits (peak)', + 'resolution', '1170 x 2532 pixels (~460 ppi)', + 'refresh_rate_hz', 60 + ), + 'hardware', jsonb_build_object( + 'chipset', 'Apple A18 (3 nm)', + 'cpu', 'Hexa-core (2x4.04 GHz + 4x2.20 GHz)', + 'gpu', 'Apple GPU (4-core graphics)', + 'ram_gb', 8, + 'storage_gb', 128, + 'os', 'iOS 18.3 (upgradable)' + ), + 'battery', jsonb_build_object('capacity_mah', 4005, 'fast_charging_w', 20), + 'camera', jsonb_build_object( + 'main_camera', '48 MP, f/1.6, 26mm (wide), PDAF, OIS', + 'front_camera', '12 MP, f/1.9, 23mm (wide), PDAF + SL 3D' + ), + 'cellular', jsonb_build_object('has_5g', true) + ), + jsonb_build_object( + 'screen', '6.1" Super Retina XDR OLED', + 'chip', 'Apple A18', + 'camera', '48 MP', + 'battery', '4005 mAh (20W)', + 'storage', '128GB', + 'ram', '8GB' + ), + COALESCE(v_offer.images, '{}'::TEXT[]), + v_offer.images[1], + '{}'::TEXT[], + 'published' + ) + ON CONFLICT (model_key) DO NOTHING; + + -- An existing apple:iphone-16e:128gb master keeps its own identity and status. + SELECT id, status INTO STRICT v_target, v_target_status + FROM rate.master_products + WHERE model_key = c_model_key + FOR UPDATE; + + IF v_offer.cluster_id = v_target THEN + RAISE NOTICE 'DOS/Rate#594: offer % already on master %', c_offer, v_target; + ELSIF v_offer.cluster_id IS DISTINCT FROM c_iphone_16 THEN + RAISE EXCEPTION 'DOS/Rate#594: offer % is on unexpected master %', c_offer, v_offer.cluster_id; + ELSE + -- submit_community_review stores a review of a master on its cheapest + -- crawled listing, which was this offer. Those reviews are about the + -- iPhone 16: keep them there, on the listing the RPC now picks. + IF EXISTS (SELECT 1 FROM rate.reviews WHERE product_id = c_offer AND origin = 'community') THEN + SELECT id INTO v_rehome + FROM rate.products + WHERE cluster_id = c_iphone_16 + AND origin = 'crawl' + AND id <> c_offer + ORDER BY price_current ASC NULLS LAST, created_at ASC + LIMIT 1; + IF v_rehome IS NULL THEN + RAISE EXCEPTION 'DOS/Rate#594: community reviews on offer % have no other iPhone 16 listing', c_offer; + END IF; + UPDATE rate.reviews + SET product_id = v_rehome + WHERE product_id = c_offer AND origin = 'community'; + GET DIAGNOSTICS v_rehomed = ROW_COUNT; + RAISE NOTICE 'DOS/Rate#594: kept % community review(s) on iPhone 16 listing %', v_rehomed, v_rehome; + END IF; + + UPDATE rate.products + SET cluster_id = v_target, + updated_at = now() + WHERE id = c_offer; + RAISE NOTICE 'DOS/Rate#594: moved offer % from % to % (status %)', + c_offer, c_iphone_16, v_target, v_target_status; + END IF; + + PERFORM rate.refresh_master_market_stats(c_iphone_16); + PERFORM rate.refresh_master_market_stats(v_target); + PERFORM rate.refresh_master_product_rating(c_iphone_16); + PERFORM rate.refresh_master_product_rating(v_target); +END $$; diff --git a/supabase/migrations/20261023090000_rate_review_vote_eligibility.sql b/supabase/migrations/20261023090000_rate_review_vote_eligibility.sql new file mode 100644 index 000000000..1a4faff04 --- /dev/null +++ b/supabase/migrations/20261023090000_rate_review_vote_eligibility.sql @@ -0,0 +1,82 @@ +-- DOS/Rate#613: enforce review-vote eligibility in the database. +-- +-- rate.review_votes grants INSERT/UPDATE to `authenticated`, and RLS only +-- checks voter_id = auth.uid(). The `rate` schema is exposed through +-- PostgREST (rate.vn writes votes with the user's JWT), so an account could +-- insert a vote directly and skip the Review Trust Model's voter rules: no +-- anonymous session, a confirmed email or phone, and an account at least 7 +-- days old. The Next.js route (apps/web/src/app/api/reviews/[id]/vote/route.ts, +-- evaluateVoterEligibility) checks a subset of them. +-- +-- The guard trigger now applies these rules to user callers: the `anon` and +-- `authenticated` roles, and SECURITY DEFINER functions they call (detected +-- through the JWT role claim). service_role (importer, admin tooling) and +-- migrations are not affected. DELETE (clearing one's own vote) stays +-- allowed. Idempotent. + +CREATE OR REPLACE FUNCTION rate.current_user_can_vote_reviews() +RETURNS boolean +LANGUAGE sql +STABLE +SECURITY DEFINER +SET search_path = '' +AS $$ + SELECT EXISTS ( + SELECT 1 + FROM auth.users u + WHERE u.id = (SELECT auth.uid()) + AND COALESCE(u.is_anonymous, false) = false + AND (u.email_confirmed_at IS NOT NULL OR u.phone_confirmed_at IS NOT NULL) + AND u.created_at <= now() - interval '7 days' + ); +$$; + +COMMENT ON FUNCTION rate.current_user_can_vote_reviews() IS + 'True when the calling user may vote on or report reviews: not anonymous, confirmed email or phone, account at least 7 days old (DOS/Rate#613). Reads only the caller''s own auth.users row.'; + +REVOKE ALL ON FUNCTION rate.current_user_can_vote_reviews() FROM PUBLIC, anon; +GRANT EXECUTE ON FUNCTION rate.current_user_can_vote_reviews() TO authenticated, service_role; + +CREATE OR REPLACE FUNCTION rate.review_votes_guard() +RETURNS trigger +LANGUAGE plpgsql +SET search_path = '' +AS $$ +DECLARE + v_recent INTEGER; +BEGIN + -- Review Trust Model voter rules (DOS/Rate#613). User callers are checked, + -- including through a SECURITY DEFINER function, where current_user is its + -- owner but the JWT role claim is still the user's. service_role and the + -- migration owner write on users' behalf and are not checked. PostgREST + -- always sets request.jwt.claims to a JSON object; a non-JSON value fails + -- closed with a cast error. + IF (current_user IN ('anon', 'authenticated') + OR COALESCE(NULLIF(current_setting('request.jwt.claims', true), '')::jsonb ->> 'role', '') + IN ('anon', 'authenticated')) + AND NOT rate.current_user_can_vote_reviews() THEN + RAISE EXCEPTION 'review_votes: account is not eligible to vote on reviews' + USING ERRCODE = '42501'; + END IF; + + IF TG_OP = 'UPDATE' THEN + IF NEW.review_id <> OLD.review_id OR NEW.voter_id <> OLD.voter_id THEN + RAISE EXCEPTION 'review_votes: review_id and voter_id are immutable'; + END IF; + NEW.updated_at := now(); + END IF; + + -- Enforce 60 vote/report actions per hour across both INSERT and UPDATE (Audit T3) + SELECT count(*) INTO v_recent + FROM rate.review_votes + WHERE voter_id = NEW.voter_id + AND updated_at > now() - interval '1 hour'; + + IF v_recent >= 60 THEN + RAISE EXCEPTION 'review_votes: rate limit exceeded (60 vote actions per hour)' + USING ERRCODE = '54000'; + END IF; + + RETURN NEW; +END; +$$; diff --git a/supabase/migrations/20261024060000_rate_master_min_price_plausible.sql b/supabase/migrations/20261024060000_rate_master_min_price_plausible.sql new file mode 100644 index 000000000..2b98db5fc --- /dev/null +++ b/supabase/migrations/20261024060000_rate_master_min_price_plausible.sql @@ -0,0 +1,85 @@ +-- Rate: a bait-price listing no longer sets a master's min_price (DOS/Rate#802) +-- +-- Purpose: a listing far below every other store's price for the same model +-- is almost never a real deal: it is a counterfeit, an accessory or a bait +-- listing. rate.vn showed the Samsung Galaxy Buds3 Pro "from 723,269 VND" on +-- catalogue cards, compare and the public API while FPT Shop, TGDD and +-- CellphoneS sold it for 3.03-3.69 M. The product page already ignores such +-- listings (DOS/Rate#797, apps/web/src/lib/offer-plausibility.ts); the stored +-- min_price did not. JOY approved this migration (2026-10-24). +-- +-- 1. rate.refresh_master_market_stats leaves implausibly low listings out of +-- min_price with the web rule: with at least 3 priced listings, a listing +-- below 40% of the lower median of the OTHER listings is implausible. +-- The lower median of the m - 1 others is position L = floor((m - 2) / 2) +-- of the others, which is position L of the sorted list before the listing +-- and L + 1 from it on. max_price and merchant_count still count every +-- priced in-stock listing. +-- The update is skipped when the stats are unchanged, so a refresh that +-- finds the same prices no longer bumps updated_at (the sitemap's lastmod). +-- 2. Backfill: every master's stats are refreshed once; thanks to (1) only +-- masters whose stats change are written. +-- +-- Body as in 20261020230000 otherwise (crawl, in-stock, priced listings). +-- CREATE OR REPLACE keeps the existing grants (service_role, rate_worker). +-- Idempotent. Touches the `rate` schema only. + +-- 1. Market stats without bait prices ------------------------------------------ + +CREATE OR REPLACE FUNCTION rate.refresh_master_market_stats(p_master_id UUID) +RETURNS void +LANGUAGE sql +SECURITY DEFINER +SET search_path = '' +AS $$ + WITH priced AS ( + SELECT + p.price_current AS price, + COALESCE(NULLIF(p.price_max, 0), p.price_current) AS price_top, + p.merchant_id, + (pg_catalog.row_number() OVER (ORDER BY p.price_current, p.id) - 1)::INTEGER AS k, + (pg_catalog.count(*) OVER ())::INTEGER AS m + FROM rate.products p + WHERE p.cluster_id = p_master_id + AND p.origin = 'crawl' + AND p.availability = 'in_stock' + AND p.price_current > 0 + ), + judged AS ( + SELECT + c.price, + c.price_top, + c.merchant_id, + (c.m >= 3 AND c.price < 0.4 * r.price) AS implausible + FROM priced c + LEFT JOIN priced r + ON r.k = CASE WHEN (c.m - 2) / 2 < c.k THEN (c.m - 2) / 2 ELSE (c.m - 2) / 2 + 1 END + ), + s AS ( + SELECT + MIN(j.price) FILTER (WHERE NOT j.implausible) AS min_price, + MAX(j.price_top) AS max_price, + COUNT(DISTINCT j.merchant_id)::INTEGER AS merchant_count + FROM judged j + ) + UPDATE rate.master_products m + SET min_price = s.min_price, + max_price = s.max_price, + merchant_count = s.merchant_count, + updated_at = now() + FROM s + WHERE m.id = p_master_id + AND (m.min_price, m.max_price, m.merchant_count) + IS DISTINCT FROM (s.min_price, s.max_price, s.merchant_count); +$$; + +-- 2. Backfill ------------------------------------------------------------------- + +DO $$ +DECLARE + v_id UUID; +BEGIN + FOR v_id IN SELECT m.id FROM rate.master_products m ORDER BY m.id LOOP + PERFORM rate.refresh_master_market_stats(v_id); + END LOOP; +END $$;