diff --git a/.env.example b/.env.example index 83f04a1..ca0beb2 100644 --- a/.env.example +++ b/.env.example @@ -19,4 +19,5 @@ GOOGLE_API_KEY=your_google_api_key GOOGLE_CSE_ID=your_google_programmable_search_engine_id SERPAPI_API_KEY=your_serpapi_api_key_for_google_shopping -DB_CONNECTION_STRING=your_database_connection_string \ No newline at end of file +# Matches the default PostgreSQL service in docker-compose.yaml. +DB_CONNECTION_STRING=postgres://asin_check:asin_check@localhost:5432/asin_check diff --git a/CLAUDE.md b/CLAUDE.md index 281e1da..750be2f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -90,6 +90,7 @@ Tracks competitor sellers across ASINs. Fetches storefronts, checks sellability | `src/config.ts` | Env var loading via `Bun.env` | | `src/db/index.ts` | Drizzle Postgres connection (shared pool) | | `src/db/schema.ts` | Drizzle schema for all tables | +| `src/db/persistence.ts` | Product, observation, unified run-item, UPC resolution, and revision persistence | | `src/integrations/keepa.ts` | Keepa API: batch ASIN fetch, UPC lookup, auto rate-limiting | | `src/integrations/sp-api.ts` | SP-API: sellability, pricing+fees, UPC catalog lookup | | `src/integrations/cache.ts` | Redis caching (24h TTL for lead-list; 12h for mid-range) | @@ -112,4 +113,6 @@ Tracks competitor sellers across ASINs. Fetches storefronts, checks sellability - The supplier UPC pipeline must not call LM Studio. - Supplier UPC files resolve UPC/EAN through SP-API catalog lookup first; Keepa UPC lookup is fallback only (no-match or request-failure cases). - Supplier workbook output must keep `Ranked Leads`, `Skipped`, and `Summary` sheets. +- Treat `products.asin` as the canonical normalized product identity; UPC values belong only in identifier and resolution records. +- Store time-varying data in observations or revisions and retain run history rather than overwriting prior analysis. - When changing UPC supplier behavior, cover SP-API UPC parsing, deterministic scoring, and workbook export with `bun test`. diff --git a/README.md b/README.md index 57b0d21..ea8ab6b 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,7 @@ ranked sourcing workbook: 2. Resolves UPCs to ASINs with SP-API catalog lookup first, then falls back to Keepa for no-match/request-failure cases. 3. Enriches resolved ASINs with Keepa demand/competition data and SP-API sellability + FBA fees. 4. Scores products with deterministic BUY/WATCH/SKIP logic; this path does not call LM Studio. -5. Writes a ranked Excel workbook and persists rows into the existing `runs` + `results` tables. +5. Writes a ranked Excel workbook and persists rows through unified runs, UPC resolution, product observation, and scoring-history tables. CLI usage: @@ -244,20 +244,28 @@ Numeric parsing accepts plain numbers as well as formatted values like `$12.50`, 4. **Keepa fetch** — batch the sellable (uncached) ASINs in a single API call (up to 100 per request) 5. **Enrich** — fetch SP-API pricing + FBA/FBM fees for sellable ASINs; combine with Keepa data and spreadsheet data 6. **LLM analysis** — send batches of 5 sellable products to LM Studio for FBA/FBM/SKIP verdict; skipped ASINs get auto-SKIP verdict (confidence 100) and bypass LLM entirely -7. **Output** — print results table to console (includes all ASINs), optionally write CSV/XLSX, and **persist results to a SQLite database**. +7. **Output** — print results table to console (includes all ASINs), optionally write CSV/XLSX, and persist products, observations, run items, and analysis revisions to PostgreSQL. -## Persistent Storage with SQLite +## Persistent Storage -Results from each run are now stored in a SQLite database named `db/results.db` by default. The SQLite implementation details are handled in `src/database.ts`. This allows you to: +PostgreSQL persistence is managed with Drizzle in `src/db/schema.ts` and `src/db/persistence.ts`. ASINs are canonical product identities: all inputs normalize to uppercase 10-character alphanumeric keys before any product reference is stored. -- Revisit past analysis results. -- Query and analyze historical data. -- Track product performance over time. +Core tables: -The database will automatically be created if it doesn't exist. Two tables are created: +- `products`: one canonical row per ASIN with latest descriptive metadata. +- `product_observations`: append-only marketplace, pricing, fee, and sellability snapshots. +- `runs` and `run_items`: unified lifecycle/history for lead, category, supplier UPC, and stalker workflows. +- `analysis_revisions` and `supplier_scores`: append-only analysis results; reanalysis does not overwrite prior decisions. +- `sourcing_inputs`, `upc_resolutions`, and `product_identifiers`: source-row and confirmed identifier data kept separate from catalog products. +- `stalker_run_details`, `stalker_scans`, and `stalker_inventory_items`: seller workflow provenance linked back to products and observations. -- `runs`: Stores metadata about each analysis run (timestamp, input file, output file, and summary counts). -- `results`: Stores detailed analysis results for each product from each run, linked to the `runs` table. +Unresolved or ambiguous supplier UPCs stay on their run item and resolution records; a UPC is never stored as an ASIN. + +Web endpoints use unified identifiers: + +- `GET /api/runs`, `GET /api/runs/:runId`, `GET /api/runs/:runId/items` +- `GET /api/products`, `GET /api/products/:asin` +- `POST /api/run-items/:itemId/reanalyze` ## Output columns diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..24b120f --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,35 @@ +services: + postgres: + image: postgres:16-alpine + restart: unless-stopped + environment: + POSTGRES_DB: asin_check + POSTGRES_USER: asin_check + POSTGRES_PASSWORD: asin_check + ports: + - "5432:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 5s + timeout: 5s + retries: 10 + + redis: + image: redis:7-alpine + restart: unless-stopped + command: ["redis-server", "--appendonly", "yes"] + ports: + - "6379:6379" + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 5s + retries: 10 + +volumes: + postgres_data: + redis_data: diff --git a/drizzle/0000_gorgeous_william_stryker.sql b/drizzle/0000_gorgeous_william_stryker.sql deleted file mode 100644 index 2949d2c..0000000 --- a/drizzle/0000_gorgeous_william_stryker.sql +++ /dev/null @@ -1,217 +0,0 @@ -CREATE TYPE "public"."run_status" AS ENUM('running', 'ok', 'empty', 'failed', 'completed');--> statement-breakpoint -CREATE TYPE "public"."run_type" AS ENUM('lead_analysis', 'category_analysis', 'supplier_upc', 'stalker');--> statement-breakpoint -CREATE TABLE "analysis_results" ( - "id" serial PRIMARY KEY NOT NULL, - "run_id" integer NOT NULL, - "asin" text NOT NULL, - "product_name" text, - "brand" text, - "category" text, - "upc" text, - "unit_cost" real, - "avg_price_90d_sheet" real, - "selling_price_sheet" real, - "fba_net_sheet" real, - "gross_profit_dollar" real, - "gross_profit_pct" real, - "net_profit_sheet" real, - "roi_sheet" real, - "moq" integer, - "moq_cost" real, - "qty_available" integer, - "supplier" text, - "source_url" text, - "asin_link" text, - "promo_coupon_code" text, - "notes" text, - "lead_date" text, - "current_price" real, - "avg_price_90d" real, - "sales_rank" integer, - "rank_avg_90d" integer, - "monthly_sold" integer, - "rank_drops_30d" integer, - "rank_drops_90d" integer, - "seller_count" integer, - "amazon_is_seller" boolean, - "amazon_buybox_share_pct_90d" real, - "fba_fee" real, - "fbm_fee" real, - "referral_percent" real, - "can_sell" text, - "sellability_status" text, - "sellability_reason" text, - "supplier_score" real, - "supplier_profit" real, - "supplier_margin" real, - "supplier_roi" real, - "supplier_reason" text, - "upc_lookup_status" text, - "upc_lookup_reason" text, - "candidate_asins" text, - "verdict" text NOT NULL, - "confidence" real, - "reasoning" text, - "fetched_at" timestamp with time zone NOT NULL -); ---> statement-breakpoint -CREATE TABLE "category_product_results" ( - "id" serial PRIMARY KEY NOT NULL, - "asin" text NOT NULL, - "run_id" integer NOT NULL, - "name" text NOT NULL, - "brand" text, - "category" text, - "unit_cost" real, - "current_price" real, - "avg_price_90d" real, - "avg_price_90d_sheet" real, - "selling_price_sheet" real, - "sales_rank" integer, - "sales_rank_avg_90d" integer, - "seller_count" integer, - "amazon_is_seller" boolean, - "amazon_buybox_share_pct_90d" real, - "monthly_sold" integer, - "rank_drops_30d" integer, - "rank_drops_90d" integer, - "fba_fee" real, - "fbm_fee" real, - "referral_percent" real, - "can_sell" text, - "sellability_status" text, - "sellability_reason" text, - "verdict" text NOT NULL, - "confidence" real NOT NULL, - "reasoning" text, - "fetched_at" timestamp with time zone NOT NULL, - CONSTRAINT "category_product_results_asin_unique" UNIQUE("asin") -); ---> statement-breakpoint -CREATE TABLE "runs" ( - "id" serial PRIMARY KEY NOT NULL, - "type" "run_type" NOT NULL, - "input_file" text, - "output_file" text, - "status" "run_status" DEFAULT 'running' NOT NULL, - "error_message" text, - "total_products" integer, - "fba_count" integer, - "fbm_count" integer, - "skip_count" integer, - "category_id" integer, - "category_label" text, - "top_asins_checked" integer, - "available_asins" integer, - "started_at" timestamp with time zone DEFAULT now() NOT NULL, - "completed_at" timestamp with time zone -); ---> statement-breakpoint -CREATE TABLE "sellers" ( - "seller_id" text PRIMARY KEY NOT NULL, - "seller_name" text, - "rating" real, - "rating_count" integer, - "storefront_asin_total" integer, - "persisted_inventory_sample_count" integer, - "last_updated_at" timestamp with time zone NOT NULL, - "raw_seller_json" text -); ---> statement-breakpoint -CREATE TABLE "stalker_asin_scans" ( - "id" serial PRIMARY KEY NOT NULL, - "run_id" integer NOT NULL, - "source_asin" text NOT NULL, - "title" text, - "offer_count" integer DEFAULT 0 NOT NULL, - "candidate_seller_count" integer DEFAULT 0 NOT NULL, - "matched_seller_count" integer DEFAULT 0 NOT NULL, - "fetched_at" timestamp with time zone NOT NULL, - "raw_product_json" text, - CONSTRAINT "uq_stalker_scans_run_asin" UNIQUE("run_id","source_asin") -); ---> statement-breakpoint -CREATE TABLE "stalker_asin_sellers" ( - "id" serial PRIMARY KEY NOT NULL, - "scan_id" integer NOT NULL, - "seller_id" text NOT NULL, - "offer_price" real, - "condition" text, - "is_fba" boolean, - "stock" integer, - "seller_rating" real, - "seller_rating_count" integer, - "raw_offer_json" text, - CONSTRAINT "uq_stalker_asin_sellers_scan_seller" UNIQUE("scan_id","seller_id") -); ---> statement-breakpoint -CREATE TABLE "stalker_runs" ( - "id" serial PRIMARY KEY NOT NULL, - "input_file" text NOT NULL, - "started_at" timestamp with time zone NOT NULL, - "completed_at" timestamp with time zone, - "requested_asins" integer DEFAULT 0 NOT NULL, - "skipped_asins" integer DEFAULT 0 NOT NULL, - "scanned_asins" integer DEFAULT 0 NOT NULL, - "source_asins_with_matches" integer DEFAULT 0 NOT NULL, - "candidate_sellers" integer DEFAULT 0 NOT NULL, - "qualifying_sellers" integer DEFAULT 0 NOT NULL, - "matched_sellers" integer DEFAULT 0 NOT NULL, - "seller_metadata_requests" integer DEFAULT 0 NOT NULL, - "seller_storefront_requests" integer DEFAULT 0 NOT NULL, - "inventory_sellability_checked_asins" integer DEFAULT 0 NOT NULL, - "inventory_sellability_available_asins" integer DEFAULT 0 NOT NULL, - "inventory_sellability_excluded_asins" integer DEFAULT 0 NOT NULL, - "persisted_inventory_asins" integer DEFAULT 0 NOT NULL, - "status" text NOT NULL, - "error_message" text -); ---> statement-breakpoint -CREATE TABLE "stalker_seller_inventory" ( - "id" serial PRIMARY KEY NOT NULL, - "run_id" integer NOT NULL, - "seller_id" text NOT NULL, - "asin" text NOT NULL, - "can_sell" boolean, - "sellability_status" text, - "sellability_reason" text, - "product_title" text, - "brand" text, - "category_tree" text, - "current_price" real, - "avg_price_90d" real, - "sales_rank" integer, - "monthly_sold" integer, - "seller_count" integer, - "amazon_is_seller" boolean, - "raw_product_json" text, - "last_seen_at" timestamp with time zone NOT NULL, - "raw_inventory_json" text, - CONSTRAINT "uq_stalker_inventory_run_seller_asin" UNIQUE("run_id","seller_id","asin") -); ---> statement-breakpoint -ALTER TABLE "analysis_results" ADD CONSTRAINT "analysis_results_run_id_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."runs"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint -ALTER TABLE "category_product_results" ADD CONSTRAINT "category_product_results_run_id_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."runs"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint -ALTER TABLE "stalker_asin_scans" ADD CONSTRAINT "stalker_asin_scans_run_id_stalker_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."stalker_runs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint -ALTER TABLE "stalker_asin_sellers" ADD CONSTRAINT "stalker_asin_sellers_scan_id_stalker_asin_scans_id_fk" FOREIGN KEY ("scan_id") REFERENCES "public"."stalker_asin_scans"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint -ALTER TABLE "stalker_asin_sellers" ADD CONSTRAINT "stalker_asin_sellers_seller_id_sellers_seller_id_fk" FOREIGN KEY ("seller_id") REFERENCES "public"."sellers"("seller_id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint -ALTER TABLE "stalker_seller_inventory" ADD CONSTRAINT "stalker_seller_inventory_run_id_stalker_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."stalker_runs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint -ALTER TABLE "stalker_seller_inventory" ADD CONSTRAINT "stalker_seller_inventory_seller_id_sellers_seller_id_fk" FOREIGN KEY ("seller_id") REFERENCES "public"."sellers"("seller_id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint -CREATE INDEX "idx_analysis_results_run_id" ON "analysis_results" USING btree ("run_id");--> statement-breakpoint -CREATE INDEX "idx_analysis_results_asin" ON "analysis_results" USING btree ("asin");--> statement-breakpoint -CREATE INDEX "idx_analysis_results_verdict" ON "analysis_results" USING btree ("verdict");--> statement-breakpoint -CREATE INDEX "idx_analysis_results_sellability_status" ON "analysis_results" USING btree ("sellability_status");--> statement-breakpoint -CREATE INDEX "idx_analysis_results_fetched_at" ON "analysis_results" USING btree ("fetched_at");--> statement-breakpoint -CREATE INDEX "idx_category_results_run_id" ON "category_product_results" USING btree ("run_id");--> statement-breakpoint -CREATE INDEX "idx_category_results_verdict" ON "category_product_results" USING btree ("verdict");--> statement-breakpoint -CREATE INDEX "idx_category_results_sellability_status" ON "category_product_results" USING btree ("sellability_status");--> statement-breakpoint -CREATE INDEX "idx_category_results_fetched_at" ON "category_product_results" USING btree ("fetched_at");--> statement-breakpoint -CREATE INDEX "idx_runs_started_at" ON "runs" USING btree ("started_at");--> statement-breakpoint -CREATE INDEX "idx_runs_type" ON "runs" USING btree ("type");--> statement-breakpoint -CREATE INDEX "idx_runs_status" ON "runs" USING btree ("status");--> statement-breakpoint -CREATE INDEX "idx_stalker_scans_run_id" ON "stalker_asin_scans" USING btree ("run_id");--> statement-breakpoint -CREATE INDEX "idx_stalker_scans_source_asin" ON "stalker_asin_scans" USING btree ("source_asin");--> statement-breakpoint -CREATE INDEX "idx_stalker_runs_started_at" ON "stalker_runs" USING btree ("started_at");--> statement-breakpoint -CREATE INDEX "idx_stalker_inventory_seller_id" ON "stalker_seller_inventory" USING btree ("seller_id");--> statement-breakpoint -CREATE INDEX "idx_stalker_inventory_asin" ON "stalker_seller_inventory" USING btree ("asin");--> statement-breakpoint -CREATE INDEX "idx_stalker_inventory_product_title" ON "stalker_seller_inventory" USING btree ("product_title"); \ No newline at end of file diff --git a/drizzle/meta/0000_snapshot.json b/drizzle/meta/0000_snapshot.json deleted file mode 100644 index 09b0018..0000000 --- a/drizzle/meta/0000_snapshot.json +++ /dev/null @@ -1,1545 +0,0 @@ -{ - "id": "5b8f629e-e65a-40a4-b261-4a460b4c23fb", - "prevId": "00000000-0000-0000-0000-000000000000", - "version": "7", - "dialect": "postgresql", - "tables": { - "public.analysis_results": { - "name": "analysis_results", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "run_id": { - "name": "run_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "asin": { - "name": "asin", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "product_name": { - "name": "product_name", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "brand": { - "name": "brand", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "category": { - "name": "category", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "upc": { - "name": "upc", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "unit_cost": { - "name": "unit_cost", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "avg_price_90d_sheet": { - "name": "avg_price_90d_sheet", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "selling_price_sheet": { - "name": "selling_price_sheet", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "fba_net_sheet": { - "name": "fba_net_sheet", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "gross_profit_dollar": { - "name": "gross_profit_dollar", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "gross_profit_pct": { - "name": "gross_profit_pct", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "net_profit_sheet": { - "name": "net_profit_sheet", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "roi_sheet": { - "name": "roi_sheet", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "moq": { - "name": "moq", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "moq_cost": { - "name": "moq_cost", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "qty_available": { - "name": "qty_available", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "supplier": { - "name": "supplier", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "source_url": { - "name": "source_url", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "asin_link": { - "name": "asin_link", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "promo_coupon_code": { - "name": "promo_coupon_code", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "notes": { - "name": "notes", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "lead_date": { - "name": "lead_date", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "current_price": { - "name": "current_price", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "avg_price_90d": { - "name": "avg_price_90d", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "sales_rank": { - "name": "sales_rank", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "rank_avg_90d": { - "name": "rank_avg_90d", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "monthly_sold": { - "name": "monthly_sold", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "rank_drops_30d": { - "name": "rank_drops_30d", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "rank_drops_90d": { - "name": "rank_drops_90d", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "seller_count": { - "name": "seller_count", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "amazon_is_seller": { - "name": "amazon_is_seller", - "type": "boolean", - "primaryKey": false, - "notNull": false - }, - "amazon_buybox_share_pct_90d": { - "name": "amazon_buybox_share_pct_90d", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "fba_fee": { - "name": "fba_fee", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "fbm_fee": { - "name": "fbm_fee", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "referral_percent": { - "name": "referral_percent", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "can_sell": { - "name": "can_sell", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "sellability_status": { - "name": "sellability_status", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "sellability_reason": { - "name": "sellability_reason", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "supplier_score": { - "name": "supplier_score", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "supplier_profit": { - "name": "supplier_profit", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "supplier_margin": { - "name": "supplier_margin", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "supplier_roi": { - "name": "supplier_roi", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "supplier_reason": { - "name": "supplier_reason", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "upc_lookup_status": { - "name": "upc_lookup_status", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "upc_lookup_reason": { - "name": "upc_lookup_reason", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "candidate_asins": { - "name": "candidate_asins", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "verdict": { - "name": "verdict", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "confidence": { - "name": "confidence", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "reasoning": { - "name": "reasoning", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "fetched_at": { - "name": "fetched_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": true - } - }, - "indexes": { - "idx_analysis_results_run_id": { - "name": "idx_analysis_results_run_id", - "columns": [ - { - "expression": "run_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "idx_analysis_results_asin": { - "name": "idx_analysis_results_asin", - "columns": [ - { - "expression": "asin", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "idx_analysis_results_verdict": { - "name": "idx_analysis_results_verdict", - "columns": [ - { - "expression": "verdict", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "idx_analysis_results_sellability_status": { - "name": "idx_analysis_results_sellability_status", - "columns": [ - { - "expression": "sellability_status", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "idx_analysis_results_fetched_at": { - "name": "idx_analysis_results_fetched_at", - "columns": [ - { - "expression": "fetched_at", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": { - "analysis_results_run_id_runs_id_fk": { - "name": "analysis_results_run_id_runs_id_fk", - "tableFrom": "analysis_results", - "tableTo": "runs", - "columnsFrom": [ - "run_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.category_product_results": { - "name": "category_product_results", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "asin": { - "name": "asin", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "run_id": { - "name": "run_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "brand": { - "name": "brand", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "category": { - "name": "category", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "unit_cost": { - "name": "unit_cost", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "current_price": { - "name": "current_price", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "avg_price_90d": { - "name": "avg_price_90d", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "avg_price_90d_sheet": { - "name": "avg_price_90d_sheet", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "selling_price_sheet": { - "name": "selling_price_sheet", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "sales_rank": { - "name": "sales_rank", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "sales_rank_avg_90d": { - "name": "sales_rank_avg_90d", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "seller_count": { - "name": "seller_count", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "amazon_is_seller": { - "name": "amazon_is_seller", - "type": "boolean", - "primaryKey": false, - "notNull": false - }, - "amazon_buybox_share_pct_90d": { - "name": "amazon_buybox_share_pct_90d", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "monthly_sold": { - "name": "monthly_sold", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "rank_drops_30d": { - "name": "rank_drops_30d", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "rank_drops_90d": { - "name": "rank_drops_90d", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "fba_fee": { - "name": "fba_fee", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "fbm_fee": { - "name": "fbm_fee", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "referral_percent": { - "name": "referral_percent", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "can_sell": { - "name": "can_sell", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "sellability_status": { - "name": "sellability_status", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "sellability_reason": { - "name": "sellability_reason", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "verdict": { - "name": "verdict", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "confidence": { - "name": "confidence", - "type": "real", - "primaryKey": false, - "notNull": true - }, - "reasoning": { - "name": "reasoning", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "fetched_at": { - "name": "fetched_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": true - } - }, - "indexes": { - "idx_category_results_run_id": { - "name": "idx_category_results_run_id", - "columns": [ - { - "expression": "run_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "idx_category_results_verdict": { - "name": "idx_category_results_verdict", - "columns": [ - { - "expression": "verdict", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "idx_category_results_sellability_status": { - "name": "idx_category_results_sellability_status", - "columns": [ - { - "expression": "sellability_status", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "idx_category_results_fetched_at": { - "name": "idx_category_results_fetched_at", - "columns": [ - { - "expression": "fetched_at", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": { - "category_product_results_run_id_runs_id_fk": { - "name": "category_product_results_run_id_runs_id_fk", - "tableFrom": "category_product_results", - "tableTo": "runs", - "columnsFrom": [ - "run_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": { - "category_product_results_asin_unique": { - "name": "category_product_results_asin_unique", - "nullsNotDistinct": false, - "columns": [ - "asin" - ] - } - }, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.runs": { - "name": "runs", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "type": { - "name": "type", - "type": "run_type", - "typeSchema": "public", - "primaryKey": false, - "notNull": true - }, - "input_file": { - "name": "input_file", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "output_file": { - "name": "output_file", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "status": { - "name": "status", - "type": "run_status", - "typeSchema": "public", - "primaryKey": false, - "notNull": true, - "default": "'running'" - }, - "error_message": { - "name": "error_message", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "total_products": { - "name": "total_products", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "fba_count": { - "name": "fba_count", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "fbm_count": { - "name": "fbm_count", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "skip_count": { - "name": "skip_count", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "category_id": { - "name": "category_id", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "category_label": { - "name": "category_label", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "top_asins_checked": { - "name": "top_asins_checked", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "available_asins": { - "name": "available_asins", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "started_at": { - "name": "started_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": true, - "default": "now()" - }, - "completed_at": { - "name": "completed_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": false - } - }, - "indexes": { - "idx_runs_started_at": { - "name": "idx_runs_started_at", - "columns": [ - { - "expression": "started_at", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "idx_runs_type": { - "name": "idx_runs_type", - "columns": [ - { - "expression": "type", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "idx_runs_status": { - "name": "idx_runs_status", - "columns": [ - { - "expression": "status", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.sellers": { - "name": "sellers", - "schema": "", - "columns": { - "seller_id": { - "name": "seller_id", - "type": "text", - "primaryKey": true, - "notNull": true - }, - "seller_name": { - "name": "seller_name", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "rating": { - "name": "rating", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "rating_count": { - "name": "rating_count", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "storefront_asin_total": { - "name": "storefront_asin_total", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "persisted_inventory_sample_count": { - "name": "persisted_inventory_sample_count", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "last_updated_at": { - "name": "last_updated_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": true - }, - "raw_seller_json": { - "name": "raw_seller_json", - "type": "text", - "primaryKey": false, - "notNull": false - } - }, - "indexes": {}, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.stalker_asin_scans": { - "name": "stalker_asin_scans", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "run_id": { - "name": "run_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "source_asin": { - "name": "source_asin", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "title": { - "name": "title", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "offer_count": { - "name": "offer_count", - "type": "integer", - "primaryKey": false, - "notNull": true, - "default": 0 - }, - "candidate_seller_count": { - "name": "candidate_seller_count", - "type": "integer", - "primaryKey": false, - "notNull": true, - "default": 0 - }, - "matched_seller_count": { - "name": "matched_seller_count", - "type": "integer", - "primaryKey": false, - "notNull": true, - "default": 0 - }, - "fetched_at": { - "name": "fetched_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": true - }, - "raw_product_json": { - "name": "raw_product_json", - "type": "text", - "primaryKey": false, - "notNull": false - } - }, - "indexes": { - "idx_stalker_scans_run_id": { - "name": "idx_stalker_scans_run_id", - "columns": [ - { - "expression": "run_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "idx_stalker_scans_source_asin": { - "name": "idx_stalker_scans_source_asin", - "columns": [ - { - "expression": "source_asin", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": { - "stalker_asin_scans_run_id_stalker_runs_id_fk": { - "name": "stalker_asin_scans_run_id_stalker_runs_id_fk", - "tableFrom": "stalker_asin_scans", - "tableTo": "stalker_runs", - "columnsFrom": [ - "run_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": { - "uq_stalker_scans_run_asin": { - "name": "uq_stalker_scans_run_asin", - "nullsNotDistinct": false, - "columns": [ - "run_id", - "source_asin" - ] - } - }, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.stalker_asin_sellers": { - "name": "stalker_asin_sellers", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "scan_id": { - "name": "scan_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "seller_id": { - "name": "seller_id", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "offer_price": { - "name": "offer_price", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "condition": { - "name": "condition", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "is_fba": { - "name": "is_fba", - "type": "boolean", - "primaryKey": false, - "notNull": false - }, - "stock": { - "name": "stock", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "seller_rating": { - "name": "seller_rating", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "seller_rating_count": { - "name": "seller_rating_count", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "raw_offer_json": { - "name": "raw_offer_json", - "type": "text", - "primaryKey": false, - "notNull": false - } - }, - "indexes": {}, - "foreignKeys": { - "stalker_asin_sellers_scan_id_stalker_asin_scans_id_fk": { - "name": "stalker_asin_sellers_scan_id_stalker_asin_scans_id_fk", - "tableFrom": "stalker_asin_sellers", - "tableTo": "stalker_asin_scans", - "columnsFrom": [ - "scan_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - }, - "stalker_asin_sellers_seller_id_sellers_seller_id_fk": { - "name": "stalker_asin_sellers_seller_id_sellers_seller_id_fk", - "tableFrom": "stalker_asin_sellers", - "tableTo": "sellers", - "columnsFrom": [ - "seller_id" - ], - "columnsTo": [ - "seller_id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": { - "uq_stalker_asin_sellers_scan_seller": { - "name": "uq_stalker_asin_sellers_scan_seller", - "nullsNotDistinct": false, - "columns": [ - "scan_id", - "seller_id" - ] - } - }, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.stalker_runs": { - "name": "stalker_runs", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "input_file": { - "name": "input_file", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "started_at": { - "name": "started_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": true - }, - "completed_at": { - "name": "completed_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": false - }, - "requested_asins": { - "name": "requested_asins", - "type": "integer", - "primaryKey": false, - "notNull": true, - "default": 0 - }, - "skipped_asins": { - "name": "skipped_asins", - "type": "integer", - "primaryKey": false, - "notNull": true, - "default": 0 - }, - "scanned_asins": { - "name": "scanned_asins", - "type": "integer", - "primaryKey": false, - "notNull": true, - "default": 0 - }, - "source_asins_with_matches": { - "name": "source_asins_with_matches", - "type": "integer", - "primaryKey": false, - "notNull": true, - "default": 0 - }, - "candidate_sellers": { - "name": "candidate_sellers", - "type": "integer", - "primaryKey": false, - "notNull": true, - "default": 0 - }, - "qualifying_sellers": { - "name": "qualifying_sellers", - "type": "integer", - "primaryKey": false, - "notNull": true, - "default": 0 - }, - "matched_sellers": { - "name": "matched_sellers", - "type": "integer", - "primaryKey": false, - "notNull": true, - "default": 0 - }, - "seller_metadata_requests": { - "name": "seller_metadata_requests", - "type": "integer", - "primaryKey": false, - "notNull": true, - "default": 0 - }, - "seller_storefront_requests": { - "name": "seller_storefront_requests", - "type": "integer", - "primaryKey": false, - "notNull": true, - "default": 0 - }, - "inventory_sellability_checked_asins": { - "name": "inventory_sellability_checked_asins", - "type": "integer", - "primaryKey": false, - "notNull": true, - "default": 0 - }, - "inventory_sellability_available_asins": { - "name": "inventory_sellability_available_asins", - "type": "integer", - "primaryKey": false, - "notNull": true, - "default": 0 - }, - "inventory_sellability_excluded_asins": { - "name": "inventory_sellability_excluded_asins", - "type": "integer", - "primaryKey": false, - "notNull": true, - "default": 0 - }, - "persisted_inventory_asins": { - "name": "persisted_inventory_asins", - "type": "integer", - "primaryKey": false, - "notNull": true, - "default": 0 - }, - "status": { - "name": "status", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "error_message": { - "name": "error_message", - "type": "text", - "primaryKey": false, - "notNull": false - } - }, - "indexes": { - "idx_stalker_runs_started_at": { - "name": "idx_stalker_runs_started_at", - "columns": [ - { - "expression": "started_at", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.stalker_seller_inventory": { - "name": "stalker_seller_inventory", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "run_id": { - "name": "run_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "seller_id": { - "name": "seller_id", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "asin": { - "name": "asin", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "can_sell": { - "name": "can_sell", - "type": "boolean", - "primaryKey": false, - "notNull": false - }, - "sellability_status": { - "name": "sellability_status", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "sellability_reason": { - "name": "sellability_reason", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "product_title": { - "name": "product_title", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "brand": { - "name": "brand", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "category_tree": { - "name": "category_tree", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "current_price": { - "name": "current_price", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "avg_price_90d": { - "name": "avg_price_90d", - "type": "real", - "primaryKey": false, - "notNull": false - }, - "sales_rank": { - "name": "sales_rank", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "monthly_sold": { - "name": "monthly_sold", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "seller_count": { - "name": "seller_count", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "amazon_is_seller": { - "name": "amazon_is_seller", - "type": "boolean", - "primaryKey": false, - "notNull": false - }, - "raw_product_json": { - "name": "raw_product_json", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "last_seen_at": { - "name": "last_seen_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": true - }, - "raw_inventory_json": { - "name": "raw_inventory_json", - "type": "text", - "primaryKey": false, - "notNull": false - } - }, - "indexes": { - "idx_stalker_inventory_seller_id": { - "name": "idx_stalker_inventory_seller_id", - "columns": [ - { - "expression": "seller_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "idx_stalker_inventory_asin": { - "name": "idx_stalker_inventory_asin", - "columns": [ - { - "expression": "asin", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "idx_stalker_inventory_product_title": { - "name": "idx_stalker_inventory_product_title", - "columns": [ - { - "expression": "product_title", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": { - "stalker_seller_inventory_run_id_stalker_runs_id_fk": { - "name": "stalker_seller_inventory_run_id_stalker_runs_id_fk", - "tableFrom": "stalker_seller_inventory", - "tableTo": "stalker_runs", - "columnsFrom": [ - "run_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - }, - "stalker_seller_inventory_seller_id_sellers_seller_id_fk": { - "name": "stalker_seller_inventory_seller_id_sellers_seller_id_fk", - "tableFrom": "stalker_seller_inventory", - "tableTo": "sellers", - "columnsFrom": [ - "seller_id" - ], - "columnsTo": [ - "seller_id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": { - "uq_stalker_inventory_run_seller_asin": { - "name": "uq_stalker_inventory_run_seller_asin", - "nullsNotDistinct": false, - "columns": [ - "run_id", - "seller_id", - "asin" - ] - } - }, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - } - }, - "enums": { - "public.run_status": { - "name": "run_status", - "schema": "public", - "values": [ - "running", - "ok", - "empty", - "failed", - "completed" - ] - }, - "public.run_type": { - "name": "run_type", - "schema": "public", - "values": [ - "lead_analysis", - "category_analysis", - "supplier_upc", - "stalker" - ] - } - }, - "schemas": {}, - "sequences": {}, - "roles": {}, - "policies": {}, - "views": {}, - "_meta": { - "columns": {}, - "schemas": {}, - "tables": {} - } -} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json deleted file mode 100644 index c1afd38..0000000 --- a/drizzle/meta/_journal.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "version": "7", - "dialect": "postgresql", - "entries": [ - { - "idx": 0, - "version": "7", - "when": 1779683900467, - "tag": "0000_gorgeous_william_stryker", - "breakpoints": true - } - ] -} \ No newline at end of file diff --git a/src/analysis-pipeline.test.ts b/src/analysis-pipeline.test.ts new file mode 100644 index 0000000..e0e446f --- /dev/null +++ b/src/analysis-pipeline.test.ts @@ -0,0 +1,114 @@ +import { beforeEach, expect, mock, test } from "bun:test"; +import { processProductChunk } from "./analysis-pipeline.ts"; +import type { ProductRecord } from "./types.ts"; + +const fetchKeepaDataBatchMock = mock(async (asins: string[]) => { + return new Map( + asins.map((asin) => [ + asin, + { + currentPrice: 20, + avgPrice90: 18, + minPrice90: null, + maxPrice90: null, + salesRank: 100, + salesRankAvg90: null, + salesRankDrops30: null, + salesRankDrops90: null, + sellerCount: 3, + amazonIsSeller: false, + amazonBuyboxSharePct90d: null, + buyBoxSeller: null, + buyBoxPrice: null, + monthlySold: 50, + categoryTree: [], + }, + ]), + ); +}); + +const fetchSellabilityBatchMock = mock(async (asins: string[]) => { + return new Map( + asins.map((asin) => [ + asin, + asin === "B000000002" + ? { + canSell: false, + sellabilityStatus: "restricted" as const, + sellabilityReason: "Approval required", + } + : { + canSell: true, + sellabilityStatus: "available" as const, + sellabilityReason: "Available", + }, + ]), + ); +}); + +const fetchSpApiPricingAndFeesMock = mock(async () => ({ + fbaFee: 4, + fbmFee: 2, + referralFeePercent: 15, + estimatedSalePrice: 20, + canSell: true, + sellabilityStatus: "available" as const, + sellabilityReason: "Available", +})); + +const analyzeProductsMock = mock(async (products: any[]) => + products.map((product) => ({ + asin: product.record.asin, + verdict: "FBA" as const, + confidence: 95, + reasoning: "Analyzed", + })), +); + +const getCacheMock = mock(async () => null); +const setCacheMock = mock(async () => undefined); + +beforeEach(() => { + fetchKeepaDataBatchMock.mockClear(); + fetchSellabilityBatchMock.mockClear(); + fetchSpApiPricingAndFeesMock.mockClear(); + analyzeProductsMock.mockClear(); + getCacheMock.mockClear(); + setCacheMock.mockClear(); +}); + +test("lead analysis retains restricted input rows as SKIP without LLM analysis", async () => { + const products: ProductRecord[] = [ + { asin: "B000000001", name: "Available", unitCost: 5 }, + { asin: "B000000002", name: "Restricted", unitCost: 6 }, + ]; + + const results = await processProductChunk(products, { + llmBatchDelayMs: 0, + llmRetryDelayMs: 0, + dependencies: { + fetchKeepaDataBatch: fetchKeepaDataBatchMock, + fetchSellabilityBatch: fetchSellabilityBatchMock, + fetchSpApiPricingAndFees: fetchSpApiPricingAndFeesMock, + analyzeProducts: analyzeProductsMock, + getCache: getCacheMock, + setCache: setCacheMock, + }, + }); + + expect(results).toHaveLength(2); + expect(results.map((result) => result.product.record.asin)).toEqual([ + "B000000001", + "B000000002", + ]); + expect(results.find((result) => result.product.record.asin === "B000000002")?.verdict) + .toEqual({ + asin: "B000000002", + verdict: "SKIP", + confidence: 100, + reasoning: "Approval required", + }); + expect(fetchKeepaDataBatchMock.mock.calls[0]?.[0]).toEqual(["B000000001"]); + expect(fetchSpApiPricingAndFeesMock.mock.calls).toHaveLength(1); + expect(analyzeProductsMock.mock.calls[0]?.[0]).toHaveLength(1); +}); diff --git a/src/analysis-pipeline.ts b/src/analysis-pipeline.ts index fdb182a..7a4fec8 100644 --- a/src/analysis-pipeline.ts +++ b/src/analysis-pipeline.ts @@ -16,6 +16,15 @@ export const DEFAULT_PRICING_CONCURRENCY = 5; export type SellabilityFilter = "available" | "all"; +type AnalysisPipelineDependencies = { + fetchKeepaDataBatch: typeof fetchKeepaDataBatch; + fetchSellabilityBatch: typeof fetchSellabilityBatch; + fetchSpApiPricingAndFees: typeof fetchSpApiPricingAndFees; + getCache: typeof getCache; + setCache: typeof setCache; + analyzeProducts: typeof analyzeProducts; +}; + export type AnalysisPipelineOptions = { llmBatchSize?: number; pricingConcurrency?: number; @@ -23,6 +32,7 @@ export type AnalysisPipelineOptions = { llmRetryDelayMs?: number; sellability?: SellabilityFilter; useClaude?: boolean; + dependencies?: Partial; }; export function chunkArray(items: T[], chunkSize: number): T[][] { @@ -62,23 +72,33 @@ export async function processProductChunk( const llmRetryDelayMs = Math.max(0, options.llmRetryDelayMs ?? 10_000); const sellabilityFilter = options.sellability ?? "available"; const useClaude = options.useClaude === true; + const dependencies: AnalysisPipelineDependencies = { + fetchKeepaDataBatch, + fetchSellabilityBatch, + fetchSpApiPricingAndFees, + getCache, + setCache, + analyzeProducts, + ...options.dependencies, + }; console.log(`\nChecking cache for ${products.length} products...`); const cached = new Map(); - const excludedCachedAsins = new Set(); + const excludedCached = new Map(); const uncachedProducts: ProductRecord[] = []; for (const p of products) { - const hit = await getCache(p.asin); + const hit = await dependencies.getCache(p.asin); if (hit) { + const currentSourceProduct = { ...hit, record: p }; if ( sellabilityFilter === "all" || hit.spApi.sellabilityStatus === "available" ) { console.log(` [cache hit] ${p.asin}`); - cached.set(p.asin, hit); + cached.set(p.asin, currentSourceProduct); } else { - excludedCachedAsins.add(p.asin); + excludedCached.set(p.asin, currentSourceProduct); console.log( ` [exclude cached] ${p.asin} - status=${hit.spApi.sellabilityStatus}`, ); @@ -89,7 +109,7 @@ export async function processProductChunk( } console.log( - `${cached.size} cached available, ${excludedCachedAsins.size} cached excluded, ${uncachedProducts.length} to fetch`, + `${cached.size} cached available, ${excludedCached.size} cached excluded, ${uncachedProducts.length} to fetch`, ); const sellabilityMap = new Map(); @@ -100,7 +120,7 @@ export async function processProductChunk( console.log( `\nChecking sellability for ${uncachedProducts.length} ASINs...`, ); - const sellResults = await fetchSellabilityBatch( + const sellResults = await dependencies.fetchSellabilityBatch( uncachedProducts.map((p) => p.asin), ); @@ -143,7 +163,7 @@ export async function processProductChunk( if (availableProducts.length > 0) { console.log(`\nFetching ${availableProducts.length} ASINs from Keepa...`); try { - keepaResults = await fetchKeepaDataBatch( + keepaResults = await dependencies.fetchKeepaDataBatch( availableProducts.map((p) => p.asin), ); } catch (err) { @@ -168,7 +188,10 @@ export async function processProductChunk( sellabilityStatus: "unknown" as const, sellabilityReason: "Sellability check returned no result", }; - const spApi = await fetchSpApiPricingAndFees(p.asin, sellability); + const spApi = await dependencies.fetchSpApiPricingAndFees( + p.asin, + sellability, + ); const keepa = keepaResults.get(p.asin); if (keepa?.currentPrice && spApi.estimatedSalePrice === 0) { @@ -196,17 +219,33 @@ export async function processProductChunk( const availableAsins = new Set(availableProducts.map((ap) => ap.asin)); for (const p of products) { - if (excludedCachedAsins.has(p.asin)) { + const excludedCachedProduct = excludedCached.get(p.asin); + if (excludedCachedProduct) { + enriched.push({ ...excludedCachedProduct, record: p }); continue; } const cachedProduct = cached.get(p.asin); if (cachedProduct) { - enriched.push(cachedProduct); + enriched.push({ ...cachedProduct, record: p }); continue; } if (!availableAsins.has(p.asin)) { + const sellability = sellabilityMap.get(p.asin); + if (sellability) { + enriched.push({ + record: p, + keepa: null, + spApi: { + ...unknownSpApiData( + sellability.sellabilityReason ?? "Product is not available", + ), + ...sellability, + }, + fetchedAt: new Date().toISOString(), + }); + } continue; } @@ -221,19 +260,41 @@ export async function processProductChunk( fetchedAt: new Date().toISOString(), }; - await setCache(p.asin, product); + await dependencies.setCache(p.asin, product); enriched.push(product); } + const resultsByProduct = new Map(); + const llmProducts: EnrichedProduct[] = []; + for (const product of enriched) { + if ( + sellabilityFilter !== "all" && + product.spApi.sellabilityStatus !== "available" + ) { + resultsByProduct.set(product, { + product, + verdict: { + asin: product.record.asin, + verdict: "SKIP", + confidence: 100, + reasoning: + product.spApi.sellabilityReason ?? + `Sellability status: ${product.spApi.sellabilityStatus}`, + }, + }); + } else { + llmProducts.push(product); + } + } + console.log( - `\nAnalyzing ${enriched.length} products via LLM (batch size: ${llmBatchSize})...\n`, + `\nAnalyzing ${llmProducts.length} products via LLM (batch size: ${llmBatchSize})...\n`, ); - const results: AnalysisResult[] = []; - for (let i = 0; i < enriched.length; i += llmBatchSize) { - const batch = enriched.slice(i, i + llmBatchSize); + for (let i = 0; i < llmProducts.length; i += llmBatchSize) { + const batch = llmProducts.slice(i, i + llmBatchSize); const batchNum = Math.floor(i / llmBatchSize) + 1; - const totalBatches = Math.ceil(enriched.length / llmBatchSize); + const totalBatches = Math.ceil(llmProducts.length / llmBatchSize); console.log(` LLM batch ${batchNum}/${totalBatches}...`); if (i > 0 && llmBatchDelayMs > 0) { @@ -242,7 +303,7 @@ export async function processProductChunk( let verdicts; try { - verdicts = await analyzeProducts(batch, { + verdicts = await dependencies.analyzeProducts(batch, { ignoreSellability: sellabilityFilter === "all", useClaude, }); @@ -251,7 +312,7 @@ export async function processProductChunk( await wait(llmRetryDelayMs); } try { - verdicts = await analyzeProducts(batch, { + verdicts = await dependencies.analyzeProducts(batch, { ignoreSellability: sellabilityFilter === "all", useClaude, }); @@ -264,7 +325,7 @@ export async function processProductChunk( const enrichedProduct = batch[j]; if (!enrichedProduct) continue; - results.push({ + resultsByProduct.set(enrichedProduct, { product: enrichedProduct, verdict: verdicts?.[j] ?? { asin: enrichedProduct.record.asin, @@ -276,5 +337,7 @@ export async function processProductChunk( } } - return results; + return enriched + .map((product) => resultsByProduct.get(product)) + .filter((result): result is AnalysisResult => result !== undefined); } diff --git a/src/asin.test.ts b/src/asin.test.ts new file mode 100644 index 0000000..5a5ee91 --- /dev/null +++ b/src/asin.test.ts @@ -0,0 +1,13 @@ +import { expect, test } from "bun:test"; +import { normalizeAsin, requireAsin } from "./asin.ts"; + +test("normalizes any valid ten-character ASIN including ISBN-style values", () => { + expect(normalizeAsin(" b07sn9bhvv ")).toBe("B07SN9BHVV"); + expect(normalizeAsin("0306406152")).toBe("0306406152"); +}); + +test("rejects values that cannot be canonical product ASIN keys", () => { + expect(normalizeAsin("short")).toBeNull(); + expect(normalizeAsin("B07SN9BHV!")).toBeNull(); + expect(() => requireAsin("012345678901")).toThrow("Invalid ASIN"); +}); diff --git a/src/asin.ts b/src/asin.ts new file mode 100644 index 0000000..653aeab --- /dev/null +++ b/src/asin.ts @@ -0,0 +1,14 @@ +export const ASIN_PATTERN = /^[A-Z0-9]{10}$/; + +export function normalizeAsin(value: unknown): string | null { + const asin = String(value ?? "").trim().toUpperCase(); + return ASIN_PATTERN.test(asin) ? asin : null; +} + +export function requireAsin(value: unknown): string { + const asin = normalizeAsin(value); + if (!asin) { + throw new Error(`Invalid ASIN: "${String(value ?? "").trim()}"`); + } + return asin; +} diff --git a/src/categories/bestsellers-by-category.ts b/src/categories/bestsellers-by-category.ts index 78ef912..0fd63e3 100644 --- a/src/categories/bestsellers-by-category.ts +++ b/src/categories/bestsellers-by-category.ts @@ -1,8 +1,11 @@ -import { existsSync, mkdirSync, readFileSync } from "node:fs"; -import path from "node:path"; -import { db } from "../db/index.ts"; -import { runs, categoryProductResults } from "../db/schema.ts"; -import { eq, sql } from "drizzle-orm"; +import { existsSync, mkdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { normalizeAsin } from "../asin.ts"; +import { + createCategoryRun, + persistLlmResults, + updateCategoryRun, +} from "../db/persistence.ts"; import { config } from "../config.ts"; import { analyzeProducts } from "../integrations/llm.ts"; import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "../integrations/sp-api.ts"; @@ -140,31 +143,12 @@ function printUsageAndExit(message: string): never { process.exit(1); } -export async function insertCategoryRunSummary( - summary: CategoryRunSummary, - runTimestamp: string, -): Promise { - const [row] = await db - .insert(runs) - .values({ - type: "category_analysis", - status: (summary.status as typeof runs.$inferInsert.status) ?? "running", - categoryId: summary.categoryId, - categoryLabel: summary.categoryLabel, - topAsinsChecked: summary.topAsinsChecked, - availableAsins: summary.availableAsins, - totalProducts: summary.topAsinsChecked, - fbaCount: summary.fba, - fbmCount: summary.fbm, - skipCount: summary.skip, - errorMessage: summary.error || null, - startedAt: new Date(runTimestamp), - }) - .returning({ id: runs.id }); - - if (!row) throw new Error("Failed to insert category run."); - return row.id; -} +export async function insertCategoryRunSummary( + summary: CategoryRunSummary, + runTimestamp: string, +): Promise { + return createCategoryRun(summary, runTimestamp); +} export async function updateCategoryRunSummary( runId: number, @@ -178,112 +162,20 @@ export async function updateCategoryRunSummary( | "status" | "error" >, -): Promise { - await db - .update(runs) - .set({ - topAsinsChecked: summary.topAsinsChecked, - availableAsins: summary.availableAsins, - totalProducts: summary.topAsinsChecked, - fbaCount: summary.fba, - fbmCount: summary.fbm, - skipCount: summary.skip, - status: summary.status as typeof runs.$inferInsert.status, - errorMessage: summary.error || null, - ...(summary.status !== "running" ? { completedAt: new Date() } : {}), - }) - .where(eq(runs.id, runId)); -} +): Promise { + await updateCategoryRun(runId, summary); +} export async function insertProductAnalysisResults( runId: number, results: AnalysisResult[], -): Promise { - if (results.length === 0) return; - - const rows = results.map((r) => { - const price = - r.product.keepa?.currentPrice ?? - r.product.record.sellingPriceFromSheet ?? - r.product.spApi.estimatedSalePrice; - const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank; - - return { - asin: r.product.record.asin, - runId, - name: r.product.record.name, - brand: r.product.record.brand ?? null, - category: - r.product.record.category ?? - r.product.keepa?.categoryTree?.join(" > ") ?? - null, - unitCost: r.product.record.unitCost ?? null, - currentPrice: price ?? null, - avgPrice90d: r.product.keepa?.avgPrice90 ?? null, - avgPrice90dSheet: r.product.record.avgPrice90FromSheet ?? null, - sellingPriceSheet: r.product.record.sellingPriceFromSheet ?? null, - salesRank: rank ?? null, - salesRankAvg90d: r.product.keepa?.salesRankAvg90 ?? null, - sellerCount: r.product.keepa?.sellerCount ?? null, - amazonIsSeller: r.product.keepa?.amazonIsSeller ?? null, - amazonBuyboxSharePct90d: r.product.keepa?.amazonBuyboxSharePct90d ?? null, - monthlySold: r.product.keepa?.monthlySold ?? null, - rankDrops30d: r.product.keepa?.salesRankDrops30 ?? null, - rankDrops90d: r.product.keepa?.salesRankDrops90 ?? null, - fbaFee: r.product.spApi.fbaFee ?? null, - fbmFee: r.product.spApi.fbmFee ?? null, - referralPercent: r.product.spApi.referralFeePercent ?? null, - canSell: - r.product.spApi.canSell == null - ? "unknown" - : r.product.spApi.canSell - ? "yes" - : "no", - sellabilityStatus: r.product.spApi.sellabilityStatus ?? null, - sellabilityReason: r.product.spApi.sellabilityReason ?? null, - verdict: r.verdict.verdict, - confidence: r.verdict.confidence, - reasoning: r.verdict.reasoning ?? null, - fetchedAt: new Date(r.product.fetchedAt), - }; - }); - - await db - .insert(categoryProductResults) - .values(rows) - .onConflictDoUpdate({ - target: categoryProductResults.asin, - set: { - runId: sql`EXCLUDED.run_id`, - name: sql`EXCLUDED.name`, - brand: sql`EXCLUDED.brand`, - category: sql`EXCLUDED.category`, - unitCost: sql`EXCLUDED.unit_cost`, - currentPrice: sql`EXCLUDED.current_price`, - avgPrice90d: sql`EXCLUDED.avg_price_90d`, - avgPrice90dSheet: sql`EXCLUDED.avg_price_90d_sheet`, - sellingPriceSheet: sql`EXCLUDED.selling_price_sheet`, - salesRank: sql`EXCLUDED.sales_rank`, - salesRankAvg90d: sql`EXCLUDED.sales_rank_avg_90d`, - sellerCount: sql`EXCLUDED.seller_count`, - amazonIsSeller: sql`EXCLUDED.amazon_is_seller`, - amazonBuyboxSharePct90d: sql`EXCLUDED.amazon_buybox_share_pct_90d`, - monthlySold: sql`EXCLUDED.monthly_sold`, - rankDrops30d: sql`EXCLUDED.rank_drops_30d`, - rankDrops90d: sql`EXCLUDED.rank_drops_90d`, - fbaFee: sql`EXCLUDED.fba_fee`, - fbmFee: sql`EXCLUDED.fbm_fee`, - referralPercent: sql`EXCLUDED.referral_percent`, - canSell: sql`EXCLUDED.can_sell`, - sellabilityStatus: sql`EXCLUDED.sellability_status`, - sellabilityReason: sql`EXCLUDED.sellability_reason`, - verdict: sql`EXCLUDED.verdict`, - confidence: sql`EXCLUDED.confidence`, - reasoning: sql`EXCLUDED.reasoning`, - fetchedAt: sql`EXCLUDED.fetched_at`, - }, - }); -} +): Promise { + if (results.length === 0) return; + await persistLlmResults(runId, results, { + source: "category_analysis", + metadataSource: "catalog", + }); +} function loadCategoryBlacklist(filePath: string): Set { const blacklist = new Set(); @@ -662,10 +554,14 @@ async function fetchCategoryBestSellerAsins( ]; for (const value of candidates) { - if (Array.isArray(value)) { - return [ - ...new Set(value.map((v) => String(v).trim()).filter(Boolean)), - ].slice(0, limit); + if (Array.isArray(value)) { + return [ + ...new Set( + value + .map((v) => normalizeAsin(v)) + .filter((asin): asin is string => asin !== null), + ), + ].slice(0, limit); } } @@ -917,10 +813,10 @@ async function fetchKeepaEnrichmentMap( `/product?key=${encodeURIComponent(config.keepaApiKey)}&domain=${DOMAIN_US}&asin=${asinParam}&stats=90&buybox=1&days=90`, ); - const products = Array.isArray(data?.products) ? data.products : []; - for (const product of products) { - const asin = String(product?.asin ?? "").trim(); - if (!asin) continue; + const products = Array.isArray(data?.products) ? data.products : []; + for (const product of products) { + const asin = normalizeAsin(product?.asin); + if (!asin) continue; out.set(asin, { keepa: parseKeepaProduct(product), title: String(product?.title ?? "").trim(), diff --git a/src/categories/mid-range-sellers-by-category.ts b/src/categories/mid-range-sellers-by-category.ts index f1ed93e..2123b44 100644 --- a/src/categories/mid-range-sellers-by-category.ts +++ b/src/categories/mid-range-sellers-by-category.ts @@ -1,10 +1,13 @@ -import { existsSync, mkdirSync, readFileSync } from "node:fs"; -import path from "node:path"; -import { createInterface } from "node:readline/promises"; -import { stdin as input, stdout as output } from "node:process"; -import { db } from "../db/index.ts"; -import { runs, categoryProductResults } from "../db/schema.ts"; -import { eq, sql } from "drizzle-orm"; +import { existsSync, mkdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { createInterface } from "node:readline/promises"; +import { stdin as input, stdout as output } from "node:process"; +import { normalizeAsin } from "../asin.ts"; +import { + createCategoryRun, + persistLlmResults, + updateCategoryRun, +} from "../db/persistence.ts"; import { config } from "../config.ts"; import { connectCache, @@ -475,31 +478,12 @@ async function promptCategoryIds( } } -export async function insertCategoryRunSummary( - summary: CategoryRunSummary, - runTimestamp: string, -): Promise { - const [row] = await db - .insert(runs) - .values({ - type: "category_analysis", - status: (summary.status as typeof runs.$inferInsert.status) ?? "running", - categoryId: summary.categoryId, - categoryLabel: summary.categoryLabel, - topAsinsChecked: summary.topAsinsChecked, - availableAsins: summary.availableAsins, - totalProducts: summary.topAsinsChecked, - fbaCount: summary.fba, - fbmCount: summary.fbm, - skipCount: summary.skip, - errorMessage: summary.error || null, - startedAt: new Date(runTimestamp), - }) - .returning({ id: runs.id }); - - if (!row) throw new Error("Failed to insert category run."); - return row.id; -} +export async function insertCategoryRunSummary( + summary: CategoryRunSummary, + runTimestamp: string, +): Promise { + return createCategoryRun(summary, runTimestamp); +} export async function updateCategoryRunSummary( runId: number, @@ -513,112 +497,20 @@ export async function updateCategoryRunSummary( | "status" | "error" >, -): Promise { - await db - .update(runs) - .set({ - topAsinsChecked: summary.topAsinsChecked, - availableAsins: summary.availableAsins, - totalProducts: summary.topAsinsChecked, - fbaCount: summary.fba, - fbmCount: summary.fbm, - skipCount: summary.skip, - status: summary.status as typeof runs.$inferInsert.status, - errorMessage: summary.error || null, - ...(summary.status !== "running" ? { completedAt: new Date() } : {}), - }) - .where(eq(runs.id, runId)); -} +): Promise { + await updateCategoryRun(runId, summary); +} export async function insertProductAnalysisResults( runId: number, results: AnalysisResult[], -): Promise { - if (results.length === 0) return; - - const rows = results.map((r) => { - const price = - r.product.keepa?.currentPrice ?? - r.product.record.sellingPriceFromSheet ?? - r.product.spApi.estimatedSalePrice; - const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank; - - return { - asin: r.product.record.asin, - runId, - name: r.product.record.name, - brand: r.product.record.brand ?? null, - category: - r.product.record.category ?? - r.product.keepa?.categoryTree?.join(" > ") ?? - null, - unitCost: r.product.record.unitCost ?? null, - currentPrice: price ?? null, - avgPrice90d: r.product.keepa?.avgPrice90 ?? null, - avgPrice90dSheet: r.product.record.avgPrice90FromSheet ?? null, - sellingPriceSheet: r.product.record.sellingPriceFromSheet ?? null, - salesRank: rank ?? null, - salesRankAvg90d: r.product.keepa?.salesRankAvg90 ?? null, - sellerCount: r.product.keepa?.sellerCount ?? null, - amazonIsSeller: r.product.keepa?.amazonIsSeller ?? null, - amazonBuyboxSharePct90d: r.product.keepa?.amazonBuyboxSharePct90d ?? null, - monthlySold: r.product.keepa?.monthlySold ?? null, - rankDrops30d: r.product.keepa?.salesRankDrops30 ?? null, - rankDrops90d: r.product.keepa?.salesRankDrops90 ?? null, - fbaFee: r.product.spApi.fbaFee ?? null, - fbmFee: r.product.spApi.fbmFee ?? null, - referralPercent: r.product.spApi.referralFeePercent ?? null, - canSell: - r.product.spApi.canSell == null - ? "unknown" - : r.product.spApi.canSell - ? "yes" - : "no", - sellabilityStatus: r.product.spApi.sellabilityStatus ?? null, - sellabilityReason: r.product.spApi.sellabilityReason ?? null, - verdict: r.verdict.verdict, - confidence: r.verdict.confidence, - reasoning: r.verdict.reasoning ?? null, - fetchedAt: new Date(r.product.fetchedAt), - }; - }); - - await db - .insert(categoryProductResults) - .values(rows) - .onConflictDoUpdate({ - target: categoryProductResults.asin, - set: { - runId: sql`EXCLUDED.run_id`, - name: sql`EXCLUDED.name`, - brand: sql`EXCLUDED.brand`, - category: sql`EXCLUDED.category`, - unitCost: sql`EXCLUDED.unit_cost`, - currentPrice: sql`EXCLUDED.current_price`, - avgPrice90d: sql`EXCLUDED.avg_price_90d`, - avgPrice90dSheet: sql`EXCLUDED.avg_price_90d_sheet`, - sellingPriceSheet: sql`EXCLUDED.selling_price_sheet`, - salesRank: sql`EXCLUDED.sales_rank`, - salesRankAvg90d: sql`EXCLUDED.sales_rank_avg_90d`, - sellerCount: sql`EXCLUDED.seller_count`, - amazonIsSeller: sql`EXCLUDED.amazon_is_seller`, - amazonBuyboxSharePct90d: sql`EXCLUDED.amazon_buybox_share_pct_90d`, - monthlySold: sql`EXCLUDED.monthly_sold`, - rankDrops30d: sql`EXCLUDED.rank_drops_30d`, - rankDrops90d: sql`EXCLUDED.rank_drops_90d`, - fbaFee: sql`EXCLUDED.fba_fee`, - fbmFee: sql`EXCLUDED.fbm_fee`, - referralPercent: sql`EXCLUDED.referral_percent`, - canSell: sql`EXCLUDED.can_sell`, - sellabilityStatus: sql`EXCLUDED.sellability_status`, - sellabilityReason: sql`EXCLUDED.sellability_reason`, - verdict: sql`EXCLUDED.verdict`, - confidence: sql`EXCLUDED.confidence`, - reasoning: sql`EXCLUDED.reasoning`, - fetchedAt: sql`EXCLUDED.fetched_at`, - }, - }); -} +): Promise { + if (results.length === 0) return; + await persistLlmResults(runId, results, { + source: "category_analysis", + metadataSource: "catalog", + }); +} function loadCategoryBlacklist(filePath: string): Set { const blacklist = new Set(); @@ -997,10 +889,14 @@ async function fetchCategoryBestSellerAsins( ]; for (const value of candidates) { - if (Array.isArray(value)) { - return [ - ...new Set(value.map((v) => String(v).trim()).filter(Boolean)), - ].slice(0, limit); + if (Array.isArray(value)) { + return [ + ...new Set( + value + .map((v) => normalizeAsin(v)) + .filter((asin): asin is string => asin !== null), + ), + ].slice(0, limit); } } @@ -1256,10 +1152,10 @@ async function fetchKeepaEnrichmentMap( `/product?key=${encodeURIComponent(config.keepaApiKey)}&domain=${DOMAIN_US}&asin=${asinParam}&stats=90&buybox=1&days=90`, ); - const products = Array.isArray(data?.products) ? data.products : []; - for (const product of products) { - const asin = String(product?.asin ?? "").trim(); - if (!asin) continue; + const products = Array.isArray(data?.products) ? data.products : []; + for (const product of products) { + const asin = normalizeAsin(product?.asin); + if (!asin) continue; const parsed = { keepa: parseKeepaProduct(product), title: String(product?.title ?? "").trim(), diff --git a/src/categories/top-monthly-sold-by-category.ts b/src/categories/top-monthly-sold-by-category.ts index 9ec8d10..41fd862 100644 --- a/src/categories/top-monthly-sold-by-category.ts +++ b/src/categories/top-monthly-sold-by-category.ts @@ -1,8 +1,11 @@ -import { existsSync, mkdirSync, readFileSync } from "node:fs"; -import path from "node:path"; -import { db } from "../db/index.ts"; -import { runs, categoryProductResults } from "../db/schema.ts"; -import { eq, sql } from "drizzle-orm"; +import { existsSync, mkdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { normalizeAsin } from "../asin.ts"; +import { + createCategoryRun, + persistLlmResults, + updateCategoryRun, +} from "../db/persistence.ts"; import { config } from "../config.ts"; import { analyzeProducts } from "../integrations/llm.ts"; import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "../integrations/sp-api.ts"; @@ -172,31 +175,12 @@ function printUsageAndExit(message: string): never { process.exit(1); } -export async function insertCategoryRunSummary( - summary: CategoryRunSummary, - runTimestamp: string, -): Promise { - const [row] = await db - .insert(runs) - .values({ - type: "category_analysis", - status: (summary.status as typeof runs.$inferInsert.status) ?? "running", - categoryId: summary.categoryId, - categoryLabel: summary.categoryLabel, - topAsinsChecked: summary.topAsinsChecked, - availableAsins: summary.availableAsins, - totalProducts: summary.topAsinsChecked, - fbaCount: summary.fba, - fbmCount: summary.fbm, - skipCount: summary.skip, - errorMessage: summary.error || null, - startedAt: new Date(runTimestamp), - }) - .returning({ id: runs.id }); - - if (!row) throw new Error("Failed to insert category run."); - return row.id; -} +export async function insertCategoryRunSummary( + summary: CategoryRunSummary, + runTimestamp: string, +): Promise { + return createCategoryRun(summary, runTimestamp); +} export async function updateCategoryRunSummary( runId: number, @@ -210,112 +194,20 @@ export async function updateCategoryRunSummary( | "status" | "error" >, -): Promise { - await db - .update(runs) - .set({ - topAsinsChecked: summary.topAsinsChecked, - availableAsins: summary.availableAsins, - totalProducts: summary.topAsinsChecked, - fbaCount: summary.fba, - fbmCount: summary.fbm, - skipCount: summary.skip, - status: summary.status as typeof runs.$inferInsert.status, - errorMessage: summary.error || null, - ...(summary.status !== "running" ? { completedAt: new Date() } : {}), - }) - .where(eq(runs.id, runId)); -} +): Promise { + await updateCategoryRun(runId, summary); +} export async function insertProductAnalysisResults( runId: number, results: AnalysisResult[], -): Promise { - if (results.length === 0) return; - - const rows = results.map((r) => { - const price = - r.product.keepa?.currentPrice ?? - r.product.record.sellingPriceFromSheet ?? - r.product.spApi.estimatedSalePrice; - const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank; - - return { - asin: r.product.record.asin, - runId, - name: r.product.record.name, - brand: r.product.record.brand ?? null, - category: - r.product.record.category ?? - r.product.keepa?.categoryTree?.join(" > ") ?? - null, - unitCost: r.product.record.unitCost ?? null, - currentPrice: price ?? null, - avgPrice90d: r.product.keepa?.avgPrice90 ?? null, - avgPrice90dSheet: r.product.record.avgPrice90FromSheet ?? null, - sellingPriceSheet: r.product.record.sellingPriceFromSheet ?? null, - salesRank: rank ?? null, - salesRankAvg90d: r.product.keepa?.salesRankAvg90 ?? null, - sellerCount: r.product.keepa?.sellerCount ?? null, - amazonIsSeller: r.product.keepa?.amazonIsSeller ?? null, - amazonBuyboxSharePct90d: r.product.keepa?.amazonBuyboxSharePct90d ?? null, - monthlySold: r.product.keepa?.monthlySold ?? null, - rankDrops30d: r.product.keepa?.salesRankDrops30 ?? null, - rankDrops90d: r.product.keepa?.salesRankDrops90 ?? null, - fbaFee: r.product.spApi.fbaFee ?? null, - fbmFee: r.product.spApi.fbmFee ?? null, - referralPercent: r.product.spApi.referralFeePercent ?? null, - canSell: - r.product.spApi.canSell == null - ? "unknown" - : r.product.spApi.canSell - ? "yes" - : "no", - sellabilityStatus: r.product.spApi.sellabilityStatus ?? null, - sellabilityReason: r.product.spApi.sellabilityReason ?? null, - verdict: r.verdict.verdict, - confidence: r.verdict.confidence, - reasoning: r.verdict.reasoning ?? null, - fetchedAt: new Date(r.product.fetchedAt), - }; - }); - - await db - .insert(categoryProductResults) - .values(rows) - .onConflictDoUpdate({ - target: categoryProductResults.asin, - set: { - runId: sql`EXCLUDED.run_id`, - name: sql`EXCLUDED.name`, - brand: sql`EXCLUDED.brand`, - category: sql`EXCLUDED.category`, - unitCost: sql`EXCLUDED.unit_cost`, - currentPrice: sql`EXCLUDED.current_price`, - avgPrice90d: sql`EXCLUDED.avg_price_90d`, - avgPrice90dSheet: sql`EXCLUDED.avg_price_90d_sheet`, - sellingPriceSheet: sql`EXCLUDED.selling_price_sheet`, - salesRank: sql`EXCLUDED.sales_rank`, - salesRankAvg90d: sql`EXCLUDED.sales_rank_avg_90d`, - sellerCount: sql`EXCLUDED.seller_count`, - amazonIsSeller: sql`EXCLUDED.amazon_is_seller`, - amazonBuyboxSharePct90d: sql`EXCLUDED.amazon_buybox_share_pct_90d`, - monthlySold: sql`EXCLUDED.monthly_sold`, - rankDrops30d: sql`EXCLUDED.rank_drops_30d`, - rankDrops90d: sql`EXCLUDED.rank_drops_90d`, - fbaFee: sql`EXCLUDED.fba_fee`, - fbmFee: sql`EXCLUDED.fbm_fee`, - referralPercent: sql`EXCLUDED.referral_percent`, - canSell: sql`EXCLUDED.can_sell`, - sellabilityStatus: sql`EXCLUDED.sellability_status`, - sellabilityReason: sql`EXCLUDED.sellability_reason`, - verdict: sql`EXCLUDED.verdict`, - confidence: sql`EXCLUDED.confidence`, - reasoning: sql`EXCLUDED.reasoning`, - fetchedAt: sql`EXCLUDED.fetched_at`, - }, - }); -} +): Promise { + if (results.length === 0) return; + await persistLlmResults(runId, results, { + source: "category_analysis", + metadataSource: "catalog", + }); +} function loadCategoryBlacklist(filePath: string): Set { const blacklist = new Set(); @@ -694,10 +586,14 @@ async function fetchCategoryBestSellerAsins( ]; for (const value of candidates) { - if (Array.isArray(value)) { - return [ - ...new Set(value.map((v) => String(v).trim()).filter(Boolean)), - ].slice(0, limit); + if (Array.isArray(value)) { + return [ + ...new Set( + value + .map((v) => normalizeAsin(v)) + .filter((asin): asin is string => asin !== null), + ), + ].slice(0, limit); } } @@ -949,10 +845,10 @@ async function fetchKeepaEnrichmentMap( `/product?key=${encodeURIComponent(config.keepaApiKey)}&domain=${DOMAIN_US}&asin=${asinParam}&stats=90&buybox=1&days=90`, ); - const products = Array.isArray(data?.products) ? data.products : []; - for (const product of products) { - const asin = String(product?.asin ?? "").trim(); - if (!asin) continue; + const products = Array.isArray(data?.products) ? data.products : []; + for (const product of products) { + const asin = normalizeAsin(product?.asin); + if (!asin) continue; out.set(asin, { keepa: parseKeepaProduct(product), title: String(product?.title ?? "").trim(), diff --git a/src/db/persistence.ts b/src/db/persistence.ts new file mode 100644 index 0000000..7d21b33 --- /dev/null +++ b/src/db/persistence.ts @@ -0,0 +1,541 @@ +import { sql } from "drizzle-orm"; +import { requireAsin, normalizeAsin } from "../asin.ts"; +import type { + AnalysisResult, + ProductRecord, + SupplierAnalysisResult, +} from "../types.ts"; +import { db } from "./index.ts"; +import { + analysisRevisions, + analysisRunStats, + categoryRunDetails, + productIdentifiers, + productObservations, + products, + runItems, + runs, + sourcingInputs, + supplierScores, + upcResolutionCandidates, + upcResolutions, +} from "./schema.ts"; + +type Executor = any; +type MetadataSource = "input" | "catalog"; + +type ProductSeed = { + asin: string; + name?: string | null; + brand?: string | null; + category?: string | null; + metadataSource?: MetadataSource; + fetchedAt?: Date; +}; + +export type CategoryRunSummaryInput = { + categoryId: number; + categoryLabel: string; + topAsinsChecked: number; + availableAsins: number; + fba: number; + fbm: number; + skip: number; + status: "running" | "ok" | "empty" | "failed"; + error: string; +}; + +export type RunCounts = { + totalProducts: number; + fbaCount: number; + fbmCount: number; + skipCount: number; +}; + +function emptyToNull(value: string | undefined | null): string | null { + const trimmed = value?.trim(); + return trimmed ? trimmed : null; +} + +function productCategory(record: ProductRecord, result: AnalysisResult): string | null { + return emptyToNull( + record.category ?? result.product.keepa?.categoryTree?.join(" > "), + ); +} + +export async function upsertProduct( + seed: ProductSeed, + executor: Executor = db, +): Promise { + const asin = requireAsin(seed.asin); + const now = seed.fetchedAt ?? new Date(); + const isCatalog = seed.metadataSource === "catalog"; + + await executor + .insert(products) + .values({ + asin, + name: emptyToNull(seed.name), + brand: emptyToNull(seed.brand), + category: emptyToNull(seed.category), + metadataFetchedAt: isCatalog ? now : null, + firstSeenAt: now, + lastSeenAt: now, + }) + .onConflictDoUpdate({ + target: products.asin, + set: { + lastSeenAt: sql`GREATEST(${products.lastSeenAt}, EXCLUDED.last_seen_at)`, + name: isCatalog + ? sql`CASE WHEN EXCLUDED.metadata_fetched_at >= COALESCE(${products.metadataFetchedAt}, '-infinity'::timestamptz) AND NULLIF(EXCLUDED.name, '') IS NOT NULL THEN EXCLUDED.name ELSE ${products.name} END` + : sql`COALESCE(${products.name}, NULLIF(EXCLUDED.name, ''))`, + brand: isCatalog + ? sql`CASE WHEN EXCLUDED.metadata_fetched_at >= COALESCE(${products.metadataFetchedAt}, '-infinity'::timestamptz) AND NULLIF(EXCLUDED.brand, '') IS NOT NULL THEN EXCLUDED.brand ELSE ${products.brand} END` + : sql`COALESCE(${products.brand}, NULLIF(EXCLUDED.brand, ''))`, + category: isCatalog + ? sql`CASE WHEN EXCLUDED.metadata_fetched_at >= COALESCE(${products.metadataFetchedAt}, '-infinity'::timestamptz) AND NULLIF(EXCLUDED.category, '') IS NOT NULL THEN EXCLUDED.category ELSE ${products.category} END` + : sql`COALESCE(${products.category}, NULLIF(EXCLUDED.category, ''))`, + metadataFetchedAt: isCatalog + ? sql`GREATEST(COALESCE(${products.metadataFetchedAt}, '-infinity'::timestamptz), EXCLUDED.metadata_fetched_at)` + : products.metadataFetchedAt, + }, + }); + + return asin; +} + +export async function insertObservation( + runId: number, + result: AnalysisResult, + source: string, + executor: Executor = db, +): Promise { + const fetchedAt = new Date(result.product.fetchedAt); + const record = result.product.record; + const keepa = result.product.keepa; + const spApi = result.product.spApi; + const asin = requireAsin(record.asin); + const [observation] = await executor + .insert(productObservations) + .values({ + productAsin: asin, + runId, + source, + currentPrice: + keepa?.currentPrice ?? + record.sellingPriceFromSheet ?? + spApi.estimatedSalePrice ?? + null, + avgPrice90d: keepa?.avgPrice90 ?? null, + salesRank: keepa?.salesRank ?? record.amazonRank ?? null, + salesRankAvg90d: keepa?.salesRankAvg90 ?? null, + monthlySold: keepa?.monthlySold ?? null, + rankDrops30d: keepa?.salesRankDrops30 ?? null, + rankDrops90d: keepa?.salesRankDrops90 ?? null, + sellerCount: keepa?.sellerCount ?? null, + amazonIsSeller: keepa?.amazonIsSeller ?? null, + amazonBuyboxSharePct90d: keepa?.amazonBuyboxSharePct90d ?? null, + fbaFee: spApi.fbaFee ?? null, + fbmFee: spApi.fbmFee ?? null, + referralPercent: spApi.referralFeePercent ?? null, + canSell: spApi.canSell, + sellabilityStatus: spApi.sellabilityStatus, + sellabilityReason: spApi.sellabilityReason ?? null, + fetchedAt, + }) + .returning({ id: productObservations.id }); + + if (!observation) throw new Error(`Failed to insert observation for ${asin}`); + return observation.id; +} + +function sourcingInputValues(runItemId: number, record: ProductRecord) { + return { + runItemId, + suppliedName: emptyToNull(record.name), + suppliedBrand: emptyToNull(record.brand), + suppliedCategory: emptyToNull(record.category), + unitCost: record.unitCost ?? null, + avgPrice90dSheet: record.avgPrice90FromSheet ?? null, + sellingPriceSheet: record.sellingPriceFromSheet ?? null, + fbaNetSheet: record.fbaNet ?? null, + grossProfitDollar: record.grossProfit ?? null, + grossProfitPct: record.grossProfitPct ?? null, + netProfitSheet: record.netProfitFromSheet ?? null, + roiSheet: record.roiFromSheet ?? null, + moq: record.moq ?? null, + moqCost: record.moqCost ?? null, + qtyAvailable: record.totalQtyAvail ?? null, + supplier: emptyToNull(record.supplier), + sourceUrl: emptyToNull(record.sourceUrl), + asinLink: emptyToNull(record.asinLink), + promoCouponCode: emptyToNull(record.promoCouponCode), + notes: emptyToNull(record.notes), + leadDate: emptyToNull(record.leadDate), + }; +} + +export async function persistLlmResults( + runId: number, + results: AnalysisResult[], + options: { + source: string; + metadataSource?: MetadataSource; + preserveSourcingInput?: boolean; + sourceInventoryIds?: Map; + }, +): Promise { + for (const result of results) { + const record = result.product.record; + const fetchedAt = new Date(result.product.fetchedAt); + const asin = await upsertProduct({ + asin: record.asin, + name: record.name, + brand: record.brand, + category: productCategory(record, result), + metadataSource: options.metadataSource ?? "input", + fetchedAt, + }); + const [item] = await db + .insert(runItems) + .values({ + runId, + productAsin: asin, + sourceInventoryItemId: options.sourceInventoryIds?.get(asin) ?? null, + }) + .returning({ id: runItems.id }); + if (!item) throw new Error(`Failed to insert run item for ${asin}`); + + if (options.preserveSourcingInput) { + await db.insert(sourcingInputs).values(sourcingInputValues(item.id, record)); + } + + const observationId = await insertObservation(runId, result, options.source); + await db.insert(analysisRevisions).values({ + runItemId: item.id, + observationId, + method: "llm", + decision: result.verdict.verdict, + confidence: result.verdict.confidence, + reasoning: result.verdict.reasoning ?? null, + analyzedAt: fetchedAt, + }); + } +} + +function supplierSourcingValues(runItemId: number, result: SupplierAnalysisResult) { + return { + runItemId, + suppliedName: emptyToNull(result.record.name), + suppliedBrand: emptyToNull(result.record.brand), + suppliedCategory: emptyToNull(result.record.category), + unitCost: result.record.unitCost ?? null, + }; +} + +async function insertSupplierObservation( + runId: number, + productAsin: string, + result: SupplierAnalysisResult, +): Promise { + const keepa = result.keepa; + const spApi = result.spApi; + if (!spApi && !keepa) return null; + + const [row] = await db + .insert(productObservations) + .values({ + productAsin, + runId, + source: "supplier_upc", + currentPrice: result.score.salePrice, + avgPrice90d: keepa?.avgPrice90 ?? null, + salesRank: keepa?.salesRank ?? null, + salesRankAvg90d: keepa?.salesRankAvg90 ?? null, + monthlySold: keepa?.monthlySold ?? null, + rankDrops30d: keepa?.salesRankDrops30 ?? null, + rankDrops90d: keepa?.salesRankDrops90 ?? null, + sellerCount: keepa?.sellerCount ?? null, + amazonIsSeller: keepa?.amazonIsSeller ?? null, + amazonBuyboxSharePct90d: keepa?.amazonBuyboxSharePct90d ?? null, + fbaFee: spApi?.fbaFee ?? null, + fbmFee: spApi?.fbmFee ?? null, + referralPercent: spApi?.referralFeePercent ?? null, + canSell: spApi?.canSell ?? null, + sellabilityStatus: spApi?.sellabilityStatus ?? null, + sellabilityReason: spApi?.sellabilityReason ?? null, + fetchedAt: new Date(result.fetchedAt), + }) + .returning({ id: productObservations.id }); + return row?.id ?? null; +} + +export async function persistSupplierResults( + runId: number, + results: SupplierAnalysisResult[], +): Promise { + for (const result of results) { + const resolvedAsin = normalizeAsin(result.lookup.asin); + if (resolvedAsin) { + await upsertProduct({ + asin: resolvedAsin, + name: result.record.name, + brand: result.record.brand, + category: result.record.category, + metadataSource: "input", + fetchedAt: new Date(result.fetchedAt), + }); + if (result.keepa?.categoryTree?.length) { + await upsertProduct({ + asin: resolvedAsin, + category: result.keepa.categoryTree.join(" > "), + metadataSource: "catalog", + fetchedAt: new Date(result.fetchedAt), + }); + } + } + + const [item] = await db + .insert(runItems) + .values({ + runId, + productAsin: resolvedAsin, + sourceRow: result.rowNumber ?? null, + }) + .returning({ id: runItems.id }); + if (!item) throw new Error("Failed to insert supplier run item"); + + await db.insert(sourcingInputs).values(supplierSourcingValues(item.id, result)); + await db.insert(upcResolutions).values({ + runItemId: item.id, + requestedUpc: result.upc, + normalizedUpc: result.lookup.normalizedUpc, + provider: result.lookup.provider ?? "unknown", + status: result.lookup.status, + reason: result.lookup.reason ?? null, + resolvedProductAsin: resolvedAsin, + resolvedAt: new Date(result.fetchedAt), + }); + + for (const candidate of result.lookup.candidateAsins) { + const candidateAsin = normalizeAsin(candidate); + if (!candidateAsin) continue; + await upsertProduct({ asin: candidateAsin, fetchedAt: new Date(result.fetchedAt) }); + await db + .insert(upcResolutionCandidates) + .values({ runItemId: item.id, productAsin: candidateAsin }) + .onConflictDoUpdate({ + target: [ + upcResolutionCandidates.runItemId, + upcResolutionCandidates.productAsin, + ], + set: { productAsin: sql`EXCLUDED.product_asin` }, + }); + } + + if (resolvedAsin) { + await db + .insert(productIdentifiers) + .values({ + productAsin: resolvedAsin, + identifierType: + result.lookup.normalizedUpc.length === 12 + ? "upc" + : result.lookup.normalizedUpc.length === 13 + ? "ean" + : "gtin", + identifierValue: result.lookup.normalizedUpc, + source: "supplier_upc", + confirmedAt: new Date(result.fetchedAt), + }) + .onConflictDoUpdate({ + target: [ + productIdentifiers.identifierType, + productIdentifiers.identifierValue, + ], + set: { + productAsin: resolvedAsin, + source: "supplier_upc", + confirmedAt: new Date(result.fetchedAt), + }, + }); + } + + const observationId = resolvedAsin + ? await insertSupplierObservation(runId, resolvedAsin, result) + : null; + const [revision] = await db + .insert(analysisRevisions) + .values({ + runItemId: item.id, + observationId, + method: "supplier_scoring", + decision: result.score.verdict, + confidence: result.score.score, + reasoning: result.score.reason, + analyzedAt: new Date(result.fetchedAt), + }) + .returning({ id: analysisRevisions.id }); + if (!revision) throw new Error("Failed to insert supplier analysis revision"); + + await db.insert(supplierScores).values({ + revisionId: revision.id, + score: result.score.score, + salePrice: result.score.salePrice, + fbaFee: result.score.fbaFee, + profit: result.score.profit, + margin: result.score.margin, + roi: result.score.roi, + reason: result.score.reason, + }); + } +} + +export async function createCategoryRun( + summary: CategoryRunSummaryInput, + runTimestamp: string, +): Promise { + const [row] = await db + .insert(runs) + .values({ + type: "category_analysis", + status: summary.status, + errorMessage: summary.error || null, + startedAt: new Date(runTimestamp), + }) + .returning({ id: runs.id }); + if (!row) throw new Error("Failed to insert category run."); + + await db.insert(categoryRunDetails).values({ + runId: row.id, + categoryId: summary.categoryId, + categoryLabel: summary.categoryLabel, + checkedAsinCount: summary.topAsinsChecked, + }); + await db.insert(analysisRunStats).values({ + runId: row.id, + processedCount: summary.topAsinsChecked, + availableCount: summary.availableAsins, + analyzedCount: summary.fba + summary.fbm + summary.skip, + fbaCount: summary.fba, + fbmCount: summary.fbm, + skipCount: summary.skip, + }); + return row.id; +} + +export async function updateCategoryRun( + runId: number, + summary: Pick< + CategoryRunSummaryInput, + | "topAsinsChecked" + | "availableAsins" + | "fba" + | "fbm" + | "skip" + | "status" + | "error" + >, +): Promise { + await db + .update(runs) + .set({ + status: summary.status, + errorMessage: summary.error || null, + ...(summary.status !== "running" ? { completedAt: new Date() } : {}), + }) + .where(sql`${runs.id} = ${runId}`); + await db + .insert(categoryRunDetails) + .values({ + runId, + categoryId: 0, + categoryLabel: "", + checkedAsinCount: summary.topAsinsChecked, + }) + .onConflictDoUpdate({ + target: categoryRunDetails.runId, + set: { checkedAsinCount: summary.topAsinsChecked }, + }); + await db + .insert(analysisRunStats) + .values({ + runId, + processedCount: summary.topAsinsChecked, + availableCount: summary.availableAsins, + analyzedCount: summary.fba + summary.fbm + summary.skip, + fbaCount: summary.fba, + fbmCount: summary.fbm, + skipCount: summary.skip, + }) + .onConflictDoUpdate({ + target: analysisRunStats.runId, + set: { + processedCount: summary.topAsinsChecked, + availableCount: summary.availableAsins, + analyzedCount: summary.fba + summary.fbm + summary.skip, + fbaCount: summary.fba, + fbmCount: summary.fbm, + skipCount: summary.skip, + }, + }); +} + +export async function refreshRunStats(runId: number): Promise { + const [stats] = await db.execute( + sql<{ + total: string; + fba: string | null; + fbm: string | null; + buy: string | null; + watch: string | null; + skip: string | null; + }>`WITH latest AS ( + SELECT DISTINCT ON (ri.id) ar.decision + FROM run_items ri + JOIN analysis_revisions ar ON ar.run_item_id = ri.id + WHERE ri.run_id = ${runId} + ORDER BY ri.id, ar.analyzed_at DESC, ar.id DESC + ) + SELECT + COUNT(*) AS total, + SUM(CASE WHEN decision = 'FBA' THEN 1 ELSE 0 END) AS fba, + SUM(CASE WHEN decision = 'FBM' THEN 1 ELSE 0 END) AS fbm, + SUM(CASE WHEN decision = 'BUY' THEN 1 ELSE 0 END) AS buy, + SUM(CASE WHEN decision = 'WATCH' THEN 1 ELSE 0 END) AS watch, + SUM(CASE WHEN decision = 'SKIP' THEN 1 ELSE 0 END) AS skip + FROM latest`, + ); + + const counts = { + totalProducts: Number(stats?.total ?? 0), + fbaCount: Number(stats?.fba ?? 0), + fbmCount: Number(stats?.fbm ?? 0), + skipCount: Number(stats?.skip ?? 0), + }; + await db + .insert(analysisRunStats) + .values({ + runId, + processedCount: counts.totalProducts, + analyzedCount: counts.totalProducts, + fbaCount: counts.fbaCount, + fbmCount: counts.fbmCount, + buyCount: Number(stats?.buy ?? 0), + watchCount: Number(stats?.watch ?? 0), + skipCount: counts.skipCount, + }) + .onConflictDoUpdate({ + target: analysisRunStats.runId, + set: { + processedCount: counts.totalProducts, + analyzedCount: counts.totalProducts, + fbaCount: counts.fbaCount, + fbmCount: counts.fbmCount, + buyCount: Number(stats?.buy ?? 0), + watchCount: Number(stats?.watch ?? 0), + skipCount: counts.skipCount, + }, + }); + return counts; +} diff --git a/src/db/schema.ts b/src/db/schema.ts index 59b77bb..6b67dfd 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -1,9 +1,13 @@ +import { sql } from "drizzle-orm"; import { + type AnyPgColumn, boolean, + check, index, integer, pgEnum, pgTable, + primaryKey, real, serial, text, @@ -11,13 +15,12 @@ import { unique, } from "drizzle-orm/pg-core"; -// ─── Enums ─────────────────────────────────────────────────────────────────── - export const runTypeEnum = pgEnum("run_type", [ "lead_analysis", "category_analysis", "supplier_upc", "stalker", + "stalker_analysis", ]); export const runStatusEnum = pgEnum("run_status", [ @@ -28,29 +31,54 @@ export const runStatusEnum = pgEnum("run_status", [ "completed", ]); -// ─── Runs ───────────────────────────────────────────────────────────────────── -// Unified run log; replaces the old `runs` and `category_analysis_runs` tables. -// Category-specific columns (categoryId, categoryLabel, …) are null for -// lead_analysis / supplier_upc runs. +export const analysisMethodEnum = pgEnum("analysis_method", [ + "llm", + "supplier_scoring", +]); + +export const analysisDecisionEnum = pgEnum("analysis_decision", [ + "FBA", + "FBM", + "BUY", + "WATCH", + "SKIP", +]); + +export const products = pgTable( + "products", + { + asin: text("asin").primaryKey(), + name: text("name"), + brand: text("brand"), + category: text("category"), + metadataFetchedAt: timestamp("metadata_fetched_at", { withTimezone: true }), + firstSeenAt: timestamp("first_seen_at", { withTimezone: true }) + .notNull() + .defaultNow(), + lastSeenAt: timestamp("last_seen_at", { withTimezone: true }) + .notNull() + .defaultNow(), + }, + (t) => [ + check("ck_products_asin", sql`${t.asin} ~ '^[A-Z0-9]{10}$'`), + index("idx_products_name").on(t.name), + index("idx_products_last_seen_at").on(t.lastSeenAt), + ], +); export const runs = pgTable( "runs", { id: serial("id").primaryKey(), type: runTypeEnum("type").notNull(), + parentRunId: integer("parent_run_id").references( + (): AnyPgColumn => runs.id, + { onDelete: "cascade" }, + ), inputFile: text("input_file"), outputFile: text("output_file"), status: runStatusEnum("status").notNull().default("running"), errorMessage: text("error_message"), - totalProducts: integer("total_products"), - fbaCount: integer("fba_count"), - fbmCount: integer("fbm_count"), - skipCount: integer("skip_count"), - // Category-pipeline only - categoryId: integer("category_id"), - categoryLabel: text("category_label"), - topAsinsChecked: integer("top_asins_checked"), - availableAsins: integer("available_asins"), startedAt: timestamp("started_at", { withTimezone: true }) .notNull() .defaultNow(), @@ -60,210 +88,262 @@ export const runs = pgTable( index("idx_runs_started_at").on(t.startedAt), index("idx_runs_type").on(t.type), index("idx_runs_status").on(t.status), + index("idx_runs_parent_run_id").on(t.parentRunId), ], ); -// ─── Analysis results ───────────────────────────────────────────────────────── -// Archival table: one row per product per run for the lead-list and supplier -// UPC pipelines. Multiple rows for the same ASIN across different runs is fine. +export const analysisRunStats = pgTable("analysis_run_stats", { + runId: integer("run_id") + .primaryKey() + .references(() => runs.id, { onDelete: "cascade" }), + processedCount: integer("processed_count").notNull().default(0), + analyzedCount: integer("analyzed_count").notNull().default(0), + availableCount: integer("available_count").notNull().default(0), + fbaCount: integer("fba_count").notNull().default(0), + fbmCount: integer("fbm_count").notNull().default(0), + buyCount: integer("buy_count").notNull().default(0), + watchCount: integer("watch_count").notNull().default(0), + skipCount: integer("skip_count").notNull().default(0), +}); -export const analysisResults = pgTable( - "analysis_results", +export const categoryRunDetails = pgTable("category_run_details", { + runId: integer("run_id") + .primaryKey() + .references(() => runs.id, { onDelete: "cascade" }), + categoryId: integer("category_id").notNull(), + categoryLabel: text("category_label").notNull(), + checkedAsinCount: integer("checked_asin_count").notNull().default(0), + selectionParametersJson: text("selection_parameters_json"), +}); + +export const stalkerRunDetails = pgTable("stalker_run_details", { + runId: integer("run_id") + .primaryKey() + .references(() => runs.id, { onDelete: "cascade" }), + requestedAsins: integer("requested_asins").notNull().default(0), + skippedAsins: integer("skipped_asins").notNull().default(0), + scannedAsins: integer("scanned_asins").notNull().default(0), + sourceAsinsWithMatches: integer("source_asins_with_matches") + .notNull() + .default(0), + candidateSellers: integer("candidate_sellers").notNull().default(0), + qualifyingSellers: integer("qualifying_sellers").notNull().default(0), + matchedSellers: integer("matched_sellers").notNull().default(0), + sellerMetadataRequests: integer("seller_metadata_requests") + .notNull() + .default(0), + sellerStorefrontRequests: integer("seller_storefront_requests") + .notNull() + .default(0), + inventorySellabilityCheckedAsins: integer( + "inventory_sellability_checked_asins", + ) + .notNull() + .default(0), + inventorySellabilityAvailableAsins: integer( + "inventory_sellability_available_asins", + ) + .notNull() + .default(0), + inventorySellabilityExcludedAsins: integer( + "inventory_sellability_excluded_asins", + ) + .notNull() + .default(0), + persistedInventoryAsins: integer("persisted_inventory_asins") + .notNull() + .default(0), +}); + +export const productIdentifiers = pgTable( + "product_identifiers", { id: serial("id").primaryKey(), - runId: integer("run_id") + productAsin: text("product_asin") .notNull() - .references(() => runs.id), - asin: text("asin").notNull(), - // Product identity - productName: text("product_name"), - brand: text("brand"), - category: text("category"), - upc: text("upc"), - // Supplier sheet data (lead_analysis only) - unitCost: real("unit_cost"), - avgPrice90dSheet: real("avg_price_90d_sheet"), - sellingPriceSheet: real("selling_price_sheet"), - fbaNetSheet: real("fba_net_sheet"), - grossProfitDollar: real("gross_profit_dollar"), - grossProfitPct: real("gross_profit_pct"), - netProfitSheet: real("net_profit_sheet"), - roiSheet: real("roi_sheet"), - moq: integer("moq"), - moqCost: real("moq_cost"), - qtyAvailable: integer("qty_available"), - supplier: text("supplier"), - sourceUrl: text("source_url"), - asinLink: text("asin_link"), - promoCouponCode: text("promo_coupon_code"), - notes: text("notes"), - leadDate: text("lead_date"), - // Market data - currentPrice: real("current_price"), - avgPrice90d: real("avg_price_90d"), - salesRank: integer("sales_rank"), - rankAvg90d: integer("rank_avg_90d"), - monthlySold: integer("monthly_sold"), - rankDrops30d: integer("rank_drops_30d"), - rankDrops90d: integer("rank_drops_90d"), - sellerCount: integer("seller_count"), - amazonIsSeller: boolean("amazon_is_seller"), - amazonBuyboxSharePct90d: real("amazon_buybox_share_pct_90d"), - // Fees - fbaFee: real("fba_fee"), - fbmFee: real("fbm_fee"), - referralPercent: real("referral_percent"), - // Sellability - canSell: text("can_sell"), - sellabilityStatus: text("sellability_status"), - sellabilityReason: text("sellability_reason"), - // Supplier-UPC scoring - supplierScore: real("supplier_score"), - supplierProfit: real("supplier_profit"), - supplierMargin: real("supplier_margin"), - supplierRoi: real("supplier_roi"), - supplierReason: text("supplier_reason"), - upcLookupStatus: text("upc_lookup_status"), - upcLookupReason: text("upc_lookup_reason"), - candidateAsins: text("candidate_asins"), - // Verdict - verdict: text("verdict").notNull(), - confidence: real("confidence"), - reasoning: text("reasoning"), - fetchedAt: timestamp("fetched_at", { withTimezone: true }).notNull(), + .references(() => products.asin), + identifierType: text("identifier_type").notNull(), + identifierValue: text("identifier_value").notNull(), + source: text("source").notNull(), + confirmedAt: timestamp("confirmed_at", { withTimezone: true }) + .notNull() + .defaultNow(), }, (t) => [ - index("idx_analysis_results_run_id").on(t.runId), - index("idx_analysis_results_asin").on(t.asin), - index("idx_analysis_results_verdict").on(t.verdict), - index("idx_analysis_results_sellability_status").on(t.sellabilityStatus), - index("idx_analysis_results_fetched_at").on(t.fetchedAt), + unique("uq_product_identifier_type_value").on( + t.identifierType, + t.identifierValue, + ), + index("idx_product_identifiers_asin").on(t.productAsin), ], ); -// ─── Category product results ────────────────────────────────────────────────── -// Latest-per-ASIN snapshot for the category pipelines (bestsellers, monthly-sold, -// mid-range, stalker analysis). Upserted on conflict so each ASIN has one row. - -export const categoryProductResults = pgTable( - "category_product_results", +export const productObservations = pgTable( + "product_observations", { id: serial("id").primaryKey(), - asin: text("asin").notNull().unique(), + productAsin: text("product_asin") + .notNull() + .references(() => products.asin), runId: integer("run_id") .notNull() - .references(() => runs.id), - name: text("name").notNull(), - brand: text("brand"), - category: text("category"), - unitCost: real("unit_cost"), + .references(() => runs.id, { onDelete: "cascade" }), + source: text("source").notNull(), + marketplace: text("marketplace").notNull().default("US"), currentPrice: real("current_price"), avgPrice90d: real("avg_price_90d"), - avgPrice90dSheet: real("avg_price_90d_sheet"), - sellingPriceSheet: real("selling_price_sheet"), salesRank: integer("sales_rank"), salesRankAvg90d: integer("sales_rank_avg_90d"), - sellerCount: integer("seller_count"), - amazonIsSeller: boolean("amazon_is_seller"), - amazonBuyboxSharePct90d: real("amazon_buybox_share_pct_90d"), monthlySold: integer("monthly_sold"), rankDrops30d: integer("rank_drops_30d"), rankDrops90d: integer("rank_drops_90d"), + sellerCount: integer("seller_count"), + amazonIsSeller: boolean("amazon_is_seller"), + amazonBuyboxSharePct90d: real("amazon_buybox_share_pct_90d"), fbaFee: real("fba_fee"), fbmFee: real("fbm_fee"), referralPercent: real("referral_percent"), - canSell: text("can_sell"), + canSell: boolean("can_sell"), sellabilityStatus: text("sellability_status"), sellabilityReason: text("sellability_reason"), - verdict: text("verdict").notNull(), - confidence: real("confidence").notNull(), - reasoning: text("reasoning"), + rawProductJson: text("raw_product_json"), fetchedAt: timestamp("fetched_at", { withTimezone: true }).notNull(), }, (t) => [ - index("idx_category_results_run_id").on(t.runId), - index("idx_category_results_verdict").on(t.verdict), - index("idx_category_results_sellability_status").on(t.sellabilityStatus), - index("idx_category_results_fetched_at").on(t.fetchedAt), + index("idx_product_observations_product_time").on( + t.productAsin, + t.fetchedAt.desc(), + ), + index("idx_product_observations_run_id").on(t.runId), + index("idx_product_observations_sellability").on(t.sellabilityStatus), ], ); -// ─── Stalker runs ───────────────────────────────────────────────────────────── - -export const stalkerRuns = pgTable( - "stalker_runs", - { - id: serial("id").primaryKey(), - inputFile: text("input_file").notNull(), - startedAt: timestamp("started_at", { withTimezone: true }).notNull(), - completedAt: timestamp("completed_at", { withTimezone: true }), - requestedAsins: integer("requested_asins").notNull().default(0), - skippedAsins: integer("skipped_asins").notNull().default(0), - scannedAsins: integer("scanned_asins").notNull().default(0), - sourceAsinsWithMatches: integer("source_asins_with_matches") - .notNull() - .default(0), - candidateSellers: integer("candidate_sellers").notNull().default(0), - qualifyingSellers: integer("qualifying_sellers").notNull().default(0), - matchedSellers: integer("matched_sellers").notNull().default(0), - sellerMetadataRequests: integer("seller_metadata_requests") - .notNull() - .default(0), - sellerStorefrontRequests: integer("seller_storefront_requests") - .notNull() - .default(0), - inventorySellabilityCheckedAsins: integer( - "inventory_sellability_checked_asins", - ) - .notNull() - .default(0), - inventorySellabilityAvailableAsins: integer( - "inventory_sellability_available_asins", - ) - .notNull() - .default(0), - inventorySellabilityExcludedAsins: integer( - "inventory_sellability_excluded_asins", - ) - .notNull() - .default(0), - persistedInventoryAsins: integer("persisted_inventory_asins") - .notNull() - .default(0), - status: text("status").notNull(), - errorMessage: text("error_message"), - }, - (t) => [index("idx_stalker_runs_started_at").on(t.startedAt)], -); - -// ─── Stalker ASIN scans ─────────────────────────────────────────────────────── - -export const stalkerAsinScans = pgTable( - "stalker_asin_scans", +export const runItems = pgTable( + "run_items", { id: serial("id").primaryKey(), runId: integer("run_id") .notNull() - .references(() => stalkerRuns.id, { onDelete: "cascade" }), - sourceAsin: text("source_asin").notNull(), - title: text("title"), - offerCount: integer("offer_count").notNull().default(0), - candidateSellerCount: integer("candidate_seller_count") + .references(() => runs.id, { onDelete: "cascade" }), + productAsin: text("product_asin").references(() => products.asin), + sourceInventoryItemId: integer("source_inventory_item_id").references( + (): AnyPgColumn => stalkerInventoryItems.id, + { onDelete: "set null" }, + ), + ordinal: integer("ordinal"), + sourceRow: integer("source_row"), + status: text("status").notNull().default("completed"), + createdAt: timestamp("created_at", { withTimezone: true }) .notNull() - .default(0), - matchedSellerCount: integer("matched_seller_count").notNull().default(0), - fetchedAt: timestamp("fetched_at", { withTimezone: true }).notNull(), - rawProductJson: text("raw_product_json"), + .defaultNow(), }, (t) => [ - unique("uq_stalker_scans_run_asin").on(t.runId, t.sourceAsin), - index("idx_stalker_scans_run_id").on(t.runId), - index("idx_stalker_scans_source_asin").on(t.sourceAsin), + index("idx_run_items_run_id").on(t.runId), + index("idx_run_items_product_asin").on(t.productAsin), ], ); -// ─── Sellers ────────────────────────────────────────────────────────────────── -// General seller registry (was stalker_sellers). +export const sourcingInputs = pgTable("sourcing_inputs", { + runItemId: integer("run_item_id") + .primaryKey() + .references(() => runItems.id, { onDelete: "cascade" }), + suppliedName: text("supplied_name"), + suppliedBrand: text("supplied_brand"), + suppliedCategory: text("supplied_category"), + unitCost: real("unit_cost"), + avgPrice90dSheet: real("avg_price_90d_sheet"), + sellingPriceSheet: real("selling_price_sheet"), + fbaNetSheet: real("fba_net_sheet"), + grossProfitDollar: real("gross_profit_dollar"), + grossProfitPct: real("gross_profit_pct"), + netProfitSheet: real("net_profit_sheet"), + roiSheet: real("roi_sheet"), + moq: integer("moq"), + moqCost: real("moq_cost"), + qtyAvailable: integer("qty_available"), + supplier: text("supplier"), + sourceUrl: text("source_url"), + asinLink: text("asin_link"), + promoCouponCode: text("promo_coupon_code"), + notes: text("notes"), + leadDate: text("lead_date"), +}); + +export const upcResolutions = pgTable( + "upc_resolutions", + { + runItemId: integer("run_item_id") + .primaryKey() + .references(() => runItems.id, { onDelete: "cascade" }), + requestedUpc: text("requested_upc").notNull(), + normalizedUpc: text("normalized_upc").notNull(), + provider: text("provider").notNull(), + status: text("status").notNull(), + reason: text("reason"), + resolvedProductAsin: text("resolved_product_asin").references( + () => products.asin, + ), + resolvedAt: timestamp("resolved_at", { withTimezone: true }) + .notNull() + .defaultNow(), + }, + (t) => [index("idx_upc_resolutions_normalized_upc").on(t.normalizedUpc)], +); + +export const upcResolutionCandidates = pgTable( + "upc_resolution_candidates", + { + runItemId: integer("run_item_id") + .notNull() + .references(() => upcResolutions.runItemId, { onDelete: "cascade" }), + productAsin: text("product_asin") + .notNull() + .references(() => products.asin), + }, + (t) => [ + primaryKey({ columns: [t.runItemId, t.productAsin] }), + index("idx_upc_candidates_product_asin").on(t.productAsin), + ], +); + +export const analysisRevisions = pgTable( + "analysis_revisions", + { + id: serial("id").primaryKey(), + runItemId: integer("run_item_id") + .notNull() + .references(() => runItems.id, { onDelete: "cascade" }), + observationId: integer("observation_id").references( + () => productObservations.id, + { onDelete: "set null" }, + ), + method: analysisMethodEnum("method").notNull(), + decision: analysisDecisionEnum("decision").notNull(), + confidence: real("confidence"), + reasoning: text("reasoning"), + analyzedAt: timestamp("analyzed_at", { withTimezone: true }) + .notNull() + .defaultNow(), + }, + (t) => [ + index("idx_analysis_revisions_run_item_time").on(t.runItemId, t.analyzedAt), + index("idx_analysis_revisions_decision").on(t.decision), + ], +); + +export const supplierScores = pgTable("supplier_scores", { + revisionId: integer("revision_id") + .primaryKey() + .references(() => analysisRevisions.id, { onDelete: "cascade" }), + score: real("score"), + salePrice: real("sale_price"), + fbaFee: real("fba_fee"), + profit: real("profit"), + margin: real("margin"), + roi: real("roi"), + reason: text("reason"), +}); export const sellers = pgTable("sellers", { sellerId: text("seller_id").primaryKey(), @@ -276,15 +356,44 @@ export const sellers = pgTable("sellers", { rawSellerJson: text("raw_seller_json"), }); -// ─── Stalker ASIN sellers ───────────────────────────────────────────────────── +export const stalkerScans = pgTable( + "stalker_scans", + { + id: serial("id").primaryKey(), + runId: integer("run_id") + .notNull() + .references(() => runs.id, { onDelete: "cascade" }), + sourceProductAsin: text("source_product_asin") + .notNull() + .references(() => products.asin), + observationId: integer("observation_id").references( + () => productObservations.id, + { onDelete: "set null" }, + ), + offerCount: integer("offer_count").notNull().default(0), + candidateSellerCount: integer("candidate_seller_count") + .notNull() + .default(0), + matchedSellerCount: integer("matched_seller_count").notNull().default(0), + fetchedAt: timestamp("fetched_at", { withTimezone: true }).notNull(), + }, + (t) => [ + unique("uq_stalker_scans_run_source_product").on( + t.runId, + t.sourceProductAsin, + ), + index("idx_stalker_scans_run_id").on(t.runId), + index("idx_stalker_scans_source_asin").on(t.sourceProductAsin), + ], +); -export const stalkerAsinSellers = pgTable( - "stalker_asin_sellers", +export const stalkerScanSellers = pgTable( + "stalker_scan_sellers", { id: serial("id").primaryKey(), scanId: integer("scan_id") .notNull() - .references(() => stalkerAsinScans.id, { onDelete: "cascade" }), + .references(() => stalkerScans.id, { onDelete: "cascade" }), sellerId: text("seller_id") .notNull() .references(() => sellers.sellerId), @@ -297,47 +406,36 @@ export const stalkerAsinSellers = pgTable( rawOfferJson: text("raw_offer_json"), }, (t) => [ - unique("uq_stalker_asin_sellers_scan_seller").on(t.scanId, t.sellerId), + unique("uq_stalker_scan_sellers_scan_seller").on(t.scanId, t.sellerId), ], ); -// ─── Stalker seller inventory ───────────────────────────────────────────────── - -export const stalkerSellerInventory = pgTable( - "stalker_seller_inventory", +export const stalkerInventoryItems = pgTable( + "stalker_inventory_items", { id: serial("id").primaryKey(), runId: integer("run_id") .notNull() - .references(() => stalkerRuns.id, { onDelete: "cascade" }), + .references(() => runs.id, { onDelete: "cascade" }), sellerId: text("seller_id") .notNull() .references(() => sellers.sellerId), - asin: text("asin").notNull(), - canSell: boolean("can_sell"), - sellabilityStatus: text("sellability_status"), - sellabilityReason: text("sellability_reason"), - productTitle: text("product_title"), - brand: text("brand"), - categoryTree: text("category_tree"), - currentPrice: real("current_price"), - avgPrice90d: real("avg_price_90d"), - salesRank: integer("sales_rank"), - monthlySold: integer("monthly_sold"), - sellerCount: integer("seller_count"), - amazonIsSeller: boolean("amazon_is_seller"), - rawProductJson: text("raw_product_json"), + productAsin: text("product_asin") + .notNull() + .references(() => products.asin), + observationId: integer("observation_id") + .notNull() + .references(() => productObservations.id, { onDelete: "cascade" }), lastSeenAt: timestamp("last_seen_at", { withTimezone: true }).notNull(), rawInventoryJson: text("raw_inventory_json"), }, (t) => [ - unique("uq_stalker_inventory_run_seller_asin").on( + unique("uq_stalker_inventory_items_run_seller_asin").on( t.runId, t.sellerId, - t.asin, + t.productAsin, ), index("idx_stalker_inventory_seller_id").on(t.sellerId), - index("idx_stalker_inventory_asin").on(t.asin), - index("idx_stalker_inventory_product_title").on(t.productTitle), + index("idx_stalker_inventory_product_asin").on(t.productAsin), ], ); diff --git a/src/integrations/keepa.test.ts b/src/integrations/keepa.test.ts index c3cb9a4..0194980 100644 --- a/src/integrations/keepa.test.ts +++ b/src/integrations/keepa.test.ts @@ -42,7 +42,7 @@ test("lookupKeepaUpcs returns found, not_found, and multiple_asins outcomes", as JSON.stringify({ products: [ { - asin: "B000FOUND01", + asin: "B000FND001", upcList: ["012345678901"], stats: { current: [null, null, null, 1234], @@ -51,7 +51,7 @@ test("lookupKeepaUpcs returns found, not_found, and multiple_asins outcomes", as csv: [[5000000, 2999, 5000100]], }, { - asin: "B000MULTI01", + asin: "B000MUL001", upcList: ["098765432109"], stats: { current: [null, null, null, 2000], @@ -60,7 +60,7 @@ test("lookupKeepaUpcs returns found, not_found, and multiple_asins outcomes", as csv: [[1, 1999]], }, { - asin: "B000MULTI02", + asin: "B000MUL002", upcList: ["098765432109"], stats: { current: [null, null, null, 2100], @@ -83,14 +83,14 @@ test("lookupKeepaUpcs returns found, not_found, and multiple_asins outcomes", as ]); expect(details.get("012345678901")?.status).toBe("found"); - expect(details.get("012345678901")?.asin).toBe("B000FOUND01"); + expect(details.get("012345678901")?.asin).toBe("B000FND001"); expect(details.get("012345678901")?.keepaData?.currentPrice).toBe(29.99); expect(details.get("012345678901")?.keepaData?.currentPrice).toBe(29.99); expect(details.get("098765432109")?.status).toBe("multiple_asins"); expect(details.get("098765432109")?.candidateAsins).toEqual([ - "B000MULTI01", - "B000MULTI02", + "B000MUL001", + "B000MUL002", ]); expect(details.get("111111111111")?.status).toBe("not_found"); @@ -100,7 +100,7 @@ test("lookupKeepaUpcs returns found, not_found, and multiple_asins outcomes", as "098765432109", "111111111111", ]); - expect(simpleMap.get("012345678901")).toBe("B000FOUND01"); + expect(simpleMap.get("012345678901")).toBe("B000FND001"); expect(simpleMap.has("098765432109")).toBe(false); expect(simpleMap.has("111111111111")).toBe(false); }); @@ -128,7 +128,7 @@ test("lookupKeepaUpcs keeps partial success when one chunk fails", async () => { JSON.stringify({ products: [ { - asin: "B000LAST001", + asin: "B000LST001", upcList: [secondChunkUpc], stats: { current: [null, null, null, 1000], @@ -148,11 +148,11 @@ test("lookupKeepaUpcs keeps partial success when one chunk fails", async () => { expect(details.get(firstChunkFirstUpc)?.status).toBe("request_failed"); expect(details.get(secondChunkUpc)?.status).toBe("found"); - expect(details.get(secondChunkUpc)?.asin).toBe("B000LAST001"); + expect(details.get(secondChunkUpc)?.asin).toBe("B000LST001"); const simpleMap = await mapUpcsToAsins(upcs); expect(simpleMap.has(firstChunkFirstUpc)).toBe(false); - expect(simpleMap.get(secondChunkUpc)).toBe("B000LAST001"); + expect(simpleMap.get(secondChunkUpc)).toBe("B000LST001"); }); test("lookupKeepaUpcs retries on 429 and succeeds after refill wait", async () => { @@ -175,7 +175,7 @@ test("lookupKeepaUpcs retries on 429 and succeeds after refill wait", async () = JSON.stringify({ products: [ { - asin: "B000RETRY01", + asin: "B000RTY001", upcList: [targetUpc], stats: { current: [null, null, null, 1111], @@ -197,7 +197,7 @@ test("lookupKeepaUpcs retries on 429 and succeeds after refill wait", async () = expect(fetchMock.mock.calls.length).toBe(2); expect(details.get(targetUpc)?.status).toBe("found"); - expect(details.get(targetUpc)?.asin).toBe("B000RETRY01"); + expect(details.get(targetUpc)?.asin).toBe("B000RTY001"); }); test("lookupKeepaUpcs uses lightweight query params for code mapping", async () => { @@ -220,7 +220,7 @@ test("lookupKeepaUpcs uses lightweight query params for code mapping", async () JSON.stringify({ products: [ { - asin: "B000LIGHT01", + asin: "B000LGT001", upcList: [targetUpc], categoryTree: [{ name: "Test Category" }], }, @@ -238,5 +238,5 @@ test("lookupKeepaUpcs uses lightweight query params for code mapping", async () expect(fetchMock.mock.calls.length).toBe(1); expect(details.get(targetUpc)?.status).toBe("found"); - expect(details.get(targetUpc)?.asin).toBe("B000LIGHT01"); + expect(details.get(targetUpc)?.asin).toBe("B000LGT001"); }); diff --git a/src/integrations/keepa.ts b/src/integrations/keepa.ts index 21bc445..128d8a3 100644 --- a/src/integrations/keepa.ts +++ b/src/integrations/keepa.ts @@ -1,5 +1,6 @@ -import { config } from "../config.ts"; -import type { KeepaData, KeepaUpcLookupDetail } from "../types.ts"; +import { config } from "../config.ts"; +import { normalizeAsin } from "../asin.ts"; +import type { KeepaData, KeepaUpcLookupDetail } from "../types.ts"; const KEEPA_BASE = "https://api.keepa.com"; const MAX_ASINS_PER_REQUEST = 100; @@ -224,14 +225,21 @@ function buildFailureDetail( }; } -export async function fetchKeepaDataBatch( - asins: string[], -): Promise> { - const results = new Map(); - - // Split into chunks of MAX_ASINS_PER_REQUEST - for (let i = 0; i < asins.length; i += MAX_ASINS_PER_REQUEST) { - const chunk = asins.slice(i, i + MAX_ASINS_PER_REQUEST); +export async function fetchKeepaDataBatch( + asins: string[], +): Promise> { + const results = new Map(); + const canonicalAsins = Array.from( + new Set( + asins + .map((asin) => normalizeAsin(asin)) + .filter((asin): asin is string => asin !== null), + ), + ); + + // Split into chunks of MAX_ASINS_PER_REQUEST + for (let i = 0; i < canonicalAsins.length; i += MAX_ASINS_PER_REQUEST) { + const chunk = canonicalAsins.slice(i, i + MAX_ASINS_PER_REQUEST); const url = buildProductUrl("asin", chunk, { includeStats: true, includeBuybox: true, @@ -248,11 +256,11 @@ export async function fetchKeepaDataBatch( `Keepa: ${data.products?.length ?? 0} products returned, ${tokensLeft} tokens remaining (refill: ${refillRate}/min)`, ); - if (data.products) { - for (const product of data.products) { - const asin = product.asin; - if (!asin) continue; - results.set(asin, parseKeepaProduct(product)); + if (data.products) { + for (const product of data.products) { + const asin = normalizeAsin(product.asin); + if (!asin) continue; + results.set(asin, parseKeepaProduct(product)); } } } @@ -307,10 +315,10 @@ export async function lookupKeepaUpcs( `Keepa: ${data.products?.length ?? 0} products returned for UPC query, ${tokensLeft} tokens remaining (refill: ${refillRate}/min)`, ); - const byUpc = new Map>(); - for (const product of data.products ?? []) { - const asin = String(product.asin ?? "").trim(); - if (!asin) continue; + const byUpc = new Map>(); + for (const product of data.products ?? []) { + const asin = normalizeAsin(product.asin); + if (!asin) continue; const keepaData = parseKeepaProduct(product); const productUpcs = extractUpcsFromProduct(product); diff --git a/src/integrations/searxng.test.ts b/src/integrations/searxng.test.ts index 6bd013f..3221adf 100644 --- a/src/integrations/searxng.test.ts +++ b/src/integrations/searxng.test.ts @@ -13,6 +13,7 @@ afterAll(() => { test("normalizeAsin uppercases and validates ASINs", () => { expect(normalizeAsin(" b07sn9bhvv ")).toBe("B07SN9BHVV"); + expect(normalizeAsin("0306406152")).toBe("0306406152"); expect(() => normalizeAsin("not-an-asin")).toThrow("Invalid ASIN"); }); diff --git a/src/integrations/searxng.ts b/src/integrations/searxng.ts index b6205e4..7c49419 100644 --- a/src/integrations/searxng.ts +++ b/src/integrations/searxng.ts @@ -1,10 +1,11 @@ +import { normalizeAsin as normalizeCanonicalAsin } from "../asin.ts"; + const DEFAULT_SEARXNG_URL = "https://searxng.nvictor.me/"; const DEFAULT_GOOGLE_CUSTOM_SEARCH_URL = "https://www.googleapis.com/customsearch/v1"; const DEFAULT_SERPAPI_URL = "https://serpapi.com/search.json"; const DEFAULT_TIMEOUT_MS = 10_000; const DEFAULT_MAX_RESULTS = 10; -const ASIN_REGEX = /^B[0-9A-Z]{9}$/; const ASIN_MATCH_REGEX = /\bB[0-9A-Z]{9}\b/gi; const PRICE_LABELS = [ "selling price", @@ -127,16 +128,15 @@ export async function searchProductOffers( } export function normalizeAsin(value: string): string { - const asin = value.trim().toUpperCase(); - if (!ASIN_REGEX.test(asin)) { + const asin = normalizeCanonicalAsin(value); + if (!asin) { throw new Error(`Invalid ASIN: ${value}`); } return asin; } function getAsinQuery(value: string): string | undefined { - const normalized = value.trim().toUpperCase(); - return ASIN_REGEX.test(normalized) ? normalized : undefined; + return normalizeCanonicalAsin(value) ?? undefined; } async function fetchSearxngResults( diff --git a/src/integrations/sp-api.test.ts b/src/integrations/sp-api.test.ts index 284d19d..69d4a51 100644 --- a/src/integrations/sp-api.test.ts +++ b/src/integrations/sp-api.test.ts @@ -20,6 +20,15 @@ test("parseCatalogUpcLookupResponse marks no match", () => { expect(detail.asin).toBeNull(); }); +test("parseCatalogUpcLookupResponse ignores invalid ASIN identifiers", () => { + const detail = parseCatalogUpcLookupResponse("012345678901", { + items: [{ asin: "012345678901" }], + }); + + expect(detail.status).toBe("not_found"); + expect(detail.asin).toBeNull(); +}); + test("parseCatalogUpcLookupResponse marks multiple ASINs", () => { const detail = parseCatalogUpcLookupResponse("012345678901", { payload: { diff --git a/src/integrations/sp-api.ts b/src/integrations/sp-api.ts index 8230258..b7549f4 100644 --- a/src/integrations/sp-api.ts +++ b/src/integrations/sp-api.ts @@ -1,4 +1,5 @@ import { SellingPartner } from "amazon-sp-api"; +import { normalizeAsin } from "../asin.ts"; import { config } from "../config.ts"; import type { KeepaUpcLookupStatus, @@ -222,8 +223,7 @@ function extractCatalogAsin(item: any): string | null { item?.identifiers?.marketplaceASIN?.asin ?? item?.Identifiers?.MarketplaceASIN?.ASIN; if (typeof raw !== "string") return null; - const asin = raw.trim().toUpperCase(); - return asin ? asin : null; + return normalizeAsin(raw); } export function parseCatalogUpcLookupResponse( diff --git a/src/reader.ts b/src/reader.ts index 0c72956..23d7d26 100644 --- a/src/reader.ts +++ b/src/reader.ts @@ -1,8 +1,7 @@ -import * as XLSX from "xlsx"; -import type { ProductRecord } from "./types.ts"; - -const ASIN_REGEX = /^B[0-9A-Z]{9}$/; - +import * as XLSX from "xlsx"; +import { normalizeAsin } from "./asin.ts"; +import type { ProductRecord } from "./types.ts"; + const COLUMN_CANDIDATES = { asin: ["asin"], name: ["name", "product name", "title", "product title"], @@ -132,14 +131,12 @@ function getKnownColumns(columns: ColumnMap): Set { return new Set(Object.values(columns).filter((column): column is string => !!column)); } -function parseAsin(value: unknown): string | undefined { - const asin = String(value ?? "") - .trim() - .toUpperCase(); - if (!asin || !ASIN_REGEX.test(asin)) { - console.warn(`Skipping invalid ASIN: "${asin}"`); - return undefined; - } +function parseAsin(value: unknown): string | undefined { + const asin = normalizeAsin(value); + if (!asin) { + console.warn(`Skipping invalid ASIN: "${String(value ?? "").trim()}"`); + return undefined; + } return asin; } diff --git a/src/server.ts b/src/server.ts index 9916a46..201a566 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,112 +1,33 @@ import index from "./web/index.html"; import * as XLSX from "xlsx"; -import { client } from "./db/index.ts"; +import { normalizeAsin } from "./asin.ts"; +import { db, client } from "./db/index.ts"; +import { analysisRevisions } from "./db/schema.ts"; +import { insertObservation, refreshRunStats } from "./db/persistence.ts"; import { fetchKeepaDataBatch, lookupKeepaUpcs, mapUpcsToAsins, } from "./integrations/keepa.ts"; -import { runUpcFileAnalysis } from "./supplier/upc-file-analysis.ts"; -import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./integrations/sp-api.ts"; import { analyzeProducts } from "./integrations/llm.ts"; +import { + fetchSellabilityBatch, + fetchSpApiPricingAndFees, +} from "./integrations/sp-api.ts"; +import { runUpcFileAnalysis } from "./supplier/upc-file-analysis.ts"; import type { + AnalysisResult, EnrichedProduct, KeepaUpcLookupDetail, ProductRecord, SpApiData, } from "./types.ts"; -type ProcessType = "lead_analysis" | "category_analysis"; - -type RunRecord = { - processType: ProcessType; - runId: number; - timestamp: string; - status: string; - jobType: string; - source: string | null; - output: string | null; - totalProducts: number; - fbaCount: number; - fbmCount: number; - skipCount: number; -}; - -type ProductListRecord = { - processType: ProcessType; - runId: number; - asin: string; - product_name: string | null; - brand: string | null; - category: string | null; - verdict: "FBA" | "FBM" | "SKIP"; - confidence: number | null; - sellability_status: string | null; - monthly_sold: number | null; - seller_count: number | null; - amazon_is_seller: boolean | null; - amazon_buybox_share_pct_90d: number | null; - sales_rank: number | null; - current_price: number | null; - avg_price_90d: number | null; - reasoning: string | null; - fetched_at: string; -}; - -type StalkerResultRecord = { - runId: number; - started_at: string; - status: string; - input_file: string; - seller_id: string; - seller_name: string | null; - rating: number | null; - rating_count: number | null; - storefront_asin_total: number | null; - persisted_inventory_sample_count: number | null; - discovered_from_count: number; - first_seen_at: string; - last_seen_at: string; - persisted_inventory_asin_count: number; - inventory_sample_asins: string | null; -}; - -type StalkerProductRecord = { - runId: number; - started_at: string; - seller_id: string; - seller_name: string | null; - rating: number | null; - rating_count: number | null; - asin: string; - can_sell: boolean; - sellability_status: string; - sellability_reason: string | null; - product_title: string | null; - brand: string | null; - category_tree: string | null; - current_price: number | null; - avg_price_90d: number | null; - sales_rank: number | null; - monthly_sold: number | null; - seller_count: number | null; - amazon_is_seller: boolean | null; - verdict: string | null; - confidence: number | null; - reasoning: string | null; - last_seen_at: string; -}; - const DEFAULT_PAGE_SIZE = 25; const MAX_PAGE_SIZE = 200; -const ASIN_PATTERN = /^[A-Z0-9]{10}$/; const MAX_UPCS_PER_REQUEST = 1000; const USE_CLAUDE = process.argv.includes("--claude"); -// --------------------------------------------------------------------------- -// Postgres helpers -// --------------------------------------------------------------------------- - function toPostgresSql(query: string): string { let n = 0; return query.replace(/\?/g, () => `$${++n}`); @@ -132,10 +53,6 @@ async function pgRun(query: string, params: unknown[] = []): Promise { return result.count; } -// --------------------------------------------------------------------------- -// Response helpers -// --------------------------------------------------------------------------- - function json(data: unknown, status = 200): Response { return new Response(JSON.stringify(data), { status, @@ -145,7 +62,6 @@ function json(data: unknown, status = 200): Response { function csv(text: string, filename: string): Response { return new Response(text, { - status: 200, headers: { "content-type": "text/csv; charset=utf-8", "content-disposition": `attachment; filename="${filename}"`, @@ -155,7 +71,6 @@ function csv(text: string, filename: string): Response { function xlsx(buffer: ArrayBuffer, filename: string): Response { return new Response(buffer, { - status: 200, headers: { "content-type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", @@ -165,144 +80,97 @@ function xlsx(buffer: ArrayBuffer, filename: string): Response { } function parseIntParam(value: string | null, fallback: number): number { - if (!value) return fallback; - const parsed = Number.parseInt(value, 10); - if (!Number.isFinite(parsed) || parsed < 1) return fallback; - return parsed; + const parsed = Number.parseInt(String(value), 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; } -function normalizeAsin(value: string): string { - return value.trim().toUpperCase(); +function pageInput(filters: URLSearchParams) { + const page = parseIntParam(filters.get("page"), 1); + const pageSize = Math.min( + parseIntParam(filters.get("pageSize"), DEFAULT_PAGE_SIZE), + MAX_PAGE_SIZE, + ); + return { page, pageSize, offset: (page - 1) * pageSize }; } -function isValidAsin(value: string): boolean { - return ASIN_PATTERN.test(value); +function escapeCsvValue(value: unknown): string { + if (value == null) return ""; + const escaped = String(value).replaceAll('"', '""'); + return /[",\n]/.test(escaped) ? `"${escaped}"` : escaped; +} + +function safeSort( + input: string | null, + columns: Record, + fallback: string, +): string { + if (!input) return fallback; + const parts = input + .split(",") + .map((part) => { + const [key, direction] = part.trim().split(":"); + const column = key ? columns[key] : undefined; + if (!column) return null; + return `${column} ${direction?.toUpperCase() === "DESC" ? "DESC" : "ASC"}`; + }) + .filter((value): value is string => value != null); + return parts.join(", ") || fallback; } function splitRawUpcValues(input: string): string[] { - return input - .split(/[\s,;|]+/) - .map((chunk) => chunk.trim()) - .filter(Boolean); + return input.split(/[\s,;|]+/).map((v) => v.trim()).filter(Boolean); } -function collectUpcsFromUnknown(value: unknown, target: string[]): void { +function collectUpcs(value: unknown, target: string[]): void { if (typeof value === "string") { target.push(...splitRawUpcValues(value)); - return; - } - - if (typeof value === "number" && Number.isFinite(value)) { + } else if (typeof value === "number" && Number.isFinite(value)) { target.push(String(Math.trunc(value))); - return; + } else if (Array.isArray(value)) { + value.forEach((entry) => collectUpcs(entry, target)); } - - if (Array.isArray(value)) { - for (const item of value) { - collectUpcsFromUnknown(item, target); - } - } -} - -function normalizeAndDedupeUpcs(values: string[]): string[] { - const seen = new Set(); - const normalized: string[] = []; - - for (const value of values) { - const upc = value.trim(); - if (!upc || seen.has(upc)) continue; - seen.add(upc); - normalized.push(upc); - } - - return normalized; -} - -function parseUpcsFromSearchParams(params: URLSearchParams): string[] { - const parsed: string[] = []; - for (const value of params.getAll("upc")) { - collectUpcsFromUnknown(value, parsed); - } - - const upcsValue = params.get("upcs"); - if (upcsValue) { - collectUpcsFromUnknown(upcsValue, parsed); - } - - return normalizeAndDedupeUpcs(parsed); } async function parseUpcsFromRequest(req: Request): Promise { + const parsed: string[] = []; if (req.method === "GET") { - const url = new URL(req.url); - return parseUpcsFromSearchParams(url.searchParams); - } - - if (req.method !== "POST") { + const params = new URL(req.url).searchParams; + params.getAll("upc").forEach((value) => collectUpcs(value, parsed)); + collectUpcs(params.get("upcs"), parsed); + } else if (req.method === "POST") { + let body: unknown; + try { + body = await req.json(); + } catch { + throw new Error("Invalid JSON body"); + } + if (body && typeof body === "object" && "upcs" in body) { + collectUpcs((body as { upcs?: unknown }).upcs, parsed); + } else { + collectUpcs(body, parsed); + } + } else { throw new Error("Method not allowed"); } - - let body: unknown; - try { - body = await req.json(); - } catch { - throw new Error("Invalid JSON body"); - } - - const parsed: string[] = []; - if (body && typeof body === "object" && "upcs" in body) { - collectUpcsFromUnknown((body as { upcs?: unknown }).upcs, parsed); - } else { - collectUpcsFromUnknown(body, parsed); - } - - return normalizeAndDedupeUpcs(parsed); + return [...new Set(parsed)]; } -function validateUpcRequest(upcs: string[]): string | null { - if (upcs.length === 0) { - return "Provide at least one UPC via query (?upc=...) or JSON body { upcs: [...] }"; - } - +function validateUpcs(upcs: string[]): string | null { + if (upcs.length === 0) return "Provide at least one UPC."; if (upcs.length > MAX_UPCS_PER_REQUEST) { return `Too many UPCs. Maximum allowed per request is ${MAX_UPCS_PER_REQUEST}.`; } - return null; } -function summarizeLookupStatuses( - details: KeepaUpcLookupDetail[], -): Record { - const counts: Record = {}; - for (const detail of details) { - counts[detail.status] = (counts[detail.status] ?? 0) + 1; - } - return counts; +function summarizeLookupStatuses(items: KeepaUpcLookupDetail[]) { + return items.reduce>((counts, item) => { + counts[item.status] = (counts[item.status] ?? 0) + 1; + return counts; + }, {}); } -function parsePositiveIntField( - value: unknown, - fieldName: string, -): number | undefined { - if (value == null) return undefined; - if (typeof value === "number") { - if (!Number.isInteger(value) || value < 1) { - throw new Error(`${fieldName} must be a positive integer`); - } - return value; - } - if (typeof value === "string" && value.trim().length > 0) { - const parsed = Number.parseInt(value, 10); - if (!Number.isFinite(parsed) || parsed < 1) { - throw new Error(`${fieldName} must be a positive integer`); - } - return parsed; - } - throw new Error(`${fieldName} must be a positive integer`); -} - -type UpcFileProcessRequest = { +type UpcFileRequest = { inputFile: string; outputFile?: string; inputBatchSize?: number; @@ -310,1316 +178,351 @@ type UpcFileProcessRequest = { maxRows?: number; }; -async function parseUpcFileProcessRequest( - req: Request, -): Promise { - if (req.method !== "POST") { - throw new Error("Method not allowed"); +function positiveField(value: unknown, name: string): number | undefined { + if (value == null) return undefined; + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed < 1) { + throw new Error(`${name} must be a positive integer`); } + return parsed; +} - let body: unknown; +async function parseUpcFileRequest(req: Request): Promise { + if (req.method !== "POST") throw new Error("Method not allowed"); + let body: Record; try { - body = await req.json(); + body = (await req.json()) as Record; } catch { throw new Error("Invalid JSON body"); } - - if (!body || typeof body !== "object") { - throw new Error("Request body must be an object"); + if (typeof body.inputFile !== "string" || !body.inputFile.trim()) { + throw new Error("inputFile is required"); } - - const parsedBody = body as Record; - const inputFileValue = parsedBody.inputFile; - if ( - typeof inputFileValue !== "string" || - inputFileValue.trim().length === 0 - ) { - throw new Error("inputFile is required and must be a non-empty string"); - } - - const outputFileValue = parsedBody.outputFile; - if ( - outputFileValue != null && - (typeof outputFileValue !== "string" || outputFileValue.trim().length === 0) - ) { - throw new Error("outputFile must be a non-empty string when provided"); - } - return { - inputFile: inputFileValue.trim(), + inputFile: body.inputFile, outputFile: - typeof outputFileValue === "string" ? outputFileValue.trim() : undefined, - inputBatchSize: parsePositiveIntField( - parsedBody.inputBatchSize, - "inputBatchSize", - ), - upcLookupBatchSize: parsePositiveIntField( - parsedBody.upcLookupBatchSize, + typeof body.outputFile === "string" ? body.outputFile : undefined, + inputBatchSize: positiveField(body.inputBatchSize, "inputBatchSize"), + upcLookupBatchSize: positiveField( + body.upcLookupBatchSize, "upcLookupBatchSize", ), - maxRows: parsePositiveIntField(parsedBody.maxRows, "maxRows"), - }; -} - -function parseSort( - sortParam: string | null, - allowed: Set, - fallback: string, -): string { - if (!sortParam) return fallback; - const clauses = sortParam - .split(",") - .map((chunk) => chunk.trim()) - .filter(Boolean) - .map((chunk) => { - const [fieldRaw, dirRaw] = chunk.split(":"); - const field = fieldRaw?.trim(); - const dir = dirRaw?.trim().toUpperCase() === "DESC" ? "DESC" : "ASC"; - if (!field || !allowed.has(field)) return null; - return `${field} ${dir}`; - }) - .filter((value): value is string => value !== null); - - return clauses.length > 0 ? clauses.join(", ") : fallback; -} - -function parseResultSort( - sortParam: string | null, - allowed: Set, - fallback: string, -): string { - if (!sortParam) return fallback; - const clauses = sortParam - .split(",") - .map((chunk) => chunk.trim()) - .filter(Boolean) - .map((chunk) => { - const [fieldRaw, dirRaw] = chunk.split(":"); - const field = fieldRaw?.trim(); - const dir = dirRaw?.trim().toUpperCase() === "DESC" ? "DESC" : "ASC"; - if (!field || !allowed.has(field)) return null; - if (field === "monthly_sold") - return `CAST(COALESCE(monthly_sold, 0) AS INTEGER) ${dir}`; - if (field === "amazon_is_seller") - return `CAST(COALESCE(amazon_is_seller, false) AS INTEGER) ${dir}`; - if (field === "amazon_buybox_share_pct_90d") { - return `CAST(COALESCE(amazon_buybox_share_pct_90d, 0) AS REAL) ${dir}`; - } - return `${field} ${dir}`; - }) - .filter((value): value is string => value !== null); - - return clauses.length > 0 ? clauses.join(", ") : fallback; -} - -function escapeCsvValue(value: unknown): string { - if (value === null || value === undefined) return ""; - const text = String(value); - const escaped = text.replaceAll('"', '""'); - return /[",\n]/.test(escaped) ? `"${escaped}"` : escaped; -} - -function parseResultFilters( - processType: ProcessType, - runId: number, - filters: URLSearchParams, -) { - const q = filters.get("q")?.trim() || ""; - const verdict = filters.get("verdict")?.trim(); - const sellabilityStatus = filters.get("sellabilityStatus")?.trim(); - const minConfidence = filters.get("minConfidence")?.trim(); - const maxConfidence = filters.get("maxConfidence")?.trim(); - const amazonIsSeller = filters.get("amazonIsSeller")?.trim(); - - const conditions: string[] = ["run_id = ?"]; - const params: Array = [runId]; - - if (verdict) { - conditions.push("verdict = ?"); - params.push(verdict); - } - - if (sellabilityStatus) { - conditions.push("sellability_status = ?"); - params.push(sellabilityStatus); - } - - if (minConfidence) { - conditions.push("confidence >= ?"); - params.push(Number(minConfidence)); - } - - if (maxConfidence) { - conditions.push("confidence <= ?"); - params.push(Number(maxConfidence)); - } - - if (amazonIsSeller === "yes") { - conditions.push("amazon_is_seller = true"); - } else if (amazonIsSeller === "no") { - conditions.push("amazon_is_seller = false"); - } - - if (q) { - const wildcard = `%${q}%`; - if (processType === "lead_analysis") { - conditions.push( - "(asin LIKE ? OR product_name LIKE ? OR brand LIKE ? OR category LIKE ? OR reasoning LIKE ?)", - ); - params.push(wildcard, wildcard, wildcard, wildcard, wildcard); - } else { - conditions.push( - "(asin LIKE ? OR name LIKE ? OR brand LIKE ? OR category LIKE ? OR reasoning LIKE ?)", - ); - params.push(wildcard, wildcard, wildcard, wildcard, wildcard); - } - } - - return { - where: `WHERE ${conditions.join(" AND ")}`, - params, + maxRows: positiveField(body.maxRows, "maxRows"), }; } async function getRuns(filters: URLSearchParams) { - const q = filters.get("q")?.trim() || ""; - const processType = filters.get("processType")?.trim(); + const { page, pageSize, offset } = pageInput(filters); + const conditions: string[] = ["TRUE"]; + const params: unknown[] = []; + const type = filters.get("processType")?.trim(); const status = filters.get("status")?.trim(); - const startDate = filters.get("startDate")?.trim(); - const endDate = filters.get("endDate")?.trim(); - const page = parseIntParam(filters.get("page"), 1); - const pageSize = Math.min( - parseIntParam(filters.get("pageSize"), DEFAULT_PAGE_SIZE), - MAX_PAGE_SIZE, - ); - const offset = (page - 1) * pageSize; - - // Map client sort keys to actual column names / expressions - const allowedSortMap: Record = { - timestamp: "started_at", - status: "status", - totalProducts: - "COALESCE(CASE WHEN type = 'lead_analysis' THEN total_products ELSE top_asins_checked END, 0)", - fbaCount: "COALESCE(fba_count, 0)", - fbmCount: "COALESCE(fbm_count, 0)", - skipCount: "COALESCE(skip_count, 0)", - runId: "id", - jobType: - "CASE type WHEN 'lead_analysis' THEN COALESCE(input_file, 'lead_file_analysis') ELSE COALESCE(category_label, 'category_analysis') END", - }; - - // Build ORDER BY using actual column expressions - const sortParam = filters.get("sort"); - let orderBy = - "started_at DESC, id DESC"; - if (sortParam) { - const clauses = sortParam - .split(",") - .map((chunk) => chunk.trim()) - .filter(Boolean) - .map((chunk) => { - const [fieldRaw, dirRaw] = chunk.split(":"); - const field = fieldRaw?.trim(); - const dir = dirRaw?.trim().toUpperCase() === "DESC" ? "DESC" : "ASC"; - if (!field || !allowedSortMap[field]) return null; - return `${allowedSortMap[field]} ${dir}`; - }) - .filter((value): value is string => value !== null); - if (clauses.length > 0) orderBy = clauses.join(", "); + const q = filters.get("q")?.trim(); + if (type) { + conditions.push("r.type::text = ?"); + params.push(type); } - - const conditions: string[] = [ - "type IN ('lead_analysis', 'category_analysis')", - ]; - const params: Array = []; - - if (processType === "lead_analysis" || processType === "category_analysis") { - conditions.push("type = ?"); - params.push(processType); - } - if (status) { - conditions.push("status = ?"); + conditions.push("r.status::text = ?"); params.push(status); } - - if (startDate) { - conditions.push("started_at >= ?"); - params.push(startDate); + if (filters.get("startDate")) { + conditions.push("r.started_at >= ?"); + params.push(filters.get("startDate")); } - - if (endDate) { - conditions.push("started_at <= ?"); - params.push(endDate); + if (filters.get("endDate")) { + conditions.push("r.started_at <= ?"); + params.push(filters.get("endDate")); } + if (q) { + conditions.push( + "(COALESCE(r.input_file, '') ILIKE ? OR COALESCE(cd.category_label, '') ILIKE ? OR CAST(r.id AS text) ILIKE ?)", + ); + params.push(`%${q}%`, `%${q}%`, `%${q}%`); + } + const where = `WHERE ${conditions.join(" AND ")}`; + const sort = safeSort( + filters.get("sort"), + { + runId: "r.id", + processType: "r.type", + timestamp: "r.started_at", + status: "r.status", + jobType: "COALESCE(cd.category_label, r.input_file, r.type::text)", + totalProducts: "COALESCE(stats.processed_count, 0)", + fbaCount: "COALESCE(stats.fba_count, 0)", + fbmCount: "COALESCE(stats.fbm_count, 0)", + skipCount: "COALESCE(stats.skip_count, 0)", + }, + "r.started_at DESC, r.id DESC", + ); + const join = ` + FROM runs r + LEFT JOIN analysis_run_stats stats ON stats.run_id = r.id + LEFT JOIN category_run_details cd ON cd.run_id = r.id`; + const totalRow = await pgGet<{ total: string }>( + `SELECT COUNT(*) AS total ${join} ${where}`, + params, + ); + const items = await pgAll( + `SELECT r.type AS "processType", r.id AS "runId", + r.started_at AS timestamp, r.status, r.input_file AS source, + r.output_file AS output, + COALESCE(cd.category_label, r.input_file, r.type::text) AS "jobType", + COALESCE(stats.processed_count, 0) AS "totalProducts", + COALESCE(stats.fba_count, 0) AS "fbaCount", + COALESCE(stats.fbm_count, 0) AS "fbmCount", + COALESCE(stats.buy_count, 0) AS "buyCount", + COALESCE(stats.watch_count, 0) AS "watchCount", + COALESCE(stats.skip_count, 0) AS "skipCount" + ${join} ${where} ORDER BY ${sort} LIMIT ? OFFSET ?`, + [...params, pageSize, offset], + ); + const total = Number(totalRow?.total ?? 0); + return { items, page, pageSize, total, totalPages: Math.max(1, Math.ceil(total / pageSize)) }; +} +async function getRun(runId: number) { + return pgGet( + `SELECT r.type AS "processType", r.id AS "runId", r.parent_run_id AS "parentRunId", + r.started_at AS timestamp, r.completed_at AS "completedAt", r.status, + r.input_file AS source, r.output_file AS output, r.error_message AS "errorMessage", + COALESCE(cd.category_label, r.input_file, r.type::text) AS "jobType", + cd.category_id AS "categoryId", cd.checked_asin_count AS "checkedAsins", + COALESCE(stats.processed_count, 0) AS "totalProducts", + COALESCE(stats.available_count, 0) AS "availableAsins", + COALESCE(stats.fba_count, 0) AS "fbaCount", + COALESCE(stats.fbm_count, 0) AS "fbmCount", + COALESCE(stats.buy_count, 0) AS "buyCount", + COALESCE(stats.watch_count, 0) AS "watchCount", + COALESCE(stats.skip_count, 0) AS "skipCount" + FROM runs r + LEFT JOIN analysis_run_stats stats ON stats.run_id = r.id + LEFT JOIN category_run_details cd ON cd.run_id = r.id + WHERE r.id = ?`, + [runId], + ); +} + +const ITEM_ROWS = ` + WITH latest_revision AS ( + SELECT DISTINCT ON (ar.run_item_id) + ar.run_item_id, ar.id AS revision_id, ar.decision, ar.confidence, + ar.reasoning, ar.analyzed_at, ar.observation_id + FROM analysis_revisions ar + ORDER BY ar.run_item_id, ar.analyzed_at DESC, ar.id DESC + ) + SELECT ri.id AS item_id, ri.run_id, r.type AS process_type, + ri.product_asin, COALESCE(ri.product_asin, ur.normalized_upc) AS asin, + COALESCE(p.name, si.supplied_name, ur.normalized_upc) AS product_name, + COALESCE(p.brand, si.supplied_brand) AS brand, + COALESCE(p.category, si.supplied_category) AS category, + si.unit_cost, si.avg_price_90d_sheet, si.selling_price_sheet, + observation.current_price, observation.avg_price_90d, + observation.sales_rank, observation.sales_rank_avg_90d, + observation.seller_count, observation.amazon_is_seller, + observation.amazon_buybox_share_pct_90d, observation.monthly_sold, + observation.rank_drops_30d, observation.rank_drops_90d, + observation.fba_fee, observation.fbm_fee, observation.referral_percent, + observation.can_sell, observation.sellability_status, + observation.sellability_reason, revision.decision AS verdict, + revision.confidence, revision.reasoning, + revision.analyzed_at AS fetched_at, ur.normalized_upc AS upc, + ur.status AS upc_lookup_status + FROM run_items ri + JOIN runs r ON r.id = ri.run_id + LEFT JOIN products p ON p.asin = ri.product_asin + LEFT JOIN sourcing_inputs si ON si.run_item_id = ri.id + LEFT JOIN upc_resolutions ur ON ur.run_item_id = ri.id + LEFT JOIN latest_revision revision ON revision.run_item_id = ri.id + LEFT JOIN product_observations observation ON observation.id = revision.observation_id +`; + +function itemFilters(filters: URLSearchParams, runId?: number) { + const conditions: string[] = []; + const params: unknown[] = []; + if (runId != null) { + conditions.push("run_id = ?"); + params.push(runId); + } + const verdict = filters.get("verdict")?.trim(); + if (verdict) { + conditions.push("verdict::text = ?"); + params.push(verdict); + } + if (filters.get("sellabilityStatus")) { + conditions.push("sellability_status = ?"); + params.push(filters.get("sellabilityStatus")); + } + const minConfidence = Number(filters.get("minConfidence")); + const maxConfidence = Number(filters.get("maxConfidence")); + if (filters.get("minConfidence") && Number.isFinite(minConfidence)) { + conditions.push("confidence >= ?"); + params.push(minConfidence); + } + if (filters.get("maxConfidence") && Number.isFinite(maxConfidence)) { + conditions.push("confidence <= ?"); + params.push(maxConfidence); + } + if (filters.get("amazonIsSeller") === "yes") { + conditions.push("amazon_is_seller = true"); + } else if (filters.get("amazonIsSeller") === "no") { + conditions.push("amazon_is_seller = false"); + } + const q = filters.get("q")?.trim(); if (q) { const wildcard = `%${q}%`; conditions.push( - `( - input_file LIKE ? - OR category_label LIKE ? - OR CAST(category_id AS TEXT) LIKE ? - OR output_file LIKE ? - OR CAST(id AS TEXT) LIKE ? - )`, + "(COALESCE(asin, '') ILIKE ? OR COALESCE(product_name, '') ILIKE ? OR COALESCE(brand, '') ILIKE ? OR COALESCE(category, '') ILIKE ? OR COALESCE(reasoning, '') ILIKE ?)", ); params.push(wildcard, wildcard, wildcard, wildcard, wildcard); } - - const where = `WHERE ${conditions.join(" AND ")}`; - - const totalRow = await pgGet<{ total: string }>( - `SELECT COUNT(*) AS total FROM runs ${where}`, - params, - ); - const total = Number(totalRow?.total ?? 0); - - const items = await pgAll>( - `SELECT - type AS "processType", - id AS "runId", - started_at AS timestamp, - status, - CASE type - WHEN 'lead_analysis' THEN COALESCE(input_file, 'lead_file_analysis') - ELSE COALESCE(category_label, 'category_analysis') - END AS "jobType", - CASE type WHEN 'lead_analysis' THEN input_file ELSE CAST(category_id AS TEXT) END AS source, - CASE type WHEN 'lead_analysis' THEN output_file ELSE NULL END AS output, - COALESCE(CASE WHEN type = 'lead_analysis' THEN total_products ELSE top_asins_checked END, 0) AS "totalProducts", - COALESCE(fba_count, 0) AS "fbaCount", - COALESCE(fbm_count, 0) AS "fbmCount", - COALESCE(skip_count, 0) AS "skipCount" - FROM runs - ${where} - ORDER BY ${orderBy} - LIMIT ? OFFSET ?`, - [...params, pageSize, offset], - ); - return { - items: items as RunRecord[], - page, - pageSize, - total, - totalPages: Math.max(1, Math.ceil(total / pageSize)), - }; -} - -async function getProductList(filters: URLSearchParams) { - const q = filters.get("q")?.trim() || ""; - const verdict = filters.get("verdict")?.trim(); - const amazonIsSeller = filters.get("amazonIsSeller")?.trim(); - const page = parseIntParam(filters.get("page"), 1); - const pageSize = Math.min( - parseIntParam(filters.get("pageSize"), DEFAULT_PAGE_SIZE), - MAX_PAGE_SIZE, - ); - const offset = (page - 1) * pageSize; - - const conditions: string[] = []; - const params: Array = []; - - if (verdict === "FBA" || verdict === "FBM" || verdict === "SKIP") { - conditions.push("verdict = ?"); - params.push(verdict); - } - - if (amazonIsSeller === "yes") { - conditions.push("amazon_is_seller = true"); - } else if (amazonIsSeller === "no") { - conditions.push("amazon_is_seller = false"); - } - - if (q) { - const wildcard = `%${q}%`; - conditions.push( - "(asin LIKE ? OR product_name LIKE ? OR brand LIKE ? OR category LIKE ?)", - ); - params.push(wildcard, wildcard, wildcard, wildcard); - } - - const where = - conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : ""; - const allowedSort = new Set([ - "asin", - "verdict", - "monthly_sold", - "seller_count", - "amazon_is_seller", - "amazon_buybox_share_pct_90d", - "sales_rank", - "current_price", - "product_name", - "brand", - "category", - "avg_price_90d", - "confidence", - "reasoning", - "fetched_at", - ]); - const orderBy = parseResultSort( - filters.get("sort"), - allowedSort, - "CAST(COALESCE(monthly_sold, 0) AS INTEGER) DESC, fetched_at DESC", - ); - - const baseUnion = ` - SELECT - 'lead_analysis' AS "processType", - run_id AS "runId", - asin, - product_name, - brand, - category, - verdict, - confidence, - sellability_status, - monthly_sold, - seller_count, - amazon_is_seller, - amazon_buybox_share_pct_90d, - sales_rank, - current_price, - avg_price_90d, - reasoning, - fetched_at - FROM analysis_results - UNION ALL - SELECT - 'category_analysis' AS "processType", - run_id AS "runId", - asin, - name AS product_name, - brand, - category, - verdict, - confidence, - sellability_status, - monthly_sold, - seller_count, - amazon_is_seller, - amazon_buybox_share_pct_90d, - sales_rank, - current_price, - avg_price_90d, - reasoning, - fetched_at - FROM category_product_results - `; - - const totalRow = await pgGet<{ total: string }>( - `SELECT COUNT(*) AS total FROM (${baseUnion}) all_products ${where}`, - params, - ); - const total = Number(totalRow?.total ?? 0); - - const items = await pgAll( - `SELECT * FROM (${baseUnion}) all_products ${where} ORDER BY ${orderBy} LIMIT ? OFFSET ?`, - [...params, pageSize, offset], - ); - - return { - items, - page, - pageSize, - total, - totalPages: Math.max(1, Math.ceil(total / pageSize)), - }; -} - -function parseStalkerFilters(filters: URLSearchParams) { - const q = filters.get("q")?.trim() || ""; - const sellerId = filters.get("sellerId")?.trim().toUpperCase() || ""; - const runIdRaw = filters.get("runId")?.trim() || ""; - const minRatingCountRaw = filters.get("minRatingCount")?.trim() || ""; - const maxRatingCountRaw = filters.get("maxRatingCount")?.trim() || ""; - - const conditions: string[] = []; - const params: Array = []; - - if (runIdRaw) { - const runId = Number(runIdRaw); - if (Number.isInteger(runId) && runId > 0) { - conditions.push("r.id = ?"); - params.push(runId); - } - } - - if (sellerId) { - conditions.push("s.seller_id = ?"); - params.push(sellerId); - } - - if (minRatingCountRaw) { - conditions.push("s.rating_count >= ?"); - params.push(Number(minRatingCountRaw)); - } - - if (maxRatingCountRaw) { - conditions.push("s.rating_count <= ?"); - params.push(Number(maxRatingCountRaw)); - } - - if (q) { - const wildcard = `%${q}%`; - conditions.push( - `(s.seller_id LIKE ? OR s.seller_name LIKE ? OR EXISTS ( - SELECT 1 FROM stalker_seller_inventory inv_q - WHERE inv_q.run_id = r.id - AND inv_q.seller_id = s.seller_id - AND inv_q.asin LIKE ? - ))`, - ); - params.push(wildcard, wildcard, wildcard); - } - - return { - where: conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "", + where: conditions.length ? `WHERE ${conditions.join(" AND ")}` : "", params, }; } -function parseStalkerSort(sortParam: string | null): string { - const allowedSort = new Set([ - "runId", - "started_at", - "seller_id", - "seller_name", - "rating", - "rating_count", - "discovered_from_count", - "persisted_inventory_asin_count", - "storefront_asin_total", - "last_seen_at", - ]); - const parsed = parseSort( - sortParam, - allowedSort, - "persisted_inventory_asin_count DESC, last_seen_at DESC, seller_id ASC", - ); - - return parsed - .replaceAll("runId", "runId") - .replaceAll("rating_count", "rating_count") - .replaceAll( - "persisted_inventory_asin_count", - "persisted_inventory_asin_count", - ) - .replaceAll("storefront_asin_total", "storefront_asin_total"); -} - -async function getStalkerResults(filters: URLSearchParams) { - const page = parseIntParam(filters.get("page"), 1); - const pageSize = Math.min( - parseIntParam(filters.get("pageSize"), DEFAULT_PAGE_SIZE), - MAX_PAGE_SIZE, - ); - const offset = (page - 1) * pageSize; - const { where, params } = parseStalkerFilters(filters); - const orderBy = parseStalkerSort(filters.get("sort")); - - const baseSelect = ` - SELECT - r.id AS "runId", - r.started_at, - r.status, - r.input_file, - s.seller_id, - s.seller_name, - s.rating, - s.rating_count, - s.storefront_asin_total, - s.persisted_inventory_sample_count, - COUNT(DISTINCT sc.source_asin) AS discovered_from_count, - MIN(sc.fetched_at) AS first_seen_at, - MAX(sc.fetched_at) AS last_seen_at, - COUNT(DISTINCT inv.asin) AS persisted_inventory_asin_count, - STRING_AGG(DISTINCT inv.asin, ',') AS inventory_sample_asins - FROM stalker_asin_sellers sas - JOIN stalker_asin_scans sc ON sc.id = sas.scan_id - JOIN stalker_runs r ON r.id = sc.run_id - JOIN sellers s ON s.seller_id = sas.seller_id - LEFT JOIN stalker_seller_inventory inv - ON inv.run_id = r.id - AND inv.seller_id = s.seller_id - ${where} - GROUP BY r.id, s.seller_id - `; - - const totalRow = await pgGet<{ total: string }>( - `SELECT COUNT(*) AS total FROM (${baseSelect}) stalker_rows`, - params, - ); - const total = Number(totalRow?.total ?? 0); - - const summary = await pgGet<{ - runs: string; - sellers: string; - persistedInventoryAsins: string; - }>( - `SELECT - COUNT(DISTINCT "runId") AS runs, - COUNT(DISTINCT seller_id) AS sellers, - COALESCE(SUM(persisted_inventory_asin_count), 0) AS "persistedInventoryAsins" - FROM (${baseSelect}) stalker_rows`, - params, - ); - - const items = await pgAll( - `SELECT * FROM (${baseSelect}) stalker_rows - ORDER BY ${orderBy} - LIMIT ? OFFSET ?`, - [...params, pageSize, offset], - ); - - return { - items, - summary: { - runs: Number(summary?.runs ?? 0), - sellers: Number(summary?.sellers ?? 0), - persistedInventoryAsins: Number(summary?.persistedInventoryAsins ?? 0), - }, - page, - pageSize, - total, - totalPages: Math.max(1, Math.ceil(total / pageSize)), - }; -} - -function parseStalkerProductFilters(filters: URLSearchParams) { - const q = filters.get("q")?.trim() || ""; - const sellerId = filters.get("sellerId")?.trim().toUpperCase() || ""; - const runIdRaw = filters.get("runId")?.trim() || ""; - const verdict = filters.get("verdict")?.trim().toUpperCase() || ""; - const amazonIsSeller = filters.get("amazonIsSeller")?.trim() || ""; - const minPriceRaw = filters.get("minPrice")?.trim() || ""; - const maxPriceRaw = filters.get("maxPrice")?.trim() || ""; - const minMonthlySoldRaw = filters.get("minMonthlySold")?.trim() || ""; - const maxMonthlySoldRaw = filters.get("maxMonthlySold")?.trim() || ""; - const minSalesRankRaw = filters.get("minSalesRank")?.trim() || ""; - const maxSalesRankRaw = filters.get("maxSalesRank")?.trim() || ""; - const minSellerCountRaw = filters.get("minSellerCount")?.trim() || ""; - const maxSellerCountRaw = filters.get("maxSellerCount")?.trim() || ""; - const minRatingCountRaw = filters.get("minRatingCount")?.trim() || ""; - const maxRatingCountRaw = filters.get("maxRatingCount")?.trim() || ""; - const minConfidenceRaw = filters.get("minConfidence")?.trim() || ""; - const maxConfidenceRaw = filters.get("maxConfidence")?.trim() || ""; - - const conditions = [ - "inv.can_sell = true", - "inv.sellability_status = 'available'", - ]; - const params: Array = []; - - if (runIdRaw) { - const runId = Number(runIdRaw); - if (Number.isInteger(runId) && runId > 0) { - conditions.push("r.id = ?"); - params.push(runId); - } - } - - if (sellerId) { - conditions.push("s.seller_id = ?"); - params.push(sellerId); - } - - if (verdict === "FBA" || verdict === "FBM" || verdict === "SKIP") { - conditions.push("analysis.verdict = ?"); - params.push(verdict); - } else if (verdict === "UNANALYZED") { - conditions.push("analysis.verdict IS NULL"); - } - - if (amazonIsSeller === "yes") { - conditions.push("inv.amazon_is_seller = true"); - } else if (amazonIsSeller === "no") { - conditions.push("inv.amazon_is_seller = false"); - } else if (amazonIsSeller === "unknown") { - conditions.push("inv.amazon_is_seller IS NULL"); - } - - const numericFilters: Array<[string, string, string]> = [ - [minPriceRaw, "inv.current_price >= ?", "minPrice"], - [maxPriceRaw, "inv.current_price <= ?", "maxPrice"], - [minMonthlySoldRaw, "inv.monthly_sold >= ?", "minMonthlySold"], - [maxMonthlySoldRaw, "inv.monthly_sold <= ?", "maxMonthlySold"], - [minSalesRankRaw, "inv.sales_rank >= ?", "minSalesRank"], - [maxSalesRankRaw, "inv.sales_rank <= ?", "maxSalesRank"], - [minSellerCountRaw, "inv.seller_count >= ?", "minSellerCount"], - [maxSellerCountRaw, "inv.seller_count <= ?", "maxSellerCount"], - [minRatingCountRaw, "s.rating_count >= ?", "minRatingCount"], - [maxRatingCountRaw, "s.rating_count <= ?", "maxRatingCount"], - [minConfidenceRaw, "analysis.confidence >= ?", "minConfidence"], - [maxConfidenceRaw, "analysis.confidence <= ?", "maxConfidence"], - ]; - - for (const [raw, condition] of numericFilters) { - if (!raw) continue; - const value = Number(raw); - if (Number.isFinite(value)) { - conditions.push(condition); - params.push(value); - } - } - - if (q) { - const wildcard = `%${q}%`; - conditions.push( - `( - inv.asin LIKE ? - OR inv.product_title LIKE ? - OR inv.brand LIKE ? - OR inv.category_tree LIKE ? - OR s.seller_id LIKE ? - OR s.seller_name LIKE ? - )`, - ); - params.push(wildcard, wildcard, wildcard, wildcard, wildcard, wildcard); - } - - return { - where: `WHERE ${conditions.join(" AND ")}`, - params, - }; -} - -function parseStalkerProductSort(sortParam: string | null): string { - const allowedSort = new Set([ - "runId", - "started_at", - "seller_id", - "seller_name", - "rating", - "rating_count", - "asin", - "product_title", - "brand", - "current_price", - "avg_price_90d", - "sales_rank", - "monthly_sold", - "seller_count", - "amazon_is_seller", - "verdict", - "confidence", - "last_seen_at", - ]); - return parseSort( - sortParam, - allowedSort, - "monthly_sold DESC, last_seen_at DESC, asin ASC", - ); -} - -async function getStalkerProducts(filters: URLSearchParams) { - const page = parseIntParam(filters.get("page"), 1); - const pageSize = Math.min( - parseIntParam(filters.get("pageSize"), DEFAULT_PAGE_SIZE), - MAX_PAGE_SIZE, - ); - const offset = (page - 1) * pageSize; - const { where, params } = parseStalkerProductFilters(filters); - const orderBy = parseStalkerProductSort(filters.get("sort")); - - const baseSelect = ` - SELECT - r.id AS "runId", - r.started_at, - s.seller_id, - s.seller_name, - s.rating, - s.rating_count, - inv.asin, - inv.can_sell, - inv.sellability_status, - inv.sellability_reason, - inv.product_title, - inv.brand, - inv.category_tree, - inv.current_price, - inv.avg_price_90d, - inv.sales_rank, - inv.monthly_sold, - inv.seller_count, - inv.amazon_is_seller, - analysis.verdict, - analysis.confidence, - analysis.reasoning, - inv.last_seen_at - FROM stalker_seller_inventory inv - JOIN stalker_runs r ON r.id = inv.run_id - JOIN sellers s ON s.seller_id = inv.seller_id - LEFT JOIN category_product_results analysis ON analysis.asin = inv.asin - ${where} - `; - - const totalRow = await pgGet<{ total: string }>( - `SELECT COUNT(*) AS total FROM (${baseSelect}) stalker_products`, - params, - ); - const total = Number(totalRow?.total ?? 0); - - const summary = await pgGet<{ - runs: string; - sellers: string; - products: string; - }>( - `SELECT - COUNT(DISTINCT "runId") AS runs, - COUNT(DISTINCT seller_id) AS sellers, - COUNT(DISTINCT asin) AS products - FROM (${baseSelect}) stalker_products`, - params, - ); - - const items = await pgAll( - `SELECT * FROM (${baseSelect}) stalker_products - ORDER BY ${orderBy} - LIMIT ? OFFSET ?`, - [...params, pageSize, offset], - ); - - return { - items, - summary: { - runs: Number(summary?.runs ?? 0), - sellers: Number(summary?.sellers ?? 0), - products: Number(summary?.products ?? 0), - }, - page, - pageSize, - total, - totalPages: Math.max(1, Math.ceil(total / pageSize)), - }; -} - -async function getStalkerProductsForExport( - filters: URLSearchParams, -): Promise { - const { where, params } = parseStalkerProductFilters(filters); - const orderBy = parseStalkerProductSort(filters.get("sort")); - - return pgAll( - `SELECT * FROM ( - SELECT - r.id AS "runId", - r.started_at, - s.seller_id, - s.seller_name, - s.rating, - s.rating_count, - inv.asin, - inv.can_sell, - inv.sellability_status, - inv.sellability_reason, - inv.product_title, - inv.brand, - inv.category_tree, - inv.current_price, - inv.avg_price_90d, - inv.sales_rank, - inv.monthly_sold, - inv.seller_count, - inv.amazon_is_seller, - analysis.verdict, - analysis.confidence, - analysis.reasoning, - inv.last_seen_at - FROM stalker_seller_inventory inv - JOIN stalker_runs r ON r.id = inv.run_id - JOIN sellers s ON s.seller_id = inv.seller_id - LEFT JOIN category_product_results analysis ON analysis.asin = inv.asin - ${where} - ) stalker_products - ORDER BY ${orderBy}`, - params, - ); -} - -function parseCategoryTreeForExport(value: string | null): string { - if (!value) return ""; - try { - const parsed = JSON.parse(value); - return Array.isArray(parsed) - ? parsed.filter((item) => typeof item === "string").join(" > ") - : ""; - } catch { - return ""; - } -} - -async function exportStalkerProductsXlsx(filters: URLSearchParams): Promise { - const rows = await getStalkerProductsForExport(filters); - const data = rows.map((row) => ({ - ASIN: row.asin, - "Amazon URL": `https://amazon.com/dp/${row.asin}`, - Product: row.product_title ?? "", - Brand: row.brand ?? "", - Category: parseCategoryTreeForExport(row.category_tree), - "Monthly Sold": row.monthly_sold ?? null, - Sellers: row.seller_count ?? null, - "Amazon Seller": - row.amazon_is_seller == null - ? "" - : row.amazon_is_seller === true - ? "Yes" - : "No", - "Sales Rank": row.sales_rank ?? null, - "Current Price": row.current_price ?? null, - "Avg 90d": row.avg_price_90d ?? null, - Verdict: row.verdict ?? "", - Confidence: row.confidence ?? null, - Reasoning: row.reasoning ?? "", - "Seller ID": row.seller_id, - Seller: row.seller_name ?? "", - "Seller Rating": row.rating ?? null, - "Seller Rating Count": row.rating_count ?? null, - "Sellability Status": row.sellability_status, - "Sellability Reason": row.sellability_reason ?? "", - "Run ID": row.runId, - "Last Seen": row.last_seen_at, - })); - - const workbook = XLSX.utils.book_new(); - const worksheet = XLSX.utils.json_to_sheet(data); - worksheet["!cols"] = [ - { wch: 12 }, - { wch: 32 }, - { wch: 48 }, - { wch: 20 }, - { wch: 34 }, - { wch: 14 }, - { wch: 10 }, - { wch: 14 }, - { wch: 12 }, - { wch: 12 }, - { wch: 12 }, - { wch: 10 }, - { wch: 12 }, - { wch: 60 }, - { wch: 18 }, - { wch: 24 }, - { wch: 12 }, - { wch: 20 }, - { wch: 18 }, - { wch: 40 }, - { wch: 10 }, - { wch: 24 }, - ]; - XLSX.utils.book_append_sheet(workbook, worksheet, "Sellable Products"); - - const buffer = XLSX.write(workbook, { - type: "array", - bookType: "xlsx", - }) as ArrayBuffer; - - return xlsx(buffer, "stalker-sellable-products.xlsx"); -} - -async function purgeStalkerData() { - const [invRow, asinSellersRow, sellersRow, scansRow, runsRow] = - await Promise.all([ - pgGet<{ count: string }>( - "SELECT COUNT(*) AS count FROM stalker_seller_inventory", - ), - pgGet<{ count: string }>( - "SELECT COUNT(*) AS count FROM stalker_asin_sellers", - ), - pgGet<{ count: string }>("SELECT COUNT(*) AS count FROM sellers"), - pgGet<{ count: string }>( - "SELECT COUNT(*) AS count FROM stalker_asin_scans", - ), - pgGet<{ count: string }>("SELECT COUNT(*) AS count FROM stalker_runs"), - ]); - - const counts = { - inventory: Number(invRow?.count ?? 0), - asinSellers: Number(asinSellersRow?.count ?? 0), - sellers: Number(sellersRow?.count ?? 0), - scans: Number(scansRow?.count ?? 0), - runs: Number(runsRow?.count ?? 0), - }; - - await client.begin(async (sql) => { - await sql`DELETE FROM stalker_seller_inventory`; - await sql`DELETE FROM stalker_asin_sellers`; - await sql`DELETE FROM sellers`; - await sql`DELETE FROM stalker_asin_scans`; - await sql`DELETE FROM stalker_runs`; - }); - - return { ok: true, deleted: counts }; -} - -async function getRun(processType: ProcessType, runId: number) { - if (processType === "lead_analysis") { - return pgGet( - `SELECT - id AS "runId", - started_at AS timestamp, - status, - 'lead_file_analysis' AS "jobType", - input_file AS source, - output_file AS output, - COALESCE(total_products, 0) AS "totalProducts", - COALESCE(fba_count, 0) AS "fbaCount", - COALESCE(fbm_count, 0) AS "fbmCount", - COALESCE(skip_count, 0) AS "skipCount" - FROM runs WHERE id = ? AND type = 'lead_analysis'`, - [runId], - ); - } - - return pgGet( - `SELECT - id AS "runId", - started_at AS timestamp, - status, - category_label AS "jobType", - CAST(category_id AS TEXT) AS source, - NULL AS output, - COALESCE(top_asins_checked, 0) AS "totalProducts", - COALESCE(fba_count, 0) AS "fbaCount", - COALESCE(fbm_count, 0) AS "fbmCount", - COALESCE(skip_count, 0) AS "skipCount", - error_message AS "errorMessage", - available_asins AS "availableAsins" - FROM runs WHERE id = ? AND type = 'category_analysis'`, - [runId], - ); -} - -async function getRunResults( - processType: ProcessType, - runId: number, - filters: URLSearchParams, -) { - const page = parseIntParam(filters.get("page"), 1); - const pageSize = Math.min( - parseIntParam(filters.get("pageSize"), DEFAULT_PAGE_SIZE), - MAX_PAGE_SIZE, - ); - const offset = (page - 1) * pageSize; - - const tableName = - processType === "lead_analysis" - ? "analysis_results" - : "category_product_results"; - const productNameSelect = - processType === "lead_analysis" ? "product_name" : "name AS product_name"; - const sellerCountSelect = "seller_count"; - const salesRankAvgSelect = - processType === "lead_analysis" - ? "rank_avg_90d AS sales_rank_avg_90d" - : "sales_rank_avg_90d"; - - const { where, params } = parseResultFilters(processType, runId, filters); - - const allowedSort = new Set([ - "asin", - "product_name", - "brand", - "category", - "current_price", - "avg_price_90d", - "sales_rank", - "seller_count", - "amazon_is_seller", - "amazon_buybox_share_pct_90d", - "monthly_sold", - "verdict", - "confidence", - "fetched_at", - ]); - const orderBy = parseResultSort( - filters.get("sort"), - allowedSort, - "CAST(COALESCE(monthly_sold, 0) AS INTEGER) DESC, asin ASC", - ); - - const totalRow = await pgGet<{ total: string }>( - `SELECT COUNT(*) AS total FROM ${tableName} ${where}`, - params, - ); - const total = Number(totalRow?.total ?? 0); - - const items = await pgAll( - `SELECT - id, - run_id, - asin, - ${productNameSelect}, - brand, - category, - unit_cost, - current_price, - avg_price_90d, - avg_price_90d_sheet, - selling_price_sheet, - sales_rank, - ${salesRankAvgSelect}, - ${sellerCountSelect}, - amazon_is_seller, - amazon_buybox_share_pct_90d, - monthly_sold, - rank_drops_30d, - rank_drops_90d, - fba_fee, - fbm_fee, - referral_percent, - can_sell, - sellability_status, - sellability_reason, - verdict, - confidence, - reasoning, - fetched_at - FROM ${tableName} - ${where} - ORDER BY ${orderBy} - LIMIT ? OFFSET ?`, - [...params, pageSize, offset], - ); - - return { - items, - page, - pageSize, - total, - totalPages: Math.max(1, Math.ceil(total / pageSize)), - }; -} - -async function deleteRun(processType: ProcessType, runId: number) { - if (processType === "lead_analysis") { - const deletedResults = await pgRun( - "DELETE FROM analysis_results WHERE run_id = ?", - [runId], - ); - const deletedRunCount = await pgRun( - "DELETE FROM runs WHERE id = ? AND type = 'lead_analysis'", - [runId], - ); - return { - deletedRun: deletedRunCount > 0, - deletedResults, - }; - } - - const deletedResults = await pgRun( - "DELETE FROM category_product_results WHERE run_id = ?", - [runId], - ); - const deletedRunCount = await pgRun( - "DELETE FROM runs WHERE id = ? AND type = 'category_analysis'", - [runId], - ); - return { - deletedRun: deletedRunCount > 0, - deletedResults, - }; -} - -async function exportRunResultsCsv( - processType: ProcessType, - runId: number, - filters: URLSearchParams, -) { - const tableName = - processType === "lead_analysis" - ? "analysis_results" - : "category_product_results"; - const productNameSelect = - processType === "lead_analysis" ? "product_name" : "name AS product_name"; - const sellerCountSelect = "seller_count"; - const salesRankAvgSelect = - processType === "lead_analysis" - ? "rank_avg_90d AS sales_rank_avg_90d" - : "sales_rank_avg_90d"; - - const { where, params } = parseResultFilters(processType, runId, filters); - - const allowedSort = new Set([ - "asin", - "product_name", - "brand", - "category", - "current_price", - "avg_price_90d", - "sales_rank", - "seller_count", - "amazon_is_seller", - "amazon_buybox_share_pct_90d", - "monthly_sold", - "verdict", - "confidence", - "fetched_at", - ]); - const orderBy = parseResultSort( - filters.get("sort"), - allowedSort, - "CAST(COALESCE(monthly_sold, 0) AS INTEGER) DESC, asin ASC", - ); - - const rows = await pgAll>( - `SELECT - run_id, - asin, - ${productNameSelect}, - brand, - category, - unit_cost, - current_price, - avg_price_90d, - ${salesRankAvgSelect}, - ${sellerCountSelect}, - amazon_is_seller, - amazon_buybox_share_pct_90d, - monthly_sold, - sellability_status, - verdict, - confidence, - reasoning, - fetched_at - FROM ${tableName} - ${where} - ORDER BY ${orderBy}`, - params, - ); - - const headers = [ - "run_id", - "asin", - "product_name", - "brand", - "category", - "unit_cost", - "current_price", - "avg_price_90d", - "sales_rank_avg_90d", - "seller_count", - "amazon_is_seller", - "amazon_buybox_share_pct_90d", - "monthly_sold", - "sellability_status", - "verdict", - "confidence", - "reasoning", - "fetched_at", - ]; - - const lines = [headers.join(",")]; - for (const row of rows) { - lines.push(headers.map((h) => escapeCsvValue(row[h])).join(",")); - } - - return lines.join("\n"); -} - -type ReanalyzeSourceRow = { - asin: string; - product_name: string | null; - brand: string | null; - category: string | null; - unit_cost: number | null; - sales_rank: number | null; - avg_price_90d_sheet: number | null; - selling_price_sheet: number | null; - fba_net_sheet: number | null; - gross_profit_dollar: number | null; - gross_profit_pct: number | null; - net_profit_sheet: number | null; - roi_sheet: number | null; - moq: number | null; - moq_cost: number | null; - qty_available: number | null; - supplier: string | null; - source_url: string | null; - asin_link: string | null; - promo_coupon_code: string | null; - notes: string | null; - lead_date: string | null; +const ITEM_SORTS: Record = { + asin: "asin", + product_name: "product_name", + brand: "brand", + category: "category", + current_price: "current_price", + avg_price_90d: "avg_price_90d", + sales_rank: "sales_rank", + seller_count: "seller_count", + amazon_is_seller: "amazon_is_seller", + amazon_buybox_share_pct_90d: "amazon_buybox_share_pct_90d", + monthly_sold: "monthly_sold", + verdict: "verdict", + confidence: "confidence", + fetched_at: "fetched_at", }; -async function getReanalyzeSourceRow( - processType: ProcessType, - runId: number, - asin: string, -): Promise { - if (processType === "lead_analysis") { - return pgGet( - `SELECT - asin, - product_name, - brand, - category, - unit_cost, - sales_rank, - avg_price_90d_sheet, - selling_price_sheet, - fba_net_sheet, - gross_profit_dollar, - gross_profit_pct, - net_profit_sheet, - roi_sheet, - moq, - moq_cost, - qty_available, - supplier, - source_url, - asin_link, - promo_coupon_code, - notes, - lead_date - FROM analysis_results - WHERE run_id = ? AND asin = ? - LIMIT 1`, - [runId, asin], - ); - } - - return pgGet( - `SELECT - asin, - name AS product_name, - brand, - category, - unit_cost, - sales_rank, - avg_price_90d_sheet, - selling_price_sheet, - NULL AS fba_net_sheet, - NULL AS gross_profit_dollar, - NULL AS gross_profit_pct, - NULL AS net_profit_sheet, - NULL AS roi_sheet, - NULL AS moq, - NULL AS moq_cost, - NULL AS qty_available, - NULL AS supplier, - NULL AS source_url, - NULL AS asin_link, - NULL AS promo_coupon_code, - NULL AS notes, - NULL AS lead_date - FROM category_product_results - WHERE run_id = ? AND asin = ? - LIMIT 1`, - [runId, asin], +async function getRunItems(runId: number, filters: URLSearchParams) { + const { page, pageSize, offset } = pageInput(filters); + const { where, params } = itemFilters(filters, runId); + const orderBy = safeSort(filters.get("sort"), ITEM_SORTS, "monthly_sold DESC NULLS LAST, asin ASC"); + const totalRow = await pgGet<{ total: string }>( + `SELECT COUNT(*) AS total FROM (${ITEM_ROWS}) item_rows ${where}`, + params, ); + const items = await pgAll( + `SELECT * FROM (${ITEM_ROWS}) item_rows ${where} ORDER BY ${orderBy} LIMIT ? OFFSET ?`, + [...params, pageSize, offset], + ); + const total = Number(totalRow?.total ?? 0); + return { items, page, pageSize, total, totalPages: Math.max(1, Math.ceil(total / pageSize)) }; } -function toProductRecord(row: ReanalyzeSourceRow): ProductRecord { - return { +async function exportRunItems(runId: number, filters: URLSearchParams) { + const { where, params } = itemFilters(filters, runId); + const orderBy = safeSort(filters.get("sort"), ITEM_SORTS, "monthly_sold DESC NULLS LAST, asin ASC"); + const rows = await pgAll>( + `SELECT * FROM (${ITEM_ROWS}) item_rows ${where} ORDER BY ${orderBy}`, + params, + ); + const headers = [ + "run_id", "asin", "product_name", "brand", "category", "unit_cost", + "current_price", "avg_price_90d", "sales_rank_avg_90d", "seller_count", + "amazon_is_seller", "amazon_buybox_share_pct_90d", "monthly_sold", + "sellability_status", "verdict", "confidence", "reasoning", "fetched_at", + ]; + return [headers.join(","), ...rows.map((row) => headers.map((h) => escapeCsvValue(row[h])).join(","))].join("\n"); +} + +async function getProducts(filters: URLSearchParams) { + const { page, pageSize, offset } = pageInput(filters); + const { where, params } = itemFilters(filters); + const orderBy = safeSort(filters.get("sort"), ITEM_SORTS, "fetched_at DESC NULLS LAST, asin ASC"); + const base = ` + SELECT product.asin, product.asin AS product_asin, + latest.item_id, latest.run_id AS "runId", latest.process_type AS "processType", + COALESCE(product.name, latest.product_name) AS product_name, + COALESCE(product.brand, latest.brand) AS brand, + COALESCE(product.category, latest.category) AS category, + latest.unit_cost, latest.current_price, latest.avg_price_90d, + latest.sales_rank, latest.sales_rank_avg_90d, latest.seller_count, + latest.amazon_is_seller, latest.amazon_buybox_share_pct_90d, + latest.monthly_sold, latest.rank_drops_30d, latest.rank_drops_90d, + latest.sellability_status, latest.verdict, latest.confidence, + latest.reasoning, latest.fetched_at + FROM products product + LEFT JOIN LATERAL ( + SELECT * + FROM (${ITEM_ROWS}) item_history + WHERE item_history.product_asin = product.asin + ORDER BY item_history.fetched_at DESC NULLS LAST, item_history.item_id DESC + LIMIT 1 + ) latest ON TRUE`; + const total = Number( + (await pgGet<{ total: string }>( + `SELECT COUNT(*) AS total FROM (${base}) products ${where}`, + params, + ))?.total ?? 0, + ); + const items = await pgAll( + `SELECT * FROM (${base}) products ${where} ORDER BY ${orderBy} LIMIT ? OFFSET ?`, + [...params, pageSize, offset], + ); + return { items, page, pageSize, total, totalPages: Math.max(1, Math.ceil(total / pageSize)) }; +} + +async function getProduct(asin: string) { + const product = await pgGet( + `SELECT * FROM products WHERE asin = ?`, + [asin], + ); + if (!product) return null; + const observations = await pgAll( + `SELECT observation.*, run.type AS run_type + FROM product_observations observation + JOIN runs run ON run.id = observation.run_id + WHERE observation.product_asin = ? + ORDER BY observation.fetched_at DESC`, + [asin], + ); + const analyses = await pgAll( + `SELECT revision.*, item.run_id, run.type AS run_type + FROM analysis_revisions revision + JOIN run_items item ON item.id = revision.run_item_id + JOIN runs run ON run.id = item.run_id + WHERE item.product_asin = ? + ORDER BY revision.analyzed_at DESC`, + [asin], + ); + return { product, observations, analyses }; +} + +async function reanalyzeRunItem(itemId: number) { + const row = await pgGet>( + `SELECT ri.id, ri.run_id, ri.product_asin AS asin, r.type, + COALESCE(p.name, si.supplied_name, ri.product_asin) AS product_name, + COALESCE(p.brand, si.supplied_brand) AS brand, + COALESCE(p.category, si.supplied_category) AS category, + si.unit_cost, si.avg_price_90d_sheet, si.selling_price_sheet, + si.fba_net_sheet, si.gross_profit_dollar, si.gross_profit_pct, + si.net_profit_sheet, si.roi_sheet, si.moq, si.moq_cost, + si.qty_available, si.supplier, si.source_url, si.asin_link, + si.promo_coupon_code, si.notes, si.lead_date + FROM run_items ri JOIN runs r ON r.id = ri.run_id + LEFT JOIN products p ON p.asin = ri.product_asin + LEFT JOIN sourcing_inputs si ON si.run_item_id = ri.id + WHERE ri.id = ? AND ri.product_asin IS NOT NULL`, + [itemId], + ); + if (!row) throw new Error("Run item not found"); + if (row.type === "supplier_upc") { + throw new Error("Supplier scoring revisions are produced by the supplier pipeline"); + } + const record: ProductRecord = { asin: row.asin, name: row.product_name ?? row.asin, unitCost: row.unit_cost ?? 0, brand: row.brand ?? undefined, category: row.category ?? undefined, - amazonRank: row.sales_rank ?? undefined, + amazonRank: undefined, avgPrice90FromSheet: row.avg_price_90d_sheet ?? undefined, sellingPriceFromSheet: row.selling_price_sheet ?? undefined, fbaNet: row.fba_net_sheet ?? undefined, @@ -1637,246 +540,286 @@ function toProductRecord(row: ReanalyzeSourceRow): ProductRecord { notes: row.notes ?? undefined, leadDate: row.lead_date ?? undefined, }; -} - -function unknownSpApiData(): SpApiData { - return { - fbaFee: 5.0, - fbmFee: 1.5, - referralFeePercent: 15, - estimatedSalePrice: 0, - canSell: null, - sellabilityStatus: "unknown", - sellabilityReason: "Sellability check returned no result", - }; -} - -async function refreshRunCounts( - processType: ProcessType, - runId: number, -): Promise { - if (processType === "lead_analysis") { - const stats = await pgGet<{ - total: string; - fba: string | null; - fbm: string | null; - skip: string | null; - }>( - `SELECT - COUNT(*) AS total, - SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba, - SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm, - SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip - FROM analysis_results - WHERE run_id = ?`, - [runId], - ); - - await pgRun( - `UPDATE runs - SET total_products = ?, fba_count = ?, fbm_count = ?, skip_count = ? - WHERE id = ?`, - [ - Number(stats?.total ?? 0), - Number(stats?.fba ?? 0), - Number(stats?.fbm ?? 0), - Number(stats?.skip ?? 0), - runId, - ], - ); - return; - } - - const stats = await pgGet<{ - available: string; - fba: string | null; - fbm: string | null; - skip: string | null; - }>( - `SELECT - COUNT(*) AS available, - SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba, - SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm, - SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip - FROM category_product_results - WHERE run_id = ?`, - [runId], - ); - - await pgRun( - `UPDATE runs - SET available_asins = ?, fba_count = ?, fbm_count = ?, skip_count = ? - WHERE id = ?`, - [ - Number(stats?.available ?? 0), - Number(stats?.fba ?? 0), - Number(stats?.fbm ?? 0), - Number(stats?.skip ?? 0), - runId, - ], - ); -} - -async function reanalyzeSingleAsin( - processType: ProcessType, - runId: number, - asin: string, -): Promise<{ - asin: string; - runId: number; - processType: ProcessType; - fetchedAt: string; -}> { - const row = await getReanalyzeSourceRow(processType, runId, asin); - if (!row) { - throw new Error("Result row not found"); - } - - const productRecord = toProductRecord(row); - - let keepa = null; - try { - const keepaMap = await fetchKeepaDataBatch([asin]); - keepa = keepaMap.get(asin) ?? null; - } catch { - keepa = null; - } - - const sellabilityMap = await fetchSellabilityBatch([asin]); - const sellability = sellabilityMap.get(asin) ?? { + const keepaMap = await fetchKeepaDataBatch([row.asin]).catch(() => new Map()); + const keepa = keepaMap.get(row.asin) ?? null; + const sellabilityMap = await fetchSellabilityBatch([row.asin]); + const sellability = sellabilityMap.get(row.asin) ?? { canSell: null, sellabilityStatus: "unknown" as const, sellabilityReason: "Sellability check returned no result", }; - const spApi = await fetchSpApiPricingAndFees(asin, sellability); - - if (keepa?.currentPrice && spApi.estimatedSalePrice === 0) { - spApi.estimatedSalePrice = keepa.currentPrice; - } - + const spApi = await fetchSpApiPricingAndFees(row.asin, sellability); const enriched: EnrichedProduct = { - record: productRecord, + record, keepa, - spApi: spApi ?? unknownSpApiData(), + spApi: spApi as SpApiData, fetchedAt: new Date().toISOString(), }; - - const verdicts = await analyzeProducts([enriched], { - useClaude: USE_CLAUDE, + const verdict = + (await analyzeProducts([enriched], { useClaude: USE_CLAUDE }))[0] ?? { + asin: row.asin, + verdict: "SKIP" as const, + confidence: 0, + reasoning: "LLM analysis returned no verdict", + }; + const result: AnalysisResult = { product: enriched, verdict }; + const observationId = await insertObservation(row.run_id, result, "reanalysis"); + await db.insert(analysisRevisions).values({ + runItemId: itemId, + observationId, + method: "llm", + decision: verdict.verdict, + confidence: verdict.confidence, + reasoning: verdict.reasoning, + analyzedAt: new Date(enriched.fetchedAt), }); - const verdict = verdicts[0] ?? { - asin, - verdict: "SKIP" as const, - confidence: 0, - reasoning: "LLM analysis returned no verdict", - }; + await refreshRunStats(row.run_id); + return { itemId, runId: row.run_id, asin: row.asin, fetchedAt: enriched.fetchedAt }; +} - const amazonIsSeller = keepa?.amazonIsSeller ?? null; - const fetchedAt = enriched.fetchedAt; - - if (processType === "lead_analysis") { - await pgRun( - `UPDATE analysis_results SET - current_price = ?, - avg_price_90d = ?, - sales_rank = ?, - rank_avg_90d = ?, - seller_count = ?, - amazon_is_seller = ?, - amazon_buybox_share_pct_90d = ?, - monthly_sold = ?, - rank_drops_30d = ?, - rank_drops_90d = ?, - fba_fee = ?, - fbm_fee = ?, - referral_percent = ?, - can_sell = ?, - sellability_status = ?, - sellability_reason = ?, - verdict = ?, - confidence = ?, - reasoning = ?, - fetched_at = ? - WHERE run_id = ? AND asin = ?`, - [ - keepa?.currentPrice ?? null, - keepa?.avgPrice90 ?? null, - keepa?.salesRank ?? row.sales_rank ?? null, - keepa?.salesRankAvg90 ?? null, - keepa?.sellerCount ?? null, - amazonIsSeller, - keepa?.amazonBuyboxSharePct90d ?? null, - keepa?.monthlySold ?? null, - keepa?.salesRankDrops30 ?? null, - keepa?.salesRankDrops90 ?? null, - spApi.fbaFee, - spApi.fbmFee, - spApi.referralFeePercent, - spApi.canSell == null ? null : spApi.canSell ? "yes" : "no", - spApi.sellabilityStatus, - spApi.sellabilityReason ?? null, - verdict.verdict, - verdict.confidence, - verdict.reasoning, - fetchedAt, - runId, - asin, - ], - ); - } else { - await pgRun( - `UPDATE category_product_results SET - current_price = ?, - avg_price_90d = ?, - sales_rank = ?, - sales_rank_avg_90d = ?, - seller_count = ?, - amazon_is_seller = ?, - amazon_buybox_share_pct_90d = ?, - monthly_sold = ?, - rank_drops_30d = ?, - rank_drops_90d = ?, - fba_fee = ?, - fbm_fee = ?, - referral_percent = ?, - can_sell = ?, - sellability_status = ?, - sellability_reason = ?, - verdict = ?, - confidence = ?, - reasoning = ?, - fetched_at = ? - WHERE run_id = ? AND asin = ?`, - [ - keepa?.currentPrice ?? null, - keepa?.avgPrice90 ?? null, - keepa?.salesRank ?? row.sales_rank ?? null, - keepa?.salesRankAvg90 ?? null, - keepa?.sellerCount ?? null, - amazonIsSeller, - keepa?.amazonBuyboxSharePct90d ?? null, - keepa?.monthlySold ?? null, - keepa?.salesRankDrops30 ?? null, - keepa?.salesRankDrops90 ?? null, - spApi.fbaFee, - spApi.fbmFee, - spApi.referralFeePercent, - spApi.canSell == null ? null : spApi.canSell, - spApi.sellabilityStatus, - spApi.sellabilityReason ?? null, - verdict.verdict, - verdict.confidence, - verdict.reasoning, - fetchedAt, - runId, - asin, - ], - ); +function stalkerBaseWhere(filters: URLSearchParams, product = false) { + const conditions = ["r.type = 'stalker'"]; + const params: unknown[] = []; + for (const [key, expression] of [ + ["runId", "r.id = ?"], + ["sellerId", "seller.seller_id = ?"], + ] as const) { + const value = filters.get(key)?.trim(); + if (value) { + conditions.push(expression); + params.push(key === "runId" ? Number(value) : value.toUpperCase()); + } } + const q = filters.get("q")?.trim(); + if (q) { + const wildcard = `%${q}%`; + if (product) { + conditions.push( + "(inventory.product_asin ILIKE ? OR COALESCE(product.name, '') ILIKE ? OR COALESCE(product.brand, '') ILIKE ? OR COALESCE(product.category, '') ILIKE ? OR seller.seller_id ILIKE ? OR COALESCE(seller.seller_name, '') ILIKE ?)", + ); + params.push(wildcard, wildcard, wildcard, wildcard, wildcard, wildcard); + } else { + conditions.push( + "(seller.seller_id ILIKE ? OR COALESCE(seller.seller_name, '') ILIKE ? OR EXISTS (SELECT 1 FROM stalker_inventory_items query_inventory WHERE query_inventory.run_id = r.id AND query_inventory.seller_id = seller.seller_id AND query_inventory.product_asin ILIKE ?))", + ); + params.push(wildcard, wildcard, wildcard); + } + } + for (const [parameter, expression] of [ + ["minRatingCount", "seller.rating_count >= ?"], + ["maxRatingCount", "seller.rating_count <= ?"], + ] as const) { + const value = Number(filters.get(parameter)); + if (filters.get(parameter) && Number.isFinite(value)) { + conditions.push(expression); + params.push(value); + } + } + if (product) { + conditions.push("observation.can_sell = true", "observation.sellability_status = 'available'"); + const verdict = filters.get("verdict")?.toUpperCase(); + if (verdict === "FBA" || verdict === "FBM" || verdict === "SKIP") { + conditions.push("analysis.decision::text = ?"); + params.push(verdict); + } else if (verdict === "UNANALYZED") { + conditions.push("analysis.decision IS NULL"); + } + if (filters.get("amazonIsSeller") === "yes") { + conditions.push("observation.amazon_is_seller = true"); + } else if (filters.get("amazonIsSeller") === "no") { + conditions.push("observation.amazon_is_seller = false"); + } else if (filters.get("amazonIsSeller") === "unknown") { + conditions.push("observation.amazon_is_seller IS NULL"); + } + for (const [parameter, expression] of [ + ["minPrice", "observation.current_price >= ?"], + ["maxPrice", "observation.current_price <= ?"], + ["minMonthlySold", "observation.monthly_sold >= ?"], + ["maxMonthlySold", "observation.monthly_sold <= ?"], + ["minSalesRank", "observation.sales_rank >= ?"], + ["maxSalesRank", "observation.sales_rank <= ?"], + ["minSellerCount", "observation.seller_count >= ?"], + ["maxSellerCount", "observation.seller_count <= ?"], + ["minConfidence", "analysis.confidence >= ?"], + ["maxConfidence", "analysis.confidence <= ?"], + ] as const) { + const value = Number(filters.get(parameter)); + if (filters.get(parameter) && Number.isFinite(value)) { + conditions.push(expression); + params.push(value); + } + } + } + return { where: `WHERE ${conditions.join(" AND ")}`, params }; +} - await refreshRunCounts(processType, runId); +async function getStalkerResults(filters: URLSearchParams) { + const { page, pageSize, offset } = pageInput(filters); + const { where, params } = stalkerBaseWhere(filters); + const base = `SELECT r.id AS "runId", r.started_at, r.status, r.input_file, + seller.seller_id, seller.seller_name, seller.rating, seller.rating_count, + seller.storefront_asin_total, seller.persisted_inventory_sample_count, + COUNT(DISTINCT scan.source_product_asin) AS discovered_from_count, + MIN(scan.fetched_at) AS first_seen_at, MAX(scan.fetched_at) AS last_seen_at, + COUNT(DISTINCT inventory.product_asin) AS persisted_inventory_asin_count, + STRING_AGG(DISTINCT inventory.product_asin, ',') AS inventory_sample_asins + FROM stalker_scan_sellers scan_seller + JOIN stalker_scans scan ON scan.id = scan_seller.scan_id + JOIN runs r ON r.id = scan.run_id + JOIN sellers seller ON seller.seller_id = scan_seller.seller_id + LEFT JOIN stalker_inventory_items inventory + ON inventory.run_id = r.id AND inventory.seller_id = seller.seller_id + ${where} GROUP BY r.id, seller.seller_id`; + const order = safeSort( + filters.get("sort"), + { + runId: '"runId"', + started_at: "started_at", + seller_id: "seller_id", + seller_name: "seller_name", + rating: "rating", + rating_count: "rating_count", + discovered_from_count: "discovered_from_count", + persisted_inventory_asin_count: "persisted_inventory_asin_count", + storefront_asin_total: "storefront_asin_total", + last_seen_at: "last_seen_at", + }, + "persisted_inventory_asin_count DESC, last_seen_at DESC, seller_id ASC", + ); + const total = Number((await pgGet<{ total: string }>(`SELECT COUNT(*) AS total FROM (${base}) rows`, params))?.total ?? 0); + const items = await pgAll( + `SELECT * FROM (${base}) rows ORDER BY ${order} LIMIT ? OFFSET ?`, + [...params, pageSize, offset], + ); + const summary = await pgGet>( + `SELECT COUNT(DISTINCT "runId") AS runs, COUNT(DISTINCT seller_id) AS sellers, + COALESCE(SUM(persisted_inventory_asin_count), 0) AS "persistedInventoryAsins" + FROM (${base}) rows`, + params, + ); + return { + items, + summary: { + runs: Number(summary?.runs ?? 0), + sellers: Number(summary?.sellers ?? 0), + persistedInventoryAsins: Number(summary?.persistedInventoryAsins ?? 0), + }, + page, pageSize, total, totalPages: Math.max(1, Math.ceil(total / pageSize)), + }; +} - return { asin, runId, processType, fetchedAt }; +function stalkerProductSql(where: string) { + return `SELECT r.id AS "runId", r.started_at, seller.seller_id, seller.seller_name, + seller.rating, seller.rating_count, inventory.product_asin AS asin, + observation.can_sell, observation.sellability_status, observation.sellability_reason, + product.name AS product_title, product.brand, + CASE WHEN product.category IS NULL THEN NULL ELSE json_build_array(product.category)::text END AS category_tree, + observation.current_price, observation.avg_price_90d, observation.sales_rank, + observation.monthly_sold, observation.seller_count, observation.amazon_is_seller, + analysis.decision AS verdict, analysis.confidence, analysis.reasoning, + inventory.last_seen_at + FROM stalker_inventory_items inventory + JOIN runs r ON r.id = inventory.run_id + JOIN sellers seller ON seller.seller_id = inventory.seller_id + JOIN products product ON product.asin = inventory.product_asin + JOIN product_observations observation ON observation.id = inventory.observation_id + LEFT JOIN LATERAL ( + SELECT revision.decision, revision.confidence, revision.reasoning + FROM run_items item + JOIN analysis_revisions revision ON revision.run_item_id = item.id + WHERE item.source_inventory_item_id = inventory.id + ORDER BY revision.analyzed_at DESC, revision.id DESC LIMIT 1 + ) analysis ON true + ${where}`; +} + +async function stalkerProducts(filters: URLSearchParams, exportOnly = false) { + const { page, pageSize, offset } = pageInput(filters); + const { where, params } = stalkerBaseWhere(filters, true); + const base = stalkerProductSql(where); + const order = safeSort( + filters.get("sort"), + { + runId: '"runId"', + started_at: "started_at", + seller_id: "seller_id", + seller_name: "seller_name", + rating: "rating", + rating_count: "rating_count", + asin: "asin", + product_title: "product_title", + brand: "brand", + current_price: "current_price", + avg_price_90d: "avg_price_90d", + sales_rank: "sales_rank", + monthly_sold: "monthly_sold", + seller_count: "seller_count", + amazon_is_seller: "amazon_is_seller", + verdict: "verdict", + confidence: "confidence", + last_seen_at: "last_seen_at", + }, + "monthly_sold DESC NULLS LAST, last_seen_at DESC, asin ASC", + ); + if (exportOnly) return pgAll(`SELECT * FROM (${base}) products ORDER BY ${order}`, params); + const total = Number((await pgGet<{ total: string }>(`SELECT COUNT(*) AS total FROM (${base}) products`, params))?.total ?? 0); + const items = await pgAll( + `SELECT * FROM (${base}) products ORDER BY ${order} LIMIT ? OFFSET ?`, + [...params, pageSize, offset], + ); + const summary = await pgGet>( + `SELECT COUNT(DISTINCT "runId") AS runs, COUNT(DISTINCT seller_id) AS sellers, + COUNT(DISTINCT asin) AS products FROM (${base}) products`, + params, + ); + return { + items, + summary: { + runs: Number(summary?.runs ?? 0), + sellers: Number(summary?.sellers ?? 0), + products: Number(summary?.products ?? 0), + }, + page, pageSize, total, totalPages: Math.max(1, Math.ceil(total / pageSize)), + }; +} + +async function exportStalkerProducts(filters: URLSearchParams): Promise { + const rows = (await stalkerProducts(filters, true)) as Array>; + const data = rows.map((row) => ({ + ASIN: row.asin, + "Amazon URL": `https://amazon.com/dp/${row.asin}`, + Product: row.product_title ?? "", + Brand: row.brand ?? "", + Category: row.category_tree ?? "", + "Monthly Sold": row.monthly_sold ?? null, + Sellers: row.seller_count ?? null, + "Sales Rank": row.sales_rank ?? null, + "Current Price": row.current_price ?? null, + Verdict: row.verdict ?? "", + Confidence: row.confidence ?? null, + "Seller ID": row.seller_id, + Seller: row.seller_name ?? "", + "Run ID": row.runId, + })); + const workbook = XLSX.utils.book_new(); + XLSX.utils.book_append_sheet(workbook, XLSX.utils.json_to_sheet(data), "Sellable Products"); + return xlsx( + XLSX.write(workbook, { type: "array", bookType: "xlsx" }) as ArrayBuffer, + "stalker-sellable-products.xlsx", + ); +} + +async function purgeStalkerData() { + const count = await pgGet<{ count: string }>( + "SELECT COUNT(*) AS count FROM runs WHERE type = 'stalker'", + ); + await client.begin(async (sql) => { + await sql`DELETE FROM runs WHERE type = 'stalker'`; + await sql`DELETE FROM sellers WHERE NOT EXISTS ( + SELECT 1 FROM stalker_scan_sellers x WHERE x.seller_id = sellers.seller_id + )`; + }); + return { ok: true, deleted: { runs: Number(count?.count ?? 0) } }; } const server = Bun.serve({ @@ -1884,235 +827,101 @@ const server = Bun.serve({ routes: { "/": index, "/products": index, + "/products/:asin": index, "/stalker": index, "/stalker/products": index, - "/runs/:processType/:runId": index, - "/api/runs": async (req) => { - const url = new URL(req.url); - return json(await getRuns(url.searchParams)); - }, - "/api/products": async (req) => { - const url = new URL(req.url); - return json(await getProductList(url.searchParams)); - }, - "/api/stalker/results": async (req) => { - const url = new URL(req.url); - return json(await getStalkerResults(url.searchParams)); - }, - "/api/stalker/products": async (req) => { - const url = new URL(req.url); - return json(await getStalkerProducts(url.searchParams)); - }, - "/api/stalker/products/export.xlsx": async (req) => { - const url = new URL(req.url); - return exportStalkerProductsXlsx(url.searchParams); - }, - "/api/stalker/purge": async (req) => { - if (req.method !== "DELETE" && req.method !== "POST") { - return json({ error: "Method not allowed" }, 405); + "/runs/:runId": index, + "/api/runs": async (req) => json(await getRuns(new URL(req.url).searchParams)), + "/api/runs/:runId": async (req) => { + const runId = Number(req.params.runId); + if (!Number.isInteger(runId)) return json({ error: "Invalid run identifier" }, 400); + if (req.method === "DELETE") { + const deleted = await pgRun("DELETE FROM runs WHERE id = ?", [runId]); + return deleted ? json({ deletedRun: true }) : json({ error: "Run not found" }, 404); } - return json(await purgeStalkerData()); + const run = await getRun(runId); + if (!run) return json({ error: "Run not found" }, 404); + return json({ + ...run, + summary: { + totalProducts: run.totalProducts, + fbaCount: run.fbaCount, + fbmCount: run.fbmCount, + buyCount: run.buyCount, + watchCount: run.watchCount, + skipCount: run.skipCount, + }, + }); }, + "/api/runs/:runId/items": async (req) => { + const runId = Number(req.params.runId); + if (!Number.isInteger(runId)) return json({ error: "Invalid run identifier" }, 400); + return json(await getRunItems(runId, new URL(req.url).searchParams)); + }, + "/api/runs/:runId/export.csv": async (req) => { + const runId = Number(req.params.runId); + if (!Number.isInteger(runId)) return json({ error: "Invalid run identifier" }, 400); + return csv(await exportRunItems(runId, new URL(req.url).searchParams), `run-${runId}.csv`); + }, + "/api/run-items/:itemId/reanalyze": async (req) => { + if (req.method !== "POST") return json({ error: "Method not allowed" }, 405); + const itemId = Number(req.params.itemId); + if (!Number.isInteger(itemId)) return json({ error: "Invalid run item identifier" }, 400); + try { + return json(await reanalyzeRunItem(itemId)); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return json({ error: message }, message === "Run item not found" ? 404 : 500); + } + }, + "/api/products": async (req) => json(await getProducts(new URL(req.url).searchParams)), + "/api/products/:asin": async (req) => { + const asin = normalizeAsin(req.params.asin); + if (!asin) return json({ error: "Invalid ASIN" }, 400); + const result = await getProduct(asin); + return result ? json(result) : json({ error: "Product not found" }, 404); + }, + "/api/stalker/results": async (req) => json(await getStalkerResults(new URL(req.url).searchParams)), + "/api/stalker/products": async (req) => json(await stalkerProducts(new URL(req.url).searchParams)), + "/api/stalker/products/export.xlsx": async (req) => exportStalkerProducts(new URL(req.url).searchParams), + "/api/stalker/purge": async (req) => + req.method === "DELETE" || req.method === "POST" + ? json(await purgeStalkerData()) + : json({ error: "Method not allowed" }, 405), "/api/upc/map": async (req) => { - let upcs: string[]; try { - upcs = await parseUpcsFromRequest(req); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - const status = message === "Method not allowed" ? 405 : 400; - return json({ error: message }, status); - } - - const validationError = validateUpcRequest(upcs); - if (validationError) { - return json({ error: validationError }, 400); - } - - try { - const mapping = await mapUpcsToAsins(upcs); - const items = Array.from(mapping.entries()).map(([upc, asin]) => ({ - upc, - asin, - })); - - return json({ - requested: upcs.length, - matched: items.length, - items, - }); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - return json({ error: message }, 500); + const upcs = await parseUpcsFromRequest(req); + const error = validateUpcs(upcs); + if (error) return json({ error }, 400); + const items = [...(await mapUpcsToAsins(upcs)).entries()].map(([upc, asin]) => ({ upc, asin })); + return json({ requested: upcs.length, matched: items.length, items }); + } catch (error) { + return json({ error: error instanceof Error ? error.message : String(error) }, 400); } }, "/api/upc/lookup": async (req) => { - let upcs: string[]; try { - upcs = await parseUpcsFromRequest(req); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - const status = message === "Method not allowed" ? 405 : 400; - return json({ error: message }, status); - } - - const validationError = validateUpcRequest(upcs); - if (validationError) { - return json({ error: validationError }, 400); - } - - try { - const detailMap = await lookupKeepaUpcs(upcs); - const items = Array.from(detailMap.values()); - return json({ - requested: upcs.length, - statusCounts: summarizeLookupStatuses(items), - items, - }); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - return json({ error: message }, 500); + const upcs = await parseUpcsFromRequest(req); + const error = validateUpcs(upcs); + if (error) return json({ error }, 400); + const items = [...(await lookupKeepaUpcs(upcs)).values()]; + return json({ requested: upcs.length, statusCounts: summarizeLookupStatuses(items), items }); + } catch (error) { + return json({ error: error instanceof Error ? error.message : String(error) }, 400); } }, "/api/process/upc-file": async (req) => { - let parsed: UpcFileProcessRequest; try { - parsed = await parseUpcFileProcessRequest(req); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - const status = - message === "Method not allowed" - ? 405 - : message === "Invalid JSON body" - ? 400 - : 400; - return json({ error: message }, status); + return json(await runUpcFileAnalysis({ ...(await parseUpcFileRequest(req)), manageResources: false })); + } catch (error) { + return json({ error: error instanceof Error ? error.message : String(error) }, 400); } - - try { - const summary = await runUpcFileAnalysis({ - inputFile: parsed.inputFile, - outputFile: parsed.outputFile, - inputBatchSize: parsed.inputBatchSize, - upcLookupBatchSize: parsed.upcLookupBatchSize, - maxRows: parsed.maxRows, - manageResources: false, - }); - return json(summary); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - return json({ error: message }, 500); - } - }, - "/api/runs/:processType/:runId": async (req) => { - const processType = req.params.processType as ProcessType; - const runId = Number(req.params.runId); - - if ( - !( - processType === "lead_analysis" || processType === "category_analysis" - ) || - !Number.isInteger(runId) - ) { - return json({ error: "Invalid run identifier" }, 400); - } - - if (req.method === "DELETE") { - const deleted = await deleteRun(processType, runId); - if (!deleted.deletedRun) return json({ error: "Run not found" }, 404); - return json(deleted); - } - - const run = await getRun(processType, runId); - if (!run) return json({ error: "Run not found" }, 404); - - const summary = { - totalProducts: (run as { totalProducts: number }).totalProducts, - fbaCount: (run as { fbaCount: number }).fbaCount, - fbmCount: (run as { fbmCount: number }).fbmCount, - skipCount: (run as { skipCount: number }).skipCount, - }; - - return json({ processType, ...run, summary }); - }, - "/api/runs/:processType/:runId/results": async (req) => { - const processType = req.params.processType as ProcessType; - const runId = Number(req.params.runId); - - if ( - !( - processType === "lead_analysis" || processType === "category_analysis" - ) || - !Number.isInteger(runId) - ) { - return json({ error: "Invalid run identifier" }, 400); - } - - const url = new URL(req.url); - const payload = await getRunResults(processType, runId, url.searchParams); - return json(payload); - }, - "/api/runs/:processType/:runId/asins/:asin/reanalyze": async (req) => { - const processType = req.params.processType as ProcessType; - const runId = Number(req.params.runId); - const asin = normalizeAsin(req.params.asin); - - if ( - !( - processType === "lead_analysis" || processType === "category_analysis" - ) || - !Number.isInteger(runId) - ) { - return json({ error: "Invalid run identifier" }, 400); - } - - if (req.method !== "POST") { - return json({ error: "Method not allowed" }, 405); - } - - if (!isValidAsin(asin)) { - return json({ error: "Invalid ASIN" }, 400); - } - - try { - const result = await reanalyzeSingleAsin(processType, runId, asin); - return json(result); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - if (message === "Result row not found") { - return json({ error: message }, 404); - } - return json({ error: message }, 500); - } - }, - "/api/runs/:processType/:runId/export.csv": async (req) => { - const processType = req.params.processType as ProcessType; - const runId = Number(req.params.runId); - - if ( - !( - processType === "lead_analysis" || processType === "category_analysis" - ) || - !Number.isInteger(runId) - ) { - return json({ error: "Invalid run identifier" }, 400); - } - - const url = new URL(req.url); - const csvText = await exportRunResultsCsv( - processType, - runId, - url.searchParams, - ); - return csv(csvText, `run-${processType}-${runId}.csv`); }, }, fetch() { return json({ error: "Not found" }, 404); }, - development: { - hmr: true, - console: true, - }, + development: { hmr: true, console: true }, }); console.log(`Results viewer running on http://localhost:${server.port}`); diff --git a/src/stalker/stalker-analyze.ts b/src/stalker/stalker-analyze.ts index 18bfa5d..ed5736f 100644 --- a/src/stalker/stalker-analyze.ts +++ b/src/stalker/stalker-analyze.ts @@ -1,6 +1,7 @@ import { db } from "../db/index.ts"; -import { categoryProductResults, runs } from "../db/schema.ts"; -import { eq, sql } from "drizzle-orm"; +import { persistLlmResults, refreshRunStats } from "../db/persistence.ts"; +import { sql } from "drizzle-orm"; +import { normalizeAsin } from "../asin.ts"; import { analyzeProducts } from "../integrations/llm.ts"; import { fetchSpApiPricingAndFees } from "../integrations/sp-api.ts"; import type { @@ -22,6 +23,7 @@ type Args = { }; type InventoryRow = { + inventoryItemId: number; asin: string; productTitle: string | null; brand: string | null; @@ -49,8 +51,8 @@ function parseArgs(argv = process.argv.slice(2)): Args { const useClaude = argv.includes("--claude"); const asins = (readFlagValue(argv, "--asins") ?? "") .split(",") - .map((asin) => asin.trim().toUpperCase()) - .filter(Boolean); + .map((asin) => normalizeAsin(asin)) + .filter((asin): asin is string => asin !== null); if (!Number.isInteger(stalkerRunId) || stalkerRunId <= 0) { throw new Error("--stalker-run-id must be a positive integer"); @@ -68,15 +70,7 @@ function wait(ms: number): Promise { } function parseCategoryTree(value: string | null): string[] { - if (!value) return []; - try { - const parsed = JSON.parse(value); - return Array.isArray(parsed) - ? parsed.filter((item): item is string => typeof item === "string") - : []; - } catch { - return []; - } + return value ? value.split(" > ").filter(Boolean) : []; } function toProductRecord(row: InventoryRow): ProductRecord { @@ -128,25 +122,29 @@ async function loadInventoryRows( ): Promise { if (asins.length === 0) return []; return db.execute( - sql`SELECT DISTINCT ON (asin) - asin, - product_title AS "productTitle", - brand, - category_tree AS "categoryTree", - current_price AS "currentPrice", - avg_price_90d AS "avgPrice90d", - sales_rank AS "salesRank", - monthly_sold AS "monthlySold", - seller_count AS "sellerCount", - amazon_is_seller AS "amazonIsSeller", - can_sell AS "canSell", - sellability_status AS "sellabilityStatus", - sellability_reason AS "sellabilityReason" - FROM stalker_seller_inventory - WHERE run_id = ${stalkerRunId} - AND can_sell = true - AND sellability_status = 'available' - AND asin = ANY(${asins})`, + sql`SELECT DISTINCT ON (inventory.product_asin) + inventory.id AS "inventoryItemId", + inventory.product_asin AS asin, + product.name AS "productTitle", + product.brand, + product.category AS "categoryTree", + observation.current_price AS "currentPrice", + observation.avg_price_90d AS "avgPrice90d", + observation.sales_rank AS "salesRank", + observation.monthly_sold AS "monthlySold", + observation.seller_count AS "sellerCount", + observation.amazon_is_seller AS "amazonIsSeller", + observation.can_sell AS "canSell", + observation.sellability_status AS "sellabilityStatus", + observation.sellability_reason AS "sellabilityReason" + FROM stalker_inventory_items inventory + JOIN products product ON product.asin = inventory.product_asin + JOIN product_observations observation ON observation.id = inventory.observation_id + WHERE inventory.run_id = ${stalkerRunId} + AND observation.can_sell = true + AND observation.sellability_status = 'available' + AND inventory.product_asin = ANY(${asins}) + ORDER BY inventory.product_asin, observation.fetched_at DESC`, ); } @@ -177,111 +175,18 @@ async function buildEnrichedProducts( async function insertProductAnalysisResults( runId: number, results: AnalysisResult[], + sourceInventoryIds: Map, ): Promise { if (results.length === 0) return; - - const rows = results.map((result) => { - const keepa = result.product.keepa; - const record = result.product.record; - const spApi = result.product.spApi; - const canSell = - spApi.canSell == null ? "unknown" : spApi.canSell ? "yes" : "no"; - - return { - asin: record.asin, - runId, - name: record.name, - brand: record.brand ?? null, - category: record.category ?? keepa?.categoryTree.join(" > ") ?? null, - unitCost: record.unitCost ?? null, - currentPrice: keepa?.currentPrice ?? spApi.estimatedSalePrice ?? null, - avgPrice90d: keepa?.avgPrice90 ?? null, - avgPrice90dSheet: record.avgPrice90FromSheet ?? null, - sellingPriceSheet: record.sellingPriceFromSheet ?? null, - salesRank: keepa?.salesRank ?? record.amazonRank ?? null, - salesRankAvg90d: keepa?.salesRankAvg90 ?? null, - sellerCount: keepa?.sellerCount ?? null, - amazonIsSeller: keepa?.amazonIsSeller ?? null, - amazonBuyboxSharePct90d: keepa?.amazonBuyboxSharePct90d ?? null, - monthlySold: keepa?.monthlySold ?? null, - rankDrops30d: keepa?.salesRankDrops30 ?? null, - rankDrops90d: keepa?.salesRankDrops90 ?? null, - fbaFee: spApi.fbaFee ?? null, - fbmFee: spApi.fbmFee ?? null, - referralPercent: spApi.referralFeePercent ?? null, - canSell, - sellabilityStatus: spApi.sellabilityStatus ?? null, - sellabilityReason: spApi.sellabilityReason ?? null, - verdict: result.verdict.verdict, - confidence: result.verdict.confidence ?? 0, - reasoning: result.verdict.reasoning ?? null, - fetchedAt: new Date(result.product.fetchedAt), - }; + await persistLlmResults(runId, results, { + source: "stalker_analysis", + metadataSource: "catalog", + sourceInventoryIds, }); - - await db - .insert(categoryProductResults) - .values(rows) - .onConflictDoUpdate({ - target: categoryProductResults.asin, - set: { - runId: sql`EXCLUDED.run_id`, - name: sql`EXCLUDED.name`, - brand: sql`EXCLUDED.brand`, - category: sql`EXCLUDED.category`, - unitCost: sql`EXCLUDED.unit_cost`, - currentPrice: sql`EXCLUDED.current_price`, - avgPrice90d: sql`EXCLUDED.avg_price_90d`, - avgPrice90dSheet: sql`EXCLUDED.avg_price_90d_sheet`, - sellingPriceSheet: sql`EXCLUDED.selling_price_sheet`, - salesRank: sql`EXCLUDED.sales_rank`, - salesRankAvg90d: sql`EXCLUDED.sales_rank_avg_90d`, - sellerCount: sql`EXCLUDED.seller_count`, - amazonIsSeller: sql`EXCLUDED.amazon_is_seller`, - amazonBuyboxSharePct90d: sql`EXCLUDED.amazon_buybox_share_pct_90d`, - monthlySold: sql`EXCLUDED.monthly_sold`, - rankDrops30d: sql`EXCLUDED.rank_drops_30d`, - rankDrops90d: sql`EXCLUDED.rank_drops_90d`, - fbaFee: sql`EXCLUDED.fba_fee`, - fbmFee: sql`EXCLUDED.fbm_fee`, - referralPercent: sql`EXCLUDED.referral_percent`, - canSell: sql`EXCLUDED.can_sell`, - sellabilityStatus: sql`EXCLUDED.sellability_status`, - sellabilityReason: sql`EXCLUDED.sellability_reason`, - verdict: sql`EXCLUDED.verdict`, - confidence: sql`EXCLUDED.confidence`, - reasoning: sql`EXCLUDED.reasoning`, - fetchedAt: sql`EXCLUDED.fetched_at`, - }, - }); } async function refreshAnalysisRun(runId: number): Promise { - const [stats] = await db.execute( - sql<{ - total: string; - fba: string | null; - fbm: string | null; - skip: string | null; - }>`SELECT - COUNT(*) AS total, - SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba, - SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm, - SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip - FROM category_product_results - WHERE run_id = ${runId}`, - ); - - await db - .update(runs) - .set({ - topAsinsChecked: Number(stats?.total ?? 0), - availableAsins: Number(stats?.total ?? 0), - fbaCount: Number(stats?.fba ?? 0), - fbmCount: Number(stats?.fbm ?? 0), - skipCount: Number(stats?.skip ?? 0), - }) - .where(eq(runs.id, runId)); + await refreshRunStats(runId); } async function analyzeInBatches( @@ -344,7 +249,14 @@ async function main(): Promise { console.log(`Stalker analysis: analyzing ${rows.length} sellable ASIN(s).`); const enriched = await buildEnrichedProducts(rows); const results = await analyzeInBatches(enriched, args.useClaude); - await insertProductAnalysisResults(args.analysisRunId, results); + const sourceInventoryIds = new Map( + rows.map((row) => [row.asin, row.inventoryItemId]), + ); + await insertProductAnalysisResults( + args.analysisRunId, + results, + sourceInventoryIds, + ); await refreshAnalysisRun(args.analysisRunId); } diff --git a/src/stalker/stalker.test.ts b/src/stalker/stalker.test.ts index 86eaf67..9068f8d 100644 --- a/src/stalker/stalker.test.ts +++ b/src/stalker/stalker.test.ts @@ -101,12 +101,17 @@ test("readAsinsFromXlsx extracts valid ASINs and deduplicates in order", () => { { ASIN: "invalid" }, { ASIN: "B000000002" }, { ASIN: "B000000001" }, + { ASIN: "0306406152" }, { ASIN: "" }, ]); XLSX.utils.book_append_sheet(workbook, sheet, "Input"); XLSX.writeFile(workbook, filePath); - expect(readAsinsFromXlsx(filePath)).toEqual(["B000000001", "B000000002"]); + expect(readAsinsFromXlsx(filePath)).toEqual([ + "B000000001", + "B000000002", + "0306406152", + ]); }); test("isQualifyingSeller accepts rating counts from 1 to 30 only", () => { diff --git a/src/stalker/stalker.ts b/src/stalker/stalker.ts index 4770589..69713f7 100644 --- a/src/stalker/stalker.ts +++ b/src/stalker/stalker.ts @@ -1,13 +1,17 @@ import * as XLSX from "xlsx"; import path from "node:path"; +import { normalizeAsin } from "../asin.ts"; import { db } from "../db/index.ts"; +import { refreshRunStats, upsertProduct } from "../db/persistence.ts"; import { + analysisRunStats, + productObservations, runs, - stalkerRuns, - stalkerAsinScans, + stalkerRunDetails, + stalkerScans, sellers, - stalkerAsinSellers, - stalkerSellerInventory, + stalkerScanSellers, + stalkerInventoryItems, } from "../db/schema.ts"; import { eq, sql } from "drizzle-orm"; import { fetchSellabilityBatch } from "../integrations/sp-api.ts"; @@ -16,7 +20,6 @@ import type { SellabilityInfo } from "../types.ts"; const KEEPA_BASE = "https://api.keepa.com"; const DOMAIN_US = "1"; const AMAZON_US_SELLER_ID = "ATVPDKIKX0DER"; -const ASIN_REGEX = /^B[0-9A-Z]{9}$/; const DEFAULT_STOREFRONT_UPDATE_HOURS = 168; const DEFAULT_OFFER_LIMIT = 100; const DEFAULT_SELLER_LIMIT = 30; @@ -333,7 +336,7 @@ export async function runStalker(args: StalkerArgs, deps: StalkerDeps = {}): Pro : await startStalkerRun(args.input, resumeFilteredAsins.length); const analysisRunId = !args.dryRun && args.analyzeSellable - ? await startStalkerAnalysisRun(args.input) + ? await startStalkerAnalysisRun(args.input, runId!) : null; const stats: StalkerRunStats = { scannedAsins: 0, @@ -841,11 +844,12 @@ async function persistAsinResult( await db.transaction(async (tx) => { const scanId = await upsertAsinScan(tx, runId, result, fetchedAt); + const observationIds = new Map(); for (const { seller, offer } of result.matchedSellers) { await upsertSeller(tx, seller, fetchedAt); await upsertAsinSeller(tx, scanId, seller, offer); - await upsertSellerInventory(tx, runId, seller, fetchedAt); + await upsertSellerInventory(tx, runId, seller, fetchedAt, observationIds); } }); } @@ -856,37 +860,58 @@ async function upsertAsinScan( result: StalkerAsinResult, fetchedAt: Date, ): Promise { - await tx - .insert(stalkerAsinScans) + const sourceProductAsin = await upsertProduct( + { + asin: result.asin, + name: result.title, + metadataSource: "catalog", + fetchedAt, + }, + tx, + ); + const [observation] = await tx + .insert(productObservations) .values({ + productAsin: sourceProductAsin, runId, - sourceAsin: result.asin, - title: result.title, - offerCount: result.offerCount, - candidateSellerCount: result.candidateSellerCount, - matchedSellerCount: result.matchedSellers.length, + source: "stalker_scan", fetchedAt, rawProductJson: JSON.stringify( result.product ?? { error: result.error ?? null }, ), }) + .returning({ id: productObservations.id }); + if (!observation) { + throw new Error(`Failed to insert stalker observation for ${result.asin}`); + } + + await tx + .insert(stalkerScans) + .values({ + runId, + sourceProductAsin, + observationId: observation.id, + offerCount: result.offerCount, + candidateSellerCount: result.candidateSellerCount, + matchedSellerCount: result.matchedSellers.length, + fetchedAt, + }) .onConflictDoUpdate({ - target: [stalkerAsinScans.runId, stalkerAsinScans.sourceAsin], + target: [stalkerScans.runId, stalkerScans.sourceProductAsin], set: { - title: sql`EXCLUDED.title`, + observationId: sql`EXCLUDED.observation_id`, offerCount: sql`EXCLUDED.offer_count`, candidateSellerCount: sql`EXCLUDED.candidate_seller_count`, matchedSellerCount: sql`EXCLUDED.matched_seller_count`, fetchedAt: sql`EXCLUDED.fetched_at`, - rawProductJson: sql`EXCLUDED.raw_product_json`, }, }); const [row] = await tx - .select({ id: stalkerAsinScans.id }) - .from(stalkerAsinScans) + .select({ id: stalkerScans.id }) + .from(stalkerScans) .where( - sql`${stalkerAsinScans.runId} = ${runId} AND ${stalkerAsinScans.sourceAsin} = ${result.asin}`, + sql`${stalkerScans.runId} = ${runId} AND ${stalkerScans.sourceProductAsin} = ${sourceProductAsin}`, ); if (!row) throw new Error(`Failed to load stalker scan row for ${result.asin}`); @@ -931,7 +956,7 @@ async function upsertAsinSeller( offer: StalkerOffer, ): Promise { await tx - .insert(stalkerAsinSellers) + .insert(stalkerScanSellers) .values({ scanId, sellerId: seller.sellerId, @@ -944,7 +969,7 @@ async function upsertAsinSeller( rawOfferJson: JSON.stringify(offer.rawOffer), }) .onConflictDoUpdate({ - target: [stalkerAsinSellers.scanId, stalkerAsinSellers.sellerId], + target: [stalkerScanSellers.scanId, stalkerScanSellers.sellerId], set: { offerPrice: sql`EXCLUDED.offer_price`, condition: sql`EXCLUDED.condition`, @@ -962,6 +987,7 @@ async function upsertSellerInventory( runId: number, seller: StalkerSeller, fetchedAt: Date, + observationIds: Map, ): Promise { const items = seller.storefrontItems.filter( (item) => @@ -971,58 +997,71 @@ async function upsertSellerInventory( if (items.length === 0) return; - await tx - .insert(stalkerSellerInventory) - .values( - items.map((item) => ({ + for (const item of items) { + let observationId = observationIds.get(item.asin); + if (observationId == null) { + const productAsin = await upsertProduct( + { + asin: item.asin, + name: item.productDetails?.title, + brand: item.productDetails?.brand, + category: item.productDetails?.categoryTree.join(" > "), + metadataSource: "catalog", + fetchedAt, + }, + tx, + ); + const [observation] = await tx + .insert(productObservations) + .values({ + productAsin, + runId, + source: "stalker_inventory", + canSell: item.sellability?.canSell ?? null, + sellabilityStatus: item.sellability?.sellabilityStatus ?? null, + sellabilityReason: item.sellability?.sellabilityReason ?? null, + currentPrice: item.productDetails?.currentPrice ?? null, + avgPrice90d: item.productDetails?.avgPrice90 ?? null, + salesRank: item.productDetails?.salesRank ?? null, + monthlySold: item.productDetails?.monthlySold ?? null, + sellerCount: item.productDetails?.sellerCount ?? null, + amazonIsSeller: item.productDetails?.amazonIsSeller ?? null, + rawProductJson: item.productDetails + ? JSON.stringify(item.productDetails.rawProduct) + : null, + fetchedAt, + }) + .returning({ id: productObservations.id }); + if (!observation) { + throw new Error(`Failed to insert inventory observation for ${item.asin}`); + } + observationId = observation.id; + observationIds.set(item.asin, observationId); + } + + await tx + .insert(stalkerInventoryItems) + .values({ runId, sellerId: seller.sellerId, - asin: item.asin, - canSell: item.sellability?.canSell ?? null, - sellabilityStatus: item.sellability?.sellabilityStatus ?? null, - sellabilityReason: item.sellability?.sellabilityReason ?? null, - productTitle: item.productDetails?.title ?? null, - brand: item.productDetails?.brand ?? null, - categoryTree: item.productDetails - ? JSON.stringify(item.productDetails.categoryTree) - : null, - currentPrice: item.productDetails?.currentPrice ?? null, - avgPrice90d: item.productDetails?.avgPrice90 ?? null, - salesRank: item.productDetails?.salesRank ?? null, - monthlySold: item.productDetails?.monthlySold ?? null, - sellerCount: item.productDetails?.sellerCount ?? null, - amazonIsSeller: item.productDetails?.amazonIsSeller ?? null, - rawProductJson: item.productDetails - ? JSON.stringify(item.productDetails.rawProduct) - : null, + productAsin: item.asin, + observationId, lastSeenAt: fetchedAt, rawInventoryJson: JSON.stringify(item.rawInventory), - })), - ) - .onConflictDoUpdate({ - target: [ - stalkerSellerInventory.runId, - stalkerSellerInventory.sellerId, - stalkerSellerInventory.asin, - ], - set: { - canSell: sql`EXCLUDED.can_sell`, - sellabilityStatus: sql`EXCLUDED.sellability_status`, - sellabilityReason: sql`EXCLUDED.sellability_reason`, - productTitle: sql`EXCLUDED.product_title`, - brand: sql`EXCLUDED.brand`, - categoryTree: sql`EXCLUDED.category_tree`, - currentPrice: sql`EXCLUDED.current_price`, - avgPrice90d: sql`EXCLUDED.avg_price_90d`, - salesRank: sql`EXCLUDED.sales_rank`, - monthlySold: sql`EXCLUDED.monthly_sold`, - sellerCount: sql`EXCLUDED.seller_count`, - amazonIsSeller: sql`EXCLUDED.amazon_is_seller`, - rawProductJson: sql`EXCLUDED.raw_product_json`, - lastSeenAt: sql`EXCLUDED.last_seen_at`, - rawInventoryJson: sql`EXCLUDED.raw_inventory_json`, - }, - }); + }) + .onConflictDoUpdate({ + target: [ + stalkerInventoryItems.runId, + stalkerInventoryItems.sellerId, + stalkerInventoryItems.productAsin, + ], + set: { + observationId: sql`EXCLUDED.observation_id`, + lastSeenAt: sql`EXCLUDED.last_seen_at`, + rawInventoryJson: sql`EXCLUDED.raw_inventory_json`, + }, + }); + } } async function startStalkerRun( @@ -1030,42 +1069,45 @@ async function startStalkerRun( totalAsins: number, ): Promise { const [row] = await db - .insert(stalkerRuns) + .insert(runs) .values({ + type: "stalker", inputFile, startedAt: new Date(), - requestedAsins: totalAsins, status: "running", }) - .returning({ id: stalkerRuns.id }); + .returning({ id: runs.id }); if (!row) throw new Error("Failed to insert stalker run record."); + await db.insert(stalkerRunDetails).values({ + runId: row.id, + requestedAsins: totalAsins, + }); return row.id; } -async function startStalkerAnalysisRun(inputFile: string): Promise { +async function startStalkerAnalysisRun( + inputFile: string, + parentRunId: number, +): Promise { const [row] = await db .insert(runs) .values({ - type: "category_analysis", - categoryId: 0, - categoryLabel: `Stalker: ${path.basename(inputFile)}`, - topAsinsChecked: 0, - availableAsins: 0, - fbaCount: 0, - fbmCount: 0, - skipCount: 0, + type: "stalker_analysis", + parentRunId, + inputFile: `Stalker: ${path.basename(inputFile)}`, status: "running", startedAt: new Date(), }) .returning({ id: runs.id }); if (!row) throw new Error("Failed to insert stalker analysis run record."); + await db.insert(analysisRunStats).values({ runId: row.id }); return row.id; } async function loadPreviouslyScannedAsins(): Promise> { const rows = await db - .selectDistinct({ sourceAsin: stalkerAsinScans.sourceAsin }) - .from(stalkerAsinScans); + .selectDistinct({ sourceAsin: stalkerScans.sourceProductAsin }) + .from(stalkerScans); return new Set(rows.map((row) => row.sourceAsin)); } @@ -1133,8 +1175,9 @@ async function refreshStalkerRun( status: string, ): Promise { await db - .update(stalkerRuns) + .update(stalkerRunDetails) .set({ + skippedAsins: stats.skippedAsins, scannedAsins: stats.scannedAsins, sourceAsinsWithMatches: stats.sourceAsinsWithMatches, candidateSellers: stats.candidateSellers, @@ -1147,10 +1190,15 @@ async function refreshStalkerRun( stats.inventorySellabilityAvailableAsins, inventorySellabilityExcludedAsins: stats.inventorySellabilityExcludedAsins, persistedInventoryAsins: stats.persistedInventoryAsins, - status, + }) + .where(eq(stalkerRunDetails.runId, runId)); + await db + .update(runs) + .set({ + status: status === "running" ? "running" : "completed", ...(status !== "running" ? { completedAt: new Date() } : {}), }) - .where(eq(stalkerRuns.id, runId)); + .where(eq(runs.id, runId)); } async function finishStalkerRunWithError( @@ -1159,8 +1207,9 @@ async function finishStalkerRunWithError( errorMessage: string, ): Promise { await db - .update(stalkerRuns) + .update(stalkerRunDetails) .set({ + skippedAsins: stats.skippedAsins, scannedAsins: stats.scannedAsins, sourceAsinsWithMatches: stats.sourceAsinsWithMatches, candidateSellers: stats.candidateSellers, @@ -1173,11 +1222,16 @@ async function finishStalkerRunWithError( stats.inventorySellabilityAvailableAsins, inventorySellabilityExcludedAsins: stats.inventorySellabilityExcludedAsins, persistedInventoryAsins: stats.persistedInventoryAsins, + }) + .where(eq(stalkerRunDetails.runId, runId)); + await db + .update(runs) + .set({ status: "failed", errorMessage, completedAt: new Date(), }) - .where(eq(stalkerRuns.id, runId)); + .where(eq(runs.id, runId)); } async function finishStalkerAnalysisRun( @@ -1185,29 +1239,10 @@ async function finishStalkerAnalysisRun( status: "completed" | "failed", errorMessage: string | null = null, ): Promise { - const [stats] = await db.execute( - sql<{ - total: string; - fba: string | null; - fbm: string | null; - skip: string | null; - }>`SELECT - COUNT(*) AS total, - SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba, - SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm, - SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip - FROM category_product_results - WHERE run_id = ${runId}`, - ); - + await refreshRunStats(runId); await db .update(runs) .set({ - topAsinsChecked: Number(stats?.total ?? 0), - availableAsins: Number(stats?.total ?? 0), - fbaCount: Number(stats?.fba ?? 0), - fbmCount: Number(stats?.fbm ?? 0), - skipCount: Number(stats?.skip ?? 0), status, errorMessage, completedAt: new Date(), @@ -1480,13 +1515,6 @@ async function runSellableAnalysisChild( } } -function normalizeAsin(value: unknown): string | null { - const asin = String(value ?? "") - .trim() - .toUpperCase(); - return ASIN_REGEX.test(asin) ? asin : null; -} - function normalizeSellerId(value: unknown): string | null { const sellerId = String(value ?? "") .trim() diff --git a/src/supplier/supplier-export.test.ts b/src/supplier/supplier-export.test.ts index e5041d3..9ccec17 100644 --- a/src/supplier/supplier-export.test.ts +++ b/src/supplier/supplier-export.test.ts @@ -16,12 +16,12 @@ function result(overrides: Partial = {}): SupplierAnalys upc: "012345678901", rowNumber: 2, record: { - asin: "B000000001", name: "Test Product", unitCost: 10, brand: "Brand", category: "Grocery", }, + product: { asin: "B000000001", name: "Test Product", unitCost: 10 }, lookup: { requestedUpc: "012345678901", normalizedUpc: "012345678901", @@ -81,7 +81,8 @@ test("writeSupplierWorkbook writes ranked, skipped, and summary sheets", async ( result(), result({ upc: "111111111111", - record: { asin: "111111111111", name: "Missing", unitCost: 0 }, + record: { name: "Missing", unitCost: 0 }, + product: null, lookup: { requestedUpc: "111111111111", normalizedUpc: "111111111111", diff --git a/src/supplier/supplier-export.ts b/src/supplier/supplier-export.ts index 4bcb9d7..7b3e932 100644 --- a/src/supplier/supplier-export.ts +++ b/src/supplier/supplier-export.ts @@ -63,7 +63,8 @@ function addRowsSheet( const sheet = workbook.addWorksheet(name); const headers = rows[0] ? Object.keys(rows[0]) : Object.keys(rowForResult({ upc: "", - record: { asin: "", name: "", unitCost: 0 }, + record: { name: "", unitCost: 0 }, + product: null, lookup: { requestedUpc: "", normalizedUpc: "", diff --git a/src/supplier/upc-file-analysis.ts b/src/supplier/upc-file-analysis.ts index fb7929e..749b5dc 100644 --- a/src/supplier/upc-file-analysis.ts +++ b/src/supplier/upc-file-analysis.ts @@ -1,4 +1,5 @@ import path from "node:path"; +import { requireAsin } from "../asin.ts"; import { fetchKeepaDataBatch, lookupKeepaUpcs } from "../integrations/keepa.ts"; import { fetchSellabilityBatch, @@ -11,6 +12,8 @@ import { } from "./upc-file-reader.ts"; import { appendSupplierResultsToRun, + completeRunInDb, + failRunInDb, refreshRunCountsInDb, startRunInDb, type RunCounts, @@ -239,8 +242,8 @@ async function lookupUpcsWithChunking( chunkDetails.set( upc, fallbackDetail && fallbackDetail.status !== "request_failed" - ? fallbackDetail - : spDetail!, + ? { ...fallbackDetail, provider: "keepa" } + : { ...spDetail!, provider: "sp_api" }, ); } @@ -266,7 +269,7 @@ function toProductRecord( const keepaCategory = detail.keepaData?.categoryTree?.[0]; return { - asin: detail.asin ?? row.upc, + asin: requireAsin(detail.asin), name: row.name ?? detail.asin ?? row.upc, unitCost: row.unitCost ?? 0, brand: row.brand, @@ -274,6 +277,15 @@ function toProductRecord( }; } +function toSupplierInputRecord(row: UpcInputRow) { + return { + name: row.name ?? row.upc, + unitCost: row.unitCost ?? 0, + brand: row.brand, + category: row.category, + }; +} + async function fetchFeesForProducts( products: ProductRecord[], keepaResults: Map>, @@ -359,7 +371,7 @@ export async function runUpcFileAnalysis( let processedRows = 0; let matchedRows = 0; - const runId = await startRunInDb(options.inputFile, outputFile); + const runId = await startRunInDb(options.inputFile, outputFile, undefined, "supplier_upc"); try { const readerSummary = await processUpcFileInBatches( @@ -382,12 +394,19 @@ export async function runUpcFileAnalysis( product: ProductRecord; }> = []; for (const row of rows) { - const detail = detailMap.get(row.upc); - if (!detail) { - unresolvedByStatus.request_failed += 1; - continue; - } - + const detail = + detailMap.get(row.upc) ?? + ({ + requestedUpc: row.upc, + normalizedUpc: row.upc, + status: "request_failed", + asin: null, + candidateAsins: [], + keepaData: null, + provider: "sp_api", + reason: "UPC lookup returned no result", + } satisfies UpcLookupDetail); + if (!detailMap.has(row.upc)) detailMap.set(row.upc, detail); unresolvedByStatus[detail.status] += 1; if (detail.status === "found" && detail.asin) { @@ -407,30 +426,15 @@ export async function runUpcFileAnalysis( const batchResults: SupplierAnalysisResult[] = []; for (const row of rows) { - const detail = detailMap.get(row.upc); - if (!detail || detail.status === "found") continue; + const detail = detailMap.get(row.upc)!; + if (detail.status === "found") continue; batchResults.push({ upc: row.upc, rowNumber: row.rowNumber, - record: { - asin: detail?.asin ?? row.upc, - name: row.name ?? row.upc, - unitCost: row.unitCost ?? 0, - brand: row.brand, - category: row.category, - }, - lookup: - detail ?? - ({ - requestedUpc: row.upc, - normalizedUpc: row.upc, - status: "request_failed", - asin: null, - candidateAsins: [], - keepaData: null, - reason: "UPC lookup returned no result", - } satisfies UpcLookupDetail), + record: toSupplierInputRecord(row), + product: null, + lookup: detail, keepa: null, spApi: null, score: skippedScore(detail?.reason ?? "UPC unresolved"), @@ -465,7 +469,8 @@ export async function runUpcFileAnalysis( batchResults.push({ upc: entry.detail.normalizedUpc, rowNumber: entry.row.rowNumber, - record: entry.product, + record: toSupplierInputRecord(entry.row), + product: entry.product, lookup: entry.detail, keepa, spApi, @@ -488,6 +493,7 @@ export async function runUpcFileAnalysis( const exportSummary = summarizeSupplierResults(allResults, unresolvedByStatus); await writeSupplierWorkbook(outputFile, allResults, exportSummary); + await completeRunInDb(runId); if (allResults.length > 0) { const ranked = allResults @@ -530,6 +536,9 @@ export async function runUpcFileAnalysis( skippedInvalidUpc: readerSummary.skippedInvalidUpc, }, }; + } catch (error) { + await failRunInDb(runId, error); + throw error; } finally { if (manageResources) { await disconnectCache(); diff --git a/src/types.ts b/src/types.ts index bc42ef4..409c01e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -58,7 +58,8 @@ export interface KeepaUpcLookupDetail { status: KeepaUpcLookupStatus; asin: string | null; candidateAsins: string[]; - keepaData: KeepaData | null; + keepaData: KeepaData | null; + provider?: "sp_api" | "keepa"; reason?: string; } @@ -114,13 +115,66 @@ export interface SupplierScore { export interface SupplierAnalysisResult { upc: string; rowNumber?: number; - record: ProductRecord; + record: SupplierInputRecord; + product: ProductRecord | null; lookup: UpcLookupDetail; keepa: KeepaData | null; spApi: SpApiData | null; score: SupplierScore; fetchedAt: string; } + +export interface SupplierInputRecord { + name: string; + unitCost: number; + brand?: string; + category?: string; +} + +export interface Product { + asin: string; + name: string | null; + brand: string | null; + category: string | null; + firstSeenAt: string; + lastSeenAt: string; +} + +export interface ProductObservation { + id: number; + productAsin: string; + runId: number; + source: string; + fetchedAt: string; +} + +export interface Run { + id: number; + type: + | "lead_analysis" + | "category_analysis" + | "supplier_upc" + | "stalker" + | "stalker_analysis"; + parentRunId?: number | null; + status: string; +} + +export interface RunItem { + id: number; + runId: number; + productAsin: string | null; + sourceRow?: number | null; +} + +export interface AnalysisRevision { + id: number; + runItemId: number; + decision: "FBA" | "FBM" | "BUY" | "WATCH" | "SKIP"; + confidence: number | null; + reasoning: string | null; + analyzedAt: string; +} export interface CategoryRunSummaryDb { categoryId: number; diff --git a/src/web/frontend.tsx b/src/web/frontend.tsx index 43f9425..8349d24 100644 --- a/src/web/frontend.tsx +++ b/src/web/frontend.tsx @@ -1,7 +1,8 @@ import { createRoot } from "react-dom/client"; import { useEffect, useMemo, useState } from "react"; -type ProcessType = "lead_analysis" | "category_analysis"; +type ProcessType = "lead_analysis" | "category_analysis" | "supplier_upc" | "stalker" | "stalker_analysis"; +type AnalysisDecision = "FBA" | "FBM" | "BUY" | "WATCH" | "SKIP"; type SortDirection = "ASC" | "DESC"; type Run = { @@ -15,6 +16,8 @@ type Run = { totalProducts: number; fbaCount: number; fbmCount: number; + buyCount: number; + watchCount: number; skipCount: number; }; @@ -37,11 +40,15 @@ type RunDetail = { totalProducts: number; fbaCount: number; fbmCount: number; + buyCount: number; + watchCount: number; skipCount: number; summary: { totalProducts: number; fbaCount: number; fbmCount: number; + buyCount: number; + watchCount: number; skipCount: number; }; errorMessage?: string; @@ -49,6 +56,8 @@ type RunDetail = { }; type ResultItem = { + item_id: number; + product_asin: string | null; id?: number; run_id: number; asin: string; @@ -60,7 +69,7 @@ type ResultItem = { sales_rank: number | null; seller_count: number | null; monthly_sold: number | null; - verdict: "FBA" | "FBM" | "SKIP"; + verdict: AnalysisDecision | null; amazon_is_seller: number | null; amazon_buybox_share_pct_90d: number | null; confidence: number | null; @@ -77,17 +86,18 @@ type ResultsResponse = { totalPages: number; }; -type VerdictFilter = "" | "FBA" | "FBM" | "SKIP"; +type VerdictFilter = "" | AnalysisDecision; type AmazonSellerFilter = "" | "yes" | "no"; type ProductListItem = { - processType: ProcessType; - runId: number; + item_id: number | null; + processType: ProcessType | null; + runId: number | null; asin: string; product_name: string | null; brand: string | null; category: string | null; - verdict: "FBA" | "FBM" | "SKIP"; + verdict: AnalysisDecision | null; confidence: number | null; sellability_status: string | null; monthly_sold: number | null; @@ -98,7 +108,7 @@ type ProductListItem = { current_price: number | null; avg_price_90d: number | null; reasoning: string | null; - fetched_at: string; + fetched_at: string | null; }; type ProductListResponse = { @@ -179,6 +189,35 @@ type StalkerProductsResponse = { totalPages: number; }; +type ProductHistoryResponse = { + product: { + asin: string; + name: string | null; + brand: string | null; + category: string | null; + first_seen_at: string; + last_seen_at: string; + }; + observations: Array<{ + id: number; + run_id: number; + source: string; + current_price: number | null; + monthly_sold: number | null; + sales_rank: number | null; + sellability_status: string | null; + fetched_at: string; + }>; + analyses: Array<{ + id: number; + run_id: number; + decision: string; + confidence: number | null; + reasoning: string | null; + analyzed_at: string; + }>; +}; + type SortState = { field: string; direction: SortDirection; @@ -362,7 +401,7 @@ function Dashboard({ setDeletingKey(key); try { - const response = await fetch(`/api/runs/${run.processType}/${run.runId}`, { method: "DELETE" }); + const response = await fetch(`/api/runs/${run.runId}`, { method: "DELETE" }); if (!response.ok) { const errorPayload = await response.json().catch(() => null) as { error?: string } | null; const message = errorPayload?.error ?? "Failed to delete run"; @@ -545,7 +584,7 @@ function RunDetails({ useEffect(() => { let cancelled = false; async function loadRun() { - const res = await fetch(`/api/runs/${processType}/${runId}`); + const res = await fetch(`/api/runs/${runId}`); const payload = (await res.json()) as RunDetail; if (!cancelled) { setRun(payload); @@ -573,7 +612,7 @@ function RunDetails({ if (minConfidence) params.set("minConfidence", minConfidence); if (maxConfidence) params.set("maxConfidence", maxConfidence); - const res = await fetch(`/api/runs/${processType}/${runId}/results?${params.toString()}`); + const res = await fetch(`/api/runs/${runId}/items?${params.toString()}`); const payload = (await res.json()) as ResultsResponse; if (!cancelled) { setResults(payload); @@ -596,12 +635,13 @@ function RunDetails({ }; }, [processType, runId]); - async function reanalyzeAsin(asin: string) { - if (reanalyzing[asin]) return; - setReanalyzing((prev) => ({ ...prev, [asin]: true })); + async function reanalyzeItem(item: ResultItem) { + const key = String(item.item_id); + if (reanalyzing[key]) return; + setReanalyzing((prev) => ({ ...prev, [key]: true })); try { const response = await fetch( - `/api/runs/${processType}/${runId}/asins/${encodeURIComponent(asin)}/reanalyze`, + `/api/run-items/${item.item_id}/reanalyze`, { method: "POST" }, ); if (!response.ok) { @@ -613,7 +653,7 @@ function RunDetails({ } finally { setReanalyzing((prev) => { const next = { ...prev }; - delete next[asin]; + delete next[key]; return next; }); } @@ -626,14 +666,14 @@ function RunDetails({

Run Detail

-
Process: {processType}
+
Process: {run?.processType ?? processType}
Run ID: {runId}
Status: {run ? {run.status} : "-"}
Timestamp: {run ? formatDate(run.timestamp) : "-"}
Job: {run?.jobType ?? "-"}
Source: {run?.source ?? "-"}
Total: {formatNumber(run?.summary.totalProducts)}
-
FBA/FBM/SKIP: {formatNumber(run?.summary.fbaCount)}/{formatNumber(run?.summary.fbmCount)}/{formatNumber(run?.summary.skipCount)}
+
FBA/FBM/BUY/WATCH/SKIP: {formatNumber(run?.summary.fbaCount)}/{formatNumber(run?.summary.fbmCount)}/{formatNumber(run?.summary.buyCount)}/{formatNumber(run?.summary.watchCount)}/{formatNumber(run?.summary.skipCount)}
@@ -647,6 +687,8 @@ function RunDetails({ + +