diff --git a/.env.example b/.env.example index 246c10e..ca0beb2 100644 --- a/.env.example +++ b/.env.example @@ -19,3 +19,5 @@ GOOGLE_API_KEY=your_google_api_key GOOGLE_CSE_ID=your_google_programmable_search_engine_id SERPAPI_API_KEY=your_serpapi_api_key_for_google_shopping +# Matches the default PostgreSQL service in docker-compose.yaml. +DB_CONNECTION_STRING=postgres://asin_check:asin_check@localhost:5432/asin_check diff --git a/CLAUDE.md b/CLAUDE.md index 5a4960b..ca79f9e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,8 +12,8 @@ Default to using Bun instead of Node.js. ## APIs -- `bun:sqlite` for SQLite. Don't use `better-sqlite3`. - `Bun.redis` for Redis. Don't use `ioredis`. +- Use Drizzle ORM with `postgres` driver for Postgres. Connection is in `src/db/index.ts`. - Prefer `Bun.file` over `node:fs`'s readFile/writeFile. - `Bun.$\`cmd\`` instead of execa. @@ -24,13 +24,13 @@ Default to using Bun instead of Node.js. bun test # Run a single test file -bun test src/supplier-scoring.test.ts +bun test src/supplier/supplier-scoring.test.ts # Type-check (no emit) ./node_modules/.bin/tsc --noEmit # ASIN lead-list pipeline (LLM-based) -bun run src/index.ts input/leads.xlsx --out output/results.xlsx +bun start leads.xlsx --out results.xlsx # Supplier UPC pipeline (deterministic) bun run upc-file --input input/supplier.xlsx --out output/supplier_ranked.xlsx @@ -40,6 +40,9 @@ bun run bestsellers bun run monthly-sold bun run mid-range +# Stalker pipeline +bun run stalker --input input/asins.xlsx + # Web API server bun run start:web # http://localhost:3000 @@ -47,29 +50,37 @@ bun run start:web # http://localhost:3000 bun run src/sp-test.ts bun run src/sp-test.ts B07SN9BHVV bun run src/sp-test.ts --sellability B07SN9BHVV + +# Database migrations (Drizzle) +bun run db:generate +bun run db:migrate ``` ## Architecture -Two distinct analysis pipelines share infrastructure (Keepa, SP-API, Redis, SQLite) but diverge in how they produce verdicts. +Two distinct analysis pipelines share infrastructure (Keepa, SP-API, Redis, Postgres) but diverge in how they produce verdicts. ### ASIN Lead-list Pipeline (`src/index.ts` → `src/analysis-pipeline.ts`) For spreadsheets containing known ASINs. Verdict is LLM-based (FBA/FBM/SKIP via LM Studio). -Flow: `reader.ts` parse → Redis cache check → `sp-api.ts` sellability gate (5 concurrent workers) → `keepa.ts` batch enrichment → `sp-api.ts` pricing + FBA fees (5 concurrent workers) → `llm.ts` batched analysis (5 products/batch) → `writer.ts` XLSX + SQLite. +Flow: `reader.ts` parse → Redis cache check → `integrations/sp-api.ts` sellability gate (5 concurrent workers) → `integrations/keepa.ts` batch enrichment → `integrations/sp-api.ts` pricing + FBA fees (5 concurrent workers) → `integrations/llm.ts` batched analysis (5 products/batch) → `writer.ts` XLSX + Postgres. -### Supplier UPC Pipeline (`src/upc-file-analysis.ts`) +### Supplier UPC Pipeline (`src/supplier/upc-file-analysis.ts`) For supplier price lists containing UPC/EAN values. Verdict is deterministic (BUY/WATCH/SKIP); never calls LM Studio. -Flow: `upc-file-reader.ts` streaming parse (`.xlsx`) or row-window parse (`.xls`) → SP-API catalog UPC lookup first, Keepa UPC lookup as fallback → `keepa.ts` demand enrichment → `sp-api.ts` sellability + FBA fees → `supplier-scoring.ts` deterministic score → `supplier-export.ts` Excel workbook (`Ranked Leads`, `Skipped`, `Summary` sheets) + SQLite. +Flow: `supplier/upc-file-reader.ts` streaming parse (`.xlsx`) or row-window parse (`.xls`) → SP-API catalog UPC lookup first, Keepa UPC lookup as fallback → `integrations/keepa.ts` demand enrichment → `integrations/sp-api.ts` sellability + FBA fees → `supplier/supplier-scoring.ts` deterministic score → `supplier/supplier-export.ts` Excel workbook (`Ranked Leads`, `Skipped`, `Summary` sheets) + Postgres. UPC resolution priority: SP-API catalog lookup → Keepa fallback (for no-match or request failure only). ### Category Pipelines -`bestsellers-by-category.ts`, `top-monthly-sold-by-category.ts`, `mid-range-sellers-by-category.ts` — Keepa category browsing → SP-API sellability gate → LLM verdict. Each saves results to SQLite. Mid-range applies configurable filters (monthly sold, price, seller count, Amazon buy box share). +`src/categories/` — Keepa category browsing → SP-API sellability gate → LLM verdict. Each saves results to Postgres. Mid-range applies configurable filters (monthly sold, price, seller count, Amazon buy box share). + +### Stalker Pipeline (`src/stalker/stalker.ts`) + +Tracks competitor sellers across ASINs. Fetches storefronts, checks sellability of inventory items, and persists matched seller data to Postgres. ### Shared Infrastructure @@ -77,18 +88,24 @@ UPC resolution priority: SP-API catalog lookup → Keepa fallback (for no-match |--------|------| | `src/types.ts` | All shared interfaces (`ProductRecord`, `KeepaData`, `SpApiData`, `SupplierScore`, etc.) | | `src/config.ts` | Env var loading via `Bun.env` | -| `src/keepa.ts` | Keepa API: batch ASIN fetch, UPC lookup, auto rate-limiting on token exhaustion | -| `src/sp-api.ts` | SP-API: sellability (`getListingsRestrictions`), pricing+fees, UPC catalog lookup | -| `src/cache.ts` | Redis caching (24h TTL for lead-list; 12h for mid-range) | -| `src/database.ts` | SQLite `runs` + `results` tables; auto-creates `db/results.db` | +| `src/db/index.ts` | Drizzle Postgres connection (shared pool) | +| `src/db/schema.ts` | Drizzle schema for all tables | +| `src/db/persistence.ts` | Product, observation, unified run-item, UPC resolution, and revision persistence | +| `src/integrations/keepa.ts` | Keepa API: batch ASIN fetch, UPC lookup, auto rate-limiting | +| `src/integrations/sp-api.ts` | SP-API: sellability, pricing+fees, UPC catalog lookup | +| `src/integrations/cache.ts` | Redis caching (24h TTL for lead-list; 12h for mid-range) | +| `src/integrations/llm.ts` | LLM integration (LM Studio / Claude) | | `src/server.ts` | Bun HTTP server exposing REST endpoints for both pipelines | ### File Layout +- `src/integrations/` — external API clients (Keepa, SP-API, Redis cache, LLM, SearXNG) +- `src/categories/` — category discovery pipelines +- `src/stalker/` — competitor seller tracking pipeline +- `src/supplier/` — supplier UPC analysis pipeline +- `src/db/` — Drizzle schema and connection - `input/` — source spreadsheets (git-ignored) - `output/` — generated workbooks (git-ignored) -- `db/` — SQLite files (git-ignored) -- `src/` — all source and test files ## Project Rules @@ -96,4 +113,6 @@ UPC resolution priority: SP-API catalog lookup → Keepa fallback (for no-match - The supplier UPC pipeline must not call LM Studio. - Supplier UPC files resolve UPC/EAN through SP-API catalog lookup first; Keepa UPC lookup is fallback only (no-match or request-failure cases). - Supplier workbook output must keep `Ranked Leads`, `Skipped`, and `Summary` sheets. +- Treat `products.asin` as the canonical normalized product identity; UPC values belong only in identifier and resolution records. +- Store time-varying data in observations or revisions and retain run history rather than overwriting prior analysis. - When changing UPC supplier behavior, cover SP-API UPC parsing, deterministic scoring, and workbook export with `bun test`. diff --git a/README.md b/README.md index 57b0d21..f418c0e 100644 --- a/README.md +++ b/README.md @@ -21,17 +21,19 @@ cp .env.example .env ## Usage ```bash -bun run src/index.ts input/ [--out output/results.xlsx] +bun start [--out results.xlsx] ``` Add `--claude` to use Anthropic Claude instead of local LM Studio for LLM analysis. +Bare input and output filenames use the `input/` and `output/` directories. Pass a path containing a directory to override those defaults. Examples: ```bash -bun run src/index.ts input/leads.xlsx -bun run src/index.ts input/leads.csv --out output/results.xlsx -bun run src/index.ts input/leads.xlsx --claude +bun start leads.xlsx +bun start leads.csv --out results.xlsx +bun start leads.xlsx --claude +bun start archive/leads.xlsx --out exports/results.xlsx ``` Large-file behavior: @@ -155,7 +157,7 @@ ranked sourcing workbook: 2. Resolves UPCs to ASINs with SP-API catalog lookup first, then falls back to Keepa for no-match/request-failure cases. 3. Enriches resolved ASINs with Keepa demand/competition data and SP-API sellability + FBA fees. 4. Scores products with deterministic BUY/WATCH/SKIP logic; this path does not call LM Studio. -5. Writes a ranked Excel workbook and persists rows into the existing `runs` + `results` tables. +5. Writes a ranked Excel workbook and persists rows through unified runs, UPC resolution, product observation, and scoring-history tables. CLI usage: @@ -244,20 +246,28 @@ Numeric parsing accepts plain numbers as well as formatted values like `$12.50`, 4. **Keepa fetch** — batch the sellable (uncached) ASINs in a single API call (up to 100 per request) 5. **Enrich** — fetch SP-API pricing + FBA/FBM fees for sellable ASINs; combine with Keepa data and spreadsheet data 6. **LLM analysis** — send batches of 5 sellable products to LM Studio for FBA/FBM/SKIP verdict; skipped ASINs get auto-SKIP verdict (confidence 100) and bypass LLM entirely -7. **Output** — print results table to console (includes all ASINs), optionally write CSV/XLSX, and **persist results to a SQLite database**. +7. **Output** — print results table to console (includes all ASINs), optionally write CSV/XLSX, and persist products, observations, run items, and analysis revisions to PostgreSQL. -## Persistent Storage with SQLite +## Persistent Storage -Results from each run are now stored in a SQLite database named `db/results.db` by default. The SQLite implementation details are handled in `src/database.ts`. This allows you to: +PostgreSQL persistence is managed with Drizzle in `src/db/schema.ts` and `src/db/persistence.ts`. ASINs are canonical product identities: all inputs normalize to uppercase 10-character alphanumeric keys before any product reference is stored. -- Revisit past analysis results. -- Query and analyze historical data. -- Track product performance over time. +Core tables: -The database will automatically be created if it doesn't exist. Two tables are created: +- `products`: one canonical row per ASIN with latest descriptive metadata. +- `product_observations`: append-only marketplace, pricing, fee, and sellability snapshots. +- `runs` and `run_items`: unified lifecycle/history for lead, category, supplier UPC, and stalker workflows. +- `analysis_revisions` and `supplier_scores`: append-only analysis results; reanalysis does not overwrite prior decisions. +- `sourcing_inputs`, `upc_resolutions`, and `product_identifiers`: source-row and confirmed identifier data kept separate from catalog products. +- `stalker_run_details`, `stalker_scans`, and `stalker_inventory_items`: seller workflow provenance linked back to products and observations. -- `runs`: Stores metadata about each analysis run (timestamp, input file, output file, and summary counts). -- `results`: Stores detailed analysis results for each product from each run, linked to the `runs` table. +Unresolved or ambiguous supplier UPCs stay on their run item and resolution records; a UPC is never stored as an ASIN. + +Web endpoints use unified identifiers: + +- `GET /api/runs`, `GET /api/runs/:runId`, `GET /api/runs/:runId/items` +- `GET /api/products`, `GET /api/products/:asin` +- `POST /api/run-items/:itemId/reanalyze` ## Output columns diff --git a/bun.lock b/bun.lock index af03c41..42de449 100644 --- a/bun.lock +++ b/bun.lock @@ -6,8 +6,10 @@ "name": "asin-check", "dependencies": { "amazon-sp-api": "^1.2.1", + "drizzle-orm": "^0.45.2", "exceljs": "^4.4.0", "ioredis": "^5.10.1", + "postgres": "^3.4.9", "react": "^19.2.0", "react-dom": "^19.2.0", "xlsx": "^0.18.5", @@ -16,11 +18,70 @@ "@types/bun": "latest", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", + "drizzle-kit": "^0.31.10", "typescript": "^6.0.3", }, }, }, "packages": { + "@drizzle-team/brocli": ["@drizzle-team/brocli@0.10.2", "", {}, "sha512-z33Il7l5dKjUgGULTqBsQBQwckHh5AbIuxhdsIxDDiZAzBOrZO6q9ogcWC65kU382AfynTfgNumVcNIjuIua6w=="], + + "@esbuild-kit/core-utils": ["@esbuild-kit/core-utils@3.3.2", "", { "dependencies": { "esbuild": "~0.18.20", "source-map-support": "^0.5.21" } }, "sha512-sPRAnw9CdSsRmEtnsl2WXWdyquogVpB3yZ3dgwJfe8zrOzTsV7cJvmwrKVa+0ma5BoiGJ+BoqkMvawbayKUsqQ=="], + + "@esbuild-kit/esm-loader": ["@esbuild-kit/esm-loader@2.6.5", "", { "dependencies": { "@esbuild-kit/core-utils": "^3.3.2", "get-tsconfig": "^4.7.0" } }, "sha512-FxEMIkJKnodyA1OaCUoEvbYRkoZlLZ4d/eXFu9Fh8CbBBgP5EmZxrfTRyN0qpXZ4vOvqnE5YdRdcrmUUXuU+dA=="], + + "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.25.12", "", { "os": "aix", "cpu": "ppc64" }, "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA=="], + + "@esbuild/android-arm": ["@esbuild/android-arm@0.25.12", "", { "os": "android", "cpu": "arm" }, "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg=="], + + "@esbuild/android-arm64": ["@esbuild/android-arm64@0.25.12", "", { "os": "android", "cpu": "arm64" }, "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg=="], + + "@esbuild/android-x64": ["@esbuild/android-x64@0.25.12", "", { "os": "android", "cpu": "x64" }, "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg=="], + + "@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.25.12", "", { "os": "darwin", "cpu": "arm64" }, "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg=="], + + "@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.25.12", "", { "os": "darwin", "cpu": "x64" }, "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA=="], + + "@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.25.12", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg=="], + + "@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.25.12", "", { "os": "freebsd", "cpu": "x64" }, "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ=="], + + "@esbuild/linux-arm": ["@esbuild/linux-arm@0.25.12", "", { "os": "linux", "cpu": "arm" }, "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw=="], + + "@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.25.12", "", { "os": "linux", "cpu": "arm64" }, "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ=="], + + "@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.25.12", "", { "os": "linux", "cpu": "ia32" }, "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA=="], + + "@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng=="], + + "@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw=="], + + "@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.25.12", "", { "os": "linux", "cpu": "ppc64" }, "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA=="], + + "@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w=="], + + "@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.25.12", "", { "os": "linux", "cpu": "s390x" }, "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg=="], + + "@esbuild/linux-x64": ["@esbuild/linux-x64@0.25.12", "", { "os": "linux", "cpu": "x64" }, "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw=="], + + "@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.25.12", "", { "os": "none", "cpu": "arm64" }, "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg=="], + + "@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.25.12", "", { "os": "none", "cpu": "x64" }, "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ=="], + + "@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.25.12", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A=="], + + "@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.25.12", "", { "os": "openbsd", "cpu": "x64" }, "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw=="], + + "@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.25.12", "", { "os": "none", "cpu": "arm64" }, "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg=="], + + "@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.25.12", "", { "os": "sunos", "cpu": "x64" }, "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w=="], + + "@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.25.12", "", { "os": "win32", "cpu": "arm64" }, "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg=="], + + "@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.25.12", "", { "os": "win32", "cpu": "ia32" }, "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ=="], + + "@esbuild/win32-x64": ["@esbuild/win32-x64@0.25.12", "", { "os": "win32", "cpu": "x64" }, "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA=="], + "@fast-csv/format": ["@fast-csv/format@4.3.5", "", { "dependencies": { "@types/node": "^14.0.1", "lodash.escaperegexp": "^4.1.2", "lodash.isboolean": "^3.0.3", "lodash.isequal": "^4.5.0", "lodash.isfunction": "^3.0.9", "lodash.isnil": "^4.0.0" } }, "sha512-8iRn6QF3I8Ak78lNAa+Gdl5MJJBM5vRHivFtMRUWINdevNo00K7OXxS2PshawLKTejVwieIlPmK5YlLu6w4u8A=="], "@fast-csv/parse": ["@fast-csv/parse@4.3.6", "", { "dependencies": { "@types/node": "^14.0.1", "lodash.escaperegexp": "^4.1.2", "lodash.groupby": "^4.6.0", "lodash.isfunction": "^3.0.9", "lodash.isnil": "^4.0.0", "lodash.isundefined": "^3.0.1", "lodash.uniq": "^4.5.0" } }, "sha512-uRsLYksqpbDmWaSmzvJcuApSEe38+6NQZBUsuAyMZKqHxH0g1wcJgsKUvN3WC8tewaqFjBMMGrkHmC+T7k8LvA=="], @@ -63,6 +124,8 @@ "buffer-crc32": ["buffer-crc32@0.2.13", "", {}, "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ=="], + "buffer-from": ["buffer-from@1.1.2", "", {}, "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="], + "buffer-indexof-polyfill": ["buffer-indexof-polyfill@1.0.2", "", {}, "sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A=="], "buffers": ["buffers@0.1.1", "", {}, "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ=="], @@ -101,6 +164,10 @@ "denque": ["denque@2.1.0", "", {}, "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw=="], + "drizzle-kit": ["drizzle-kit@0.31.10", "", { "dependencies": { "@drizzle-team/brocli": "^0.10.2", "@esbuild-kit/esm-loader": "^2.5.5", "esbuild": "^0.25.4", "tsx": "^4.21.0" }, "bin": { "drizzle-kit": "bin.cjs" } }, "sha512-7OZcmQUrdGI+DUNNsKBn1aW8qSoKuTH7d0mYgSP8bAzdFzKoovxEFnoGQp2dVs82EOJeYycqRtciopszwUf8bw=="], + + "drizzle-orm": ["drizzle-orm@0.45.2", "", { "peerDependencies": { "@aws-sdk/client-rds-data": ">=3", "@cloudflare/workers-types": ">=4", "@electric-sql/pglite": ">=0.2.0", "@libsql/client": ">=0.10.0", "@libsql/client-wasm": ">=0.10.0", "@neondatabase/serverless": ">=0.10.0", "@op-engineering/op-sqlite": ">=2", "@opentelemetry/api": "^1.4.1", "@planetscale/database": ">=1.13", "@prisma/client": "*", "@tidbcloud/serverless": "*", "@types/better-sqlite3": "*", "@types/pg": "*", "@types/sql.js": "*", "@upstash/redis": ">=1.34.7", "@vercel/postgres": ">=0.8.0", "@xata.io/client": "*", "better-sqlite3": ">=7", "bun-types": "*", "expo-sqlite": ">=14.0.0", "gel": ">=2", "knex": "*", "kysely": "*", "mysql2": ">=2", "pg": ">=8", "postgres": ">=3", "prisma": "*", "sql.js": ">=1", "sqlite3": ">=5" }, "optionalPeers": ["@aws-sdk/client-rds-data", "@cloudflare/workers-types", "@electric-sql/pglite", "@libsql/client", "@libsql/client-wasm", "@neondatabase/serverless", "@op-engineering/op-sqlite", "@opentelemetry/api", "@planetscale/database", "@prisma/client", "@tidbcloud/serverless", "@types/better-sqlite3", "@types/pg", "@types/sql.js", "@upstash/redis", "@vercel/postgres", "@xata.io/client", "better-sqlite3", "bun-types", "expo-sqlite", "gel", "knex", "kysely", "mysql2", "pg", "postgres", "prisma", "sql.js", "sqlite3"] }, "sha512-kY0BSaTNYWnoDMVoyY8uxmyHjpJW1geOmBMdSSicKo9CIIWkSxMIj2rkeSR51b8KAPB7m+qysjuHme5nKP+E5Q=="], + "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="], "duplexer2": ["duplexer2@0.1.4", "", { "dependencies": { "readable-stream": "^2.0.2" } }, "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA=="], @@ -113,6 +180,8 @@ "es-object-atoms": ["es-object-atoms@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0" } }, "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA=="], + "esbuild": ["esbuild@0.25.12", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.25.12", "@esbuild/android-arm": "0.25.12", "@esbuild/android-arm64": "0.25.12", "@esbuild/android-x64": "0.25.12", "@esbuild/darwin-arm64": "0.25.12", "@esbuild/darwin-x64": "0.25.12", "@esbuild/freebsd-arm64": "0.25.12", "@esbuild/freebsd-x64": "0.25.12", "@esbuild/linux-arm": "0.25.12", "@esbuild/linux-arm64": "0.25.12", "@esbuild/linux-ia32": "0.25.12", "@esbuild/linux-loong64": "0.25.12", "@esbuild/linux-mips64el": "0.25.12", "@esbuild/linux-ppc64": "0.25.12", "@esbuild/linux-riscv64": "0.25.12", "@esbuild/linux-s390x": "0.25.12", "@esbuild/linux-x64": "0.25.12", "@esbuild/netbsd-arm64": "0.25.12", "@esbuild/netbsd-x64": "0.25.12", "@esbuild/openbsd-arm64": "0.25.12", "@esbuild/openbsd-x64": "0.25.12", "@esbuild/openharmony-arm64": "0.25.12", "@esbuild/sunos-x64": "0.25.12", "@esbuild/win32-arm64": "0.25.12", "@esbuild/win32-ia32": "0.25.12", "@esbuild/win32-x64": "0.25.12" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg=="], + "exceljs": ["exceljs@4.4.0", "", { "dependencies": { "archiver": "^5.0.0", "dayjs": "^1.8.34", "fast-csv": "^4.3.1", "jszip": "^3.10.1", "readable-stream": "^3.6.0", "saxes": "^5.0.1", "tmp": "^0.2.0", "unzipper": "^0.10.11", "uuid": "^8.3.0" } }, "sha512-XctvKaEMaj1Ii9oDOqbW/6e1gXknSY4g/aLCDicOXqBE4M0nRWkUu0PTp++UPNzoFY12BNHMfs/VadKIS6llvg=="], "fast-csv": ["fast-csv@4.3.6", "", { "dependencies": { "@fast-csv/format": "4.3.5", "@fast-csv/parse": "4.3.6" } }, "sha512-2RNSpuwwsJGP0frGsOmTb9oUF+VkFSM4SyLTDgwf2ciHWTarN0lQTC+F2f/t5J9QjW+c65VFIAAu85GsvMIusw=="], @@ -127,6 +196,8 @@ "fs.realpath": ["fs.realpath@1.0.0", "", {}, "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw=="], + "fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], + "fstream": ["fstream@1.0.12", "", { "dependencies": { "graceful-fs": "^4.1.2", "inherits": "~2.0.0", "mkdirp": ">=0.5 0", "rimraf": "2" } }, "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg=="], "function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="], @@ -135,6 +206,8 @@ "get-proto": ["get-proto@1.0.1", "", { "dependencies": { "dunder-proto": "^1.0.1", "es-object-atoms": "^1.0.0" } }, "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g=="], + "get-tsconfig": ["get-tsconfig@4.14.0", "", { "dependencies": { "resolve-pkg-maps": "^1.0.0" } }, "sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA=="], + "glob": ["glob@7.2.3", "", { "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", "minimatch": "^3.1.1", "once": "^1.3.0", "path-is-absolute": "^1.0.0" } }, "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q=="], "gopd": ["gopd@1.2.0", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="], @@ -217,6 +290,8 @@ "path-is-absolute": ["path-is-absolute@1.0.1", "", {}, "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg=="], + "postgres": ["postgres@3.4.9", "", {}, "sha512-GD3qdB0x1z9xgFI6cdRD6xu2Sp2WCOEoe3mtnyB5Ee0XrrL5Pe+e4CCnJrRMnL1zYtRDZmQQVbvOttLnKDLnaw=="], + "process-nextick-args": ["process-nextick-args@2.0.1", "", {}, "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="], "qs": ["qs@6.15.1", "", { "dependencies": { "side-channel": "^1.1.0" } }, "sha512-6YHEFRL9mfgcAvql/XhwTvf5jKcOiiupt2FiJxHkiX1z4j7WL8J/jRHYLluORvc1XxB5rV20KoeK00gVJamspg=="], @@ -233,6 +308,8 @@ "redis-parser": ["redis-parser@3.0.0", "", { "dependencies": { "redis-errors": "^1.0.0" } }, "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A=="], + "resolve-pkg-maps": ["resolve-pkg-maps@1.0.0", "", {}, "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw=="], + "rimraf": ["rimraf@2.7.1", "", { "dependencies": { "glob": "^7.1.3" }, "bin": { "rimraf": "./bin.js" } }, "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w=="], "safe-buffer": ["safe-buffer@5.1.2", "", {}, "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="], @@ -253,6 +330,10 @@ "side-channel-weakmap": ["side-channel-weakmap@1.0.2", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3", "side-channel-map": "^1.0.1" } }, "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A=="], + "source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="], + + "source-map-support": ["source-map-support@0.5.21", "", { "dependencies": { "buffer-from": "^1.0.0", "source-map": "^0.6.0" } }, "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w=="], + "ssf": ["ssf@0.11.2", "", { "dependencies": { "frac": "~1.1.2" } }, "sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g=="], "standard-as-callback": ["standard-as-callback@2.1.0", "", {}, "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A=="], @@ -267,6 +348,8 @@ "traverse": ["traverse@0.3.9", "", {}, "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ=="], + "tsx": ["tsx@4.22.3", "", { "dependencies": { "esbuild": "~0.28.0" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "bin": { "tsx": "dist/cli.mjs" } }, "sha512-mdoNxBC/cSQObGGVQ5Bpn5i+yv7j68gk3Nfm3wFjcJg3Z0Mix9jzAFfP12prmm5eVGmDKtp0yyArrs0Q+8gZHg=="], + "typescript": ["typescript@6.0.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw=="], "undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="], @@ -289,6 +372,8 @@ "zip-stream": ["zip-stream@4.1.1", "", { "dependencies": { "archiver-utils": "^3.0.4", "compress-commons": "^4.1.2", "readable-stream": "^3.6.0" } }, "sha512-9qv4rlDiopXg4E69k+vMHjNN63YFMe9sZMrdlvKnCjlCRWeCBswPPMPUfx+ipsAWq1LXHe70RcbaHdJJpS6hyQ=="], + "@esbuild-kit/core-utils/esbuild": ["esbuild@0.18.20", "", { "optionalDependencies": { "@esbuild/android-arm": "0.18.20", "@esbuild/android-arm64": "0.18.20", "@esbuild/android-x64": "0.18.20", "@esbuild/darwin-arm64": "0.18.20", "@esbuild/darwin-x64": "0.18.20", "@esbuild/freebsd-arm64": "0.18.20", "@esbuild/freebsd-x64": "0.18.20", "@esbuild/linux-arm": "0.18.20", "@esbuild/linux-arm64": "0.18.20", "@esbuild/linux-ia32": "0.18.20", "@esbuild/linux-loong64": "0.18.20", "@esbuild/linux-mips64el": "0.18.20", "@esbuild/linux-ppc64": "0.18.20", "@esbuild/linux-riscv64": "0.18.20", "@esbuild/linux-s390x": "0.18.20", "@esbuild/linux-x64": "0.18.20", "@esbuild/netbsd-x64": "0.18.20", "@esbuild/openbsd-x64": "0.18.20", "@esbuild/sunos-x64": "0.18.20", "@esbuild/win32-arm64": "0.18.20", "@esbuild/win32-ia32": "0.18.20", "@esbuild/win32-x64": "0.18.20" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA=="], + "@fast-csv/format/@types/node": ["@types/node@14.18.63", "", {}, "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ=="], "@fast-csv/parse/@types/node": ["@types/node@14.18.63", "", {}, "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ=="], @@ -305,10 +390,56 @@ "string_decoder/safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="], + "tsx/esbuild": ["esbuild@0.28.0", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.28.0", "@esbuild/android-arm": "0.28.0", "@esbuild/android-arm64": "0.28.0", "@esbuild/android-x64": "0.28.0", "@esbuild/darwin-arm64": "0.28.0", "@esbuild/darwin-x64": "0.28.0", "@esbuild/freebsd-arm64": "0.28.0", "@esbuild/freebsd-x64": "0.28.0", "@esbuild/linux-arm": "0.28.0", "@esbuild/linux-arm64": "0.28.0", "@esbuild/linux-ia32": "0.28.0", "@esbuild/linux-loong64": "0.28.0", "@esbuild/linux-mips64el": "0.28.0", "@esbuild/linux-ppc64": "0.28.0", "@esbuild/linux-riscv64": "0.28.0", "@esbuild/linux-s390x": "0.28.0", "@esbuild/linux-x64": "0.28.0", "@esbuild/netbsd-arm64": "0.28.0", "@esbuild/netbsd-x64": "0.28.0", "@esbuild/openbsd-arm64": "0.28.0", "@esbuild/openbsd-x64": "0.28.0", "@esbuild/openharmony-arm64": "0.28.0", "@esbuild/sunos-x64": "0.28.0", "@esbuild/win32-arm64": "0.28.0", "@esbuild/win32-ia32": "0.28.0", "@esbuild/win32-x64": "0.28.0" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-sNR9MHpXSUV/XB4zmsFKN+QgVG82Cc7+/aaxJ8Adi8hyOac+EXptIp45QBPaVyX3N70664wRbTcLTOemCAnyqw=="], + "unzipper/readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="], "zip-stream/archiver-utils": ["archiver-utils@3.0.4", "", { "dependencies": { "glob": "^7.2.3", "graceful-fs": "^4.2.0", "lazystream": "^1.0.0", "lodash.defaults": "^4.2.0", "lodash.difference": "^4.5.0", "lodash.flatten": "^4.4.0", "lodash.isplainobject": "^4.0.6", "lodash.union": "^4.6.0", "normalize-path": "^3.0.0", "readable-stream": "^3.6.0" } }, "sha512-KVgf4XQVrTjhyWmx6cte4RxonPLR9onExufI1jhvw/MQ4BB6IsZD5gT8Lq+u/+pRkWna/6JoHpiQioaqFP5Rzw=="], + "@esbuild-kit/core-utils/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.18.20", "", { "os": "android", "cpu": "arm" }, "sha512-fyi7TDI/ijKKNZTUJAQqiG5T7YjJXgnzkURqmGj13C6dCqckZBLdl4h7bkhHt/t0WP+zO9/zwroDvANaOqO5Sw=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/android-arm64": ["@esbuild/android-arm64@0.18.20", "", { "os": "android", "cpu": "arm64" }, "sha512-Nz4rJcchGDtENV0eMKUNa6L12zz2zBDXuhj/Vjh18zGqB44Bi7MBMSXjgunJgjRhCmKOjnPuZp4Mb6OKqtMHLQ=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/android-x64": ["@esbuild/android-x64@0.18.20", "", { "os": "android", "cpu": "x64" }, "sha512-8GDdlePJA8D6zlZYJV/jnrRAi6rOiNaCC/JclcXpB+KIuvfBN4owLtgzY2bsxnx666XjJx2kDPUmnTtR8qKQUg=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.18.20", "", { "os": "darwin", "cpu": "arm64" }, "sha512-bxRHW5kHU38zS2lPTPOyuyTm+S+eobPUnTNkdJEfAddYgEcll4xkT8DB9d2008DtTbl7uJag2HuE5NZAZgnNEA=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.18.20", "", { "os": "darwin", "cpu": "x64" }, "sha512-pc5gxlMDxzm513qPGbCbDukOdsGtKhfxD1zJKXjCCcU7ju50O7MeAZ8c4krSJcOIJGFR+qx21yMMVYwiQvyTyQ=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.18.20", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-yqDQHy4QHevpMAaxhhIwYPMv1NECwOvIpGCZkECn8w2WFHXjEwrBn3CeNIYsibZ/iZEUemj++M26W3cNR5h+Tw=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.18.20", "", { "os": "freebsd", "cpu": "x64" }, "sha512-tgWRPPuQsd3RmBZwarGVHZQvtzfEBOreNuxEMKFcd5DaDn2PbBxfwLcj4+aenoh7ctXcbXmOQIn8HI6mCSw5MQ=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/linux-arm": ["@esbuild/linux-arm@0.18.20", "", { "os": "linux", "cpu": "arm" }, "sha512-/5bHkMWnq1EgKr1V+Ybz3s1hWXok7mDFUMQ4cG10AfW3wL02PSZi5kFpYKrptDsgb2WAJIvRcDm+qIvXf/apvg=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.18.20", "", { "os": "linux", "cpu": "arm64" }, "sha512-2YbscF+UL7SQAVIpnWvYwM+3LskyDmPhe31pE7/aoTMFKKzIc9lLbyGUpmmb8a8AixOL61sQ/mFh3jEjHYFvdA=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.18.20", "", { "os": "linux", "cpu": "ia32" }, "sha512-P4etWwq6IsReT0E1KHU40bOnzMHoH73aXp96Fs8TIT6z9Hu8G6+0SHSw9i2isWrD2nbx2qo5yUqACgdfVGx7TA=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.18.20", "", { "os": "linux", "cpu": "none" }, "sha512-nXW8nqBTrOpDLPgPY9uV+/1DjxoQ7DoB2N8eocyq8I9XuqJ7BiAMDMf9n1xZM9TgW0J8zrquIb/A7s3BJv7rjg=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.18.20", "", { "os": "linux", "cpu": "none" }, "sha512-d5NeaXZcHp8PzYy5VnXV3VSd2D328Zb+9dEq5HE6bw6+N86JVPExrA6O68OPwobntbNJ0pzCpUFZTo3w0GyetQ=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.18.20", "", { "os": "linux", "cpu": "ppc64" }, "sha512-WHPyeScRNcmANnLQkq6AfyXRFr5D6N2sKgkFo2FqguP44Nw2eyDlbTdZwd9GYk98DZG9QItIiTlFLHJHjxP3FA=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.18.20", "", { "os": "linux", "cpu": "none" }, "sha512-WSxo6h5ecI5XH34KC7w5veNnKkju3zBRLEQNY7mv5mtBmrP/MjNBCAlsM2u5hDBlS3NGcTQpoBvRzqBcRtpq1A=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.18.20", "", { "os": "linux", "cpu": "s390x" }, "sha512-+8231GMs3mAEth6Ja1iK0a1sQ3ohfcpzpRLH8uuc5/KVDFneH6jtAJLFGafpzpMRO6DzJ6AvXKze9LfFMrIHVQ=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/linux-x64": ["@esbuild/linux-x64@0.18.20", "", { "os": "linux", "cpu": "x64" }, "sha512-UYqiqemphJcNsFEskc73jQ7B9jgwjWrSayxawS6UVFZGWrAAtkzjxSqnoclCXxWtfwLdzU+vTpcNYhpn43uP1w=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.18.20", "", { "os": "none", "cpu": "x64" }, "sha512-iO1c++VP6xUBUmltHZoMtCUdPlnPGdBom6IrO4gyKPFFVBKioIImVooR5I83nTew5UOYrk3gIJhbZh8X44y06A=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.18.20", "", { "os": "openbsd", "cpu": "x64" }, "sha512-e5e4YSsuQfX4cxcygw/UCPIEP6wbIL+se3sxPdCiMbFLBWu0eiZOJ7WoD+ptCLrmjZBK1Wk7I6D/I3NglUGOxg=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.18.20", "", { "os": "sunos", "cpu": "x64" }, "sha512-kDbFRFp0YpTQVVrqUd5FTYmWo45zGaXe0X8E1G/LKFC0v8x0vWrhOWSLITcCn63lmZIxfOMXtCfti/RxN/0wnQ=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.18.20", "", { "os": "win32", "cpu": "arm64" }, "sha512-ddYFR6ItYgoaq4v4JmQQaAI5s7npztfV4Ag6NrhiaW0RrnOXqBkgwZLofVTlq1daVTQNhtI5oieTvkRPfZrePg=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.18.20", "", { "os": "win32", "cpu": "ia32" }, "sha512-Wv7QBi3ID/rROT08SABTS7eV4hX26sVduqDOTe1MvGMjNd3EjOz4b7zeexIR62GTIEKrfJXKL9LFxTYgkyeu7g=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.18.20", "", { "os": "win32", "cpu": "x64" }, "sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ=="], + "archiver-utils/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], "duplexer2/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], @@ -319,6 +450,58 @@ "lazystream/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], + "tsx/esbuild/@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.28.0", "", { "os": "aix", "cpu": "ppc64" }, "sha512-lhRUCeuOyJQURhTxl4WkpFTjIsbDayJHih5kZC1giwE+MhIzAb7mEsQMqMf18rHLsrb5qI1tafG20mLxEWcWlA=="], + + "tsx/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.28.0", "", { "os": "android", "cpu": "arm" }, "sha512-wqh0ByljabXLKHeWXYLqoJ5jKC4XBaw6Hk08OfMrCRd2nP2ZQ5eleDZC41XHyCNgktBGYMbqnrJKq/K/lzPMSQ=="], + + "tsx/esbuild/@esbuild/android-arm64": ["@esbuild/android-arm64@0.28.0", "", { "os": "android", "cpu": "arm64" }, "sha512-+WzIXQOSaGs33tLEgYPYe/yQHf0WTU0X42Jca3y8NWMbUVhp7rUnw+vAsRC/QiDrdD31IszMrZy+qwPOPjd+rw=="], + + "tsx/esbuild/@esbuild/android-x64": ["@esbuild/android-x64@0.28.0", "", { "os": "android", "cpu": "x64" }, "sha512-+VJggoaKhk2VNNqVL7f6S189UzShHC/mR9EE8rDdSkdpN0KflSwWY/gWjDrNxxisg8Fp1ZCD9jLMo4m0OUfeUA=="], + + "tsx/esbuild/@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.28.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-0T+A9WZm+bZ84nZBtk1ckYsOvyA3x7e2Acj1KdVfV4/2tdG4fzUp91YHx+GArWLtwqp77pBXVCPn2We7Letr0Q=="], + + "tsx/esbuild/@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.28.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-fyzLm/DLDl/84OCfp2f/XQ4flmORsjU7VKt8HLjvIXChJoFFOIL6pLJPH4Yhd1n1gGFF9mPwtlN5Wf82DZs+LQ=="], + + "tsx/esbuild/@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.28.0", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-l9GeW5UZBT9k9brBYI+0WDffcRxgHQD8ShN2Ur4xWq/NFzUKm3k5lsH4PdaRgb2w7mI9u61nr2gI2mLI27Nh3Q=="], + + "tsx/esbuild/@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.28.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-BXoQai/A0wPO6Es3yFJ7APCiKGc1tdAEOgeTNy3SsB491S3aHn4S4r3e976eUnPdU+NbdtmBuLncYir2tMU9Nw=="], + + "tsx/esbuild/@esbuild/linux-arm": ["@esbuild/linux-arm@0.28.0", "", { "os": "linux", "cpu": "arm" }, "sha512-CjaaREJagqJp7iTaNQjjidaNbCKYcd4IDkzbwwxtSvjI7NZm79qiHc8HqciMddQ6CKvJT6aBd8lO9kN/ZudLlw=="], + + "tsx/esbuild/@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.28.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-RVyzfb3FWsGA55n6WY0MEIEPURL1FcbhFE6BffZEMEekfCzCIMtB5yyDcFnVbTnwk+CLAgTujmV/Lgvih56W+A=="], + + "tsx/esbuild/@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.28.0", "", { "os": "linux", "cpu": "ia32" }, "sha512-KBnSTt1kxl9x70q+ydterVdl+Cn0H18ngRMRCEQfrbqdUuntQQ0LoMZv47uB97NljZFzY6HcfqEZ2SAyIUTQBQ=="], + + "tsx/esbuild/@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.28.0", "", { "os": "linux", "cpu": "none" }, "sha512-zpSlUce1mnxzgBADvxKXX5sl8aYQHo2ezvMNI8I0lbblJtp8V4odlm3Yzlj7gPyt3T8ReksE6bK+pT3WD+aJRg=="], + + "tsx/esbuild/@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.28.0", "", { "os": "linux", "cpu": "none" }, "sha512-2jIfP6mmjkdmeTlsX/9vmdmhBmKADrWqN7zcdtHIeNSCH1SqIoNI63cYsjQR8J+wGa4Y5izRcSHSm8K3QWmk3w=="], + + "tsx/esbuild/@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.28.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-bc0FE9wWeC0WBm49IQMPSPILRocGTQt3j5KPCA8os6VprfuJ7KD+5PzESSrJ6GmPIPJK965ZJHTUlSA6GNYEhg=="], + + "tsx/esbuild/@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.28.0", "", { "os": "linux", "cpu": "none" }, "sha512-SQPZOwoTTT/HXFXQJG/vBX8sOFagGqvZyXcgLA3NhIqcBv1BJU1d46c0rGcrij2B56Z2rNiSLaZOYW5cUk7yLQ=="], + + "tsx/esbuild/@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.28.0", "", { "os": "linux", "cpu": "s390x" }, "sha512-SCfR0HN8CEEjnYnySJTd2cw0k9OHB/YFzt5zgJEwa+wL/T/raGWYMBqwDNAC6dqFKmJYZoQBRfHjgwLHGSrn3Q=="], + + "tsx/esbuild/@esbuild/linux-x64": ["@esbuild/linux-x64@0.28.0", "", { "os": "linux", "cpu": "x64" }, "sha512-us0dSb9iFxIi8srnpl931Nvs65it/Jd2a2K3qs7fz2WfGPHqzfzZTfec7oxZJRNPXPnNYZtanmRc4AL/JwVzHQ=="], + + "tsx/esbuild/@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.28.0", "", { "os": "none", "cpu": "arm64" }, "sha512-CR/RYotgtCKwtftMwJlUU7xCVNg3lMYZ0RzTmAHSfLCXw3NtZtNpswLEj/Kkf6kEL3Gw+BpOekRX0BYCtklhUw=="], + + "tsx/esbuild/@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.28.0", "", { "os": "none", "cpu": "x64" }, "sha512-nU1yhmYutL+fQ71Kxnhg8uEOdC0pwEW9entHykTgEbna2pw2dkbFSMeqjjyHZoCmt8SBkOSvV+yNmm94aUrrqw=="], + + "tsx/esbuild/@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.28.0", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-cXb5vApOsRsxsEl4mcZ1XY3D4DzcoMxR/nnc4IyqYs0rTI8ZKmW6kyyg+11Z8yvgMfAEldKzP7AdP64HnSC/6g=="], + + "tsx/esbuild/@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.28.0", "", { "os": "openbsd", "cpu": "x64" }, "sha512-8wZM2qqtv9UP3mzy7HiGYNH/zjTA355mpeuA+859TyR+e+Tc08IHYpLJuMsfpDJwoLo1ikIJI8jC3GFjnRClzA=="], + + "tsx/esbuild/@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.28.0", "", { "os": "none", "cpu": "arm64" }, "sha512-FLGfyizszcef5C3YtoyQDACyg95+dndv79i2EekILBofh5wpCa1KuBqOWKrEHZg3zrL3t5ouE5jgr94vA+Wb2w=="], + + "tsx/esbuild/@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.28.0", "", { "os": "sunos", "cpu": "x64" }, "sha512-1ZgjUoEdHZZl/YlV76TSCz9Hqj9h9YmMGAgAPYd+q4SicWNX3G5GCyx9uhQWSLcbvPW8Ni7lj4gDa1T40akdlw=="], + + "tsx/esbuild/@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.28.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-Q9StnDmQ/enxnpxCCLSg0oo4+34B9TdXpuyPeTedN/6+iXBJ4J+zwfQI28u/Jl40nOYAxGoNi7mFP40RUtkmUA=="], + + "tsx/esbuild/@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.28.0", "", { "os": "win32", "cpu": "ia32" }, "sha512-zF3ag/gfiCe6U2iczcRzSYJKH1DCI+ByzSENHlM2FcDbEeo5Zd2C86Aq0tKUYAJJ1obRP84ymxIAksZUcdztHA=="], + + "tsx/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.28.0", "", { "os": "win32", "cpu": "x64" }, "sha512-pEl1bO9mfAmIC+tW5btTmrKaujg3zGtUmWNdCw/xs70FBjwAL3o9OEKNHvNmnyylD6ubxUERiEhdsL0xBQ9efw=="], + "unzipper/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], } } diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..24b120f --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,35 @@ +services: + postgres: + image: postgres:16-alpine + restart: unless-stopped + environment: + POSTGRES_DB: asin_check + POSTGRES_USER: asin_check + POSTGRES_PASSWORD: asin_check + ports: + - "5432:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 5s + timeout: 5s + retries: 10 + + redis: + image: redis:7-alpine + restart: unless-stopped + command: ["redis-server", "--appendonly", "yes"] + ports: + - "6379:6379" + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 5s + retries: 10 + +volumes: + postgres_data: + redis_data: diff --git a/drizzle.config.ts b/drizzle.config.ts new file mode 100644 index 0000000..bcbd67a --- /dev/null +++ b/drizzle.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from "drizzle-kit"; + +export default defineConfig({ + schema: "./src/db/schema.ts", + out: "./drizzle", + dialect: "postgresql", + dbCredentials: { + url: process.env.DB_CONNECTION_STRING!, + }, +}); diff --git a/drizzle/0000_adorable_shiver_man.sql b/drizzle/0000_adorable_shiver_man.sql new file mode 100644 index 0000000..0ddc7ae --- /dev/null +++ b/drizzle/0000_adorable_shiver_man.sql @@ -0,0 +1,269 @@ +CREATE TYPE "public"."analysis_decision" AS ENUM('FBA', 'FBM', 'BUY', 'WATCH', 'SKIP');--> statement-breakpoint +CREATE TYPE "public"."analysis_method" AS ENUM('llm', 'supplier_scoring');--> statement-breakpoint +CREATE TYPE "public"."run_status" AS ENUM('running', 'ok', 'empty', 'failed', 'completed');--> statement-breakpoint +CREATE TYPE "public"."run_type" AS ENUM('lead_analysis', 'category_analysis', 'supplier_upc', 'stalker', 'stalker_analysis');--> statement-breakpoint +CREATE TABLE "analysis_revisions" ( + "id" serial PRIMARY KEY NOT NULL, + "run_item_id" integer NOT NULL, + "observation_id" integer, + "method" "analysis_method" NOT NULL, + "decision" "analysis_decision" NOT NULL, + "confidence" real, + "reasoning" text, + "analyzed_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE TABLE "analysis_run_stats" ( + "run_id" integer PRIMARY KEY NOT NULL, + "processed_count" integer DEFAULT 0 NOT NULL, + "analyzed_count" integer DEFAULT 0 NOT NULL, + "available_count" integer DEFAULT 0 NOT NULL, + "fba_count" integer DEFAULT 0 NOT NULL, + "fbm_count" integer DEFAULT 0 NOT NULL, + "buy_count" integer DEFAULT 0 NOT NULL, + "watch_count" integer DEFAULT 0 NOT NULL, + "skip_count" integer DEFAULT 0 NOT NULL +); +--> statement-breakpoint +CREATE TABLE "category_run_details" ( + "run_id" integer PRIMARY KEY NOT NULL, + "category_id" integer NOT NULL, + "category_label" text NOT NULL, + "checked_asin_count" integer DEFAULT 0 NOT NULL, + "selection_parameters_json" text +); +--> statement-breakpoint +CREATE TABLE "product_identifiers" ( + "id" serial PRIMARY KEY NOT NULL, + "product_asin" text NOT NULL, + "identifier_type" text NOT NULL, + "identifier_value" text NOT NULL, + "source" text NOT NULL, + "confirmed_at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "uq_product_identifier_type_value" UNIQUE("identifier_type","identifier_value") +); +--> statement-breakpoint +CREATE TABLE "product_observations" ( + "id" serial PRIMARY KEY NOT NULL, + "product_asin" text NOT NULL, + "run_id" integer NOT NULL, + "source" text NOT NULL, + "marketplace" text DEFAULT 'US' NOT NULL, + "current_price" real, + "avg_price_90d" real, + "sales_rank" integer, + "sales_rank_avg_90d" integer, + "monthly_sold" integer, + "rank_drops_30d" integer, + "rank_drops_90d" integer, + "seller_count" integer, + "amazon_is_seller" boolean, + "amazon_buybox_share_pct_90d" real, + "fba_fee" real, + "fbm_fee" real, + "referral_percent" real, + "can_sell" boolean, + "sellability_status" text, + "sellability_reason" text, + "raw_product_json" text, + "fetched_at" timestamp with time zone NOT NULL +); +--> statement-breakpoint +CREATE TABLE "products" ( + "asin" text PRIMARY KEY NOT NULL, + "name" text, + "brand" text, + "category" text, + "metadata_fetched_at" timestamp with time zone, + "first_seen_at" timestamp with time zone DEFAULT now() NOT NULL, + "last_seen_at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "ck_products_asin" CHECK ("products"."asin" ~ '^[A-Z0-9]{10}$') +); +--> statement-breakpoint +CREATE TABLE "run_items" ( + "id" serial PRIMARY KEY NOT NULL, + "run_id" integer NOT NULL, + "product_asin" text, + "source_inventory_item_id" integer, + "ordinal" integer, + "source_row" integer, + "status" text DEFAULT 'completed' NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE TABLE "runs" ( + "id" serial PRIMARY KEY NOT NULL, + "type" "run_type" NOT NULL, + "parent_run_id" integer, + "input_file" text, + "output_file" text, + "status" "run_status" DEFAULT 'running' NOT NULL, + "error_message" text, + "started_at" timestamp with time zone DEFAULT now() NOT NULL, + "completed_at" timestamp with time zone +); +--> statement-breakpoint +CREATE TABLE "sellers" ( + "seller_id" text PRIMARY KEY NOT NULL, + "seller_name" text, + "rating" real, + "rating_count" integer, + "storefront_asin_total" integer, + "persisted_inventory_sample_count" integer, + "last_updated_at" timestamp with time zone NOT NULL, + "raw_seller_json" text +); +--> statement-breakpoint +CREATE TABLE "sourcing_inputs" ( + "run_item_id" integer PRIMARY KEY NOT NULL, + "supplied_name" text, + "supplied_brand" text, + "supplied_category" text, + "unit_cost" real, + "avg_price_90d_sheet" real, + "selling_price_sheet" real, + "fba_net_sheet" real, + "gross_profit_dollar" real, + "gross_profit_pct" real, + "net_profit_sheet" real, + "roi_sheet" real, + "moq" integer, + "moq_cost" real, + "qty_available" integer, + "supplier" text, + "source_url" text, + "asin_link" text, + "promo_coupon_code" text, + "notes" text, + "lead_date" text +); +--> statement-breakpoint +CREATE TABLE "stalker_inventory_items" ( + "id" serial PRIMARY KEY NOT NULL, + "run_id" integer NOT NULL, + "seller_id" text NOT NULL, + "product_asin" text NOT NULL, + "observation_id" integer NOT NULL, + "last_seen_at" timestamp with time zone NOT NULL, + "raw_inventory_json" text, + CONSTRAINT "uq_stalker_inventory_items_run_seller_asin" UNIQUE("run_id","seller_id","product_asin") +); +--> statement-breakpoint +CREATE TABLE "stalker_run_details" ( + "run_id" integer PRIMARY KEY NOT NULL, + "requested_asins" integer DEFAULT 0 NOT NULL, + "skipped_asins" integer DEFAULT 0 NOT NULL, + "scanned_asins" integer DEFAULT 0 NOT NULL, + "source_asins_with_matches" integer DEFAULT 0 NOT NULL, + "candidate_sellers" integer DEFAULT 0 NOT NULL, + "qualifying_sellers" integer DEFAULT 0 NOT NULL, + "matched_sellers" integer DEFAULT 0 NOT NULL, + "seller_metadata_requests" integer DEFAULT 0 NOT NULL, + "seller_storefront_requests" integer DEFAULT 0 NOT NULL, + "inventory_sellability_checked_asins" integer DEFAULT 0 NOT NULL, + "inventory_sellability_available_asins" integer DEFAULT 0 NOT NULL, + "inventory_sellability_excluded_asins" integer DEFAULT 0 NOT NULL, + "persisted_inventory_asins" integer DEFAULT 0 NOT NULL +); +--> statement-breakpoint +CREATE TABLE "stalker_scan_sellers" ( + "id" serial PRIMARY KEY NOT NULL, + "scan_id" integer NOT NULL, + "seller_id" text NOT NULL, + "offer_price" real, + "condition" text, + "is_fba" boolean, + "stock" integer, + "seller_rating" real, + "seller_rating_count" integer, + "raw_offer_json" text, + CONSTRAINT "uq_stalker_scan_sellers_scan_seller" UNIQUE("scan_id","seller_id") +); +--> statement-breakpoint +CREATE TABLE "stalker_scans" ( + "id" serial PRIMARY KEY NOT NULL, + "run_id" integer NOT NULL, + "source_product_asin" text NOT NULL, + "observation_id" integer, + "offer_count" integer DEFAULT 0 NOT NULL, + "candidate_seller_count" integer DEFAULT 0 NOT NULL, + "matched_seller_count" integer DEFAULT 0 NOT NULL, + "fetched_at" timestamp with time zone NOT NULL, + CONSTRAINT "uq_stalker_scans_run_source_product" UNIQUE("run_id","source_product_asin") +); +--> statement-breakpoint +CREATE TABLE "supplier_scores" ( + "revision_id" integer PRIMARY KEY NOT NULL, + "score" real, + "sale_price" real, + "fba_fee" real, + "profit" real, + "margin" real, + "roi" real, + "reason" text +); +--> statement-breakpoint +CREATE TABLE "upc_resolution_candidates" ( + "run_item_id" integer NOT NULL, + "product_asin" text NOT NULL, + CONSTRAINT "upc_resolution_candidates_run_item_id_product_asin_pk" PRIMARY KEY("run_item_id","product_asin") +); +--> statement-breakpoint +CREATE TABLE "upc_resolutions" ( + "run_item_id" integer PRIMARY KEY NOT NULL, + "requested_upc" text NOT NULL, + "normalized_upc" text NOT NULL, + "provider" text NOT NULL, + "status" text NOT NULL, + "reason" text, + "resolved_product_asin" text, + "resolved_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "analysis_revisions" ADD CONSTRAINT "analysis_revisions_run_item_id_run_items_id_fk" FOREIGN KEY ("run_item_id") REFERENCES "public"."run_items"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "analysis_revisions" ADD CONSTRAINT "analysis_revisions_observation_id_product_observations_id_fk" FOREIGN KEY ("observation_id") REFERENCES "public"."product_observations"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "analysis_run_stats" ADD CONSTRAINT "analysis_run_stats_run_id_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."runs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "category_run_details" ADD CONSTRAINT "category_run_details_run_id_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."runs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "product_identifiers" ADD CONSTRAINT "product_identifiers_product_asin_products_asin_fk" FOREIGN KEY ("product_asin") REFERENCES "public"."products"("asin") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "product_observations" ADD CONSTRAINT "product_observations_product_asin_products_asin_fk" FOREIGN KEY ("product_asin") REFERENCES "public"."products"("asin") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "product_observations" ADD CONSTRAINT "product_observations_run_id_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."runs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "run_items" ADD CONSTRAINT "run_items_run_id_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."runs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "run_items" ADD CONSTRAINT "run_items_product_asin_products_asin_fk" FOREIGN KEY ("product_asin") REFERENCES "public"."products"("asin") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "run_items" ADD CONSTRAINT "run_items_source_inventory_item_id_stalker_inventory_items_id_fk" FOREIGN KEY ("source_inventory_item_id") REFERENCES "public"."stalker_inventory_items"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "runs" ADD CONSTRAINT "runs_parent_run_id_runs_id_fk" FOREIGN KEY ("parent_run_id") REFERENCES "public"."runs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "sourcing_inputs" ADD CONSTRAINT "sourcing_inputs_run_item_id_run_items_id_fk" FOREIGN KEY ("run_item_id") REFERENCES "public"."run_items"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "stalker_inventory_items" ADD CONSTRAINT "stalker_inventory_items_run_id_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."runs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "stalker_inventory_items" ADD CONSTRAINT "stalker_inventory_items_seller_id_sellers_seller_id_fk" FOREIGN KEY ("seller_id") REFERENCES "public"."sellers"("seller_id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "stalker_inventory_items" ADD CONSTRAINT "stalker_inventory_items_product_asin_products_asin_fk" FOREIGN KEY ("product_asin") REFERENCES "public"."products"("asin") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "stalker_inventory_items" ADD CONSTRAINT "stalker_inventory_items_observation_id_product_observations_id_fk" FOREIGN KEY ("observation_id") REFERENCES "public"."product_observations"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "stalker_run_details" ADD CONSTRAINT "stalker_run_details_run_id_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."runs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "stalker_scan_sellers" ADD CONSTRAINT "stalker_scan_sellers_scan_id_stalker_scans_id_fk" FOREIGN KEY ("scan_id") REFERENCES "public"."stalker_scans"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "stalker_scan_sellers" ADD CONSTRAINT "stalker_scan_sellers_seller_id_sellers_seller_id_fk" FOREIGN KEY ("seller_id") REFERENCES "public"."sellers"("seller_id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "stalker_scans" ADD CONSTRAINT "stalker_scans_run_id_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."runs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "stalker_scans" ADD CONSTRAINT "stalker_scans_source_product_asin_products_asin_fk" FOREIGN KEY ("source_product_asin") REFERENCES "public"."products"("asin") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "stalker_scans" ADD CONSTRAINT "stalker_scans_observation_id_product_observations_id_fk" FOREIGN KEY ("observation_id") REFERENCES "public"."product_observations"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "supplier_scores" ADD CONSTRAINT "supplier_scores_revision_id_analysis_revisions_id_fk" FOREIGN KEY ("revision_id") REFERENCES "public"."analysis_revisions"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "upc_resolution_candidates" ADD CONSTRAINT "upc_resolution_candidates_run_item_id_upc_resolutions_run_item_id_fk" FOREIGN KEY ("run_item_id") REFERENCES "public"."upc_resolutions"("run_item_id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "upc_resolution_candidates" ADD CONSTRAINT "upc_resolution_candidates_product_asin_products_asin_fk" FOREIGN KEY ("product_asin") REFERENCES "public"."products"("asin") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "upc_resolutions" ADD CONSTRAINT "upc_resolutions_run_item_id_run_items_id_fk" FOREIGN KEY ("run_item_id") REFERENCES "public"."run_items"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "upc_resolutions" ADD CONSTRAINT "upc_resolutions_resolved_product_asin_products_asin_fk" FOREIGN KEY ("resolved_product_asin") REFERENCES "public"."products"("asin") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "idx_analysis_revisions_run_item_time" ON "analysis_revisions" USING btree ("run_item_id","analyzed_at");--> statement-breakpoint +CREATE INDEX "idx_analysis_revisions_decision" ON "analysis_revisions" USING btree ("decision");--> statement-breakpoint +CREATE INDEX "idx_product_identifiers_asin" ON "product_identifiers" USING btree ("product_asin");--> statement-breakpoint +CREATE INDEX "idx_product_observations_product_time" ON "product_observations" USING btree ("product_asin","fetched_at" DESC NULLS LAST);--> statement-breakpoint +CREATE INDEX "idx_product_observations_run_id" ON "product_observations" USING btree ("run_id");--> statement-breakpoint +CREATE INDEX "idx_product_observations_sellability" ON "product_observations" USING btree ("sellability_status");--> statement-breakpoint +CREATE INDEX "idx_products_name" ON "products" USING btree ("name");--> statement-breakpoint +CREATE INDEX "idx_products_last_seen_at" ON "products" USING btree ("last_seen_at");--> statement-breakpoint +CREATE INDEX "idx_run_items_run_id" ON "run_items" USING btree ("run_id");--> statement-breakpoint +CREATE INDEX "idx_run_items_product_asin" ON "run_items" USING btree ("product_asin");--> statement-breakpoint +CREATE INDEX "idx_runs_started_at" ON "runs" USING btree ("started_at");--> statement-breakpoint +CREATE INDEX "idx_runs_type" ON "runs" USING btree ("type");--> statement-breakpoint +CREATE INDEX "idx_runs_status" ON "runs" USING btree ("status");--> statement-breakpoint +CREATE INDEX "idx_runs_parent_run_id" ON "runs" USING btree ("parent_run_id");--> statement-breakpoint +CREATE INDEX "idx_stalker_inventory_seller_id" ON "stalker_inventory_items" USING btree ("seller_id");--> statement-breakpoint +CREATE INDEX "idx_stalker_inventory_product_asin" ON "stalker_inventory_items" USING btree ("product_asin");--> statement-breakpoint +CREATE INDEX "idx_stalker_scans_run_id" ON "stalker_scans" USING btree ("run_id");--> statement-breakpoint +CREATE INDEX "idx_stalker_scans_source_asin" ON "stalker_scans" USING btree ("source_product_asin");--> statement-breakpoint +CREATE INDEX "idx_upc_candidates_product_asin" ON "upc_resolution_candidates" USING btree ("product_asin");--> statement-breakpoint +CREATE INDEX "idx_upc_resolutions_normalized_upc" ON "upc_resolutions" USING btree ("normalized_upc"); \ No newline at end of file diff --git a/drizzle/meta/0000_snapshot.json b/drizzle/meta/0000_snapshot.json new file mode 100644 index 0000000..4088654 --- /dev/null +++ b/drizzle/meta/0000_snapshot.json @@ -0,0 +1,2025 @@ +{ + "id": "f3a73b1a-820c-4905-b998-55a9b70bc49d", + "prevId": "00000000-0000-0000-0000-000000000000", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.analysis_revisions": { + "name": "analysis_revisions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "run_item_id": { + "name": "run_item_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "observation_id": { + "name": "observation_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "method": { + "name": "method", + "type": "analysis_method", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "decision": { + "name": "decision", + "type": "analysis_decision", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "confidence": { + "name": "confidence", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "reasoning": { + "name": "reasoning", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "analyzed_at": { + "name": "analyzed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_analysis_revisions_run_item_time": { + "name": "idx_analysis_revisions_run_item_time", + "columns": [ + { + "expression": "run_item_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "analyzed_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_analysis_revisions_decision": { + "name": "idx_analysis_revisions_decision", + "columns": [ + { + "expression": "decision", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "analysis_revisions_run_item_id_run_items_id_fk": { + "name": "analysis_revisions_run_item_id_run_items_id_fk", + "tableFrom": "analysis_revisions", + "tableTo": "run_items", + "columnsFrom": [ + "run_item_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "analysis_revisions_observation_id_product_observations_id_fk": { + "name": "analysis_revisions_observation_id_product_observations_id_fk", + "tableFrom": "analysis_revisions", + "tableTo": "product_observations", + "columnsFrom": [ + "observation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.analysis_run_stats": { + "name": "analysis_run_stats", + "schema": "", + "columns": { + "run_id": { + "name": "run_id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "processed_count": { + "name": "processed_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "analyzed_count": { + "name": "analyzed_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "available_count": { + "name": "available_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "fba_count": { + "name": "fba_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "fbm_count": { + "name": "fbm_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "buy_count": { + "name": "buy_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "watch_count": { + "name": "watch_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "skip_count": { + "name": "skip_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": { + "analysis_run_stats_run_id_runs_id_fk": { + "name": "analysis_run_stats_run_id_runs_id_fk", + "tableFrom": "analysis_run_stats", + "tableTo": "runs", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.category_run_details": { + "name": "category_run_details", + "schema": "", + "columns": { + "run_id": { + "name": "run_id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "category_id": { + "name": "category_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "category_label": { + "name": "category_label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "checked_asin_count": { + "name": "checked_asin_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "selection_parameters_json": { + "name": "selection_parameters_json", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "category_run_details_run_id_runs_id_fk": { + "name": "category_run_details_run_id_runs_id_fk", + "tableFrom": "category_run_details", + "tableTo": "runs", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.product_identifiers": { + "name": "product_identifiers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "product_asin": { + "name": "product_asin", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "identifier_type": { + "name": "identifier_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "identifier_value": { + "name": "identifier_value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "confirmed_at": { + "name": "confirmed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_product_identifiers_asin": { + "name": "idx_product_identifiers_asin", + "columns": [ + { + "expression": "product_asin", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "product_identifiers_product_asin_products_asin_fk": { + "name": "product_identifiers_product_asin_products_asin_fk", + "tableFrom": "product_identifiers", + "tableTo": "products", + "columnsFrom": [ + "product_asin" + ], + "columnsTo": [ + "asin" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "uq_product_identifier_type_value": { + "name": "uq_product_identifier_type_value", + "nullsNotDistinct": false, + "columns": [ + "identifier_type", + "identifier_value" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.product_observations": { + "name": "product_observations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "product_asin": { + "name": "product_asin", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "marketplace": { + "name": "marketplace", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'US'" + }, + "current_price": { + "name": "current_price", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "avg_price_90d": { + "name": "avg_price_90d", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "sales_rank": { + "name": "sales_rank", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "sales_rank_avg_90d": { + "name": "sales_rank_avg_90d", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "monthly_sold": { + "name": "monthly_sold", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "rank_drops_30d": { + "name": "rank_drops_30d", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "rank_drops_90d": { + "name": "rank_drops_90d", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "seller_count": { + "name": "seller_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "amazon_is_seller": { + "name": "amazon_is_seller", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "amazon_buybox_share_pct_90d": { + "name": "amazon_buybox_share_pct_90d", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "fba_fee": { + "name": "fba_fee", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "fbm_fee": { + "name": "fbm_fee", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "referral_percent": { + "name": "referral_percent", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "can_sell": { + "name": "can_sell", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "sellability_status": { + "name": "sellability_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sellability_reason": { + "name": "sellability_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "raw_product_json": { + "name": "raw_product_json", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "fetched_at": { + "name": "fetched_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "idx_product_observations_product_time": { + "name": "idx_product_observations_product_time", + "columns": [ + { + "expression": "product_asin", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "fetched_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_product_observations_run_id": { + "name": "idx_product_observations_run_id", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_product_observations_sellability": { + "name": "idx_product_observations_sellability", + "columns": [ + { + "expression": "sellability_status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "product_observations_product_asin_products_asin_fk": { + "name": "product_observations_product_asin_products_asin_fk", + "tableFrom": "product_observations", + "tableTo": "products", + "columnsFrom": [ + "product_asin" + ], + "columnsTo": [ + "asin" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "product_observations_run_id_runs_id_fk": { + "name": "product_observations_run_id_runs_id_fk", + "tableFrom": "product_observations", + "tableTo": "runs", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.products": { + "name": "products", + "schema": "", + "columns": { + "asin": { + "name": "asin", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "brand": { + "name": "brand", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "category": { + "name": "category", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata_fetched_at": { + "name": "metadata_fetched_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "first_seen_at": { + "name": "first_seen_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_products_name": { + "name": "idx_products_name", + "columns": [ + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_products_last_seen_at": { + "name": "idx_products_last_seen_at", + "columns": [ + { + "expression": "last_seen_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "ck_products_asin": { + "name": "ck_products_asin", + "value": "\"products\".\"asin\" ~ '^[A-Z0-9]{10}$'" + } + }, + "isRLSEnabled": false + }, + "public.run_items": { + "name": "run_items", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "product_asin": { + "name": "product_asin", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_inventory_item_id": { + "name": "source_inventory_item_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "ordinal": { + "name": "ordinal", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "source_row": { + "name": "source_row", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'completed'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_run_items_run_id": { + "name": "idx_run_items_run_id", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_run_items_product_asin": { + "name": "idx_run_items_product_asin", + "columns": [ + { + "expression": "product_asin", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "run_items_run_id_runs_id_fk": { + "name": "run_items_run_id_runs_id_fk", + "tableFrom": "run_items", + "tableTo": "runs", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "run_items_product_asin_products_asin_fk": { + "name": "run_items_product_asin_products_asin_fk", + "tableFrom": "run_items", + "tableTo": "products", + "columnsFrom": [ + "product_asin" + ], + "columnsTo": [ + "asin" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "run_items_source_inventory_item_id_stalker_inventory_items_id_fk": { + "name": "run_items_source_inventory_item_id_stalker_inventory_items_id_fk", + "tableFrom": "run_items", + "tableTo": "stalker_inventory_items", + "columnsFrom": [ + "source_inventory_item_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.runs": { + "name": "runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "run_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "parent_run_id": { + "name": "parent_run_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "input_file": { + "name": "input_file", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "output_file": { + "name": "output_file", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "run_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'running'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_runs_started_at": { + "name": "idx_runs_started_at", + "columns": [ + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_runs_type": { + "name": "idx_runs_type", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_runs_status": { + "name": "idx_runs_status", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_runs_parent_run_id": { + "name": "idx_runs_parent_run_id", + "columns": [ + { + "expression": "parent_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "runs_parent_run_id_runs_id_fk": { + "name": "runs_parent_run_id_runs_id_fk", + "tableFrom": "runs", + "tableTo": "runs", + "columnsFrom": [ + "parent_run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sellers": { + "name": "sellers", + "schema": "", + "columns": { + "seller_id": { + "name": "seller_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "seller_name": { + "name": "seller_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "rating": { + "name": "rating", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "rating_count": { + "name": "rating_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "storefront_asin_total": { + "name": "storefront_asin_total", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "persisted_inventory_sample_count": { + "name": "persisted_inventory_sample_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_updated_at": { + "name": "last_updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "raw_seller_json": { + "name": "raw_seller_json", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sourcing_inputs": { + "name": "sourcing_inputs", + "schema": "", + "columns": { + "run_item_id": { + "name": "run_item_id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "supplied_name": { + "name": "supplied_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "supplied_brand": { + "name": "supplied_brand", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "supplied_category": { + "name": "supplied_category", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "unit_cost": { + "name": "unit_cost", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "avg_price_90d_sheet": { + "name": "avg_price_90d_sheet", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "selling_price_sheet": { + "name": "selling_price_sheet", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "fba_net_sheet": { + "name": "fba_net_sheet", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "gross_profit_dollar": { + "name": "gross_profit_dollar", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "gross_profit_pct": { + "name": "gross_profit_pct", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "net_profit_sheet": { + "name": "net_profit_sheet", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "roi_sheet": { + "name": "roi_sheet", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "moq": { + "name": "moq", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "moq_cost": { + "name": "moq_cost", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "qty_available": { + "name": "qty_available", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "supplier": { + "name": "supplier", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "asin_link": { + "name": "asin_link", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "promo_coupon_code": { + "name": "promo_coupon_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lead_date": { + "name": "lead_date", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "sourcing_inputs_run_item_id_run_items_id_fk": { + "name": "sourcing_inputs_run_item_id_run_items_id_fk", + "tableFrom": "sourcing_inputs", + "tableTo": "run_items", + "columnsFrom": [ + "run_item_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.stalker_inventory_items": { + "name": "stalker_inventory_items", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "seller_id": { + "name": "seller_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "product_asin": { + "name": "product_asin", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "observation_id": { + "name": "observation_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "raw_inventory_json": { + "name": "raw_inventory_json", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_stalker_inventory_seller_id": { + "name": "idx_stalker_inventory_seller_id", + "columns": [ + { + "expression": "seller_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_stalker_inventory_product_asin": { + "name": "idx_stalker_inventory_product_asin", + "columns": [ + { + "expression": "product_asin", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "stalker_inventory_items_run_id_runs_id_fk": { + "name": "stalker_inventory_items_run_id_runs_id_fk", + "tableFrom": "stalker_inventory_items", + "tableTo": "runs", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "stalker_inventory_items_seller_id_sellers_seller_id_fk": { + "name": "stalker_inventory_items_seller_id_sellers_seller_id_fk", + "tableFrom": "stalker_inventory_items", + "tableTo": "sellers", + "columnsFrom": [ + "seller_id" + ], + "columnsTo": [ + "seller_id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "stalker_inventory_items_product_asin_products_asin_fk": { + "name": "stalker_inventory_items_product_asin_products_asin_fk", + "tableFrom": "stalker_inventory_items", + "tableTo": "products", + "columnsFrom": [ + "product_asin" + ], + "columnsTo": [ + "asin" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "stalker_inventory_items_observation_id_product_observations_id_fk": { + "name": "stalker_inventory_items_observation_id_product_observations_id_fk", + "tableFrom": "stalker_inventory_items", + "tableTo": "product_observations", + "columnsFrom": [ + "observation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "uq_stalker_inventory_items_run_seller_asin": { + "name": "uq_stalker_inventory_items_run_seller_asin", + "nullsNotDistinct": false, + "columns": [ + "run_id", + "seller_id", + "product_asin" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.stalker_run_details": { + "name": "stalker_run_details", + "schema": "", + "columns": { + "run_id": { + "name": "run_id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "requested_asins": { + "name": "requested_asins", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "skipped_asins": { + "name": "skipped_asins", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "scanned_asins": { + "name": "scanned_asins", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "source_asins_with_matches": { + "name": "source_asins_with_matches", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "candidate_sellers": { + "name": "candidate_sellers", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "qualifying_sellers": { + "name": "qualifying_sellers", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "matched_sellers": { + "name": "matched_sellers", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "seller_metadata_requests": { + "name": "seller_metadata_requests", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "seller_storefront_requests": { + "name": "seller_storefront_requests", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "inventory_sellability_checked_asins": { + "name": "inventory_sellability_checked_asins", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "inventory_sellability_available_asins": { + "name": "inventory_sellability_available_asins", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "inventory_sellability_excluded_asins": { + "name": "inventory_sellability_excluded_asins", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "persisted_inventory_asins": { + "name": "persisted_inventory_asins", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": { + "stalker_run_details_run_id_runs_id_fk": { + "name": "stalker_run_details_run_id_runs_id_fk", + "tableFrom": "stalker_run_details", + "tableTo": "runs", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.stalker_scan_sellers": { + "name": "stalker_scan_sellers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "scan_id": { + "name": "scan_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "seller_id": { + "name": "seller_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "offer_price": { + "name": "offer_price", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "condition": { + "name": "condition", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_fba": { + "name": "is_fba", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "stock": { + "name": "stock", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "seller_rating": { + "name": "seller_rating", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "seller_rating_count": { + "name": "seller_rating_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "raw_offer_json": { + "name": "raw_offer_json", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "stalker_scan_sellers_scan_id_stalker_scans_id_fk": { + "name": "stalker_scan_sellers_scan_id_stalker_scans_id_fk", + "tableFrom": "stalker_scan_sellers", + "tableTo": "stalker_scans", + "columnsFrom": [ + "scan_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "stalker_scan_sellers_seller_id_sellers_seller_id_fk": { + "name": "stalker_scan_sellers_seller_id_sellers_seller_id_fk", + "tableFrom": "stalker_scan_sellers", + "tableTo": "sellers", + "columnsFrom": [ + "seller_id" + ], + "columnsTo": [ + "seller_id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "uq_stalker_scan_sellers_scan_seller": { + "name": "uq_stalker_scan_sellers_scan_seller", + "nullsNotDistinct": false, + "columns": [ + "scan_id", + "seller_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.stalker_scans": { + "name": "stalker_scans", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "source_product_asin": { + "name": "source_product_asin", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "observation_id": { + "name": "observation_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "offer_count": { + "name": "offer_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "candidate_seller_count": { + "name": "candidate_seller_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "matched_seller_count": { + "name": "matched_seller_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "fetched_at": { + "name": "fetched_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "idx_stalker_scans_run_id": { + "name": "idx_stalker_scans_run_id", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_stalker_scans_source_asin": { + "name": "idx_stalker_scans_source_asin", + "columns": [ + { + "expression": "source_product_asin", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "stalker_scans_run_id_runs_id_fk": { + "name": "stalker_scans_run_id_runs_id_fk", + "tableFrom": "stalker_scans", + "tableTo": "runs", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "stalker_scans_source_product_asin_products_asin_fk": { + "name": "stalker_scans_source_product_asin_products_asin_fk", + "tableFrom": "stalker_scans", + "tableTo": "products", + "columnsFrom": [ + "source_product_asin" + ], + "columnsTo": [ + "asin" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "stalker_scans_observation_id_product_observations_id_fk": { + "name": "stalker_scans_observation_id_product_observations_id_fk", + "tableFrom": "stalker_scans", + "tableTo": "product_observations", + "columnsFrom": [ + "observation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "uq_stalker_scans_run_source_product": { + "name": "uq_stalker_scans_run_source_product", + "nullsNotDistinct": false, + "columns": [ + "run_id", + "source_product_asin" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.supplier_scores": { + "name": "supplier_scores", + "schema": "", + "columns": { + "revision_id": { + "name": "revision_id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "score": { + "name": "score", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "sale_price": { + "name": "sale_price", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "fba_fee": { + "name": "fba_fee", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "profit": { + "name": "profit", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "margin": { + "name": "margin", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "roi": { + "name": "roi", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "supplier_scores_revision_id_analysis_revisions_id_fk": { + "name": "supplier_scores_revision_id_analysis_revisions_id_fk", + "tableFrom": "supplier_scores", + "tableTo": "analysis_revisions", + "columnsFrom": [ + "revision_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.upc_resolution_candidates": { + "name": "upc_resolution_candidates", + "schema": "", + "columns": { + "run_item_id": { + "name": "run_item_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "product_asin": { + "name": "product_asin", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "idx_upc_candidates_product_asin": { + "name": "idx_upc_candidates_product_asin", + "columns": [ + { + "expression": "product_asin", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "upc_resolution_candidates_run_item_id_upc_resolutions_run_item_id_fk": { + "name": "upc_resolution_candidates_run_item_id_upc_resolutions_run_item_id_fk", + "tableFrom": "upc_resolution_candidates", + "tableTo": "upc_resolutions", + "columnsFrom": [ + "run_item_id" + ], + "columnsTo": [ + "run_item_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "upc_resolution_candidates_product_asin_products_asin_fk": { + "name": "upc_resolution_candidates_product_asin_products_asin_fk", + "tableFrom": "upc_resolution_candidates", + "tableTo": "products", + "columnsFrom": [ + "product_asin" + ], + "columnsTo": [ + "asin" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "upc_resolution_candidates_run_item_id_product_asin_pk": { + "name": "upc_resolution_candidates_run_item_id_product_asin_pk", + "columns": [ + "run_item_id", + "product_asin" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.upc_resolutions": { + "name": "upc_resolutions", + "schema": "", + "columns": { + "run_item_id": { + "name": "run_item_id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "requested_upc": { + "name": "requested_upc", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "normalized_upc": { + "name": "normalized_upc", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resolved_product_asin": { + "name": "resolved_product_asin", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_upc_resolutions_normalized_upc": { + "name": "idx_upc_resolutions_normalized_upc", + "columns": [ + { + "expression": "normalized_upc", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "upc_resolutions_run_item_id_run_items_id_fk": { + "name": "upc_resolutions_run_item_id_run_items_id_fk", + "tableFrom": "upc_resolutions", + "tableTo": "run_items", + "columnsFrom": [ + "run_item_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "upc_resolutions_resolved_product_asin_products_asin_fk": { + "name": "upc_resolutions_resolved_product_asin_products_asin_fk", + "tableFrom": "upc_resolutions", + "tableTo": "products", + "columnsFrom": [ + "resolved_product_asin" + ], + "columnsTo": [ + "asin" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.analysis_decision": { + "name": "analysis_decision", + "schema": "public", + "values": [ + "FBA", + "FBM", + "BUY", + "WATCH", + "SKIP" + ] + }, + "public.analysis_method": { + "name": "analysis_method", + "schema": "public", + "values": [ + "llm", + "supplier_scoring" + ] + }, + "public.run_status": { + "name": "run_status", + "schema": "public", + "values": [ + "running", + "ok", + "empty", + "failed", + "completed" + ] + }, + "public.run_type": { + "name": "run_type", + "schema": "public", + "values": [ + "lead_analysis", + "category_analysis", + "supplier_upc", + "stalker", + "stalker_analysis" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json new file mode 100644 index 0000000..96ebeac --- /dev/null +++ b/drizzle/meta/_journal.json @@ -0,0 +1,13 @@ +{ + "version": "7", + "dialect": "postgresql", + "entries": [ + { + "idx": 0, + "version": "7", + "when": 1779726518779, + "tag": "0000_adorable_shiver_man", + "breakpoints": true + } + ] +} \ No newline at end of file diff --git a/package.json b/package.json index 3e898aa..e10d4bc 100644 --- a/package.json +++ b/package.json @@ -4,28 +4,33 @@ "type": "module", "private": true, "scripts": { - "bestsellers": "bun run src/bestsellers-by-category.ts", - "monthly-sold": "bun run src/top-monthly-sold-by-category.ts", - "mid-range": "bun run src/mid-range-sellers-by-category.ts", - "stalker": "bun run src/stalker.ts", + "bestsellers": "bun run src/categories/bestsellers-by-category.ts", + "monthly-sold": "bun run src/categories/top-monthly-sold-by-category.ts", + "mid-range": "bun run src/categories/mid-range-sellers-by-category.ts", + "stalker": "bun run src/stalker/stalker.ts", "search-offers": "bun run src/asin-offer-search.ts", - "upc": "bun run src/upc-lookup.ts", - "upc-file": "bun run src/upc-file-analysis.ts", + "upc": "bun run src/supplier/upc-lookup.ts", + "upc-file": "bun run src/supplier/upc-file-analysis.ts", "start": "bun run src/index.ts", "start:web": "bun --hot src/server.ts", "build:web": "bun build src/web/index.html --outdir dist", - "test": "bun test" + "test": "bun test", + "db:generate": "drizzle-kit generate", + "db:migrate": "drizzle-kit migrate" }, "devDependencies": { "@types/bun": "latest", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", + "drizzle-kit": "^0.31.10", "typescript": "^6.0.3" }, "dependencies": { "amazon-sp-api": "^1.2.1", + "drizzle-orm": "^0.45.2", "exceljs": "^4.4.0", "ioredis": "^5.10.1", + "postgres": "^3.4.9", "react": "^19.2.0", "react-dom": "^19.2.0", "xlsx": "^0.18.5" diff --git a/src/analysis-pipeline.test.ts b/src/analysis-pipeline.test.ts new file mode 100644 index 0000000..e0e446f --- /dev/null +++ b/src/analysis-pipeline.test.ts @@ -0,0 +1,114 @@ +import { beforeEach, expect, mock, test } from "bun:test"; +import { processProductChunk } from "./analysis-pipeline.ts"; +import type { ProductRecord } from "./types.ts"; + +const fetchKeepaDataBatchMock = mock(async (asins: string[]) => { + return new Map( + asins.map((asin) => [ + asin, + { + currentPrice: 20, + avgPrice90: 18, + minPrice90: null, + maxPrice90: null, + salesRank: 100, + salesRankAvg90: null, + salesRankDrops30: null, + salesRankDrops90: null, + sellerCount: 3, + amazonIsSeller: false, + amazonBuyboxSharePct90d: null, + buyBoxSeller: null, + buyBoxPrice: null, + monthlySold: 50, + categoryTree: [], + }, + ]), + ); +}); + +const fetchSellabilityBatchMock = mock(async (asins: string[]) => { + return new Map( + asins.map((asin) => [ + asin, + asin === "B000000002" + ? { + canSell: false, + sellabilityStatus: "restricted" as const, + sellabilityReason: "Approval required", + } + : { + canSell: true, + sellabilityStatus: "available" as const, + sellabilityReason: "Available", + }, + ]), + ); +}); + +const fetchSpApiPricingAndFeesMock = mock(async () => ({ + fbaFee: 4, + fbmFee: 2, + referralFeePercent: 15, + estimatedSalePrice: 20, + canSell: true, + sellabilityStatus: "available" as const, + sellabilityReason: "Available", +})); + +const analyzeProductsMock = mock(async (products: any[]) => + products.map((product) => ({ + asin: product.record.asin, + verdict: "FBA" as const, + confidence: 95, + reasoning: "Analyzed", + })), +); + +const getCacheMock = mock(async () => null); +const setCacheMock = mock(async () => undefined); + +beforeEach(() => { + fetchKeepaDataBatchMock.mockClear(); + fetchSellabilityBatchMock.mockClear(); + fetchSpApiPricingAndFeesMock.mockClear(); + analyzeProductsMock.mockClear(); + getCacheMock.mockClear(); + setCacheMock.mockClear(); +}); + +test("lead analysis retains restricted input rows as SKIP without LLM analysis", async () => { + const products: ProductRecord[] = [ + { asin: "B000000001", name: "Available", unitCost: 5 }, + { asin: "B000000002", name: "Restricted", unitCost: 6 }, + ]; + + const results = await processProductChunk(products, { + llmBatchDelayMs: 0, + llmRetryDelayMs: 0, + dependencies: { + fetchKeepaDataBatch: fetchKeepaDataBatchMock, + fetchSellabilityBatch: fetchSellabilityBatchMock, + fetchSpApiPricingAndFees: fetchSpApiPricingAndFeesMock, + analyzeProducts: analyzeProductsMock, + getCache: getCacheMock, + setCache: setCacheMock, + }, + }); + + expect(results).toHaveLength(2); + expect(results.map((result) => result.product.record.asin)).toEqual([ + "B000000001", + "B000000002", + ]); + expect(results.find((result) => result.product.record.asin === "B000000002")?.verdict) + .toEqual({ + asin: "B000000002", + verdict: "SKIP", + confidence: 100, + reasoning: "Approval required", + }); + expect(fetchKeepaDataBatchMock.mock.calls[0]?.[0]).toEqual(["B000000001"]); + expect(fetchSpApiPricingAndFeesMock.mock.calls).toHaveLength(1); + expect(analyzeProductsMock.mock.calls[0]?.[0]).toHaveLength(1); +}); diff --git a/src/analysis-pipeline.ts b/src/analysis-pipeline.ts index b78f5c9..7a4fec8 100644 --- a/src/analysis-pipeline.ts +++ b/src/analysis-pipeline.ts @@ -1,7 +1,7 @@ -import { fetchKeepaDataBatch } from "./keepa.ts"; -import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts"; -import { getCache, setCache } from "./cache.ts"; -import { analyzeProducts } from "./llm.ts"; +import { fetchKeepaDataBatch } from "./integrations/keepa.ts"; +import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./integrations/sp-api.ts"; +import { getCache, setCache } from "./integrations/cache.ts"; +import { analyzeProducts } from "./integrations/llm.ts"; import type { AnalysisResult, EnrichedProduct, @@ -16,6 +16,15 @@ export const DEFAULT_PRICING_CONCURRENCY = 5; export type SellabilityFilter = "available" | "all"; +type AnalysisPipelineDependencies = { + fetchKeepaDataBatch: typeof fetchKeepaDataBatch; + fetchSellabilityBatch: typeof fetchSellabilityBatch; + fetchSpApiPricingAndFees: typeof fetchSpApiPricingAndFees; + getCache: typeof getCache; + setCache: typeof setCache; + analyzeProducts: typeof analyzeProducts; +}; + export type AnalysisPipelineOptions = { llmBatchSize?: number; pricingConcurrency?: number; @@ -23,6 +32,7 @@ export type AnalysisPipelineOptions = { llmRetryDelayMs?: number; sellability?: SellabilityFilter; useClaude?: boolean; + dependencies?: Partial; }; export function chunkArray(items: T[], chunkSize: number): T[][] { @@ -62,23 +72,33 @@ export async function processProductChunk( const llmRetryDelayMs = Math.max(0, options.llmRetryDelayMs ?? 10_000); const sellabilityFilter = options.sellability ?? "available"; const useClaude = options.useClaude === true; + const dependencies: AnalysisPipelineDependencies = { + fetchKeepaDataBatch, + fetchSellabilityBatch, + fetchSpApiPricingAndFees, + getCache, + setCache, + analyzeProducts, + ...options.dependencies, + }; console.log(`\nChecking cache for ${products.length} products...`); const cached = new Map(); - const excludedCachedAsins = new Set(); + const excludedCached = new Map(); const uncachedProducts: ProductRecord[] = []; for (const p of products) { - const hit = await getCache(p.asin); + const hit = await dependencies.getCache(p.asin); if (hit) { + const currentSourceProduct = { ...hit, record: p }; if ( sellabilityFilter === "all" || hit.spApi.sellabilityStatus === "available" ) { console.log(` [cache hit] ${p.asin}`); - cached.set(p.asin, hit); + cached.set(p.asin, currentSourceProduct); } else { - excludedCachedAsins.add(p.asin); + excludedCached.set(p.asin, currentSourceProduct); console.log( ` [exclude cached] ${p.asin} - status=${hit.spApi.sellabilityStatus}`, ); @@ -89,7 +109,7 @@ export async function processProductChunk( } console.log( - `${cached.size} cached available, ${excludedCachedAsins.size} cached excluded, ${uncachedProducts.length} to fetch`, + `${cached.size} cached available, ${excludedCached.size} cached excluded, ${uncachedProducts.length} to fetch`, ); const sellabilityMap = new Map(); @@ -100,7 +120,7 @@ export async function processProductChunk( console.log( `\nChecking sellability for ${uncachedProducts.length} ASINs...`, ); - const sellResults = await fetchSellabilityBatch( + const sellResults = await dependencies.fetchSellabilityBatch( uncachedProducts.map((p) => p.asin), ); @@ -143,7 +163,7 @@ export async function processProductChunk( if (availableProducts.length > 0) { console.log(`\nFetching ${availableProducts.length} ASINs from Keepa...`); try { - keepaResults = await fetchKeepaDataBatch( + keepaResults = await dependencies.fetchKeepaDataBatch( availableProducts.map((p) => p.asin), ); } catch (err) { @@ -168,7 +188,10 @@ export async function processProductChunk( sellabilityStatus: "unknown" as const, sellabilityReason: "Sellability check returned no result", }; - const spApi = await fetchSpApiPricingAndFees(p.asin, sellability); + const spApi = await dependencies.fetchSpApiPricingAndFees( + p.asin, + sellability, + ); const keepa = keepaResults.get(p.asin); if (keepa?.currentPrice && spApi.estimatedSalePrice === 0) { @@ -196,17 +219,33 @@ export async function processProductChunk( const availableAsins = new Set(availableProducts.map((ap) => ap.asin)); for (const p of products) { - if (excludedCachedAsins.has(p.asin)) { + const excludedCachedProduct = excludedCached.get(p.asin); + if (excludedCachedProduct) { + enriched.push({ ...excludedCachedProduct, record: p }); continue; } const cachedProduct = cached.get(p.asin); if (cachedProduct) { - enriched.push(cachedProduct); + enriched.push({ ...cachedProduct, record: p }); continue; } if (!availableAsins.has(p.asin)) { + const sellability = sellabilityMap.get(p.asin); + if (sellability) { + enriched.push({ + record: p, + keepa: null, + spApi: { + ...unknownSpApiData( + sellability.sellabilityReason ?? "Product is not available", + ), + ...sellability, + }, + fetchedAt: new Date().toISOString(), + }); + } continue; } @@ -221,19 +260,41 @@ export async function processProductChunk( fetchedAt: new Date().toISOString(), }; - await setCache(p.asin, product); + await dependencies.setCache(p.asin, product); enriched.push(product); } + const resultsByProduct = new Map(); + const llmProducts: EnrichedProduct[] = []; + for (const product of enriched) { + if ( + sellabilityFilter !== "all" && + product.spApi.sellabilityStatus !== "available" + ) { + resultsByProduct.set(product, { + product, + verdict: { + asin: product.record.asin, + verdict: "SKIP", + confidence: 100, + reasoning: + product.spApi.sellabilityReason ?? + `Sellability status: ${product.spApi.sellabilityStatus}`, + }, + }); + } else { + llmProducts.push(product); + } + } + console.log( - `\nAnalyzing ${enriched.length} products via LLM (batch size: ${llmBatchSize})...\n`, + `\nAnalyzing ${llmProducts.length} products via LLM (batch size: ${llmBatchSize})...\n`, ); - const results: AnalysisResult[] = []; - for (let i = 0; i < enriched.length; i += llmBatchSize) { - const batch = enriched.slice(i, i + llmBatchSize); + for (let i = 0; i < llmProducts.length; i += llmBatchSize) { + const batch = llmProducts.slice(i, i + llmBatchSize); const batchNum = Math.floor(i / llmBatchSize) + 1; - const totalBatches = Math.ceil(enriched.length / llmBatchSize); + const totalBatches = Math.ceil(llmProducts.length / llmBatchSize); console.log(` LLM batch ${batchNum}/${totalBatches}...`); if (i > 0 && llmBatchDelayMs > 0) { @@ -242,7 +303,7 @@ export async function processProductChunk( let verdicts; try { - verdicts = await analyzeProducts(batch, { + verdicts = await dependencies.analyzeProducts(batch, { ignoreSellability: sellabilityFilter === "all", useClaude, }); @@ -251,7 +312,7 @@ export async function processProductChunk( await wait(llmRetryDelayMs); } try { - verdicts = await analyzeProducts(batch, { + verdicts = await dependencies.analyzeProducts(batch, { ignoreSellability: sellabilityFilter === "all", useClaude, }); @@ -264,7 +325,7 @@ export async function processProductChunk( const enrichedProduct = batch[j]; if (!enrichedProduct) continue; - results.push({ + resultsByProduct.set(enrichedProduct, { product: enrichedProduct, verdict: verdicts?.[j] ?? { asin: enrichedProduct.record.asin, @@ -276,5 +337,7 @@ export async function processProductChunk( } } - return results; + return enriched + .map((product) => resultsByProduct.get(product)) + .filter((result): result is AnalysisResult => result !== undefined); } diff --git a/src/asin-offer-search.ts b/src/asin-offer-search.ts index cea369b..fb1626a 100644 --- a/src/asin-offer-search.ts +++ b/src/asin-offer-search.ts @@ -1,4 +1,4 @@ -import { searchProductOffers, type SearxngOfferSearchResult } from "./searxng.ts"; +import { searchProductOffers, type SearxngOfferSearchResult } from "./integrations/searxng.ts"; type CliArgs = { query: string; diff --git a/src/asin.test.ts b/src/asin.test.ts new file mode 100644 index 0000000..5a5ee91 --- /dev/null +++ b/src/asin.test.ts @@ -0,0 +1,13 @@ +import { expect, test } from "bun:test"; +import { normalizeAsin, requireAsin } from "./asin.ts"; + +test("normalizes any valid ten-character ASIN including ISBN-style values", () => { + expect(normalizeAsin(" b07sn9bhvv ")).toBe("B07SN9BHVV"); + expect(normalizeAsin("0306406152")).toBe("0306406152"); +}); + +test("rejects values that cannot be canonical product ASIN keys", () => { + expect(normalizeAsin("short")).toBeNull(); + expect(normalizeAsin("B07SN9BHV!")).toBeNull(); + expect(() => requireAsin("012345678901")).toThrow("Invalid ASIN"); +}); diff --git a/src/asin.ts b/src/asin.ts new file mode 100644 index 0000000..653aeab --- /dev/null +++ b/src/asin.ts @@ -0,0 +1,14 @@ +export const ASIN_PATTERN = /^[A-Z0-9]{10}$/; + +export function normalizeAsin(value: unknown): string | null { + const asin = String(value ?? "").trim().toUpperCase(); + return ASIN_PATTERN.test(asin) ? asin : null; +} + +export function requireAsin(value: unknown): string { + const asin = normalizeAsin(value); + if (!asin) { + throw new Error(`Invalid ASIN: "${String(value ?? "").trim()}"`); + } + return asin; +} diff --git a/src/bestsellers-by-category.test.ts b/src/categories/bestsellers-by-category.test.ts similarity index 51% rename from src/bestsellers-by-category.test.ts rename to src/categories/bestsellers-by-category.test.ts index bd75b79..f94ee13 100644 --- a/src/bestsellers-by-category.test.ts +++ b/src/categories/bestsellers-by-category.test.ts @@ -1,8 +1,41 @@ -import { test, expect, beforeAll, afterAll, beforeEach, mock } from "bun:test"; -import { Database } from "bun:sqlite"; -import { getDb, initDb, closeDb } from "./database.ts"; -import path from "node:path"; -import { rmSync, mkdirSync } from "node:fs"; +import { test, expect, beforeEach, mock } from "bun:test"; + +let nextId = 0; + +function chainable(resolveWith: any[] = []): any { + const p: any = Promise.resolve(resolveWith); + p.limit = (_n: any) => chainable(resolveWith); + p.where = (_cond: any) => chainable(resolveWith); + p.from = (_table: any) => chainable(resolveWith); + return p; +} + +const makeMockDb = (): any => ({ + insert: (_table: any) => ({ + values: (_vals: any) => ({ + returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]), + onConflictDoUpdate: (_conf: any) => Promise.resolve([]), + }), + }), + update: (_table: any) => ({ + set: (_vals: any) => ({ + where: (_cond: any) => Promise.resolve([]), + }), + }), + select: (_sel?: any) => ({ + from: (_table: any) => ({ + where: (_cond: any) => chainable(), + limit: (_n: any) => chainable(), + }), + }), + selectDistinct: (_sel: any) => ({ + from: (_table: any) => chainable(), + }), + execute: (_query: any) => Promise.resolve([]), + transaction: async (fn: (tx: any) => Promise) => fn(makeMockDb()), +}); + +mock.module("../db/index.ts", () => ({ db: makeMockDb(), client: {} })); const fetchSellabilityBatchMock = mock(async (asins: string[]) => { return new Map( @@ -36,51 +69,28 @@ const analyzeProductsMock = mock(async (products: any[]) => { })); }); -mock.module("./sp-api.ts", () => ({ +mock.module("../integrations/sp-api.ts", () => ({ fetchSellabilityBatch: fetchSellabilityBatchMock, fetchSpApiPricingAndFees: fetchSpApiPricingAndFeesMock, })); -mock.module("./llm.ts", () => ({ +mock.module("../integrations/llm.ts", () => ({ analyzeProducts: analyzeProductsMock, })); const modulePromise = import("./bestsellers-by-category.ts"); -const DB_TEST_PATH = path.join( - process.cwd(), - "test_output", - "test_analysis.sqlite", -); - -let db: Database; -let processCategory: (db: Database, runId: number, category: any, perCategoryTop: number) => Promise; -let insertCategoryRunSummary: (db: Database, summary: any, runTimestamp: string) => Promise; +let processCategory: (runId: number, category: any, perCategoryTop: number) => Promise; +let insertCategoryRunSummary: (summary: any, runTimestamp: string) => Promise; let originalFetch: typeof globalThis.fetch; -beforeAll(async () => { - const mod = await modulePromise; - processCategory = mod.processCategory; - insertCategoryRunSummary = mod.insertCategoryRunSummary; - - rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true }); - mkdirSync(path.dirname(DB_TEST_PATH), { recursive: true }); - initDb(DB_TEST_PATH); - db = getDb(DB_TEST_PATH); - - originalFetch = globalThis.fetch; -}); - -afterAll(() => { - globalThis.fetch = originalFetch; - closeDb(); - rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true }); -}); +const mod = await modulePromise; +processCategory = mod.processCategory; +insertCategoryRunSummary = mod.insertCategoryRunSummary; +originalFetch = globalThis.fetch; beforeEach(() => { - db.run("DELETE FROM product_analysis_results"); - db.run("DELETE FROM category_analysis_runs"); - + nextId = 0; globalThis.fetch = mock(async (input: string | URL | Request) => { const rawUrl = typeof input === "string" @@ -139,39 +149,34 @@ test("processCategory function test", async () => { childCount: 0, }; - const runId = await insertCategoryRunSummary(db, { - categoryId: mockCategory.id, - categoryLabel: mockCategory.label, - topAsinsChecked: 0, - availableAsins: 0, - fba: 0, - fbm: 0, - skip: 0, - status: "running", - error: "", - results: [], - }, new Date().toISOString()); - const summary = await processCategory(db, runId, mockCategory, 2); + const runId = await insertCategoryRunSummary( + { + categoryId: mockCategory.id, + categoryLabel: mockCategory.label, + topAsinsChecked: 0, + availableAsins: 0, + fba: 0, + fbm: 0, + skip: 0, + status: "running", + error: "", + results: [], + }, + new Date().toISOString(), + ); - const categoryRun = db.query("SELECT * FROM category_analysis_runs").all() as any[]; - expect(categoryRun.length).toBe(1); - expect(categoryRun[0].category_label).toBe("Category 1"); - expect(categoryRun[0].top_asins_checked).toBe(2); - expect(categoryRun[0].available_asins).toBe(2); - expect(categoryRun[0].fba_count).toBe(1); - expect(categoryRun[0].fbm_count).toBe(1); - expect(categoryRun[0].status).toBe("ok"); + const summary = await processCategory(runId, mockCategory, 2); - const productResults = db.query("SELECT * FROM product_analysis_results ORDER BY asin").all() as any[]; - expect(productResults.length).toBe(2); + expect(summary.status).toBe("ok"); + expect(summary.topAsinsChecked).toBe(2); + expect(summary.availableAsins).toBe(2); + expect(summary.fba).toBe(1); + expect(summary.fbm).toBe(1); + expect(summary.results?.length).toBe(2); + expect(summary.results?.[0]?.product.record.asin).toBe("B000000001"); + expect(summary.results?.[0]?.verdict.verdict).toBe("FBA"); + expect(summary.results?.[1]?.product.record.asin).toBe("B000000002"); + expect(summary.results?.[1]?.verdict.verdict).toBe("FBM"); - expect(productResults[0].asin).toBe("B000000001"); - expect(productResults[0].name).toBe("Product One"); - expect(productResults[0].verdict).toBe("FBA"); - expect(productResults[0].run_id).toBe(categoryRun[0].id); - - expect(productResults[1].asin).toBe("B000000002"); - expect(productResults[1].name).toBe("Product Two"); - expect(productResults[1].verdict).toBe("FBM"); - expect(productResults[1].run_id).toBe(categoryRun[0].id); + globalThis.fetch = originalFetch; }); diff --git a/src/bestsellers-by-category.ts b/src/categories/bestsellers-by-category.ts similarity index 81% rename from src/bestsellers-by-category.ts rename to src/categories/bestsellers-by-category.ts index ed0721c..0fd63e3 100644 --- a/src/bestsellers-by-category.ts +++ b/src/categories/bestsellers-by-category.ts @@ -1,9 +1,14 @@ -import { existsSync, mkdirSync, readFileSync } from "node:fs"; -import path from "node:path"; -import { type Database, getDb, initDb } from "./database.ts"; -import { config } from "./config.ts"; -import { analyzeProducts } from "./llm.ts"; -import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts"; +import { existsSync, mkdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { normalizeAsin } from "../asin.ts"; +import { + createCategoryRun, + persistLlmResults, + updateCategoryRun, +} from "../db/persistence.ts"; +import { config } from "../config.ts"; +import { analyzeProducts } from "../integrations/llm.ts"; +import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "../integrations/sp-api.ts"; import type { AnalysisResult, EnrichedProduct, @@ -12,7 +17,7 @@ import type { ProductRecord, SellabilityInfo, SpApiData, -} from "./types.ts"; +} from "../types.ts"; type CategoryInfo = { id: number; @@ -138,37 +143,14 @@ function printUsageAndExit(message: string): never { process.exit(1); } -export async function insertCategoryRunSummary( - db: Database, - summary: CategoryRunSummary, - runTimestamp: string, -): Promise { - const query = ` - INSERT INTO category_analysis_runs ( - category_id, category_label, run_timestamp, - top_asins_checked, available_asins, - fba_count, fbm_count, skip_count, - status, error_message - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); - `; - const result = db.run(query, [ - summary.categoryId, - summary.categoryLabel, - runTimestamp, - summary.topAsinsChecked, - summary.availableAsins, - summary.fba, - summary.fbm, - summary.skip, - summary.status, - summary.error, - ]); - // Bun's SQLite client returns { changes: number, lastInsertRowid: number | bigint } - return Number(result.lastInsertRowid); -} +export async function insertCategoryRunSummary( + summary: CategoryRunSummary, + runTimestamp: string, +): Promise { + return createCategoryRun(summary, runTimestamp); +} export async function updateCategoryRunSummary( - db: Database, runId: number, summary: Pick< CategoryRunSummary, @@ -180,138 +162,20 @@ export async function updateCategoryRunSummary( | "status" | "error" >, -): Promise { - db.run( - ` - UPDATE category_analysis_runs - SET - top_asins_checked = ?, - available_asins = ?, - fba_count = ?, - fbm_count = ?, - skip_count = ?, - status = ?, - error_message = ? - WHERE id = ? - `, - [ - summary.topAsinsChecked, - summary.availableAsins, - summary.fba, - summary.fbm, - summary.skip, - summary.status, - summary.error, - runId, - ], - ); -} +): Promise { + await updateCategoryRun(runId, summary); +} export async function insertProductAnalysisResults( - db: Database, runId: number, results: AnalysisResult[], -): Promise { - if (results.length === 0) { - return; - } - - const insertStmt = db.prepare(` - INSERT INTO product_analysis_results ( - asin, run_id, name, brand, category, unit_cost, - current_price, avg_price_90d, avg_price_90d_sheet, - selling_price_sheet, sales_rank, sales_rank_avg_90d, - seller_count, amazon_is_seller, amazon_buybox_share_pct_90d, - monthly_sold, rank_drops_30d, rank_drops_90d, - fba_fee, fbm_fee, referral_percent, can_sell, - sellability_status, sellability_reason, - verdict, confidence, reasoning, fetched_at - ) VALUES ( - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ?, ?, ?, ? - ) - ON CONFLICT(asin) DO UPDATE SET - run_id = excluded.run_id, - name = excluded.name, - brand = excluded.brand, - category = excluded.category, - unit_cost = excluded.unit_cost, - current_price = excluded.current_price, - avg_price_90d = excluded.avg_price_90d, - avg_price_90d_sheet = excluded.avg_price_90d_sheet, - selling_price_sheet = excluded.selling_price_sheet, - sales_rank = excluded.sales_rank, - sales_rank_avg_90d = excluded.sales_rank_avg_90d, - seller_count = excluded.seller_count, - amazon_is_seller = excluded.amazon_is_seller, - amazon_buybox_share_pct_90d = excluded.amazon_buybox_share_pct_90d, - monthly_sold = excluded.monthly_sold, - rank_drops_30d = excluded.rank_drops_30d, - rank_drops_90d = excluded.rank_drops_90d, - fba_fee = excluded.fba_fee, - fbm_fee = excluded.fbm_fee, - referral_percent = excluded.referral_percent, - can_sell = excluded.can_sell, - sellability_status = excluded.sellability_status, - sellability_reason = excluded.sellability_reason, - verdict = excluded.verdict, - confidence = excluded.confidence, - reasoning = excluded.reasoning, - fetched_at = excluded.fetched_at; - `); - - db.transaction((resultsBatch: AnalysisResult[]) => { - for (const r of resultsBatch) { - const price = - r.product.keepa?.currentPrice ?? - r.product.record.sellingPriceFromSheet ?? - r.product.spApi.estimatedSalePrice; - const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank; - - insertStmt.run( - r.product.record.asin, - runId, - r.product.record.name, - r.product.record.brand ?? null, - r.product.record.category ?? - r.product.keepa?.categoryTree?.join(" > ") ?? - null, - r.product.record.unitCost ?? null, - price ?? null, - r.product.keepa?.avgPrice90 ?? null, - r.product.record.avgPrice90FromSheet ?? null, - r.product.record.sellingPriceFromSheet ?? null, - rank ?? null, - r.product.keepa?.salesRankAvg90 ?? null, - r.product.keepa?.sellerCount ?? null, - r.product.keepa?.amazonIsSeller == null - ? null - : r.product.keepa.amazonIsSeller - ? 1 - : 0, - r.product.keepa?.amazonBuyboxSharePct90d ?? null, - r.product.keepa?.monthlySold ?? null, - r.product.keepa?.salesRankDrops30 ?? null, - r.product.keepa?.salesRankDrops90 ?? null, - r.product.spApi.fbaFee ?? null, - r.product.spApi.fbmFee ?? null, - r.product.spApi.referralFeePercent ?? null, - r.product.spApi.canSell == null - ? "unknown" - : r.product.spApi.canSell - ? "yes" - : "no", - r.product.spApi.sellabilityStatus ?? null, - r.product.spApi.sellabilityReason ?? null, - r.verdict.verdict, - r.verdict.confidence, - r.verdict.reasoning ?? null, - r.product.fetchedAt, - ); - } - })(results); // Execute the transaction with the results batch -} +): Promise { + if (results.length === 0) return; + await persistLlmResults(runId, results, { + source: "category_analysis", + metadataSource: "catalog", + }); +} function loadCategoryBlacklist(filePath: string): Set { const blacklist = new Set(); @@ -690,10 +554,14 @@ async function fetchCategoryBestSellerAsins( ]; for (const value of candidates) { - if (Array.isArray(value)) { - return [ - ...new Set(value.map((v) => String(v).trim()).filter(Boolean)), - ].slice(0, limit); + if (Array.isArray(value)) { + return [ + ...new Set( + value + .map((v) => normalizeAsin(v)) + .filter((asin): asin is string => asin !== null), + ), + ].slice(0, limit); } } @@ -945,10 +813,10 @@ async function fetchKeepaEnrichmentMap( `/product?key=${encodeURIComponent(config.keepaApiKey)}&domain=${DOMAIN_US}&asin=${asinParam}&stats=90&buybox=1&days=90`, ); - const products = Array.isArray(data?.products) ? data.products : []; - for (const product of products) { - const asin = String(product?.asin ?? "").trim(); - if (!asin) continue; + const products = Array.isArray(data?.products) ? data.products : []; + for (const product of products) { + const asin = normalizeAsin(product?.asin); + if (!asin) continue; out.set(asin, { keepa: parseKeepaProduct(product), title: String(product?.title ?? "").trim(), @@ -1014,7 +882,6 @@ function buildEnrichedProducts( } export async function processCategory( - db: Database, runId: number, category: CategoryInfo, perCategoryTop: number, @@ -1025,7 +892,7 @@ export async function processCategory( const topAsins = await fetchCategoryBestSellerAsins(category, perCategoryTop); if (topAsins.length === 0) { log("info", " Keepa returned no ASINs for this category."); - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: 0, availableAsins: 0, fba: 0, @@ -1069,7 +936,7 @@ export async function processCategory( ` Sellable ASINs: ${availableAsins.length}/${uniqueTopAsins.length}`, ); if (availableAsins.length === 0) { - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: uniqueTopAsins.length, availableAsins: 0, fba: 0, @@ -1137,7 +1004,7 @@ export async function processCategory( }, })); - await insertProductAnalysisResults(db, runId, batchResults); + await insertProductAnalysisResults(runId, batchResults); for (const result of batchResults) { results.push(result); @@ -1150,7 +1017,7 @@ export async function processCategory( } } - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: uniqueTopAsins.length, availableAsins: availableAsins.length, fba, @@ -1170,7 +1037,7 @@ export async function processCategory( } } - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: uniqueTopAsins.length, availableAsins: availableAsins.length, fba, @@ -1199,10 +1066,6 @@ export async function main(): Promise { assertSpApiPrerequisites(); mkdirSync(args.outputDir, { recursive: true }); - const DB_PATH = - process.env.RESULTS_DB_PATH || path.join(process.cwd(), "db", "results.db"); - initDb(DB_PATH); - const db = getDb(DB_PATH); log("info", "Starting per-category bestseller pipeline"); log("info", `Marketplace: ${config.spApiMarketplaceId}`); @@ -1236,7 +1099,6 @@ export async function main(): Promise { let runId: number | undefined; try { runId = await insertCategoryRunSummary( - db, { categoryId: category.id, categoryLabel: category.label, @@ -1253,7 +1115,6 @@ export async function main(): Promise { ); categorySummary = await processCategory( - db, runId, category, args.perCategoryTop, @@ -1283,7 +1144,7 @@ export async function main(): Promise { results: [], }; if (runId) { - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: 0, availableAsins: 0, fba: 0, diff --git a/src/mid-range-sellers-by-category.test.ts b/src/categories/mid-range-sellers-by-category.test.ts similarity index 59% rename from src/mid-range-sellers-by-category.test.ts rename to src/categories/mid-range-sellers-by-category.test.ts index 7c53d8b..71668dc 100644 --- a/src/mid-range-sellers-by-category.test.ts +++ b/src/categories/mid-range-sellers-by-category.test.ts @@ -1,8 +1,41 @@ -import { test, expect, beforeAll, afterAll, beforeEach, mock } from "bun:test"; -import { Database } from "bun:sqlite"; -import { getDb, initDb, closeDb } from "./database.ts"; -import path from "node:path"; -import { rmSync, mkdirSync } from "node:fs"; +import { test, expect, beforeEach, mock } from "bun:test"; + +let nextId = 0; + +function chainable(resolveWith: any[] = []): any { + const p: any = Promise.resolve(resolveWith); + p.limit = (_n: any) => chainable(resolveWith); + p.where = (_cond: any) => chainable(resolveWith); + p.from = (_table: any) => chainable(resolveWith); + return p; +} + +const makeMockDb = (): any => ({ + insert: (_table: any) => ({ + values: (_vals: any) => ({ + returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]), + onConflictDoUpdate: (_conf: any) => Promise.resolve([]), + }), + }), + update: (_table: any) => ({ + set: (_vals: any) => ({ + where: (_cond: any) => Promise.resolve([]), + }), + }), + select: (_sel?: any) => ({ + from: (_table: any) => ({ + where: (_cond: any) => chainable(), + limit: (_n: any) => chainable(), + }), + }), + selectDistinct: (_sel: any) => ({ + from: (_table: any) => chainable(), + }), + execute: (_query: any) => Promise.resolve([]), + transaction: async (fn: (tx: any) => Promise) => fn(makeMockDb()), +}); + +mock.module("../db/index.ts", () => ({ db: makeMockDb(), client: {} })); const fetchSellabilityBatchMock = mock(async (asins: string[]) => { return new Map( @@ -51,55 +84,28 @@ const analyzeProductsMock = mock(async (products: any[]) => { })); }); -mock.module("./sp-api.ts", () => ({ +mock.module("../integrations/sp-api.ts", () => ({ fetchSellabilityBatch: fetchSellabilityBatchMock, fetchSpApiPricingAndFees: fetchSpApiPricingAndFeesMock, })); -mock.module("./llm.ts", () => ({ +mock.module("../integrations/llm.ts", () => ({ analyzeProducts: analyzeProductsMock, })); const modulePromise = import("./mid-range-sellers-by-category.ts"); -const DB_TEST_PATH = path.join( - process.cwd(), - "test_output", - "test_mid_range_analysis.sqlite", -); - -let db: Database; let processCategory: any; -let insertCategoryRunSummary: ( - db: Database, - summary: any, - runTimestamp: string, -) => Promise; +let insertCategoryRunSummary: (summary: any, runTimestamp: string) => Promise; let originalFetch: typeof globalThis.fetch; -beforeAll(async () => { - const mod = await modulePromise; - processCategory = mod.processCategory; - insertCategoryRunSummary = mod.insertCategoryRunSummary; - - rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true }); - mkdirSync(path.dirname(DB_TEST_PATH), { recursive: true }); - initDb(DB_TEST_PATH); - db = getDb(DB_TEST_PATH); - - originalFetch = globalThis.fetch; -}); - -afterAll(() => { - globalThis.fetch = originalFetch; - closeDb(); - rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true }); -}); +const mod = await modulePromise; +processCategory = mod.processCategory; +insertCategoryRunSummary = mod.insertCategoryRunSummary; +originalFetch = globalThis.fetch; beforeEach(() => { - db.run("DELETE FROM product_analysis_results"); - db.run("DELETE FROM category_analysis_runs"); - + nextId = 0; globalThis.fetch = mock(async (input: string | URL | Request) => { const rawUrl = typeof input === "string" @@ -138,25 +144,8 @@ beforeEach(() => { buyBoxStatsAmazon90: 40, stats: { current: [ - null, - null, - null, - 1000, - null, - null, - null, - null, - null, - null, - null, - 5, - null, - null, - null, - null, - null, - null, - 2599, + null, null, null, 1000, null, null, null, null, null, null, null, 5, + null, null, null, null, null, null, 2599, ], avg: [2400, null, null, 1200], }, @@ -171,25 +160,8 @@ beforeEach(() => { buyBoxStatsAmazon90: 50, stats: { current: [ - null, - null, - null, - 2000, - null, - null, - null, - null, - null, - null, - null, - 3, - null, - null, - null, - null, - null, - null, - 1999, + null, null, null, 2000, null, null, null, null, null, null, null, 3, + null, null, null, null, null, null, 1999, ], avg: [1800, null, null, 2200], }, @@ -204,25 +176,8 @@ beforeEach(() => { buyBoxStatsAmazon90: 50, stats: { current: [ - null, - null, - null, - 1500, - null, - null, - null, - null, - null, - null, - null, - 4, - null, - null, - null, - null, - null, - null, - 2099, + null, null, null, 1500, null, null, null, null, null, null, null, 4, + null, null, null, null, null, null, 2099, ], avg: [2000, null, null, 1800], }, @@ -237,25 +192,8 @@ beforeEach(() => { buyBoxStatsAmazon90: 95, stats: { current: [ - null, - null, - null, - 3000, - null, - null, - null, - null, - null, - null, - null, - 4, - null, - null, - null, - null, - null, - null, - 2899, + null, null, null, 3000, null, null, null, null, null, null, null, 4, + null, null, null, null, null, null, 2899, ], avg: [2600, null, null, 2800], }, @@ -269,25 +207,8 @@ beforeEach(() => { isAmazonSeller: false, stats: { current: [ - null, - null, - null, - 3200, - null, - null, - null, - null, - null, - null, - null, - 25, - null, - null, - null, - null, - null, - null, - 3500, + null, null, null, 3200, null, null, null, null, null, null, null, 25, + null, null, null, null, null, null, 3500, ], avg: [3200, null, null, 3200], }, @@ -315,7 +236,6 @@ test("processCategory only analyzes sellable mid-range matches", async () => { }; const runId = await insertCategoryRunSummary( - db, { categoryId: mockCategory.id, categoryLabel: mockCategory.label, @@ -332,7 +252,6 @@ test("processCategory only analyzes sellable mid-range matches", async () => { ); const summary = await processCategory( - db, runId, mockCategory, 3, @@ -345,6 +264,7 @@ test("processCategory only analyzes sellable mid-range matches", async () => { 20, 15, 85, + "strict", ); expect(summary.status).toBe("ok"); @@ -352,23 +272,7 @@ test("processCategory only analyzes sellable mid-range matches", async () => { expect(summary.availableAsins).toBe(1); expect(summary.results?.length).toBe(1); - const productResults = db - .query( - "SELECT asin, monthly_sold, can_sell, sellability_status FROM product_analysis_results ORDER BY monthly_sold DESC", - ) - .all() as Array<{ - asin: string; - monthly_sold: number; - can_sell: string; - sellability_status: string; - }>; - - expect(productResults.length).toBe(1); - expect(productResults.map((row) => row.asin)).toEqual(["B000000001"]); - - const sellable = productResults.find((row) => row.asin === "B000000001"); - expect(sellable?.can_sell).toBe("yes"); - expect(sellable?.sellability_status).toBe("available"); + globalThis.fetch = originalFetch; }); test("processCategory returns empty when no products match mid-range criteria", async () => { @@ -380,7 +284,6 @@ test("processCategory returns empty when no products match mid-range criteria", }; const runId = await insertCategoryRunSummary( - db, { categoryId: mockCategory.id, categoryLabel: mockCategory.label, @@ -397,7 +300,6 @@ test("processCategory returns empty when no products match mid-range criteria", ); const summary = await processCategory( - db, runId, mockCategory, 3, @@ -410,6 +312,7 @@ test("processCategory returns empty when no products match mid-range criteria", 20, 15, 85, + "strict", ); expect(summary.status).toBe("empty"); @@ -417,8 +320,5 @@ test("processCategory returns empty when no products match mid-range criteria", expect(summary.availableAsins).toBe(0); expect(summary.results?.length).toBe(0); - const rows = db - .query("SELECT COUNT(*) as c FROM product_analysis_results") - .all() as Array<{ c: number }>; - expect(rows[0]?.c).toBe(0); + globalThis.fetch = originalFetch; }); diff --git a/src/mid-range-sellers-by-category.ts b/src/categories/mid-range-sellers-by-category.ts similarity index 87% rename from src/mid-range-sellers-by-category.ts rename to src/categories/mid-range-sellers-by-category.ts index 154e01b..2123b44 100644 --- a/src/mid-range-sellers-by-category.ts +++ b/src/categories/mid-range-sellers-by-category.ts @@ -1,17 +1,22 @@ -import { existsSync, mkdirSync, readFileSync } from "node:fs"; -import path from "node:path"; -import { createInterface } from "node:readline/promises"; -import { stdin as input, stdout as output } from "node:process"; -import { type Database, getDb, initDb } from "./database.ts"; -import { config } from "./config.ts"; +import { existsSync, mkdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { createInterface } from "node:readline/promises"; +import { stdin as input, stdout as output } from "node:process"; +import { normalizeAsin } from "../asin.ts"; +import { + createCategoryRun, + persistLlmResults, + updateCategoryRun, +} from "../db/persistence.ts"; +import { config } from "../config.ts"; import { connectCache, disconnectCache, getApiCache, setApiCache, -} from "./cache.ts"; -import { analyzeProducts } from "./llm.ts"; -import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts"; +} from "../integrations/cache.ts"; +import { analyzeProducts } from "../integrations/llm.ts"; +import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "../integrations/sp-api.ts"; import type { AnalysisResult, EnrichedProduct, @@ -20,7 +25,7 @@ import type { ProductRecord, SellabilityInfo, SpApiData, -} from "./types.ts"; +} from "../types.ts"; type CategoryInfo = { id: number; @@ -473,37 +478,14 @@ async function promptCategoryIds( } } -export async function insertCategoryRunSummary( - db: Database, - summary: CategoryRunSummary, - runTimestamp: string, -): Promise { - const query = ` - INSERT INTO category_analysis_runs ( - category_id, category_label, run_timestamp, - top_asins_checked, available_asins, - fba_count, fbm_count, skip_count, - status, error_message - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); - `; - const result = db.run(query, [ - summary.categoryId, - summary.categoryLabel, - runTimestamp, - summary.topAsinsChecked, - summary.availableAsins, - summary.fba, - summary.fbm, - summary.skip, - summary.status, - summary.error, - ]); - // Bun's SQLite client returns { changes: number, lastInsertRowid: number | bigint } - return Number(result.lastInsertRowid); -} +export async function insertCategoryRunSummary( + summary: CategoryRunSummary, + runTimestamp: string, +): Promise { + return createCategoryRun(summary, runTimestamp); +} export async function updateCategoryRunSummary( - db: Database, runId: number, summary: Pick< CategoryRunSummary, @@ -515,138 +497,20 @@ export async function updateCategoryRunSummary( | "status" | "error" >, -): Promise { - db.run( - ` - UPDATE category_analysis_runs - SET - top_asins_checked = ?, - available_asins = ?, - fba_count = ?, - fbm_count = ?, - skip_count = ?, - status = ?, - error_message = ? - WHERE id = ? - `, - [ - summary.topAsinsChecked, - summary.availableAsins, - summary.fba, - summary.fbm, - summary.skip, - summary.status, - summary.error, - runId, - ], - ); -} +): Promise { + await updateCategoryRun(runId, summary); +} export async function insertProductAnalysisResults( - db: Database, runId: number, results: AnalysisResult[], -): Promise { - if (results.length === 0) { - return; - } - - const insertStmt = db.prepare(` - INSERT INTO product_analysis_results ( - asin, run_id, name, brand, category, unit_cost, - current_price, avg_price_90d, avg_price_90d_sheet, - selling_price_sheet, sales_rank, sales_rank_avg_90d, - seller_count, amazon_is_seller, amazon_buybox_share_pct_90d, - monthly_sold, rank_drops_30d, rank_drops_90d, - fba_fee, fbm_fee, referral_percent, can_sell, - sellability_status, sellability_reason, - verdict, confidence, reasoning, fetched_at - ) VALUES ( - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ?, ?, ?, ? - ) - ON CONFLICT(asin) DO UPDATE SET - run_id = excluded.run_id, - name = excluded.name, - brand = excluded.brand, - category = excluded.category, - unit_cost = excluded.unit_cost, - current_price = excluded.current_price, - avg_price_90d = excluded.avg_price_90d, - avg_price_90d_sheet = excluded.avg_price_90d_sheet, - selling_price_sheet = excluded.selling_price_sheet, - sales_rank = excluded.sales_rank, - sales_rank_avg_90d = excluded.sales_rank_avg_90d, - seller_count = excluded.seller_count, - amazon_is_seller = excluded.amazon_is_seller, - amazon_buybox_share_pct_90d = excluded.amazon_buybox_share_pct_90d, - monthly_sold = excluded.monthly_sold, - rank_drops_30d = excluded.rank_drops_30d, - rank_drops_90d = excluded.rank_drops_90d, - fba_fee = excluded.fba_fee, - fbm_fee = excluded.fbm_fee, - referral_percent = excluded.referral_percent, - can_sell = excluded.can_sell, - sellability_status = excluded.sellability_status, - sellability_reason = excluded.sellability_reason, - verdict = excluded.verdict, - confidence = excluded.confidence, - reasoning = excluded.reasoning, - fetched_at = excluded.fetched_at; - `); - - db.transaction((resultsBatch: AnalysisResult[]) => { - for (const r of resultsBatch) { - const price = - r.product.keepa?.currentPrice ?? - r.product.record.sellingPriceFromSheet ?? - r.product.spApi.estimatedSalePrice; - const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank; - - insertStmt.run( - r.product.record.asin, - runId, - r.product.record.name, - r.product.record.brand ?? null, - r.product.record.category ?? - r.product.keepa?.categoryTree?.join(" > ") ?? - null, - r.product.record.unitCost ?? null, - price ?? null, - r.product.keepa?.avgPrice90 ?? null, - r.product.record.avgPrice90FromSheet ?? null, - r.product.record.sellingPriceFromSheet ?? null, - rank ?? null, - r.product.keepa?.salesRankAvg90 ?? null, - r.product.keepa?.sellerCount ?? null, - r.product.keepa?.amazonIsSeller == null - ? null - : r.product.keepa.amazonIsSeller - ? 1 - : 0, - r.product.keepa?.amazonBuyboxSharePct90d ?? null, - r.product.keepa?.monthlySold ?? null, - r.product.keepa?.salesRankDrops30 ?? null, - r.product.keepa?.salesRankDrops90 ?? null, - r.product.spApi.fbaFee ?? null, - r.product.spApi.fbmFee ?? null, - r.product.spApi.referralFeePercent ?? null, - r.product.spApi.canSell == null - ? "unknown" - : r.product.spApi.canSell - ? "yes" - : "no", - r.product.spApi.sellabilityStatus ?? null, - r.product.spApi.sellabilityReason ?? null, - r.verdict.verdict, - r.verdict.confidence, - r.verdict.reasoning ?? null, - r.product.fetchedAt, - ); - } - })(results); // Execute the transaction with the results batch -} +): Promise { + if (results.length === 0) return; + await persistLlmResults(runId, results, { + source: "category_analysis", + metadataSource: "catalog", + }); +} function loadCategoryBlacklist(filePath: string): Set { const blacklist = new Set(); @@ -1025,10 +889,14 @@ async function fetchCategoryBestSellerAsins( ]; for (const value of candidates) { - if (Array.isArray(value)) { - return [ - ...new Set(value.map((v) => String(v).trim()).filter(Boolean)), - ].slice(0, limit); + if (Array.isArray(value)) { + return [ + ...new Set( + value + .map((v) => normalizeAsin(v)) + .filter((asin): asin is string => asin !== null), + ), + ].slice(0, limit); } } @@ -1284,10 +1152,10 @@ async function fetchKeepaEnrichmentMap( `/product?key=${encodeURIComponent(config.keepaApiKey)}&domain=${DOMAIN_US}&asin=${asinParam}&stats=90&buybox=1&days=90`, ); - const products = Array.isArray(data?.products) ? data.products : []; - for (const product of products) { - const asin = String(product?.asin ?? "").trim(); - if (!asin) continue; + const products = Array.isArray(data?.products) ? data.products : []; + for (const product of products) { + const asin = normalizeAsin(product?.asin); + if (!asin) continue; const parsed = { keepa: parseKeepaProduct(product), title: String(product?.title ?? "").trim(), @@ -1471,7 +1339,6 @@ function shouldKeepCandidateBySellability( } export async function processCategory( - db: Database, runId: number, category: CategoryInfo, perCategoryTop: number, @@ -1505,7 +1372,7 @@ export async function processCategory( ); if (topAsins.length === 0) { log("info", " Keepa returned no ASINs for this category."); - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: 0, availableAsins: 0, fba: 0, @@ -1766,7 +1633,7 @@ export async function processCategory( }, })); - await insertProductAnalysisResults(db, runId, batchResults); + await insertProductAnalysisResults(runId, batchResults); for (const result of batchResults) { if (result.verdict.verdict === "FBA") { @@ -1781,7 +1648,7 @@ export async function processCategory( budget.analyzedAsins += batchResults.length; results.push(...batchResults); - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: checkedAsins, availableAsins: results.length, fba, @@ -1802,7 +1669,7 @@ export async function processCategory( const emptyReason = budget.stopReason || "No sellable ASINs matched the configured mid-range criteria"; - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: checkedAsins, availableAsins: 0, fba, @@ -1830,7 +1697,7 @@ export async function processCategory( ` Category stream totals: checked=${checkedAsins}, sellable=${sellableAsins}, keepaEnriched=${keepaEnrichedAsins}, analyzed=${results.length}`, ); - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: checkedAsins, availableAsins: results.length, fba, @@ -1923,11 +1790,6 @@ export async function main(): Promise { await connectCache(); try { mkdirSync(args.outputDir, { recursive: true }); - const DB_PATH = - process.env.RESULTS_DB_PATH || - path.join(process.cwd(), "db", "results.db"); - initDb(DB_PATH); - const db = getDb(DB_PATH); log("info", "Starting per-category mid-range pipeline"); log("info", `Marketplace: ${config.spApiMarketplaceId}`); @@ -1987,7 +1849,6 @@ export async function main(): Promise { let runId: number | undefined; try { runId = await insertCategoryRunSummary( - db, { categoryId: category.id, categoryLabel: category.label, @@ -2004,7 +1865,6 @@ export async function main(): Promise { ); categorySummary = await processCategory( - db, runId, category, args.perCategoryTop, @@ -2046,7 +1906,7 @@ export async function main(): Promise { results: [], }; if (runId) { - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: 0, availableAsins: 0, fba: 0, diff --git a/src/top-monthly-sold-by-category.test.ts b/src/categories/top-monthly-sold-by-category.test.ts similarity index 56% rename from src/top-monthly-sold-by-category.test.ts rename to src/categories/top-monthly-sold-by-category.test.ts index fce48c5..4a324e9 100644 --- a/src/top-monthly-sold-by-category.test.ts +++ b/src/categories/top-monthly-sold-by-category.test.ts @@ -1,8 +1,41 @@ -import { test, expect, beforeAll, afterAll, beforeEach, mock } from "bun:test"; -import { Database } from "bun:sqlite"; -import { getDb, initDb, closeDb } from "./database.ts"; -import path from "node:path"; -import { rmSync, mkdirSync } from "node:fs"; +import { test, expect, beforeEach, mock } from "bun:test"; + +let nextId = 0; + +function chainable(resolveWith: any[] = []): any { + const p: any = Promise.resolve(resolveWith); + p.limit = (_n: any) => chainable(resolveWith); + p.where = (_cond: any) => chainable(resolveWith); + p.from = (_table: any) => chainable(resolveWith); + return p; +} + +const makeMockDb = (): any => ({ + insert: (_table: any) => ({ + values: (_vals: any) => ({ + returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]), + onConflictDoUpdate: (_conf: any) => Promise.resolve([]), + }), + }), + update: (_table: any) => ({ + set: (_vals: any) => ({ + where: (_cond: any) => Promise.resolve([]), + }), + }), + select: (_sel?: any) => ({ + from: (_table: any) => ({ + where: (_cond: any) => chainable(), + limit: (_n: any) => chainable(), + }), + }), + selectDistinct: (_sel: any) => ({ + from: (_table: any) => chainable(), + }), + execute: (_query: any) => Promise.resolve([]), + transaction: async (fn: (tx: any) => Promise) => fn(makeMockDb()), +}); + +mock.module("../db/index.ts", () => ({ db: makeMockDb(), client: {} })); const fetchSellabilityBatchMock = mock(async (asins: string[]) => { return new Map( @@ -49,62 +82,34 @@ const analyzeProductsMock = mock(async (products: any[]) => { })); }); -mock.module("./sp-api.ts", () => ({ +mock.module("../integrations/sp-api.ts", () => ({ fetchSellabilityBatch: fetchSellabilityBatchMock, fetchSpApiPricingAndFees: fetchSpApiPricingAndFeesMock, })); -mock.module("./llm.ts", () => ({ +mock.module("../integrations/llm.ts", () => ({ analyzeProducts: analyzeProductsMock, })); const modulePromise = import("./top-monthly-sold-by-category.ts"); -const DB_TEST_PATH = path.join( - process.cwd(), - "test_output", - "test_monthly_sold_analysis.sqlite", -); - -let db: Database; let processCategory: ( - db: Database, runId: number, category: any, perCategoryTop: number, categoryCandidatePool: number, minMonthlySold: number, ) => Promise; -let insertCategoryRunSummary: ( - db: Database, - summary: any, - runTimestamp: string, -) => Promise; +let insertCategoryRunSummary: (summary: any, runTimestamp: string) => Promise; let originalFetch: typeof globalThis.fetch; -beforeAll(async () => { - const mod = await modulePromise; - processCategory = mod.processCategory; - insertCategoryRunSummary = mod.insertCategoryRunSummary; - - rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true }); - mkdirSync(path.dirname(DB_TEST_PATH), { recursive: true }); - initDb(DB_TEST_PATH); - db = getDb(DB_TEST_PATH); - - originalFetch = globalThis.fetch; -}); - -afterAll(() => { - globalThis.fetch = originalFetch; - closeDb(); - rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true }); -}); +const mod = await modulePromise; +processCategory = mod.processCategory; +insertCategoryRunSummary = mod.insertCategoryRunSummary; +originalFetch = globalThis.fetch; beforeEach(() => { - db.run("DELETE FROM product_analysis_results"); - db.run("DELETE FROM category_analysis_runs"); - + nextId = 0; globalThis.fetch = mock(async (input: string | URL | Request) => { const rawUrl = typeof input === "string" @@ -140,25 +145,8 @@ beforeEach(() => { monthlySold: 600, stats: { current: [ - null, - null, - null, - 1000, - null, - null, - null, - null, - null, - null, - null, - 2, - null, - null, - null, - null, - null, - null, - 2599, + null, null, null, 1000, null, null, null, null, null, null, null, 2, + null, null, null, null, null, null, 2599, ], avg: [2400, null, null, 1200], }, @@ -171,25 +159,8 @@ beforeEach(() => { monthlySold: 250, stats: { current: [ - null, - null, - null, - 2000, - null, - null, - null, - null, - null, - null, - null, - 3, - null, - null, - null, - null, - null, - null, - 1999, + null, null, null, 2000, null, null, null, null, null, null, null, 3, + null, null, null, null, null, null, 1999, ], avg: [1800, null, null, 2200], }, @@ -202,25 +173,8 @@ beforeEach(() => { monthlySold: 800, stats: { current: [ - null, - null, - null, - 1500, - null, - null, - null, - null, - null, - null, - null, - 1, - null, - null, - null, - null, - null, - null, - 2099, + null, null, null, 1500, null, null, null, null, null, null, null, 1, + null, null, null, null, null, null, 2099, ], avg: [2000, null, null, 1800], }, @@ -233,25 +187,8 @@ beforeEach(() => { monthlySold: 400, stats: { current: [ - null, - null, - null, - 3000, - null, - null, - null, - null, - null, - null, - null, - 4, - null, - null, - null, - null, - null, - null, - 2899, + null, null, null, 3000, null, null, null, null, null, null, null, 4, + null, null, null, null, null, null, 2899, ], avg: [2600, null, null, 2800], }, @@ -279,7 +216,6 @@ test("processCategory filters to sellable ASINs with monthly sold >= threshold a }; const runId = await insertCategoryRunSummary( - db, { categoryId: mockCategory.id, categoryLabel: mockCategory.label, @@ -295,22 +231,16 @@ test("processCategory filters to sellable ASINs with monthly sold >= threshold a new Date().toISOString(), ); - const summary = await processCategory(db, runId, mockCategory, 2, 4, 300); + const summary = await processCategory(runId, mockCategory, 2, 4, 300); expect(summary.status).toBe("ok"); expect(summary.topAsinsChecked).toBe(4); expect(summary.availableAsins).toBe(2); expect(summary.results?.length).toBe(2); - const productResults = db - .query( - "SELECT asin, monthly_sold FROM product_analysis_results ORDER BY monthly_sold DESC", - ) - .all() as Array<{ asin: string; monthly_sold: number }>; + const asins = summary.results?.map((r: any) => r.product.record.asin) ?? []; + expect(asins).toContain("B000000001"); + expect(asins).toContain("B000000004"); - expect(productResults.length).toBe(2); - expect(productResults[0]?.asin).toBe("B000000001"); - expect(productResults[0]?.monthly_sold).toBe(600); - expect(productResults[1]?.asin).toBe("B000000004"); - expect(productResults[1]?.monthly_sold).toBe(400); + globalThis.fetch = originalFetch; }); diff --git a/src/top-monthly-sold-by-category.ts b/src/categories/top-monthly-sold-by-category.ts similarity index 82% rename from src/top-monthly-sold-by-category.ts rename to src/categories/top-monthly-sold-by-category.ts index 547aa5c..41fd862 100644 --- a/src/top-monthly-sold-by-category.ts +++ b/src/categories/top-monthly-sold-by-category.ts @@ -1,9 +1,14 @@ -import { existsSync, mkdirSync, readFileSync } from "node:fs"; -import path from "node:path"; -import { type Database, getDb, initDb } from "./database.ts"; -import { config } from "./config.ts"; -import { analyzeProducts } from "./llm.ts"; -import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts"; +import { existsSync, mkdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { normalizeAsin } from "../asin.ts"; +import { + createCategoryRun, + persistLlmResults, + updateCategoryRun, +} from "../db/persistence.ts"; +import { config } from "../config.ts"; +import { analyzeProducts } from "../integrations/llm.ts"; +import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "../integrations/sp-api.ts"; import type { AnalysisResult, EnrichedProduct, @@ -12,7 +17,7 @@ import type { ProductRecord, SellabilityInfo, SpApiData, -} from "./types.ts"; +} from "../types.ts"; type CategoryInfo = { id: number; @@ -170,37 +175,14 @@ function printUsageAndExit(message: string): never { process.exit(1); } -export async function insertCategoryRunSummary( - db: Database, - summary: CategoryRunSummary, - runTimestamp: string, -): Promise { - const query = ` - INSERT INTO category_analysis_runs ( - category_id, category_label, run_timestamp, - top_asins_checked, available_asins, - fba_count, fbm_count, skip_count, - status, error_message - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); - `; - const result = db.run(query, [ - summary.categoryId, - summary.categoryLabel, - runTimestamp, - summary.topAsinsChecked, - summary.availableAsins, - summary.fba, - summary.fbm, - summary.skip, - summary.status, - summary.error, - ]); - // Bun's SQLite client returns { changes: number, lastInsertRowid: number | bigint } - return Number(result.lastInsertRowid); -} +export async function insertCategoryRunSummary( + summary: CategoryRunSummary, + runTimestamp: string, +): Promise { + return createCategoryRun(summary, runTimestamp); +} export async function updateCategoryRunSummary( - db: Database, runId: number, summary: Pick< CategoryRunSummary, @@ -212,138 +194,20 @@ export async function updateCategoryRunSummary( | "status" | "error" >, -): Promise { - db.run( - ` - UPDATE category_analysis_runs - SET - top_asins_checked = ?, - available_asins = ?, - fba_count = ?, - fbm_count = ?, - skip_count = ?, - status = ?, - error_message = ? - WHERE id = ? - `, - [ - summary.topAsinsChecked, - summary.availableAsins, - summary.fba, - summary.fbm, - summary.skip, - summary.status, - summary.error, - runId, - ], - ); -} +): Promise { + await updateCategoryRun(runId, summary); +} export async function insertProductAnalysisResults( - db: Database, runId: number, results: AnalysisResult[], -): Promise { - if (results.length === 0) { - return; - } - - const insertStmt = db.prepare(` - INSERT INTO product_analysis_results ( - asin, run_id, name, brand, category, unit_cost, - current_price, avg_price_90d, avg_price_90d_sheet, - selling_price_sheet, sales_rank, sales_rank_avg_90d, - seller_count, amazon_is_seller, amazon_buybox_share_pct_90d, - monthly_sold, rank_drops_30d, rank_drops_90d, - fba_fee, fbm_fee, referral_percent, can_sell, - sellability_status, sellability_reason, - verdict, confidence, reasoning, fetched_at - ) VALUES ( - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ?, ?, ?, ? - ) - ON CONFLICT(asin) DO UPDATE SET - run_id = excluded.run_id, - name = excluded.name, - brand = excluded.brand, - category = excluded.category, - unit_cost = excluded.unit_cost, - current_price = excluded.current_price, - avg_price_90d = excluded.avg_price_90d, - avg_price_90d_sheet = excluded.avg_price_90d_sheet, - selling_price_sheet = excluded.selling_price_sheet, - sales_rank = excluded.sales_rank, - sales_rank_avg_90d = excluded.sales_rank_avg_90d, - seller_count = excluded.seller_count, - amazon_is_seller = excluded.amazon_is_seller, - amazon_buybox_share_pct_90d = excluded.amazon_buybox_share_pct_90d, - monthly_sold = excluded.monthly_sold, - rank_drops_30d = excluded.rank_drops_30d, - rank_drops_90d = excluded.rank_drops_90d, - fba_fee = excluded.fba_fee, - fbm_fee = excluded.fbm_fee, - referral_percent = excluded.referral_percent, - can_sell = excluded.can_sell, - sellability_status = excluded.sellability_status, - sellability_reason = excluded.sellability_reason, - verdict = excluded.verdict, - confidence = excluded.confidence, - reasoning = excluded.reasoning, - fetched_at = excluded.fetched_at; - `); - - db.transaction((resultsBatch: AnalysisResult[]) => { - for (const r of resultsBatch) { - const price = - r.product.keepa?.currentPrice ?? - r.product.record.sellingPriceFromSheet ?? - r.product.spApi.estimatedSalePrice; - const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank; - - insertStmt.run( - r.product.record.asin, - runId, - r.product.record.name, - r.product.record.brand ?? null, - r.product.record.category ?? - r.product.keepa?.categoryTree?.join(" > ") ?? - null, - r.product.record.unitCost ?? null, - price ?? null, - r.product.keepa?.avgPrice90 ?? null, - r.product.record.avgPrice90FromSheet ?? null, - r.product.record.sellingPriceFromSheet ?? null, - rank ?? null, - r.product.keepa?.salesRankAvg90 ?? null, - r.product.keepa?.sellerCount ?? null, - r.product.keepa?.amazonIsSeller == null - ? null - : r.product.keepa.amazonIsSeller - ? 1 - : 0, - r.product.keepa?.amazonBuyboxSharePct90d ?? null, - r.product.keepa?.monthlySold ?? null, - r.product.keepa?.salesRankDrops30 ?? null, - r.product.keepa?.salesRankDrops90 ?? null, - r.product.spApi.fbaFee ?? null, - r.product.spApi.fbmFee ?? null, - r.product.spApi.referralFeePercent ?? null, - r.product.spApi.canSell == null - ? "unknown" - : r.product.spApi.canSell - ? "yes" - : "no", - r.product.spApi.sellabilityStatus ?? null, - r.product.spApi.sellabilityReason ?? null, - r.verdict.verdict, - r.verdict.confidence, - r.verdict.reasoning ?? null, - r.product.fetchedAt, - ); - } - })(results); // Execute the transaction with the results batch -} +): Promise { + if (results.length === 0) return; + await persistLlmResults(runId, results, { + source: "category_analysis", + metadataSource: "catalog", + }); +} function loadCategoryBlacklist(filePath: string): Set { const blacklist = new Set(); @@ -722,10 +586,14 @@ async function fetchCategoryBestSellerAsins( ]; for (const value of candidates) { - if (Array.isArray(value)) { - return [ - ...new Set(value.map((v) => String(v).trim()).filter(Boolean)), - ].slice(0, limit); + if (Array.isArray(value)) { + return [ + ...new Set( + value + .map((v) => normalizeAsin(v)) + .filter((asin): asin is string => asin !== null), + ), + ].slice(0, limit); } } @@ -977,10 +845,10 @@ async function fetchKeepaEnrichmentMap( `/product?key=${encodeURIComponent(config.keepaApiKey)}&domain=${DOMAIN_US}&asin=${asinParam}&stats=90&buybox=1&days=90`, ); - const products = Array.isArray(data?.products) ? data.products : []; - for (const product of products) { - const asin = String(product?.asin ?? "").trim(); - if (!asin) continue; + const products = Array.isArray(data?.products) ? data.products : []; + for (const product of products) { + const asin = normalizeAsin(product?.asin); + if (!asin) continue; out.set(asin, { keepa: parseKeepaProduct(product), title: String(product?.title ?? "").trim(), @@ -1067,7 +935,6 @@ function buildEnrichedProducts( } export async function processCategory( - db: Database, runId: number, category: CategoryInfo, perCategoryTop: number, @@ -1083,7 +950,7 @@ export async function processCategory( ); if (topAsins.length === 0) { log("info", " Keepa returned no ASINs for this category."); - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: 0, availableAsins: 0, fba: 0, @@ -1127,7 +994,7 @@ export async function processCategory( ` Sellable ASINs: ${availableAsins.length}/${uniqueTopAsins.length}`, ); if (availableAsins.length === 0) { - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: uniqueTopAsins.length, availableAsins: 0, fba: 0, @@ -1164,7 +1031,7 @@ export async function processCategory( ); if (selectedAsins.length === 0) { - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: uniqueTopAsins.length, availableAsins: 0, fba: 0, @@ -1231,7 +1098,7 @@ export async function processCategory( }, })); - await insertProductAnalysisResults(db, runId, batchResults); + await insertProductAnalysisResults(runId, batchResults); for (const result of batchResults) { results.push(result); @@ -1244,7 +1111,7 @@ export async function processCategory( } } - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: uniqueTopAsins.length, availableAsins: selectedAsins.length, fba, @@ -1264,7 +1131,7 @@ export async function processCategory( } } - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: uniqueTopAsins.length, availableAsins: selectedAsins.length, fba, @@ -1293,10 +1160,6 @@ export async function main(): Promise { assertSpApiPrerequisites(); mkdirSync(args.outputDir, { recursive: true }); - const DB_PATH = - process.env.RESULTS_DB_PATH || path.join(process.cwd(), "db", "results.db"); - initDb(DB_PATH); - const db = getDb(DB_PATH); log("info", "Starting per-category monthly-sold pipeline"); log("info", `Marketplace: ${config.spApiMarketplaceId}`); @@ -1333,7 +1196,6 @@ export async function main(): Promise { let runId: number | undefined; try { runId = await insertCategoryRunSummary( - db, { categoryId: category.id, categoryLabel: category.label, @@ -1350,7 +1212,6 @@ export async function main(): Promise { ); categorySummary = await processCategory( - db, runId, category, args.perCategoryTop, @@ -1382,7 +1243,7 @@ export async function main(): Promise { results: [], }; if (runId) { - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: 0, availableAsins: 0, fba: 0, diff --git a/src/check_db.ts b/src/check_db.ts index fa086c8..3b2636c 100644 --- a/src/check_db.ts +++ b/src/check_db.ts @@ -1,21 +1,16 @@ -import { getDb } from "./database.ts"; -import path from "node:path"; +import { db } from "./db/index.ts"; +import { runs } from "./db/schema.ts"; +import { eq } from "drizzle-orm"; async function checkDb() { - const DB_PATH = - process.env.RESULTS_DB_PATH || path.join(process.cwd(), "db", "results.db"); - const db = getDb(DB_PATH); - try { - const query = db.query( - "SELECT * FROM category_analysis_runs WHERE category_id = ?", - ); - const result = query.all(19419898011); + const result = await db + .select() + .from(runs) + .where(eq(runs.type, "category_analysis")); console.log(JSON.stringify(result, null, 2)); } catch (error) { console.error("Database query failed:", error); - } finally { - db.close(); } } diff --git a/src/database.ts b/src/database.ts deleted file mode 100644 index 8d4413f..0000000 --- a/src/database.ts +++ /dev/null @@ -1,494 +0,0 @@ -import { Database } from "bun:sqlite"; -import { dirname } from "node:path"; -import { mkdirSync } from "node:fs"; -export { Database } from "bun:sqlite"; - -let db: Database | null = null; - -export function getDb(dbPath: string): Database { - if (!db) { - const dbDir = dirname(dbPath); - if (dbDir && dbDir !== ".") { - mkdirSync(dbDir, { recursive: true }); - } - db = new Database(dbPath); - db.run("PRAGMA journal_mode = WAL;"); // Enable WAL mode for better performance - db.run("PRAGMA foreign_keys = ON;"); // Enforce foreign key constraints - } - return db; -} - -export function closeDb(): void { - if (db) { - db.close(); - db = null; - } -} - -function createProductAnalysisResultsTable(database: Database): void { - database.run(` - CREATE TABLE IF NOT EXISTS product_analysis_results ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - asin TEXT NOT NULL, - run_id INTEGER NOT NULL, - name TEXT NOT NULL, - brand TEXT, - category TEXT, - unit_cost REAL, - current_price REAL, - avg_price_90d REAL, - avg_price_90d_sheet REAL, - selling_price_sheet REAL, - sales_rank INTEGER, - sales_rank_avg_90d INTEGER, - seller_count INTEGER, - amazon_is_seller INTEGER, - amazon_buybox_share_pct_90d REAL, - monthly_sold INTEGER, - rank_drops_30d INTEGER, - rank_drops_90d INTEGER, - fba_fee REAL, - fbm_fee REAL, - referral_percent REAL, - can_sell TEXT, - sellability_status TEXT, - sellability_reason TEXT, - verdict TEXT NOT NULL, - confidence REAL NOT NULL, - reasoning TEXT, - fetched_at TEXT NOT NULL, - UNIQUE(asin), - FOREIGN KEY (run_id) REFERENCES category_analysis_runs(id) - ); - `); -} - -function ensureProductAnalysisResultsTable(database: Database): void { - const tableInfo = database - .query("PRAGMA table_info(product_analysis_results)") - .all() as Array<{ name: string; pk: number }>; - - if (tableInfo.length === 0) { - createProductAnalysisResultsTable(database); - return; - } - - const hasIdColumn = tableInfo.some((col) => col.name === "id"); - const hasAsinPrimaryKey = tableInfo.some( - (col) => col.name === "asin" && col.pk === 1, - ); - - const indexList = database - .query("PRAGMA index_list(product_analysis_results)") - .all() as Array<{ name: string; unique: number }>; - const hasUniqueAsinConstraint = indexList.some((idx) => { - if (idx.unique !== 1) return false; - const columns = database - .query(`PRAGMA index_info(${JSON.stringify(idx.name)})`) - .all() as Array<{ name: string }>; - return columns.length === 1 && columns[0]?.name === "asin"; - }); - - if (!hasIdColumn || hasAsinPrimaryKey || !hasUniqueAsinConstraint) { - database.run( - "ALTER TABLE product_analysis_results RENAME TO product_analysis_results_legacy", - ); - createProductAnalysisResultsTable(database); - database.run(` - WITH ranked AS ( - SELECT - asin, run_id, name, brand, category, unit_cost, - current_price, avg_price_90d, avg_price_90d_sheet, - selling_price_sheet, sales_rank, sales_rank_avg_90d, - seller_count, NULL AS amazon_is_seller, - NULL AS amazon_buybox_share_pct_90d, - monthly_sold, rank_drops_30d, rank_drops_90d, - fba_fee, fbm_fee, referral_percent, can_sell, - sellability_status, sellability_reason, - verdict, confidence, reasoning, fetched_at, - ROW_NUMBER() OVER ( - PARTITION BY asin - ORDER BY datetime(fetched_at) DESC, run_id DESC, id DESC - ) AS row_num - FROM product_analysis_results_legacy - ) - INSERT INTO product_analysis_results ( - asin, run_id, name, brand, category, unit_cost, - current_price, avg_price_90d, avg_price_90d_sheet, - selling_price_sheet, sales_rank, sales_rank_avg_90d, - seller_count, amazon_is_seller, amazon_buybox_share_pct_90d, - monthly_sold, rank_drops_30d, rank_drops_90d, - fba_fee, fbm_fee, referral_percent, can_sell, - sellability_status, sellability_reason, - verdict, confidence, reasoning, fetched_at - ) - SELECT - asin, run_id, name, brand, category, unit_cost, - current_price, avg_price_90d, avg_price_90d_sheet, - selling_price_sheet, sales_rank, sales_rank_avg_90d, - seller_count, amazon_is_seller, amazon_buybox_share_pct_90d, - monthly_sold, rank_drops_30d, rank_drops_90d, - fba_fee, fbm_fee, referral_percent, can_sell, - sellability_status, sellability_reason, - verdict, confidence, reasoning, fetched_at - FROM ranked - WHERE row_num = 1 - `); - database.run("DROP TABLE product_analysis_results_legacy"); - } -} - -function ensureProductAnalysisResultsColumns(database: Database): void { - const tableInfo = database - .query("PRAGMA table_info(product_analysis_results)") - .all() as Array<{ name: string }>; - - if (tableInfo.length === 0) { - return; - } - - const existingColumns = new Set(tableInfo.map((col) => col.name)); - const requiredColumns: Array<{ name: string; type: string }> = [ - { name: "amazon_is_seller", type: "INTEGER" }, - { name: "amazon_buybox_share_pct_90d", type: "REAL" }, - ]; - - for (const column of requiredColumns) { - if (!existingColumns.has(column.name)) { - database.run( - `ALTER TABLE product_analysis_results ADD COLUMN ${column.name} ${column.type}`, - ); - } - } -} - -function ensureResultsTableColumns(database: Database): void { - const tableInfo = database - .query("PRAGMA table_info(results)") - .all() as Array<{ name: string }>; - - if (tableInfo.length === 0) { - return; - } - - const existingColumns = new Set(tableInfo.map((col) => col.name)); - const requiredColumns: Array<{ name: string; type: string }> = [ - { name: "fba_net_sheet", type: "REAL" }, - { name: "gross_profit_dollar", type: "REAL" }, - { name: "gross_profit_pct", type: "REAL" }, - { name: "net_profit_sheet", type: "REAL" }, - { name: "roi_sheet", type: "REAL" }, - { name: "moq", type: "INTEGER" }, - { name: "moq_cost", type: "REAL" }, - { name: "qty_available", type: "INTEGER" }, - { name: "supplier", type: "TEXT" }, - { name: "source_url", type: "TEXT" }, - { name: "asin_link", type: "TEXT" }, - { name: "promo_coupon_code", type: "TEXT" }, - { name: "notes", type: "TEXT" }, - { name: "lead_date", type: "TEXT" }, - { name: "amazon_is_seller", type: "INTEGER" }, - { name: "amazon_buybox_share_pct_90d", type: "REAL" }, - { name: "upc", type: "TEXT" }, - { name: "supplier_score", type: "REAL" }, - { name: "supplier_profit", type: "REAL" }, - { name: "supplier_margin", type: "REAL" }, - { name: "supplier_roi", type: "REAL" }, - { name: "supplier_reason", type: "TEXT" }, - { name: "upc_lookup_status", type: "TEXT" }, - { name: "upc_lookup_reason", type: "TEXT" }, - { name: "candidate_asins", type: "TEXT" }, - ]; - - for (const column of requiredColumns) { - if (!existingColumns.has(column.name)) { - database.run( - `ALTER TABLE results ADD COLUMN ${column.name} ${column.type}`, - ); - } - } -} - -export function initDb(dbPath: string): void { - const database = getDb(dbPath); - database.run(` - CREATE TABLE IF NOT EXISTS runs ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - timestamp TEXT NOT NULL, - input_file TEXT NOT NULL, - output_file TEXT, - total_products INTEGER, - fba_count INTEGER, - fbm_count INTEGER, - skip_count INTEGER - ); - `); - database.run(` - CREATE TABLE IF NOT EXISTS results ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - run_id INTEGER NOT NULL, - asin TEXT NOT NULL, - product_name TEXT, - brand TEXT, - category TEXT, - unit_cost REAL, - current_price REAL, - avg_price_90d REAL, - avg_price_90d_sheet REAL, - selling_price_sheet REAL, - sales_rank INTEGER, - rank_avg_90d INTEGER, - sellers INTEGER, - amazon_is_seller INTEGER, - amazon_buybox_share_pct_90d REAL, - monthly_sold INTEGER, - rank_drops_30d INTEGER, - rank_drops_90d INTEGER, - fba_net_sheet REAL, - gross_profit_dollar REAL, - gross_profit_pct REAL, - net_profit_sheet REAL, - roi_sheet REAL, - moq INTEGER, - moq_cost REAL, - qty_available INTEGER, - supplier TEXT, - source_url TEXT, - asin_link TEXT, - promo_coupon_code TEXT, - notes TEXT, - lead_date TEXT, - upc TEXT, - fba_fee REAL, - fbm_fee REAL, - referral_percent REAL, - supplier_score REAL, - supplier_profit REAL, - supplier_margin REAL, - supplier_roi REAL, - supplier_reason TEXT, - upc_lookup_status TEXT, - upc_lookup_reason TEXT, - candidate_asins TEXT, - can_sell TEXT, - sellability_status TEXT, - sellability_reason TEXT, - verdict TEXT NOT NULL, - confidence INTEGER, - reasoning TEXT, - fetched_at TEXT NOT NULL, - FOREIGN KEY (run_id) REFERENCES runs(id) - ); - `); - ensureResultsTableColumns(database); - database.run(` - CREATE TABLE IF NOT EXISTS category_analysis_runs ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - category_id INTEGER NOT NULL, - category_label TEXT NOT NULL, - run_timestamp TEXT NOT NULL, - top_asins_checked INTEGER NOT NULL, - available_asins INTEGER NOT NULL, - fba_count INTEGER NOT NULL, - fbm_count INTEGER NOT NULL, - skip_count INTEGER NOT NULL, - status TEXT NOT NULL, - error_message TEXT - ); - `); - ensureProductAnalysisResultsTable(database); - ensureProductAnalysisResultsColumns(database); - - database.run( - `CREATE INDEX IF NOT EXISTS idx_runs_timestamp ON runs(timestamp DESC);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_results_run_id ON results(run_id);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_results_verdict ON results(verdict);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_results_sellability_status ON results(sellability_status);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_results_fetched_at ON results(fetched_at DESC);`, - ); - database.run(`CREATE INDEX IF NOT EXISTS idx_results_asin ON results(asin);`); - database.run( - `CREATE INDEX IF NOT EXISTS idx_category_runs_timestamp ON category_analysis_runs(run_timestamp DESC);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_category_runs_status ON category_analysis_runs(status);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_product_results_run_id ON product_analysis_results(run_id);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_product_results_verdict ON product_analysis_results(verdict);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_product_results_sellability_status ON product_analysis_results(sellability_status);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_product_results_fetched_at ON product_analysis_results(fetched_at DESC);`, - ); - initStalkerDb(database); -} - -export function initStalkerDb(database: Database): void { - resetLegacyStalkerSchema(database); - - database.run(` - CREATE TABLE IF NOT EXISTS stalker_runs ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - input_file TEXT NOT NULL, - started_at TEXT NOT NULL, - completed_at TEXT, - requested_asins INTEGER NOT NULL DEFAULT 0, - skipped_asins INTEGER NOT NULL DEFAULT 0, - scanned_asins INTEGER NOT NULL DEFAULT 0, - source_asins_with_matches INTEGER NOT NULL DEFAULT 0, - candidate_sellers INTEGER NOT NULL DEFAULT 0, - qualifying_sellers INTEGER NOT NULL DEFAULT 0, - matched_sellers INTEGER NOT NULL DEFAULT 0, - seller_metadata_requests INTEGER NOT NULL DEFAULT 0, - seller_storefront_requests INTEGER NOT NULL DEFAULT 0, - inventory_sellability_checked_asins INTEGER NOT NULL DEFAULT 0, - inventory_sellability_available_asins INTEGER NOT NULL DEFAULT 0, - inventory_sellability_excluded_asins INTEGER NOT NULL DEFAULT 0, - persisted_inventory_asins INTEGER NOT NULL DEFAULT 0, - status TEXT NOT NULL, - error_message TEXT - ); - `); - - database.run(` - CREATE TABLE IF NOT EXISTS stalker_asin_scans ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - run_id INTEGER NOT NULL, - source_asin TEXT NOT NULL, - title TEXT, - offer_count INTEGER NOT NULL DEFAULT 0, - candidate_seller_count INTEGER NOT NULL DEFAULT 0, - matched_seller_count INTEGER NOT NULL DEFAULT 0, - fetched_at TEXT NOT NULL, - raw_product_json TEXT, - UNIQUE(run_id, source_asin), - FOREIGN KEY (run_id) REFERENCES stalker_runs(id) ON DELETE CASCADE - ); - `); - - database.run(` - CREATE TABLE IF NOT EXISTS stalker_sellers ( - seller_id TEXT PRIMARY KEY, - seller_name TEXT, - rating REAL, - rating_count INTEGER, - storefront_asin_total INTEGER, - persisted_inventory_sample_count INTEGER, - last_updated_at TEXT NOT NULL, - raw_seller_json TEXT - ); - `); - - database.run(` - CREATE TABLE IF NOT EXISTS stalker_asin_sellers ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - scan_id INTEGER NOT NULL, - seller_id TEXT NOT NULL, - offer_price REAL, - condition TEXT, - is_fba INTEGER, - stock INTEGER, - seller_rating REAL, - seller_rating_count INTEGER, - raw_offer_json TEXT, - UNIQUE(scan_id, seller_id), - FOREIGN KEY (scan_id) REFERENCES stalker_asin_scans(id) ON DELETE CASCADE, - FOREIGN KEY (seller_id) REFERENCES stalker_sellers(seller_id) - ); - `); - - database.run(` - CREATE TABLE IF NOT EXISTS stalker_seller_inventory ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - run_id INTEGER NOT NULL, - seller_id TEXT NOT NULL, - asin TEXT NOT NULL, - can_sell INTEGER, - sellability_status TEXT, - sellability_reason TEXT, - product_title TEXT, - brand TEXT, - category_tree TEXT, - current_price REAL, - avg_price_90d REAL, - sales_rank INTEGER, - monthly_sold INTEGER, - seller_count INTEGER, - amazon_is_seller INTEGER, - raw_product_json TEXT, - last_seen_at TEXT NOT NULL, - raw_inventory_json TEXT, - UNIQUE(run_id, seller_id, asin), - FOREIGN KEY (run_id) REFERENCES stalker_runs(id) ON DELETE CASCADE, - FOREIGN KEY (seller_id) REFERENCES stalker_sellers(seller_id) - ); - `); - - database.run( - `CREATE INDEX IF NOT EXISTS idx_stalker_runs_started_at ON stalker_runs(started_at DESC);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_stalker_scans_run_id ON stalker_asin_scans(run_id);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_stalker_scans_source_asin ON stalker_asin_scans(source_asin);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_stalker_asin_sellers_seller_id ON stalker_asin_sellers(seller_id);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_stalker_inventory_seller_id ON stalker_seller_inventory(seller_id);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_stalker_inventory_asin ON stalker_seller_inventory(asin);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_stalker_inventory_product_title ON stalker_seller_inventory(product_title);`, - ); -} - -function resetLegacyStalkerSchema(database: Database): void { - const runColumns = database - .query("PRAGMA table_info(stalker_runs)") - .all() as Array<{ name: string }>; - if (runColumns.length === 0) return; - - const columnNames = new Set(runColumns.map((column) => column.name)); - if ( - columnNames.has("scanned_asins") && - columnNames.has("inventory_sellability_checked_asins") && - inventoryColumnsHaveSellability(database) - ) { - return; - } - - database.run("DROP TABLE IF EXISTS stalker_seller_inventory"); - database.run("DROP TABLE IF EXISTS stalker_asin_sellers"); - database.run("DROP TABLE IF EXISTS stalker_sellers"); - database.run("DROP TABLE IF EXISTS stalker_asin_scans"); - database.run("DROP TABLE IF EXISTS stalker_runs"); -} - -function inventoryColumnsHaveSellability(database: Database): boolean { - const inventoryColumns = database - .query("PRAGMA table_info(stalker_seller_inventory)") - .all() as Array<{ name: string }>; - const columnNames = new Set(inventoryColumns.map((column) => column.name)); - return ( - columnNames.has("sellability_status") && - columnNames.has("product_title") - ); -} diff --git a/src/db/index.ts b/src/db/index.ts new file mode 100644 index 0000000..f1b2e9a --- /dev/null +++ b/src/db/index.ts @@ -0,0 +1,15 @@ +import { drizzle } from "drizzle-orm/postgres-js"; +import postgres from "postgres"; +import * as schema from "./schema.ts"; + +const url = Bun.env.DB_CONNECTION_STRING; +if (!url) { + throw new Error("Missing required env var: DB_CONNECTION_STRING"); +} + +// Shared connection pool — imported once and reused across the process. +export const client = postgres(url); + +export const db = drizzle(client, { schema }); + +export type Db = typeof db; diff --git a/src/db/persistence.ts b/src/db/persistence.ts new file mode 100644 index 0000000..7d21b33 --- /dev/null +++ b/src/db/persistence.ts @@ -0,0 +1,541 @@ +import { sql } from "drizzle-orm"; +import { requireAsin, normalizeAsin } from "../asin.ts"; +import type { + AnalysisResult, + ProductRecord, + SupplierAnalysisResult, +} from "../types.ts"; +import { db } from "./index.ts"; +import { + analysisRevisions, + analysisRunStats, + categoryRunDetails, + productIdentifiers, + productObservations, + products, + runItems, + runs, + sourcingInputs, + supplierScores, + upcResolutionCandidates, + upcResolutions, +} from "./schema.ts"; + +type Executor = any; +type MetadataSource = "input" | "catalog"; + +type ProductSeed = { + asin: string; + name?: string | null; + brand?: string | null; + category?: string | null; + metadataSource?: MetadataSource; + fetchedAt?: Date; +}; + +export type CategoryRunSummaryInput = { + categoryId: number; + categoryLabel: string; + topAsinsChecked: number; + availableAsins: number; + fba: number; + fbm: number; + skip: number; + status: "running" | "ok" | "empty" | "failed"; + error: string; +}; + +export type RunCounts = { + totalProducts: number; + fbaCount: number; + fbmCount: number; + skipCount: number; +}; + +function emptyToNull(value: string | undefined | null): string | null { + const trimmed = value?.trim(); + return trimmed ? trimmed : null; +} + +function productCategory(record: ProductRecord, result: AnalysisResult): string | null { + return emptyToNull( + record.category ?? result.product.keepa?.categoryTree?.join(" > "), + ); +} + +export async function upsertProduct( + seed: ProductSeed, + executor: Executor = db, +): Promise { + const asin = requireAsin(seed.asin); + const now = seed.fetchedAt ?? new Date(); + const isCatalog = seed.metadataSource === "catalog"; + + await executor + .insert(products) + .values({ + asin, + name: emptyToNull(seed.name), + brand: emptyToNull(seed.brand), + category: emptyToNull(seed.category), + metadataFetchedAt: isCatalog ? now : null, + firstSeenAt: now, + lastSeenAt: now, + }) + .onConflictDoUpdate({ + target: products.asin, + set: { + lastSeenAt: sql`GREATEST(${products.lastSeenAt}, EXCLUDED.last_seen_at)`, + name: isCatalog + ? sql`CASE WHEN EXCLUDED.metadata_fetched_at >= COALESCE(${products.metadataFetchedAt}, '-infinity'::timestamptz) AND NULLIF(EXCLUDED.name, '') IS NOT NULL THEN EXCLUDED.name ELSE ${products.name} END` + : sql`COALESCE(${products.name}, NULLIF(EXCLUDED.name, ''))`, + brand: isCatalog + ? sql`CASE WHEN EXCLUDED.metadata_fetched_at >= COALESCE(${products.metadataFetchedAt}, '-infinity'::timestamptz) AND NULLIF(EXCLUDED.brand, '') IS NOT NULL THEN EXCLUDED.brand ELSE ${products.brand} END` + : sql`COALESCE(${products.brand}, NULLIF(EXCLUDED.brand, ''))`, + category: isCatalog + ? sql`CASE WHEN EXCLUDED.metadata_fetched_at >= COALESCE(${products.metadataFetchedAt}, '-infinity'::timestamptz) AND NULLIF(EXCLUDED.category, '') IS NOT NULL THEN EXCLUDED.category ELSE ${products.category} END` + : sql`COALESCE(${products.category}, NULLIF(EXCLUDED.category, ''))`, + metadataFetchedAt: isCatalog + ? sql`GREATEST(COALESCE(${products.metadataFetchedAt}, '-infinity'::timestamptz), EXCLUDED.metadata_fetched_at)` + : products.metadataFetchedAt, + }, + }); + + return asin; +} + +export async function insertObservation( + runId: number, + result: AnalysisResult, + source: string, + executor: Executor = db, +): Promise { + const fetchedAt = new Date(result.product.fetchedAt); + const record = result.product.record; + const keepa = result.product.keepa; + const spApi = result.product.spApi; + const asin = requireAsin(record.asin); + const [observation] = await executor + .insert(productObservations) + .values({ + productAsin: asin, + runId, + source, + currentPrice: + keepa?.currentPrice ?? + record.sellingPriceFromSheet ?? + spApi.estimatedSalePrice ?? + null, + avgPrice90d: keepa?.avgPrice90 ?? null, + salesRank: keepa?.salesRank ?? record.amazonRank ?? null, + salesRankAvg90d: keepa?.salesRankAvg90 ?? null, + monthlySold: keepa?.monthlySold ?? null, + rankDrops30d: keepa?.salesRankDrops30 ?? null, + rankDrops90d: keepa?.salesRankDrops90 ?? null, + sellerCount: keepa?.sellerCount ?? null, + amazonIsSeller: keepa?.amazonIsSeller ?? null, + amazonBuyboxSharePct90d: keepa?.amazonBuyboxSharePct90d ?? null, + fbaFee: spApi.fbaFee ?? null, + fbmFee: spApi.fbmFee ?? null, + referralPercent: spApi.referralFeePercent ?? null, + canSell: spApi.canSell, + sellabilityStatus: spApi.sellabilityStatus, + sellabilityReason: spApi.sellabilityReason ?? null, + fetchedAt, + }) + .returning({ id: productObservations.id }); + + if (!observation) throw new Error(`Failed to insert observation for ${asin}`); + return observation.id; +} + +function sourcingInputValues(runItemId: number, record: ProductRecord) { + return { + runItemId, + suppliedName: emptyToNull(record.name), + suppliedBrand: emptyToNull(record.brand), + suppliedCategory: emptyToNull(record.category), + unitCost: record.unitCost ?? null, + avgPrice90dSheet: record.avgPrice90FromSheet ?? null, + sellingPriceSheet: record.sellingPriceFromSheet ?? null, + fbaNetSheet: record.fbaNet ?? null, + grossProfitDollar: record.grossProfit ?? null, + grossProfitPct: record.grossProfitPct ?? null, + netProfitSheet: record.netProfitFromSheet ?? null, + roiSheet: record.roiFromSheet ?? null, + moq: record.moq ?? null, + moqCost: record.moqCost ?? null, + qtyAvailable: record.totalQtyAvail ?? null, + supplier: emptyToNull(record.supplier), + sourceUrl: emptyToNull(record.sourceUrl), + asinLink: emptyToNull(record.asinLink), + promoCouponCode: emptyToNull(record.promoCouponCode), + notes: emptyToNull(record.notes), + leadDate: emptyToNull(record.leadDate), + }; +} + +export async function persistLlmResults( + runId: number, + results: AnalysisResult[], + options: { + source: string; + metadataSource?: MetadataSource; + preserveSourcingInput?: boolean; + sourceInventoryIds?: Map; + }, +): Promise { + for (const result of results) { + const record = result.product.record; + const fetchedAt = new Date(result.product.fetchedAt); + const asin = await upsertProduct({ + asin: record.asin, + name: record.name, + brand: record.brand, + category: productCategory(record, result), + metadataSource: options.metadataSource ?? "input", + fetchedAt, + }); + const [item] = await db + .insert(runItems) + .values({ + runId, + productAsin: asin, + sourceInventoryItemId: options.sourceInventoryIds?.get(asin) ?? null, + }) + .returning({ id: runItems.id }); + if (!item) throw new Error(`Failed to insert run item for ${asin}`); + + if (options.preserveSourcingInput) { + await db.insert(sourcingInputs).values(sourcingInputValues(item.id, record)); + } + + const observationId = await insertObservation(runId, result, options.source); + await db.insert(analysisRevisions).values({ + runItemId: item.id, + observationId, + method: "llm", + decision: result.verdict.verdict, + confidence: result.verdict.confidence, + reasoning: result.verdict.reasoning ?? null, + analyzedAt: fetchedAt, + }); + } +} + +function supplierSourcingValues(runItemId: number, result: SupplierAnalysisResult) { + return { + runItemId, + suppliedName: emptyToNull(result.record.name), + suppliedBrand: emptyToNull(result.record.brand), + suppliedCategory: emptyToNull(result.record.category), + unitCost: result.record.unitCost ?? null, + }; +} + +async function insertSupplierObservation( + runId: number, + productAsin: string, + result: SupplierAnalysisResult, +): Promise { + const keepa = result.keepa; + const spApi = result.spApi; + if (!spApi && !keepa) return null; + + const [row] = await db + .insert(productObservations) + .values({ + productAsin, + runId, + source: "supplier_upc", + currentPrice: result.score.salePrice, + avgPrice90d: keepa?.avgPrice90 ?? null, + salesRank: keepa?.salesRank ?? null, + salesRankAvg90d: keepa?.salesRankAvg90 ?? null, + monthlySold: keepa?.monthlySold ?? null, + rankDrops30d: keepa?.salesRankDrops30 ?? null, + rankDrops90d: keepa?.salesRankDrops90 ?? null, + sellerCount: keepa?.sellerCount ?? null, + amazonIsSeller: keepa?.amazonIsSeller ?? null, + amazonBuyboxSharePct90d: keepa?.amazonBuyboxSharePct90d ?? null, + fbaFee: spApi?.fbaFee ?? null, + fbmFee: spApi?.fbmFee ?? null, + referralPercent: spApi?.referralFeePercent ?? null, + canSell: spApi?.canSell ?? null, + sellabilityStatus: spApi?.sellabilityStatus ?? null, + sellabilityReason: spApi?.sellabilityReason ?? null, + fetchedAt: new Date(result.fetchedAt), + }) + .returning({ id: productObservations.id }); + return row?.id ?? null; +} + +export async function persistSupplierResults( + runId: number, + results: SupplierAnalysisResult[], +): Promise { + for (const result of results) { + const resolvedAsin = normalizeAsin(result.lookup.asin); + if (resolvedAsin) { + await upsertProduct({ + asin: resolvedAsin, + name: result.record.name, + brand: result.record.brand, + category: result.record.category, + metadataSource: "input", + fetchedAt: new Date(result.fetchedAt), + }); + if (result.keepa?.categoryTree?.length) { + await upsertProduct({ + asin: resolvedAsin, + category: result.keepa.categoryTree.join(" > "), + metadataSource: "catalog", + fetchedAt: new Date(result.fetchedAt), + }); + } + } + + const [item] = await db + .insert(runItems) + .values({ + runId, + productAsin: resolvedAsin, + sourceRow: result.rowNumber ?? null, + }) + .returning({ id: runItems.id }); + if (!item) throw new Error("Failed to insert supplier run item"); + + await db.insert(sourcingInputs).values(supplierSourcingValues(item.id, result)); + await db.insert(upcResolutions).values({ + runItemId: item.id, + requestedUpc: result.upc, + normalizedUpc: result.lookup.normalizedUpc, + provider: result.lookup.provider ?? "unknown", + status: result.lookup.status, + reason: result.lookup.reason ?? null, + resolvedProductAsin: resolvedAsin, + resolvedAt: new Date(result.fetchedAt), + }); + + for (const candidate of result.lookup.candidateAsins) { + const candidateAsin = normalizeAsin(candidate); + if (!candidateAsin) continue; + await upsertProduct({ asin: candidateAsin, fetchedAt: new Date(result.fetchedAt) }); + await db + .insert(upcResolutionCandidates) + .values({ runItemId: item.id, productAsin: candidateAsin }) + .onConflictDoUpdate({ + target: [ + upcResolutionCandidates.runItemId, + upcResolutionCandidates.productAsin, + ], + set: { productAsin: sql`EXCLUDED.product_asin` }, + }); + } + + if (resolvedAsin) { + await db + .insert(productIdentifiers) + .values({ + productAsin: resolvedAsin, + identifierType: + result.lookup.normalizedUpc.length === 12 + ? "upc" + : result.lookup.normalizedUpc.length === 13 + ? "ean" + : "gtin", + identifierValue: result.lookup.normalizedUpc, + source: "supplier_upc", + confirmedAt: new Date(result.fetchedAt), + }) + .onConflictDoUpdate({ + target: [ + productIdentifiers.identifierType, + productIdentifiers.identifierValue, + ], + set: { + productAsin: resolvedAsin, + source: "supplier_upc", + confirmedAt: new Date(result.fetchedAt), + }, + }); + } + + const observationId = resolvedAsin + ? await insertSupplierObservation(runId, resolvedAsin, result) + : null; + const [revision] = await db + .insert(analysisRevisions) + .values({ + runItemId: item.id, + observationId, + method: "supplier_scoring", + decision: result.score.verdict, + confidence: result.score.score, + reasoning: result.score.reason, + analyzedAt: new Date(result.fetchedAt), + }) + .returning({ id: analysisRevisions.id }); + if (!revision) throw new Error("Failed to insert supplier analysis revision"); + + await db.insert(supplierScores).values({ + revisionId: revision.id, + score: result.score.score, + salePrice: result.score.salePrice, + fbaFee: result.score.fbaFee, + profit: result.score.profit, + margin: result.score.margin, + roi: result.score.roi, + reason: result.score.reason, + }); + } +} + +export async function createCategoryRun( + summary: CategoryRunSummaryInput, + runTimestamp: string, +): Promise { + const [row] = await db + .insert(runs) + .values({ + type: "category_analysis", + status: summary.status, + errorMessage: summary.error || null, + startedAt: new Date(runTimestamp), + }) + .returning({ id: runs.id }); + if (!row) throw new Error("Failed to insert category run."); + + await db.insert(categoryRunDetails).values({ + runId: row.id, + categoryId: summary.categoryId, + categoryLabel: summary.categoryLabel, + checkedAsinCount: summary.topAsinsChecked, + }); + await db.insert(analysisRunStats).values({ + runId: row.id, + processedCount: summary.topAsinsChecked, + availableCount: summary.availableAsins, + analyzedCount: summary.fba + summary.fbm + summary.skip, + fbaCount: summary.fba, + fbmCount: summary.fbm, + skipCount: summary.skip, + }); + return row.id; +} + +export async function updateCategoryRun( + runId: number, + summary: Pick< + CategoryRunSummaryInput, + | "topAsinsChecked" + | "availableAsins" + | "fba" + | "fbm" + | "skip" + | "status" + | "error" + >, +): Promise { + await db + .update(runs) + .set({ + status: summary.status, + errorMessage: summary.error || null, + ...(summary.status !== "running" ? { completedAt: new Date() } : {}), + }) + .where(sql`${runs.id} = ${runId}`); + await db + .insert(categoryRunDetails) + .values({ + runId, + categoryId: 0, + categoryLabel: "", + checkedAsinCount: summary.topAsinsChecked, + }) + .onConflictDoUpdate({ + target: categoryRunDetails.runId, + set: { checkedAsinCount: summary.topAsinsChecked }, + }); + await db + .insert(analysisRunStats) + .values({ + runId, + processedCount: summary.topAsinsChecked, + availableCount: summary.availableAsins, + analyzedCount: summary.fba + summary.fbm + summary.skip, + fbaCount: summary.fba, + fbmCount: summary.fbm, + skipCount: summary.skip, + }) + .onConflictDoUpdate({ + target: analysisRunStats.runId, + set: { + processedCount: summary.topAsinsChecked, + availableCount: summary.availableAsins, + analyzedCount: summary.fba + summary.fbm + summary.skip, + fbaCount: summary.fba, + fbmCount: summary.fbm, + skipCount: summary.skip, + }, + }); +} + +export async function refreshRunStats(runId: number): Promise { + const [stats] = await db.execute( + sql<{ + total: string; + fba: string | null; + fbm: string | null; + buy: string | null; + watch: string | null; + skip: string | null; + }>`WITH latest AS ( + SELECT DISTINCT ON (ri.id) ar.decision + FROM run_items ri + JOIN analysis_revisions ar ON ar.run_item_id = ri.id + WHERE ri.run_id = ${runId} + ORDER BY ri.id, ar.analyzed_at DESC, ar.id DESC + ) + SELECT + COUNT(*) AS total, + SUM(CASE WHEN decision = 'FBA' THEN 1 ELSE 0 END) AS fba, + SUM(CASE WHEN decision = 'FBM' THEN 1 ELSE 0 END) AS fbm, + SUM(CASE WHEN decision = 'BUY' THEN 1 ELSE 0 END) AS buy, + SUM(CASE WHEN decision = 'WATCH' THEN 1 ELSE 0 END) AS watch, + SUM(CASE WHEN decision = 'SKIP' THEN 1 ELSE 0 END) AS skip + FROM latest`, + ); + + const counts = { + totalProducts: Number(stats?.total ?? 0), + fbaCount: Number(stats?.fba ?? 0), + fbmCount: Number(stats?.fbm ?? 0), + skipCount: Number(stats?.skip ?? 0), + }; + await db + .insert(analysisRunStats) + .values({ + runId, + processedCount: counts.totalProducts, + analyzedCount: counts.totalProducts, + fbaCount: counts.fbaCount, + fbmCount: counts.fbmCount, + buyCount: Number(stats?.buy ?? 0), + watchCount: Number(stats?.watch ?? 0), + skipCount: counts.skipCount, + }) + .onConflictDoUpdate({ + target: analysisRunStats.runId, + set: { + processedCount: counts.totalProducts, + analyzedCount: counts.totalProducts, + fbaCount: counts.fbaCount, + fbmCount: counts.fbmCount, + buyCount: Number(stats?.buy ?? 0), + watchCount: Number(stats?.watch ?? 0), + skipCount: counts.skipCount, + }, + }); + return counts; +} diff --git a/src/db/schema.ts b/src/db/schema.ts new file mode 100644 index 0000000..6b67dfd --- /dev/null +++ b/src/db/schema.ts @@ -0,0 +1,441 @@ +import { sql } from "drizzle-orm"; +import { + type AnyPgColumn, + boolean, + check, + index, + integer, + pgEnum, + pgTable, + primaryKey, + real, + serial, + text, + timestamp, + unique, +} from "drizzle-orm/pg-core"; + +export const runTypeEnum = pgEnum("run_type", [ + "lead_analysis", + "category_analysis", + "supplier_upc", + "stalker", + "stalker_analysis", +]); + +export const runStatusEnum = pgEnum("run_status", [ + "running", + "ok", + "empty", + "failed", + "completed", +]); + +export const analysisMethodEnum = pgEnum("analysis_method", [ + "llm", + "supplier_scoring", +]); + +export const analysisDecisionEnum = pgEnum("analysis_decision", [ + "FBA", + "FBM", + "BUY", + "WATCH", + "SKIP", +]); + +export const products = pgTable( + "products", + { + asin: text("asin").primaryKey(), + name: text("name"), + brand: text("brand"), + category: text("category"), + metadataFetchedAt: timestamp("metadata_fetched_at", { withTimezone: true }), + firstSeenAt: timestamp("first_seen_at", { withTimezone: true }) + .notNull() + .defaultNow(), + lastSeenAt: timestamp("last_seen_at", { withTimezone: true }) + .notNull() + .defaultNow(), + }, + (t) => [ + check("ck_products_asin", sql`${t.asin} ~ '^[A-Z0-9]{10}$'`), + index("idx_products_name").on(t.name), + index("idx_products_last_seen_at").on(t.lastSeenAt), + ], +); + +export const runs = pgTable( + "runs", + { + id: serial("id").primaryKey(), + type: runTypeEnum("type").notNull(), + parentRunId: integer("parent_run_id").references( + (): AnyPgColumn => runs.id, + { onDelete: "cascade" }, + ), + inputFile: text("input_file"), + outputFile: text("output_file"), + status: runStatusEnum("status").notNull().default("running"), + errorMessage: text("error_message"), + startedAt: timestamp("started_at", { withTimezone: true }) + .notNull() + .defaultNow(), + completedAt: timestamp("completed_at", { withTimezone: true }), + }, + (t) => [ + index("idx_runs_started_at").on(t.startedAt), + index("idx_runs_type").on(t.type), + index("idx_runs_status").on(t.status), + index("idx_runs_parent_run_id").on(t.parentRunId), + ], +); + +export const analysisRunStats = pgTable("analysis_run_stats", { + runId: integer("run_id") + .primaryKey() + .references(() => runs.id, { onDelete: "cascade" }), + processedCount: integer("processed_count").notNull().default(0), + analyzedCount: integer("analyzed_count").notNull().default(0), + availableCount: integer("available_count").notNull().default(0), + fbaCount: integer("fba_count").notNull().default(0), + fbmCount: integer("fbm_count").notNull().default(0), + buyCount: integer("buy_count").notNull().default(0), + watchCount: integer("watch_count").notNull().default(0), + skipCount: integer("skip_count").notNull().default(0), +}); + +export const categoryRunDetails = pgTable("category_run_details", { + runId: integer("run_id") + .primaryKey() + .references(() => runs.id, { onDelete: "cascade" }), + categoryId: integer("category_id").notNull(), + categoryLabel: text("category_label").notNull(), + checkedAsinCount: integer("checked_asin_count").notNull().default(0), + selectionParametersJson: text("selection_parameters_json"), +}); + +export const stalkerRunDetails = pgTable("stalker_run_details", { + runId: integer("run_id") + .primaryKey() + .references(() => runs.id, { onDelete: "cascade" }), + requestedAsins: integer("requested_asins").notNull().default(0), + skippedAsins: integer("skipped_asins").notNull().default(0), + scannedAsins: integer("scanned_asins").notNull().default(0), + sourceAsinsWithMatches: integer("source_asins_with_matches") + .notNull() + .default(0), + candidateSellers: integer("candidate_sellers").notNull().default(0), + qualifyingSellers: integer("qualifying_sellers").notNull().default(0), + matchedSellers: integer("matched_sellers").notNull().default(0), + sellerMetadataRequests: integer("seller_metadata_requests") + .notNull() + .default(0), + sellerStorefrontRequests: integer("seller_storefront_requests") + .notNull() + .default(0), + inventorySellabilityCheckedAsins: integer( + "inventory_sellability_checked_asins", + ) + .notNull() + .default(0), + inventorySellabilityAvailableAsins: integer( + "inventory_sellability_available_asins", + ) + .notNull() + .default(0), + inventorySellabilityExcludedAsins: integer( + "inventory_sellability_excluded_asins", + ) + .notNull() + .default(0), + persistedInventoryAsins: integer("persisted_inventory_asins") + .notNull() + .default(0), +}); + +export const productIdentifiers = pgTable( + "product_identifiers", + { + id: serial("id").primaryKey(), + productAsin: text("product_asin") + .notNull() + .references(() => products.asin), + identifierType: text("identifier_type").notNull(), + identifierValue: text("identifier_value").notNull(), + source: text("source").notNull(), + confirmedAt: timestamp("confirmed_at", { withTimezone: true }) + .notNull() + .defaultNow(), + }, + (t) => [ + unique("uq_product_identifier_type_value").on( + t.identifierType, + t.identifierValue, + ), + index("idx_product_identifiers_asin").on(t.productAsin), + ], +); + +export const productObservations = pgTable( + "product_observations", + { + id: serial("id").primaryKey(), + productAsin: text("product_asin") + .notNull() + .references(() => products.asin), + runId: integer("run_id") + .notNull() + .references(() => runs.id, { onDelete: "cascade" }), + source: text("source").notNull(), + marketplace: text("marketplace").notNull().default("US"), + currentPrice: real("current_price"), + avgPrice90d: real("avg_price_90d"), + salesRank: integer("sales_rank"), + salesRankAvg90d: integer("sales_rank_avg_90d"), + monthlySold: integer("monthly_sold"), + rankDrops30d: integer("rank_drops_30d"), + rankDrops90d: integer("rank_drops_90d"), + sellerCount: integer("seller_count"), + amazonIsSeller: boolean("amazon_is_seller"), + amazonBuyboxSharePct90d: real("amazon_buybox_share_pct_90d"), + fbaFee: real("fba_fee"), + fbmFee: real("fbm_fee"), + referralPercent: real("referral_percent"), + canSell: boolean("can_sell"), + sellabilityStatus: text("sellability_status"), + sellabilityReason: text("sellability_reason"), + rawProductJson: text("raw_product_json"), + fetchedAt: timestamp("fetched_at", { withTimezone: true }).notNull(), + }, + (t) => [ + index("idx_product_observations_product_time").on( + t.productAsin, + t.fetchedAt.desc(), + ), + index("idx_product_observations_run_id").on(t.runId), + index("idx_product_observations_sellability").on(t.sellabilityStatus), + ], +); + +export const runItems = pgTable( + "run_items", + { + id: serial("id").primaryKey(), + runId: integer("run_id") + .notNull() + .references(() => runs.id, { onDelete: "cascade" }), + productAsin: text("product_asin").references(() => products.asin), + sourceInventoryItemId: integer("source_inventory_item_id").references( + (): AnyPgColumn => stalkerInventoryItems.id, + { onDelete: "set null" }, + ), + ordinal: integer("ordinal"), + sourceRow: integer("source_row"), + status: text("status").notNull().default("completed"), + createdAt: timestamp("created_at", { withTimezone: true }) + .notNull() + .defaultNow(), + }, + (t) => [ + index("idx_run_items_run_id").on(t.runId), + index("idx_run_items_product_asin").on(t.productAsin), + ], +); + +export const sourcingInputs = pgTable("sourcing_inputs", { + runItemId: integer("run_item_id") + .primaryKey() + .references(() => runItems.id, { onDelete: "cascade" }), + suppliedName: text("supplied_name"), + suppliedBrand: text("supplied_brand"), + suppliedCategory: text("supplied_category"), + unitCost: real("unit_cost"), + avgPrice90dSheet: real("avg_price_90d_sheet"), + sellingPriceSheet: real("selling_price_sheet"), + fbaNetSheet: real("fba_net_sheet"), + grossProfitDollar: real("gross_profit_dollar"), + grossProfitPct: real("gross_profit_pct"), + netProfitSheet: real("net_profit_sheet"), + roiSheet: real("roi_sheet"), + moq: integer("moq"), + moqCost: real("moq_cost"), + qtyAvailable: integer("qty_available"), + supplier: text("supplier"), + sourceUrl: text("source_url"), + asinLink: text("asin_link"), + promoCouponCode: text("promo_coupon_code"), + notes: text("notes"), + leadDate: text("lead_date"), +}); + +export const upcResolutions = pgTable( + "upc_resolutions", + { + runItemId: integer("run_item_id") + .primaryKey() + .references(() => runItems.id, { onDelete: "cascade" }), + requestedUpc: text("requested_upc").notNull(), + normalizedUpc: text("normalized_upc").notNull(), + provider: text("provider").notNull(), + status: text("status").notNull(), + reason: text("reason"), + resolvedProductAsin: text("resolved_product_asin").references( + () => products.asin, + ), + resolvedAt: timestamp("resolved_at", { withTimezone: true }) + .notNull() + .defaultNow(), + }, + (t) => [index("idx_upc_resolutions_normalized_upc").on(t.normalizedUpc)], +); + +export const upcResolutionCandidates = pgTable( + "upc_resolution_candidates", + { + runItemId: integer("run_item_id") + .notNull() + .references(() => upcResolutions.runItemId, { onDelete: "cascade" }), + productAsin: text("product_asin") + .notNull() + .references(() => products.asin), + }, + (t) => [ + primaryKey({ columns: [t.runItemId, t.productAsin] }), + index("idx_upc_candidates_product_asin").on(t.productAsin), + ], +); + +export const analysisRevisions = pgTable( + "analysis_revisions", + { + id: serial("id").primaryKey(), + runItemId: integer("run_item_id") + .notNull() + .references(() => runItems.id, { onDelete: "cascade" }), + observationId: integer("observation_id").references( + () => productObservations.id, + { onDelete: "set null" }, + ), + method: analysisMethodEnum("method").notNull(), + decision: analysisDecisionEnum("decision").notNull(), + confidence: real("confidence"), + reasoning: text("reasoning"), + analyzedAt: timestamp("analyzed_at", { withTimezone: true }) + .notNull() + .defaultNow(), + }, + (t) => [ + index("idx_analysis_revisions_run_item_time").on(t.runItemId, t.analyzedAt), + index("idx_analysis_revisions_decision").on(t.decision), + ], +); + +export const supplierScores = pgTable("supplier_scores", { + revisionId: integer("revision_id") + .primaryKey() + .references(() => analysisRevisions.id, { onDelete: "cascade" }), + score: real("score"), + salePrice: real("sale_price"), + fbaFee: real("fba_fee"), + profit: real("profit"), + margin: real("margin"), + roi: real("roi"), + reason: text("reason"), +}); + +export const sellers = pgTable("sellers", { + sellerId: text("seller_id").primaryKey(), + sellerName: text("seller_name"), + rating: real("rating"), + ratingCount: integer("rating_count"), + storefrontAsinTotal: integer("storefront_asin_total"), + persistedInventorySampleCount: integer("persisted_inventory_sample_count"), + lastUpdatedAt: timestamp("last_updated_at", { withTimezone: true }).notNull(), + rawSellerJson: text("raw_seller_json"), +}); + +export const stalkerScans = pgTable( + "stalker_scans", + { + id: serial("id").primaryKey(), + runId: integer("run_id") + .notNull() + .references(() => runs.id, { onDelete: "cascade" }), + sourceProductAsin: text("source_product_asin") + .notNull() + .references(() => products.asin), + observationId: integer("observation_id").references( + () => productObservations.id, + { onDelete: "set null" }, + ), + offerCount: integer("offer_count").notNull().default(0), + candidateSellerCount: integer("candidate_seller_count") + .notNull() + .default(0), + matchedSellerCount: integer("matched_seller_count").notNull().default(0), + fetchedAt: timestamp("fetched_at", { withTimezone: true }).notNull(), + }, + (t) => [ + unique("uq_stalker_scans_run_source_product").on( + t.runId, + t.sourceProductAsin, + ), + index("idx_stalker_scans_run_id").on(t.runId), + index("idx_stalker_scans_source_asin").on(t.sourceProductAsin), + ], +); + +export const stalkerScanSellers = pgTable( + "stalker_scan_sellers", + { + id: serial("id").primaryKey(), + scanId: integer("scan_id") + .notNull() + .references(() => stalkerScans.id, { onDelete: "cascade" }), + sellerId: text("seller_id") + .notNull() + .references(() => sellers.sellerId), + offerPrice: real("offer_price"), + condition: text("condition"), + isFba: boolean("is_fba"), + stock: integer("stock"), + sellerRating: real("seller_rating"), + sellerRatingCount: integer("seller_rating_count"), + rawOfferJson: text("raw_offer_json"), + }, + (t) => [ + unique("uq_stalker_scan_sellers_scan_seller").on(t.scanId, t.sellerId), + ], +); + +export const stalkerInventoryItems = pgTable( + "stalker_inventory_items", + { + id: serial("id").primaryKey(), + runId: integer("run_id") + .notNull() + .references(() => runs.id, { onDelete: "cascade" }), + sellerId: text("seller_id") + .notNull() + .references(() => sellers.sellerId), + productAsin: text("product_asin") + .notNull() + .references(() => products.asin), + observationId: integer("observation_id") + .notNull() + .references(() => productObservations.id, { onDelete: "cascade" }), + lastSeenAt: timestamp("last_seen_at", { withTimezone: true }).notNull(), + rawInventoryJson: text("raw_inventory_json"), + }, + (t) => [ + unique("uq_stalker_inventory_items_run_seller_asin").on( + t.runId, + t.sellerId, + t.productAsin, + ), + index("idx_stalker_inventory_seller_id").on(t.sellerId), + index("idx_stalker_inventory_product_asin").on(t.productAsin), + ], +); diff --git a/src/index.ts b/src/index.ts index ec197cc..7d89f54 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,11 +1,10 @@ import { readProducts } from "./reader.ts"; -import { connectCache, disconnectCache } from "./cache.ts"; +import { connectCache, disconnectCache } from "./integrations/cache.ts"; import { printResults, writeResultsToDb, writeResultsWorkbook, } from "./writer.ts"; -import { initDb, closeDb } from "./database.ts"; import { chunkArray, processProductChunk, @@ -14,8 +13,9 @@ import { import path from "node:path"; import type { AnalysisResult } from "./types.ts"; -const DB_PATH = process.env.RESULTS_DB_PATH || path.join("db", "results.db"); const INPUT_BATCH_SIZE = 50; +const INPUT_DIR = "input"; +const OUTPUT_DIR = "output"; function parseSellabilityArg(args: string[]): SellabilityFilter { const sellabilityArg = args.find((a) => a.startsWith("--sellability=")); @@ -47,7 +47,7 @@ function parseArgs(): { const args = process.argv.slice(2); const outputFile = readFlagValue(args, "--out", "--output"); const useClaude = args.includes("--claude"); - const inputFile = readInputFileArg( + const inputFileArg = readInputFileArg( args, "--out", "--output", @@ -55,14 +55,19 @@ function parseArgs(): { ); const sellability = parseSellabilityArg(args); - if (!inputFile) { + if (!inputFileArg) { console.error( - "Usage: bun run src/index.ts [--out results.xlsx|--output results.xlsx] [--sellability available|all] [--claude]", + "Usage: bun run src/index.ts [--out results.xlsx|--output results.xlsx] [--sellability available|all] [--claude]\nBare filenames are read from input/ and written to output/.", ); process.exit(1); } - return { inputFile, outputFile, sellability, useClaude }; + return { + inputFile: resolveInputPath(inputFileArg), + outputFile, + sellability, + useClaude, + }; } function readFlagValue(args: string[], ...flags: string[]): string | undefined { @@ -103,11 +108,25 @@ function readInputFileArg( return undefined; } +function isBareFilename(filePath: string): boolean { + return !path.isAbsolute(filePath) && !/[\\/]/.test(filePath); +} + +function resolveInputPath(inputFile: string): string { + return isBareFilename(inputFile) + ? path.join(INPUT_DIR, inputFile) + : inputFile; +} + function resolveBaseOutputPath(inputFile: string, outputFile?: string): string { - if (outputFile) return outputFile; + if (outputFile) { + return isBareFilename(outputFile) + ? path.join(OUTPUT_DIR, outputFile) + : outputFile; + } const parsedInput = path.parse(inputFile); - return path.join("output", `${parsedInput.name}_results.xlsx`); + return path.join(OUTPUT_DIR, `${parsedInput.name}_results.xlsx`); } async function main() { @@ -119,9 +138,6 @@ async function main() { console.log("Connecting to Redis..."); await connectCache(); - console.log("Initializing SQLite database..."); - initDb(DB_PATH); - try { console.log(`\nReading ${inputFile}...`); const products = readProducts(inputFile); @@ -156,10 +172,9 @@ async function main() { printResults(allResults); writeResultsWorkbook(allResults, resolvedBaseOutputPath); - writeResultsToDb(allResults, DB_PATH, inputFile, resolvedBaseOutputPath); + await writeResultsToDb(allResults, inputFile, resolvedBaseOutputPath); } finally { await disconnectCache(); - closeDb(); } } diff --git a/src/cache.ts b/src/integrations/cache.ts similarity index 91% rename from src/cache.ts rename to src/integrations/cache.ts index 9b2f908..cec53d1 100644 --- a/src/cache.ts +++ b/src/integrations/cache.ts @@ -1,6 +1,6 @@ import Redis from "ioredis"; -import { config } from "./config.ts"; -import type { EnrichedProduct, KeepaData, SpApiData } from "./types.ts"; +import { config } from "../config.ts"; +import type { EnrichedProduct, KeepaData, SpApiData } from "../types.ts"; let redis: Redis | null = null; let disabled = false; diff --git a/src/keepa.test.ts b/src/integrations/keepa.test.ts similarity index 91% rename from src/keepa.test.ts rename to src/integrations/keepa.test.ts index c3cb9a4..0194980 100644 --- a/src/keepa.test.ts +++ b/src/integrations/keepa.test.ts @@ -42,7 +42,7 @@ test("lookupKeepaUpcs returns found, not_found, and multiple_asins outcomes", as JSON.stringify({ products: [ { - asin: "B000FOUND01", + asin: "B000FND001", upcList: ["012345678901"], stats: { current: [null, null, null, 1234], @@ -51,7 +51,7 @@ test("lookupKeepaUpcs returns found, not_found, and multiple_asins outcomes", as csv: [[5000000, 2999, 5000100]], }, { - asin: "B000MULTI01", + asin: "B000MUL001", upcList: ["098765432109"], stats: { current: [null, null, null, 2000], @@ -60,7 +60,7 @@ test("lookupKeepaUpcs returns found, not_found, and multiple_asins outcomes", as csv: [[1, 1999]], }, { - asin: "B000MULTI02", + asin: "B000MUL002", upcList: ["098765432109"], stats: { current: [null, null, null, 2100], @@ -83,14 +83,14 @@ test("lookupKeepaUpcs returns found, not_found, and multiple_asins outcomes", as ]); expect(details.get("012345678901")?.status).toBe("found"); - expect(details.get("012345678901")?.asin).toBe("B000FOUND01"); + expect(details.get("012345678901")?.asin).toBe("B000FND001"); expect(details.get("012345678901")?.keepaData?.currentPrice).toBe(29.99); expect(details.get("012345678901")?.keepaData?.currentPrice).toBe(29.99); expect(details.get("098765432109")?.status).toBe("multiple_asins"); expect(details.get("098765432109")?.candidateAsins).toEqual([ - "B000MULTI01", - "B000MULTI02", + "B000MUL001", + "B000MUL002", ]); expect(details.get("111111111111")?.status).toBe("not_found"); @@ -100,7 +100,7 @@ test("lookupKeepaUpcs returns found, not_found, and multiple_asins outcomes", as "098765432109", "111111111111", ]); - expect(simpleMap.get("012345678901")).toBe("B000FOUND01"); + expect(simpleMap.get("012345678901")).toBe("B000FND001"); expect(simpleMap.has("098765432109")).toBe(false); expect(simpleMap.has("111111111111")).toBe(false); }); @@ -128,7 +128,7 @@ test("lookupKeepaUpcs keeps partial success when one chunk fails", async () => { JSON.stringify({ products: [ { - asin: "B000LAST001", + asin: "B000LST001", upcList: [secondChunkUpc], stats: { current: [null, null, null, 1000], @@ -148,11 +148,11 @@ test("lookupKeepaUpcs keeps partial success when one chunk fails", async () => { expect(details.get(firstChunkFirstUpc)?.status).toBe("request_failed"); expect(details.get(secondChunkUpc)?.status).toBe("found"); - expect(details.get(secondChunkUpc)?.asin).toBe("B000LAST001"); + expect(details.get(secondChunkUpc)?.asin).toBe("B000LST001"); const simpleMap = await mapUpcsToAsins(upcs); expect(simpleMap.has(firstChunkFirstUpc)).toBe(false); - expect(simpleMap.get(secondChunkUpc)).toBe("B000LAST001"); + expect(simpleMap.get(secondChunkUpc)).toBe("B000LST001"); }); test("lookupKeepaUpcs retries on 429 and succeeds after refill wait", async () => { @@ -175,7 +175,7 @@ test("lookupKeepaUpcs retries on 429 and succeeds after refill wait", async () = JSON.stringify({ products: [ { - asin: "B000RETRY01", + asin: "B000RTY001", upcList: [targetUpc], stats: { current: [null, null, null, 1111], @@ -197,7 +197,7 @@ test("lookupKeepaUpcs retries on 429 and succeeds after refill wait", async () = expect(fetchMock.mock.calls.length).toBe(2); expect(details.get(targetUpc)?.status).toBe("found"); - expect(details.get(targetUpc)?.asin).toBe("B000RETRY01"); + expect(details.get(targetUpc)?.asin).toBe("B000RTY001"); }); test("lookupKeepaUpcs uses lightweight query params for code mapping", async () => { @@ -220,7 +220,7 @@ test("lookupKeepaUpcs uses lightweight query params for code mapping", async () JSON.stringify({ products: [ { - asin: "B000LIGHT01", + asin: "B000LGT001", upcList: [targetUpc], categoryTree: [{ name: "Test Category" }], }, @@ -238,5 +238,5 @@ test("lookupKeepaUpcs uses lightweight query params for code mapping", async () expect(fetchMock.mock.calls.length).toBe(1); expect(details.get(targetUpc)?.status).toBe("found"); - expect(details.get(targetUpc)?.asin).toBe("B000LIGHT01"); + expect(details.get(targetUpc)?.asin).toBe("B000LGT001"); }); diff --git a/src/keepa.ts b/src/integrations/keepa.ts similarity index 93% rename from src/keepa.ts rename to src/integrations/keepa.ts index 81d4632..128d8a3 100644 --- a/src/keepa.ts +++ b/src/integrations/keepa.ts @@ -1,5 +1,6 @@ -import { config } from "./config.ts"; -import type { KeepaData, KeepaUpcLookupDetail } from "./types.ts"; +import { config } from "../config.ts"; +import { normalizeAsin } from "../asin.ts"; +import type { KeepaData, KeepaUpcLookupDetail } from "../types.ts"; const KEEPA_BASE = "https://api.keepa.com"; const MAX_ASINS_PER_REQUEST = 100; @@ -224,14 +225,21 @@ function buildFailureDetail( }; } -export async function fetchKeepaDataBatch( - asins: string[], -): Promise> { - const results = new Map(); - - // Split into chunks of MAX_ASINS_PER_REQUEST - for (let i = 0; i < asins.length; i += MAX_ASINS_PER_REQUEST) { - const chunk = asins.slice(i, i + MAX_ASINS_PER_REQUEST); +export async function fetchKeepaDataBatch( + asins: string[], +): Promise> { + const results = new Map(); + const canonicalAsins = Array.from( + new Set( + asins + .map((asin) => normalizeAsin(asin)) + .filter((asin): asin is string => asin !== null), + ), + ); + + // Split into chunks of MAX_ASINS_PER_REQUEST + for (let i = 0; i < canonicalAsins.length; i += MAX_ASINS_PER_REQUEST) { + const chunk = canonicalAsins.slice(i, i + MAX_ASINS_PER_REQUEST); const url = buildProductUrl("asin", chunk, { includeStats: true, includeBuybox: true, @@ -248,11 +256,11 @@ export async function fetchKeepaDataBatch( `Keepa: ${data.products?.length ?? 0} products returned, ${tokensLeft} tokens remaining (refill: ${refillRate}/min)`, ); - if (data.products) { - for (const product of data.products) { - const asin = product.asin; - if (!asin) continue; - results.set(asin, parseKeepaProduct(product)); + if (data.products) { + for (const product of data.products) { + const asin = normalizeAsin(product.asin); + if (!asin) continue; + results.set(asin, parseKeepaProduct(product)); } } } @@ -307,10 +315,10 @@ export async function lookupKeepaUpcs( `Keepa: ${data.products?.length ?? 0} products returned for UPC query, ${tokensLeft} tokens remaining (refill: ${refillRate}/min)`, ); - const byUpc = new Map>(); - for (const product of data.products ?? []) { - const asin = String(product.asin ?? "").trim(); - if (!asin) continue; + const byUpc = new Map>(); + for (const product of data.products ?? []) { + const asin = normalizeAsin(product.asin); + if (!asin) continue; const keepaData = parseKeepaProduct(product); const productUpcs = extractUpcsFromProduct(product); diff --git a/src/llm.ts b/src/integrations/llm.ts similarity index 96% rename from src/llm.ts rename to src/integrations/llm.ts index e96c5ff..1908902 100644 --- a/src/llm.ts +++ b/src/integrations/llm.ts @@ -1,5 +1,5 @@ -import { config } from "./config.ts"; -import type { EnrichedProduct, LlmVerdict } from "./types.ts"; +import { config } from "../config.ts"; +import type { EnrichedProduct, LlmVerdict } from "../types.ts"; const SYSTEM_PROMPT_STRICT = `You are an expert Amazon product analyst specializing in FBA and FBM fulfillment strategy. diff --git a/src/searxng.test.ts b/src/integrations/searxng.test.ts similarity index 99% rename from src/searxng.test.ts rename to src/integrations/searxng.test.ts index 6bd013f..3221adf 100644 --- a/src/searxng.test.ts +++ b/src/integrations/searxng.test.ts @@ -13,6 +13,7 @@ afterAll(() => { test("normalizeAsin uppercases and validates ASINs", () => { expect(normalizeAsin(" b07sn9bhvv ")).toBe("B07SN9BHVV"); + expect(normalizeAsin("0306406152")).toBe("0306406152"); expect(() => normalizeAsin("not-an-asin")).toThrow("Invalid ASIN"); }); diff --git a/src/searxng.ts b/src/integrations/searxng.ts similarity index 99% rename from src/searxng.ts rename to src/integrations/searxng.ts index b6205e4..7c49419 100644 --- a/src/searxng.ts +++ b/src/integrations/searxng.ts @@ -1,10 +1,11 @@ +import { normalizeAsin as normalizeCanonicalAsin } from "../asin.ts"; + const DEFAULT_SEARXNG_URL = "https://searxng.nvictor.me/"; const DEFAULT_GOOGLE_CUSTOM_SEARCH_URL = "https://www.googleapis.com/customsearch/v1"; const DEFAULT_SERPAPI_URL = "https://serpapi.com/search.json"; const DEFAULT_TIMEOUT_MS = 10_000; const DEFAULT_MAX_RESULTS = 10; -const ASIN_REGEX = /^B[0-9A-Z]{9}$/; const ASIN_MATCH_REGEX = /\bB[0-9A-Z]{9}\b/gi; const PRICE_LABELS = [ "selling price", @@ -127,16 +128,15 @@ export async function searchProductOffers( } export function normalizeAsin(value: string): string { - const asin = value.trim().toUpperCase(); - if (!ASIN_REGEX.test(asin)) { + const asin = normalizeCanonicalAsin(value); + if (!asin) { throw new Error(`Invalid ASIN: ${value}`); } return asin; } function getAsinQuery(value: string): string | undefined { - const normalized = value.trim().toUpperCase(); - return ASIN_REGEX.test(normalized) ? normalized : undefined; + return normalizeCanonicalAsin(value) ?? undefined; } async function fetchSearxngResults( diff --git a/src/sp-api.test.ts b/src/integrations/sp-api.test.ts similarity index 83% rename from src/sp-api.test.ts rename to src/integrations/sp-api.test.ts index 284d19d..69d4a51 100644 --- a/src/sp-api.test.ts +++ b/src/integrations/sp-api.test.ts @@ -20,6 +20,15 @@ test("parseCatalogUpcLookupResponse marks no match", () => { expect(detail.asin).toBeNull(); }); +test("parseCatalogUpcLookupResponse ignores invalid ASIN identifiers", () => { + const detail = parseCatalogUpcLookupResponse("012345678901", { + items: [{ asin: "012345678901" }], + }); + + expect(detail.status).toBe("not_found"); + expect(detail.asin).toBeNull(); +}); + test("parseCatalogUpcLookupResponse marks multiple ASINs", () => { const detail = parseCatalogUpcLookupResponse("012345678901", { payload: { diff --git a/src/sp-api.ts b/src/integrations/sp-api.ts similarity index 96% rename from src/sp-api.ts rename to src/integrations/sp-api.ts index eb2b9a8..b7549f4 100644 --- a/src/sp-api.ts +++ b/src/integrations/sp-api.ts @@ -1,11 +1,12 @@ import { SellingPartner } from "amazon-sp-api"; -import { config } from "./config.ts"; +import { normalizeAsin } from "../asin.ts"; +import { config } from "../config.ts"; import type { KeepaUpcLookupStatus, SpApiData, SellabilityInfo, UpcLookupDetail, -} from "./types.ts"; +} from "../types.ts"; type RegionCode = "na" | "eu" | "fe"; @@ -222,8 +223,7 @@ function extractCatalogAsin(item: any): string | null { item?.identifiers?.marketplaceASIN?.asin ?? item?.Identifiers?.MarketplaceASIN?.ASIN; if (typeof raw !== "string") return null; - const asin = raw.trim().toUpperCase(); - return asin ? asin : null; + return normalizeAsin(raw); } export function parseCatalogUpcLookupResponse( diff --git a/src/reader.ts b/src/reader.ts index 0c72956..23d7d26 100644 --- a/src/reader.ts +++ b/src/reader.ts @@ -1,8 +1,7 @@ -import * as XLSX from "xlsx"; -import type { ProductRecord } from "./types.ts"; - -const ASIN_REGEX = /^B[0-9A-Z]{9}$/; - +import * as XLSX from "xlsx"; +import { normalizeAsin } from "./asin.ts"; +import type { ProductRecord } from "./types.ts"; + const COLUMN_CANDIDATES = { asin: ["asin"], name: ["name", "product name", "title", "product title"], @@ -132,14 +131,12 @@ function getKnownColumns(columns: ColumnMap): Set { return new Set(Object.values(columns).filter((column): column is string => !!column)); } -function parseAsin(value: unknown): string | undefined { - const asin = String(value ?? "") - .trim() - .toUpperCase(); - if (!asin || !ASIN_REGEX.test(asin)) { - console.warn(`Skipping invalid ASIN: "${asin}"`); - return undefined; - } +function parseAsin(value: unknown): string | undefined { + const asin = normalizeAsin(value); + if (!asin) { + console.warn(`Skipping invalid ASIN: "${String(value ?? "").trim()}"`); + return undefined; + } return asin; } diff --git a/src/server.ts b/src/server.ts index 6f275b0..201a566 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,112 +1,57 @@ import index from "./web/index.html"; -import path from "node:path"; import * as XLSX from "xlsx"; -import { getDb, initDb } from "./database.ts"; +import { normalizeAsin } from "./asin.ts"; +import { db, client } from "./db/index.ts"; +import { analysisRevisions } from "./db/schema.ts"; +import { insertObservation, refreshRunStats } from "./db/persistence.ts"; import { fetchKeepaDataBatch, lookupKeepaUpcs, mapUpcsToAsins, -} from "./keepa.ts"; -import { runUpcFileAnalysis } from "./upc-file-analysis.ts"; -import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts"; -import { analyzeProducts } from "./llm.ts"; +} from "./integrations/keepa.ts"; +import { analyzeProducts } from "./integrations/llm.ts"; +import { + fetchSellabilityBatch, + fetchSpApiPricingAndFees, +} from "./integrations/sp-api.ts"; +import { runUpcFileAnalysis } from "./supplier/upc-file-analysis.ts"; import type { + AnalysisResult, EnrichedProduct, KeepaUpcLookupDetail, ProductRecord, SpApiData, } from "./types.ts"; -type ProcessType = "lead_analysis" | "category_analysis"; - -type RunRecord = { - processType: ProcessType; - runId: number; - timestamp: string; - status: string; - jobType: string; - source: string | null; - output: string | null; - totalProducts: number; - fbaCount: number; - fbmCount: number; - skipCount: number; -}; - -type ProductListRecord = { - processType: ProcessType; - runId: number; - asin: string; - product_name: string | null; - brand: string | null; - category: string | null; - verdict: "FBA" | "FBM" | "SKIP"; - confidence: number | null; - sellability_status: string | null; - monthly_sold: number | null; - seller_count: number | null; - amazon_is_seller: number | null; - amazon_buybox_share_pct_90d: number | null; - sales_rank: number | null; - current_price: number | null; - avg_price_90d: number | null; - reasoning: string | null; - fetched_at: string; -}; - -type StalkerResultRecord = { - runId: number; - started_at: string; - status: string; - input_file: string; - seller_id: string; - seller_name: string | null; - rating: number | null; - rating_count: number | null; - storefront_asin_total: number | null; - persisted_inventory_sample_count: number | null; - discovered_from_count: number; - first_seen_at: string; - last_seen_at: string; - persisted_inventory_asin_count: number; - inventory_sample_asins: string | null; -}; - -type StalkerProductRecord = { - runId: number; - started_at: string; - seller_id: string; - seller_name: string | null; - rating: number | null; - rating_count: number | null; - asin: string; - can_sell: number; - sellability_status: string; - sellability_reason: string | null; - product_title: string | null; - brand: string | null; - category_tree: string | null; - current_price: number | null; - avg_price_90d: number | null; - sales_rank: number | null; - monthly_sold: number | null; - seller_count: number | null; - amazon_is_seller: number | null; - verdict: string | null; - confidence: number | null; - reasoning: string | null; - last_seen_at: string; -}; - -const DB_PATH = process.env.RESULTS_DB_PATH || path.join("db", "results.db"); const DEFAULT_PAGE_SIZE = 25; const MAX_PAGE_SIZE = 200; -const ASIN_PATTERN = /^[A-Z0-9]{10}$/; const MAX_UPCS_PER_REQUEST = 1000; const USE_CLAUDE = process.argv.includes("--claude"); -initDb(DB_PATH); -const db = getDb(DB_PATH); +function toPostgresSql(query: string): string { + let n = 0; + return query.replace(/\?/g, () => `$${++n}`); +} + +async function pgGet>( + query: string, + params: unknown[] = [], +): Promise { + const rows = await client.unsafe(toPostgresSql(query), params as never[]); + return (rows[0] as T) ?? null; +} + +async function pgAll>( + query: string, + params: unknown[] = [], +): Promise { + return client.unsafe(toPostgresSql(query), params as never[]) as unknown as T[]; +} + +async function pgRun(query: string, params: unknown[] = []): Promise { + const result = await client.unsafe(toPostgresSql(query), params as never[]); + return result.count; +} function json(data: unknown, status = 200): Response { return new Response(JSON.stringify(data), { @@ -117,7 +62,6 @@ function json(data: unknown, status = 200): Response { function csv(text: string, filename: string): Response { return new Response(text, { - status: 200, headers: { "content-type": "text/csv; charset=utf-8", "content-disposition": `attachment; filename="${filename}"`, @@ -127,7 +71,6 @@ function csv(text: string, filename: string): Response { function xlsx(buffer: ArrayBuffer, filename: string): Response { return new Response(buffer, { - status: 200, headers: { "content-type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", @@ -137,144 +80,97 @@ function xlsx(buffer: ArrayBuffer, filename: string): Response { } function parseIntParam(value: string | null, fallback: number): number { - if (!value) return fallback; - const parsed = Number.parseInt(value, 10); - if (!Number.isFinite(parsed) || parsed < 1) return fallback; - return parsed; + const parsed = Number.parseInt(String(value), 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; } -function normalizeAsin(value: string): string { - return value.trim().toUpperCase(); +function pageInput(filters: URLSearchParams) { + const page = parseIntParam(filters.get("page"), 1); + const pageSize = Math.min( + parseIntParam(filters.get("pageSize"), DEFAULT_PAGE_SIZE), + MAX_PAGE_SIZE, + ); + return { page, pageSize, offset: (page - 1) * pageSize }; } -function isValidAsin(value: string): boolean { - return ASIN_PATTERN.test(value); +function escapeCsvValue(value: unknown): string { + if (value == null) return ""; + const escaped = String(value).replaceAll('"', '""'); + return /[",\n]/.test(escaped) ? `"${escaped}"` : escaped; +} + +function safeSort( + input: string | null, + columns: Record, + fallback: string, +): string { + if (!input) return fallback; + const parts = input + .split(",") + .map((part) => { + const [key, direction] = part.trim().split(":"); + const column = key ? columns[key] : undefined; + if (!column) return null; + return `${column} ${direction?.toUpperCase() === "DESC" ? "DESC" : "ASC"}`; + }) + .filter((value): value is string => value != null); + return parts.join(", ") || fallback; } function splitRawUpcValues(input: string): string[] { - return input - .split(/[\s,;|]+/) - .map((chunk) => chunk.trim()) - .filter(Boolean); + return input.split(/[\s,;|]+/).map((v) => v.trim()).filter(Boolean); } -function collectUpcsFromUnknown(value: unknown, target: string[]): void { +function collectUpcs(value: unknown, target: string[]): void { if (typeof value === "string") { target.push(...splitRawUpcValues(value)); - return; - } - - if (typeof value === "number" && Number.isFinite(value)) { + } else if (typeof value === "number" && Number.isFinite(value)) { target.push(String(Math.trunc(value))); - return; + } else if (Array.isArray(value)) { + value.forEach((entry) => collectUpcs(entry, target)); } - - if (Array.isArray(value)) { - for (const item of value) { - collectUpcsFromUnknown(item, target); - } - } -} - -function normalizeAndDedupeUpcs(values: string[]): string[] { - const seen = new Set(); - const normalized: string[] = []; - - for (const value of values) { - const upc = value.trim(); - if (!upc || seen.has(upc)) continue; - seen.add(upc); - normalized.push(upc); - } - - return normalized; -} - -function parseUpcsFromSearchParams(params: URLSearchParams): string[] { - const parsed: string[] = []; - for (const value of params.getAll("upc")) { - collectUpcsFromUnknown(value, parsed); - } - - const upcsValue = params.get("upcs"); - if (upcsValue) { - collectUpcsFromUnknown(upcsValue, parsed); - } - - return normalizeAndDedupeUpcs(parsed); } async function parseUpcsFromRequest(req: Request): Promise { + const parsed: string[] = []; if (req.method === "GET") { - const url = new URL(req.url); - return parseUpcsFromSearchParams(url.searchParams); - } - - if (req.method !== "POST") { + const params = new URL(req.url).searchParams; + params.getAll("upc").forEach((value) => collectUpcs(value, parsed)); + collectUpcs(params.get("upcs"), parsed); + } else if (req.method === "POST") { + let body: unknown; + try { + body = await req.json(); + } catch { + throw new Error("Invalid JSON body"); + } + if (body && typeof body === "object" && "upcs" in body) { + collectUpcs((body as { upcs?: unknown }).upcs, parsed); + } else { + collectUpcs(body, parsed); + } + } else { throw new Error("Method not allowed"); } - - let body: unknown; - try { - body = await req.json(); - } catch { - throw new Error("Invalid JSON body"); - } - - const parsed: string[] = []; - if (body && typeof body === "object" && "upcs" in body) { - collectUpcsFromUnknown((body as { upcs?: unknown }).upcs, parsed); - } else { - collectUpcsFromUnknown(body, parsed); - } - - return normalizeAndDedupeUpcs(parsed); + return [...new Set(parsed)]; } -function validateUpcRequest(upcs: string[]): string | null { - if (upcs.length === 0) { - return "Provide at least one UPC via query (?upc=...) or JSON body { upcs: [...] }"; - } - +function validateUpcs(upcs: string[]): string | null { + if (upcs.length === 0) return "Provide at least one UPC."; if (upcs.length > MAX_UPCS_PER_REQUEST) { return `Too many UPCs. Maximum allowed per request is ${MAX_UPCS_PER_REQUEST}.`; } - return null; } -function summarizeLookupStatuses( - details: KeepaUpcLookupDetail[], -): Record { - const counts: Record = {}; - for (const detail of details) { - counts[detail.status] = (counts[detail.status] ?? 0) + 1; - } - return counts; +function summarizeLookupStatuses(items: KeepaUpcLookupDetail[]) { + return items.reduce>((counts, item) => { + counts[item.status] = (counts[item.status] ?? 0) + 1; + return counts; + }, {}); } -function parsePositiveIntField( - value: unknown, - fieldName: string, -): number | undefined { - if (value == null) return undefined; - if (typeof value === "number") { - if (!Number.isInteger(value) || value < 1) { - throw new Error(`${fieldName} must be a positive integer`); - } - return value; - } - if (typeof value === "string" && value.trim().length > 0) { - const parsed = Number.parseInt(value, 10); - if (!Number.isFinite(parsed) || parsed < 1) { - throw new Error(`${fieldName} must be a positive integer`); - } - return parsed; - } - throw new Error(`${fieldName} must be a positive integer`); -} - -type UpcFileProcessRequest = { +type UpcFileRequest = { inputFile: string; outputFile?: string; inputBatchSize?: number; @@ -282,1305 +178,351 @@ type UpcFileProcessRequest = { maxRows?: number; }; -async function parseUpcFileProcessRequest( - req: Request, -): Promise { - if (req.method !== "POST") { - throw new Error("Method not allowed"); +function positiveField(value: unknown, name: string): number | undefined { + if (value == null) return undefined; + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed < 1) { + throw new Error(`${name} must be a positive integer`); } + return parsed; +} - let body: unknown; +async function parseUpcFileRequest(req: Request): Promise { + if (req.method !== "POST") throw new Error("Method not allowed"); + let body: Record; try { - body = await req.json(); + body = (await req.json()) as Record; } catch { throw new Error("Invalid JSON body"); } - - if (!body || typeof body !== "object") { - throw new Error("Request body must be an object"); + if (typeof body.inputFile !== "string" || !body.inputFile.trim()) { + throw new Error("inputFile is required"); } - - const parsedBody = body as Record; - const inputFileValue = parsedBody.inputFile; - if ( - typeof inputFileValue !== "string" || - inputFileValue.trim().length === 0 - ) { - throw new Error("inputFile is required and must be a non-empty string"); - } - - const outputFileValue = parsedBody.outputFile; - if ( - outputFileValue != null && - (typeof outputFileValue !== "string" || outputFileValue.trim().length === 0) - ) { - throw new Error("outputFile must be a non-empty string when provided"); - } - return { - inputFile: inputFileValue.trim(), + inputFile: body.inputFile, outputFile: - typeof outputFileValue === "string" ? outputFileValue.trim() : undefined, - inputBatchSize: parsePositiveIntField( - parsedBody.inputBatchSize, - "inputBatchSize", - ), - upcLookupBatchSize: parsePositiveIntField( - parsedBody.upcLookupBatchSize, + typeof body.outputFile === "string" ? body.outputFile : undefined, + inputBatchSize: positiveField(body.inputBatchSize, "inputBatchSize"), + upcLookupBatchSize: positiveField( + body.upcLookupBatchSize, "upcLookupBatchSize", ), - maxRows: parsePositiveIntField(parsedBody.maxRows, "maxRows"), + maxRows: positiveField(body.maxRows, "maxRows"), }; } -function parseSort( - sortParam: string | null, - allowed: Set, - fallback: string, -): string { - if (!sortParam) return fallback; - const clauses = sortParam - .split(",") - .map((chunk) => chunk.trim()) - .filter(Boolean) - .map((chunk) => { - const [fieldRaw, dirRaw] = chunk.split(":"); - const field = fieldRaw?.trim(); - const dir = dirRaw?.trim().toUpperCase() === "DESC" ? "DESC" : "ASC"; - if (!field || !allowed.has(field)) return null; - return `${field} ${dir}`; - }) - .filter((value): value is string => value !== null); - - return clauses.length > 0 ? clauses.join(", ") : fallback; -} - -function parseResultSort( - sortParam: string | null, - allowed: Set, - fallback: string, -): string { - if (!sortParam) return fallback; - const clauses = sortParam - .split(",") - .map((chunk) => chunk.trim()) - .filter(Boolean) - .map((chunk) => { - const [fieldRaw, dirRaw] = chunk.split(":"); - const field = fieldRaw?.trim(); - const dir = dirRaw?.trim().toUpperCase() === "DESC" ? "DESC" : "ASC"; - if (!field || !allowed.has(field)) return null; - if (field === "monthly_sold") - return `CAST(COALESCE(monthly_sold, 0) AS INTEGER) ${dir}`; - if (field === "amazon_is_seller") - return `CAST(COALESCE(amazon_is_seller, 0) AS INTEGER) ${dir}`; - if (field === "amazon_buybox_share_pct_90d") { - return `CAST(COALESCE(amazon_buybox_share_pct_90d, 0) AS REAL) ${dir}`; - } - return `${field} ${dir}`; - }) - .filter((value): value is string => value !== null); - - return clauses.length > 0 ? clauses.join(", ") : fallback; -} - -function escapeCsvValue(value: unknown): string { - if (value === null || value === undefined) return ""; - const text = String(value); - const escaped = text.replaceAll('"', '""'); - return /[",\n]/.test(escaped) ? `"${escaped}"` : escaped; -} - -function parseResultFilters( - processType: ProcessType, - runId: number, - filters: URLSearchParams, -) { - const q = filters.get("q")?.trim() || ""; - const verdict = filters.get("verdict")?.trim(); - const sellabilityStatus = filters.get("sellabilityStatus")?.trim(); - const minConfidence = filters.get("minConfidence")?.trim(); - const maxConfidence = filters.get("maxConfidence")?.trim(); - const amazonIsSeller = filters.get("amazonIsSeller")?.trim(); - - const conditions: string[] = ["run_id = ?"]; - const params: Array = [runId]; - - if (verdict) { - conditions.push("verdict = ?"); - params.push(verdict); - } - - if (sellabilityStatus) { - conditions.push("sellability_status = ?"); - params.push(sellabilityStatus); - } - - if (minConfidence) { - conditions.push("confidence >= ?"); - params.push(Number(minConfidence)); - } - - if (maxConfidence) { - conditions.push("confidence <= ?"); - params.push(Number(maxConfidence)); - } - - if (amazonIsSeller === "yes") { - conditions.push("amazon_is_seller = 1"); - } else if (amazonIsSeller === "no") { - conditions.push("amazon_is_seller = 0"); - } - - if (q) { - const wildcard = `%${q}%`; - if (processType === "lead_analysis") { - conditions.push( - "(asin LIKE ? OR product_name LIKE ? OR brand LIKE ? OR category LIKE ? OR reasoning LIKE ?)", - ); - params.push(wildcard, wildcard, wildcard, wildcard, wildcard); - } else { - conditions.push( - "(asin LIKE ? OR name LIKE ? OR brand LIKE ? OR category LIKE ? OR reasoning LIKE ?)", - ); - params.push(wildcard, wildcard, wildcard, wildcard, wildcard); - } - } - - return { - where: `WHERE ${conditions.join(" AND ")}`, - params, - }; -} - -function getRuns(filters: URLSearchParams) { - const q = filters.get("q")?.trim() || ""; - const processType = filters.get("processType")?.trim(); +async function getRuns(filters: URLSearchParams) { + const { page, pageSize, offset } = pageInput(filters); + const conditions: string[] = ["TRUE"]; + const params: unknown[] = []; + const type = filters.get("processType")?.trim(); const status = filters.get("status")?.trim(); - const startDate = filters.get("startDate")?.trim(); - const endDate = filters.get("endDate")?.trim(); - const page = parseIntParam(filters.get("page"), 1); - const pageSize = Math.min( - parseIntParam(filters.get("pageSize"), DEFAULT_PAGE_SIZE), - MAX_PAGE_SIZE, - ); - const offset = (page - 1) * pageSize; - - const allowedSort = new Set([ - "timestamp", - "status", - "totalProducts", - "fbaCount", - "fbmCount", - "skipCount", - "runId", - "jobType", - ]); - const orderBy = parseSort( - filters.get("sort"), - allowedSort, - "timestamp DESC, runId DESC", - ); - - const conditions: string[] = []; - const params: Array = []; - - if (processType === "lead_analysis" || processType === "category_analysis") { - conditions.push("processType = ?"); - params.push(processType); + const q = filters.get("q")?.trim(); + if (type) { + conditions.push("r.type::text = ?"); + params.push(type); } - if (status) { - conditions.push("status = ?"); + conditions.push("r.status::text = ?"); params.push(status); } - - if (startDate) { - conditions.push("timestamp >= ?"); - params.push(startDate); + if (filters.get("startDate")) { + conditions.push("r.started_at >= ?"); + params.push(filters.get("startDate")); } - - if (endDate) { - conditions.push("timestamp <= ?"); - params.push(endDate); + if (filters.get("endDate")) { + conditions.push("r.started_at <= ?"); + params.push(filters.get("endDate")); } - if (q) { conditions.push( - "(jobType LIKE ? OR source LIKE ? OR output LIKE ? OR CAST(runId AS TEXT) LIKE ?)", + "(COALESCE(r.input_file, '') ILIKE ? OR COALESCE(cd.category_label, '') ILIKE ? OR CAST(r.id AS text) ILIKE ?)", ); - const wildcard = `%${q}%`; - params.push(wildcard, wildcard, wildcard, wildcard); + params.push(`%${q}%`, `%${q}%`, `%${q}%`); } - - const where = - conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : ""; - - const baseUnion = ` - SELECT - 'lead_analysis' AS processType, - id AS runId, - timestamp, - 'completed' AS status, - 'lead_file_analysis' AS jobType, - input_file AS source, - output_file AS output, - COALESCE(total_products, 0) AS totalProducts, - COALESCE(fba_count, 0) AS fbaCount, - COALESCE(fbm_count, 0) AS fbmCount, - COALESCE(skip_count, 0) AS skipCount - FROM runs - UNION ALL - SELECT - 'category_analysis' AS processType, - id AS runId, - run_timestamp AS timestamp, - status, - category_label AS jobType, - CAST(category_id AS TEXT) AS source, - NULL AS output, - COALESCE(top_asins_checked, 0) AS totalProducts, - COALESCE(fba_count, 0) AS fbaCount, - COALESCE(fbm_count, 0) AS fbmCount, - COALESCE(skip_count, 0) AS skipCount - FROM category_analysis_runs - `; - - const totalRow = db - .query(`SELECT COUNT(*) as total FROM (${baseUnion}) all_runs ${where}`) - .get(...params) as { total: number }; - - const items = db - .query( - `SELECT * FROM (${baseUnion}) all_runs ${where} ORDER BY ${orderBy} LIMIT ? OFFSET ?`, - ) - .all(...params, pageSize, offset) as RunRecord[]; - - return { - items, - page, - pageSize, - total: totalRow.total, - totalPages: Math.max(1, Math.ceil(totalRow.total / pageSize)), - }; + const where = `WHERE ${conditions.join(" AND ")}`; + const sort = safeSort( + filters.get("sort"), + { + runId: "r.id", + processType: "r.type", + timestamp: "r.started_at", + status: "r.status", + jobType: "COALESCE(cd.category_label, r.input_file, r.type::text)", + totalProducts: "COALESCE(stats.processed_count, 0)", + fbaCount: "COALESCE(stats.fba_count, 0)", + fbmCount: "COALESCE(stats.fbm_count, 0)", + skipCount: "COALESCE(stats.skip_count, 0)", + }, + "r.started_at DESC, r.id DESC", + ); + const join = ` + FROM runs r + LEFT JOIN analysis_run_stats stats ON stats.run_id = r.id + LEFT JOIN category_run_details cd ON cd.run_id = r.id`; + const totalRow = await pgGet<{ total: string }>( + `SELECT COUNT(*) AS total ${join} ${where}`, + params, + ); + const items = await pgAll( + `SELECT r.type AS "processType", r.id AS "runId", + r.started_at AS timestamp, r.status, r.input_file AS source, + r.output_file AS output, + COALESCE(cd.category_label, r.input_file, r.type::text) AS "jobType", + COALESCE(stats.processed_count, 0) AS "totalProducts", + COALESCE(stats.fba_count, 0) AS "fbaCount", + COALESCE(stats.fbm_count, 0) AS "fbmCount", + COALESCE(stats.buy_count, 0) AS "buyCount", + COALESCE(stats.watch_count, 0) AS "watchCount", + COALESCE(stats.skip_count, 0) AS "skipCount" + ${join} ${where} ORDER BY ${sort} LIMIT ? OFFSET ?`, + [...params, pageSize, offset], + ); + const total = Number(totalRow?.total ?? 0); + return { items, page, pageSize, total, totalPages: Math.max(1, Math.ceil(total / pageSize)) }; } -function getProductList(filters: URLSearchParams) { - const q = filters.get("q")?.trim() || ""; +async function getRun(runId: number) { + return pgGet( + `SELECT r.type AS "processType", r.id AS "runId", r.parent_run_id AS "parentRunId", + r.started_at AS timestamp, r.completed_at AS "completedAt", r.status, + r.input_file AS source, r.output_file AS output, r.error_message AS "errorMessage", + COALESCE(cd.category_label, r.input_file, r.type::text) AS "jobType", + cd.category_id AS "categoryId", cd.checked_asin_count AS "checkedAsins", + COALESCE(stats.processed_count, 0) AS "totalProducts", + COALESCE(stats.available_count, 0) AS "availableAsins", + COALESCE(stats.fba_count, 0) AS "fbaCount", + COALESCE(stats.fbm_count, 0) AS "fbmCount", + COALESCE(stats.buy_count, 0) AS "buyCount", + COALESCE(stats.watch_count, 0) AS "watchCount", + COALESCE(stats.skip_count, 0) AS "skipCount" + FROM runs r + LEFT JOIN analysis_run_stats stats ON stats.run_id = r.id + LEFT JOIN category_run_details cd ON cd.run_id = r.id + WHERE r.id = ?`, + [runId], + ); +} + +const ITEM_ROWS = ` + WITH latest_revision AS ( + SELECT DISTINCT ON (ar.run_item_id) + ar.run_item_id, ar.id AS revision_id, ar.decision, ar.confidence, + ar.reasoning, ar.analyzed_at, ar.observation_id + FROM analysis_revisions ar + ORDER BY ar.run_item_id, ar.analyzed_at DESC, ar.id DESC + ) + SELECT ri.id AS item_id, ri.run_id, r.type AS process_type, + ri.product_asin, COALESCE(ri.product_asin, ur.normalized_upc) AS asin, + COALESCE(p.name, si.supplied_name, ur.normalized_upc) AS product_name, + COALESCE(p.brand, si.supplied_brand) AS brand, + COALESCE(p.category, si.supplied_category) AS category, + si.unit_cost, si.avg_price_90d_sheet, si.selling_price_sheet, + observation.current_price, observation.avg_price_90d, + observation.sales_rank, observation.sales_rank_avg_90d, + observation.seller_count, observation.amazon_is_seller, + observation.amazon_buybox_share_pct_90d, observation.monthly_sold, + observation.rank_drops_30d, observation.rank_drops_90d, + observation.fba_fee, observation.fbm_fee, observation.referral_percent, + observation.can_sell, observation.sellability_status, + observation.sellability_reason, revision.decision AS verdict, + revision.confidence, revision.reasoning, + revision.analyzed_at AS fetched_at, ur.normalized_upc AS upc, + ur.status AS upc_lookup_status + FROM run_items ri + JOIN runs r ON r.id = ri.run_id + LEFT JOIN products p ON p.asin = ri.product_asin + LEFT JOIN sourcing_inputs si ON si.run_item_id = ri.id + LEFT JOIN upc_resolutions ur ON ur.run_item_id = ri.id + LEFT JOIN latest_revision revision ON revision.run_item_id = ri.id + LEFT JOIN product_observations observation ON observation.id = revision.observation_id +`; + +function itemFilters(filters: URLSearchParams, runId?: number) { + const conditions: string[] = []; + const params: unknown[] = []; + if (runId != null) { + conditions.push("run_id = ?"); + params.push(runId); + } const verdict = filters.get("verdict")?.trim(); - const amazonIsSeller = filters.get("amazonIsSeller")?.trim(); - const page = parseIntParam(filters.get("page"), 1); - const pageSize = Math.min( - parseIntParam(filters.get("pageSize"), DEFAULT_PAGE_SIZE), - MAX_PAGE_SIZE, - ); - const offset = (page - 1) * pageSize; - - const conditions: string[] = []; - const params: Array = []; - - if (verdict === "FBA" || verdict === "FBM" || verdict === "SKIP") { - conditions.push("verdict = ?"); + if (verdict) { + conditions.push("verdict::text = ?"); params.push(verdict); } - - if (amazonIsSeller === "yes") { - conditions.push("amazon_is_seller = 1"); - } else if (amazonIsSeller === "no") { - conditions.push("amazon_is_seller = 0"); + if (filters.get("sellabilityStatus")) { + conditions.push("sellability_status = ?"); + params.push(filters.get("sellabilityStatus")); } - + const minConfidence = Number(filters.get("minConfidence")); + const maxConfidence = Number(filters.get("maxConfidence")); + if (filters.get("minConfidence") && Number.isFinite(minConfidence)) { + conditions.push("confidence >= ?"); + params.push(minConfidence); + } + if (filters.get("maxConfidence") && Number.isFinite(maxConfidence)) { + conditions.push("confidence <= ?"); + params.push(maxConfidence); + } + if (filters.get("amazonIsSeller") === "yes") { + conditions.push("amazon_is_seller = true"); + } else if (filters.get("amazonIsSeller") === "no") { + conditions.push("amazon_is_seller = false"); + } + const q = filters.get("q")?.trim(); if (q) { const wildcard = `%${q}%`; conditions.push( - "(asin LIKE ? OR product_name LIKE ? OR brand LIKE ? OR category LIKE ?)", + "(COALESCE(asin, '') ILIKE ? OR COALESCE(product_name, '') ILIKE ? OR COALESCE(brand, '') ILIKE ? OR COALESCE(category, '') ILIKE ? OR COALESCE(reasoning, '') ILIKE ?)", ); - params.push(wildcard, wildcard, wildcard, wildcard); + params.push(wildcard, wildcard, wildcard, wildcard, wildcard); } - - const where = - conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : ""; - const allowedSort = new Set([ - "asin", - "verdict", - "monthly_sold", - "seller_count", - "amazon_is_seller", - "amazon_buybox_share_pct_90d", - "sales_rank", - "current_price", - "product_name", - "brand", - "category", - "avg_price_90d", - "confidence", - "reasoning", - "fetched_at", - ]); - const orderBy = parseResultSort( - filters.get("sort"), - allowedSort, - "CAST(COALESCE(monthly_sold, 0) AS INTEGER) DESC, fetched_at DESC", - ); - - const baseUnion = ` - SELECT - 'lead_analysis' AS processType, - run_id AS runId, - asin, - product_name, - brand, - category, - verdict, - confidence, - sellability_status, - monthly_sold, - sellers AS seller_count, - amazon_is_seller, - amazon_buybox_share_pct_90d, - sales_rank, - current_price, - avg_price_90d, - reasoning, - fetched_at - FROM results - UNION ALL - SELECT - 'category_analysis' AS processType, - run_id AS runId, - asin, - name AS product_name, - brand, - category, - verdict, - confidence, - sellability_status, - monthly_sold, - seller_count, - amazon_is_seller, - amazon_buybox_share_pct_90d, - sales_rank, - current_price, - avg_price_90d, - reasoning, - fetched_at - FROM product_analysis_results - `; - - const totalRow = db - .query(`SELECT COUNT(*) as total FROM (${baseUnion}) all_products ${where}`) - .get(...params) as { total: number }; - - const items = db - .query( - `SELECT * FROM (${baseUnion}) all_products ${where} ORDER BY ${orderBy} LIMIT ? OFFSET ?`, - ) - .all(...params, pageSize, offset) as ProductListRecord[]; - return { - items, - page, - pageSize, - total: totalRow.total, - totalPages: Math.max(1, Math.ceil(totalRow.total / pageSize)), - }; -} - -function parseStalkerFilters(filters: URLSearchParams) { - const q = filters.get("q")?.trim() || ""; - const sellerId = filters.get("sellerId")?.trim().toUpperCase() || ""; - const runIdRaw = filters.get("runId")?.trim() || ""; - const minRatingCountRaw = filters.get("minRatingCount")?.trim() || ""; - const maxRatingCountRaw = filters.get("maxRatingCount")?.trim() || ""; - - const conditions: string[] = []; - const params: Array = []; - - if (runIdRaw) { - const runId = Number(runIdRaw); - if (Number.isInteger(runId) && runId > 0) { - conditions.push("r.id = ?"); - params.push(runId); - } - } - - if (sellerId) { - conditions.push("s.seller_id = ?"); - params.push(sellerId); - } - - if (minRatingCountRaw) { - conditions.push("s.rating_count >= ?"); - params.push(Number(minRatingCountRaw)); - } - - if (maxRatingCountRaw) { - conditions.push("s.rating_count <= ?"); - params.push(Number(maxRatingCountRaw)); - } - - if (q) { - const wildcard = `%${q}%`; - conditions.push( - `(s.seller_id LIKE ? OR s.seller_name LIKE ? OR EXISTS ( - SELECT 1 FROM stalker_seller_inventory inv_q - WHERE inv_q.run_id = r.id - AND inv_q.seller_id = s.seller_id - AND inv_q.asin LIKE ? - ))`, - ); - params.push(wildcard, wildcard, wildcard); - } - - return { - where: conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "", + where: conditions.length ? `WHERE ${conditions.join(" AND ")}` : "", params, }; } -function parseStalkerSort(sortParam: string | null): string { - const allowedSort = new Set([ - "runId", - "started_at", - "seller_id", - "seller_name", - "rating", - "rating_count", - "discovered_from_count", - "persisted_inventory_asin_count", - "storefront_asin_total", - "last_seen_at", - ]); - const parsed = parseSort( - sortParam, - allowedSort, - "persisted_inventory_asin_count DESC, last_seen_at DESC, seller_id ASC", - ); - - return parsed - .replaceAll("runId", "runId") - .replaceAll("rating_count", "rating_count") - .replaceAll( - "persisted_inventory_asin_count", - "persisted_inventory_asin_count", - ) - .replaceAll("storefront_asin_total", "storefront_asin_total"); -} - -function getStalkerResults(filters: URLSearchParams) { - const page = parseIntParam(filters.get("page"), 1); - const pageSize = Math.min( - parseIntParam(filters.get("pageSize"), DEFAULT_PAGE_SIZE), - MAX_PAGE_SIZE, - ); - const offset = (page - 1) * pageSize; - const { where, params } = parseStalkerFilters(filters); - const orderBy = parseStalkerSort(filters.get("sort")); - - const baseSelect = ` - SELECT - r.id AS runId, - r.started_at, - r.status, - r.input_file, - s.seller_id, - s.seller_name, - s.rating, - s.rating_count, - s.storefront_asin_total, - s.persisted_inventory_sample_count, - COUNT(DISTINCT sc.source_asin) AS discovered_from_count, - MIN(sc.fetched_at) AS first_seen_at, - MAX(sc.fetched_at) AS last_seen_at, - COUNT(DISTINCT inv.asin) AS persisted_inventory_asin_count, - GROUP_CONCAT(DISTINCT inv.asin) AS inventory_sample_asins - FROM stalker_asin_sellers sas - JOIN stalker_asin_scans sc ON sc.id = sas.scan_id - JOIN stalker_runs r ON r.id = sc.run_id - JOIN stalker_sellers s ON s.seller_id = sas.seller_id - LEFT JOIN stalker_seller_inventory inv - ON inv.run_id = r.id - AND inv.seller_id = s.seller_id - ${where} - GROUP BY r.id, s.seller_id - `; - - const totalRow = db - .query(`SELECT COUNT(*) AS total FROM (${baseSelect}) stalker_rows`) - .get(...params) as { total: number }; - - const summary = db - .query( - `SELECT - COUNT(DISTINCT runId) AS runs, - COUNT(DISTINCT seller_id) AS sellers, - COALESCE(SUM(persisted_inventory_asin_count), 0) AS persistedInventoryAsins - FROM (${baseSelect}) stalker_rows`, - ) - .get(...params) as { - runs: number; - sellers: number; - persistedInventoryAsins: number; - }; - - const items = db - .query( - `SELECT * FROM (${baseSelect}) stalker_rows - ORDER BY ${orderBy} - LIMIT ? OFFSET ?`, - ) - .all(...params, pageSize, offset) as StalkerResultRecord[]; - - return { - items, - summary, - page, - pageSize, - total: totalRow.total, - totalPages: Math.max(1, Math.ceil(totalRow.total / pageSize)), - }; -} - -function parseStalkerProductFilters(filters: URLSearchParams) { - const q = filters.get("q")?.trim() || ""; - const sellerId = filters.get("sellerId")?.trim().toUpperCase() || ""; - const runIdRaw = filters.get("runId")?.trim() || ""; - const verdict = filters.get("verdict")?.trim().toUpperCase() || ""; - const amazonIsSeller = filters.get("amazonIsSeller")?.trim() || ""; - const minPriceRaw = filters.get("minPrice")?.trim() || ""; - const maxPriceRaw = filters.get("maxPrice")?.trim() || ""; - const minMonthlySoldRaw = filters.get("minMonthlySold")?.trim() || ""; - const maxMonthlySoldRaw = filters.get("maxMonthlySold")?.trim() || ""; - const minSalesRankRaw = filters.get("minSalesRank")?.trim() || ""; - const maxSalesRankRaw = filters.get("maxSalesRank")?.trim() || ""; - const minSellerCountRaw = filters.get("minSellerCount")?.trim() || ""; - const maxSellerCountRaw = filters.get("maxSellerCount")?.trim() || ""; - const minRatingCountRaw = filters.get("minRatingCount")?.trim() || ""; - const maxRatingCountRaw = filters.get("maxRatingCount")?.trim() || ""; - const minConfidenceRaw = filters.get("minConfidence")?.trim() || ""; - const maxConfidenceRaw = filters.get("maxConfidence")?.trim() || ""; - - const conditions = [ - "inv.can_sell = 1", - "inv.sellability_status = 'available'", - ]; - const params: Array = []; - - if (runIdRaw) { - const runId = Number(runIdRaw); - if (Number.isInteger(runId) && runId > 0) { - conditions.push("r.id = ?"); - params.push(runId); - } - } - - if (sellerId) { - conditions.push("s.seller_id = ?"); - params.push(sellerId); - } - - if (verdict === "FBA" || verdict === "FBM" || verdict === "SKIP") { - conditions.push("analysis.verdict = ?"); - params.push(verdict); - } else if (verdict === "UNANALYZED") { - conditions.push("analysis.verdict IS NULL"); - } - - if (amazonIsSeller === "yes") { - conditions.push("inv.amazon_is_seller = 1"); - } else if (amazonIsSeller === "no") { - conditions.push("inv.amazon_is_seller = 0"); - } else if (amazonIsSeller === "unknown") { - conditions.push("inv.amazon_is_seller IS NULL"); - } - - const numericFilters: Array<[string, string, string]> = [ - [minPriceRaw, "inv.current_price >= ?", "minPrice"], - [maxPriceRaw, "inv.current_price <= ?", "maxPrice"], - [minMonthlySoldRaw, "inv.monthly_sold >= ?", "minMonthlySold"], - [maxMonthlySoldRaw, "inv.monthly_sold <= ?", "maxMonthlySold"], - [minSalesRankRaw, "inv.sales_rank >= ?", "minSalesRank"], - [maxSalesRankRaw, "inv.sales_rank <= ?", "maxSalesRank"], - [minSellerCountRaw, "inv.seller_count >= ?", "minSellerCount"], - [maxSellerCountRaw, "inv.seller_count <= ?", "maxSellerCount"], - [minRatingCountRaw, "s.rating_count >= ?", "minRatingCount"], - [maxRatingCountRaw, "s.rating_count <= ?", "maxRatingCount"], - [minConfidenceRaw, "analysis.confidence >= ?", "minConfidence"], - [maxConfidenceRaw, "analysis.confidence <= ?", "maxConfidence"], - ]; - - for (const [raw, condition] of numericFilters) { - if (!raw) continue; - const value = Number(raw); - if (Number.isFinite(value)) { - conditions.push(condition); - params.push(value); - } - } - - if (q) { - const wildcard = `%${q}%`; - conditions.push( - `( - inv.asin LIKE ? - OR inv.product_title LIKE ? - OR inv.brand LIKE ? - OR inv.category_tree LIKE ? - OR s.seller_id LIKE ? - OR s.seller_name LIKE ? - )`, - ); - params.push(wildcard, wildcard, wildcard, wildcard, wildcard, wildcard); - } - - return { - where: `WHERE ${conditions.join(" AND ")}`, - params, - }; -} - -function parseStalkerProductSort(sortParam: string | null): string { - const allowedSort = new Set([ - "runId", - "started_at", - "seller_id", - "seller_name", - "rating", - "rating_count", - "asin", - "product_title", - "brand", - "current_price", - "avg_price_90d", - "sales_rank", - "monthly_sold", - "seller_count", - "amazon_is_seller", - "verdict", - "confidence", - "last_seen_at", - ]); - return parseSort( - sortParam, - allowedSort, - "monthly_sold DESC, last_seen_at DESC, asin ASC", - ); -} - -function getStalkerProducts(filters: URLSearchParams) { - const page = parseIntParam(filters.get("page"), 1); - const pageSize = Math.min( - parseIntParam(filters.get("pageSize"), DEFAULT_PAGE_SIZE), - MAX_PAGE_SIZE, - ); - const offset = (page - 1) * pageSize; - const { where, params } = parseStalkerProductFilters(filters); - const orderBy = parseStalkerProductSort(filters.get("sort")); - - const baseSelect = ` - SELECT - r.id AS runId, - r.started_at, - s.seller_id, - s.seller_name, - s.rating, - s.rating_count, - inv.asin, - inv.can_sell, - inv.sellability_status, - inv.sellability_reason, - inv.product_title, - inv.brand, - inv.category_tree, - inv.current_price, - inv.avg_price_90d, - inv.sales_rank, - inv.monthly_sold, - inv.seller_count, - inv.amazon_is_seller, - analysis.verdict, - analysis.confidence, - analysis.reasoning, - inv.last_seen_at - FROM stalker_seller_inventory inv - JOIN stalker_runs r ON r.id = inv.run_id - JOIN stalker_sellers s ON s.seller_id = inv.seller_id - LEFT JOIN product_analysis_results analysis ON analysis.asin = inv.asin - ${where} - `; - - const totalRow = db - .query(`SELECT COUNT(*) AS total FROM (${baseSelect}) stalker_products`) - .get(...params) as { total: number }; - - const summary = db - .query( - `SELECT - COUNT(DISTINCT runId) AS runs, - COUNT(DISTINCT seller_id) AS sellers, - COUNT(DISTINCT asin) AS products - FROM (${baseSelect}) stalker_products`, - ) - .get(...params) as { - runs: number; - sellers: number; - products: number; - }; - - const items = db - .query( - `SELECT * FROM (${baseSelect}) stalker_products - ORDER BY ${orderBy} - LIMIT ? OFFSET ?`, - ) - .all(...params, pageSize, offset) as StalkerProductRecord[]; - - return { - items, - summary, - page, - pageSize, - total: totalRow.total, - totalPages: Math.max(1, Math.ceil(totalRow.total / pageSize)), - }; -} - -function getStalkerProductsForExport( - filters: URLSearchParams, -): StalkerProductRecord[] { - const { where, params } = parseStalkerProductFilters(filters); - const orderBy = parseStalkerProductSort(filters.get("sort")); - - return db - .query( - `SELECT * FROM ( - SELECT - r.id AS runId, - r.started_at, - s.seller_id, - s.seller_name, - s.rating, - s.rating_count, - inv.asin, - inv.can_sell, - inv.sellability_status, - inv.sellability_reason, - inv.product_title, - inv.brand, - inv.category_tree, - inv.current_price, - inv.avg_price_90d, - inv.sales_rank, - inv.monthly_sold, - inv.seller_count, - inv.amazon_is_seller, - analysis.verdict, - analysis.confidence, - analysis.reasoning, - inv.last_seen_at - FROM stalker_seller_inventory inv - JOIN stalker_runs r ON r.id = inv.run_id - JOIN stalker_sellers s ON s.seller_id = inv.seller_id - LEFT JOIN product_analysis_results analysis ON analysis.asin = inv.asin - ${where} - ) stalker_products - ORDER BY ${orderBy}`, - ) - .all(...params) as StalkerProductRecord[]; -} - -function parseCategoryTreeForExport(value: string | null): string { - if (!value) return ""; - try { - const parsed = JSON.parse(value); - return Array.isArray(parsed) - ? parsed.filter((item) => typeof item === "string").join(" > ") - : ""; - } catch { - return ""; - } -} - -function exportStalkerProductsXlsx(filters: URLSearchParams): Response { - const rows = getStalkerProductsForExport(filters); - const data = rows.map((row) => ({ - ASIN: row.asin, - "Amazon URL": `https://amazon.com/dp/${row.asin}`, - Product: row.product_title ?? "", - Brand: row.brand ?? "", - Category: parseCategoryTreeForExport(row.category_tree), - "Monthly Sold": row.monthly_sold ?? null, - Sellers: row.seller_count ?? null, - "Amazon Seller": - row.amazon_is_seller == null - ? "" - : row.amazon_is_seller === 1 - ? "Yes" - : "No", - "Sales Rank": row.sales_rank ?? null, - "Current Price": row.current_price ?? null, - "Avg 90d": row.avg_price_90d ?? null, - Verdict: row.verdict ?? "", - Confidence: row.confidence ?? null, - Reasoning: row.reasoning ?? "", - "Seller ID": row.seller_id, - Seller: row.seller_name ?? "", - "Seller Rating": row.rating ?? null, - "Seller Rating Count": row.rating_count ?? null, - "Sellability Status": row.sellability_status, - "Sellability Reason": row.sellability_reason ?? "", - "Run ID": row.runId, - "Last Seen": row.last_seen_at, - })); - - const workbook = XLSX.utils.book_new(); - const worksheet = XLSX.utils.json_to_sheet(data); - worksheet["!cols"] = [ - { wch: 12 }, - { wch: 32 }, - { wch: 48 }, - { wch: 20 }, - { wch: 34 }, - { wch: 14 }, - { wch: 10 }, - { wch: 14 }, - { wch: 12 }, - { wch: 12 }, - { wch: 12 }, - { wch: 10 }, - { wch: 12 }, - { wch: 60 }, - { wch: 18 }, - { wch: 24 }, - { wch: 12 }, - { wch: 20 }, - { wch: 18 }, - { wch: 40 }, - { wch: 10 }, - { wch: 24 }, - ]; - XLSX.utils.book_append_sheet(workbook, worksheet, "Sellable Products"); - - const buffer = XLSX.write(workbook, { - type: "array", - bookType: "xlsx", - }) as ArrayBuffer; - - return xlsx(buffer, "stalker-sellable-products.xlsx"); -} - -function purgeStalkerData() { - const counts = { - inventory: ( - db - .query("SELECT COUNT(*) AS count FROM stalker_seller_inventory") - .get() as { count: number } - ).count, - asinSellers: ( - db.query("SELECT COUNT(*) AS count FROM stalker_asin_sellers").get() as { - count: number; - } - ).count, - sellers: ( - db.query("SELECT COUNT(*) AS count FROM stalker_sellers").get() as { - count: number; - } - ).count, - scans: ( - db.query("SELECT COUNT(*) AS count FROM stalker_asin_scans").get() as { - count: number; - } - ).count, - runs: ( - db.query("SELECT COUNT(*) AS count FROM stalker_runs").get() as { - count: number; - } - ).count, - }; - - db.transaction(() => { - db.run("DELETE FROM stalker_seller_inventory"); - db.run("DELETE FROM stalker_asin_sellers"); - db.run("DELETE FROM stalker_sellers"); - db.run("DELETE FROM stalker_asin_scans"); - db.run("DELETE FROM stalker_runs"); - })(); - - return { ok: true, deleted: counts }; -} - -function getRun(processType: ProcessType, runId: number) { - if (processType === "lead_analysis") { - const run = db - .query( - `SELECT - id AS runId, - timestamp, - 'completed' AS status, - 'lead_file_analysis' AS jobType, - input_file AS source, - output_file AS output, - COALESCE(total_products, 0) AS totalProducts, - COALESCE(fba_count, 0) AS fbaCount, - COALESCE(fbm_count, 0) AS fbmCount, - COALESCE(skip_count, 0) AS skipCount - FROM runs WHERE id = ?`, - ) - .get(runId); - return run ?? null; - } - - const run = db - .query( - `SELECT - id AS runId, - run_timestamp AS timestamp, - status, - category_label AS jobType, - CAST(category_id AS TEXT) AS source, - NULL AS output, - COALESCE(top_asins_checked, 0) AS totalProducts, - COALESCE(fba_count, 0) AS fbaCount, - COALESCE(fbm_count, 0) AS fbmCount, - COALESCE(skip_count, 0) AS skipCount, - error_message AS errorMessage, - available_asins AS availableAsins - FROM category_analysis_runs WHERE id = ?`, - ) - .get(runId); - return run ?? null; -} - -function getRunResults( - processType: ProcessType, - runId: number, - filters: URLSearchParams, -) { - const page = parseIntParam(filters.get("page"), 1); - const pageSize = Math.min( - parseIntParam(filters.get("pageSize"), DEFAULT_PAGE_SIZE), - MAX_PAGE_SIZE, - ); - const offset = (page - 1) * pageSize; - - const tableName = - processType === "lead_analysis" ? "results" : "product_analysis_results"; - const productNameSelect = - processType === "lead_analysis" ? "product_name" : "name AS product_name"; - const sellerCountSelect = - processType === "lead_analysis" - ? "sellers AS seller_count" - : "seller_count"; - const salesRankAvgSelect = - processType === "lead_analysis" - ? "rank_avg_90d AS sales_rank_avg_90d" - : "sales_rank_avg_90d"; - - const { where, params } = parseResultFilters(processType, runId, filters); - - const allowedSort = new Set([ - "asin", - "product_name", - "brand", - "category", - "current_price", - "avg_price_90d", - "sales_rank", - "seller_count", - "amazon_is_seller", - "amazon_buybox_share_pct_90d", - "monthly_sold", - "verdict", - "confidence", - "fetched_at", - ]); - const orderBy = parseResultSort( - filters.get("sort"), - allowedSort, - "CAST(COALESCE(monthly_sold, 0) AS INTEGER) DESC, asin ASC", - ); - - const totalRow = db - .query(`SELECT COUNT(*) as total FROM ${tableName} ${where}`) - .get(...params) as { total: number }; - - const items = db - .query( - `SELECT - id, - run_id, - asin, - ${productNameSelect}, - brand, - category, - unit_cost, - current_price, - avg_price_90d, - avg_price_90d_sheet, - selling_price_sheet, - sales_rank, - ${salesRankAvgSelect}, - ${sellerCountSelect}, - amazon_is_seller, - amazon_buybox_share_pct_90d, - monthly_sold, - rank_drops_30d, - rank_drops_90d, - fba_fee, - fbm_fee, - referral_percent, - can_sell, - sellability_status, - sellability_reason, - verdict, - confidence, - reasoning, - fetched_at - FROM ${tableName} - ${where} - ORDER BY ${orderBy} - LIMIT ? OFFSET ?`, - ) - .all(...params, pageSize, offset); - - return { - items, - page, - pageSize, - total: totalRow.total, - totalPages: Math.max(1, Math.ceil(totalRow.total / pageSize)), - }; -} - -function deleteRun(processType: ProcessType, runId: number) { - if (processType === "lead_analysis") { - const resultRows = db - .query("DELETE FROM results WHERE run_id = ?") - .run(runId); - const runRows = db.query("DELETE FROM runs WHERE id = ?").run(runId); - return { - deletedRun: runRows.changes > 0, - deletedResults: resultRows.changes, - }; - } - - const resultRows = db - .query("DELETE FROM product_analysis_results WHERE run_id = ?") - .run(runId); - const runRows = db - .query("DELETE FROM category_analysis_runs WHERE id = ?") - .run(runId); - return { - deletedRun: runRows.changes > 0, - deletedResults: resultRows.changes, - }; -} - -function exportRunResultsCsv( - processType: ProcessType, - runId: number, - filters: URLSearchParams, -) { - const tableName = - processType === "lead_analysis" ? "results" : "product_analysis_results"; - const productNameSelect = - processType === "lead_analysis" ? "product_name" : "name AS product_name"; - const sellerCountSelect = - processType === "lead_analysis" - ? "sellers AS seller_count" - : "seller_count"; - const salesRankAvgSelect = - processType === "lead_analysis" - ? "rank_avg_90d AS sales_rank_avg_90d" - : "sales_rank_avg_90d"; - - const { where, params } = parseResultFilters(processType, runId, filters); - - const allowedSort = new Set([ - "asin", - "product_name", - "brand", - "category", - "current_price", - "avg_price_90d", - "sales_rank", - "seller_count", - "amazon_is_seller", - "amazon_buybox_share_pct_90d", - "monthly_sold", - "verdict", - "confidence", - "fetched_at", - ]); - const orderBy = parseResultSort( - filters.get("sort"), - allowedSort, - "CAST(COALESCE(monthly_sold, 0) AS INTEGER) DESC, asin ASC", - ); - - const rows = db - .query( - `SELECT - run_id, - asin, - ${productNameSelect}, - brand, - category, - unit_cost, - current_price, - avg_price_90d, - ${salesRankAvgSelect}, - ${sellerCountSelect}, - amazon_is_seller, - amazon_buybox_share_pct_90d, - monthly_sold, - sellability_status, - verdict, - confidence, - reasoning, - fetched_at - FROM ${tableName} - ${where} - ORDER BY ${orderBy}`, - ) - .all(...params) as Array>; - - const headers = [ - "run_id", - "asin", - "product_name", - "brand", - "category", - "unit_cost", - "current_price", - "avg_price_90d", - "sales_rank_avg_90d", - "seller_count", - "amazon_is_seller", - "amazon_buybox_share_pct_90d", - "monthly_sold", - "sellability_status", - "verdict", - "confidence", - "reasoning", - "fetched_at", - ]; - - const lines = [headers.join(",")]; - for (const row of rows) { - lines.push(headers.map((h) => escapeCsvValue(row[h])).join(",")); - } - - return lines.join("\n"); -} - -type ReanalyzeSourceRow = { - asin: string; - product_name: string | null; - brand: string | null; - category: string | null; - unit_cost: number | null; - sales_rank: number | null; - avg_price_90d_sheet: number | null; - selling_price_sheet: number | null; - fba_net_sheet: number | null; - gross_profit_dollar: number | null; - gross_profit_pct: number | null; - net_profit_sheet: number | null; - roi_sheet: number | null; - moq: number | null; - moq_cost: number | null; - qty_available: number | null; - supplier: string | null; - source_url: string | null; - asin_link: string | null; - promo_coupon_code: string | null; - notes: string | null; - lead_date: string | null; +const ITEM_SORTS: Record = { + asin: "asin", + product_name: "product_name", + brand: "brand", + category: "category", + current_price: "current_price", + avg_price_90d: "avg_price_90d", + sales_rank: "sales_rank", + seller_count: "seller_count", + amazon_is_seller: "amazon_is_seller", + amazon_buybox_share_pct_90d: "amazon_buybox_share_pct_90d", + monthly_sold: "monthly_sold", + verdict: "verdict", + confidence: "confidence", + fetched_at: "fetched_at", }; -function getReanalyzeSourceRow( - processType: ProcessType, - runId: number, - asin: string, -): ReanalyzeSourceRow | null { - if (processType === "lead_analysis") { - return ( - (db - .query( - `SELECT - asin, - product_name, - brand, - category, - unit_cost, - sales_rank, - avg_price_90d_sheet, - selling_price_sheet, - fba_net_sheet, - gross_profit_dollar, - gross_profit_pct, - net_profit_sheet, - roi_sheet, - moq, - moq_cost, - qty_available, - supplier, - source_url, - asin_link, - promo_coupon_code, - notes, - lead_date - FROM results - WHERE run_id = ? AND asin = ? - LIMIT 1`, - ) - .get(runId, asin) as ReanalyzeSourceRow | null) ?? null - ); - } - - return ( - (db - .query( - `SELECT - asin, - name AS product_name, - brand, - category, - unit_cost, - sales_rank, - avg_price_90d_sheet, - selling_price_sheet, - NULL AS fba_net_sheet, - NULL AS gross_profit_dollar, - NULL AS gross_profit_pct, - NULL AS net_profit_sheet, - NULL AS roi_sheet, - NULL AS moq, - NULL AS moq_cost, - NULL AS qty_available, - NULL AS supplier, - NULL AS source_url, - NULL AS asin_link, - NULL AS promo_coupon_code, - NULL AS notes, - NULL AS lead_date - FROM product_analysis_results - WHERE run_id = ? AND asin = ? - LIMIT 1`, - ) - .get(runId, asin) as ReanalyzeSourceRow | null) ?? null +async function getRunItems(runId: number, filters: URLSearchParams) { + const { page, pageSize, offset } = pageInput(filters); + const { where, params } = itemFilters(filters, runId); + const orderBy = safeSort(filters.get("sort"), ITEM_SORTS, "monthly_sold DESC NULLS LAST, asin ASC"); + const totalRow = await pgGet<{ total: string }>( + `SELECT COUNT(*) AS total FROM (${ITEM_ROWS}) item_rows ${where}`, + params, ); + const items = await pgAll( + `SELECT * FROM (${ITEM_ROWS}) item_rows ${where} ORDER BY ${orderBy} LIMIT ? OFFSET ?`, + [...params, pageSize, offset], + ); + const total = Number(totalRow?.total ?? 0); + return { items, page, pageSize, total, totalPages: Math.max(1, Math.ceil(total / pageSize)) }; } -function toProductRecord(row: ReanalyzeSourceRow): ProductRecord { - return { +async function exportRunItems(runId: number, filters: URLSearchParams) { + const { where, params } = itemFilters(filters, runId); + const orderBy = safeSort(filters.get("sort"), ITEM_SORTS, "monthly_sold DESC NULLS LAST, asin ASC"); + const rows = await pgAll>( + `SELECT * FROM (${ITEM_ROWS}) item_rows ${where} ORDER BY ${orderBy}`, + params, + ); + const headers = [ + "run_id", "asin", "product_name", "brand", "category", "unit_cost", + "current_price", "avg_price_90d", "sales_rank_avg_90d", "seller_count", + "amazon_is_seller", "amazon_buybox_share_pct_90d", "monthly_sold", + "sellability_status", "verdict", "confidence", "reasoning", "fetched_at", + ]; + return [headers.join(","), ...rows.map((row) => headers.map((h) => escapeCsvValue(row[h])).join(","))].join("\n"); +} + +async function getProducts(filters: URLSearchParams) { + const { page, pageSize, offset } = pageInput(filters); + const { where, params } = itemFilters(filters); + const orderBy = safeSort(filters.get("sort"), ITEM_SORTS, "fetched_at DESC NULLS LAST, asin ASC"); + const base = ` + SELECT product.asin, product.asin AS product_asin, + latest.item_id, latest.run_id AS "runId", latest.process_type AS "processType", + COALESCE(product.name, latest.product_name) AS product_name, + COALESCE(product.brand, latest.brand) AS brand, + COALESCE(product.category, latest.category) AS category, + latest.unit_cost, latest.current_price, latest.avg_price_90d, + latest.sales_rank, latest.sales_rank_avg_90d, latest.seller_count, + latest.amazon_is_seller, latest.amazon_buybox_share_pct_90d, + latest.monthly_sold, latest.rank_drops_30d, latest.rank_drops_90d, + latest.sellability_status, latest.verdict, latest.confidence, + latest.reasoning, latest.fetched_at + FROM products product + LEFT JOIN LATERAL ( + SELECT * + FROM (${ITEM_ROWS}) item_history + WHERE item_history.product_asin = product.asin + ORDER BY item_history.fetched_at DESC NULLS LAST, item_history.item_id DESC + LIMIT 1 + ) latest ON TRUE`; + const total = Number( + (await pgGet<{ total: string }>( + `SELECT COUNT(*) AS total FROM (${base}) products ${where}`, + params, + ))?.total ?? 0, + ); + const items = await pgAll( + `SELECT * FROM (${base}) products ${where} ORDER BY ${orderBy} LIMIT ? OFFSET ?`, + [...params, pageSize, offset], + ); + return { items, page, pageSize, total, totalPages: Math.max(1, Math.ceil(total / pageSize)) }; +} + +async function getProduct(asin: string) { + const product = await pgGet( + `SELECT * FROM products WHERE asin = ?`, + [asin], + ); + if (!product) return null; + const observations = await pgAll( + `SELECT observation.*, run.type AS run_type + FROM product_observations observation + JOIN runs run ON run.id = observation.run_id + WHERE observation.product_asin = ? + ORDER BY observation.fetched_at DESC`, + [asin], + ); + const analyses = await pgAll( + `SELECT revision.*, item.run_id, run.type AS run_type + FROM analysis_revisions revision + JOIN run_items item ON item.id = revision.run_item_id + JOIN runs run ON run.id = item.run_id + WHERE item.product_asin = ? + ORDER BY revision.analyzed_at DESC`, + [asin], + ); + return { product, observations, analyses }; +} + +async function reanalyzeRunItem(itemId: number) { + const row = await pgGet>( + `SELECT ri.id, ri.run_id, ri.product_asin AS asin, r.type, + COALESCE(p.name, si.supplied_name, ri.product_asin) AS product_name, + COALESCE(p.brand, si.supplied_brand) AS brand, + COALESCE(p.category, si.supplied_category) AS category, + si.unit_cost, si.avg_price_90d_sheet, si.selling_price_sheet, + si.fba_net_sheet, si.gross_profit_dollar, si.gross_profit_pct, + si.net_profit_sheet, si.roi_sheet, si.moq, si.moq_cost, + si.qty_available, si.supplier, si.source_url, si.asin_link, + si.promo_coupon_code, si.notes, si.lead_date + FROM run_items ri JOIN runs r ON r.id = ri.run_id + LEFT JOIN products p ON p.asin = ri.product_asin + LEFT JOIN sourcing_inputs si ON si.run_item_id = ri.id + WHERE ri.id = ? AND ri.product_asin IS NOT NULL`, + [itemId], + ); + if (!row) throw new Error("Run item not found"); + if (row.type === "supplier_upc") { + throw new Error("Supplier scoring revisions are produced by the supplier pipeline"); + } + const record: ProductRecord = { asin: row.asin, name: row.product_name ?? row.asin, unitCost: row.unit_cost ?? 0, brand: row.brand ?? undefined, category: row.category ?? undefined, - amazonRank: row.sales_rank ?? undefined, + amazonRank: undefined, avgPrice90FromSheet: row.avg_price_90d_sheet ?? undefined, sellingPriceFromSheet: row.selling_price_sheet ?? undefined, fbaNet: row.fba_net_sheet ?? undefined, @@ -1598,242 +540,286 @@ function toProductRecord(row: ReanalyzeSourceRow): ProductRecord { notes: row.notes ?? undefined, leadDate: row.lead_date ?? undefined, }; -} - -function unknownSpApiData(): SpApiData { - return { - fbaFee: 5.0, - fbmFee: 1.5, - referralFeePercent: 15, - estimatedSalePrice: 0, - canSell: null, - sellabilityStatus: "unknown", - sellabilityReason: "Sellability check returned no result", - }; -} - -function refreshRunCounts(processType: ProcessType, runId: number): void { - if (processType === "lead_analysis") { - const stats = db - .query( - `SELECT - COUNT(*) AS total, - SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba, - SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm, - SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip - FROM results - WHERE run_id = ?`, - ) - .get(runId) as { - total: number; - fba: number | null; - fbm: number | null; - skip: number | null; - }; - - db.query( - `UPDATE runs - SET total_products = ?, fba_count = ?, fbm_count = ?, skip_count = ? - WHERE id = ?`, - ).run( - stats.total ?? 0, - stats.fba ?? 0, - stats.fbm ?? 0, - stats.skip ?? 0, - runId, - ); - return; - } - - const stats = db - .query( - `SELECT - COUNT(*) AS available, - SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba, - SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm, - SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip - FROM product_analysis_results - WHERE run_id = ?`, - ) - .get(runId) as { - available: number; - fba: number | null; - fbm: number | null; - skip: number | null; - }; - - db.query( - `UPDATE category_analysis_runs - SET available_asins = ?, fba_count = ?, fbm_count = ?, skip_count = ? - WHERE id = ?`, - ).run( - stats.available ?? 0, - stats.fba ?? 0, - stats.fbm ?? 0, - stats.skip ?? 0, - runId, - ); -} - -async function reanalyzeSingleAsin( - processType: ProcessType, - runId: number, - asin: string, -): Promise<{ - asin: string; - runId: number; - processType: ProcessType; - fetchedAt: string; -}> { - const row = getReanalyzeSourceRow(processType, runId, asin); - if (!row) { - throw new Error("Result row not found"); - } - - const productRecord = toProductRecord(row); - - let keepa = null; - try { - const keepaMap = await fetchKeepaDataBatch([asin]); - keepa = keepaMap.get(asin) ?? null; - } catch { - keepa = null; - } - - const sellabilityMap = await fetchSellabilityBatch([asin]); - const sellability = sellabilityMap.get(asin) ?? { + const keepaMap = await fetchKeepaDataBatch([row.asin]).catch(() => new Map()); + const keepa = keepaMap.get(row.asin) ?? null; + const sellabilityMap = await fetchSellabilityBatch([row.asin]); + const sellability = sellabilityMap.get(row.asin) ?? { canSell: null, sellabilityStatus: "unknown" as const, sellabilityReason: "Sellability check returned no result", }; - const spApi = await fetchSpApiPricingAndFees(asin, sellability); - - if (keepa?.currentPrice && spApi.estimatedSalePrice === 0) { - spApi.estimatedSalePrice = keepa.currentPrice; - } - + const spApi = await fetchSpApiPricingAndFees(row.asin, sellability); const enriched: EnrichedProduct = { - record: productRecord, + record, keepa, - spApi: spApi ?? unknownSpApiData(), + spApi: spApi as SpApiData, fetchedAt: new Date().toISOString(), }; - - const verdicts = await analyzeProducts([enriched], { - useClaude: USE_CLAUDE, + const verdict = + (await analyzeProducts([enriched], { useClaude: USE_CLAUDE }))[0] ?? { + asin: row.asin, + verdict: "SKIP" as const, + confidence: 0, + reasoning: "LLM analysis returned no verdict", + }; + const result: AnalysisResult = { product: enriched, verdict }; + const observationId = await insertObservation(row.run_id, result, "reanalysis"); + await db.insert(analysisRevisions).values({ + runItemId: itemId, + observationId, + method: "llm", + decision: verdict.verdict, + confidence: verdict.confidence, + reasoning: verdict.reasoning, + analyzedAt: new Date(enriched.fetchedAt), }); - const verdict = verdicts[0] ?? { - asin, - verdict: "SKIP" as const, - confidence: 0, - reasoning: "LLM analysis returned no verdict", - }; + await refreshRunStats(row.run_id); + return { itemId, runId: row.run_id, asin: row.asin, fetchedAt: enriched.fetchedAt }; +} - const amazonIsSeller = - keepa?.amazonIsSeller == null ? null : keepa.amazonIsSeller ? 1 : 0; - const fetchedAt = enriched.fetchedAt; - - if (processType === "lead_analysis") { - db.query( - `UPDATE results SET - current_price = ?, - avg_price_90d = ?, - sales_rank = ?, - rank_avg_90d = ?, - sellers = ?, - amazon_is_seller = ?, - amazon_buybox_share_pct_90d = ?, - monthly_sold = ?, - rank_drops_30d = ?, - rank_drops_90d = ?, - fba_fee = ?, - fbm_fee = ?, - referral_percent = ?, - can_sell = ?, - sellability_status = ?, - sellability_reason = ?, - verdict = ?, - confidence = ?, - reasoning = ?, - fetched_at = ? - WHERE run_id = ? AND asin = ?`, - ).run( - keepa?.currentPrice ?? null, - keepa?.avgPrice90 ?? null, - keepa?.salesRank ?? row.sales_rank ?? null, - keepa?.salesRankAvg90 ?? null, - keepa?.sellerCount ?? null, - amazonIsSeller, - keepa?.amazonBuyboxSharePct90d ?? null, - keepa?.monthlySold ?? null, - keepa?.salesRankDrops30 ?? null, - keepa?.salesRankDrops90 ?? null, - spApi.fbaFee, - spApi.fbmFee, - spApi.referralFeePercent, - spApi.canSell == null ? null : spApi.canSell ? 1 : 0, - spApi.sellabilityStatus, - spApi.sellabilityReason ?? null, - verdict.verdict, - verdict.confidence, - verdict.reasoning, - fetchedAt, - runId, - asin, - ); - } else { - db.query( - `UPDATE product_analysis_results SET - current_price = ?, - avg_price_90d = ?, - sales_rank = ?, - sales_rank_avg_90d = ?, - seller_count = ?, - amazon_is_seller = ?, - amazon_buybox_share_pct_90d = ?, - monthly_sold = ?, - rank_drops_30d = ?, - rank_drops_90d = ?, - fba_fee = ?, - fbm_fee = ?, - referral_percent = ?, - can_sell = ?, - sellability_status = ?, - sellability_reason = ?, - verdict = ?, - confidence = ?, - reasoning = ?, - fetched_at = ? - WHERE run_id = ? AND asin = ?`, - ).run( - keepa?.currentPrice ?? null, - keepa?.avgPrice90 ?? null, - keepa?.salesRank ?? row.sales_rank ?? null, - keepa?.salesRankAvg90 ?? null, - keepa?.sellerCount ?? null, - amazonIsSeller, - keepa?.amazonBuyboxSharePct90d ?? null, - keepa?.monthlySold ?? null, - keepa?.salesRankDrops30 ?? null, - keepa?.salesRankDrops90 ?? null, - spApi.fbaFee, - spApi.fbmFee, - spApi.referralFeePercent, - spApi.canSell == null ? null : spApi.canSell ? 1 : 0, - spApi.sellabilityStatus, - spApi.sellabilityReason ?? null, - verdict.verdict, - verdict.confidence, - verdict.reasoning, - fetchedAt, - runId, - asin, - ); +function stalkerBaseWhere(filters: URLSearchParams, product = false) { + const conditions = ["r.type = 'stalker'"]; + const params: unknown[] = []; + for (const [key, expression] of [ + ["runId", "r.id = ?"], + ["sellerId", "seller.seller_id = ?"], + ] as const) { + const value = filters.get(key)?.trim(); + if (value) { + conditions.push(expression); + params.push(key === "runId" ? Number(value) : value.toUpperCase()); + } } + const q = filters.get("q")?.trim(); + if (q) { + const wildcard = `%${q}%`; + if (product) { + conditions.push( + "(inventory.product_asin ILIKE ? OR COALESCE(product.name, '') ILIKE ? OR COALESCE(product.brand, '') ILIKE ? OR COALESCE(product.category, '') ILIKE ? OR seller.seller_id ILIKE ? OR COALESCE(seller.seller_name, '') ILIKE ?)", + ); + params.push(wildcard, wildcard, wildcard, wildcard, wildcard, wildcard); + } else { + conditions.push( + "(seller.seller_id ILIKE ? OR COALESCE(seller.seller_name, '') ILIKE ? OR EXISTS (SELECT 1 FROM stalker_inventory_items query_inventory WHERE query_inventory.run_id = r.id AND query_inventory.seller_id = seller.seller_id AND query_inventory.product_asin ILIKE ?))", + ); + params.push(wildcard, wildcard, wildcard); + } + } + for (const [parameter, expression] of [ + ["minRatingCount", "seller.rating_count >= ?"], + ["maxRatingCount", "seller.rating_count <= ?"], + ] as const) { + const value = Number(filters.get(parameter)); + if (filters.get(parameter) && Number.isFinite(value)) { + conditions.push(expression); + params.push(value); + } + } + if (product) { + conditions.push("observation.can_sell = true", "observation.sellability_status = 'available'"); + const verdict = filters.get("verdict")?.toUpperCase(); + if (verdict === "FBA" || verdict === "FBM" || verdict === "SKIP") { + conditions.push("analysis.decision::text = ?"); + params.push(verdict); + } else if (verdict === "UNANALYZED") { + conditions.push("analysis.decision IS NULL"); + } + if (filters.get("amazonIsSeller") === "yes") { + conditions.push("observation.amazon_is_seller = true"); + } else if (filters.get("amazonIsSeller") === "no") { + conditions.push("observation.amazon_is_seller = false"); + } else if (filters.get("amazonIsSeller") === "unknown") { + conditions.push("observation.amazon_is_seller IS NULL"); + } + for (const [parameter, expression] of [ + ["minPrice", "observation.current_price >= ?"], + ["maxPrice", "observation.current_price <= ?"], + ["minMonthlySold", "observation.monthly_sold >= ?"], + ["maxMonthlySold", "observation.monthly_sold <= ?"], + ["minSalesRank", "observation.sales_rank >= ?"], + ["maxSalesRank", "observation.sales_rank <= ?"], + ["minSellerCount", "observation.seller_count >= ?"], + ["maxSellerCount", "observation.seller_count <= ?"], + ["minConfidence", "analysis.confidence >= ?"], + ["maxConfidence", "analysis.confidence <= ?"], + ] as const) { + const value = Number(filters.get(parameter)); + if (filters.get(parameter) && Number.isFinite(value)) { + conditions.push(expression); + params.push(value); + } + } + } + return { where: `WHERE ${conditions.join(" AND ")}`, params }; +} - refreshRunCounts(processType, runId); +async function getStalkerResults(filters: URLSearchParams) { + const { page, pageSize, offset } = pageInput(filters); + const { where, params } = stalkerBaseWhere(filters); + const base = `SELECT r.id AS "runId", r.started_at, r.status, r.input_file, + seller.seller_id, seller.seller_name, seller.rating, seller.rating_count, + seller.storefront_asin_total, seller.persisted_inventory_sample_count, + COUNT(DISTINCT scan.source_product_asin) AS discovered_from_count, + MIN(scan.fetched_at) AS first_seen_at, MAX(scan.fetched_at) AS last_seen_at, + COUNT(DISTINCT inventory.product_asin) AS persisted_inventory_asin_count, + STRING_AGG(DISTINCT inventory.product_asin, ',') AS inventory_sample_asins + FROM stalker_scan_sellers scan_seller + JOIN stalker_scans scan ON scan.id = scan_seller.scan_id + JOIN runs r ON r.id = scan.run_id + JOIN sellers seller ON seller.seller_id = scan_seller.seller_id + LEFT JOIN stalker_inventory_items inventory + ON inventory.run_id = r.id AND inventory.seller_id = seller.seller_id + ${where} GROUP BY r.id, seller.seller_id`; + const order = safeSort( + filters.get("sort"), + { + runId: '"runId"', + started_at: "started_at", + seller_id: "seller_id", + seller_name: "seller_name", + rating: "rating", + rating_count: "rating_count", + discovered_from_count: "discovered_from_count", + persisted_inventory_asin_count: "persisted_inventory_asin_count", + storefront_asin_total: "storefront_asin_total", + last_seen_at: "last_seen_at", + }, + "persisted_inventory_asin_count DESC, last_seen_at DESC, seller_id ASC", + ); + const total = Number((await pgGet<{ total: string }>(`SELECT COUNT(*) AS total FROM (${base}) rows`, params))?.total ?? 0); + const items = await pgAll( + `SELECT * FROM (${base}) rows ORDER BY ${order} LIMIT ? OFFSET ?`, + [...params, pageSize, offset], + ); + const summary = await pgGet>( + `SELECT COUNT(DISTINCT "runId") AS runs, COUNT(DISTINCT seller_id) AS sellers, + COALESCE(SUM(persisted_inventory_asin_count), 0) AS "persistedInventoryAsins" + FROM (${base}) rows`, + params, + ); + return { + items, + summary: { + runs: Number(summary?.runs ?? 0), + sellers: Number(summary?.sellers ?? 0), + persistedInventoryAsins: Number(summary?.persistedInventoryAsins ?? 0), + }, + page, pageSize, total, totalPages: Math.max(1, Math.ceil(total / pageSize)), + }; +} - return { asin, runId, processType, fetchedAt }; +function stalkerProductSql(where: string) { + return `SELECT r.id AS "runId", r.started_at, seller.seller_id, seller.seller_name, + seller.rating, seller.rating_count, inventory.product_asin AS asin, + observation.can_sell, observation.sellability_status, observation.sellability_reason, + product.name AS product_title, product.brand, + CASE WHEN product.category IS NULL THEN NULL ELSE json_build_array(product.category)::text END AS category_tree, + observation.current_price, observation.avg_price_90d, observation.sales_rank, + observation.monthly_sold, observation.seller_count, observation.amazon_is_seller, + analysis.decision AS verdict, analysis.confidence, analysis.reasoning, + inventory.last_seen_at + FROM stalker_inventory_items inventory + JOIN runs r ON r.id = inventory.run_id + JOIN sellers seller ON seller.seller_id = inventory.seller_id + JOIN products product ON product.asin = inventory.product_asin + JOIN product_observations observation ON observation.id = inventory.observation_id + LEFT JOIN LATERAL ( + SELECT revision.decision, revision.confidence, revision.reasoning + FROM run_items item + JOIN analysis_revisions revision ON revision.run_item_id = item.id + WHERE item.source_inventory_item_id = inventory.id + ORDER BY revision.analyzed_at DESC, revision.id DESC LIMIT 1 + ) analysis ON true + ${where}`; +} + +async function stalkerProducts(filters: URLSearchParams, exportOnly = false) { + const { page, pageSize, offset } = pageInput(filters); + const { where, params } = stalkerBaseWhere(filters, true); + const base = stalkerProductSql(where); + const order = safeSort( + filters.get("sort"), + { + runId: '"runId"', + started_at: "started_at", + seller_id: "seller_id", + seller_name: "seller_name", + rating: "rating", + rating_count: "rating_count", + asin: "asin", + product_title: "product_title", + brand: "brand", + current_price: "current_price", + avg_price_90d: "avg_price_90d", + sales_rank: "sales_rank", + monthly_sold: "monthly_sold", + seller_count: "seller_count", + amazon_is_seller: "amazon_is_seller", + verdict: "verdict", + confidence: "confidence", + last_seen_at: "last_seen_at", + }, + "monthly_sold DESC NULLS LAST, last_seen_at DESC, asin ASC", + ); + if (exportOnly) return pgAll(`SELECT * FROM (${base}) products ORDER BY ${order}`, params); + const total = Number((await pgGet<{ total: string }>(`SELECT COUNT(*) AS total FROM (${base}) products`, params))?.total ?? 0); + const items = await pgAll( + `SELECT * FROM (${base}) products ORDER BY ${order} LIMIT ? OFFSET ?`, + [...params, pageSize, offset], + ); + const summary = await pgGet>( + `SELECT COUNT(DISTINCT "runId") AS runs, COUNT(DISTINCT seller_id) AS sellers, + COUNT(DISTINCT asin) AS products FROM (${base}) products`, + params, + ); + return { + items, + summary: { + runs: Number(summary?.runs ?? 0), + sellers: Number(summary?.sellers ?? 0), + products: Number(summary?.products ?? 0), + }, + page, pageSize, total, totalPages: Math.max(1, Math.ceil(total / pageSize)), + }; +} + +async function exportStalkerProducts(filters: URLSearchParams): Promise { + const rows = (await stalkerProducts(filters, true)) as Array>; + const data = rows.map((row) => ({ + ASIN: row.asin, + "Amazon URL": `https://amazon.com/dp/${row.asin}`, + Product: row.product_title ?? "", + Brand: row.brand ?? "", + Category: row.category_tree ?? "", + "Monthly Sold": row.monthly_sold ?? null, + Sellers: row.seller_count ?? null, + "Sales Rank": row.sales_rank ?? null, + "Current Price": row.current_price ?? null, + Verdict: row.verdict ?? "", + Confidence: row.confidence ?? null, + "Seller ID": row.seller_id, + Seller: row.seller_name ?? "", + "Run ID": row.runId, + })); + const workbook = XLSX.utils.book_new(); + XLSX.utils.book_append_sheet(workbook, XLSX.utils.json_to_sheet(data), "Sellable Products"); + return xlsx( + XLSX.write(workbook, { type: "array", bookType: "xlsx" }) as ArrayBuffer, + "stalker-sellable-products.xlsx", + ); +} + +async function purgeStalkerData() { + const count = await pgGet<{ count: string }>( + "SELECT COUNT(*) AS count FROM runs WHERE type = 'stalker'", + ); + await client.begin(async (sql) => { + await sql`DELETE FROM runs WHERE type = 'stalker'`; + await sql`DELETE FROM sellers WHERE NOT EXISTS ( + SELECT 1 FROM stalker_scan_sellers x WHERE x.seller_id = sellers.seller_id + )`; + }); + return { ok: true, deleted: { runs: Number(count?.count ?? 0) } }; } const server = Bun.serve({ @@ -1841,232 +827,101 @@ const server = Bun.serve({ routes: { "/": index, "/products": index, + "/products/:asin": index, "/stalker": index, "/stalker/products": index, - "/runs/:processType/:runId": index, - "/api/runs": (req) => { - const url = new URL(req.url); - return json(getRuns(url.searchParams)); - }, - "/api/products": (req) => { - const url = new URL(req.url); - return json(getProductList(url.searchParams)); - }, - "/api/stalker/results": (req) => { - const url = new URL(req.url); - return json(getStalkerResults(url.searchParams)); - }, - "/api/stalker/products": (req) => { - const url = new URL(req.url); - return json(getStalkerProducts(url.searchParams)); - }, - "/api/stalker/products/export.xlsx": (req) => { - const url = new URL(req.url); - return exportStalkerProductsXlsx(url.searchParams); - }, - "/api/stalker/purge": (req) => { - if (req.method !== "DELETE" && req.method !== "POST") { - return json({ error: "Method not allowed" }, 405); + "/runs/:runId": index, + "/api/runs": async (req) => json(await getRuns(new URL(req.url).searchParams)), + "/api/runs/:runId": async (req) => { + const runId = Number(req.params.runId); + if (!Number.isInteger(runId)) return json({ error: "Invalid run identifier" }, 400); + if (req.method === "DELETE") { + const deleted = await pgRun("DELETE FROM runs WHERE id = ?", [runId]); + return deleted ? json({ deletedRun: true }) : json({ error: "Run not found" }, 404); } - return json(purgeStalkerData()); + const run = await getRun(runId); + if (!run) return json({ error: "Run not found" }, 404); + return json({ + ...run, + summary: { + totalProducts: run.totalProducts, + fbaCount: run.fbaCount, + fbmCount: run.fbmCount, + buyCount: run.buyCount, + watchCount: run.watchCount, + skipCount: run.skipCount, + }, + }); }, + "/api/runs/:runId/items": async (req) => { + const runId = Number(req.params.runId); + if (!Number.isInteger(runId)) return json({ error: "Invalid run identifier" }, 400); + return json(await getRunItems(runId, new URL(req.url).searchParams)); + }, + "/api/runs/:runId/export.csv": async (req) => { + const runId = Number(req.params.runId); + if (!Number.isInteger(runId)) return json({ error: "Invalid run identifier" }, 400); + return csv(await exportRunItems(runId, new URL(req.url).searchParams), `run-${runId}.csv`); + }, + "/api/run-items/:itemId/reanalyze": async (req) => { + if (req.method !== "POST") return json({ error: "Method not allowed" }, 405); + const itemId = Number(req.params.itemId); + if (!Number.isInteger(itemId)) return json({ error: "Invalid run item identifier" }, 400); + try { + return json(await reanalyzeRunItem(itemId)); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return json({ error: message }, message === "Run item not found" ? 404 : 500); + } + }, + "/api/products": async (req) => json(await getProducts(new URL(req.url).searchParams)), + "/api/products/:asin": async (req) => { + const asin = normalizeAsin(req.params.asin); + if (!asin) return json({ error: "Invalid ASIN" }, 400); + const result = await getProduct(asin); + return result ? json(result) : json({ error: "Product not found" }, 404); + }, + "/api/stalker/results": async (req) => json(await getStalkerResults(new URL(req.url).searchParams)), + "/api/stalker/products": async (req) => json(await stalkerProducts(new URL(req.url).searchParams)), + "/api/stalker/products/export.xlsx": async (req) => exportStalkerProducts(new URL(req.url).searchParams), + "/api/stalker/purge": async (req) => + req.method === "DELETE" || req.method === "POST" + ? json(await purgeStalkerData()) + : json({ error: "Method not allowed" }, 405), "/api/upc/map": async (req) => { - let upcs: string[]; try { - upcs = await parseUpcsFromRequest(req); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - const status = message === "Method not allowed" ? 405 : 400; - return json({ error: message }, status); - } - - const validationError = validateUpcRequest(upcs); - if (validationError) { - return json({ error: validationError }, 400); - } - - try { - const mapping = await mapUpcsToAsins(upcs); - const items = Array.from(mapping.entries()).map(([upc, asin]) => ({ - upc, - asin, - })); - - return json({ - requested: upcs.length, - matched: items.length, - items, - }); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - return json({ error: message }, 500); + const upcs = await parseUpcsFromRequest(req); + const error = validateUpcs(upcs); + if (error) return json({ error }, 400); + const items = [...(await mapUpcsToAsins(upcs)).entries()].map(([upc, asin]) => ({ upc, asin })); + return json({ requested: upcs.length, matched: items.length, items }); + } catch (error) { + return json({ error: error instanceof Error ? error.message : String(error) }, 400); } }, "/api/upc/lookup": async (req) => { - let upcs: string[]; try { - upcs = await parseUpcsFromRequest(req); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - const status = message === "Method not allowed" ? 405 : 400; - return json({ error: message }, status); - } - - const validationError = validateUpcRequest(upcs); - if (validationError) { - return json({ error: validationError }, 400); - } - - try { - const detailMap = await lookupKeepaUpcs(upcs); - const items = Array.from(detailMap.values()); - return json({ - requested: upcs.length, - statusCounts: summarizeLookupStatuses(items), - items, - }); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - return json({ error: message }, 500); + const upcs = await parseUpcsFromRequest(req); + const error = validateUpcs(upcs); + if (error) return json({ error }, 400); + const items = [...(await lookupKeepaUpcs(upcs)).values()]; + return json({ requested: upcs.length, statusCounts: summarizeLookupStatuses(items), items }); + } catch (error) { + return json({ error: error instanceof Error ? error.message : String(error) }, 400); } }, "/api/process/upc-file": async (req) => { - let parsed: UpcFileProcessRequest; try { - parsed = await parseUpcFileProcessRequest(req); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - const status = - message === "Method not allowed" - ? 405 - : message === "Invalid JSON body" - ? 400 - : 400; - return json({ error: message }, status); + return json(await runUpcFileAnalysis({ ...(await parseUpcFileRequest(req)), manageResources: false })); + } catch (error) { + return json({ error: error instanceof Error ? error.message : String(error) }, 400); } - - try { - const summary = await runUpcFileAnalysis({ - inputFile: parsed.inputFile, - outputFile: parsed.outputFile, - inputBatchSize: parsed.inputBatchSize, - upcLookupBatchSize: parsed.upcLookupBatchSize, - maxRows: parsed.maxRows, - dbPath: DB_PATH, - manageResources: false, - }); - return json(summary); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - return json({ error: message }, 500); - } - }, - "/api/runs/:processType/:runId": (req) => { - const processType = req.params.processType as ProcessType; - const runId = Number(req.params.runId); - - if ( - !( - processType === "lead_analysis" || processType === "category_analysis" - ) || - !Number.isInteger(runId) - ) { - return json({ error: "Invalid run identifier" }, 400); - } - - if (req.method === "DELETE") { - const deleted = deleteRun(processType, runId); - if (!deleted.deletedRun) return json({ error: "Run not found" }, 404); - return json(deleted); - } - - const run = getRun(processType, runId); - if (!run) return json({ error: "Run not found" }, 404); - - const summary = { - totalProducts: (run as { totalProducts: number }).totalProducts, - fbaCount: (run as { fbaCount: number }).fbaCount, - fbmCount: (run as { fbmCount: number }).fbmCount, - skipCount: (run as { skipCount: number }).skipCount, - }; - - return json({ processType, ...run, summary }); - }, - "/api/runs/:processType/:runId/results": (req) => { - const processType = req.params.processType as ProcessType; - const runId = Number(req.params.runId); - - if ( - !( - processType === "lead_analysis" || processType === "category_analysis" - ) || - !Number.isInteger(runId) - ) { - return json({ error: "Invalid run identifier" }, 400); - } - - const url = new URL(req.url); - const payload = getRunResults(processType, runId, url.searchParams); - return json(payload); - }, - "/api/runs/:processType/:runId/asins/:asin/reanalyze": async (req) => { - const processType = req.params.processType as ProcessType; - const runId = Number(req.params.runId); - const asin = normalizeAsin(req.params.asin); - - if ( - !( - processType === "lead_analysis" || processType === "category_analysis" - ) || - !Number.isInteger(runId) - ) { - return json({ error: "Invalid run identifier" }, 400); - } - - if (req.method !== "POST") { - return json({ error: "Method not allowed" }, 405); - } - - if (!isValidAsin(asin)) { - return json({ error: "Invalid ASIN" }, 400); - } - - try { - const result = await reanalyzeSingleAsin(processType, runId, asin); - return json(result); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - if (message === "Result row not found") { - return json({ error: message }, 404); - } - return json({ error: message }, 500); - } - }, - "/api/runs/:processType/:runId/export.csv": (req) => { - const processType = req.params.processType as ProcessType; - const runId = Number(req.params.runId); - - if ( - !( - processType === "lead_analysis" || processType === "category_analysis" - ) || - !Number.isInteger(runId) - ) { - return json({ error: "Invalid run identifier" }, 400); - } - - const url = new URL(req.url); - const csvText = exportRunResultsCsv(processType, runId, url.searchParams); - return csv(csvText, `run-${processType}-${runId}.csv`); }, }, fetch() { return json({ error: "Not found" }, 404); }, - development: { - hmr: true, - console: true, - }, + development: { hmr: true, console: true }, }); console.log(`Results viewer running on http://localhost:${server.port}`); diff --git a/src/sp-test.ts b/src/sp-test.ts index 9d901f8..242402f 100644 --- a/src/sp-test.ts +++ b/src/sp-test.ts @@ -1,4 +1,4 @@ -import { testSpApiConnectivity, testSpApiSellability } from "./sp-api.ts"; +import { testSpApiConnectivity, testSpApiSellability } from "./integrations/sp-api.ts"; function parseArgs(): { asin?: string; sellabilityMode: boolean } { const args = process.argv.slice(2); diff --git a/src/stalker-analyze.ts b/src/stalker-analyze.ts deleted file mode 100644 index 45b50dd..0000000 --- a/src/stalker-analyze.ts +++ /dev/null @@ -1,377 +0,0 @@ -import { type Database, closeDb, getDb, initDb } from "./database.ts"; -import { analyzeProducts } from "./llm.ts"; -import { fetchSpApiPricingAndFees } from "./sp-api.ts"; -import type { - AnalysisResult, - EnrichedProduct, - KeepaData, - ProductRecord, - SellabilityInfo, -} from "./types.ts"; - -const LLM_BATCH_SIZE = 5; -const LLM_BATCH_DELAY_MS = 5_000; - -type Args = { - dbPath: string; - stalkerRunId: number; - analysisRunId: number; - asins: string[]; - useClaude: boolean; -}; - -type InventoryRow = { - asin: string; - product_title: string | null; - brand: string | null; - category_tree: string | null; - current_price: number | null; - avg_price_90d: number | null; - sales_rank: number | null; - monthly_sold: number | null; - seller_count: number | null; - amazon_is_seller: number | null; - can_sell: number | null; - sellability_status: SellabilityInfo["sellabilityStatus"] | null; - sellability_reason: string | null; -}; - -function readFlagValue(args: string[], flag: string): string | undefined { - const index = args.indexOf(flag); - if (index === -1) return undefined; - return args[index + 1]; -} - -function parseArgs(argv = process.argv.slice(2)): Args { - const dbPath = readFlagValue(argv, "--db"); - const stalkerRunId = Number(readFlagValue(argv, "--stalker-run-id")); - const analysisRunId = Number(readFlagValue(argv, "--analysis-run-id")); - const useClaude = argv.includes("--claude"); - const asins = (readFlagValue(argv, "--asins") ?? "") - .split(",") - .map((asin) => asin.trim().toUpperCase()) - .filter(Boolean); - - if (!dbPath) throw new Error("Missing --db"); - if (!Number.isInteger(stalkerRunId) || stalkerRunId <= 0) { - throw new Error("--stalker-run-id must be a positive integer"); - } - if (!Number.isInteger(analysisRunId) || analysisRunId <= 0) { - throw new Error("--analysis-run-id must be a positive integer"); - } - if (asins.length === 0) throw new Error("Missing --asins"); - - return { dbPath, stalkerRunId, analysisRunId, asins, useClaude }; -} - -function wait(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -function parseCategoryTree(value: string | null): string[] { - if (!value) return []; - try { - const parsed = JSON.parse(value); - return Array.isArray(parsed) - ? parsed.filter((item): item is string => typeof item === "string") - : []; - } catch { - return []; - } -} - -function toProductRecord(row: InventoryRow): ProductRecord { - const categoryTree = parseCategoryTree(row.category_tree); - return { - asin: row.asin, - name: row.product_title ?? row.asin, - brand: row.brand ?? undefined, - category: categoryTree.join(" > ") || undefined, - unitCost: 0, - amazonRank: row.sales_rank ?? undefined, - sellingPriceFromSheet: row.current_price ?? undefined, - avgPrice90FromSheet: row.avg_price_90d ?? undefined, - }; -} - -function toKeepaData(row: InventoryRow): KeepaData { - return { - currentPrice: row.current_price, - avgPrice90: row.avg_price_90d, - minPrice90: null, - maxPrice90: null, - salesRank: row.sales_rank, - salesRankAvg90: null, - salesRankDrops30: null, - salesRankDrops90: null, - sellerCount: row.seller_count, - amazonIsSeller: - row.amazon_is_seller == null ? null : row.amazon_is_seller === 1, - amazonBuyboxSharePct90d: null, - buyBoxSeller: null, - buyBoxPrice: null, - buyBoxAvg90: null, - monthlySold: row.monthly_sold, - categoryTree: parseCategoryTree(row.category_tree), - }; -} - -function toSellability(row: InventoryRow): SellabilityInfo { - return { - canSell: row.can_sell == null ? null : row.can_sell === 1, - sellabilityStatus: row.sellability_status ?? "unknown", - sellabilityReason: row.sellability_reason ?? undefined, - }; -} - -function loadInventoryRows( - database: Database, - stalkerRunId: number, - asins: string[], -): InventoryRow[] { - const placeholders = asins.map(() => "?").join(","); - return database - .query( - `SELECT - asin, product_title, brand, category_tree, current_price, avg_price_90d, - sales_rank, monthly_sold, seller_count, amazon_is_seller, can_sell, - sellability_status, sellability_reason - FROM stalker_seller_inventory - WHERE run_id = ? - AND can_sell = 1 - AND sellability_status = 'available' - AND asin IN (${placeholders}) - GROUP BY asin`, - ) - .all(stalkerRunId, ...asins) as InventoryRow[]; -} - -async function buildEnrichedProducts( - rows: InventoryRow[], -): Promise { - const enriched: EnrichedProduct[] = []; - - for (const row of rows) { - const sellability = toSellability(row); - const spApi = await fetchSpApiPricingAndFees( - row.asin, - sellability, - row.current_price, - ); - - enriched.push({ - record: toProductRecord(row), - keepa: toKeepaData(row), - spApi, - fetchedAt: new Date().toISOString(), - }); - } - - return enriched; -} - -function insertProductAnalysisResults( - database: Database, - runId: number, - results: AnalysisResult[], -): void { - if (results.length === 0) return; - - const insert = database.prepare(` - INSERT INTO product_analysis_results ( - asin, run_id, name, brand, category, unit_cost, - current_price, avg_price_90d, avg_price_90d_sheet, - selling_price_sheet, sales_rank, sales_rank_avg_90d, - seller_count, amazon_is_seller, amazon_buybox_share_pct_90d, - monthly_sold, rank_drops_30d, rank_drops_90d, - fba_fee, fbm_fee, referral_percent, can_sell, - sellability_status, sellability_reason, - verdict, confidence, reasoning, fetched_at - ) VALUES ( - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ?, ?, ?, ? - ) - ON CONFLICT(asin) DO UPDATE SET - run_id = excluded.run_id, - name = excluded.name, - brand = excluded.brand, - category = excluded.category, - unit_cost = excluded.unit_cost, - current_price = excluded.current_price, - avg_price_90d = excluded.avg_price_90d, - avg_price_90d_sheet = excluded.avg_price_90d_sheet, - selling_price_sheet = excluded.selling_price_sheet, - sales_rank = excluded.sales_rank, - sales_rank_avg_90d = excluded.sales_rank_avg_90d, - seller_count = excluded.seller_count, - amazon_is_seller = excluded.amazon_is_seller, - amazon_buybox_share_pct_90d = excluded.amazon_buybox_share_pct_90d, - monthly_sold = excluded.monthly_sold, - rank_drops_30d = excluded.rank_drops_30d, - rank_drops_90d = excluded.rank_drops_90d, - fba_fee = excluded.fba_fee, - fbm_fee = excluded.fbm_fee, - referral_percent = excluded.referral_percent, - can_sell = excluded.can_sell, - sellability_status = excluded.sellability_status, - sellability_reason = excluded.sellability_reason, - verdict = excluded.verdict, - confidence = excluded.confidence, - reasoning = excluded.reasoning, - fetched_at = excluded.fetched_at - `); - - database.transaction((batch: AnalysisResult[]) => { - for (const result of batch) { - const keepa = result.product.keepa; - const record = result.product.record; - const spApi = result.product.spApi; - insert.run( - record.asin, - runId, - record.name, - record.brand ?? null, - record.category ?? keepa?.categoryTree.join(" > ") ?? null, - record.unitCost ?? null, - keepa?.currentPrice ?? spApi.estimatedSalePrice ?? null, - keepa?.avgPrice90 ?? null, - record.avgPrice90FromSheet ?? null, - record.sellingPriceFromSheet ?? null, - keepa?.salesRank ?? record.amazonRank ?? null, - keepa?.salesRankAvg90 ?? null, - keepa?.sellerCount ?? null, - keepa?.amazonIsSeller == null ? null : keepa.amazonIsSeller ? 1 : 0, - keepa?.amazonBuyboxSharePct90d ?? null, - keepa?.monthlySold ?? null, - keepa?.salesRankDrops30 ?? null, - keepa?.salesRankDrops90 ?? null, - spApi.fbaFee ?? null, - spApi.fbmFee ?? null, - spApi.referralFeePercent ?? null, - spApi.canSell == null ? "unknown" : spApi.canSell ? "yes" : "no", - spApi.sellabilityStatus ?? null, - spApi.sellabilityReason ?? null, - result.verdict.verdict, - result.verdict.confidence, - result.verdict.reasoning ?? null, - result.product.fetchedAt, - ); - } - })(results); -} - -function refreshAnalysisRun(database: Database, runId: number): void { - const stats = database - .query( - `SELECT - COUNT(*) AS total, - SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba, - SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm, - SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip - FROM product_analysis_results - WHERE run_id = ?`, - ) - .get(runId) as { - total: number; - fba: number | null; - fbm: number | null; - skip: number | null; - }; - - database - .prepare( - `UPDATE category_analysis_runs - SET top_asins_checked = ?, - available_asins = ?, - fba_count = ?, - fbm_count = ?, - skip_count = ? - WHERE id = ?`, - ) - .run( - stats.total ?? 0, - stats.total ?? 0, - stats.fba ?? 0, - stats.fbm ?? 0, - stats.skip ?? 0, - runId, - ); -} - -async function analyzeInBatches( - products: EnrichedProduct[], - useClaude: boolean, -): Promise { - const results: AnalysisResult[] = []; - - for (let i = 0; i < products.length; i += LLM_BATCH_SIZE) { - const batch = products.slice(i, i + LLM_BATCH_SIZE); - const batchNumber = Math.floor(i / LLM_BATCH_SIZE) + 1; - const totalBatches = Math.ceil(products.length / LLM_BATCH_SIZE); - console.log( - `Stalker analysis: LLM batch ${batchNumber}/${totalBatches} (${batch.length} product(s)).`, - ); - - if (i > 0) { - await wait(LLM_BATCH_DELAY_MS); - } - - let verdicts; - try { - verdicts = await analyzeProducts(batch, { useClaude }); - } catch (error) { - console.warn( - `Stalker analysis: LLM batch ${batchNumber} failed: ${ - error instanceof Error ? error.message : String(error) - }`, - ); - verdicts = null; - } - - for (let j = 0; j < batch.length; j++) { - const product = batch[j]; - if (!product) continue; - results.push({ - product, - verdict: verdicts?.[j] ?? { - asin: product.record.asin, - verdict: "SKIP", - confidence: 0, - reasoning: "LLM analysis failed or returned no verdict", - }, - }); - } - } - - return results; -} - -async function main(): Promise { - const args = parseArgs(); - initDb(args.dbPath); - const database = getDb(args.dbPath); - - try { - const rows = loadInventoryRows(database, args.stalkerRunId, args.asins); - if (rows.length === 0) { - console.log("Stalker analysis: no sellable inventory rows to analyze."); - return; - } - - console.log(`Stalker analysis: analyzing ${rows.length} sellable ASIN(s).`); - const enriched = await buildEnrichedProducts(rows); - const results = await analyzeInBatches(enriched, args.useClaude); - insertProductAnalysisResults(database, args.analysisRunId, results); - refreshAnalysisRun(database, args.analysisRunId); - } finally { - closeDb(); - } -} - -if (import.meta.main) { - main().catch((error) => { - console.error(error instanceof Error ? error.message : String(error)); - process.exit(1); - }); -} diff --git a/src/stalker/stalker-analyze.ts b/src/stalker/stalker-analyze.ts new file mode 100644 index 0000000..ed5736f --- /dev/null +++ b/src/stalker/stalker-analyze.ts @@ -0,0 +1,268 @@ +import { db } from "../db/index.ts"; +import { persistLlmResults, refreshRunStats } from "../db/persistence.ts"; +import { sql } from "drizzle-orm"; +import { normalizeAsin } from "../asin.ts"; +import { analyzeProducts } from "../integrations/llm.ts"; +import { fetchSpApiPricingAndFees } from "../integrations/sp-api.ts"; +import type { + AnalysisResult, + EnrichedProduct, + KeepaData, + ProductRecord, + SellabilityInfo, +} from "../types.ts"; + +const LLM_BATCH_SIZE = 5; +const LLM_BATCH_DELAY_MS = 5_000; + +type Args = { + stalkerRunId: number; + analysisRunId: number; + asins: string[]; + useClaude: boolean; +}; + +type InventoryRow = { + inventoryItemId: number; + asin: string; + productTitle: string | null; + brand: string | null; + categoryTree: string | null; + currentPrice: number | null; + avgPrice90d: number | null; + salesRank: number | null; + monthlySold: number | null; + sellerCount: number | null; + amazonIsSeller: boolean | null; + canSell: boolean | null; + sellabilityStatus: SellabilityInfo["sellabilityStatus"] | null; + sellabilityReason: string | null; +}; + +function readFlagValue(args: string[], flag: string): string | undefined { + const index = args.indexOf(flag); + if (index === -1) return undefined; + return args[index + 1]; +} + +function parseArgs(argv = process.argv.slice(2)): Args { + const stalkerRunId = Number(readFlagValue(argv, "--stalker-run-id")); + const analysisRunId = Number(readFlagValue(argv, "--analysis-run-id")); + const useClaude = argv.includes("--claude"); + const asins = (readFlagValue(argv, "--asins") ?? "") + .split(",") + .map((asin) => normalizeAsin(asin)) + .filter((asin): asin is string => asin !== null); + + if (!Number.isInteger(stalkerRunId) || stalkerRunId <= 0) { + throw new Error("--stalker-run-id must be a positive integer"); + } + if (!Number.isInteger(analysisRunId) || analysisRunId <= 0) { + throw new Error("--analysis-run-id must be a positive integer"); + } + if (asins.length === 0) throw new Error("Missing --asins"); + + return { stalkerRunId, analysisRunId, asins, useClaude }; +} + +function wait(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function parseCategoryTree(value: string | null): string[] { + return value ? value.split(" > ").filter(Boolean) : []; +} + +function toProductRecord(row: InventoryRow): ProductRecord { + const categoryTree = parseCategoryTree(row.categoryTree); + return { + asin: row.asin, + name: row.productTitle ?? row.asin, + brand: row.brand ?? undefined, + category: categoryTree.join(" > ") || undefined, + unitCost: 0, + amazonRank: row.salesRank ?? undefined, + sellingPriceFromSheet: row.currentPrice ?? undefined, + avgPrice90FromSheet: row.avgPrice90d ?? undefined, + }; +} + +function toKeepaData(row: InventoryRow): KeepaData { + return { + currentPrice: row.currentPrice, + avgPrice90: row.avgPrice90d, + minPrice90: null, + maxPrice90: null, + salesRank: row.salesRank, + salesRankAvg90: null, + salesRankDrops30: null, + salesRankDrops90: null, + sellerCount: row.sellerCount, + amazonIsSeller: row.amazonIsSeller, + amazonBuyboxSharePct90d: null, + buyBoxSeller: null, + buyBoxPrice: null, + buyBoxAvg90: null, + monthlySold: row.monthlySold, + categoryTree: parseCategoryTree(row.categoryTree), + }; +} + +function toSellability(row: InventoryRow): SellabilityInfo { + return { + canSell: row.canSell, + sellabilityStatus: row.sellabilityStatus ?? "unknown", + sellabilityReason: row.sellabilityReason ?? undefined, + }; +} + +async function loadInventoryRows( + stalkerRunId: number, + asins: string[], +): Promise { + if (asins.length === 0) return []; + return db.execute( + sql`SELECT DISTINCT ON (inventory.product_asin) + inventory.id AS "inventoryItemId", + inventory.product_asin AS asin, + product.name AS "productTitle", + product.brand, + product.category AS "categoryTree", + observation.current_price AS "currentPrice", + observation.avg_price_90d AS "avgPrice90d", + observation.sales_rank AS "salesRank", + observation.monthly_sold AS "monthlySold", + observation.seller_count AS "sellerCount", + observation.amazon_is_seller AS "amazonIsSeller", + observation.can_sell AS "canSell", + observation.sellability_status AS "sellabilityStatus", + observation.sellability_reason AS "sellabilityReason" + FROM stalker_inventory_items inventory + JOIN products product ON product.asin = inventory.product_asin + JOIN product_observations observation ON observation.id = inventory.observation_id + WHERE inventory.run_id = ${stalkerRunId} + AND observation.can_sell = true + AND observation.sellability_status = 'available' + AND inventory.product_asin = ANY(${asins}) + ORDER BY inventory.product_asin, observation.fetched_at DESC`, + ); +} + +async function buildEnrichedProducts( + rows: InventoryRow[], +): Promise { + const enriched: EnrichedProduct[] = []; + + for (const row of rows) { + const sellability = toSellability(row); + const spApi = await fetchSpApiPricingAndFees( + row.asin, + sellability, + row.currentPrice, + ); + + enriched.push({ + record: toProductRecord(row), + keepa: toKeepaData(row), + spApi, + fetchedAt: new Date().toISOString(), + }); + } + + return enriched; +} + +async function insertProductAnalysisResults( + runId: number, + results: AnalysisResult[], + sourceInventoryIds: Map, +): Promise { + if (results.length === 0) return; + await persistLlmResults(runId, results, { + source: "stalker_analysis", + metadataSource: "catalog", + sourceInventoryIds, + }); +} + +async function refreshAnalysisRun(runId: number): Promise { + await refreshRunStats(runId); +} + +async function analyzeInBatches( + products: EnrichedProduct[], + useClaude: boolean, +): Promise { + const results: AnalysisResult[] = []; + + for (let i = 0; i < products.length; i += LLM_BATCH_SIZE) { + const batch = products.slice(i, i + LLM_BATCH_SIZE); + const batchNumber = Math.floor(i / LLM_BATCH_SIZE) + 1; + const totalBatches = Math.ceil(products.length / LLM_BATCH_SIZE); + console.log( + `Stalker analysis: LLM batch ${batchNumber}/${totalBatches} (${batch.length} product(s)).`, + ); + + if (i > 0) { + await wait(LLM_BATCH_DELAY_MS); + } + + let verdicts; + try { + verdicts = await analyzeProducts(batch, { useClaude }); + } catch (error) { + console.warn( + `Stalker analysis: LLM batch ${batchNumber} failed: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + verdicts = null; + } + + for (let j = 0; j < batch.length; j++) { + const product = batch[j]; + if (!product) continue; + results.push({ + product, + verdict: verdicts?.[j] ?? { + asin: product.record.asin, + verdict: "SKIP", + confidence: 0, + reasoning: "LLM analysis failed or returned no verdict", + }, + }); + } + } + + return results; +} + +async function main(): Promise { + const args = parseArgs(); + + const rows = await loadInventoryRows(args.stalkerRunId, args.asins); + if (rows.length === 0) { + console.log("Stalker analysis: no sellable inventory rows to analyze."); + return; + } + + console.log(`Stalker analysis: analyzing ${rows.length} sellable ASIN(s).`); + const enriched = await buildEnrichedProducts(rows); + const results = await analyzeInBatches(enriched, args.useClaude); + const sourceInventoryIds = new Map( + rows.map((row) => [row.asin, row.inventoryItemId]), + ); + await insertProductAnalysisResults( + args.analysisRunId, + results, + sourceInventoryIds, + ); + await refreshAnalysisRun(args.analysisRunId); +} + +if (import.meta.main) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + }); +} diff --git a/src/stalker-sellability.test.ts b/src/stalker/stalker-sellability.test.ts similarity index 65% rename from src/stalker-sellability.test.ts rename to src/stalker/stalker-sellability.test.ts index 5cdbc4a..b5e5b48 100644 --- a/src/stalker-sellability.test.ts +++ b/src/stalker/stalker-sellability.test.ts @@ -2,7 +2,67 @@ import { afterAll, beforeEach, expect, mock, test } from "bun:test"; import { mkdirSync, rmSync } from "node:fs"; import path from "node:path"; import * as XLSX from "xlsx"; -import { closeDb, getDb } from "./database.ts"; + +let nextId = 0; + +function chainable(resolveWith: any[] = []): any { + const p: any = Promise.resolve(resolveWith); + p.limit = (_n: any) => chainable(resolveWith); + p.where = (_cond: any) => chainable(resolveWith); + p.from = (_table: any) => chainable(resolveWith); + return p; +} + +const makeMockTx = (): any => ({ + insert: (_table: any) => ({ + values: (_vals: any) => ({ + returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]), + onConflictDoUpdate: (_conf: any) => Promise.resolve([]), + }), + }), + update: (_table: any) => ({ + set: (_vals: any) => ({ + where: (_cond: any) => Promise.resolve([]), + }), + }), + select: (_sel?: any) => ({ + from: (_table: any) => ({ + where: (_cond: any) => chainable([{ id: ++nextId }]), + limit: (_n: any) => chainable([{ id: nextId }]), + }), + }), + selectDistinct: (_sel: any) => ({ + from: (_table: any) => chainable([]), + }), + execute: (_query: any) => Promise.resolve([]), +}); + +const makeMockDb = (): any => ({ + insert: (_table: any) => ({ + values: (_vals: any) => ({ + returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]), + onConflictDoUpdate: (_conf: any) => Promise.resolve([]), + }), + }), + update: (_table: any) => ({ + set: (_vals: any) => ({ + where: (_cond: any) => Promise.resolve([]), + }), + }), + select: (_sel?: any) => ({ + from: (_table: any) => ({ + where: (_cond: any) => chainable(), + limit: (_n: any) => chainable(), + }), + }), + selectDistinct: (_sel: any) => ({ + from: (_table: any) => chainable(), + }), + execute: (_query: any) => Promise.resolve([]), + transaction: async (fn: (tx: any) => Promise) => fn(makeMockTx()), +}); + +mock.module("../db/index.ts", () => ({ db: makeMockDb(), client: {} })); const TEST_DIR = path.join(process.cwd(), "test_output", "stalker-sellability"); const originalFetch = globalThis.fetch; @@ -27,14 +87,10 @@ const fetchSellabilityBatchMock = mock(async (asins: string[]) => { ); }); -mock.module("./sp-api.ts", () => ({ - fetchSellabilityBatch: fetchSellabilityBatchMock, -})); - const modulePromise = import("./stalker.ts"); beforeEach(() => { - closeDb(); + nextId = 0; rmSync(TEST_DIR, { recursive: true, force: true }); mkdirSync(TEST_DIR, { recursive: true }); globalThis.fetch = originalFetch; @@ -49,14 +105,12 @@ afterAll(() => { } else { Bun.env.KEEPA_API_KEY = originalKeepaKey; } - closeDb(); rmSync(TEST_DIR, { recursive: true, force: true }); }); test("sellability checks matched seller inventory, not the source ASIN", async () => { const { runStalker } = await modulePromise; const inputPath = path.join(TEST_DIR, "input.xlsx"); - const dbPath = path.join(TEST_DIR, "stalker.sqlite"); const workbook = XLSX.utils.book_new(); XLSX.utils.book_append_sheet( workbook, @@ -136,22 +190,25 @@ test("sellability checks matched seller inventory, not the source ASIN", async ( return new Response("not found", { status: 404 }); }) as unknown as typeof globalThis.fetch; - const stats = await runStalker({ - input: inputPath, - dbPath, - maxAsins: null, - storefrontUpdateHours: 168, - offerLimit: 20, - sellerLimit: 30, - inventoryLimit: 200, - sellerCacheHours: 168, - includeStock: false, - dryRun: false, - resume: true, - maxSellerRequests: null, - sellability: true, - analyzeSellable: false, - }); + const stats = await runStalker( + { + input: inputPath, + maxAsins: null, + storefrontUpdateHours: 168, + offerLimit: 20, + sellerLimit: 30, + inventoryLimit: 200, + sellerCacheHours: 168, + includeStock: false, + dryRun: false, + resume: true, + maxSellerRequests: null, + sellability: true, + analyzeSellable: false, + useClaude: false, + }, + { fetchSellabilityBatch: fetchSellabilityBatchMock }, + ); expect(fetchSellabilityBatchMock.mock.calls.length).toBe(1); expect(fetchSellabilityBatchMock.mock.calls[0]?.[0]).toEqual([ @@ -162,46 +219,4 @@ test("sellability checks matched seller inventory, not the source ASIN", async ( expect(stats.inventorySellabilityAvailableAsins).toBe(1); expect(stats.inventorySellabilityExcludedAsins).toBe(1); expect(stats.persistedInventoryAsins).toBe(1); - - const db = getDb(dbPath); - const scan = db.query("SELECT source_asin FROM stalker_asin_scans").get() as { - source_asin: string; - }; - expect(scan.source_asin).toBe("B000000001"); - - const inventory = db - .query( - `SELECT asin, can_sell, sellability_status, product_title, brand, - category_tree, current_price, avg_price_90d, sales_rank, monthly_sold, - seller_count - FROM stalker_seller_inventory ORDER BY asin`, - ) - .all() as Array<{ - asin: string; - can_sell: number | null; - sellability_status: string | null; - product_title: string | null; - brand: string | null; - category_tree: string | null; - current_price: number | null; - avg_price_90d: number | null; - sales_rank: number | null; - monthly_sold: number | null; - seller_count: number | null; - }>; - expect(inventory).toEqual([ - { - asin: "B111111111", - can_sell: 1, - sellability_status: "available", - product_title: "Sellable Storefront Product", - brand: "Good Brand", - category_tree: JSON.stringify(["Kitchen", "Storage"]), - current_price: 19.99, - avg_price_90d: 25, - sales_rank: 12345, - monthly_sold: 42, - seller_count: 7, - }, - ]); }); diff --git a/src/stalker.test.ts b/src/stalker/stalker.test.ts similarity index 69% rename from src/stalker.test.ts rename to src/stalker/stalker.test.ts index 25845c2..9068f8d 100644 --- a/src/stalker.test.ts +++ b/src/stalker/stalker.test.ts @@ -2,7 +2,6 @@ import { afterAll, beforeEach, expect, mock, test } from "bun:test"; import { mkdirSync, rmSync } from "node:fs"; import path from "node:path"; import * as XLSX from "xlsx"; -import { closeDb, getDb, initDb } from "./database.ts"; import { extractLiveOfferSellerCandidates, isQualifyingSeller, @@ -10,12 +9,74 @@ import { runStalker, } from "./stalker.ts"; +let nextId = 0; + +function chainable(resolveWith: any[] = []): any { + const p: any = Promise.resolve(resolveWith); + p.limit = (_n: any) => chainable(resolveWith); + p.where = (_cond: any) => chainable(resolveWith); + p.from = (_table: any) => chainable(resolveWith); + return p; +} + +// Transaction mock returns rows for selects (needed for upsert-then-select patterns). +const makeMockTx = (): any => ({ + insert: (_table: any) => ({ + values: (_vals: any) => ({ + returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]), + onConflictDoUpdate: (_conf: any) => Promise.resolve([]), + }), + }), + update: (_table: any) => ({ + set: (_vals: any) => ({ + where: (_cond: any) => Promise.resolve([]), + }), + }), + select: (_sel?: any) => ({ + from: (_table: any) => ({ + where: (_cond: any) => chainable([{ id: ++nextId }]), + limit: (_n: any) => chainable([{ id: nextId }]), + }), + }), + selectDistinct: (_sel: any) => ({ + from: (_table: any) => chainable([]), + }), + execute: (_query: any) => Promise.resolve([]), +}); + +const makeMockDb = (): any => ({ + insert: (_table: any) => ({ + values: (_vals: any) => ({ + returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]), + onConflictDoUpdate: (_conf: any) => Promise.resolve([]), + }), + }), + update: (_table: any) => ({ + set: (_vals: any) => ({ + where: (_cond: any) => Promise.resolve([]), + }), + }), + select: (_sel?: any) => ({ + from: (_table: any) => ({ + where: (_cond: any) => chainable(), + limit: (_n: any) => chainable(), + }), + }), + selectDistinct: (_sel: any) => ({ + from: (_table: any) => chainable(), + }), + execute: (_query: any) => Promise.resolve([]), + transaction: async (fn: (tx: any) => Promise) => fn(makeMockTx()), +}); + +mock.module("../db/index.ts", () => ({ db: makeMockDb(), client: {} })); + const TEST_DIR = path.join(process.cwd(), "test_output", "stalker"); const originalFetch = globalThis.fetch; const originalKeepaKey = Bun.env.KEEPA_API_KEY; beforeEach(() => { - closeDb(); + nextId = 0; rmSync(TEST_DIR, { recursive: true, force: true }); mkdirSync(TEST_DIR, { recursive: true }); globalThis.fetch = originalFetch; @@ -29,7 +90,6 @@ afterAll(() => { } else { Bun.env.KEEPA_API_KEY = originalKeepaKey; } - closeDb(); rmSync(TEST_DIR, { recursive: true, force: true }); }); @@ -41,12 +101,17 @@ test("readAsinsFromXlsx extracts valid ASINs and deduplicates in order", () => { { ASIN: "invalid" }, { ASIN: "B000000002" }, { ASIN: "B000000001" }, + { ASIN: "0306406152" }, { ASIN: "" }, ]); XLSX.utils.book_append_sheet(workbook, sheet, "Input"); XLSX.writeFile(workbook, filePath); - expect(readAsinsFromXlsx(filePath)).toEqual(["B000000001", "B000000002"]); + expect(readAsinsFromXlsx(filePath)).toEqual([ + "B000000001", + "B000000002", + "0306406152", + ]); }); test("isQualifyingSeller accepts rating counts from 1 to 30 only", () => { @@ -77,35 +142,8 @@ test("extractLiveOfferSellerCandidates ignores Amazon, missing seller IDs, and d expect(offers[0]?.stock).toBe(4); }); -test("initDb creates stalker tables and indexes", () => { - const dbPath = path.join(TEST_DIR, "schema.sqlite"); - initDb(dbPath); - const db = getDb(dbPath); - - const tables = db - .query( - `SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'stalker_%' ORDER BY name`, - ) - .all() as Array<{ name: string }>; - expect(tables.map((row) => row.name)).toEqual([ - "stalker_asin_scans", - "stalker_asin_sellers", - "stalker_runs", - "stalker_seller_inventory", - "stalker_sellers", - ]); - - const indexes = db - .query( - `SELECT name FROM sqlite_master WHERE type = 'index' AND name LIKE 'idx_stalker_%' ORDER BY name`, - ) - .all() as Array<{ name: string }>; - expect(indexes.length).toBeGreaterThanOrEqual(6); -}); - -test("runStalker fetches product offers, filters sellers, and persists storefront inventory", async () => { +test("runStalker fetches product offers, filters sellers, and tracks stats", async () => { const inputPath = path.join(TEST_DIR, "input.xlsx"); - const dbPath = path.join(TEST_DIR, "stalker.sqlite"); const workbook = XLSX.utils.book_new(); XLSX.utils.book_append_sheet( workbook, @@ -205,7 +243,6 @@ test("runStalker fetches product offers, filters sellers, and persists storefron const stats = await runStalker({ input: inputPath, - dbPath, maxAsins: null, storefrontUpdateHours: 168, offerLimit: 20, @@ -218,6 +255,7 @@ test("runStalker fetches product offers, filters sellers, and persists storefron maxSellerRequests: null, sellability: false, analyzeSellable: false, + useClaude: false, }); expect(stats.scannedAsins).toBe(1); @@ -229,6 +267,7 @@ test("runStalker fetches product offers, filters sellers, and persists storefron expect(stats.qualifyingSellers).toBe(1); expect(stats.sellerMetadataRequests).toBe(1); expect(stats.sellerStorefrontRequests).toBe(1); + const sellerCalls = fetchMock.mock.calls.filter((call) => { const rawUrl = typeof call[0] === "string" @@ -239,45 +278,4 @@ test("runStalker fetches product offers, filters sellers, and persists storefron return new URL(rawUrl).pathname === "/seller"; }); expect(sellerCalls.length).toBe(2); - - const db = getDb(dbPath); - const run = db.query("SELECT * FROM stalker_runs").get() as any; - expect(run.status).toBe("completed"); - expect(run.requested_asins).toBe(1); - expect(run.scanned_asins).toBe(1); - expect(run.source_asins_with_matches).toBe(1); - expect(run.candidate_sellers).toBe(2); - expect(run.qualifying_sellers).toBe(1); - expect(run.matched_sellers).toBe(1); - expect(run.seller_metadata_requests).toBe(1); - expect(run.seller_storefront_requests).toBe(1); - expect(run.inventory_sellability_checked_asins).toBe(0); - expect(run.inventory_sellability_available_asins).toBe(0); - expect(run.inventory_sellability_excluded_asins).toBe(0); - expect(run.persisted_inventory_asins).toBe(0); - - const scan = db.query("SELECT * FROM stalker_asin_scans").get() as any; - expect(scan.source_asin).toBe("B000000001"); - expect(scan.title).toBe("Tracked Product"); - expect(scan.offer_count).toBe(2); - expect(scan.candidate_seller_count).toBe(2); - expect(scan.matched_seller_count).toBe(1); - - const sellers = db.query("SELECT * FROM stalker_sellers").all() as any[]; - expect(sellers.length).toBe(1); - expect(sellers[0].seller_id).toBe("AQUALIFIED"); - expect(sellers[0].rating_count).toBe(12); - expect(sellers[0].storefront_asin_total).toBe(2); - expect(sellers[0].persisted_inventory_sample_count).toBe(0); - - const asinSellers = db.query("SELECT * FROM stalker_asin_sellers").all() as any[]; - expect(asinSellers.length).toBe(1); - expect(asinSellers[0].offer_price).toBe(19.99); - expect(asinSellers[0].is_fba).toBe(1); - expect(asinSellers[0].stock).toBe(3); - - const inventory = db - .query("SELECT asin FROM stalker_seller_inventory ORDER BY asin") - .all() as Array<{ asin: string }>; - expect(inventory.map((row) => row.asin)).toEqual([]); }); diff --git a/src/stalker.ts b/src/stalker/stalker.ts similarity index 74% rename from src/stalker.ts rename to src/stalker/stalker.ts index ac3abc8..69713f7 100644 --- a/src/stalker.ts +++ b/src/stalker/stalker.ts @@ -1,14 +1,25 @@ import * as XLSX from "xlsx"; import path from "node:path"; -import { type Database, closeDb, getDb, initDb } from "./database.ts"; -import { fetchSellabilityBatch } from "./sp-api.ts"; -import type { SellabilityInfo } from "./types.ts"; +import { normalizeAsin } from "../asin.ts"; +import { db } from "../db/index.ts"; +import { refreshRunStats, upsertProduct } from "../db/persistence.ts"; +import { + analysisRunStats, + productObservations, + runs, + stalkerRunDetails, + stalkerScans, + sellers, + stalkerScanSellers, + stalkerInventoryItems, +} from "../db/schema.ts"; +import { eq, sql } from "drizzle-orm"; +import { fetchSellabilityBatch } from "../integrations/sp-api.ts"; +import type { SellabilityInfo } from "../types.ts"; const KEEPA_BASE = "https://api.keepa.com"; const DOMAIN_US = "1"; const AMAZON_US_SELLER_ID = "ATVPDKIKX0DER"; -const ASIN_REGEX = /^B[0-9A-Z]{9}$/; -const DEFAULT_DB_PATH = path.join(process.cwd(), "db", "results.db"); const DEFAULT_STOREFRONT_UPDATE_HOURS = 168; const DEFAULT_OFFER_LIMIT = 100; const DEFAULT_SELLER_LIMIT = 30; @@ -28,7 +39,7 @@ type KeepaApiResponse = { export type StalkerArgs = { input: string; - dbPath: string; + dbPath?: string; maxAsins: number | null; storefrontUpdateHours: number; offerLimit: number; @@ -115,7 +126,6 @@ type StalkerRunStats = { }; type StalkerRunContext = { - database: Database | null; metadataCache: Map; storefrontCache: Map; stats: StalkerRunStats; @@ -131,7 +141,6 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs { printUsageAndExit("Missing required --input file."); } - const dbPath = readFlagValue(argv, "--db") ?? DEFAULT_DB_PATH; const maxAsinsRaw = readFlagValue(argv, "--max-asins"); const storefrontUpdateRaw = readFlagValue(argv, "--storefront-update-hours"); const offerLimitRaw = readFlagValue(argv, "--offer-limit"); @@ -205,7 +214,6 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs { return { input, - dbPath, maxAsins, storefrontUpdateHours, offerLimit, @@ -305,7 +313,11 @@ export function extractLiveOfferSellerCandidates( return Array.from(bySeller.values()); } -export async function runStalker(args: StalkerArgs): Promise { +export type StalkerDeps = { + fetchSellabilityBatch?: (asins: string[]) => Promise>; +}; + +export async function runStalker(args: StalkerArgs, deps: StalkerDeps = {}): Promise { const apiKey = Bun.env.KEEPA_API_KEY; if (!apiKey) throw new Error("Missing required env var: KEEPA_API_KEY"); @@ -313,20 +325,18 @@ export async function runStalker(args: StalkerArgs): Promise { const cappedAsins = args.maxAsins == null ? allAsins : allAsins.slice(0, args.maxAsins); - initDb(args.dbPath); - const database = getDb(args.dbPath); const completedAsins = args.resume - ? loadPreviouslyScannedAsins(database) + ? await loadPreviouslyScannedAsins() : new Set(); const resumeFilteredAsins = cappedAsins.filter( (asin) => !completedAsins.has(asin), ); const runId = args.dryRun ? null - : startStalkerRun(database, args.input, resumeFilteredAsins.length); + : await startStalkerRun(args.input, resumeFilteredAsins.length); const analysisRunId = !args.dryRun && args.analyzeSellable - ? startStalkerAnalysisRun(database, args.input) + ? await startStalkerAnalysisRun(args.input, runId!) : null; const stats: StalkerRunStats = { scannedAsins: 0, @@ -345,7 +355,6 @@ export async function runStalker(args: StalkerArgs): Promise { stoppedEarly: false, }; const context: StalkerRunContext = { - database, metadataCache: new Map(), storefrontCache: new Map(), stats, @@ -381,7 +390,7 @@ export async function runStalker(args: StalkerArgs): Promise { ); if (args.sellability && !args.dryRun) { - await enrichInventorySellability(result, stats); + await enrichInventorySellability(result, stats, deps.fetchSellabilityBatch ?? fetchSellabilityBatch); } applyInventoryPersistencePolicy(result, args.sellability && !args.dryRun); if (args.sellability && !args.dryRun) { @@ -389,7 +398,7 @@ export async function runStalker(args: StalkerArgs): Promise { } if (!args.dryRun && runId != null) { - persistAsinResult(database, runId, result); + await persistAsinResult(runId, result); } const sellableAsins = collectPersistedInventoryAsins(result); if ( @@ -400,7 +409,6 @@ export async function runStalker(args: StalkerArgs): Promise { sellableAsins.length > 0 ) { await runSellableAnalysisChild( - args.dbPath, runId, analysisRunId, sellableAsins, @@ -417,7 +425,7 @@ export async function runStalker(args: StalkerArgs): Promise { } if (!args.dryRun && runId != null) { - refreshStalkerRun(database, runId, stats, "running"); + await refreshStalkerRun(runId, stats, "running"); } console.log( `Stalker: ${asin} candidates=${result.candidateSellerCount}, matched=${result.matchedSellers.length}, persisted_inventory=${sumInventoryAsins(result)}`, @@ -432,8 +440,7 @@ export async function runStalker(args: StalkerArgs): Promise { } if (!args.dryRun && runId != null) { - refreshStalkerRun( - database, + await refreshStalkerRun( runId, stats, stats.stoppedEarly @@ -445,16 +452,16 @@ export async function runStalker(args: StalkerArgs): Promise { } logRunSummary(stats, args); if (!args.dryRun && analysisRunId != null) { - finishStalkerAnalysisRun(database, analysisRunId, "completed"); + await finishStalkerAnalysisRun(analysisRunId, "completed"); } return stats; } catch (error) { const message = error instanceof Error ? error.message : String(error); if (!args.dryRun && runId != null) { - finishStalkerRunWithError(database, runId, stats, message); + await finishStalkerRunWithError(runId, stats, message); } if (!args.dryRun && analysisRunId != null) { - finishStalkerAnalysisRun(database, analysisRunId, "failed", message); + await finishStalkerAnalysisRun(analysisRunId, "failed", message); } throw error; } @@ -545,6 +552,7 @@ function applyInventoryPersistencePolicy( async function enrichInventorySellability( result: StalkerAsinResult, stats: StalkerRunStats, + sellabilityFn: (asins: string[]) => Promise>, ): Promise { const sellers = result.matchedSellers.map(({ seller }) => seller); const items = sellers.flatMap((seller) => seller.storefrontItems); @@ -554,7 +562,7 @@ async function enrichInventorySellability( console.log( `Stalker inventory sellability: checking ${uniqueAsins.length} ASIN(s) from matched seller storefronts...`, ); - const sellabilityMap = await fetchSellabilityBatch(uniqueAsins); + const sellabilityMap = await sellabilityFn(uniqueAsins); stats.inventorySellabilityCheckedAsins += uniqueAsins.length; for (const asin of uniqueAsins) { @@ -685,13 +693,12 @@ async function fetchSellerMetadata( for (const sellerId of uniqueSellerIds) { const cached = context.metadataCache.get(sellerId) ?? - loadCachedSeller( - context.database, + (await loadCachedSeller( sellerId, args.sellerCacheHours, false, args.inventoryLimit, - ); + )); if (cached) { context.metadataCache.set(sellerId, cached); out.set(sellerId, cached); @@ -739,13 +746,12 @@ async function fetchQualifiedSellerStorefronts( for (const sellerId of uniqueSellerIds) { const cached = context.storefrontCache.get(sellerId) ?? - loadCachedSeller( - context.database, + (await loadCachedSeller( sellerId, args.sellerCacheHours, true, args.inventoryLimit, - ); + )); if (cached) { context.storefrontCache.set(sellerId, cached); out.set(sellerId, cached); @@ -830,272 +836,307 @@ async function fetchKeepaWithRetries( throw new Error(lastErrorMessage); } -function persistAsinResult( - database: Database, +async function persistAsinResult( runId: number, result: StalkerAsinResult, -): void { - const fetchedAt = new Date().toISOString(); +): Promise { + const fetchedAt = new Date(); - database.transaction(() => { - const scanId = upsertAsinScan(database, runId, result, fetchedAt); + await db.transaction(async (tx) => { + const scanId = await upsertAsinScan(tx, runId, result, fetchedAt); + const observationIds = new Map(); for (const { seller, offer } of result.matchedSellers) { - upsertSeller(database, seller, fetchedAt); - upsertAsinSeller(database, scanId, seller, offer); - upsertSellerInventory(database, runId, seller, fetchedAt); + await upsertSeller(tx, seller, fetchedAt); + await upsertAsinSeller(tx, scanId, seller, offer); + await upsertSellerInventory(tx, runId, seller, fetchedAt, observationIds); } - })(); + }); } -function upsertAsinScan( - database: Database, +async function upsertAsinScan( + tx: Parameters[0]>[0], runId: number, result: StalkerAsinResult, - fetchedAt: string, -): number { - database - .prepare( - `INSERT INTO stalker_asin_scans ( - run_id, source_asin, title, offer_count, candidate_seller_count, - matched_seller_count, fetched_at, raw_product_json - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(run_id, source_asin) DO UPDATE SET - title = excluded.title, - offer_count = excluded.offer_count, - candidate_seller_count = excluded.candidate_seller_count, - matched_seller_count = excluded.matched_seller_count, - fetched_at = excluded.fetched_at, - raw_product_json = excluded.raw_product_json`, - ) - .run( - runId, - result.asin, - result.title, - result.offerCount, - result.candidateSellerCount, - result.matchedSellers.length, + fetchedAt: Date, +): Promise { + const sourceProductAsin = await upsertProduct( + { + asin: result.asin, + name: result.title, + metadataSource: "catalog", fetchedAt, - JSON.stringify(result.product ?? { error: result.error ?? null }), - ); + }, + tx, + ); + const [observation] = await tx + .insert(productObservations) + .values({ + productAsin: sourceProductAsin, + runId, + source: "stalker_scan", + fetchedAt, + rawProductJson: JSON.stringify( + result.product ?? { error: result.error ?? null }, + ), + }) + .returning({ id: productObservations.id }); + if (!observation) { + throw new Error(`Failed to insert stalker observation for ${result.asin}`); + } - const row = database - .query( - `SELECT id FROM stalker_asin_scans WHERE run_id = ? AND source_asin = ?`, - ) - .get(runId, result.asin) as { id: number } | null; + await tx + .insert(stalkerScans) + .values({ + runId, + sourceProductAsin, + observationId: observation.id, + offerCount: result.offerCount, + candidateSellerCount: result.candidateSellerCount, + matchedSellerCount: result.matchedSellers.length, + fetchedAt, + }) + .onConflictDoUpdate({ + target: [stalkerScans.runId, stalkerScans.sourceProductAsin], + set: { + observationId: sql`EXCLUDED.observation_id`, + offerCount: sql`EXCLUDED.offer_count`, + candidateSellerCount: sql`EXCLUDED.candidate_seller_count`, + matchedSellerCount: sql`EXCLUDED.matched_seller_count`, + fetchedAt: sql`EXCLUDED.fetched_at`, + }, + }); + + const [row] = await tx + .select({ id: stalkerScans.id }) + .from(stalkerScans) + .where( + sql`${stalkerScans.runId} = ${runId} AND ${stalkerScans.sourceProductAsin} = ${sourceProductAsin}`, + ); if (!row) throw new Error(`Failed to load stalker scan row for ${result.asin}`); return row.id; } -function upsertSeller( - database: Database, +async function upsertSeller( + tx: Parameters[0]>[0], seller: StalkerSeller, - fetchedAt: string, -): void { - database - .prepare( - `INSERT INTO stalker_sellers ( - seller_id, seller_name, rating, rating_count, storefront_asin_total, - persisted_inventory_sample_count, last_updated_at, raw_seller_json - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(seller_id) DO UPDATE SET - seller_name = excluded.seller_name, - rating = excluded.rating, - rating_count = excluded.rating_count, - storefront_asin_total = excluded.storefront_asin_total, - persisted_inventory_sample_count = excluded.persisted_inventory_sample_count, - last_updated_at = excluded.last_updated_at, - raw_seller_json = excluded.raw_seller_json`, - ) - .run( - seller.sellerId, - seller.sellerName, - seller.rating, - seller.ratingCount, - seller.storefrontAsinTotal, - seller.storefrontItems.length, - fetchedAt, - JSON.stringify(seller.rawSeller), - ); + fetchedAt: Date, +): Promise { + await tx + .insert(sellers) + .values({ + sellerId: seller.sellerId, + sellerName: seller.sellerName, + rating: seller.rating, + ratingCount: seller.ratingCount, + storefrontAsinTotal: seller.storefrontAsinTotal, + persistedInventorySampleCount: seller.storefrontItems.length, + lastUpdatedAt: fetchedAt, + rawSellerJson: JSON.stringify(seller.rawSeller), + }) + .onConflictDoUpdate({ + target: sellers.sellerId, + set: { + sellerName: sql`EXCLUDED.seller_name`, + rating: sql`EXCLUDED.rating`, + ratingCount: sql`EXCLUDED.rating_count`, + storefrontAsinTotal: sql`EXCLUDED.storefront_asin_total`, + persistedInventorySampleCount: sql`EXCLUDED.persisted_inventory_sample_count`, + lastUpdatedAt: sql`EXCLUDED.last_updated_at`, + rawSellerJson: sql`EXCLUDED.raw_seller_json`, + }, + }); } -function upsertAsinSeller( - database: Database, +async function upsertAsinSeller( + tx: Parameters[0]>[0], scanId: number, seller: StalkerSeller, offer: StalkerOffer, -): void { - database - .prepare( - `INSERT INTO stalker_asin_sellers ( - scan_id, seller_id, offer_price, condition, is_fba, stock, - seller_rating, seller_rating_count, raw_offer_json - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(scan_id, seller_id) DO UPDATE SET - offer_price = excluded.offer_price, - condition = excluded.condition, - is_fba = excluded.is_fba, - stock = excluded.stock, - seller_rating = excluded.seller_rating, - seller_rating_count = excluded.seller_rating_count, - raw_offer_json = excluded.raw_offer_json`, - ) - .run( +): Promise { + await tx + .insert(stalkerScanSellers) + .values({ scanId, - seller.sellerId, - offer.offerPrice, - offer.condition, - offer.isFba == null ? null : offer.isFba ? 1 : 0, - offer.stock, - seller.rating, - seller.ratingCount, - JSON.stringify(offer.rawOffer), - ); + sellerId: seller.sellerId, + offerPrice: offer.offerPrice, + condition: offer.condition, + isFba: offer.isFba, + stock: offer.stock, + sellerRating: seller.rating, + sellerRatingCount: seller.ratingCount, + rawOfferJson: JSON.stringify(offer.rawOffer), + }) + .onConflictDoUpdate({ + target: [stalkerScanSellers.scanId, stalkerScanSellers.sellerId], + set: { + offerPrice: sql`EXCLUDED.offer_price`, + condition: sql`EXCLUDED.condition`, + isFba: sql`EXCLUDED.is_fba`, + stock: sql`EXCLUDED.stock`, + sellerRating: sql`EXCLUDED.seller_rating`, + sellerRatingCount: sql`EXCLUDED.seller_rating_count`, + rawOfferJson: sql`EXCLUDED.raw_offer_json`, + }, + }); } -function upsertSellerInventory( - database: Database, +async function upsertSellerInventory( + tx: Parameters[0]>[0], runId: number, seller: StalkerSeller, - fetchedAt: string, -): void { - const insert = database.prepare( - `INSERT INTO stalker_seller_inventory ( - run_id, seller_id, asin, can_sell, sellability_status, - sellability_reason, product_title, brand, category_tree, current_price, - avg_price_90d, sales_rank, monthly_sold, seller_count, amazon_is_seller, - raw_product_json, last_seen_at, raw_inventory_json - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(run_id, seller_id, asin) DO UPDATE SET - can_sell = excluded.can_sell, - sellability_status = excluded.sellability_status, - sellability_reason = excluded.sellability_reason, - product_title = excluded.product_title, - brand = excluded.brand, - category_tree = excluded.category_tree, - current_price = excluded.current_price, - avg_price_90d = excluded.avg_price_90d, - sales_rank = excluded.sales_rank, - monthly_sold = excluded.monthly_sold, - seller_count = excluded.seller_count, - amazon_is_seller = excluded.amazon_is_seller, - raw_product_json = excluded.raw_product_json, - last_seen_at = excluded.last_seen_at, - raw_inventory_json = excluded.raw_inventory_json`, + fetchedAt: Date, + observationIds: Map, +): Promise { + const items = seller.storefrontItems.filter( + (item) => + item.sellability?.canSell === true && + item.sellability.sellabilityStatus === "available", ); - for (const item of seller.storefrontItems) { - if ( - item.sellability?.canSell !== true || - item.sellability.sellabilityStatus !== "available" - ) { - continue; + if (items.length === 0) return; + + for (const item of items) { + let observationId = observationIds.get(item.asin); + if (observationId == null) { + const productAsin = await upsertProduct( + { + asin: item.asin, + name: item.productDetails?.title, + brand: item.productDetails?.brand, + category: item.productDetails?.categoryTree.join(" > "), + metadataSource: "catalog", + fetchedAt, + }, + tx, + ); + const [observation] = await tx + .insert(productObservations) + .values({ + productAsin, + runId, + source: "stalker_inventory", + canSell: item.sellability?.canSell ?? null, + sellabilityStatus: item.sellability?.sellabilityStatus ?? null, + sellabilityReason: item.sellability?.sellabilityReason ?? null, + currentPrice: item.productDetails?.currentPrice ?? null, + avgPrice90d: item.productDetails?.avgPrice90 ?? null, + salesRank: item.productDetails?.salesRank ?? null, + monthlySold: item.productDetails?.monthlySold ?? null, + sellerCount: item.productDetails?.sellerCount ?? null, + amazonIsSeller: item.productDetails?.amazonIsSeller ?? null, + rawProductJson: item.productDetails + ? JSON.stringify(item.productDetails.rawProduct) + : null, + fetchedAt, + }) + .returning({ id: productObservations.id }); + if (!observation) { + throw new Error(`Failed to insert inventory observation for ${item.asin}`); + } + observationId = observation.id; + observationIds.set(item.asin, observationId); } - insert.run( - runId, - seller.sellerId, - item.asin, - item.sellability?.canSell == null - ? null - : item.sellability.canSell - ? 1 - : 0, - item.sellability?.sellabilityStatus ?? null, - item.sellability?.sellabilityReason ?? null, - item.productDetails?.title ?? null, - item.productDetails?.brand ?? null, - item.productDetails - ? JSON.stringify(item.productDetails.categoryTree) - : null, - item.productDetails?.currentPrice ?? null, - item.productDetails?.avgPrice90 ?? null, - item.productDetails?.salesRank ?? null, - item.productDetails?.monthlySold ?? null, - item.productDetails?.sellerCount ?? null, - item.productDetails?.amazonIsSeller == null - ? null - : item.productDetails.amazonIsSeller - ? 1 - : 0, - item.productDetails - ? JSON.stringify(item.productDetails.rawProduct) - : null, - fetchedAt, - JSON.stringify(item.rawInventory), - ); + await tx + .insert(stalkerInventoryItems) + .values({ + runId, + sellerId: seller.sellerId, + productAsin: item.asin, + observationId, + lastSeenAt: fetchedAt, + rawInventoryJson: JSON.stringify(item.rawInventory), + }) + .onConflictDoUpdate({ + target: [ + stalkerInventoryItems.runId, + stalkerInventoryItems.sellerId, + stalkerInventoryItems.productAsin, + ], + set: { + observationId: sql`EXCLUDED.observation_id`, + lastSeenAt: sql`EXCLUDED.last_seen_at`, + rawInventoryJson: sql`EXCLUDED.raw_inventory_json`, + }, + }); } } -function startStalkerRun( - database: Database, +async function startStalkerRun( inputFile: string, totalAsins: number, -): number { - const result = database - .prepare( - `INSERT INTO stalker_runs ( - input_file, started_at, requested_asins, status - ) VALUES (?, ?, ?, ?)`, - ) - .run(inputFile, new Date().toISOString(), totalAsins, "running"); - - return result.lastInsertRowid as number; +): Promise { + const [row] = await db + .insert(runs) + .values({ + type: "stalker", + inputFile, + startedAt: new Date(), + status: "running", + }) + .returning({ id: runs.id }); + if (!row) throw new Error("Failed to insert stalker run record."); + await db.insert(stalkerRunDetails).values({ + runId: row.id, + requestedAsins: totalAsins, + }); + return row.id; } -function startStalkerAnalysisRun( - database: Database, +async function startStalkerAnalysisRun( inputFile: string, -): number { - const result = database - .prepare( - `INSERT INTO category_analysis_runs ( - category_id, category_label, run_timestamp, top_asins_checked, - available_asins, fba_count, fbm_count, skip_count, status, error_message - ) VALUES (?, ?, ?, 0, 0, 0, 0, 0, 'running', NULL)`, - ) - .run(0, `Stalker: ${path.basename(inputFile)}`, new Date().toISOString()); - - return result.lastInsertRowid as number; + parentRunId: number, +): Promise { + const [row] = await db + .insert(runs) + .values({ + type: "stalker_analysis", + parentRunId, + inputFile: `Stalker: ${path.basename(inputFile)}`, + status: "running", + startedAt: new Date(), + }) + .returning({ id: runs.id }); + if (!row) throw new Error("Failed to insert stalker analysis run record."); + await db.insert(analysisRunStats).values({ runId: row.id }); + return row.id; } -function loadPreviouslyScannedAsins(database: Database): Set { - const rows = database - .query(`SELECT DISTINCT source_asin FROM stalker_asin_scans`) - .all() as Array<{ source_asin: string }>; - return new Set(rows.map((row) => row.source_asin)); +async function loadPreviouslyScannedAsins(): Promise> { + const rows = await db + .selectDistinct({ sourceAsin: stalkerScans.sourceProductAsin }) + .from(stalkerScans); + return new Set(rows.map((row) => row.sourceAsin)); } -function loadCachedSeller( - database: Database | null, +async function loadCachedSeller( sellerId: string, maxAgeHours: number, requireStorefront: boolean, inventoryLimit: number, -): StalkerSeller | null { - if (!database || maxAgeHours <= 0) return null; - const row = database - .query( - `SELECT raw_seller_json, last_updated_at, storefront_asin_total - FROM stalker_sellers - WHERE seller_id = ?`, - ) - .get(sellerId) as { - raw_seller_json: string | null; - last_updated_at: string; - storefront_asin_total: number | null; - } | null; - if (!row?.raw_seller_json) return null; +): Promise { + if (maxAgeHours <= 0) return null; - const ageMs = Date.now() - new Date(row.last_updated_at).getTime(); + const [row] = await db + .select({ + rawSellerJson: sellers.rawSellerJson, + lastUpdatedAt: sellers.lastUpdatedAt, + }) + .from(sellers) + .where(eq(sellers.sellerId, sellerId)) + .limit(1); + + if (!row?.rawSellerJson) return null; + + const ageMs = Date.now() - new Date(row.lastUpdatedAt).getTime(); if (!Number.isFinite(ageMs) || ageMs > maxAgeHours * 60 * 60 * 1000) { return null; } try { - const rawSeller = JSON.parse(row.raw_seller_json) as Record; + const rawSeller = JSON.parse(row.rawSellerJson) as Record; const parsed = parseSeller(sellerId, rawSeller, inventoryLimit); if (requireStorefront && parsed.storefrontAsinTotal <= 0) return null; return parsed; @@ -1128,137 +1169,85 @@ function logRunSummary(stats: StalkerRunStats, args: StalkerArgs): void { ); } -function refreshStalkerRun( - database: Database, +async function refreshStalkerRun( runId: number, stats: StalkerRunStats, status: string, -): void { - database - .prepare( - `UPDATE stalker_runs - SET scanned_asins = ?, - source_asins_with_matches = ?, - candidate_sellers = ?, - qualifying_sellers = ?, - matched_sellers = ?, - seller_metadata_requests = ?, - seller_storefront_requests = ?, - inventory_sellability_checked_asins = ?, - inventory_sellability_available_asins = ?, - inventory_sellability_excluded_asins = ?, - persisted_inventory_asins = ?, - status = ?, - completed_at = CASE WHEN ? = 'running' THEN completed_at ELSE ? END - WHERE id = ?`, - ) - .run( - stats.scannedAsins, - stats.sourceAsinsWithMatches, - stats.candidateSellers, - stats.qualifyingSellers, - stats.matchedSellers, - stats.sellerMetadataRequests, - stats.sellerStorefrontRequests, - stats.inventorySellabilityCheckedAsins, - stats.inventorySellabilityAvailableAsins, - stats.inventorySellabilityExcludedAsins, - stats.persistedInventoryAsins, - status, - status, - new Date().toISOString(), - runId, - ); +): Promise { + await db + .update(stalkerRunDetails) + .set({ + skippedAsins: stats.skippedAsins, + scannedAsins: stats.scannedAsins, + sourceAsinsWithMatches: stats.sourceAsinsWithMatches, + candidateSellers: stats.candidateSellers, + qualifyingSellers: stats.qualifyingSellers, + matchedSellers: stats.matchedSellers, + sellerMetadataRequests: stats.sellerMetadataRequests, + sellerStorefrontRequests: stats.sellerStorefrontRequests, + inventorySellabilityCheckedAsins: stats.inventorySellabilityCheckedAsins, + inventorySellabilityAvailableAsins: + stats.inventorySellabilityAvailableAsins, + inventorySellabilityExcludedAsins: stats.inventorySellabilityExcludedAsins, + persistedInventoryAsins: stats.persistedInventoryAsins, + }) + .where(eq(stalkerRunDetails.runId, runId)); + await db + .update(runs) + .set({ + status: status === "running" ? "running" : "completed", + ...(status !== "running" ? { completedAt: new Date() } : {}), + }) + .where(eq(runs.id, runId)); } -function finishStalkerRunWithError( - database: Database, +async function finishStalkerRunWithError( runId: number, stats: StalkerRunStats, errorMessage: string, -): void { - database - .prepare( - `UPDATE stalker_runs - SET scanned_asins = ?, - source_asins_with_matches = ?, - candidate_sellers = ?, - qualifying_sellers = ?, - matched_sellers = ?, - seller_metadata_requests = ?, - seller_storefront_requests = ?, - inventory_sellability_checked_asins = ?, - inventory_sellability_available_asins = ?, - inventory_sellability_excluded_asins = ?, - persisted_inventory_asins = ?, - status = 'failed', - error_message = ?, - completed_at = ? - WHERE id = ?`, - ) - .run( - stats.scannedAsins, - stats.sourceAsinsWithMatches, - stats.candidateSellers, - stats.qualifyingSellers, - stats.matchedSellers, - stats.sellerMetadataRequests, - stats.sellerStorefrontRequests, - stats.inventorySellabilityCheckedAsins, - stats.inventorySellabilityAvailableAsins, - stats.inventorySellabilityExcludedAsins, - stats.persistedInventoryAsins, +): Promise { + await db + .update(stalkerRunDetails) + .set({ + skippedAsins: stats.skippedAsins, + scannedAsins: stats.scannedAsins, + sourceAsinsWithMatches: stats.sourceAsinsWithMatches, + candidateSellers: stats.candidateSellers, + qualifyingSellers: stats.qualifyingSellers, + matchedSellers: stats.matchedSellers, + sellerMetadataRequests: stats.sellerMetadataRequests, + sellerStorefrontRequests: stats.sellerStorefrontRequests, + inventorySellabilityCheckedAsins: stats.inventorySellabilityCheckedAsins, + inventorySellabilityAvailableAsins: + stats.inventorySellabilityAvailableAsins, + inventorySellabilityExcludedAsins: stats.inventorySellabilityExcludedAsins, + persistedInventoryAsins: stats.persistedInventoryAsins, + }) + .where(eq(stalkerRunDetails.runId, runId)); + await db + .update(runs) + .set({ + status: "failed", errorMessage, - new Date().toISOString(), - runId, - ); + completedAt: new Date(), + }) + .where(eq(runs.id, runId)); } -function finishStalkerAnalysisRun( - database: Database, +async function finishStalkerAnalysisRun( runId: number, status: "completed" | "failed", errorMessage: string | null = null, -): void { - const stats = database - .query( - `SELECT - COUNT(*) AS total, - SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba, - SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm, - SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip - FROM product_analysis_results - WHERE run_id = ?`, - ) - .get(runId) as { - total: number; - fba: number | null; - fbm: number | null; - skip: number | null; - }; - - database - .prepare( - `UPDATE category_analysis_runs - SET top_asins_checked = ?, - available_asins = ?, - fba_count = ?, - fbm_count = ?, - skip_count = ?, - status = ?, - error_message = ? - WHERE id = ?`, - ) - .run( - stats.total ?? 0, - stats.total ?? 0, - stats.fba ?? 0, - stats.fbm ?? 0, - stats.skip ?? 0, +): Promise { + await refreshRunStats(runId); + await db + .update(runs) + .set({ status, errorMessage, - runId, - ); + completedAt: new Date(), + }) + .where(eq(runs.id, runId)); } function normalizeSellerResponse( @@ -1492,7 +1481,6 @@ function collectPersistedInventoryAsins(result: StalkerAsinResult): string[] { } async function runSellableAnalysisChild( - dbPath: string, stalkerRunId: number, analysisRunId: number, asins: string[], @@ -1502,8 +1490,6 @@ async function runSellableAnalysisChild( "bun", "run", "src/stalker-analyze.ts", - "--db", - dbPath, "--stalker-run-id", String(stalkerRunId), "--analysis-run-id", @@ -1529,13 +1515,6 @@ async function runSellableAnalysisChild( } } -function normalizeAsin(value: unknown): string | null { - const asin = String(value ?? "") - .trim() - .toUpperCase(); - return ASIN_REGEX.test(asin) ? asin : null; -} - function normalizeSellerId(value: unknown): string | null { const sellerId = String(value ?? "") .trim() @@ -1660,8 +1639,5 @@ if (import.meta.main) { .catch((error) => { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; - }) - .finally(() => { - closeDb(); }); } diff --git a/src/supplier-export.test.ts b/src/supplier/supplier-export.test.ts similarity index 94% rename from src/supplier-export.test.ts rename to src/supplier/supplier-export.test.ts index c4d163f..9ccec17 100644 --- a/src/supplier-export.test.ts +++ b/src/supplier/supplier-export.test.ts @@ -3,7 +3,7 @@ import path from "node:path"; import { rmSync } from "node:fs"; import ExcelJS from "exceljs"; import { writeSupplierWorkbook } from "./supplier-export.ts"; -import type { SupplierAnalysisResult } from "./types.ts"; +import type { SupplierAnalysisResult } from "../types.ts"; const OUTPUT_FILE = path.join("/private/tmp", "asin-check-supplier-export-test.xlsx"); @@ -16,12 +16,12 @@ function result(overrides: Partial = {}): SupplierAnalys upc: "012345678901", rowNumber: 2, record: { - asin: "B000000001", name: "Test Product", unitCost: 10, brand: "Brand", category: "Grocery", }, + product: { asin: "B000000001", name: "Test Product", unitCost: 10 }, lookup: { requestedUpc: "012345678901", normalizedUpc: "012345678901", @@ -81,7 +81,8 @@ test("writeSupplierWorkbook writes ranked, skipped, and summary sheets", async ( result(), result({ upc: "111111111111", - record: { asin: "111111111111", name: "Missing", unitCost: 0 }, + record: { name: "Missing", unitCost: 0 }, + product: null, lookup: { requestedUpc: "111111111111", normalizedUpc: "111111111111", diff --git a/src/supplier-export.ts b/src/supplier/supplier-export.ts similarity index 98% rename from src/supplier-export.ts rename to src/supplier/supplier-export.ts index 61be0d9..7b3e932 100644 --- a/src/supplier-export.ts +++ b/src/supplier/supplier-export.ts @@ -5,7 +5,7 @@ import type { KeepaUpcLookupStatus, SupplierAnalysisResult, SupplierVerdict, -} from "./types.ts"; +} from "../types.ts"; export type SupplierExportSummary = { processedRows: number; @@ -63,7 +63,8 @@ function addRowsSheet( const sheet = workbook.addWorksheet(name); const headers = rows[0] ? Object.keys(rows[0]) : Object.keys(rowForResult({ upc: "", - record: { asin: "", name: "", unitCost: 0 }, + record: { name: "", unitCost: 0 }, + product: null, lookup: { requestedUpc: "", normalizedUpc: "", diff --git a/src/supplier-scoring.test.ts b/src/supplier/supplier-scoring.test.ts similarity index 96% rename from src/supplier-scoring.test.ts rename to src/supplier/supplier-scoring.test.ts index 297ae70..17a1c86 100644 --- a/src/supplier-scoring.test.ts +++ b/src/supplier/supplier-scoring.test.ts @@ -1,6 +1,6 @@ import { expect, test } from "bun:test"; import { scoreSupplierProduct } from "./supplier-scoring.ts"; -import type { KeepaData, ProductRecord, SpApiData } from "./types.ts"; +import type { KeepaData, ProductRecord, SpApiData } from "../types.ts"; function record(overrides: Partial = {}): ProductRecord { return { diff --git a/src/supplier-scoring.ts b/src/supplier/supplier-scoring.ts similarity index 99% rename from src/supplier-scoring.ts rename to src/supplier/supplier-scoring.ts index 01275e3..47e397b 100644 --- a/src/supplier-scoring.ts +++ b/src/supplier/supplier-scoring.ts @@ -3,7 +3,7 @@ import type { ProductRecord, SpApiData, SupplierScore, -} from "./types.ts"; +} from "../types.ts"; function round2(value: number): number { return Math.round(value * 100) / 100; diff --git a/src/upc-file-analysis.ts b/src/supplier/upc-file-analysis.ts similarity index 89% rename from src/upc-file-analysis.ts rename to src/supplier/upc-file-analysis.ts index 6d6903e..749b5dc 100644 --- a/src/upc-file-analysis.ts +++ b/src/supplier/upc-file-analysis.ts @@ -1,22 +1,24 @@ import path from "node:path"; -import { fetchKeepaDataBatch, lookupKeepaUpcs } from "./keepa.ts"; +import { requireAsin } from "../asin.ts"; +import { fetchKeepaDataBatch, lookupKeepaUpcs } from "../integrations/keepa.ts"; import { fetchSellabilityBatch, fetchSpApiPricingAndFees, lookupSpApiUpcs, -} from "./sp-api.ts"; +} from "../integrations/sp-api.ts"; import { processUpcFileInBatches, type UpcInputRow, } from "./upc-file-reader.ts"; import { appendSupplierResultsToRun, + completeRunInDb, + failRunInDb, refreshRunCountsInDb, startRunInDb, type RunCounts, -} from "./writer.ts"; -import { initDb, closeDb } from "./database.ts"; -import { connectCache, disconnectCache } from "./cache.ts"; +} from "../writer.ts"; +import { connectCache, disconnectCache } from "../integrations/cache.ts"; import { scoreSupplierProduct, resolveSupplierSalePrice } from "./supplier-scoring.ts"; import { writeSupplierWorkbook, @@ -29,9 +31,8 @@ import type { SupplierAnalysisResult, SupplierScore, UpcLookupDetail, -} from "./types.ts"; +} from "../types.ts"; -const DB_PATH = process.env.RESULTS_DB_PATH || path.join("db", "results.db"); const DEFAULT_INPUT_BATCH_SIZE = 200; const DEFAULT_UPC_LOOKUP_BATCH_SIZE = 100; const DEFAULT_PRICING_CONCURRENCY = 5; @@ -48,7 +49,6 @@ export type UpcFileAnalysisOptions = { export type UpcFileAnalysisSummary = { runId: number; - dbPath: string; inputFile: string; outputFile?: string; processedRows: number; @@ -242,8 +242,8 @@ async function lookupUpcsWithChunking( chunkDetails.set( upc, fallbackDetail && fallbackDetail.status !== "request_failed" - ? fallbackDetail - : spDetail!, + ? { ...fallbackDetail, provider: "keepa" } + : { ...spDetail!, provider: "sp_api" }, ); } @@ -269,7 +269,7 @@ function toProductRecord( const keepaCategory = detail.keepaData?.categoryTree?.[0]; return { - asin: detail.asin ?? row.upc, + asin: requireAsin(detail.asin), name: row.name ?? detail.asin ?? row.upc, unitCost: row.unitCost ?? 0, brand: row.brand, @@ -277,6 +277,15 @@ function toProductRecord( }; } +function toSupplierInputRecord(row: UpcInputRow) { + return { + name: row.name ?? row.upc, + unitCost: row.unitCost ?? 0, + brand: row.brand, + category: row.category, + }; +} + async function fetchFeesForProducts( products: ProductRecord[], keepaResults: Map>, @@ -339,7 +348,6 @@ function summarizeSupplierResults( export async function runUpcFileAnalysis( options: UpcFileAnalysisOptions, ): Promise { - const dbPath = options.dbPath ?? DB_PATH; const inputBatchSize = Math.max( 1, options.inputBatchSize ?? DEFAULT_INPUT_BATCH_SIZE, @@ -355,8 +363,6 @@ export async function runUpcFileAnalysis( if (manageResources) { console.log("Connecting to Redis..."); await connectCache(); - console.log("Initializing SQLite database..."); - initDb(dbPath); } const unresolvedByStatus = createStatusCounter(); @@ -365,7 +371,7 @@ export async function runUpcFileAnalysis( let processedRows = 0; let matchedRows = 0; - const runId = startRunInDb(dbPath, options.inputFile, outputFile); + const runId = await startRunInDb(options.inputFile, outputFile, undefined, "supplier_upc"); try { const readerSummary = await processUpcFileInBatches( @@ -388,12 +394,19 @@ export async function runUpcFileAnalysis( product: ProductRecord; }> = []; for (const row of rows) { - const detail = detailMap.get(row.upc); - if (!detail) { - unresolvedByStatus.request_failed += 1; - continue; - } - + const detail = + detailMap.get(row.upc) ?? + ({ + requestedUpc: row.upc, + normalizedUpc: row.upc, + status: "request_failed", + asin: null, + candidateAsins: [], + keepaData: null, + provider: "sp_api", + reason: "UPC lookup returned no result", + } satisfies UpcLookupDetail); + if (!detailMap.has(row.upc)) detailMap.set(row.upc, detail); unresolvedByStatus[detail.status] += 1; if (detail.status === "found" && detail.asin) { @@ -413,30 +426,15 @@ export async function runUpcFileAnalysis( const batchResults: SupplierAnalysisResult[] = []; for (const row of rows) { - const detail = detailMap.get(row.upc); - if (!detail || detail.status === "found") continue; + const detail = detailMap.get(row.upc)!; + if (detail.status === "found") continue; batchResults.push({ upc: row.upc, rowNumber: row.rowNumber, - record: { - asin: detail?.asin ?? row.upc, - name: row.name ?? row.upc, - unitCost: row.unitCost ?? 0, - brand: row.brand, - category: row.category, - }, - lookup: - detail ?? - ({ - requestedUpc: row.upc, - normalizedUpc: row.upc, - status: "request_failed", - asin: null, - candidateAsins: [], - keepaData: null, - reason: "UPC lookup returned no result", - } satisfies UpcLookupDetail), + record: toSupplierInputRecord(row), + product: null, + lookup: detail, keepa: null, spApi: null, score: skippedScore(detail?.reason ?? "UPC unresolved"), @@ -471,7 +469,8 @@ export async function runUpcFileAnalysis( batchResults.push({ upc: entry.detail.normalizedUpc, rowNumber: entry.row.rowNumber, - record: entry.product, + record: toSupplierInputRecord(entry.row), + product: entry.product, lookup: entry.detail, keepa, spApi, @@ -481,7 +480,7 @@ export async function runUpcFileAnalysis( } } - appendSupplierResultsToRun(dbPath, runId, batchResults); + await appendSupplierResultsToRun(runId, batchResults); allResults.push(...batchResults); }, { @@ -490,10 +489,11 @@ export async function runUpcFileAnalysis( }, ); - const runCounts = refreshRunCountsInDb(dbPath, runId); + const runCounts = await refreshRunCountsInDb(runId); const exportSummary = summarizeSupplierResults(allResults, unresolvedByStatus); await writeSupplierWorkbook(outputFile, allResults, exportSummary); + await completeRunInDb(runId); if (allResults.length > 0) { const ranked = allResults @@ -522,7 +522,6 @@ export async function runUpcFileAnalysis( return { runId, - dbPath, inputFile: options.inputFile, outputFile, processedRows, @@ -537,10 +536,12 @@ export async function runUpcFileAnalysis( skippedInvalidUpc: readerSummary.skippedInvalidUpc, }, }; + } catch (error) { + await failRunInDb(runId, error); + throw error; } finally { if (manageResources) { await disconnectCache(); - closeDb(); } } } diff --git a/src/upc-file-reader.ts b/src/supplier/upc-file-reader.ts similarity index 100% rename from src/upc-file-reader.ts rename to src/supplier/upc-file-reader.ts diff --git a/src/upc-lookup.ts b/src/supplier/upc-lookup.ts similarity index 97% rename from src/upc-lookup.ts rename to src/supplier/upc-lookup.ts index afeaa60..3346d86 100644 --- a/src/upc-lookup.ts +++ b/src/supplier/upc-lookup.ts @@ -1,4 +1,4 @@ -import { lookupKeepaUpcs, mapUpcsToAsins } from "./keepa.ts"; +import { lookupKeepaUpcs, mapUpcsToAsins } from "../integrations/keepa.ts"; function printUsage(): void { console.log("Usage:"); diff --git a/src/types.ts b/src/types.ts index bc42ef4..409c01e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -58,7 +58,8 @@ export interface KeepaUpcLookupDetail { status: KeepaUpcLookupStatus; asin: string | null; candidateAsins: string[]; - keepaData: KeepaData | null; + keepaData: KeepaData | null; + provider?: "sp_api" | "keepa"; reason?: string; } @@ -114,13 +115,66 @@ export interface SupplierScore { export interface SupplierAnalysisResult { upc: string; rowNumber?: number; - record: ProductRecord; + record: SupplierInputRecord; + product: ProductRecord | null; lookup: UpcLookupDetail; keepa: KeepaData | null; spApi: SpApiData | null; score: SupplierScore; fetchedAt: string; } + +export interface SupplierInputRecord { + name: string; + unitCost: number; + brand?: string; + category?: string; +} + +export interface Product { + asin: string; + name: string | null; + brand: string | null; + category: string | null; + firstSeenAt: string; + lastSeenAt: string; +} + +export interface ProductObservation { + id: number; + productAsin: string; + runId: number; + source: string; + fetchedAt: string; +} + +export interface Run { + id: number; + type: + | "lead_analysis" + | "category_analysis" + | "supplier_upc" + | "stalker" + | "stalker_analysis"; + parentRunId?: number | null; + status: string; +} + +export interface RunItem { + id: number; + runId: number; + productAsin: string | null; + sourceRow?: number | null; +} + +export interface AnalysisRevision { + id: number; + runItemId: number; + decision: "FBA" | "FBM" | "BUY" | "WATCH" | "SKIP"; + confidence: number | null; + reasoning: string | null; + analyzedAt: string; +} export interface CategoryRunSummaryDb { categoryId: number; diff --git a/src/web/frontend.tsx b/src/web/frontend.tsx index 43f9425..46debfd 100644 --- a/src/web/frontend.tsx +++ b/src/web/frontend.tsx @@ -1,7 +1,8 @@ import { createRoot } from "react-dom/client"; import { useEffect, useMemo, useState } from "react"; -type ProcessType = "lead_analysis" | "category_analysis"; +type ProcessType = "lead_analysis" | "category_analysis" | "supplier_upc" | "stalker" | "stalker_analysis"; +type AnalysisDecision = "FBA" | "FBM" | "BUY" | "WATCH" | "SKIP"; type SortDirection = "ASC" | "DESC"; type Run = { @@ -15,6 +16,8 @@ type Run = { totalProducts: number; fbaCount: number; fbmCount: number; + buyCount: number; + watchCount: number; skipCount: number; }; @@ -37,11 +40,15 @@ type RunDetail = { totalProducts: number; fbaCount: number; fbmCount: number; + buyCount: number; + watchCount: number; skipCount: number; summary: { totalProducts: number; fbaCount: number; fbmCount: number; + buyCount: number; + watchCount: number; skipCount: number; }; errorMessage?: string; @@ -49,6 +56,8 @@ type RunDetail = { }; type ResultItem = { + item_id: number; + product_asin: string | null; id?: number; run_id: number; asin: string; @@ -60,7 +69,7 @@ type ResultItem = { sales_rank: number | null; seller_count: number | null; monthly_sold: number | null; - verdict: "FBA" | "FBM" | "SKIP"; + verdict: AnalysisDecision | null; amazon_is_seller: number | null; amazon_buybox_share_pct_90d: number | null; confidence: number | null; @@ -77,17 +86,18 @@ type ResultsResponse = { totalPages: number; }; -type VerdictFilter = "" | "FBA" | "FBM" | "SKIP"; +type VerdictFilter = "" | AnalysisDecision; type AmazonSellerFilter = "" | "yes" | "no"; type ProductListItem = { - processType: ProcessType; - runId: number; + item_id: number | null; + processType: ProcessType | null; + runId: number | null; asin: string; product_name: string | null; brand: string | null; category: string | null; - verdict: "FBA" | "FBM" | "SKIP"; + verdict: AnalysisDecision | null; confidence: number | null; sellability_status: string | null; monthly_sold: number | null; @@ -98,7 +108,7 @@ type ProductListItem = { current_price: number | null; avg_price_90d: number | null; reasoning: string | null; - fetched_at: string; + fetched_at: string | null; }; type ProductListResponse = { @@ -179,6 +189,35 @@ type StalkerProductsResponse = { totalPages: number; }; +type ProductHistoryResponse = { + product: { + asin: string; + name: string | null; + brand: string | null; + category: string | null; + first_seen_at: string; + last_seen_at: string; + }; + observations: Array<{ + id: number; + run_id: number; + source: string; + current_price: number | null; + monthly_sold: number | null; + sales_rank: number | null; + sellability_status: string | null; + fetched_at: string; + }>; + analyses: Array<{ + id: number; + run_id: number; + decision: string; + confidence: number | null; + reasoning: string | null; + analyzed_at: string; + }>; +}; + type SortState = { field: string; direction: SortDirection; @@ -362,7 +401,7 @@ function Dashboard({ setDeletingKey(key); try { - const response = await fetch(`/api/runs/${run.processType}/${run.runId}`, { method: "DELETE" }); + const response = await fetch(`/api/runs/${run.runId}`, { method: "DELETE" }); if (!response.ok) { const errorPayload = await response.json().catch(() => null) as { error?: string } | null; const message = errorPayload?.error ?? "Failed to delete run"; @@ -523,7 +562,7 @@ function RunDetails({ }) { const [run, setRun] = useState(null); const [results, setResults] = useState(null); - const [loading, setLoading] = useState(false); + const [loading, setLoading] = useState(true); const [search, setSearch] = useState(""); const [verdict, setVerdict] = useState(""); const [sellabilityStatus, setSellabilityStatus] = useState(""); @@ -545,7 +584,7 @@ function RunDetails({ useEffect(() => { let cancelled = false; async function loadRun() { - const res = await fetch(`/api/runs/${processType}/${runId}`); + const res = await fetch(`/api/runs/${runId}`); const payload = (await res.json()) as RunDetail; if (!cancelled) { setRun(payload); @@ -560,7 +599,6 @@ function RunDetails({ useEffect(() => { let cancelled = false; async function loadResults() { - setLoading(true); const params = new URLSearchParams({ page: String(page), pageSize: String(pageSize), @@ -573,7 +611,7 @@ function RunDetails({ if (minConfidence) params.set("minConfidence", minConfidence); if (maxConfidence) params.set("maxConfidence", maxConfidence); - const res = await fetch(`/api/runs/${processType}/${runId}/results?${params.toString()}`); + const res = await fetch(`/api/runs/${runId}/items?${params.toString()}`); const payload = (await res.json()) as ResultsResponse; if (!cancelled) { setResults(payload); @@ -588,20 +626,25 @@ function RunDetails({ }, [processType, runId, search, verdict, sellabilityStatus, amazonSellerFilter, minConfidence, maxConfidence, page, pageSize, sort, refreshTick]); useEffect(() => { + if (run && run.status !== "running") { + return; + } + const interval = window.setInterval(() => { setRefreshTick((tick) => tick + 1); }, 4000); return () => { window.clearInterval(interval); }; - }, [processType, runId]); + }, [processType, runId, run?.status]); - async function reanalyzeAsin(asin: string) { - if (reanalyzing[asin]) return; - setReanalyzing((prev) => ({ ...prev, [asin]: true })); + async function reanalyzeItem(item: ResultItem) { + const key = String(item.item_id); + if (reanalyzing[key]) return; + setReanalyzing((prev) => ({ ...prev, [key]: true })); try { const response = await fetch( - `/api/runs/${processType}/${runId}/asins/${encodeURIComponent(asin)}/reanalyze`, + `/api/run-items/${item.item_id}/reanalyze`, { method: "POST" }, ); if (!response.ok) { @@ -613,7 +656,7 @@ function RunDetails({ } finally { setReanalyzing((prev) => { const next = { ...prev }; - delete next[asin]; + delete next[key]; return next; }); } @@ -626,14 +669,14 @@ function RunDetails({

Run Detail

-
Process: {processType}
+
Process: {run?.processType ?? processType}
Run ID: {runId}
Status: {run ? {run.status} : "-"}
Timestamp: {run ? formatDate(run.timestamp) : "-"}
Job: {run?.jobType ?? "-"}
Source: {run?.source ?? "-"}
Total: {formatNumber(run?.summary.totalProducts)}
-
FBA/FBM/SKIP: {formatNumber(run?.summary.fbaCount)}/{formatNumber(run?.summary.fbmCount)}/{formatNumber(run?.summary.skipCount)}
+
FBA/FBM/BUY/WATCH/SKIP: {formatNumber(run?.summary.fbaCount)}/{formatNumber(run?.summary.fbmCount)}/{formatNumber(run?.summary.buyCount)}/{formatNumber(run?.summary.watchCount)}/{formatNumber(run?.summary.skipCount)}
@@ -647,6 +690,8 @@ function RunDetails({ + +