Merge branch 'postgres'
This commit is contained in:
@@ -19,3 +19,5 @@ GOOGLE_API_KEY=your_google_api_key
|
||||
GOOGLE_CSE_ID=your_google_programmable_search_engine_id
|
||||
SERPAPI_API_KEY=your_serpapi_api_key_for_google_shopping
|
||||
|
||||
# Matches the default PostgreSQL service in docker-compose.yaml.
|
||||
DB_CONNECTION_STRING=postgres://asin_check:asin_check@localhost:5432/asin_check
|
||||
|
||||
47
CLAUDE.md
47
CLAUDE.md
@@ -12,8 +12,8 @@ Default to using Bun instead of Node.js.
|
||||
|
||||
## APIs
|
||||
|
||||
- `bun:sqlite` for SQLite. Don't use `better-sqlite3`.
|
||||
- `Bun.redis` for Redis. Don't use `ioredis`.
|
||||
- Use Drizzle ORM with `postgres` driver for Postgres. Connection is in `src/db/index.ts`.
|
||||
- Prefer `Bun.file` over `node:fs`'s readFile/writeFile.
|
||||
- `Bun.$\`cmd\`` instead of execa.
|
||||
|
||||
@@ -24,13 +24,13 @@ Default to using Bun instead of Node.js.
|
||||
bun test
|
||||
|
||||
# Run a single test file
|
||||
bun test src/supplier-scoring.test.ts
|
||||
bun test src/supplier/supplier-scoring.test.ts
|
||||
|
||||
# Type-check (no emit)
|
||||
./node_modules/.bin/tsc --noEmit
|
||||
|
||||
# ASIN lead-list pipeline (LLM-based)
|
||||
bun run src/index.ts input/leads.xlsx --out output/results.xlsx
|
||||
bun start leads.xlsx --out results.xlsx
|
||||
|
||||
# Supplier UPC pipeline (deterministic)
|
||||
bun run upc-file --input input/supplier.xlsx --out output/supplier_ranked.xlsx
|
||||
@@ -40,6 +40,9 @@ bun run bestsellers
|
||||
bun run monthly-sold
|
||||
bun run mid-range
|
||||
|
||||
# Stalker pipeline
|
||||
bun run stalker --input input/asins.xlsx
|
||||
|
||||
# Web API server
|
||||
bun run start:web # http://localhost:3000
|
||||
|
||||
@@ -47,29 +50,37 @@ bun run start:web # http://localhost:3000
|
||||
bun run src/sp-test.ts
|
||||
bun run src/sp-test.ts B07SN9BHVV
|
||||
bun run src/sp-test.ts --sellability B07SN9BHVV
|
||||
|
||||
# Database migrations (Drizzle)
|
||||
bun run db:generate
|
||||
bun run db:migrate
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
||||
Two distinct analysis pipelines share infrastructure (Keepa, SP-API, Redis, SQLite) but diverge in how they produce verdicts.
|
||||
Two distinct analysis pipelines share infrastructure (Keepa, SP-API, Redis, Postgres) but diverge in how they produce verdicts.
|
||||
|
||||
### ASIN Lead-list Pipeline (`src/index.ts` → `src/analysis-pipeline.ts`)
|
||||
|
||||
For spreadsheets containing known ASINs. Verdict is LLM-based (FBA/FBM/SKIP via LM Studio).
|
||||
|
||||
Flow: `reader.ts` parse → Redis cache check → `sp-api.ts` sellability gate (5 concurrent workers) → `keepa.ts` batch enrichment → `sp-api.ts` pricing + FBA fees (5 concurrent workers) → `llm.ts` batched analysis (5 products/batch) → `writer.ts` XLSX + SQLite.
|
||||
Flow: `reader.ts` parse → Redis cache check → `integrations/sp-api.ts` sellability gate (5 concurrent workers) → `integrations/keepa.ts` batch enrichment → `integrations/sp-api.ts` pricing + FBA fees (5 concurrent workers) → `integrations/llm.ts` batched analysis (5 products/batch) → `writer.ts` XLSX + Postgres.
|
||||
|
||||
### Supplier UPC Pipeline (`src/upc-file-analysis.ts`)
|
||||
### Supplier UPC Pipeline (`src/supplier/upc-file-analysis.ts`)
|
||||
|
||||
For supplier price lists containing UPC/EAN values. Verdict is deterministic (BUY/WATCH/SKIP); never calls LM Studio.
|
||||
|
||||
Flow: `upc-file-reader.ts` streaming parse (`.xlsx`) or row-window parse (`.xls`) → SP-API catalog UPC lookup first, Keepa UPC lookup as fallback → `keepa.ts` demand enrichment → `sp-api.ts` sellability + FBA fees → `supplier-scoring.ts` deterministic score → `supplier-export.ts` Excel workbook (`Ranked Leads`, `Skipped`, `Summary` sheets) + SQLite.
|
||||
Flow: `supplier/upc-file-reader.ts` streaming parse (`.xlsx`) or row-window parse (`.xls`) → SP-API catalog UPC lookup first, Keepa UPC lookup as fallback → `integrations/keepa.ts` demand enrichment → `integrations/sp-api.ts` sellability + FBA fees → `supplier/supplier-scoring.ts` deterministic score → `supplier/supplier-export.ts` Excel workbook (`Ranked Leads`, `Skipped`, `Summary` sheets) + Postgres.
|
||||
|
||||
UPC resolution priority: SP-API catalog lookup → Keepa fallback (for no-match or request failure only).
|
||||
|
||||
### Category Pipelines
|
||||
|
||||
`bestsellers-by-category.ts`, `top-monthly-sold-by-category.ts`, `mid-range-sellers-by-category.ts` — Keepa category browsing → SP-API sellability gate → LLM verdict. Each saves results to SQLite. Mid-range applies configurable filters (monthly sold, price, seller count, Amazon buy box share).
|
||||
`src/categories/` — Keepa category browsing → SP-API sellability gate → LLM verdict. Each saves results to Postgres. Mid-range applies configurable filters (monthly sold, price, seller count, Amazon buy box share).
|
||||
|
||||
### Stalker Pipeline (`src/stalker/stalker.ts`)
|
||||
|
||||
Tracks competitor sellers across ASINs. Fetches storefronts, checks sellability of inventory items, and persists matched seller data to Postgres.
|
||||
|
||||
### Shared Infrastructure
|
||||
|
||||
@@ -77,18 +88,24 @@ UPC resolution priority: SP-API catalog lookup → Keepa fallback (for no-match
|
||||
|--------|------|
|
||||
| `src/types.ts` | All shared interfaces (`ProductRecord`, `KeepaData`, `SpApiData`, `SupplierScore`, etc.) |
|
||||
| `src/config.ts` | Env var loading via `Bun.env` |
|
||||
| `src/keepa.ts` | Keepa API: batch ASIN fetch, UPC lookup, auto rate-limiting on token exhaustion |
|
||||
| `src/sp-api.ts` | SP-API: sellability (`getListingsRestrictions`), pricing+fees, UPC catalog lookup |
|
||||
| `src/cache.ts` | Redis caching (24h TTL for lead-list; 12h for mid-range) |
|
||||
| `src/database.ts` | SQLite `runs` + `results` tables; auto-creates `db/results.db` |
|
||||
| `src/db/index.ts` | Drizzle Postgres connection (shared pool) |
|
||||
| `src/db/schema.ts` | Drizzle schema for all tables |
|
||||
| `src/db/persistence.ts` | Product, observation, unified run-item, UPC resolution, and revision persistence |
|
||||
| `src/integrations/keepa.ts` | Keepa API: batch ASIN fetch, UPC lookup, auto rate-limiting |
|
||||
| `src/integrations/sp-api.ts` | SP-API: sellability, pricing+fees, UPC catalog lookup |
|
||||
| `src/integrations/cache.ts` | Redis caching (24h TTL for lead-list; 12h for mid-range) |
|
||||
| `src/integrations/llm.ts` | LLM integration (LM Studio / Claude) |
|
||||
| `src/server.ts` | Bun HTTP server exposing REST endpoints for both pipelines |
|
||||
|
||||
### File Layout
|
||||
|
||||
- `src/integrations/` — external API clients (Keepa, SP-API, Redis cache, LLM, SearXNG)
|
||||
- `src/categories/` — category discovery pipelines
|
||||
- `src/stalker/` — competitor seller tracking pipeline
|
||||
- `src/supplier/` — supplier UPC analysis pipeline
|
||||
- `src/db/` — Drizzle schema and connection
|
||||
- `input/` — source spreadsheets (git-ignored)
|
||||
- `output/` — generated workbooks (git-ignored)
|
||||
- `db/` — SQLite files (git-ignored)
|
||||
- `src/` — all source and test files
|
||||
|
||||
## Project Rules
|
||||
|
||||
@@ -96,4 +113,6 @@ UPC resolution priority: SP-API catalog lookup → Keepa fallback (for no-match
|
||||
- The supplier UPC pipeline must not call LM Studio.
|
||||
- Supplier UPC files resolve UPC/EAN through SP-API catalog lookup first; Keepa UPC lookup is fallback only (no-match or request-failure cases).
|
||||
- Supplier workbook output must keep `Ranked Leads`, `Skipped`, and `Summary` sheets.
|
||||
- Treat `products.asin` as the canonical normalized product identity; UPC values belong only in identifier and resolution records.
|
||||
- Store time-varying data in observations or revisions and retain run history rather than overwriting prior analysis.
|
||||
- When changing UPC supplier behavior, cover SP-API UPC parsing, deterministic scoring, and workbook export with `bun test`.
|
||||
|
||||
38
README.md
38
README.md
@@ -21,17 +21,19 @@ cp .env.example .env
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
bun run src/index.ts input/<input.csv|xlsx> [--out output/results.xlsx]
|
||||
bun start <input.csv|xlsx> [--out results.xlsx]
|
||||
```
|
||||
|
||||
Add `--claude` to use Anthropic Claude instead of local LM Studio for LLM analysis.
|
||||
Bare input and output filenames use the `input/` and `output/` directories. Pass a path containing a directory to override those defaults.
|
||||
|
||||
Examples:
|
||||
|
||||
```bash
|
||||
bun run src/index.ts input/leads.xlsx
|
||||
bun run src/index.ts input/leads.csv --out output/results.xlsx
|
||||
bun run src/index.ts input/leads.xlsx --claude
|
||||
bun start leads.xlsx
|
||||
bun start leads.csv --out results.xlsx
|
||||
bun start leads.xlsx --claude
|
||||
bun start archive/leads.xlsx --out exports/results.xlsx
|
||||
```
|
||||
|
||||
Large-file behavior:
|
||||
@@ -155,7 +157,7 @@ ranked sourcing workbook:
|
||||
2. Resolves UPCs to ASINs with SP-API catalog lookup first, then falls back to Keepa for no-match/request-failure cases.
|
||||
3. Enriches resolved ASINs with Keepa demand/competition data and SP-API sellability + FBA fees.
|
||||
4. Scores products with deterministic BUY/WATCH/SKIP logic; this path does not call LM Studio.
|
||||
5. Writes a ranked Excel workbook and persists rows into the existing `runs` + `results` tables.
|
||||
5. Writes a ranked Excel workbook and persists rows through unified runs, UPC resolution, product observation, and scoring-history tables.
|
||||
|
||||
CLI usage:
|
||||
|
||||
@@ -244,20 +246,28 @@ Numeric parsing accepts plain numbers as well as formatted values like `$12.50`,
|
||||
4. **Keepa fetch** — batch the sellable (uncached) ASINs in a single API call (up to 100 per request)
|
||||
5. **Enrich** — fetch SP-API pricing + FBA/FBM fees for sellable ASINs; combine with Keepa data and spreadsheet data
|
||||
6. **LLM analysis** — send batches of 5 sellable products to LM Studio for FBA/FBM/SKIP verdict; skipped ASINs get auto-SKIP verdict (confidence 100) and bypass LLM entirely
|
||||
7. **Output** — print results table to console (includes all ASINs), optionally write CSV/XLSX, and **persist results to a SQLite database**.
|
||||
7. **Output** — print results table to console (includes all ASINs), optionally write CSV/XLSX, and persist products, observations, run items, and analysis revisions to PostgreSQL.
|
||||
|
||||
## Persistent Storage with SQLite
|
||||
## Persistent Storage
|
||||
|
||||
Results from each run are now stored in a SQLite database named `db/results.db` by default. The SQLite implementation details are handled in `src/database.ts`. This allows you to:
|
||||
PostgreSQL persistence is managed with Drizzle in `src/db/schema.ts` and `src/db/persistence.ts`. ASINs are canonical product identities: all inputs normalize to uppercase 10-character alphanumeric keys before any product reference is stored.
|
||||
|
||||
- Revisit past analysis results.
|
||||
- Query and analyze historical data.
|
||||
- Track product performance over time.
|
||||
Core tables:
|
||||
|
||||
The database will automatically be created if it doesn't exist. Two tables are created:
|
||||
- `products`: one canonical row per ASIN with latest descriptive metadata.
|
||||
- `product_observations`: append-only marketplace, pricing, fee, and sellability snapshots.
|
||||
- `runs` and `run_items`: unified lifecycle/history for lead, category, supplier UPC, and stalker workflows.
|
||||
- `analysis_revisions` and `supplier_scores`: append-only analysis results; reanalysis does not overwrite prior decisions.
|
||||
- `sourcing_inputs`, `upc_resolutions`, and `product_identifiers`: source-row and confirmed identifier data kept separate from catalog products.
|
||||
- `stalker_run_details`, `stalker_scans`, and `stalker_inventory_items`: seller workflow provenance linked back to products and observations.
|
||||
|
||||
- `runs`: Stores metadata about each analysis run (timestamp, input file, output file, and summary counts).
|
||||
- `results`: Stores detailed analysis results for each product from each run, linked to the `runs` table.
|
||||
Unresolved or ambiguous supplier UPCs stay on their run item and resolution records; a UPC is never stored as an ASIN.
|
||||
|
||||
Web endpoints use unified identifiers:
|
||||
|
||||
- `GET /api/runs`, `GET /api/runs/:runId`, `GET /api/runs/:runId/items`
|
||||
- `GET /api/products`, `GET /api/products/:asin`
|
||||
- `POST /api/run-items/:itemId/reanalyze`
|
||||
|
||||
## Output columns
|
||||
|
||||
|
||||
183
bun.lock
183
bun.lock
@@ -6,8 +6,10 @@
|
||||
"name": "asin-check",
|
||||
"dependencies": {
|
||||
"amazon-sp-api": "^1.2.1",
|
||||
"drizzle-orm": "^0.45.2",
|
||||
"exceljs": "^4.4.0",
|
||||
"ioredis": "^5.10.1",
|
||||
"postgres": "^3.4.9",
|
||||
"react": "^19.2.0",
|
||||
"react-dom": "^19.2.0",
|
||||
"xlsx": "^0.18.5",
|
||||
@@ -16,11 +18,70 @@
|
||||
"@types/bun": "latest",
|
||||
"@types/react": "^19.2.14",
|
||||
"@types/react-dom": "^19.2.3",
|
||||
"drizzle-kit": "^0.31.10",
|
||||
"typescript": "^6.0.3",
|
||||
},
|
||||
},
|
||||
},
|
||||
"packages": {
|
||||
"@drizzle-team/brocli": ["@drizzle-team/brocli@0.10.2", "", {}, "sha512-z33Il7l5dKjUgGULTqBsQBQwckHh5AbIuxhdsIxDDiZAzBOrZO6q9ogcWC65kU382AfynTfgNumVcNIjuIua6w=="],
|
||||
|
||||
"@esbuild-kit/core-utils": ["@esbuild-kit/core-utils@3.3.2", "", { "dependencies": { "esbuild": "~0.18.20", "source-map-support": "^0.5.21" } }, "sha512-sPRAnw9CdSsRmEtnsl2WXWdyquogVpB3yZ3dgwJfe8zrOzTsV7cJvmwrKVa+0ma5BoiGJ+BoqkMvawbayKUsqQ=="],
|
||||
|
||||
"@esbuild-kit/esm-loader": ["@esbuild-kit/esm-loader@2.6.5", "", { "dependencies": { "@esbuild-kit/core-utils": "^3.3.2", "get-tsconfig": "^4.7.0" } }, "sha512-FxEMIkJKnodyA1OaCUoEvbYRkoZlLZ4d/eXFu9Fh8CbBBgP5EmZxrfTRyN0qpXZ4vOvqnE5YdRdcrmUUXuU+dA=="],
|
||||
|
||||
"@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.25.12", "", { "os": "aix", "cpu": "ppc64" }, "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA=="],
|
||||
|
||||
"@esbuild/android-arm": ["@esbuild/android-arm@0.25.12", "", { "os": "android", "cpu": "arm" }, "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg=="],
|
||||
|
||||
"@esbuild/android-arm64": ["@esbuild/android-arm64@0.25.12", "", { "os": "android", "cpu": "arm64" }, "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg=="],
|
||||
|
||||
"@esbuild/android-x64": ["@esbuild/android-x64@0.25.12", "", { "os": "android", "cpu": "x64" }, "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg=="],
|
||||
|
||||
"@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.25.12", "", { "os": "darwin", "cpu": "arm64" }, "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg=="],
|
||||
|
||||
"@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.25.12", "", { "os": "darwin", "cpu": "x64" }, "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA=="],
|
||||
|
||||
"@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.25.12", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg=="],
|
||||
|
||||
"@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.25.12", "", { "os": "freebsd", "cpu": "x64" }, "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ=="],
|
||||
|
||||
"@esbuild/linux-arm": ["@esbuild/linux-arm@0.25.12", "", { "os": "linux", "cpu": "arm" }, "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw=="],
|
||||
|
||||
"@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.25.12", "", { "os": "linux", "cpu": "arm64" }, "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ=="],
|
||||
|
||||
"@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.25.12", "", { "os": "linux", "cpu": "ia32" }, "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA=="],
|
||||
|
||||
"@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng=="],
|
||||
|
||||
"@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw=="],
|
||||
|
||||
"@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.25.12", "", { "os": "linux", "cpu": "ppc64" }, "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA=="],
|
||||
|
||||
"@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w=="],
|
||||
|
||||
"@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.25.12", "", { "os": "linux", "cpu": "s390x" }, "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg=="],
|
||||
|
||||
"@esbuild/linux-x64": ["@esbuild/linux-x64@0.25.12", "", { "os": "linux", "cpu": "x64" }, "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw=="],
|
||||
|
||||
"@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.25.12", "", { "os": "none", "cpu": "arm64" }, "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg=="],
|
||||
|
||||
"@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.25.12", "", { "os": "none", "cpu": "x64" }, "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ=="],
|
||||
|
||||
"@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.25.12", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A=="],
|
||||
|
||||
"@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.25.12", "", { "os": "openbsd", "cpu": "x64" }, "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw=="],
|
||||
|
||||
"@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.25.12", "", { "os": "none", "cpu": "arm64" }, "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg=="],
|
||||
|
||||
"@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.25.12", "", { "os": "sunos", "cpu": "x64" }, "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w=="],
|
||||
|
||||
"@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.25.12", "", { "os": "win32", "cpu": "arm64" }, "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg=="],
|
||||
|
||||
"@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.25.12", "", { "os": "win32", "cpu": "ia32" }, "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ=="],
|
||||
|
||||
"@esbuild/win32-x64": ["@esbuild/win32-x64@0.25.12", "", { "os": "win32", "cpu": "x64" }, "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA=="],
|
||||
|
||||
"@fast-csv/format": ["@fast-csv/format@4.3.5", "", { "dependencies": { "@types/node": "^14.0.1", "lodash.escaperegexp": "^4.1.2", "lodash.isboolean": "^3.0.3", "lodash.isequal": "^4.5.0", "lodash.isfunction": "^3.0.9", "lodash.isnil": "^4.0.0" } }, "sha512-8iRn6QF3I8Ak78lNAa+Gdl5MJJBM5vRHivFtMRUWINdevNo00K7OXxS2PshawLKTejVwieIlPmK5YlLu6w4u8A=="],
|
||||
|
||||
"@fast-csv/parse": ["@fast-csv/parse@4.3.6", "", { "dependencies": { "@types/node": "^14.0.1", "lodash.escaperegexp": "^4.1.2", "lodash.groupby": "^4.6.0", "lodash.isfunction": "^3.0.9", "lodash.isnil": "^4.0.0", "lodash.isundefined": "^3.0.1", "lodash.uniq": "^4.5.0" } }, "sha512-uRsLYksqpbDmWaSmzvJcuApSEe38+6NQZBUsuAyMZKqHxH0g1wcJgsKUvN3WC8tewaqFjBMMGrkHmC+T7k8LvA=="],
|
||||
@@ -63,6 +124,8 @@
|
||||
|
||||
"buffer-crc32": ["buffer-crc32@0.2.13", "", {}, "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ=="],
|
||||
|
||||
"buffer-from": ["buffer-from@1.1.2", "", {}, "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="],
|
||||
|
||||
"buffer-indexof-polyfill": ["buffer-indexof-polyfill@1.0.2", "", {}, "sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A=="],
|
||||
|
||||
"buffers": ["buffers@0.1.1", "", {}, "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ=="],
|
||||
@@ -101,6 +164,10 @@
|
||||
|
||||
"denque": ["denque@2.1.0", "", {}, "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw=="],
|
||||
|
||||
"drizzle-kit": ["drizzle-kit@0.31.10", "", { "dependencies": { "@drizzle-team/brocli": "^0.10.2", "@esbuild-kit/esm-loader": "^2.5.5", "esbuild": "^0.25.4", "tsx": "^4.21.0" }, "bin": { "drizzle-kit": "bin.cjs" } }, "sha512-7OZcmQUrdGI+DUNNsKBn1aW8qSoKuTH7d0mYgSP8bAzdFzKoovxEFnoGQp2dVs82EOJeYycqRtciopszwUf8bw=="],
|
||||
|
||||
"drizzle-orm": ["drizzle-orm@0.45.2", "", { "peerDependencies": { "@aws-sdk/client-rds-data": ">=3", "@cloudflare/workers-types": ">=4", "@electric-sql/pglite": ">=0.2.0", "@libsql/client": ">=0.10.0", "@libsql/client-wasm": ">=0.10.0", "@neondatabase/serverless": ">=0.10.0", "@op-engineering/op-sqlite": ">=2", "@opentelemetry/api": "^1.4.1", "@planetscale/database": ">=1.13", "@prisma/client": "*", "@tidbcloud/serverless": "*", "@types/better-sqlite3": "*", "@types/pg": "*", "@types/sql.js": "*", "@upstash/redis": ">=1.34.7", "@vercel/postgres": ">=0.8.0", "@xata.io/client": "*", "better-sqlite3": ">=7", "bun-types": "*", "expo-sqlite": ">=14.0.0", "gel": ">=2", "knex": "*", "kysely": "*", "mysql2": ">=2", "pg": ">=8", "postgres": ">=3", "prisma": "*", "sql.js": ">=1", "sqlite3": ">=5" }, "optionalPeers": ["@aws-sdk/client-rds-data", "@cloudflare/workers-types", "@electric-sql/pglite", "@libsql/client", "@libsql/client-wasm", "@neondatabase/serverless", "@op-engineering/op-sqlite", "@opentelemetry/api", "@planetscale/database", "@prisma/client", "@tidbcloud/serverless", "@types/better-sqlite3", "@types/pg", "@types/sql.js", "@upstash/redis", "@vercel/postgres", "@xata.io/client", "better-sqlite3", "bun-types", "expo-sqlite", "gel", "knex", "kysely", "mysql2", "pg", "postgres", "prisma", "sql.js", "sqlite3"] }, "sha512-kY0BSaTNYWnoDMVoyY8uxmyHjpJW1geOmBMdSSicKo9CIIWkSxMIj2rkeSR51b8KAPB7m+qysjuHme5nKP+E5Q=="],
|
||||
|
||||
"dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="],
|
||||
|
||||
"duplexer2": ["duplexer2@0.1.4", "", { "dependencies": { "readable-stream": "^2.0.2" } }, "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA=="],
|
||||
@@ -113,6 +180,8 @@
|
||||
|
||||
"es-object-atoms": ["es-object-atoms@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0" } }, "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA=="],
|
||||
|
||||
"esbuild": ["esbuild@0.25.12", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.25.12", "@esbuild/android-arm": "0.25.12", "@esbuild/android-arm64": "0.25.12", "@esbuild/android-x64": "0.25.12", "@esbuild/darwin-arm64": "0.25.12", "@esbuild/darwin-x64": "0.25.12", "@esbuild/freebsd-arm64": "0.25.12", "@esbuild/freebsd-x64": "0.25.12", "@esbuild/linux-arm": "0.25.12", "@esbuild/linux-arm64": "0.25.12", "@esbuild/linux-ia32": "0.25.12", "@esbuild/linux-loong64": "0.25.12", "@esbuild/linux-mips64el": "0.25.12", "@esbuild/linux-ppc64": "0.25.12", "@esbuild/linux-riscv64": "0.25.12", "@esbuild/linux-s390x": "0.25.12", "@esbuild/linux-x64": "0.25.12", "@esbuild/netbsd-arm64": "0.25.12", "@esbuild/netbsd-x64": "0.25.12", "@esbuild/openbsd-arm64": "0.25.12", "@esbuild/openbsd-x64": "0.25.12", "@esbuild/openharmony-arm64": "0.25.12", "@esbuild/sunos-x64": "0.25.12", "@esbuild/win32-arm64": "0.25.12", "@esbuild/win32-ia32": "0.25.12", "@esbuild/win32-x64": "0.25.12" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg=="],
|
||||
|
||||
"exceljs": ["exceljs@4.4.0", "", { "dependencies": { "archiver": "^5.0.0", "dayjs": "^1.8.34", "fast-csv": "^4.3.1", "jszip": "^3.10.1", "readable-stream": "^3.6.0", "saxes": "^5.0.1", "tmp": "^0.2.0", "unzipper": "^0.10.11", "uuid": "^8.3.0" } }, "sha512-XctvKaEMaj1Ii9oDOqbW/6e1gXknSY4g/aLCDicOXqBE4M0nRWkUu0PTp++UPNzoFY12BNHMfs/VadKIS6llvg=="],
|
||||
|
||||
"fast-csv": ["fast-csv@4.3.6", "", { "dependencies": { "@fast-csv/format": "4.3.5", "@fast-csv/parse": "4.3.6" } }, "sha512-2RNSpuwwsJGP0frGsOmTb9oUF+VkFSM4SyLTDgwf2ciHWTarN0lQTC+F2f/t5J9QjW+c65VFIAAu85GsvMIusw=="],
|
||||
@@ -127,6 +196,8 @@
|
||||
|
||||
"fs.realpath": ["fs.realpath@1.0.0", "", {}, "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw=="],
|
||||
|
||||
"fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="],
|
||||
|
||||
"fstream": ["fstream@1.0.12", "", { "dependencies": { "graceful-fs": "^4.1.2", "inherits": "~2.0.0", "mkdirp": ">=0.5 0", "rimraf": "2" } }, "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg=="],
|
||||
|
||||
"function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="],
|
||||
@@ -135,6 +206,8 @@
|
||||
|
||||
"get-proto": ["get-proto@1.0.1", "", { "dependencies": { "dunder-proto": "^1.0.1", "es-object-atoms": "^1.0.0" } }, "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g=="],
|
||||
|
||||
"get-tsconfig": ["get-tsconfig@4.14.0", "", { "dependencies": { "resolve-pkg-maps": "^1.0.0" } }, "sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA=="],
|
||||
|
||||
"glob": ["glob@7.2.3", "", { "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", "minimatch": "^3.1.1", "once": "^1.3.0", "path-is-absolute": "^1.0.0" } }, "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q=="],
|
||||
|
||||
"gopd": ["gopd@1.2.0", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="],
|
||||
@@ -217,6 +290,8 @@
|
||||
|
||||
"path-is-absolute": ["path-is-absolute@1.0.1", "", {}, "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg=="],
|
||||
|
||||
"postgres": ["postgres@3.4.9", "", {}, "sha512-GD3qdB0x1z9xgFI6cdRD6xu2Sp2WCOEoe3mtnyB5Ee0XrrL5Pe+e4CCnJrRMnL1zYtRDZmQQVbvOttLnKDLnaw=="],
|
||||
|
||||
"process-nextick-args": ["process-nextick-args@2.0.1", "", {}, "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="],
|
||||
|
||||
"qs": ["qs@6.15.1", "", { "dependencies": { "side-channel": "^1.1.0" } }, "sha512-6YHEFRL9mfgcAvql/XhwTvf5jKcOiiupt2FiJxHkiX1z4j7WL8J/jRHYLluORvc1XxB5rV20KoeK00gVJamspg=="],
|
||||
@@ -233,6 +308,8 @@
|
||||
|
||||
"redis-parser": ["redis-parser@3.0.0", "", { "dependencies": { "redis-errors": "^1.0.0" } }, "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A=="],
|
||||
|
||||
"resolve-pkg-maps": ["resolve-pkg-maps@1.0.0", "", {}, "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw=="],
|
||||
|
||||
"rimraf": ["rimraf@2.7.1", "", { "dependencies": { "glob": "^7.1.3" }, "bin": { "rimraf": "./bin.js" } }, "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w=="],
|
||||
|
||||
"safe-buffer": ["safe-buffer@5.1.2", "", {}, "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="],
|
||||
@@ -253,6 +330,10 @@
|
||||
|
||||
"side-channel-weakmap": ["side-channel-weakmap@1.0.2", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3", "side-channel-map": "^1.0.1" } }, "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A=="],
|
||||
|
||||
"source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="],
|
||||
|
||||
"source-map-support": ["source-map-support@0.5.21", "", { "dependencies": { "buffer-from": "^1.0.0", "source-map": "^0.6.0" } }, "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w=="],
|
||||
|
||||
"ssf": ["ssf@0.11.2", "", { "dependencies": { "frac": "~1.1.2" } }, "sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g=="],
|
||||
|
||||
"standard-as-callback": ["standard-as-callback@2.1.0", "", {}, "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A=="],
|
||||
@@ -267,6 +348,8 @@
|
||||
|
||||
"traverse": ["traverse@0.3.9", "", {}, "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ=="],
|
||||
|
||||
"tsx": ["tsx@4.22.3", "", { "dependencies": { "esbuild": "~0.28.0" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "bin": { "tsx": "dist/cli.mjs" } }, "sha512-mdoNxBC/cSQObGGVQ5Bpn5i+yv7j68gk3Nfm3wFjcJg3Z0Mix9jzAFfP12prmm5eVGmDKtp0yyArrs0Q+8gZHg=="],
|
||||
|
||||
"typescript": ["typescript@6.0.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw=="],
|
||||
|
||||
"undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="],
|
||||
@@ -289,6 +372,8 @@
|
||||
|
||||
"zip-stream": ["zip-stream@4.1.1", "", { "dependencies": { "archiver-utils": "^3.0.4", "compress-commons": "^4.1.2", "readable-stream": "^3.6.0" } }, "sha512-9qv4rlDiopXg4E69k+vMHjNN63YFMe9sZMrdlvKnCjlCRWeCBswPPMPUfx+ipsAWq1LXHe70RcbaHdJJpS6hyQ=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild": ["esbuild@0.18.20", "", { "optionalDependencies": { "@esbuild/android-arm": "0.18.20", "@esbuild/android-arm64": "0.18.20", "@esbuild/android-x64": "0.18.20", "@esbuild/darwin-arm64": "0.18.20", "@esbuild/darwin-x64": "0.18.20", "@esbuild/freebsd-arm64": "0.18.20", "@esbuild/freebsd-x64": "0.18.20", "@esbuild/linux-arm": "0.18.20", "@esbuild/linux-arm64": "0.18.20", "@esbuild/linux-ia32": "0.18.20", "@esbuild/linux-loong64": "0.18.20", "@esbuild/linux-mips64el": "0.18.20", "@esbuild/linux-ppc64": "0.18.20", "@esbuild/linux-riscv64": "0.18.20", "@esbuild/linux-s390x": "0.18.20", "@esbuild/linux-x64": "0.18.20", "@esbuild/netbsd-x64": "0.18.20", "@esbuild/openbsd-x64": "0.18.20", "@esbuild/sunos-x64": "0.18.20", "@esbuild/win32-arm64": "0.18.20", "@esbuild/win32-ia32": "0.18.20", "@esbuild/win32-x64": "0.18.20" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA=="],
|
||||
|
||||
"@fast-csv/format/@types/node": ["@types/node@14.18.63", "", {}, "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ=="],
|
||||
|
||||
"@fast-csv/parse/@types/node": ["@types/node@14.18.63", "", {}, "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ=="],
|
||||
@@ -305,10 +390,56 @@
|
||||
|
||||
"string_decoder/safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="],
|
||||
|
||||
"tsx/esbuild": ["esbuild@0.28.0", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.28.0", "@esbuild/android-arm": "0.28.0", "@esbuild/android-arm64": "0.28.0", "@esbuild/android-x64": "0.28.0", "@esbuild/darwin-arm64": "0.28.0", "@esbuild/darwin-x64": "0.28.0", "@esbuild/freebsd-arm64": "0.28.0", "@esbuild/freebsd-x64": "0.28.0", "@esbuild/linux-arm": "0.28.0", "@esbuild/linux-arm64": "0.28.0", "@esbuild/linux-ia32": "0.28.0", "@esbuild/linux-loong64": "0.28.0", "@esbuild/linux-mips64el": "0.28.0", "@esbuild/linux-ppc64": "0.28.0", "@esbuild/linux-riscv64": "0.28.0", "@esbuild/linux-s390x": "0.28.0", "@esbuild/linux-x64": "0.28.0", "@esbuild/netbsd-arm64": "0.28.0", "@esbuild/netbsd-x64": "0.28.0", "@esbuild/openbsd-arm64": "0.28.0", "@esbuild/openbsd-x64": "0.28.0", "@esbuild/openharmony-arm64": "0.28.0", "@esbuild/sunos-x64": "0.28.0", "@esbuild/win32-arm64": "0.28.0", "@esbuild/win32-ia32": "0.28.0", "@esbuild/win32-x64": "0.28.0" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-sNR9MHpXSUV/XB4zmsFKN+QgVG82Cc7+/aaxJ8Adi8hyOac+EXptIp45QBPaVyX3N70664wRbTcLTOemCAnyqw=="],
|
||||
|
||||
"unzipper/readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="],
|
||||
|
||||
"zip-stream/archiver-utils": ["archiver-utils@3.0.4", "", { "dependencies": { "glob": "^7.2.3", "graceful-fs": "^4.2.0", "lazystream": "^1.0.0", "lodash.defaults": "^4.2.0", "lodash.difference": "^4.5.0", "lodash.flatten": "^4.4.0", "lodash.isplainobject": "^4.0.6", "lodash.union": "^4.6.0", "normalize-path": "^3.0.0", "readable-stream": "^3.6.0" } }, "sha512-KVgf4XQVrTjhyWmx6cte4RxonPLR9onExufI1jhvw/MQ4BB6IsZD5gT8Lq+u/+pRkWna/6JoHpiQioaqFP5Rzw=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.18.20", "", { "os": "android", "cpu": "arm" }, "sha512-fyi7TDI/ijKKNZTUJAQqiG5T7YjJXgnzkURqmGj13C6dCqckZBLdl4h7bkhHt/t0WP+zO9/zwroDvANaOqO5Sw=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/android-arm64": ["@esbuild/android-arm64@0.18.20", "", { "os": "android", "cpu": "arm64" }, "sha512-Nz4rJcchGDtENV0eMKUNa6L12zz2zBDXuhj/Vjh18zGqB44Bi7MBMSXjgunJgjRhCmKOjnPuZp4Mb6OKqtMHLQ=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/android-x64": ["@esbuild/android-x64@0.18.20", "", { "os": "android", "cpu": "x64" }, "sha512-8GDdlePJA8D6zlZYJV/jnrRAi6rOiNaCC/JclcXpB+KIuvfBN4owLtgzY2bsxnx666XjJx2kDPUmnTtR8qKQUg=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.18.20", "", { "os": "darwin", "cpu": "arm64" }, "sha512-bxRHW5kHU38zS2lPTPOyuyTm+S+eobPUnTNkdJEfAddYgEcll4xkT8DB9d2008DtTbl7uJag2HuE5NZAZgnNEA=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.18.20", "", { "os": "darwin", "cpu": "x64" }, "sha512-pc5gxlMDxzm513qPGbCbDukOdsGtKhfxD1zJKXjCCcU7ju50O7MeAZ8c4krSJcOIJGFR+qx21yMMVYwiQvyTyQ=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.18.20", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-yqDQHy4QHevpMAaxhhIwYPMv1NECwOvIpGCZkECn8w2WFHXjEwrBn3CeNIYsibZ/iZEUemj++M26W3cNR5h+Tw=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.18.20", "", { "os": "freebsd", "cpu": "x64" }, "sha512-tgWRPPuQsd3RmBZwarGVHZQvtzfEBOreNuxEMKFcd5DaDn2PbBxfwLcj4+aenoh7ctXcbXmOQIn8HI6mCSw5MQ=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/linux-arm": ["@esbuild/linux-arm@0.18.20", "", { "os": "linux", "cpu": "arm" }, "sha512-/5bHkMWnq1EgKr1V+Ybz3s1hWXok7mDFUMQ4cG10AfW3wL02PSZi5kFpYKrptDsgb2WAJIvRcDm+qIvXf/apvg=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.18.20", "", { "os": "linux", "cpu": "arm64" }, "sha512-2YbscF+UL7SQAVIpnWvYwM+3LskyDmPhe31pE7/aoTMFKKzIc9lLbyGUpmmb8a8AixOL61sQ/mFh3jEjHYFvdA=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.18.20", "", { "os": "linux", "cpu": "ia32" }, "sha512-P4etWwq6IsReT0E1KHU40bOnzMHoH73aXp96Fs8TIT6z9Hu8G6+0SHSw9i2isWrD2nbx2qo5yUqACgdfVGx7TA=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.18.20", "", { "os": "linux", "cpu": "none" }, "sha512-nXW8nqBTrOpDLPgPY9uV+/1DjxoQ7DoB2N8eocyq8I9XuqJ7BiAMDMf9n1xZM9TgW0J8zrquIb/A7s3BJv7rjg=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.18.20", "", { "os": "linux", "cpu": "none" }, "sha512-d5NeaXZcHp8PzYy5VnXV3VSd2D328Zb+9dEq5HE6bw6+N86JVPExrA6O68OPwobntbNJ0pzCpUFZTo3w0GyetQ=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.18.20", "", { "os": "linux", "cpu": "ppc64" }, "sha512-WHPyeScRNcmANnLQkq6AfyXRFr5D6N2sKgkFo2FqguP44Nw2eyDlbTdZwd9GYk98DZG9QItIiTlFLHJHjxP3FA=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.18.20", "", { "os": "linux", "cpu": "none" }, "sha512-WSxo6h5ecI5XH34KC7w5veNnKkju3zBRLEQNY7mv5mtBmrP/MjNBCAlsM2u5hDBlS3NGcTQpoBvRzqBcRtpq1A=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.18.20", "", { "os": "linux", "cpu": "s390x" }, "sha512-+8231GMs3mAEth6Ja1iK0a1sQ3ohfcpzpRLH8uuc5/KVDFneH6jtAJLFGafpzpMRO6DzJ6AvXKze9LfFMrIHVQ=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/linux-x64": ["@esbuild/linux-x64@0.18.20", "", { "os": "linux", "cpu": "x64" }, "sha512-UYqiqemphJcNsFEskc73jQ7B9jgwjWrSayxawS6UVFZGWrAAtkzjxSqnoclCXxWtfwLdzU+vTpcNYhpn43uP1w=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.18.20", "", { "os": "none", "cpu": "x64" }, "sha512-iO1c++VP6xUBUmltHZoMtCUdPlnPGdBom6IrO4gyKPFFVBKioIImVooR5I83nTew5UOYrk3gIJhbZh8X44y06A=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.18.20", "", { "os": "openbsd", "cpu": "x64" }, "sha512-e5e4YSsuQfX4cxcygw/UCPIEP6wbIL+se3sxPdCiMbFLBWu0eiZOJ7WoD+ptCLrmjZBK1Wk7I6D/I3NglUGOxg=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.18.20", "", { "os": "sunos", "cpu": "x64" }, "sha512-kDbFRFp0YpTQVVrqUd5FTYmWo45zGaXe0X8E1G/LKFC0v8x0vWrhOWSLITcCn63lmZIxfOMXtCfti/RxN/0wnQ=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.18.20", "", { "os": "win32", "cpu": "arm64" }, "sha512-ddYFR6ItYgoaq4v4JmQQaAI5s7npztfV4Ag6NrhiaW0RrnOXqBkgwZLofVTlq1daVTQNhtI5oieTvkRPfZrePg=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.18.20", "", { "os": "win32", "cpu": "ia32" }, "sha512-Wv7QBi3ID/rROT08SABTS7eV4hX26sVduqDOTe1MvGMjNd3EjOz4b7zeexIR62GTIEKrfJXKL9LFxTYgkyeu7g=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.18.20", "", { "os": "win32", "cpu": "x64" }, "sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ=="],
|
||||
|
||||
"archiver-utils/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="],
|
||||
|
||||
"duplexer2/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="],
|
||||
@@ -319,6 +450,58 @@
|
||||
|
||||
"lazystream/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.28.0", "", { "os": "aix", "cpu": "ppc64" }, "sha512-lhRUCeuOyJQURhTxl4WkpFTjIsbDayJHih5kZC1giwE+MhIzAb7mEsQMqMf18rHLsrb5qI1tafG20mLxEWcWlA=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.28.0", "", { "os": "android", "cpu": "arm" }, "sha512-wqh0ByljabXLKHeWXYLqoJ5jKC4XBaw6Hk08OfMrCRd2nP2ZQ5eleDZC41XHyCNgktBGYMbqnrJKq/K/lzPMSQ=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/android-arm64": ["@esbuild/android-arm64@0.28.0", "", { "os": "android", "cpu": "arm64" }, "sha512-+WzIXQOSaGs33tLEgYPYe/yQHf0WTU0X42Jca3y8NWMbUVhp7rUnw+vAsRC/QiDrdD31IszMrZy+qwPOPjd+rw=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/android-x64": ["@esbuild/android-x64@0.28.0", "", { "os": "android", "cpu": "x64" }, "sha512-+VJggoaKhk2VNNqVL7f6S189UzShHC/mR9EE8rDdSkdpN0KflSwWY/gWjDrNxxisg8Fp1ZCD9jLMo4m0OUfeUA=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.28.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-0T+A9WZm+bZ84nZBtk1ckYsOvyA3x7e2Acj1KdVfV4/2tdG4fzUp91YHx+GArWLtwqp77pBXVCPn2We7Letr0Q=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.28.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-fyzLm/DLDl/84OCfp2f/XQ4flmORsjU7VKt8HLjvIXChJoFFOIL6pLJPH4Yhd1n1gGFF9mPwtlN5Wf82DZs+LQ=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.28.0", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-l9GeW5UZBT9k9brBYI+0WDffcRxgHQD8ShN2Ur4xWq/NFzUKm3k5lsH4PdaRgb2w7mI9u61nr2gI2mLI27Nh3Q=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.28.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-BXoQai/A0wPO6Es3yFJ7APCiKGc1tdAEOgeTNy3SsB491S3aHn4S4r3e976eUnPdU+NbdtmBuLncYir2tMU9Nw=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/linux-arm": ["@esbuild/linux-arm@0.28.0", "", { "os": "linux", "cpu": "arm" }, "sha512-CjaaREJagqJp7iTaNQjjidaNbCKYcd4IDkzbwwxtSvjI7NZm79qiHc8HqciMddQ6CKvJT6aBd8lO9kN/ZudLlw=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.28.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-RVyzfb3FWsGA55n6WY0MEIEPURL1FcbhFE6BffZEMEekfCzCIMtB5yyDcFnVbTnwk+CLAgTujmV/Lgvih56W+A=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.28.0", "", { "os": "linux", "cpu": "ia32" }, "sha512-KBnSTt1kxl9x70q+ydterVdl+Cn0H18ngRMRCEQfrbqdUuntQQ0LoMZv47uB97NljZFzY6HcfqEZ2SAyIUTQBQ=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.28.0", "", { "os": "linux", "cpu": "none" }, "sha512-zpSlUce1mnxzgBADvxKXX5sl8aYQHo2ezvMNI8I0lbblJtp8V4odlm3Yzlj7gPyt3T8ReksE6bK+pT3WD+aJRg=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.28.0", "", { "os": "linux", "cpu": "none" }, "sha512-2jIfP6mmjkdmeTlsX/9vmdmhBmKADrWqN7zcdtHIeNSCH1SqIoNI63cYsjQR8J+wGa4Y5izRcSHSm8K3QWmk3w=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.28.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-bc0FE9wWeC0WBm49IQMPSPILRocGTQt3j5KPCA8os6VprfuJ7KD+5PzESSrJ6GmPIPJK965ZJHTUlSA6GNYEhg=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.28.0", "", { "os": "linux", "cpu": "none" }, "sha512-SQPZOwoTTT/HXFXQJG/vBX8sOFagGqvZyXcgLA3NhIqcBv1BJU1d46c0rGcrij2B56Z2rNiSLaZOYW5cUk7yLQ=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.28.0", "", { "os": "linux", "cpu": "s390x" }, "sha512-SCfR0HN8CEEjnYnySJTd2cw0k9OHB/YFzt5zgJEwa+wL/T/raGWYMBqwDNAC6dqFKmJYZoQBRfHjgwLHGSrn3Q=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/linux-x64": ["@esbuild/linux-x64@0.28.0", "", { "os": "linux", "cpu": "x64" }, "sha512-us0dSb9iFxIi8srnpl931Nvs65it/Jd2a2K3qs7fz2WfGPHqzfzZTfec7oxZJRNPXPnNYZtanmRc4AL/JwVzHQ=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.28.0", "", { "os": "none", "cpu": "arm64" }, "sha512-CR/RYotgtCKwtftMwJlUU7xCVNg3lMYZ0RzTmAHSfLCXw3NtZtNpswLEj/Kkf6kEL3Gw+BpOekRX0BYCtklhUw=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.28.0", "", { "os": "none", "cpu": "x64" }, "sha512-nU1yhmYutL+fQ71Kxnhg8uEOdC0pwEW9entHykTgEbna2pw2dkbFSMeqjjyHZoCmt8SBkOSvV+yNmm94aUrrqw=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.28.0", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-cXb5vApOsRsxsEl4mcZ1XY3D4DzcoMxR/nnc4IyqYs0rTI8ZKmW6kyyg+11Z8yvgMfAEldKzP7AdP64HnSC/6g=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.28.0", "", { "os": "openbsd", "cpu": "x64" }, "sha512-8wZM2qqtv9UP3mzy7HiGYNH/zjTA355mpeuA+859TyR+e+Tc08IHYpLJuMsfpDJwoLo1ikIJI8jC3GFjnRClzA=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.28.0", "", { "os": "none", "cpu": "arm64" }, "sha512-FLGfyizszcef5C3YtoyQDACyg95+dndv79i2EekILBofh5wpCa1KuBqOWKrEHZg3zrL3t5ouE5jgr94vA+Wb2w=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.28.0", "", { "os": "sunos", "cpu": "x64" }, "sha512-1ZgjUoEdHZZl/YlV76TSCz9Hqj9h9YmMGAgAPYd+q4SicWNX3G5GCyx9uhQWSLcbvPW8Ni7lj4gDa1T40akdlw=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.28.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-Q9StnDmQ/enxnpxCCLSg0oo4+34B9TdXpuyPeTedN/6+iXBJ4J+zwfQI28u/Jl40nOYAxGoNi7mFP40RUtkmUA=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.28.0", "", { "os": "win32", "cpu": "ia32" }, "sha512-zF3ag/gfiCe6U2iczcRzSYJKH1DCI+ByzSENHlM2FcDbEeo5Zd2C86Aq0tKUYAJJ1obRP84ymxIAksZUcdztHA=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.28.0", "", { "os": "win32", "cpu": "x64" }, "sha512-pEl1bO9mfAmIC+tW5btTmrKaujg3zGtUmWNdCw/xs70FBjwAL3o9OEKNHvNmnyylD6ubxUERiEhdsL0xBQ9efw=="],
|
||||
|
||||
"unzipper/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="],
|
||||
}
|
||||
}
|
||||
|
||||
35
docker-compose.yaml
Normal file
35
docker-compose.yaml
Normal file
@@ -0,0 +1,35 @@
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_DB: asin_check
|
||||
POSTGRES_USER: asin_check
|
||||
POSTGRES_PASSWORD: asin_check
|
||||
ports:
|
||||
- "5432:5432"
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
restart: unless-stopped
|
||||
command: ["redis-server", "--appendonly", "yes"]
|
||||
ports:
|
||||
- "6379:6379"
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
10
drizzle.config.ts
Normal file
10
drizzle.config.ts
Normal file
@@ -0,0 +1,10 @@
|
||||
import { defineConfig } from "drizzle-kit";
|
||||
|
||||
export default defineConfig({
|
||||
schema: "./src/db/schema.ts",
|
||||
out: "./drizzle",
|
||||
dialect: "postgresql",
|
||||
dbCredentials: {
|
||||
url: process.env.DB_CONNECTION_STRING!,
|
||||
},
|
||||
});
|
||||
269
drizzle/0000_adorable_shiver_man.sql
Normal file
269
drizzle/0000_adorable_shiver_man.sql
Normal file
@@ -0,0 +1,269 @@
|
||||
CREATE TYPE "public"."analysis_decision" AS ENUM('FBA', 'FBM', 'BUY', 'WATCH', 'SKIP');--> statement-breakpoint
|
||||
CREATE TYPE "public"."analysis_method" AS ENUM('llm', 'supplier_scoring');--> statement-breakpoint
|
||||
CREATE TYPE "public"."run_status" AS ENUM('running', 'ok', 'empty', 'failed', 'completed');--> statement-breakpoint
|
||||
CREATE TYPE "public"."run_type" AS ENUM('lead_analysis', 'category_analysis', 'supplier_upc', 'stalker', 'stalker_analysis');--> statement-breakpoint
|
||||
CREATE TABLE "analysis_revisions" (
|
||||
"id" serial PRIMARY KEY NOT NULL,
|
||||
"run_item_id" integer NOT NULL,
|
||||
"observation_id" integer,
|
||||
"method" "analysis_method" NOT NULL,
|
||||
"decision" "analysis_decision" NOT NULL,
|
||||
"confidence" real,
|
||||
"reasoning" text,
|
||||
"analyzed_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "analysis_run_stats" (
|
||||
"run_id" integer PRIMARY KEY NOT NULL,
|
||||
"processed_count" integer DEFAULT 0 NOT NULL,
|
||||
"analyzed_count" integer DEFAULT 0 NOT NULL,
|
||||
"available_count" integer DEFAULT 0 NOT NULL,
|
||||
"fba_count" integer DEFAULT 0 NOT NULL,
|
||||
"fbm_count" integer DEFAULT 0 NOT NULL,
|
||||
"buy_count" integer DEFAULT 0 NOT NULL,
|
||||
"watch_count" integer DEFAULT 0 NOT NULL,
|
||||
"skip_count" integer DEFAULT 0 NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "category_run_details" (
|
||||
"run_id" integer PRIMARY KEY NOT NULL,
|
||||
"category_id" integer NOT NULL,
|
||||
"category_label" text NOT NULL,
|
||||
"checked_asin_count" integer DEFAULT 0 NOT NULL,
|
||||
"selection_parameters_json" text
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "product_identifiers" (
|
||||
"id" serial PRIMARY KEY NOT NULL,
|
||||
"product_asin" text NOT NULL,
|
||||
"identifier_type" text NOT NULL,
|
||||
"identifier_value" text NOT NULL,
|
||||
"source" text NOT NULL,
|
||||
"confirmed_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "uq_product_identifier_type_value" UNIQUE("identifier_type","identifier_value")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "product_observations" (
|
||||
"id" serial PRIMARY KEY NOT NULL,
|
||||
"product_asin" text NOT NULL,
|
||||
"run_id" integer NOT NULL,
|
||||
"source" text NOT NULL,
|
||||
"marketplace" text DEFAULT 'US' NOT NULL,
|
||||
"current_price" real,
|
||||
"avg_price_90d" real,
|
||||
"sales_rank" integer,
|
||||
"sales_rank_avg_90d" integer,
|
||||
"monthly_sold" integer,
|
||||
"rank_drops_30d" integer,
|
||||
"rank_drops_90d" integer,
|
||||
"seller_count" integer,
|
||||
"amazon_is_seller" boolean,
|
||||
"amazon_buybox_share_pct_90d" real,
|
||||
"fba_fee" real,
|
||||
"fbm_fee" real,
|
||||
"referral_percent" real,
|
||||
"can_sell" boolean,
|
||||
"sellability_status" text,
|
||||
"sellability_reason" text,
|
||||
"raw_product_json" text,
|
||||
"fetched_at" timestamp with time zone NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "products" (
|
||||
"asin" text PRIMARY KEY NOT NULL,
|
||||
"name" text,
|
||||
"brand" text,
|
||||
"category" text,
|
||||
"metadata_fetched_at" timestamp with time zone,
|
||||
"first_seen_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"last_seen_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "ck_products_asin" CHECK ("products"."asin" ~ '^[A-Z0-9]{10}$')
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "run_items" (
|
||||
"id" serial PRIMARY KEY NOT NULL,
|
||||
"run_id" integer NOT NULL,
|
||||
"product_asin" text,
|
||||
"source_inventory_item_id" integer,
|
||||
"ordinal" integer,
|
||||
"source_row" integer,
|
||||
"status" text DEFAULT 'completed' NOT NULL,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "runs" (
|
||||
"id" serial PRIMARY KEY NOT NULL,
|
||||
"type" "run_type" NOT NULL,
|
||||
"parent_run_id" integer,
|
||||
"input_file" text,
|
||||
"output_file" text,
|
||||
"status" "run_status" DEFAULT 'running' NOT NULL,
|
||||
"error_message" text,
|
||||
"started_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"completed_at" timestamp with time zone
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "sellers" (
|
||||
"seller_id" text PRIMARY KEY NOT NULL,
|
||||
"seller_name" text,
|
||||
"rating" real,
|
||||
"rating_count" integer,
|
||||
"storefront_asin_total" integer,
|
||||
"persisted_inventory_sample_count" integer,
|
||||
"last_updated_at" timestamp with time zone NOT NULL,
|
||||
"raw_seller_json" text
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "sourcing_inputs" (
|
||||
"run_item_id" integer PRIMARY KEY NOT NULL,
|
||||
"supplied_name" text,
|
||||
"supplied_brand" text,
|
||||
"supplied_category" text,
|
||||
"unit_cost" real,
|
||||
"avg_price_90d_sheet" real,
|
||||
"selling_price_sheet" real,
|
||||
"fba_net_sheet" real,
|
||||
"gross_profit_dollar" real,
|
||||
"gross_profit_pct" real,
|
||||
"net_profit_sheet" real,
|
||||
"roi_sheet" real,
|
||||
"moq" integer,
|
||||
"moq_cost" real,
|
||||
"qty_available" integer,
|
||||
"supplier" text,
|
||||
"source_url" text,
|
||||
"asin_link" text,
|
||||
"promo_coupon_code" text,
|
||||
"notes" text,
|
||||
"lead_date" text
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "stalker_inventory_items" (
|
||||
"id" serial PRIMARY KEY NOT NULL,
|
||||
"run_id" integer NOT NULL,
|
||||
"seller_id" text NOT NULL,
|
||||
"product_asin" text NOT NULL,
|
||||
"observation_id" integer NOT NULL,
|
||||
"last_seen_at" timestamp with time zone NOT NULL,
|
||||
"raw_inventory_json" text,
|
||||
CONSTRAINT "uq_stalker_inventory_items_run_seller_asin" UNIQUE("run_id","seller_id","product_asin")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "stalker_run_details" (
|
||||
"run_id" integer PRIMARY KEY NOT NULL,
|
||||
"requested_asins" integer DEFAULT 0 NOT NULL,
|
||||
"skipped_asins" integer DEFAULT 0 NOT NULL,
|
||||
"scanned_asins" integer DEFAULT 0 NOT NULL,
|
||||
"source_asins_with_matches" integer DEFAULT 0 NOT NULL,
|
||||
"candidate_sellers" integer DEFAULT 0 NOT NULL,
|
||||
"qualifying_sellers" integer DEFAULT 0 NOT NULL,
|
||||
"matched_sellers" integer DEFAULT 0 NOT NULL,
|
||||
"seller_metadata_requests" integer DEFAULT 0 NOT NULL,
|
||||
"seller_storefront_requests" integer DEFAULT 0 NOT NULL,
|
||||
"inventory_sellability_checked_asins" integer DEFAULT 0 NOT NULL,
|
||||
"inventory_sellability_available_asins" integer DEFAULT 0 NOT NULL,
|
||||
"inventory_sellability_excluded_asins" integer DEFAULT 0 NOT NULL,
|
||||
"persisted_inventory_asins" integer DEFAULT 0 NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "stalker_scan_sellers" (
|
||||
"id" serial PRIMARY KEY NOT NULL,
|
||||
"scan_id" integer NOT NULL,
|
||||
"seller_id" text NOT NULL,
|
||||
"offer_price" real,
|
||||
"condition" text,
|
||||
"is_fba" boolean,
|
||||
"stock" integer,
|
||||
"seller_rating" real,
|
||||
"seller_rating_count" integer,
|
||||
"raw_offer_json" text,
|
||||
CONSTRAINT "uq_stalker_scan_sellers_scan_seller" UNIQUE("scan_id","seller_id")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "stalker_scans" (
|
||||
"id" serial PRIMARY KEY NOT NULL,
|
||||
"run_id" integer NOT NULL,
|
||||
"source_product_asin" text NOT NULL,
|
||||
"observation_id" integer,
|
||||
"offer_count" integer DEFAULT 0 NOT NULL,
|
||||
"candidate_seller_count" integer DEFAULT 0 NOT NULL,
|
||||
"matched_seller_count" integer DEFAULT 0 NOT NULL,
|
||||
"fetched_at" timestamp with time zone NOT NULL,
|
||||
CONSTRAINT "uq_stalker_scans_run_source_product" UNIQUE("run_id","source_product_asin")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "supplier_scores" (
|
||||
"revision_id" integer PRIMARY KEY NOT NULL,
|
||||
"score" real,
|
||||
"sale_price" real,
|
||||
"fba_fee" real,
|
||||
"profit" real,
|
||||
"margin" real,
|
||||
"roi" real,
|
||||
"reason" text
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "upc_resolution_candidates" (
|
||||
"run_item_id" integer NOT NULL,
|
||||
"product_asin" text NOT NULL,
|
||||
CONSTRAINT "upc_resolution_candidates_run_item_id_product_asin_pk" PRIMARY KEY("run_item_id","product_asin")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "upc_resolutions" (
|
||||
"run_item_id" integer PRIMARY KEY NOT NULL,
|
||||
"requested_upc" text NOT NULL,
|
||||
"normalized_upc" text NOT NULL,
|
||||
"provider" text NOT NULL,
|
||||
"status" text NOT NULL,
|
||||
"reason" text,
|
||||
"resolved_product_asin" text,
|
||||
"resolved_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
ALTER TABLE "analysis_revisions" ADD CONSTRAINT "analysis_revisions_run_item_id_run_items_id_fk" FOREIGN KEY ("run_item_id") REFERENCES "public"."run_items"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "analysis_revisions" ADD CONSTRAINT "analysis_revisions_observation_id_product_observations_id_fk" FOREIGN KEY ("observation_id") REFERENCES "public"."product_observations"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "analysis_run_stats" ADD CONSTRAINT "analysis_run_stats_run_id_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."runs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "category_run_details" ADD CONSTRAINT "category_run_details_run_id_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."runs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "product_identifiers" ADD CONSTRAINT "product_identifiers_product_asin_products_asin_fk" FOREIGN KEY ("product_asin") REFERENCES "public"."products"("asin") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "product_observations" ADD CONSTRAINT "product_observations_product_asin_products_asin_fk" FOREIGN KEY ("product_asin") REFERENCES "public"."products"("asin") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "product_observations" ADD CONSTRAINT "product_observations_run_id_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."runs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "run_items" ADD CONSTRAINT "run_items_run_id_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."runs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "run_items" ADD CONSTRAINT "run_items_product_asin_products_asin_fk" FOREIGN KEY ("product_asin") REFERENCES "public"."products"("asin") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "run_items" ADD CONSTRAINT "run_items_source_inventory_item_id_stalker_inventory_items_id_fk" FOREIGN KEY ("source_inventory_item_id") REFERENCES "public"."stalker_inventory_items"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "runs" ADD CONSTRAINT "runs_parent_run_id_runs_id_fk" FOREIGN KEY ("parent_run_id") REFERENCES "public"."runs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "sourcing_inputs" ADD CONSTRAINT "sourcing_inputs_run_item_id_run_items_id_fk" FOREIGN KEY ("run_item_id") REFERENCES "public"."run_items"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "stalker_inventory_items" ADD CONSTRAINT "stalker_inventory_items_run_id_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."runs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "stalker_inventory_items" ADD CONSTRAINT "stalker_inventory_items_seller_id_sellers_seller_id_fk" FOREIGN KEY ("seller_id") REFERENCES "public"."sellers"("seller_id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "stalker_inventory_items" ADD CONSTRAINT "stalker_inventory_items_product_asin_products_asin_fk" FOREIGN KEY ("product_asin") REFERENCES "public"."products"("asin") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "stalker_inventory_items" ADD CONSTRAINT "stalker_inventory_items_observation_id_product_observations_id_fk" FOREIGN KEY ("observation_id") REFERENCES "public"."product_observations"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "stalker_run_details" ADD CONSTRAINT "stalker_run_details_run_id_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."runs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "stalker_scan_sellers" ADD CONSTRAINT "stalker_scan_sellers_scan_id_stalker_scans_id_fk" FOREIGN KEY ("scan_id") REFERENCES "public"."stalker_scans"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "stalker_scan_sellers" ADD CONSTRAINT "stalker_scan_sellers_seller_id_sellers_seller_id_fk" FOREIGN KEY ("seller_id") REFERENCES "public"."sellers"("seller_id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "stalker_scans" ADD CONSTRAINT "stalker_scans_run_id_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."runs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "stalker_scans" ADD CONSTRAINT "stalker_scans_source_product_asin_products_asin_fk" FOREIGN KEY ("source_product_asin") REFERENCES "public"."products"("asin") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "stalker_scans" ADD CONSTRAINT "stalker_scans_observation_id_product_observations_id_fk" FOREIGN KEY ("observation_id") REFERENCES "public"."product_observations"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "supplier_scores" ADD CONSTRAINT "supplier_scores_revision_id_analysis_revisions_id_fk" FOREIGN KEY ("revision_id") REFERENCES "public"."analysis_revisions"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "upc_resolution_candidates" ADD CONSTRAINT "upc_resolution_candidates_run_item_id_upc_resolutions_run_item_id_fk" FOREIGN KEY ("run_item_id") REFERENCES "public"."upc_resolutions"("run_item_id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "upc_resolution_candidates" ADD CONSTRAINT "upc_resolution_candidates_product_asin_products_asin_fk" FOREIGN KEY ("product_asin") REFERENCES "public"."products"("asin") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "upc_resolutions" ADD CONSTRAINT "upc_resolutions_run_item_id_run_items_id_fk" FOREIGN KEY ("run_item_id") REFERENCES "public"."run_items"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "upc_resolutions" ADD CONSTRAINT "upc_resolutions_resolved_product_asin_products_asin_fk" FOREIGN KEY ("resolved_product_asin") REFERENCES "public"."products"("asin") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
|
||||
CREATE INDEX "idx_analysis_revisions_run_item_time" ON "analysis_revisions" USING btree ("run_item_id","analyzed_at");--> statement-breakpoint
|
||||
CREATE INDEX "idx_analysis_revisions_decision" ON "analysis_revisions" USING btree ("decision");--> statement-breakpoint
|
||||
CREATE INDEX "idx_product_identifiers_asin" ON "product_identifiers" USING btree ("product_asin");--> statement-breakpoint
|
||||
CREATE INDEX "idx_product_observations_product_time" ON "product_observations" USING btree ("product_asin","fetched_at" DESC NULLS LAST);--> statement-breakpoint
|
||||
CREATE INDEX "idx_product_observations_run_id" ON "product_observations" USING btree ("run_id");--> statement-breakpoint
|
||||
CREATE INDEX "idx_product_observations_sellability" ON "product_observations" USING btree ("sellability_status");--> statement-breakpoint
|
||||
CREATE INDEX "idx_products_name" ON "products" USING btree ("name");--> statement-breakpoint
|
||||
CREATE INDEX "idx_products_last_seen_at" ON "products" USING btree ("last_seen_at");--> statement-breakpoint
|
||||
CREATE INDEX "idx_run_items_run_id" ON "run_items" USING btree ("run_id");--> statement-breakpoint
|
||||
CREATE INDEX "idx_run_items_product_asin" ON "run_items" USING btree ("product_asin");--> statement-breakpoint
|
||||
CREATE INDEX "idx_runs_started_at" ON "runs" USING btree ("started_at");--> statement-breakpoint
|
||||
CREATE INDEX "idx_runs_type" ON "runs" USING btree ("type");--> statement-breakpoint
|
||||
CREATE INDEX "idx_runs_status" ON "runs" USING btree ("status");--> statement-breakpoint
|
||||
CREATE INDEX "idx_runs_parent_run_id" ON "runs" USING btree ("parent_run_id");--> statement-breakpoint
|
||||
CREATE INDEX "idx_stalker_inventory_seller_id" ON "stalker_inventory_items" USING btree ("seller_id");--> statement-breakpoint
|
||||
CREATE INDEX "idx_stalker_inventory_product_asin" ON "stalker_inventory_items" USING btree ("product_asin");--> statement-breakpoint
|
||||
CREATE INDEX "idx_stalker_scans_run_id" ON "stalker_scans" USING btree ("run_id");--> statement-breakpoint
|
||||
CREATE INDEX "idx_stalker_scans_source_asin" ON "stalker_scans" USING btree ("source_product_asin");--> statement-breakpoint
|
||||
CREATE INDEX "idx_upc_candidates_product_asin" ON "upc_resolution_candidates" USING btree ("product_asin");--> statement-breakpoint
|
||||
CREATE INDEX "idx_upc_resolutions_normalized_upc" ON "upc_resolutions" USING btree ("normalized_upc");
|
||||
2025
drizzle/meta/0000_snapshot.json
Normal file
2025
drizzle/meta/0000_snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
13
drizzle/meta/_journal.json
Normal file
13
drizzle/meta/_journal.json
Normal file
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"version": "7",
|
||||
"dialect": "postgresql",
|
||||
"entries": [
|
||||
{
|
||||
"idx": 0,
|
||||
"version": "7",
|
||||
"when": 1779726518779,
|
||||
"tag": "0000_adorable_shiver_man",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
19
package.json
19
package.json
@@ -4,28 +4,33 @@
|
||||
"type": "module",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"bestsellers": "bun run src/bestsellers-by-category.ts",
|
||||
"monthly-sold": "bun run src/top-monthly-sold-by-category.ts",
|
||||
"mid-range": "bun run src/mid-range-sellers-by-category.ts",
|
||||
"stalker": "bun run src/stalker.ts",
|
||||
"bestsellers": "bun run src/categories/bestsellers-by-category.ts",
|
||||
"monthly-sold": "bun run src/categories/top-monthly-sold-by-category.ts",
|
||||
"mid-range": "bun run src/categories/mid-range-sellers-by-category.ts",
|
||||
"stalker": "bun run src/stalker/stalker.ts",
|
||||
"search-offers": "bun run src/asin-offer-search.ts",
|
||||
"upc": "bun run src/upc-lookup.ts",
|
||||
"upc-file": "bun run src/upc-file-analysis.ts",
|
||||
"upc": "bun run src/supplier/upc-lookup.ts",
|
||||
"upc-file": "bun run src/supplier/upc-file-analysis.ts",
|
||||
"start": "bun run src/index.ts",
|
||||
"start:web": "bun --hot src/server.ts",
|
||||
"build:web": "bun build src/web/index.html --outdir dist",
|
||||
"test": "bun test"
|
||||
"test": "bun test",
|
||||
"db:generate": "drizzle-kit generate",
|
||||
"db:migrate": "drizzle-kit migrate"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "latest",
|
||||
"@types/react": "^19.2.14",
|
||||
"@types/react-dom": "^19.2.3",
|
||||
"drizzle-kit": "^0.31.10",
|
||||
"typescript": "^6.0.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"amazon-sp-api": "^1.2.1",
|
||||
"drizzle-orm": "^0.45.2",
|
||||
"exceljs": "^4.4.0",
|
||||
"ioredis": "^5.10.1",
|
||||
"postgres": "^3.4.9",
|
||||
"react": "^19.2.0",
|
||||
"react-dom": "^19.2.0",
|
||||
"xlsx": "^0.18.5"
|
||||
|
||||
114
src/analysis-pipeline.test.ts
Normal file
114
src/analysis-pipeline.test.ts
Normal file
@@ -0,0 +1,114 @@
|
||||
import { beforeEach, expect, mock, test } from "bun:test";
|
||||
import { processProductChunk } from "./analysis-pipeline.ts";
|
||||
import type { ProductRecord } from "./types.ts";
|
||||
|
||||
const fetchKeepaDataBatchMock = mock(async (asins: string[]) => {
|
||||
return new Map(
|
||||
asins.map((asin) => [
|
||||
asin,
|
||||
{
|
||||
currentPrice: 20,
|
||||
avgPrice90: 18,
|
||||
minPrice90: null,
|
||||
maxPrice90: null,
|
||||
salesRank: 100,
|
||||
salesRankAvg90: null,
|
||||
salesRankDrops30: null,
|
||||
salesRankDrops90: null,
|
||||
sellerCount: 3,
|
||||
amazonIsSeller: false,
|
||||
amazonBuyboxSharePct90d: null,
|
||||
buyBoxSeller: null,
|
||||
buyBoxPrice: null,
|
||||
monthlySold: 50,
|
||||
categoryTree: [],
|
||||
},
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
const fetchSellabilityBatchMock = mock(async (asins: string[]) => {
|
||||
return new Map(
|
||||
asins.map((asin) => [
|
||||
asin,
|
||||
asin === "B000000002"
|
||||
? {
|
||||
canSell: false,
|
||||
sellabilityStatus: "restricted" as const,
|
||||
sellabilityReason: "Approval required",
|
||||
}
|
||||
: {
|
||||
canSell: true,
|
||||
sellabilityStatus: "available" as const,
|
||||
sellabilityReason: "Available",
|
||||
},
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
const fetchSpApiPricingAndFeesMock = mock(async () => ({
|
||||
fbaFee: 4,
|
||||
fbmFee: 2,
|
||||
referralFeePercent: 15,
|
||||
estimatedSalePrice: 20,
|
||||
canSell: true,
|
||||
sellabilityStatus: "available" as const,
|
||||
sellabilityReason: "Available",
|
||||
}));
|
||||
|
||||
const analyzeProductsMock = mock(async (products: any[]) =>
|
||||
products.map((product) => ({
|
||||
asin: product.record.asin,
|
||||
verdict: "FBA" as const,
|
||||
confidence: 95,
|
||||
reasoning: "Analyzed",
|
||||
})),
|
||||
);
|
||||
|
||||
const getCacheMock = mock(async () => null);
|
||||
const setCacheMock = mock(async () => undefined);
|
||||
|
||||
beforeEach(() => {
|
||||
fetchKeepaDataBatchMock.mockClear();
|
||||
fetchSellabilityBatchMock.mockClear();
|
||||
fetchSpApiPricingAndFeesMock.mockClear();
|
||||
analyzeProductsMock.mockClear();
|
||||
getCacheMock.mockClear();
|
||||
setCacheMock.mockClear();
|
||||
});
|
||||
|
||||
test("lead analysis retains restricted input rows as SKIP without LLM analysis", async () => {
|
||||
const products: ProductRecord[] = [
|
||||
{ asin: "B000000001", name: "Available", unitCost: 5 },
|
||||
{ asin: "B000000002", name: "Restricted", unitCost: 6 },
|
||||
];
|
||||
|
||||
const results = await processProductChunk(products, {
|
||||
llmBatchDelayMs: 0,
|
||||
llmRetryDelayMs: 0,
|
||||
dependencies: {
|
||||
fetchKeepaDataBatch: fetchKeepaDataBatchMock,
|
||||
fetchSellabilityBatch: fetchSellabilityBatchMock,
|
||||
fetchSpApiPricingAndFees: fetchSpApiPricingAndFeesMock,
|
||||
analyzeProducts: analyzeProductsMock,
|
||||
getCache: getCacheMock,
|
||||
setCache: setCacheMock,
|
||||
},
|
||||
});
|
||||
|
||||
expect(results).toHaveLength(2);
|
||||
expect(results.map((result) => result.product.record.asin)).toEqual([
|
||||
"B000000001",
|
||||
"B000000002",
|
||||
]);
|
||||
expect(results.find((result) => result.product.record.asin === "B000000002")?.verdict)
|
||||
.toEqual({
|
||||
asin: "B000000002",
|
||||
verdict: "SKIP",
|
||||
confidence: 100,
|
||||
reasoning: "Approval required",
|
||||
});
|
||||
expect(fetchKeepaDataBatchMock.mock.calls[0]?.[0]).toEqual(["B000000001"]);
|
||||
expect(fetchSpApiPricingAndFeesMock.mock.calls).toHaveLength(1);
|
||||
expect(analyzeProductsMock.mock.calls[0]?.[0]).toHaveLength(1);
|
||||
});
|
||||
@@ -1,7 +1,7 @@
|
||||
import { fetchKeepaDataBatch } from "./keepa.ts";
|
||||
import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts";
|
||||
import { getCache, setCache } from "./cache.ts";
|
||||
import { analyzeProducts } from "./llm.ts";
|
||||
import { fetchKeepaDataBatch } from "./integrations/keepa.ts";
|
||||
import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./integrations/sp-api.ts";
|
||||
import { getCache, setCache } from "./integrations/cache.ts";
|
||||
import { analyzeProducts } from "./integrations/llm.ts";
|
||||
import type {
|
||||
AnalysisResult,
|
||||
EnrichedProduct,
|
||||
@@ -16,6 +16,15 @@ export const DEFAULT_PRICING_CONCURRENCY = 5;
|
||||
|
||||
export type SellabilityFilter = "available" | "all";
|
||||
|
||||
type AnalysisPipelineDependencies = {
|
||||
fetchKeepaDataBatch: typeof fetchKeepaDataBatch;
|
||||
fetchSellabilityBatch: typeof fetchSellabilityBatch;
|
||||
fetchSpApiPricingAndFees: typeof fetchSpApiPricingAndFees;
|
||||
getCache: typeof getCache;
|
||||
setCache: typeof setCache;
|
||||
analyzeProducts: typeof analyzeProducts;
|
||||
};
|
||||
|
||||
export type AnalysisPipelineOptions = {
|
||||
llmBatchSize?: number;
|
||||
pricingConcurrency?: number;
|
||||
@@ -23,6 +32,7 @@ export type AnalysisPipelineOptions = {
|
||||
llmRetryDelayMs?: number;
|
||||
sellability?: SellabilityFilter;
|
||||
useClaude?: boolean;
|
||||
dependencies?: Partial<AnalysisPipelineDependencies>;
|
||||
};
|
||||
|
||||
export function chunkArray<T>(items: T[], chunkSize: number): T[][] {
|
||||
@@ -62,23 +72,33 @@ export async function processProductChunk(
|
||||
const llmRetryDelayMs = Math.max(0, options.llmRetryDelayMs ?? 10_000);
|
||||
const sellabilityFilter = options.sellability ?? "available";
|
||||
const useClaude = options.useClaude === true;
|
||||
const dependencies: AnalysisPipelineDependencies = {
|
||||
fetchKeepaDataBatch,
|
||||
fetchSellabilityBatch,
|
||||
fetchSpApiPricingAndFees,
|
||||
getCache,
|
||||
setCache,
|
||||
analyzeProducts,
|
||||
...options.dependencies,
|
||||
};
|
||||
|
||||
console.log(`\nChecking cache for ${products.length} products...`);
|
||||
const cached = new Map<string, EnrichedProduct>();
|
||||
const excludedCachedAsins = new Set<string>();
|
||||
const excludedCached = new Map<string, EnrichedProduct>();
|
||||
const uncachedProducts: ProductRecord[] = [];
|
||||
|
||||
for (const p of products) {
|
||||
const hit = await getCache(p.asin);
|
||||
const hit = await dependencies.getCache(p.asin);
|
||||
if (hit) {
|
||||
const currentSourceProduct = { ...hit, record: p };
|
||||
if (
|
||||
sellabilityFilter === "all" ||
|
||||
hit.spApi.sellabilityStatus === "available"
|
||||
) {
|
||||
console.log(` [cache hit] ${p.asin}`);
|
||||
cached.set(p.asin, hit);
|
||||
cached.set(p.asin, currentSourceProduct);
|
||||
} else {
|
||||
excludedCachedAsins.add(p.asin);
|
||||
excludedCached.set(p.asin, currentSourceProduct);
|
||||
console.log(
|
||||
` [exclude cached] ${p.asin} - status=${hit.spApi.sellabilityStatus}`,
|
||||
);
|
||||
@@ -89,7 +109,7 @@ export async function processProductChunk(
|
||||
}
|
||||
|
||||
console.log(
|
||||
`${cached.size} cached available, ${excludedCachedAsins.size} cached excluded, ${uncachedProducts.length} to fetch`,
|
||||
`${cached.size} cached available, ${excludedCached.size} cached excluded, ${uncachedProducts.length} to fetch`,
|
||||
);
|
||||
|
||||
const sellabilityMap = new Map<string, SellabilityInfo>();
|
||||
@@ -100,7 +120,7 @@ export async function processProductChunk(
|
||||
console.log(
|
||||
`\nChecking sellability for ${uncachedProducts.length} ASINs...`,
|
||||
);
|
||||
const sellResults = await fetchSellabilityBatch(
|
||||
const sellResults = await dependencies.fetchSellabilityBatch(
|
||||
uncachedProducts.map((p) => p.asin),
|
||||
);
|
||||
|
||||
@@ -143,7 +163,7 @@ export async function processProductChunk(
|
||||
if (availableProducts.length > 0) {
|
||||
console.log(`\nFetching ${availableProducts.length} ASINs from Keepa...`);
|
||||
try {
|
||||
keepaResults = await fetchKeepaDataBatch(
|
||||
keepaResults = await dependencies.fetchKeepaDataBatch(
|
||||
availableProducts.map((p) => p.asin),
|
||||
);
|
||||
} catch (err) {
|
||||
@@ -168,7 +188,10 @@ export async function processProductChunk(
|
||||
sellabilityStatus: "unknown" as const,
|
||||
sellabilityReason: "Sellability check returned no result",
|
||||
};
|
||||
const spApi = await fetchSpApiPricingAndFees(p.asin, sellability);
|
||||
const spApi = await dependencies.fetchSpApiPricingAndFees(
|
||||
p.asin,
|
||||
sellability,
|
||||
);
|
||||
|
||||
const keepa = keepaResults.get(p.asin);
|
||||
if (keepa?.currentPrice && spApi.estimatedSalePrice === 0) {
|
||||
@@ -196,17 +219,33 @@ export async function processProductChunk(
|
||||
const availableAsins = new Set(availableProducts.map((ap) => ap.asin));
|
||||
|
||||
for (const p of products) {
|
||||
if (excludedCachedAsins.has(p.asin)) {
|
||||
const excludedCachedProduct = excludedCached.get(p.asin);
|
||||
if (excludedCachedProduct) {
|
||||
enriched.push({ ...excludedCachedProduct, record: p });
|
||||
continue;
|
||||
}
|
||||
|
||||
const cachedProduct = cached.get(p.asin);
|
||||
if (cachedProduct) {
|
||||
enriched.push(cachedProduct);
|
||||
enriched.push({ ...cachedProduct, record: p });
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!availableAsins.has(p.asin)) {
|
||||
const sellability = sellabilityMap.get(p.asin);
|
||||
if (sellability) {
|
||||
enriched.push({
|
||||
record: p,
|
||||
keepa: null,
|
||||
spApi: {
|
||||
...unknownSpApiData(
|
||||
sellability.sellabilityReason ?? "Product is not available",
|
||||
),
|
||||
...sellability,
|
||||
},
|
||||
fetchedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -221,19 +260,41 @@ export async function processProductChunk(
|
||||
fetchedAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
await setCache(p.asin, product);
|
||||
await dependencies.setCache(p.asin, product);
|
||||
enriched.push(product);
|
||||
}
|
||||
|
||||
const resultsByProduct = new Map<EnrichedProduct, AnalysisResult>();
|
||||
const llmProducts: EnrichedProduct[] = [];
|
||||
for (const product of enriched) {
|
||||
if (
|
||||
sellabilityFilter !== "all" &&
|
||||
product.spApi.sellabilityStatus !== "available"
|
||||
) {
|
||||
resultsByProduct.set(product, {
|
||||
product,
|
||||
verdict: {
|
||||
asin: product.record.asin,
|
||||
verdict: "SKIP",
|
||||
confidence: 100,
|
||||
reasoning:
|
||||
product.spApi.sellabilityReason ??
|
||||
`Sellability status: ${product.spApi.sellabilityStatus}`,
|
||||
},
|
||||
});
|
||||
} else {
|
||||
llmProducts.push(product);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(
|
||||
`\nAnalyzing ${enriched.length} products via LLM (batch size: ${llmBatchSize})...\n`,
|
||||
`\nAnalyzing ${llmProducts.length} products via LLM (batch size: ${llmBatchSize})...\n`,
|
||||
);
|
||||
|
||||
const results: AnalysisResult[] = [];
|
||||
for (let i = 0; i < enriched.length; i += llmBatchSize) {
|
||||
const batch = enriched.slice(i, i + llmBatchSize);
|
||||
for (let i = 0; i < llmProducts.length; i += llmBatchSize) {
|
||||
const batch = llmProducts.slice(i, i + llmBatchSize);
|
||||
const batchNum = Math.floor(i / llmBatchSize) + 1;
|
||||
const totalBatches = Math.ceil(enriched.length / llmBatchSize);
|
||||
const totalBatches = Math.ceil(llmProducts.length / llmBatchSize);
|
||||
console.log(` LLM batch ${batchNum}/${totalBatches}...`);
|
||||
|
||||
if (i > 0 && llmBatchDelayMs > 0) {
|
||||
@@ -242,7 +303,7 @@ export async function processProductChunk(
|
||||
|
||||
let verdicts;
|
||||
try {
|
||||
verdicts = await analyzeProducts(batch, {
|
||||
verdicts = await dependencies.analyzeProducts(batch, {
|
||||
ignoreSellability: sellabilityFilter === "all",
|
||||
useClaude,
|
||||
});
|
||||
@@ -251,7 +312,7 @@ export async function processProductChunk(
|
||||
await wait(llmRetryDelayMs);
|
||||
}
|
||||
try {
|
||||
verdicts = await analyzeProducts(batch, {
|
||||
verdicts = await dependencies.analyzeProducts(batch, {
|
||||
ignoreSellability: sellabilityFilter === "all",
|
||||
useClaude,
|
||||
});
|
||||
@@ -264,7 +325,7 @@ export async function processProductChunk(
|
||||
const enrichedProduct = batch[j];
|
||||
if (!enrichedProduct) continue;
|
||||
|
||||
results.push({
|
||||
resultsByProduct.set(enrichedProduct, {
|
||||
product: enrichedProduct,
|
||||
verdict: verdicts?.[j] ?? {
|
||||
asin: enrichedProduct.record.asin,
|
||||
@@ -276,5 +337,7 @@ export async function processProductChunk(
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
return enriched
|
||||
.map((product) => resultsByProduct.get(product))
|
||||
.filter((result): result is AnalysisResult => result !== undefined);
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { searchProductOffers, type SearxngOfferSearchResult } from "./searxng.ts";
|
||||
import { searchProductOffers, type SearxngOfferSearchResult } from "./integrations/searxng.ts";
|
||||
|
||||
type CliArgs = {
|
||||
query: string;
|
||||
|
||||
13
src/asin.test.ts
Normal file
13
src/asin.test.ts
Normal file
@@ -0,0 +1,13 @@
|
||||
import { expect, test } from "bun:test";
|
||||
import { normalizeAsin, requireAsin } from "./asin.ts";
|
||||
|
||||
test("normalizes any valid ten-character ASIN including ISBN-style values", () => {
|
||||
expect(normalizeAsin(" b07sn9bhvv ")).toBe("B07SN9BHVV");
|
||||
expect(normalizeAsin("0306406152")).toBe("0306406152");
|
||||
});
|
||||
|
||||
test("rejects values that cannot be canonical product ASIN keys", () => {
|
||||
expect(normalizeAsin("short")).toBeNull();
|
||||
expect(normalizeAsin("B07SN9BHV!")).toBeNull();
|
||||
expect(() => requireAsin("012345678901")).toThrow("Invalid ASIN");
|
||||
});
|
||||
14
src/asin.ts
Normal file
14
src/asin.ts
Normal file
@@ -0,0 +1,14 @@
|
||||
export const ASIN_PATTERN = /^[A-Z0-9]{10}$/;
|
||||
|
||||
export function normalizeAsin(value: unknown): string | null {
|
||||
const asin = String(value ?? "").trim().toUpperCase();
|
||||
return ASIN_PATTERN.test(asin) ? asin : null;
|
||||
}
|
||||
|
||||
export function requireAsin(value: unknown): string {
|
||||
const asin = normalizeAsin(value);
|
||||
if (!asin) {
|
||||
throw new Error(`Invalid ASIN: "${String(value ?? "").trim()}"`);
|
||||
}
|
||||
return asin;
|
||||
}
|
||||
@@ -1,8 +1,41 @@
|
||||
import { test, expect, beforeAll, afterAll, beforeEach, mock } from "bun:test";
|
||||
import { Database } from "bun:sqlite";
|
||||
import { getDb, initDb, closeDb } from "./database.ts";
|
||||
import path from "node:path";
|
||||
import { rmSync, mkdirSync } from "node:fs";
|
||||
import { test, expect, beforeEach, mock } from "bun:test";
|
||||
|
||||
let nextId = 0;
|
||||
|
||||
function chainable(resolveWith: any[] = []): any {
|
||||
const p: any = Promise.resolve(resolveWith);
|
||||
p.limit = (_n: any) => chainable(resolveWith);
|
||||
p.where = (_cond: any) => chainable(resolveWith);
|
||||
p.from = (_table: any) => chainable(resolveWith);
|
||||
return p;
|
||||
}
|
||||
|
||||
const makeMockDb = (): any => ({
|
||||
insert: (_table: any) => ({
|
||||
values: (_vals: any) => ({
|
||||
returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]),
|
||||
onConflictDoUpdate: (_conf: any) => Promise.resolve([]),
|
||||
}),
|
||||
}),
|
||||
update: (_table: any) => ({
|
||||
set: (_vals: any) => ({
|
||||
where: (_cond: any) => Promise.resolve([]),
|
||||
}),
|
||||
}),
|
||||
select: (_sel?: any) => ({
|
||||
from: (_table: any) => ({
|
||||
where: (_cond: any) => chainable(),
|
||||
limit: (_n: any) => chainable(),
|
||||
}),
|
||||
}),
|
||||
selectDistinct: (_sel: any) => ({
|
||||
from: (_table: any) => chainable(),
|
||||
}),
|
||||
execute: (_query: any) => Promise.resolve([]),
|
||||
transaction: async (fn: (tx: any) => Promise<any>) => fn(makeMockDb()),
|
||||
});
|
||||
|
||||
mock.module("../db/index.ts", () => ({ db: makeMockDb(), client: {} }));
|
||||
|
||||
const fetchSellabilityBatchMock = mock(async (asins: string[]) => {
|
||||
return new Map(
|
||||
@@ -36,51 +69,28 @@ const analyzeProductsMock = mock(async (products: any[]) => {
|
||||
}));
|
||||
});
|
||||
|
||||
mock.module("./sp-api.ts", () => ({
|
||||
mock.module("../integrations/sp-api.ts", () => ({
|
||||
fetchSellabilityBatch: fetchSellabilityBatchMock,
|
||||
fetchSpApiPricingAndFees: fetchSpApiPricingAndFeesMock,
|
||||
}));
|
||||
|
||||
mock.module("./llm.ts", () => ({
|
||||
mock.module("../integrations/llm.ts", () => ({
|
||||
analyzeProducts: analyzeProductsMock,
|
||||
}));
|
||||
|
||||
const modulePromise = import("./bestsellers-by-category.ts");
|
||||
|
||||
const DB_TEST_PATH = path.join(
|
||||
process.cwd(),
|
||||
"test_output",
|
||||
"test_analysis.sqlite",
|
||||
);
|
||||
|
||||
let db: Database;
|
||||
let processCategory: (db: Database, runId: number, category: any, perCategoryTop: number) => Promise<any>;
|
||||
let insertCategoryRunSummary: (db: Database, summary: any, runTimestamp: string) => Promise<number>;
|
||||
let processCategory: (runId: number, category: any, perCategoryTop: number) => Promise<any>;
|
||||
let insertCategoryRunSummary: (summary: any, runTimestamp: string) => Promise<number>;
|
||||
let originalFetch: typeof globalThis.fetch;
|
||||
|
||||
beforeAll(async () => {
|
||||
const mod = await modulePromise;
|
||||
processCategory = mod.processCategory;
|
||||
insertCategoryRunSummary = mod.insertCategoryRunSummary;
|
||||
|
||||
rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true });
|
||||
mkdirSync(path.dirname(DB_TEST_PATH), { recursive: true });
|
||||
initDb(DB_TEST_PATH);
|
||||
db = getDb(DB_TEST_PATH);
|
||||
|
||||
originalFetch = globalThis.fetch;
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
closeDb();
|
||||
rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true });
|
||||
});
|
||||
const mod = await modulePromise;
|
||||
processCategory = mod.processCategory;
|
||||
insertCategoryRunSummary = mod.insertCategoryRunSummary;
|
||||
originalFetch = globalThis.fetch;
|
||||
|
||||
beforeEach(() => {
|
||||
db.run("DELETE FROM product_analysis_results");
|
||||
db.run("DELETE FROM category_analysis_runs");
|
||||
|
||||
nextId = 0;
|
||||
globalThis.fetch = mock(async (input: string | URL | Request) => {
|
||||
const rawUrl =
|
||||
typeof input === "string"
|
||||
@@ -139,39 +149,34 @@ test("processCategory function test", async () => {
|
||||
childCount: 0,
|
||||
};
|
||||
|
||||
const runId = await insertCategoryRunSummary(db, {
|
||||
categoryId: mockCategory.id,
|
||||
categoryLabel: mockCategory.label,
|
||||
topAsinsChecked: 0,
|
||||
availableAsins: 0,
|
||||
fba: 0,
|
||||
fbm: 0,
|
||||
skip: 0,
|
||||
status: "running",
|
||||
error: "",
|
||||
results: [],
|
||||
}, new Date().toISOString());
|
||||
const summary = await processCategory(db, runId, mockCategory, 2);
|
||||
const runId = await insertCategoryRunSummary(
|
||||
{
|
||||
categoryId: mockCategory.id,
|
||||
categoryLabel: mockCategory.label,
|
||||
topAsinsChecked: 0,
|
||||
availableAsins: 0,
|
||||
fba: 0,
|
||||
fbm: 0,
|
||||
skip: 0,
|
||||
status: "running",
|
||||
error: "",
|
||||
results: [],
|
||||
},
|
||||
new Date().toISOString(),
|
||||
);
|
||||
|
||||
const categoryRun = db.query("SELECT * FROM category_analysis_runs").all() as any[];
|
||||
expect(categoryRun.length).toBe(1);
|
||||
expect(categoryRun[0].category_label).toBe("Category 1");
|
||||
expect(categoryRun[0].top_asins_checked).toBe(2);
|
||||
expect(categoryRun[0].available_asins).toBe(2);
|
||||
expect(categoryRun[0].fba_count).toBe(1);
|
||||
expect(categoryRun[0].fbm_count).toBe(1);
|
||||
expect(categoryRun[0].status).toBe("ok");
|
||||
const summary = await processCategory(runId, mockCategory, 2);
|
||||
|
||||
const productResults = db.query("SELECT * FROM product_analysis_results ORDER BY asin").all() as any[];
|
||||
expect(productResults.length).toBe(2);
|
||||
expect(summary.status).toBe("ok");
|
||||
expect(summary.topAsinsChecked).toBe(2);
|
||||
expect(summary.availableAsins).toBe(2);
|
||||
expect(summary.fba).toBe(1);
|
||||
expect(summary.fbm).toBe(1);
|
||||
expect(summary.results?.length).toBe(2);
|
||||
expect(summary.results?.[0]?.product.record.asin).toBe("B000000001");
|
||||
expect(summary.results?.[0]?.verdict.verdict).toBe("FBA");
|
||||
expect(summary.results?.[1]?.product.record.asin).toBe("B000000002");
|
||||
expect(summary.results?.[1]?.verdict.verdict).toBe("FBM");
|
||||
|
||||
expect(productResults[0].asin).toBe("B000000001");
|
||||
expect(productResults[0].name).toBe("Product One");
|
||||
expect(productResults[0].verdict).toBe("FBA");
|
||||
expect(productResults[0].run_id).toBe(categoryRun[0].id);
|
||||
|
||||
expect(productResults[1].asin).toBe("B000000002");
|
||||
expect(productResults[1].name).toBe("Product Two");
|
||||
expect(productResults[1].verdict).toBe("FBM");
|
||||
expect(productResults[1].run_id).toBe(categoryRun[0].id);
|
||||
globalThis.fetch = originalFetch;
|
||||
});
|
||||
@@ -1,9 +1,14 @@
|
||||
import { existsSync, mkdirSync, readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { type Database, getDb, initDb } from "./database.ts";
|
||||
import { config } from "./config.ts";
|
||||
import { analyzeProducts } from "./llm.ts";
|
||||
import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts";
|
||||
import { existsSync, mkdirSync, readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { normalizeAsin } from "../asin.ts";
|
||||
import {
|
||||
createCategoryRun,
|
||||
persistLlmResults,
|
||||
updateCategoryRun,
|
||||
} from "../db/persistence.ts";
|
||||
import { config } from "../config.ts";
|
||||
import { analyzeProducts } from "../integrations/llm.ts";
|
||||
import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "../integrations/sp-api.ts";
|
||||
import type {
|
||||
AnalysisResult,
|
||||
EnrichedProduct,
|
||||
@@ -12,7 +17,7 @@ import type {
|
||||
ProductRecord,
|
||||
SellabilityInfo,
|
||||
SpApiData,
|
||||
} from "./types.ts";
|
||||
} from "../types.ts";
|
||||
|
||||
type CategoryInfo = {
|
||||
id: number;
|
||||
@@ -138,37 +143,14 @@ function printUsageAndExit(message: string): never {
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
export async function insertCategoryRunSummary(
|
||||
db: Database,
|
||||
summary: CategoryRunSummary,
|
||||
runTimestamp: string,
|
||||
): Promise<number> {
|
||||
const query = `
|
||||
INSERT INTO category_analysis_runs (
|
||||
category_id, category_label, run_timestamp,
|
||||
top_asins_checked, available_asins,
|
||||
fba_count, fbm_count, skip_count,
|
||||
status, error_message
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
|
||||
`;
|
||||
const result = db.run(query, [
|
||||
summary.categoryId,
|
||||
summary.categoryLabel,
|
||||
runTimestamp,
|
||||
summary.topAsinsChecked,
|
||||
summary.availableAsins,
|
||||
summary.fba,
|
||||
summary.fbm,
|
||||
summary.skip,
|
||||
summary.status,
|
||||
summary.error,
|
||||
]);
|
||||
// Bun's SQLite client returns { changes: number, lastInsertRowid: number | bigint }
|
||||
return Number(result.lastInsertRowid);
|
||||
}
|
||||
export async function insertCategoryRunSummary(
|
||||
summary: CategoryRunSummary,
|
||||
runTimestamp: string,
|
||||
): Promise<number> {
|
||||
return createCategoryRun(summary, runTimestamp);
|
||||
}
|
||||
|
||||
export async function updateCategoryRunSummary(
|
||||
db: Database,
|
||||
runId: number,
|
||||
summary: Pick<
|
||||
CategoryRunSummary,
|
||||
@@ -180,138 +162,20 @@ export async function updateCategoryRunSummary(
|
||||
| "status"
|
||||
| "error"
|
||||
>,
|
||||
): Promise<void> {
|
||||
db.run(
|
||||
`
|
||||
UPDATE category_analysis_runs
|
||||
SET
|
||||
top_asins_checked = ?,
|
||||
available_asins = ?,
|
||||
fba_count = ?,
|
||||
fbm_count = ?,
|
||||
skip_count = ?,
|
||||
status = ?,
|
||||
error_message = ?
|
||||
WHERE id = ?
|
||||
`,
|
||||
[
|
||||
summary.topAsinsChecked,
|
||||
summary.availableAsins,
|
||||
summary.fba,
|
||||
summary.fbm,
|
||||
summary.skip,
|
||||
summary.status,
|
||||
summary.error,
|
||||
runId,
|
||||
],
|
||||
);
|
||||
}
|
||||
): Promise<void> {
|
||||
await updateCategoryRun(runId, summary);
|
||||
}
|
||||
|
||||
export async function insertProductAnalysisResults(
|
||||
db: Database,
|
||||
runId: number,
|
||||
results: AnalysisResult[],
|
||||
): Promise<void> {
|
||||
if (results.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const insertStmt = db.prepare(`
|
||||
INSERT INTO product_analysis_results (
|
||||
asin, run_id, name, brand, category, unit_cost,
|
||||
current_price, avg_price_90d, avg_price_90d_sheet,
|
||||
selling_price_sheet, sales_rank, sales_rank_avg_90d,
|
||||
seller_count, amazon_is_seller, amazon_buybox_share_pct_90d,
|
||||
monthly_sold, rank_drops_30d, rank_drops_90d,
|
||||
fba_fee, fbm_fee, referral_percent, can_sell,
|
||||
sellability_status, sellability_reason,
|
||||
verdict, confidence, reasoning, fetched_at
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?
|
||||
)
|
||||
ON CONFLICT(asin) DO UPDATE SET
|
||||
run_id = excluded.run_id,
|
||||
name = excluded.name,
|
||||
brand = excluded.brand,
|
||||
category = excluded.category,
|
||||
unit_cost = excluded.unit_cost,
|
||||
current_price = excluded.current_price,
|
||||
avg_price_90d = excluded.avg_price_90d,
|
||||
avg_price_90d_sheet = excluded.avg_price_90d_sheet,
|
||||
selling_price_sheet = excluded.selling_price_sheet,
|
||||
sales_rank = excluded.sales_rank,
|
||||
sales_rank_avg_90d = excluded.sales_rank_avg_90d,
|
||||
seller_count = excluded.seller_count,
|
||||
amazon_is_seller = excluded.amazon_is_seller,
|
||||
amazon_buybox_share_pct_90d = excluded.amazon_buybox_share_pct_90d,
|
||||
monthly_sold = excluded.monthly_sold,
|
||||
rank_drops_30d = excluded.rank_drops_30d,
|
||||
rank_drops_90d = excluded.rank_drops_90d,
|
||||
fba_fee = excluded.fba_fee,
|
||||
fbm_fee = excluded.fbm_fee,
|
||||
referral_percent = excluded.referral_percent,
|
||||
can_sell = excluded.can_sell,
|
||||
sellability_status = excluded.sellability_status,
|
||||
sellability_reason = excluded.sellability_reason,
|
||||
verdict = excluded.verdict,
|
||||
confidence = excluded.confidence,
|
||||
reasoning = excluded.reasoning,
|
||||
fetched_at = excluded.fetched_at;
|
||||
`);
|
||||
|
||||
db.transaction((resultsBatch: AnalysisResult[]) => {
|
||||
for (const r of resultsBatch) {
|
||||
const price =
|
||||
r.product.keepa?.currentPrice ??
|
||||
r.product.record.sellingPriceFromSheet ??
|
||||
r.product.spApi.estimatedSalePrice;
|
||||
const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank;
|
||||
|
||||
insertStmt.run(
|
||||
r.product.record.asin,
|
||||
runId,
|
||||
r.product.record.name,
|
||||
r.product.record.brand ?? null,
|
||||
r.product.record.category ??
|
||||
r.product.keepa?.categoryTree?.join(" > ") ??
|
||||
null,
|
||||
r.product.record.unitCost ?? null,
|
||||
price ?? null,
|
||||
r.product.keepa?.avgPrice90 ?? null,
|
||||
r.product.record.avgPrice90FromSheet ?? null,
|
||||
r.product.record.sellingPriceFromSheet ?? null,
|
||||
rank ?? null,
|
||||
r.product.keepa?.salesRankAvg90 ?? null,
|
||||
r.product.keepa?.sellerCount ?? null,
|
||||
r.product.keepa?.amazonIsSeller == null
|
||||
? null
|
||||
: r.product.keepa.amazonIsSeller
|
||||
? 1
|
||||
: 0,
|
||||
r.product.keepa?.amazonBuyboxSharePct90d ?? null,
|
||||
r.product.keepa?.monthlySold ?? null,
|
||||
r.product.keepa?.salesRankDrops30 ?? null,
|
||||
r.product.keepa?.salesRankDrops90 ?? null,
|
||||
r.product.spApi.fbaFee ?? null,
|
||||
r.product.spApi.fbmFee ?? null,
|
||||
r.product.spApi.referralFeePercent ?? null,
|
||||
r.product.spApi.canSell == null
|
||||
? "unknown"
|
||||
: r.product.spApi.canSell
|
||||
? "yes"
|
||||
: "no",
|
||||
r.product.spApi.sellabilityStatus ?? null,
|
||||
r.product.spApi.sellabilityReason ?? null,
|
||||
r.verdict.verdict,
|
||||
r.verdict.confidence,
|
||||
r.verdict.reasoning ?? null,
|
||||
r.product.fetchedAt,
|
||||
);
|
||||
}
|
||||
})(results); // Execute the transaction with the results batch
|
||||
}
|
||||
): Promise<void> {
|
||||
if (results.length === 0) return;
|
||||
await persistLlmResults(runId, results, {
|
||||
source: "category_analysis",
|
||||
metadataSource: "catalog",
|
||||
});
|
||||
}
|
||||
|
||||
function loadCategoryBlacklist(filePath: string): Set<number> {
|
||||
const blacklist = new Set<number>();
|
||||
@@ -690,10 +554,14 @@ async function fetchCategoryBestSellerAsins(
|
||||
];
|
||||
|
||||
for (const value of candidates) {
|
||||
if (Array.isArray(value)) {
|
||||
return [
|
||||
...new Set(value.map((v) => String(v).trim()).filter(Boolean)),
|
||||
].slice(0, limit);
|
||||
if (Array.isArray(value)) {
|
||||
return [
|
||||
...new Set(
|
||||
value
|
||||
.map((v) => normalizeAsin(v))
|
||||
.filter((asin): asin is string => asin !== null),
|
||||
),
|
||||
].slice(0, limit);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -945,10 +813,10 @@ async function fetchKeepaEnrichmentMap(
|
||||
`/product?key=${encodeURIComponent(config.keepaApiKey)}&domain=${DOMAIN_US}&asin=${asinParam}&stats=90&buybox=1&days=90`,
|
||||
);
|
||||
|
||||
const products = Array.isArray(data?.products) ? data.products : [];
|
||||
for (const product of products) {
|
||||
const asin = String(product?.asin ?? "").trim();
|
||||
if (!asin) continue;
|
||||
const products = Array.isArray(data?.products) ? data.products : [];
|
||||
for (const product of products) {
|
||||
const asin = normalizeAsin(product?.asin);
|
||||
if (!asin) continue;
|
||||
out.set(asin, {
|
||||
keepa: parseKeepaProduct(product),
|
||||
title: String(product?.title ?? "").trim(),
|
||||
@@ -1014,7 +882,6 @@ function buildEnrichedProducts(
|
||||
}
|
||||
|
||||
export async function processCategory(
|
||||
db: Database,
|
||||
runId: number,
|
||||
category: CategoryInfo,
|
||||
perCategoryTop: number,
|
||||
@@ -1025,7 +892,7 @@ export async function processCategory(
|
||||
const topAsins = await fetchCategoryBestSellerAsins(category, perCategoryTop);
|
||||
if (topAsins.length === 0) {
|
||||
log("info", " Keepa returned no ASINs for this category.");
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: 0,
|
||||
availableAsins: 0,
|
||||
fba: 0,
|
||||
@@ -1069,7 +936,7 @@ export async function processCategory(
|
||||
` Sellable ASINs: ${availableAsins.length}/${uniqueTopAsins.length}`,
|
||||
);
|
||||
if (availableAsins.length === 0) {
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: uniqueTopAsins.length,
|
||||
availableAsins: 0,
|
||||
fba: 0,
|
||||
@@ -1137,7 +1004,7 @@ export async function processCategory(
|
||||
},
|
||||
}));
|
||||
|
||||
await insertProductAnalysisResults(db, runId, batchResults);
|
||||
await insertProductAnalysisResults(runId, batchResults);
|
||||
|
||||
for (const result of batchResults) {
|
||||
results.push(result);
|
||||
@@ -1150,7 +1017,7 @@ export async function processCategory(
|
||||
}
|
||||
}
|
||||
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: uniqueTopAsins.length,
|
||||
availableAsins: availableAsins.length,
|
||||
fba,
|
||||
@@ -1170,7 +1037,7 @@ export async function processCategory(
|
||||
}
|
||||
}
|
||||
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: uniqueTopAsins.length,
|
||||
availableAsins: availableAsins.length,
|
||||
fba,
|
||||
@@ -1199,10 +1066,6 @@ export async function main(): Promise<void> {
|
||||
assertSpApiPrerequisites();
|
||||
|
||||
mkdirSync(args.outputDir, { recursive: true });
|
||||
const DB_PATH =
|
||||
process.env.RESULTS_DB_PATH || path.join(process.cwd(), "db", "results.db");
|
||||
initDb(DB_PATH);
|
||||
const db = getDb(DB_PATH);
|
||||
|
||||
log("info", "Starting per-category bestseller pipeline");
|
||||
log("info", `Marketplace: ${config.spApiMarketplaceId}`);
|
||||
@@ -1236,7 +1099,6 @@ export async function main(): Promise<void> {
|
||||
let runId: number | undefined;
|
||||
try {
|
||||
runId = await insertCategoryRunSummary(
|
||||
db,
|
||||
{
|
||||
categoryId: category.id,
|
||||
categoryLabel: category.label,
|
||||
@@ -1253,7 +1115,6 @@ export async function main(): Promise<void> {
|
||||
);
|
||||
|
||||
categorySummary = await processCategory(
|
||||
db,
|
||||
runId,
|
||||
category,
|
||||
args.perCategoryTop,
|
||||
@@ -1283,7 +1144,7 @@ export async function main(): Promise<void> {
|
||||
results: [],
|
||||
};
|
||||
if (runId) {
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: 0,
|
||||
availableAsins: 0,
|
||||
fba: 0,
|
||||
@@ -1,8 +1,41 @@
|
||||
import { test, expect, beforeAll, afterAll, beforeEach, mock } from "bun:test";
|
||||
import { Database } from "bun:sqlite";
|
||||
import { getDb, initDb, closeDb } from "./database.ts";
|
||||
import path from "node:path";
|
||||
import { rmSync, mkdirSync } from "node:fs";
|
||||
import { test, expect, beforeEach, mock } from "bun:test";
|
||||
|
||||
let nextId = 0;
|
||||
|
||||
function chainable(resolveWith: any[] = []): any {
|
||||
const p: any = Promise.resolve(resolveWith);
|
||||
p.limit = (_n: any) => chainable(resolveWith);
|
||||
p.where = (_cond: any) => chainable(resolveWith);
|
||||
p.from = (_table: any) => chainable(resolveWith);
|
||||
return p;
|
||||
}
|
||||
|
||||
const makeMockDb = (): any => ({
|
||||
insert: (_table: any) => ({
|
||||
values: (_vals: any) => ({
|
||||
returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]),
|
||||
onConflictDoUpdate: (_conf: any) => Promise.resolve([]),
|
||||
}),
|
||||
}),
|
||||
update: (_table: any) => ({
|
||||
set: (_vals: any) => ({
|
||||
where: (_cond: any) => Promise.resolve([]),
|
||||
}),
|
||||
}),
|
||||
select: (_sel?: any) => ({
|
||||
from: (_table: any) => ({
|
||||
where: (_cond: any) => chainable(),
|
||||
limit: (_n: any) => chainable(),
|
||||
}),
|
||||
}),
|
||||
selectDistinct: (_sel: any) => ({
|
||||
from: (_table: any) => chainable(),
|
||||
}),
|
||||
execute: (_query: any) => Promise.resolve([]),
|
||||
transaction: async (fn: (tx: any) => Promise<any>) => fn(makeMockDb()),
|
||||
});
|
||||
|
||||
mock.module("../db/index.ts", () => ({ db: makeMockDb(), client: {} }));
|
||||
|
||||
const fetchSellabilityBatchMock = mock(async (asins: string[]) => {
|
||||
return new Map<string, any>(
|
||||
@@ -51,55 +84,28 @@ const analyzeProductsMock = mock(async (products: any[]) => {
|
||||
}));
|
||||
});
|
||||
|
||||
mock.module("./sp-api.ts", () => ({
|
||||
mock.module("../integrations/sp-api.ts", () => ({
|
||||
fetchSellabilityBatch: fetchSellabilityBatchMock,
|
||||
fetchSpApiPricingAndFees: fetchSpApiPricingAndFeesMock,
|
||||
}));
|
||||
|
||||
mock.module("./llm.ts", () => ({
|
||||
mock.module("../integrations/llm.ts", () => ({
|
||||
analyzeProducts: analyzeProductsMock,
|
||||
}));
|
||||
|
||||
const modulePromise = import("./mid-range-sellers-by-category.ts");
|
||||
|
||||
const DB_TEST_PATH = path.join(
|
||||
process.cwd(),
|
||||
"test_output",
|
||||
"test_mid_range_analysis.sqlite",
|
||||
);
|
||||
|
||||
let db: Database;
|
||||
let processCategory: any;
|
||||
let insertCategoryRunSummary: (
|
||||
db: Database,
|
||||
summary: any,
|
||||
runTimestamp: string,
|
||||
) => Promise<number>;
|
||||
let insertCategoryRunSummary: (summary: any, runTimestamp: string) => Promise<number>;
|
||||
let originalFetch: typeof globalThis.fetch;
|
||||
|
||||
beforeAll(async () => {
|
||||
const mod = await modulePromise;
|
||||
processCategory = mod.processCategory;
|
||||
insertCategoryRunSummary = mod.insertCategoryRunSummary;
|
||||
|
||||
rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true });
|
||||
mkdirSync(path.dirname(DB_TEST_PATH), { recursive: true });
|
||||
initDb(DB_TEST_PATH);
|
||||
db = getDb(DB_TEST_PATH);
|
||||
|
||||
originalFetch = globalThis.fetch;
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
closeDb();
|
||||
rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true });
|
||||
});
|
||||
const mod = await modulePromise;
|
||||
processCategory = mod.processCategory;
|
||||
insertCategoryRunSummary = mod.insertCategoryRunSummary;
|
||||
originalFetch = globalThis.fetch;
|
||||
|
||||
beforeEach(() => {
|
||||
db.run("DELETE FROM product_analysis_results");
|
||||
db.run("DELETE FROM category_analysis_runs");
|
||||
|
||||
nextId = 0;
|
||||
globalThis.fetch = mock(async (input: string | URL | Request) => {
|
||||
const rawUrl =
|
||||
typeof input === "string"
|
||||
@@ -138,25 +144,8 @@ beforeEach(() => {
|
||||
buyBoxStatsAmazon90: 40,
|
||||
stats: {
|
||||
current: [
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
1000,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
5,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
2599,
|
||||
null, null, null, 1000, null, null, null, null, null, null, null, 5,
|
||||
null, null, null, null, null, null, 2599,
|
||||
],
|
||||
avg: [2400, null, null, 1200],
|
||||
},
|
||||
@@ -171,25 +160,8 @@ beforeEach(() => {
|
||||
buyBoxStatsAmazon90: 50,
|
||||
stats: {
|
||||
current: [
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
2000,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
3,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
1999,
|
||||
null, null, null, 2000, null, null, null, null, null, null, null, 3,
|
||||
null, null, null, null, null, null, 1999,
|
||||
],
|
||||
avg: [1800, null, null, 2200],
|
||||
},
|
||||
@@ -204,25 +176,8 @@ beforeEach(() => {
|
||||
buyBoxStatsAmazon90: 50,
|
||||
stats: {
|
||||
current: [
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
1500,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
4,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
2099,
|
||||
null, null, null, 1500, null, null, null, null, null, null, null, 4,
|
||||
null, null, null, null, null, null, 2099,
|
||||
],
|
||||
avg: [2000, null, null, 1800],
|
||||
},
|
||||
@@ -237,25 +192,8 @@ beforeEach(() => {
|
||||
buyBoxStatsAmazon90: 95,
|
||||
stats: {
|
||||
current: [
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
3000,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
4,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
2899,
|
||||
null, null, null, 3000, null, null, null, null, null, null, null, 4,
|
||||
null, null, null, null, null, null, 2899,
|
||||
],
|
||||
avg: [2600, null, null, 2800],
|
||||
},
|
||||
@@ -269,25 +207,8 @@ beforeEach(() => {
|
||||
isAmazonSeller: false,
|
||||
stats: {
|
||||
current: [
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
3200,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
25,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
3500,
|
||||
null, null, null, 3200, null, null, null, null, null, null, null, 25,
|
||||
null, null, null, null, null, null, 3500,
|
||||
],
|
||||
avg: [3200, null, null, 3200],
|
||||
},
|
||||
@@ -315,7 +236,6 @@ test("processCategory only analyzes sellable mid-range matches", async () => {
|
||||
};
|
||||
|
||||
const runId = await insertCategoryRunSummary(
|
||||
db,
|
||||
{
|
||||
categoryId: mockCategory.id,
|
||||
categoryLabel: mockCategory.label,
|
||||
@@ -332,7 +252,6 @@ test("processCategory only analyzes sellable mid-range matches", async () => {
|
||||
);
|
||||
|
||||
const summary = await processCategory(
|
||||
db,
|
||||
runId,
|
||||
mockCategory,
|
||||
3,
|
||||
@@ -345,6 +264,7 @@ test("processCategory only analyzes sellable mid-range matches", async () => {
|
||||
20,
|
||||
15,
|
||||
85,
|
||||
"strict",
|
||||
);
|
||||
|
||||
expect(summary.status).toBe("ok");
|
||||
@@ -352,23 +272,7 @@ test("processCategory only analyzes sellable mid-range matches", async () => {
|
||||
expect(summary.availableAsins).toBe(1);
|
||||
expect(summary.results?.length).toBe(1);
|
||||
|
||||
const productResults = db
|
||||
.query(
|
||||
"SELECT asin, monthly_sold, can_sell, sellability_status FROM product_analysis_results ORDER BY monthly_sold DESC",
|
||||
)
|
||||
.all() as Array<{
|
||||
asin: string;
|
||||
monthly_sold: number;
|
||||
can_sell: string;
|
||||
sellability_status: string;
|
||||
}>;
|
||||
|
||||
expect(productResults.length).toBe(1);
|
||||
expect(productResults.map((row) => row.asin)).toEqual(["B000000001"]);
|
||||
|
||||
const sellable = productResults.find((row) => row.asin === "B000000001");
|
||||
expect(sellable?.can_sell).toBe("yes");
|
||||
expect(sellable?.sellability_status).toBe("available");
|
||||
globalThis.fetch = originalFetch;
|
||||
});
|
||||
|
||||
test("processCategory returns empty when no products match mid-range criteria", async () => {
|
||||
@@ -380,7 +284,6 @@ test("processCategory returns empty when no products match mid-range criteria",
|
||||
};
|
||||
|
||||
const runId = await insertCategoryRunSummary(
|
||||
db,
|
||||
{
|
||||
categoryId: mockCategory.id,
|
||||
categoryLabel: mockCategory.label,
|
||||
@@ -397,7 +300,6 @@ test("processCategory returns empty when no products match mid-range criteria",
|
||||
);
|
||||
|
||||
const summary = await processCategory(
|
||||
db,
|
||||
runId,
|
||||
mockCategory,
|
||||
3,
|
||||
@@ -410,6 +312,7 @@ test("processCategory returns empty when no products match mid-range criteria",
|
||||
20,
|
||||
15,
|
||||
85,
|
||||
"strict",
|
||||
);
|
||||
|
||||
expect(summary.status).toBe("empty");
|
||||
@@ -417,8 +320,5 @@ test("processCategory returns empty when no products match mid-range criteria",
|
||||
expect(summary.availableAsins).toBe(0);
|
||||
expect(summary.results?.length).toBe(0);
|
||||
|
||||
const rows = db
|
||||
.query("SELECT COUNT(*) as c FROM product_analysis_results")
|
||||
.all() as Array<{ c: number }>;
|
||||
expect(rows[0]?.c).toBe(0);
|
||||
globalThis.fetch = originalFetch;
|
||||
});
|
||||
@@ -1,17 +1,22 @@
|
||||
import { existsSync, mkdirSync, readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { createInterface } from "node:readline/promises";
|
||||
import { stdin as input, stdout as output } from "node:process";
|
||||
import { type Database, getDb, initDb } from "./database.ts";
|
||||
import { config } from "./config.ts";
|
||||
import { existsSync, mkdirSync, readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { createInterface } from "node:readline/promises";
|
||||
import { stdin as input, stdout as output } from "node:process";
|
||||
import { normalizeAsin } from "../asin.ts";
|
||||
import {
|
||||
createCategoryRun,
|
||||
persistLlmResults,
|
||||
updateCategoryRun,
|
||||
} from "../db/persistence.ts";
|
||||
import { config } from "../config.ts";
|
||||
import {
|
||||
connectCache,
|
||||
disconnectCache,
|
||||
getApiCache,
|
||||
setApiCache,
|
||||
} from "./cache.ts";
|
||||
import { analyzeProducts } from "./llm.ts";
|
||||
import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts";
|
||||
} from "../integrations/cache.ts";
|
||||
import { analyzeProducts } from "../integrations/llm.ts";
|
||||
import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "../integrations/sp-api.ts";
|
||||
import type {
|
||||
AnalysisResult,
|
||||
EnrichedProduct,
|
||||
@@ -20,7 +25,7 @@ import type {
|
||||
ProductRecord,
|
||||
SellabilityInfo,
|
||||
SpApiData,
|
||||
} from "./types.ts";
|
||||
} from "../types.ts";
|
||||
|
||||
type CategoryInfo = {
|
||||
id: number;
|
||||
@@ -473,37 +478,14 @@ async function promptCategoryIds(
|
||||
}
|
||||
}
|
||||
|
||||
export async function insertCategoryRunSummary(
|
||||
db: Database,
|
||||
summary: CategoryRunSummary,
|
||||
runTimestamp: string,
|
||||
): Promise<number> {
|
||||
const query = `
|
||||
INSERT INTO category_analysis_runs (
|
||||
category_id, category_label, run_timestamp,
|
||||
top_asins_checked, available_asins,
|
||||
fba_count, fbm_count, skip_count,
|
||||
status, error_message
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
|
||||
`;
|
||||
const result = db.run(query, [
|
||||
summary.categoryId,
|
||||
summary.categoryLabel,
|
||||
runTimestamp,
|
||||
summary.topAsinsChecked,
|
||||
summary.availableAsins,
|
||||
summary.fba,
|
||||
summary.fbm,
|
||||
summary.skip,
|
||||
summary.status,
|
||||
summary.error,
|
||||
]);
|
||||
// Bun's SQLite client returns { changes: number, lastInsertRowid: number | bigint }
|
||||
return Number(result.lastInsertRowid);
|
||||
}
|
||||
export async function insertCategoryRunSummary(
|
||||
summary: CategoryRunSummary,
|
||||
runTimestamp: string,
|
||||
): Promise<number> {
|
||||
return createCategoryRun(summary, runTimestamp);
|
||||
}
|
||||
|
||||
export async function updateCategoryRunSummary(
|
||||
db: Database,
|
||||
runId: number,
|
||||
summary: Pick<
|
||||
CategoryRunSummary,
|
||||
@@ -515,138 +497,20 @@ export async function updateCategoryRunSummary(
|
||||
| "status"
|
||||
| "error"
|
||||
>,
|
||||
): Promise<void> {
|
||||
db.run(
|
||||
`
|
||||
UPDATE category_analysis_runs
|
||||
SET
|
||||
top_asins_checked = ?,
|
||||
available_asins = ?,
|
||||
fba_count = ?,
|
||||
fbm_count = ?,
|
||||
skip_count = ?,
|
||||
status = ?,
|
||||
error_message = ?
|
||||
WHERE id = ?
|
||||
`,
|
||||
[
|
||||
summary.topAsinsChecked,
|
||||
summary.availableAsins,
|
||||
summary.fba,
|
||||
summary.fbm,
|
||||
summary.skip,
|
||||
summary.status,
|
||||
summary.error,
|
||||
runId,
|
||||
],
|
||||
);
|
||||
}
|
||||
): Promise<void> {
|
||||
await updateCategoryRun(runId, summary);
|
||||
}
|
||||
|
||||
export async function insertProductAnalysisResults(
|
||||
db: Database,
|
||||
runId: number,
|
||||
results: AnalysisResult[],
|
||||
): Promise<void> {
|
||||
if (results.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const insertStmt = db.prepare(`
|
||||
INSERT INTO product_analysis_results (
|
||||
asin, run_id, name, brand, category, unit_cost,
|
||||
current_price, avg_price_90d, avg_price_90d_sheet,
|
||||
selling_price_sheet, sales_rank, sales_rank_avg_90d,
|
||||
seller_count, amazon_is_seller, amazon_buybox_share_pct_90d,
|
||||
monthly_sold, rank_drops_30d, rank_drops_90d,
|
||||
fba_fee, fbm_fee, referral_percent, can_sell,
|
||||
sellability_status, sellability_reason,
|
||||
verdict, confidence, reasoning, fetched_at
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?
|
||||
)
|
||||
ON CONFLICT(asin) DO UPDATE SET
|
||||
run_id = excluded.run_id,
|
||||
name = excluded.name,
|
||||
brand = excluded.brand,
|
||||
category = excluded.category,
|
||||
unit_cost = excluded.unit_cost,
|
||||
current_price = excluded.current_price,
|
||||
avg_price_90d = excluded.avg_price_90d,
|
||||
avg_price_90d_sheet = excluded.avg_price_90d_sheet,
|
||||
selling_price_sheet = excluded.selling_price_sheet,
|
||||
sales_rank = excluded.sales_rank,
|
||||
sales_rank_avg_90d = excluded.sales_rank_avg_90d,
|
||||
seller_count = excluded.seller_count,
|
||||
amazon_is_seller = excluded.amazon_is_seller,
|
||||
amazon_buybox_share_pct_90d = excluded.amazon_buybox_share_pct_90d,
|
||||
monthly_sold = excluded.monthly_sold,
|
||||
rank_drops_30d = excluded.rank_drops_30d,
|
||||
rank_drops_90d = excluded.rank_drops_90d,
|
||||
fba_fee = excluded.fba_fee,
|
||||
fbm_fee = excluded.fbm_fee,
|
||||
referral_percent = excluded.referral_percent,
|
||||
can_sell = excluded.can_sell,
|
||||
sellability_status = excluded.sellability_status,
|
||||
sellability_reason = excluded.sellability_reason,
|
||||
verdict = excluded.verdict,
|
||||
confidence = excluded.confidence,
|
||||
reasoning = excluded.reasoning,
|
||||
fetched_at = excluded.fetched_at;
|
||||
`);
|
||||
|
||||
db.transaction((resultsBatch: AnalysisResult[]) => {
|
||||
for (const r of resultsBatch) {
|
||||
const price =
|
||||
r.product.keepa?.currentPrice ??
|
||||
r.product.record.sellingPriceFromSheet ??
|
||||
r.product.spApi.estimatedSalePrice;
|
||||
const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank;
|
||||
|
||||
insertStmt.run(
|
||||
r.product.record.asin,
|
||||
runId,
|
||||
r.product.record.name,
|
||||
r.product.record.brand ?? null,
|
||||
r.product.record.category ??
|
||||
r.product.keepa?.categoryTree?.join(" > ") ??
|
||||
null,
|
||||
r.product.record.unitCost ?? null,
|
||||
price ?? null,
|
||||
r.product.keepa?.avgPrice90 ?? null,
|
||||
r.product.record.avgPrice90FromSheet ?? null,
|
||||
r.product.record.sellingPriceFromSheet ?? null,
|
||||
rank ?? null,
|
||||
r.product.keepa?.salesRankAvg90 ?? null,
|
||||
r.product.keepa?.sellerCount ?? null,
|
||||
r.product.keepa?.amazonIsSeller == null
|
||||
? null
|
||||
: r.product.keepa.amazonIsSeller
|
||||
? 1
|
||||
: 0,
|
||||
r.product.keepa?.amazonBuyboxSharePct90d ?? null,
|
||||
r.product.keepa?.monthlySold ?? null,
|
||||
r.product.keepa?.salesRankDrops30 ?? null,
|
||||
r.product.keepa?.salesRankDrops90 ?? null,
|
||||
r.product.spApi.fbaFee ?? null,
|
||||
r.product.spApi.fbmFee ?? null,
|
||||
r.product.spApi.referralFeePercent ?? null,
|
||||
r.product.spApi.canSell == null
|
||||
? "unknown"
|
||||
: r.product.spApi.canSell
|
||||
? "yes"
|
||||
: "no",
|
||||
r.product.spApi.sellabilityStatus ?? null,
|
||||
r.product.spApi.sellabilityReason ?? null,
|
||||
r.verdict.verdict,
|
||||
r.verdict.confidence,
|
||||
r.verdict.reasoning ?? null,
|
||||
r.product.fetchedAt,
|
||||
);
|
||||
}
|
||||
})(results); // Execute the transaction with the results batch
|
||||
}
|
||||
): Promise<void> {
|
||||
if (results.length === 0) return;
|
||||
await persistLlmResults(runId, results, {
|
||||
source: "category_analysis",
|
||||
metadataSource: "catalog",
|
||||
});
|
||||
}
|
||||
|
||||
function loadCategoryBlacklist(filePath: string): Set<number> {
|
||||
const blacklist = new Set<number>();
|
||||
@@ -1025,10 +889,14 @@ async function fetchCategoryBestSellerAsins(
|
||||
];
|
||||
|
||||
for (const value of candidates) {
|
||||
if (Array.isArray(value)) {
|
||||
return [
|
||||
...new Set(value.map((v) => String(v).trim()).filter(Boolean)),
|
||||
].slice(0, limit);
|
||||
if (Array.isArray(value)) {
|
||||
return [
|
||||
...new Set(
|
||||
value
|
||||
.map((v) => normalizeAsin(v))
|
||||
.filter((asin): asin is string => asin !== null),
|
||||
),
|
||||
].slice(0, limit);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1284,10 +1152,10 @@ async function fetchKeepaEnrichmentMap(
|
||||
`/product?key=${encodeURIComponent(config.keepaApiKey)}&domain=${DOMAIN_US}&asin=${asinParam}&stats=90&buybox=1&days=90`,
|
||||
);
|
||||
|
||||
const products = Array.isArray(data?.products) ? data.products : [];
|
||||
for (const product of products) {
|
||||
const asin = String(product?.asin ?? "").trim();
|
||||
if (!asin) continue;
|
||||
const products = Array.isArray(data?.products) ? data.products : [];
|
||||
for (const product of products) {
|
||||
const asin = normalizeAsin(product?.asin);
|
||||
if (!asin) continue;
|
||||
const parsed = {
|
||||
keepa: parseKeepaProduct(product),
|
||||
title: String(product?.title ?? "").trim(),
|
||||
@@ -1471,7 +1339,6 @@ function shouldKeepCandidateBySellability(
|
||||
}
|
||||
|
||||
export async function processCategory(
|
||||
db: Database,
|
||||
runId: number,
|
||||
category: CategoryInfo,
|
||||
perCategoryTop: number,
|
||||
@@ -1505,7 +1372,7 @@ export async function processCategory(
|
||||
);
|
||||
if (topAsins.length === 0) {
|
||||
log("info", " Keepa returned no ASINs for this category.");
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: 0,
|
||||
availableAsins: 0,
|
||||
fba: 0,
|
||||
@@ -1766,7 +1633,7 @@ export async function processCategory(
|
||||
},
|
||||
}));
|
||||
|
||||
await insertProductAnalysisResults(db, runId, batchResults);
|
||||
await insertProductAnalysisResults(runId, batchResults);
|
||||
|
||||
for (const result of batchResults) {
|
||||
if (result.verdict.verdict === "FBA") {
|
||||
@@ -1781,7 +1648,7 @@ export async function processCategory(
|
||||
budget.analyzedAsins += batchResults.length;
|
||||
results.push(...batchResults);
|
||||
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: checkedAsins,
|
||||
availableAsins: results.length,
|
||||
fba,
|
||||
@@ -1802,7 +1669,7 @@ export async function processCategory(
|
||||
const emptyReason =
|
||||
budget.stopReason ||
|
||||
"No sellable ASINs matched the configured mid-range criteria";
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: checkedAsins,
|
||||
availableAsins: 0,
|
||||
fba,
|
||||
@@ -1830,7 +1697,7 @@ export async function processCategory(
|
||||
` Category stream totals: checked=${checkedAsins}, sellable=${sellableAsins}, keepaEnriched=${keepaEnrichedAsins}, analyzed=${results.length}`,
|
||||
);
|
||||
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: checkedAsins,
|
||||
availableAsins: results.length,
|
||||
fba,
|
||||
@@ -1923,11 +1790,6 @@ export async function main(): Promise<void> {
|
||||
await connectCache();
|
||||
try {
|
||||
mkdirSync(args.outputDir, { recursive: true });
|
||||
const DB_PATH =
|
||||
process.env.RESULTS_DB_PATH ||
|
||||
path.join(process.cwd(), "db", "results.db");
|
||||
initDb(DB_PATH);
|
||||
const db = getDb(DB_PATH);
|
||||
|
||||
log("info", "Starting per-category mid-range pipeline");
|
||||
log("info", `Marketplace: ${config.spApiMarketplaceId}`);
|
||||
@@ -1987,7 +1849,6 @@ export async function main(): Promise<void> {
|
||||
let runId: number | undefined;
|
||||
try {
|
||||
runId = await insertCategoryRunSummary(
|
||||
db,
|
||||
{
|
||||
categoryId: category.id,
|
||||
categoryLabel: category.label,
|
||||
@@ -2004,7 +1865,6 @@ export async function main(): Promise<void> {
|
||||
);
|
||||
|
||||
categorySummary = await processCategory(
|
||||
db,
|
||||
runId,
|
||||
category,
|
||||
args.perCategoryTop,
|
||||
@@ -2046,7 +1906,7 @@ export async function main(): Promise<void> {
|
||||
results: [],
|
||||
};
|
||||
if (runId) {
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: 0,
|
||||
availableAsins: 0,
|
||||
fba: 0,
|
||||
@@ -1,8 +1,41 @@
|
||||
import { test, expect, beforeAll, afterAll, beforeEach, mock } from "bun:test";
|
||||
import { Database } from "bun:sqlite";
|
||||
import { getDb, initDb, closeDb } from "./database.ts";
|
||||
import path from "node:path";
|
||||
import { rmSync, mkdirSync } from "node:fs";
|
||||
import { test, expect, beforeEach, mock } from "bun:test";
|
||||
|
||||
let nextId = 0;
|
||||
|
||||
function chainable(resolveWith: any[] = []): any {
|
||||
const p: any = Promise.resolve(resolveWith);
|
||||
p.limit = (_n: any) => chainable(resolveWith);
|
||||
p.where = (_cond: any) => chainable(resolveWith);
|
||||
p.from = (_table: any) => chainable(resolveWith);
|
||||
return p;
|
||||
}
|
||||
|
||||
const makeMockDb = (): any => ({
|
||||
insert: (_table: any) => ({
|
||||
values: (_vals: any) => ({
|
||||
returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]),
|
||||
onConflictDoUpdate: (_conf: any) => Promise.resolve([]),
|
||||
}),
|
||||
}),
|
||||
update: (_table: any) => ({
|
||||
set: (_vals: any) => ({
|
||||
where: (_cond: any) => Promise.resolve([]),
|
||||
}),
|
||||
}),
|
||||
select: (_sel?: any) => ({
|
||||
from: (_table: any) => ({
|
||||
where: (_cond: any) => chainable(),
|
||||
limit: (_n: any) => chainable(),
|
||||
}),
|
||||
}),
|
||||
selectDistinct: (_sel: any) => ({
|
||||
from: (_table: any) => chainable(),
|
||||
}),
|
||||
execute: (_query: any) => Promise.resolve([]),
|
||||
transaction: async (fn: (tx: any) => Promise<any>) => fn(makeMockDb()),
|
||||
});
|
||||
|
||||
mock.module("../db/index.ts", () => ({ db: makeMockDb(), client: {} }));
|
||||
|
||||
const fetchSellabilityBatchMock = mock(async (asins: string[]) => {
|
||||
return new Map<string, any>(
|
||||
@@ -49,62 +82,34 @@ const analyzeProductsMock = mock(async (products: any[]) => {
|
||||
}));
|
||||
});
|
||||
|
||||
mock.module("./sp-api.ts", () => ({
|
||||
mock.module("../integrations/sp-api.ts", () => ({
|
||||
fetchSellabilityBatch: fetchSellabilityBatchMock,
|
||||
fetchSpApiPricingAndFees: fetchSpApiPricingAndFeesMock,
|
||||
}));
|
||||
|
||||
mock.module("./llm.ts", () => ({
|
||||
mock.module("../integrations/llm.ts", () => ({
|
||||
analyzeProducts: analyzeProductsMock,
|
||||
}));
|
||||
|
||||
const modulePromise = import("./top-monthly-sold-by-category.ts");
|
||||
|
||||
const DB_TEST_PATH = path.join(
|
||||
process.cwd(),
|
||||
"test_output",
|
||||
"test_monthly_sold_analysis.sqlite",
|
||||
);
|
||||
|
||||
let db: Database;
|
||||
let processCategory: (
|
||||
db: Database,
|
||||
runId: number,
|
||||
category: any,
|
||||
perCategoryTop: number,
|
||||
categoryCandidatePool: number,
|
||||
minMonthlySold: number,
|
||||
) => Promise<any>;
|
||||
let insertCategoryRunSummary: (
|
||||
db: Database,
|
||||
summary: any,
|
||||
runTimestamp: string,
|
||||
) => Promise<number>;
|
||||
let insertCategoryRunSummary: (summary: any, runTimestamp: string) => Promise<number>;
|
||||
let originalFetch: typeof globalThis.fetch;
|
||||
|
||||
beforeAll(async () => {
|
||||
const mod = await modulePromise;
|
||||
processCategory = mod.processCategory;
|
||||
insertCategoryRunSummary = mod.insertCategoryRunSummary;
|
||||
|
||||
rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true });
|
||||
mkdirSync(path.dirname(DB_TEST_PATH), { recursive: true });
|
||||
initDb(DB_TEST_PATH);
|
||||
db = getDb(DB_TEST_PATH);
|
||||
|
||||
originalFetch = globalThis.fetch;
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
closeDb();
|
||||
rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true });
|
||||
});
|
||||
const mod = await modulePromise;
|
||||
processCategory = mod.processCategory;
|
||||
insertCategoryRunSummary = mod.insertCategoryRunSummary;
|
||||
originalFetch = globalThis.fetch;
|
||||
|
||||
beforeEach(() => {
|
||||
db.run("DELETE FROM product_analysis_results");
|
||||
db.run("DELETE FROM category_analysis_runs");
|
||||
|
||||
nextId = 0;
|
||||
globalThis.fetch = mock(async (input: string | URL | Request) => {
|
||||
const rawUrl =
|
||||
typeof input === "string"
|
||||
@@ -140,25 +145,8 @@ beforeEach(() => {
|
||||
monthlySold: 600,
|
||||
stats: {
|
||||
current: [
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
1000,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
2,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
2599,
|
||||
null, null, null, 1000, null, null, null, null, null, null, null, 2,
|
||||
null, null, null, null, null, null, 2599,
|
||||
],
|
||||
avg: [2400, null, null, 1200],
|
||||
},
|
||||
@@ -171,25 +159,8 @@ beforeEach(() => {
|
||||
monthlySold: 250,
|
||||
stats: {
|
||||
current: [
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
2000,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
3,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
1999,
|
||||
null, null, null, 2000, null, null, null, null, null, null, null, 3,
|
||||
null, null, null, null, null, null, 1999,
|
||||
],
|
||||
avg: [1800, null, null, 2200],
|
||||
},
|
||||
@@ -202,25 +173,8 @@ beforeEach(() => {
|
||||
monthlySold: 800,
|
||||
stats: {
|
||||
current: [
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
1500,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
1,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
2099,
|
||||
null, null, null, 1500, null, null, null, null, null, null, null, 1,
|
||||
null, null, null, null, null, null, 2099,
|
||||
],
|
||||
avg: [2000, null, null, 1800],
|
||||
},
|
||||
@@ -233,25 +187,8 @@ beforeEach(() => {
|
||||
monthlySold: 400,
|
||||
stats: {
|
||||
current: [
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
3000,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
4,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
2899,
|
||||
null, null, null, 3000, null, null, null, null, null, null, null, 4,
|
||||
null, null, null, null, null, null, 2899,
|
||||
],
|
||||
avg: [2600, null, null, 2800],
|
||||
},
|
||||
@@ -279,7 +216,6 @@ test("processCategory filters to sellable ASINs with monthly sold >= threshold a
|
||||
};
|
||||
|
||||
const runId = await insertCategoryRunSummary(
|
||||
db,
|
||||
{
|
||||
categoryId: mockCategory.id,
|
||||
categoryLabel: mockCategory.label,
|
||||
@@ -295,22 +231,16 @@ test("processCategory filters to sellable ASINs with monthly sold >= threshold a
|
||||
new Date().toISOString(),
|
||||
);
|
||||
|
||||
const summary = await processCategory(db, runId, mockCategory, 2, 4, 300);
|
||||
const summary = await processCategory(runId, mockCategory, 2, 4, 300);
|
||||
|
||||
expect(summary.status).toBe("ok");
|
||||
expect(summary.topAsinsChecked).toBe(4);
|
||||
expect(summary.availableAsins).toBe(2);
|
||||
expect(summary.results?.length).toBe(2);
|
||||
|
||||
const productResults = db
|
||||
.query(
|
||||
"SELECT asin, monthly_sold FROM product_analysis_results ORDER BY monthly_sold DESC",
|
||||
)
|
||||
.all() as Array<{ asin: string; monthly_sold: number }>;
|
||||
const asins = summary.results?.map((r: any) => r.product.record.asin) ?? [];
|
||||
expect(asins).toContain("B000000001");
|
||||
expect(asins).toContain("B000000004");
|
||||
|
||||
expect(productResults.length).toBe(2);
|
||||
expect(productResults[0]?.asin).toBe("B000000001");
|
||||
expect(productResults[0]?.monthly_sold).toBe(600);
|
||||
expect(productResults[1]?.asin).toBe("B000000004");
|
||||
expect(productResults[1]?.monthly_sold).toBe(400);
|
||||
globalThis.fetch = originalFetch;
|
||||
});
|
||||
@@ -1,9 +1,14 @@
|
||||
import { existsSync, mkdirSync, readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { type Database, getDb, initDb } from "./database.ts";
|
||||
import { config } from "./config.ts";
|
||||
import { analyzeProducts } from "./llm.ts";
|
||||
import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts";
|
||||
import { existsSync, mkdirSync, readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { normalizeAsin } from "../asin.ts";
|
||||
import {
|
||||
createCategoryRun,
|
||||
persistLlmResults,
|
||||
updateCategoryRun,
|
||||
} from "../db/persistence.ts";
|
||||
import { config } from "../config.ts";
|
||||
import { analyzeProducts } from "../integrations/llm.ts";
|
||||
import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "../integrations/sp-api.ts";
|
||||
import type {
|
||||
AnalysisResult,
|
||||
EnrichedProduct,
|
||||
@@ -12,7 +17,7 @@ import type {
|
||||
ProductRecord,
|
||||
SellabilityInfo,
|
||||
SpApiData,
|
||||
} from "./types.ts";
|
||||
} from "../types.ts";
|
||||
|
||||
type CategoryInfo = {
|
||||
id: number;
|
||||
@@ -170,37 +175,14 @@ function printUsageAndExit(message: string): never {
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
export async function insertCategoryRunSummary(
|
||||
db: Database,
|
||||
summary: CategoryRunSummary,
|
||||
runTimestamp: string,
|
||||
): Promise<number> {
|
||||
const query = `
|
||||
INSERT INTO category_analysis_runs (
|
||||
category_id, category_label, run_timestamp,
|
||||
top_asins_checked, available_asins,
|
||||
fba_count, fbm_count, skip_count,
|
||||
status, error_message
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
|
||||
`;
|
||||
const result = db.run(query, [
|
||||
summary.categoryId,
|
||||
summary.categoryLabel,
|
||||
runTimestamp,
|
||||
summary.topAsinsChecked,
|
||||
summary.availableAsins,
|
||||
summary.fba,
|
||||
summary.fbm,
|
||||
summary.skip,
|
||||
summary.status,
|
||||
summary.error,
|
||||
]);
|
||||
// Bun's SQLite client returns { changes: number, lastInsertRowid: number | bigint }
|
||||
return Number(result.lastInsertRowid);
|
||||
}
|
||||
export async function insertCategoryRunSummary(
|
||||
summary: CategoryRunSummary,
|
||||
runTimestamp: string,
|
||||
): Promise<number> {
|
||||
return createCategoryRun(summary, runTimestamp);
|
||||
}
|
||||
|
||||
export async function updateCategoryRunSummary(
|
||||
db: Database,
|
||||
runId: number,
|
||||
summary: Pick<
|
||||
CategoryRunSummary,
|
||||
@@ -212,138 +194,20 @@ export async function updateCategoryRunSummary(
|
||||
| "status"
|
||||
| "error"
|
||||
>,
|
||||
): Promise<void> {
|
||||
db.run(
|
||||
`
|
||||
UPDATE category_analysis_runs
|
||||
SET
|
||||
top_asins_checked = ?,
|
||||
available_asins = ?,
|
||||
fba_count = ?,
|
||||
fbm_count = ?,
|
||||
skip_count = ?,
|
||||
status = ?,
|
||||
error_message = ?
|
||||
WHERE id = ?
|
||||
`,
|
||||
[
|
||||
summary.topAsinsChecked,
|
||||
summary.availableAsins,
|
||||
summary.fba,
|
||||
summary.fbm,
|
||||
summary.skip,
|
||||
summary.status,
|
||||
summary.error,
|
||||
runId,
|
||||
],
|
||||
);
|
||||
}
|
||||
): Promise<void> {
|
||||
await updateCategoryRun(runId, summary);
|
||||
}
|
||||
|
||||
export async function insertProductAnalysisResults(
|
||||
db: Database,
|
||||
runId: number,
|
||||
results: AnalysisResult[],
|
||||
): Promise<void> {
|
||||
if (results.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const insertStmt = db.prepare(`
|
||||
INSERT INTO product_analysis_results (
|
||||
asin, run_id, name, brand, category, unit_cost,
|
||||
current_price, avg_price_90d, avg_price_90d_sheet,
|
||||
selling_price_sheet, sales_rank, sales_rank_avg_90d,
|
||||
seller_count, amazon_is_seller, amazon_buybox_share_pct_90d,
|
||||
monthly_sold, rank_drops_30d, rank_drops_90d,
|
||||
fba_fee, fbm_fee, referral_percent, can_sell,
|
||||
sellability_status, sellability_reason,
|
||||
verdict, confidence, reasoning, fetched_at
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?
|
||||
)
|
||||
ON CONFLICT(asin) DO UPDATE SET
|
||||
run_id = excluded.run_id,
|
||||
name = excluded.name,
|
||||
brand = excluded.brand,
|
||||
category = excluded.category,
|
||||
unit_cost = excluded.unit_cost,
|
||||
current_price = excluded.current_price,
|
||||
avg_price_90d = excluded.avg_price_90d,
|
||||
avg_price_90d_sheet = excluded.avg_price_90d_sheet,
|
||||
selling_price_sheet = excluded.selling_price_sheet,
|
||||
sales_rank = excluded.sales_rank,
|
||||
sales_rank_avg_90d = excluded.sales_rank_avg_90d,
|
||||
seller_count = excluded.seller_count,
|
||||
amazon_is_seller = excluded.amazon_is_seller,
|
||||
amazon_buybox_share_pct_90d = excluded.amazon_buybox_share_pct_90d,
|
||||
monthly_sold = excluded.monthly_sold,
|
||||
rank_drops_30d = excluded.rank_drops_30d,
|
||||
rank_drops_90d = excluded.rank_drops_90d,
|
||||
fba_fee = excluded.fba_fee,
|
||||
fbm_fee = excluded.fbm_fee,
|
||||
referral_percent = excluded.referral_percent,
|
||||
can_sell = excluded.can_sell,
|
||||
sellability_status = excluded.sellability_status,
|
||||
sellability_reason = excluded.sellability_reason,
|
||||
verdict = excluded.verdict,
|
||||
confidence = excluded.confidence,
|
||||
reasoning = excluded.reasoning,
|
||||
fetched_at = excluded.fetched_at;
|
||||
`);
|
||||
|
||||
db.transaction((resultsBatch: AnalysisResult[]) => {
|
||||
for (const r of resultsBatch) {
|
||||
const price =
|
||||
r.product.keepa?.currentPrice ??
|
||||
r.product.record.sellingPriceFromSheet ??
|
||||
r.product.spApi.estimatedSalePrice;
|
||||
const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank;
|
||||
|
||||
insertStmt.run(
|
||||
r.product.record.asin,
|
||||
runId,
|
||||
r.product.record.name,
|
||||
r.product.record.brand ?? null,
|
||||
r.product.record.category ??
|
||||
r.product.keepa?.categoryTree?.join(" > ") ??
|
||||
null,
|
||||
r.product.record.unitCost ?? null,
|
||||
price ?? null,
|
||||
r.product.keepa?.avgPrice90 ?? null,
|
||||
r.product.record.avgPrice90FromSheet ?? null,
|
||||
r.product.record.sellingPriceFromSheet ?? null,
|
||||
rank ?? null,
|
||||
r.product.keepa?.salesRankAvg90 ?? null,
|
||||
r.product.keepa?.sellerCount ?? null,
|
||||
r.product.keepa?.amazonIsSeller == null
|
||||
? null
|
||||
: r.product.keepa.amazonIsSeller
|
||||
? 1
|
||||
: 0,
|
||||
r.product.keepa?.amazonBuyboxSharePct90d ?? null,
|
||||
r.product.keepa?.monthlySold ?? null,
|
||||
r.product.keepa?.salesRankDrops30 ?? null,
|
||||
r.product.keepa?.salesRankDrops90 ?? null,
|
||||
r.product.spApi.fbaFee ?? null,
|
||||
r.product.spApi.fbmFee ?? null,
|
||||
r.product.spApi.referralFeePercent ?? null,
|
||||
r.product.spApi.canSell == null
|
||||
? "unknown"
|
||||
: r.product.spApi.canSell
|
||||
? "yes"
|
||||
: "no",
|
||||
r.product.spApi.sellabilityStatus ?? null,
|
||||
r.product.spApi.sellabilityReason ?? null,
|
||||
r.verdict.verdict,
|
||||
r.verdict.confidence,
|
||||
r.verdict.reasoning ?? null,
|
||||
r.product.fetchedAt,
|
||||
);
|
||||
}
|
||||
})(results); // Execute the transaction with the results batch
|
||||
}
|
||||
): Promise<void> {
|
||||
if (results.length === 0) return;
|
||||
await persistLlmResults(runId, results, {
|
||||
source: "category_analysis",
|
||||
metadataSource: "catalog",
|
||||
});
|
||||
}
|
||||
|
||||
function loadCategoryBlacklist(filePath: string): Set<number> {
|
||||
const blacklist = new Set<number>();
|
||||
@@ -722,10 +586,14 @@ async function fetchCategoryBestSellerAsins(
|
||||
];
|
||||
|
||||
for (const value of candidates) {
|
||||
if (Array.isArray(value)) {
|
||||
return [
|
||||
...new Set(value.map((v) => String(v).trim()).filter(Boolean)),
|
||||
].slice(0, limit);
|
||||
if (Array.isArray(value)) {
|
||||
return [
|
||||
...new Set(
|
||||
value
|
||||
.map((v) => normalizeAsin(v))
|
||||
.filter((asin): asin is string => asin !== null),
|
||||
),
|
||||
].slice(0, limit);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -977,10 +845,10 @@ async function fetchKeepaEnrichmentMap(
|
||||
`/product?key=${encodeURIComponent(config.keepaApiKey)}&domain=${DOMAIN_US}&asin=${asinParam}&stats=90&buybox=1&days=90`,
|
||||
);
|
||||
|
||||
const products = Array.isArray(data?.products) ? data.products : [];
|
||||
for (const product of products) {
|
||||
const asin = String(product?.asin ?? "").trim();
|
||||
if (!asin) continue;
|
||||
const products = Array.isArray(data?.products) ? data.products : [];
|
||||
for (const product of products) {
|
||||
const asin = normalizeAsin(product?.asin);
|
||||
if (!asin) continue;
|
||||
out.set(asin, {
|
||||
keepa: parseKeepaProduct(product),
|
||||
title: String(product?.title ?? "").trim(),
|
||||
@@ -1067,7 +935,6 @@ function buildEnrichedProducts(
|
||||
}
|
||||
|
||||
export async function processCategory(
|
||||
db: Database,
|
||||
runId: number,
|
||||
category: CategoryInfo,
|
||||
perCategoryTop: number,
|
||||
@@ -1083,7 +950,7 @@ export async function processCategory(
|
||||
);
|
||||
if (topAsins.length === 0) {
|
||||
log("info", " Keepa returned no ASINs for this category.");
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: 0,
|
||||
availableAsins: 0,
|
||||
fba: 0,
|
||||
@@ -1127,7 +994,7 @@ export async function processCategory(
|
||||
` Sellable ASINs: ${availableAsins.length}/${uniqueTopAsins.length}`,
|
||||
);
|
||||
if (availableAsins.length === 0) {
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: uniqueTopAsins.length,
|
||||
availableAsins: 0,
|
||||
fba: 0,
|
||||
@@ -1164,7 +1031,7 @@ export async function processCategory(
|
||||
);
|
||||
|
||||
if (selectedAsins.length === 0) {
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: uniqueTopAsins.length,
|
||||
availableAsins: 0,
|
||||
fba: 0,
|
||||
@@ -1231,7 +1098,7 @@ export async function processCategory(
|
||||
},
|
||||
}));
|
||||
|
||||
await insertProductAnalysisResults(db, runId, batchResults);
|
||||
await insertProductAnalysisResults(runId, batchResults);
|
||||
|
||||
for (const result of batchResults) {
|
||||
results.push(result);
|
||||
@@ -1244,7 +1111,7 @@ export async function processCategory(
|
||||
}
|
||||
}
|
||||
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: uniqueTopAsins.length,
|
||||
availableAsins: selectedAsins.length,
|
||||
fba,
|
||||
@@ -1264,7 +1131,7 @@ export async function processCategory(
|
||||
}
|
||||
}
|
||||
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: uniqueTopAsins.length,
|
||||
availableAsins: selectedAsins.length,
|
||||
fba,
|
||||
@@ -1293,10 +1160,6 @@ export async function main(): Promise<void> {
|
||||
assertSpApiPrerequisites();
|
||||
|
||||
mkdirSync(args.outputDir, { recursive: true });
|
||||
const DB_PATH =
|
||||
process.env.RESULTS_DB_PATH || path.join(process.cwd(), "db", "results.db");
|
||||
initDb(DB_PATH);
|
||||
const db = getDb(DB_PATH);
|
||||
|
||||
log("info", "Starting per-category monthly-sold pipeline");
|
||||
log("info", `Marketplace: ${config.spApiMarketplaceId}`);
|
||||
@@ -1333,7 +1196,6 @@ export async function main(): Promise<void> {
|
||||
let runId: number | undefined;
|
||||
try {
|
||||
runId = await insertCategoryRunSummary(
|
||||
db,
|
||||
{
|
||||
categoryId: category.id,
|
||||
categoryLabel: category.label,
|
||||
@@ -1350,7 +1212,6 @@ export async function main(): Promise<void> {
|
||||
);
|
||||
|
||||
categorySummary = await processCategory(
|
||||
db,
|
||||
runId,
|
||||
category,
|
||||
args.perCategoryTop,
|
||||
@@ -1382,7 +1243,7 @@ export async function main(): Promise<void> {
|
||||
results: [],
|
||||
};
|
||||
if (runId) {
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: 0,
|
||||
availableAsins: 0,
|
||||
fba: 0,
|
||||
@@ -1,21 +1,16 @@
|
||||
import { getDb } from "./database.ts";
|
||||
import path from "node:path";
|
||||
import { db } from "./db/index.ts";
|
||||
import { runs } from "./db/schema.ts";
|
||||
import { eq } from "drizzle-orm";
|
||||
|
||||
async function checkDb() {
|
||||
const DB_PATH =
|
||||
process.env.RESULTS_DB_PATH || path.join(process.cwd(), "db", "results.db");
|
||||
const db = getDb(DB_PATH);
|
||||
|
||||
try {
|
||||
const query = db.query(
|
||||
"SELECT * FROM category_analysis_runs WHERE category_id = ?",
|
||||
);
|
||||
const result = query.all(19419898011);
|
||||
const result = await db
|
||||
.select()
|
||||
.from(runs)
|
||||
.where(eq(runs.type, "category_analysis"));
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
} catch (error) {
|
||||
console.error("Database query failed:", error);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
494
src/database.ts
494
src/database.ts
@@ -1,494 +0,0 @@
|
||||
import { Database } from "bun:sqlite";
|
||||
import { dirname } from "node:path";
|
||||
import { mkdirSync } from "node:fs";
|
||||
export { Database } from "bun:sqlite";
|
||||
|
||||
let db: Database | null = null;
|
||||
|
||||
export function getDb(dbPath: string): Database {
|
||||
if (!db) {
|
||||
const dbDir = dirname(dbPath);
|
||||
if (dbDir && dbDir !== ".") {
|
||||
mkdirSync(dbDir, { recursive: true });
|
||||
}
|
||||
db = new Database(dbPath);
|
||||
db.run("PRAGMA journal_mode = WAL;"); // Enable WAL mode for better performance
|
||||
db.run("PRAGMA foreign_keys = ON;"); // Enforce foreign key constraints
|
||||
}
|
||||
return db;
|
||||
}
|
||||
|
||||
export function closeDb(): void {
|
||||
if (db) {
|
||||
db.close();
|
||||
db = null;
|
||||
}
|
||||
}
|
||||
|
||||
function createProductAnalysisResultsTable(database: Database): void {
|
||||
database.run(`
|
||||
CREATE TABLE IF NOT EXISTS product_analysis_results (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
asin TEXT NOT NULL,
|
||||
run_id INTEGER NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
brand TEXT,
|
||||
category TEXT,
|
||||
unit_cost REAL,
|
||||
current_price REAL,
|
||||
avg_price_90d REAL,
|
||||
avg_price_90d_sheet REAL,
|
||||
selling_price_sheet REAL,
|
||||
sales_rank INTEGER,
|
||||
sales_rank_avg_90d INTEGER,
|
||||
seller_count INTEGER,
|
||||
amazon_is_seller INTEGER,
|
||||
amazon_buybox_share_pct_90d REAL,
|
||||
monthly_sold INTEGER,
|
||||
rank_drops_30d INTEGER,
|
||||
rank_drops_90d INTEGER,
|
||||
fba_fee REAL,
|
||||
fbm_fee REAL,
|
||||
referral_percent REAL,
|
||||
can_sell TEXT,
|
||||
sellability_status TEXT,
|
||||
sellability_reason TEXT,
|
||||
verdict TEXT NOT NULL,
|
||||
confidence REAL NOT NULL,
|
||||
reasoning TEXT,
|
||||
fetched_at TEXT NOT NULL,
|
||||
UNIQUE(asin),
|
||||
FOREIGN KEY (run_id) REFERENCES category_analysis_runs(id)
|
||||
);
|
||||
`);
|
||||
}
|
||||
|
||||
function ensureProductAnalysisResultsTable(database: Database): void {
|
||||
const tableInfo = database
|
||||
.query("PRAGMA table_info(product_analysis_results)")
|
||||
.all() as Array<{ name: string; pk: number }>;
|
||||
|
||||
if (tableInfo.length === 0) {
|
||||
createProductAnalysisResultsTable(database);
|
||||
return;
|
||||
}
|
||||
|
||||
const hasIdColumn = tableInfo.some((col) => col.name === "id");
|
||||
const hasAsinPrimaryKey = tableInfo.some(
|
||||
(col) => col.name === "asin" && col.pk === 1,
|
||||
);
|
||||
|
||||
const indexList = database
|
||||
.query("PRAGMA index_list(product_analysis_results)")
|
||||
.all() as Array<{ name: string; unique: number }>;
|
||||
const hasUniqueAsinConstraint = indexList.some((idx) => {
|
||||
if (idx.unique !== 1) return false;
|
||||
const columns = database
|
||||
.query(`PRAGMA index_info(${JSON.stringify(idx.name)})`)
|
||||
.all() as Array<{ name: string }>;
|
||||
return columns.length === 1 && columns[0]?.name === "asin";
|
||||
});
|
||||
|
||||
if (!hasIdColumn || hasAsinPrimaryKey || !hasUniqueAsinConstraint) {
|
||||
database.run(
|
||||
"ALTER TABLE product_analysis_results RENAME TO product_analysis_results_legacy",
|
||||
);
|
||||
createProductAnalysisResultsTable(database);
|
||||
database.run(`
|
||||
WITH ranked AS (
|
||||
SELECT
|
||||
asin, run_id, name, brand, category, unit_cost,
|
||||
current_price, avg_price_90d, avg_price_90d_sheet,
|
||||
selling_price_sheet, sales_rank, sales_rank_avg_90d,
|
||||
seller_count, NULL AS amazon_is_seller,
|
||||
NULL AS amazon_buybox_share_pct_90d,
|
||||
monthly_sold, rank_drops_30d, rank_drops_90d,
|
||||
fba_fee, fbm_fee, referral_percent, can_sell,
|
||||
sellability_status, sellability_reason,
|
||||
verdict, confidence, reasoning, fetched_at,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY asin
|
||||
ORDER BY datetime(fetched_at) DESC, run_id DESC, id DESC
|
||||
) AS row_num
|
||||
FROM product_analysis_results_legacy
|
||||
)
|
||||
INSERT INTO product_analysis_results (
|
||||
asin, run_id, name, brand, category, unit_cost,
|
||||
current_price, avg_price_90d, avg_price_90d_sheet,
|
||||
selling_price_sheet, sales_rank, sales_rank_avg_90d,
|
||||
seller_count, amazon_is_seller, amazon_buybox_share_pct_90d,
|
||||
monthly_sold, rank_drops_30d, rank_drops_90d,
|
||||
fba_fee, fbm_fee, referral_percent, can_sell,
|
||||
sellability_status, sellability_reason,
|
||||
verdict, confidence, reasoning, fetched_at
|
||||
)
|
||||
SELECT
|
||||
asin, run_id, name, brand, category, unit_cost,
|
||||
current_price, avg_price_90d, avg_price_90d_sheet,
|
||||
selling_price_sheet, sales_rank, sales_rank_avg_90d,
|
||||
seller_count, amazon_is_seller, amazon_buybox_share_pct_90d,
|
||||
monthly_sold, rank_drops_30d, rank_drops_90d,
|
||||
fba_fee, fbm_fee, referral_percent, can_sell,
|
||||
sellability_status, sellability_reason,
|
||||
verdict, confidence, reasoning, fetched_at
|
||||
FROM ranked
|
||||
WHERE row_num = 1
|
||||
`);
|
||||
database.run("DROP TABLE product_analysis_results_legacy");
|
||||
}
|
||||
}
|
||||
|
||||
function ensureProductAnalysisResultsColumns(database: Database): void {
|
||||
const tableInfo = database
|
||||
.query("PRAGMA table_info(product_analysis_results)")
|
||||
.all() as Array<{ name: string }>;
|
||||
|
||||
if (tableInfo.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const existingColumns = new Set(tableInfo.map((col) => col.name));
|
||||
const requiredColumns: Array<{ name: string; type: string }> = [
|
||||
{ name: "amazon_is_seller", type: "INTEGER" },
|
||||
{ name: "amazon_buybox_share_pct_90d", type: "REAL" },
|
||||
];
|
||||
|
||||
for (const column of requiredColumns) {
|
||||
if (!existingColumns.has(column.name)) {
|
||||
database.run(
|
||||
`ALTER TABLE product_analysis_results ADD COLUMN ${column.name} ${column.type}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function ensureResultsTableColumns(database: Database): void {
|
||||
const tableInfo = database
|
||||
.query("PRAGMA table_info(results)")
|
||||
.all() as Array<{ name: string }>;
|
||||
|
||||
if (tableInfo.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const existingColumns = new Set(tableInfo.map((col) => col.name));
|
||||
const requiredColumns: Array<{ name: string; type: string }> = [
|
||||
{ name: "fba_net_sheet", type: "REAL" },
|
||||
{ name: "gross_profit_dollar", type: "REAL" },
|
||||
{ name: "gross_profit_pct", type: "REAL" },
|
||||
{ name: "net_profit_sheet", type: "REAL" },
|
||||
{ name: "roi_sheet", type: "REAL" },
|
||||
{ name: "moq", type: "INTEGER" },
|
||||
{ name: "moq_cost", type: "REAL" },
|
||||
{ name: "qty_available", type: "INTEGER" },
|
||||
{ name: "supplier", type: "TEXT" },
|
||||
{ name: "source_url", type: "TEXT" },
|
||||
{ name: "asin_link", type: "TEXT" },
|
||||
{ name: "promo_coupon_code", type: "TEXT" },
|
||||
{ name: "notes", type: "TEXT" },
|
||||
{ name: "lead_date", type: "TEXT" },
|
||||
{ name: "amazon_is_seller", type: "INTEGER" },
|
||||
{ name: "amazon_buybox_share_pct_90d", type: "REAL" },
|
||||
{ name: "upc", type: "TEXT" },
|
||||
{ name: "supplier_score", type: "REAL" },
|
||||
{ name: "supplier_profit", type: "REAL" },
|
||||
{ name: "supplier_margin", type: "REAL" },
|
||||
{ name: "supplier_roi", type: "REAL" },
|
||||
{ name: "supplier_reason", type: "TEXT" },
|
||||
{ name: "upc_lookup_status", type: "TEXT" },
|
||||
{ name: "upc_lookup_reason", type: "TEXT" },
|
||||
{ name: "candidate_asins", type: "TEXT" },
|
||||
];
|
||||
|
||||
for (const column of requiredColumns) {
|
||||
if (!existingColumns.has(column.name)) {
|
||||
database.run(
|
||||
`ALTER TABLE results ADD COLUMN ${column.name} ${column.type}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function initDb(dbPath: string): void {
|
||||
const database = getDb(dbPath);
|
||||
database.run(`
|
||||
CREATE TABLE IF NOT EXISTS runs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
timestamp TEXT NOT NULL,
|
||||
input_file TEXT NOT NULL,
|
||||
output_file TEXT,
|
||||
total_products INTEGER,
|
||||
fba_count INTEGER,
|
||||
fbm_count INTEGER,
|
||||
skip_count INTEGER
|
||||
);
|
||||
`);
|
||||
database.run(`
|
||||
CREATE TABLE IF NOT EXISTS results (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
run_id INTEGER NOT NULL,
|
||||
asin TEXT NOT NULL,
|
||||
product_name TEXT,
|
||||
brand TEXT,
|
||||
category TEXT,
|
||||
unit_cost REAL,
|
||||
current_price REAL,
|
||||
avg_price_90d REAL,
|
||||
avg_price_90d_sheet REAL,
|
||||
selling_price_sheet REAL,
|
||||
sales_rank INTEGER,
|
||||
rank_avg_90d INTEGER,
|
||||
sellers INTEGER,
|
||||
amazon_is_seller INTEGER,
|
||||
amazon_buybox_share_pct_90d REAL,
|
||||
monthly_sold INTEGER,
|
||||
rank_drops_30d INTEGER,
|
||||
rank_drops_90d INTEGER,
|
||||
fba_net_sheet REAL,
|
||||
gross_profit_dollar REAL,
|
||||
gross_profit_pct REAL,
|
||||
net_profit_sheet REAL,
|
||||
roi_sheet REAL,
|
||||
moq INTEGER,
|
||||
moq_cost REAL,
|
||||
qty_available INTEGER,
|
||||
supplier TEXT,
|
||||
source_url TEXT,
|
||||
asin_link TEXT,
|
||||
promo_coupon_code TEXT,
|
||||
notes TEXT,
|
||||
lead_date TEXT,
|
||||
upc TEXT,
|
||||
fba_fee REAL,
|
||||
fbm_fee REAL,
|
||||
referral_percent REAL,
|
||||
supplier_score REAL,
|
||||
supplier_profit REAL,
|
||||
supplier_margin REAL,
|
||||
supplier_roi REAL,
|
||||
supplier_reason TEXT,
|
||||
upc_lookup_status TEXT,
|
||||
upc_lookup_reason TEXT,
|
||||
candidate_asins TEXT,
|
||||
can_sell TEXT,
|
||||
sellability_status TEXT,
|
||||
sellability_reason TEXT,
|
||||
verdict TEXT NOT NULL,
|
||||
confidence INTEGER,
|
||||
reasoning TEXT,
|
||||
fetched_at TEXT NOT NULL,
|
||||
FOREIGN KEY (run_id) REFERENCES runs(id)
|
||||
);
|
||||
`);
|
||||
ensureResultsTableColumns(database);
|
||||
database.run(`
|
||||
CREATE TABLE IF NOT EXISTS category_analysis_runs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
category_id INTEGER NOT NULL,
|
||||
category_label TEXT NOT NULL,
|
||||
run_timestamp TEXT NOT NULL,
|
||||
top_asins_checked INTEGER NOT NULL,
|
||||
available_asins INTEGER NOT NULL,
|
||||
fba_count INTEGER NOT NULL,
|
||||
fbm_count INTEGER NOT NULL,
|
||||
skip_count INTEGER NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
error_message TEXT
|
||||
);
|
||||
`);
|
||||
ensureProductAnalysisResultsTable(database);
|
||||
ensureProductAnalysisResultsColumns(database);
|
||||
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_runs_timestamp ON runs(timestamp DESC);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_results_run_id ON results(run_id);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_results_verdict ON results(verdict);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_results_sellability_status ON results(sellability_status);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_results_fetched_at ON results(fetched_at DESC);`,
|
||||
);
|
||||
database.run(`CREATE INDEX IF NOT EXISTS idx_results_asin ON results(asin);`);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_category_runs_timestamp ON category_analysis_runs(run_timestamp DESC);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_category_runs_status ON category_analysis_runs(status);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_product_results_run_id ON product_analysis_results(run_id);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_product_results_verdict ON product_analysis_results(verdict);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_product_results_sellability_status ON product_analysis_results(sellability_status);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_product_results_fetched_at ON product_analysis_results(fetched_at DESC);`,
|
||||
);
|
||||
initStalkerDb(database);
|
||||
}
|
||||
|
||||
export function initStalkerDb(database: Database): void {
|
||||
resetLegacyStalkerSchema(database);
|
||||
|
||||
database.run(`
|
||||
CREATE TABLE IF NOT EXISTS stalker_runs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
input_file TEXT NOT NULL,
|
||||
started_at TEXT NOT NULL,
|
||||
completed_at TEXT,
|
||||
requested_asins INTEGER NOT NULL DEFAULT 0,
|
||||
skipped_asins INTEGER NOT NULL DEFAULT 0,
|
||||
scanned_asins INTEGER NOT NULL DEFAULT 0,
|
||||
source_asins_with_matches INTEGER NOT NULL DEFAULT 0,
|
||||
candidate_sellers INTEGER NOT NULL DEFAULT 0,
|
||||
qualifying_sellers INTEGER NOT NULL DEFAULT 0,
|
||||
matched_sellers INTEGER NOT NULL DEFAULT 0,
|
||||
seller_metadata_requests INTEGER NOT NULL DEFAULT 0,
|
||||
seller_storefront_requests INTEGER NOT NULL DEFAULT 0,
|
||||
inventory_sellability_checked_asins INTEGER NOT NULL DEFAULT 0,
|
||||
inventory_sellability_available_asins INTEGER NOT NULL DEFAULT 0,
|
||||
inventory_sellability_excluded_asins INTEGER NOT NULL DEFAULT 0,
|
||||
persisted_inventory_asins INTEGER NOT NULL DEFAULT 0,
|
||||
status TEXT NOT NULL,
|
||||
error_message TEXT
|
||||
);
|
||||
`);
|
||||
|
||||
database.run(`
|
||||
CREATE TABLE IF NOT EXISTS stalker_asin_scans (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
run_id INTEGER NOT NULL,
|
||||
source_asin TEXT NOT NULL,
|
||||
title TEXT,
|
||||
offer_count INTEGER NOT NULL DEFAULT 0,
|
||||
candidate_seller_count INTEGER NOT NULL DEFAULT 0,
|
||||
matched_seller_count INTEGER NOT NULL DEFAULT 0,
|
||||
fetched_at TEXT NOT NULL,
|
||||
raw_product_json TEXT,
|
||||
UNIQUE(run_id, source_asin),
|
||||
FOREIGN KEY (run_id) REFERENCES stalker_runs(id) ON DELETE CASCADE
|
||||
);
|
||||
`);
|
||||
|
||||
database.run(`
|
||||
CREATE TABLE IF NOT EXISTS stalker_sellers (
|
||||
seller_id TEXT PRIMARY KEY,
|
||||
seller_name TEXT,
|
||||
rating REAL,
|
||||
rating_count INTEGER,
|
||||
storefront_asin_total INTEGER,
|
||||
persisted_inventory_sample_count INTEGER,
|
||||
last_updated_at TEXT NOT NULL,
|
||||
raw_seller_json TEXT
|
||||
);
|
||||
`);
|
||||
|
||||
database.run(`
|
||||
CREATE TABLE IF NOT EXISTS stalker_asin_sellers (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
scan_id INTEGER NOT NULL,
|
||||
seller_id TEXT NOT NULL,
|
||||
offer_price REAL,
|
||||
condition TEXT,
|
||||
is_fba INTEGER,
|
||||
stock INTEGER,
|
||||
seller_rating REAL,
|
||||
seller_rating_count INTEGER,
|
||||
raw_offer_json TEXT,
|
||||
UNIQUE(scan_id, seller_id),
|
||||
FOREIGN KEY (scan_id) REFERENCES stalker_asin_scans(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (seller_id) REFERENCES stalker_sellers(seller_id)
|
||||
);
|
||||
`);
|
||||
|
||||
database.run(`
|
||||
CREATE TABLE IF NOT EXISTS stalker_seller_inventory (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
run_id INTEGER NOT NULL,
|
||||
seller_id TEXT NOT NULL,
|
||||
asin TEXT NOT NULL,
|
||||
can_sell INTEGER,
|
||||
sellability_status TEXT,
|
||||
sellability_reason TEXT,
|
||||
product_title TEXT,
|
||||
brand TEXT,
|
||||
category_tree TEXT,
|
||||
current_price REAL,
|
||||
avg_price_90d REAL,
|
||||
sales_rank INTEGER,
|
||||
monthly_sold INTEGER,
|
||||
seller_count INTEGER,
|
||||
amazon_is_seller INTEGER,
|
||||
raw_product_json TEXT,
|
||||
last_seen_at TEXT NOT NULL,
|
||||
raw_inventory_json TEXT,
|
||||
UNIQUE(run_id, seller_id, asin),
|
||||
FOREIGN KEY (run_id) REFERENCES stalker_runs(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (seller_id) REFERENCES stalker_sellers(seller_id)
|
||||
);
|
||||
`);
|
||||
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_stalker_runs_started_at ON stalker_runs(started_at DESC);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_stalker_scans_run_id ON stalker_asin_scans(run_id);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_stalker_scans_source_asin ON stalker_asin_scans(source_asin);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_stalker_asin_sellers_seller_id ON stalker_asin_sellers(seller_id);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_stalker_inventory_seller_id ON stalker_seller_inventory(seller_id);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_stalker_inventory_asin ON stalker_seller_inventory(asin);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_stalker_inventory_product_title ON stalker_seller_inventory(product_title);`,
|
||||
);
|
||||
}
|
||||
|
||||
function resetLegacyStalkerSchema(database: Database): void {
|
||||
const runColumns = database
|
||||
.query("PRAGMA table_info(stalker_runs)")
|
||||
.all() as Array<{ name: string }>;
|
||||
if (runColumns.length === 0) return;
|
||||
|
||||
const columnNames = new Set(runColumns.map((column) => column.name));
|
||||
if (
|
||||
columnNames.has("scanned_asins") &&
|
||||
columnNames.has("inventory_sellability_checked_asins") &&
|
||||
inventoryColumnsHaveSellability(database)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
database.run("DROP TABLE IF EXISTS stalker_seller_inventory");
|
||||
database.run("DROP TABLE IF EXISTS stalker_asin_sellers");
|
||||
database.run("DROP TABLE IF EXISTS stalker_sellers");
|
||||
database.run("DROP TABLE IF EXISTS stalker_asin_scans");
|
||||
database.run("DROP TABLE IF EXISTS stalker_runs");
|
||||
}
|
||||
|
||||
function inventoryColumnsHaveSellability(database: Database): boolean {
|
||||
const inventoryColumns = database
|
||||
.query("PRAGMA table_info(stalker_seller_inventory)")
|
||||
.all() as Array<{ name: string }>;
|
||||
const columnNames = new Set(inventoryColumns.map((column) => column.name));
|
||||
return (
|
||||
columnNames.has("sellability_status") &&
|
||||
columnNames.has("product_title")
|
||||
);
|
||||
}
|
||||
15
src/db/index.ts
Normal file
15
src/db/index.ts
Normal file
@@ -0,0 +1,15 @@
|
||||
import { drizzle } from "drizzle-orm/postgres-js";
|
||||
import postgres from "postgres";
|
||||
import * as schema from "./schema.ts";
|
||||
|
||||
const url = Bun.env.DB_CONNECTION_STRING;
|
||||
if (!url) {
|
||||
throw new Error("Missing required env var: DB_CONNECTION_STRING");
|
||||
}
|
||||
|
||||
// Shared connection pool — imported once and reused across the process.
|
||||
export const client = postgres(url);
|
||||
|
||||
export const db = drizzle(client, { schema });
|
||||
|
||||
export type Db = typeof db;
|
||||
541
src/db/persistence.ts
Normal file
541
src/db/persistence.ts
Normal file
@@ -0,0 +1,541 @@
|
||||
import { sql } from "drizzle-orm";
|
||||
import { requireAsin, normalizeAsin } from "../asin.ts";
|
||||
import type {
|
||||
AnalysisResult,
|
||||
ProductRecord,
|
||||
SupplierAnalysisResult,
|
||||
} from "../types.ts";
|
||||
import { db } from "./index.ts";
|
||||
import {
|
||||
analysisRevisions,
|
||||
analysisRunStats,
|
||||
categoryRunDetails,
|
||||
productIdentifiers,
|
||||
productObservations,
|
||||
products,
|
||||
runItems,
|
||||
runs,
|
||||
sourcingInputs,
|
||||
supplierScores,
|
||||
upcResolutionCandidates,
|
||||
upcResolutions,
|
||||
} from "./schema.ts";
|
||||
|
||||
type Executor = any;
|
||||
type MetadataSource = "input" | "catalog";
|
||||
|
||||
type ProductSeed = {
|
||||
asin: string;
|
||||
name?: string | null;
|
||||
brand?: string | null;
|
||||
category?: string | null;
|
||||
metadataSource?: MetadataSource;
|
||||
fetchedAt?: Date;
|
||||
};
|
||||
|
||||
export type CategoryRunSummaryInput = {
|
||||
categoryId: number;
|
||||
categoryLabel: string;
|
||||
topAsinsChecked: number;
|
||||
availableAsins: number;
|
||||
fba: number;
|
||||
fbm: number;
|
||||
skip: number;
|
||||
status: "running" | "ok" | "empty" | "failed";
|
||||
error: string;
|
||||
};
|
||||
|
||||
export type RunCounts = {
|
||||
totalProducts: number;
|
||||
fbaCount: number;
|
||||
fbmCount: number;
|
||||
skipCount: number;
|
||||
};
|
||||
|
||||
function emptyToNull(value: string | undefined | null): string | null {
|
||||
const trimmed = value?.trim();
|
||||
return trimmed ? trimmed : null;
|
||||
}
|
||||
|
||||
function productCategory(record: ProductRecord, result: AnalysisResult): string | null {
|
||||
return emptyToNull(
|
||||
record.category ?? result.product.keepa?.categoryTree?.join(" > "),
|
||||
);
|
||||
}
|
||||
|
||||
export async function upsertProduct(
|
||||
seed: ProductSeed,
|
||||
executor: Executor = db,
|
||||
): Promise<string> {
|
||||
const asin = requireAsin(seed.asin);
|
||||
const now = seed.fetchedAt ?? new Date();
|
||||
const isCatalog = seed.metadataSource === "catalog";
|
||||
|
||||
await executor
|
||||
.insert(products)
|
||||
.values({
|
||||
asin,
|
||||
name: emptyToNull(seed.name),
|
||||
brand: emptyToNull(seed.brand),
|
||||
category: emptyToNull(seed.category),
|
||||
metadataFetchedAt: isCatalog ? now : null,
|
||||
firstSeenAt: now,
|
||||
lastSeenAt: now,
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: products.asin,
|
||||
set: {
|
||||
lastSeenAt: sql`GREATEST(${products.lastSeenAt}, EXCLUDED.last_seen_at)`,
|
||||
name: isCatalog
|
||||
? sql`CASE WHEN EXCLUDED.metadata_fetched_at >= COALESCE(${products.metadataFetchedAt}, '-infinity'::timestamptz) AND NULLIF(EXCLUDED.name, '') IS NOT NULL THEN EXCLUDED.name ELSE ${products.name} END`
|
||||
: sql`COALESCE(${products.name}, NULLIF(EXCLUDED.name, ''))`,
|
||||
brand: isCatalog
|
||||
? sql`CASE WHEN EXCLUDED.metadata_fetched_at >= COALESCE(${products.metadataFetchedAt}, '-infinity'::timestamptz) AND NULLIF(EXCLUDED.brand, '') IS NOT NULL THEN EXCLUDED.brand ELSE ${products.brand} END`
|
||||
: sql`COALESCE(${products.brand}, NULLIF(EXCLUDED.brand, ''))`,
|
||||
category: isCatalog
|
||||
? sql`CASE WHEN EXCLUDED.metadata_fetched_at >= COALESCE(${products.metadataFetchedAt}, '-infinity'::timestamptz) AND NULLIF(EXCLUDED.category, '') IS NOT NULL THEN EXCLUDED.category ELSE ${products.category} END`
|
||||
: sql`COALESCE(${products.category}, NULLIF(EXCLUDED.category, ''))`,
|
||||
metadataFetchedAt: isCatalog
|
||||
? sql`GREATEST(COALESCE(${products.metadataFetchedAt}, '-infinity'::timestamptz), EXCLUDED.metadata_fetched_at)`
|
||||
: products.metadataFetchedAt,
|
||||
},
|
||||
});
|
||||
|
||||
return asin;
|
||||
}
|
||||
|
||||
export async function insertObservation(
|
||||
runId: number,
|
||||
result: AnalysisResult,
|
||||
source: string,
|
||||
executor: Executor = db,
|
||||
): Promise<number> {
|
||||
const fetchedAt = new Date(result.product.fetchedAt);
|
||||
const record = result.product.record;
|
||||
const keepa = result.product.keepa;
|
||||
const spApi = result.product.spApi;
|
||||
const asin = requireAsin(record.asin);
|
||||
const [observation] = await executor
|
||||
.insert(productObservations)
|
||||
.values({
|
||||
productAsin: asin,
|
||||
runId,
|
||||
source,
|
||||
currentPrice:
|
||||
keepa?.currentPrice ??
|
||||
record.sellingPriceFromSheet ??
|
||||
spApi.estimatedSalePrice ??
|
||||
null,
|
||||
avgPrice90d: keepa?.avgPrice90 ?? null,
|
||||
salesRank: keepa?.salesRank ?? record.amazonRank ?? null,
|
||||
salesRankAvg90d: keepa?.salesRankAvg90 ?? null,
|
||||
monthlySold: keepa?.monthlySold ?? null,
|
||||
rankDrops30d: keepa?.salesRankDrops30 ?? null,
|
||||
rankDrops90d: keepa?.salesRankDrops90 ?? null,
|
||||
sellerCount: keepa?.sellerCount ?? null,
|
||||
amazonIsSeller: keepa?.amazonIsSeller ?? null,
|
||||
amazonBuyboxSharePct90d: keepa?.amazonBuyboxSharePct90d ?? null,
|
||||
fbaFee: spApi.fbaFee ?? null,
|
||||
fbmFee: spApi.fbmFee ?? null,
|
||||
referralPercent: spApi.referralFeePercent ?? null,
|
||||
canSell: spApi.canSell,
|
||||
sellabilityStatus: spApi.sellabilityStatus,
|
||||
sellabilityReason: spApi.sellabilityReason ?? null,
|
||||
fetchedAt,
|
||||
})
|
||||
.returning({ id: productObservations.id });
|
||||
|
||||
if (!observation) throw new Error(`Failed to insert observation for ${asin}`);
|
||||
return observation.id;
|
||||
}
|
||||
|
||||
function sourcingInputValues(runItemId: number, record: ProductRecord) {
|
||||
return {
|
||||
runItemId,
|
||||
suppliedName: emptyToNull(record.name),
|
||||
suppliedBrand: emptyToNull(record.brand),
|
||||
suppliedCategory: emptyToNull(record.category),
|
||||
unitCost: record.unitCost ?? null,
|
||||
avgPrice90dSheet: record.avgPrice90FromSheet ?? null,
|
||||
sellingPriceSheet: record.sellingPriceFromSheet ?? null,
|
||||
fbaNetSheet: record.fbaNet ?? null,
|
||||
grossProfitDollar: record.grossProfit ?? null,
|
||||
grossProfitPct: record.grossProfitPct ?? null,
|
||||
netProfitSheet: record.netProfitFromSheet ?? null,
|
||||
roiSheet: record.roiFromSheet ?? null,
|
||||
moq: record.moq ?? null,
|
||||
moqCost: record.moqCost ?? null,
|
||||
qtyAvailable: record.totalQtyAvail ?? null,
|
||||
supplier: emptyToNull(record.supplier),
|
||||
sourceUrl: emptyToNull(record.sourceUrl),
|
||||
asinLink: emptyToNull(record.asinLink),
|
||||
promoCouponCode: emptyToNull(record.promoCouponCode),
|
||||
notes: emptyToNull(record.notes),
|
||||
leadDate: emptyToNull(record.leadDate),
|
||||
};
|
||||
}
|
||||
|
||||
export async function persistLlmResults(
|
||||
runId: number,
|
||||
results: AnalysisResult[],
|
||||
options: {
|
||||
source: string;
|
||||
metadataSource?: MetadataSource;
|
||||
preserveSourcingInput?: boolean;
|
||||
sourceInventoryIds?: Map<string, number>;
|
||||
},
|
||||
): Promise<void> {
|
||||
for (const result of results) {
|
||||
const record = result.product.record;
|
||||
const fetchedAt = new Date(result.product.fetchedAt);
|
||||
const asin = await upsertProduct({
|
||||
asin: record.asin,
|
||||
name: record.name,
|
||||
brand: record.brand,
|
||||
category: productCategory(record, result),
|
||||
metadataSource: options.metadataSource ?? "input",
|
||||
fetchedAt,
|
||||
});
|
||||
const [item] = await db
|
||||
.insert(runItems)
|
||||
.values({
|
||||
runId,
|
||||
productAsin: asin,
|
||||
sourceInventoryItemId: options.sourceInventoryIds?.get(asin) ?? null,
|
||||
})
|
||||
.returning({ id: runItems.id });
|
||||
if (!item) throw new Error(`Failed to insert run item for ${asin}`);
|
||||
|
||||
if (options.preserveSourcingInput) {
|
||||
await db.insert(sourcingInputs).values(sourcingInputValues(item.id, record));
|
||||
}
|
||||
|
||||
const observationId = await insertObservation(runId, result, options.source);
|
||||
await db.insert(analysisRevisions).values({
|
||||
runItemId: item.id,
|
||||
observationId,
|
||||
method: "llm",
|
||||
decision: result.verdict.verdict,
|
||||
confidence: result.verdict.confidence,
|
||||
reasoning: result.verdict.reasoning ?? null,
|
||||
analyzedAt: fetchedAt,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function supplierSourcingValues(runItemId: number, result: SupplierAnalysisResult) {
|
||||
return {
|
||||
runItemId,
|
||||
suppliedName: emptyToNull(result.record.name),
|
||||
suppliedBrand: emptyToNull(result.record.brand),
|
||||
suppliedCategory: emptyToNull(result.record.category),
|
||||
unitCost: result.record.unitCost ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
async function insertSupplierObservation(
|
||||
runId: number,
|
||||
productAsin: string,
|
||||
result: SupplierAnalysisResult,
|
||||
): Promise<number | null> {
|
||||
const keepa = result.keepa;
|
||||
const spApi = result.spApi;
|
||||
if (!spApi && !keepa) return null;
|
||||
|
||||
const [row] = await db
|
||||
.insert(productObservations)
|
||||
.values({
|
||||
productAsin,
|
||||
runId,
|
||||
source: "supplier_upc",
|
||||
currentPrice: result.score.salePrice,
|
||||
avgPrice90d: keepa?.avgPrice90 ?? null,
|
||||
salesRank: keepa?.salesRank ?? null,
|
||||
salesRankAvg90d: keepa?.salesRankAvg90 ?? null,
|
||||
monthlySold: keepa?.monthlySold ?? null,
|
||||
rankDrops30d: keepa?.salesRankDrops30 ?? null,
|
||||
rankDrops90d: keepa?.salesRankDrops90 ?? null,
|
||||
sellerCount: keepa?.sellerCount ?? null,
|
||||
amazonIsSeller: keepa?.amazonIsSeller ?? null,
|
||||
amazonBuyboxSharePct90d: keepa?.amazonBuyboxSharePct90d ?? null,
|
||||
fbaFee: spApi?.fbaFee ?? null,
|
||||
fbmFee: spApi?.fbmFee ?? null,
|
||||
referralPercent: spApi?.referralFeePercent ?? null,
|
||||
canSell: spApi?.canSell ?? null,
|
||||
sellabilityStatus: spApi?.sellabilityStatus ?? null,
|
||||
sellabilityReason: spApi?.sellabilityReason ?? null,
|
||||
fetchedAt: new Date(result.fetchedAt),
|
||||
})
|
||||
.returning({ id: productObservations.id });
|
||||
return row?.id ?? null;
|
||||
}
|
||||
|
||||
export async function persistSupplierResults(
|
||||
runId: number,
|
||||
results: SupplierAnalysisResult[],
|
||||
): Promise<void> {
|
||||
for (const result of results) {
|
||||
const resolvedAsin = normalizeAsin(result.lookup.asin);
|
||||
if (resolvedAsin) {
|
||||
await upsertProduct({
|
||||
asin: resolvedAsin,
|
||||
name: result.record.name,
|
||||
brand: result.record.brand,
|
||||
category: result.record.category,
|
||||
metadataSource: "input",
|
||||
fetchedAt: new Date(result.fetchedAt),
|
||||
});
|
||||
if (result.keepa?.categoryTree?.length) {
|
||||
await upsertProduct({
|
||||
asin: resolvedAsin,
|
||||
category: result.keepa.categoryTree.join(" > "),
|
||||
metadataSource: "catalog",
|
||||
fetchedAt: new Date(result.fetchedAt),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const [item] = await db
|
||||
.insert(runItems)
|
||||
.values({
|
||||
runId,
|
||||
productAsin: resolvedAsin,
|
||||
sourceRow: result.rowNumber ?? null,
|
||||
})
|
||||
.returning({ id: runItems.id });
|
||||
if (!item) throw new Error("Failed to insert supplier run item");
|
||||
|
||||
await db.insert(sourcingInputs).values(supplierSourcingValues(item.id, result));
|
||||
await db.insert(upcResolutions).values({
|
||||
runItemId: item.id,
|
||||
requestedUpc: result.upc,
|
||||
normalizedUpc: result.lookup.normalizedUpc,
|
||||
provider: result.lookup.provider ?? "unknown",
|
||||
status: result.lookup.status,
|
||||
reason: result.lookup.reason ?? null,
|
||||
resolvedProductAsin: resolvedAsin,
|
||||
resolvedAt: new Date(result.fetchedAt),
|
||||
});
|
||||
|
||||
for (const candidate of result.lookup.candidateAsins) {
|
||||
const candidateAsin = normalizeAsin(candidate);
|
||||
if (!candidateAsin) continue;
|
||||
await upsertProduct({ asin: candidateAsin, fetchedAt: new Date(result.fetchedAt) });
|
||||
await db
|
||||
.insert(upcResolutionCandidates)
|
||||
.values({ runItemId: item.id, productAsin: candidateAsin })
|
||||
.onConflictDoUpdate({
|
||||
target: [
|
||||
upcResolutionCandidates.runItemId,
|
||||
upcResolutionCandidates.productAsin,
|
||||
],
|
||||
set: { productAsin: sql`EXCLUDED.product_asin` },
|
||||
});
|
||||
}
|
||||
|
||||
if (resolvedAsin) {
|
||||
await db
|
||||
.insert(productIdentifiers)
|
||||
.values({
|
||||
productAsin: resolvedAsin,
|
||||
identifierType:
|
||||
result.lookup.normalizedUpc.length === 12
|
||||
? "upc"
|
||||
: result.lookup.normalizedUpc.length === 13
|
||||
? "ean"
|
||||
: "gtin",
|
||||
identifierValue: result.lookup.normalizedUpc,
|
||||
source: "supplier_upc",
|
||||
confirmedAt: new Date(result.fetchedAt),
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: [
|
||||
productIdentifiers.identifierType,
|
||||
productIdentifiers.identifierValue,
|
||||
],
|
||||
set: {
|
||||
productAsin: resolvedAsin,
|
||||
source: "supplier_upc",
|
||||
confirmedAt: new Date(result.fetchedAt),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const observationId = resolvedAsin
|
||||
? await insertSupplierObservation(runId, resolvedAsin, result)
|
||||
: null;
|
||||
const [revision] = await db
|
||||
.insert(analysisRevisions)
|
||||
.values({
|
||||
runItemId: item.id,
|
||||
observationId,
|
||||
method: "supplier_scoring",
|
||||
decision: result.score.verdict,
|
||||
confidence: result.score.score,
|
||||
reasoning: result.score.reason,
|
||||
analyzedAt: new Date(result.fetchedAt),
|
||||
})
|
||||
.returning({ id: analysisRevisions.id });
|
||||
if (!revision) throw new Error("Failed to insert supplier analysis revision");
|
||||
|
||||
await db.insert(supplierScores).values({
|
||||
revisionId: revision.id,
|
||||
score: result.score.score,
|
||||
salePrice: result.score.salePrice,
|
||||
fbaFee: result.score.fbaFee,
|
||||
profit: result.score.profit,
|
||||
margin: result.score.margin,
|
||||
roi: result.score.roi,
|
||||
reason: result.score.reason,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export async function createCategoryRun(
|
||||
summary: CategoryRunSummaryInput,
|
||||
runTimestamp: string,
|
||||
): Promise<number> {
|
||||
const [row] = await db
|
||||
.insert(runs)
|
||||
.values({
|
||||
type: "category_analysis",
|
||||
status: summary.status,
|
||||
errorMessage: summary.error || null,
|
||||
startedAt: new Date(runTimestamp),
|
||||
})
|
||||
.returning({ id: runs.id });
|
||||
if (!row) throw new Error("Failed to insert category run.");
|
||||
|
||||
await db.insert(categoryRunDetails).values({
|
||||
runId: row.id,
|
||||
categoryId: summary.categoryId,
|
||||
categoryLabel: summary.categoryLabel,
|
||||
checkedAsinCount: summary.topAsinsChecked,
|
||||
});
|
||||
await db.insert(analysisRunStats).values({
|
||||
runId: row.id,
|
||||
processedCount: summary.topAsinsChecked,
|
||||
availableCount: summary.availableAsins,
|
||||
analyzedCount: summary.fba + summary.fbm + summary.skip,
|
||||
fbaCount: summary.fba,
|
||||
fbmCount: summary.fbm,
|
||||
skipCount: summary.skip,
|
||||
});
|
||||
return row.id;
|
||||
}
|
||||
|
||||
export async function updateCategoryRun(
|
||||
runId: number,
|
||||
summary: Pick<
|
||||
CategoryRunSummaryInput,
|
||||
| "topAsinsChecked"
|
||||
| "availableAsins"
|
||||
| "fba"
|
||||
| "fbm"
|
||||
| "skip"
|
||||
| "status"
|
||||
| "error"
|
||||
>,
|
||||
): Promise<void> {
|
||||
await db
|
||||
.update(runs)
|
||||
.set({
|
||||
status: summary.status,
|
||||
errorMessage: summary.error || null,
|
||||
...(summary.status !== "running" ? { completedAt: new Date() } : {}),
|
||||
})
|
||||
.where(sql`${runs.id} = ${runId}`);
|
||||
await db
|
||||
.insert(categoryRunDetails)
|
||||
.values({
|
||||
runId,
|
||||
categoryId: 0,
|
||||
categoryLabel: "",
|
||||
checkedAsinCount: summary.topAsinsChecked,
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: categoryRunDetails.runId,
|
||||
set: { checkedAsinCount: summary.topAsinsChecked },
|
||||
});
|
||||
await db
|
||||
.insert(analysisRunStats)
|
||||
.values({
|
||||
runId,
|
||||
processedCount: summary.topAsinsChecked,
|
||||
availableCount: summary.availableAsins,
|
||||
analyzedCount: summary.fba + summary.fbm + summary.skip,
|
||||
fbaCount: summary.fba,
|
||||
fbmCount: summary.fbm,
|
||||
skipCount: summary.skip,
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: analysisRunStats.runId,
|
||||
set: {
|
||||
processedCount: summary.topAsinsChecked,
|
||||
availableCount: summary.availableAsins,
|
||||
analyzedCount: summary.fba + summary.fbm + summary.skip,
|
||||
fbaCount: summary.fba,
|
||||
fbmCount: summary.fbm,
|
||||
skipCount: summary.skip,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function refreshRunStats(runId: number): Promise<RunCounts> {
|
||||
const [stats] = await db.execute(
|
||||
sql<{
|
||||
total: string;
|
||||
fba: string | null;
|
||||
fbm: string | null;
|
||||
buy: string | null;
|
||||
watch: string | null;
|
||||
skip: string | null;
|
||||
}>`WITH latest AS (
|
||||
SELECT DISTINCT ON (ri.id) ar.decision
|
||||
FROM run_items ri
|
||||
JOIN analysis_revisions ar ON ar.run_item_id = ri.id
|
||||
WHERE ri.run_id = ${runId}
|
||||
ORDER BY ri.id, ar.analyzed_at DESC, ar.id DESC
|
||||
)
|
||||
SELECT
|
||||
COUNT(*) AS total,
|
||||
SUM(CASE WHEN decision = 'FBA' THEN 1 ELSE 0 END) AS fba,
|
||||
SUM(CASE WHEN decision = 'FBM' THEN 1 ELSE 0 END) AS fbm,
|
||||
SUM(CASE WHEN decision = 'BUY' THEN 1 ELSE 0 END) AS buy,
|
||||
SUM(CASE WHEN decision = 'WATCH' THEN 1 ELSE 0 END) AS watch,
|
||||
SUM(CASE WHEN decision = 'SKIP' THEN 1 ELSE 0 END) AS skip
|
||||
FROM latest`,
|
||||
);
|
||||
|
||||
const counts = {
|
||||
totalProducts: Number(stats?.total ?? 0),
|
||||
fbaCount: Number(stats?.fba ?? 0),
|
||||
fbmCount: Number(stats?.fbm ?? 0),
|
||||
skipCount: Number(stats?.skip ?? 0),
|
||||
};
|
||||
await db
|
||||
.insert(analysisRunStats)
|
||||
.values({
|
||||
runId,
|
||||
processedCount: counts.totalProducts,
|
||||
analyzedCount: counts.totalProducts,
|
||||
fbaCount: counts.fbaCount,
|
||||
fbmCount: counts.fbmCount,
|
||||
buyCount: Number(stats?.buy ?? 0),
|
||||
watchCount: Number(stats?.watch ?? 0),
|
||||
skipCount: counts.skipCount,
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: analysisRunStats.runId,
|
||||
set: {
|
||||
processedCount: counts.totalProducts,
|
||||
analyzedCount: counts.totalProducts,
|
||||
fbaCount: counts.fbaCount,
|
||||
fbmCount: counts.fbmCount,
|
||||
buyCount: Number(stats?.buy ?? 0),
|
||||
watchCount: Number(stats?.watch ?? 0),
|
||||
skipCount: counts.skipCount,
|
||||
},
|
||||
});
|
||||
return counts;
|
||||
}
|
||||
441
src/db/schema.ts
Normal file
441
src/db/schema.ts
Normal file
@@ -0,0 +1,441 @@
|
||||
import { sql } from "drizzle-orm";
|
||||
import {
|
||||
type AnyPgColumn,
|
||||
boolean,
|
||||
check,
|
||||
index,
|
||||
integer,
|
||||
pgEnum,
|
||||
pgTable,
|
||||
primaryKey,
|
||||
real,
|
||||
serial,
|
||||
text,
|
||||
timestamp,
|
||||
unique,
|
||||
} from "drizzle-orm/pg-core";
|
||||
|
||||
export const runTypeEnum = pgEnum("run_type", [
|
||||
"lead_analysis",
|
||||
"category_analysis",
|
||||
"supplier_upc",
|
||||
"stalker",
|
||||
"stalker_analysis",
|
||||
]);
|
||||
|
||||
export const runStatusEnum = pgEnum("run_status", [
|
||||
"running",
|
||||
"ok",
|
||||
"empty",
|
||||
"failed",
|
||||
"completed",
|
||||
]);
|
||||
|
||||
export const analysisMethodEnum = pgEnum("analysis_method", [
|
||||
"llm",
|
||||
"supplier_scoring",
|
||||
]);
|
||||
|
||||
export const analysisDecisionEnum = pgEnum("analysis_decision", [
|
||||
"FBA",
|
||||
"FBM",
|
||||
"BUY",
|
||||
"WATCH",
|
||||
"SKIP",
|
||||
]);
|
||||
|
||||
export const products = pgTable(
|
||||
"products",
|
||||
{
|
||||
asin: text("asin").primaryKey(),
|
||||
name: text("name"),
|
||||
brand: text("brand"),
|
||||
category: text("category"),
|
||||
metadataFetchedAt: timestamp("metadata_fetched_at", { withTimezone: true }),
|
||||
firstSeenAt: timestamp("first_seen_at", { withTimezone: true })
|
||||
.notNull()
|
||||
.defaultNow(),
|
||||
lastSeenAt: timestamp("last_seen_at", { withTimezone: true })
|
||||
.notNull()
|
||||
.defaultNow(),
|
||||
},
|
||||
(t) => [
|
||||
check("ck_products_asin", sql`${t.asin} ~ '^[A-Z0-9]{10}$'`),
|
||||
index("idx_products_name").on(t.name),
|
||||
index("idx_products_last_seen_at").on(t.lastSeenAt),
|
||||
],
|
||||
);
|
||||
|
||||
export const runs = pgTable(
|
||||
"runs",
|
||||
{
|
||||
id: serial("id").primaryKey(),
|
||||
type: runTypeEnum("type").notNull(),
|
||||
parentRunId: integer("parent_run_id").references(
|
||||
(): AnyPgColumn => runs.id,
|
||||
{ onDelete: "cascade" },
|
||||
),
|
||||
inputFile: text("input_file"),
|
||||
outputFile: text("output_file"),
|
||||
status: runStatusEnum("status").notNull().default("running"),
|
||||
errorMessage: text("error_message"),
|
||||
startedAt: timestamp("started_at", { withTimezone: true })
|
||||
.notNull()
|
||||
.defaultNow(),
|
||||
completedAt: timestamp("completed_at", { withTimezone: true }),
|
||||
},
|
||||
(t) => [
|
||||
index("idx_runs_started_at").on(t.startedAt),
|
||||
index("idx_runs_type").on(t.type),
|
||||
index("idx_runs_status").on(t.status),
|
||||
index("idx_runs_parent_run_id").on(t.parentRunId),
|
||||
],
|
||||
);
|
||||
|
||||
export const analysisRunStats = pgTable("analysis_run_stats", {
|
||||
runId: integer("run_id")
|
||||
.primaryKey()
|
||||
.references(() => runs.id, { onDelete: "cascade" }),
|
||||
processedCount: integer("processed_count").notNull().default(0),
|
||||
analyzedCount: integer("analyzed_count").notNull().default(0),
|
||||
availableCount: integer("available_count").notNull().default(0),
|
||||
fbaCount: integer("fba_count").notNull().default(0),
|
||||
fbmCount: integer("fbm_count").notNull().default(0),
|
||||
buyCount: integer("buy_count").notNull().default(0),
|
||||
watchCount: integer("watch_count").notNull().default(0),
|
||||
skipCount: integer("skip_count").notNull().default(0),
|
||||
});
|
||||
|
||||
export const categoryRunDetails = pgTable("category_run_details", {
|
||||
runId: integer("run_id")
|
||||
.primaryKey()
|
||||
.references(() => runs.id, { onDelete: "cascade" }),
|
||||
categoryId: integer("category_id").notNull(),
|
||||
categoryLabel: text("category_label").notNull(),
|
||||
checkedAsinCount: integer("checked_asin_count").notNull().default(0),
|
||||
selectionParametersJson: text("selection_parameters_json"),
|
||||
});
|
||||
|
||||
export const stalkerRunDetails = pgTable("stalker_run_details", {
|
||||
runId: integer("run_id")
|
||||
.primaryKey()
|
||||
.references(() => runs.id, { onDelete: "cascade" }),
|
||||
requestedAsins: integer("requested_asins").notNull().default(0),
|
||||
skippedAsins: integer("skipped_asins").notNull().default(0),
|
||||
scannedAsins: integer("scanned_asins").notNull().default(0),
|
||||
sourceAsinsWithMatches: integer("source_asins_with_matches")
|
||||
.notNull()
|
||||
.default(0),
|
||||
candidateSellers: integer("candidate_sellers").notNull().default(0),
|
||||
qualifyingSellers: integer("qualifying_sellers").notNull().default(0),
|
||||
matchedSellers: integer("matched_sellers").notNull().default(0),
|
||||
sellerMetadataRequests: integer("seller_metadata_requests")
|
||||
.notNull()
|
||||
.default(0),
|
||||
sellerStorefrontRequests: integer("seller_storefront_requests")
|
||||
.notNull()
|
||||
.default(0),
|
||||
inventorySellabilityCheckedAsins: integer(
|
||||
"inventory_sellability_checked_asins",
|
||||
)
|
||||
.notNull()
|
||||
.default(0),
|
||||
inventorySellabilityAvailableAsins: integer(
|
||||
"inventory_sellability_available_asins",
|
||||
)
|
||||
.notNull()
|
||||
.default(0),
|
||||
inventorySellabilityExcludedAsins: integer(
|
||||
"inventory_sellability_excluded_asins",
|
||||
)
|
||||
.notNull()
|
||||
.default(0),
|
||||
persistedInventoryAsins: integer("persisted_inventory_asins")
|
||||
.notNull()
|
||||
.default(0),
|
||||
});
|
||||
|
||||
export const productIdentifiers = pgTable(
|
||||
"product_identifiers",
|
||||
{
|
||||
id: serial("id").primaryKey(),
|
||||
productAsin: text("product_asin")
|
||||
.notNull()
|
||||
.references(() => products.asin),
|
||||
identifierType: text("identifier_type").notNull(),
|
||||
identifierValue: text("identifier_value").notNull(),
|
||||
source: text("source").notNull(),
|
||||
confirmedAt: timestamp("confirmed_at", { withTimezone: true })
|
||||
.notNull()
|
||||
.defaultNow(),
|
||||
},
|
||||
(t) => [
|
||||
unique("uq_product_identifier_type_value").on(
|
||||
t.identifierType,
|
||||
t.identifierValue,
|
||||
),
|
||||
index("idx_product_identifiers_asin").on(t.productAsin),
|
||||
],
|
||||
);
|
||||
|
||||
export const productObservations = pgTable(
|
||||
"product_observations",
|
||||
{
|
||||
id: serial("id").primaryKey(),
|
||||
productAsin: text("product_asin")
|
||||
.notNull()
|
||||
.references(() => products.asin),
|
||||
runId: integer("run_id")
|
||||
.notNull()
|
||||
.references(() => runs.id, { onDelete: "cascade" }),
|
||||
source: text("source").notNull(),
|
||||
marketplace: text("marketplace").notNull().default("US"),
|
||||
currentPrice: real("current_price"),
|
||||
avgPrice90d: real("avg_price_90d"),
|
||||
salesRank: integer("sales_rank"),
|
||||
salesRankAvg90d: integer("sales_rank_avg_90d"),
|
||||
monthlySold: integer("monthly_sold"),
|
||||
rankDrops30d: integer("rank_drops_30d"),
|
||||
rankDrops90d: integer("rank_drops_90d"),
|
||||
sellerCount: integer("seller_count"),
|
||||
amazonIsSeller: boolean("amazon_is_seller"),
|
||||
amazonBuyboxSharePct90d: real("amazon_buybox_share_pct_90d"),
|
||||
fbaFee: real("fba_fee"),
|
||||
fbmFee: real("fbm_fee"),
|
||||
referralPercent: real("referral_percent"),
|
||||
canSell: boolean("can_sell"),
|
||||
sellabilityStatus: text("sellability_status"),
|
||||
sellabilityReason: text("sellability_reason"),
|
||||
rawProductJson: text("raw_product_json"),
|
||||
fetchedAt: timestamp("fetched_at", { withTimezone: true }).notNull(),
|
||||
},
|
||||
(t) => [
|
||||
index("idx_product_observations_product_time").on(
|
||||
t.productAsin,
|
||||
t.fetchedAt.desc(),
|
||||
),
|
||||
index("idx_product_observations_run_id").on(t.runId),
|
||||
index("idx_product_observations_sellability").on(t.sellabilityStatus),
|
||||
],
|
||||
);
|
||||
|
||||
export const runItems = pgTable(
|
||||
"run_items",
|
||||
{
|
||||
id: serial("id").primaryKey(),
|
||||
runId: integer("run_id")
|
||||
.notNull()
|
||||
.references(() => runs.id, { onDelete: "cascade" }),
|
||||
productAsin: text("product_asin").references(() => products.asin),
|
||||
sourceInventoryItemId: integer("source_inventory_item_id").references(
|
||||
(): AnyPgColumn => stalkerInventoryItems.id,
|
||||
{ onDelete: "set null" },
|
||||
),
|
||||
ordinal: integer("ordinal"),
|
||||
sourceRow: integer("source_row"),
|
||||
status: text("status").notNull().default("completed"),
|
||||
createdAt: timestamp("created_at", { withTimezone: true })
|
||||
.notNull()
|
||||
.defaultNow(),
|
||||
},
|
||||
(t) => [
|
||||
index("idx_run_items_run_id").on(t.runId),
|
||||
index("idx_run_items_product_asin").on(t.productAsin),
|
||||
],
|
||||
);
|
||||
|
||||
export const sourcingInputs = pgTable("sourcing_inputs", {
|
||||
runItemId: integer("run_item_id")
|
||||
.primaryKey()
|
||||
.references(() => runItems.id, { onDelete: "cascade" }),
|
||||
suppliedName: text("supplied_name"),
|
||||
suppliedBrand: text("supplied_brand"),
|
||||
suppliedCategory: text("supplied_category"),
|
||||
unitCost: real("unit_cost"),
|
||||
avgPrice90dSheet: real("avg_price_90d_sheet"),
|
||||
sellingPriceSheet: real("selling_price_sheet"),
|
||||
fbaNetSheet: real("fba_net_sheet"),
|
||||
grossProfitDollar: real("gross_profit_dollar"),
|
||||
grossProfitPct: real("gross_profit_pct"),
|
||||
netProfitSheet: real("net_profit_sheet"),
|
||||
roiSheet: real("roi_sheet"),
|
||||
moq: integer("moq"),
|
||||
moqCost: real("moq_cost"),
|
||||
qtyAvailable: integer("qty_available"),
|
||||
supplier: text("supplier"),
|
||||
sourceUrl: text("source_url"),
|
||||
asinLink: text("asin_link"),
|
||||
promoCouponCode: text("promo_coupon_code"),
|
||||
notes: text("notes"),
|
||||
leadDate: text("lead_date"),
|
||||
});
|
||||
|
||||
export const upcResolutions = pgTable(
|
||||
"upc_resolutions",
|
||||
{
|
||||
runItemId: integer("run_item_id")
|
||||
.primaryKey()
|
||||
.references(() => runItems.id, { onDelete: "cascade" }),
|
||||
requestedUpc: text("requested_upc").notNull(),
|
||||
normalizedUpc: text("normalized_upc").notNull(),
|
||||
provider: text("provider").notNull(),
|
||||
status: text("status").notNull(),
|
||||
reason: text("reason"),
|
||||
resolvedProductAsin: text("resolved_product_asin").references(
|
||||
() => products.asin,
|
||||
),
|
||||
resolvedAt: timestamp("resolved_at", { withTimezone: true })
|
||||
.notNull()
|
||||
.defaultNow(),
|
||||
},
|
||||
(t) => [index("idx_upc_resolutions_normalized_upc").on(t.normalizedUpc)],
|
||||
);
|
||||
|
||||
export const upcResolutionCandidates = pgTable(
|
||||
"upc_resolution_candidates",
|
||||
{
|
||||
runItemId: integer("run_item_id")
|
||||
.notNull()
|
||||
.references(() => upcResolutions.runItemId, { onDelete: "cascade" }),
|
||||
productAsin: text("product_asin")
|
||||
.notNull()
|
||||
.references(() => products.asin),
|
||||
},
|
||||
(t) => [
|
||||
primaryKey({ columns: [t.runItemId, t.productAsin] }),
|
||||
index("idx_upc_candidates_product_asin").on(t.productAsin),
|
||||
],
|
||||
);
|
||||
|
||||
export const analysisRevisions = pgTable(
|
||||
"analysis_revisions",
|
||||
{
|
||||
id: serial("id").primaryKey(),
|
||||
runItemId: integer("run_item_id")
|
||||
.notNull()
|
||||
.references(() => runItems.id, { onDelete: "cascade" }),
|
||||
observationId: integer("observation_id").references(
|
||||
() => productObservations.id,
|
||||
{ onDelete: "set null" },
|
||||
),
|
||||
method: analysisMethodEnum("method").notNull(),
|
||||
decision: analysisDecisionEnum("decision").notNull(),
|
||||
confidence: real("confidence"),
|
||||
reasoning: text("reasoning"),
|
||||
analyzedAt: timestamp("analyzed_at", { withTimezone: true })
|
||||
.notNull()
|
||||
.defaultNow(),
|
||||
},
|
||||
(t) => [
|
||||
index("idx_analysis_revisions_run_item_time").on(t.runItemId, t.analyzedAt),
|
||||
index("idx_analysis_revisions_decision").on(t.decision),
|
||||
],
|
||||
);
|
||||
|
||||
export const supplierScores = pgTable("supplier_scores", {
|
||||
revisionId: integer("revision_id")
|
||||
.primaryKey()
|
||||
.references(() => analysisRevisions.id, { onDelete: "cascade" }),
|
||||
score: real("score"),
|
||||
salePrice: real("sale_price"),
|
||||
fbaFee: real("fba_fee"),
|
||||
profit: real("profit"),
|
||||
margin: real("margin"),
|
||||
roi: real("roi"),
|
||||
reason: text("reason"),
|
||||
});
|
||||
|
||||
export const sellers = pgTable("sellers", {
|
||||
sellerId: text("seller_id").primaryKey(),
|
||||
sellerName: text("seller_name"),
|
||||
rating: real("rating"),
|
||||
ratingCount: integer("rating_count"),
|
||||
storefrontAsinTotal: integer("storefront_asin_total"),
|
||||
persistedInventorySampleCount: integer("persisted_inventory_sample_count"),
|
||||
lastUpdatedAt: timestamp("last_updated_at", { withTimezone: true }).notNull(),
|
||||
rawSellerJson: text("raw_seller_json"),
|
||||
});
|
||||
|
||||
export const stalkerScans = pgTable(
|
||||
"stalker_scans",
|
||||
{
|
||||
id: serial("id").primaryKey(),
|
||||
runId: integer("run_id")
|
||||
.notNull()
|
||||
.references(() => runs.id, { onDelete: "cascade" }),
|
||||
sourceProductAsin: text("source_product_asin")
|
||||
.notNull()
|
||||
.references(() => products.asin),
|
||||
observationId: integer("observation_id").references(
|
||||
() => productObservations.id,
|
||||
{ onDelete: "set null" },
|
||||
),
|
||||
offerCount: integer("offer_count").notNull().default(0),
|
||||
candidateSellerCount: integer("candidate_seller_count")
|
||||
.notNull()
|
||||
.default(0),
|
||||
matchedSellerCount: integer("matched_seller_count").notNull().default(0),
|
||||
fetchedAt: timestamp("fetched_at", { withTimezone: true }).notNull(),
|
||||
},
|
||||
(t) => [
|
||||
unique("uq_stalker_scans_run_source_product").on(
|
||||
t.runId,
|
||||
t.sourceProductAsin,
|
||||
),
|
||||
index("idx_stalker_scans_run_id").on(t.runId),
|
||||
index("idx_stalker_scans_source_asin").on(t.sourceProductAsin),
|
||||
],
|
||||
);
|
||||
|
||||
export const stalkerScanSellers = pgTable(
|
||||
"stalker_scan_sellers",
|
||||
{
|
||||
id: serial("id").primaryKey(),
|
||||
scanId: integer("scan_id")
|
||||
.notNull()
|
||||
.references(() => stalkerScans.id, { onDelete: "cascade" }),
|
||||
sellerId: text("seller_id")
|
||||
.notNull()
|
||||
.references(() => sellers.sellerId),
|
||||
offerPrice: real("offer_price"),
|
||||
condition: text("condition"),
|
||||
isFba: boolean("is_fba"),
|
||||
stock: integer("stock"),
|
||||
sellerRating: real("seller_rating"),
|
||||
sellerRatingCount: integer("seller_rating_count"),
|
||||
rawOfferJson: text("raw_offer_json"),
|
||||
},
|
||||
(t) => [
|
||||
unique("uq_stalker_scan_sellers_scan_seller").on(t.scanId, t.sellerId),
|
||||
],
|
||||
);
|
||||
|
||||
export const stalkerInventoryItems = pgTable(
|
||||
"stalker_inventory_items",
|
||||
{
|
||||
id: serial("id").primaryKey(),
|
||||
runId: integer("run_id")
|
||||
.notNull()
|
||||
.references(() => runs.id, { onDelete: "cascade" }),
|
||||
sellerId: text("seller_id")
|
||||
.notNull()
|
||||
.references(() => sellers.sellerId),
|
||||
productAsin: text("product_asin")
|
||||
.notNull()
|
||||
.references(() => products.asin),
|
||||
observationId: integer("observation_id")
|
||||
.notNull()
|
||||
.references(() => productObservations.id, { onDelete: "cascade" }),
|
||||
lastSeenAt: timestamp("last_seen_at", { withTimezone: true }).notNull(),
|
||||
rawInventoryJson: text("raw_inventory_json"),
|
||||
},
|
||||
(t) => [
|
||||
unique("uq_stalker_inventory_items_run_seller_asin").on(
|
||||
t.runId,
|
||||
t.sellerId,
|
||||
t.productAsin,
|
||||
),
|
||||
index("idx_stalker_inventory_seller_id").on(t.sellerId),
|
||||
index("idx_stalker_inventory_product_asin").on(t.productAsin),
|
||||
],
|
||||
);
|
||||
43
src/index.ts
43
src/index.ts
@@ -1,11 +1,10 @@
|
||||
import { readProducts } from "./reader.ts";
|
||||
import { connectCache, disconnectCache } from "./cache.ts";
|
||||
import { connectCache, disconnectCache } from "./integrations/cache.ts";
|
||||
import {
|
||||
printResults,
|
||||
writeResultsToDb,
|
||||
writeResultsWorkbook,
|
||||
} from "./writer.ts";
|
||||
import { initDb, closeDb } from "./database.ts";
|
||||
import {
|
||||
chunkArray,
|
||||
processProductChunk,
|
||||
@@ -14,8 +13,9 @@ import {
|
||||
import path from "node:path";
|
||||
import type { AnalysisResult } from "./types.ts";
|
||||
|
||||
const DB_PATH = process.env.RESULTS_DB_PATH || path.join("db", "results.db");
|
||||
const INPUT_BATCH_SIZE = 50;
|
||||
const INPUT_DIR = "input";
|
||||
const OUTPUT_DIR = "output";
|
||||
|
||||
function parseSellabilityArg(args: string[]): SellabilityFilter {
|
||||
const sellabilityArg = args.find((a) => a.startsWith("--sellability="));
|
||||
@@ -47,7 +47,7 @@ function parseArgs(): {
|
||||
const args = process.argv.slice(2);
|
||||
const outputFile = readFlagValue(args, "--out", "--output");
|
||||
const useClaude = args.includes("--claude");
|
||||
const inputFile = readInputFileArg(
|
||||
const inputFileArg = readInputFileArg(
|
||||
args,
|
||||
"--out",
|
||||
"--output",
|
||||
@@ -55,14 +55,19 @@ function parseArgs(): {
|
||||
);
|
||||
const sellability = parseSellabilityArg(args);
|
||||
|
||||
if (!inputFile) {
|
||||
if (!inputFileArg) {
|
||||
console.error(
|
||||
"Usage: bun run src/index.ts <input.csv|xlsx> [--out results.xlsx|--output results.xlsx] [--sellability available|all] [--claude]",
|
||||
"Usage: bun run src/index.ts <input.csv|xlsx> [--out results.xlsx|--output results.xlsx] [--sellability available|all] [--claude]\nBare filenames are read from input/ and written to output/.",
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
return { inputFile, outputFile, sellability, useClaude };
|
||||
return {
|
||||
inputFile: resolveInputPath(inputFileArg),
|
||||
outputFile,
|
||||
sellability,
|
||||
useClaude,
|
||||
};
|
||||
}
|
||||
|
||||
function readFlagValue(args: string[], ...flags: string[]): string | undefined {
|
||||
@@ -103,11 +108,25 @@ function readInputFileArg(
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function isBareFilename(filePath: string): boolean {
|
||||
return !path.isAbsolute(filePath) && !/[\\/]/.test(filePath);
|
||||
}
|
||||
|
||||
function resolveInputPath(inputFile: string): string {
|
||||
return isBareFilename(inputFile)
|
||||
? path.join(INPUT_DIR, inputFile)
|
||||
: inputFile;
|
||||
}
|
||||
|
||||
function resolveBaseOutputPath(inputFile: string, outputFile?: string): string {
|
||||
if (outputFile) return outputFile;
|
||||
if (outputFile) {
|
||||
return isBareFilename(outputFile)
|
||||
? path.join(OUTPUT_DIR, outputFile)
|
||||
: outputFile;
|
||||
}
|
||||
|
||||
const parsedInput = path.parse(inputFile);
|
||||
return path.join("output", `${parsedInput.name}_results.xlsx`);
|
||||
return path.join(OUTPUT_DIR, `${parsedInput.name}_results.xlsx`);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
@@ -119,9 +138,6 @@ async function main() {
|
||||
console.log("Connecting to Redis...");
|
||||
await connectCache();
|
||||
|
||||
console.log("Initializing SQLite database...");
|
||||
initDb(DB_PATH);
|
||||
|
||||
try {
|
||||
console.log(`\nReading ${inputFile}...`);
|
||||
const products = readProducts(inputFile);
|
||||
@@ -156,10 +172,9 @@ async function main() {
|
||||
|
||||
printResults(allResults);
|
||||
writeResultsWorkbook(allResults, resolvedBaseOutputPath);
|
||||
writeResultsToDb(allResults, DB_PATH, inputFile, resolvedBaseOutputPath);
|
||||
await writeResultsToDb(allResults, inputFile, resolvedBaseOutputPath);
|
||||
} finally {
|
||||
await disconnectCache();
|
||||
closeDb();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import Redis from "ioredis";
|
||||
import { config } from "./config.ts";
|
||||
import type { EnrichedProduct, KeepaData, SpApiData } from "./types.ts";
|
||||
import { config } from "../config.ts";
|
||||
import type { EnrichedProduct, KeepaData, SpApiData } from "../types.ts";
|
||||
|
||||
let redis: Redis | null = null;
|
||||
let disabled = false;
|
||||
@@ -42,7 +42,7 @@ test("lookupKeepaUpcs returns found, not_found, and multiple_asins outcomes", as
|
||||
JSON.stringify({
|
||||
products: [
|
||||
{
|
||||
asin: "B000FOUND01",
|
||||
asin: "B000FND001",
|
||||
upcList: ["012345678901"],
|
||||
stats: {
|
||||
current: [null, null, null, 1234],
|
||||
@@ -51,7 +51,7 @@ test("lookupKeepaUpcs returns found, not_found, and multiple_asins outcomes", as
|
||||
csv: [[5000000, 2999, 5000100]],
|
||||
},
|
||||
{
|
||||
asin: "B000MULTI01",
|
||||
asin: "B000MUL001",
|
||||
upcList: ["098765432109"],
|
||||
stats: {
|
||||
current: [null, null, null, 2000],
|
||||
@@ -60,7 +60,7 @@ test("lookupKeepaUpcs returns found, not_found, and multiple_asins outcomes", as
|
||||
csv: [[1, 1999]],
|
||||
},
|
||||
{
|
||||
asin: "B000MULTI02",
|
||||
asin: "B000MUL002",
|
||||
upcList: ["098765432109"],
|
||||
stats: {
|
||||
current: [null, null, null, 2100],
|
||||
@@ -83,14 +83,14 @@ test("lookupKeepaUpcs returns found, not_found, and multiple_asins outcomes", as
|
||||
]);
|
||||
|
||||
expect(details.get("012345678901")?.status).toBe("found");
|
||||
expect(details.get("012345678901")?.asin).toBe("B000FOUND01");
|
||||
expect(details.get("012345678901")?.asin).toBe("B000FND001");
|
||||
expect(details.get("012345678901")?.keepaData?.currentPrice).toBe(29.99);
|
||||
expect(details.get("012345678901")?.keepaData?.currentPrice).toBe(29.99);
|
||||
|
||||
expect(details.get("098765432109")?.status).toBe("multiple_asins");
|
||||
expect(details.get("098765432109")?.candidateAsins).toEqual([
|
||||
"B000MULTI01",
|
||||
"B000MULTI02",
|
||||
"B000MUL001",
|
||||
"B000MUL002",
|
||||
]);
|
||||
|
||||
expect(details.get("111111111111")?.status).toBe("not_found");
|
||||
@@ -100,7 +100,7 @@ test("lookupKeepaUpcs returns found, not_found, and multiple_asins outcomes", as
|
||||
"098765432109",
|
||||
"111111111111",
|
||||
]);
|
||||
expect(simpleMap.get("012345678901")).toBe("B000FOUND01");
|
||||
expect(simpleMap.get("012345678901")).toBe("B000FND001");
|
||||
expect(simpleMap.has("098765432109")).toBe(false);
|
||||
expect(simpleMap.has("111111111111")).toBe(false);
|
||||
});
|
||||
@@ -128,7 +128,7 @@ test("lookupKeepaUpcs keeps partial success when one chunk fails", async () => {
|
||||
JSON.stringify({
|
||||
products: [
|
||||
{
|
||||
asin: "B000LAST001",
|
||||
asin: "B000LST001",
|
||||
upcList: [secondChunkUpc],
|
||||
stats: {
|
||||
current: [null, null, null, 1000],
|
||||
@@ -148,11 +148,11 @@ test("lookupKeepaUpcs keeps partial success when one chunk fails", async () => {
|
||||
|
||||
expect(details.get(firstChunkFirstUpc)?.status).toBe("request_failed");
|
||||
expect(details.get(secondChunkUpc)?.status).toBe("found");
|
||||
expect(details.get(secondChunkUpc)?.asin).toBe("B000LAST001");
|
||||
expect(details.get(secondChunkUpc)?.asin).toBe("B000LST001");
|
||||
|
||||
const simpleMap = await mapUpcsToAsins(upcs);
|
||||
expect(simpleMap.has(firstChunkFirstUpc)).toBe(false);
|
||||
expect(simpleMap.get(secondChunkUpc)).toBe("B000LAST001");
|
||||
expect(simpleMap.get(secondChunkUpc)).toBe("B000LST001");
|
||||
});
|
||||
|
||||
test("lookupKeepaUpcs retries on 429 and succeeds after refill wait", async () => {
|
||||
@@ -175,7 +175,7 @@ test("lookupKeepaUpcs retries on 429 and succeeds after refill wait", async () =
|
||||
JSON.stringify({
|
||||
products: [
|
||||
{
|
||||
asin: "B000RETRY01",
|
||||
asin: "B000RTY001",
|
||||
upcList: [targetUpc],
|
||||
stats: {
|
||||
current: [null, null, null, 1111],
|
||||
@@ -197,7 +197,7 @@ test("lookupKeepaUpcs retries on 429 and succeeds after refill wait", async () =
|
||||
|
||||
expect(fetchMock.mock.calls.length).toBe(2);
|
||||
expect(details.get(targetUpc)?.status).toBe("found");
|
||||
expect(details.get(targetUpc)?.asin).toBe("B000RETRY01");
|
||||
expect(details.get(targetUpc)?.asin).toBe("B000RTY001");
|
||||
});
|
||||
|
||||
test("lookupKeepaUpcs uses lightweight query params for code mapping", async () => {
|
||||
@@ -220,7 +220,7 @@ test("lookupKeepaUpcs uses lightweight query params for code mapping", async ()
|
||||
JSON.stringify({
|
||||
products: [
|
||||
{
|
||||
asin: "B000LIGHT01",
|
||||
asin: "B000LGT001",
|
||||
upcList: [targetUpc],
|
||||
categoryTree: [{ name: "Test Category" }],
|
||||
},
|
||||
@@ -238,5 +238,5 @@ test("lookupKeepaUpcs uses lightweight query params for code mapping", async ()
|
||||
|
||||
expect(fetchMock.mock.calls.length).toBe(1);
|
||||
expect(details.get(targetUpc)?.status).toBe("found");
|
||||
expect(details.get(targetUpc)?.asin).toBe("B000LIGHT01");
|
||||
expect(details.get(targetUpc)?.asin).toBe("B000LGT001");
|
||||
});
|
||||
@@ -1,5 +1,6 @@
|
||||
import { config } from "./config.ts";
|
||||
import type { KeepaData, KeepaUpcLookupDetail } from "./types.ts";
|
||||
import { config } from "../config.ts";
|
||||
import { normalizeAsin } from "../asin.ts";
|
||||
import type { KeepaData, KeepaUpcLookupDetail } from "../types.ts";
|
||||
|
||||
const KEEPA_BASE = "https://api.keepa.com";
|
||||
const MAX_ASINS_PER_REQUEST = 100;
|
||||
@@ -224,14 +225,21 @@ function buildFailureDetail(
|
||||
};
|
||||
}
|
||||
|
||||
export async function fetchKeepaDataBatch(
|
||||
asins: string[],
|
||||
): Promise<Map<string, KeepaData>> {
|
||||
const results = new Map<string, KeepaData>();
|
||||
|
||||
// Split into chunks of MAX_ASINS_PER_REQUEST
|
||||
for (let i = 0; i < asins.length; i += MAX_ASINS_PER_REQUEST) {
|
||||
const chunk = asins.slice(i, i + MAX_ASINS_PER_REQUEST);
|
||||
export async function fetchKeepaDataBatch(
|
||||
asins: string[],
|
||||
): Promise<Map<string, KeepaData>> {
|
||||
const results = new Map<string, KeepaData>();
|
||||
const canonicalAsins = Array.from(
|
||||
new Set(
|
||||
asins
|
||||
.map((asin) => normalizeAsin(asin))
|
||||
.filter((asin): asin is string => asin !== null),
|
||||
),
|
||||
);
|
||||
|
||||
// Split into chunks of MAX_ASINS_PER_REQUEST
|
||||
for (let i = 0; i < canonicalAsins.length; i += MAX_ASINS_PER_REQUEST) {
|
||||
const chunk = canonicalAsins.slice(i, i + MAX_ASINS_PER_REQUEST);
|
||||
const url = buildProductUrl("asin", chunk, {
|
||||
includeStats: true,
|
||||
includeBuybox: true,
|
||||
@@ -248,11 +256,11 @@ export async function fetchKeepaDataBatch(
|
||||
`Keepa: ${data.products?.length ?? 0} products returned, ${tokensLeft} tokens remaining (refill: ${refillRate}/min)`,
|
||||
);
|
||||
|
||||
if (data.products) {
|
||||
for (const product of data.products) {
|
||||
const asin = product.asin;
|
||||
if (!asin) continue;
|
||||
results.set(asin, parseKeepaProduct(product));
|
||||
if (data.products) {
|
||||
for (const product of data.products) {
|
||||
const asin = normalizeAsin(product.asin);
|
||||
if (!asin) continue;
|
||||
results.set(asin, parseKeepaProduct(product));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -307,10 +315,10 @@ export async function lookupKeepaUpcs(
|
||||
`Keepa: ${data.products?.length ?? 0} products returned for UPC query, ${tokensLeft} tokens remaining (refill: ${refillRate}/min)`,
|
||||
);
|
||||
|
||||
const byUpc = new Map<string, Map<string, KeepaData>>();
|
||||
for (const product of data.products ?? []) {
|
||||
const asin = String(product.asin ?? "").trim();
|
||||
if (!asin) continue;
|
||||
const byUpc = new Map<string, Map<string, KeepaData>>();
|
||||
for (const product of data.products ?? []) {
|
||||
const asin = normalizeAsin(product.asin);
|
||||
if (!asin) continue;
|
||||
|
||||
const keepaData = parseKeepaProduct(product);
|
||||
const productUpcs = extractUpcsFromProduct(product);
|
||||
@@ -1,5 +1,5 @@
|
||||
import { config } from "./config.ts";
|
||||
import type { EnrichedProduct, LlmVerdict } from "./types.ts";
|
||||
import { config } from "../config.ts";
|
||||
import type { EnrichedProduct, LlmVerdict } from "../types.ts";
|
||||
|
||||
const SYSTEM_PROMPT_STRICT = `You are an expert Amazon product analyst specializing in FBA and FBM fulfillment strategy.
|
||||
|
||||
@@ -13,6 +13,7 @@ afterAll(() => {
|
||||
|
||||
test("normalizeAsin uppercases and validates ASINs", () => {
|
||||
expect(normalizeAsin(" b07sn9bhvv ")).toBe("B07SN9BHVV");
|
||||
expect(normalizeAsin("0306406152")).toBe("0306406152");
|
||||
expect(() => normalizeAsin("not-an-asin")).toThrow("Invalid ASIN");
|
||||
});
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
import { normalizeAsin as normalizeCanonicalAsin } from "../asin.ts";
|
||||
|
||||
const DEFAULT_SEARXNG_URL = "https://searxng.nvictor.me/";
|
||||
const DEFAULT_GOOGLE_CUSTOM_SEARCH_URL =
|
||||
"https://www.googleapis.com/customsearch/v1";
|
||||
const DEFAULT_SERPAPI_URL = "https://serpapi.com/search.json";
|
||||
const DEFAULT_TIMEOUT_MS = 10_000;
|
||||
const DEFAULT_MAX_RESULTS = 10;
|
||||
const ASIN_REGEX = /^B[0-9A-Z]{9}$/;
|
||||
const ASIN_MATCH_REGEX = /\bB[0-9A-Z]{9}\b/gi;
|
||||
const PRICE_LABELS = [
|
||||
"selling price",
|
||||
@@ -127,16 +128,15 @@ export async function searchProductOffers(
|
||||
}
|
||||
|
||||
export function normalizeAsin(value: string): string {
|
||||
const asin = value.trim().toUpperCase();
|
||||
if (!ASIN_REGEX.test(asin)) {
|
||||
const asin = normalizeCanonicalAsin(value);
|
||||
if (!asin) {
|
||||
throw new Error(`Invalid ASIN: ${value}`);
|
||||
}
|
||||
return asin;
|
||||
}
|
||||
|
||||
function getAsinQuery(value: string): string | undefined {
|
||||
const normalized = value.trim().toUpperCase();
|
||||
return ASIN_REGEX.test(normalized) ? normalized : undefined;
|
||||
return normalizeCanonicalAsin(value) ?? undefined;
|
||||
}
|
||||
|
||||
async function fetchSearxngResults(
|
||||
@@ -20,6 +20,15 @@ test("parseCatalogUpcLookupResponse marks no match", () => {
|
||||
expect(detail.asin).toBeNull();
|
||||
});
|
||||
|
||||
test("parseCatalogUpcLookupResponse ignores invalid ASIN identifiers", () => {
|
||||
const detail = parseCatalogUpcLookupResponse("012345678901", {
|
||||
items: [{ asin: "012345678901" }],
|
||||
});
|
||||
|
||||
expect(detail.status).toBe("not_found");
|
||||
expect(detail.asin).toBeNull();
|
||||
});
|
||||
|
||||
test("parseCatalogUpcLookupResponse marks multiple ASINs", () => {
|
||||
const detail = parseCatalogUpcLookupResponse("012345678901", {
|
||||
payload: {
|
||||
@@ -1,11 +1,12 @@
|
||||
import { SellingPartner } from "amazon-sp-api";
|
||||
import { config } from "./config.ts";
|
||||
import { normalizeAsin } from "../asin.ts";
|
||||
import { config } from "../config.ts";
|
||||
import type {
|
||||
KeepaUpcLookupStatus,
|
||||
SpApiData,
|
||||
SellabilityInfo,
|
||||
UpcLookupDetail,
|
||||
} from "./types.ts";
|
||||
} from "../types.ts";
|
||||
|
||||
type RegionCode = "na" | "eu" | "fe";
|
||||
|
||||
@@ -222,8 +223,7 @@ function extractCatalogAsin(item: any): string | null {
|
||||
item?.identifiers?.marketplaceASIN?.asin ??
|
||||
item?.Identifiers?.MarketplaceASIN?.ASIN;
|
||||
if (typeof raw !== "string") return null;
|
||||
const asin = raw.trim().toUpperCase();
|
||||
return asin ? asin : null;
|
||||
return normalizeAsin(raw);
|
||||
}
|
||||
|
||||
export function parseCatalogUpcLookupResponse(
|
||||
@@ -1,8 +1,7 @@
|
||||
import * as XLSX from "xlsx";
|
||||
import type { ProductRecord } from "./types.ts";
|
||||
|
||||
const ASIN_REGEX = /^B[0-9A-Z]{9}$/;
|
||||
|
||||
import * as XLSX from "xlsx";
|
||||
import { normalizeAsin } from "./asin.ts";
|
||||
import type { ProductRecord } from "./types.ts";
|
||||
|
||||
const COLUMN_CANDIDATES = {
|
||||
asin: ["asin"],
|
||||
name: ["name", "product name", "title", "product title"],
|
||||
@@ -132,14 +131,12 @@ function getKnownColumns(columns: ColumnMap): Set<string> {
|
||||
return new Set(Object.values(columns).filter((column): column is string => !!column));
|
||||
}
|
||||
|
||||
function parseAsin(value: unknown): string | undefined {
|
||||
const asin = String(value ?? "")
|
||||
.trim()
|
||||
.toUpperCase();
|
||||
if (!asin || !ASIN_REGEX.test(asin)) {
|
||||
console.warn(`Skipping invalid ASIN: "${asin}"`);
|
||||
return undefined;
|
||||
}
|
||||
function parseAsin(value: unknown): string | undefined {
|
||||
const asin = normalizeAsin(value);
|
||||
if (!asin) {
|
||||
console.warn(`Skipping invalid ASIN: "${String(value ?? "").trim()}"`);
|
||||
return undefined;
|
||||
}
|
||||
return asin;
|
||||
}
|
||||
|
||||
|
||||
2613
src/server.ts
2613
src/server.ts
File diff suppressed because it is too large
Load Diff
@@ -1,4 +1,4 @@
|
||||
import { testSpApiConnectivity, testSpApiSellability } from "./sp-api.ts";
|
||||
import { testSpApiConnectivity, testSpApiSellability } from "./integrations/sp-api.ts";
|
||||
|
||||
function parseArgs(): { asin?: string; sellabilityMode: boolean } {
|
||||
const args = process.argv.slice(2);
|
||||
|
||||
@@ -1,377 +0,0 @@
|
||||
import { type Database, closeDb, getDb, initDb } from "./database.ts";
|
||||
import { analyzeProducts } from "./llm.ts";
|
||||
import { fetchSpApiPricingAndFees } from "./sp-api.ts";
|
||||
import type {
|
||||
AnalysisResult,
|
||||
EnrichedProduct,
|
||||
KeepaData,
|
||||
ProductRecord,
|
||||
SellabilityInfo,
|
||||
} from "./types.ts";
|
||||
|
||||
const LLM_BATCH_SIZE = 5;
|
||||
const LLM_BATCH_DELAY_MS = 5_000;
|
||||
|
||||
type Args = {
|
||||
dbPath: string;
|
||||
stalkerRunId: number;
|
||||
analysisRunId: number;
|
||||
asins: string[];
|
||||
useClaude: boolean;
|
||||
};
|
||||
|
||||
type InventoryRow = {
|
||||
asin: string;
|
||||
product_title: string | null;
|
||||
brand: string | null;
|
||||
category_tree: string | null;
|
||||
current_price: number | null;
|
||||
avg_price_90d: number | null;
|
||||
sales_rank: number | null;
|
||||
monthly_sold: number | null;
|
||||
seller_count: number | null;
|
||||
amazon_is_seller: number | null;
|
||||
can_sell: number | null;
|
||||
sellability_status: SellabilityInfo["sellabilityStatus"] | null;
|
||||
sellability_reason: string | null;
|
||||
};
|
||||
|
||||
function readFlagValue(args: string[], flag: string): string | undefined {
|
||||
const index = args.indexOf(flag);
|
||||
if (index === -1) return undefined;
|
||||
return args[index + 1];
|
||||
}
|
||||
|
||||
function parseArgs(argv = process.argv.slice(2)): Args {
|
||||
const dbPath = readFlagValue(argv, "--db");
|
||||
const stalkerRunId = Number(readFlagValue(argv, "--stalker-run-id"));
|
||||
const analysisRunId = Number(readFlagValue(argv, "--analysis-run-id"));
|
||||
const useClaude = argv.includes("--claude");
|
||||
const asins = (readFlagValue(argv, "--asins") ?? "")
|
||||
.split(",")
|
||||
.map((asin) => asin.trim().toUpperCase())
|
||||
.filter(Boolean);
|
||||
|
||||
if (!dbPath) throw new Error("Missing --db");
|
||||
if (!Number.isInteger(stalkerRunId) || stalkerRunId <= 0) {
|
||||
throw new Error("--stalker-run-id must be a positive integer");
|
||||
}
|
||||
if (!Number.isInteger(analysisRunId) || analysisRunId <= 0) {
|
||||
throw new Error("--analysis-run-id must be a positive integer");
|
||||
}
|
||||
if (asins.length === 0) throw new Error("Missing --asins");
|
||||
|
||||
return { dbPath, stalkerRunId, analysisRunId, asins, useClaude };
|
||||
}
|
||||
|
||||
function wait(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function parseCategoryTree(value: string | null): string[] {
|
||||
if (!value) return [];
|
||||
try {
|
||||
const parsed = JSON.parse(value);
|
||||
return Array.isArray(parsed)
|
||||
? parsed.filter((item): item is string => typeof item === "string")
|
||||
: [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function toProductRecord(row: InventoryRow): ProductRecord {
|
||||
const categoryTree = parseCategoryTree(row.category_tree);
|
||||
return {
|
||||
asin: row.asin,
|
||||
name: row.product_title ?? row.asin,
|
||||
brand: row.brand ?? undefined,
|
||||
category: categoryTree.join(" > ") || undefined,
|
||||
unitCost: 0,
|
||||
amazonRank: row.sales_rank ?? undefined,
|
||||
sellingPriceFromSheet: row.current_price ?? undefined,
|
||||
avgPrice90FromSheet: row.avg_price_90d ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function toKeepaData(row: InventoryRow): KeepaData {
|
||||
return {
|
||||
currentPrice: row.current_price,
|
||||
avgPrice90: row.avg_price_90d,
|
||||
minPrice90: null,
|
||||
maxPrice90: null,
|
||||
salesRank: row.sales_rank,
|
||||
salesRankAvg90: null,
|
||||
salesRankDrops30: null,
|
||||
salesRankDrops90: null,
|
||||
sellerCount: row.seller_count,
|
||||
amazonIsSeller:
|
||||
row.amazon_is_seller == null ? null : row.amazon_is_seller === 1,
|
||||
amazonBuyboxSharePct90d: null,
|
||||
buyBoxSeller: null,
|
||||
buyBoxPrice: null,
|
||||
buyBoxAvg90: null,
|
||||
monthlySold: row.monthly_sold,
|
||||
categoryTree: parseCategoryTree(row.category_tree),
|
||||
};
|
||||
}
|
||||
|
||||
function toSellability(row: InventoryRow): SellabilityInfo {
|
||||
return {
|
||||
canSell: row.can_sell == null ? null : row.can_sell === 1,
|
||||
sellabilityStatus: row.sellability_status ?? "unknown",
|
||||
sellabilityReason: row.sellability_reason ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function loadInventoryRows(
|
||||
database: Database,
|
||||
stalkerRunId: number,
|
||||
asins: string[],
|
||||
): InventoryRow[] {
|
||||
const placeholders = asins.map(() => "?").join(",");
|
||||
return database
|
||||
.query(
|
||||
`SELECT
|
||||
asin, product_title, brand, category_tree, current_price, avg_price_90d,
|
||||
sales_rank, monthly_sold, seller_count, amazon_is_seller, can_sell,
|
||||
sellability_status, sellability_reason
|
||||
FROM stalker_seller_inventory
|
||||
WHERE run_id = ?
|
||||
AND can_sell = 1
|
||||
AND sellability_status = 'available'
|
||||
AND asin IN (${placeholders})
|
||||
GROUP BY asin`,
|
||||
)
|
||||
.all(stalkerRunId, ...asins) as InventoryRow[];
|
||||
}
|
||||
|
||||
async function buildEnrichedProducts(
|
||||
rows: InventoryRow[],
|
||||
): Promise<EnrichedProduct[]> {
|
||||
const enriched: EnrichedProduct[] = [];
|
||||
|
||||
for (const row of rows) {
|
||||
const sellability = toSellability(row);
|
||||
const spApi = await fetchSpApiPricingAndFees(
|
||||
row.asin,
|
||||
sellability,
|
||||
row.current_price,
|
||||
);
|
||||
|
||||
enriched.push({
|
||||
record: toProductRecord(row),
|
||||
keepa: toKeepaData(row),
|
||||
spApi,
|
||||
fetchedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
return enriched;
|
||||
}
|
||||
|
||||
function insertProductAnalysisResults(
|
||||
database: Database,
|
||||
runId: number,
|
||||
results: AnalysisResult[],
|
||||
): void {
|
||||
if (results.length === 0) return;
|
||||
|
||||
const insert = database.prepare(`
|
||||
INSERT INTO product_analysis_results (
|
||||
asin, run_id, name, brand, category, unit_cost,
|
||||
current_price, avg_price_90d, avg_price_90d_sheet,
|
||||
selling_price_sheet, sales_rank, sales_rank_avg_90d,
|
||||
seller_count, amazon_is_seller, amazon_buybox_share_pct_90d,
|
||||
monthly_sold, rank_drops_30d, rank_drops_90d,
|
||||
fba_fee, fbm_fee, referral_percent, can_sell,
|
||||
sellability_status, sellability_reason,
|
||||
verdict, confidence, reasoning, fetched_at
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?
|
||||
)
|
||||
ON CONFLICT(asin) DO UPDATE SET
|
||||
run_id = excluded.run_id,
|
||||
name = excluded.name,
|
||||
brand = excluded.brand,
|
||||
category = excluded.category,
|
||||
unit_cost = excluded.unit_cost,
|
||||
current_price = excluded.current_price,
|
||||
avg_price_90d = excluded.avg_price_90d,
|
||||
avg_price_90d_sheet = excluded.avg_price_90d_sheet,
|
||||
selling_price_sheet = excluded.selling_price_sheet,
|
||||
sales_rank = excluded.sales_rank,
|
||||
sales_rank_avg_90d = excluded.sales_rank_avg_90d,
|
||||
seller_count = excluded.seller_count,
|
||||
amazon_is_seller = excluded.amazon_is_seller,
|
||||
amazon_buybox_share_pct_90d = excluded.amazon_buybox_share_pct_90d,
|
||||
monthly_sold = excluded.monthly_sold,
|
||||
rank_drops_30d = excluded.rank_drops_30d,
|
||||
rank_drops_90d = excluded.rank_drops_90d,
|
||||
fba_fee = excluded.fba_fee,
|
||||
fbm_fee = excluded.fbm_fee,
|
||||
referral_percent = excluded.referral_percent,
|
||||
can_sell = excluded.can_sell,
|
||||
sellability_status = excluded.sellability_status,
|
||||
sellability_reason = excluded.sellability_reason,
|
||||
verdict = excluded.verdict,
|
||||
confidence = excluded.confidence,
|
||||
reasoning = excluded.reasoning,
|
||||
fetched_at = excluded.fetched_at
|
||||
`);
|
||||
|
||||
database.transaction((batch: AnalysisResult[]) => {
|
||||
for (const result of batch) {
|
||||
const keepa = result.product.keepa;
|
||||
const record = result.product.record;
|
||||
const spApi = result.product.spApi;
|
||||
insert.run(
|
||||
record.asin,
|
||||
runId,
|
||||
record.name,
|
||||
record.brand ?? null,
|
||||
record.category ?? keepa?.categoryTree.join(" > ") ?? null,
|
||||
record.unitCost ?? null,
|
||||
keepa?.currentPrice ?? spApi.estimatedSalePrice ?? null,
|
||||
keepa?.avgPrice90 ?? null,
|
||||
record.avgPrice90FromSheet ?? null,
|
||||
record.sellingPriceFromSheet ?? null,
|
||||
keepa?.salesRank ?? record.amazonRank ?? null,
|
||||
keepa?.salesRankAvg90 ?? null,
|
||||
keepa?.sellerCount ?? null,
|
||||
keepa?.amazonIsSeller == null ? null : keepa.amazonIsSeller ? 1 : 0,
|
||||
keepa?.amazonBuyboxSharePct90d ?? null,
|
||||
keepa?.monthlySold ?? null,
|
||||
keepa?.salesRankDrops30 ?? null,
|
||||
keepa?.salesRankDrops90 ?? null,
|
||||
spApi.fbaFee ?? null,
|
||||
spApi.fbmFee ?? null,
|
||||
spApi.referralFeePercent ?? null,
|
||||
spApi.canSell == null ? "unknown" : spApi.canSell ? "yes" : "no",
|
||||
spApi.sellabilityStatus ?? null,
|
||||
spApi.sellabilityReason ?? null,
|
||||
result.verdict.verdict,
|
||||
result.verdict.confidence,
|
||||
result.verdict.reasoning ?? null,
|
||||
result.product.fetchedAt,
|
||||
);
|
||||
}
|
||||
})(results);
|
||||
}
|
||||
|
||||
function refreshAnalysisRun(database: Database, runId: number): void {
|
||||
const stats = database
|
||||
.query(
|
||||
`SELECT
|
||||
COUNT(*) AS total,
|
||||
SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba,
|
||||
SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm,
|
||||
SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip
|
||||
FROM product_analysis_results
|
||||
WHERE run_id = ?`,
|
||||
)
|
||||
.get(runId) as {
|
||||
total: number;
|
||||
fba: number | null;
|
||||
fbm: number | null;
|
||||
skip: number | null;
|
||||
};
|
||||
|
||||
database
|
||||
.prepare(
|
||||
`UPDATE category_analysis_runs
|
||||
SET top_asins_checked = ?,
|
||||
available_asins = ?,
|
||||
fba_count = ?,
|
||||
fbm_count = ?,
|
||||
skip_count = ?
|
||||
WHERE id = ?`,
|
||||
)
|
||||
.run(
|
||||
stats.total ?? 0,
|
||||
stats.total ?? 0,
|
||||
stats.fba ?? 0,
|
||||
stats.fbm ?? 0,
|
||||
stats.skip ?? 0,
|
||||
runId,
|
||||
);
|
||||
}
|
||||
|
||||
async function analyzeInBatches(
|
||||
products: EnrichedProduct[],
|
||||
useClaude: boolean,
|
||||
): Promise<AnalysisResult[]> {
|
||||
const results: AnalysisResult[] = [];
|
||||
|
||||
for (let i = 0; i < products.length; i += LLM_BATCH_SIZE) {
|
||||
const batch = products.slice(i, i + LLM_BATCH_SIZE);
|
||||
const batchNumber = Math.floor(i / LLM_BATCH_SIZE) + 1;
|
||||
const totalBatches = Math.ceil(products.length / LLM_BATCH_SIZE);
|
||||
console.log(
|
||||
`Stalker analysis: LLM batch ${batchNumber}/${totalBatches} (${batch.length} product(s)).`,
|
||||
);
|
||||
|
||||
if (i > 0) {
|
||||
await wait(LLM_BATCH_DELAY_MS);
|
||||
}
|
||||
|
||||
let verdicts;
|
||||
try {
|
||||
verdicts = await analyzeProducts(batch, { useClaude });
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
`Stalker analysis: LLM batch ${batchNumber} failed: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`,
|
||||
);
|
||||
verdicts = null;
|
||||
}
|
||||
|
||||
for (let j = 0; j < batch.length; j++) {
|
||||
const product = batch[j];
|
||||
if (!product) continue;
|
||||
results.push({
|
||||
product,
|
||||
verdict: verdicts?.[j] ?? {
|
||||
asin: product.record.asin,
|
||||
verdict: "SKIP",
|
||||
confidence: 0,
|
||||
reasoning: "LLM analysis failed or returned no verdict",
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = parseArgs();
|
||||
initDb(args.dbPath);
|
||||
const database = getDb(args.dbPath);
|
||||
|
||||
try {
|
||||
const rows = loadInventoryRows(database, args.stalkerRunId, args.asins);
|
||||
if (rows.length === 0) {
|
||||
console.log("Stalker analysis: no sellable inventory rows to analyze.");
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`Stalker analysis: analyzing ${rows.length} sellable ASIN(s).`);
|
||||
const enriched = await buildEnrichedProducts(rows);
|
||||
const results = await analyzeInBatches(enriched, args.useClaude);
|
||||
insertProductAnalysisResults(database, args.analysisRunId, results);
|
||||
refreshAnalysisRun(database, args.analysisRunId);
|
||||
} finally {
|
||||
closeDb();
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
268
src/stalker/stalker-analyze.ts
Normal file
268
src/stalker/stalker-analyze.ts
Normal file
@@ -0,0 +1,268 @@
|
||||
import { db } from "../db/index.ts";
|
||||
import { persistLlmResults, refreshRunStats } from "../db/persistence.ts";
|
||||
import { sql } from "drizzle-orm";
|
||||
import { normalizeAsin } from "../asin.ts";
|
||||
import { analyzeProducts } from "../integrations/llm.ts";
|
||||
import { fetchSpApiPricingAndFees } from "../integrations/sp-api.ts";
|
||||
import type {
|
||||
AnalysisResult,
|
||||
EnrichedProduct,
|
||||
KeepaData,
|
||||
ProductRecord,
|
||||
SellabilityInfo,
|
||||
} from "../types.ts";
|
||||
|
||||
const LLM_BATCH_SIZE = 5;
|
||||
const LLM_BATCH_DELAY_MS = 5_000;
|
||||
|
||||
type Args = {
|
||||
stalkerRunId: number;
|
||||
analysisRunId: number;
|
||||
asins: string[];
|
||||
useClaude: boolean;
|
||||
};
|
||||
|
||||
type InventoryRow = {
|
||||
inventoryItemId: number;
|
||||
asin: string;
|
||||
productTitle: string | null;
|
||||
brand: string | null;
|
||||
categoryTree: string | null;
|
||||
currentPrice: number | null;
|
||||
avgPrice90d: number | null;
|
||||
salesRank: number | null;
|
||||
monthlySold: number | null;
|
||||
sellerCount: number | null;
|
||||
amazonIsSeller: boolean | null;
|
||||
canSell: boolean | null;
|
||||
sellabilityStatus: SellabilityInfo["sellabilityStatus"] | null;
|
||||
sellabilityReason: string | null;
|
||||
};
|
||||
|
||||
function readFlagValue(args: string[], flag: string): string | undefined {
|
||||
const index = args.indexOf(flag);
|
||||
if (index === -1) return undefined;
|
||||
return args[index + 1];
|
||||
}
|
||||
|
||||
function parseArgs(argv = process.argv.slice(2)): Args {
|
||||
const stalkerRunId = Number(readFlagValue(argv, "--stalker-run-id"));
|
||||
const analysisRunId = Number(readFlagValue(argv, "--analysis-run-id"));
|
||||
const useClaude = argv.includes("--claude");
|
||||
const asins = (readFlagValue(argv, "--asins") ?? "")
|
||||
.split(",")
|
||||
.map((asin) => normalizeAsin(asin))
|
||||
.filter((asin): asin is string => asin !== null);
|
||||
|
||||
if (!Number.isInteger(stalkerRunId) || stalkerRunId <= 0) {
|
||||
throw new Error("--stalker-run-id must be a positive integer");
|
||||
}
|
||||
if (!Number.isInteger(analysisRunId) || analysisRunId <= 0) {
|
||||
throw new Error("--analysis-run-id must be a positive integer");
|
||||
}
|
||||
if (asins.length === 0) throw new Error("Missing --asins");
|
||||
|
||||
return { stalkerRunId, analysisRunId, asins, useClaude };
|
||||
}
|
||||
|
||||
function wait(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function parseCategoryTree(value: string | null): string[] {
|
||||
return value ? value.split(" > ").filter(Boolean) : [];
|
||||
}
|
||||
|
||||
function toProductRecord(row: InventoryRow): ProductRecord {
|
||||
const categoryTree = parseCategoryTree(row.categoryTree);
|
||||
return {
|
||||
asin: row.asin,
|
||||
name: row.productTitle ?? row.asin,
|
||||
brand: row.brand ?? undefined,
|
||||
category: categoryTree.join(" > ") || undefined,
|
||||
unitCost: 0,
|
||||
amazonRank: row.salesRank ?? undefined,
|
||||
sellingPriceFromSheet: row.currentPrice ?? undefined,
|
||||
avgPrice90FromSheet: row.avgPrice90d ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function toKeepaData(row: InventoryRow): KeepaData {
|
||||
return {
|
||||
currentPrice: row.currentPrice,
|
||||
avgPrice90: row.avgPrice90d,
|
||||
minPrice90: null,
|
||||
maxPrice90: null,
|
||||
salesRank: row.salesRank,
|
||||
salesRankAvg90: null,
|
||||
salesRankDrops30: null,
|
||||
salesRankDrops90: null,
|
||||
sellerCount: row.sellerCount,
|
||||
amazonIsSeller: row.amazonIsSeller,
|
||||
amazonBuyboxSharePct90d: null,
|
||||
buyBoxSeller: null,
|
||||
buyBoxPrice: null,
|
||||
buyBoxAvg90: null,
|
||||
monthlySold: row.monthlySold,
|
||||
categoryTree: parseCategoryTree(row.categoryTree),
|
||||
};
|
||||
}
|
||||
|
||||
function toSellability(row: InventoryRow): SellabilityInfo {
|
||||
return {
|
||||
canSell: row.canSell,
|
||||
sellabilityStatus: row.sellabilityStatus ?? "unknown",
|
||||
sellabilityReason: row.sellabilityReason ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
async function loadInventoryRows(
|
||||
stalkerRunId: number,
|
||||
asins: string[],
|
||||
): Promise<InventoryRow[]> {
|
||||
if (asins.length === 0) return [];
|
||||
return db.execute(
|
||||
sql<InventoryRow>`SELECT DISTINCT ON (inventory.product_asin)
|
||||
inventory.id AS "inventoryItemId",
|
||||
inventory.product_asin AS asin,
|
||||
product.name AS "productTitle",
|
||||
product.brand,
|
||||
product.category AS "categoryTree",
|
||||
observation.current_price AS "currentPrice",
|
||||
observation.avg_price_90d AS "avgPrice90d",
|
||||
observation.sales_rank AS "salesRank",
|
||||
observation.monthly_sold AS "monthlySold",
|
||||
observation.seller_count AS "sellerCount",
|
||||
observation.amazon_is_seller AS "amazonIsSeller",
|
||||
observation.can_sell AS "canSell",
|
||||
observation.sellability_status AS "sellabilityStatus",
|
||||
observation.sellability_reason AS "sellabilityReason"
|
||||
FROM stalker_inventory_items inventory
|
||||
JOIN products product ON product.asin = inventory.product_asin
|
||||
JOIN product_observations observation ON observation.id = inventory.observation_id
|
||||
WHERE inventory.run_id = ${stalkerRunId}
|
||||
AND observation.can_sell = true
|
||||
AND observation.sellability_status = 'available'
|
||||
AND inventory.product_asin = ANY(${asins})
|
||||
ORDER BY inventory.product_asin, observation.fetched_at DESC`,
|
||||
);
|
||||
}
|
||||
|
||||
async function buildEnrichedProducts(
|
||||
rows: InventoryRow[],
|
||||
): Promise<EnrichedProduct[]> {
|
||||
const enriched: EnrichedProduct[] = [];
|
||||
|
||||
for (const row of rows) {
|
||||
const sellability = toSellability(row);
|
||||
const spApi = await fetchSpApiPricingAndFees(
|
||||
row.asin,
|
||||
sellability,
|
||||
row.currentPrice,
|
||||
);
|
||||
|
||||
enriched.push({
|
||||
record: toProductRecord(row),
|
||||
keepa: toKeepaData(row),
|
||||
spApi,
|
||||
fetchedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
return enriched;
|
||||
}
|
||||
|
||||
async function insertProductAnalysisResults(
|
||||
runId: number,
|
||||
results: AnalysisResult[],
|
||||
sourceInventoryIds: Map<string, number>,
|
||||
): Promise<void> {
|
||||
if (results.length === 0) return;
|
||||
await persistLlmResults(runId, results, {
|
||||
source: "stalker_analysis",
|
||||
metadataSource: "catalog",
|
||||
sourceInventoryIds,
|
||||
});
|
||||
}
|
||||
|
||||
async function refreshAnalysisRun(runId: number): Promise<void> {
|
||||
await refreshRunStats(runId);
|
||||
}
|
||||
|
||||
async function analyzeInBatches(
|
||||
products: EnrichedProduct[],
|
||||
useClaude: boolean,
|
||||
): Promise<AnalysisResult[]> {
|
||||
const results: AnalysisResult[] = [];
|
||||
|
||||
for (let i = 0; i < products.length; i += LLM_BATCH_SIZE) {
|
||||
const batch = products.slice(i, i + LLM_BATCH_SIZE);
|
||||
const batchNumber = Math.floor(i / LLM_BATCH_SIZE) + 1;
|
||||
const totalBatches = Math.ceil(products.length / LLM_BATCH_SIZE);
|
||||
console.log(
|
||||
`Stalker analysis: LLM batch ${batchNumber}/${totalBatches} (${batch.length} product(s)).`,
|
||||
);
|
||||
|
||||
if (i > 0) {
|
||||
await wait(LLM_BATCH_DELAY_MS);
|
||||
}
|
||||
|
||||
let verdicts;
|
||||
try {
|
||||
verdicts = await analyzeProducts(batch, { useClaude });
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
`Stalker analysis: LLM batch ${batchNumber} failed: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`,
|
||||
);
|
||||
verdicts = null;
|
||||
}
|
||||
|
||||
for (let j = 0; j < batch.length; j++) {
|
||||
const product = batch[j];
|
||||
if (!product) continue;
|
||||
results.push({
|
||||
product,
|
||||
verdict: verdicts?.[j] ?? {
|
||||
asin: product.record.asin,
|
||||
verdict: "SKIP",
|
||||
confidence: 0,
|
||||
reasoning: "LLM analysis failed or returned no verdict",
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = parseArgs();
|
||||
|
||||
const rows = await loadInventoryRows(args.stalkerRunId, args.asins);
|
||||
if (rows.length === 0) {
|
||||
console.log("Stalker analysis: no sellable inventory rows to analyze.");
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`Stalker analysis: analyzing ${rows.length} sellable ASIN(s).`);
|
||||
const enriched = await buildEnrichedProducts(rows);
|
||||
const results = await analyzeInBatches(enriched, args.useClaude);
|
||||
const sourceInventoryIds = new Map(
|
||||
rows.map((row) => [row.asin, row.inventoryItemId]),
|
||||
);
|
||||
await insertProductAnalysisResults(
|
||||
args.analysisRunId,
|
||||
results,
|
||||
sourceInventoryIds,
|
||||
);
|
||||
await refreshAnalysisRun(args.analysisRunId);
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -2,7 +2,67 @@ import { afterAll, beforeEach, expect, mock, test } from "bun:test";
|
||||
import { mkdirSync, rmSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import * as XLSX from "xlsx";
|
||||
import { closeDb, getDb } from "./database.ts";
|
||||
|
||||
let nextId = 0;
|
||||
|
||||
function chainable(resolveWith: any[] = []): any {
|
||||
const p: any = Promise.resolve(resolveWith);
|
||||
p.limit = (_n: any) => chainable(resolveWith);
|
||||
p.where = (_cond: any) => chainable(resolveWith);
|
||||
p.from = (_table: any) => chainable(resolveWith);
|
||||
return p;
|
||||
}
|
||||
|
||||
const makeMockTx = (): any => ({
|
||||
insert: (_table: any) => ({
|
||||
values: (_vals: any) => ({
|
||||
returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]),
|
||||
onConflictDoUpdate: (_conf: any) => Promise.resolve([]),
|
||||
}),
|
||||
}),
|
||||
update: (_table: any) => ({
|
||||
set: (_vals: any) => ({
|
||||
where: (_cond: any) => Promise.resolve([]),
|
||||
}),
|
||||
}),
|
||||
select: (_sel?: any) => ({
|
||||
from: (_table: any) => ({
|
||||
where: (_cond: any) => chainable([{ id: ++nextId }]),
|
||||
limit: (_n: any) => chainable([{ id: nextId }]),
|
||||
}),
|
||||
}),
|
||||
selectDistinct: (_sel: any) => ({
|
||||
from: (_table: any) => chainable([]),
|
||||
}),
|
||||
execute: (_query: any) => Promise.resolve([]),
|
||||
});
|
||||
|
||||
const makeMockDb = (): any => ({
|
||||
insert: (_table: any) => ({
|
||||
values: (_vals: any) => ({
|
||||
returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]),
|
||||
onConflictDoUpdate: (_conf: any) => Promise.resolve([]),
|
||||
}),
|
||||
}),
|
||||
update: (_table: any) => ({
|
||||
set: (_vals: any) => ({
|
||||
where: (_cond: any) => Promise.resolve([]),
|
||||
}),
|
||||
}),
|
||||
select: (_sel?: any) => ({
|
||||
from: (_table: any) => ({
|
||||
where: (_cond: any) => chainable(),
|
||||
limit: (_n: any) => chainable(),
|
||||
}),
|
||||
}),
|
||||
selectDistinct: (_sel: any) => ({
|
||||
from: (_table: any) => chainable(),
|
||||
}),
|
||||
execute: (_query: any) => Promise.resolve([]),
|
||||
transaction: async (fn: (tx: any) => Promise<any>) => fn(makeMockTx()),
|
||||
});
|
||||
|
||||
mock.module("../db/index.ts", () => ({ db: makeMockDb(), client: {} }));
|
||||
|
||||
const TEST_DIR = path.join(process.cwd(), "test_output", "stalker-sellability");
|
||||
const originalFetch = globalThis.fetch;
|
||||
@@ -27,14 +87,10 @@ const fetchSellabilityBatchMock = mock(async (asins: string[]) => {
|
||||
);
|
||||
});
|
||||
|
||||
mock.module("./sp-api.ts", () => ({
|
||||
fetchSellabilityBatch: fetchSellabilityBatchMock,
|
||||
}));
|
||||
|
||||
const modulePromise = import("./stalker.ts");
|
||||
|
||||
beforeEach(() => {
|
||||
closeDb();
|
||||
nextId = 0;
|
||||
rmSync(TEST_DIR, { recursive: true, force: true });
|
||||
mkdirSync(TEST_DIR, { recursive: true });
|
||||
globalThis.fetch = originalFetch;
|
||||
@@ -49,14 +105,12 @@ afterAll(() => {
|
||||
} else {
|
||||
Bun.env.KEEPA_API_KEY = originalKeepaKey;
|
||||
}
|
||||
closeDb();
|
||||
rmSync(TEST_DIR, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("sellability checks matched seller inventory, not the source ASIN", async () => {
|
||||
const { runStalker } = await modulePromise;
|
||||
const inputPath = path.join(TEST_DIR, "input.xlsx");
|
||||
const dbPath = path.join(TEST_DIR, "stalker.sqlite");
|
||||
const workbook = XLSX.utils.book_new();
|
||||
XLSX.utils.book_append_sheet(
|
||||
workbook,
|
||||
@@ -136,22 +190,25 @@ test("sellability checks matched seller inventory, not the source ASIN", async (
|
||||
return new Response("not found", { status: 404 });
|
||||
}) as unknown as typeof globalThis.fetch;
|
||||
|
||||
const stats = await runStalker({
|
||||
input: inputPath,
|
||||
dbPath,
|
||||
maxAsins: null,
|
||||
storefrontUpdateHours: 168,
|
||||
offerLimit: 20,
|
||||
sellerLimit: 30,
|
||||
inventoryLimit: 200,
|
||||
sellerCacheHours: 168,
|
||||
includeStock: false,
|
||||
dryRun: false,
|
||||
resume: true,
|
||||
maxSellerRequests: null,
|
||||
sellability: true,
|
||||
analyzeSellable: false,
|
||||
});
|
||||
const stats = await runStalker(
|
||||
{
|
||||
input: inputPath,
|
||||
maxAsins: null,
|
||||
storefrontUpdateHours: 168,
|
||||
offerLimit: 20,
|
||||
sellerLimit: 30,
|
||||
inventoryLimit: 200,
|
||||
sellerCacheHours: 168,
|
||||
includeStock: false,
|
||||
dryRun: false,
|
||||
resume: true,
|
||||
maxSellerRequests: null,
|
||||
sellability: true,
|
||||
analyzeSellable: false,
|
||||
useClaude: false,
|
||||
},
|
||||
{ fetchSellabilityBatch: fetchSellabilityBatchMock },
|
||||
);
|
||||
|
||||
expect(fetchSellabilityBatchMock.mock.calls.length).toBe(1);
|
||||
expect(fetchSellabilityBatchMock.mock.calls[0]?.[0]).toEqual([
|
||||
@@ -162,46 +219,4 @@ test("sellability checks matched seller inventory, not the source ASIN", async (
|
||||
expect(stats.inventorySellabilityAvailableAsins).toBe(1);
|
||||
expect(stats.inventorySellabilityExcludedAsins).toBe(1);
|
||||
expect(stats.persistedInventoryAsins).toBe(1);
|
||||
|
||||
const db = getDb(dbPath);
|
||||
const scan = db.query("SELECT source_asin FROM stalker_asin_scans").get() as {
|
||||
source_asin: string;
|
||||
};
|
||||
expect(scan.source_asin).toBe("B000000001");
|
||||
|
||||
const inventory = db
|
||||
.query(
|
||||
`SELECT asin, can_sell, sellability_status, product_title, brand,
|
||||
category_tree, current_price, avg_price_90d, sales_rank, monthly_sold,
|
||||
seller_count
|
||||
FROM stalker_seller_inventory ORDER BY asin`,
|
||||
)
|
||||
.all() as Array<{
|
||||
asin: string;
|
||||
can_sell: number | null;
|
||||
sellability_status: string | null;
|
||||
product_title: string | null;
|
||||
brand: string | null;
|
||||
category_tree: string | null;
|
||||
current_price: number | null;
|
||||
avg_price_90d: number | null;
|
||||
sales_rank: number | null;
|
||||
monthly_sold: number | null;
|
||||
seller_count: number | null;
|
||||
}>;
|
||||
expect(inventory).toEqual([
|
||||
{
|
||||
asin: "B111111111",
|
||||
can_sell: 1,
|
||||
sellability_status: "available",
|
||||
product_title: "Sellable Storefront Product",
|
||||
brand: "Good Brand",
|
||||
category_tree: JSON.stringify(["Kitchen", "Storage"]),
|
||||
current_price: 19.99,
|
||||
avg_price_90d: 25,
|
||||
sales_rank: 12345,
|
||||
monthly_sold: 42,
|
||||
seller_count: 7,
|
||||
},
|
||||
]);
|
||||
});
|
||||
@@ -2,7 +2,6 @@ import { afterAll, beforeEach, expect, mock, test } from "bun:test";
|
||||
import { mkdirSync, rmSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import * as XLSX from "xlsx";
|
||||
import { closeDb, getDb, initDb } from "./database.ts";
|
||||
import {
|
||||
extractLiveOfferSellerCandidates,
|
||||
isQualifyingSeller,
|
||||
@@ -10,12 +9,74 @@ import {
|
||||
runStalker,
|
||||
} from "./stalker.ts";
|
||||
|
||||
let nextId = 0;
|
||||
|
||||
function chainable(resolveWith: any[] = []): any {
|
||||
const p: any = Promise.resolve(resolveWith);
|
||||
p.limit = (_n: any) => chainable(resolveWith);
|
||||
p.where = (_cond: any) => chainable(resolveWith);
|
||||
p.from = (_table: any) => chainable(resolveWith);
|
||||
return p;
|
||||
}
|
||||
|
||||
// Transaction mock returns rows for selects (needed for upsert-then-select patterns).
|
||||
const makeMockTx = (): any => ({
|
||||
insert: (_table: any) => ({
|
||||
values: (_vals: any) => ({
|
||||
returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]),
|
||||
onConflictDoUpdate: (_conf: any) => Promise.resolve([]),
|
||||
}),
|
||||
}),
|
||||
update: (_table: any) => ({
|
||||
set: (_vals: any) => ({
|
||||
where: (_cond: any) => Promise.resolve([]),
|
||||
}),
|
||||
}),
|
||||
select: (_sel?: any) => ({
|
||||
from: (_table: any) => ({
|
||||
where: (_cond: any) => chainable([{ id: ++nextId }]),
|
||||
limit: (_n: any) => chainable([{ id: nextId }]),
|
||||
}),
|
||||
}),
|
||||
selectDistinct: (_sel: any) => ({
|
||||
from: (_table: any) => chainable([]),
|
||||
}),
|
||||
execute: (_query: any) => Promise.resolve([]),
|
||||
});
|
||||
|
||||
const makeMockDb = (): any => ({
|
||||
insert: (_table: any) => ({
|
||||
values: (_vals: any) => ({
|
||||
returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]),
|
||||
onConflictDoUpdate: (_conf: any) => Promise.resolve([]),
|
||||
}),
|
||||
}),
|
||||
update: (_table: any) => ({
|
||||
set: (_vals: any) => ({
|
||||
where: (_cond: any) => Promise.resolve([]),
|
||||
}),
|
||||
}),
|
||||
select: (_sel?: any) => ({
|
||||
from: (_table: any) => ({
|
||||
where: (_cond: any) => chainable(),
|
||||
limit: (_n: any) => chainable(),
|
||||
}),
|
||||
}),
|
||||
selectDistinct: (_sel: any) => ({
|
||||
from: (_table: any) => chainable(),
|
||||
}),
|
||||
execute: (_query: any) => Promise.resolve([]),
|
||||
transaction: async (fn: (tx: any) => Promise<any>) => fn(makeMockTx()),
|
||||
});
|
||||
|
||||
mock.module("../db/index.ts", () => ({ db: makeMockDb(), client: {} }));
|
||||
|
||||
const TEST_DIR = path.join(process.cwd(), "test_output", "stalker");
|
||||
const originalFetch = globalThis.fetch;
|
||||
const originalKeepaKey = Bun.env.KEEPA_API_KEY;
|
||||
|
||||
beforeEach(() => {
|
||||
closeDb();
|
||||
nextId = 0;
|
||||
rmSync(TEST_DIR, { recursive: true, force: true });
|
||||
mkdirSync(TEST_DIR, { recursive: true });
|
||||
globalThis.fetch = originalFetch;
|
||||
@@ -29,7 +90,6 @@ afterAll(() => {
|
||||
} else {
|
||||
Bun.env.KEEPA_API_KEY = originalKeepaKey;
|
||||
}
|
||||
closeDb();
|
||||
rmSync(TEST_DIR, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
@@ -41,12 +101,17 @@ test("readAsinsFromXlsx extracts valid ASINs and deduplicates in order", () => {
|
||||
{ ASIN: "invalid" },
|
||||
{ ASIN: "B000000002" },
|
||||
{ ASIN: "B000000001" },
|
||||
{ ASIN: "0306406152" },
|
||||
{ ASIN: "" },
|
||||
]);
|
||||
XLSX.utils.book_append_sheet(workbook, sheet, "Input");
|
||||
XLSX.writeFile(workbook, filePath);
|
||||
|
||||
expect(readAsinsFromXlsx(filePath)).toEqual(["B000000001", "B000000002"]);
|
||||
expect(readAsinsFromXlsx(filePath)).toEqual([
|
||||
"B000000001",
|
||||
"B000000002",
|
||||
"0306406152",
|
||||
]);
|
||||
});
|
||||
|
||||
test("isQualifyingSeller accepts rating counts from 1 to 30 only", () => {
|
||||
@@ -77,35 +142,8 @@ test("extractLiveOfferSellerCandidates ignores Amazon, missing seller IDs, and d
|
||||
expect(offers[0]?.stock).toBe(4);
|
||||
});
|
||||
|
||||
test("initDb creates stalker tables and indexes", () => {
|
||||
const dbPath = path.join(TEST_DIR, "schema.sqlite");
|
||||
initDb(dbPath);
|
||||
const db = getDb(dbPath);
|
||||
|
||||
const tables = db
|
||||
.query(
|
||||
`SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'stalker_%' ORDER BY name`,
|
||||
)
|
||||
.all() as Array<{ name: string }>;
|
||||
expect(tables.map((row) => row.name)).toEqual([
|
||||
"stalker_asin_scans",
|
||||
"stalker_asin_sellers",
|
||||
"stalker_runs",
|
||||
"stalker_seller_inventory",
|
||||
"stalker_sellers",
|
||||
]);
|
||||
|
||||
const indexes = db
|
||||
.query(
|
||||
`SELECT name FROM sqlite_master WHERE type = 'index' AND name LIKE 'idx_stalker_%' ORDER BY name`,
|
||||
)
|
||||
.all() as Array<{ name: string }>;
|
||||
expect(indexes.length).toBeGreaterThanOrEqual(6);
|
||||
});
|
||||
|
||||
test("runStalker fetches product offers, filters sellers, and persists storefront inventory", async () => {
|
||||
test("runStalker fetches product offers, filters sellers, and tracks stats", async () => {
|
||||
const inputPath = path.join(TEST_DIR, "input.xlsx");
|
||||
const dbPath = path.join(TEST_DIR, "stalker.sqlite");
|
||||
const workbook = XLSX.utils.book_new();
|
||||
XLSX.utils.book_append_sheet(
|
||||
workbook,
|
||||
@@ -205,7 +243,6 @@ test("runStalker fetches product offers, filters sellers, and persists storefron
|
||||
|
||||
const stats = await runStalker({
|
||||
input: inputPath,
|
||||
dbPath,
|
||||
maxAsins: null,
|
||||
storefrontUpdateHours: 168,
|
||||
offerLimit: 20,
|
||||
@@ -218,6 +255,7 @@ test("runStalker fetches product offers, filters sellers, and persists storefron
|
||||
maxSellerRequests: null,
|
||||
sellability: false,
|
||||
analyzeSellable: false,
|
||||
useClaude: false,
|
||||
});
|
||||
|
||||
expect(stats.scannedAsins).toBe(1);
|
||||
@@ -229,6 +267,7 @@ test("runStalker fetches product offers, filters sellers, and persists storefron
|
||||
expect(stats.qualifyingSellers).toBe(1);
|
||||
expect(stats.sellerMetadataRequests).toBe(1);
|
||||
expect(stats.sellerStorefrontRequests).toBe(1);
|
||||
|
||||
const sellerCalls = fetchMock.mock.calls.filter((call) => {
|
||||
const rawUrl =
|
||||
typeof call[0] === "string"
|
||||
@@ -239,45 +278,4 @@ test("runStalker fetches product offers, filters sellers, and persists storefron
|
||||
return new URL(rawUrl).pathname === "/seller";
|
||||
});
|
||||
expect(sellerCalls.length).toBe(2);
|
||||
|
||||
const db = getDb(dbPath);
|
||||
const run = db.query("SELECT * FROM stalker_runs").get() as any;
|
||||
expect(run.status).toBe("completed");
|
||||
expect(run.requested_asins).toBe(1);
|
||||
expect(run.scanned_asins).toBe(1);
|
||||
expect(run.source_asins_with_matches).toBe(1);
|
||||
expect(run.candidate_sellers).toBe(2);
|
||||
expect(run.qualifying_sellers).toBe(1);
|
||||
expect(run.matched_sellers).toBe(1);
|
||||
expect(run.seller_metadata_requests).toBe(1);
|
||||
expect(run.seller_storefront_requests).toBe(1);
|
||||
expect(run.inventory_sellability_checked_asins).toBe(0);
|
||||
expect(run.inventory_sellability_available_asins).toBe(0);
|
||||
expect(run.inventory_sellability_excluded_asins).toBe(0);
|
||||
expect(run.persisted_inventory_asins).toBe(0);
|
||||
|
||||
const scan = db.query("SELECT * FROM stalker_asin_scans").get() as any;
|
||||
expect(scan.source_asin).toBe("B000000001");
|
||||
expect(scan.title).toBe("Tracked Product");
|
||||
expect(scan.offer_count).toBe(2);
|
||||
expect(scan.candidate_seller_count).toBe(2);
|
||||
expect(scan.matched_seller_count).toBe(1);
|
||||
|
||||
const sellers = db.query("SELECT * FROM stalker_sellers").all() as any[];
|
||||
expect(sellers.length).toBe(1);
|
||||
expect(sellers[0].seller_id).toBe("AQUALIFIED");
|
||||
expect(sellers[0].rating_count).toBe(12);
|
||||
expect(sellers[0].storefront_asin_total).toBe(2);
|
||||
expect(sellers[0].persisted_inventory_sample_count).toBe(0);
|
||||
|
||||
const asinSellers = db.query("SELECT * FROM stalker_asin_sellers").all() as any[];
|
||||
expect(asinSellers.length).toBe(1);
|
||||
expect(asinSellers[0].offer_price).toBe(19.99);
|
||||
expect(asinSellers[0].is_fba).toBe(1);
|
||||
expect(asinSellers[0].stock).toBe(3);
|
||||
|
||||
const inventory = db
|
||||
.query("SELECT asin FROM stalker_seller_inventory ORDER BY asin")
|
||||
.all() as Array<{ asin: string }>;
|
||||
expect(inventory.map((row) => row.asin)).toEqual([]);
|
||||
});
|
||||
@@ -1,14 +1,25 @@
|
||||
import * as XLSX from "xlsx";
|
||||
import path from "node:path";
|
||||
import { type Database, closeDb, getDb, initDb } from "./database.ts";
|
||||
import { fetchSellabilityBatch } from "./sp-api.ts";
|
||||
import type { SellabilityInfo } from "./types.ts";
|
||||
import { normalizeAsin } from "../asin.ts";
|
||||
import { db } from "../db/index.ts";
|
||||
import { refreshRunStats, upsertProduct } from "../db/persistence.ts";
|
||||
import {
|
||||
analysisRunStats,
|
||||
productObservations,
|
||||
runs,
|
||||
stalkerRunDetails,
|
||||
stalkerScans,
|
||||
sellers,
|
||||
stalkerScanSellers,
|
||||
stalkerInventoryItems,
|
||||
} from "../db/schema.ts";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { fetchSellabilityBatch } from "../integrations/sp-api.ts";
|
||||
import type { SellabilityInfo } from "../types.ts";
|
||||
|
||||
const KEEPA_BASE = "https://api.keepa.com";
|
||||
const DOMAIN_US = "1";
|
||||
const AMAZON_US_SELLER_ID = "ATVPDKIKX0DER";
|
||||
const ASIN_REGEX = /^B[0-9A-Z]{9}$/;
|
||||
const DEFAULT_DB_PATH = path.join(process.cwd(), "db", "results.db");
|
||||
const DEFAULT_STOREFRONT_UPDATE_HOURS = 168;
|
||||
const DEFAULT_OFFER_LIMIT = 100;
|
||||
const DEFAULT_SELLER_LIMIT = 30;
|
||||
@@ -28,7 +39,7 @@ type KeepaApiResponse = {
|
||||
|
||||
export type StalkerArgs = {
|
||||
input: string;
|
||||
dbPath: string;
|
||||
dbPath?: string;
|
||||
maxAsins: number | null;
|
||||
storefrontUpdateHours: number;
|
||||
offerLimit: number;
|
||||
@@ -115,7 +126,6 @@ type StalkerRunStats = {
|
||||
};
|
||||
|
||||
type StalkerRunContext = {
|
||||
database: Database | null;
|
||||
metadataCache: Map<string, StalkerSeller>;
|
||||
storefrontCache: Map<string, StalkerSeller>;
|
||||
stats: StalkerRunStats;
|
||||
@@ -131,7 +141,6 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs {
|
||||
printUsageAndExit("Missing required --input file.");
|
||||
}
|
||||
|
||||
const dbPath = readFlagValue(argv, "--db") ?? DEFAULT_DB_PATH;
|
||||
const maxAsinsRaw = readFlagValue(argv, "--max-asins");
|
||||
const storefrontUpdateRaw = readFlagValue(argv, "--storefront-update-hours");
|
||||
const offerLimitRaw = readFlagValue(argv, "--offer-limit");
|
||||
@@ -205,7 +214,6 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs {
|
||||
|
||||
return {
|
||||
input,
|
||||
dbPath,
|
||||
maxAsins,
|
||||
storefrontUpdateHours,
|
||||
offerLimit,
|
||||
@@ -305,7 +313,11 @@ export function extractLiveOfferSellerCandidates(
|
||||
return Array.from(bySeller.values());
|
||||
}
|
||||
|
||||
export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
|
||||
export type StalkerDeps = {
|
||||
fetchSellabilityBatch?: (asins: string[]) => Promise<Map<string, SellabilityInfo>>;
|
||||
};
|
||||
|
||||
export async function runStalker(args: StalkerArgs, deps: StalkerDeps = {}): Promise<StalkerRunStats> {
|
||||
const apiKey = Bun.env.KEEPA_API_KEY;
|
||||
if (!apiKey) throw new Error("Missing required env var: KEEPA_API_KEY");
|
||||
|
||||
@@ -313,20 +325,18 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
|
||||
const cappedAsins =
|
||||
args.maxAsins == null ? allAsins : allAsins.slice(0, args.maxAsins);
|
||||
|
||||
initDb(args.dbPath);
|
||||
const database = getDb(args.dbPath);
|
||||
const completedAsins = args.resume
|
||||
? loadPreviouslyScannedAsins(database)
|
||||
? await loadPreviouslyScannedAsins()
|
||||
: new Set<string>();
|
||||
const resumeFilteredAsins = cappedAsins.filter(
|
||||
(asin) => !completedAsins.has(asin),
|
||||
);
|
||||
const runId = args.dryRun
|
||||
? null
|
||||
: startStalkerRun(database, args.input, resumeFilteredAsins.length);
|
||||
: await startStalkerRun(args.input, resumeFilteredAsins.length);
|
||||
const analysisRunId =
|
||||
!args.dryRun && args.analyzeSellable
|
||||
? startStalkerAnalysisRun(database, args.input)
|
||||
? await startStalkerAnalysisRun(args.input, runId!)
|
||||
: null;
|
||||
const stats: StalkerRunStats = {
|
||||
scannedAsins: 0,
|
||||
@@ -345,7 +355,6 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
|
||||
stoppedEarly: false,
|
||||
};
|
||||
const context: StalkerRunContext = {
|
||||
database,
|
||||
metadataCache: new Map(),
|
||||
storefrontCache: new Map(),
|
||||
stats,
|
||||
@@ -381,7 +390,7 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
|
||||
);
|
||||
|
||||
if (args.sellability && !args.dryRun) {
|
||||
await enrichInventorySellability(result, stats);
|
||||
await enrichInventorySellability(result, stats, deps.fetchSellabilityBatch ?? fetchSellabilityBatch);
|
||||
}
|
||||
applyInventoryPersistencePolicy(result, args.sellability && !args.dryRun);
|
||||
if (args.sellability && !args.dryRun) {
|
||||
@@ -389,7 +398,7 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
|
||||
}
|
||||
|
||||
if (!args.dryRun && runId != null) {
|
||||
persistAsinResult(database, runId, result);
|
||||
await persistAsinResult(runId, result);
|
||||
}
|
||||
const sellableAsins = collectPersistedInventoryAsins(result);
|
||||
if (
|
||||
@@ -400,7 +409,6 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
|
||||
sellableAsins.length > 0
|
||||
) {
|
||||
await runSellableAnalysisChild(
|
||||
args.dbPath,
|
||||
runId,
|
||||
analysisRunId,
|
||||
sellableAsins,
|
||||
@@ -417,7 +425,7 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
|
||||
}
|
||||
|
||||
if (!args.dryRun && runId != null) {
|
||||
refreshStalkerRun(database, runId, stats, "running");
|
||||
await refreshStalkerRun(runId, stats, "running");
|
||||
}
|
||||
console.log(
|
||||
`Stalker: ${asin} candidates=${result.candidateSellerCount}, matched=${result.matchedSellers.length}, persisted_inventory=${sumInventoryAsins(result)}`,
|
||||
@@ -432,8 +440,7 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
|
||||
}
|
||||
|
||||
if (!args.dryRun && runId != null) {
|
||||
refreshStalkerRun(
|
||||
database,
|
||||
await refreshStalkerRun(
|
||||
runId,
|
||||
stats,
|
||||
stats.stoppedEarly
|
||||
@@ -445,16 +452,16 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
|
||||
}
|
||||
logRunSummary(stats, args);
|
||||
if (!args.dryRun && analysisRunId != null) {
|
||||
finishStalkerAnalysisRun(database, analysisRunId, "completed");
|
||||
await finishStalkerAnalysisRun(analysisRunId, "completed");
|
||||
}
|
||||
return stats;
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
if (!args.dryRun && runId != null) {
|
||||
finishStalkerRunWithError(database, runId, stats, message);
|
||||
await finishStalkerRunWithError(runId, stats, message);
|
||||
}
|
||||
if (!args.dryRun && analysisRunId != null) {
|
||||
finishStalkerAnalysisRun(database, analysisRunId, "failed", message);
|
||||
await finishStalkerAnalysisRun(analysisRunId, "failed", message);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
@@ -545,6 +552,7 @@ function applyInventoryPersistencePolicy(
|
||||
async function enrichInventorySellability(
|
||||
result: StalkerAsinResult,
|
||||
stats: StalkerRunStats,
|
||||
sellabilityFn: (asins: string[]) => Promise<Map<string, SellabilityInfo>>,
|
||||
): Promise<void> {
|
||||
const sellers = result.matchedSellers.map(({ seller }) => seller);
|
||||
const items = sellers.flatMap((seller) => seller.storefrontItems);
|
||||
@@ -554,7 +562,7 @@ async function enrichInventorySellability(
|
||||
console.log(
|
||||
`Stalker inventory sellability: checking ${uniqueAsins.length} ASIN(s) from matched seller storefronts...`,
|
||||
);
|
||||
const sellabilityMap = await fetchSellabilityBatch(uniqueAsins);
|
||||
const sellabilityMap = await sellabilityFn(uniqueAsins);
|
||||
stats.inventorySellabilityCheckedAsins += uniqueAsins.length;
|
||||
|
||||
for (const asin of uniqueAsins) {
|
||||
@@ -685,13 +693,12 @@ async function fetchSellerMetadata(
|
||||
for (const sellerId of uniqueSellerIds) {
|
||||
const cached =
|
||||
context.metadataCache.get(sellerId) ??
|
||||
loadCachedSeller(
|
||||
context.database,
|
||||
(await loadCachedSeller(
|
||||
sellerId,
|
||||
args.sellerCacheHours,
|
||||
false,
|
||||
args.inventoryLimit,
|
||||
);
|
||||
));
|
||||
if (cached) {
|
||||
context.metadataCache.set(sellerId, cached);
|
||||
out.set(sellerId, cached);
|
||||
@@ -739,13 +746,12 @@ async function fetchQualifiedSellerStorefronts(
|
||||
for (const sellerId of uniqueSellerIds) {
|
||||
const cached =
|
||||
context.storefrontCache.get(sellerId) ??
|
||||
loadCachedSeller(
|
||||
context.database,
|
||||
(await loadCachedSeller(
|
||||
sellerId,
|
||||
args.sellerCacheHours,
|
||||
true,
|
||||
args.inventoryLimit,
|
||||
);
|
||||
));
|
||||
if (cached) {
|
||||
context.storefrontCache.set(sellerId, cached);
|
||||
out.set(sellerId, cached);
|
||||
@@ -830,272 +836,307 @@ async function fetchKeepaWithRetries(
|
||||
throw new Error(lastErrorMessage);
|
||||
}
|
||||
|
||||
function persistAsinResult(
|
||||
database: Database,
|
||||
async function persistAsinResult(
|
||||
runId: number,
|
||||
result: StalkerAsinResult,
|
||||
): void {
|
||||
const fetchedAt = new Date().toISOString();
|
||||
): Promise<void> {
|
||||
const fetchedAt = new Date();
|
||||
|
||||
database.transaction(() => {
|
||||
const scanId = upsertAsinScan(database, runId, result, fetchedAt);
|
||||
await db.transaction(async (tx) => {
|
||||
const scanId = await upsertAsinScan(tx, runId, result, fetchedAt);
|
||||
const observationIds = new Map<string, number>();
|
||||
|
||||
for (const { seller, offer } of result.matchedSellers) {
|
||||
upsertSeller(database, seller, fetchedAt);
|
||||
upsertAsinSeller(database, scanId, seller, offer);
|
||||
upsertSellerInventory(database, runId, seller, fetchedAt);
|
||||
await upsertSeller(tx, seller, fetchedAt);
|
||||
await upsertAsinSeller(tx, scanId, seller, offer);
|
||||
await upsertSellerInventory(tx, runId, seller, fetchedAt, observationIds);
|
||||
}
|
||||
})();
|
||||
});
|
||||
}
|
||||
|
||||
function upsertAsinScan(
|
||||
database: Database,
|
||||
async function upsertAsinScan(
|
||||
tx: Parameters<Parameters<typeof db.transaction>[0]>[0],
|
||||
runId: number,
|
||||
result: StalkerAsinResult,
|
||||
fetchedAt: string,
|
||||
): number {
|
||||
database
|
||||
.prepare(
|
||||
`INSERT INTO stalker_asin_scans (
|
||||
run_id, source_asin, title, offer_count, candidate_seller_count,
|
||||
matched_seller_count, fetched_at, raw_product_json
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(run_id, source_asin) DO UPDATE SET
|
||||
title = excluded.title,
|
||||
offer_count = excluded.offer_count,
|
||||
candidate_seller_count = excluded.candidate_seller_count,
|
||||
matched_seller_count = excluded.matched_seller_count,
|
||||
fetched_at = excluded.fetched_at,
|
||||
raw_product_json = excluded.raw_product_json`,
|
||||
)
|
||||
.run(
|
||||
runId,
|
||||
result.asin,
|
||||
result.title,
|
||||
result.offerCount,
|
||||
result.candidateSellerCount,
|
||||
result.matchedSellers.length,
|
||||
fetchedAt: Date,
|
||||
): Promise<number> {
|
||||
const sourceProductAsin = await upsertProduct(
|
||||
{
|
||||
asin: result.asin,
|
||||
name: result.title,
|
||||
metadataSource: "catalog",
|
||||
fetchedAt,
|
||||
JSON.stringify(result.product ?? { error: result.error ?? null }),
|
||||
);
|
||||
},
|
||||
tx,
|
||||
);
|
||||
const [observation] = await tx
|
||||
.insert(productObservations)
|
||||
.values({
|
||||
productAsin: sourceProductAsin,
|
||||
runId,
|
||||
source: "stalker_scan",
|
||||
fetchedAt,
|
||||
rawProductJson: JSON.stringify(
|
||||
result.product ?? { error: result.error ?? null },
|
||||
),
|
||||
})
|
||||
.returning({ id: productObservations.id });
|
||||
if (!observation) {
|
||||
throw new Error(`Failed to insert stalker observation for ${result.asin}`);
|
||||
}
|
||||
|
||||
const row = database
|
||||
.query(
|
||||
`SELECT id FROM stalker_asin_scans WHERE run_id = ? AND source_asin = ?`,
|
||||
)
|
||||
.get(runId, result.asin) as { id: number } | null;
|
||||
await tx
|
||||
.insert(stalkerScans)
|
||||
.values({
|
||||
runId,
|
||||
sourceProductAsin,
|
||||
observationId: observation.id,
|
||||
offerCount: result.offerCount,
|
||||
candidateSellerCount: result.candidateSellerCount,
|
||||
matchedSellerCount: result.matchedSellers.length,
|
||||
fetchedAt,
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: [stalkerScans.runId, stalkerScans.sourceProductAsin],
|
||||
set: {
|
||||
observationId: sql`EXCLUDED.observation_id`,
|
||||
offerCount: sql`EXCLUDED.offer_count`,
|
||||
candidateSellerCount: sql`EXCLUDED.candidate_seller_count`,
|
||||
matchedSellerCount: sql`EXCLUDED.matched_seller_count`,
|
||||
fetchedAt: sql`EXCLUDED.fetched_at`,
|
||||
},
|
||||
});
|
||||
|
||||
const [row] = await tx
|
||||
.select({ id: stalkerScans.id })
|
||||
.from(stalkerScans)
|
||||
.where(
|
||||
sql`${stalkerScans.runId} = ${runId} AND ${stalkerScans.sourceProductAsin} = ${sourceProductAsin}`,
|
||||
);
|
||||
if (!row)
|
||||
throw new Error(`Failed to load stalker scan row for ${result.asin}`);
|
||||
return row.id;
|
||||
}
|
||||
|
||||
function upsertSeller(
|
||||
database: Database,
|
||||
async function upsertSeller(
|
||||
tx: Parameters<Parameters<typeof db.transaction>[0]>[0],
|
||||
seller: StalkerSeller,
|
||||
fetchedAt: string,
|
||||
): void {
|
||||
database
|
||||
.prepare(
|
||||
`INSERT INTO stalker_sellers (
|
||||
seller_id, seller_name, rating, rating_count, storefront_asin_total,
|
||||
persisted_inventory_sample_count, last_updated_at, raw_seller_json
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(seller_id) DO UPDATE SET
|
||||
seller_name = excluded.seller_name,
|
||||
rating = excluded.rating,
|
||||
rating_count = excluded.rating_count,
|
||||
storefront_asin_total = excluded.storefront_asin_total,
|
||||
persisted_inventory_sample_count = excluded.persisted_inventory_sample_count,
|
||||
last_updated_at = excluded.last_updated_at,
|
||||
raw_seller_json = excluded.raw_seller_json`,
|
||||
)
|
||||
.run(
|
||||
seller.sellerId,
|
||||
seller.sellerName,
|
||||
seller.rating,
|
||||
seller.ratingCount,
|
||||
seller.storefrontAsinTotal,
|
||||
seller.storefrontItems.length,
|
||||
fetchedAt,
|
||||
JSON.stringify(seller.rawSeller),
|
||||
);
|
||||
fetchedAt: Date,
|
||||
): Promise<void> {
|
||||
await tx
|
||||
.insert(sellers)
|
||||
.values({
|
||||
sellerId: seller.sellerId,
|
||||
sellerName: seller.sellerName,
|
||||
rating: seller.rating,
|
||||
ratingCount: seller.ratingCount,
|
||||
storefrontAsinTotal: seller.storefrontAsinTotal,
|
||||
persistedInventorySampleCount: seller.storefrontItems.length,
|
||||
lastUpdatedAt: fetchedAt,
|
||||
rawSellerJson: JSON.stringify(seller.rawSeller),
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: sellers.sellerId,
|
||||
set: {
|
||||
sellerName: sql`EXCLUDED.seller_name`,
|
||||
rating: sql`EXCLUDED.rating`,
|
||||
ratingCount: sql`EXCLUDED.rating_count`,
|
||||
storefrontAsinTotal: sql`EXCLUDED.storefront_asin_total`,
|
||||
persistedInventorySampleCount: sql`EXCLUDED.persisted_inventory_sample_count`,
|
||||
lastUpdatedAt: sql`EXCLUDED.last_updated_at`,
|
||||
rawSellerJson: sql`EXCLUDED.raw_seller_json`,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function upsertAsinSeller(
|
||||
database: Database,
|
||||
async function upsertAsinSeller(
|
||||
tx: Parameters<Parameters<typeof db.transaction>[0]>[0],
|
||||
scanId: number,
|
||||
seller: StalkerSeller,
|
||||
offer: StalkerOffer,
|
||||
): void {
|
||||
database
|
||||
.prepare(
|
||||
`INSERT INTO stalker_asin_sellers (
|
||||
scan_id, seller_id, offer_price, condition, is_fba, stock,
|
||||
seller_rating, seller_rating_count, raw_offer_json
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(scan_id, seller_id) DO UPDATE SET
|
||||
offer_price = excluded.offer_price,
|
||||
condition = excluded.condition,
|
||||
is_fba = excluded.is_fba,
|
||||
stock = excluded.stock,
|
||||
seller_rating = excluded.seller_rating,
|
||||
seller_rating_count = excluded.seller_rating_count,
|
||||
raw_offer_json = excluded.raw_offer_json`,
|
||||
)
|
||||
.run(
|
||||
): Promise<void> {
|
||||
await tx
|
||||
.insert(stalkerScanSellers)
|
||||
.values({
|
||||
scanId,
|
||||
seller.sellerId,
|
||||
offer.offerPrice,
|
||||
offer.condition,
|
||||
offer.isFba == null ? null : offer.isFba ? 1 : 0,
|
||||
offer.stock,
|
||||
seller.rating,
|
||||
seller.ratingCount,
|
||||
JSON.stringify(offer.rawOffer),
|
||||
);
|
||||
sellerId: seller.sellerId,
|
||||
offerPrice: offer.offerPrice,
|
||||
condition: offer.condition,
|
||||
isFba: offer.isFba,
|
||||
stock: offer.stock,
|
||||
sellerRating: seller.rating,
|
||||
sellerRatingCount: seller.ratingCount,
|
||||
rawOfferJson: JSON.stringify(offer.rawOffer),
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: [stalkerScanSellers.scanId, stalkerScanSellers.sellerId],
|
||||
set: {
|
||||
offerPrice: sql`EXCLUDED.offer_price`,
|
||||
condition: sql`EXCLUDED.condition`,
|
||||
isFba: sql`EXCLUDED.is_fba`,
|
||||
stock: sql`EXCLUDED.stock`,
|
||||
sellerRating: sql`EXCLUDED.seller_rating`,
|
||||
sellerRatingCount: sql`EXCLUDED.seller_rating_count`,
|
||||
rawOfferJson: sql`EXCLUDED.raw_offer_json`,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function upsertSellerInventory(
|
||||
database: Database,
|
||||
async function upsertSellerInventory(
|
||||
tx: Parameters<Parameters<typeof db.transaction>[0]>[0],
|
||||
runId: number,
|
||||
seller: StalkerSeller,
|
||||
fetchedAt: string,
|
||||
): void {
|
||||
const insert = database.prepare(
|
||||
`INSERT INTO stalker_seller_inventory (
|
||||
run_id, seller_id, asin, can_sell, sellability_status,
|
||||
sellability_reason, product_title, brand, category_tree, current_price,
|
||||
avg_price_90d, sales_rank, monthly_sold, seller_count, amazon_is_seller,
|
||||
raw_product_json, last_seen_at, raw_inventory_json
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(run_id, seller_id, asin) DO UPDATE SET
|
||||
can_sell = excluded.can_sell,
|
||||
sellability_status = excluded.sellability_status,
|
||||
sellability_reason = excluded.sellability_reason,
|
||||
product_title = excluded.product_title,
|
||||
brand = excluded.brand,
|
||||
category_tree = excluded.category_tree,
|
||||
current_price = excluded.current_price,
|
||||
avg_price_90d = excluded.avg_price_90d,
|
||||
sales_rank = excluded.sales_rank,
|
||||
monthly_sold = excluded.monthly_sold,
|
||||
seller_count = excluded.seller_count,
|
||||
amazon_is_seller = excluded.amazon_is_seller,
|
||||
raw_product_json = excluded.raw_product_json,
|
||||
last_seen_at = excluded.last_seen_at,
|
||||
raw_inventory_json = excluded.raw_inventory_json`,
|
||||
fetchedAt: Date,
|
||||
observationIds: Map<string, number>,
|
||||
): Promise<void> {
|
||||
const items = seller.storefrontItems.filter(
|
||||
(item) =>
|
||||
item.sellability?.canSell === true &&
|
||||
item.sellability.sellabilityStatus === "available",
|
||||
);
|
||||
|
||||
for (const item of seller.storefrontItems) {
|
||||
if (
|
||||
item.sellability?.canSell !== true ||
|
||||
item.sellability.sellabilityStatus !== "available"
|
||||
) {
|
||||
continue;
|
||||
if (items.length === 0) return;
|
||||
|
||||
for (const item of items) {
|
||||
let observationId = observationIds.get(item.asin);
|
||||
if (observationId == null) {
|
||||
const productAsin = await upsertProduct(
|
||||
{
|
||||
asin: item.asin,
|
||||
name: item.productDetails?.title,
|
||||
brand: item.productDetails?.brand,
|
||||
category: item.productDetails?.categoryTree.join(" > "),
|
||||
metadataSource: "catalog",
|
||||
fetchedAt,
|
||||
},
|
||||
tx,
|
||||
);
|
||||
const [observation] = await tx
|
||||
.insert(productObservations)
|
||||
.values({
|
||||
productAsin,
|
||||
runId,
|
||||
source: "stalker_inventory",
|
||||
canSell: item.sellability?.canSell ?? null,
|
||||
sellabilityStatus: item.sellability?.sellabilityStatus ?? null,
|
||||
sellabilityReason: item.sellability?.sellabilityReason ?? null,
|
||||
currentPrice: item.productDetails?.currentPrice ?? null,
|
||||
avgPrice90d: item.productDetails?.avgPrice90 ?? null,
|
||||
salesRank: item.productDetails?.salesRank ?? null,
|
||||
monthlySold: item.productDetails?.monthlySold ?? null,
|
||||
sellerCount: item.productDetails?.sellerCount ?? null,
|
||||
amazonIsSeller: item.productDetails?.amazonIsSeller ?? null,
|
||||
rawProductJson: item.productDetails
|
||||
? JSON.stringify(item.productDetails.rawProduct)
|
||||
: null,
|
||||
fetchedAt,
|
||||
})
|
||||
.returning({ id: productObservations.id });
|
||||
if (!observation) {
|
||||
throw new Error(`Failed to insert inventory observation for ${item.asin}`);
|
||||
}
|
||||
observationId = observation.id;
|
||||
observationIds.set(item.asin, observationId);
|
||||
}
|
||||
|
||||
insert.run(
|
||||
runId,
|
||||
seller.sellerId,
|
||||
item.asin,
|
||||
item.sellability?.canSell == null
|
||||
? null
|
||||
: item.sellability.canSell
|
||||
? 1
|
||||
: 0,
|
||||
item.sellability?.sellabilityStatus ?? null,
|
||||
item.sellability?.sellabilityReason ?? null,
|
||||
item.productDetails?.title ?? null,
|
||||
item.productDetails?.brand ?? null,
|
||||
item.productDetails
|
||||
? JSON.stringify(item.productDetails.categoryTree)
|
||||
: null,
|
||||
item.productDetails?.currentPrice ?? null,
|
||||
item.productDetails?.avgPrice90 ?? null,
|
||||
item.productDetails?.salesRank ?? null,
|
||||
item.productDetails?.monthlySold ?? null,
|
||||
item.productDetails?.sellerCount ?? null,
|
||||
item.productDetails?.amazonIsSeller == null
|
||||
? null
|
||||
: item.productDetails.amazonIsSeller
|
||||
? 1
|
||||
: 0,
|
||||
item.productDetails
|
||||
? JSON.stringify(item.productDetails.rawProduct)
|
||||
: null,
|
||||
fetchedAt,
|
||||
JSON.stringify(item.rawInventory),
|
||||
);
|
||||
await tx
|
||||
.insert(stalkerInventoryItems)
|
||||
.values({
|
||||
runId,
|
||||
sellerId: seller.sellerId,
|
||||
productAsin: item.asin,
|
||||
observationId,
|
||||
lastSeenAt: fetchedAt,
|
||||
rawInventoryJson: JSON.stringify(item.rawInventory),
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: [
|
||||
stalkerInventoryItems.runId,
|
||||
stalkerInventoryItems.sellerId,
|
||||
stalkerInventoryItems.productAsin,
|
||||
],
|
||||
set: {
|
||||
observationId: sql`EXCLUDED.observation_id`,
|
||||
lastSeenAt: sql`EXCLUDED.last_seen_at`,
|
||||
rawInventoryJson: sql`EXCLUDED.raw_inventory_json`,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function startStalkerRun(
|
||||
database: Database,
|
||||
async function startStalkerRun(
|
||||
inputFile: string,
|
||||
totalAsins: number,
|
||||
): number {
|
||||
const result = database
|
||||
.prepare(
|
||||
`INSERT INTO stalker_runs (
|
||||
input_file, started_at, requested_asins, status
|
||||
) VALUES (?, ?, ?, ?)`,
|
||||
)
|
||||
.run(inputFile, new Date().toISOString(), totalAsins, "running");
|
||||
|
||||
return result.lastInsertRowid as number;
|
||||
): Promise<number> {
|
||||
const [row] = await db
|
||||
.insert(runs)
|
||||
.values({
|
||||
type: "stalker",
|
||||
inputFile,
|
||||
startedAt: new Date(),
|
||||
status: "running",
|
||||
})
|
||||
.returning({ id: runs.id });
|
||||
if (!row) throw new Error("Failed to insert stalker run record.");
|
||||
await db.insert(stalkerRunDetails).values({
|
||||
runId: row.id,
|
||||
requestedAsins: totalAsins,
|
||||
});
|
||||
return row.id;
|
||||
}
|
||||
|
||||
function startStalkerAnalysisRun(
|
||||
database: Database,
|
||||
async function startStalkerAnalysisRun(
|
||||
inputFile: string,
|
||||
): number {
|
||||
const result = database
|
||||
.prepare(
|
||||
`INSERT INTO category_analysis_runs (
|
||||
category_id, category_label, run_timestamp, top_asins_checked,
|
||||
available_asins, fba_count, fbm_count, skip_count, status, error_message
|
||||
) VALUES (?, ?, ?, 0, 0, 0, 0, 0, 'running', NULL)`,
|
||||
)
|
||||
.run(0, `Stalker: ${path.basename(inputFile)}`, new Date().toISOString());
|
||||
|
||||
return result.lastInsertRowid as number;
|
||||
parentRunId: number,
|
||||
): Promise<number> {
|
||||
const [row] = await db
|
||||
.insert(runs)
|
||||
.values({
|
||||
type: "stalker_analysis",
|
||||
parentRunId,
|
||||
inputFile: `Stalker: ${path.basename(inputFile)}`,
|
||||
status: "running",
|
||||
startedAt: new Date(),
|
||||
})
|
||||
.returning({ id: runs.id });
|
||||
if (!row) throw new Error("Failed to insert stalker analysis run record.");
|
||||
await db.insert(analysisRunStats).values({ runId: row.id });
|
||||
return row.id;
|
||||
}
|
||||
|
||||
function loadPreviouslyScannedAsins(database: Database): Set<string> {
|
||||
const rows = database
|
||||
.query(`SELECT DISTINCT source_asin FROM stalker_asin_scans`)
|
||||
.all() as Array<{ source_asin: string }>;
|
||||
return new Set(rows.map((row) => row.source_asin));
|
||||
async function loadPreviouslyScannedAsins(): Promise<Set<string>> {
|
||||
const rows = await db
|
||||
.selectDistinct({ sourceAsin: stalkerScans.sourceProductAsin })
|
||||
.from(stalkerScans);
|
||||
return new Set(rows.map((row) => row.sourceAsin));
|
||||
}
|
||||
|
||||
function loadCachedSeller(
|
||||
database: Database | null,
|
||||
async function loadCachedSeller(
|
||||
sellerId: string,
|
||||
maxAgeHours: number,
|
||||
requireStorefront: boolean,
|
||||
inventoryLimit: number,
|
||||
): StalkerSeller | null {
|
||||
if (!database || maxAgeHours <= 0) return null;
|
||||
const row = database
|
||||
.query(
|
||||
`SELECT raw_seller_json, last_updated_at, storefront_asin_total
|
||||
FROM stalker_sellers
|
||||
WHERE seller_id = ?`,
|
||||
)
|
||||
.get(sellerId) as {
|
||||
raw_seller_json: string | null;
|
||||
last_updated_at: string;
|
||||
storefront_asin_total: number | null;
|
||||
} | null;
|
||||
if (!row?.raw_seller_json) return null;
|
||||
): Promise<StalkerSeller | null> {
|
||||
if (maxAgeHours <= 0) return null;
|
||||
|
||||
const ageMs = Date.now() - new Date(row.last_updated_at).getTime();
|
||||
const [row] = await db
|
||||
.select({
|
||||
rawSellerJson: sellers.rawSellerJson,
|
||||
lastUpdatedAt: sellers.lastUpdatedAt,
|
||||
})
|
||||
.from(sellers)
|
||||
.where(eq(sellers.sellerId, sellerId))
|
||||
.limit(1);
|
||||
|
||||
if (!row?.rawSellerJson) return null;
|
||||
|
||||
const ageMs = Date.now() - new Date(row.lastUpdatedAt).getTime();
|
||||
if (!Number.isFinite(ageMs) || ageMs > maxAgeHours * 60 * 60 * 1000) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const rawSeller = JSON.parse(row.raw_seller_json) as Record<string, any>;
|
||||
const rawSeller = JSON.parse(row.rawSellerJson) as Record<string, any>;
|
||||
const parsed = parseSeller(sellerId, rawSeller, inventoryLimit);
|
||||
if (requireStorefront && parsed.storefrontAsinTotal <= 0) return null;
|
||||
return parsed;
|
||||
@@ -1128,137 +1169,85 @@ function logRunSummary(stats: StalkerRunStats, args: StalkerArgs): void {
|
||||
);
|
||||
}
|
||||
|
||||
function refreshStalkerRun(
|
||||
database: Database,
|
||||
async function refreshStalkerRun(
|
||||
runId: number,
|
||||
stats: StalkerRunStats,
|
||||
status: string,
|
||||
): void {
|
||||
database
|
||||
.prepare(
|
||||
`UPDATE stalker_runs
|
||||
SET scanned_asins = ?,
|
||||
source_asins_with_matches = ?,
|
||||
candidate_sellers = ?,
|
||||
qualifying_sellers = ?,
|
||||
matched_sellers = ?,
|
||||
seller_metadata_requests = ?,
|
||||
seller_storefront_requests = ?,
|
||||
inventory_sellability_checked_asins = ?,
|
||||
inventory_sellability_available_asins = ?,
|
||||
inventory_sellability_excluded_asins = ?,
|
||||
persisted_inventory_asins = ?,
|
||||
status = ?,
|
||||
completed_at = CASE WHEN ? = 'running' THEN completed_at ELSE ? END
|
||||
WHERE id = ?`,
|
||||
)
|
||||
.run(
|
||||
stats.scannedAsins,
|
||||
stats.sourceAsinsWithMatches,
|
||||
stats.candidateSellers,
|
||||
stats.qualifyingSellers,
|
||||
stats.matchedSellers,
|
||||
stats.sellerMetadataRequests,
|
||||
stats.sellerStorefrontRequests,
|
||||
stats.inventorySellabilityCheckedAsins,
|
||||
stats.inventorySellabilityAvailableAsins,
|
||||
stats.inventorySellabilityExcludedAsins,
|
||||
stats.persistedInventoryAsins,
|
||||
status,
|
||||
status,
|
||||
new Date().toISOString(),
|
||||
runId,
|
||||
);
|
||||
): Promise<void> {
|
||||
await db
|
||||
.update(stalkerRunDetails)
|
||||
.set({
|
||||
skippedAsins: stats.skippedAsins,
|
||||
scannedAsins: stats.scannedAsins,
|
||||
sourceAsinsWithMatches: stats.sourceAsinsWithMatches,
|
||||
candidateSellers: stats.candidateSellers,
|
||||
qualifyingSellers: stats.qualifyingSellers,
|
||||
matchedSellers: stats.matchedSellers,
|
||||
sellerMetadataRequests: stats.sellerMetadataRequests,
|
||||
sellerStorefrontRequests: stats.sellerStorefrontRequests,
|
||||
inventorySellabilityCheckedAsins: stats.inventorySellabilityCheckedAsins,
|
||||
inventorySellabilityAvailableAsins:
|
||||
stats.inventorySellabilityAvailableAsins,
|
||||
inventorySellabilityExcludedAsins: stats.inventorySellabilityExcludedAsins,
|
||||
persistedInventoryAsins: stats.persistedInventoryAsins,
|
||||
})
|
||||
.where(eq(stalkerRunDetails.runId, runId));
|
||||
await db
|
||||
.update(runs)
|
||||
.set({
|
||||
status: status === "running" ? "running" : "completed",
|
||||
...(status !== "running" ? { completedAt: new Date() } : {}),
|
||||
})
|
||||
.where(eq(runs.id, runId));
|
||||
}
|
||||
|
||||
function finishStalkerRunWithError(
|
||||
database: Database,
|
||||
async function finishStalkerRunWithError(
|
||||
runId: number,
|
||||
stats: StalkerRunStats,
|
||||
errorMessage: string,
|
||||
): void {
|
||||
database
|
||||
.prepare(
|
||||
`UPDATE stalker_runs
|
||||
SET scanned_asins = ?,
|
||||
source_asins_with_matches = ?,
|
||||
candidate_sellers = ?,
|
||||
qualifying_sellers = ?,
|
||||
matched_sellers = ?,
|
||||
seller_metadata_requests = ?,
|
||||
seller_storefront_requests = ?,
|
||||
inventory_sellability_checked_asins = ?,
|
||||
inventory_sellability_available_asins = ?,
|
||||
inventory_sellability_excluded_asins = ?,
|
||||
persisted_inventory_asins = ?,
|
||||
status = 'failed',
|
||||
error_message = ?,
|
||||
completed_at = ?
|
||||
WHERE id = ?`,
|
||||
)
|
||||
.run(
|
||||
stats.scannedAsins,
|
||||
stats.sourceAsinsWithMatches,
|
||||
stats.candidateSellers,
|
||||
stats.qualifyingSellers,
|
||||
stats.matchedSellers,
|
||||
stats.sellerMetadataRequests,
|
||||
stats.sellerStorefrontRequests,
|
||||
stats.inventorySellabilityCheckedAsins,
|
||||
stats.inventorySellabilityAvailableAsins,
|
||||
stats.inventorySellabilityExcludedAsins,
|
||||
stats.persistedInventoryAsins,
|
||||
): Promise<void> {
|
||||
await db
|
||||
.update(stalkerRunDetails)
|
||||
.set({
|
||||
skippedAsins: stats.skippedAsins,
|
||||
scannedAsins: stats.scannedAsins,
|
||||
sourceAsinsWithMatches: stats.sourceAsinsWithMatches,
|
||||
candidateSellers: stats.candidateSellers,
|
||||
qualifyingSellers: stats.qualifyingSellers,
|
||||
matchedSellers: stats.matchedSellers,
|
||||
sellerMetadataRequests: stats.sellerMetadataRequests,
|
||||
sellerStorefrontRequests: stats.sellerStorefrontRequests,
|
||||
inventorySellabilityCheckedAsins: stats.inventorySellabilityCheckedAsins,
|
||||
inventorySellabilityAvailableAsins:
|
||||
stats.inventorySellabilityAvailableAsins,
|
||||
inventorySellabilityExcludedAsins: stats.inventorySellabilityExcludedAsins,
|
||||
persistedInventoryAsins: stats.persistedInventoryAsins,
|
||||
})
|
||||
.where(eq(stalkerRunDetails.runId, runId));
|
||||
await db
|
||||
.update(runs)
|
||||
.set({
|
||||
status: "failed",
|
||||
errorMessage,
|
||||
new Date().toISOString(),
|
||||
runId,
|
||||
);
|
||||
completedAt: new Date(),
|
||||
})
|
||||
.where(eq(runs.id, runId));
|
||||
}
|
||||
|
||||
function finishStalkerAnalysisRun(
|
||||
database: Database,
|
||||
async function finishStalkerAnalysisRun(
|
||||
runId: number,
|
||||
status: "completed" | "failed",
|
||||
errorMessage: string | null = null,
|
||||
): void {
|
||||
const stats = database
|
||||
.query(
|
||||
`SELECT
|
||||
COUNT(*) AS total,
|
||||
SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba,
|
||||
SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm,
|
||||
SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip
|
||||
FROM product_analysis_results
|
||||
WHERE run_id = ?`,
|
||||
)
|
||||
.get(runId) as {
|
||||
total: number;
|
||||
fba: number | null;
|
||||
fbm: number | null;
|
||||
skip: number | null;
|
||||
};
|
||||
|
||||
database
|
||||
.prepare(
|
||||
`UPDATE category_analysis_runs
|
||||
SET top_asins_checked = ?,
|
||||
available_asins = ?,
|
||||
fba_count = ?,
|
||||
fbm_count = ?,
|
||||
skip_count = ?,
|
||||
status = ?,
|
||||
error_message = ?
|
||||
WHERE id = ?`,
|
||||
)
|
||||
.run(
|
||||
stats.total ?? 0,
|
||||
stats.total ?? 0,
|
||||
stats.fba ?? 0,
|
||||
stats.fbm ?? 0,
|
||||
stats.skip ?? 0,
|
||||
): Promise<void> {
|
||||
await refreshRunStats(runId);
|
||||
await db
|
||||
.update(runs)
|
||||
.set({
|
||||
status,
|
||||
errorMessage,
|
||||
runId,
|
||||
);
|
||||
completedAt: new Date(),
|
||||
})
|
||||
.where(eq(runs.id, runId));
|
||||
}
|
||||
|
||||
function normalizeSellerResponse(
|
||||
@@ -1492,7 +1481,6 @@ function collectPersistedInventoryAsins(result: StalkerAsinResult): string[] {
|
||||
}
|
||||
|
||||
async function runSellableAnalysisChild(
|
||||
dbPath: string,
|
||||
stalkerRunId: number,
|
||||
analysisRunId: number,
|
||||
asins: string[],
|
||||
@@ -1502,8 +1490,6 @@ async function runSellableAnalysisChild(
|
||||
"bun",
|
||||
"run",
|
||||
"src/stalker-analyze.ts",
|
||||
"--db",
|
||||
dbPath,
|
||||
"--stalker-run-id",
|
||||
String(stalkerRunId),
|
||||
"--analysis-run-id",
|
||||
@@ -1529,13 +1515,6 @@ async function runSellableAnalysisChild(
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeAsin(value: unknown): string | null {
|
||||
const asin = String(value ?? "")
|
||||
.trim()
|
||||
.toUpperCase();
|
||||
return ASIN_REGEX.test(asin) ? asin : null;
|
||||
}
|
||||
|
||||
function normalizeSellerId(value: unknown): string | null {
|
||||
const sellerId = String(value ?? "")
|
||||
.trim()
|
||||
@@ -1660,8 +1639,5 @@ if (import.meta.main) {
|
||||
.catch((error) => {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exitCode = 1;
|
||||
})
|
||||
.finally(() => {
|
||||
closeDb();
|
||||
});
|
||||
}
|
||||
@@ -3,7 +3,7 @@ import path from "node:path";
|
||||
import { rmSync } from "node:fs";
|
||||
import ExcelJS from "exceljs";
|
||||
import { writeSupplierWorkbook } from "./supplier-export.ts";
|
||||
import type { SupplierAnalysisResult } from "./types.ts";
|
||||
import type { SupplierAnalysisResult } from "../types.ts";
|
||||
|
||||
const OUTPUT_FILE = path.join("/private/tmp", "asin-check-supplier-export-test.xlsx");
|
||||
|
||||
@@ -16,12 +16,12 @@ function result(overrides: Partial<SupplierAnalysisResult> = {}): SupplierAnalys
|
||||
upc: "012345678901",
|
||||
rowNumber: 2,
|
||||
record: {
|
||||
asin: "B000000001",
|
||||
name: "Test Product",
|
||||
unitCost: 10,
|
||||
brand: "Brand",
|
||||
category: "Grocery",
|
||||
},
|
||||
product: { asin: "B000000001", name: "Test Product", unitCost: 10 },
|
||||
lookup: {
|
||||
requestedUpc: "012345678901",
|
||||
normalizedUpc: "012345678901",
|
||||
@@ -81,7 +81,8 @@ test("writeSupplierWorkbook writes ranked, skipped, and summary sheets", async (
|
||||
result(),
|
||||
result({
|
||||
upc: "111111111111",
|
||||
record: { asin: "111111111111", name: "Missing", unitCost: 0 },
|
||||
record: { name: "Missing", unitCost: 0 },
|
||||
product: null,
|
||||
lookup: {
|
||||
requestedUpc: "111111111111",
|
||||
normalizedUpc: "111111111111",
|
||||
@@ -5,7 +5,7 @@ import type {
|
||||
KeepaUpcLookupStatus,
|
||||
SupplierAnalysisResult,
|
||||
SupplierVerdict,
|
||||
} from "./types.ts";
|
||||
} from "../types.ts";
|
||||
|
||||
export type SupplierExportSummary = {
|
||||
processedRows: number;
|
||||
@@ -63,7 +63,8 @@ function addRowsSheet(
|
||||
const sheet = workbook.addWorksheet(name);
|
||||
const headers = rows[0] ? Object.keys(rows[0]) : Object.keys(rowForResult({
|
||||
upc: "",
|
||||
record: { asin: "", name: "", unitCost: 0 },
|
||||
record: { name: "", unitCost: 0 },
|
||||
product: null,
|
||||
lookup: {
|
||||
requestedUpc: "",
|
||||
normalizedUpc: "",
|
||||
@@ -1,6 +1,6 @@
|
||||
import { expect, test } from "bun:test";
|
||||
import { scoreSupplierProduct } from "./supplier-scoring.ts";
|
||||
import type { KeepaData, ProductRecord, SpApiData } from "./types.ts";
|
||||
import type { KeepaData, ProductRecord, SpApiData } from "../types.ts";
|
||||
|
||||
function record(overrides: Partial<ProductRecord> = {}): ProductRecord {
|
||||
return {
|
||||
@@ -3,7 +3,7 @@ import type {
|
||||
ProductRecord,
|
||||
SpApiData,
|
||||
SupplierScore,
|
||||
} from "./types.ts";
|
||||
} from "../types.ts";
|
||||
|
||||
function round2(value: number): number {
|
||||
return Math.round(value * 100) / 100;
|
||||
@@ -1,22 +1,24 @@
|
||||
import path from "node:path";
|
||||
import { fetchKeepaDataBatch, lookupKeepaUpcs } from "./keepa.ts";
|
||||
import { requireAsin } from "../asin.ts";
|
||||
import { fetchKeepaDataBatch, lookupKeepaUpcs } from "../integrations/keepa.ts";
|
||||
import {
|
||||
fetchSellabilityBatch,
|
||||
fetchSpApiPricingAndFees,
|
||||
lookupSpApiUpcs,
|
||||
} from "./sp-api.ts";
|
||||
} from "../integrations/sp-api.ts";
|
||||
import {
|
||||
processUpcFileInBatches,
|
||||
type UpcInputRow,
|
||||
} from "./upc-file-reader.ts";
|
||||
import {
|
||||
appendSupplierResultsToRun,
|
||||
completeRunInDb,
|
||||
failRunInDb,
|
||||
refreshRunCountsInDb,
|
||||
startRunInDb,
|
||||
type RunCounts,
|
||||
} from "./writer.ts";
|
||||
import { initDb, closeDb } from "./database.ts";
|
||||
import { connectCache, disconnectCache } from "./cache.ts";
|
||||
} from "../writer.ts";
|
||||
import { connectCache, disconnectCache } from "../integrations/cache.ts";
|
||||
import { scoreSupplierProduct, resolveSupplierSalePrice } from "./supplier-scoring.ts";
|
||||
import {
|
||||
writeSupplierWorkbook,
|
||||
@@ -29,9 +31,8 @@ import type {
|
||||
SupplierAnalysisResult,
|
||||
SupplierScore,
|
||||
UpcLookupDetail,
|
||||
} from "./types.ts";
|
||||
} from "../types.ts";
|
||||
|
||||
const DB_PATH = process.env.RESULTS_DB_PATH || path.join("db", "results.db");
|
||||
const DEFAULT_INPUT_BATCH_SIZE = 200;
|
||||
const DEFAULT_UPC_LOOKUP_BATCH_SIZE = 100;
|
||||
const DEFAULT_PRICING_CONCURRENCY = 5;
|
||||
@@ -48,7 +49,6 @@ export type UpcFileAnalysisOptions = {
|
||||
|
||||
export type UpcFileAnalysisSummary = {
|
||||
runId: number;
|
||||
dbPath: string;
|
||||
inputFile: string;
|
||||
outputFile?: string;
|
||||
processedRows: number;
|
||||
@@ -242,8 +242,8 @@ async function lookupUpcsWithChunking(
|
||||
chunkDetails.set(
|
||||
upc,
|
||||
fallbackDetail && fallbackDetail.status !== "request_failed"
|
||||
? fallbackDetail
|
||||
: spDetail!,
|
||||
? { ...fallbackDetail, provider: "keepa" }
|
||||
: { ...spDetail!, provider: "sp_api" },
|
||||
);
|
||||
}
|
||||
|
||||
@@ -269,7 +269,7 @@ function toProductRecord(
|
||||
const keepaCategory = detail.keepaData?.categoryTree?.[0];
|
||||
|
||||
return {
|
||||
asin: detail.asin ?? row.upc,
|
||||
asin: requireAsin(detail.asin),
|
||||
name: row.name ?? detail.asin ?? row.upc,
|
||||
unitCost: row.unitCost ?? 0,
|
||||
brand: row.brand,
|
||||
@@ -277,6 +277,15 @@ function toProductRecord(
|
||||
};
|
||||
}
|
||||
|
||||
function toSupplierInputRecord(row: UpcInputRow) {
|
||||
return {
|
||||
name: row.name ?? row.upc,
|
||||
unitCost: row.unitCost ?? 0,
|
||||
brand: row.brand,
|
||||
category: row.category,
|
||||
};
|
||||
}
|
||||
|
||||
async function fetchFeesForProducts(
|
||||
products: ProductRecord[],
|
||||
keepaResults: Map<string, NonNullable<SupplierAnalysisResult["keepa"]>>,
|
||||
@@ -339,7 +348,6 @@ function summarizeSupplierResults(
|
||||
export async function runUpcFileAnalysis(
|
||||
options: UpcFileAnalysisOptions,
|
||||
): Promise<UpcFileAnalysisSummary> {
|
||||
const dbPath = options.dbPath ?? DB_PATH;
|
||||
const inputBatchSize = Math.max(
|
||||
1,
|
||||
options.inputBatchSize ?? DEFAULT_INPUT_BATCH_SIZE,
|
||||
@@ -355,8 +363,6 @@ export async function runUpcFileAnalysis(
|
||||
if (manageResources) {
|
||||
console.log("Connecting to Redis...");
|
||||
await connectCache();
|
||||
console.log("Initializing SQLite database...");
|
||||
initDb(dbPath);
|
||||
}
|
||||
|
||||
const unresolvedByStatus = createStatusCounter();
|
||||
@@ -365,7 +371,7 @@ export async function runUpcFileAnalysis(
|
||||
let processedRows = 0;
|
||||
let matchedRows = 0;
|
||||
|
||||
const runId = startRunInDb(dbPath, options.inputFile, outputFile);
|
||||
const runId = await startRunInDb(options.inputFile, outputFile, undefined, "supplier_upc");
|
||||
|
||||
try {
|
||||
const readerSummary = await processUpcFileInBatches(
|
||||
@@ -388,12 +394,19 @@ export async function runUpcFileAnalysis(
|
||||
product: ProductRecord;
|
||||
}> = [];
|
||||
for (const row of rows) {
|
||||
const detail = detailMap.get(row.upc);
|
||||
if (!detail) {
|
||||
unresolvedByStatus.request_failed += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
const detail =
|
||||
detailMap.get(row.upc) ??
|
||||
({
|
||||
requestedUpc: row.upc,
|
||||
normalizedUpc: row.upc,
|
||||
status: "request_failed",
|
||||
asin: null,
|
||||
candidateAsins: [],
|
||||
keepaData: null,
|
||||
provider: "sp_api",
|
||||
reason: "UPC lookup returned no result",
|
||||
} satisfies UpcLookupDetail);
|
||||
if (!detailMap.has(row.upc)) detailMap.set(row.upc, detail);
|
||||
unresolvedByStatus[detail.status] += 1;
|
||||
|
||||
if (detail.status === "found" && detail.asin) {
|
||||
@@ -413,30 +426,15 @@ export async function runUpcFileAnalysis(
|
||||
|
||||
const batchResults: SupplierAnalysisResult[] = [];
|
||||
for (const row of rows) {
|
||||
const detail = detailMap.get(row.upc);
|
||||
if (!detail || detail.status === "found") continue;
|
||||
const detail = detailMap.get(row.upc)!;
|
||||
if (detail.status === "found") continue;
|
||||
|
||||
batchResults.push({
|
||||
upc: row.upc,
|
||||
rowNumber: row.rowNumber,
|
||||
record: {
|
||||
asin: detail?.asin ?? row.upc,
|
||||
name: row.name ?? row.upc,
|
||||
unitCost: row.unitCost ?? 0,
|
||||
brand: row.brand,
|
||||
category: row.category,
|
||||
},
|
||||
lookup:
|
||||
detail ??
|
||||
({
|
||||
requestedUpc: row.upc,
|
||||
normalizedUpc: row.upc,
|
||||
status: "request_failed",
|
||||
asin: null,
|
||||
candidateAsins: [],
|
||||
keepaData: null,
|
||||
reason: "UPC lookup returned no result",
|
||||
} satisfies UpcLookupDetail),
|
||||
record: toSupplierInputRecord(row),
|
||||
product: null,
|
||||
lookup: detail,
|
||||
keepa: null,
|
||||
spApi: null,
|
||||
score: skippedScore(detail?.reason ?? "UPC unresolved"),
|
||||
@@ -471,7 +469,8 @@ export async function runUpcFileAnalysis(
|
||||
batchResults.push({
|
||||
upc: entry.detail.normalizedUpc,
|
||||
rowNumber: entry.row.rowNumber,
|
||||
record: entry.product,
|
||||
record: toSupplierInputRecord(entry.row),
|
||||
product: entry.product,
|
||||
lookup: entry.detail,
|
||||
keepa,
|
||||
spApi,
|
||||
@@ -481,7 +480,7 @@ export async function runUpcFileAnalysis(
|
||||
}
|
||||
}
|
||||
|
||||
appendSupplierResultsToRun(dbPath, runId, batchResults);
|
||||
await appendSupplierResultsToRun(runId, batchResults);
|
||||
allResults.push(...batchResults);
|
||||
},
|
||||
{
|
||||
@@ -490,10 +489,11 @@ export async function runUpcFileAnalysis(
|
||||
},
|
||||
);
|
||||
|
||||
const runCounts = refreshRunCountsInDb(dbPath, runId);
|
||||
const runCounts = await refreshRunCountsInDb(runId);
|
||||
|
||||
const exportSummary = summarizeSupplierResults(allResults, unresolvedByStatus);
|
||||
await writeSupplierWorkbook(outputFile, allResults, exportSummary);
|
||||
await completeRunInDb(runId);
|
||||
|
||||
if (allResults.length > 0) {
|
||||
const ranked = allResults
|
||||
@@ -522,7 +522,6 @@ export async function runUpcFileAnalysis(
|
||||
|
||||
return {
|
||||
runId,
|
||||
dbPath,
|
||||
inputFile: options.inputFile,
|
||||
outputFile,
|
||||
processedRows,
|
||||
@@ -537,10 +536,12 @@ export async function runUpcFileAnalysis(
|
||||
skippedInvalidUpc: readerSummary.skippedInvalidUpc,
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
await failRunInDb(runId, error);
|
||||
throw error;
|
||||
} finally {
|
||||
if (manageResources) {
|
||||
await disconnectCache();
|
||||
closeDb();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import { lookupKeepaUpcs, mapUpcsToAsins } from "./keepa.ts";
|
||||
import { lookupKeepaUpcs, mapUpcsToAsins } from "../integrations/keepa.ts";
|
||||
|
||||
function printUsage(): void {
|
||||
console.log("Usage:");
|
||||
58
src/types.ts
58
src/types.ts
@@ -58,7 +58,8 @@ export interface KeepaUpcLookupDetail {
|
||||
status: KeepaUpcLookupStatus;
|
||||
asin: string | null;
|
||||
candidateAsins: string[];
|
||||
keepaData: KeepaData | null;
|
||||
keepaData: KeepaData | null;
|
||||
provider?: "sp_api" | "keepa";
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
@@ -114,13 +115,66 @@ export interface SupplierScore {
|
||||
export interface SupplierAnalysisResult {
|
||||
upc: string;
|
||||
rowNumber?: number;
|
||||
record: ProductRecord;
|
||||
record: SupplierInputRecord;
|
||||
product: ProductRecord | null;
|
||||
lookup: UpcLookupDetail;
|
||||
keepa: KeepaData | null;
|
||||
spApi: SpApiData | null;
|
||||
score: SupplierScore;
|
||||
fetchedAt: string;
|
||||
}
|
||||
|
||||
export interface SupplierInputRecord {
|
||||
name: string;
|
||||
unitCost: number;
|
||||
brand?: string;
|
||||
category?: string;
|
||||
}
|
||||
|
||||
export interface Product {
|
||||
asin: string;
|
||||
name: string | null;
|
||||
brand: string | null;
|
||||
category: string | null;
|
||||
firstSeenAt: string;
|
||||
lastSeenAt: string;
|
||||
}
|
||||
|
||||
export interface ProductObservation {
|
||||
id: number;
|
||||
productAsin: string;
|
||||
runId: number;
|
||||
source: string;
|
||||
fetchedAt: string;
|
||||
}
|
||||
|
||||
export interface Run {
|
||||
id: number;
|
||||
type:
|
||||
| "lead_analysis"
|
||||
| "category_analysis"
|
||||
| "supplier_upc"
|
||||
| "stalker"
|
||||
| "stalker_analysis";
|
||||
parentRunId?: number | null;
|
||||
status: string;
|
||||
}
|
||||
|
||||
export interface RunItem {
|
||||
id: number;
|
||||
runId: number;
|
||||
productAsin: string | null;
|
||||
sourceRow?: number | null;
|
||||
}
|
||||
|
||||
export interface AnalysisRevision {
|
||||
id: number;
|
||||
runItemId: number;
|
||||
decision: "FBA" | "FBM" | "BUY" | "WATCH" | "SKIP";
|
||||
confidence: number | null;
|
||||
reasoning: string | null;
|
||||
analyzedAt: string;
|
||||
}
|
||||
|
||||
export interface CategoryRunSummaryDb {
|
||||
categoryId: number;
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import { createRoot } from "react-dom/client";
|
||||
import { useEffect, useMemo, useState } from "react";
|
||||
|
||||
type ProcessType = "lead_analysis" | "category_analysis";
|
||||
type ProcessType = "lead_analysis" | "category_analysis" | "supplier_upc" | "stalker" | "stalker_analysis";
|
||||
type AnalysisDecision = "FBA" | "FBM" | "BUY" | "WATCH" | "SKIP";
|
||||
type SortDirection = "ASC" | "DESC";
|
||||
|
||||
type Run = {
|
||||
@@ -15,6 +16,8 @@ type Run = {
|
||||
totalProducts: number;
|
||||
fbaCount: number;
|
||||
fbmCount: number;
|
||||
buyCount: number;
|
||||
watchCount: number;
|
||||
skipCount: number;
|
||||
};
|
||||
|
||||
@@ -37,11 +40,15 @@ type RunDetail = {
|
||||
totalProducts: number;
|
||||
fbaCount: number;
|
||||
fbmCount: number;
|
||||
buyCount: number;
|
||||
watchCount: number;
|
||||
skipCount: number;
|
||||
summary: {
|
||||
totalProducts: number;
|
||||
fbaCount: number;
|
||||
fbmCount: number;
|
||||
buyCount: number;
|
||||
watchCount: number;
|
||||
skipCount: number;
|
||||
};
|
||||
errorMessage?: string;
|
||||
@@ -49,6 +56,8 @@ type RunDetail = {
|
||||
};
|
||||
|
||||
type ResultItem = {
|
||||
item_id: number;
|
||||
product_asin: string | null;
|
||||
id?: number;
|
||||
run_id: number;
|
||||
asin: string;
|
||||
@@ -60,7 +69,7 @@ type ResultItem = {
|
||||
sales_rank: number | null;
|
||||
seller_count: number | null;
|
||||
monthly_sold: number | null;
|
||||
verdict: "FBA" | "FBM" | "SKIP";
|
||||
verdict: AnalysisDecision | null;
|
||||
amazon_is_seller: number | null;
|
||||
amazon_buybox_share_pct_90d: number | null;
|
||||
confidence: number | null;
|
||||
@@ -77,17 +86,18 @@ type ResultsResponse = {
|
||||
totalPages: number;
|
||||
};
|
||||
|
||||
type VerdictFilter = "" | "FBA" | "FBM" | "SKIP";
|
||||
type VerdictFilter = "" | AnalysisDecision;
|
||||
type AmazonSellerFilter = "" | "yes" | "no";
|
||||
|
||||
type ProductListItem = {
|
||||
processType: ProcessType;
|
||||
runId: number;
|
||||
item_id: number | null;
|
||||
processType: ProcessType | null;
|
||||
runId: number | null;
|
||||
asin: string;
|
||||
product_name: string | null;
|
||||
brand: string | null;
|
||||
category: string | null;
|
||||
verdict: "FBA" | "FBM" | "SKIP";
|
||||
verdict: AnalysisDecision | null;
|
||||
confidence: number | null;
|
||||
sellability_status: string | null;
|
||||
monthly_sold: number | null;
|
||||
@@ -98,7 +108,7 @@ type ProductListItem = {
|
||||
current_price: number | null;
|
||||
avg_price_90d: number | null;
|
||||
reasoning: string | null;
|
||||
fetched_at: string;
|
||||
fetched_at: string | null;
|
||||
};
|
||||
|
||||
type ProductListResponse = {
|
||||
@@ -179,6 +189,35 @@ type StalkerProductsResponse = {
|
||||
totalPages: number;
|
||||
};
|
||||
|
||||
type ProductHistoryResponse = {
|
||||
product: {
|
||||
asin: string;
|
||||
name: string | null;
|
||||
brand: string | null;
|
||||
category: string | null;
|
||||
first_seen_at: string;
|
||||
last_seen_at: string;
|
||||
};
|
||||
observations: Array<{
|
||||
id: number;
|
||||
run_id: number;
|
||||
source: string;
|
||||
current_price: number | null;
|
||||
monthly_sold: number | null;
|
||||
sales_rank: number | null;
|
||||
sellability_status: string | null;
|
||||
fetched_at: string;
|
||||
}>;
|
||||
analyses: Array<{
|
||||
id: number;
|
||||
run_id: number;
|
||||
decision: string;
|
||||
confidence: number | null;
|
||||
reasoning: string | null;
|
||||
analyzed_at: string;
|
||||
}>;
|
||||
};
|
||||
|
||||
type SortState = {
|
||||
field: string;
|
||||
direction: SortDirection;
|
||||
@@ -362,7 +401,7 @@ function Dashboard({
|
||||
|
||||
setDeletingKey(key);
|
||||
try {
|
||||
const response = await fetch(`/api/runs/${run.processType}/${run.runId}`, { method: "DELETE" });
|
||||
const response = await fetch(`/api/runs/${run.runId}`, { method: "DELETE" });
|
||||
if (!response.ok) {
|
||||
const errorPayload = await response.json().catch(() => null) as { error?: string } | null;
|
||||
const message = errorPayload?.error ?? "Failed to delete run";
|
||||
@@ -523,7 +562,7 @@ function RunDetails({
|
||||
}) {
|
||||
const [run, setRun] = useState<RunDetail | null>(null);
|
||||
const [results, setResults] = useState<ResultsResponse | null>(null);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [search, setSearch] = useState("");
|
||||
const [verdict, setVerdict] = useState("");
|
||||
const [sellabilityStatus, setSellabilityStatus] = useState("");
|
||||
@@ -545,7 +584,7 @@ function RunDetails({
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
async function loadRun() {
|
||||
const res = await fetch(`/api/runs/${processType}/${runId}`);
|
||||
const res = await fetch(`/api/runs/${runId}`);
|
||||
const payload = (await res.json()) as RunDetail;
|
||||
if (!cancelled) {
|
||||
setRun(payload);
|
||||
@@ -560,7 +599,6 @@ function RunDetails({
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
async function loadResults() {
|
||||
setLoading(true);
|
||||
const params = new URLSearchParams({
|
||||
page: String(page),
|
||||
pageSize: String(pageSize),
|
||||
@@ -573,7 +611,7 @@ function RunDetails({
|
||||
if (minConfidence) params.set("minConfidence", minConfidence);
|
||||
if (maxConfidence) params.set("maxConfidence", maxConfidence);
|
||||
|
||||
const res = await fetch(`/api/runs/${processType}/${runId}/results?${params.toString()}`);
|
||||
const res = await fetch(`/api/runs/${runId}/items?${params.toString()}`);
|
||||
const payload = (await res.json()) as ResultsResponse;
|
||||
if (!cancelled) {
|
||||
setResults(payload);
|
||||
@@ -588,20 +626,25 @@ function RunDetails({
|
||||
}, [processType, runId, search, verdict, sellabilityStatus, amazonSellerFilter, minConfidence, maxConfidence, page, pageSize, sort, refreshTick]);
|
||||
|
||||
useEffect(() => {
|
||||
if (run && run.status !== "running") {
|
||||
return;
|
||||
}
|
||||
|
||||
const interval = window.setInterval(() => {
|
||||
setRefreshTick((tick) => tick + 1);
|
||||
}, 4000);
|
||||
return () => {
|
||||
window.clearInterval(interval);
|
||||
};
|
||||
}, [processType, runId]);
|
||||
}, [processType, runId, run?.status]);
|
||||
|
||||
async function reanalyzeAsin(asin: string) {
|
||||
if (reanalyzing[asin]) return;
|
||||
setReanalyzing((prev) => ({ ...prev, [asin]: true }));
|
||||
async function reanalyzeItem(item: ResultItem) {
|
||||
const key = String(item.item_id);
|
||||
if (reanalyzing[key]) return;
|
||||
setReanalyzing((prev) => ({ ...prev, [key]: true }));
|
||||
try {
|
||||
const response = await fetch(
|
||||
`/api/runs/${processType}/${runId}/asins/${encodeURIComponent(asin)}/reanalyze`,
|
||||
`/api/run-items/${item.item_id}/reanalyze`,
|
||||
{ method: "POST" },
|
||||
);
|
||||
if (!response.ok) {
|
||||
@@ -613,7 +656,7 @@ function RunDetails({
|
||||
} finally {
|
||||
setReanalyzing((prev) => {
|
||||
const next = { ...prev };
|
||||
delete next[asin];
|
||||
delete next[key];
|
||||
return next;
|
||||
});
|
||||
}
|
||||
@@ -626,14 +669,14 @@ function RunDetails({
|
||||
<div className="card">
|
||||
<h2>Run Detail</h2>
|
||||
<div className="meta-grid" style={{ marginTop: 12 }}>
|
||||
<div className="meta"><strong>Process:</strong> {processType}</div>
|
||||
<div className="meta"><strong>Process:</strong> {run?.processType ?? processType}</div>
|
||||
<div className="meta"><strong>Run ID:</strong> {runId}</div>
|
||||
<div className="meta"><strong>Status:</strong> {run ? <span className={statusBadgeClass(run.status)}>{run.status}</span> : "-"}</div>
|
||||
<div className="meta"><strong>Timestamp:</strong> {run ? formatDate(run.timestamp) : "-"}</div>
|
||||
<div className="meta"><strong>Job:</strong> {run?.jobType ?? "-"}</div>
|
||||
<div className="meta"><strong>Source:</strong> {run?.source ?? "-"}</div>
|
||||
<div className="meta"><strong>Total:</strong> {formatNumber(run?.summary.totalProducts)}</div>
|
||||
<div className="meta"><strong>FBA/FBM/SKIP:</strong> {formatNumber(run?.summary.fbaCount)}/{formatNumber(run?.summary.fbmCount)}/{formatNumber(run?.summary.skipCount)}</div>
|
||||
<div className="meta"><strong>FBA/FBM/BUY/WATCH/SKIP:</strong> {formatNumber(run?.summary.fbaCount)}/{formatNumber(run?.summary.fbmCount)}/{formatNumber(run?.summary.buyCount)}/{formatNumber(run?.summary.watchCount)}/{formatNumber(run?.summary.skipCount)}</div>
|
||||
</div>
|
||||
<div style={{ marginTop: 10 }}>
|
||||
<TinyBar fba={run?.summary.fbaCount ?? 0} fbm={run?.summary.fbmCount ?? 0} skip={run?.summary.skipCount ?? 0} />
|
||||
@@ -647,6 +690,8 @@ function RunDetails({
|
||||
<option value="">All verdicts</option>
|
||||
<option value="FBA">FBA</option>
|
||||
<option value="FBM">FBM</option>
|
||||
<option value="BUY">BUY</option>
|
||||
<option value="WATCH">WATCH</option>
|
||||
<option value="SKIP">SKIP</option>
|
||||
</select>
|
||||
<select value={sellabilityStatus} onChange={(e) => { setPage(1); setSellabilityStatus(e.target.value); }}>
|
||||
@@ -677,7 +722,7 @@ function RunDetails({
|
||||
</div>
|
||||
<div style={{ marginTop: 10 }}>
|
||||
<a
|
||||
href={`/api/runs/${processType}/${runId}/export.csv?q=${encodeURIComponent(search)}&verdict=${encodeURIComponent(verdict)}&sellabilityStatus=${encodeURIComponent(sellabilityStatus)}&amazonIsSeller=${encodeURIComponent(amazonSellerFilter)}&minConfidence=${encodeURIComponent(minConfidence)}&maxConfidence=${encodeURIComponent(maxConfidence)}&sort=${encodeURIComponent(buildSortValue(sort))}`}
|
||||
href={`/api/runs/${runId}/export.csv?q=${encodeURIComponent(search)}&verdict=${encodeURIComponent(verdict)}&sellabilityStatus=${encodeURIComponent(sellabilityStatus)}&amazonIsSeller=${encodeURIComponent(amazonSellerFilter)}&minConfidence=${encodeURIComponent(minConfidence)}&maxConfidence=${encodeURIComponent(maxConfidence)}&sort=${encodeURIComponent(buildSortValue(sort))}`}
|
||||
>
|
||||
<button>Export filtered CSV</button>
|
||||
</a>
|
||||
@@ -692,8 +737,8 @@ function RunDetails({
|
||||
<div className="anomaly-list" style={{ marginTop: 8 }}>
|
||||
{anomalies.slice(0, 8).map((item) => (
|
||||
<div key={`anom-${item.asin}-${item.fetched_at}`} className="anomaly-item">
|
||||
<a href={`http://amazon.com/dp/${item.asin}`} target="_blank" rel="noreferrer">{item.asin}</a>
|
||||
<span className={verdictBadgeClass(item.verdict)}>{item.verdict}</span>
|
||||
{item.product_asin ? <a href={`/products/${item.product_asin}`}>{item.asin}</a> : item.asin}
|
||||
{item.verdict ? <span className={verdictBadgeClass(item.verdict)}>{item.verdict}</span> : "-"}
|
||||
<span>{detectAnomaly(item)}</span>
|
||||
</div>
|
||||
))}
|
||||
@@ -729,8 +774,8 @@ function RunDetails({
|
||||
) : results?.items.length ? (
|
||||
results.items.map((item) => (
|
||||
<tr key={`${item.asin}-${item.fetched_at}`}>
|
||||
<td><a href={`http://amazon.com/dp/${item.asin}`} target="_blank" rel="noreferrer">{item.asin}</a></td>
|
||||
<td><span className={verdictBadgeClass(item.verdict)}>{item.verdict}</span></td>
|
||||
<td>{item.product_asin ? <a href={`/products/${item.product_asin}`}>{item.asin}</a> : item.asin}</td>
|
||||
<td>{item.verdict ? <span className={verdictBadgeClass(item.verdict)}>{item.verdict}</span> : "-"}</td>
|
||||
<td>{formatNumber(item.monthly_sold)}</td>
|
||||
<td>{formatNumber(item.seller_count)}</td>
|
||||
<td>{formatAmazonSeller(item.amazon_is_seller)}</td>
|
||||
@@ -744,12 +789,14 @@ function RunDetails({
|
||||
<td>{formatNumber(item.confidence)}</td>
|
||||
<td className="reason-col" title={item.reasoning || undefined}>{item.reasoning || "-"}</td>
|
||||
<td>
|
||||
<button
|
||||
onClick={() => reanalyzeAsin(item.asin)}
|
||||
disabled={Boolean(reanalyzing[item.asin])}
|
||||
>
|
||||
{reanalyzing[item.asin] ? "Re-analyzing..." : "Re-analyze"}
|
||||
</button>
|
||||
{item.product_asin && run?.processType !== "supplier_upc" ? (
|
||||
<button
|
||||
onClick={() => reanalyzeItem(item)}
|
||||
disabled={Boolean(reanalyzing[String(item.item_id)])}
|
||||
>
|
||||
{reanalyzing[String(item.item_id)] ? "Re-analyzing..." : "Re-analyze"}
|
||||
</button>
|
||||
) : "-"}
|
||||
</td>
|
||||
</tr>
|
||||
))
|
||||
@@ -813,12 +860,13 @@ function ProductList({ verdict, onBack }: { verdict: VerdictFilter; onBack: () =
|
||||
}, [search, activeVerdict, amazonSellerFilter, page, pageSize, sort]);
|
||||
|
||||
async function reanalyzeAsin(item: ProductListItem) {
|
||||
const key = `${item.processType}:${item.runId}:${item.asin}`;
|
||||
if (item.item_id == null) return;
|
||||
const key = String(item.item_id);
|
||||
if (reanalyzing[key]) return;
|
||||
setReanalyzing((prev) => ({ ...prev, [key]: true }));
|
||||
try {
|
||||
const response = await fetch(
|
||||
`/api/runs/${item.processType}/${item.runId}/asins/${encodeURIComponent(item.asin)}/reanalyze`,
|
||||
`/api/run-items/${item.item_id}/reanalyze`,
|
||||
{ method: "POST" },
|
||||
);
|
||||
if (!response.ok) {
|
||||
@@ -854,6 +902,8 @@ function ProductList({ verdict, onBack }: { verdict: VerdictFilter; onBack: () =
|
||||
<option value="">All verdicts</option>
|
||||
<option value="FBA">FBA</option>
|
||||
<option value="FBM">FBM</option>
|
||||
<option value="BUY">BUY</option>
|
||||
<option value="WATCH">WATCH</option>
|
||||
<option value="SKIP">SKIP</option>
|
||||
</select>
|
||||
<select
|
||||
@@ -902,8 +952,8 @@ function ProductList({ verdict, onBack }: { verdict: VerdictFilter; onBack: () =
|
||||
) : items?.items.length ? (
|
||||
items.items.map((item) => (
|
||||
<tr key={`${item.processType}-${item.runId}-${item.asin}-${item.fetched_at}`}>
|
||||
<td><a href={`http://amazon.com/dp/${item.asin}`} target="_blank" rel="noreferrer">{item.asin}</a></td>
|
||||
<td><span className={verdictBadgeClass(item.verdict)}>{item.verdict}</span></td>
|
||||
<td><a href={`/products/${item.asin}`}>{item.asin}</a></td>
|
||||
<td>{item.verdict ? <span className={verdictBadgeClass(item.verdict)}>{item.verdict}</span> : "-"}</td>
|
||||
<td>{formatNumber(item.monthly_sold)}</td>
|
||||
<td>{formatNumber(item.seller_count)}</td>
|
||||
<td>{formatAmazonSeller(item.amazon_is_seller)}</td>
|
||||
@@ -917,12 +967,14 @@ function ProductList({ verdict, onBack }: { verdict: VerdictFilter; onBack: () =
|
||||
<td>{formatNumber(item.confidence)}</td>
|
||||
<td className="reason-col" title={item.reasoning || undefined}>{item.reasoning || "-"}</td>
|
||||
<td>
|
||||
<button
|
||||
onClick={() => reanalyzeAsin(item)}
|
||||
disabled={Boolean(reanalyzing[`${item.processType}:${item.runId}:${item.asin}`])}
|
||||
>
|
||||
{reanalyzing[`${item.processType}:${item.runId}:${item.asin}`] ? "Re-analyzing..." : "Re-analyze"}
|
||||
</button>
|
||||
{item.item_id == null || item.processType === "supplier_upc" ? "-" : (
|
||||
<button
|
||||
onClick={() => reanalyzeAsin(item)}
|
||||
disabled={Boolean(reanalyzing[String(item.item_id)])}
|
||||
>
|
||||
{reanalyzing[String(item.item_id)] ? "Re-analyzing..." : "Re-analyze"}
|
||||
</button>
|
||||
)}
|
||||
</td>
|
||||
</tr>
|
||||
))
|
||||
@@ -995,7 +1047,7 @@ function StalkerExplorer({
|
||||
}, [search, sellerId, runId, minRatingCount, maxRatingCount, page, pageSize, sort, refreshTick]);
|
||||
|
||||
async function purgeStalkerData() {
|
||||
const confirmed = window.confirm("Permanently delete all Stalker runs, sellers, and sellable products from the database?");
|
||||
const confirmed = window.confirm("Permanently delete all Stalker runs and unreferenced seller records? Canonical products are retained.");
|
||||
if (!confirmed) return;
|
||||
|
||||
setPurging(true);
|
||||
@@ -1384,17 +1436,95 @@ function StalkerProductsExplorer({
|
||||
);
|
||||
}
|
||||
|
||||
function ProductDetails({
|
||||
asin,
|
||||
onBack,
|
||||
}: {
|
||||
asin: string;
|
||||
onBack: () => void;
|
||||
}) {
|
||||
const [data, setData] = useState<ProductHistoryResponse | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
fetch(`/api/products/${encodeURIComponent(asin)}`)
|
||||
.then((response) => response.json())
|
||||
.then((payload: ProductHistoryResponse) => {
|
||||
if (!cancelled) setData(payload);
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [asin]);
|
||||
|
||||
return (
|
||||
<div className="page">
|
||||
<button className="back" onClick={onBack}>Back</button>
|
||||
<div className="card">
|
||||
<h2>{data?.product.name ?? asin}</h2>
|
||||
<div className="meta-grid" style={{ marginTop: 12 }}>
|
||||
<div className="meta"><strong>ASIN:</strong> <a href={`https://amazon.com/dp/${asin}`} target="_blank" rel="noreferrer">{asin}</a></div>
|
||||
<div className="meta"><strong>Brand:</strong> {data?.product.brand ?? "-"}</div>
|
||||
<div className="meta"><strong>Category:</strong> {data?.product.category ?? "-"}</div>
|
||||
<div className="meta"><strong>Last seen:</strong> {data ? formatDate(data.product.last_seen_at) : "-"}</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className="card">
|
||||
<h3>Analysis History</h3>
|
||||
<div className="table-wrap">
|
||||
<table>
|
||||
<thead><tr><th>Run</th><th>Decision</th><th>Confidence</th><th>Reasoning</th><th>Analyzed</th></tr></thead>
|
||||
<tbody>
|
||||
{data?.analyses.length ? data.analyses.map((analysis) => (
|
||||
<tr key={analysis.id}>
|
||||
<td>{analysis.run_id}</td>
|
||||
<td>{analysis.decision}</td>
|
||||
<td>{formatNumber(analysis.confidence)}</td>
|
||||
<td>{analysis.reasoning ?? "-"}</td>
|
||||
<td>{formatDate(analysis.analyzed_at)}</td>
|
||||
</tr>
|
||||
)) : <tr><td colSpan={5}>No analysis history</td></tr>}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
<div className="card">
|
||||
<h3>Observations</h3>
|
||||
<div className="table-wrap">
|
||||
<table>
|
||||
<thead><tr><th>Run</th><th>Source</th><th>Price</th><th>Monthly Sold</th><th>Sales Rank</th><th>Sellability</th><th>Fetched</th></tr></thead>
|
||||
<tbody>
|
||||
{data?.observations.length ? data.observations.map((observation) => (
|
||||
<tr key={observation.id}>
|
||||
<td>{observation.run_id}</td>
|
||||
<td>{observation.source}</td>
|
||||
<td>{formatCurrency(observation.current_price)}</td>
|
||||
<td>{formatNumber(observation.monthly_sold)}</td>
|
||||
<td>{formatNumber(observation.sales_rank)}</td>
|
||||
<td>{observation.sellability_status ?? "-"}</td>
|
||||
<td>{formatDate(observation.fetched_at)}</td>
|
||||
</tr>
|
||||
)) : <tr><td colSpan={7}>No observations</td></tr>}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
type AppRoute =
|
||||
| { kind: "dashboard" }
|
||||
| { kind: "run"; processType: ProcessType; runId: number }
|
||||
| { kind: "products"; verdict: VerdictFilter }
|
||||
| { kind: "product"; asin: string }
|
||||
| { kind: "stalker" }
|
||||
| { kind: "stalker-products" };
|
||||
|
||||
function parseRoute(pathname: string, search: string): AppRoute {
|
||||
const runMatch = pathname.match(/^\/runs\/(lead_analysis|category_analysis)\/(\d+)$/);
|
||||
const runMatch = pathname.match(/^\/runs\/(\d+)$/);
|
||||
if (runMatch) {
|
||||
return { kind: "run", processType: runMatch[1] as ProcessType, runId: Number(runMatch[2]) };
|
||||
return { kind: "run", processType: "lead_analysis", runId: Number(runMatch[1]) };
|
||||
}
|
||||
|
||||
if (pathname === "/products") {
|
||||
@@ -1404,6 +1534,11 @@ function parseRoute(pathname: string, search: string): AppRoute {
|
||||
return { kind: "products", verdict };
|
||||
}
|
||||
|
||||
const productMatch = pathname.match(/^\/products\/([A-Z0-9]{10})$/i);
|
||||
if (productMatch) {
|
||||
return { kind: "product", asin: productMatch[1]!.toUpperCase() };
|
||||
}
|
||||
|
||||
if (pathname === "/stalker") {
|
||||
return { kind: "stalker" };
|
||||
}
|
||||
@@ -1425,7 +1560,7 @@ function App() {
|
||||
}, []);
|
||||
|
||||
function openRun(run: Run) {
|
||||
const path = `/runs/${run.processType}/${run.runId}`;
|
||||
const path = `/runs/${run.runId}`;
|
||||
history.pushState({}, "", path);
|
||||
setRoute({ kind: "run", processType: run.processType, runId: run.runId });
|
||||
}
|
||||
@@ -1459,6 +1594,10 @@ function App() {
|
||||
return <ProductList verdict={route.verdict} onBack={backToDashboard} />;
|
||||
}
|
||||
|
||||
if (route.kind === "product") {
|
||||
return <ProductDetails asin={route.asin} onBack={backToDashboard} />;
|
||||
}
|
||||
|
||||
if (route.kind === "stalker") {
|
||||
return <StalkerExplorer onBack={backToDashboard} onOpenProducts={openStalkerProducts} />;
|
||||
}
|
||||
|
||||
324
src/writer.ts
324
src/writer.ts
@@ -1,4 +1,11 @@
|
||||
import { getDb } from "./database.ts";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "./db/index.ts";
|
||||
import { analysisRunStats, runs } from "./db/schema.ts";
|
||||
import {
|
||||
persistLlmResults,
|
||||
persistSupplierResults,
|
||||
refreshRunStats,
|
||||
} from "./db/persistence.ts";
|
||||
import type { AnalysisResult, SupplierAnalysisResult } from "./types.ts";
|
||||
import { mkdirSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
@@ -11,18 +18,6 @@ export type RunCounts = {
|
||||
skipCount: number;
|
||||
};
|
||||
|
||||
function computeRunCountsFromResults(results: AnalysisResult[]): RunCounts {
|
||||
const fbaCount = results.filter((r) => r.verdict.verdict === "FBA").length;
|
||||
const fbmCount = results.filter((r) => r.verdict.verdict === "FBM").length;
|
||||
const skipCount = results.filter((r) => r.verdict.verdict === "SKIP").length;
|
||||
return {
|
||||
totalProducts: results.length,
|
||||
fbaCount,
|
||||
fbmCount,
|
||||
skipCount,
|
||||
};
|
||||
}
|
||||
|
||||
function buildRow(r: AnalysisResult) {
|
||||
const price =
|
||||
r.product.keepa?.currentPrice ??
|
||||
@@ -84,16 +79,21 @@ function buildRow(r: AnalysisResult) {
|
||||
};
|
||||
}
|
||||
|
||||
export function writeResultsToDb(
|
||||
export async function writeResultsToDb(
|
||||
results: AnalysisResult[],
|
||||
dbPath: string,
|
||||
inputFile: string,
|
||||
outputFile: string | undefined,
|
||||
): void {
|
||||
const runCounts = computeRunCountsFromResults(results);
|
||||
const runId = startRunInDb(dbPath, inputFile, outputFile, runCounts);
|
||||
appendResultsToRun(dbPath, runId, results);
|
||||
console.log(`Results written to SQLite database for run_id: ${runId}`);
|
||||
): Promise<void> {
|
||||
const runId = await startRunInDb(inputFile, outputFile);
|
||||
try {
|
||||
await appendResultsToRun(runId, results);
|
||||
await refreshRunCountsInDb(runId);
|
||||
await completeRunInDb(runId);
|
||||
} catch (error) {
|
||||
await failRunInDb(runId, error);
|
||||
throw error;
|
||||
}
|
||||
console.log(`Results written to database for run_id: ${runId}`);
|
||||
}
|
||||
|
||||
export function writeResultsWorkbook(
|
||||
@@ -112,8 +112,7 @@ export function writeResultsWorkbook(
|
||||
console.log(`Results workbook written: ${outputFile}`);
|
||||
}
|
||||
|
||||
export function startRunInDb(
|
||||
dbPath: string,
|
||||
export async function startRunInDb(
|
||||
inputFile: string,
|
||||
outputFile: string | undefined,
|
||||
counts: RunCounts = {
|
||||
@@ -122,244 +121,73 @@ export function startRunInDb(
|
||||
fbmCount: 0,
|
||||
skipCount: 0,
|
||||
},
|
||||
): number {
|
||||
const database = getDb(dbPath);
|
||||
const timestamp = new Date().toISOString();
|
||||
type: "lead_analysis" | "supplier_upc" = "lead_analysis",
|
||||
): Promise<number> {
|
||||
const [row] = await db
|
||||
.insert(runs)
|
||||
.values({
|
||||
type,
|
||||
inputFile,
|
||||
outputFile: outputFile ?? null,
|
||||
status: "running",
|
||||
startedAt: new Date(),
|
||||
})
|
||||
.returning({ id: runs.id });
|
||||
|
||||
const insertRun = database.prepare(
|
||||
`INSERT INTO runs (
|
||||
timestamp,
|
||||
input_file,
|
||||
output_file,
|
||||
total_products,
|
||||
fba_count,
|
||||
fbm_count,
|
||||
skip_count
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
||||
);
|
||||
|
||||
const runInfo = insertRun.run(
|
||||
timestamp,
|
||||
inputFile,
|
||||
outputFile ?? null,
|
||||
counts.totalProducts,
|
||||
counts.fbaCount,
|
||||
counts.fbmCount,
|
||||
counts.skipCount,
|
||||
);
|
||||
|
||||
const runId =
|
||||
(runInfo.changes as number) > 0
|
||||
? (runInfo.lastInsertRowid as number)
|
||||
: null;
|
||||
|
||||
if (runId === null) {
|
||||
throw new Error("Failed to insert run record into SQLite.");
|
||||
}
|
||||
|
||||
return runId;
|
||||
if (!row) throw new Error("Failed to insert run record.");
|
||||
await db.insert(analysisRunStats).values({
|
||||
runId: row.id,
|
||||
processedCount: counts.totalProducts,
|
||||
analyzedCount: counts.totalProducts,
|
||||
fbaCount: counts.fbaCount,
|
||||
fbmCount: counts.fbmCount,
|
||||
skipCount: counts.skipCount,
|
||||
});
|
||||
return row.id;
|
||||
}
|
||||
|
||||
export function appendResultsToRun(
|
||||
dbPath: string,
|
||||
export async function appendResultsToRun(
|
||||
runId: number,
|
||||
results: AnalysisResult[],
|
||||
): void {
|
||||
if (results.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const database = getDb(dbPath);
|
||||
const insertResult = database.prepare(
|
||||
`INSERT INTO results (
|
||||
run_id, asin, product_name, brand, category, unit_cost, current_price,
|
||||
avg_price_90d, avg_price_90d_sheet, selling_price_sheet, sales_rank, rank_avg_90d,
|
||||
sellers, amazon_is_seller, amazon_buybox_share_pct_90d,
|
||||
monthly_sold, rank_drops_30d, rank_drops_90d, fba_net_sheet,
|
||||
gross_profit_dollar, gross_profit_pct, net_profit_sheet, roi_sheet, moq, moq_cost,
|
||||
qty_available, supplier, source_url, asin_link, promo_coupon_code, notes, lead_date,
|
||||
fba_fee, fbm_fee, referral_percent, can_sell, sellability_status, sellability_reason,
|
||||
verdict, confidence, reasoning, fetched_at
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
|
||||
)`,
|
||||
);
|
||||
|
||||
database.transaction(() => {
|
||||
for (const r of results) {
|
||||
const row = buildRow(r);
|
||||
insertResult.run(
|
||||
runId,
|
||||
row.ASIN,
|
||||
row.Name,
|
||||
row.Brand,
|
||||
row.Category,
|
||||
row["Unit Cost"] ?? null,
|
||||
row["Current Price"] ?? null,
|
||||
row["Avg Price 90d"] ?? null,
|
||||
row["Avg Price 90d (sheet)"] ?? null,
|
||||
row["Selling Price (sheet)"] ?? null,
|
||||
row["Sales Rank"] ?? null,
|
||||
row["Rank Avg 90d"] ?? null,
|
||||
row.Sellers ?? null,
|
||||
row["Amazon Is Seller"] == null
|
||||
? null
|
||||
: row["Amazon Is Seller"]
|
||||
? 1
|
||||
: 0,
|
||||
row["Amazon Buy Box Share 90d %"] ?? null,
|
||||
row["Monthly Sold"] ?? null,
|
||||
row["Rank Drops 30d"] ?? null,
|
||||
row["Rank Drops 90d"] ?? null,
|
||||
row["FBA Net (sheet)"] ?? null,
|
||||
row["Gross Profit $"] ?? null,
|
||||
row["Gross Profit %"] ?? null,
|
||||
row["Net Profit (sheet)"] ?? null,
|
||||
row["ROI (sheet)"] ?? null,
|
||||
row.MOQ ?? null,
|
||||
row["MOQ Cost"] ?? null,
|
||||
row["Qty Available"] ?? null,
|
||||
row.Supplier ?? null,
|
||||
row["Source URL"] ?? null,
|
||||
row["ASIN Link"] ?? null,
|
||||
row["Promo/Coupon Code"] ?? null,
|
||||
row.Notes ?? null,
|
||||
row["Lead Date"] ?? null,
|
||||
row["FBA Fee"] ?? null,
|
||||
row["FBM Fee"] ?? null,
|
||||
row["Referral %"] ?? null,
|
||||
row["Can Sell"],
|
||||
row.Sellability,
|
||||
row["Sellability Reason"] ?? null,
|
||||
row.Verdict,
|
||||
row.Confidence ?? null,
|
||||
row.Reasoning,
|
||||
r.product.fetchedAt,
|
||||
);
|
||||
}
|
||||
})();
|
||||
): Promise<void> {
|
||||
if (results.length === 0) return;
|
||||
await persistLlmResults(runId, results, {
|
||||
source: "lead_analysis",
|
||||
metadataSource: "input",
|
||||
preserveSourcingInput: true,
|
||||
});
|
||||
}
|
||||
|
||||
export function appendSupplierResultsToRun(
|
||||
dbPath: string,
|
||||
export async function appendSupplierResultsToRun(
|
||||
runId: number,
|
||||
results: SupplierAnalysisResult[],
|
||||
): void {
|
||||
if (results.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const database = getDb(dbPath);
|
||||
const insertResult = database.prepare(
|
||||
`INSERT INTO results (
|
||||
run_id, asin, product_name, brand, category, unit_cost, current_price,
|
||||
avg_price_90d, sales_rank, rank_avg_90d, sellers,
|
||||
amazon_is_seller, amazon_buybox_share_pct_90d, monthly_sold,
|
||||
rank_drops_30d, rank_drops_90d, upc, fba_fee, fbm_fee,
|
||||
referral_percent, supplier_score, supplier_profit, supplier_margin,
|
||||
supplier_roi, supplier_reason, upc_lookup_status, upc_lookup_reason,
|
||||
candidate_asins, can_sell, sellability_status, sellability_reason,
|
||||
verdict, confidence, reasoning, fetched_at
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
|
||||
)`,
|
||||
);
|
||||
|
||||
database.transaction(() => {
|
||||
for (const result of results) {
|
||||
const keepa = result.keepa;
|
||||
const spApi = result.spApi;
|
||||
const asin = result.lookup.asin ?? result.record.asin ?? result.upc;
|
||||
const category =
|
||||
result.record.category ?? keepa?.categoryTree?.join(" > ") ?? null;
|
||||
const canSell =
|
||||
spApi?.canSell == null ? null : spApi.canSell ? "yes" : "no";
|
||||
|
||||
insertResult.run(
|
||||
runId,
|
||||
asin,
|
||||
result.record.name,
|
||||
result.record.brand ?? null,
|
||||
category,
|
||||
result.record.unitCost || null,
|
||||
result.score.salePrice,
|
||||
keepa?.avgPrice90 ?? null,
|
||||
keepa?.salesRank ?? null,
|
||||
keepa?.salesRankAvg90 ?? null,
|
||||
keepa?.sellerCount ?? null,
|
||||
keepa?.amazonIsSeller == null ? null : keepa.amazonIsSeller ? 1 : 0,
|
||||
keepa?.amazonBuyboxSharePct90d ?? null,
|
||||
keepa?.monthlySold ?? null,
|
||||
keepa?.salesRankDrops30 ?? null,
|
||||
keepa?.salesRankDrops90 ?? null,
|
||||
result.upc,
|
||||
result.score.fbaFee,
|
||||
spApi?.fbmFee ?? null,
|
||||
spApi?.referralFeePercent ?? null,
|
||||
result.score.score,
|
||||
result.score.profit,
|
||||
result.score.margin,
|
||||
result.score.roi,
|
||||
result.score.reason,
|
||||
result.lookup.status,
|
||||
result.lookup.reason ?? null,
|
||||
result.lookup.candidateAsins.join(","),
|
||||
canSell,
|
||||
spApi?.sellabilityStatus ?? null,
|
||||
spApi?.sellabilityReason ?? null,
|
||||
result.score.verdict,
|
||||
Math.round(result.score.score),
|
||||
result.score.reason,
|
||||
result.fetchedAt,
|
||||
);
|
||||
}
|
||||
})();
|
||||
): Promise<void> {
|
||||
if (results.length === 0) return;
|
||||
await persistSupplierResults(runId, results);
|
||||
}
|
||||
|
||||
export function refreshRunCountsInDb(dbPath: string, runId: number): RunCounts {
|
||||
const database = getDb(dbPath);
|
||||
const stats = database
|
||||
.query(
|
||||
`SELECT
|
||||
COUNT(*) AS total,
|
||||
SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba,
|
||||
SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm,
|
||||
SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip
|
||||
FROM results
|
||||
WHERE run_id = ?`,
|
||||
)
|
||||
.get(runId) as {
|
||||
total: number;
|
||||
fba: number | null;
|
||||
fbm: number | null;
|
||||
skip: number | null;
|
||||
};
|
||||
|
||||
const counts: RunCounts = {
|
||||
totalProducts: stats.total ?? 0,
|
||||
fbaCount: stats.fba ?? 0,
|
||||
fbmCount: stats.fbm ?? 0,
|
||||
skipCount: stats.skip ?? 0,
|
||||
};
|
||||
|
||||
database
|
||||
.query(
|
||||
`UPDATE runs
|
||||
SET total_products = ?, fba_count = ?, fbm_count = ?, skip_count = ?
|
||||
WHERE id = ?`,
|
||||
)
|
||||
.run(
|
||||
counts.totalProducts,
|
||||
counts.fbaCount,
|
||||
counts.fbmCount,
|
||||
counts.skipCount,
|
||||
runId,
|
||||
);
|
||||
|
||||
return counts;
|
||||
export async function refreshRunCountsInDb(runId: number): Promise<RunCounts> {
|
||||
return refreshRunStats(runId);
|
||||
}
|
||||
|
||||
export async function completeRunInDb(runId: number): Promise<void> {
|
||||
await db
|
||||
.update(runs)
|
||||
.set({ status: "completed", completedAt: new Date(), errorMessage: null })
|
||||
.where(eq(runs.id, runId));
|
||||
}
|
||||
|
||||
export async function failRunInDb(
|
||||
runId: number,
|
||||
error: unknown,
|
||||
): Promise<void> {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
await db
|
||||
.update(runs)
|
||||
.set({ status: "failed", completedAt: new Date(), errorMessage })
|
||||
.where(eq(runs.id, runId));
|
||||
}
|
||||
|
||||
export function printResults(results: AnalysisResult[]): void {
|
||||
const rows = results
|
||||
.filter((r) => r.verdict.verdict === "FBA" || r.verdict.verdict === "FBM")
|
||||
|
||||
Reference in New Issue
Block a user