feat: add supplier scoring and UPC file analysis functionality
- Implemented supplier scoring logic in `supplier-scoring.ts` with functions to compute demand score, competition penalty, and overall supplier product score. - Created unit tests for supplier scoring in `supplier-scoring.test.ts` to validate scoring logic against various scenarios. - Developed UPC file analysis tool in `upc-file-analysis.ts` to process UPCs in batches, fetch product data from Keepa and SP-API, and generate supplier results. - Added UPC input reading functionality in `upc-file-reader.ts` to handle XLSX and XLS files, including validation for UPC formats. - Introduced a command-line tool in `upc-lookup.ts` for looking up UPCs and displaying detailed results or mappings to ASINs. - Enhanced error handling and logging throughout the new modules for better traceability and user feedback.
This commit is contained in:
42
CLAUDE.md
42
CLAUDE.md
@@ -12,8 +12,8 @@ Default to using Bun instead of Node.js.
|
||||
|
||||
## APIs
|
||||
|
||||
- `bun:sqlite` for SQLite. Don't use `better-sqlite3`.
|
||||
- `Bun.redis` for Redis. Don't use `ioredis`.
|
||||
- Use Drizzle ORM with `postgres` driver for Postgres. Connection is in `src/db/index.ts`.
|
||||
- Prefer `Bun.file` over `node:fs`'s readFile/writeFile.
|
||||
- `Bun.$\`cmd\`` instead of execa.
|
||||
|
||||
@@ -24,7 +24,7 @@ Default to using Bun instead of Node.js.
|
||||
bun test
|
||||
|
||||
# Run a single test file
|
||||
bun test src/supplier-scoring.test.ts
|
||||
bun test src/supplier/supplier-scoring.test.ts
|
||||
|
||||
# Type-check (no emit)
|
||||
./node_modules/.bin/tsc --noEmit
|
||||
@@ -40,6 +40,9 @@ bun run bestsellers
|
||||
bun run monthly-sold
|
||||
bun run mid-range
|
||||
|
||||
# Stalker pipeline
|
||||
bun run stalker --input input/asins.xlsx
|
||||
|
||||
# Web API server
|
||||
bun run start:web # http://localhost:3000
|
||||
|
||||
@@ -47,29 +50,37 @@ bun run start:web # http://localhost:3000
|
||||
bun run src/sp-test.ts
|
||||
bun run src/sp-test.ts B07SN9BHVV
|
||||
bun run src/sp-test.ts --sellability B07SN9BHVV
|
||||
|
||||
# Database migrations (Drizzle)
|
||||
bun run db:generate
|
||||
bun run db:migrate
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
||||
Two distinct analysis pipelines share infrastructure (Keepa, SP-API, Redis, SQLite) but diverge in how they produce verdicts.
|
||||
Two distinct analysis pipelines share infrastructure (Keepa, SP-API, Redis, Postgres) but diverge in how they produce verdicts.
|
||||
|
||||
### ASIN Lead-list Pipeline (`src/index.ts` → `src/analysis-pipeline.ts`)
|
||||
|
||||
For spreadsheets containing known ASINs. Verdict is LLM-based (FBA/FBM/SKIP via LM Studio).
|
||||
|
||||
Flow: `reader.ts` parse → Redis cache check → `sp-api.ts` sellability gate (5 concurrent workers) → `keepa.ts` batch enrichment → `sp-api.ts` pricing + FBA fees (5 concurrent workers) → `llm.ts` batched analysis (5 products/batch) → `writer.ts` XLSX + SQLite.
|
||||
Flow: `reader.ts` parse → Redis cache check → `integrations/sp-api.ts` sellability gate (5 concurrent workers) → `integrations/keepa.ts` batch enrichment → `integrations/sp-api.ts` pricing + FBA fees (5 concurrent workers) → `integrations/llm.ts` batched analysis (5 products/batch) → `writer.ts` XLSX + Postgres.
|
||||
|
||||
### Supplier UPC Pipeline (`src/upc-file-analysis.ts`)
|
||||
### Supplier UPC Pipeline (`src/supplier/upc-file-analysis.ts`)
|
||||
|
||||
For supplier price lists containing UPC/EAN values. Verdict is deterministic (BUY/WATCH/SKIP); never calls LM Studio.
|
||||
|
||||
Flow: `upc-file-reader.ts` streaming parse (`.xlsx`) or row-window parse (`.xls`) → SP-API catalog UPC lookup first, Keepa UPC lookup as fallback → `keepa.ts` demand enrichment → `sp-api.ts` sellability + FBA fees → `supplier-scoring.ts` deterministic score → `supplier-export.ts` Excel workbook (`Ranked Leads`, `Skipped`, `Summary` sheets) + SQLite.
|
||||
Flow: `supplier/upc-file-reader.ts` streaming parse (`.xlsx`) or row-window parse (`.xls`) → SP-API catalog UPC lookup first, Keepa UPC lookup as fallback → `integrations/keepa.ts` demand enrichment → `integrations/sp-api.ts` sellability + FBA fees → `supplier/supplier-scoring.ts` deterministic score → `supplier/supplier-export.ts` Excel workbook (`Ranked Leads`, `Skipped`, `Summary` sheets) + Postgres.
|
||||
|
||||
UPC resolution priority: SP-API catalog lookup → Keepa fallback (for no-match or request failure only).
|
||||
|
||||
### Category Pipelines
|
||||
|
||||
`bestsellers-by-category.ts`, `top-monthly-sold-by-category.ts`, `mid-range-sellers-by-category.ts` — Keepa category browsing → SP-API sellability gate → LLM verdict. Each saves results to SQLite. Mid-range applies configurable filters (monthly sold, price, seller count, Amazon buy box share).
|
||||
`src/categories/` — Keepa category browsing → SP-API sellability gate → LLM verdict. Each saves results to Postgres. Mid-range applies configurable filters (monthly sold, price, seller count, Amazon buy box share).
|
||||
|
||||
### Stalker Pipeline (`src/stalker/stalker.ts`)
|
||||
|
||||
Tracks competitor sellers across ASINs. Fetches storefronts, checks sellability of inventory items, and persists matched seller data to Postgres.
|
||||
|
||||
### Shared Infrastructure
|
||||
|
||||
@@ -77,18 +88,23 @@ UPC resolution priority: SP-API catalog lookup → Keepa fallback (for no-match
|
||||
|--------|------|
|
||||
| `src/types.ts` | All shared interfaces (`ProductRecord`, `KeepaData`, `SpApiData`, `SupplierScore`, etc.) |
|
||||
| `src/config.ts` | Env var loading via `Bun.env` |
|
||||
| `src/keepa.ts` | Keepa API: batch ASIN fetch, UPC lookup, auto rate-limiting on token exhaustion |
|
||||
| `src/sp-api.ts` | SP-API: sellability (`getListingsRestrictions`), pricing+fees, UPC catalog lookup |
|
||||
| `src/cache.ts` | Redis caching (24h TTL for lead-list; 12h for mid-range) |
|
||||
| `src/database.ts` | SQLite `runs` + `results` tables; auto-creates `db/results.db` |
|
||||
| `src/db/index.ts` | Drizzle Postgres connection (shared pool) |
|
||||
| `src/db/schema.ts` | Drizzle schema for all tables |
|
||||
| `src/integrations/keepa.ts` | Keepa API: batch ASIN fetch, UPC lookup, auto rate-limiting |
|
||||
| `src/integrations/sp-api.ts` | SP-API: sellability, pricing+fees, UPC catalog lookup |
|
||||
| `src/integrations/cache.ts` | Redis caching (24h TTL for lead-list; 12h for mid-range) |
|
||||
| `src/integrations/llm.ts` | LLM integration (LM Studio / Claude) |
|
||||
| `src/server.ts` | Bun HTTP server exposing REST endpoints for both pipelines |
|
||||
|
||||
### File Layout
|
||||
|
||||
- `src/integrations/` — external API clients (Keepa, SP-API, Redis cache, LLM, SearXNG)
|
||||
- `src/categories/` — category discovery pipelines
|
||||
- `src/stalker/` — competitor seller tracking pipeline
|
||||
- `src/supplier/` — supplier UPC analysis pipeline
|
||||
- `src/db/` — Drizzle schema and connection
|
||||
- `input/` — source spreadsheets (git-ignored)
|
||||
- `output/` — generated workbooks (git-ignored)
|
||||
- `db/` — SQLite files (git-ignored)
|
||||
- `src/` — all source and test files
|
||||
|
||||
## Project Rules
|
||||
|
||||
|
||||
217
drizzle/0000_gorgeous_william_stryker.sql
Normal file
217
drizzle/0000_gorgeous_william_stryker.sql
Normal file
@@ -0,0 +1,217 @@
|
||||
CREATE TYPE "public"."run_status" AS ENUM('running', 'ok', 'empty', 'failed', 'completed');--> statement-breakpoint
|
||||
CREATE TYPE "public"."run_type" AS ENUM('lead_analysis', 'category_analysis', 'supplier_upc', 'stalker');--> statement-breakpoint
|
||||
CREATE TABLE "analysis_results" (
|
||||
"id" serial PRIMARY KEY NOT NULL,
|
||||
"run_id" integer NOT NULL,
|
||||
"asin" text NOT NULL,
|
||||
"product_name" text,
|
||||
"brand" text,
|
||||
"category" text,
|
||||
"upc" text,
|
||||
"unit_cost" real,
|
||||
"avg_price_90d_sheet" real,
|
||||
"selling_price_sheet" real,
|
||||
"fba_net_sheet" real,
|
||||
"gross_profit_dollar" real,
|
||||
"gross_profit_pct" real,
|
||||
"net_profit_sheet" real,
|
||||
"roi_sheet" real,
|
||||
"moq" integer,
|
||||
"moq_cost" real,
|
||||
"qty_available" integer,
|
||||
"supplier" text,
|
||||
"source_url" text,
|
||||
"asin_link" text,
|
||||
"promo_coupon_code" text,
|
||||
"notes" text,
|
||||
"lead_date" text,
|
||||
"current_price" real,
|
||||
"avg_price_90d" real,
|
||||
"sales_rank" integer,
|
||||
"rank_avg_90d" integer,
|
||||
"monthly_sold" integer,
|
||||
"rank_drops_30d" integer,
|
||||
"rank_drops_90d" integer,
|
||||
"seller_count" integer,
|
||||
"amazon_is_seller" boolean,
|
||||
"amazon_buybox_share_pct_90d" real,
|
||||
"fba_fee" real,
|
||||
"fbm_fee" real,
|
||||
"referral_percent" real,
|
||||
"can_sell" text,
|
||||
"sellability_status" text,
|
||||
"sellability_reason" text,
|
||||
"supplier_score" real,
|
||||
"supplier_profit" real,
|
||||
"supplier_margin" real,
|
||||
"supplier_roi" real,
|
||||
"supplier_reason" text,
|
||||
"upc_lookup_status" text,
|
||||
"upc_lookup_reason" text,
|
||||
"candidate_asins" text,
|
||||
"verdict" text NOT NULL,
|
||||
"confidence" real,
|
||||
"reasoning" text,
|
||||
"fetched_at" timestamp with time zone NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "category_product_results" (
|
||||
"id" serial PRIMARY KEY NOT NULL,
|
||||
"asin" text NOT NULL,
|
||||
"run_id" integer NOT NULL,
|
||||
"name" text NOT NULL,
|
||||
"brand" text,
|
||||
"category" text,
|
||||
"unit_cost" real,
|
||||
"current_price" real,
|
||||
"avg_price_90d" real,
|
||||
"avg_price_90d_sheet" real,
|
||||
"selling_price_sheet" real,
|
||||
"sales_rank" integer,
|
||||
"sales_rank_avg_90d" integer,
|
||||
"seller_count" integer,
|
||||
"amazon_is_seller" boolean,
|
||||
"amazon_buybox_share_pct_90d" real,
|
||||
"monthly_sold" integer,
|
||||
"rank_drops_30d" integer,
|
||||
"rank_drops_90d" integer,
|
||||
"fba_fee" real,
|
||||
"fbm_fee" real,
|
||||
"referral_percent" real,
|
||||
"can_sell" text,
|
||||
"sellability_status" text,
|
||||
"sellability_reason" text,
|
||||
"verdict" text NOT NULL,
|
||||
"confidence" real NOT NULL,
|
||||
"reasoning" text,
|
||||
"fetched_at" timestamp with time zone NOT NULL,
|
||||
CONSTRAINT "category_product_results_asin_unique" UNIQUE("asin")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "runs" (
|
||||
"id" serial PRIMARY KEY NOT NULL,
|
||||
"type" "run_type" NOT NULL,
|
||||
"input_file" text,
|
||||
"output_file" text,
|
||||
"status" "run_status" DEFAULT 'running' NOT NULL,
|
||||
"error_message" text,
|
||||
"total_products" integer,
|
||||
"fba_count" integer,
|
||||
"fbm_count" integer,
|
||||
"skip_count" integer,
|
||||
"category_id" integer,
|
||||
"category_label" text,
|
||||
"top_asins_checked" integer,
|
||||
"available_asins" integer,
|
||||
"started_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"completed_at" timestamp with time zone
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "sellers" (
|
||||
"seller_id" text PRIMARY KEY NOT NULL,
|
||||
"seller_name" text,
|
||||
"rating" real,
|
||||
"rating_count" integer,
|
||||
"storefront_asin_total" integer,
|
||||
"persisted_inventory_sample_count" integer,
|
||||
"last_updated_at" timestamp with time zone NOT NULL,
|
||||
"raw_seller_json" text
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "stalker_asin_scans" (
|
||||
"id" serial PRIMARY KEY NOT NULL,
|
||||
"run_id" integer NOT NULL,
|
||||
"source_asin" text NOT NULL,
|
||||
"title" text,
|
||||
"offer_count" integer DEFAULT 0 NOT NULL,
|
||||
"candidate_seller_count" integer DEFAULT 0 NOT NULL,
|
||||
"matched_seller_count" integer DEFAULT 0 NOT NULL,
|
||||
"fetched_at" timestamp with time zone NOT NULL,
|
||||
"raw_product_json" text,
|
||||
CONSTRAINT "uq_stalker_scans_run_asin" UNIQUE("run_id","source_asin")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "stalker_asin_sellers" (
|
||||
"id" serial PRIMARY KEY NOT NULL,
|
||||
"scan_id" integer NOT NULL,
|
||||
"seller_id" text NOT NULL,
|
||||
"offer_price" real,
|
||||
"condition" text,
|
||||
"is_fba" boolean,
|
||||
"stock" integer,
|
||||
"seller_rating" real,
|
||||
"seller_rating_count" integer,
|
||||
"raw_offer_json" text,
|
||||
CONSTRAINT "uq_stalker_asin_sellers_scan_seller" UNIQUE("scan_id","seller_id")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "stalker_runs" (
|
||||
"id" serial PRIMARY KEY NOT NULL,
|
||||
"input_file" text NOT NULL,
|
||||
"started_at" timestamp with time zone NOT NULL,
|
||||
"completed_at" timestamp with time zone,
|
||||
"requested_asins" integer DEFAULT 0 NOT NULL,
|
||||
"skipped_asins" integer DEFAULT 0 NOT NULL,
|
||||
"scanned_asins" integer DEFAULT 0 NOT NULL,
|
||||
"source_asins_with_matches" integer DEFAULT 0 NOT NULL,
|
||||
"candidate_sellers" integer DEFAULT 0 NOT NULL,
|
||||
"qualifying_sellers" integer DEFAULT 0 NOT NULL,
|
||||
"matched_sellers" integer DEFAULT 0 NOT NULL,
|
||||
"seller_metadata_requests" integer DEFAULT 0 NOT NULL,
|
||||
"seller_storefront_requests" integer DEFAULT 0 NOT NULL,
|
||||
"inventory_sellability_checked_asins" integer DEFAULT 0 NOT NULL,
|
||||
"inventory_sellability_available_asins" integer DEFAULT 0 NOT NULL,
|
||||
"inventory_sellability_excluded_asins" integer DEFAULT 0 NOT NULL,
|
||||
"persisted_inventory_asins" integer DEFAULT 0 NOT NULL,
|
||||
"status" text NOT NULL,
|
||||
"error_message" text
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "stalker_seller_inventory" (
|
||||
"id" serial PRIMARY KEY NOT NULL,
|
||||
"run_id" integer NOT NULL,
|
||||
"seller_id" text NOT NULL,
|
||||
"asin" text NOT NULL,
|
||||
"can_sell" boolean,
|
||||
"sellability_status" text,
|
||||
"sellability_reason" text,
|
||||
"product_title" text,
|
||||
"brand" text,
|
||||
"category_tree" text,
|
||||
"current_price" real,
|
||||
"avg_price_90d" real,
|
||||
"sales_rank" integer,
|
||||
"monthly_sold" integer,
|
||||
"seller_count" integer,
|
||||
"amazon_is_seller" boolean,
|
||||
"raw_product_json" text,
|
||||
"last_seen_at" timestamp with time zone NOT NULL,
|
||||
"raw_inventory_json" text,
|
||||
CONSTRAINT "uq_stalker_inventory_run_seller_asin" UNIQUE("run_id","seller_id","asin")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
ALTER TABLE "analysis_results" ADD CONSTRAINT "analysis_results_run_id_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."runs"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "category_product_results" ADD CONSTRAINT "category_product_results_run_id_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."runs"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "stalker_asin_scans" ADD CONSTRAINT "stalker_asin_scans_run_id_stalker_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."stalker_runs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "stalker_asin_sellers" ADD CONSTRAINT "stalker_asin_sellers_scan_id_stalker_asin_scans_id_fk" FOREIGN KEY ("scan_id") REFERENCES "public"."stalker_asin_scans"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "stalker_asin_sellers" ADD CONSTRAINT "stalker_asin_sellers_seller_id_sellers_seller_id_fk" FOREIGN KEY ("seller_id") REFERENCES "public"."sellers"("seller_id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "stalker_seller_inventory" ADD CONSTRAINT "stalker_seller_inventory_run_id_stalker_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."stalker_runs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "stalker_seller_inventory" ADD CONSTRAINT "stalker_seller_inventory_seller_id_sellers_seller_id_fk" FOREIGN KEY ("seller_id") REFERENCES "public"."sellers"("seller_id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
|
||||
CREATE INDEX "idx_analysis_results_run_id" ON "analysis_results" USING btree ("run_id");--> statement-breakpoint
|
||||
CREATE INDEX "idx_analysis_results_asin" ON "analysis_results" USING btree ("asin");--> statement-breakpoint
|
||||
CREATE INDEX "idx_analysis_results_verdict" ON "analysis_results" USING btree ("verdict");--> statement-breakpoint
|
||||
CREATE INDEX "idx_analysis_results_sellability_status" ON "analysis_results" USING btree ("sellability_status");--> statement-breakpoint
|
||||
CREATE INDEX "idx_analysis_results_fetched_at" ON "analysis_results" USING btree ("fetched_at");--> statement-breakpoint
|
||||
CREATE INDEX "idx_category_results_run_id" ON "category_product_results" USING btree ("run_id");--> statement-breakpoint
|
||||
CREATE INDEX "idx_category_results_verdict" ON "category_product_results" USING btree ("verdict");--> statement-breakpoint
|
||||
CREATE INDEX "idx_category_results_sellability_status" ON "category_product_results" USING btree ("sellability_status");--> statement-breakpoint
|
||||
CREATE INDEX "idx_category_results_fetched_at" ON "category_product_results" USING btree ("fetched_at");--> statement-breakpoint
|
||||
CREATE INDEX "idx_runs_started_at" ON "runs" USING btree ("started_at");--> statement-breakpoint
|
||||
CREATE INDEX "idx_runs_type" ON "runs" USING btree ("type");--> statement-breakpoint
|
||||
CREATE INDEX "idx_runs_status" ON "runs" USING btree ("status");--> statement-breakpoint
|
||||
CREATE INDEX "idx_stalker_scans_run_id" ON "stalker_asin_scans" USING btree ("run_id");--> statement-breakpoint
|
||||
CREATE INDEX "idx_stalker_scans_source_asin" ON "stalker_asin_scans" USING btree ("source_asin");--> statement-breakpoint
|
||||
CREATE INDEX "idx_stalker_runs_started_at" ON "stalker_runs" USING btree ("started_at");--> statement-breakpoint
|
||||
CREATE INDEX "idx_stalker_inventory_seller_id" ON "stalker_seller_inventory" USING btree ("seller_id");--> statement-breakpoint
|
||||
CREATE INDEX "idx_stalker_inventory_asin" ON "stalker_seller_inventory" USING btree ("asin");--> statement-breakpoint
|
||||
CREATE INDEX "idx_stalker_inventory_product_title" ON "stalker_seller_inventory" USING btree ("product_title");
|
||||
1545
drizzle/meta/0000_snapshot.json
Normal file
1545
drizzle/meta/0000_snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
13
drizzle/meta/_journal.json
Normal file
13
drizzle/meta/_journal.json
Normal file
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"version": "7",
|
||||
"dialect": "postgresql",
|
||||
"entries": [
|
||||
{
|
||||
"idx": 0,
|
||||
"version": "7",
|
||||
"when": 1779683900467,
|
||||
"tag": "0000_gorgeous_william_stryker",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
12
package.json
12
package.json
@@ -4,13 +4,13 @@
|
||||
"type": "module",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"bestsellers": "bun run src/bestsellers-by-category.ts",
|
||||
"monthly-sold": "bun run src/top-monthly-sold-by-category.ts",
|
||||
"mid-range": "bun run src/mid-range-sellers-by-category.ts",
|
||||
"stalker": "bun run src/stalker.ts",
|
||||
"bestsellers": "bun run src/categories/bestsellers-by-category.ts",
|
||||
"monthly-sold": "bun run src/categories/top-monthly-sold-by-category.ts",
|
||||
"mid-range": "bun run src/categories/mid-range-sellers-by-category.ts",
|
||||
"stalker": "bun run src/stalker/stalker.ts",
|
||||
"search-offers": "bun run src/asin-offer-search.ts",
|
||||
"upc": "bun run src/upc-lookup.ts",
|
||||
"upc-file": "bun run src/upc-file-analysis.ts",
|
||||
"upc": "bun run src/supplier/upc-lookup.ts",
|
||||
"upc-file": "bun run src/supplier/upc-file-analysis.ts",
|
||||
"start": "bun run src/index.ts",
|
||||
"start:web": "bun --hot src/server.ts",
|
||||
"build:web": "bun build src/web/index.html --outdir dist",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { fetchKeepaDataBatch } from "./keepa.ts";
|
||||
import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts";
|
||||
import { getCache, setCache } from "./cache.ts";
|
||||
import { analyzeProducts } from "./llm.ts";
|
||||
import { fetchKeepaDataBatch } from "./integrations/keepa.ts";
|
||||
import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./integrations/sp-api.ts";
|
||||
import { getCache, setCache } from "./integrations/cache.ts";
|
||||
import { analyzeProducts } from "./integrations/llm.ts";
|
||||
import type {
|
||||
AnalysisResult,
|
||||
EnrichedProduct,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { searchProductOffers, type SearxngOfferSearchResult } from "./searxng.ts";
|
||||
import { searchProductOffers, type SearxngOfferSearchResult } from "./integrations/searxng.ts";
|
||||
|
||||
type CliArgs = {
|
||||
query: string;
|
||||
|
||||
@@ -35,7 +35,7 @@ const makeMockDb = (): any => ({
|
||||
transaction: async (fn: (tx: any) => Promise<any>) => fn(makeMockDb()),
|
||||
});
|
||||
|
||||
mock.module("./db/index.ts", () => ({ db: makeMockDb(), client: {} }));
|
||||
mock.module("../db/index.ts", () => ({ db: makeMockDb(), client: {} }));
|
||||
|
||||
const fetchSellabilityBatchMock = mock(async (asins: string[]) => {
|
||||
return new Map(
|
||||
@@ -69,12 +69,12 @@ const analyzeProductsMock = mock(async (products: any[]) => {
|
||||
}));
|
||||
});
|
||||
|
||||
mock.module("./sp-api.ts", () => ({
|
||||
mock.module("../integrations/sp-api.ts", () => ({
|
||||
fetchSellabilityBatch: fetchSellabilityBatchMock,
|
||||
fetchSpApiPricingAndFees: fetchSpApiPricingAndFeesMock,
|
||||
}));
|
||||
|
||||
mock.module("./llm.ts", () => ({
|
||||
mock.module("../integrations/llm.ts", () => ({
|
||||
analyzeProducts: analyzeProductsMock,
|
||||
}));
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import { existsSync, mkdirSync, readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { db } from "./db/index.ts";
|
||||
import { runs, categoryProductResults } from "./db/schema.ts";
|
||||
import { db } from "../db/index.ts";
|
||||
import { runs, categoryProductResults } from "../db/schema.ts";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { config } from "./config.ts";
|
||||
import { analyzeProducts } from "./llm.ts";
|
||||
import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts";
|
||||
import { config } from "../config.ts";
|
||||
import { analyzeProducts } from "../integrations/llm.ts";
|
||||
import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "../integrations/sp-api.ts";
|
||||
import type {
|
||||
AnalysisResult,
|
||||
EnrichedProduct,
|
||||
@@ -14,7 +14,7 @@ import type {
|
||||
ProductRecord,
|
||||
SellabilityInfo,
|
||||
SpApiData,
|
||||
} from "./types.ts";
|
||||
} from "../types.ts";
|
||||
|
||||
type CategoryInfo = {
|
||||
id: number;
|
||||
@@ -35,7 +35,7 @@ const makeMockDb = (): any => ({
|
||||
transaction: async (fn: (tx: any) => Promise<any>) => fn(makeMockDb()),
|
||||
});
|
||||
|
||||
mock.module("./db/index.ts", () => ({ db: makeMockDb(), client: {} }));
|
||||
mock.module("../db/index.ts", () => ({ db: makeMockDb(), client: {} }));
|
||||
|
||||
const fetchSellabilityBatchMock = mock(async (asins: string[]) => {
|
||||
return new Map<string, any>(
|
||||
@@ -84,12 +84,12 @@ const analyzeProductsMock = mock(async (products: any[]) => {
|
||||
}));
|
||||
});
|
||||
|
||||
mock.module("./sp-api.ts", () => ({
|
||||
mock.module("../integrations/sp-api.ts", () => ({
|
||||
fetchSellabilityBatch: fetchSellabilityBatchMock,
|
||||
fetchSpApiPricingAndFees: fetchSpApiPricingAndFeesMock,
|
||||
}));
|
||||
|
||||
mock.module("./llm.ts", () => ({
|
||||
mock.module("../integrations/llm.ts", () => ({
|
||||
analyzeProducts: analyzeProductsMock,
|
||||
}));
|
||||
|
||||
@@ -2,18 +2,18 @@ import { existsSync, mkdirSync, readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { createInterface } from "node:readline/promises";
|
||||
import { stdin as input, stdout as output } from "node:process";
|
||||
import { db } from "./db/index.ts";
|
||||
import { runs, categoryProductResults } from "./db/schema.ts";
|
||||
import { db } from "../db/index.ts";
|
||||
import { runs, categoryProductResults } from "../db/schema.ts";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { config } from "./config.ts";
|
||||
import { config } from "../config.ts";
|
||||
import {
|
||||
connectCache,
|
||||
disconnectCache,
|
||||
getApiCache,
|
||||
setApiCache,
|
||||
} from "./cache.ts";
|
||||
import { analyzeProducts } from "./llm.ts";
|
||||
import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts";
|
||||
} from "../integrations/cache.ts";
|
||||
import { analyzeProducts } from "../integrations/llm.ts";
|
||||
import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "../integrations/sp-api.ts";
|
||||
import type {
|
||||
AnalysisResult,
|
||||
EnrichedProduct,
|
||||
@@ -22,7 +22,7 @@ import type {
|
||||
ProductRecord,
|
||||
SellabilityInfo,
|
||||
SpApiData,
|
||||
} from "./types.ts";
|
||||
} from "../types.ts";
|
||||
|
||||
type CategoryInfo = {
|
||||
id: number;
|
||||
@@ -35,7 +35,7 @@ const makeMockDb = (): any => ({
|
||||
transaction: async (fn: (tx: any) => Promise<any>) => fn(makeMockDb()),
|
||||
});
|
||||
|
||||
mock.module("./db/index.ts", () => ({ db: makeMockDb(), client: {} }));
|
||||
mock.module("../db/index.ts", () => ({ db: makeMockDb(), client: {} }));
|
||||
|
||||
const fetchSellabilityBatchMock = mock(async (asins: string[]) => {
|
||||
return new Map<string, any>(
|
||||
@@ -82,12 +82,12 @@ const analyzeProductsMock = mock(async (products: any[]) => {
|
||||
}));
|
||||
});
|
||||
|
||||
mock.module("./sp-api.ts", () => ({
|
||||
mock.module("../integrations/sp-api.ts", () => ({
|
||||
fetchSellabilityBatch: fetchSellabilityBatchMock,
|
||||
fetchSpApiPricingAndFees: fetchSpApiPricingAndFeesMock,
|
||||
}));
|
||||
|
||||
mock.module("./llm.ts", () => ({
|
||||
mock.module("../integrations/llm.ts", () => ({
|
||||
analyzeProducts: analyzeProductsMock,
|
||||
}));
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import { existsSync, mkdirSync, readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { db } from "./db/index.ts";
|
||||
import { runs, categoryProductResults } from "./db/schema.ts";
|
||||
import { db } from "../db/index.ts";
|
||||
import { runs, categoryProductResults } from "../db/schema.ts";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { config } from "./config.ts";
|
||||
import { analyzeProducts } from "./llm.ts";
|
||||
import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts";
|
||||
import { config } from "../config.ts";
|
||||
import { analyzeProducts } from "../integrations/llm.ts";
|
||||
import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "../integrations/sp-api.ts";
|
||||
import type {
|
||||
AnalysisResult,
|
||||
EnrichedProduct,
|
||||
@@ -14,7 +14,7 @@ import type {
|
||||
ProductRecord,
|
||||
SellabilityInfo,
|
||||
SpApiData,
|
||||
} from "./types.ts";
|
||||
} from "../types.ts";
|
||||
|
||||
type CategoryInfo = {
|
||||
id: number;
|
||||
@@ -1,3 +0,0 @@
|
||||
// Central re-export so existing `import { db } from "./database.ts"` keeps working.
|
||||
export { db, type Db } from "./db/index.ts";
|
||||
export * as schema from "./db/schema.ts";
|
||||
@@ -1,5 +1,5 @@
|
||||
import { readProducts } from "./reader.ts";
|
||||
import { connectCache, disconnectCache } from "./cache.ts";
|
||||
import { connectCache, disconnectCache } from "./integrations/cache.ts";
|
||||
import {
|
||||
printResults,
|
||||
writeResultsToDb,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import Redis from "ioredis";
|
||||
import { config } from "./config.ts";
|
||||
import type { EnrichedProduct, KeepaData, SpApiData } from "./types.ts";
|
||||
import { config } from "../config.ts";
|
||||
import type { EnrichedProduct, KeepaData, SpApiData } from "../types.ts";
|
||||
|
||||
let redis: Redis | null = null;
|
||||
let disabled = false;
|
||||
@@ -1,5 +1,5 @@
|
||||
import { config } from "./config.ts";
|
||||
import type { KeepaData, KeepaUpcLookupDetail } from "./types.ts";
|
||||
import { config } from "../config.ts";
|
||||
import type { KeepaData, KeepaUpcLookupDetail } from "../types.ts";
|
||||
|
||||
const KEEPA_BASE = "https://api.keepa.com";
|
||||
const MAX_ASINS_PER_REQUEST = 100;
|
||||
@@ -1,5 +1,5 @@
|
||||
import { config } from "./config.ts";
|
||||
import type { EnrichedProduct, LlmVerdict } from "./types.ts";
|
||||
import { config } from "../config.ts";
|
||||
import type { EnrichedProduct, LlmVerdict } from "../types.ts";
|
||||
|
||||
const SYSTEM_PROMPT_STRICT = `You are an expert Amazon product analyst specializing in FBA and FBM fulfillment strategy.
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import { SellingPartner } from "amazon-sp-api";
|
||||
import { config } from "./config.ts";
|
||||
import { config } from "../config.ts";
|
||||
import type {
|
||||
KeepaUpcLookupStatus,
|
||||
SpApiData,
|
||||
SellabilityInfo,
|
||||
UpcLookupDetail,
|
||||
} from "./types.ts";
|
||||
} from "../types.ts";
|
||||
|
||||
type RegionCode = "na" | "eu" | "fe";
|
||||
|
||||
@@ -5,10 +5,10 @@ import {
|
||||
fetchKeepaDataBatch,
|
||||
lookupKeepaUpcs,
|
||||
mapUpcsToAsins,
|
||||
} from "./keepa.ts";
|
||||
import { runUpcFileAnalysis } from "./upc-file-analysis.ts";
|
||||
import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts";
|
||||
import { analyzeProducts } from "./llm.ts";
|
||||
} from "./integrations/keepa.ts";
|
||||
import { runUpcFileAnalysis } from "./supplier/upc-file-analysis.ts";
|
||||
import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./integrations/sp-api.ts";
|
||||
import { analyzeProducts } from "./integrations/llm.ts";
|
||||
import type {
|
||||
EnrichedProduct,
|
||||
KeepaUpcLookupDetail,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { testSpApiConnectivity, testSpApiSellability } from "./sp-api.ts";
|
||||
import { testSpApiConnectivity, testSpApiSellability } from "./integrations/sp-api.ts";
|
||||
|
||||
function parseArgs(): { asin?: string; sellabilityMode: boolean } {
|
||||
const args = process.argv.slice(2);
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
import { db } from "./db/index.ts";
|
||||
import { categoryProductResults, runs } from "./db/schema.ts";
|
||||
import { db } from "../db/index.ts";
|
||||
import { categoryProductResults, runs } from "../db/schema.ts";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { analyzeProducts } from "./llm.ts";
|
||||
import { fetchSpApiPricingAndFees } from "./sp-api.ts";
|
||||
import { analyzeProducts } from "../integrations/llm.ts";
|
||||
import { fetchSpApiPricingAndFees } from "../integrations/sp-api.ts";
|
||||
import type {
|
||||
AnalysisResult,
|
||||
EnrichedProduct,
|
||||
KeepaData,
|
||||
ProductRecord,
|
||||
SellabilityInfo,
|
||||
} from "./types.ts";
|
||||
} from "../types.ts";
|
||||
|
||||
const LLM_BATCH_SIZE = 5;
|
||||
const LLM_BATCH_DELAY_MS = 5_000;
|
||||
@@ -62,7 +62,7 @@ const makeMockDb = (): any => ({
|
||||
transaction: async (fn: (tx: any) => Promise<any>) => fn(makeMockTx()),
|
||||
});
|
||||
|
||||
mock.module("./db/index.ts", () => ({ db: makeMockDb(), client: {} }));
|
||||
mock.module("../db/index.ts", () => ({ db: makeMockDb(), client: {} }));
|
||||
|
||||
const TEST_DIR = path.join(process.cwd(), "test_output", "stalker-sellability");
|
||||
const originalFetch = globalThis.fetch;
|
||||
@@ -87,10 +87,6 @@ const fetchSellabilityBatchMock = mock(async (asins: string[]) => {
|
||||
);
|
||||
});
|
||||
|
||||
mock.module("./sp-api.ts", () => ({
|
||||
fetchSellabilityBatch: fetchSellabilityBatchMock,
|
||||
}));
|
||||
|
||||
const modulePromise = import("./stalker.ts");
|
||||
|
||||
beforeEach(() => {
|
||||
@@ -194,7 +190,8 @@ test("sellability checks matched seller inventory, not the source ASIN", async (
|
||||
return new Response("not found", { status: 404 });
|
||||
}) as unknown as typeof globalThis.fetch;
|
||||
|
||||
const stats = await runStalker({
|
||||
const stats = await runStalker(
|
||||
{
|
||||
input: inputPath,
|
||||
maxAsins: null,
|
||||
storefrontUpdateHours: 168,
|
||||
@@ -209,7 +206,9 @@ test("sellability checks matched seller inventory, not the source ASIN", async (
|
||||
sellability: true,
|
||||
analyzeSellable: false,
|
||||
useClaude: false,
|
||||
});
|
||||
},
|
||||
{ fetchSellabilityBatch: fetchSellabilityBatchMock },
|
||||
);
|
||||
|
||||
expect(fetchSellabilityBatchMock.mock.calls.length).toBe(1);
|
||||
expect(fetchSellabilityBatchMock.mock.calls[0]?.[0]).toEqual([
|
||||
@@ -69,7 +69,7 @@ const makeMockDb = (): any => ({
|
||||
transaction: async (fn: (tx: any) => Promise<any>) => fn(makeMockTx()),
|
||||
});
|
||||
|
||||
mock.module("./db/index.ts", () => ({ db: makeMockDb(), client: {} }));
|
||||
mock.module("../db/index.ts", () => ({ db: makeMockDb(), client: {} }));
|
||||
|
||||
const TEST_DIR = path.join(process.cwd(), "test_output", "stalker");
|
||||
const originalFetch = globalThis.fetch;
|
||||
@@ -1,6 +1,6 @@
|
||||
import * as XLSX from "xlsx";
|
||||
import path from "node:path";
|
||||
import { db } from "./db/index.ts";
|
||||
import { db } from "../db/index.ts";
|
||||
import {
|
||||
runs,
|
||||
stalkerRuns,
|
||||
@@ -8,10 +8,10 @@ import {
|
||||
sellers,
|
||||
stalkerAsinSellers,
|
||||
stalkerSellerInventory,
|
||||
} from "./db/schema.ts";
|
||||
} from "../db/schema.ts";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { fetchSellabilityBatch } from "./sp-api.ts";
|
||||
import type { SellabilityInfo } from "./types.ts";
|
||||
import { fetchSellabilityBatch } from "../integrations/sp-api.ts";
|
||||
import type { SellabilityInfo } from "../types.ts";
|
||||
|
||||
const KEEPA_BASE = "https://api.keepa.com";
|
||||
const DOMAIN_US = "1";
|
||||
@@ -310,7 +310,11 @@ export function extractLiveOfferSellerCandidates(
|
||||
return Array.from(bySeller.values());
|
||||
}
|
||||
|
||||
export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
|
||||
export type StalkerDeps = {
|
||||
fetchSellabilityBatch?: (asins: string[]) => Promise<Map<string, SellabilityInfo>>;
|
||||
};
|
||||
|
||||
export async function runStalker(args: StalkerArgs, deps: StalkerDeps = {}): Promise<StalkerRunStats> {
|
||||
const apiKey = Bun.env.KEEPA_API_KEY;
|
||||
if (!apiKey) throw new Error("Missing required env var: KEEPA_API_KEY");
|
||||
|
||||
@@ -383,7 +387,7 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
|
||||
);
|
||||
|
||||
if (args.sellability && !args.dryRun) {
|
||||
await enrichInventorySellability(result, stats);
|
||||
await enrichInventorySellability(result, stats, deps.fetchSellabilityBatch ?? fetchSellabilityBatch);
|
||||
}
|
||||
applyInventoryPersistencePolicy(result, args.sellability && !args.dryRun);
|
||||
if (args.sellability && !args.dryRun) {
|
||||
@@ -545,6 +549,7 @@ function applyInventoryPersistencePolicy(
|
||||
async function enrichInventorySellability(
|
||||
result: StalkerAsinResult,
|
||||
stats: StalkerRunStats,
|
||||
sellabilityFn: (asins: string[]) => Promise<Map<string, SellabilityInfo>>,
|
||||
): Promise<void> {
|
||||
const sellers = result.matchedSellers.map(({ seller }) => seller);
|
||||
const items = sellers.flatMap((seller) => seller.storefrontItems);
|
||||
@@ -554,7 +559,7 @@ async function enrichInventorySellability(
|
||||
console.log(
|
||||
`Stalker inventory sellability: checking ${uniqueAsins.length} ASIN(s) from matched seller storefronts...`,
|
||||
);
|
||||
const sellabilityMap = await fetchSellabilityBatch(uniqueAsins);
|
||||
const sellabilityMap = await sellabilityFn(uniqueAsins);
|
||||
stats.inventorySellabilityCheckedAsins += uniqueAsins.length;
|
||||
|
||||
for (const asin of uniqueAsins) {
|
||||
@@ -3,7 +3,7 @@ import path from "node:path";
|
||||
import { rmSync } from "node:fs";
|
||||
import ExcelJS from "exceljs";
|
||||
import { writeSupplierWorkbook } from "./supplier-export.ts";
|
||||
import type { SupplierAnalysisResult } from "./types.ts";
|
||||
import type { SupplierAnalysisResult } from "../types.ts";
|
||||
|
||||
const OUTPUT_FILE = path.join("/private/tmp", "asin-check-supplier-export-test.xlsx");
|
||||
|
||||
@@ -5,7 +5,7 @@ import type {
|
||||
KeepaUpcLookupStatus,
|
||||
SupplierAnalysisResult,
|
||||
SupplierVerdict,
|
||||
} from "./types.ts";
|
||||
} from "../types.ts";
|
||||
|
||||
export type SupplierExportSummary = {
|
||||
processedRows: number;
|
||||
@@ -1,6 +1,6 @@
|
||||
import { expect, test } from "bun:test";
|
||||
import { scoreSupplierProduct } from "./supplier-scoring.ts";
|
||||
import type { KeepaData, ProductRecord, SpApiData } from "./types.ts";
|
||||
import type { KeepaData, ProductRecord, SpApiData } from "../types.ts";
|
||||
|
||||
function record(overrides: Partial<ProductRecord> = {}): ProductRecord {
|
||||
return {
|
||||
@@ -3,7 +3,7 @@ import type {
|
||||
ProductRecord,
|
||||
SpApiData,
|
||||
SupplierScore,
|
||||
} from "./types.ts";
|
||||
} from "../types.ts";
|
||||
|
||||
function round2(value: number): number {
|
||||
return Math.round(value * 100) / 100;
|
||||
@@ -1,10 +1,10 @@
|
||||
import path from "node:path";
|
||||
import { fetchKeepaDataBatch, lookupKeepaUpcs } from "./keepa.ts";
|
||||
import { fetchKeepaDataBatch, lookupKeepaUpcs } from "../integrations/keepa.ts";
|
||||
import {
|
||||
fetchSellabilityBatch,
|
||||
fetchSpApiPricingAndFees,
|
||||
lookupSpApiUpcs,
|
||||
} from "./sp-api.ts";
|
||||
} from "../integrations/sp-api.ts";
|
||||
import {
|
||||
processUpcFileInBatches,
|
||||
type UpcInputRow,
|
||||
@@ -14,8 +14,8 @@ import {
|
||||
refreshRunCountsInDb,
|
||||
startRunInDb,
|
||||
type RunCounts,
|
||||
} from "./writer.ts";
|
||||
import { connectCache, disconnectCache } from "./cache.ts";
|
||||
} from "../writer.ts";
|
||||
import { connectCache, disconnectCache } from "../integrations/cache.ts";
|
||||
import { scoreSupplierProduct, resolveSupplierSalePrice } from "./supplier-scoring.ts";
|
||||
import {
|
||||
writeSupplierWorkbook,
|
||||
@@ -28,7 +28,7 @@ import type {
|
||||
SupplierAnalysisResult,
|
||||
SupplierScore,
|
||||
UpcLookupDetail,
|
||||
} from "./types.ts";
|
||||
} from "../types.ts";
|
||||
|
||||
const DEFAULT_INPUT_BATCH_SIZE = 200;
|
||||
const DEFAULT_UPC_LOOKUP_BATCH_SIZE = 100;
|
||||
@@ -1,4 +1,4 @@
|
||||
import { lookupKeepaUpcs, mapUpcsToAsins } from "./keepa.ts";
|
||||
import { lookupKeepaUpcs, mapUpcsToAsins } from "../integrations/keepa.ts";
|
||||
|
||||
function printUsage(): void {
|
||||
console.log("Usage:");
|
||||
Reference in New Issue
Block a user