diff --git a/.claude/settings.local.json b/.claude/settings.local.json index e452c17..1255e9d 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -1,10 +1,10 @@ -{ - "permissions": { - "allow": [ - "WebSearch", - "Bash(bun init:*)", - "Bash(bunx tsc:*)", - "Bash(bun -e ':*)" - ] - } -} +{ + "permissions": { + "allow": [ + "WebSearch", + "Bash(bun init:*)", + "Bash(bunx tsc:*)", + "Bash(bun -e ':*)" + ] + } +} diff --git a/.env.example b/.env.example index e6a3e8c..eb6f582 100644 --- a/.env.example +++ b/.env.example @@ -1,15 +1,15 @@ -KEEPA_API_KEY=your_keepa_api_key_here -SP_API_CLIENT_ID=your_sp_api_client_id -SP_API_CLIENT_SECRET=your_sp_api_client_secret -SP_API_REFRESH_TOKEN=your_sp_api_refresh_token -SP_API_REGION=na -SP_API_MARKETPLACE_ID=ATVPDKIKX0DER -SP_API_SELLER_ID=your_seller_id -SP_API_USE_SANDBOX=false -AWS_ACCESS_KEY_ID=your_aws_access_key_id -AWS_SECRET_ACCESS_KEY=your_aws_secret_access_key -# AWS_SESSION_TOKEN=optional_if_using_sts -REDIS_URL=redis://localhost:6379 -LLM_URL=http://localhost:1234/v1 -LLM_MODEL=default -CACHE_TTL=86400 +KEEPA_API_KEY=your_keepa_api_key_here +SP_API_CLIENT_ID=your_sp_api_client_id +SP_API_CLIENT_SECRET=your_sp_api_client_secret +SP_API_REFRESH_TOKEN=your_sp_api_refresh_token +SP_API_REGION=na +SP_API_MARKETPLACE_ID=ATVPDKIKX0DER +SP_API_SELLER_ID=your_seller_id +SP_API_USE_SANDBOX=false +AWS_ACCESS_KEY_ID=your_aws_access_key_id +AWS_SECRET_ACCESS_KEY=your_aws_secret_access_key +# AWS_SESSION_TOKEN=optional_if_using_sts +REDIS_URL=redis://localhost:6379 +LLM_URL=http://localhost:1234/v1 +LLM_MODEL=default +CACHE_TTL=86400 diff --git a/.gitignore b/.gitignore index 694ac49..60bcf2f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,37 +1,37 @@ -# dependencies (bun install) -node_modules - -# output -out -dist -*.tgz - -# code coverage -coverage -*.lcov - -# logs -logs -_.log -report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json - -# dotenv environment variable files -.env -.env.development.local -.env.test.local -.env.production.local -.env.local - -# caches -.eslintcache -.cache -*.tsbuildinfo - -# IntelliJ based IDEs -.idea - -# Finder (MacOS) folder config -.DS_Store -*.xlsx -*.csv - +# dependencies (bun install) +node_modules + +# output +out +dist +*.tgz + +# code coverage +coverage +*.lcov + +# logs +logs +_.log +report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json + +# dotenv environment variable files +.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# caches +.eslintcache +.cache +*.tsbuildinfo + +# IntelliJ based IDEs +.idea + +# Finder (MacOS) folder config +.DS_Store +*.xlsx +*.csv + diff --git a/CLAUDE.md b/CLAUDE.md index 764c1dd..a3f4b68 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,106 +1,106 @@ - -Default to using Bun instead of Node.js. - -- Use `bun ` instead of `node ` or `ts-node ` -- Use `bun test` instead of `jest` or `vitest` -- Use `bun build ` instead of `webpack` or `esbuild` -- Use `bun install` instead of `npm install` or `yarn install` or `pnpm install` -- Use `bun run - - -``` - -With the following `frontend.tsx`: - -```tsx#frontend.tsx -import React from "react"; -import { createRoot } from "react-dom/client"; - -// import .css files directly and it works -import './index.css'; - -const root = createRoot(document.body); - -export default function Frontend() { - return

Hello, world!

; -} - -root.render(); -``` - -Then, run index.ts - -```sh -bun --hot ./index.ts -``` - -For more information, read the Bun API docs in `node_modules/bun-types/docs/**.mdx`. + +Default to using Bun instead of Node.js. + +- Use `bun ` instead of `node ` or `ts-node ` +- Use `bun test` instead of `jest` or `vitest` +- Use `bun build ` instead of `webpack` or `esbuild` +- Use `bun install` instead of `npm install` or `yarn install` or `pnpm install` +- Use `bun run + + +``` + +With the following `frontend.tsx`: + +```tsx#frontend.tsx +import React from "react"; +import { createRoot } from "react-dom/client"; + +// import .css files directly and it works +import './index.css'; + +const root = createRoot(document.body); + +export default function Frontend() { + return

Hello, world!

; +} + +root.render(); +``` + +Then, run index.ts + +```sh +bun --hot ./index.ts +``` + +For more information, read the Bun API docs in `node_modules/bun-types/docs/**.mdx`. diff --git a/README.md b/README.md index 46c3298..a52720b 100644 --- a/README.md +++ b/README.md @@ -1,135 +1,135 @@ -# asin-check - -Amazon product analysis and lead finder agent. Reads product leads from a CSV/XLSX file, enriches them with Keepa pricing and sales data, caches results in Redis, and runs each product through a local LLM to get an FBA/FBM/SKIP verdict. - -## Requirements - -- [Bun](https://bun.com) runtime -- Redis (local or Docker) -- [LM Studio](https://lmstudio.ai) running locally with a model loaded -- Keepa API key ([keepa.com](https://keepa.com)) -- Amazon SP-API private app credentials (LWA + refresh token + IAM) - -## Setup - -```bash -bun install -cp .env.example .env -# Edit .env and set your KEEPA_API_KEY and SP-API credentials -``` - -## Usage - -```bash -bun run src/index.ts [--out results.csv] -``` - -Examples: - -```bash -bun run src/index.ts leads.xlsx -bun run src/index.ts leads.csv --out results.xlsx -``` - -Large-file behavior: - -- If the input has more than 50 products, processing is done in chunks of 50. -- Each chunk is analyzed and written to a numbered output file, for example: `results_part_001.xlsx`, `results_part_002.xlsx`, ... -- If `--out` is omitted for large files, the base output name defaults to `_results.xlsx` and chunk files are still written with numbered suffixes. - -Quick SP-API connectivity tests: - -```bash -bun run src/sp-test.ts # Auth + sellers endpoint -bun run src/sp-test.ts B07SN9BHVV # Auth + sellers endpoint + pricing offer check -bun run src/sp-test.ts --sellability B07SN9BHVV # Standalone sellability check -``` - -## Input file format - -Accepts `.csv` or `.xlsx` files. Column names are matched case-insensitively. Required column: - -| Column | Aliases | -| ------ | ------- | -| ASIN | — | - -Optional but recommended: - -| Column | Aliases | -| --------------- | ---------------------------- | -| Product Name | Name, Title | -| Unit Cost | Cost, Price, Buy Cost | -| Brand | — | -| Category | — | -| Amazon Rank | Amazon Rank, BSR, Sales Rank | -| FBA NET | — | -| Gross Profit $ | Gross Profit | -| Gross Profit % | — | -| MOQ | Min Order Qty | -| MOQ Cost | — | -| Total Qty Avail | Qty Available | -| Link | URL, Source | - -Lead-list format aliases (supported): - -| Column | Aliases | -| ----------------- | ------------------------------------------ | -| Name | Product Name, Title, Product Title | -| ASIN Link | ASIN URL, Amazon Link | -| Source URL | Source Link, Supplier URL | -| 90 Day Average | 90-day Average, Avg Price 90d, 90d Average | -| Cost | Unit Cost, Buy Cost, Price | -| Selling Price | Sale Price, Sell Price | -| Net Profit | Gross Profit | -| ROI | Gross Profit %, Return on Investment | -| Supplier | Vendor | -| Promo/Coupon Code | Promo Code, Coupon Code | -| Notes | Note | -| Date | Lead Date | - -Numeric parsing accepts plain numbers as well as formatted values like `$12.50`, `1,209.60`, and `27.5%`. - -## Pipeline - -1. **Read** — parse input file, validate ASINs -2. **Cache check** — look up each ASIN in Redis (24h TTL by default) -3. **Sellability gate** — check all uncached ASINs against SP-API `getListingsRestrictions` (concurrency: 5 workers); immediately skip ASINs with status `not_available` and `canSell=false` (no Keepa/fees wasted) -4. **Keepa fetch** — batch the sellable (uncached) ASINs in a single API call (up to 100 per request) -5. **Enrich** — fetch SP-API pricing + FBA/FBM fees for sellable ASINs; combine with Keepa data and spreadsheet data -6. **LLM analysis** — send batches of 5 available products to LM Studio for FBA/FBM/SKIP verdict -7. **Chunk orchestration** — if input size is greater than 50, run phases 2-6 for each 50-item chunk sequentially -8. **Output** — print results table to console (includes all ASINs); for chunked runs, always write seriated chunk files (`*_part_001`, `*_part_002`, ...); for non-chunked runs, write a single file only when `--out` is provided - -## Output columns - -ASIN, Name, Brand, Category, Unit Cost, Current Price, Avg Price 90d, Sales Rank, Rank Avg 90d, Sellers, Monthly Sold, Rank Drops 30d, Rank Drops 90d, FBA Net (sheet), Gross Profit $, Gross Profit %, MOQ, MOQ Cost, Qty Available, FBA Fee, FBM Fee, Referral %, Verdict, Confidence, Reasoning - -## Environment variables - -| Variable | Default | Description | -| ----------------------- | -------------------------- | ----------------------------------------------------------------------- | -| `KEEPA_API_KEY` | — | **Required.** Keepa API key | -| `SP_API_CLIENT_ID` | — | LWA app client id from Solution Provider Portal | -| `SP_API_CLIENT_SECRET` | — | LWA app client secret from Solution Provider Portal | -| `SP_API_REFRESH_TOKEN` | — | Refresh token from self-authorization | -| `SP_API_REGION` | `na` | SP-API endpoint region (`na`, `eu`, `fe`; `us` is accepted as `na`) | -| `SP_API_MARKETPLACE_ID` | `ATVPDKIKX0DER` | Marketplace id used for pricing and fee calls (default: US) | -| `SP_API_SELLER_ID` | — | Seller ID used for listing restrictions eligibility checks | -| `SP_API_USE_SANDBOX` | `false` | Enable SP-API sandbox mode (`true`/`false`) | -| `AWS_ACCESS_KEY_ID` | — | AWS credentials for SigV4 signing (required in most private app setups) | -| `AWS_SECRET_ACCESS_KEY` | — | AWS credentials for SigV4 signing | -| `AWS_SESSION_TOKEN` | — | Optional session token when using STS credentials | -| `REDIS_URL` | `redis://localhost:6379` | Redis connection URL | -| `LLM_URL` | `http://localhost:1234/v1` | LM Studio API base URL | -| `LLM_MODEL` | `default` | Model name to pass to LM Studio | -| `CACHE_TTL` | `86400` | Redis cache TTL in seconds | - -## Notes - -- **Available-only processing**: SP-API `getListingsRestrictions` is checked first and only ASINs with `sellabilityStatus=available` are enriched, analyzed, and included in outputs. Restricted, not_available, and unknown items are excluded. -- **SP-API concurrency**: `fetchSellabilityBatch` limits concurrent requests to 5 workers to avoid 429 throttling. Pricing+fees fetches also use 5 concurrent workers. -- **No batch endpoint**: Amazon SP-API does not provide batch endpoints for `getListingsRestrictions` or `getMyFeesEstimate*`. Concurrency limiting with the library's built-in `auto_request_throttled` safety net prevents overwhelming the API. -- **Keepa rate limiting**: The client reads `tokensLeft` and `refillRate` from each API response and waits automatically when tokens are exhausted. With a Pro subscription (1 token/min), all 100 ASINs in a batch cost 1 token. -- **Redis is optional**: If Redis is unavailable the tool runs without caching — every run re-fetches from Keepa. -- **SP-API**: `src/sp-api.ts` provides `fetchSellability`, `fetchSellabilityBatch`, and `fetchSpApiPricingAndFees` functions. If SP-API credentials are missing or a call fails, the tool falls back to conservative fee defaults and keeps processing. -- **Sandbox vs production**: When `SP_API_USE_SANDBOX=true`, production ASIN calls can be denied. Use sandbox-compatible test data or set it to `false` for live marketplace connectivity. +# asin-check + +Amazon product analysis and lead finder agent. Reads product leads from a CSV/XLSX file, enriches them with Keepa pricing and sales data, caches results in Redis, and runs each product through a local LLM to get an FBA/FBM/SKIP verdict. + +## Requirements + +- [Bun](https://bun.com) runtime +- Redis (local or Docker) +- [LM Studio](https://lmstudio.ai) running locally with a model loaded +- Keepa API key ([keepa.com](https://keepa.com)) +- Amazon SP-API private app credentials (LWA + refresh token + IAM) + +## Setup + +```bash +bun install +cp .env.example .env +# Edit .env and set your KEEPA_API_KEY and SP-API credentials +``` + +## Usage + +```bash +bun run src/index.ts [--out results.csv] +``` + +Examples: + +```bash +bun run src/index.ts leads.xlsx +bun run src/index.ts leads.csv --out results.xlsx +``` + +Large-file behavior: + +- If the input has more than 50 products, processing is done in chunks of 50. +- Each chunk is analyzed and written to a numbered output file, for example: `results_part_001.xlsx`, `results_part_002.xlsx`, ... +- If `--out` is omitted for large files, the base output name defaults to `_results.xlsx` and chunk files are still written with numbered suffixes. + +Quick SP-API connectivity tests: + +```bash +bun run src/sp-test.ts # Auth + sellers endpoint +bun run src/sp-test.ts B07SN9BHVV # Auth + sellers endpoint + pricing offer check +bun run src/sp-test.ts --sellability B07SN9BHVV # Standalone sellability check +``` + +## Input file format + +Accepts `.csv` or `.xlsx` files. Column names are matched case-insensitively. Required column: + +| Column | Aliases | +| ------ | ------- | +| ASIN | — | + +Optional but recommended: + +| Column | Aliases | +| --------------- | ---------------------------- | +| Product Name | Name, Title | +| Unit Cost | Cost, Price, Buy Cost | +| Brand | — | +| Category | — | +| Amazon Rank | Amazon Rank, BSR, Sales Rank | +| FBA NET | — | +| Gross Profit $ | Gross Profit | +| Gross Profit % | — | +| MOQ | Min Order Qty | +| MOQ Cost | — | +| Total Qty Avail | Qty Available | +| Link | URL, Source | + +Lead-list format aliases (supported): + +| Column | Aliases | +| ----------------- | ------------------------------------------ | +| Name | Product Name, Title, Product Title | +| ASIN Link | ASIN URL, Amazon Link | +| Source URL | Source Link, Supplier URL | +| 90 Day Average | 90-day Average, Avg Price 90d, 90d Average | +| Cost | Unit Cost, Buy Cost, Price | +| Selling Price | Sale Price, Sell Price | +| Net Profit | Gross Profit | +| ROI | Gross Profit %, Return on Investment | +| Supplier | Vendor | +| Promo/Coupon Code | Promo Code, Coupon Code | +| Notes | Note | +| Date | Lead Date | + +Numeric parsing accepts plain numbers as well as formatted values like `$12.50`, `1,209.60`, and `27.5%`. + +## Pipeline + +1. **Read** — parse input file, validate ASINs +2. **Cache check** — look up each ASIN in Redis (24h TTL by default) +3. **Sellability gate** — check all uncached ASINs against SP-API `getListingsRestrictions` (concurrency: 5 workers); immediately skip ASINs with status `not_available` and `canSell=false` (no Keepa/fees wasted) +4. **Keepa fetch** — batch the sellable (uncached) ASINs in a single API call (up to 100 per request) +5. **Enrich** — fetch SP-API pricing + FBA/FBM fees for sellable ASINs; combine with Keepa data and spreadsheet data +6. **LLM analysis** — send batches of 5 available products to LM Studio for FBA/FBM/SKIP verdict +7. **Chunk orchestration** — if input size is greater than 50, run phases 2-6 for each 50-item chunk sequentially +8. **Output** — print results table to console (includes all ASINs); for chunked runs, always write seriated chunk files (`*_part_001`, `*_part_002`, ...); for non-chunked runs, write a single file only when `--out` is provided + +## Output columns + +ASIN, Name, Brand, Category, Unit Cost, Current Price, Avg Price 90d, Sales Rank, Rank Avg 90d, Sellers, Monthly Sold, Rank Drops 30d, Rank Drops 90d, FBA Net (sheet), Gross Profit $, Gross Profit %, MOQ, MOQ Cost, Qty Available, FBA Fee, FBM Fee, Referral %, Verdict, Confidence, Reasoning + +## Environment variables + +| Variable | Default | Description | +| ----------------------- | -------------------------- | ----------------------------------------------------------------------- | +| `KEEPA_API_KEY` | — | **Required.** Keepa API key | +| `SP_API_CLIENT_ID` | — | LWA app client id from Solution Provider Portal | +| `SP_API_CLIENT_SECRET` | — | LWA app client secret from Solution Provider Portal | +| `SP_API_REFRESH_TOKEN` | — | Refresh token from self-authorization | +| `SP_API_REGION` | `na` | SP-API endpoint region (`na`, `eu`, `fe`; `us` is accepted as `na`) | +| `SP_API_MARKETPLACE_ID` | `ATVPDKIKX0DER` | Marketplace id used for pricing and fee calls (default: US) | +| `SP_API_SELLER_ID` | — | Seller ID used for listing restrictions eligibility checks | +| `SP_API_USE_SANDBOX` | `false` | Enable SP-API sandbox mode (`true`/`false`) | +| `AWS_ACCESS_KEY_ID` | — | AWS credentials for SigV4 signing (required in most private app setups) | +| `AWS_SECRET_ACCESS_KEY` | — | AWS credentials for SigV4 signing | +| `AWS_SESSION_TOKEN` | — | Optional session token when using STS credentials | +| `REDIS_URL` | `redis://localhost:6379` | Redis connection URL | +| `LLM_URL` | `http://localhost:1234/v1` | LM Studio API base URL | +| `LLM_MODEL` | `default` | Model name to pass to LM Studio | +| `CACHE_TTL` | `86400` | Redis cache TTL in seconds | + +## Notes + +- **Available-only processing**: SP-API `getListingsRestrictions` is checked first and only ASINs with `sellabilityStatus=available` are enriched, analyzed, and included in outputs. Restricted, not_available, and unknown items are excluded. +- **SP-API concurrency**: `fetchSellabilityBatch` limits concurrent requests to 5 workers to avoid 429 throttling. Pricing+fees fetches also use 5 concurrent workers. +- **No batch endpoint**: Amazon SP-API does not provide batch endpoints for `getListingsRestrictions` or `getMyFeesEstimate*`. Concurrency limiting with the library's built-in `auto_request_throttled` safety net prevents overwhelming the API. +- **Keepa rate limiting**: The client reads `tokensLeft` and `refillRate` from each API response and waits automatically when tokens are exhausted. With a Pro subscription (1 token/min), all 100 ASINs in a batch cost 1 token. +- **Redis is optional**: If Redis is unavailable the tool runs without caching — every run re-fetches from Keepa. +- **SP-API**: `src/sp-api.ts` provides `fetchSellability`, `fetchSellabilityBatch`, and `fetchSpApiPricingAndFees` functions. If SP-API credentials are missing or a call fails, the tool falls back to conservative fee defaults and keeps processing. +- **Sandbox vs production**: When `SP_API_USE_SANDBOX=true`, production ASIN calls can be denied. Use sandbox-compatible test data or set it to `false` for live marketplace connectivity. diff --git a/package.json b/package.json index f6fcded..93473a1 100644 --- a/package.json +++ b/package.json @@ -1,17 +1,17 @@ -{ - "name": "asin-check", - "module": "src/index.ts", - "type": "module", - "private": true, - "devDependencies": { - "@types/bun": "latest" - }, - "peerDependencies": { - "typescript": "^5" - }, - "dependencies": { - "amazon-sp-api": "^1.2.1", - "ioredis": "^5.10.1", - "xlsx": "^0.18.5" - } -} +{ + "name": "asin-check", + "module": "src/index.ts", + "type": "module", + "private": true, + "devDependencies": { + "@types/bun": "latest" + }, + "peerDependencies": { + "typescript": "^5" + }, + "dependencies": { + "amazon-sp-api": "^1.2.1", + "ioredis": "^5.10.1", + "xlsx": "^0.18.5" + } +} diff --git a/src/bestsellers-by-category.ts b/src/bestsellers-by-category.ts new file mode 100644 index 0000000..54be858 --- /dev/null +++ b/src/bestsellers-by-category.ts @@ -0,0 +1,1068 @@ +import { existsSync, mkdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import * as XLSX from "xlsx"; +import { config } from "./config.ts"; +import { analyzeProducts } from "./llm.ts"; +import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts"; +import type { + AnalysisResult, + EnrichedProduct, + KeepaData, + LlmVerdict, + ProductRecord, + SellabilityInfo, + SpApiData, +} from "./types.ts"; + +type CategoryInfo = { + id: number; + label: string; + parentId: number; + childCount: number; +}; + +type ParsedArgs = { + outputDir: string; + categoryLimit: number; + perCategoryTop: number; + blacklistFile: string; +}; + +type CategoryRunSummary = { + categoryId: number; + categoryLabel: string; + topAsinsChecked: number; + availableAsins: number; + fba: number; + fbm: number; + skip: number; + outputFile: string; + status: "ok" | "empty" | "failed"; + error: string; +}; + +const KEEPA_BASE = "https://api.keepa.com"; +const DOMAIN_US = 1; +const DEFAULT_CATEGORY_LIMIT = 32; +const DEFAULT_PER_CATEGORY_TOP = 100; +const SELLABILITY_BATCH_SIZE = 60; +const LLM_BATCH_SIZE = 10; +const PRICING_CONCURRENCY = 5; +const KEEPA_PRODUCT_CHUNK_SIZE = 100; +const DEFAULT_BLACKLIST_FILE = path.join( + process.cwd(), + "category-blacklist.csv", +); + +let keepaTokensLeft = 1; +let keepaRefillRate = 1; +let keepaLastRequestMs = 0; + +function parseArgs(): ParsedArgs { + const args = process.argv.slice(2); + const outputDir = + readFlagValue(args, "--out-dir") ?? path.join(process.cwd(), "output"); + const blacklistFile = + readFlagValue(args, "--blacklist-file") ?? DEFAULT_BLACKLIST_FILE; + + const categoryLimitRaw = readFlagValue(args, "--category-limit"); + const perCategoryTopRaw = readFlagValue(args, "--per-category-top"); + + const categoryLimit = categoryLimitRaw + ? Number(categoryLimitRaw) + : DEFAULT_CATEGORY_LIMIT; + const perCategoryTop = perCategoryTopRaw + ? Number(perCategoryTopRaw) + : DEFAULT_PER_CATEGORY_TOP; + + if (!Number.isInteger(categoryLimit) || categoryLimit <= 0) { + printUsageAndExit("--category-limit must be a positive integer."); + } + + if (!Number.isInteger(perCategoryTop) || perCategoryTop <= 0) { + printUsageAndExit("--per-category-top must be a positive integer."); + } + + return { + outputDir, + categoryLimit, + perCategoryTop, + blacklistFile, + }; +} + +function readFlagValue(args: string[], flag: string): string | undefined { + const idx = args.indexOf(flag); + if (idx === -1) return undefined; + return args[idx + 1]; +} + +function printUsageAndExit(message: string): never { + if (message) { + console.error(message); + } + + console.error( + [ + "Usage:", + " bun run src/bestsellers-by-category.ts [--category-limit 32] [--per-category-top 100] [--out-dir output] [--blacklist-file category-blacklist.csv]", + "", + "Flow:", + " 1) Discover categories and round-robin selection.", + " 2) For each category: check sellability for top N (default 100).", + " 3) Enrich sellable ASINs with Keepa + SP-API pricing/fees.", + " 4) LLM-analyze and write one output file per category.", + ].join("\n"), + ); + + process.exit(1); +} + +function loadCategoryBlacklist(filePath: string): Set { + const blacklist = new Set(); + + if (!existsSync(filePath)) { + console.warn( + `Blacklist file not found at ${filePath}; continuing with no excluded categories.`, + ); + return blacklist; + } + + const raw = readFileSync(filePath, "utf8"); + const lines = raw.split(/\r?\n/); + + for (let i = 0; i < lines.length; i++) { + const lineNumber = i + 1; + const line = lines[i] ?? ""; + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) continue; + + const [idPart, namePart] = trimmed.split(",", 2); + const idToken = idPart?.trim() ?? ""; + const nameToken = namePart?.trim() ?? ""; + + // Allow header row: id,name + if (idToken.toLowerCase() === "id") { + continue; + } + + if (!idToken) { + console.warn( + `Blacklist CSV line ${lineNumber}: missing id, row ignored (${trimmed}).`, + ); + continue; + } + + const id = Number(idToken); + if (!Number.isInteger(id) || id <= 0) { + console.warn( + `Blacklist CSV line ${lineNumber}: invalid id '${idToken}', row ignored (${trimmed}).`, + ); + continue; + } + + if (!nameToken) { + console.warn( + `Blacklist CSV line ${lineNumber}: missing name for id ${id}; accepted but please add name.`, + ); + } + + if (blacklist.has(id)) { + console.warn( + `Blacklist CSV line ${lineNumber}: duplicate id ${id}, keeping first occurrence.`, + ); + continue; + } + + blacklist.add(id); + } + + return blacklist; +} + +function assertSpApiPrerequisites(): void { + const missing: string[] = []; + if (!config.spApiClientId) missing.push("SP_API_CLIENT_ID"); + if (!config.spApiClientSecret) missing.push("SP_API_CLIENT_SECRET"); + if (!config.spApiRefreshToken) missing.push("SP_API_REFRESH_TOKEN"); + if (!config.spApiSellerId) missing.push("SP_API_SELLER_ID"); + + if (missing.length > 0) { + throw new Error(`Missing required SP-API env vars: ${missing.join(", ")}`); + } +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function sanitizeFileSegment(value: string): string { + const compact = value.trim().toLowerCase().replace(/\s+/g, "-"); + const safe = compact.replace(/[^a-z0-9-_]+/g, "-").replace(/-+/g, "-"); + return safe.replace(/^-|-$/g, "") || "category"; +} + +function parseKeepaRateLimitPayload(text: string): { + refillInMs?: number; + tokensLeft?: number; + refillRate?: number; +} { + try { + const parsed = JSON.parse(text); + return { + refillInMs: + typeof parsed?.refillIn === "number" && Number.isFinite(parsed.refillIn) + ? Math.max(0, parsed.refillIn) + : undefined, + tokensLeft: + typeof parsed?.tokensLeft === "number" && + Number.isFinite(parsed.tokensLeft) + ? parsed.tokensLeft + : undefined, + refillRate: + typeof parsed?.refillRate === "number" && + Number.isFinite(parsed.refillRate) + ? parsed.refillRate + : undefined, + }; + } catch { + return {}; + } +} + +function computeBackoffMs(attempt: number, refillInMs?: number): number { + const refillBased = refillInMs != null ? refillInMs + 1500 : 0; + const exponential = Math.min(60_000, 2 ** attempt * 1000); + const base = Math.max(refillBased, exponential); + return base + Math.floor(Math.random() * 750); +} + +async function waitForKeepaToken(): Promise { + if (keepaTokensLeft > 0) return; + + const elapsedMinutes = (Date.now() - keepaLastRequestMs) / 60_000; + const regenerated = Math.floor(elapsedMinutes * keepaRefillRate); + if (regenerated > 0) { + keepaTokensLeft += regenerated; + return; + } + + const waitMs = + Math.ceil((1 / keepaRefillRate) * 60_000) - + (Date.now() - keepaLastRequestMs); + + if (waitMs > 0) { + console.log( + `Keepa tokens depleted; waiting ${Math.ceil(waitMs / 1000)}s...`, + ); + await sleep(waitMs); + } + + keepaTokensLeft = 1; +} + +async function keepaGetJson(pathAndQuery: string): Promise { + let rateLimitHits = 0; + + while (true) { + await waitForKeepaToken(); + + const response = await fetch(`${KEEPA_BASE}${pathAndQuery}`); + keepaLastRequestMs = Date.now(); + + if (response.ok) { + const data = (await response.json()) as any; + if (typeof data?.tokensLeft === "number") { + keepaTokensLeft = data.tokensLeft; + } + if (typeof data?.refillRate === "number" && data.refillRate > 0) { + keepaRefillRate = data.refillRate; + } + return data; + } + + const text = await response.text(); + + if (response.status === 429) { + const rate = parseKeepaRateLimitPayload(text); + if (typeof rate.tokensLeft === "number") { + keepaTokensLeft = rate.tokensLeft; + } + if (typeof rate.refillRate === "number" && rate.refillRate > 0) { + keepaRefillRate = rate.refillRate; + } + + rateLimitHits++; + const waitMs = computeBackoffMs(rateLimitHits, rate.refillInMs); + console.warn( + `Keepa rate limited (429). Retry ${rateLimitHits} in ${Math.ceil(waitMs / 1000)}s...`, + ); + await sleep(waitMs); + continue; + } + + throw new Error(`Keepa HTTP ${response.status}: ${text}`); + } +} + +function normalizeCategoryList(data: any): CategoryInfo[] { + const deduped = new Map(); + + const addRawCategory = (value: any): void => { + const id = Number( + value?.catId ?? value?.categoryId ?? value?.id ?? value?.nodeId, + ); + if (!Number.isInteger(id) || id <= 0) return; + + const label = String( + value?.name ?? value?.label ?? `Category ${id}`, + ).trim(); + const parentId = Number(value?.parent ?? value?.parentId ?? -1); + const childCount = Array.isArray(value?.children) + ? value.children.length + : Number.isInteger(value?.childCount) + ? Number(value.childCount) + : 0; + + if (id === 0 || label.toLowerCase() === "root" || parentId === -1) { + return; + } + + if (!deduped.has(id)) { + deduped.set(id, { + id, + label: label || `Category ${id}`, + parentId, + childCount: Math.max(0, childCount), + }); + } + }; + + if (Array.isArray(data?.categories)) { + for (const value of data.categories) { + addRawCategory(value); + } + } + + if (data?.categories && typeof data.categories === "object") { + for (const value of Object.values(data.categories)) { + addRawCategory(value); + } + } + + if (Array.isArray(data?.categoryList)) { + for (const value of data.categoryList) { + addRawCategory(value); + } + } + + return [...deduped.values()]; +} + +function prioritizeLikelyBestsellerCategories( + categories: CategoryInfo[], +): CategoryInfo[] { + const leaves: CategoryInfo[] = []; + const nonLeaves: CategoryInfo[] = []; + + for (const category of categories) { + if (category.childCount === 0) { + leaves.push(category); + } else { + nonLeaves.push(category); + } + } + + const withNamedLabels = (list: CategoryInfo[]) => + list.filter((c) => !/^Category\s+\d+$/i.test(c.label)); + + const withFallbackLabels = (list: CategoryInfo[]) => + list.filter((c) => /^Category\s+\d+$/i.test(c.label)); + + return [ + ...withNamedLabels(leaves), + ...withFallbackLabels(leaves), + ...withNamedLabels(nonLeaves), + ...withFallbackLabels(nonLeaves), + ]; +} + +function resolveRootCategory( + category: CategoryInfo, + byId: Map, +): CategoryInfo { + let current = category; + const seen = new Set(); + + while (current.parentId > 0 && !seen.has(current.id)) { + seen.add(current.id); + const parent = byId.get(current.parentId); + if (!parent) break; + current = parent; + } + + return current; +} + +function selectCategoriesAcrossRoots( + categories: CategoryInfo[], + maxCategories: number, +): CategoryInfo[] { + const byId = new Map(categories.map((c) => [c.id, c])); + const grouped = new Map(); + + for (const category of categories) { + const root = resolveRootCategory(category, byId); + const bucket = grouped.get(root.id) ?? []; + bucket.push(category); + grouped.set(root.id, bucket); + } + + const rootIds = [...grouped.keys()]; + const selected: CategoryInfo[] = []; + let depth = 0; + + while (selected.length < maxCategories) { + let progressed = false; + + for (const rootId of rootIds) { + const bucket = grouped.get(rootId) ?? []; + if (depth >= bucket.length) continue; + selected.push(bucket[depth]!); + progressed = true; + if (selected.length >= maxCategories) break; + } + + if (!progressed) break; + depth++; + } + + return selected; +} + +async function discoverCategories( + maxCategories: number, +): Promise { + const data = await keepaGetJson( + `/category?key=${encodeURIComponent(config.keepaApiKey)}&domain=${DOMAIN_US}&category=0`, + ); + + const categories = normalizeCategoryList(data); + if (categories.length === 0) { + throw new Error("Keepa category discovery returned no usable categories."); + } + + const prioritized = prioritizeLikelyBestsellerCategories(categories); + return selectCategoriesAcrossRoots(prioritized, maxCategories); +} + +async function fetchCategoryBestSellerAsins( + category: CategoryInfo, + limit: number, +): Promise { + const query = new URLSearchParams({ + key: config.keepaApiKey, + domain: String(DOMAIN_US), + category: String(category.id), + range: "0", + variations: "0", + sublist: category.parentId > 0 ? "1" : "0", + }); + + const data = await keepaGetJson(`/bestsellers?${query.toString()}`); + + const bestSellersList = data?.bestSellersList; + const candidates = [ + bestSellersList, + bestSellersList?.asinList, + bestSellersList?.asins, + bestSellersList?.bestSellers, + bestSellersList?.bestSellerAsins, + data?.asinList, + data?.asins, + data?.bestsellers, + data?.bestSellers, + data?.bestSellerAsins, + data?.bestsellerList?.asinList, + data?.categories?.[String(category.id)]?.asinList, + ]; + + for (const value of candidates) { + if (Array.isArray(value)) { + return [ + ...new Set(value.map((v) => String(v).trim()).filter(Boolean)), + ].slice(0, limit); + } + } + + return []; +} + +async function fetchSellabilityMap( + asins: string[], +): Promise> { + const sellability = new Map(); + + for (let i = 0; i < asins.length; i += SELLABILITY_BATCH_SIZE) { + const chunk = asins.slice(i, i + SELLABILITY_BATCH_SIZE); + const chunkResults = await fetchSellabilityBatch(chunk); + + for (const asin of chunk) { + const info = chunkResults.get(asin) ?? { + canSell: null, + sellabilityStatus: "unknown" as const, + sellabilityReason: "Sellability check returned no result", + }; + sellability.set(asin, info); + } + + console.log( + ` Sellability progress: ${Math.min(i + chunk.length, asins.length)}/${asins.length}`, + ); + } + + return sellability; +} + +async function fetchSpApiMap( + asins: string[], + sellabilityMap: Map, +): Promise> { + const pricingQueue = [...asins]; + const spApiMap = new Map(); + let done = 0; + + async function worker(): Promise { + while (pricingQueue.length > 0) { + const asin = pricingQueue.shift(); + if (!asin) return; + + const sellability = sellabilityMap.get(asin) ?? { + canSell: null, + sellabilityStatus: "unknown", + sellabilityReason: "Sellability missing", + }; + + const spApi = await fetchSpApiPricingAndFees(asin, sellability); + spApiMap.set(asin, spApi); + + done++; + if (done % 10 === 0 || done === asins.length) { + console.log(` Pricing progress: ${done}/${asins.length}`); + } + } + } + + const workers = Array.from( + { length: Math.min(PRICING_CONCURRENCY, asins.length || 1) }, + () => worker(), + ); + + await Promise.all(workers); + return spApiMap; +} + +function pickKeepaNumber(...values: unknown[]): number | null { + for (const value of values) { + if (typeof value !== "number" || !Number.isFinite(value)) continue; + if (value < 0) continue; + return value; + } + return null; +} + +function extractCurrentPrice(csv: number[][] | undefined): number | null { + if (!Array.isArray(csv)) return null; + + for (const series of [csv[0], csv[1]]) { + if (Array.isArray(series) && series.length >= 2) { + const lastPrice = series[series.length - 1]; + if (typeof lastPrice === "number" && lastPrice > 0) { + return Math.round((lastPrice / 100) * 100) / 100; + } + } + } + + return null; +} + +function parseKeepaProduct(product: Record): KeepaData { + const stats = product.stats; + const csv = product.csv; + const salesRankDrops30 = pickKeepaNumber( + product.salesRankDrops30, + stats?.salesRankDrops30, + ); + const salesRankDrops90 = + pickKeepaNumber(product.salesRankDrops90, stats?.salesRankDrops90) ?? + (salesRankDrops30 != null ? salesRankDrops30 * 3 : null); + const monthlySold = + pickKeepaNumber(product.monthlySold, stats?.monthlySold) ?? + salesRankDrops30; + + return { + currentPrice: extractCurrentPrice(csv), + avgPrice90: stats?.avg?.[0] != null ? stats.avg[0] / 100 : null, + minPrice90: stats?.min?.[0] != null ? stats.min[0] / 100 : null, + maxPrice90: stats?.max?.[0] != null ? stats.max[0] / 100 : null, + salesRank: stats?.current?.[3] ?? null, + salesRankAvg90: stats?.avg?.[3] ?? null, + salesRankDrops30, + salesRankDrops90, + sellerCount: stats?.current?.[11] ?? null, + buyBoxSeller: product.buyBoxSellerId ?? null, + buyBoxPrice: stats?.current?.[18] != null ? stats.current[18] / 100 : null, + monthlySold, + categoryTree: + product.categoryTree?.map((c: { name: string }) => c.name) ?? [], + }; +} + +async function fetchKeepaEnrichmentMap( + asins: string[], +): Promise> { + const out = new Map(); + + for (let i = 0; i < asins.length; i += KEEPA_PRODUCT_CHUNK_SIZE) { + const chunk = asins.slice(i, i + KEEPA_PRODUCT_CHUNK_SIZE); + const asinParam = encodeURIComponent(chunk.join(",")); + const data = await keepaGetJson( + `/product?key=${encodeURIComponent(config.keepaApiKey)}&domain=${DOMAIN_US}&asin=${asinParam}&stats=90`, + ); + + const products = Array.isArray(data?.products) ? data.products : []; + for (const product of products) { + const asin = String(product?.asin ?? "").trim(); + if (!asin) continue; + out.set(asin, { + keepa: parseKeepaProduct(product), + title: String(product?.title ?? "").trim(), + }); + } + + console.log( + ` Keepa enrichment progress: ${Math.min(i + chunk.length, asins.length)}/${asins.length}`, + ); + } + + return out; +} + +function buildEnrichedProducts( + asins: string[], + sellabilityMap: Map, + spApiMap: Map, + titleByAsin: Map, +): EnrichedProduct[] { + return asins.map((asin) => { + const keepa = null; + const sellability = sellabilityMap.get(asin) ?? { + canSell: null, + sellabilityStatus: "unknown" as const, + sellabilityReason: "Sellability missing", + }; + + const spApi = spApiMap.get(asin) ?? { + fbaFee: 0, + fbmFee: 0, + referralFeePercent: 15, + estimatedSalePrice: 0, + canSell: sellability.canSell, + sellabilityStatus: sellability.sellabilityStatus, + sellabilityReason: sellability.sellabilityReason, + }; + + const record: ProductRecord = { + asin, + name: titleByAsin.get(asin) ?? asin, + unitCost: 0, + category: undefined, + brand: undefined, + supplier: undefined, + }; + + return { + record, + keepa, + spApi, + fetchedAt: new Date().toISOString(), + }; + }); +} + +async function runLlmInBatches( + products: EnrichedProduct[], +): Promise { + const verdicts: LlmVerdict[] = []; + + for (let i = 0; i < products.length; i += LLM_BATCH_SIZE) { + const batch = products.slice(i, i + LLM_BATCH_SIZE); + const batchNum = Math.floor(i / LLM_BATCH_SIZE) + 1; + const totalBatches = Math.ceil(products.length / LLM_BATCH_SIZE); + + console.log(` LLM batch ${batchNum}/${totalBatches}...`); + + let batchVerdicts: LlmVerdict[]; + try { + batchVerdicts = await analyzeProducts(batch); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + console.warn(` LLM batch failed: ${message}`); + batchVerdicts = batch.map((p) => ({ + asin: p.record.asin, + verdict: "SKIP", + confidence: 0, + reasoning: "LLM analysis failed", + })); + } + + verdicts.push(...batchVerdicts); + + if (i + LLM_BATCH_SIZE < products.length) { + await sleep(1500); + } + } + + return verdicts; +} + +async function processCategory( + category: CategoryInfo, + perCategoryTop: number, + outputDir: string, +): Promise { + console.log(`\nCategory ${category.label} (${category.id})`); + + const topAsins = await fetchCategoryBestSellerAsins(category, perCategoryTop); + if (topAsins.length === 0) { + console.log(" Keepa returned no ASINs for this category."); + return { + categoryId: category.id, + categoryLabel: category.label, + topAsinsChecked: 0, + availableAsins: 0, + fba: 0, + fbm: 0, + skip: 0, + outputFile: "", + status: "empty", + error: "No ASINs returned by Keepa", + }; + } + + console.log(` Top ASINs fetched: ${topAsins.length}`); + const sellabilityMap = await fetchSellabilityMap(topAsins); + + const availableAsins = topAsins.filter((asin) => { + const info = sellabilityMap.get(asin); + return info?.canSell === true && info.sellabilityStatus === "available"; + }); + + console.log(` Sellable ASINs: ${availableAsins.length}/${topAsins.length}`); + if (availableAsins.length === 0) { + return { + categoryId: category.id, + categoryLabel: category.label, + topAsinsChecked: topAsins.length, + availableAsins: 0, + fba: 0, + fbm: 0, + skip: 0, + outputFile: "", + status: "empty", + error: "No sellable ASINs", + }; + } + + const keepaEnrichment = await fetchKeepaEnrichmentMap(availableAsins); + const spApiMap = await fetchSpApiMap(availableAsins, sellabilityMap); + const titleByAsin = new Map(); + const keepaMap = new Map(); + + for (const asin of availableAsins) { + const enriched = keepaEnrichment.get(asin); + if (enriched?.title) { + titleByAsin.set(asin, enriched.title); + } + if (enriched?.keepa) { + keepaMap.set(asin, enriched.keepa); + } + } + + const enrichedProducts = buildEnrichedProducts( + availableAsins, + sellabilityMap, + spApiMap, + titleByAsin, + ).map((product) => { + const keepa = keepaMap.get(product.record.asin) ?? null; + const spApi = product.spApi; + + if (keepa?.currentPrice && spApi.estimatedSalePrice === 0) { + spApi.estimatedSalePrice = keepa.currentPrice; + } + + return { + ...product, + keepa, + spApi, + }; + }); + + const verdicts = await runLlmInBatches(enrichedProducts); + const verdictByAsin = new Map(verdicts.map((v) => [v.asin, v])); + + const results: AnalysisResult[] = enrichedProducts.map((product) => ({ + product, + verdict: verdictByAsin.get(product.record.asin) ?? { + asin: product.record.asin, + verdict: "SKIP", + confidence: 0, + reasoning: "LLM returned no verdict", + }, + })); + + const outputName = `${sanitizeFileSegment(category.label)}_${category.id}.xlsx`; + const outputPath = path.join(outputDir, outputName); + writeCategoryResultsWorkbook(results, outputPath); + + const fba = results.filter((r) => r.verdict.verdict === "FBA").length; + const fbm = results.filter((r) => r.verdict.verdict === "FBM").length; + const skip = results.filter((r) => r.verdict.verdict === "SKIP").length; + + return { + categoryId: category.id, + categoryLabel: category.label, + topAsinsChecked: topAsins.length, + availableAsins: availableAsins.length, + fba, + fbm, + skip, + outputFile: path.basename(outputPath), + status: "ok", + error: "", + }; +} + +function buildCategoryOutputRow(r: AnalysisResult) { + const price = + r.product.keepa?.currentPrice ?? + r.product.record.sellingPriceFromSheet ?? + r.product.spApi.estimatedSalePrice; + const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank; + + return { + ASIN: r.product.record.asin, + Name: r.product.record.name, + Brand: r.product.record.brand ?? "", + Category: + r.product.record.category ?? + r.product.keepa?.categoryTree?.join(" > ") ?? + "", + "Unit Cost": r.product.record.unitCost, + "Current Price": price ?? "", + "Avg Price 90d": r.product.keepa?.avgPrice90 ?? "", + "Avg Price 90d (sheet)": r.product.record.avgPrice90FromSheet ?? "", + "Selling Price (sheet)": r.product.record.sellingPriceFromSheet ?? "", + "Sales Rank": rank ?? "", + "Rank Avg 90d": r.product.keepa?.salesRankAvg90 ?? "", + Sellers: r.product.keepa?.sellerCount ?? "", + "Monthly Sold": r.product.keepa?.monthlySold ?? "", + "Rank Drops 30d": r.product.keepa?.salesRankDrops30 ?? "", + "Rank Drops 90d": r.product.keepa?.salesRankDrops90 ?? "", + "FBA Fee": r.product.spApi.fbaFee, + "FBM Fee": r.product.spApi.fbmFee, + "Referral %": r.product.spApi.referralFeePercent, + "Can Sell": + r.product.spApi.canSell == null + ? "unknown" + : r.product.spApi.canSell + ? "yes" + : "no", + Sellability: r.product.spApi.sellabilityStatus, + "Sellability Reason": r.product.spApi.sellabilityReason ?? "", + Verdict: r.verdict.verdict, + Confidence: r.verdict.confidence, + Reasoning: r.verdict.reasoning, + }; +} + +function writeCategoryResultsWorkbook( + results: AnalysisResult[], + outputPath: string, +): void { + const rows = results.map(buildCategoryOutputRow); + const ws = XLSX.utils.json_to_sheet(rows); + const wb = XLSX.utils.book_new(); + XLSX.utils.book_append_sheet(wb, ws, "Results"); + XLSX.writeFile(wb, outputPath); + console.log(`Results written to ${outputPath}`); +} + +function writeConsolidatedWorkbook( + summaries: CategoryRunSummary[], + outputDir: string, +): string { + const workbook = XLSX.utils.book_new(); + + const summaryRows = summaries.map((row) => ({ + "Category ID": row.categoryId, + "Category Label": row.categoryLabel, + "Top ASINs Checked": row.topAsinsChecked, + "Sellable ASINs": row.availableAsins, + FBA: row.fba, + FBM: row.fbm, + SKIP: row.skip, + Status: row.status, + "Output File": row.outputFile, + Error: row.error, + })); + + const totals = summaries.reduce( + (acc, row) => { + acc.topAsinsChecked += row.topAsinsChecked; + acc.availableAsins += row.availableAsins; + acc.fba += row.fba; + acc.fbm += row.fbm; + acc.skip += row.skip; + if (row.status === "ok") acc.ok += 1; + if (row.status === "empty") acc.empty += 1; + if (row.status === "failed") acc.failed += 1; + return acc; + }, + { + topAsinsChecked: 0, + availableAsins: 0, + fba: 0, + fbm: 0, + skip: 0, + ok: 0, + empty: 0, + failed: 0, + }, + ); + + const overviewRows = [ + { Metric: "Categories total", Value: summaries.length }, + { Metric: "Categories with output", Value: totals.ok }, + { Metric: "Categories empty", Value: totals.empty }, + { Metric: "Categories failed", Value: totals.failed }, + { Metric: "Top ASINs checked", Value: totals.topAsinsChecked }, + { Metric: "Sellable ASINs", Value: totals.availableAsins }, + { Metric: "Total FBA verdicts", Value: totals.fba }, + { Metric: "Total FBM verdicts", Value: totals.fbm }, + { Metric: "Total SKIP verdicts", Value: totals.skip }, + ]; + + const summarySheet = XLSX.utils.json_to_sheet(summaryRows); + const overviewSheet = XLSX.utils.json_to_sheet(overviewRows); + XLSX.utils.book_append_sheet(workbook, overviewSheet, "Overview"); + XLSX.utils.book_append_sheet(workbook, summarySheet, "ByCategory"); + + const outputPath = path.join( + outputDir, + "consolidated_bestsellers_summary.xlsx", + ); + XLSX.writeFile(workbook, outputPath); + return outputPath; +} + +function printSummary( + categories: CategoryInfo[], + processed: number, + generatedFiles: number, + totalTopAsins: number, + totalAvailableAsins: number, +): void { + console.log("\nRun summary"); + console.log(`Categories discovered/selected: ${categories.length}`); + console.log(`Categories processed: ${processed}`); + console.log(`Category files written: ${generatedFiles}`); + console.log(`Top ASINs checked: ${totalTopAsins}`); + console.log(`Sellable ASINs enriched: ${totalAvailableAsins}`); +} + +async function main(): Promise { + const args = parseArgs(); + assertSpApiPrerequisites(); + + mkdirSync(args.outputDir, { recursive: true }); + + console.log("Starting per-category bestseller pipeline"); + console.log(`Marketplace: ${config.spApiMarketplaceId}`); + console.log(`SP-API region: ${config.spApiRegion}`); + console.log(`Category limit: ${args.categoryLimit}`); + console.log(`Top ASINs per category: ${args.perCategoryTop}`); + console.log(`Output directory: ${args.outputDir}`); + console.log(`Blacklist file: ${args.blacklistFile}`); + + const categoryBlacklist = loadCategoryBlacklist(args.blacklistFile); + console.log(`Loaded ${categoryBlacklist.size} blacklisted category IDs.`); + + const categories = await discoverCategories(args.categoryLimit); + const allowedCategories = categories.filter( + (c) => !categoryBlacklist.has(c.id), + ); + const blacklistedCount = categories.length - allowedCategories.length; + console.log( + `Discovered ${categories.length} categories (${blacklistedCount} blacklisted, ${allowedCategories.length} to process).`, + ); + + let processed = 0; + let generatedFiles = 0; + let totalTopAsins = 0; + let totalAvailableAsins = 0; + const categorySummaries: CategoryRunSummary[] = []; + + for (const category of allowedCategories) { + try { + const outcome = await processCategory( + category, + args.perCategoryTop, + args.outputDir, + ); + + processed++; + totalTopAsins += outcome.topAsinsChecked; + totalAvailableAsins += outcome.availableAsins; + if (outcome.status === "ok") { + generatedFiles++; + } + categorySummaries.push(outcome); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + console.warn( + `Skipping category ${category.label} (${category.id}) due to error: ${message}`, + ); + processed++; + categorySummaries.push({ + categoryId: category.id, + categoryLabel: category.label, + topAsinsChecked: 0, + availableAsins: 0, + fba: 0, + fbm: 0, + skip: 0, + outputFile: "", + status: "failed", + error: message, + }); + } + } + + const consolidatedPath = writeConsolidatedWorkbook( + categorySummaries, + args.outputDir, + ); + console.log(`Consolidated workbook written: ${consolidatedPath}`); + + printSummary( + allowedCategories, + processed, + generatedFiles, + totalTopAsins, + totalAvailableAsins, + ); +} + +await main(); diff --git a/src/cache.ts b/src/cache.ts index 93fa618..5de7710 100644 --- a/src/cache.ts +++ b/src/cache.ts @@ -1,66 +1,66 @@ -import Redis from "ioredis"; -import { config } from "./config.ts"; -import type { EnrichedProduct } from "./types.ts"; - -let redis: Redis | null = null; -let disabled = false; - -export async function connectCache(): Promise { - if (disabled) return; - try { - redis = new Redis(config.redisUrl, { - maxRetriesPerRequest: 1, - connectTimeout: 3000, - lazyConnect: true, - retryStrategy: () => null, - reconnectOnError: () => false, - }); - // Swallow connection-level errors after we intentionally disable cache. - redis.on("error", () => { - // no-op - }); - await redis.connect(); - console.log("Redis connected"); - } catch (err) { - console.warn(`Redis unavailable, running without cache: ${err}`); - if (redis) { - redis.disconnect(); - } - redis = null; - disabled = true; - } -} - -export async function getCache(asin: string): Promise { - if (!redis) return null; - try { - const data = await redis.get(`asin:${asin}`); - return data ? JSON.parse(data) : null; - } catch { - return null; - } -} - -export async function setCache( - asin: string, - data: EnrichedProduct, -): Promise { - if (!redis) return; - try { - await redis.set( - `asin:${asin}`, - JSON.stringify(data), - "EX", - config.cacheTtl, - ); - } catch { - // Non-critical, continue without caching - } -} - -export async function disconnectCache(): Promise { - if (redis) { - await redis.quit(); - redis = null; - } -} +import Redis from "ioredis"; +import { config } from "./config.ts"; +import type { EnrichedProduct } from "./types.ts"; + +let redis: Redis | null = null; +let disabled = false; + +export async function connectCache(): Promise { + if (disabled) return; + try { + redis = new Redis(config.redisUrl, { + maxRetriesPerRequest: 1, + connectTimeout: 3000, + lazyConnect: true, + retryStrategy: () => null, + reconnectOnError: () => false, + }); + // Swallow connection-level errors after we intentionally disable cache. + redis.on("error", () => { + // no-op + }); + await redis.connect(); + console.log("Redis connected"); + } catch (err) { + console.warn(`Redis unavailable, running without cache: ${err}`); + if (redis) { + redis.disconnect(); + } + redis = null; + disabled = true; + } +} + +export async function getCache(asin: string): Promise { + if (!redis) return null; + try { + const data = await redis.get(`asin:${asin}`); + return data ? JSON.parse(data) : null; + } catch { + return null; + } +} + +export async function setCache( + asin: string, + data: EnrichedProduct, +): Promise { + if (!redis) return; + try { + await redis.set( + `asin:${asin}`, + JSON.stringify(data), + "EX", + config.cacheTtl, + ); + } catch { + // Non-critical, continue without caching + } +} + +export async function disconnectCache(): Promise { + if (redis) { + await redis.quit(); + redis = null; + } +} diff --git a/src/config.ts b/src/config.ts index c658af0..7aa4692 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,34 +1,34 @@ -function required(key: string): string { - const val = Bun.env[key]; - if (!val) throw new Error(`Missing required env var: ${key}`); - return val; -} - -function optional(key: string, fallback: string): string { - return Bun.env[key] || fallback; -} - -function optionalBoolean(key: string, fallback: boolean): boolean { - const raw = Bun.env[key]; - if (!raw) return fallback; - const value = raw.trim().toLowerCase(); - return value === "1" || value === "true" || value === "yes"; -} - -export const config = { - keepaApiKey: required("KEEPA_API_KEY"), - redisUrl: optional("REDIS_URL", "redis://localhost:6379"), - llmUrl: optional("LLM_URL", "http://localhost:1234/v1"), - llmModel: optional("LLM_MODEL", "default"), - cacheTtl: parseInt(optional("CACHE_TTL", "86400"), 10), - spApiClientId: Bun.env.SP_API_CLIENT_ID, - spApiClientSecret: Bun.env.SP_API_CLIENT_SECRET, - spApiRefreshToken: Bun.env.SP_API_REFRESH_TOKEN, - spApiRegion: optional("SP_API_REGION", "na"), - spApiMarketplaceId: optional("SP_API_MARKETPLACE_ID", "ATVPDKIKX0DER"), - spApiSellerId: Bun.env.SP_API_SELLER_ID, - spApiUseSandbox: optionalBoolean("SP_API_USE_SANDBOX", false), - awsAccessKeyId: Bun.env.AWS_ACCESS_KEY_ID, - awsSecretAccessKey: Bun.env.AWS_SECRET_ACCESS_KEY, - awsSessionToken: Bun.env.AWS_SESSION_TOKEN, -} as const; +function required(key: string): string { + const val = Bun.env[key]; + if (!val) throw new Error(`Missing required env var: ${key}`); + return val; +} + +function optional(key: string, fallback: string): string { + return Bun.env[key] || fallback; +} + +function optionalBoolean(key: string, fallback: boolean): boolean { + const raw = Bun.env[key]; + if (!raw) return fallback; + const value = raw.trim().toLowerCase(); + return value === "1" || value === "true" || value === "yes"; +} + +export const config = { + keepaApiKey: required("KEEPA_API_KEY"), + redisUrl: optional("REDIS_URL", "redis://localhost:6379"), + llmUrl: optional("LLM_URL", "http://localhost:1234/v1"), + llmModel: optional("LLM_MODEL", "default"), + cacheTtl: parseInt(optional("CACHE_TTL", "86400"), 10), + spApiClientId: Bun.env.SP_API_CLIENT_ID, + spApiClientSecret: Bun.env.SP_API_CLIENT_SECRET, + spApiRefreshToken: Bun.env.SP_API_REFRESH_TOKEN, + spApiRegion: optional("SP_API_REGION", "na"), + spApiMarketplaceId: optional("SP_API_MARKETPLACE_ID", "ATVPDKIKX0DER"), + spApiSellerId: Bun.env.SP_API_SELLER_ID, + spApiUseSandbox: optionalBoolean("SP_API_USE_SANDBOX", false), + awsAccessKeyId: Bun.env.AWS_ACCESS_KEY_ID, + awsSecretAccessKey: Bun.env.AWS_SECRET_ACCESS_KEY, + awsSessionToken: Bun.env.AWS_SESSION_TOKEN, +} as const; diff --git a/src/index.ts b/src/index.ts index e60ab2d..44ba2d5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,345 +1,345 @@ -import { readProducts } from "./reader.ts"; -import { fetchKeepaDataBatch } from "./keepa.ts"; -import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts"; -import { connectCache, getCache, setCache, disconnectCache } from "./cache.ts"; -import { analyzeProducts } from "./llm.ts"; -import { printResults, writeResultsCsv } from "./writer.ts"; -import path from "node:path"; -import type { - EnrichedProduct, - AnalysisResult, - KeepaData, - ProductRecord, - SellabilityInfo, - SpApiData, -} from "./types.ts"; - -const LLM_BATCH_SIZE = 5; -const INPUT_BATCH_SIZE = 50; - -function parseArgs(): { inputFile: string; outputFile?: string } { - const args = process.argv.slice(2); - const inputFile = args.find((a) => !a.startsWith("--")); - const outIdx = args.indexOf("--out"); - const outputFile = outIdx !== -1 ? args[outIdx + 1] : undefined; - - if (!inputFile) { - console.error( - "Usage: bun run src/index.ts [--out results.csv]", - ); - process.exit(1); - } - return { inputFile, outputFile }; -} - -function chunkArray(items: T[], chunkSize: number): T[][] { - const chunks: T[][] = []; - for (let i = 0; i < items.length; i += chunkSize) { - chunks.push(items.slice(i, i + chunkSize)); - } - return chunks; -} - -function resolveBaseOutputPath(inputFile: string, outputFile?: string): string { - if (outputFile) return outputFile; - - const parsedInput = path.parse(inputFile); - return path.join(parsedInput.dir, `${parsedInput.name}_results.xlsx`); -} - -function buildChunkOutputPath( - baseOutputPath: string, - chunkIndex: number, -): string { - const parsed = path.parse(baseOutputPath); - const extension = parsed.ext || ".xlsx"; - const chunkSuffix = String(chunkIndex + 1).padStart(3, "0"); - return path.join( - parsed.dir, - `${parsed.name}_part_${chunkSuffix}${extension}`, - ); -} - -async function processProductChunk( - products: ProductRecord[], -): Promise { - // Phase 2: Check cache for all ASINs in chunk - console.log(`\nChecking cache for ${products.length} products...`); - const cached = new Map(); - const excludedCachedAsins = new Set(); - const uncachedProducts: ProductRecord[] = []; - - for (const p of products) { - const hit = await getCache(p.asin); - if (hit) { - if (hit.spApi.sellabilityStatus === "available") { - console.log(` [cache hit] ${p.asin}`); - cached.set(p.asin, hit); - } else { - excludedCachedAsins.add(p.asin); - console.log( - ` [exclude cached] ${p.asin} — status=${hit.spApi.sellabilityStatus}`, - ); - } - } else { - uncachedProducts.push(p); - } - } - console.log( - `${cached.size} cached available, ${excludedCachedAsins.size} cached excluded, ${uncachedProducts.length} to fetch`, - ); - - // Phase 3: Sellability gate — check uncached ASINs before anything else - const sellabilityMap = new Map(); - const availableProducts: ProductRecord[] = []; - const unavailableProducts: ProductRecord[] = []; - - if (uncachedProducts.length > 0) { - console.log( - `\nChecking sellability for ${uncachedProducts.length} ASINs...`, - ); - const sellResults = await fetchSellabilityBatch( - uncachedProducts.map((p) => p.asin), - ); - - for (const p of uncachedProducts) { - const info = sellResults.get(p.asin) ?? { - canSell: null, - sellabilityStatus: "unknown" as const, - sellabilityReason: "Sellability check returned no result", - }; - sellabilityMap.set(p.asin, info); - - // Keep only ASINs that are explicitly available. - if (info.sellabilityStatus === "available") { - availableProducts.push(p); - console.log( - ` [available] ${p.asin} — status=${info.sellabilityStatus}`, - ); - } else { - unavailableProducts.push(p); - console.log( - ` [exclude] ${p.asin} — status=${info.sellabilityStatus}, reason=${info.sellabilityReason ?? "n/a"}`, - ); - } - } - - console.log( - `\nSellability gate: ${availableProducts.length} available, ${unavailableProducts.length} excluded`, - ); - } - - // Phase 4: Keepa batch fetch — only for available (uncached) ASINs - let keepaResults = new Map(); - if (availableProducts.length > 0) { - console.log(`\nFetching ${availableProducts.length} ASINs from Keepa...`); - try { - keepaResults = await fetchKeepaDataBatch( - availableProducts.map((p) => p.asin), - ); - } catch (err) { - console.warn(`Keepa batch fetch failed: ${err}`); - } - } - - // Phase 5: SP-API pricing + fees — only for available ASINs - console.log( - `\nFetching pricing & fees for ${availableProducts.length} ASINs...`, - ); - const spApiResults = new Map(); - - // Concurrency-limited pricing+fees fetches - const pricingQueue = [...availableProducts]; - let pricingDone = 0; - - async function fetchNextPricing(): Promise { - while (pricingQueue.length > 0) { - const p = pricingQueue.shift()!; - const sellability = sellabilityMap.get(p.asin)!; - const spApi = await fetchSpApiPricingAndFees(p.asin, sellability); - - const keepa = keepaResults.get(p.asin); - if (keepa?.currentPrice && spApi.estimatedSalePrice === 0) { - spApi.estimatedSalePrice = keepa.currentPrice; - } - - spApiResults.set(p.asin, spApi); - pricingDone++; - if (pricingDone % 10 === 0 || pricingDone === availableProducts.length) { - console.log( - ` [pricing] ${pricingDone}/${availableProducts.length} fetched`, - ); - } - } - } - - const pricingWorkers = Array.from( - { length: Math.min(5, availableProducts.length || 1) }, - () => fetchNextPricing(), - ); - await Promise.all(pricingWorkers); - - // Phase 6: Build enriched products - console.log(`\nEnriching products...`); - const enriched: EnrichedProduct[] = []; - const availableAsins = new Set(availableProducts.map((ap) => ap.asin)); - - for (const p of products) { - if (excludedCachedAsins.has(p.asin)) { - continue; - } - - // Cached products — already enriched - const cachedProduct = cached.get(p.asin); - if (cachedProduct) { - enriched.push(cachedProduct); - continue; - } - - // Exclude products that are not explicitly available. - if (!availableAsins.has(p.asin)) { - continue; - } - - // Available products — full enrichment - const keepa = keepaResults.get(p.asin) ?? null; - const spApi = spApiResults.get(p.asin) ?? { - fbaFee: 5.0, - fbmFee: 1.5, - referralFeePercent: 15, - estimatedSalePrice: 0, - canSell: null, - sellabilityStatus: "unknown" as const, - sellabilityReason: "SP-API data missing", - }; - - const product: EnrichedProduct = { - record: p, - keepa, - spApi, - fetchedAt: new Date().toISOString(), - }; - - await setCache(p.asin, product); - enriched.push(product); - - if (keepa) { - console.log( - ` [enriched] ${p.asin} — price: $${keepa.currentPrice ?? "N/A"}, rank: ${keepa.salesRank ?? "N/A"}`, - ); - } else { - console.log(` [no keepa] ${p.asin} — using spreadsheet data only`); - } - } - - // Phase 7: LLM analysis in batches — only for enriched available products - console.log( - `\nAnalyzing ${enriched.length} products via LLM (batch size: ${LLM_BATCH_SIZE})...\n`, - ); - - const results: AnalysisResult[] = []; - for (let i = 0; i < enriched.length; i += LLM_BATCH_SIZE) { - const batch = enriched.slice(i, i + LLM_BATCH_SIZE); - const batchNum = Math.floor(i / LLM_BATCH_SIZE) + 1; - const totalBatches = Math.ceil(enriched.length / LLM_BATCH_SIZE); - console.log(` LLM batch ${batchNum}/${totalBatches}...`); - - // Wait between batches to avoid overwhelming LM Studio - if (i > 0) { - console.log(` Waiting 5s before next batch...`); - await new Promise((r) => setTimeout(r, 5000)); - } - - let verdicts; - try { - verdicts = await analyzeProducts(batch); - } catch { - console.warn(` LLM batch error, retrying after 10s...`); - await new Promise((r) => setTimeout(r, 10_000)); - try { - verdicts = await analyzeProducts(batch); - } catch (retryErr) { - console.error(` LLM analysis failed: ${retryErr}`); - verdicts = null; - } - } - - for (let j = 0; j < batch.length; j++) { - results.push({ - product: batch[j]!, - verdict: verdicts?.[j] ?? { - asin: batch[j]!.record.asin, - verdict: "SKIP", - confidence: 0, - reasoning: "LLM analysis failed", - }, - }); - } - } - - return results; -} - -async function main() { - const { inputFile, outputFile } = parseArgs(); - - console.log("Connecting to Redis..."); - await connectCache(); - - try { - // Phase 1: Read input file - console.log(`\nReading ${inputFile}...`); - const products = readProducts(inputFile); - - if (products.length === 0) { - console.error("No valid products found in input file."); - process.exit(1); - } - - const productChunks = chunkArray(products, INPUT_BATCH_SIZE); - const hasMultipleChunks = productChunks.length > 1; - const shouldWriteChunkFiles = hasMultipleChunks; - const resolvedBaseOutputPath = resolveBaseOutputPath(inputFile, outputFile); - const allResults: AnalysisResult[] = []; - - if (hasMultipleChunks) { - console.log( - `\nLarge input detected (${products.length} products). Processing in chunks of ${INPUT_BATCH_SIZE}.`, - ); - console.log( - `Chunk outputs will be written as numbered files using base: ${resolvedBaseOutputPath}`, - ); - } - - for (let chunkIndex = 0; chunkIndex < productChunks.length; chunkIndex++) { - const chunk = productChunks[chunkIndex]!; - console.log( - `\n=== Input chunk ${chunkIndex + 1}/${productChunks.length} (${chunk.length} products) ===`, - ); - - const chunkResults = await processProductChunk(chunk); - allResults.push(...chunkResults); - - if (shouldWriteChunkFiles) { - const chunkOutputPath = buildChunkOutputPath( - resolvedBaseOutputPath, - chunkIndex, - ); - writeResultsCsv(chunkResults, chunkOutputPath); - } - } - - printResults(allResults); - - if (!hasMultipleChunks && outputFile) { - writeResultsCsv(allResults, outputFile); - } - } finally { - await disconnectCache(); - } -} - -main().catch((err) => { - console.error("Fatal error:", err); - process.exit(1); -}); +import { readProducts } from "./reader.ts"; +import { fetchKeepaDataBatch } from "./keepa.ts"; +import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts"; +import { connectCache, getCache, setCache, disconnectCache } from "./cache.ts"; +import { analyzeProducts } from "./llm.ts"; +import { printResults, writeResultsCsv } from "./writer.ts"; +import path from "node:path"; +import type { + EnrichedProduct, + AnalysisResult, + KeepaData, + ProductRecord, + SellabilityInfo, + SpApiData, +} from "./types.ts"; + +const LLM_BATCH_SIZE = 5; +const INPUT_BATCH_SIZE = 50; + +function parseArgs(): { inputFile: string; outputFile?: string } { + const args = process.argv.slice(2); + const inputFile = args.find((a) => !a.startsWith("--")); + const outIdx = args.indexOf("--out"); + const outputFile = outIdx !== -1 ? args[outIdx + 1] : undefined; + + if (!inputFile) { + console.error( + "Usage: bun run src/index.ts [--out results.csv]", + ); + process.exit(1); + } + return { inputFile, outputFile }; +} + +function chunkArray(items: T[], chunkSize: number): T[][] { + const chunks: T[][] = []; + for (let i = 0; i < items.length; i += chunkSize) { + chunks.push(items.slice(i, i + chunkSize)); + } + return chunks; +} + +function resolveBaseOutputPath(inputFile: string, outputFile?: string): string { + if (outputFile) return outputFile; + + const parsedInput = path.parse(inputFile); + return path.join(parsedInput.dir, `${parsedInput.name}_results.xlsx`); +} + +function buildChunkOutputPath( + baseOutputPath: string, + chunkIndex: number, +): string { + const parsed = path.parse(baseOutputPath); + const extension = parsed.ext || ".xlsx"; + const chunkSuffix = String(chunkIndex + 1).padStart(3, "0"); + return path.join( + parsed.dir, + `${parsed.name}_part_${chunkSuffix}${extension}`, + ); +} + +async function processProductChunk( + products: ProductRecord[], +): Promise { + // Phase 2: Check cache for all ASINs in chunk + console.log(`\nChecking cache for ${products.length} products...`); + const cached = new Map(); + const excludedCachedAsins = new Set(); + const uncachedProducts: ProductRecord[] = []; + + for (const p of products) { + const hit = await getCache(p.asin); + if (hit) { + if (hit.spApi.sellabilityStatus === "available") { + console.log(` [cache hit] ${p.asin}`); + cached.set(p.asin, hit); + } else { + excludedCachedAsins.add(p.asin); + console.log( + ` [exclude cached] ${p.asin} — status=${hit.spApi.sellabilityStatus}`, + ); + } + } else { + uncachedProducts.push(p); + } + } + console.log( + `${cached.size} cached available, ${excludedCachedAsins.size} cached excluded, ${uncachedProducts.length} to fetch`, + ); + + // Phase 3: Sellability gate — check uncached ASINs before anything else + const sellabilityMap = new Map(); + const availableProducts: ProductRecord[] = []; + const unavailableProducts: ProductRecord[] = []; + + if (uncachedProducts.length > 0) { + console.log( + `\nChecking sellability for ${uncachedProducts.length} ASINs...`, + ); + const sellResults = await fetchSellabilityBatch( + uncachedProducts.map((p) => p.asin), + ); + + for (const p of uncachedProducts) { + const info = sellResults.get(p.asin) ?? { + canSell: null, + sellabilityStatus: "unknown" as const, + sellabilityReason: "Sellability check returned no result", + }; + sellabilityMap.set(p.asin, info); + + // Keep only ASINs that are explicitly available. + if (info.sellabilityStatus === "available") { + availableProducts.push(p); + console.log( + ` [available] ${p.asin} — status=${info.sellabilityStatus}`, + ); + } else { + unavailableProducts.push(p); + console.log( + ` [exclude] ${p.asin} — status=${info.sellabilityStatus}, reason=${info.sellabilityReason ?? "n/a"}`, + ); + } + } + + console.log( + `\nSellability gate: ${availableProducts.length} available, ${unavailableProducts.length} excluded`, + ); + } + + // Phase 4: Keepa batch fetch — only for available (uncached) ASINs + let keepaResults = new Map(); + if (availableProducts.length > 0) { + console.log(`\nFetching ${availableProducts.length} ASINs from Keepa...`); + try { + keepaResults = await fetchKeepaDataBatch( + availableProducts.map((p) => p.asin), + ); + } catch (err) { + console.warn(`Keepa batch fetch failed: ${err}`); + } + } + + // Phase 5: SP-API pricing + fees — only for available ASINs + console.log( + `\nFetching pricing & fees for ${availableProducts.length} ASINs...`, + ); + const spApiResults = new Map(); + + // Concurrency-limited pricing+fees fetches + const pricingQueue = [...availableProducts]; + let pricingDone = 0; + + async function fetchNextPricing(): Promise { + while (pricingQueue.length > 0) { + const p = pricingQueue.shift()!; + const sellability = sellabilityMap.get(p.asin)!; + const spApi = await fetchSpApiPricingAndFees(p.asin, sellability); + + const keepa = keepaResults.get(p.asin); + if (keepa?.currentPrice && spApi.estimatedSalePrice === 0) { + spApi.estimatedSalePrice = keepa.currentPrice; + } + + spApiResults.set(p.asin, spApi); + pricingDone++; + if (pricingDone % 10 === 0 || pricingDone === availableProducts.length) { + console.log( + ` [pricing] ${pricingDone}/${availableProducts.length} fetched`, + ); + } + } + } + + const pricingWorkers = Array.from( + { length: Math.min(5, availableProducts.length || 1) }, + () => fetchNextPricing(), + ); + await Promise.all(pricingWorkers); + + // Phase 6: Build enriched products + console.log(`\nEnriching products...`); + const enriched: EnrichedProduct[] = []; + const availableAsins = new Set(availableProducts.map((ap) => ap.asin)); + + for (const p of products) { + if (excludedCachedAsins.has(p.asin)) { + continue; + } + + // Cached products — already enriched + const cachedProduct = cached.get(p.asin); + if (cachedProduct) { + enriched.push(cachedProduct); + continue; + } + + // Exclude products that are not explicitly available. + if (!availableAsins.has(p.asin)) { + continue; + } + + // Available products — full enrichment + const keepa = keepaResults.get(p.asin) ?? null; + const spApi = spApiResults.get(p.asin) ?? { + fbaFee: 5.0, + fbmFee: 1.5, + referralFeePercent: 15, + estimatedSalePrice: 0, + canSell: null, + sellabilityStatus: "unknown" as const, + sellabilityReason: "SP-API data missing", + }; + + const product: EnrichedProduct = { + record: p, + keepa, + spApi, + fetchedAt: new Date().toISOString(), + }; + + await setCache(p.asin, product); + enriched.push(product); + + if (keepa) { + console.log( + ` [enriched] ${p.asin} — price: $${keepa.currentPrice ?? "N/A"}, rank: ${keepa.salesRank ?? "N/A"}`, + ); + } else { + console.log(` [no keepa] ${p.asin} — using spreadsheet data only`); + } + } + + // Phase 7: LLM analysis in batches — only for enriched available products + console.log( + `\nAnalyzing ${enriched.length} products via LLM (batch size: ${LLM_BATCH_SIZE})...\n`, + ); + + const results: AnalysisResult[] = []; + for (let i = 0; i < enriched.length; i += LLM_BATCH_SIZE) { + const batch = enriched.slice(i, i + LLM_BATCH_SIZE); + const batchNum = Math.floor(i / LLM_BATCH_SIZE) + 1; + const totalBatches = Math.ceil(enriched.length / LLM_BATCH_SIZE); + console.log(` LLM batch ${batchNum}/${totalBatches}...`); + + // Wait between batches to avoid overwhelming LM Studio + if (i > 0) { + console.log(` Waiting 5s before next batch...`); + await new Promise((r) => setTimeout(r, 5000)); + } + + let verdicts; + try { + verdicts = await analyzeProducts(batch); + } catch { + console.warn(` LLM batch error, retrying after 10s...`); + await new Promise((r) => setTimeout(r, 10_000)); + try { + verdicts = await analyzeProducts(batch); + } catch (retryErr) { + console.error(` LLM analysis failed: ${retryErr}`); + verdicts = null; + } + } + + for (let j = 0; j < batch.length; j++) { + results.push({ + product: batch[j]!, + verdict: verdicts?.[j] ?? { + asin: batch[j]!.record.asin, + verdict: "SKIP", + confidence: 0, + reasoning: "LLM analysis failed", + }, + }); + } + } + + return results; +} + +async function main() { + const { inputFile, outputFile } = parseArgs(); + + console.log("Connecting to Redis..."); + await connectCache(); + + try { + // Phase 1: Read input file + console.log(`\nReading ${inputFile}...`); + const products = readProducts(inputFile); + + if (products.length === 0) { + console.error("No valid products found in input file."); + process.exit(1); + } + + const productChunks = chunkArray(products, INPUT_BATCH_SIZE); + const hasMultipleChunks = productChunks.length > 1; + const shouldWriteChunkFiles = hasMultipleChunks; + const resolvedBaseOutputPath = resolveBaseOutputPath(inputFile, outputFile); + const allResults: AnalysisResult[] = []; + + if (hasMultipleChunks) { + console.log( + `\nLarge input detected (${products.length} products). Processing in chunks of ${INPUT_BATCH_SIZE}.`, + ); + console.log( + `Chunk outputs will be written as numbered files using base: ${resolvedBaseOutputPath}`, + ); + } + + for (let chunkIndex = 0; chunkIndex < productChunks.length; chunkIndex++) { + const chunk = productChunks[chunkIndex]!; + console.log( + `\n=== Input chunk ${chunkIndex + 1}/${productChunks.length} (${chunk.length} products) ===`, + ); + + const chunkResults = await processProductChunk(chunk); + allResults.push(...chunkResults); + + if (shouldWriteChunkFiles) { + const chunkOutputPath = buildChunkOutputPath( + resolvedBaseOutputPath, + chunkIndex, + ); + writeResultsCsv(chunkResults, chunkOutputPath); + } + } + + printResults(allResults); + + if (!hasMultipleChunks && outputFile) { + writeResultsCsv(allResults, outputFile); + } + } finally { + await disconnectCache(); + } +} + +main().catch((err) => { + console.error("Fatal error:", err); + process.exit(1); +}); diff --git a/src/keepa.ts b/src/keepa.ts index e086cde..2f2f49d 100644 --- a/src/keepa.ts +++ b/src/keepa.ts @@ -1,141 +1,141 @@ -import { config } from "./config.ts"; -import type { KeepaData } from "./types.ts"; - -const KEEPA_BASE = "https://api.keepa.com"; -const MAX_ASINS_PER_REQUEST = 100; - -// Token-based rate limiting: Keepa Pro = 1 token/min regeneration. -// Each product request costs 1 token regardless of ASIN count (up to 100). -// The API response includes tokensLeft and refillRate — we use those to pace. -let tokensLeft = 1; // Conservative start; updated from API response -let refillRate = 1; // tokens per minute, updated from API response -let lastRequestTime = 0; - -async function waitForToken(): Promise { - if (tokensLeft > 0) return; - - const elapsed = (Date.now() - lastRequestTime) / 60_000; // minutes - const regenerated = Math.floor(elapsed * refillRate); - if (regenerated > 0) { - tokensLeft += regenerated; - return; - } - - // Wait until we regenerate at least 1 token - const waitMs = - Math.ceil((1 / refillRate) * 60_000) - (Date.now() - lastRequestTime); - if (waitMs > 0) { - console.log( - `Keepa tokens exhausted. Waiting ${Math.ceil(waitMs / 1000)}s for token regeneration...`, - ); - await new Promise((r) => setTimeout(r, waitMs)); - } - tokensLeft = 1; -} - -export async function fetchKeepaDataBatch( - asins: string[], -): Promise> { - const results = new Map(); - - // Split into chunks of MAX_ASINS_PER_REQUEST - for (let i = 0; i < asins.length; i += MAX_ASINS_PER_REQUEST) { - const chunk = asins.slice(i, i + MAX_ASINS_PER_REQUEST); - await waitForToken(); - - const asinParam = chunk.join(","); - const url = `${KEEPA_BASE}/product?key=${config.keepaApiKey}&domain=1&asin=${asinParam}&stats=90`; - - console.log( - `Keepa: fetching ${chunk.length} ASINs (tokens left: ${tokensLeft})...`, - ); - - const res = await fetch(url); - lastRequestTime = Date.now(); - - if (!res.ok) { - const text = await res.text(); - throw new Error(`Keepa API error ${res.status}: ${text}`); - } - - const data = (await res.json()) as { - products?: Record[]; - tokensLeft?: number; - refillRate?: number; - }; - - // Update token state from API response - if (data.tokensLeft != null) tokensLeft = data.tokensLeft; - if (data.refillRate != null) refillRate = data.refillRate; - - console.log( - `Keepa: ${data.products?.length ?? 0} products returned, ${tokensLeft} tokens remaining (refill: ${refillRate}/min)`, - ); - - if (data.products) { - for (const product of data.products) { - const asin = product.asin; - if (!asin) continue; - results.set(asin, parseKeepaProduct(product)); - } - } - } - - return results; -} - -function parseKeepaProduct(product: Record): KeepaData { - const stats = product.stats; - const csv = product.csv; - const salesRankDrops30 = pickKeepaNumber( - product.salesRankDrops30, - stats?.salesRankDrops30, - ); - const salesRankDrops90 = - pickKeepaNumber(product.salesRankDrops90, stats?.salesRankDrops90) ?? - (salesRankDrops30 != null ? salesRankDrops30 * 3 : null); - const monthlySold = - pickKeepaNumber(product.monthlySold, stats?.monthlySold) ?? - salesRankDrops30; - - return { - currentPrice: extractCurrentPrice(csv), - avgPrice90: stats?.avg?.[0] != null ? stats.avg[0] / 100 : null, - minPrice90: stats?.min?.[0] != null ? stats.min[0] / 100 : null, - maxPrice90: stats?.max?.[0] != null ? stats.max[0] / 100 : null, - salesRank: stats?.current?.[3] ?? null, - salesRankAvg90: stats?.avg?.[3] ?? null, - salesRankDrops30, - salesRankDrops90, - sellerCount: stats?.current?.[11] ?? null, - buyBoxSeller: product.buyBoxSellerId ?? null, - buyBoxPrice: stats?.current?.[18] != null ? stats.current[18] / 100 : null, - monthlySold, - categoryTree: - product.categoryTree?.map((c: { name: string }) => c.name) ?? [], - }; -} - -function pickKeepaNumber(...values: unknown[]): number | null { - for (const value of values) { - if (typeof value !== "number" || !Number.isFinite(value)) continue; - // Keepa often uses -1 as "not available". - if (value < 0) continue; - return value; - } - return null; -} - -function extractCurrentPrice(csv: number[][] | undefined): number | null { - if (!csv) return null; - - // csv[0] = Amazon price history, csv[1] = Marketplace new price history - // Each is [time, price, time, price, ...] — last value is most recent - for (const series of [csv[0], csv[1]]) { - if (series && series.length >= 2) { - const lastPrice = series[series.length - 1]!; - if (lastPrice > 0) return lastPrice / 100; - } - } - return null; -} +import { config } from "./config.ts"; +import type { KeepaData } from "./types.ts"; + +const KEEPA_BASE = "https://api.keepa.com"; +const MAX_ASINS_PER_REQUEST = 100; + +// Token-based rate limiting: Keepa Pro = 1 token/min regeneration. +// Each product request costs 1 token regardless of ASIN count (up to 100). +// The API response includes tokensLeft and refillRate — we use those to pace. +let tokensLeft = 1; // Conservative start; updated from API response +let refillRate = 1; // tokens per minute, updated from API response +let lastRequestTime = 0; + +async function waitForToken(): Promise { + if (tokensLeft > 0) return; + + const elapsed = (Date.now() - lastRequestTime) / 60_000; // minutes + const regenerated = Math.floor(elapsed * refillRate); + if (regenerated > 0) { + tokensLeft += regenerated; + return; + } + + // Wait until we regenerate at least 1 token + const waitMs = + Math.ceil((1 / refillRate) * 60_000) - (Date.now() - lastRequestTime); + if (waitMs > 0) { + console.log( + `Keepa tokens exhausted. Waiting ${Math.ceil(waitMs / 1000)}s for token regeneration...`, + ); + await new Promise((r) => setTimeout(r, waitMs)); + } + tokensLeft = 1; +} + +export async function fetchKeepaDataBatch( + asins: string[], +): Promise> { + const results = new Map(); + + // Split into chunks of MAX_ASINS_PER_REQUEST + for (let i = 0; i < asins.length; i += MAX_ASINS_PER_REQUEST) { + const chunk = asins.slice(i, i + MAX_ASINS_PER_REQUEST); + await waitForToken(); + + const asinParam = chunk.join(","); + const url = `${KEEPA_BASE}/product?key=${config.keepaApiKey}&domain=1&asin=${asinParam}&stats=90`; + + console.log( + `Keepa: fetching ${chunk.length} ASINs (tokens left: ${tokensLeft})...`, + ); + + const res = await fetch(url); + lastRequestTime = Date.now(); + + if (!res.ok) { + const text = await res.text(); + throw new Error(`Keepa API error ${res.status}: ${text}`); + } + + const data = (await res.json()) as { + products?: Record[]; + tokensLeft?: number; + refillRate?: number; + }; + + // Update token state from API response + if (data.tokensLeft != null) tokensLeft = data.tokensLeft; + if (data.refillRate != null) refillRate = data.refillRate; + + console.log( + `Keepa: ${data.products?.length ?? 0} products returned, ${tokensLeft} tokens remaining (refill: ${refillRate}/min)`, + ); + + if (data.products) { + for (const product of data.products) { + const asin = product.asin; + if (!asin) continue; + results.set(asin, parseKeepaProduct(product)); + } + } + } + + return results; +} + +function parseKeepaProduct(product: Record): KeepaData { + const stats = product.stats; + const csv = product.csv; + const salesRankDrops30 = pickKeepaNumber( + product.salesRankDrops30, + stats?.salesRankDrops30, + ); + const salesRankDrops90 = + pickKeepaNumber(product.salesRankDrops90, stats?.salesRankDrops90) ?? + (salesRankDrops30 != null ? salesRankDrops30 * 3 : null); + const monthlySold = + pickKeepaNumber(product.monthlySold, stats?.monthlySold) ?? + salesRankDrops30; + + return { + currentPrice: extractCurrentPrice(csv), + avgPrice90: stats?.avg?.[0] != null ? stats.avg[0] / 100 : null, + minPrice90: stats?.min?.[0] != null ? stats.min[0] / 100 : null, + maxPrice90: stats?.max?.[0] != null ? stats.max[0] / 100 : null, + salesRank: stats?.current?.[3] ?? null, + salesRankAvg90: stats?.avg?.[3] ?? null, + salesRankDrops30, + salesRankDrops90, + sellerCount: stats?.current?.[11] ?? null, + buyBoxSeller: product.buyBoxSellerId ?? null, + buyBoxPrice: stats?.current?.[18] != null ? stats.current[18] / 100 : null, + monthlySold, + categoryTree: + product.categoryTree?.map((c: { name: string }) => c.name) ?? [], + }; +} + +function pickKeepaNumber(...values: unknown[]): number | null { + for (const value of values) { + if (typeof value !== "number" || !Number.isFinite(value)) continue; + // Keepa often uses -1 as "not available". + if (value < 0) continue; + return value; + } + return null; +} + +function extractCurrentPrice(csv: number[][] | undefined): number | null { + if (!csv) return null; + + // csv[0] = Amazon price history, csv[1] = Marketplace new price history + // Each is [time, price, time, price, ...] — last value is most recent + for (const series of [csv[0], csv[1]]) { + if (series && series.length >= 2) { + const lastPrice = series[series.length - 1]!; + if (lastPrice > 0) return lastPrice / 100; + } + } + return null; +} diff --git a/src/llm.ts b/src/llm.ts index a3dfd45..8fa69bf 100644 --- a/src/llm.ts +++ b/src/llm.ts @@ -1,353 +1,353 @@ -import { config } from "./config.ts"; -import type { EnrichedProduct, LlmVerdict } from "./types.ts"; - -const SYSTEM_PROMPT = `You are an expert Amazon product analyst specializing in FBA and FBM fulfillment strategy. - -Given product data, evaluate each product's viability for selling on Amazon. Consider: - -1. **Sales Velocity**: monthlySold and salesRankDrops30 are the most important signals. A product that doesn't sell is worthless regardless of margin. salesRankDrops30 = approximate units sold in 30 days. monthlySold is Keepa's estimate. -2. **Margin Analysis**: Sale price minus unit cost minus fees (FBA or FBM). Aim for >30% ROI minimum. The spreadsheet may include FBA NET and gross profit estimates — cross-check against Keepa pricing data. -3. **Sales Rank (BSR)**: Lower rank = higher demand. Rank <50,000 is good, <1,000 is excellent. -4. **Sales Rank Trend**: Compare current rank vs 90d average. Lower current = improving demand. -5. **Competition**: Number of sellers and Buy Box dynamics. Fewer sellers = easier entry. -6. **Price Stability**: Large price swings (high max vs low min over 90d) = volatile/risky. -7. **FBA vs FBM**: FBA preferred for fast-selling, small/light items. FBM for oversized, slow-moving, or high-margin items where fee savings matter. -8. **MOQ & Capital**: High MOQ with thin margins is risky. -9. **Supply Availability**: Total quantity available from supplier — low stock means limited runway. -10. **Seller Eligibility (critical)**: - - If sellerEligibility.status is "restricted" or "not_available", return verdict = "SKIP". - - If sellerEligibility.status is "unknown", treat as elevated risk and only allow FBA/FBM with clearly strong economics + demand. - - If canSell is false, return "SKIP" regardless of margin. - -Decision policy: -- Do not recommend products that cannot be listed by this seller account. -- Prioritize profitable + high-velocity + listable products. -- Use "SKIP" when data quality is poor or risk is high. - -Return ONLY a raw JSON array (no markdown, no code fences, no explanation before or after). One verdict per product: -[{ "asin": "B...", "verdict": "FBA" | "FBM" | "SKIP", "confidence": 0-100, "reasoning": "..." }] - -Keep each reasoning under 100 characters to stay within output limits and mention key blocker if skipped (e.g., restricted, low demand, thin margin).`; - -export async function analyzeProducts( - products: EnrichedProduct[], -): Promise { - try { - return await analyzeProductsInternal(products); - } catch (err) { - const msg = String(err); - if (products.length > 1 && msg.includes("Context size has been exceeded")) { - console.warn( - `LLM context exceeded for batch of ${products.length}, retrying one product at a time...`, - ); - - const fallback: LlmVerdict[] = []; - for (const product of products) { - try { - const single = await analyzeProductsInternal([product]); - fallback.push( - single[0] ?? { - asin: product.record.asin, - verdict: "SKIP", - confidence: 0, - reasoning: "LLM returned empty verdict", - }, - ); - } catch { - fallback.push({ - asin: product.record.asin, - verdict: "SKIP", - confidence: 0, - reasoning: "LLM context overflow on single-item fallback", - }); - } - } - return fallback; - } - throw err; - } -} - -async function analyzeProductsInternal( - products: EnrichedProduct[], -): Promise { - const productSummaries = products.map(summarizeForLlm); - - const res = await fetch(`${config.llmUrl}/chat/completions`, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: "Bearer lm-studio", - }, - body: JSON.stringify({ - model: config.llmModel, - messages: [ - { role: "system", content: SYSTEM_PROMPT }, - { role: "user", content: JSON.stringify(productSummaries, null, 2) }, - ], - temperature: 0.3, - max_tokens: 2048, - }), - }); - - if (!res.ok) { - throw new Error(`LLM API error ${res.status}: ${await res.text()}`); - } - - const data = (await res.json()) as { - choices?: { message?: { content?: string } }[]; - }; - const content = data.choices?.[0]?.message?.content ?? ""; - - return parseVerdicts(content, products); -} - -function summarizeForLlm(p: EnrichedProduct) { - const salePrice = - p.keepa?.currentPrice ?? - p.record.sellingPriceFromSheet ?? - p.spApi.estimatedSalePrice; - const referralFee = salePrice * (p.spApi.referralFeePercent / 100); - const fbaProfit = - salePrice - p.record.unitCost - p.spApi.fbaFee - referralFee; - const fbmProfit = - salePrice - p.record.unitCost - p.spApi.fbmFee - referralFee; - - return { - asin: p.record.asin, - name: clampText(p.record.name, 80), - brand: p.record.brand, - category: clampText( - p.record.category ?? p.keepa?.categoryTree?.join(" > "), - 60, - ), - unitCost: p.record.unitCost, - currentPrice: salePrice, - priceRange90d: p.keepa - ? { - min: p.keepa.minPrice90, - max: p.keepa.maxPrice90, - avg: p.keepa.avgPrice90, - } - : null, - salesRank: p.keepa?.salesRank ?? p.record.amazonRank, - salesRankAvg90d: p.keepa?.salesRankAvg90, - sellerCount: p.keepa?.sellerCount, - salesVelocity: { - monthlySold: p.keepa?.monthlySold, - salesRankDrops30: p.keepa?.salesRankDrops30, - salesRankDrops90: p.keepa?.salesRankDrops90, - }, - spreadsheetEstimates: { - avgPrice90: p.record.avgPrice90FromSheet, - sellingPrice: p.record.sellingPriceFromSheet, - fbaNet: p.record.fbaNet, - grossProfit: p.record.grossProfit, - grossProfitPct: p.record.grossProfitPct, - netProfit: p.record.netProfitFromSheet, - roi: p.record.roiFromSheet, - }, - supplier: clampText(p.record.supplier, 40), - moq: p.record.moq, - moqCost: p.record.moqCost, - totalQtyAvail: p.record.totalQtyAvail, - fees: { - fbaFee: p.spApi.fbaFee, - fbmFee: p.spApi.fbmFee, - referralFeePercent: p.spApi.referralFeePercent, - referralFee: Math.round(referralFee * 100) / 100, - }, - sellerEligibility: { - canSell: p.spApi.canSell, - status: p.spApi.sellabilityStatus, - reason: clampText(p.spApi.sellabilityReason, 120), - }, - estimatedProfit: { - fba: Math.round(fbaProfit * 100) / 100, - fbm: Math.round(fbmProfit * 100) / 100, - }, - estimatedROI: { - fba: - p.record.unitCost > 0 - ? Math.round((fbaProfit / p.record.unitCost) * 100) - : null, - fbm: - p.record.unitCost > 0 - ? Math.round((fbmProfit / p.record.unitCost) * 100) - : null, - }, - }; -} - -function clampText(value: unknown, maxLen: number): string | undefined { - if (value == null) return undefined; - const s = String(value).trim(); - if (!s) return undefined; - return s.length > maxLen ? `${s.slice(0, maxLen - 1)}.` : s; -} - -function cleanLlmJson(text: string): string { - // Remove ```json ... ``` or ``` ... ``` wrapping - const fenceMatch = text.match(/```(?:json)?\s*\n?([\s\S]*?)```/); - let cleaned = fenceMatch ? fenceMatch[1]!.trim() : text.trim(); - - // Strip any non-JSON wrapper text by taking the largest JSON-looking segment - const firstArray = cleaned.indexOf("["); - const firstObject = cleaned.indexOf("{"); - const startCandidates = [firstArray, firstObject].filter((i) => i >= 0); - const start = startCandidates.length > 0 ? Math.min(...startCandidates) : -1; - const endArray = cleaned.lastIndexOf("]"); - const endObject = cleaned.lastIndexOf("}"); - const end = Math.max(endArray, endObject); - if (start >= 0 && end > start) { - cleaned = cleaned.slice(start, end + 1); - } - - // Fix trailing comma-quote before closing brace: ,"} → "} - cleaned = cleaned.replace(/,"\s*}/g, '"}'); - - // Fix malformed comma-quote before a closing bracket/brace: ,"} or ,"] - cleaned = cleaned.replace(/,\s*"\s*([}\]])/g, "$1"); - - // Fix malformed quote-comma before a closing bracket/brace: ",} or ",] - cleaned = cleaned.replace(/"\s*,\s*([}\]])/g, '"$1'); - - // Fix trailing commas before ] or } - cleaned = cleaned.replace(/,\s*([}\]])/g, "$1"); - - return cleaned; -} - -function parseVerdicts( - content: string, - products: EnrichedProduct[], -): LlmVerdict[] { - const cleaned = cleanLlmJson(content); - - try { - const parsed = JSON.parse(cleaned) as unknown; - return alignVerdicts(products, normalizeVerdicts(parsed)); - } catch (err) { - const salvaged = extractVerdictsLoosely(cleaned); - if (salvaged.length > 0) { - console.warn( - `LLM response was invalid JSON; salvaged ${salvaged.length} verdict(s) with loose parsing.`, - ); - return alignVerdicts(products, salvaged); - } - - console.warn( - "Failed to parse LLM response, marking all as ANALYSIS_FAILED", - ); - console.warn("Raw LLM content:", content.slice(0, 500)); - return products.map((p) => ({ - asin: p.record.asin, - verdict: "SKIP" as const, - confidence: 0, - reasoning: `Analysis failed: could not parse LLM output`, - })); - } -} - -function normalizeVerdicts(parsed: unknown): LlmVerdict[] { - const container = - parsed && typeof parsed === "object" - ? (parsed as Record) - : undefined; - const nested = container?.verdicts ?? container?.results; - - const arr: unknown[] = Array.isArray(parsed) - ? parsed - : Array.isArray(nested) - ? nested - : [parsed]; - - return arr - .filter((v): v is Record => !!v && typeof v === "object") - .map((v) => ({ - asin: String(v.asin ?? "") - .trim() - .toUpperCase(), - verdict: (String(v.verdict).toUpperCase() === "FBA" || - String(v.verdict).toUpperCase() === "FBM" || - String(v.verdict).toUpperCase() === "SKIP" - ? String(v.verdict).toUpperCase() - : "SKIP") as LlmVerdict["verdict"], - confidence: clampConfidence( - typeof v.confidence === "number" - ? v.confidence - : Number(v.confidence ?? 0), - ), - reasoning: String(v.reasoning ?? "No reasoning provided"), - })); -} - -function extractVerdictsLoosely(text: string): LlmVerdict[] { - const objectMatches = text.match(/\{[\s\S]*?\}/g) ?? []; - const verdicts: LlmVerdict[] = []; - - for (const chunk of objectMatches) { - const asin = extractField(chunk, /"asin"\s*:\s*"?([A-Z0-9]{10})"?/i) ?? ""; - const verdictRaw = - extractField(chunk, /"verdict"\s*:\s*"?([A-Z]+)"?/i) ?? "SKIP"; - const confidenceRaw = - extractField(chunk, /"confidence"\s*:\s*([0-9]+(?:\.[0-9]+)?)/i) ?? "0"; - const reasoning = - extractField(chunk, /"reasoning"\s*:\s*"([\s\S]*?)"\s*(?:,|})/i) ?? - "No reasoning provided"; - - const normalizedVerdict = verdictRaw.toUpperCase(); - if (!asin) continue; - - verdicts.push({ - asin, - verdict: (normalizedVerdict === "FBA" || - normalizedVerdict === "FBM" || - normalizedVerdict === "SKIP" - ? normalizedVerdict - : "SKIP") as LlmVerdict["verdict"], - confidence: clampConfidence(Number(confidenceRaw)), - reasoning, - }); - } - - return verdicts; -} - -function extractField(text: string, regex: RegExp): string | undefined { - const match = text.match(regex); - return match?.[1]?.trim(); -} - -function clampConfidence(value: number): number { - if (!Number.isFinite(value)) return 0; - return Math.max(0, Math.min(100, Math.round(value))); -} - -function alignVerdicts( - products: EnrichedProduct[], - verdicts: LlmVerdict[], -): LlmVerdict[] { - const byAsin = new Map(); - for (const verdict of verdicts) { - if (verdict.asin && !byAsin.has(verdict.asin)) { - byAsin.set(verdict.asin, verdict); - } - } - - return products.map((product, index) => { - const asin = product.record.asin; - const byAsinVerdict = byAsin.get(asin); - if (byAsinVerdict) return { ...byAsinVerdict, asin }; - - const byIndexVerdict = verdicts[index]; - if (byIndexVerdict) return { ...byIndexVerdict, asin }; - - return { - asin, - verdict: "SKIP" as const, - confidence: 0, - reasoning: "LLM returned no verdict for this product", - }; - }); -} +import { config } from "./config.ts"; +import type { EnrichedProduct, LlmVerdict } from "./types.ts"; + +const SYSTEM_PROMPT = `You are an expert Amazon product analyst specializing in FBA and FBM fulfillment strategy. + +Given product data, evaluate each product's viability for selling on Amazon. Consider: + +1. **Sales Velocity**: monthlySold and salesRankDrops30 are the most important signals. A product that doesn't sell is worthless regardless of margin. salesRankDrops30 = approximate units sold in 30 days. monthlySold is Keepa's estimate. +2. **Margin Analysis**: Sale price minus unit cost minus fees (FBA or FBM). Aim for >30% ROI minimum. The spreadsheet may include FBA NET and gross profit estimates — cross-check against Keepa pricing data. +3. **Sales Rank (BSR)**: Lower rank = higher demand. Rank <50,000 is good, <1,000 is excellent. +4. **Sales Rank Trend**: Compare current rank vs 90d average. Lower current = improving demand. +5. **Competition**: Number of sellers and Buy Box dynamics. Fewer sellers = easier entry. +6. **Price Stability**: Large price swings (high max vs low min over 90d) = volatile/risky. +7. **FBA vs FBM**: FBA preferred for fast-selling, small/light items. FBM for oversized, slow-moving, or high-margin items where fee savings matter. +8. **MOQ & Capital**: High MOQ with thin margins is risky. +9. **Supply Availability**: Total quantity available from supplier — low stock means limited runway. +10. **Seller Eligibility (critical)**: + - If sellerEligibility.status is "restricted" or "not_available", return verdict = "SKIP". + - If sellerEligibility.status is "unknown", treat as elevated risk and only allow FBA/FBM with clearly strong economics + demand. + - If canSell is false, return "SKIP" regardless of margin. + +Decision policy: +- Do not recommend products that cannot be listed by this seller account. +- Prioritize profitable + high-velocity + listable products. +- Use "SKIP" when data quality is poor or risk is high. + +Return ONLY a raw JSON array (no markdown, no code fences, no explanation before or after). One verdict per product: +[{ "asin": "B...", "verdict": "FBA" | "FBM" | "SKIP", "confidence": 0-100, "reasoning": "..." }] + +Keep each reasoning under 100 characters to stay within output limits and mention key blocker if skipped (e.g., restricted, low demand, thin margin).`; + +export async function analyzeProducts( + products: EnrichedProduct[], +): Promise { + try { + return await analyzeProductsInternal(products); + } catch (err) { + const msg = String(err); + if (products.length > 1 && msg.includes("Context size has been exceeded")) { + console.warn( + `LLM context exceeded for batch of ${products.length}, retrying one product at a time...`, + ); + + const fallback: LlmVerdict[] = []; + for (const product of products) { + try { + const single = await analyzeProductsInternal([product]); + fallback.push( + single[0] ?? { + asin: product.record.asin, + verdict: "SKIP", + confidence: 0, + reasoning: "LLM returned empty verdict", + }, + ); + } catch { + fallback.push({ + asin: product.record.asin, + verdict: "SKIP", + confidence: 0, + reasoning: "LLM context overflow on single-item fallback", + }); + } + } + return fallback; + } + throw err; + } +} + +async function analyzeProductsInternal( + products: EnrichedProduct[], +): Promise { + const productSummaries = products.map(summarizeForLlm); + + const res = await fetch(`${config.llmUrl}/chat/completions`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: "Bearer lm-studio", + }, + body: JSON.stringify({ + model: config.llmModel, + messages: [ + { role: "system", content: SYSTEM_PROMPT }, + { role: "user", content: JSON.stringify(productSummaries, null, 2) }, + ], + temperature: 0.3, + max_tokens: 2048, + }), + }); + + if (!res.ok) { + throw new Error(`LLM API error ${res.status}: ${await res.text()}`); + } + + const data = (await res.json()) as { + choices?: { message?: { content?: string } }[]; + }; + const content = data.choices?.[0]?.message?.content ?? ""; + + return parseVerdicts(content, products); +} + +function summarizeForLlm(p: EnrichedProduct) { + const salePrice = + p.keepa?.currentPrice ?? + p.record.sellingPriceFromSheet ?? + p.spApi.estimatedSalePrice; + const referralFee = salePrice * (p.spApi.referralFeePercent / 100); + const fbaProfit = + salePrice - p.record.unitCost - p.spApi.fbaFee - referralFee; + const fbmProfit = + salePrice - p.record.unitCost - p.spApi.fbmFee - referralFee; + + return { + asin: p.record.asin, + name: clampText(p.record.name, 80), + brand: p.record.brand, + category: clampText( + p.record.category ?? p.keepa?.categoryTree?.join(" > "), + 60, + ), + unitCost: p.record.unitCost, + currentPrice: salePrice, + priceRange90d: p.keepa + ? { + min: p.keepa.minPrice90, + max: p.keepa.maxPrice90, + avg: p.keepa.avgPrice90, + } + : null, + salesRank: p.keepa?.salesRank ?? p.record.amazonRank, + salesRankAvg90d: p.keepa?.salesRankAvg90, + sellerCount: p.keepa?.sellerCount, + salesVelocity: { + monthlySold: p.keepa?.monthlySold, + salesRankDrops30: p.keepa?.salesRankDrops30, + salesRankDrops90: p.keepa?.salesRankDrops90, + }, + spreadsheetEstimates: { + avgPrice90: p.record.avgPrice90FromSheet, + sellingPrice: p.record.sellingPriceFromSheet, + fbaNet: p.record.fbaNet, + grossProfit: p.record.grossProfit, + grossProfitPct: p.record.grossProfitPct, + netProfit: p.record.netProfitFromSheet, + roi: p.record.roiFromSheet, + }, + supplier: clampText(p.record.supplier, 40), + moq: p.record.moq, + moqCost: p.record.moqCost, + totalQtyAvail: p.record.totalQtyAvail, + fees: { + fbaFee: p.spApi.fbaFee, + fbmFee: p.spApi.fbmFee, + referralFeePercent: p.spApi.referralFeePercent, + referralFee: Math.round(referralFee * 100) / 100, + }, + sellerEligibility: { + canSell: p.spApi.canSell, + status: p.spApi.sellabilityStatus, + reason: clampText(p.spApi.sellabilityReason, 120), + }, + estimatedProfit: { + fba: Math.round(fbaProfit * 100) / 100, + fbm: Math.round(fbmProfit * 100) / 100, + }, + estimatedROI: { + fba: + p.record.unitCost > 0 + ? Math.round((fbaProfit / p.record.unitCost) * 100) + : null, + fbm: + p.record.unitCost > 0 + ? Math.round((fbmProfit / p.record.unitCost) * 100) + : null, + }, + }; +} + +function clampText(value: unknown, maxLen: number): string | undefined { + if (value == null) return undefined; + const s = String(value).trim(); + if (!s) return undefined; + return s.length > maxLen ? `${s.slice(0, maxLen - 1)}.` : s; +} + +function cleanLlmJson(text: string): string { + // Remove ```json ... ``` or ``` ... ``` wrapping + const fenceMatch = text.match(/```(?:json)?\s*\n?([\s\S]*?)```/); + let cleaned = fenceMatch ? fenceMatch[1]!.trim() : text.trim(); + + // Strip any non-JSON wrapper text by taking the largest JSON-looking segment + const firstArray = cleaned.indexOf("["); + const firstObject = cleaned.indexOf("{"); + const startCandidates = [firstArray, firstObject].filter((i) => i >= 0); + const start = startCandidates.length > 0 ? Math.min(...startCandidates) : -1; + const endArray = cleaned.lastIndexOf("]"); + const endObject = cleaned.lastIndexOf("}"); + const end = Math.max(endArray, endObject); + if (start >= 0 && end > start) { + cleaned = cleaned.slice(start, end + 1); + } + + // Fix trailing comma-quote before closing brace: ,"} → "} + cleaned = cleaned.replace(/,"\s*}/g, '"}'); + + // Fix malformed comma-quote before a closing bracket/brace: ,"} or ,"] + cleaned = cleaned.replace(/,\s*"\s*([}\]])/g, "$1"); + + // Fix malformed quote-comma before a closing bracket/brace: ",} or ",] + cleaned = cleaned.replace(/"\s*,\s*([}\]])/g, '"$1'); + + // Fix trailing commas before ] or } + cleaned = cleaned.replace(/,\s*([}\]])/g, "$1"); + + return cleaned; +} + +function parseVerdicts( + content: string, + products: EnrichedProduct[], +): LlmVerdict[] { + const cleaned = cleanLlmJson(content); + + try { + const parsed = JSON.parse(cleaned) as unknown; + return alignVerdicts(products, normalizeVerdicts(parsed)); + } catch (err) { + const salvaged = extractVerdictsLoosely(cleaned); + if (salvaged.length > 0) { + console.warn( + `LLM response was invalid JSON; salvaged ${salvaged.length} verdict(s) with loose parsing.`, + ); + return alignVerdicts(products, salvaged); + } + + console.warn( + "Failed to parse LLM response, marking all as ANALYSIS_FAILED", + ); + console.warn("Raw LLM content:", content.slice(0, 500)); + return products.map((p) => ({ + asin: p.record.asin, + verdict: "SKIP" as const, + confidence: 0, + reasoning: `Analysis failed: could not parse LLM output`, + })); + } +} + +function normalizeVerdicts(parsed: unknown): LlmVerdict[] { + const container = + parsed && typeof parsed === "object" + ? (parsed as Record) + : undefined; + const nested = container?.verdicts ?? container?.results; + + const arr: unknown[] = Array.isArray(parsed) + ? parsed + : Array.isArray(nested) + ? nested + : [parsed]; + + return arr + .filter((v): v is Record => !!v && typeof v === "object") + .map((v) => ({ + asin: String(v.asin ?? "") + .trim() + .toUpperCase(), + verdict: (String(v.verdict).toUpperCase() === "FBA" || + String(v.verdict).toUpperCase() === "FBM" || + String(v.verdict).toUpperCase() === "SKIP" + ? String(v.verdict).toUpperCase() + : "SKIP") as LlmVerdict["verdict"], + confidence: clampConfidence( + typeof v.confidence === "number" + ? v.confidence + : Number(v.confidence ?? 0), + ), + reasoning: String(v.reasoning ?? "No reasoning provided"), + })); +} + +function extractVerdictsLoosely(text: string): LlmVerdict[] { + const objectMatches = text.match(/\{[\s\S]*?\}/g) ?? []; + const verdicts: LlmVerdict[] = []; + + for (const chunk of objectMatches) { + const asin = extractField(chunk, /"asin"\s*:\s*"?([A-Z0-9]{10})"?/i) ?? ""; + const verdictRaw = + extractField(chunk, /"verdict"\s*:\s*"?([A-Z]+)"?/i) ?? "SKIP"; + const confidenceRaw = + extractField(chunk, /"confidence"\s*:\s*([0-9]+(?:\.[0-9]+)?)/i) ?? "0"; + const reasoning = + extractField(chunk, /"reasoning"\s*:\s*"([\s\S]*?)"\s*(?:,|})/i) ?? + "No reasoning provided"; + + const normalizedVerdict = verdictRaw.toUpperCase(); + if (!asin) continue; + + verdicts.push({ + asin, + verdict: (normalizedVerdict === "FBA" || + normalizedVerdict === "FBM" || + normalizedVerdict === "SKIP" + ? normalizedVerdict + : "SKIP") as LlmVerdict["verdict"], + confidence: clampConfidence(Number(confidenceRaw)), + reasoning, + }); + } + + return verdicts; +} + +function extractField(text: string, regex: RegExp): string | undefined { + const match = text.match(regex); + return match?.[1]?.trim(); +} + +function clampConfidence(value: number): number { + if (!Number.isFinite(value)) return 0; + return Math.max(0, Math.min(100, Math.round(value))); +} + +function alignVerdicts( + products: EnrichedProduct[], + verdicts: LlmVerdict[], +): LlmVerdict[] { + const byAsin = new Map(); + for (const verdict of verdicts) { + if (verdict.asin && !byAsin.has(verdict.asin)) { + byAsin.set(verdict.asin, verdict); + } + } + + return products.map((product, index) => { + const asin = product.record.asin; + const byAsinVerdict = byAsin.get(asin); + if (byAsinVerdict) return { ...byAsinVerdict, asin }; + + const byIndexVerdict = verdicts[index]; + if (byIndexVerdict) return { ...byIndexVerdict, asin }; + + return { + asin, + verdict: "SKIP" as const, + confidence: 0, + reasoning: "LLM returned no verdict for this product", + }; + }); +} diff --git a/src/reader.ts b/src/reader.ts index e48939c..0c72956 100644 --- a/src/reader.ts +++ b/src/reader.ts @@ -1,209 +1,209 @@ -import * as XLSX from "xlsx"; -import type { ProductRecord } from "./types.ts"; - -const ASIN_REGEX = /^B[0-9A-Z]{9}$/; - -const COLUMN_CANDIDATES = { - asin: ["asin"], - name: ["name", "product name", "title", "product title"], - cost: ["unit cost", "cost", "unitcost", "unit_cost", "price", "buy cost"], - brand: ["brand"], - category: ["category"], - amazonRank: ["amazon rank", "amazonrank", "sales rank", "bsr"], - avgPrice90: [ - "90 day average", - "90-day average", - "avg price 90d", - "avg 90 day", - "90d average", - ], - sellingPrice: ["selling price", "sale price", "sell price"], - fbaNet: ["fba net", "fbanet", "fba_net"], - grossProfit: ["gross profit $", "gross profit", "grossprofit"], - grossProfitPct: ["gross profit %", "gross profit pct", "grossprofitpct"], - netProfit: ["net profit", "netprofit"], - roi: ["roi", "return on investment"], - moq: ["moq", "min order qty", "minimum order quantity"], - moqCost: ["moq cost", "moqcost", "moq_cost"], - totalQty: ["total qty avail", "totalqtyavail", "qty available", "quantity"], - link: ["link", "url", "source"], - asinLink: ["asin link", "amazon link", "asin url"], - sourceUrl: ["source url", "supplier url", "source link"], - supplier: ["supplier", "vendor"], - promoCouponCode: [ - "promo/coupon code", - "promo coupon code", - "coupon code", - "promo code", - ], - notes: ["notes", "note"], - leadDate: ["date", "lead date"], -} as const; - -type ColumnKey = keyof typeof COLUMN_CANDIDATES; -type ColumnMap = Record; - -export function readProducts(filePath: string): ProductRecord[] { - const workbook = XLSX.readFile(filePath); - const sheetName = workbook.SheetNames[0]; - if (!sheetName) throw new Error("No sheets found in file"); - - const sheet = workbook.Sheets[sheetName]!; - const rows = XLSX.utils.sheet_to_json>(sheet); - - if (rows.length === 0) throw new Error("File contains no data rows"); - - const headers = Object.keys(rows[0]!); - const columns = detectColumns(headers); - const asinColumn = columns.asin; - - if (!asinColumn) - throw new Error( - `No ASIN column found. Available columns: ${headers.join(", ")}`, - ); - - logColumnDetection(headers, columns); - - const knownCols = getKnownColumns(columns); - - const products: ProductRecord[] = []; - - for (const row of rows) { - const asin = parseAsin(row[asinColumn]); - if (!asin) continue; - - const sourceUrl = getOptionalString(row, columns.sourceUrl); - const asinLink = getOptionalString(row, columns.asinLink); - const link = sourceUrl ?? asinLink ?? getOptionalString(row, columns.link); - - const extra = getExtraFields(row, headers, knownCols); - const netProfitFromSheet = getOptionalNumber(row, columns.netProfit); - const roiFromSheet = getOptionalNumber(row, columns.roi); - - products.push({ - asin, - name: getOptionalString(row, columns.name) ?? "", - unitCost: getOptionalNumber(row, columns.cost) ?? 0, - brand: getOptionalString(row, columns.brand), - category: getOptionalString(row, columns.category), - amazonRank: getOptionalNumber(row, columns.amazonRank), - avgPrice90FromSheet: getOptionalNumber(row, columns.avgPrice90), - sellingPriceFromSheet: getOptionalNumber(row, columns.sellingPrice), - fbaNet: getOptionalNumber(row, columns.fbaNet), - grossProfit: getOptionalNumber(row, columns.grossProfit) ?? netProfitFromSheet, - grossProfitPct: - getOptionalNumber(row, columns.grossProfitPct) ?? roiFromSheet, - netProfitFromSheet, - roiFromSheet, - moq: getOptionalNumber(row, columns.moq), - moqCost: getOptionalNumber(row, columns.moqCost), - totalQtyAvail: getOptionalNumber(row, columns.totalQty), - link, - asinLink, - sourceUrl, - supplier: getOptionalString(row, columns.supplier), - promoCouponCode: getOptionalString(row, columns.promoCouponCode), - notes: getOptionalString(row, columns.notes), - leadDate: getOptionalString(row, columns.leadDate), - ...extra, - }); - } - - console.log(`Read ${products.length} valid products from ${filePath}`); - return products; -} - -function detectColumns(headers: string[]): ColumnMap { - const columns = {} as ColumnMap; - for (const key of Object.keys(COLUMN_CANDIDATES) as ColumnKey[]) { - columns[key] = findColumn(headers, [...COLUMN_CANDIDATES[key]]); - } - return columns; -} - -function logColumnDetection(headers: string[], columns: ColumnMap): void { - console.log(`Found columns: ${headers.join(", ")}`); - console.log( - `Detected columns -> ASIN: ${columns.asin ?? "n/a"}, Name: ${columns.name ?? "n/a"}, Cost: ${columns.cost ?? "n/a"}, 90d Avg: ${columns.avgPrice90 ?? "n/a"}, Selling Price: ${columns.sellingPrice ?? "n/a"}, Net Profit: ${columns.netProfit ?? columns.grossProfit ?? "n/a"}, ROI: ${columns.roi ?? columns.grossProfitPct ?? "n/a"}, Source URL: ${columns.sourceUrl ?? "n/a"}, ASIN Link: ${columns.asinLink ?? "n/a"}`, - ); -} - -function getKnownColumns(columns: ColumnMap): Set { - return new Set(Object.values(columns).filter((column): column is string => !!column)); -} - -function parseAsin(value: unknown): string | undefined { - const asin = String(value ?? "") - .trim() - .toUpperCase(); - if (!asin || !ASIN_REGEX.test(asin)) { - console.warn(`Skipping invalid ASIN: "${asin}"`); - return undefined; - } - return asin; -} - -function getOptionalString( - row: Record, - column: string | undefined, -): string | undefined { - if (!column) return undefined; - return normalizeOptionalString(row[column]); -} - -function getOptionalNumber( - row: Record, - column: string | undefined, -): number | undefined { - if (!column) return undefined; - return parseOptionalNumber(row[column]); -} - -function getExtraFields( - row: Record, - headers: string[], - knownCols: Set, -): Record { - const extra: Record = {}; - for (const header of headers) { - if (!knownCols.has(header)) extra[header] = row[header]; - } - return extra; -} - -function findColumn( - headers: string[], - candidates: string[], -): string | undefined { - const normalizedCandidates = new Set(candidates.map(normalizeHeader)); - - for (const header of headers) { - if (normalizedCandidates.has(normalizeHeader(header))) { - return header; - } - } - - return undefined; -} - -function normalizeHeader(value: string): string { - return value - .toLowerCase() - .trim() - .replace(/%/g, " pct ") - .replace(/\$/g, " usd ") - .replace(/[^a-z0-9]/g, ""); -} - -function normalizeOptionalString(value: unknown): string | undefined { - if (value == null) return undefined; - const s = String(value).trim(); - return s.length > 0 ? s : undefined; -} - -function parseOptionalNumber(value: unknown): number | undefined { - if (value == null || value === "") return undefined; - const cleaned = String(value).trim().replace(/[$,%]/g, "").replace(/,/g, ""); - const parsed = Number(cleaned); - return Number.isFinite(parsed) ? parsed : undefined; -} +import * as XLSX from "xlsx"; +import type { ProductRecord } from "./types.ts"; + +const ASIN_REGEX = /^B[0-9A-Z]{9}$/; + +const COLUMN_CANDIDATES = { + asin: ["asin"], + name: ["name", "product name", "title", "product title"], + cost: ["unit cost", "cost", "unitcost", "unit_cost", "price", "buy cost"], + brand: ["brand"], + category: ["category"], + amazonRank: ["amazon rank", "amazonrank", "sales rank", "bsr"], + avgPrice90: [ + "90 day average", + "90-day average", + "avg price 90d", + "avg 90 day", + "90d average", + ], + sellingPrice: ["selling price", "sale price", "sell price"], + fbaNet: ["fba net", "fbanet", "fba_net"], + grossProfit: ["gross profit $", "gross profit", "grossprofit"], + grossProfitPct: ["gross profit %", "gross profit pct", "grossprofitpct"], + netProfit: ["net profit", "netprofit"], + roi: ["roi", "return on investment"], + moq: ["moq", "min order qty", "minimum order quantity"], + moqCost: ["moq cost", "moqcost", "moq_cost"], + totalQty: ["total qty avail", "totalqtyavail", "qty available", "quantity"], + link: ["link", "url", "source"], + asinLink: ["asin link", "amazon link", "asin url"], + sourceUrl: ["source url", "supplier url", "source link"], + supplier: ["supplier", "vendor"], + promoCouponCode: [ + "promo/coupon code", + "promo coupon code", + "coupon code", + "promo code", + ], + notes: ["notes", "note"], + leadDate: ["date", "lead date"], +} as const; + +type ColumnKey = keyof typeof COLUMN_CANDIDATES; +type ColumnMap = Record; + +export function readProducts(filePath: string): ProductRecord[] { + const workbook = XLSX.readFile(filePath); + const sheetName = workbook.SheetNames[0]; + if (!sheetName) throw new Error("No sheets found in file"); + + const sheet = workbook.Sheets[sheetName]!; + const rows = XLSX.utils.sheet_to_json>(sheet); + + if (rows.length === 0) throw new Error("File contains no data rows"); + + const headers = Object.keys(rows[0]!); + const columns = detectColumns(headers); + const asinColumn = columns.asin; + + if (!asinColumn) + throw new Error( + `No ASIN column found. Available columns: ${headers.join(", ")}`, + ); + + logColumnDetection(headers, columns); + + const knownCols = getKnownColumns(columns); + + const products: ProductRecord[] = []; + + for (const row of rows) { + const asin = parseAsin(row[asinColumn]); + if (!asin) continue; + + const sourceUrl = getOptionalString(row, columns.sourceUrl); + const asinLink = getOptionalString(row, columns.asinLink); + const link = sourceUrl ?? asinLink ?? getOptionalString(row, columns.link); + + const extra = getExtraFields(row, headers, knownCols); + const netProfitFromSheet = getOptionalNumber(row, columns.netProfit); + const roiFromSheet = getOptionalNumber(row, columns.roi); + + products.push({ + asin, + name: getOptionalString(row, columns.name) ?? "", + unitCost: getOptionalNumber(row, columns.cost) ?? 0, + brand: getOptionalString(row, columns.brand), + category: getOptionalString(row, columns.category), + amazonRank: getOptionalNumber(row, columns.amazonRank), + avgPrice90FromSheet: getOptionalNumber(row, columns.avgPrice90), + sellingPriceFromSheet: getOptionalNumber(row, columns.sellingPrice), + fbaNet: getOptionalNumber(row, columns.fbaNet), + grossProfit: getOptionalNumber(row, columns.grossProfit) ?? netProfitFromSheet, + grossProfitPct: + getOptionalNumber(row, columns.grossProfitPct) ?? roiFromSheet, + netProfitFromSheet, + roiFromSheet, + moq: getOptionalNumber(row, columns.moq), + moqCost: getOptionalNumber(row, columns.moqCost), + totalQtyAvail: getOptionalNumber(row, columns.totalQty), + link, + asinLink, + sourceUrl, + supplier: getOptionalString(row, columns.supplier), + promoCouponCode: getOptionalString(row, columns.promoCouponCode), + notes: getOptionalString(row, columns.notes), + leadDate: getOptionalString(row, columns.leadDate), + ...extra, + }); + } + + console.log(`Read ${products.length} valid products from ${filePath}`); + return products; +} + +function detectColumns(headers: string[]): ColumnMap { + const columns = {} as ColumnMap; + for (const key of Object.keys(COLUMN_CANDIDATES) as ColumnKey[]) { + columns[key] = findColumn(headers, [...COLUMN_CANDIDATES[key]]); + } + return columns; +} + +function logColumnDetection(headers: string[], columns: ColumnMap): void { + console.log(`Found columns: ${headers.join(", ")}`); + console.log( + `Detected columns -> ASIN: ${columns.asin ?? "n/a"}, Name: ${columns.name ?? "n/a"}, Cost: ${columns.cost ?? "n/a"}, 90d Avg: ${columns.avgPrice90 ?? "n/a"}, Selling Price: ${columns.sellingPrice ?? "n/a"}, Net Profit: ${columns.netProfit ?? columns.grossProfit ?? "n/a"}, ROI: ${columns.roi ?? columns.grossProfitPct ?? "n/a"}, Source URL: ${columns.sourceUrl ?? "n/a"}, ASIN Link: ${columns.asinLink ?? "n/a"}`, + ); +} + +function getKnownColumns(columns: ColumnMap): Set { + return new Set(Object.values(columns).filter((column): column is string => !!column)); +} + +function parseAsin(value: unknown): string | undefined { + const asin = String(value ?? "") + .trim() + .toUpperCase(); + if (!asin || !ASIN_REGEX.test(asin)) { + console.warn(`Skipping invalid ASIN: "${asin}"`); + return undefined; + } + return asin; +} + +function getOptionalString( + row: Record, + column: string | undefined, +): string | undefined { + if (!column) return undefined; + return normalizeOptionalString(row[column]); +} + +function getOptionalNumber( + row: Record, + column: string | undefined, +): number | undefined { + if (!column) return undefined; + return parseOptionalNumber(row[column]); +} + +function getExtraFields( + row: Record, + headers: string[], + knownCols: Set, +): Record { + const extra: Record = {}; + for (const header of headers) { + if (!knownCols.has(header)) extra[header] = row[header]; + } + return extra; +} + +function findColumn( + headers: string[], + candidates: string[], +): string | undefined { + const normalizedCandidates = new Set(candidates.map(normalizeHeader)); + + for (const header of headers) { + if (normalizedCandidates.has(normalizeHeader(header))) { + return header; + } + } + + return undefined; +} + +function normalizeHeader(value: string): string { + return value + .toLowerCase() + .trim() + .replace(/%/g, " pct ") + .replace(/\$/g, " usd ") + .replace(/[^a-z0-9]/g, ""); +} + +function normalizeOptionalString(value: unknown): string | undefined { + if (value == null) return undefined; + const s = String(value).trim(); + return s.length > 0 ? s : undefined; +} + +function parseOptionalNumber(value: unknown): number | undefined { + if (value == null || value === "") return undefined; + const cleaned = String(value).trim().replace(/[$,%]/g, "").replace(/,/g, ""); + const parsed = Number(cleaned); + return Number.isFinite(parsed) ? parsed : undefined; +} diff --git a/src/sp-api.ts b/src/sp-api.ts index 7c9d9ed..cd3d4a6 100644 --- a/src/sp-api.ts +++ b/src/sp-api.ts @@ -1,650 +1,650 @@ -import { SellingPartner } from "amazon-sp-api"; -import { config } from "./config.ts"; -import type { SpApiData, SellabilityInfo } from "./types.ts"; - -type RegionCode = "na" | "eu" | "fe"; - -let client: SellingPartner | null = null; -let loggedMissingCreds = false; -let loggedSandboxMode = false; - -function normalizeRegion(region: string): RegionCode { - const value = region.trim().toLowerCase(); - if (value === "us") return "na"; - if (value === "na" || value === "eu" || value === "fe") return value; - console.warn(`Unknown SP_API_REGION \"${region}\", defaulting to \"na\".`); - return "na"; -} - -function hasSpApiCredentials(): boolean { - return !!( - config.spApiClientId && - config.spApiClientSecret && - config.spApiRefreshToken - ); -} - -function getSpClient(): SellingPartner | null { - if (!hasSpApiCredentials()) { - if (!loggedMissingCreds) { - console.warn( - "SP-API credentials not configured; falling back to fee defaults. Set SP_API_CLIENT_ID, SP_API_CLIENT_SECRET, and SP_API_REFRESH_TOKEN.", - ); - loggedMissingCreds = true; - } - return null; - } - - if (client) return client; - - if (config.spApiUseSandbox && !loggedSandboxMode) { - console.warn( - "SP-API sandbox mode is enabled (SP_API_USE_SANDBOX=true). Production ASIN calls may be denied.", - ); - loggedSandboxMode = true; - } - - client = new SellingPartner({ - region: normalizeRegion(config.spApiRegion), - refresh_token: config.spApiRefreshToken!, - credentials: { - SELLING_PARTNER_APP_CLIENT_ID: config.spApiClientId!, - SELLING_PARTNER_APP_CLIENT_SECRET: config.spApiClientSecret!, - ...(config.awsAccessKeyId && config.awsSecretAccessKey - ? { - AWS_ACCESS_KEY_ID: config.awsAccessKeyId, - AWS_SECRET_ACCESS_KEY: config.awsSecretAccessKey, - } - : {}), - ...(config.awsSessionToken - ? { AWS_SESSION_TOKEN: config.awsSessionToken } - : {}), - }, - options: { - auto_request_tokens: true, - auto_request_throttled: true, - use_sandbox: config.spApiUseSandbox, - debug_log: false, - }, - }); - - return client; -} - -function getAmount(value: unknown): number | undefined { - if (!value || typeof value !== "object") return undefined; - const amount = (value as { Amount?: unknown }).Amount; - return typeof amount === "number" && Number.isFinite(amount) - ? amount - : undefined; -} - -function extractEstimatedSalePrice(pricing: any): number { - const buyBox = pricing?.Summary?.BuyBoxPrices?.[0]; - const lowest = pricing?.Summary?.LowestPrices?.[0]; - const buyBoxLanded = getAmount(buyBox?.LandedPrice); - if (buyBoxLanded != null) return buyBoxLanded; - - const lowestLanded = getAmount(lowest?.LandedPrice); - if (lowestLanded != null) return lowestLanded; - - const firstOffer = pricing?.Offers?.[0]; - const listing = getAmount(firstOffer?.ListingPrice) ?? 0; - const shipping = getAmount(firstOffer?.Shipping) ?? 0; - return listing + shipping; -} - -function extractFeeResult(feesResponse: any): { - totalFee: number; - referralFee?: number; -} { - const result = feesResponse?.payload?.FeesEstimateResult; - const total = getAmount(result?.FeesEstimate?.TotalFeesEstimate) ?? 0; - const feeDetails = result?.FeesEstimate?.FeeDetailList; - - const referralDetail = Array.isArray(feeDetails) - ? feeDetails.find((f: any) => - String(f?.FeeType ?? "") - .toLowerCase() - .includes("referral"), - ) - : undefined; - - const referralFee = getAmount(referralDetail?.FinalFee); - return { totalFee: total, referralFee }; -} - -function round2(value: number): number { - return Math.round(value * 100) / 100; -} - -const SELLABILITY_CONCURRENCY = 5; -const PRICING_CONCURRENCY = 5; - -function parseSellabilityResponse(response: any): SellabilityInfo { - const restrictions = Array.isArray(response?.restrictions) - ? response.restrictions - : Array.isArray(response?.payload?.restrictions) - ? response.payload.restrictions - : null; - - if (!Array.isArray(restrictions)) { - return { - canSell: null, - sellabilityStatus: "unknown", - sellabilityReason: "Unexpected restrictions response shape", - }; - } - - if (restrictions.length === 0) { - return { - canSell: true, - sellabilityStatus: "available", - sellabilityReason: "No listing restrictions reported", - }; - } - - const reasons = restrictions.flatMap((r: any) => - Array.isArray(r?.reasons) ? r.reasons : [], - ); - const reasonCodes = reasons - .map((r: any) => String(r?.reasonCode ?? "").trim()) - .filter((r: string) => r.length > 0); - const reasonMessages = reasons - .map((r: any) => String(r?.message ?? "").trim()) - .filter((m: string) => m.length > 0); - - const allReasonText = [...reasonCodes, ...reasonMessages] - .join(" | ") - .toLowerCase(); - const status = - allReasonText.includes("not_eligible") || - allReasonText.includes("not eligible") || - allReasonText.includes("not available") - ? "not_available" - : "restricted"; - - return { - canSell: false, - sellabilityStatus: status, - sellabilityReason: - [...reasonCodes, ...reasonMessages].join(" | ") || - "Listing restrictions reported", - }; -} - -async function fetchSellabilityInternal( - spClient: SellingPartner, - asin: string, -): Promise { - if (!config.spApiSellerId) { - return { - canSell: null, - sellabilityStatus: "unknown", - sellabilityReason: "Missing SP_API_SELLER_ID", - }; - } - - try { - const restrictionResponse = await spClient.callAPI({ - operation: "getListingsRestrictions", - endpoint: "listingsRestrictions", - query: { - asin, - sellerId: config.spApiSellerId, - marketplaceIds: [config.spApiMarketplaceId], - conditionType: "new_new", - }, - }); - return parseSellabilityResponse(restrictionResponse); - } catch (err) { - return { - canSell: null, - sellabilityStatus: "unknown", - sellabilityReason: `Restrictions check failed: ${extractErrorMessage(err)}`, - }; - } -} - -function missingSpApiEnvVars(): string[] { - const missing: string[] = []; - if (!config.spApiClientId) missing.push("SP_API_CLIENT_ID"); - if (!config.spApiClientSecret) missing.push("SP_API_CLIENT_SECRET"); - if (!config.spApiRefreshToken) missing.push("SP_API_REFRESH_TOKEN"); - return missing; -} - -function extractErrorMessage(err: unknown): string { - if (err instanceof Error) return err.message; - return String(err); -} - -function isAccessDeniedMessage(message: string): boolean { - const value = message.toLowerCase(); - return ( - value.includes("access to requested resource is denied") || - value.includes("access denied") || - value.includes("forbidden") || - value.includes("unauthorized") - ); -} - -function deniedHint(operation: string): string { - const sandboxHint = config.spApiUseSandbox - ? " Sandbox mode is enabled; production data calls are often denied in sandbox." - : ""; - - if (operation === "sellers") { - return ( - " Check app authorization, role grants, and that refresh token belongs to the intended seller account." + - sandboxHint - ); - } - - return ( - " Check that Product Pricing role is enabled and re-authorize app to mint a new refresh token." + - sandboxHint - ); -} - -export async function testSpApiConnectivity( - asin?: string, -): Promise<{ ok: boolean; message: string }> { - const missingVars = missingSpApiEnvVars(); - if (missingVars.length > 0) { - return { - ok: false, - message: `Missing required SP-API env vars: ${missingVars.join(", ")}`, - }; - } - - const spClient = getSpClient(); - if (!spClient) { - return { - ok: false, - message: "SP-API client could not be initialized.", - }; - } - - try { - let sellersRes: { payload?: unknown[] }; - try { - sellersRes = (await spClient.callAPI({ - operation: "getMarketplaceParticipations", - endpoint: "sellers", - })) as { payload?: unknown[] }; - } catch (err) { - const message = extractErrorMessage(err); - const hint = isAccessDeniedMessage(message) ? deniedHint("sellers") : ""; - return { - ok: false, - message: `Auth probe failed (sellers.getMarketplaceParticipations): ${message}.${hint}`, - }; - } - - const participationCount = Array.isArray(sellersRes?.payload) - ? sellersRes.payload.length - : 0; - - if (!asin) { - return { - ok: true, - message: `SP-API auth OK. Seller participations returned: ${participationCount}.`, - }; - } - - let pricingRes: { status?: string; ASIN?: string }; - try { - pricingRes = (await spClient.callAPI({ - operation: "getItemOffers", - endpoint: "productPricing", - path: { Asin: asin }, - query: { - MarketplaceId: config.spApiMarketplaceId, - ItemCondition: "New", - }, - })) as { status?: string; ASIN?: string }; - } catch (err) { - const message = extractErrorMessage(err); - const hint = isAccessDeniedMessage(message) ? deniedHint("pricing") : ""; - return { - ok: false, - message: `Pricing probe failed (productPricing.getItemOffers): ${message}.${hint}`, - }; - } - - return { - ok: true, - message: - `SP-API auth OK. Seller participations: ${participationCount}. ` + - `Pricing check for ${asin} returned status=${pricingRes?.status ?? "unknown"} asin=${pricingRes?.ASIN ?? "unknown"}.`, - }; - } catch (err) { - return { - ok: false, - message: `SP-API connectivity failed unexpectedly: ${extractErrorMessage(err)}`, - }; - } -} - -export async function testSpApiSellability( - asin: string, -): Promise<{ ok: boolean; message: string }> { - const missingVars = missingSpApiEnvVars(); - if (missingVars.length > 0) { - return { - ok: false, - message: `Missing required SP-API env vars: ${missingVars.join(", ")}`, - }; - } - - if (!config.spApiSellerId) { - return { - ok: false, - message: "Missing required env var: SP_API_SELLER_ID", - }; - } - - const spClient = getSpClient(); - if (!spClient) { - return { - ok: false, - message: "SP-API client could not be initialized.", - }; - } - - const sellability = await fetchSellabilityInternal(spClient, asin); - if (sellability.sellabilityStatus === "unknown") { - return { - ok: false, - message: - `Sellability probe failed for ${asin}: ${sellability.sellabilityReason ?? "unknown reason"}. ` + - "This usually means sellerId is missing/incorrect or the app lacks Listings Restrictions permission.", - }; - } - - const canSell = - sellability.canSell == null - ? "unknown" - : sellability.canSell - ? "yes" - : "no"; - - return { - ok: true, - message: - `Sellability probe OK for ${asin}: status=${sellability.sellabilityStatus}, canSell=${canSell}. ` + - `${sellability.sellabilityReason ?? ""}`, - }; -} - -export async function fetchSpApiData(asin: string): Promise { - const fallback: SpApiData = { - fbaFee: 5.0, - fbmFee: 1.5, - referralFeePercent: 15, - estimatedSalePrice: 0, - canSell: null, - sellabilityStatus: "unknown", - sellabilityReason: "SP-API fallback values in use", - }; - - const spClient = getSpClient(); - if (!spClient) { - console.log(` [sp-api:fallback] ${asin} reason=missing_credentials`); - return fallback; - } - - try { - const pricing = (await spClient.callAPI({ - operation: "getItemOffers", - endpoint: "productPricing", - path: { Asin: asin }, - query: { - MarketplaceId: config.spApiMarketplaceId, - ItemCondition: "New", - }, - })) as any; - const sellability = await fetchSellabilityInternal(spClient, asin); - - const estimatedSalePrice = extractEstimatedSalePrice(pricing); - if (!Number.isFinite(estimatedSalePrice) || estimatedSalePrice <= 0) { - console.log( - ` [sp-api:fallback] ${asin} reason=no_valid_price_from_offers`, - ); - return { - ...fallback, - ...sellability, - }; - } - - const [fbaFeesRes, fbmFeesRes] = await Promise.all([ - spClient.callAPI({ - operation: "getMyFeesEstimateForASIN", - endpoint: "productFees", - path: { Asin: asin }, - body: { - FeesEstimateRequest: { - MarketplaceId: config.spApiMarketplaceId, - IsAmazonFulfilled: true, - Identifier: `${asin}-fba`, - PriceToEstimateFees: { - ListingPrice: { - CurrencyCode: "USD", - Amount: estimatedSalePrice, - }, - }, - }, - }, - }), - spClient.callAPI({ - operation: "getMyFeesEstimateForASIN", - endpoint: "productFees", - path: { Asin: asin }, - body: { - FeesEstimateRequest: { - MarketplaceId: config.spApiMarketplaceId, - IsAmazonFulfilled: false, - Identifier: `${asin}-fbm`, - PriceToEstimateFees: { - ListingPrice: { - CurrencyCode: "USD", - Amount: estimatedSalePrice, - }, - }, - }, - }, - }), - ]); - - const fba = extractFeeResult(fbaFeesRes); - const fbm = extractFeeResult(fbmFeesRes); - const referralFee = - fba.referralFee ?? fbm.referralFee ?? (estimatedSalePrice * 15) / 100; - const referralFeePercent = round2((referralFee / estimatedSalePrice) * 100); - - const result: SpApiData = { - fbaFee: round2(fba.totalFee || fallback.fbaFee), - fbmFee: round2(fbm.totalFee || fallback.fbmFee), - referralFeePercent: - Number.isFinite(referralFeePercent) && referralFeePercent > 0 - ? referralFeePercent - : fallback.referralFeePercent, - estimatedSalePrice: round2(estimatedSalePrice), - ...sellability, - }; - - console.log( - ` [sp-api:live] ${asin} price=$${result.estimatedSalePrice} fbaFee=$${result.fbaFee} fbmFee=$${result.fbmFee} referralPct=${result.referralFeePercent} sellability=${result.sellabilityStatus}`, - ); - - return result; - } catch (err) { - console.warn(`SP-API fetch failed for ${asin}: ${String(err)}`); - console.log(` [sp-api:fallback] ${asin} reason=request_failed`); - return fallback; - } -} - -// --------------------------------------------------------------------------- -// Public sellability + pricing/fees functions for the new pipeline -// --------------------------------------------------------------------------- - -export async function fetchSellability(asin: string): Promise { - const spClient = getSpClient(); - if (!spClient) { - return { - canSell: null, - sellabilityStatus: "unknown", - sellabilityReason: "SP-API credentials not configured", - }; - } - return fetchSellabilityInternal(spClient, asin); -} - -export async function fetchSellabilityBatch( - asins: string[], -): Promise> { - const results = new Map(); - const spClient = getSpClient(); - - if (!spClient) { - for (const asin of asins) { - results.set(asin, { - canSell: null, - sellabilityStatus: "unknown", - sellabilityReason: "SP-API credentials not configured", - }); - } - return results; - } - - let completed = 0; - let running = 0; - const queue = [...asins]; - - async function next(): Promise { - while (queue.length > 0) { - const asin = queue.shift()!; - const info = await fetchSellabilityInternal(spClient!, asin); - results.set(asin, info); - completed++; - if (completed % 10 === 0 || completed === asins.length) { - console.log(` [sellability] ${completed}/${asins.length} checked`); - } - } - } - - const workers = Array.from( - { length: Math.min(SELLABILITY_CONCURRENCY, asins.length) }, - () => next(), - ); - await Promise.all(workers); - - return results; -} - -export async function fetchSpApiPricingAndFees( - asin: string, - sellability: SellabilityInfo, -): Promise { - const fallback: SpApiData = { - fbaFee: 5.0, - fbmFee: 1.5, - referralFeePercent: 15, - estimatedSalePrice: 0, - ...sellability, - }; - - const spClient = getSpClient(); - if (!spClient) { - console.log(` [sp-api:fallback] ${asin} reason=missing_credentials`); - return fallback; - } - - try { - const pricing = (await spClient.callAPI({ - operation: "getItemOffers", - endpoint: "productPricing", - path: { Asin: asin }, - query: { - MarketplaceId: config.spApiMarketplaceId, - ItemCondition: "New", - }, - })) as any; - - const estimatedSalePrice = extractEstimatedSalePrice(pricing); - if (!Number.isFinite(estimatedSalePrice) || estimatedSalePrice <= 0) { - console.log( - ` [sp-api:fallback] ${asin} reason=no_valid_price_from_offers`, - ); - return fallback; - } - - const [fbaFeesRes, fbmFeesRes] = await Promise.all([ - spClient.callAPI({ - operation: "getMyFeesEstimateForASIN", - endpoint: "productFees", - path: { Asin: asin }, - body: { - FeesEstimateRequest: { - MarketplaceId: config.spApiMarketplaceId, - IsAmazonFulfilled: true, - Identifier: `${asin}-fba`, - PriceToEstimateFees: { - ListingPrice: { - CurrencyCode: "USD", - Amount: estimatedSalePrice, - }, - }, - }, - }, - }), - spClient.callAPI({ - operation: "getMyFeesEstimateForASIN", - endpoint: "productFees", - path: { Asin: asin }, - body: { - FeesEstimateRequest: { - MarketplaceId: config.spApiMarketplaceId, - IsAmazonFulfilled: false, - Identifier: `${asin}-fbm`, - PriceToEstimateFees: { - ListingPrice: { - CurrencyCode: "USD", - Amount: estimatedSalePrice, - }, - }, - }, - }, - }), - ]); - - const fba = extractFeeResult(fbaFeesRes); - const fbm = extractFeeResult(fbmFeesRes); - const referralFee = - fba.referralFee ?? fbm.referralFee ?? (estimatedSalePrice * 15) / 100; - const referralFeePercent = round2((referralFee / estimatedSalePrice) * 100); - - const result: SpApiData = { - fbaFee: round2(fba.totalFee || fallback.fbaFee), - fbmFee: round2(fbm.totalFee || fallback.fbmFee), - referralFeePercent: - Number.isFinite(referralFeePercent) && referralFeePercent > 0 - ? referralFeePercent - : fallback.referralFeePercent, - estimatedSalePrice: round2(estimatedSalePrice), - ...sellability, - }; - - console.log( - ` [sp-api:live] ${asin} price=$${result.estimatedSalePrice} fbaFee=$${result.fbaFee} fbmFee=$${result.fbmFee} referralPct=${result.referralFeePercent}`, - ); - - return result; - } catch (err) { - console.warn(`SP-API pricing/fees failed for ${asin}: ${String(err)}`); - console.log(` [sp-api:fallback] ${asin} reason=request_failed`); - return fallback; - } -} +import { SellingPartner } from "amazon-sp-api"; +import { config } from "./config.ts"; +import type { SpApiData, SellabilityInfo } from "./types.ts"; + +type RegionCode = "na" | "eu" | "fe"; + +let client: SellingPartner | null = null; +let loggedMissingCreds = false; +let loggedSandboxMode = false; + +function normalizeRegion(region: string): RegionCode { + const value = region.trim().toLowerCase(); + if (value === "us") return "na"; + if (value === "na" || value === "eu" || value === "fe") return value; + console.warn(`Unknown SP_API_REGION \"${region}\", defaulting to \"na\".`); + return "na"; +} + +function hasSpApiCredentials(): boolean { + return !!( + config.spApiClientId && + config.spApiClientSecret && + config.spApiRefreshToken + ); +} + +function getSpClient(): SellingPartner | null { + if (!hasSpApiCredentials()) { + if (!loggedMissingCreds) { + console.warn( + "SP-API credentials not configured; falling back to fee defaults. Set SP_API_CLIENT_ID, SP_API_CLIENT_SECRET, and SP_API_REFRESH_TOKEN.", + ); + loggedMissingCreds = true; + } + return null; + } + + if (client) return client; + + if (config.spApiUseSandbox && !loggedSandboxMode) { + console.warn( + "SP-API sandbox mode is enabled (SP_API_USE_SANDBOX=true). Production ASIN calls may be denied.", + ); + loggedSandboxMode = true; + } + + client = new SellingPartner({ + region: normalizeRegion(config.spApiRegion), + refresh_token: config.spApiRefreshToken!, + credentials: { + SELLING_PARTNER_APP_CLIENT_ID: config.spApiClientId!, + SELLING_PARTNER_APP_CLIENT_SECRET: config.spApiClientSecret!, + ...(config.awsAccessKeyId && config.awsSecretAccessKey + ? { + AWS_ACCESS_KEY_ID: config.awsAccessKeyId, + AWS_SECRET_ACCESS_KEY: config.awsSecretAccessKey, + } + : {}), + ...(config.awsSessionToken + ? { AWS_SESSION_TOKEN: config.awsSessionToken } + : {}), + }, + options: { + auto_request_tokens: true, + auto_request_throttled: true, + use_sandbox: config.spApiUseSandbox, + debug_log: false, + }, + }); + + return client; +} + +function getAmount(value: unknown): number | undefined { + if (!value || typeof value !== "object") return undefined; + const amount = (value as { Amount?: unknown }).Amount; + return typeof amount === "number" && Number.isFinite(amount) + ? amount + : undefined; +} + +function extractEstimatedSalePrice(pricing: any): number { + const buyBox = pricing?.Summary?.BuyBoxPrices?.[0]; + const lowest = pricing?.Summary?.LowestPrices?.[0]; + const buyBoxLanded = getAmount(buyBox?.LandedPrice); + if (buyBoxLanded != null) return buyBoxLanded; + + const lowestLanded = getAmount(lowest?.LandedPrice); + if (lowestLanded != null) return lowestLanded; + + const firstOffer = pricing?.Offers?.[0]; + const listing = getAmount(firstOffer?.ListingPrice) ?? 0; + const shipping = getAmount(firstOffer?.Shipping) ?? 0; + return listing + shipping; +} + +function extractFeeResult(feesResponse: any): { + totalFee: number; + referralFee?: number; +} { + const result = feesResponse?.payload?.FeesEstimateResult; + const total = getAmount(result?.FeesEstimate?.TotalFeesEstimate) ?? 0; + const feeDetails = result?.FeesEstimate?.FeeDetailList; + + const referralDetail = Array.isArray(feeDetails) + ? feeDetails.find((f: any) => + String(f?.FeeType ?? "") + .toLowerCase() + .includes("referral"), + ) + : undefined; + + const referralFee = getAmount(referralDetail?.FinalFee); + return { totalFee: total, referralFee }; +} + +function round2(value: number): number { + return Math.round(value * 100) / 100; +} + +const SELLABILITY_CONCURRENCY = 5; +const PRICING_CONCURRENCY = 5; + +function parseSellabilityResponse(response: any): SellabilityInfo { + const restrictions = Array.isArray(response?.restrictions) + ? response.restrictions + : Array.isArray(response?.payload?.restrictions) + ? response.payload.restrictions + : null; + + if (!Array.isArray(restrictions)) { + return { + canSell: null, + sellabilityStatus: "unknown", + sellabilityReason: "Unexpected restrictions response shape", + }; + } + + if (restrictions.length === 0) { + return { + canSell: true, + sellabilityStatus: "available", + sellabilityReason: "No listing restrictions reported", + }; + } + + const reasons = restrictions.flatMap((r: any) => + Array.isArray(r?.reasons) ? r.reasons : [], + ); + const reasonCodes = reasons + .map((r: any) => String(r?.reasonCode ?? "").trim()) + .filter((r: string) => r.length > 0); + const reasonMessages = reasons + .map((r: any) => String(r?.message ?? "").trim()) + .filter((m: string) => m.length > 0); + + const allReasonText = [...reasonCodes, ...reasonMessages] + .join(" | ") + .toLowerCase(); + const status = + allReasonText.includes("not_eligible") || + allReasonText.includes("not eligible") || + allReasonText.includes("not available") + ? "not_available" + : "restricted"; + + return { + canSell: false, + sellabilityStatus: status, + sellabilityReason: + [...reasonCodes, ...reasonMessages].join(" | ") || + "Listing restrictions reported", + }; +} + +async function fetchSellabilityInternal( + spClient: SellingPartner, + asin: string, +): Promise { + if (!config.spApiSellerId) { + return { + canSell: null, + sellabilityStatus: "unknown", + sellabilityReason: "Missing SP_API_SELLER_ID", + }; + } + + try { + const restrictionResponse = await spClient.callAPI({ + operation: "getListingsRestrictions", + endpoint: "listingsRestrictions", + query: { + asin, + sellerId: config.spApiSellerId, + marketplaceIds: [config.spApiMarketplaceId], + conditionType: "new_new", + }, + }); + return parseSellabilityResponse(restrictionResponse); + } catch (err) { + return { + canSell: null, + sellabilityStatus: "unknown", + sellabilityReason: `Restrictions check failed: ${extractErrorMessage(err)}`, + }; + } +} + +function missingSpApiEnvVars(): string[] { + const missing: string[] = []; + if (!config.spApiClientId) missing.push("SP_API_CLIENT_ID"); + if (!config.spApiClientSecret) missing.push("SP_API_CLIENT_SECRET"); + if (!config.spApiRefreshToken) missing.push("SP_API_REFRESH_TOKEN"); + return missing; +} + +function extractErrorMessage(err: unknown): string { + if (err instanceof Error) return err.message; + return String(err); +} + +function isAccessDeniedMessage(message: string): boolean { + const value = message.toLowerCase(); + return ( + value.includes("access to requested resource is denied") || + value.includes("access denied") || + value.includes("forbidden") || + value.includes("unauthorized") + ); +} + +function deniedHint(operation: string): string { + const sandboxHint = config.spApiUseSandbox + ? " Sandbox mode is enabled; production data calls are often denied in sandbox." + : ""; + + if (operation === "sellers") { + return ( + " Check app authorization, role grants, and that refresh token belongs to the intended seller account." + + sandboxHint + ); + } + + return ( + " Check that Product Pricing role is enabled and re-authorize app to mint a new refresh token." + + sandboxHint + ); +} + +export async function testSpApiConnectivity( + asin?: string, +): Promise<{ ok: boolean; message: string }> { + const missingVars = missingSpApiEnvVars(); + if (missingVars.length > 0) { + return { + ok: false, + message: `Missing required SP-API env vars: ${missingVars.join(", ")}`, + }; + } + + const spClient = getSpClient(); + if (!spClient) { + return { + ok: false, + message: "SP-API client could not be initialized.", + }; + } + + try { + let sellersRes: { payload?: unknown[] }; + try { + sellersRes = (await spClient.callAPI({ + operation: "getMarketplaceParticipations", + endpoint: "sellers", + })) as { payload?: unknown[] }; + } catch (err) { + const message = extractErrorMessage(err); + const hint = isAccessDeniedMessage(message) ? deniedHint("sellers") : ""; + return { + ok: false, + message: `Auth probe failed (sellers.getMarketplaceParticipations): ${message}.${hint}`, + }; + } + + const participationCount = Array.isArray(sellersRes?.payload) + ? sellersRes.payload.length + : 0; + + if (!asin) { + return { + ok: true, + message: `SP-API auth OK. Seller participations returned: ${participationCount}.`, + }; + } + + let pricingRes: { status?: string; ASIN?: string }; + try { + pricingRes = (await spClient.callAPI({ + operation: "getItemOffers", + endpoint: "productPricing", + path: { Asin: asin }, + query: { + MarketplaceId: config.spApiMarketplaceId, + ItemCondition: "New", + }, + })) as { status?: string; ASIN?: string }; + } catch (err) { + const message = extractErrorMessage(err); + const hint = isAccessDeniedMessage(message) ? deniedHint("pricing") : ""; + return { + ok: false, + message: `Pricing probe failed (productPricing.getItemOffers): ${message}.${hint}`, + }; + } + + return { + ok: true, + message: + `SP-API auth OK. Seller participations: ${participationCount}. ` + + `Pricing check for ${asin} returned status=${pricingRes?.status ?? "unknown"} asin=${pricingRes?.ASIN ?? "unknown"}.`, + }; + } catch (err) { + return { + ok: false, + message: `SP-API connectivity failed unexpectedly: ${extractErrorMessage(err)}`, + }; + } +} + +export async function testSpApiSellability( + asin: string, +): Promise<{ ok: boolean; message: string }> { + const missingVars = missingSpApiEnvVars(); + if (missingVars.length > 0) { + return { + ok: false, + message: `Missing required SP-API env vars: ${missingVars.join(", ")}`, + }; + } + + if (!config.spApiSellerId) { + return { + ok: false, + message: "Missing required env var: SP_API_SELLER_ID", + }; + } + + const spClient = getSpClient(); + if (!spClient) { + return { + ok: false, + message: "SP-API client could not be initialized.", + }; + } + + const sellability = await fetchSellabilityInternal(spClient, asin); + if (sellability.sellabilityStatus === "unknown") { + return { + ok: false, + message: + `Sellability probe failed for ${asin}: ${sellability.sellabilityReason ?? "unknown reason"}. ` + + "This usually means sellerId is missing/incorrect or the app lacks Listings Restrictions permission.", + }; + } + + const canSell = + sellability.canSell == null + ? "unknown" + : sellability.canSell + ? "yes" + : "no"; + + return { + ok: true, + message: + `Sellability probe OK for ${asin}: status=${sellability.sellabilityStatus}, canSell=${canSell}. ` + + `${sellability.sellabilityReason ?? ""}`, + }; +} + +export async function fetchSpApiData(asin: string): Promise { + const fallback: SpApiData = { + fbaFee: 5.0, + fbmFee: 1.5, + referralFeePercent: 15, + estimatedSalePrice: 0, + canSell: null, + sellabilityStatus: "unknown", + sellabilityReason: "SP-API fallback values in use", + }; + + const spClient = getSpClient(); + if (!spClient) { + console.log(` [sp-api:fallback] ${asin} reason=missing_credentials`); + return fallback; + } + + try { + const pricing = (await spClient.callAPI({ + operation: "getItemOffers", + endpoint: "productPricing", + path: { Asin: asin }, + query: { + MarketplaceId: config.spApiMarketplaceId, + ItemCondition: "New", + }, + })) as any; + const sellability = await fetchSellabilityInternal(spClient, asin); + + const estimatedSalePrice = extractEstimatedSalePrice(pricing); + if (!Number.isFinite(estimatedSalePrice) || estimatedSalePrice <= 0) { + console.log( + ` [sp-api:fallback] ${asin} reason=no_valid_price_from_offers`, + ); + return { + ...fallback, + ...sellability, + }; + } + + const [fbaFeesRes, fbmFeesRes] = await Promise.all([ + spClient.callAPI({ + operation: "getMyFeesEstimateForASIN", + endpoint: "productFees", + path: { Asin: asin }, + body: { + FeesEstimateRequest: { + MarketplaceId: config.spApiMarketplaceId, + IsAmazonFulfilled: true, + Identifier: `${asin}-fba`, + PriceToEstimateFees: { + ListingPrice: { + CurrencyCode: "USD", + Amount: estimatedSalePrice, + }, + }, + }, + }, + }), + spClient.callAPI({ + operation: "getMyFeesEstimateForASIN", + endpoint: "productFees", + path: { Asin: asin }, + body: { + FeesEstimateRequest: { + MarketplaceId: config.spApiMarketplaceId, + IsAmazonFulfilled: false, + Identifier: `${asin}-fbm`, + PriceToEstimateFees: { + ListingPrice: { + CurrencyCode: "USD", + Amount: estimatedSalePrice, + }, + }, + }, + }, + }), + ]); + + const fba = extractFeeResult(fbaFeesRes); + const fbm = extractFeeResult(fbmFeesRes); + const referralFee = + fba.referralFee ?? fbm.referralFee ?? (estimatedSalePrice * 15) / 100; + const referralFeePercent = round2((referralFee / estimatedSalePrice) * 100); + + const result: SpApiData = { + fbaFee: round2(fba.totalFee || fallback.fbaFee), + fbmFee: round2(fbm.totalFee || fallback.fbmFee), + referralFeePercent: + Number.isFinite(referralFeePercent) && referralFeePercent > 0 + ? referralFeePercent + : fallback.referralFeePercent, + estimatedSalePrice: round2(estimatedSalePrice), + ...sellability, + }; + + console.log( + ` [sp-api:live] ${asin} price=$${result.estimatedSalePrice} fbaFee=$${result.fbaFee} fbmFee=$${result.fbmFee} referralPct=${result.referralFeePercent} sellability=${result.sellabilityStatus}`, + ); + + return result; + } catch (err) { + console.warn(`SP-API fetch failed for ${asin}: ${String(err)}`); + console.log(` [sp-api:fallback] ${asin} reason=request_failed`); + return fallback; + } +} + +// --------------------------------------------------------------------------- +// Public sellability + pricing/fees functions for the new pipeline +// --------------------------------------------------------------------------- + +export async function fetchSellability(asin: string): Promise { + const spClient = getSpClient(); + if (!spClient) { + return { + canSell: null, + sellabilityStatus: "unknown", + sellabilityReason: "SP-API credentials not configured", + }; + } + return fetchSellabilityInternal(spClient, asin); +} + +export async function fetchSellabilityBatch( + asins: string[], +): Promise> { + const results = new Map(); + const spClient = getSpClient(); + + if (!spClient) { + for (const asin of asins) { + results.set(asin, { + canSell: null, + sellabilityStatus: "unknown", + sellabilityReason: "SP-API credentials not configured", + }); + } + return results; + } + + let completed = 0; + let running = 0; + const queue = [...asins]; + + async function next(): Promise { + while (queue.length > 0) { + const asin = queue.shift()!; + const info = await fetchSellabilityInternal(spClient!, asin); + results.set(asin, info); + completed++; + if (completed % 10 === 0 || completed === asins.length) { + console.log(` [sellability] ${completed}/${asins.length} checked`); + } + } + } + + const workers = Array.from( + { length: Math.min(SELLABILITY_CONCURRENCY, asins.length) }, + () => next(), + ); + await Promise.all(workers); + + return results; +} + +export async function fetchSpApiPricingAndFees( + asin: string, + sellability: SellabilityInfo, +): Promise { + const fallback: SpApiData = { + fbaFee: 5.0, + fbmFee: 1.5, + referralFeePercent: 15, + estimatedSalePrice: 0, + ...sellability, + }; + + const spClient = getSpClient(); + if (!spClient) { + console.log(` [sp-api:fallback] ${asin} reason=missing_credentials`); + return fallback; + } + + try { + const pricing = (await spClient.callAPI({ + operation: "getItemOffers", + endpoint: "productPricing", + path: { Asin: asin }, + query: { + MarketplaceId: config.spApiMarketplaceId, + ItemCondition: "New", + }, + })) as any; + + const estimatedSalePrice = extractEstimatedSalePrice(pricing); + if (!Number.isFinite(estimatedSalePrice) || estimatedSalePrice <= 0) { + console.log( + ` [sp-api:fallback] ${asin} reason=no_valid_price_from_offers`, + ); + return fallback; + } + + const [fbaFeesRes, fbmFeesRes] = await Promise.all([ + spClient.callAPI({ + operation: "getMyFeesEstimateForASIN", + endpoint: "productFees", + path: { Asin: asin }, + body: { + FeesEstimateRequest: { + MarketplaceId: config.spApiMarketplaceId, + IsAmazonFulfilled: true, + Identifier: `${asin}-fba`, + PriceToEstimateFees: { + ListingPrice: { + CurrencyCode: "USD", + Amount: estimatedSalePrice, + }, + }, + }, + }, + }), + spClient.callAPI({ + operation: "getMyFeesEstimateForASIN", + endpoint: "productFees", + path: { Asin: asin }, + body: { + FeesEstimateRequest: { + MarketplaceId: config.spApiMarketplaceId, + IsAmazonFulfilled: false, + Identifier: `${asin}-fbm`, + PriceToEstimateFees: { + ListingPrice: { + CurrencyCode: "USD", + Amount: estimatedSalePrice, + }, + }, + }, + }, + }), + ]); + + const fba = extractFeeResult(fbaFeesRes); + const fbm = extractFeeResult(fbmFeesRes); + const referralFee = + fba.referralFee ?? fbm.referralFee ?? (estimatedSalePrice * 15) / 100; + const referralFeePercent = round2((referralFee / estimatedSalePrice) * 100); + + const result: SpApiData = { + fbaFee: round2(fba.totalFee || fallback.fbaFee), + fbmFee: round2(fbm.totalFee || fallback.fbmFee), + referralFeePercent: + Number.isFinite(referralFeePercent) && referralFeePercent > 0 + ? referralFeePercent + : fallback.referralFeePercent, + estimatedSalePrice: round2(estimatedSalePrice), + ...sellability, + }; + + console.log( + ` [sp-api:live] ${asin} price=$${result.estimatedSalePrice} fbaFee=$${result.fbaFee} fbmFee=$${result.fbmFee} referralPct=${result.referralFeePercent}`, + ); + + return result; + } catch (err) { + console.warn(`SP-API pricing/fees failed for ${asin}: ${String(err)}`); + console.log(` [sp-api:fallback] ${asin} reason=request_failed`); + return fallback; + } +} diff --git a/src/sp-test.ts b/src/sp-test.ts index 2d94e37..9d901f8 100644 --- a/src/sp-test.ts +++ b/src/sp-test.ts @@ -1,48 +1,48 @@ -import { testSpApiConnectivity, testSpApiSellability } from "./sp-api.ts"; - -function parseArgs(): { asin?: string; sellabilityMode: boolean } { - const args = process.argv.slice(2); - const sellabilityMode = args.includes("--sellability"); - const asin = args.find((arg) => !arg.startsWith("--")); - return { asin, sellabilityMode }; -} - -async function main() { - const { asin, sellabilityMode } = parseArgs(); - - console.log("Running SP-API connectivity test..."); - - if (sellabilityMode) { - if (!asin) { - console.error("Usage: bun run src/sp-test.ts --sellability "); - process.exit(1); - } - - console.log(`Running sellability check for ASIN: ${asin}`); - const sellability = await testSpApiSellability(asin); - if (!sellability.ok) { - console.error(`SP-API sellability test failed: ${sellability.message}`); - process.exit(1); - } - - console.log(`SP-API sellability test passed: ${sellability.message}`); - return; - } - - if (asin) { - console.log(`Including pricing connectivity check for ASIN: ${asin}`); - } - - const result = await testSpApiConnectivity(asin); - if (!result.ok) { - console.error(`SP-API test failed: ${result.message}`); - process.exit(1); - } - - console.log(`SP-API test passed: ${result.message}`); -} - -main().catch((err) => { - console.error(`SP-API test crashed: ${String(err)}`); - process.exit(1); -}); +import { testSpApiConnectivity, testSpApiSellability } from "./sp-api.ts"; + +function parseArgs(): { asin?: string; sellabilityMode: boolean } { + const args = process.argv.slice(2); + const sellabilityMode = args.includes("--sellability"); + const asin = args.find((arg) => !arg.startsWith("--")); + return { asin, sellabilityMode }; +} + +async function main() { + const { asin, sellabilityMode } = parseArgs(); + + console.log("Running SP-API connectivity test..."); + + if (sellabilityMode) { + if (!asin) { + console.error("Usage: bun run src/sp-test.ts --sellability "); + process.exit(1); + } + + console.log(`Running sellability check for ASIN: ${asin}`); + const sellability = await testSpApiSellability(asin); + if (!sellability.ok) { + console.error(`SP-API sellability test failed: ${sellability.message}`); + process.exit(1); + } + + console.log(`SP-API sellability test passed: ${sellability.message}`); + return; + } + + if (asin) { + console.log(`Including pricing connectivity check for ASIN: ${asin}`); + } + + const result = await testSpApiConnectivity(asin); + if (!result.ok) { + console.error(`SP-API test failed: ${result.message}`); + process.exit(1); + } + + console.log(`SP-API test passed: ${result.message}`); +} + +main().catch((err) => { + console.error(`SP-API test crashed: ${String(err)}`); + process.exit(1); +}); diff --git a/src/types.ts b/src/types.ts index b142a1b..cc9bc4d 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,75 +1,75 @@ -export interface ProductRecord { - asin: string; - name: string; - unitCost: number; - brand?: string; - category?: string; - amazonRank?: number; - avgPrice90FromSheet?: number; - sellingPriceFromSheet?: number; - fbaNet?: number; - grossProfit?: number; - grossProfitPct?: number; - netProfitFromSheet?: number; - roiFromSheet?: number; - moq?: number; - moqCost?: number; - totalQtyAvail?: number; - - link?: string; - asinLink?: string; - sourceUrl?: string; - supplier?: string; - promoCouponCode?: string; - notes?: string; - leadDate?: string; - [key: string]: unknown; -} - -export interface KeepaData { - currentPrice: number | null; - avgPrice90: number | null; - minPrice90: number | null; - maxPrice90: number | null; - salesRank: number | null; - salesRankAvg90: number | null; - salesRankDrops30: number | null; - salesRankDrops90: number | null; - sellerCount: number | null; - buyBoxSeller: string | null; - buyBoxPrice: number | null; - monthlySold: number | null; - categoryTree: string[]; -} - -export type SellabilityInfo = { - canSell: boolean | null; - sellabilityStatus: "available" | "restricted" | "not_available" | "unknown"; - sellabilityReason?: string; -}; - -export interface SpApiData extends SellabilityInfo { - fbaFee: number; - fbmFee: number; - referralFeePercent: number; - estimatedSalePrice: number; -} - -export interface EnrichedProduct { - record: ProductRecord; - keepa: KeepaData | null; - spApi: SpApiData; - fetchedAt: string; -} - -export interface LlmVerdict { - asin: string; - verdict: "FBA" | "FBM" | "SKIP"; - confidence: number; - reasoning: string; -} - -export interface AnalysisResult { - product: EnrichedProduct; - verdict: LlmVerdict; -} +export interface ProductRecord { + asin: string; + name: string; + unitCost: number; + brand?: string; + category?: string; + amazonRank?: number; + avgPrice90FromSheet?: number; + sellingPriceFromSheet?: number; + fbaNet?: number; + grossProfit?: number; + grossProfitPct?: number; + netProfitFromSheet?: number; + roiFromSheet?: number; + moq?: number; + moqCost?: number; + totalQtyAvail?: number; + + link?: string; + asinLink?: string; + sourceUrl?: string; + supplier?: string; + promoCouponCode?: string; + notes?: string; + leadDate?: string; + [key: string]: unknown; +} + +export interface KeepaData { + currentPrice: number | null; + avgPrice90: number | null; + minPrice90: number | null; + maxPrice90: number | null; + salesRank: number | null; + salesRankAvg90: number | null; + salesRankDrops30: number | null; + salesRankDrops90: number | null; + sellerCount: number | null; + buyBoxSeller: string | null; + buyBoxPrice: number | null; + monthlySold: number | null; + categoryTree: string[]; +} + +export type SellabilityInfo = { + canSell: boolean | null; + sellabilityStatus: "available" | "restricted" | "not_available" | "unknown"; + sellabilityReason?: string; +}; + +export interface SpApiData extends SellabilityInfo { + fbaFee: number; + fbmFee: number; + referralFeePercent: number; + estimatedSalePrice: number; +} + +export interface EnrichedProduct { + record: ProductRecord; + keepa: KeepaData | null; + spApi: SpApiData; + fetchedAt: string; +} + +export interface LlmVerdict { + asin: string; + verdict: "FBA" | "FBM" | "SKIP"; + confidence: number; + reasoning: string; +} + +export interface AnalysisResult { + product: EnrichedProduct; + verdict: LlmVerdict; +} diff --git a/src/writer.ts b/src/writer.ts index 2172af9..cd725d5 100644 --- a/src/writer.ts +++ b/src/writer.ts @@ -1,159 +1,159 @@ -import * as XLSX from "xlsx"; -import type { AnalysisResult } from "./types.ts"; - -function buildRow(r: AnalysisResult) { - const price = - r.product.keepa?.currentPrice ?? - r.product.record.sellingPriceFromSheet ?? - r.product.spApi.estimatedSalePrice; - const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank; - - return { - ASIN: r.product.record.asin, - Name: r.product.record.name, - Brand: r.product.record.brand ?? "", - Category: - r.product.record.category ?? - r.product.keepa?.categoryTree?.join(" > ") ?? - "", - "Unit Cost": r.product.record.unitCost, - "Current Price": price ?? "", - "Avg Price 90d": r.product.keepa?.avgPrice90 ?? "", - "Avg Price 90d (sheet)": r.product.record.avgPrice90FromSheet ?? "", - "Selling Price (sheet)": r.product.record.sellingPriceFromSheet ?? "", - "Sales Rank": rank ?? "", - "Rank Avg 90d": r.product.keepa?.salesRankAvg90 ?? "", - Sellers: r.product.keepa?.sellerCount ?? "", - "Monthly Sold": r.product.keepa?.monthlySold ?? "", - "Rank Drops 30d": r.product.keepa?.salesRankDrops30 ?? "", - "Rank Drops 90d": r.product.keepa?.salesRankDrops90 ?? "", - "FBA Net (sheet)": r.product.record.fbaNet ?? "", - "Gross Profit $": r.product.record.grossProfit ?? "", - "Gross Profit %": r.product.record.grossProfitPct ?? "", - "Net Profit (sheet)": r.product.record.netProfitFromSheet ?? "", - "ROI (sheet)": r.product.record.roiFromSheet ?? "", - MOQ: r.product.record.moq ?? "", - "MOQ Cost": r.product.record.moqCost ?? "", - "Qty Available": r.product.record.totalQtyAvail ?? "", - Supplier: r.product.record.supplier ?? "", - "Source URL": r.product.record.sourceUrl ?? "", - "ASIN Link": r.product.record.asinLink ?? "", - "Promo/Coupon Code": r.product.record.promoCouponCode ?? "", - Notes: r.product.record.notes ?? "", - "Lead Date": r.product.record.leadDate ?? "", - "FBA Fee": r.product.spApi.fbaFee, - "FBM Fee": r.product.spApi.fbmFee, - "Referral %": r.product.spApi.referralFeePercent, - "Can Sell": - r.product.spApi.canSell == null - ? "unknown" - : r.product.spApi.canSell - ? "yes" - : "no", - Sellability: r.product.spApi.sellabilityStatus, - "Sellability Reason": r.product.spApi.sellabilityReason ?? "", - Verdict: r.verdict.verdict, - Confidence: r.verdict.confidence, - Reasoning: r.verdict.reasoning, - }; -} - -export function printResults(results: AnalysisResult[]): void { - const rows = results - .filter((r) => r.verdict.verdict === "FBA" || r.verdict.verdict === "FBM") - .map((r) => { - const sellingPrice = - r.product.keepa?.currentPrice ?? - r.product.record.sellingPriceFromSheet ?? - r.product.spApi.estimatedSalePrice; - const referralFee = - sellingPrice != null - ? sellingPrice * (r.product.spApi.referralFeePercent / 100) - : null; - const fulfillmentFee = - r.verdict.verdict === "FBA" - ? r.product.spApi.fbaFee - : r.product.spApi.fbmFee; - const netProfit = - sellingPrice != null - ? Math.round( - (sellingPrice - - r.product.record.unitCost - - fulfillmentFee - - (referralFee ?? 0)) * - 100, - ) / 100 - : ""; - - return { - ASIN: r.product.record.asin, - Name: r.product.record.name.slice(0, 40), - Category: String( - r.product.record.category ?? - r.product.keepa?.categoryTree?.join(" > ") ?? - "", - ).slice(0, 20), - "Unit Cost": r.product.record.unitCost, - "Selling Price": sellingPrice ?? "", - "Net Profit": netProfit, - "Monthly Sold": r.product.keepa?.monthlySold ?? "", - "Sold 90 Day": r.product.keepa?.salesRankDrops90 ?? "", - "Can Sell": - r.product.spApi.canSell == null - ? "unknown" - : r.product.spApi.canSell - ? "yes" - : "no", - Sellability: r.product.spApi.sellabilityStatus, - "Sellability Reason": String( - r.product.spApi.sellabilityReason ?? "", - ).slice(0, 60), - Confidence: r.verdict.confidence, - Reasoning: r.verdict.reasoning.slice(0, 60), - }; - }); - - console.log("\n=== Analysis Results ===\n"); - if (rows.length === 0) { - console.log("No FBA/FBM leads found."); - } else { - console.table(rows); - } - - const summary = { - FBA: results.filter((r) => r.verdict.verdict === "FBA").length, - FBM: results.filter((r) => r.verdict.verdict === "FBM").length, - SKIP: results.filter((r) => r.verdict.verdict === "SKIP").length, - Available: results.filter( - (r) => r.product.spApi.sellabilityStatus === "available", - ).length, - Restricted: results.filter( - (r) => r.product.spApi.sellabilityStatus === "restricted", - ).length, - NotAvailable: results.filter( - (r) => r.product.spApi.sellabilityStatus === "not_available", - ).length, - Unknown: results.filter( - (r) => r.product.spApi.sellabilityStatus === "unknown", - ).length, - }; - console.log( - `\nSummary: ${summary.FBA} FBA | ${summary.FBM} FBM | ${summary.SKIP} SKIP out of ${results.length} products\n`, - ); - console.log( - `Sellability: ${summary.Available} available | ${summary.Restricted} restricted | ${summary.NotAvailable} not_available | ${summary.Unknown} unknown\n`, - ); -} - -export function writeResultsCsv( - results: AnalysisResult[], - outputPath: string, -): void { - const rows = results.map(buildRow); - - const ws = XLSX.utils.json_to_sheet(rows); - const wb = XLSX.utils.book_new(); - XLSX.utils.book_append_sheet(wb, ws, "Results"); - XLSX.writeFile(wb, outputPath); - console.log(`Results written to ${outputPath}`); -} +import * as XLSX from "xlsx"; +import type { AnalysisResult } from "./types.ts"; + +function buildRow(r: AnalysisResult) { + const price = + r.product.keepa?.currentPrice ?? + r.product.record.sellingPriceFromSheet ?? + r.product.spApi.estimatedSalePrice; + const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank; + + return { + ASIN: r.product.record.asin, + Name: r.product.record.name, + Brand: r.product.record.brand ?? "", + Category: + r.product.record.category ?? + r.product.keepa?.categoryTree?.join(" > ") ?? + "", + "Unit Cost": r.product.record.unitCost, + "Current Price": price ?? "", + "Avg Price 90d": r.product.keepa?.avgPrice90 ?? "", + "Avg Price 90d (sheet)": r.product.record.avgPrice90FromSheet ?? "", + "Selling Price (sheet)": r.product.record.sellingPriceFromSheet ?? "", + "Sales Rank": rank ?? "", + "Rank Avg 90d": r.product.keepa?.salesRankAvg90 ?? "", + Sellers: r.product.keepa?.sellerCount ?? "", + "Monthly Sold": r.product.keepa?.monthlySold ?? "", + "Rank Drops 30d": r.product.keepa?.salesRankDrops30 ?? "", + "Rank Drops 90d": r.product.keepa?.salesRankDrops90 ?? "", + "FBA Net (sheet)": r.product.record.fbaNet ?? "", + "Gross Profit $": r.product.record.grossProfit ?? "", + "Gross Profit %": r.product.record.grossProfitPct ?? "", + "Net Profit (sheet)": r.product.record.netProfitFromSheet ?? "", + "ROI (sheet)": r.product.record.roiFromSheet ?? "", + MOQ: r.product.record.moq ?? "", + "MOQ Cost": r.product.record.moqCost ?? "", + "Qty Available": r.product.record.totalQtyAvail ?? "", + Supplier: r.product.record.supplier ?? "", + "Source URL": r.product.record.sourceUrl ?? "", + "ASIN Link": r.product.record.asinLink ?? "", + "Promo/Coupon Code": r.product.record.promoCouponCode ?? "", + Notes: r.product.record.notes ?? "", + "Lead Date": r.product.record.leadDate ?? "", + "FBA Fee": r.product.spApi.fbaFee, + "FBM Fee": r.product.spApi.fbmFee, + "Referral %": r.product.spApi.referralFeePercent, + "Can Sell": + r.product.spApi.canSell == null + ? "unknown" + : r.product.spApi.canSell + ? "yes" + : "no", + Sellability: r.product.spApi.sellabilityStatus, + "Sellability Reason": r.product.spApi.sellabilityReason ?? "", + Verdict: r.verdict.verdict, + Confidence: r.verdict.confidence, + Reasoning: r.verdict.reasoning, + }; +} + +export function printResults(results: AnalysisResult[]): void { + const rows = results + .filter((r) => r.verdict.verdict === "FBA" || r.verdict.verdict === "FBM") + .map((r) => { + const sellingPrice = + r.product.keepa?.currentPrice ?? + r.product.record.sellingPriceFromSheet ?? + r.product.spApi.estimatedSalePrice; + const referralFee = + sellingPrice != null + ? sellingPrice * (r.product.spApi.referralFeePercent / 100) + : null; + const fulfillmentFee = + r.verdict.verdict === "FBA" + ? r.product.spApi.fbaFee + : r.product.spApi.fbmFee; + const netProfit = + sellingPrice != null + ? Math.round( + (sellingPrice - + r.product.record.unitCost - + fulfillmentFee - + (referralFee ?? 0)) * + 100, + ) / 100 + : ""; + + return { + ASIN: r.product.record.asin, + Name: r.product.record.name.slice(0, 40), + Category: String( + r.product.record.category ?? + r.product.keepa?.categoryTree?.join(" > ") ?? + "", + ).slice(0, 20), + "Unit Cost": r.product.record.unitCost, + "Selling Price": sellingPrice ?? "", + "Net Profit": netProfit, + "Monthly Sold": r.product.keepa?.monthlySold ?? "", + "Sold 90 Day": r.product.keepa?.salesRankDrops90 ?? "", + "Can Sell": + r.product.spApi.canSell == null + ? "unknown" + : r.product.spApi.canSell + ? "yes" + : "no", + Sellability: r.product.spApi.sellabilityStatus, + "Sellability Reason": String( + r.product.spApi.sellabilityReason ?? "", + ).slice(0, 60), + Confidence: r.verdict.confidence, + Reasoning: r.verdict.reasoning.slice(0, 60), + }; + }); + + console.log("\n=== Analysis Results ===\n"); + if (rows.length === 0) { + console.log("No FBA/FBM leads found."); + } else { + console.table(rows); + } + + const summary = { + FBA: results.filter((r) => r.verdict.verdict === "FBA").length, + FBM: results.filter((r) => r.verdict.verdict === "FBM").length, + SKIP: results.filter((r) => r.verdict.verdict === "SKIP").length, + Available: results.filter( + (r) => r.product.spApi.sellabilityStatus === "available", + ).length, + Restricted: results.filter( + (r) => r.product.spApi.sellabilityStatus === "restricted", + ).length, + NotAvailable: results.filter( + (r) => r.product.spApi.sellabilityStatus === "not_available", + ).length, + Unknown: results.filter( + (r) => r.product.spApi.sellabilityStatus === "unknown", + ).length, + }; + console.log( + `\nSummary: ${summary.FBA} FBA | ${summary.FBM} FBM | ${summary.SKIP} SKIP out of ${results.length} products\n`, + ); + console.log( + `Sellability: ${summary.Available} available | ${summary.Restricted} restricted | ${summary.NotAvailable} not_available | ${summary.Unknown} unknown\n`, + ); +} + +export function writeResultsCsv( + results: AnalysisResult[], + outputPath: string, +): void { + const rows = results.map(buildRow); + + const ws = XLSX.utils.json_to_sheet(rows); + const wb = XLSX.utils.book_new(); + XLSX.utils.book_append_sheet(wb, ws, "Results"); + XLSX.writeFile(wb, outputPath); + console.log(`Results written to ${outputPath}`); +} diff --git a/tsconfig.json b/tsconfig.json index bfa0fea..370696a 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,29 +1,29 @@ -{ - "compilerOptions": { - // Environment setup & latest features - "lib": ["ESNext"], - "target": "ESNext", - "module": "Preserve", - "moduleDetection": "force", - "jsx": "react-jsx", - "allowJs": true, - - // Bundler mode - "moduleResolution": "bundler", - "allowImportingTsExtensions": true, - "verbatimModuleSyntax": true, - "noEmit": true, - - // Best practices - "strict": true, - "skipLibCheck": true, - "noFallthroughCasesInSwitch": true, - "noUncheckedIndexedAccess": true, - "noImplicitOverride": true, - - // Some stricter flags (disabled by default) - "noUnusedLocals": false, - "noUnusedParameters": false, - "noPropertyAccessFromIndexSignature": false - } -} +{ + "compilerOptions": { + // Environment setup & latest features + "lib": ["ESNext"], + "target": "ESNext", + "module": "Preserve", + "moduleDetection": "force", + "jsx": "react-jsx", + "allowJs": true, + + // Bundler mode + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + + // Best practices + "strict": true, + "skipLibCheck": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedIndexedAccess": true, + "noImplicitOverride": true, + + // Some stricter flags (disabled by default) + "noUnusedLocals": false, + "noUnusedParameters": false, + "noPropertyAccessFromIndexSignature": false + } +}