diff --git a/.gitignore b/.gitignore index 6daa5a1..dbc34d3 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,11 @@ out dist *.tgz +# local data directories +input/* +output/* +db/* + # code coverage coverage *.lcov @@ -32,18 +37,6 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json # Finder (MacOS) folder config .DS_Store -*.xlsx - -results.db - -results.db-shm - -results.db-wal - -output/ - temp_output/ dist-server/ - -*.xls diff --git a/CLAUDE.md b/CLAUDE.md index a3f4b68..74290d7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,9 +19,15 @@ Default to using Bun instead of Node.js. - Prefer `Bun.file` over `node:fs`'s readFile/writeFile - Bun.$`ls` instead of execa. -## Testing - -Use `bun test` to run tests. +## Testing + +Use `bun test` to run tests. + +For this project, also use TypeScript's local compiler for type-checking: + +```sh +./node_modules/.bin/tsc --noEmit +``` ```ts#index.test.ts import { test, expect } from "bun:test"; @@ -103,4 +109,14 @@ Then, run index.ts bun --hot ./index.ts ``` -For more information, read the Bun API docs in `node_modules/bun-types/docs/**.mdx`. +For more information, read the Bun API docs in `node_modules/bun-types/docs/**.mdx`. + +## asin-check Project Notes + +- Keep the existing ASIN lead-list and category flows compatible with their current LLM-based FBA/FBM/SKIP analysis. +- The supplier UPC workflow is deterministic and runs through `bun run upc-file --input input/supplier.xlsx --out output/supplier_ranked.xlsx`. +- Keep supplier spreadsheets in `input/`, generated workbooks in `output/`, and SQLite files in `db/`; folder contents are ignored by git. +- Supplier UPC files should resolve UPC/EAN values through SP-API catalog lookup first, with Keepa UPC lookup only as fallback for no-match or request-failure cases. +- The supplier pipeline should not call LM Studio. It should enrich with Keepa + SP-API sellability/fees, score BUY/WATCH/SKIP numerically, write an Excel workbook, and persist rows to SQLite. +- Supplier workbook output should keep the `Ranked Leads`, `Skipped`, and `Summary` sheets. +- When changing UPC supplier behavior, cover SP-API UPC parsing, deterministic scoring, and workbook export with `bun test`. diff --git a/README.md b/README.md index 2cb1988..623a477 100644 --- a/README.md +++ b/README.md @@ -21,21 +21,21 @@ cp .env.example .env ## Usage ```bash -bun run src/index.ts [--out results.csv] +bun run src/index.ts input/ [--out output/results.xlsx] ``` Examples: ```bash -bun run src/index.ts leads.xlsx -bun run src/index.ts leads.csv --out results.xlsx +bun run src/index.ts input/leads.xlsx +bun run src/index.ts input/leads.csv --out output/results.xlsx ``` Large-file behavior: - If the input has more than 50 products, processing is done in chunks of 50. -- Each chunk is analyzed and written to a numbered output file, for example: `results_part_001.xlsx`, `results_part_002.xlsx`, ... -- If `--out` is omitted for large files, the base output name defaults to `_results.xlsx` and chunk files are still written with numbered suffixes. +- Each chunk is analyzed and written to a numbered output file under `output/`, for example: `output/results_part_001.xlsx`, `output/results_part_002.xlsx`, ... +- If `--out` is omitted for large files, the base output name defaults to `output/_results.xlsx` and chunk files are still written with numbered suffixes. Quick SP-API connectivity tests: @@ -130,27 +130,36 @@ curl -X POST "http://localhost:3000/api/upc/lookup" \ ## Large UPC File Analysis (XLS/XLSX) -For very large Excel files that contain UPC values, use the dedicated UPC-file process. It runs in batches: +For supplier price lists that contain UPC/EAN values and unit cost, use the +dedicated UPC-file process. It runs in batches and produces a deterministic +ranked sourcing workbook: 1. Reads UPC rows in batches (`.xlsx` uses streaming reader, `.xls` uses fallback row-window parsing). -2. Resolves UPCs to ASINs with Keepa. -3. Runs the same sellability + Keepa/SP-API enrichment + LLM verdict pipeline as lead analysis. -4. Persists output into existing `runs` + `results` tables, so it appears in current reporting APIs/UI. +2. Resolves UPCs to ASINs with SP-API catalog lookup first, then falls back to Keepa for no-match/request-failure cases. +3. Enriches resolved ASINs with Keepa demand/competition data and SP-API sellability + FBA fees. +4. Scores products with deterministic BUY/WATCH/SKIP logic; this path does not call LM Studio. +5. Writes a ranked Excel workbook and persists rows into the existing `runs` + `results` tables. CLI usage: ```bash -bun run upc-file --input huge-upcs.xlsx -bun run upc-file --input huge-upcs.xls --input-batch-size 500 --upc-lookup-batch-size 100 --max-rows 10000 +bun run upc-file --input input/huge-upcs.xlsx +bun run upc-file --input input/supplier.xlsx --out output/supplier_ranked.xlsx +bun run upc-file --input input/huge-upcs.xls --input-batch-size 500 --upc-lookup-batch-size 100 --max-rows 10000 ``` +Workbook output includes `Ranked Leads`, `Skipped`, and `Summary` sheets with +UPC, ASIN, cost, sale price, FBA fee, profit, margin, ROI, BSR, rank drops, +monthly sold, seller count, Amazon Buy Box share, sellability, score, verdict, +and reason columns. + API usage (when `bun run start:web` is running): ```bash curl -X POST "http://localhost:3000/api/process/upc-file" \ -H "content-type: application/json" \ -d '{ - "inputFile": "/absolute/path/to/huge-upcs.xlsx", + "inputFile": "/absolute/path/to/input/huge-upcs.xlsx", "inputBatchSize": 300, "upcLookupBatchSize": 100 }' @@ -222,7 +231,7 @@ Numeric parsing accepts plain numbers as well as formatted values like `$12.50`, ## Persistent Storage with SQLite -Results from each run are now stored in a SQLite database named `results.db` in the project root. The SQLite implementation details are handled in `src/database.ts`. This allows you to: +Results from each run are now stored in a SQLite database named `db/results.db` by default. The SQLite implementation details are handled in `src/database.ts`. This allows you to: - Revisit past analysis results. - Query and analyze historical data. diff --git a/bun.lock b/bun.lock index 1c4de3f..af03c41 100644 --- a/bun.lock +++ b/bun.lock @@ -16,9 +16,7 @@ "@types/bun": "latest", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", - }, - "peerDependencies": { - "typescript": "^5", + "typescript": "^6.0.3", }, }, }, @@ -269,7 +267,7 @@ "traverse": ["traverse@0.3.9", "", {}, "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ=="], - "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + "typescript": ["typescript@6.0.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw=="], "undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="], diff --git a/package.json b/package.json index ebad1d4..b3374ab 100644 --- a/package.json +++ b/package.json @@ -17,10 +17,8 @@ "devDependencies": { "@types/bun": "latest", "@types/react": "^19.2.14", - "@types/react-dom": "^19.2.3" - }, - "peerDependencies": { - "typescript": "^5" + "@types/react-dom": "^19.2.3", + "typescript": "^6.0.3" }, "dependencies": { "amazon-sp-api": "^1.2.1", diff --git a/src/bestsellers-by-category.ts b/src/bestsellers-by-category.ts index 64ab5a3..5a7b1d3 100644 --- a/src/bestsellers-by-category.ts +++ b/src/bestsellers-by-category.ts @@ -1192,7 +1192,7 @@ export async function main(): Promise { mkdirSync(args.outputDir, { recursive: true }); const DB_PATH = - process.env.RESULTS_DB_PATH || path.join(process.cwd(), "results.db"); + process.env.RESULTS_DB_PATH || path.join(process.cwd(), "db", "results.db"); initDb(DB_PATH); const db = getDb(DB_PATH); diff --git a/src/check_db.ts b/src/check_db.ts index a1bc87e..fa086c8 100644 --- a/src/check_db.ts +++ b/src/check_db.ts @@ -2,7 +2,8 @@ import { getDb } from "./database.ts"; import path from "node:path"; async function checkDb() { - const DB_PATH = path.join(process.cwd(), "temp_output", "analysis.sqlite"); + const DB_PATH = + process.env.RESULTS_DB_PATH || path.join(process.cwd(), "db", "results.db"); const db = getDb(DB_PATH); try { diff --git a/src/database.ts b/src/database.ts index fc4c3bb..526a220 100644 --- a/src/database.ts +++ b/src/database.ts @@ -1,10 +1,16 @@ import { Database } from "bun:sqlite"; +import { dirname } from "node:path"; +import { mkdirSync } from "node:fs"; export { Database } from "bun:sqlite"; let db: Database | null = null; export function getDb(dbPath: string): Database { if (!db) { + const dbDir = dirname(dbPath); + if (dbDir && dbDir !== ".") { + mkdirSync(dbDir, { recursive: true }); + } db = new Database(dbPath); db.run("PRAGMA journal_mode = WAL;"); // Enable WAL mode for better performance db.run("PRAGMA foreign_keys = ON;"); // Enforce foreign key constraints @@ -183,6 +189,15 @@ function ensureResultsTableColumns(database: Database): void { { name: "lead_date", type: "TEXT" }, { name: "amazon_is_seller", type: "INTEGER" }, { name: "amazon_buybox_share_pct_90d", type: "REAL" }, + { name: "upc", type: "TEXT" }, + { name: "supplier_score", type: "REAL" }, + { name: "supplier_profit", type: "REAL" }, + { name: "supplier_margin", type: "REAL" }, + { name: "supplier_roi", type: "REAL" }, + { name: "supplier_reason", type: "TEXT" }, + { name: "upc_lookup_status", type: "TEXT" }, + { name: "upc_lookup_reason", type: "TEXT" }, + { name: "candidate_asins", type: "TEXT" }, ]; for (const column of requiredColumns) { @@ -243,9 +258,18 @@ export function initDb(dbPath: string): void { promo_coupon_code TEXT, notes TEXT, lead_date TEXT, + upc TEXT, fba_fee REAL, fbm_fee REAL, referral_percent REAL, + supplier_score REAL, + supplier_profit REAL, + supplier_margin REAL, + supplier_roi REAL, + supplier_reason TEXT, + upc_lookup_status TEXT, + upc_lookup_reason TEXT, + candidate_asins TEXT, can_sell TEXT, sellability_status TEXT, sellability_reason TEXT, diff --git a/src/index.ts b/src/index.ts index 419df3a..b6b9efd 100644 --- a/src/index.ts +++ b/src/index.ts @@ -10,7 +10,7 @@ import { import path from "node:path"; import type { AnalysisResult } from "./types.ts"; -const DB_PATH = "./results.db"; +const DB_PATH = process.env.RESULTS_DB_PATH || path.join("db", "results.db"); const INPUT_BATCH_SIZE = 50; function parseSellabilityArg(args: string[]): SellabilityFilter { @@ -59,7 +59,7 @@ function resolveBaseOutputPath(inputFile: string, outputFile?: string): string { if (outputFile) return outputFile; const parsedInput = path.parse(inputFile); - return path.join(parsedInput.dir, `${parsedInput.name}_results.xlsx`); + return path.join("output", `${parsedInput.name}_results.xlsx`); } async function main() { diff --git a/src/keepa.ts b/src/keepa.ts index 19c7f42..a230393 100644 --- a/src/keepa.ts +++ b/src/keepa.ts @@ -423,14 +423,15 @@ function parseKeepaProduct(product: Record): KeepaData { salesRankDrops90, sellerCount: stats?.current?.[11] ?? null, amazonIsSeller, - amazonBuyboxSharePct90d, - buyBoxSeller: product.buyBoxSellerId ?? null, - buyBoxPrice: stats?.current?.[18] != null ? stats.current[18] / 100 : null, - monthlySold, - categoryTree: - product.categoryTree?.map((c: { name: string }) => c.name) ?? [], - }; -} + amazonBuyboxSharePct90d, + buyBoxSeller: product.buyBoxSellerId ?? null, + buyBoxPrice: stats?.current?.[18] != null ? stats.current[18] / 100 : null, + buyBoxAvg90: stats?.avg?.[18] != null ? stats.avg[18] / 100 : null, + monthlySold, + categoryTree: + product.categoryTree?.map((c: { name: string }) => c.name) ?? [], + }; +} function resolveAmazonIsSeller( product: Record, diff --git a/src/mid-range-sellers-by-category.test.ts b/src/mid-range-sellers-by-category.test.ts index 6e2e1ac..7c53d8b 100644 --- a/src/mid-range-sellers-by-category.test.ts +++ b/src/mid-range-sellers-by-category.test.ts @@ -5,7 +5,7 @@ import path from "node:path"; import { rmSync, mkdirSync } from "node:fs"; const fetchSellabilityBatchMock = mock(async (asins: string[]) => { - return new Map( + return new Map( asins.map((asin) => { if (asin === "B000000003") { return [ @@ -69,21 +69,7 @@ const DB_TEST_PATH = path.join( ); let db: Database; -let processCategory: ( - db: Database, - runId: number, - category: any, - perCategoryTop: number, - categoryCandidatePool: number, - minMonthlySold: number, - maxMonthlySold: number, - minPrice: number, - maxPrice: number, - minSellerCount: number, - maxSellerCount: number, - minAmazonBuyboxSharePct: number, - maxAmazonBuyboxSharePct: number, -) => Promise; +let processCategory: any; let insertCategoryRunSummary: ( db: Database, summary: any, diff --git a/src/mid-range-sellers-by-category.ts b/src/mid-range-sellers-by-category.ts index 90b5f61..6c4fd94 100644 --- a/src/mid-range-sellers-by-category.ts +++ b/src/mid-range-sellers-by-category.ts @@ -1920,7 +1920,8 @@ export async function main(): Promise { try { mkdirSync(args.outputDir, { recursive: true }); const DB_PATH = - process.env.RESULTS_DB_PATH || path.join(process.cwd(), "results.db"); + process.env.RESULTS_DB_PATH || + path.join(process.cwd(), "db", "results.db"); initDb(DB_PATH); const db = getDb(DB_PATH); diff --git a/src/server.ts b/src/server.ts index 29fbb3a..9fef320 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,4 +1,5 @@ import index from "./web/index.html"; +import path from "node:path"; import { getDb, initDb } from "./database.ts"; import { fetchKeepaDataBatch, @@ -52,7 +53,7 @@ type ProductListRecord = { fetched_at: string; }; -const DB_PATH = process.env.RESULTS_DB_PATH || "./results.db"; +const DB_PATH = process.env.RESULTS_DB_PATH || path.join("db", "results.db"); const DEFAULT_PAGE_SIZE = 25; const MAX_PAGE_SIZE = 200; const ASIN_PATTERN = /^[A-Z0-9]{10}$/; diff --git a/src/sp-api.test.ts b/src/sp-api.test.ts new file mode 100644 index 0000000..284d19d --- /dev/null +++ b/src/sp-api.test.ts @@ -0,0 +1,46 @@ +import { expect, test } from "bun:test"; +import { parseCatalogUpcLookupResponse } from "./sp-api.ts"; + +test("parseCatalogUpcLookupResponse resolves one ASIN", () => { + const detail = parseCatalogUpcLookupResponse("012345678901", { + items: [{ asin: "b000found1" }], + }); + + expect(detail.status).toBe("found"); + expect(detail.asin).toBe("B000FOUND1"); + expect(detail.candidateAsins).toEqual(["B000FOUND1"]); +}); + +test("parseCatalogUpcLookupResponse marks no match", () => { + const detail = parseCatalogUpcLookupResponse("012345678901", { + payload: { items: [] }, + }); + + expect(detail.status).toBe("not_found"); + expect(detail.asin).toBeNull(); +}); + +test("parseCatalogUpcLookupResponse marks multiple ASINs", () => { + const detail = parseCatalogUpcLookupResponse("012345678901", { + payload: { + items: [{ asin: "B000000001" }, { asin: "B000000002" }], + }, + }); + + expect(detail.status).toBe("multiple_asins"); + expect(detail.candidateAsins).toEqual(["B000000001", "B000000002"]); +}); + +test("parseCatalogUpcLookupResponse marks invalid UPCs", () => { + const detail = parseCatalogUpcLookupResponse("123", { items: [] }); + + expect(detail.status).toBe("invalid_upc"); +}); + +test("parseCatalogUpcLookupResponse marks malformed response as failed", () => { + const detail = parseCatalogUpcLookupResponse("012345678901", { + unexpected: true, + }); + + expect(detail.status).toBe("request_failed"); +}); diff --git a/src/sp-api.ts b/src/sp-api.ts index cd3d4a6..eace361 100644 --- a/src/sp-api.ts +++ b/src/sp-api.ts @@ -1,6 +1,11 @@ -import { SellingPartner } from "amazon-sp-api"; -import { config } from "./config.ts"; -import type { SpApiData, SellabilityInfo } from "./types.ts"; +import { SellingPartner } from "amazon-sp-api"; +import { config } from "./config.ts"; +import type { + KeepaUpcLookupStatus, + SpApiData, + SellabilityInfo, + UpcLookupDetail, +} from "./types.ts"; type RegionCode = "na" | "eu" | "fe"; @@ -118,10 +123,11 @@ function round2(value: number): number { return Math.round(value * 100) / 100; } -const SELLABILITY_CONCURRENCY = 5; -const PRICING_CONCURRENCY = 5; +const SELLABILITY_CONCURRENCY = 5; +const PRICING_CONCURRENCY = 5; +const UPC_PATTERN = /^\d{12,14}$/; -function parseSellabilityResponse(response: any): SellabilityInfo { +function parseSellabilityResponse(response: any): SellabilityInfo { const restrictions = Array.isArray(response?.restrictions) ? response.restrictions : Array.isArray(response?.payload?.restrictions) @@ -171,7 +177,102 @@ function parseSellabilityResponse(response: any): SellabilityInfo { [...reasonCodes, ...reasonMessages].join(" | ") || "Listing restrictions reported", }; -} +} + +function buildUpcLookupDetail( + upc: string, + status: KeepaUpcLookupStatus, + reason: string, + candidateAsins: string[] = [], +): UpcLookupDetail { + const asin = status === "found" ? candidateAsins[0] ?? null : null; + return { + requestedUpc: upc, + normalizedUpc: upc, + status, + asin, + candidateAsins, + keepaData: null, + reason, + }; +} + +function collectCatalogItems(response: any): any[] | null { + const candidates = [ + response?.items, + response?.payload?.items, + response?.payload, + response?.Items, + ]; + + for (const candidate of candidates) { + if (Array.isArray(candidate)) return candidate; + } + + return null; +} + +function extractCatalogAsin(item: any): string | null { + const raw = + item?.asin ?? + item?.ASIN ?? + item?.identifiers?.marketplaceASIN?.asin ?? + item?.Identifiers?.MarketplaceASIN?.ASIN; + if (typeof raw !== "string") return null; + const asin = raw.trim().toUpperCase(); + return asin ? asin : null; +} + +export function parseCatalogUpcLookupResponse( + upc: string, + response: unknown, +): UpcLookupDetail { + const normalizedUpc = upc.trim(); + if (!UPC_PATTERN.test(normalizedUpc)) { + return buildUpcLookupDetail( + normalizedUpc, + "invalid_upc", + "UPC must be 12, 13, or 14 digits", + ); + } + + const items = collectCatalogItems(response); + if (!items) { + return buildUpcLookupDetail( + normalizedUpc, + "request_failed", + "Unexpected catalog response shape", + ); + } + + const candidateAsins = Array.from( + new Set(items.map(extractCatalogAsin).filter((asin): asin is string => !!asin)), + ); + + if (candidateAsins.length === 0) { + return buildUpcLookupDetail( + normalizedUpc, + "not_found", + "No SP-API catalog item matched this UPC", + ); + } + + if (candidateAsins.length > 1) { + return buildUpcLookupDetail( + normalizedUpc, + "multiple_asins", + `UPC matched multiple ASINs (${candidateAsins.length})`, + candidateAsins, + ); + } + + return buildUpcLookupDetail( + normalizedUpc, + "found", + "Matched by SP-API catalog", + candidateAsins, + ); +} async function fetchSellabilityInternal( spClient: SellingPartner, @@ -502,9 +603,9 @@ export async function fetchSellability(asin: string): Promise { return fetchSellabilityInternal(spClient, asin); } -export async function fetchSellabilityBatch( - asins: string[], -): Promise> { +export async function fetchSellabilityBatch( + asins: string[], +): Promise> { const results = new Map(); const spClient = getSpClient(); @@ -540,14 +641,74 @@ export async function fetchSellabilityBatch( () => next(), ); await Promise.all(workers); - - return results; -} - -export async function fetchSpApiPricingAndFees( - asin: string, - sellability: SellabilityInfo, -): Promise { + + return results; +} + +export async function lookupSpApiUpc(upc: string): Promise { + const normalizedUpc = upc.trim(); + if (!UPC_PATTERN.test(normalizedUpc)) { + return buildUpcLookupDetail( + normalizedUpc, + "invalid_upc", + "UPC must be 12, 13, or 14 digits", + ); + } + + const spClient = getSpClient(); + if (!spClient) { + return buildUpcLookupDetail( + normalizedUpc, + "request_failed", + "SP-API credentials not configured", + ); + } + + try { + const response = await spClient.callAPI({ + operation: "searchCatalogItems", + endpoint: "catalogItems", + query: { + marketplaceIds: [config.spApiMarketplaceId], + identifiers: [normalizedUpc], + identifiersType: "UPC", + includedData: ["identifiers", "summaries"], + }, + }); + return parseCatalogUpcLookupResponse(normalizedUpc, response); + } catch (err) { + return buildUpcLookupDetail( + normalizedUpc, + "request_failed", + `SP-API catalog lookup failed: ${extractErrorMessage(err)}`, + ); + } +} + +export async function lookupSpApiUpcs( + upcs: string[], +): Promise> { + const results = new Map(); + const uniqueUpcs = Array.from(new Set(upcs.map((upc) => upc.trim()))); + + let completed = 0; + for (const upc of uniqueUpcs) { + const detail = await lookupSpApiUpc(upc); + results.set(upc, detail); + completed++; + if (completed % 10 === 0 || completed === uniqueUpcs.length) { + console.log(` [sp-api:catalog] ${completed}/${uniqueUpcs.length} UPCs checked`); + } + } + + return results; +} + +export async function fetchSpApiPricingAndFees( + asin: string, + sellability: SellabilityInfo, + priceOverride?: number | null, +): Promise { const fallback: SpApiData = { fbaFee: 5.0, fbmFee: 1.5, @@ -561,22 +722,28 @@ export async function fetchSpApiPricingAndFees( console.log(` [sp-api:fallback] ${asin} reason=missing_credentials`); return fallback; } - - try { - const pricing = (await spClient.callAPI({ - operation: "getItemOffers", - endpoint: "productPricing", - path: { Asin: asin }, - query: { - MarketplaceId: config.spApiMarketplaceId, - ItemCondition: "New", - }, - })) as any; - - const estimatedSalePrice = extractEstimatedSalePrice(pricing); - if (!Number.isFinite(estimatedSalePrice) || estimatedSalePrice <= 0) { - console.log( - ` [sp-api:fallback] ${asin} reason=no_valid_price_from_offers`, + + try { + let estimatedSalePrice = + typeof priceOverride === "number" && Number.isFinite(priceOverride) + ? priceOverride + : 0; + if (estimatedSalePrice <= 0) { + const pricing = (await spClient.callAPI({ + operation: "getItemOffers", + endpoint: "productPricing", + path: { Asin: asin }, + query: { + MarketplaceId: config.spApiMarketplaceId, + ItemCondition: "New", + }, + })) as any; + + estimatedSalePrice = extractEstimatedSalePrice(pricing); + } + if (!Number.isFinite(estimatedSalePrice) || estimatedSalePrice <= 0) { + console.log( + ` [sp-api:fallback] ${asin} reason=no_valid_price_from_offers`, ); return fallback; } diff --git a/src/supplier-export.test.ts b/src/supplier-export.test.ts new file mode 100644 index 0000000..c4d163f --- /dev/null +++ b/src/supplier-export.test.ts @@ -0,0 +1,134 @@ +import { afterEach, expect, test } from "bun:test"; +import path from "node:path"; +import { rmSync } from "node:fs"; +import ExcelJS from "exceljs"; +import { writeSupplierWorkbook } from "./supplier-export.ts"; +import type { SupplierAnalysisResult } from "./types.ts"; + +const OUTPUT_FILE = path.join("/private/tmp", "asin-check-supplier-export-test.xlsx"); + +afterEach(() => { + rmSync(OUTPUT_FILE, { force: true }); +}); + +function result(overrides: Partial = {}): SupplierAnalysisResult { + return { + upc: "012345678901", + rowNumber: 2, + record: { + asin: "B000000001", + name: "Test Product", + unitCost: 10, + brand: "Brand", + category: "Grocery", + }, + lookup: { + requestedUpc: "012345678901", + normalizedUpc: "012345678901", + status: "found", + asin: "B000000001", + candidateAsins: ["B000000001"], + keepaData: null, + }, + keepa: { + currentPrice: 30, + avgPrice90: 29, + minPrice90: 25, + maxPrice90: 35, + salesRank: 1000, + salesRankAvg90: 1200, + salesRankDrops30: 60, + salesRankDrops90: 180, + sellerCount: 4, + amazonIsSeller: false, + amazonBuyboxSharePct90d: 0, + buyBoxSeller: "SELLER", + buyBoxPrice: 30, + buyBoxAvg90: 29, + monthlySold: 300, + categoryTree: ["Grocery"], + }, + spApi: { + fbaFee: 5, + fbmFee: 3, + referralFeePercent: 15, + estimatedSalePrice: 30, + canSell: true, + sellabilityStatus: "available", + sellabilityReason: "ok", + }, + score: { + salePrice: 30, + fbaFee: 5, + profit: 15, + margin: 0.5, + roi: 1.5, + demandScore: 1, + competitionPenalty: 1, + score: 70, + verdict: "BUY", + reason: "Profitable with demand", + }, + fetchedAt: "2026-05-19T00:00:00.000Z", + ...overrides, + }; +} + +test("writeSupplierWorkbook writes ranked, skipped, and summary sheets", async () => { + await writeSupplierWorkbook( + OUTPUT_FILE, + [ + result(), + result({ + upc: "111111111111", + record: { asin: "111111111111", name: "Missing", unitCost: 0 }, + lookup: { + requestedUpc: "111111111111", + normalizedUpc: "111111111111", + status: "not_found", + asin: null, + candidateAsins: [], + keepaData: null, + reason: "No match", + }, + keepa: null, + spApi: null, + score: { + salePrice: null, + fbaFee: null, + profit: null, + margin: null, + roi: null, + demandScore: 0, + competitionPenalty: 1, + score: 0, + verdict: "SKIP", + reason: "No match", + }, + }), + ], + { + processedRows: 2, + resolvedRows: 1, + eligibleRows: 1, + verdictCounts: { BUY: 1, WATCH: 0, SKIP: 1 }, + unresolvedByStatus: { + found: 1, + invalid_upc: 0, + not_found: 1, + multiple_asins: 0, + request_failed: 0, + }, + }, + ); + + const workbook = new ExcelJS.Workbook(); + await workbook.xlsx.readFile(OUTPUT_FILE); + + expect(workbook.getWorksheet("Ranked Leads")).toBeDefined(); + expect(workbook.getWorksheet("Skipped")).toBeDefined(); + expect(workbook.getWorksheet("Summary")).toBeDefined(); + expect(workbook.getWorksheet("Ranked Leads")?.getCell("A1").value).toBe("UPC"); + expect(workbook.getWorksheet("Ranked Leads")?.getCell("B2").value).toBe("B000000001"); + expect(workbook.getWorksheet("Skipped")?.getCell("A2").value).toBe("111111111111"); +}); diff --git a/src/supplier-export.ts b/src/supplier-export.ts new file mode 100644 index 0000000..61be0d9 --- /dev/null +++ b/src/supplier-export.ts @@ -0,0 +1,158 @@ +import ExcelJS from "exceljs"; +import { dirname } from "node:path"; +import { mkdirSync } from "node:fs"; +import type { + KeepaUpcLookupStatus, + SupplierAnalysisResult, + SupplierVerdict, +} from "./types.ts"; + +export type SupplierExportSummary = { + processedRows: number; + resolvedRows: number; + eligibleRows: number; + verdictCounts: Record; + unresolvedByStatus: Record; +}; + +function pct(value: number | null): number | "" { + return value == null ? "" : Math.round(value * 10_000) / 100; +} + +function rowForResult(result: SupplierAnalysisResult) { + const category = + result.record.category ?? result.keepa?.categoryTree?.join(" > ") ?? ""; + const canSell = + result.spApi?.canSell == null ? "" : result.spApi.canSell ? "yes" : "no"; + + return { + UPC: result.upc, + ASIN: result.lookup.asin ?? "", + Name: result.record.name, + Brand: result.record.brand ?? "", + Category: category, + "Unit Cost": result.record.unitCost || "", + "Sale Price": result.score.salePrice ?? "", + "FBA Fee": result.score.fbaFee ?? "", + Profit: result.score.profit ?? "", + "Margin %": pct(result.score.margin), + "ROI %": pct(result.score.roi), + "BSR Current": result.keepa?.salesRank ?? "", + "BSR 90d": result.keepa?.salesRankAvg90 ?? "", + "Rank Drops 30d": result.keepa?.salesRankDrops30 ?? "", + "Rank Drops 90d": result.keepa?.salesRankDrops90 ?? "", + "Monthly Sold": result.keepa?.monthlySold ?? "", + "Seller Count": result.keepa?.sellerCount ?? "", + "Amazon Share 90d %": result.keepa?.amazonBuyboxSharePct90d ?? "", + "Can Sell": canSell, + Sellability: result.spApi?.sellabilityStatus ?? "", + Score: result.score.score, + Verdict: result.score.verdict, + Reason: result.score.reason, + "Lookup Status": result.lookup.status, + "Candidate ASINs": result.lookup.candidateAsins.join(","), + "Lookup Reason": result.lookup.reason ?? "", + }; +} + +function addRowsSheet( + workbook: ExcelJS.Workbook, + name: string, + rows: ReturnType[], +): void { + const sheet = workbook.addWorksheet(name); + const headers = rows[0] ? Object.keys(rows[0]) : Object.keys(rowForResult({ + upc: "", + record: { asin: "", name: "", unitCost: 0 }, + lookup: { + requestedUpc: "", + normalizedUpc: "", + status: "not_found", + asin: null, + candidateAsins: [], + keepaData: null, + }, + keepa: null, + spApi: null, + score: { + salePrice: null, + fbaFee: null, + profit: null, + margin: null, + roi: null, + demandScore: 0, + competitionPenalty: 1, + score: 0, + verdict: "SKIP", + reason: "", + }, + fetchedAt: "", + })); + + sheet.columns = headers.map((header) => ({ + header, + key: header, + width: Math.min(Math.max(header.length + 4, 12), 28), + })); + sheet.addRows(rows); + sheet.views = [{ state: "frozen", ySplit: 1 }]; + sheet.autoFilter = { + from: { row: 1, column: 1 }, + to: { row: 1, column: headers.length }, + }; + sheet.getRow(1).font = { bold: true }; +} + +function addSummarySheet( + workbook: ExcelJS.Workbook, + summary: SupplierExportSummary, +): void { + const sheet = workbook.addWorksheet("Summary"); + sheet.columns = [ + { header: "Metric", key: "Metric", width: 28 }, + { header: "Value", key: "Value", width: 18 }, + ]; + + sheet.addRows([ + { Metric: "Processed Rows", Value: summary.processedRows }, + { Metric: "Resolved Rows", Value: summary.resolvedRows }, + { Metric: "Eligible Rows", Value: summary.eligibleRows }, + { Metric: "BUY", Value: summary.verdictCounts.BUY }, + { Metric: "WATCH", Value: summary.verdictCounts.WATCH }, + { Metric: "SKIP", Value: summary.verdictCounts.SKIP }, + { Metric: "Unresolved invalid_upc", Value: summary.unresolvedByStatus.invalid_upc }, + { Metric: "Unresolved not_found", Value: summary.unresolvedByStatus.not_found }, + { Metric: "Unresolved multiple_asins", Value: summary.unresolvedByStatus.multiple_asins }, + { Metric: "Unresolved request_failed", Value: summary.unresolvedByStatus.request_failed }, + ]); + sheet.getRow(1).font = { bold: true }; +} + +export async function writeSupplierWorkbook( + outputFile: string, + results: SupplierAnalysisResult[], + summary: SupplierExportSummary, +): Promise { + const outputDir = dirname(outputFile); + if (outputDir && outputDir !== ".") { + mkdirSync(outputDir, { recursive: true }); + } + + const workbook = new ExcelJS.Workbook(); + workbook.creator = "asin-check"; + workbook.created = new Date(); + + const ranked = results + .filter((result) => result.score.verdict !== "SKIP") + .sort((a, b) => b.score.score - a.score.score) + .map(rowForResult); + const skipped = results + .filter((result) => result.score.verdict === "SKIP") + .map(rowForResult); + + addRowsSheet(workbook, "Ranked Leads", ranked); + addRowsSheet(workbook, "Skipped", skipped); + addSummarySheet(workbook, summary); + + await workbook.xlsx.writeFile(outputFile); +} diff --git a/src/supplier-scoring.test.ts b/src/supplier-scoring.test.ts new file mode 100644 index 0000000..297ae70 --- /dev/null +++ b/src/supplier-scoring.test.ts @@ -0,0 +1,97 @@ +import { expect, test } from "bun:test"; +import { scoreSupplierProduct } from "./supplier-scoring.ts"; +import type { KeepaData, ProductRecord, SpApiData } from "./types.ts"; + +function record(overrides: Partial = {}): ProductRecord { + return { + asin: "B000000001", + name: "Test Product", + unitCost: 10, + ...overrides, + }; +} + +function keepa(overrides: Partial = {}): KeepaData { + return { + currentPrice: 30, + avgPrice90: 29, + minPrice90: 25, + maxPrice90: 35, + salesRank: 8_000, + salesRankAvg90: 10_000, + salesRankDrops30: 80, + salesRankDrops90: 220, + sellerCount: 4, + amazonIsSeller: false, + amazonBuyboxSharePct90d: 0, + buyBoxSeller: "SELLER", + buyBoxPrice: 30, + buyBoxAvg90: 29, + monthlySold: 350, + categoryTree: ["Grocery"], + ...overrides, + }; +} + +function spApi(overrides: Partial = {}): SpApiData { + return { + fbaFee: 5, + fbmFee: 3, + referralFeePercent: 15, + estimatedSalePrice: 30, + canSell: true, + sellabilityStatus: "available", + sellabilityReason: "ok", + ...overrides, + }; +} + +test("profitable high-demand product ranks above competitive product", () => { + const strong = scoreSupplierProduct(record(), keepa(), spApi()); + const competitive = scoreSupplierProduct( + record(), + keepa({ + sellerCount: 35, + amazonIsSeller: true, + amazonBuyboxSharePct90d: 90, + }), + spApi(), + ); + + expect(strong.verdict).toBe("BUY"); + expect(strong.score).toBeGreaterThan(competitive.score); +}); + +test("missing cost skips", () => { + const score = scoreSupplierProduct(record({ unitCost: 0 }), keepa(), spApi()); + + expect(score.verdict).toBe("SKIP"); + expect(score.reason).toContain("unit cost"); +}); + +test("restricted ASIN skips", () => { + const score = scoreSupplierProduct( + record(), + keepa(), + spApi({ canSell: false, sellabilityStatus: "restricted" }), + ); + + expect(score.verdict).toBe("SKIP"); + expect(score.reason).toContain("restricted"); +}); + +test("missing price skips", () => { + const score = scoreSupplierProduct( + record(), + keepa({ + currentPrice: null, + avgPrice90: null, + buyBoxPrice: null, + buyBoxAvg90: null, + }), + spApi({ estimatedSalePrice: 0 }), + ); + + expect(score.verdict).toBe("SKIP"); + expect(score.reason).toContain("price"); +}); diff --git a/src/supplier-scoring.ts b/src/supplier-scoring.ts new file mode 100644 index 0000000..01275e3 --- /dev/null +++ b/src/supplier-scoring.ts @@ -0,0 +1,224 @@ +import type { + KeepaData, + ProductRecord, + SpApiData, + SupplierScore, +} from "./types.ts"; + +function round2(value: number): number { + return Math.round(value * 100) / 100; +} + +function clamp(value: number, min: number, max: number): number { + return Math.max(min, Math.min(max, value)); +} + +export function resolveSupplierSalePrice( + keepa: KeepaData | null, + spApi: SpApiData | null, +): number | null { + const candidates = [ + keepa?.buyBoxPrice, + keepa?.buyBoxAvg90, + keepa?.currentPrice, + keepa?.avgPrice90, + spApi?.estimatedSalePrice, + ]; + + for (const candidate of candidates) { + if (typeof candidate === "number" && Number.isFinite(candidate) && candidate > 0) { + return round2(candidate); + } + } + + return null; +} + +export function computeDemandScore(keepa: KeepaData | null): number { + if (!keepa) return 0; + + const monthlySold = keepa.monthlySold ?? 0; + const rankDrops30 = keepa.salesRankDrops30 ?? 0; + const rankDrops90 = keepa.salesRankDrops90 ?? 0; + const velocityScore = clamp( + Math.max(monthlySold / 300, rankDrops30 / 60, rankDrops90 / 180), + 0, + 1, + ); + + const rankCandidates = [keepa.salesRank, keepa.salesRankAvg90].filter( + (value): value is number => + typeof value === "number" && Number.isFinite(value) && value > 0, + ); + const bestRank = rankCandidates.length > 0 ? Math.min(...rankCandidates) : null; + const rankScore = + bestRank == null + ? 0 + : bestRank <= 10_000 + ? 1 + : bestRank <= 50_000 + ? 0.8 + : bestRank <= 100_000 + ? 0.55 + : bestRank <= 250_000 + ? 0.3 + : 0.1; + + return round2(clamp(velocityScore * 0.65 + rankScore * 0.35, 0, 1)); +} + +export function computeCompetitionPenalty(keepa: KeepaData | null): number { + if (!keepa) return 1; + + const sellerCount = keepa.sellerCount ?? 0; + const sellerPenalty = + sellerCount <= 3 + ? 0.85 + : sellerCount <= 8 + ? 1 + : sellerCount <= 15 + ? 1.25 + : sellerCount <= 30 + ? 1.6 + : 2; + + const amazonShare = keepa.amazonBuyboxSharePct90d ?? 0; + const amazonPenalty = + keepa.amazonIsSeller === true + ? 1.35 + : amazonShare >= 75 + ? 1.45 + : amazonShare >= 35 + ? 1.2 + : 1; + + return round2(clamp(sellerPenalty * amazonPenalty, 0.75, 2.5)); +} + +export function scoreSupplierProduct( + record: ProductRecord, + keepa: KeepaData | null, + spApi: SpApiData | null, +): SupplierScore { + const salePrice = resolveSupplierSalePrice(keepa, spApi); + const fbaFee = spApi?.fbaFee ?? null; + const demandScore = computeDemandScore(keepa); + const competitionPenalty = computeCompetitionPenalty(keepa); + + if (spApi && spApi.sellabilityStatus !== "available") { + return { + salePrice, + fbaFee, + profit: null, + margin: null, + roi: null, + demandScore, + competitionPenalty, + score: 0, + verdict: "SKIP", + reason: `Not sellable: ${spApi.sellabilityStatus}`, + }; + } + + if (!salePrice) { + return { + salePrice, + fbaFee, + profit: null, + margin: null, + roi: null, + demandScore, + competitionPenalty, + score: 0, + verdict: "SKIP", + reason: "Missing sale price", + }; + } + + if (!record.unitCost || record.unitCost <= 0) { + return { + salePrice, + fbaFee, + profit: null, + margin: null, + roi: null, + demandScore, + competitionPenalty, + score: 0, + verdict: "SKIP", + reason: "Missing or invalid unit cost", + }; + } + + if (fbaFee == null || fbaFee < 0) { + return { + salePrice, + fbaFee, + profit: null, + margin: null, + roi: null, + demandScore, + competitionPenalty, + score: 0, + verdict: "SKIP", + reason: "Missing FBA fee", + }; + } + + const profit = round2(salePrice - record.unitCost - fbaFee); + const margin = round2(profit / salePrice); + const roi = round2(profit / record.unitCost); + + if (profit <= 0 || margin <= 0 || roi <= 0) { + return { + salePrice, + fbaFee, + profit, + margin, + roi, + demandScore, + competitionPenalty, + score: 0, + verdict: "SKIP", + reason: "Non-positive profit", + }; + } + + if (demandScore < 0.15) { + return { + salePrice, + fbaFee, + profit, + margin, + roi, + demandScore, + competitionPenalty, + score: 0, + verdict: "SKIP", + reason: "Weak demand signals", + }; + } + + const rawScore = + ((margin * 0.55 + clamp(roi, 0, 2) * 0.45) * demandScore * 100) / + competitionPenalty; + const score = round2(clamp(rawScore, 0, 100)); + const verdict = score >= 18 && margin >= 0.18 && roi >= 0.3 ? "BUY" : "WATCH"; + const reason = + verdict === "BUY" + ? "Profitable with demand" + : "Viable but needs review"; + + return { + salePrice, + fbaFee, + profit, + margin, + roi, + demandScore, + competitionPenalty, + score, + verdict, + reason, + }; +} diff --git a/src/top-monthly-sold-by-category.test.ts b/src/top-monthly-sold-by-category.test.ts index 98e3d51..fce48c5 100644 --- a/src/top-monthly-sold-by-category.test.ts +++ b/src/top-monthly-sold-by-category.test.ts @@ -5,7 +5,7 @@ import path from "node:path"; import { rmSync, mkdirSync } from "node:fs"; const fetchSellabilityBatchMock = mock(async (asins: string[]) => { - return new Map( + return new Map( asins.map((asin) => { if (asin === "B000000003") { return [ diff --git a/src/top-monthly-sold-by-category.ts b/src/top-monthly-sold-by-category.ts index 01f0a41..0645119 100644 --- a/src/top-monthly-sold-by-category.ts +++ b/src/top-monthly-sold-by-category.ts @@ -1286,7 +1286,7 @@ export async function main(): Promise { mkdirSync(args.outputDir, { recursive: true }); const DB_PATH = - process.env.RESULTS_DB_PATH || path.join(process.cwd(), "results.db"); + process.env.RESULTS_DB_PATH || path.join(process.cwd(), "db", "results.db"); initDb(DB_PATH); const db = getDb(DB_PATH); diff --git a/src/types.ts b/src/types.ts index 8be360c..bc42ef4 100644 --- a/src/types.ts +++ b/src/types.ts @@ -26,23 +26,24 @@ export interface ProductRecord { [key: string]: unknown; } -export interface KeepaData { - currentPrice: number | null; - avgPrice90: number | null; - minPrice90: number | null; - maxPrice90: number | null; +export interface KeepaData { + currentPrice: number | null; + avgPrice90: number | null; + minPrice90: number | null; + maxPrice90: number | null; salesRank: number | null; salesRankAvg90: number | null; salesRankDrops30: number | null; salesRankDrops90: number | null; sellerCount: number | null; amazonIsSeller: boolean | null; - amazonBuyboxSharePct90d: number | null; - buyBoxSeller: string | null; - buyBoxPrice: number | null; - monthlySold: number | null; - categoryTree: string[]; -} + amazonBuyboxSharePct90d: number | null; + buyBoxSeller: string | null; + buyBoxPrice: number | null; + buyBoxAvg90?: number | null; + monthlySold: number | null; + categoryTree: string[]; +} export type KeepaUpcLookupStatus = | "found" @@ -51,15 +52,17 @@ export type KeepaUpcLookupStatus = | "multiple_asins" | "request_failed"; -export interface KeepaUpcLookupDetail { - requestedUpc: string; - normalizedUpc: string; - status: KeepaUpcLookupStatus; +export interface KeepaUpcLookupDetail { + requestedUpc: string; + normalizedUpc: string; + status: KeepaUpcLookupStatus; asin: string | null; candidateAsins: string[]; keepaData: KeepaData | null; - reason?: string; -} + reason?: string; +} + +export type UpcLookupDetail = KeepaUpcLookupDetail; export type SellabilityInfo = { canSell: boolean | null; @@ -88,10 +91,36 @@ export interface LlmVerdict { reasoning: string; } -export interface AnalysisResult { - product: EnrichedProduct; - verdict: LlmVerdict; -} +export interface AnalysisResult { + product: EnrichedProduct; + verdict: LlmVerdict; +} + +export type SupplierVerdict = "BUY" | "WATCH" | "SKIP"; + +export interface SupplierScore { + salePrice: number | null; + fbaFee: number | null; + profit: number | null; + margin: number | null; + roi: number | null; + demandScore: number; + competitionPenalty: number; + score: number; + verdict: SupplierVerdict; + reason: string; +} + +export interface SupplierAnalysisResult { + upc: string; + rowNumber?: number; + record: ProductRecord; + lookup: UpcLookupDetail; + keepa: KeepaData | null; + spApi: SpApiData | null; + score: SupplierScore; + fetchedAt: string; +} export interface CategoryRunSummaryDb { categoryId: number; diff --git a/src/upc-file-analysis.ts b/src/upc-file-analysis.ts index fe84c71..6d6903e 100644 --- a/src/upc-file-analysis.ts +++ b/src/upc-file-analysis.ts @@ -1,28 +1,40 @@ import path from "node:path"; -import { lookupKeepaUpcs } from "./keepa.ts"; -import { processProductChunk, chunkArray } from "./analysis-pipeline.ts"; +import { fetchKeepaDataBatch, lookupKeepaUpcs } from "./keepa.ts"; +import { + fetchSellabilityBatch, + fetchSpApiPricingAndFees, + lookupSpApiUpcs, +} from "./sp-api.ts"; import { processUpcFileInBatches, type UpcInputRow, } from "./upc-file-reader.ts"; import { - appendResultsToRun, - printResults, + appendSupplierResultsToRun, refreshRunCountsInDb, startRunInDb, type RunCounts, } from "./writer.ts"; import { initDb, closeDb } from "./database.ts"; import { connectCache, disconnectCache } from "./cache.ts"; +import { scoreSupplierProduct, resolveSupplierSalePrice } from "./supplier-scoring.ts"; +import { + writeSupplierWorkbook, + type SupplierExportSummary, +} from "./supplier-export.ts"; import type { KeepaUpcLookupDetail, KeepaUpcLookupStatus, ProductRecord, + SupplierAnalysisResult, + SupplierScore, + UpcLookupDetail, } from "./types.ts"; -const DB_PATH = process.env.RESULTS_DB_PATH || "./results.db"; +const DB_PATH = process.env.RESULTS_DB_PATH || path.join("db", "results.db"); const DEFAULT_INPUT_BATCH_SIZE = 200; const DEFAULT_UPC_LOOKUP_BATCH_SIZE = 100; +const DEFAULT_PRICING_CONCURRENCY = 5; export type UpcFileAnalysisOptions = { inputFile: string; @@ -55,7 +67,7 @@ export type UpcFileAnalysisSummary = { function printUsage(): void { console.log("Usage:"); console.log( - " bun run src/upc-file-analysis.ts --input [--out output.xlsx] [--input-batch-size 200] [--upc-lookup-batch-size 100] [--max-rows 1000]", + " bun run src/upc-file-analysis.ts --input input/ [--out output/results.xlsx] [--input-batch-size 200] [--upc-lookup-batch-size 100] [--max-rows 1000]", ); } @@ -146,7 +158,7 @@ function parseArgs(argv: string[]): UpcFileAnalysisOptions { function resolveDefaultOutputPath(inputFile: string): string { const parsedInput = path.parse(inputFile); - return path.join(parsedInput.dir, `${parsedInput.name}_upc_results.xlsx`); + return path.join("output", `${parsedInput.name}_upc_results.xlsx`); } function createStatusCounter(): Record { @@ -159,15 +171,38 @@ function createStatusCounter(): Record { }; } +function chunkArray(items: T[], chunkSize: number): T[][] { + const chunks: T[][] = []; + for (let i = 0; i < items.length; i += chunkSize) { + chunks.push(items.slice(i, i + chunkSize)); + } + return chunks; +} + +function skippedScore(reason: string): SupplierScore { + return { + salePrice: null, + fbaFee: null, + profit: null, + margin: null, + roi: null, + demandScore: 0, + competitionPenalty: 1, + score: 0, + verdict: "SKIP", + reason, + }; +} + async function lookupUpcsWithChunking( rows: UpcInputRow[], lookupBatchSize: number, runCache: Map, -): Promise> { +): Promise> { const uniqueUpcs = Array.from(new Set(rows.map((row) => row.upc))); const missingUpcs = uniqueUpcs.filter((upc) => !runCache.has(upc)); const chunks = chunkArray(missingUpcs, lookupBatchSize); - const details = new Map(); + const details = new Map(); const cacheHits = uniqueUpcs.length - missingUpcs.length; if (cacheHits > 0) { @@ -187,10 +222,31 @@ async function lookupUpcsWithChunking( for (let i = 0; i < chunks.length; i++) { const chunk = chunks[i]!; console.log( - ` Keepa UPC lookup chunk ${i + 1}/${chunks.length} (${chunk.length} UPCs)...`, + ` SP-API UPC lookup chunk ${i + 1}/${chunks.length} (${chunk.length} UPCs)...`, ); - const chunkDetails = await lookupKeepaUpcs(chunk); + const spDetails = await lookupSpApiUpcs(chunk); + const fallbackUpcs = Array.from(spDetails.values()) + .filter( + (detail) => + detail.status === "not_found" || detail.status === "request_failed", + ) + .map((detail) => detail.normalizedUpc); + const fallbackDetails = + fallbackUpcs.length > 0 ? await lookupKeepaUpcs(fallbackUpcs) : new Map(); + + const chunkDetails = new Map(); + for (const upc of chunk) { + const spDetail = spDetails.get(upc); + const fallbackDetail = fallbackDetails.get(upc); + chunkDetails.set( + upc, + fallbackDetail && fallbackDetail.status !== "request_failed" + ? fallbackDetail + : spDetail!, + ); + } + for (const [upc, detail] of chunkDetails.entries()) { runCache.set(upc, detail); } @@ -208,7 +264,7 @@ async function lookupUpcsWithChunking( function toProductRecord( row: UpcInputRow, - detail: KeepaUpcLookupDetail, + detail: UpcLookupDetail, ): ProductRecord { const keepaCategory = detail.keepaData?.categoryTree?.[0]; @@ -221,6 +277,65 @@ function toProductRecord( }; } +async function fetchFeesForProducts( + products: ProductRecord[], + keepaResults: Map>, + sellabilityMap: Awaited>, +): Promise>> { + const spApiResults = new Map>(); + const queue = [...products]; + let completed = 0; + + async function next(): Promise { + while (queue.length > 0) { + const product = queue.shift(); + if (!product) return; + const sellability = + sellabilityMap.get(product.asin) ?? { + canSell: null, + sellabilityStatus: "unknown" as const, + sellabilityReason: "Sellability check returned no result", + }; + const price = resolveSupplierSalePrice( + keepaResults.get(product.asin) ?? null, + null, + ); + const spApi = await fetchSpApiPricingAndFees(product.asin, sellability, price); + spApiResults.set(product.asin, spApi); + completed++; + if (completed % 10 === 0 || completed === products.length) { + console.log(` [fees] ${completed}/${products.length} fetched`); + } + } + } + + const workers = Array.from( + { length: Math.min(DEFAULT_PRICING_CONCURRENCY, products.length || 1) }, + () => next(), + ); + await Promise.all(workers); + return spApiResults; +} + +function summarizeSupplierResults( + results: SupplierAnalysisResult[], + unresolvedByStatus: Record, +): SupplierExportSummary { + return { + processedRows: results.length, + resolvedRows: results.filter((result) => result.lookup.status === "found").length, + eligibleRows: results.filter( + (result) => result.spApi?.sellabilityStatus === "available", + ).length, + verdictCounts: { + BUY: results.filter((result) => result.score.verdict === "BUY").length, + WATCH: results.filter((result) => result.score.verdict === "WATCH").length, + SKIP: results.filter((result) => result.score.verdict === "SKIP").length, + }, + unresolvedByStatus, + }; +} + export async function runUpcFileAnalysis( options: UpcFileAnalysisOptions, ): Promise { @@ -245,7 +360,7 @@ export async function runUpcFileAnalysis( } const unresolvedByStatus = createStatusCounter(); - const printableSample = []; + const allResults: SupplierAnalysisResult[] = []; const upcLookupCache = new Map(); let processedRows = 0; let matchedRows = 0; @@ -267,7 +382,11 @@ export async function runUpcFileAnalysis( upcLookupCache, ); - const matchedProducts: ProductRecord[] = []; + const matchedEntries: Array<{ + row: UpcInputRow; + detail: UpcLookupDetail; + product: ProductRecord; + }> = []; for (const row of rows) { const detail = detailMap.get(row.upc); if (!detail) { @@ -279,25 +398,91 @@ export async function runUpcFileAnalysis( if (detail.status === "found" && detail.asin) { matchedRows += 1; - matchedProducts.push(toProductRecord(row, detail)); + matchedEntries.push({ + row, + detail, + product: toProductRecord(row, detail), + }); } } + const matchedProducts = matchedEntries.map((entry) => entry.product); console.log( `Batch ${batchNumber}: ${matchedProducts.length}/${rows.length} rows resolved to single ASINs`, ); - if (matchedProducts.length === 0) { - return; + const batchResults: SupplierAnalysisResult[] = []; + for (const row of rows) { + const detail = detailMap.get(row.upc); + if (!detail || detail.status === "found") continue; + + batchResults.push({ + upc: row.upc, + rowNumber: row.rowNumber, + record: { + asin: detail?.asin ?? row.upc, + name: row.name ?? row.upc, + unitCost: row.unitCost ?? 0, + brand: row.brand, + category: row.category, + }, + lookup: + detail ?? + ({ + requestedUpc: row.upc, + normalizedUpc: row.upc, + status: "request_failed", + asin: null, + candidateAsins: [], + keepaData: null, + reason: "UPC lookup returned no result", + } satisfies UpcLookupDetail), + keepa: null, + spApi: null, + score: skippedScore(detail?.reason ?? "UPC unresolved"), + fetchedAt: new Date().toISOString(), + }); } - const analyzed = await processProductChunk(matchedProducts); - appendResultsToRun(dbPath, runId, analyzed); + if (matchedProducts.length > 0) { + console.log(`Fetching ${matchedProducts.length} ASINs from Keepa...`); + const keepaResults = await fetchKeepaDataBatch( + matchedProducts.map((product) => product.asin), + ); - if (printableSample.length < 200) { - const remaining = 200 - printableSample.length; - printableSample.push(...analyzed.slice(0, remaining)); + console.log(`Checking sellability for ${matchedProducts.length} ASINs...`); + const sellabilityMap = await fetchSellabilityBatch( + matchedProducts.map((product) => product.asin), + ); + + console.log(`Fetching fees for ${matchedProducts.length} ASINs...`); + const spApiResults = await fetchFeesForProducts( + matchedProducts, + keepaResults, + sellabilityMap, + ); + + for (const entry of matchedEntries) { + const keepa = + keepaResults.get(entry.product.asin) ?? + entry.detail.keepaData ?? + null; + const spApi = spApiResults.get(entry.product.asin) ?? null; + batchResults.push({ + upc: entry.detail.normalizedUpc, + rowNumber: entry.row.rowNumber, + record: entry.product, + lookup: entry.detail, + keepa, + spApi, + score: scoreSupplierProduct(entry.product, keepa, spApi), + fetchedAt: new Date().toISOString(), + }); + } } + + appendSupplierResultsToRun(dbPath, runId, batchResults); + allResults.push(...batchResults); }, { batchSize: inputBatchSize, @@ -307,17 +492,34 @@ export async function runUpcFileAnalysis( const runCounts = refreshRunCountsInDb(dbPath, runId); - if (printableSample.length > 0) { - printResults(printableSample); - if (runCounts.totalProducts > printableSample.length) { - console.log( - `Printed ${printableSample.length} sampled results out of ${runCounts.totalProducts} analyzed products.`, - ); - } + const exportSummary = summarizeSupplierResults(allResults, unresolvedByStatus); + await writeSupplierWorkbook(outputFile, allResults, exportSummary); + + if (allResults.length > 0) { + const ranked = allResults + .filter((result) => result.score.verdict !== "SKIP") + .sort((a, b) => b.score.score - a.score.score) + .slice(0, 25) + .map((result) => ({ + UPC: result.upc, + ASIN: result.lookup.asin ?? "", + Name: result.record.name.slice(0, 40), + Cost: result.record.unitCost, + Price: result.score.salePrice ?? "", + Profit: result.score.profit ?? "", + ROI: result.score.roi == null ? "" : `${Math.round(result.score.roi * 100)}%`, + Score: result.score.score, + Verdict: result.score.verdict, + Reason: result.score.reason, + })); + console.log("\n=== Top Supplier Leads ===\n"); + console.table(ranked); } else { - console.log("No products were eligible for analysis after UPC mapping."); + console.log("No supplier rows were analyzed."); } + console.log(`Ranked workbook written: ${outputFile}`); + return { runId, dbPath, diff --git a/src/upc-file-reader.ts b/src/upc-file-reader.ts index bca05a5..d2cf93c 100644 --- a/src/upc-file-reader.ts +++ b/src/upc-file-reader.ts @@ -123,6 +123,9 @@ async function processXlsxStreaming( } seenRows += 1; + if (!columns) { + throw new Error("UPC reader columns were not initialized."); + } const parsed = parseUpcInputRow(values, columns, row.number); if (!parsed) { skippedMissingUpc += 1; diff --git a/src/writer.ts b/src/writer.ts index 21d22e2..477d918 100644 --- a/src/writer.ts +++ b/src/writer.ts @@ -1,5 +1,5 @@ import { getDb } from "./database.ts"; -import type { AnalysisResult } from "./types.ts"; +import type { AnalysisResult, SupplierAnalysisResult } from "./types.ts"; export type RunCounts = { totalProducts: number; @@ -222,6 +222,83 @@ export function appendResultsToRun( })(); } +export function appendSupplierResultsToRun( + dbPath: string, + runId: number, + results: SupplierAnalysisResult[], +): void { + if (results.length === 0) { + return; + } + + const database = getDb(dbPath); + const insertResult = database.prepare( + `INSERT INTO results ( + run_id, asin, product_name, brand, category, unit_cost, current_price, + avg_price_90d, sales_rank, rank_avg_90d, sellers, + amazon_is_seller, amazon_buybox_share_pct_90d, monthly_sold, + rank_drops_30d, rank_drops_90d, upc, fba_fee, fbm_fee, + referral_percent, supplier_score, supplier_profit, supplier_margin, + supplier_roi, supplier_reason, upc_lookup_status, upc_lookup_reason, + candidate_asins, can_sell, sellability_status, sellability_reason, + verdict, confidence, reasoning, fetched_at + ) VALUES ( + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + )`, + ); + + database.transaction(() => { + for (const result of results) { + const keepa = result.keepa; + const spApi = result.spApi; + const asin = result.lookup.asin ?? result.record.asin ?? result.upc; + const category = + result.record.category ?? keepa?.categoryTree?.join(" > ") ?? null; + const canSell = + spApi?.canSell == null ? null : spApi.canSell ? "yes" : "no"; + + insertResult.run( + runId, + asin, + result.record.name, + result.record.brand ?? null, + category, + result.record.unitCost || null, + result.score.salePrice, + keepa?.avgPrice90 ?? null, + keepa?.salesRank ?? null, + keepa?.salesRankAvg90 ?? null, + keepa?.sellerCount ?? null, + keepa?.amazonIsSeller == null ? null : keepa.amazonIsSeller ? 1 : 0, + keepa?.amazonBuyboxSharePct90d ?? null, + keepa?.monthlySold ?? null, + keepa?.salesRankDrops30 ?? null, + keepa?.salesRankDrops90 ?? null, + result.upc, + result.score.fbaFee, + spApi?.fbmFee ?? null, + spApi?.referralFeePercent ?? null, + result.score.score, + result.score.profit, + result.score.margin, + result.score.roi, + result.score.reason, + result.lookup.status, + result.lookup.reason ?? null, + result.lookup.candidateAsins.join(","), + canSell, + spApi?.sellabilityStatus ?? null, + spApi?.sellabilityReason ?? null, + result.score.verdict, + Math.round(result.score.score), + result.score.reason, + result.fetchedAt, + ); + } + })(); +} + export function refreshRunCountsInDb(dbPath: string, runId: number): RunCounts { const database = getDb(dbPath); const stats = database