diff --git a/package.json b/package.json index b3374ab..4c95933 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "bestsellers": "bun run src/bestsellers-by-category.ts", "monthly-sold": "bun run src/top-monthly-sold-by-category.ts", "mid-range": "bun run src/mid-range-sellers-by-category.ts", + "stalker": "bun run src/stalker.ts", "upc": "bun run src/upc-lookup.ts", "upc-file": "bun run src/upc-file-analysis.ts", "start": "bun run src/index.ts", diff --git a/src/database.ts b/src/database.ts index 526a220..cdef940 100644 --- a/src/database.ts +++ b/src/database.ts @@ -333,4 +333,126 @@ export function initDb(dbPath: string): void { database.run( `CREATE INDEX IF NOT EXISTS idx_product_results_fetched_at ON product_analysis_results(fetched_at DESC);`, ); + initStalkerDb(database); +} + +export function initStalkerDb(database: Database): void { + resetLegacyStalkerSchema(database); + + database.run(` + CREATE TABLE IF NOT EXISTS stalker_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + input_file TEXT NOT NULL, + started_at TEXT NOT NULL, + completed_at TEXT, + requested_asins INTEGER NOT NULL DEFAULT 0, + skipped_asins INTEGER NOT NULL DEFAULT 0, + scanned_asins INTEGER NOT NULL DEFAULT 0, + source_asins_with_matches INTEGER NOT NULL DEFAULT 0, + candidate_sellers INTEGER NOT NULL DEFAULT 0, + qualifying_sellers INTEGER NOT NULL DEFAULT 0, + matched_sellers INTEGER NOT NULL DEFAULT 0, + seller_metadata_requests INTEGER NOT NULL DEFAULT 0, + seller_storefront_requests INTEGER NOT NULL DEFAULT 0, + persisted_inventory_asins INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL, + error_message TEXT + ); + `); + + database.run(` + CREATE TABLE IF NOT EXISTS stalker_asin_scans ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_id INTEGER NOT NULL, + source_asin TEXT NOT NULL, + title TEXT, + offer_count INTEGER NOT NULL DEFAULT 0, + candidate_seller_count INTEGER NOT NULL DEFAULT 0, + matched_seller_count INTEGER NOT NULL DEFAULT 0, + fetched_at TEXT NOT NULL, + raw_product_json TEXT, + UNIQUE(run_id, source_asin), + FOREIGN KEY (run_id) REFERENCES stalker_runs(id) ON DELETE CASCADE + ); + `); + + database.run(` + CREATE TABLE IF NOT EXISTS stalker_sellers ( + seller_id TEXT PRIMARY KEY, + seller_name TEXT, + rating REAL, + rating_count INTEGER, + storefront_asin_total INTEGER, + persisted_inventory_sample_count INTEGER, + last_updated_at TEXT NOT NULL, + raw_seller_json TEXT + ); + `); + + database.run(` + CREATE TABLE IF NOT EXISTS stalker_asin_sellers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + scan_id INTEGER NOT NULL, + seller_id TEXT NOT NULL, + offer_price REAL, + condition TEXT, + is_fba INTEGER, + stock INTEGER, + seller_rating REAL, + seller_rating_count INTEGER, + raw_offer_json TEXT, + UNIQUE(scan_id, seller_id), + FOREIGN KEY (scan_id) REFERENCES stalker_asin_scans(id) ON DELETE CASCADE, + FOREIGN KEY (seller_id) REFERENCES stalker_sellers(seller_id) + ); + `); + + database.run(` + CREATE TABLE IF NOT EXISTS stalker_seller_inventory ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_id INTEGER NOT NULL, + seller_id TEXT NOT NULL, + asin TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + raw_inventory_json TEXT, + UNIQUE(run_id, seller_id, asin), + FOREIGN KEY (run_id) REFERENCES stalker_runs(id) ON DELETE CASCADE, + FOREIGN KEY (seller_id) REFERENCES stalker_sellers(seller_id) + ); + `); + + database.run( + `CREATE INDEX IF NOT EXISTS idx_stalker_runs_started_at ON stalker_runs(started_at DESC);`, + ); + database.run( + `CREATE INDEX IF NOT EXISTS idx_stalker_scans_run_id ON stalker_asin_scans(run_id);`, + ); + database.run( + `CREATE INDEX IF NOT EXISTS idx_stalker_scans_source_asin ON stalker_asin_scans(source_asin);`, + ); + database.run( + `CREATE INDEX IF NOT EXISTS idx_stalker_asin_sellers_seller_id ON stalker_asin_sellers(seller_id);`, + ); + database.run( + `CREATE INDEX IF NOT EXISTS idx_stalker_inventory_seller_id ON stalker_seller_inventory(seller_id);`, + ); + database.run( + `CREATE INDEX IF NOT EXISTS idx_stalker_inventory_asin ON stalker_seller_inventory(asin);`, + ); +} + +function resetLegacyStalkerSchema(database: Database): void { + const runColumns = database + .query("PRAGMA table_info(stalker_runs)") + .all() as Array<{ name: string }>; + if (runColumns.length === 0) return; + + const columnNames = new Set(runColumns.map((column) => column.name)); + if (columnNames.has("scanned_asins")) return; + + database.run("DROP TABLE IF EXISTS stalker_seller_inventory"); + database.run("DROP TABLE IF EXISTS stalker_asin_sellers"); + database.run("DROP TABLE IF EXISTS stalker_sellers"); + database.run("DROP TABLE IF EXISTS stalker_asin_scans"); + database.run("DROP TABLE IF EXISTS stalker_runs"); } diff --git a/src/server.ts b/src/server.ts index 9fef320..2221df6 100644 --- a/src/server.ts +++ b/src/server.ts @@ -53,6 +53,31 @@ type ProductListRecord = { fetched_at: string; }; +type StalkerResultRecord = { + runId: number; + started_at: string; + status: string; + input_file: string; + source_asin: string; + title: string | null; + offer_count: number; + candidate_seller_count: number; + matched_seller_count: number; + scan_fetched_at: string; + seller_id: string; + seller_name: string | null; + rating: number | null; + rating_count: number | null; + storefront_asin_total: number | null; + persisted_inventory_sample_count: number | null; + offer_price: number | null; + condition: string | null; + is_fba: number | null; + stock: number | null; + persisted_inventory_asin_count: number; + inventory_sample_asins: string | null; +}; + const DB_PATH = process.env.RESULTS_DB_PATH || path.join("db", "results.db"); const DEFAULT_PAGE_SIZE = 25; const MAX_PAGE_SIZE = 200; @@ -629,6 +654,170 @@ function getProductList(filters: URLSearchParams) { }; } +function parseStalkerFilters(filters: URLSearchParams) { + const q = filters.get("q")?.trim() || ""; + const sellerId = filters.get("sellerId")?.trim().toUpperCase() || ""; + const runIdRaw = filters.get("runId")?.trim() || ""; + const minRatingCountRaw = filters.get("minRatingCount")?.trim() || ""; + const maxRatingCountRaw = filters.get("maxRatingCount")?.trim() || ""; + + const conditions: string[] = []; + const params: Array = []; + + if (runIdRaw) { + const runId = Number(runIdRaw); + if (Number.isInteger(runId) && runId > 0) { + conditions.push("r.id = ?"); + params.push(runId); + } + } + + if (sellerId) { + conditions.push("s.seller_id = ?"); + params.push(sellerId); + } + + if (minRatingCountRaw) { + conditions.push("s.rating_count >= ?"); + params.push(Number(minRatingCountRaw)); + } + + if (maxRatingCountRaw) { + conditions.push("s.rating_count <= ?"); + params.push(Number(maxRatingCountRaw)); + } + + if (q) { + const wildcard = `%${q}%`; + conditions.push( + `(sc.source_asin LIKE ? OR sc.title LIKE ? OR s.seller_id LIKE ? OR s.seller_name LIKE ? OR EXISTS ( + SELECT 1 FROM stalker_seller_inventory inv_q + WHERE inv_q.run_id = r.id + AND inv_q.seller_id = s.seller_id + AND inv_q.asin LIKE ? + ))`, + ); + params.push(wildcard, wildcard, wildcard, wildcard, wildcard); + } + + return { + where: conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "", + params, + }; +} + +function parseStalkerSort(sortParam: string | null): string { + const allowedSort = new Set([ + "runId", + "started_at", + "source_asin", + "title", + "seller_id", + "seller_name", + "rating", + "rating_count", + "offer_price", + "stock", + "persisted_inventory_asin_count", + "storefront_asin_total", + "scan_fetched_at", + ]); + const parsed = parseSort( + sortParam, + allowedSort, + "started_at DESC, runId DESC, source_asin ASC", + ); + + return parsed + .replaceAll("runId", "runId") + .replaceAll("rating_count", "rating_count") + .replaceAll("persisted_inventory_asin_count", "persisted_inventory_asin_count") + .replaceAll("storefront_asin_total", "storefront_asin_total"); +} + +function getStalkerResults(filters: URLSearchParams) { + const page = parseIntParam(filters.get("page"), 1); + const pageSize = Math.min( + parseIntParam(filters.get("pageSize"), DEFAULT_PAGE_SIZE), + MAX_PAGE_SIZE, + ); + const offset = (page - 1) * pageSize; + const { where, params } = parseStalkerFilters(filters); + const orderBy = parseStalkerSort(filters.get("sort")); + + const baseSelect = ` + SELECT + r.id AS runId, + r.started_at, + r.status, + r.input_file, + sc.source_asin, + sc.title, + sc.offer_count, + sc.candidate_seller_count, + sc.matched_seller_count, + sc.fetched_at AS scan_fetched_at, + s.seller_id, + s.seller_name, + s.rating, + s.rating_count, + s.storefront_asin_total, + s.persisted_inventory_sample_count, + sas.offer_price, + sas.condition, + sas.is_fba, + sas.stock, + COUNT(DISTINCT inv.asin) AS persisted_inventory_asin_count, + GROUP_CONCAT(DISTINCT inv.asin) AS inventory_sample_asins + FROM stalker_asin_sellers sas + JOIN stalker_asin_scans sc ON sc.id = sas.scan_id + JOIN stalker_runs r ON r.id = sc.run_id + JOIN stalker_sellers s ON s.seller_id = sas.seller_id + LEFT JOIN stalker_seller_inventory inv + ON inv.run_id = r.id + AND inv.seller_id = s.seller_id + ${where} + GROUP BY sas.id + `; + + const totalRow = db + .query(`SELECT COUNT(*) AS total FROM (${baseSelect}) stalker_rows`) + .get(...params) as { total: number }; + + const summary = db + .query( + `SELECT + COUNT(DISTINCT runId) AS runs, + COUNT(DISTINCT source_asin) AS sourceAsins, + COUNT(DISTINCT seller_id) AS sellers, + COALESCE(SUM(persisted_inventory_asin_count), 0) AS persistedInventoryAsins + FROM (${baseSelect}) stalker_rows`, + ) + .get(...params) as { + runs: number; + sourceAsins: number; + sellers: number; + persistedInventoryAsins: number; + }; + + const items = db + .query( + `SELECT * FROM (${baseSelect}) stalker_rows + ORDER BY ${orderBy} + LIMIT ? OFFSET ?`, + ) + .all(...params, pageSize, offset) as StalkerResultRecord[]; + + return { + items, + summary, + page, + pageSize, + total: totalRow.total, + totalPages: Math.max(1, Math.ceil(totalRow.total / pageSize)), + }; +} + function getRun(processType: ProcessType, runId: number) { if (processType === "lead_analysis") { const run = db @@ -1259,6 +1448,7 @@ const server = Bun.serve({ routes: { "/": index, "/products": index, + "/stalker": index, "/runs/:processType/:runId": index, "/api/runs": (req) => { const url = new URL(req.url); @@ -1268,6 +1458,10 @@ const server = Bun.serve({ const url = new URL(req.url); return json(getProductList(url.searchParams)); }, + "/api/stalker/results": (req) => { + const url = new URL(req.url); + return json(getStalkerResults(url.searchParams)); + }, "/api/upc/map": async (req) => { let upcs: string[]; try { diff --git a/src/stalker.test.ts b/src/stalker.test.ts new file mode 100644 index 0000000..42ffc08 --- /dev/null +++ b/src/stalker.test.ts @@ -0,0 +1,281 @@ +import { afterAll, beforeEach, expect, mock, test } from "bun:test"; +import { mkdirSync, rmSync } from "node:fs"; +import path from "node:path"; +import * as XLSX from "xlsx"; +import { closeDb, getDb, initDb } from "./database.ts"; +import { + extractLiveOfferSellerCandidates, + isQualifyingSeller, + readAsinsFromXlsx, + runStalker, +} from "./stalker.ts"; + +const TEST_DIR = path.join(process.cwd(), "test_output", "stalker"); +const originalFetch = globalThis.fetch; +const originalKeepaKey = Bun.env.KEEPA_API_KEY; + +beforeEach(() => { + closeDb(); + rmSync(TEST_DIR, { recursive: true, force: true }); + mkdirSync(TEST_DIR, { recursive: true }); + globalThis.fetch = originalFetch; + Bun.env.KEEPA_API_KEY = "test-keepa-key"; +}); + +afterAll(() => { + globalThis.fetch = originalFetch; + if (originalKeepaKey == null) { + delete Bun.env.KEEPA_API_KEY; + } else { + Bun.env.KEEPA_API_KEY = originalKeepaKey; + } + closeDb(); + rmSync(TEST_DIR, { recursive: true, force: true }); +}); + +test("readAsinsFromXlsx extracts valid ASINs and deduplicates in order", () => { + const filePath = path.join(TEST_DIR, "asins.xlsx"); + const workbook = XLSX.utils.book_new(); + const sheet = XLSX.utils.json_to_sheet([ + { ASIN: "b000000001" }, + { ASIN: "invalid" }, + { ASIN: "B000000002" }, + { ASIN: "B000000001" }, + { ASIN: "" }, + ]); + XLSX.utils.book_append_sheet(workbook, sheet, "Input"); + XLSX.writeFile(workbook, filePath); + + expect(readAsinsFromXlsx(filePath)).toEqual(["B000000001", "B000000002"]); +}); + +test("isQualifyingSeller accepts rating counts from 1 to 30 only", () => { + expect(isQualifyingSeller({ ratingCount: null })).toBe(false); + expect(isQualifyingSeller({ ratingCount: 0 })).toBe(false); + expect(isQualifyingSeller({ ratingCount: 1 })).toBe(true); + expect(isQualifyingSeller({ ratingCount: 30 })).toBe(true); + expect(isQualifyingSeller({ ratingCount: 31 })).toBe(false); +}); + +test("extractLiveOfferSellerCandidates ignores Amazon, missing seller IDs, and duplicates", () => { + const offers = extractLiveOfferSellerCandidates({ + offers: [ + { sellerId: "ATVPDKIKX0DER", price: 1999 }, + { price: 1899 }, + { sellerId: "A1SELLER", price: 1599, isFBA: true, stock: 4 }, + { sellerId: "A1SELLER", price: 1499 }, + { sellerID: "A2SELLER", currentPrice: 2499, isFba: false }, + ], + }); + + expect(offers.map((offer) => offer.sellerId)).toEqual([ + "A1SELLER", + "A2SELLER", + ]); + expect(offers[0]?.offerPrice).toBe(15.99); + expect(offers[0]?.isFba).toBe(true); + expect(offers[0]?.stock).toBe(4); +}); + +test("initDb creates stalker tables and indexes", () => { + const dbPath = path.join(TEST_DIR, "schema.sqlite"); + initDb(dbPath); + const db = getDb(dbPath); + + const tables = db + .query( + `SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'stalker_%' ORDER BY name`, + ) + .all() as Array<{ name: string }>; + expect(tables.map((row) => row.name)).toEqual([ + "stalker_asin_scans", + "stalker_asin_sellers", + "stalker_runs", + "stalker_seller_inventory", + "stalker_sellers", + ]); + + const indexes = db + .query( + `SELECT name FROM sqlite_master WHERE type = 'index' AND name LIKE 'idx_stalker_%' ORDER BY name`, + ) + .all() as Array<{ name: string }>; + expect(indexes.length).toBeGreaterThanOrEqual(6); +}); + +test("runStalker fetches product offers, filters sellers, and persists storefront inventory", async () => { + const inputPath = path.join(TEST_DIR, "input.xlsx"); + const dbPath = path.join(TEST_DIR, "stalker.sqlite"); + const workbook = XLSX.utils.book_new(); + XLSX.utils.book_append_sheet( + workbook, + XLSX.utils.json_to_sheet([{ asin: "B000000001" }]), + "Input", + ); + XLSX.writeFile(workbook, inputPath); + + const fetchMock = mock(async (input: string | URL | Request) => { + const rawUrl = + typeof input === "string" + ? input + : input instanceof URL + ? input.toString() + : input.url; + const url = new URL(rawUrl); + + if (url.pathname === "/product") { + expect(url.searchParams.get("asin")).toBe("B000000001"); + expect(url.searchParams.get("offers")).toBe("20"); + expect(url.searchParams.get("only-live-offers")).toBe("1"); + expect(url.searchParams.has("stock")).toBe(false); + + return new Response( + JSON.stringify({ + products: [ + { + asin: "B000000001", + title: "Tracked Product", + offers: [ + { + sellerId: "AQUALIFIED", + price: 1999, + condition: "New", + isFBA: true, + stock: 3, + }, + { + sellerId: "AOLDSELLER", + price: 2099, + }, + ], + }, + ], + tokensLeft: 10, + refillRate: 10, + }), + { status: 200 }, + ); + } + + if (url.pathname === "/seller") { + const wantsStorefront = url.searchParams.get("storefront") === "1"; + if (wantsStorefront) { + expect(url.searchParams.get("update")).toBe("168"); + } + const sellerId = url.searchParams.get("seller"); + + return new Response( + JSON.stringify({ + sellers: { + ...(!wantsStorefront && sellerId === "AQUALIFIED,AOLDSELLER" + ? { + AQUALIFIED: { + sellerName: "New Seller", + currentRating: 96, + currentRatingCount: 12, + }, + AOLDSELLER: { + sellerName: "Old Seller", + currentRating: 99, + currentRatingCount: 120, + }, + } + : {}), + ...(wantsStorefront && sellerId === "AQUALIFIED" + ? { + AQUALIFIED: { + sellerName: "New Seller", + currentRating: 96, + currentRatingCount: 12, + asinList: ["B111111111", "B222222222"], + }, + } + : {}), + }, + tokensLeft: 10, + refillRate: 10, + }), + { status: 200 }, + ); + } + + return new Response("not found", { status: 404 }); + }); + globalThis.fetch = fetchMock as unknown as typeof globalThis.fetch; + + const stats = await runStalker({ + input: inputPath, + dbPath, + maxAsins: null, + storefrontUpdateHours: 168, + offerLimit: 20, + sellerLimit: 30, + inventoryLimit: 200, + sellerCacheHours: 168, + includeStock: false, + dryRun: false, + resume: true, + maxSellerRequests: null, + }); + + expect(stats.scannedAsins).toBe(1); + expect(stats.sourceAsinsWithMatches).toBe(1); + expect(stats.matchedSellers).toBe(1); + expect(stats.persistedInventoryAsins).toBe(2); + expect(stats.failedAsins).toBe(0); + expect(stats.candidateSellers).toBe(2); + expect(stats.qualifyingSellers).toBe(1); + expect(stats.sellerMetadataRequests).toBe(1); + expect(stats.sellerStorefrontRequests).toBe(1); + const sellerCalls = fetchMock.mock.calls.filter((call) => { + const rawUrl = + typeof call[0] === "string" + ? call[0] + : call[0] instanceof URL + ? call[0].toString() + : (call[0] as Request).url; + return new URL(rawUrl).pathname === "/seller"; + }); + expect(sellerCalls.length).toBe(2); + + const db = getDb(dbPath); + const run = db.query("SELECT * FROM stalker_runs").get() as any; + expect(run.status).toBe("completed"); + expect(run.requested_asins).toBe(1); + expect(run.scanned_asins).toBe(1); + expect(run.source_asins_with_matches).toBe(1); + expect(run.candidate_sellers).toBe(2); + expect(run.qualifying_sellers).toBe(1); + expect(run.matched_sellers).toBe(1); + expect(run.seller_metadata_requests).toBe(1); + expect(run.seller_storefront_requests).toBe(1); + expect(run.persisted_inventory_asins).toBe(2); + + const scan = db.query("SELECT * FROM stalker_asin_scans").get() as any; + expect(scan.source_asin).toBe("B000000001"); + expect(scan.title).toBe("Tracked Product"); + expect(scan.offer_count).toBe(2); + expect(scan.candidate_seller_count).toBe(2); + expect(scan.matched_seller_count).toBe(1); + + const sellers = db.query("SELECT * FROM stalker_sellers").all() as any[]; + expect(sellers.length).toBe(1); + expect(sellers[0].seller_id).toBe("AQUALIFIED"); + expect(sellers[0].rating_count).toBe(12); + expect(sellers[0].storefront_asin_total).toBe(2); + expect(sellers[0].persisted_inventory_sample_count).toBe(2); + + const asinSellers = db.query("SELECT * FROM stalker_asin_sellers").all() as any[]; + expect(asinSellers.length).toBe(1); + expect(asinSellers[0].offer_price).toBe(19.99); + expect(asinSellers[0].is_fba).toBe(1); + expect(asinSellers[0].stock).toBe(3); + + const inventory = db + .query("SELECT asin FROM stalker_seller_inventory ORDER BY asin") + .all() as Array<{ asin: string }>; + expect(inventory.map((row) => row.asin)).toEqual([ + "B111111111", + "B222222222", + ]); +}); diff --git a/src/stalker.ts b/src/stalker.ts new file mode 100644 index 0000000..f4b6563 --- /dev/null +++ b/src/stalker.ts @@ -0,0 +1,1189 @@ +import * as XLSX from "xlsx"; +import path from "node:path"; +import { type Database, closeDb, getDb, initDb } from "./database.ts"; + +const KEEPA_BASE = "https://api.keepa.com"; +const DOMAIN_US = "1"; +const AMAZON_US_SELLER_ID = "ATVPDKIKX0DER"; +const ASIN_REGEX = /^B[0-9A-Z]{9}$/; +const DEFAULT_DB_PATH = path.join(process.cwd(), "db", "results.db"); +const DEFAULT_STOREFRONT_UPDATE_HOURS = 168; +const DEFAULT_OFFER_LIMIT = 100; +const DEFAULT_SELLER_LIMIT = 30; +const DEFAULT_INVENTORY_LIMIT = 200; +const DEFAULT_SELLER_CACHE_HOURS = 168; +const MAX_SELLERS_PER_METADATA_REQUEST = 100; +const MAX_KEEPA_RETRIES = 4; +const KEEP_RETRY_BUFFER_MS = 250; + +type KeepaApiResponse = { + products?: Record[]; + sellers?: Record | Record[]; + tokensLeft?: number; + refillRate?: number; + refillIn?: number; +}; + +export type StalkerArgs = { + input: string; + dbPath: string; + maxAsins: number | null; + storefrontUpdateHours: number; + offerLimit: number; + sellerLimit: number; + inventoryLimit: number; + sellerCacheHours: number; + includeStock: boolean; + dryRun: boolean; + resume: boolean; + maxSellerRequests: number | null; +}; + +export type StalkerOffer = { + sellerId: string; + offerPrice: number | null; + condition: string | null; + isFba: boolean | null; + stock: number | null; + rawOffer: Record; +}; + +export type StalkerSeller = { + sellerId: string; + sellerName: string | null; + rating: number | null; + ratingCount: number | null; + storefrontAsins: string[]; + storefrontItems: StalkerInventoryItem[]; + storefrontAsinTotal: number; + rawSeller: Record; +}; + +type StalkerInventoryItem = { + asin: string; + rawInventory: unknown; +}; + +type StalkerAsinResult = { + asin: string; + title: string | null; + offerCount: number; + candidateSellerCount: number; + matchedSellers: Array<{ + seller: StalkerSeller; + offer: StalkerOffer; + }>; + product: Record | null; + error?: string; +}; + +type StalkerRunStats = { + scannedAsins: number; + sourceAsinsWithMatches: number; + matchedSellers: number; + persistedInventoryAsins: number; + failedAsins: number; + skippedAsins: number; + candidateSellers: number; + qualifyingSellers: number; + sellerMetadataRequests: number; + sellerStorefrontRequests: number; + stoppedEarly: boolean; +}; + +type StalkerRunContext = { + database: Database | null; + metadataCache: Map; + storefrontCache: Map; + stats: StalkerRunStats; +}; + +let tokensLeft = 1; +let refillRate = 1; +let lastRequestTime = 0; + +export function parseArgs(argv = process.argv.slice(2)): StalkerArgs { + const input = readFlagValue(argv, "--input"); + if (!input) { + printUsageAndExit("Missing required --input file."); + } + + const dbPath = readFlagValue(argv, "--db") ?? DEFAULT_DB_PATH; + const maxAsinsRaw = readFlagValue(argv, "--max-asins"); + const storefrontUpdateRaw = readFlagValue(argv, "--storefront-update-hours"); + const offerLimitRaw = readFlagValue(argv, "--offer-limit"); + const sellerLimitRaw = readFlagValue(argv, "--seller-limit"); + const inventoryLimitRaw = readFlagValue(argv, "--inventory-limit"); + const sellerCacheHoursRaw = readFlagValue(argv, "--seller-cache-hours"); + const maxSellerRequestsRaw = readFlagValue(argv, "--max-seller-requests"); + + const maxAsins = maxAsinsRaw ? Number(maxAsinsRaw) : null; + const storefrontUpdateHours = storefrontUpdateRaw + ? Number(storefrontUpdateRaw) + : DEFAULT_STOREFRONT_UPDATE_HOURS; + const offerLimit = offerLimitRaw ? Number(offerLimitRaw) : DEFAULT_OFFER_LIMIT; + const sellerLimit = sellerLimitRaw ? Number(sellerLimitRaw) : DEFAULT_SELLER_LIMIT; + const inventoryLimit = inventoryLimitRaw + ? Number(inventoryLimitRaw) + : DEFAULT_INVENTORY_LIMIT; + const sellerCacheHours = sellerCacheHoursRaw + ? Number(sellerCacheHoursRaw) + : DEFAULT_SELLER_CACHE_HOURS; + const maxSellerRequests = maxSellerRequestsRaw + ? Number(maxSellerRequestsRaw) + : null; + const includeStock = hasFlag(argv, "--include-stock"); + const dryRun = hasFlag(argv, "--dry-run"); + const resume = !hasFlag(argv, "--no-resume"); + + if (maxAsins != null && (!Number.isInteger(maxAsins) || maxAsins <= 0)) { + printUsageAndExit("--max-asins must be a positive integer."); + } + + if ( + !Number.isInteger(storefrontUpdateHours) || + storefrontUpdateHours < 0 + ) { + printUsageAndExit( + "--storefront-update-hours must be a non-negative integer.", + ); + } + + if (!Number.isInteger(offerLimit) || offerLimit < 20 || offerLimit > 100) { + printUsageAndExit("--offer-limit must be an integer from 20 to 100."); + } + + if (!Number.isInteger(sellerLimit) || sellerLimit <= 0) { + printUsageAndExit("--seller-limit must be a positive integer."); + } + + if (!Number.isInteger(inventoryLimit) || inventoryLimit < 0) { + printUsageAndExit("--inventory-limit must be a non-negative integer."); + } + + if (!Number.isInteger(sellerCacheHours) || sellerCacheHours < 0) { + printUsageAndExit("--seller-cache-hours must be a non-negative integer."); + } + + if ( + maxSellerRequests != null && + (!Number.isInteger(maxSellerRequests) || maxSellerRequests <= 0) + ) { + printUsageAndExit("--max-seller-requests must be a positive integer."); + } + + return { + input, + dbPath, + maxAsins, + storefrontUpdateHours, + offerLimit, + sellerLimit, + inventoryLimit, + sellerCacheHours, + includeStock, + dryRun, + resume, + maxSellerRequests, + }; +} + +export function readAsinsFromXlsx(filePath: string): string[] { + const workbook = XLSX.readFile(filePath); + const sheetName = workbook.SheetNames[0]; + if (!sheetName) throw new Error("No sheets found in file"); + + const sheet = workbook.Sheets[sheetName]; + if (!sheet) throw new Error("First sheet is missing"); + + const rows = XLSX.utils.sheet_to_json>(sheet, { + defval: "", + }); + if (rows.length === 0) throw new Error("File contains no data rows"); + + const headers = Object.keys(rows[0]!); + const asinColumn = headers.find((header) => normalizeHeader(header) === "asin"); + if (!asinColumn) { + throw new Error(`No ASIN column found. Available columns: ${headers.join(", ")}`); + } + + return extractAsinsFromRows(rows, asinColumn); +} + +export function extractAsinsFromRows( + rows: Array>, + asinColumn = "asin", +): string[] { + const asins: string[] = []; + const seen = new Set(); + + for (const row of rows) { + const asin = normalizeAsin(row[asinColumn]); + if (!asin || seen.has(asin)) continue; + seen.add(asin); + asins.push(asin); + } + + return asins; +} + +export function isQualifyingSeller(seller: { + ratingCount?: number | null; +}): boolean { + return ( + typeof seller.ratingCount === "number" && + Number.isFinite(seller.ratingCount) && + seller.ratingCount >= 1 && + seller.ratingCount <= 30 + ); +} + +export function extractLiveOfferSellerCandidates( + product: Record, +): StalkerOffer[] { + const offers = Array.isArray(product.offers) ? product.offers : []; + const bySeller = new Map(); + + for (const offer of offers) { + if (!offer || typeof offer !== "object") continue; + const sellerId = normalizeSellerId( + offer.sellerId ?? offer.sellerID ?? offer.seller_id, + ); + if (!sellerId || sellerId === AMAZON_US_SELLER_ID) continue; + if (bySeller.has(sellerId)) continue; + + bySeller.set(sellerId, { + sellerId, + offerPrice: extractOfferPrice(offer), + condition: extractString(offer.condition ?? offer.conditionComment), + isFba: extractBoolean(offer.isFBA ?? offer.isFba ?? offer.fba), + stock: extractNumber(offer.stock ?? offer.stockCount ?? offer.currentStock), + rawOffer: offer, + }); + } + + return Array.from(bySeller.values()); +} + +export async function runStalker(args: StalkerArgs): Promise { + const apiKey = Bun.env.KEEPA_API_KEY; + if (!apiKey) throw new Error("Missing required env var: KEEPA_API_KEY"); + + const allAsins = readAsinsFromXlsx(args.input); + const cappedAsins = + args.maxAsins == null ? allAsins : allAsins.slice(0, args.maxAsins); + + initDb(args.dbPath); + const database = getDb(args.dbPath); + const completedAsins = args.resume ? loadPreviouslyScannedAsins(database) : new Set(); + const asins = cappedAsins.filter((asin) => !completedAsins.has(asin)); + const runId = args.dryRun + ? null + : startStalkerRun(database, args.input, asins.length); + const stats: StalkerRunStats = { + scannedAsins: 0, + sourceAsinsWithMatches: 0, + matchedSellers: 0, + persistedInventoryAsins: 0, + failedAsins: 0, + skippedAsins: cappedAsins.length - asins.length, + candidateSellers: 0, + qualifyingSellers: 0, + sellerMetadataRequests: 0, + sellerStorefrontRequests: 0, + stoppedEarly: false, + }; + const context: StalkerRunContext = { + database, + metadataCache: new Map(), + storefrontCache: new Map(), + stats, + }; + + try { + if (args.dryRun) { + console.log("Stalker dry-run: product and seller metadata will be fetched, storefronts will not be fetched or persisted."); + } + if (stats.skippedAsins > 0) { + console.log(`Stalker resume: skipped ${stats.skippedAsins} previously scanned ASIN(s).`); + } + + for (const asin of asins) { + console.log(`Stalker: scanning ${asin} (${stats.scannedAsins + 1}/${asins.length})`); + + const result = await scanAsin(asin, args, apiKey, context).catch((error) => ({ + asin, + title: null, + offerCount: 0, + candidateSellerCount: 0, + matchedSellers: [], + product: null, + error: error instanceof Error ? error.message : String(error), + })); + + if (!args.dryRun && runId != null) { + persistAsinResult(database, runId, result); + } + stats.scannedAsins += 1; + stats.matchedSellers += result.matchedSellers.length; + stats.persistedInventoryAsins += sumInventoryAsins(result); + if (result.matchedSellers.length > 0) stats.sourceAsinsWithMatches += 1; + if (result.error) { + stats.failedAsins += 1; + console.warn(`Stalker: ${asin} failed: ${result.error}`); + } + + if (!args.dryRun && runId != null) { + refreshStalkerRun(database, runId, stats, "running"); + } + console.log( + `Stalker: ${asin} candidates=${result.candidateSellerCount}, matched=${result.matchedSellers.length}, persisted_inventory=${sumInventoryAsins(result)}`, + ); + + if (stats.stoppedEarly) { + console.log("Stalker: stopping early because max seller request budget was reached."); + break; + } + } + + if (!args.dryRun && runId != null) { + refreshStalkerRun( + database, + runId, + stats, + stats.stoppedEarly + ? "stopped" + : stats.failedAsins > 0 + ? "completed_with_errors" + : "completed", + ); + } + logRunSummary(stats, args); + return stats; + } catch (error) { + if (!args.dryRun && runId != null) { + finishStalkerRunWithError( + database, + runId, + stats, + error instanceof Error ? error.message : String(error), + ); + } + throw error; + } +} + +async function scanAsin( + asin: string, + args: StalkerArgs, + apiKey: string, + context: StalkerRunContext, +): Promise { + const product = await fetchKeepaProduct( + asin, + apiKey, + args.offerLimit, + args.includeStock, + ); + const offers = extractLiveOfferSellerCandidates(product).slice( + 0, + args.sellerLimit, + ); + context.stats.candidateSellers += offers.length; + + const metadata = await fetchSellerMetadata( + offers.map((offer) => offer.sellerId), + apiKey, + args, + context, + ); + const qualifyingOffers = offers.filter((offer) => { + const seller = metadata.get(offer.sellerId); + return seller ? isQualifyingSeller(seller) : false; + }); + context.stats.qualifyingSellers += qualifyingOffers.length; + + if (args.dryRun) { + console.log( + `Stalker dry-run estimate for ${asin}: storefront requests needed=${qualifyingOffers.length}, candidate sellers=${offers.length}`, + ); + } + + const storefronts = args.dryRun + ? new Map() + : await fetchQualifiedSellerStorefronts( + qualifyingOffers.map((offer) => offer.sellerId), + apiKey, + args, + context, + ); + + const matchedSellers = qualifyingOffers + .map((offer) => { + const seller = storefronts.get(offer.sellerId); + if (!seller || !isQualifyingSeller(seller)) return null; + return { seller, offer }; + }) + .filter( + (entry): entry is { seller: StalkerSeller; offer: StalkerOffer } => + entry != null, + ); + + return { + asin, + title: extractString(product.title), + offerCount: Array.isArray(product.offers) ? product.offers.length : 0, + candidateSellerCount: offers.length, + matchedSellers, + product, + }; +} + +async function fetchKeepaProduct( + asin: string, + apiKey: string, + offerLimit: number, + includeStock: boolean, +): Promise> { + const params = new URLSearchParams({ + key: apiKey, + domain: DOMAIN_US, + asin, + offers: String(offerLimit), + "only-live-offers": "1", + stats: "30", + days: "30", + }); + if (includeStock) { + params.set("stock", "1"); + } + const data = await fetchKeepaWithRetries( + `${KEEPA_BASE}/product?${params.toString()}`, + `product ${asin}`, + ); + const product = data.products?.[0]; + if (!product) throw new Error("Keepa returned no product"); + return product; +} + +async function fetchSellerMetadata( + sellerIds: string[], + apiKey: string, + args: StalkerArgs, + context: StalkerRunContext, +): Promise> { + const out = new Map(); + const uniqueSellerIds = Array.from(new Set(sellerIds)); + const missing: string[] = []; + + for (const sellerId of uniqueSellerIds) { + const cached = + context.metadataCache.get(sellerId) ?? + loadCachedSeller( + context.database, + sellerId, + args.sellerCacheHours, + false, + args.inventoryLimit, + ); + if (cached) { + context.metadataCache.set(sellerId, cached); + out.set(sellerId, cached); + continue; + } + missing.push(sellerId); + } + + for (let i = 0; i < missing.length; i += MAX_SELLERS_PER_METADATA_REQUEST) { + const chunk = missing.slice(i, i + MAX_SELLERS_PER_METADATA_REQUEST); + if (!canSpendSellerRequests(context, args, 1)) break; + + const params = new URLSearchParams({ + key: apiKey, + domain: DOMAIN_US, + seller: chunk.join(","), + }); + + context.stats.sellerMetadataRequests += 1; + const data = await fetchKeepaWithRetries( + `${KEEPA_BASE}/seller?${params.toString()}`, + `seller metadata batch ${i / MAX_SELLERS_PER_METADATA_REQUEST + 1}`, + ); + + for (const [sellerId, seller] of normalizeSellerResponse(data.sellers)) { + const parsed = parseSeller(sellerId, seller, args.inventoryLimit); + context.metadataCache.set(sellerId, parsed); + out.set(sellerId, parsed); + } + } + + return out; +} + +async function fetchQualifiedSellerStorefronts( + sellerIds: string[], + apiKey: string, + args: StalkerArgs, + context: StalkerRunContext, +): Promise> { + const out = new Map(); + const uniqueSellerIds = Array.from(new Set(sellerIds)); + + // Keepa only allows a single sellerId per request when storefront=1. + for (const sellerId of uniqueSellerIds) { + const cached = + context.storefrontCache.get(sellerId) ?? + loadCachedSeller( + context.database, + sellerId, + args.sellerCacheHours, + true, + args.inventoryLimit, + ); + if (cached) { + context.storefrontCache.set(sellerId, cached); + out.set(sellerId, cached); + continue; + } + + if (!canSpendSellerRequests(context, args, 1)) break; + + const params = new URLSearchParams({ + key: apiKey, + domain: DOMAIN_US, + seller: sellerId, + storefront: "1", + update: String(args.storefrontUpdateHours), + }); + + context.stats.sellerStorefrontRequests += 1; + const data = await fetchKeepaWithRetries( + `${KEEPA_BASE}/seller?${params.toString()}`, + `seller ${sellerId}`, + ); + + for (const [returnedSellerId, seller] of normalizeSellerResponse( + data.sellers, + )) { + const parsed = parseSeller(returnedSellerId, seller, args.inventoryLimit); + context.metadataCache.set(returnedSellerId, parsed); + context.storefrontCache.set(returnedSellerId, parsed); + out.set(returnedSellerId, parsed); + } + } + + return out; +} + +function canSpendSellerRequests( + context: StalkerRunContext, + args: StalkerArgs, + nextRequests: number, +): boolean { + if (args.maxSellerRequests == null) return true; + const spent = + context.stats.sellerMetadataRequests + context.stats.sellerStorefrontRequests; + if (spent + nextRequests <= args.maxSellerRequests) return true; + context.stats.stoppedEarly = true; + return false; +} + +async function fetchKeepaWithRetries( + url: string, + operationLabel: string, +): Promise { + let lastErrorMessage = "Unknown Keepa error"; + + for (let attempt = 1; attempt <= MAX_KEEPA_RETRIES; attempt++) { + await waitForToken(); + const response = await fetch(url); + lastRequestTime = Date.now(); + + if (response.ok) { + const data = (await response.json()) as KeepaApiResponse; + updateTokenState(data); + return data; + } + + const text = await response.text(); + const payload = parseErrorPayload(text); + if (payload) updateTokenState(payload); + lastErrorMessage = `Keepa API error ${response.status}: ${text}`; + + if (response.status !== 429 || attempt === MAX_KEEPA_RETRIES) break; + + const waitMs = computeWaitMsFromRefill(payload?.refillIn); + tokensLeft = Math.min(tokensLeft, 0); + console.warn( + `Keepa throttled during ${operationLabel} (attempt ${attempt}/${MAX_KEEPA_RETRIES}). Waiting ${Math.ceil(waitMs / 1000)}s before retry...`, + ); + await wait(waitMs); + } + + throw new Error(lastErrorMessage); +} + +function persistAsinResult( + database: Database, + runId: number, + result: StalkerAsinResult, +): void { + const fetchedAt = new Date().toISOString(); + + database.transaction(() => { + const scanId = upsertAsinScan(database, runId, result, fetchedAt); + + for (const { seller, offer } of result.matchedSellers) { + upsertSeller(database, seller, fetchedAt); + upsertAsinSeller(database, scanId, seller, offer); + upsertSellerInventory(database, runId, seller, fetchedAt); + } + })(); +} + +function upsertAsinScan( + database: Database, + runId: number, + result: StalkerAsinResult, + fetchedAt: string, +): number { + database + .prepare( + `INSERT INTO stalker_asin_scans ( + run_id, source_asin, title, offer_count, candidate_seller_count, + matched_seller_count, fetched_at, raw_product_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(run_id, source_asin) DO UPDATE SET + title = excluded.title, + offer_count = excluded.offer_count, + candidate_seller_count = excluded.candidate_seller_count, + matched_seller_count = excluded.matched_seller_count, + fetched_at = excluded.fetched_at, + raw_product_json = excluded.raw_product_json`, + ) + .run( + runId, + result.asin, + result.title, + result.offerCount, + result.candidateSellerCount, + result.matchedSellers.length, + fetchedAt, + JSON.stringify(result.product ?? { error: result.error ?? null }), + ); + + const row = database + .query( + `SELECT id FROM stalker_asin_scans WHERE run_id = ? AND source_asin = ?`, + ) + .get(runId, result.asin) as { id: number } | null; + if (!row) throw new Error(`Failed to load stalker scan row for ${result.asin}`); + return row.id; +} + +function upsertSeller( + database: Database, + seller: StalkerSeller, + fetchedAt: string, +): void { + database + .prepare( + `INSERT INTO stalker_sellers ( + seller_id, seller_name, rating, rating_count, storefront_asin_total, + persisted_inventory_sample_count, last_updated_at, raw_seller_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(seller_id) DO UPDATE SET + seller_name = excluded.seller_name, + rating = excluded.rating, + rating_count = excluded.rating_count, + storefront_asin_total = excluded.storefront_asin_total, + persisted_inventory_sample_count = excluded.persisted_inventory_sample_count, + last_updated_at = excluded.last_updated_at, + raw_seller_json = excluded.raw_seller_json`, + ) + .run( + seller.sellerId, + seller.sellerName, + seller.rating, + seller.ratingCount, + seller.storefrontAsinTotal, + seller.storefrontItems.length, + fetchedAt, + JSON.stringify(seller.rawSeller), + ); +} + +function upsertAsinSeller( + database: Database, + scanId: number, + seller: StalkerSeller, + offer: StalkerOffer, +): void { + database + .prepare( + `INSERT INTO stalker_asin_sellers ( + scan_id, seller_id, offer_price, condition, is_fba, stock, + seller_rating, seller_rating_count, raw_offer_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(scan_id, seller_id) DO UPDATE SET + offer_price = excluded.offer_price, + condition = excluded.condition, + is_fba = excluded.is_fba, + stock = excluded.stock, + seller_rating = excluded.seller_rating, + seller_rating_count = excluded.seller_rating_count, + raw_offer_json = excluded.raw_offer_json`, + ) + .run( + scanId, + seller.sellerId, + offer.offerPrice, + offer.condition, + offer.isFba == null ? null : offer.isFba ? 1 : 0, + offer.stock, + seller.rating, + seller.ratingCount, + JSON.stringify(offer.rawOffer), + ); +} + +function upsertSellerInventory( + database: Database, + runId: number, + seller: StalkerSeller, + fetchedAt: string, +): void { + const insert = database.prepare( + `INSERT INTO stalker_seller_inventory ( + run_id, seller_id, asin, last_seen_at, raw_inventory_json + ) VALUES (?, ?, ?, ?, ?) + ON CONFLICT(run_id, seller_id, asin) DO UPDATE SET + last_seen_at = excluded.last_seen_at, + raw_inventory_json = excluded.raw_inventory_json`, + ); + + for (const item of seller.storefrontItems) { + insert.run( + runId, + seller.sellerId, + item.asin, + fetchedAt, + JSON.stringify(item.rawInventory), + ); + } +} + +function startStalkerRun( + database: Database, + inputFile: string, + totalAsins: number, +): number { + const result = database + .prepare( + `INSERT INTO stalker_runs ( + input_file, started_at, requested_asins, status + ) VALUES (?, ?, ?, ?)`, + ) + .run(inputFile, new Date().toISOString(), totalAsins, "running"); + + return result.lastInsertRowid as number; +} + +function loadPreviouslyScannedAsins(database: Database): Set { + const rows = database + .query(`SELECT DISTINCT source_asin FROM stalker_asin_scans`) + .all() as Array<{ source_asin: string }>; + return new Set(rows.map((row) => row.source_asin)); +} + +function loadCachedSeller( + database: Database | null, + sellerId: string, + maxAgeHours: number, + requireStorefront: boolean, + inventoryLimit: number, +): StalkerSeller | null { + if (!database || maxAgeHours <= 0) return null; + const row = database + .query( + `SELECT raw_seller_json, last_updated_at, storefront_asin_total + FROM stalker_sellers + WHERE seller_id = ?`, + ) + .get(sellerId) as { + raw_seller_json: string | null; + last_updated_at: string; + storefront_asin_total: number | null; + } | null; + if (!row?.raw_seller_json) return null; + + const ageMs = Date.now() - new Date(row.last_updated_at).getTime(); + if (!Number.isFinite(ageMs) || ageMs > maxAgeHours * 60 * 60 * 1000) { + return null; + } + + try { + const rawSeller = JSON.parse(row.raw_seller_json) as Record; + const parsed = parseSeller(sellerId, rawSeller, inventoryLimit); + if (requireStorefront && parsed.storefrontAsinTotal <= 0) return null; + return parsed; + } catch { + return null; + } +} + +function logRunSummary(stats: StalkerRunStats, args: StalkerArgs): void { + const estimatedStorefrontRequestsSaved = Math.max( + 0, + stats.candidateSellers - stats.qualifyingSellers, + ); + console.log( + [ + "Stalker summary:", + `processed=${stats.scannedAsins}`, + `skipped=${stats.skippedAsins}`, + `candidate_sellers=${stats.candidateSellers}`, + `qualifying_sellers=${stats.qualifyingSellers}`, + `metadata_requests=${stats.sellerMetadataRequests}`, + `storefront_requests=${stats.sellerStorefrontRequests}`, + `storefront_requests_saved_by_two_phase=${estimatedStorefrontRequestsSaved}`, + `persisted_inventory=${stats.persistedInventoryAsins}`, + `dry_run=${args.dryRun ? "yes" : "no"}`, + ].join(" "), + ); +} + +function refreshStalkerRun( + database: Database, + runId: number, + stats: StalkerRunStats, + status: string, +): void { + database + .prepare( + `UPDATE stalker_runs + SET scanned_asins = ?, + source_asins_with_matches = ?, + candidate_sellers = ?, + qualifying_sellers = ?, + matched_sellers = ?, + seller_metadata_requests = ?, + seller_storefront_requests = ?, + persisted_inventory_asins = ?, + status = ?, + completed_at = CASE WHEN ? = 'running' THEN completed_at ELSE ? END + WHERE id = ?`, + ) + .run( + stats.scannedAsins, + stats.sourceAsinsWithMatches, + stats.candidateSellers, + stats.qualifyingSellers, + stats.matchedSellers, + stats.sellerMetadataRequests, + stats.sellerStorefrontRequests, + stats.persistedInventoryAsins, + status, + status, + new Date().toISOString(), + runId, + ); +} + +function finishStalkerRunWithError( + database: Database, + runId: number, + stats: StalkerRunStats, + errorMessage: string, +): void { + database + .prepare( + `UPDATE stalker_runs + SET scanned_asins = ?, + source_asins_with_matches = ?, + candidate_sellers = ?, + qualifying_sellers = ?, + matched_sellers = ?, + seller_metadata_requests = ?, + seller_storefront_requests = ?, + persisted_inventory_asins = ?, + status = 'failed', + error_message = ?, + completed_at = ? + WHERE id = ?`, + ) + .run( + stats.scannedAsins, + stats.sourceAsinsWithMatches, + stats.candidateSellers, + stats.qualifyingSellers, + stats.matchedSellers, + stats.sellerMetadataRequests, + stats.sellerStorefrontRequests, + stats.persistedInventoryAsins, + errorMessage, + new Date().toISOString(), + runId, + ); +} + +function normalizeSellerResponse( + sellers: KeepaApiResponse["sellers"], +): Array<[string, Record]> { + if (!sellers) return []; + if (Array.isArray(sellers)) { + return sellers + .map((seller) => [ + normalizeSellerId(seller.sellerId ?? seller.sellerID ?? seller.id), + seller, + ] as [string | null, Record]) + .filter((entry): entry is [string, Record] => !!entry[0]); + } + + return Object.entries(sellers) + .map(([sellerId, seller]) => [ + normalizeSellerId(sellerId), + seller, + ] as [string | null, Record]) + .filter((entry): entry is [string, Record] => !!entry[0]); +} + +function parseSeller( + sellerId: string, + seller: Record, + inventoryLimit: number, +): StalkerSeller { + const allStorefrontItems = extractStorefrontItems(seller); + const storefrontItems = + inventoryLimit === 0 + ? [] + : allStorefrontItems.slice(0, inventoryLimit); + const storefrontAsins = storefrontItems.map((item) => item.asin); + return { + sellerId, + sellerName: extractString( + seller.sellerName ?? seller.name ?? seller.storeName ?? seller.businessName, + ), + rating: extractNumber( + seller.currentRating ?? seller.rating ?? seller.feedbackRating, + ), + ratingCount: extractSellerRatingCount(seller), + storefrontAsins, + storefrontItems, + storefrontAsinTotal: + extractNumber( + seller.totalStorefrontAsinCount ?? + seller.storefrontAsinCount ?? + seller.asinListCount ?? + seller.totalStorefrontProducts, + ) ?? allStorefrontItems.length, + rawSeller: seller, + }; +} + +function extractStorefrontItems(seller: Record): StalkerInventoryItem[] { + const candidates = [ + seller.asinList, + seller.asins, + seller.storefront, + seller.storefrontAsins, + seller.inventory, + ]; + const items: StalkerInventoryItem[] = []; + const seen = new Set(); + + for (const candidate of candidates) { + collectStorefrontItems(candidate, items, seen); + } + + return items; +} + +function collectStorefrontItems( + value: unknown, + items: StalkerInventoryItem[], + seen: Set, +): void { + if (Array.isArray(value)) { + for (const item of value) collectStorefrontItems(item, items, seen); + return; + } + + if (value && typeof value === "object") { + const asin = normalizeAsin((value as Record).asin); + if (asin && !seen.has(asin)) { + seen.add(asin); + items.push({ asin, rawInventory: value }); + } + return; + } + + const asin = normalizeAsin(value); + if (!asin || seen.has(asin)) return; + seen.add(asin); + items.push({ asin, rawInventory: { asin } }); +} + +function extractSellerRatingCount(seller: Record): number | null { + const direct = extractNumber( + seller.currentRatingCount ?? + seller.ratingCount ?? + seller.ratingsCount ?? + seller.feedbackCount ?? + seller.reviewCount, + ); + if (direct != null) return direct; + + const ratingCountHistory = seller.ratingCountHistory ?? seller.ratingCountCSV; + if (Array.isArray(ratingCountHistory) && ratingCountHistory.length > 0) { + return extractNumber(ratingCountHistory[ratingCountHistory.length - 1]); + } + + return null; +} + +function extractOfferPrice(offer: Record): number | null { + const raw = extractNumber( + offer.price ?? offer.currentPrice ?? offer.offerPrice ?? offer.newPrice, + ); + if (raw == null) return null; + return raw > 100 ? Math.round(raw) / 100 : raw; +} + +function sumInventoryAsins(result: StalkerAsinResult): number { + return result.matchedSellers.reduce( + (sum, entry) => sum + entry.seller.storefrontAsins.length, + 0, + ); +} + +function normalizeAsin(value: unknown): string | null { + const asin = String(value ?? "") + .trim() + .toUpperCase(); + return ASIN_REGEX.test(asin) ? asin : null; +} + +function normalizeSellerId(value: unknown): string | null { + const sellerId = String(value ?? "") + .trim() + .toUpperCase(); + return sellerId.length > 0 ? sellerId : null; +} + +function extractString(value: unknown): string | null { + if (value == null) return null; + const text = String(value).trim(); + return text.length > 0 ? text : null; +} + +function extractNumber(value: unknown): number | null { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value !== "string") return null; + const parsed = Number(value.trim().replace(/[$,%]/g, "").replace(/,/g, "")); + return Number.isFinite(parsed) ? parsed : null; +} + +function extractBoolean(value: unknown): boolean | null { + if (typeof value === "boolean") return value; + if (typeof value === "number") return value === 1 ? true : value === 0 ? false : null; + if (typeof value !== "string") return null; + const normalized = value.trim().toLowerCase(); + if (["1", "true", "yes"].includes(normalized)) return true; + if (["0", "false", "no"].includes(normalized)) return false; + return null; +} + +function normalizeHeader(value: string): string { + return value.toLowerCase().trim().replace(/[^a-z0-9]/g, ""); +} + +function readFlagValue(args: string[], flag: string): string | undefined { + const index = args.indexOf(flag); + if (index === -1) return undefined; + return args[index + 1]; +} + +function hasFlag(args: string[], flag: string): boolean { + return args.includes(flag); +} + +function printUsageAndExit(message: string): never { + console.error(message); + console.error( + "Usage: bun run stalker --input input/asins.xlsx [--db db/results.db] [--max-asins N] [--offer-limit 100] [--seller-limit 30] [--inventory-limit 200] [--storefront-update-hours 168] [--seller-cache-hours 168] [--max-seller-requests N] [--include-stock] [--dry-run] [--no-resume]", + ); + process.exit(1); +} + +function updateTokenState(data: KeepaApiResponse): void { + if (data.tokensLeft != null) tokensLeft = data.tokensLeft; + if (data.refillRate != null) refillRate = data.refillRate; +} + +async function waitForToken(): Promise { + if (tokensLeft > 0) return; + + const elapsed = (Date.now() - lastRequestTime) / 60_000; + const regenerated = Math.floor(elapsed * Math.max(1, refillRate)); + if (regenerated > 0) { + tokensLeft += regenerated; + return; + } + + const waitMs = + Math.ceil((1 / Math.max(1, refillRate)) * 60_000) - + (Date.now() - lastRequestTime); + if (waitMs > 0) { + console.log( + `Keepa tokens exhausted. Waiting ${Math.ceil(waitMs / 1000)}s for token regeneration...`, + ); + await wait(waitMs); + } + tokensLeft = 1; +} + +function computeWaitMsFromRefill(refillIn?: number): number { + if ( + typeof refillIn === "number" && + Number.isFinite(refillIn) && + refillIn >= 0 + ) { + return Math.max( + Math.ceil(refillIn) + KEEP_RETRY_BUFFER_MS, + KEEP_RETRY_BUFFER_MS, + ); + } + + return Math.ceil((1 / Math.max(1, refillRate)) * 60_000) + KEEP_RETRY_BUFFER_MS; +} + +function parseErrorPayload(text: string): KeepaApiResponse | null { + try { + const parsed = JSON.parse(text) as KeepaApiResponse; + return parsed && typeof parsed === "object" ? parsed : null; + } catch { + return null; + } +} + +function wait(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +if (import.meta.main) { + const args = parseArgs(); + runStalker(args) + .then((stats) => { + console.log( + `Stalker complete: scanned=${stats.scannedAsins}, matched_sellers=${stats.matchedSellers}, persisted_inventory_asins=${stats.persistedInventoryAsins}, failed=${stats.failedAsins}`, + ); + }) + .catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }) + .finally(() => { + closeDb(); + }); +} diff --git a/src/web/frontend.tsx b/src/web/frontend.tsx index 17a6df7..19aa424 100644 --- a/src/web/frontend.tsx +++ b/src/web/frontend.tsx @@ -109,6 +109,45 @@ type ProductListResponse = { totalPages: number; }; +type StalkerResultItem = { + runId: number; + started_at: string; + status: string; + input_file: string; + source_asin: string; + title: string | null; + offer_count: number; + candidate_seller_count: number; + matched_seller_count: number; + scan_fetched_at: string; + seller_id: string; + seller_name: string | null; + rating: number | null; + rating_count: number | null; + storefront_asin_total: number | null; + persisted_inventory_sample_count: number | null; + offer_price: number | null; + condition: string | null; + is_fba: number | null; + stock: number | null; + persisted_inventory_asin_count: number; + inventory_sample_asins: string | null; +}; + +type StalkerResultsResponse = { + items: StalkerResultItem[]; + summary: { + runs: number; + sourceAsins: number; + sellers: number; + persistedInventoryAsins: number; + }; + page: number; + pageSize: number; + total: number; + totalPages: number; +}; + type SortState = { field: string; direction: SortDirection; @@ -139,6 +178,11 @@ function formatAmazonSeller(value: number | null | undefined): string { return value === 1 ? "Yes" : "No"; } +function formatBoolean(value: number | null | undefined): string { + if (value === null || value === undefined) return "-"; + return value === 1 ? "Yes" : "No"; +} + function buildSortValue(sort: SortState): string { return `${sort.field}:${sort.direction}`; } @@ -197,9 +241,11 @@ function detectAnomaly(item: ResultItem): string { function Dashboard({ onOpenRun, onOpenProducts, + onOpenStalker, }: { onOpenRun: (run: Run) => void; onOpenProducts: (verdict: VerdictFilter) => void; + onOpenStalker: () => void; }) { const [runs, setRuns] = useState(null); const [loading, setLoading] = useState(false); @@ -288,7 +334,10 @@ function Dashboard({ return (
-

Runs Dashboard

+
+

Runs Dashboard

+ +
@@ -848,10 +897,166 @@ function ProductList({ verdict, onBack }: { verdict: VerdictFilter; onBack: () = ); } +function StalkerExplorer({ onBack }: { onBack: () => void }) { + const [results, setResults] = useState(null); + const [loading, setLoading] = useState(false); + const [search, setSearch] = useState(""); + const [sellerId, setSellerId] = useState(""); + const [runId, setRunId] = useState(""); + const [minRatingCount, setMinRatingCount] = useState("1"); + const [maxRatingCount, setMaxRatingCount] = useState("30"); + const [page, setPage] = useState(1); + const [pageSize, setPageSize] = useState(25); + const [sort, setSort] = useState({ field: "started_at", direction: "DESC" }); + + useEffect(() => { + let cancelled = false; + async function load() { + setLoading(true); + const params = new URLSearchParams({ + page: String(page), + pageSize: String(pageSize), + sort: buildSortValue(sort), + }); + if (search) params.set("q", search); + if (sellerId) params.set("sellerId", sellerId); + if (runId) params.set("runId", runId); + if (minRatingCount) params.set("minRatingCount", minRatingCount); + if (maxRatingCount) params.set("maxRatingCount", maxRatingCount); + + const res = await fetch(`/api/stalker/results?${params.toString()}`); + const payload = (await res.json()) as StalkerResultsResponse; + if (!cancelled) { + setResults(payload); + setLoading(false); + } + } + + load(); + return () => { + cancelled = true; + }; + }, [search, sellerId, runId, minRatingCount, maxRatingCount, page, pageSize, sort]); + + return ( +
+ + +
+

Stalker Results

+
+ +
+
+
Runs
+
{formatNumber(results?.summary.runs)}
+
+
+
Source ASINs
+
{formatNumber(results?.summary.sourceAsins)}
+
+
+
Matched sellers
+
{formatNumber(results?.summary.sellers)}
+
+
+
Persisted inventory ASINs
+
{formatNumber(results?.summary.persistedInventoryAsins)}
+
+
+ +
+
+ { setPage(1); setSearch(e.target.value); }} placeholder="Search source ASIN/title/seller/inventory" /> + { setPage(1); setSellerId(e.target.value.toUpperCase()); }} placeholder="Seller ID" /> + { setPage(1); setRunId(e.target.value); }} placeholder="Run ID" /> + { setPage(1); setMinRatingCount(e.target.value); }} placeholder="Min rating count" /> + { setPage(1); setMaxRatingCount(e.target.value); }} placeholder="Max rating count" /> + +
+
+ +
+
+ + + + + + + + + + + + + + + + + + + + + {loading ? ( + + ) : results?.items.length ? ( + results.items.map((item) => { + const inventorySample = (item.inventory_sample_asins ?? "") + .split(",") + .filter(Boolean) + .slice(0, 20); + return ( + + + + + + + + + + + + + + + + + ); + }) + ) : ( + + )} + +
FBAInventory ASIN sample
Loading...
{item.runId}{formatDate(item.started_at)}{item.source_asin}{item.title || "-"}{item.seller_id}{item.seller_name || "-"}{formatNumber(item.rating)}{formatNumber(item.rating_count)}{formatCurrency(item.offer_price)}{formatBoolean(item.is_fba)}{formatNumber(item.stock)}{formatNumber(item.storefront_asin_total)}{formatNumber(item.persisted_inventory_asin_count)} + {inventorySample.length === 0 ? "-" : inventorySample.map((asin) => ( + {asin} + ))} +
No stalker results found
+
+
+
Showing {results?.items.length ?? 0} of {results?.total ?? 0}
+
+ + Page {results?.page ?? page} / {results?.totalPages ?? 1} + +
+
+
+
+ ); +} + type AppRoute = | { kind: "dashboard" } | { kind: "run"; processType: ProcessType; runId: number } - | { kind: "products"; verdict: VerdictFilter }; + | { kind: "products"; verdict: VerdictFilter } + | { kind: "stalker" }; function parseRoute(pathname: string, search: string): AppRoute { const runMatch = pathname.match(/^\/runs\/(lead_analysis|category_analysis)\/(\d+)$/); @@ -866,6 +1071,10 @@ function parseRoute(pathname: string, search: string): AppRoute { return { kind: "products", verdict }; } + if (pathname === "/stalker") { + return { kind: "stalker" }; + } + return { kind: "dashboard" }; } @@ -890,6 +1099,11 @@ function App() { setRoute({ kind: "products", verdict }); } + function openStalker() { + history.pushState({}, "", "/stalker"); + setRoute({ kind: "stalker" }); + } + function backToDashboard() { history.pushState({}, "", "/"); setRoute({ kind: "dashboard" }); @@ -903,7 +1117,11 @@ function App() { return ; } - return ; + if (route.kind === "stalker") { + return ; + } + + return ; } const root = document.getElementById("root"); diff --git a/src/web/styles.css b/src/web/styles.css index b1893a9..205f4dd 100644 --- a/src/web/styles.css +++ b/src/web/styles.css @@ -41,6 +41,13 @@ p { gap: 10px; } +.section-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; +} + .toolbar input, .toolbar select, button { @@ -91,6 +98,23 @@ td { overflow-wrap: anywhere; } +.inventory-col { + min-width: 360px; + max-width: 520px; + white-space: normal; + overflow-wrap: anywhere; +} + +.inventory-col a { + display: inline-block; + margin-right: 8px; + margin-bottom: 4px; +} + +.stalker-table { + min-width: 1320px; +} + th { background: #fafafb; font-weight: 600; @@ -262,4 +286,9 @@ th button { .spark-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } + + .section-header { + align-items: flex-start; + flex-direction: column; + } }