From a7c0e44e3d2c79d2c1a44d3ed784427a8703dc4c Mon Sep 17 00:00:00 2001 From: Victor Noguera Date: Tue, 19 May 2026 18:10:01 -0400 Subject: [PATCH 1/9] feat: add Stalker results page with filtering and pagination - Introduced StalkerResultItem and StalkerResultsResponse types for handling API responses. - Implemented StalkerExplorer component for displaying Stalker results with search and filter options. - Added sorting functionality for Stalker results table. - Enhanced Dashboard to include a button for navigating to Stalker results. - Updated routing to support Stalker results page. - Improved styles for section headers and inventory columns in the results table. --- package.json | 1 + src/database.ts | 122 +++++ src/server.ts | 194 +++++++ src/stalker.test.ts | 281 ++++++++++ src/stalker.ts | 1189 ++++++++++++++++++++++++++++++++++++++++++ src/web/frontend.tsx | 224 +++++++- src/web/styles.css | 29 ++ 7 files changed, 2037 insertions(+), 3 deletions(-) create mode 100644 src/stalker.test.ts create mode 100644 src/stalker.ts diff --git a/package.json b/package.json index b3374ab..4c95933 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "bestsellers": "bun run src/bestsellers-by-category.ts", "monthly-sold": "bun run src/top-monthly-sold-by-category.ts", "mid-range": "bun run src/mid-range-sellers-by-category.ts", + "stalker": "bun run src/stalker.ts", "upc": "bun run src/upc-lookup.ts", "upc-file": "bun run src/upc-file-analysis.ts", "start": "bun run src/index.ts", diff --git a/src/database.ts b/src/database.ts index 526a220..cdef940 100644 --- a/src/database.ts +++ b/src/database.ts @@ -333,4 +333,126 @@ export function initDb(dbPath: string): void { database.run( `CREATE INDEX IF NOT EXISTS idx_product_results_fetched_at ON product_analysis_results(fetched_at DESC);`, ); + initStalkerDb(database); +} + +export function initStalkerDb(database: Database): void { + resetLegacyStalkerSchema(database); + + database.run(` + CREATE TABLE IF NOT EXISTS stalker_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + input_file TEXT NOT NULL, + started_at TEXT NOT NULL, + completed_at TEXT, + requested_asins INTEGER NOT NULL DEFAULT 0, + skipped_asins INTEGER NOT NULL DEFAULT 0, + scanned_asins INTEGER NOT NULL DEFAULT 0, + source_asins_with_matches INTEGER NOT NULL DEFAULT 0, + candidate_sellers INTEGER NOT NULL DEFAULT 0, + qualifying_sellers INTEGER NOT NULL DEFAULT 0, + matched_sellers INTEGER NOT NULL DEFAULT 0, + seller_metadata_requests INTEGER NOT NULL DEFAULT 0, + seller_storefront_requests INTEGER NOT NULL DEFAULT 0, + persisted_inventory_asins INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL, + error_message TEXT + ); + `); + + database.run(` + CREATE TABLE IF NOT EXISTS stalker_asin_scans ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_id INTEGER NOT NULL, + source_asin TEXT NOT NULL, + title TEXT, + offer_count INTEGER NOT NULL DEFAULT 0, + candidate_seller_count INTEGER NOT NULL DEFAULT 0, + matched_seller_count INTEGER NOT NULL DEFAULT 0, + fetched_at TEXT NOT NULL, + raw_product_json TEXT, + UNIQUE(run_id, source_asin), + FOREIGN KEY (run_id) REFERENCES stalker_runs(id) ON DELETE CASCADE + ); + `); + + database.run(` + CREATE TABLE IF NOT EXISTS stalker_sellers ( + seller_id TEXT PRIMARY KEY, + seller_name TEXT, + rating REAL, + rating_count INTEGER, + storefront_asin_total INTEGER, + persisted_inventory_sample_count INTEGER, + last_updated_at TEXT NOT NULL, + raw_seller_json TEXT + ); + `); + + database.run(` + CREATE TABLE IF NOT EXISTS stalker_asin_sellers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + scan_id INTEGER NOT NULL, + seller_id TEXT NOT NULL, + offer_price REAL, + condition TEXT, + is_fba INTEGER, + stock INTEGER, + seller_rating REAL, + seller_rating_count INTEGER, + raw_offer_json TEXT, + UNIQUE(scan_id, seller_id), + FOREIGN KEY (scan_id) REFERENCES stalker_asin_scans(id) ON DELETE CASCADE, + FOREIGN KEY (seller_id) REFERENCES stalker_sellers(seller_id) + ); + `); + + database.run(` + CREATE TABLE IF NOT EXISTS stalker_seller_inventory ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_id INTEGER NOT NULL, + seller_id TEXT NOT NULL, + asin TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + raw_inventory_json TEXT, + UNIQUE(run_id, seller_id, asin), + FOREIGN KEY (run_id) REFERENCES stalker_runs(id) ON DELETE CASCADE, + FOREIGN KEY (seller_id) REFERENCES stalker_sellers(seller_id) + ); + `); + + database.run( + `CREATE INDEX IF NOT EXISTS idx_stalker_runs_started_at ON stalker_runs(started_at DESC);`, + ); + database.run( + `CREATE INDEX IF NOT EXISTS idx_stalker_scans_run_id ON stalker_asin_scans(run_id);`, + ); + database.run( + `CREATE INDEX IF NOT EXISTS idx_stalker_scans_source_asin ON stalker_asin_scans(source_asin);`, + ); + database.run( + `CREATE INDEX IF NOT EXISTS idx_stalker_asin_sellers_seller_id ON stalker_asin_sellers(seller_id);`, + ); + database.run( + `CREATE INDEX IF NOT EXISTS idx_stalker_inventory_seller_id ON stalker_seller_inventory(seller_id);`, + ); + database.run( + `CREATE INDEX IF NOT EXISTS idx_stalker_inventory_asin ON stalker_seller_inventory(asin);`, + ); +} + +function resetLegacyStalkerSchema(database: Database): void { + const runColumns = database + .query("PRAGMA table_info(stalker_runs)") + .all() as Array<{ name: string }>; + if (runColumns.length === 0) return; + + const columnNames = new Set(runColumns.map((column) => column.name)); + if (columnNames.has("scanned_asins")) return; + + database.run("DROP TABLE IF EXISTS stalker_seller_inventory"); + database.run("DROP TABLE IF EXISTS stalker_asin_sellers"); + database.run("DROP TABLE IF EXISTS stalker_sellers"); + database.run("DROP TABLE IF EXISTS stalker_asin_scans"); + database.run("DROP TABLE IF EXISTS stalker_runs"); } diff --git a/src/server.ts b/src/server.ts index 9fef320..2221df6 100644 --- a/src/server.ts +++ b/src/server.ts @@ -53,6 +53,31 @@ type ProductListRecord = { fetched_at: string; }; +type StalkerResultRecord = { + runId: number; + started_at: string; + status: string; + input_file: string; + source_asin: string; + title: string | null; + offer_count: number; + candidate_seller_count: number; + matched_seller_count: number; + scan_fetched_at: string; + seller_id: string; + seller_name: string | null; + rating: number | null; + rating_count: number | null; + storefront_asin_total: number | null; + persisted_inventory_sample_count: number | null; + offer_price: number | null; + condition: string | null; + is_fba: number | null; + stock: number | null; + persisted_inventory_asin_count: number; + inventory_sample_asins: string | null; +}; + const DB_PATH = process.env.RESULTS_DB_PATH || path.join("db", "results.db"); const DEFAULT_PAGE_SIZE = 25; const MAX_PAGE_SIZE = 200; @@ -629,6 +654,170 @@ function getProductList(filters: URLSearchParams) { }; } +function parseStalkerFilters(filters: URLSearchParams) { + const q = filters.get("q")?.trim() || ""; + const sellerId = filters.get("sellerId")?.trim().toUpperCase() || ""; + const runIdRaw = filters.get("runId")?.trim() || ""; + const minRatingCountRaw = filters.get("minRatingCount")?.trim() || ""; + const maxRatingCountRaw = filters.get("maxRatingCount")?.trim() || ""; + + const conditions: string[] = []; + const params: Array = []; + + if (runIdRaw) { + const runId = Number(runIdRaw); + if (Number.isInteger(runId) && runId > 0) { + conditions.push("r.id = ?"); + params.push(runId); + } + } + + if (sellerId) { + conditions.push("s.seller_id = ?"); + params.push(sellerId); + } + + if (minRatingCountRaw) { + conditions.push("s.rating_count >= ?"); + params.push(Number(minRatingCountRaw)); + } + + if (maxRatingCountRaw) { + conditions.push("s.rating_count <= ?"); + params.push(Number(maxRatingCountRaw)); + } + + if (q) { + const wildcard = `%${q}%`; + conditions.push( + `(sc.source_asin LIKE ? OR sc.title LIKE ? OR s.seller_id LIKE ? OR s.seller_name LIKE ? OR EXISTS ( + SELECT 1 FROM stalker_seller_inventory inv_q + WHERE inv_q.run_id = r.id + AND inv_q.seller_id = s.seller_id + AND inv_q.asin LIKE ? + ))`, + ); + params.push(wildcard, wildcard, wildcard, wildcard, wildcard); + } + + return { + where: conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "", + params, + }; +} + +function parseStalkerSort(sortParam: string | null): string { + const allowedSort = new Set([ + "runId", + "started_at", + "source_asin", + "title", + "seller_id", + "seller_name", + "rating", + "rating_count", + "offer_price", + "stock", + "persisted_inventory_asin_count", + "storefront_asin_total", + "scan_fetched_at", + ]); + const parsed = parseSort( + sortParam, + allowedSort, + "started_at DESC, runId DESC, source_asin ASC", + ); + + return parsed + .replaceAll("runId", "runId") + .replaceAll("rating_count", "rating_count") + .replaceAll("persisted_inventory_asin_count", "persisted_inventory_asin_count") + .replaceAll("storefront_asin_total", "storefront_asin_total"); +} + +function getStalkerResults(filters: URLSearchParams) { + const page = parseIntParam(filters.get("page"), 1); + const pageSize = Math.min( + parseIntParam(filters.get("pageSize"), DEFAULT_PAGE_SIZE), + MAX_PAGE_SIZE, + ); + const offset = (page - 1) * pageSize; + const { where, params } = parseStalkerFilters(filters); + const orderBy = parseStalkerSort(filters.get("sort")); + + const baseSelect = ` + SELECT + r.id AS runId, + r.started_at, + r.status, + r.input_file, + sc.source_asin, + sc.title, + sc.offer_count, + sc.candidate_seller_count, + sc.matched_seller_count, + sc.fetched_at AS scan_fetched_at, + s.seller_id, + s.seller_name, + s.rating, + s.rating_count, + s.storefront_asin_total, + s.persisted_inventory_sample_count, + sas.offer_price, + sas.condition, + sas.is_fba, + sas.stock, + COUNT(DISTINCT inv.asin) AS persisted_inventory_asin_count, + GROUP_CONCAT(DISTINCT inv.asin) AS inventory_sample_asins + FROM stalker_asin_sellers sas + JOIN stalker_asin_scans sc ON sc.id = sas.scan_id + JOIN stalker_runs r ON r.id = sc.run_id + JOIN stalker_sellers s ON s.seller_id = sas.seller_id + LEFT JOIN stalker_seller_inventory inv + ON inv.run_id = r.id + AND inv.seller_id = s.seller_id + ${where} + GROUP BY sas.id + `; + + const totalRow = db + .query(`SELECT COUNT(*) AS total FROM (${baseSelect}) stalker_rows`) + .get(...params) as { total: number }; + + const summary = db + .query( + `SELECT + COUNT(DISTINCT runId) AS runs, + COUNT(DISTINCT source_asin) AS sourceAsins, + COUNT(DISTINCT seller_id) AS sellers, + COALESCE(SUM(persisted_inventory_asin_count), 0) AS persistedInventoryAsins + FROM (${baseSelect}) stalker_rows`, + ) + .get(...params) as { + runs: number; + sourceAsins: number; + sellers: number; + persistedInventoryAsins: number; + }; + + const items = db + .query( + `SELECT * FROM (${baseSelect}) stalker_rows + ORDER BY ${orderBy} + LIMIT ? OFFSET ?`, + ) + .all(...params, pageSize, offset) as StalkerResultRecord[]; + + return { + items, + summary, + page, + pageSize, + total: totalRow.total, + totalPages: Math.max(1, Math.ceil(totalRow.total / pageSize)), + }; +} + function getRun(processType: ProcessType, runId: number) { if (processType === "lead_analysis") { const run = db @@ -1259,6 +1448,7 @@ const server = Bun.serve({ routes: { "/": index, "/products": index, + "/stalker": index, "/runs/:processType/:runId": index, "/api/runs": (req) => { const url = new URL(req.url); @@ -1268,6 +1458,10 @@ const server = Bun.serve({ const url = new URL(req.url); return json(getProductList(url.searchParams)); }, + "/api/stalker/results": (req) => { + const url = new URL(req.url); + return json(getStalkerResults(url.searchParams)); + }, "/api/upc/map": async (req) => { let upcs: string[]; try { diff --git a/src/stalker.test.ts b/src/stalker.test.ts new file mode 100644 index 0000000..42ffc08 --- /dev/null +++ b/src/stalker.test.ts @@ -0,0 +1,281 @@ +import { afterAll, beforeEach, expect, mock, test } from "bun:test"; +import { mkdirSync, rmSync } from "node:fs"; +import path from "node:path"; +import * as XLSX from "xlsx"; +import { closeDb, getDb, initDb } from "./database.ts"; +import { + extractLiveOfferSellerCandidates, + isQualifyingSeller, + readAsinsFromXlsx, + runStalker, +} from "./stalker.ts"; + +const TEST_DIR = path.join(process.cwd(), "test_output", "stalker"); +const originalFetch = globalThis.fetch; +const originalKeepaKey = Bun.env.KEEPA_API_KEY; + +beforeEach(() => { + closeDb(); + rmSync(TEST_DIR, { recursive: true, force: true }); + mkdirSync(TEST_DIR, { recursive: true }); + globalThis.fetch = originalFetch; + Bun.env.KEEPA_API_KEY = "test-keepa-key"; +}); + +afterAll(() => { + globalThis.fetch = originalFetch; + if (originalKeepaKey == null) { + delete Bun.env.KEEPA_API_KEY; + } else { + Bun.env.KEEPA_API_KEY = originalKeepaKey; + } + closeDb(); + rmSync(TEST_DIR, { recursive: true, force: true }); +}); + +test("readAsinsFromXlsx extracts valid ASINs and deduplicates in order", () => { + const filePath = path.join(TEST_DIR, "asins.xlsx"); + const workbook = XLSX.utils.book_new(); + const sheet = XLSX.utils.json_to_sheet([ + { ASIN: "b000000001" }, + { ASIN: "invalid" }, + { ASIN: "B000000002" }, + { ASIN: "B000000001" }, + { ASIN: "" }, + ]); + XLSX.utils.book_append_sheet(workbook, sheet, "Input"); + XLSX.writeFile(workbook, filePath); + + expect(readAsinsFromXlsx(filePath)).toEqual(["B000000001", "B000000002"]); +}); + +test("isQualifyingSeller accepts rating counts from 1 to 30 only", () => { + expect(isQualifyingSeller({ ratingCount: null })).toBe(false); + expect(isQualifyingSeller({ ratingCount: 0 })).toBe(false); + expect(isQualifyingSeller({ ratingCount: 1 })).toBe(true); + expect(isQualifyingSeller({ ratingCount: 30 })).toBe(true); + expect(isQualifyingSeller({ ratingCount: 31 })).toBe(false); +}); + +test("extractLiveOfferSellerCandidates ignores Amazon, missing seller IDs, and duplicates", () => { + const offers = extractLiveOfferSellerCandidates({ + offers: [ + { sellerId: "ATVPDKIKX0DER", price: 1999 }, + { price: 1899 }, + { sellerId: "A1SELLER", price: 1599, isFBA: true, stock: 4 }, + { sellerId: "A1SELLER", price: 1499 }, + { sellerID: "A2SELLER", currentPrice: 2499, isFba: false }, + ], + }); + + expect(offers.map((offer) => offer.sellerId)).toEqual([ + "A1SELLER", + "A2SELLER", + ]); + expect(offers[0]?.offerPrice).toBe(15.99); + expect(offers[0]?.isFba).toBe(true); + expect(offers[0]?.stock).toBe(4); +}); + +test("initDb creates stalker tables and indexes", () => { + const dbPath = path.join(TEST_DIR, "schema.sqlite"); + initDb(dbPath); + const db = getDb(dbPath); + + const tables = db + .query( + `SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'stalker_%' ORDER BY name`, + ) + .all() as Array<{ name: string }>; + expect(tables.map((row) => row.name)).toEqual([ + "stalker_asin_scans", + "stalker_asin_sellers", + "stalker_runs", + "stalker_seller_inventory", + "stalker_sellers", + ]); + + const indexes = db + .query( + `SELECT name FROM sqlite_master WHERE type = 'index' AND name LIKE 'idx_stalker_%' ORDER BY name`, + ) + .all() as Array<{ name: string }>; + expect(indexes.length).toBeGreaterThanOrEqual(6); +}); + +test("runStalker fetches product offers, filters sellers, and persists storefront inventory", async () => { + const inputPath = path.join(TEST_DIR, "input.xlsx"); + const dbPath = path.join(TEST_DIR, "stalker.sqlite"); + const workbook = XLSX.utils.book_new(); + XLSX.utils.book_append_sheet( + workbook, + XLSX.utils.json_to_sheet([{ asin: "B000000001" }]), + "Input", + ); + XLSX.writeFile(workbook, inputPath); + + const fetchMock = mock(async (input: string | URL | Request) => { + const rawUrl = + typeof input === "string" + ? input + : input instanceof URL + ? input.toString() + : input.url; + const url = new URL(rawUrl); + + if (url.pathname === "/product") { + expect(url.searchParams.get("asin")).toBe("B000000001"); + expect(url.searchParams.get("offers")).toBe("20"); + expect(url.searchParams.get("only-live-offers")).toBe("1"); + expect(url.searchParams.has("stock")).toBe(false); + + return new Response( + JSON.stringify({ + products: [ + { + asin: "B000000001", + title: "Tracked Product", + offers: [ + { + sellerId: "AQUALIFIED", + price: 1999, + condition: "New", + isFBA: true, + stock: 3, + }, + { + sellerId: "AOLDSELLER", + price: 2099, + }, + ], + }, + ], + tokensLeft: 10, + refillRate: 10, + }), + { status: 200 }, + ); + } + + if (url.pathname === "/seller") { + const wantsStorefront = url.searchParams.get("storefront") === "1"; + if (wantsStorefront) { + expect(url.searchParams.get("update")).toBe("168"); + } + const sellerId = url.searchParams.get("seller"); + + return new Response( + JSON.stringify({ + sellers: { + ...(!wantsStorefront && sellerId === "AQUALIFIED,AOLDSELLER" + ? { + AQUALIFIED: { + sellerName: "New Seller", + currentRating: 96, + currentRatingCount: 12, + }, + AOLDSELLER: { + sellerName: "Old Seller", + currentRating: 99, + currentRatingCount: 120, + }, + } + : {}), + ...(wantsStorefront && sellerId === "AQUALIFIED" + ? { + AQUALIFIED: { + sellerName: "New Seller", + currentRating: 96, + currentRatingCount: 12, + asinList: ["B111111111", "B222222222"], + }, + } + : {}), + }, + tokensLeft: 10, + refillRate: 10, + }), + { status: 200 }, + ); + } + + return new Response("not found", { status: 404 }); + }); + globalThis.fetch = fetchMock as unknown as typeof globalThis.fetch; + + const stats = await runStalker({ + input: inputPath, + dbPath, + maxAsins: null, + storefrontUpdateHours: 168, + offerLimit: 20, + sellerLimit: 30, + inventoryLimit: 200, + sellerCacheHours: 168, + includeStock: false, + dryRun: false, + resume: true, + maxSellerRequests: null, + }); + + expect(stats.scannedAsins).toBe(1); + expect(stats.sourceAsinsWithMatches).toBe(1); + expect(stats.matchedSellers).toBe(1); + expect(stats.persistedInventoryAsins).toBe(2); + expect(stats.failedAsins).toBe(0); + expect(stats.candidateSellers).toBe(2); + expect(stats.qualifyingSellers).toBe(1); + expect(stats.sellerMetadataRequests).toBe(1); + expect(stats.sellerStorefrontRequests).toBe(1); + const sellerCalls = fetchMock.mock.calls.filter((call) => { + const rawUrl = + typeof call[0] === "string" + ? call[0] + : call[0] instanceof URL + ? call[0].toString() + : (call[0] as Request).url; + return new URL(rawUrl).pathname === "/seller"; + }); + expect(sellerCalls.length).toBe(2); + + const db = getDb(dbPath); + const run = db.query("SELECT * FROM stalker_runs").get() as any; + expect(run.status).toBe("completed"); + expect(run.requested_asins).toBe(1); + expect(run.scanned_asins).toBe(1); + expect(run.source_asins_with_matches).toBe(1); + expect(run.candidate_sellers).toBe(2); + expect(run.qualifying_sellers).toBe(1); + expect(run.matched_sellers).toBe(1); + expect(run.seller_metadata_requests).toBe(1); + expect(run.seller_storefront_requests).toBe(1); + expect(run.persisted_inventory_asins).toBe(2); + + const scan = db.query("SELECT * FROM stalker_asin_scans").get() as any; + expect(scan.source_asin).toBe("B000000001"); + expect(scan.title).toBe("Tracked Product"); + expect(scan.offer_count).toBe(2); + expect(scan.candidate_seller_count).toBe(2); + expect(scan.matched_seller_count).toBe(1); + + const sellers = db.query("SELECT * FROM stalker_sellers").all() as any[]; + expect(sellers.length).toBe(1); + expect(sellers[0].seller_id).toBe("AQUALIFIED"); + expect(sellers[0].rating_count).toBe(12); + expect(sellers[0].storefront_asin_total).toBe(2); + expect(sellers[0].persisted_inventory_sample_count).toBe(2); + + const asinSellers = db.query("SELECT * FROM stalker_asin_sellers").all() as any[]; + expect(asinSellers.length).toBe(1); + expect(asinSellers[0].offer_price).toBe(19.99); + expect(asinSellers[0].is_fba).toBe(1); + expect(asinSellers[0].stock).toBe(3); + + const inventory = db + .query("SELECT asin FROM stalker_seller_inventory ORDER BY asin") + .all() as Array<{ asin: string }>; + expect(inventory.map((row) => row.asin)).toEqual([ + "B111111111", + "B222222222", + ]); +}); diff --git a/src/stalker.ts b/src/stalker.ts new file mode 100644 index 0000000..f4b6563 --- /dev/null +++ b/src/stalker.ts @@ -0,0 +1,1189 @@ +import * as XLSX from "xlsx"; +import path from "node:path"; +import { type Database, closeDb, getDb, initDb } from "./database.ts"; + +const KEEPA_BASE = "https://api.keepa.com"; +const DOMAIN_US = "1"; +const AMAZON_US_SELLER_ID = "ATVPDKIKX0DER"; +const ASIN_REGEX = /^B[0-9A-Z]{9}$/; +const DEFAULT_DB_PATH = path.join(process.cwd(), "db", "results.db"); +const DEFAULT_STOREFRONT_UPDATE_HOURS = 168; +const DEFAULT_OFFER_LIMIT = 100; +const DEFAULT_SELLER_LIMIT = 30; +const DEFAULT_INVENTORY_LIMIT = 200; +const DEFAULT_SELLER_CACHE_HOURS = 168; +const MAX_SELLERS_PER_METADATA_REQUEST = 100; +const MAX_KEEPA_RETRIES = 4; +const KEEP_RETRY_BUFFER_MS = 250; + +type KeepaApiResponse = { + products?: Record[]; + sellers?: Record | Record[]; + tokensLeft?: number; + refillRate?: number; + refillIn?: number; +}; + +export type StalkerArgs = { + input: string; + dbPath: string; + maxAsins: number | null; + storefrontUpdateHours: number; + offerLimit: number; + sellerLimit: number; + inventoryLimit: number; + sellerCacheHours: number; + includeStock: boolean; + dryRun: boolean; + resume: boolean; + maxSellerRequests: number | null; +}; + +export type StalkerOffer = { + sellerId: string; + offerPrice: number | null; + condition: string | null; + isFba: boolean | null; + stock: number | null; + rawOffer: Record; +}; + +export type StalkerSeller = { + sellerId: string; + sellerName: string | null; + rating: number | null; + ratingCount: number | null; + storefrontAsins: string[]; + storefrontItems: StalkerInventoryItem[]; + storefrontAsinTotal: number; + rawSeller: Record; +}; + +type StalkerInventoryItem = { + asin: string; + rawInventory: unknown; +}; + +type StalkerAsinResult = { + asin: string; + title: string | null; + offerCount: number; + candidateSellerCount: number; + matchedSellers: Array<{ + seller: StalkerSeller; + offer: StalkerOffer; + }>; + product: Record | null; + error?: string; +}; + +type StalkerRunStats = { + scannedAsins: number; + sourceAsinsWithMatches: number; + matchedSellers: number; + persistedInventoryAsins: number; + failedAsins: number; + skippedAsins: number; + candidateSellers: number; + qualifyingSellers: number; + sellerMetadataRequests: number; + sellerStorefrontRequests: number; + stoppedEarly: boolean; +}; + +type StalkerRunContext = { + database: Database | null; + metadataCache: Map; + storefrontCache: Map; + stats: StalkerRunStats; +}; + +let tokensLeft = 1; +let refillRate = 1; +let lastRequestTime = 0; + +export function parseArgs(argv = process.argv.slice(2)): StalkerArgs { + const input = readFlagValue(argv, "--input"); + if (!input) { + printUsageAndExit("Missing required --input file."); + } + + const dbPath = readFlagValue(argv, "--db") ?? DEFAULT_DB_PATH; + const maxAsinsRaw = readFlagValue(argv, "--max-asins"); + const storefrontUpdateRaw = readFlagValue(argv, "--storefront-update-hours"); + const offerLimitRaw = readFlagValue(argv, "--offer-limit"); + const sellerLimitRaw = readFlagValue(argv, "--seller-limit"); + const inventoryLimitRaw = readFlagValue(argv, "--inventory-limit"); + const sellerCacheHoursRaw = readFlagValue(argv, "--seller-cache-hours"); + const maxSellerRequestsRaw = readFlagValue(argv, "--max-seller-requests"); + + const maxAsins = maxAsinsRaw ? Number(maxAsinsRaw) : null; + const storefrontUpdateHours = storefrontUpdateRaw + ? Number(storefrontUpdateRaw) + : DEFAULT_STOREFRONT_UPDATE_HOURS; + const offerLimit = offerLimitRaw ? Number(offerLimitRaw) : DEFAULT_OFFER_LIMIT; + const sellerLimit = sellerLimitRaw ? Number(sellerLimitRaw) : DEFAULT_SELLER_LIMIT; + const inventoryLimit = inventoryLimitRaw + ? Number(inventoryLimitRaw) + : DEFAULT_INVENTORY_LIMIT; + const sellerCacheHours = sellerCacheHoursRaw + ? Number(sellerCacheHoursRaw) + : DEFAULT_SELLER_CACHE_HOURS; + const maxSellerRequests = maxSellerRequestsRaw + ? Number(maxSellerRequestsRaw) + : null; + const includeStock = hasFlag(argv, "--include-stock"); + const dryRun = hasFlag(argv, "--dry-run"); + const resume = !hasFlag(argv, "--no-resume"); + + if (maxAsins != null && (!Number.isInteger(maxAsins) || maxAsins <= 0)) { + printUsageAndExit("--max-asins must be a positive integer."); + } + + if ( + !Number.isInteger(storefrontUpdateHours) || + storefrontUpdateHours < 0 + ) { + printUsageAndExit( + "--storefront-update-hours must be a non-negative integer.", + ); + } + + if (!Number.isInteger(offerLimit) || offerLimit < 20 || offerLimit > 100) { + printUsageAndExit("--offer-limit must be an integer from 20 to 100."); + } + + if (!Number.isInteger(sellerLimit) || sellerLimit <= 0) { + printUsageAndExit("--seller-limit must be a positive integer."); + } + + if (!Number.isInteger(inventoryLimit) || inventoryLimit < 0) { + printUsageAndExit("--inventory-limit must be a non-negative integer."); + } + + if (!Number.isInteger(sellerCacheHours) || sellerCacheHours < 0) { + printUsageAndExit("--seller-cache-hours must be a non-negative integer."); + } + + if ( + maxSellerRequests != null && + (!Number.isInteger(maxSellerRequests) || maxSellerRequests <= 0) + ) { + printUsageAndExit("--max-seller-requests must be a positive integer."); + } + + return { + input, + dbPath, + maxAsins, + storefrontUpdateHours, + offerLimit, + sellerLimit, + inventoryLimit, + sellerCacheHours, + includeStock, + dryRun, + resume, + maxSellerRequests, + }; +} + +export function readAsinsFromXlsx(filePath: string): string[] { + const workbook = XLSX.readFile(filePath); + const sheetName = workbook.SheetNames[0]; + if (!sheetName) throw new Error("No sheets found in file"); + + const sheet = workbook.Sheets[sheetName]; + if (!sheet) throw new Error("First sheet is missing"); + + const rows = XLSX.utils.sheet_to_json>(sheet, { + defval: "", + }); + if (rows.length === 0) throw new Error("File contains no data rows"); + + const headers = Object.keys(rows[0]!); + const asinColumn = headers.find((header) => normalizeHeader(header) === "asin"); + if (!asinColumn) { + throw new Error(`No ASIN column found. Available columns: ${headers.join(", ")}`); + } + + return extractAsinsFromRows(rows, asinColumn); +} + +export function extractAsinsFromRows( + rows: Array>, + asinColumn = "asin", +): string[] { + const asins: string[] = []; + const seen = new Set(); + + for (const row of rows) { + const asin = normalizeAsin(row[asinColumn]); + if (!asin || seen.has(asin)) continue; + seen.add(asin); + asins.push(asin); + } + + return asins; +} + +export function isQualifyingSeller(seller: { + ratingCount?: number | null; +}): boolean { + return ( + typeof seller.ratingCount === "number" && + Number.isFinite(seller.ratingCount) && + seller.ratingCount >= 1 && + seller.ratingCount <= 30 + ); +} + +export function extractLiveOfferSellerCandidates( + product: Record, +): StalkerOffer[] { + const offers = Array.isArray(product.offers) ? product.offers : []; + const bySeller = new Map(); + + for (const offer of offers) { + if (!offer || typeof offer !== "object") continue; + const sellerId = normalizeSellerId( + offer.sellerId ?? offer.sellerID ?? offer.seller_id, + ); + if (!sellerId || sellerId === AMAZON_US_SELLER_ID) continue; + if (bySeller.has(sellerId)) continue; + + bySeller.set(sellerId, { + sellerId, + offerPrice: extractOfferPrice(offer), + condition: extractString(offer.condition ?? offer.conditionComment), + isFba: extractBoolean(offer.isFBA ?? offer.isFba ?? offer.fba), + stock: extractNumber(offer.stock ?? offer.stockCount ?? offer.currentStock), + rawOffer: offer, + }); + } + + return Array.from(bySeller.values()); +} + +export async function runStalker(args: StalkerArgs): Promise { + const apiKey = Bun.env.KEEPA_API_KEY; + if (!apiKey) throw new Error("Missing required env var: KEEPA_API_KEY"); + + const allAsins = readAsinsFromXlsx(args.input); + const cappedAsins = + args.maxAsins == null ? allAsins : allAsins.slice(0, args.maxAsins); + + initDb(args.dbPath); + const database = getDb(args.dbPath); + const completedAsins = args.resume ? loadPreviouslyScannedAsins(database) : new Set(); + const asins = cappedAsins.filter((asin) => !completedAsins.has(asin)); + const runId = args.dryRun + ? null + : startStalkerRun(database, args.input, asins.length); + const stats: StalkerRunStats = { + scannedAsins: 0, + sourceAsinsWithMatches: 0, + matchedSellers: 0, + persistedInventoryAsins: 0, + failedAsins: 0, + skippedAsins: cappedAsins.length - asins.length, + candidateSellers: 0, + qualifyingSellers: 0, + sellerMetadataRequests: 0, + sellerStorefrontRequests: 0, + stoppedEarly: false, + }; + const context: StalkerRunContext = { + database, + metadataCache: new Map(), + storefrontCache: new Map(), + stats, + }; + + try { + if (args.dryRun) { + console.log("Stalker dry-run: product and seller metadata will be fetched, storefronts will not be fetched or persisted."); + } + if (stats.skippedAsins > 0) { + console.log(`Stalker resume: skipped ${stats.skippedAsins} previously scanned ASIN(s).`); + } + + for (const asin of asins) { + console.log(`Stalker: scanning ${asin} (${stats.scannedAsins + 1}/${asins.length})`); + + const result = await scanAsin(asin, args, apiKey, context).catch((error) => ({ + asin, + title: null, + offerCount: 0, + candidateSellerCount: 0, + matchedSellers: [], + product: null, + error: error instanceof Error ? error.message : String(error), + })); + + if (!args.dryRun && runId != null) { + persistAsinResult(database, runId, result); + } + stats.scannedAsins += 1; + stats.matchedSellers += result.matchedSellers.length; + stats.persistedInventoryAsins += sumInventoryAsins(result); + if (result.matchedSellers.length > 0) stats.sourceAsinsWithMatches += 1; + if (result.error) { + stats.failedAsins += 1; + console.warn(`Stalker: ${asin} failed: ${result.error}`); + } + + if (!args.dryRun && runId != null) { + refreshStalkerRun(database, runId, stats, "running"); + } + console.log( + `Stalker: ${asin} candidates=${result.candidateSellerCount}, matched=${result.matchedSellers.length}, persisted_inventory=${sumInventoryAsins(result)}`, + ); + + if (stats.stoppedEarly) { + console.log("Stalker: stopping early because max seller request budget was reached."); + break; + } + } + + if (!args.dryRun && runId != null) { + refreshStalkerRun( + database, + runId, + stats, + stats.stoppedEarly + ? "stopped" + : stats.failedAsins > 0 + ? "completed_with_errors" + : "completed", + ); + } + logRunSummary(stats, args); + return stats; + } catch (error) { + if (!args.dryRun && runId != null) { + finishStalkerRunWithError( + database, + runId, + stats, + error instanceof Error ? error.message : String(error), + ); + } + throw error; + } +} + +async function scanAsin( + asin: string, + args: StalkerArgs, + apiKey: string, + context: StalkerRunContext, +): Promise { + const product = await fetchKeepaProduct( + asin, + apiKey, + args.offerLimit, + args.includeStock, + ); + const offers = extractLiveOfferSellerCandidates(product).slice( + 0, + args.sellerLimit, + ); + context.stats.candidateSellers += offers.length; + + const metadata = await fetchSellerMetadata( + offers.map((offer) => offer.sellerId), + apiKey, + args, + context, + ); + const qualifyingOffers = offers.filter((offer) => { + const seller = metadata.get(offer.sellerId); + return seller ? isQualifyingSeller(seller) : false; + }); + context.stats.qualifyingSellers += qualifyingOffers.length; + + if (args.dryRun) { + console.log( + `Stalker dry-run estimate for ${asin}: storefront requests needed=${qualifyingOffers.length}, candidate sellers=${offers.length}`, + ); + } + + const storefronts = args.dryRun + ? new Map() + : await fetchQualifiedSellerStorefronts( + qualifyingOffers.map((offer) => offer.sellerId), + apiKey, + args, + context, + ); + + const matchedSellers = qualifyingOffers + .map((offer) => { + const seller = storefronts.get(offer.sellerId); + if (!seller || !isQualifyingSeller(seller)) return null; + return { seller, offer }; + }) + .filter( + (entry): entry is { seller: StalkerSeller; offer: StalkerOffer } => + entry != null, + ); + + return { + asin, + title: extractString(product.title), + offerCount: Array.isArray(product.offers) ? product.offers.length : 0, + candidateSellerCount: offers.length, + matchedSellers, + product, + }; +} + +async function fetchKeepaProduct( + asin: string, + apiKey: string, + offerLimit: number, + includeStock: boolean, +): Promise> { + const params = new URLSearchParams({ + key: apiKey, + domain: DOMAIN_US, + asin, + offers: String(offerLimit), + "only-live-offers": "1", + stats: "30", + days: "30", + }); + if (includeStock) { + params.set("stock", "1"); + } + const data = await fetchKeepaWithRetries( + `${KEEPA_BASE}/product?${params.toString()}`, + `product ${asin}`, + ); + const product = data.products?.[0]; + if (!product) throw new Error("Keepa returned no product"); + return product; +} + +async function fetchSellerMetadata( + sellerIds: string[], + apiKey: string, + args: StalkerArgs, + context: StalkerRunContext, +): Promise> { + const out = new Map(); + const uniqueSellerIds = Array.from(new Set(sellerIds)); + const missing: string[] = []; + + for (const sellerId of uniqueSellerIds) { + const cached = + context.metadataCache.get(sellerId) ?? + loadCachedSeller( + context.database, + sellerId, + args.sellerCacheHours, + false, + args.inventoryLimit, + ); + if (cached) { + context.metadataCache.set(sellerId, cached); + out.set(sellerId, cached); + continue; + } + missing.push(sellerId); + } + + for (let i = 0; i < missing.length; i += MAX_SELLERS_PER_METADATA_REQUEST) { + const chunk = missing.slice(i, i + MAX_SELLERS_PER_METADATA_REQUEST); + if (!canSpendSellerRequests(context, args, 1)) break; + + const params = new URLSearchParams({ + key: apiKey, + domain: DOMAIN_US, + seller: chunk.join(","), + }); + + context.stats.sellerMetadataRequests += 1; + const data = await fetchKeepaWithRetries( + `${KEEPA_BASE}/seller?${params.toString()}`, + `seller metadata batch ${i / MAX_SELLERS_PER_METADATA_REQUEST + 1}`, + ); + + for (const [sellerId, seller] of normalizeSellerResponse(data.sellers)) { + const parsed = parseSeller(sellerId, seller, args.inventoryLimit); + context.metadataCache.set(sellerId, parsed); + out.set(sellerId, parsed); + } + } + + return out; +} + +async function fetchQualifiedSellerStorefronts( + sellerIds: string[], + apiKey: string, + args: StalkerArgs, + context: StalkerRunContext, +): Promise> { + const out = new Map(); + const uniqueSellerIds = Array.from(new Set(sellerIds)); + + // Keepa only allows a single sellerId per request when storefront=1. + for (const sellerId of uniqueSellerIds) { + const cached = + context.storefrontCache.get(sellerId) ?? + loadCachedSeller( + context.database, + sellerId, + args.sellerCacheHours, + true, + args.inventoryLimit, + ); + if (cached) { + context.storefrontCache.set(sellerId, cached); + out.set(sellerId, cached); + continue; + } + + if (!canSpendSellerRequests(context, args, 1)) break; + + const params = new URLSearchParams({ + key: apiKey, + domain: DOMAIN_US, + seller: sellerId, + storefront: "1", + update: String(args.storefrontUpdateHours), + }); + + context.stats.sellerStorefrontRequests += 1; + const data = await fetchKeepaWithRetries( + `${KEEPA_BASE}/seller?${params.toString()}`, + `seller ${sellerId}`, + ); + + for (const [returnedSellerId, seller] of normalizeSellerResponse( + data.sellers, + )) { + const parsed = parseSeller(returnedSellerId, seller, args.inventoryLimit); + context.metadataCache.set(returnedSellerId, parsed); + context.storefrontCache.set(returnedSellerId, parsed); + out.set(returnedSellerId, parsed); + } + } + + return out; +} + +function canSpendSellerRequests( + context: StalkerRunContext, + args: StalkerArgs, + nextRequests: number, +): boolean { + if (args.maxSellerRequests == null) return true; + const spent = + context.stats.sellerMetadataRequests + context.stats.sellerStorefrontRequests; + if (spent + nextRequests <= args.maxSellerRequests) return true; + context.stats.stoppedEarly = true; + return false; +} + +async function fetchKeepaWithRetries( + url: string, + operationLabel: string, +): Promise { + let lastErrorMessage = "Unknown Keepa error"; + + for (let attempt = 1; attempt <= MAX_KEEPA_RETRIES; attempt++) { + await waitForToken(); + const response = await fetch(url); + lastRequestTime = Date.now(); + + if (response.ok) { + const data = (await response.json()) as KeepaApiResponse; + updateTokenState(data); + return data; + } + + const text = await response.text(); + const payload = parseErrorPayload(text); + if (payload) updateTokenState(payload); + lastErrorMessage = `Keepa API error ${response.status}: ${text}`; + + if (response.status !== 429 || attempt === MAX_KEEPA_RETRIES) break; + + const waitMs = computeWaitMsFromRefill(payload?.refillIn); + tokensLeft = Math.min(tokensLeft, 0); + console.warn( + `Keepa throttled during ${operationLabel} (attempt ${attempt}/${MAX_KEEPA_RETRIES}). Waiting ${Math.ceil(waitMs / 1000)}s before retry...`, + ); + await wait(waitMs); + } + + throw new Error(lastErrorMessage); +} + +function persistAsinResult( + database: Database, + runId: number, + result: StalkerAsinResult, +): void { + const fetchedAt = new Date().toISOString(); + + database.transaction(() => { + const scanId = upsertAsinScan(database, runId, result, fetchedAt); + + for (const { seller, offer } of result.matchedSellers) { + upsertSeller(database, seller, fetchedAt); + upsertAsinSeller(database, scanId, seller, offer); + upsertSellerInventory(database, runId, seller, fetchedAt); + } + })(); +} + +function upsertAsinScan( + database: Database, + runId: number, + result: StalkerAsinResult, + fetchedAt: string, +): number { + database + .prepare( + `INSERT INTO stalker_asin_scans ( + run_id, source_asin, title, offer_count, candidate_seller_count, + matched_seller_count, fetched_at, raw_product_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(run_id, source_asin) DO UPDATE SET + title = excluded.title, + offer_count = excluded.offer_count, + candidate_seller_count = excluded.candidate_seller_count, + matched_seller_count = excluded.matched_seller_count, + fetched_at = excluded.fetched_at, + raw_product_json = excluded.raw_product_json`, + ) + .run( + runId, + result.asin, + result.title, + result.offerCount, + result.candidateSellerCount, + result.matchedSellers.length, + fetchedAt, + JSON.stringify(result.product ?? { error: result.error ?? null }), + ); + + const row = database + .query( + `SELECT id FROM stalker_asin_scans WHERE run_id = ? AND source_asin = ?`, + ) + .get(runId, result.asin) as { id: number } | null; + if (!row) throw new Error(`Failed to load stalker scan row for ${result.asin}`); + return row.id; +} + +function upsertSeller( + database: Database, + seller: StalkerSeller, + fetchedAt: string, +): void { + database + .prepare( + `INSERT INTO stalker_sellers ( + seller_id, seller_name, rating, rating_count, storefront_asin_total, + persisted_inventory_sample_count, last_updated_at, raw_seller_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(seller_id) DO UPDATE SET + seller_name = excluded.seller_name, + rating = excluded.rating, + rating_count = excluded.rating_count, + storefront_asin_total = excluded.storefront_asin_total, + persisted_inventory_sample_count = excluded.persisted_inventory_sample_count, + last_updated_at = excluded.last_updated_at, + raw_seller_json = excluded.raw_seller_json`, + ) + .run( + seller.sellerId, + seller.sellerName, + seller.rating, + seller.ratingCount, + seller.storefrontAsinTotal, + seller.storefrontItems.length, + fetchedAt, + JSON.stringify(seller.rawSeller), + ); +} + +function upsertAsinSeller( + database: Database, + scanId: number, + seller: StalkerSeller, + offer: StalkerOffer, +): void { + database + .prepare( + `INSERT INTO stalker_asin_sellers ( + scan_id, seller_id, offer_price, condition, is_fba, stock, + seller_rating, seller_rating_count, raw_offer_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(scan_id, seller_id) DO UPDATE SET + offer_price = excluded.offer_price, + condition = excluded.condition, + is_fba = excluded.is_fba, + stock = excluded.stock, + seller_rating = excluded.seller_rating, + seller_rating_count = excluded.seller_rating_count, + raw_offer_json = excluded.raw_offer_json`, + ) + .run( + scanId, + seller.sellerId, + offer.offerPrice, + offer.condition, + offer.isFba == null ? null : offer.isFba ? 1 : 0, + offer.stock, + seller.rating, + seller.ratingCount, + JSON.stringify(offer.rawOffer), + ); +} + +function upsertSellerInventory( + database: Database, + runId: number, + seller: StalkerSeller, + fetchedAt: string, +): void { + const insert = database.prepare( + `INSERT INTO stalker_seller_inventory ( + run_id, seller_id, asin, last_seen_at, raw_inventory_json + ) VALUES (?, ?, ?, ?, ?) + ON CONFLICT(run_id, seller_id, asin) DO UPDATE SET + last_seen_at = excluded.last_seen_at, + raw_inventory_json = excluded.raw_inventory_json`, + ); + + for (const item of seller.storefrontItems) { + insert.run( + runId, + seller.sellerId, + item.asin, + fetchedAt, + JSON.stringify(item.rawInventory), + ); + } +} + +function startStalkerRun( + database: Database, + inputFile: string, + totalAsins: number, +): number { + const result = database + .prepare( + `INSERT INTO stalker_runs ( + input_file, started_at, requested_asins, status + ) VALUES (?, ?, ?, ?)`, + ) + .run(inputFile, new Date().toISOString(), totalAsins, "running"); + + return result.lastInsertRowid as number; +} + +function loadPreviouslyScannedAsins(database: Database): Set { + const rows = database + .query(`SELECT DISTINCT source_asin FROM stalker_asin_scans`) + .all() as Array<{ source_asin: string }>; + return new Set(rows.map((row) => row.source_asin)); +} + +function loadCachedSeller( + database: Database | null, + sellerId: string, + maxAgeHours: number, + requireStorefront: boolean, + inventoryLimit: number, +): StalkerSeller | null { + if (!database || maxAgeHours <= 0) return null; + const row = database + .query( + `SELECT raw_seller_json, last_updated_at, storefront_asin_total + FROM stalker_sellers + WHERE seller_id = ?`, + ) + .get(sellerId) as { + raw_seller_json: string | null; + last_updated_at: string; + storefront_asin_total: number | null; + } | null; + if (!row?.raw_seller_json) return null; + + const ageMs = Date.now() - new Date(row.last_updated_at).getTime(); + if (!Number.isFinite(ageMs) || ageMs > maxAgeHours * 60 * 60 * 1000) { + return null; + } + + try { + const rawSeller = JSON.parse(row.raw_seller_json) as Record; + const parsed = parseSeller(sellerId, rawSeller, inventoryLimit); + if (requireStorefront && parsed.storefrontAsinTotal <= 0) return null; + return parsed; + } catch { + return null; + } +} + +function logRunSummary(stats: StalkerRunStats, args: StalkerArgs): void { + const estimatedStorefrontRequestsSaved = Math.max( + 0, + stats.candidateSellers - stats.qualifyingSellers, + ); + console.log( + [ + "Stalker summary:", + `processed=${stats.scannedAsins}`, + `skipped=${stats.skippedAsins}`, + `candidate_sellers=${stats.candidateSellers}`, + `qualifying_sellers=${stats.qualifyingSellers}`, + `metadata_requests=${stats.sellerMetadataRequests}`, + `storefront_requests=${stats.sellerStorefrontRequests}`, + `storefront_requests_saved_by_two_phase=${estimatedStorefrontRequestsSaved}`, + `persisted_inventory=${stats.persistedInventoryAsins}`, + `dry_run=${args.dryRun ? "yes" : "no"}`, + ].join(" "), + ); +} + +function refreshStalkerRun( + database: Database, + runId: number, + stats: StalkerRunStats, + status: string, +): void { + database + .prepare( + `UPDATE stalker_runs + SET scanned_asins = ?, + source_asins_with_matches = ?, + candidate_sellers = ?, + qualifying_sellers = ?, + matched_sellers = ?, + seller_metadata_requests = ?, + seller_storefront_requests = ?, + persisted_inventory_asins = ?, + status = ?, + completed_at = CASE WHEN ? = 'running' THEN completed_at ELSE ? END + WHERE id = ?`, + ) + .run( + stats.scannedAsins, + stats.sourceAsinsWithMatches, + stats.candidateSellers, + stats.qualifyingSellers, + stats.matchedSellers, + stats.sellerMetadataRequests, + stats.sellerStorefrontRequests, + stats.persistedInventoryAsins, + status, + status, + new Date().toISOString(), + runId, + ); +} + +function finishStalkerRunWithError( + database: Database, + runId: number, + stats: StalkerRunStats, + errorMessage: string, +): void { + database + .prepare( + `UPDATE stalker_runs + SET scanned_asins = ?, + source_asins_with_matches = ?, + candidate_sellers = ?, + qualifying_sellers = ?, + matched_sellers = ?, + seller_metadata_requests = ?, + seller_storefront_requests = ?, + persisted_inventory_asins = ?, + status = 'failed', + error_message = ?, + completed_at = ? + WHERE id = ?`, + ) + .run( + stats.scannedAsins, + stats.sourceAsinsWithMatches, + stats.candidateSellers, + stats.qualifyingSellers, + stats.matchedSellers, + stats.sellerMetadataRequests, + stats.sellerStorefrontRequests, + stats.persistedInventoryAsins, + errorMessage, + new Date().toISOString(), + runId, + ); +} + +function normalizeSellerResponse( + sellers: KeepaApiResponse["sellers"], +): Array<[string, Record]> { + if (!sellers) return []; + if (Array.isArray(sellers)) { + return sellers + .map((seller) => [ + normalizeSellerId(seller.sellerId ?? seller.sellerID ?? seller.id), + seller, + ] as [string | null, Record]) + .filter((entry): entry is [string, Record] => !!entry[0]); + } + + return Object.entries(sellers) + .map(([sellerId, seller]) => [ + normalizeSellerId(sellerId), + seller, + ] as [string | null, Record]) + .filter((entry): entry is [string, Record] => !!entry[0]); +} + +function parseSeller( + sellerId: string, + seller: Record, + inventoryLimit: number, +): StalkerSeller { + const allStorefrontItems = extractStorefrontItems(seller); + const storefrontItems = + inventoryLimit === 0 + ? [] + : allStorefrontItems.slice(0, inventoryLimit); + const storefrontAsins = storefrontItems.map((item) => item.asin); + return { + sellerId, + sellerName: extractString( + seller.sellerName ?? seller.name ?? seller.storeName ?? seller.businessName, + ), + rating: extractNumber( + seller.currentRating ?? seller.rating ?? seller.feedbackRating, + ), + ratingCount: extractSellerRatingCount(seller), + storefrontAsins, + storefrontItems, + storefrontAsinTotal: + extractNumber( + seller.totalStorefrontAsinCount ?? + seller.storefrontAsinCount ?? + seller.asinListCount ?? + seller.totalStorefrontProducts, + ) ?? allStorefrontItems.length, + rawSeller: seller, + }; +} + +function extractStorefrontItems(seller: Record): StalkerInventoryItem[] { + const candidates = [ + seller.asinList, + seller.asins, + seller.storefront, + seller.storefrontAsins, + seller.inventory, + ]; + const items: StalkerInventoryItem[] = []; + const seen = new Set(); + + for (const candidate of candidates) { + collectStorefrontItems(candidate, items, seen); + } + + return items; +} + +function collectStorefrontItems( + value: unknown, + items: StalkerInventoryItem[], + seen: Set, +): void { + if (Array.isArray(value)) { + for (const item of value) collectStorefrontItems(item, items, seen); + return; + } + + if (value && typeof value === "object") { + const asin = normalizeAsin((value as Record).asin); + if (asin && !seen.has(asin)) { + seen.add(asin); + items.push({ asin, rawInventory: value }); + } + return; + } + + const asin = normalizeAsin(value); + if (!asin || seen.has(asin)) return; + seen.add(asin); + items.push({ asin, rawInventory: { asin } }); +} + +function extractSellerRatingCount(seller: Record): number | null { + const direct = extractNumber( + seller.currentRatingCount ?? + seller.ratingCount ?? + seller.ratingsCount ?? + seller.feedbackCount ?? + seller.reviewCount, + ); + if (direct != null) return direct; + + const ratingCountHistory = seller.ratingCountHistory ?? seller.ratingCountCSV; + if (Array.isArray(ratingCountHistory) && ratingCountHistory.length > 0) { + return extractNumber(ratingCountHistory[ratingCountHistory.length - 1]); + } + + return null; +} + +function extractOfferPrice(offer: Record): number | null { + const raw = extractNumber( + offer.price ?? offer.currentPrice ?? offer.offerPrice ?? offer.newPrice, + ); + if (raw == null) return null; + return raw > 100 ? Math.round(raw) / 100 : raw; +} + +function sumInventoryAsins(result: StalkerAsinResult): number { + return result.matchedSellers.reduce( + (sum, entry) => sum + entry.seller.storefrontAsins.length, + 0, + ); +} + +function normalizeAsin(value: unknown): string | null { + const asin = String(value ?? "") + .trim() + .toUpperCase(); + return ASIN_REGEX.test(asin) ? asin : null; +} + +function normalizeSellerId(value: unknown): string | null { + const sellerId = String(value ?? "") + .trim() + .toUpperCase(); + return sellerId.length > 0 ? sellerId : null; +} + +function extractString(value: unknown): string | null { + if (value == null) return null; + const text = String(value).trim(); + return text.length > 0 ? text : null; +} + +function extractNumber(value: unknown): number | null { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value !== "string") return null; + const parsed = Number(value.trim().replace(/[$,%]/g, "").replace(/,/g, "")); + return Number.isFinite(parsed) ? parsed : null; +} + +function extractBoolean(value: unknown): boolean | null { + if (typeof value === "boolean") return value; + if (typeof value === "number") return value === 1 ? true : value === 0 ? false : null; + if (typeof value !== "string") return null; + const normalized = value.trim().toLowerCase(); + if (["1", "true", "yes"].includes(normalized)) return true; + if (["0", "false", "no"].includes(normalized)) return false; + return null; +} + +function normalizeHeader(value: string): string { + return value.toLowerCase().trim().replace(/[^a-z0-9]/g, ""); +} + +function readFlagValue(args: string[], flag: string): string | undefined { + const index = args.indexOf(flag); + if (index === -1) return undefined; + return args[index + 1]; +} + +function hasFlag(args: string[], flag: string): boolean { + return args.includes(flag); +} + +function printUsageAndExit(message: string): never { + console.error(message); + console.error( + "Usage: bun run stalker --input input/asins.xlsx [--db db/results.db] [--max-asins N] [--offer-limit 100] [--seller-limit 30] [--inventory-limit 200] [--storefront-update-hours 168] [--seller-cache-hours 168] [--max-seller-requests N] [--include-stock] [--dry-run] [--no-resume]", + ); + process.exit(1); +} + +function updateTokenState(data: KeepaApiResponse): void { + if (data.tokensLeft != null) tokensLeft = data.tokensLeft; + if (data.refillRate != null) refillRate = data.refillRate; +} + +async function waitForToken(): Promise { + if (tokensLeft > 0) return; + + const elapsed = (Date.now() - lastRequestTime) / 60_000; + const regenerated = Math.floor(elapsed * Math.max(1, refillRate)); + if (regenerated > 0) { + tokensLeft += regenerated; + return; + } + + const waitMs = + Math.ceil((1 / Math.max(1, refillRate)) * 60_000) - + (Date.now() - lastRequestTime); + if (waitMs > 0) { + console.log( + `Keepa tokens exhausted. Waiting ${Math.ceil(waitMs / 1000)}s for token regeneration...`, + ); + await wait(waitMs); + } + tokensLeft = 1; +} + +function computeWaitMsFromRefill(refillIn?: number): number { + if ( + typeof refillIn === "number" && + Number.isFinite(refillIn) && + refillIn >= 0 + ) { + return Math.max( + Math.ceil(refillIn) + KEEP_RETRY_BUFFER_MS, + KEEP_RETRY_BUFFER_MS, + ); + } + + return Math.ceil((1 / Math.max(1, refillRate)) * 60_000) + KEEP_RETRY_BUFFER_MS; +} + +function parseErrorPayload(text: string): KeepaApiResponse | null { + try { + const parsed = JSON.parse(text) as KeepaApiResponse; + return parsed && typeof parsed === "object" ? parsed : null; + } catch { + return null; + } +} + +function wait(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +if (import.meta.main) { + const args = parseArgs(); + runStalker(args) + .then((stats) => { + console.log( + `Stalker complete: scanned=${stats.scannedAsins}, matched_sellers=${stats.matchedSellers}, persisted_inventory_asins=${stats.persistedInventoryAsins}, failed=${stats.failedAsins}`, + ); + }) + .catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }) + .finally(() => { + closeDb(); + }); +} diff --git a/src/web/frontend.tsx b/src/web/frontend.tsx index 17a6df7..19aa424 100644 --- a/src/web/frontend.tsx +++ b/src/web/frontend.tsx @@ -109,6 +109,45 @@ type ProductListResponse = { totalPages: number; }; +type StalkerResultItem = { + runId: number; + started_at: string; + status: string; + input_file: string; + source_asin: string; + title: string | null; + offer_count: number; + candidate_seller_count: number; + matched_seller_count: number; + scan_fetched_at: string; + seller_id: string; + seller_name: string | null; + rating: number | null; + rating_count: number | null; + storefront_asin_total: number | null; + persisted_inventory_sample_count: number | null; + offer_price: number | null; + condition: string | null; + is_fba: number | null; + stock: number | null; + persisted_inventory_asin_count: number; + inventory_sample_asins: string | null; +}; + +type StalkerResultsResponse = { + items: StalkerResultItem[]; + summary: { + runs: number; + sourceAsins: number; + sellers: number; + persistedInventoryAsins: number; + }; + page: number; + pageSize: number; + total: number; + totalPages: number; +}; + type SortState = { field: string; direction: SortDirection; @@ -139,6 +178,11 @@ function formatAmazonSeller(value: number | null | undefined): string { return value === 1 ? "Yes" : "No"; } +function formatBoolean(value: number | null | undefined): string { + if (value === null || value === undefined) return "-"; + return value === 1 ? "Yes" : "No"; +} + function buildSortValue(sort: SortState): string { return `${sort.field}:${sort.direction}`; } @@ -197,9 +241,11 @@ function detectAnomaly(item: ResultItem): string { function Dashboard({ onOpenRun, onOpenProducts, + onOpenStalker, }: { onOpenRun: (run: Run) => void; onOpenProducts: (verdict: VerdictFilter) => void; + onOpenStalker: () => void; }) { const [runs, setRuns] = useState(null); const [loading, setLoading] = useState(false); @@ -288,7 +334,10 @@ function Dashboard({ return (
-

Runs Dashboard

+
+

Runs Dashboard

+ +
@@ -848,10 +897,166 @@ function ProductList({ verdict, onBack }: { verdict: VerdictFilter; onBack: () = ); } +function StalkerExplorer({ onBack }: { onBack: () => void }) { + const [results, setResults] = useState(null); + const [loading, setLoading] = useState(false); + const [search, setSearch] = useState(""); + const [sellerId, setSellerId] = useState(""); + const [runId, setRunId] = useState(""); + const [minRatingCount, setMinRatingCount] = useState("1"); + const [maxRatingCount, setMaxRatingCount] = useState("30"); + const [page, setPage] = useState(1); + const [pageSize, setPageSize] = useState(25); + const [sort, setSort] = useState({ field: "started_at", direction: "DESC" }); + + useEffect(() => { + let cancelled = false; + async function load() { + setLoading(true); + const params = new URLSearchParams({ + page: String(page), + pageSize: String(pageSize), + sort: buildSortValue(sort), + }); + if (search) params.set("q", search); + if (sellerId) params.set("sellerId", sellerId); + if (runId) params.set("runId", runId); + if (minRatingCount) params.set("minRatingCount", minRatingCount); + if (maxRatingCount) params.set("maxRatingCount", maxRatingCount); + + const res = await fetch(`/api/stalker/results?${params.toString()}`); + const payload = (await res.json()) as StalkerResultsResponse; + if (!cancelled) { + setResults(payload); + setLoading(false); + } + } + + load(); + return () => { + cancelled = true; + }; + }, [search, sellerId, runId, minRatingCount, maxRatingCount, page, pageSize, sort]); + + return ( +
+ + +
+

Stalker Results

+
+ +
+
+
Runs
+
{formatNumber(results?.summary.runs)}
+
+
+
Source ASINs
+
{formatNumber(results?.summary.sourceAsins)}
+
+
+
Matched sellers
+
{formatNumber(results?.summary.sellers)}
+
+
+
Persisted inventory ASINs
+
{formatNumber(results?.summary.persistedInventoryAsins)}
+
+
+ +
+
+ { setPage(1); setSearch(e.target.value); }} placeholder="Search source ASIN/title/seller/inventory" /> + { setPage(1); setSellerId(e.target.value.toUpperCase()); }} placeholder="Seller ID" /> + { setPage(1); setRunId(e.target.value); }} placeholder="Run ID" /> + { setPage(1); setMinRatingCount(e.target.value); }} placeholder="Min rating count" /> + { setPage(1); setMaxRatingCount(e.target.value); }} placeholder="Max rating count" /> + +
+
+ +
+
+ + + + + + + + + + + + + + + + + + + + + {loading ? ( + + ) : results?.items.length ? ( + results.items.map((item) => { + const inventorySample = (item.inventory_sample_asins ?? "") + .split(",") + .filter(Boolean) + .slice(0, 20); + return ( + + + + + + + + + + + + + + + + + ); + }) + ) : ( + + )} + +
FBAInventory ASIN sample
Loading...
{item.runId}{formatDate(item.started_at)}{item.source_asin}{item.title || "-"}{item.seller_id}{item.seller_name || "-"}{formatNumber(item.rating)}{formatNumber(item.rating_count)}{formatCurrency(item.offer_price)}{formatBoolean(item.is_fba)}{formatNumber(item.stock)}{formatNumber(item.storefront_asin_total)}{formatNumber(item.persisted_inventory_asin_count)} + {inventorySample.length === 0 ? "-" : inventorySample.map((asin) => ( + {asin} + ))} +
No stalker results found
+
+
+
Showing {results?.items.length ?? 0} of {results?.total ?? 0}
+
+ + Page {results?.page ?? page} / {results?.totalPages ?? 1} + +
+
+
+
+ ); +} + type AppRoute = | { kind: "dashboard" } | { kind: "run"; processType: ProcessType; runId: number } - | { kind: "products"; verdict: VerdictFilter }; + | { kind: "products"; verdict: VerdictFilter } + | { kind: "stalker" }; function parseRoute(pathname: string, search: string): AppRoute { const runMatch = pathname.match(/^\/runs\/(lead_analysis|category_analysis)\/(\d+)$/); @@ -866,6 +1071,10 @@ function parseRoute(pathname: string, search: string): AppRoute { return { kind: "products", verdict }; } + if (pathname === "/stalker") { + return { kind: "stalker" }; + } + return { kind: "dashboard" }; } @@ -890,6 +1099,11 @@ function App() { setRoute({ kind: "products", verdict }); } + function openStalker() { + history.pushState({}, "", "/stalker"); + setRoute({ kind: "stalker" }); + } + function backToDashboard() { history.pushState({}, "", "/"); setRoute({ kind: "dashboard" }); @@ -903,7 +1117,11 @@ function App() { return ; } - return ; + if (route.kind === "stalker") { + return ; + } + + return ; } const root = document.getElementById("root"); diff --git a/src/web/styles.css b/src/web/styles.css index b1893a9..205f4dd 100644 --- a/src/web/styles.css +++ b/src/web/styles.css @@ -41,6 +41,13 @@ p { gap: 10px; } +.section-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; +} + .toolbar input, .toolbar select, button { @@ -91,6 +98,23 @@ td { overflow-wrap: anywhere; } +.inventory-col { + min-width: 360px; + max-width: 520px; + white-space: normal; + overflow-wrap: anywhere; +} + +.inventory-col a { + display: inline-block; + margin-right: 8px; + margin-bottom: 4px; +} + +.stalker-table { + min-width: 1320px; +} + th { background: #fafafb; font-weight: 600; @@ -262,4 +286,9 @@ th button { .spark-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } + + .section-header { + align-items: flex-start; + flex-direction: column; + } } From aed0c1101767ac16d3b70c47edc2f0bf066b5552 Mon Sep 17 00:00:00 2001 From: Victor Noguera Date: Tue, 19 May 2026 18:35:55 -0400 Subject: [PATCH 2/9] feat: enhance stalker functionality with inventory sellability checks and update frontend display --- src/database.ts | 21 +++- src/server.ts | 43 +++------ src/stalker-sellability.test.ts | 163 ++++++++++++++++++++++++++++++++ src/stalker.test.ts | 4 + src/stalker.ts | 112 ++++++++++++++++++++-- src/web/frontend.tsx | 52 ++++------ 6 files changed, 317 insertions(+), 78 deletions(-) create mode 100644 src/stalker-sellability.test.ts diff --git a/src/database.ts b/src/database.ts index cdef940..35a1074 100644 --- a/src/database.ts +++ b/src/database.ts @@ -354,6 +354,9 @@ export function initStalkerDb(database: Database): void { matched_sellers INTEGER NOT NULL DEFAULT 0, seller_metadata_requests INTEGER NOT NULL DEFAULT 0, seller_storefront_requests INTEGER NOT NULL DEFAULT 0, + inventory_sellability_checked_asins INTEGER NOT NULL DEFAULT 0, + inventory_sellability_available_asins INTEGER NOT NULL DEFAULT 0, + inventory_sellability_excluded_asins INTEGER NOT NULL DEFAULT 0, persisted_inventory_asins INTEGER NOT NULL DEFAULT 0, status TEXT NOT NULL, error_message TEXT @@ -413,6 +416,9 @@ export function initStalkerDb(database: Database): void { run_id INTEGER NOT NULL, seller_id TEXT NOT NULL, asin TEXT NOT NULL, + can_sell INTEGER, + sellability_status TEXT, + sellability_reason TEXT, last_seen_at TEXT NOT NULL, raw_inventory_json TEXT, UNIQUE(run_id, seller_id, asin), @@ -448,7 +454,13 @@ function resetLegacyStalkerSchema(database: Database): void { if (runColumns.length === 0) return; const columnNames = new Set(runColumns.map((column) => column.name)); - if (columnNames.has("scanned_asins")) return; + if ( + columnNames.has("scanned_asins") && + columnNames.has("inventory_sellability_checked_asins") && + inventoryColumnsHaveSellability(database) + ) { + return; + } database.run("DROP TABLE IF EXISTS stalker_seller_inventory"); database.run("DROP TABLE IF EXISTS stalker_asin_sellers"); @@ -456,3 +468,10 @@ function resetLegacyStalkerSchema(database: Database): void { database.run("DROP TABLE IF EXISTS stalker_asin_scans"); database.run("DROP TABLE IF EXISTS stalker_runs"); } + +function inventoryColumnsHaveSellability(database: Database): boolean { + const inventoryColumns = database + .query("PRAGMA table_info(stalker_seller_inventory)") + .all() as Array<{ name: string }>; + return inventoryColumns.some((column) => column.name === "sellability_status"); +} diff --git a/src/server.ts b/src/server.ts index 2221df6..92a3a2e 100644 --- a/src/server.ts +++ b/src/server.ts @@ -58,22 +58,15 @@ type StalkerResultRecord = { started_at: string; status: string; input_file: string; - source_asin: string; - title: string | null; - offer_count: number; - candidate_seller_count: number; - matched_seller_count: number; - scan_fetched_at: string; seller_id: string; seller_name: string | null; rating: number | null; rating_count: number | null; storefront_asin_total: number | null; persisted_inventory_sample_count: number | null; - offer_price: number | null; - condition: string | null; - is_fba: number | null; - stock: number | null; + discovered_from_count: number; + first_seen_at: string; + last_seen_at: string; persisted_inventory_asin_count: number; inventory_sample_asins: string | null; }; @@ -690,14 +683,14 @@ function parseStalkerFilters(filters: URLSearchParams) { if (q) { const wildcard = `%${q}%`; conditions.push( - `(sc.source_asin LIKE ? OR sc.title LIKE ? OR s.seller_id LIKE ? OR s.seller_name LIKE ? OR EXISTS ( + `(s.seller_id LIKE ? OR s.seller_name LIKE ? OR EXISTS ( SELECT 1 FROM stalker_seller_inventory inv_q WHERE inv_q.run_id = r.id AND inv_q.seller_id = s.seller_id AND inv_q.asin LIKE ? ))`, ); - params.push(wildcard, wildcard, wildcard, wildcard, wildcard); + params.push(wildcard, wildcard, wildcard); } return { @@ -710,22 +703,19 @@ function parseStalkerSort(sortParam: string | null): string { const allowedSort = new Set([ "runId", "started_at", - "source_asin", - "title", "seller_id", "seller_name", "rating", "rating_count", - "offer_price", - "stock", + "discovered_from_count", "persisted_inventory_asin_count", "storefront_asin_total", - "scan_fetched_at", + "last_seen_at", ]); const parsed = parseSort( sortParam, allowedSort, - "started_at DESC, runId DESC, source_asin ASC", + "persisted_inventory_asin_count DESC, last_seen_at DESC, seller_id ASC", ); return parsed @@ -751,22 +741,15 @@ function getStalkerResults(filters: URLSearchParams) { r.started_at, r.status, r.input_file, - sc.source_asin, - sc.title, - sc.offer_count, - sc.candidate_seller_count, - sc.matched_seller_count, - sc.fetched_at AS scan_fetched_at, s.seller_id, s.seller_name, s.rating, s.rating_count, s.storefront_asin_total, s.persisted_inventory_sample_count, - sas.offer_price, - sas.condition, - sas.is_fba, - sas.stock, + COUNT(DISTINCT sc.source_asin) AS discovered_from_count, + MIN(sc.fetched_at) AS first_seen_at, + MAX(sc.fetched_at) AS last_seen_at, COUNT(DISTINCT inv.asin) AS persisted_inventory_asin_count, GROUP_CONCAT(DISTINCT inv.asin) AS inventory_sample_asins FROM stalker_asin_sellers sas @@ -777,7 +760,7 @@ function getStalkerResults(filters: URLSearchParams) { ON inv.run_id = r.id AND inv.seller_id = s.seller_id ${where} - GROUP BY sas.id + GROUP BY r.id, s.seller_id `; const totalRow = db @@ -788,14 +771,12 @@ function getStalkerResults(filters: URLSearchParams) { .query( `SELECT COUNT(DISTINCT runId) AS runs, - COUNT(DISTINCT source_asin) AS sourceAsins, COUNT(DISTINCT seller_id) AS sellers, COALESCE(SUM(persisted_inventory_asin_count), 0) AS persistedInventoryAsins FROM (${baseSelect}) stalker_rows`, ) .get(...params) as { runs: number; - sourceAsins: number; sellers: number; persistedInventoryAsins: number; }; diff --git a/src/stalker-sellability.test.ts b/src/stalker-sellability.test.ts new file mode 100644 index 0000000..33d2dbd --- /dev/null +++ b/src/stalker-sellability.test.ts @@ -0,0 +1,163 @@ +import { afterAll, beforeEach, expect, mock, test } from "bun:test"; +import { mkdirSync, rmSync } from "node:fs"; +import path from "node:path"; +import * as XLSX from "xlsx"; +import { closeDb, getDb } from "./database.ts"; + +const TEST_DIR = path.join(process.cwd(), "test_output", "stalker-sellability"); +const originalFetch = globalThis.fetch; +const originalKeepaKey = Bun.env.KEEPA_API_KEY; + +const fetchSellabilityBatchMock = mock(async (asins: string[]) => { + return new Map( + asins.map((asin) => [ + asin, + asin === "B111111111" + ? { + canSell: true, + sellabilityStatus: "available" as const, + sellabilityReason: "No listing restrictions reported", + } + : { + canSell: false, + sellabilityStatus: "restricted" as const, + sellabilityReason: "approval required", + }, + ]), + ); +}); + +mock.module("./sp-api.ts", () => ({ + fetchSellabilityBatch: fetchSellabilityBatchMock, +})); + +const modulePromise = import("./stalker.ts"); + +beforeEach(() => { + closeDb(); + rmSync(TEST_DIR, { recursive: true, force: true }); + mkdirSync(TEST_DIR, { recursive: true }); + globalThis.fetch = originalFetch; + Bun.env.KEEPA_API_KEY = "test-keepa-key"; + fetchSellabilityBatchMock.mockClear(); +}); + +afterAll(() => { + globalThis.fetch = originalFetch; + if (originalKeepaKey == null) { + delete Bun.env.KEEPA_API_KEY; + } else { + Bun.env.KEEPA_API_KEY = originalKeepaKey; + } + closeDb(); + rmSync(TEST_DIR, { recursive: true, force: true }); +}); + +test("sellability checks matched seller inventory, not the source ASIN", async () => { + const { runStalker } = await modulePromise; + const inputPath = path.join(TEST_DIR, "input.xlsx"); + const dbPath = path.join(TEST_DIR, "stalker.sqlite"); + const workbook = XLSX.utils.book_new(); + XLSX.utils.book_append_sheet( + workbook, + XLSX.utils.json_to_sheet([{ asin: "B000000001" }]), + "Input", + ); + XLSX.writeFile(workbook, inputPath); + + globalThis.fetch = mock(async (input: string | URL | Request) => { + const rawUrl = + typeof input === "string" + ? input + : input instanceof URL + ? input.toString() + : input.url; + const url = new URL(rawUrl); + + if (url.pathname === "/product") { + return new Response( + JSON.stringify({ + products: [ + { + asin: "B000000001", + title: "Source Product", + offers: [{ sellerId: "AQUALIFIED", price: 1999 }], + }, + ], + tokensLeft: 10, + refillRate: 10, + }), + { status: 200 }, + ); + } + + if (url.pathname === "/seller") { + const wantsStorefront = url.searchParams.get("storefront") === "1"; + return new Response( + JSON.stringify({ + sellers: { + AQUALIFIED: { + sellerName: "New Seller", + currentRatingCount: 12, + asinList: wantsStorefront ? ["B111111111", "B222222222"] : [], + }, + }, + tokensLeft: 10, + refillRate: 10, + }), + { status: 200 }, + ); + } + + return new Response("not found", { status: 404 }); + }) as unknown as typeof globalThis.fetch; + + const stats = await runStalker({ + input: inputPath, + dbPath, + maxAsins: null, + storefrontUpdateHours: 168, + offerLimit: 20, + sellerLimit: 30, + inventoryLimit: 200, + sellerCacheHours: 168, + includeStock: false, + dryRun: false, + resume: true, + maxSellerRequests: null, + sellability: true, + }); + + expect(fetchSellabilityBatchMock.mock.calls.length).toBe(1); + expect(fetchSellabilityBatchMock.mock.calls[0]?.[0]).toEqual([ + "B111111111", + "B222222222", + ]); + expect(stats.inventorySellabilityCheckedAsins).toBe(2); + expect(stats.inventorySellabilityAvailableAsins).toBe(1); + expect(stats.inventorySellabilityExcludedAsins).toBe(1); + expect(stats.persistedInventoryAsins).toBe(1); + + const db = getDb(dbPath); + const scan = db.query("SELECT source_asin FROM stalker_asin_scans").get() as { + source_asin: string; + }; + expect(scan.source_asin).toBe("B000000001"); + + const inventory = db + .query( + "SELECT asin, can_sell, sellability_status FROM stalker_seller_inventory ORDER BY asin", + ) + .all() as Array<{ + asin: string; + can_sell: number | null; + sellability_status: string | null; + }>; + expect(inventory).toEqual([ + { + asin: "B111111111", + can_sell: 1, + sellability_status: "available", + }, + ]); +}); diff --git a/src/stalker.test.ts b/src/stalker.test.ts index 42ffc08..6c5ad7e 100644 --- a/src/stalker.test.ts +++ b/src/stalker.test.ts @@ -216,6 +216,7 @@ test("runStalker fetches product offers, filters sellers, and persists storefron dryRun: false, resume: true, maxSellerRequests: null, + sellability: false, }); expect(stats.scannedAsins).toBe(1); @@ -249,6 +250,9 @@ test("runStalker fetches product offers, filters sellers, and persists storefron expect(run.matched_sellers).toBe(1); expect(run.seller_metadata_requests).toBe(1); expect(run.seller_storefront_requests).toBe(1); + expect(run.inventory_sellability_checked_asins).toBe(0); + expect(run.inventory_sellability_available_asins).toBe(0); + expect(run.inventory_sellability_excluded_asins).toBe(0); expect(run.persisted_inventory_asins).toBe(2); const scan = db.query("SELECT * FROM stalker_asin_scans").get() as any; diff --git a/src/stalker.ts b/src/stalker.ts index f4b6563..8fba937 100644 --- a/src/stalker.ts +++ b/src/stalker.ts @@ -1,6 +1,8 @@ import * as XLSX from "xlsx"; import path from "node:path"; import { type Database, closeDb, getDb, initDb } from "./database.ts"; +import { fetchSellabilityBatch } from "./sp-api.ts"; +import type { SellabilityInfo } from "./types.ts"; const KEEPA_BASE = "https://api.keepa.com"; const DOMAIN_US = "1"; @@ -37,6 +39,7 @@ export type StalkerArgs = { dryRun: boolean; resume: boolean; maxSellerRequests: number | null; + sellability: boolean; }; export type StalkerOffer = { @@ -62,6 +65,7 @@ export type StalkerSeller = { type StalkerInventoryItem = { asin: string; rawInventory: unknown; + sellability: SellabilityInfo | null; }; type StalkerAsinResult = { @@ -88,6 +92,9 @@ type StalkerRunStats = { qualifyingSellers: number; sellerMetadataRequests: number; sellerStorefrontRequests: number; + inventorySellabilityCheckedAsins: number; + inventorySellabilityAvailableAsins: number; + inventorySellabilityExcludedAsins: number; stoppedEarly: boolean; }; @@ -135,6 +142,7 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs { const includeStock = hasFlag(argv, "--include-stock"); const dryRun = hasFlag(argv, "--dry-run"); const resume = !hasFlag(argv, "--no-resume"); + const sellability = hasFlag(argv, "--sellability"); if (maxAsins != null && (!Number.isInteger(maxAsins) || maxAsins <= 0)) { printUsageAndExit("--max-asins must be a positive integer."); @@ -185,6 +193,7 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs { dryRun, resume, maxSellerRequests, + sellability, }; } @@ -276,21 +285,26 @@ export async function runStalker(args: StalkerArgs): Promise { initDb(args.dbPath); const database = getDb(args.dbPath); const completedAsins = args.resume ? loadPreviouslyScannedAsins(database) : new Set(); - const asins = cappedAsins.filter((asin) => !completedAsins.has(asin)); + const resumeFilteredAsins = cappedAsins.filter( + (asin) => !completedAsins.has(asin), + ); const runId = args.dryRun ? null - : startStalkerRun(database, args.input, asins.length); + : startStalkerRun(database, args.input, resumeFilteredAsins.length); const stats: StalkerRunStats = { scannedAsins: 0, sourceAsinsWithMatches: 0, matchedSellers: 0, persistedInventoryAsins: 0, failedAsins: 0, - skippedAsins: cappedAsins.length - asins.length, + skippedAsins: cappedAsins.length - resumeFilteredAsins.length, candidateSellers: 0, qualifyingSellers: 0, sellerMetadataRequests: 0, sellerStorefrontRequests: 0, + inventorySellabilityCheckedAsins: 0, + inventorySellabilityAvailableAsins: 0, + inventorySellabilityExcludedAsins: 0, stoppedEarly: false, }; const context: StalkerRunContext = { @@ -308,8 +322,8 @@ export async function runStalker(args: StalkerArgs): Promise { console.log(`Stalker resume: skipped ${stats.skippedAsins} previously scanned ASIN(s).`); } - for (const asin of asins) { - console.log(`Stalker: scanning ${asin} (${stats.scannedAsins + 1}/${asins.length})`); + for (const asin of resumeFilteredAsins) { + console.log(`Stalker: scanning ${asin} (${stats.scannedAsins + 1}/${resumeFilteredAsins.length})`); const result = await scanAsin(asin, args, apiKey, context).catch((error) => ({ asin, @@ -321,6 +335,10 @@ export async function runStalker(args: StalkerArgs): Promise { error: error instanceof Error ? error.message : String(error), })); + if (args.sellability && !args.dryRun) { + await enrichInventorySellability(result, stats); + } + if (!args.dryRun && runId != null) { persistAsinResult(database, runId, result); } @@ -439,6 +457,54 @@ async function scanAsin( }; } +async function enrichInventorySellability( + result: StalkerAsinResult, + stats: StalkerRunStats, +): Promise { + const sellers = result.matchedSellers.map(({ seller }) => seller); + const items = sellers.flatMap((seller) => seller.storefrontItems); + const uniqueAsins = Array.from(new Set(items.map((item) => item.asin))); + if (uniqueAsins.length === 0) return; + + console.log( + `Stalker inventory sellability: checking ${uniqueAsins.length} ASIN(s) from matched seller storefronts...`, + ); + const sellabilityMap = await fetchSellabilityBatch(uniqueAsins); + stats.inventorySellabilityCheckedAsins += uniqueAsins.length; + + for (const asin of uniqueAsins) { + const info = sellabilityMap.get(asin) ?? { + canSell: null, + sellabilityStatus: "unknown" as const, + sellabilityReason: "Sellability check returned no result", + }; + + if (info.sellabilityStatus === "available" && info.canSell === true) { + stats.inventorySellabilityAvailableAsins += 1; + } else { + stats.inventorySellabilityExcludedAsins += 1; + } + } + + for (const item of items) { + item.sellability = + sellabilityMap.get(item.asin) ?? { + canSell: null, + sellabilityStatus: "unknown", + sellabilityReason: "Sellability check returned no result", + }; + } + + for (const seller of sellers) { + seller.storefrontItems = seller.storefrontItems.filter( + (item) => + item.sellability?.canSell === true && + item.sellability.sellabilityStatus === "available", + ); + seller.storefrontAsins = seller.storefrontItems.map((item) => item.asin); + } +} + async function fetchKeepaProduct( asin: string, apiKey: string, @@ -755,9 +821,13 @@ function upsertSellerInventory( ): void { const insert = database.prepare( `INSERT INTO stalker_seller_inventory ( - run_id, seller_id, asin, last_seen_at, raw_inventory_json - ) VALUES (?, ?, ?, ?, ?) + run_id, seller_id, asin, can_sell, sellability_status, + sellability_reason, last_seen_at, raw_inventory_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(run_id, seller_id, asin) DO UPDATE SET + can_sell = excluded.can_sell, + sellability_status = excluded.sellability_status, + sellability_reason = excluded.sellability_reason, last_seen_at = excluded.last_seen_at, raw_inventory_json = excluded.raw_inventory_json`, ); @@ -767,6 +837,13 @@ function upsertSellerInventory( runId, seller.sellerId, item.asin, + item.sellability?.canSell == null + ? null + : item.sellability.canSell + ? 1 + : 0, + item.sellability?.sellabilityStatus ?? null, + item.sellability?.sellabilityReason ?? null, fetchedAt, JSON.stringify(item.rawInventory), ); @@ -846,6 +923,9 @@ function logRunSummary(stats: StalkerRunStats, args: StalkerArgs): void { `qualifying_sellers=${stats.qualifyingSellers}`, `metadata_requests=${stats.sellerMetadataRequests}`, `storefront_requests=${stats.sellerStorefrontRequests}`, + `sellability_checked=${stats.inventorySellabilityCheckedAsins}`, + `sellability_available=${stats.inventorySellabilityAvailableAsins}`, + `sellability_excluded=${stats.inventorySellabilityExcludedAsins}`, `storefront_requests_saved_by_two_phase=${estimatedStorefrontRequestsSaved}`, `persisted_inventory=${stats.persistedInventoryAsins}`, `dry_run=${args.dryRun ? "yes" : "no"}`, @@ -869,6 +949,9 @@ function refreshStalkerRun( matched_sellers = ?, seller_metadata_requests = ?, seller_storefront_requests = ?, + inventory_sellability_checked_asins = ?, + inventory_sellability_available_asins = ?, + inventory_sellability_excluded_asins = ?, persisted_inventory_asins = ?, status = ?, completed_at = CASE WHEN ? = 'running' THEN completed_at ELSE ? END @@ -882,6 +965,9 @@ function refreshStalkerRun( stats.matchedSellers, stats.sellerMetadataRequests, stats.sellerStorefrontRequests, + stats.inventorySellabilityCheckedAsins, + stats.inventorySellabilityAvailableAsins, + stats.inventorySellabilityExcludedAsins, stats.persistedInventoryAsins, status, status, @@ -906,6 +992,9 @@ function finishStalkerRunWithError( matched_sellers = ?, seller_metadata_requests = ?, seller_storefront_requests = ?, + inventory_sellability_checked_asins = ?, + inventory_sellability_available_asins = ?, + inventory_sellability_excluded_asins = ?, persisted_inventory_asins = ?, status = 'failed', error_message = ?, @@ -920,6 +1009,9 @@ function finishStalkerRunWithError( stats.matchedSellers, stats.sellerMetadataRequests, stats.sellerStorefrontRequests, + stats.inventorySellabilityCheckedAsins, + stats.inventorySellabilityAvailableAsins, + stats.inventorySellabilityExcludedAsins, stats.persistedInventoryAsins, errorMessage, new Date().toISOString(), @@ -1013,7 +1105,7 @@ function collectStorefrontItems( const asin = normalizeAsin((value as Record).asin); if (asin && !seen.has(asin)) { seen.add(asin); - items.push({ asin, rawInventory: value }); + items.push({ asin, rawInventory: value, sellability: null }); } return; } @@ -1021,7 +1113,7 @@ function collectStorefrontItems( const asin = normalizeAsin(value); if (!asin || seen.has(asin)) return; seen.add(asin); - items.push({ asin, rawInventory: { asin } }); + items.push({ asin, rawInventory: { asin }, sellability: null }); } function extractSellerRatingCount(seller: Record): number | null { @@ -1111,7 +1203,7 @@ function hasFlag(args: string[], flag: string): boolean { function printUsageAndExit(message: string): never { console.error(message); console.error( - "Usage: bun run stalker --input input/asins.xlsx [--db db/results.db] [--max-asins N] [--offer-limit 100] [--seller-limit 30] [--inventory-limit 200] [--storefront-update-hours 168] [--seller-cache-hours 168] [--max-seller-requests N] [--include-stock] [--dry-run] [--no-resume]", + "Usage: bun run stalker --input input/asins.xlsx [--db db/results.db] [--max-asins N] [--offer-limit 100] [--seller-limit 30] [--inventory-limit 200] [--storefront-update-hours 168] [--seller-cache-hours 168] [--max-seller-requests N] [--sellability] [--include-stock] [--dry-run] [--no-resume]", ); process.exit(1); } diff --git a/src/web/frontend.tsx b/src/web/frontend.tsx index 19aa424..f4bf2ad 100644 --- a/src/web/frontend.tsx +++ b/src/web/frontend.tsx @@ -114,22 +114,15 @@ type StalkerResultItem = { started_at: string; status: string; input_file: string; - source_asin: string; - title: string | null; - offer_count: number; - candidate_seller_count: number; - matched_seller_count: number; - scan_fetched_at: string; seller_id: string; seller_name: string | null; rating: number | null; rating_count: number | null; storefront_asin_total: number | null; persisted_inventory_sample_count: number | null; - offer_price: number | null; - condition: string | null; - is_fba: number | null; - stock: number | null; + discovered_from_count: number; + first_seen_at: string; + last_seen_at: string; persisted_inventory_asin_count: number; inventory_sample_asins: string | null; }; @@ -138,7 +131,6 @@ type StalkerResultsResponse = { items: StalkerResultItem[]; summary: { runs: number; - sourceAsins: number; sellers: number; persistedInventoryAsins: number; }; @@ -907,7 +899,7 @@ function StalkerExplorer({ onBack }: { onBack: () => void }) { const [maxRatingCount, setMaxRatingCount] = useState("30"); const [page, setPage] = useState(1); const [pageSize, setPageSize] = useState(25); - const [sort, setSort] = useState({ field: "started_at", direction: "DESC" }); + const [sort, setSort] = useState({ field: "persisted_inventory_asin_count", direction: "DESC" }); useEffect(() => { let cancelled = false; @@ -943,7 +935,7 @@ function StalkerExplorer({ onBack }: { onBack: () => void }) {
-

Stalker Results

+

Seller Storefronts

@@ -951,23 +943,19 @@ function StalkerExplorer({ onBack }: { onBack: () => void }) {
Runs
{formatNumber(results?.summary.runs)}
-
-
Source ASINs
-
{formatNumber(results?.summary.sourceAsins)}
-
Matched sellers
{formatNumber(results?.summary.sellers)}
-
Persisted inventory ASINs
+
Sellable inventory ASINs
{formatNumber(results?.summary.persistedInventoryAsins)}
- { setPage(1); setSearch(e.target.value); }} placeholder="Search source ASIN/title/seller/inventory" /> + { setPage(1); setSearch(e.target.value); }} placeholder="Search seller or sellable ASIN" /> { setPage(1); setSellerId(e.target.value.toUpperCase()); }} placeholder="Seller ID" /> { setPage(1); setRunId(e.target.value); }} placeholder="Run ID" /> { setPage(1); setMinRatingCount(e.target.value); }} placeholder="Min rating count" /> @@ -986,24 +974,20 @@ function StalkerExplorer({ onBack }: { onBack: () => void }) { - - - - - FBA - + - - Inventory ASIN sample + + + Sellable inventory ASIN sample {loading ? ( - Loading... + Loading... ) : results?.items.length ? ( results.items.map((item) => { const inventorySample = (item.inventory_sample_asins ?? "") @@ -1011,20 +995,16 @@ function StalkerExplorer({ onBack }: { onBack: () => void }) { .filter(Boolean) .slice(0, 20); return ( - + {item.runId} - {formatDate(item.started_at)} - {item.source_asin} - {item.title || "-"} {item.seller_id} {item.seller_name || "-"} {formatNumber(item.rating)} {formatNumber(item.rating_count)} - {formatCurrency(item.offer_price)} - {formatBoolean(item.is_fba)} - {formatNumber(item.stock)} + {formatNumber(item.discovered_from_count)} {formatNumber(item.storefront_asin_total)} {formatNumber(item.persisted_inventory_asin_count)} + {formatDate(item.last_seen_at)} {inventorySample.length === 0 ? "-" : inventorySample.map((asin) => ( {asin} @@ -1034,7 +1014,7 @@ function StalkerExplorer({ onBack }: { onBack: () => void }) { ); }) ) : ( - No stalker results found + No seller storefronts found )} From f6178a665c26fb48c1376da186421d3a5690c487 Mon Sep 17 00:00:00 2001 From: Victor Noguera Date: Tue, 19 May 2026 19:37:05 -0400 Subject: [PATCH 3/9] feat: add Stalker products functionality with filtering, pagination, and purge option --- src/server.ts | 162 ++++++++++++++++++++++++++++++ src/sp-api.ts | 17 ++-- src/stalker.test.ts | 11 +- src/stalker.ts | 24 +++++ src/web/frontend.tsx | 232 +++++++++++++++++++++++++++++++++++++++++-- src/web/styles.css | 18 ++++ 6 files changed, 444 insertions(+), 20 deletions(-) diff --git a/src/server.ts b/src/server.ts index 92a3a2e..8d6957e 100644 --- a/src/server.ts +++ b/src/server.ts @@ -71,6 +71,20 @@ type StalkerResultRecord = { inventory_sample_asins: string | null; }; +type StalkerProductRecord = { + runId: number; + started_at: string; + seller_id: string; + seller_name: string | null; + rating: number | null; + rating_count: number | null; + asin: string; + can_sell: number; + sellability_status: string; + sellability_reason: string | null; + last_seen_at: string; +}; + const DB_PATH = process.env.RESULTS_DB_PATH || path.join("db", "results.db"); const DEFAULT_PAGE_SIZE = 25; const MAX_PAGE_SIZE = 200; @@ -799,6 +813,143 @@ function getStalkerResults(filters: URLSearchParams) { }; } +function parseStalkerProductFilters(filters: URLSearchParams) { + const q = filters.get("q")?.trim() || ""; + const sellerId = filters.get("sellerId")?.trim().toUpperCase() || ""; + const runIdRaw = filters.get("runId")?.trim() || ""; + + const conditions = [ + "inv.can_sell = 1", + "inv.sellability_status = 'available'", + ]; + const params: Array = []; + + if (runIdRaw) { + const runId = Number(runIdRaw); + if (Number.isInteger(runId) && runId > 0) { + conditions.push("r.id = ?"); + params.push(runId); + } + } + + if (sellerId) { + conditions.push("s.seller_id = ?"); + params.push(sellerId); + } + + if (q) { + const wildcard = `%${q}%`; + conditions.push( + "(inv.asin LIKE ? OR s.seller_id LIKE ? OR s.seller_name LIKE ?)", + ); + params.push(wildcard, wildcard, wildcard); + } + + return { + where: `WHERE ${conditions.join(" AND ")}`, + params, + }; +} + +function parseStalkerProductSort(sortParam: string | null): string { + const allowedSort = new Set([ + "runId", + "started_at", + "seller_id", + "seller_name", + "rating", + "rating_count", + "asin", + "last_seen_at", + ]); + return parseSort(sortParam, allowedSort, "last_seen_at DESC, asin ASC"); +} + +function getStalkerProducts(filters: URLSearchParams) { + const page = parseIntParam(filters.get("page"), 1); + const pageSize = Math.min( + parseIntParam(filters.get("pageSize"), DEFAULT_PAGE_SIZE), + MAX_PAGE_SIZE, + ); + const offset = (page - 1) * pageSize; + const { where, params } = parseStalkerProductFilters(filters); + const orderBy = parseStalkerProductSort(filters.get("sort")); + + const baseSelect = ` + SELECT + r.id AS runId, + r.started_at, + s.seller_id, + s.seller_name, + s.rating, + s.rating_count, + inv.asin, + inv.can_sell, + inv.sellability_status, + inv.sellability_reason, + inv.last_seen_at + FROM stalker_seller_inventory inv + JOIN stalker_runs r ON r.id = inv.run_id + JOIN stalker_sellers s ON s.seller_id = inv.seller_id + ${where} + `; + + const totalRow = db + .query(`SELECT COUNT(*) AS total FROM (${baseSelect}) stalker_products`) + .get(...params) as { total: number }; + + const summary = db + .query( + `SELECT + COUNT(DISTINCT runId) AS runs, + COUNT(DISTINCT seller_id) AS sellers, + COUNT(DISTINCT asin) AS products + FROM (${baseSelect}) stalker_products`, + ) + .get(...params) as { + runs: number; + sellers: number; + products: number; + }; + + const items = db + .query( + `SELECT * FROM (${baseSelect}) stalker_products + ORDER BY ${orderBy} + LIMIT ? OFFSET ?`, + ) + .all(...params, pageSize, offset) as StalkerProductRecord[]; + + return { + items, + summary, + page, + pageSize, + total: totalRow.total, + totalPages: Math.max(1, Math.ceil(totalRow.total / pageSize)), + }; +} + +function purgeStalkerData() { + const counts = { + inventory: (db.query("SELECT COUNT(*) AS count FROM stalker_seller_inventory").get() as { count: number }).count, + asinSellers: (db.query("SELECT COUNT(*) AS count FROM stalker_asin_sellers").get() as { count: number }).count, + sellers: (db.query("SELECT COUNT(*) AS count FROM stalker_sellers").get() as { count: number }).count, + scans: (db.query("SELECT COUNT(*) AS count FROM stalker_asin_scans").get() as { count: number }).count, + runs: (db.query("SELECT COUNT(*) AS count FROM stalker_runs").get() as { count: number }).count, + }; + + db.transaction(() => { + db.run("DELETE FROM stalker_seller_inventory"); + db.run("DELETE FROM stalker_asin_sellers"); + db.run("DELETE FROM stalker_sellers"); + db.run("DELETE FROM stalker_asin_scans"); + db.run("DELETE FROM stalker_runs"); + })(); + + return { ok: true, deleted: counts }; +} + function getRun(processType: ProcessType, runId: number) { if (processType === "lead_analysis") { const run = db @@ -1430,6 +1581,7 @@ const server = Bun.serve({ "/": index, "/products": index, "/stalker": index, + "/stalker/products": index, "/runs/:processType/:runId": index, "/api/runs": (req) => { const url = new URL(req.url); @@ -1443,6 +1595,16 @@ const server = Bun.serve({ const url = new URL(req.url); return json(getStalkerResults(url.searchParams)); }, + "/api/stalker/products": (req) => { + const url = new URL(req.url); + return json(getStalkerProducts(url.searchParams)); + }, + "/api/stalker/purge": (req) => { + if (req.method !== "DELETE" && req.method !== "POST") { + return json({ error: "Method not allowed" }, 405); + } + return json(purgeStalkerData()); + }, "/api/upc/map": async (req) => { let upcs: string[]; try { diff --git a/src/sp-api.ts b/src/sp-api.ts index eace361..eb2b9a8 100644 --- a/src/sp-api.ts +++ b/src/sp-api.ts @@ -123,7 +123,10 @@ function round2(value: number): number { return Math.round(value * 100) / 100; } -const SELLABILITY_CONCURRENCY = 5; +const LISTINGS_RESTRICTIONS_RATE_LIMIT_PER_SECOND = 5; +const LISTINGS_RESTRICTIONS_BURST_REQUESTS = 10; +const SELLABILITY_CONCURRENCY = LISTINGS_RESTRICTIONS_RATE_LIMIT_PER_SECOND; +const SELLABILITY_PROGRESS_INTERVAL = LISTINGS_RESTRICTIONS_BURST_REQUESTS; const PRICING_CONCURRENCY = 5; const UPC_PATTERN = /^\d{12,14}$/; @@ -621,8 +624,7 @@ export async function fetchSellabilityBatch( } let completed = 0; - let running = 0; - const queue = [...asins]; + const queue = [...asins]; async function next(): Promise { while (queue.length > 0) { @@ -630,9 +632,12 @@ export async function fetchSellabilityBatch( const info = await fetchSellabilityInternal(spClient!, asin); results.set(asin, info); completed++; - if (completed % 10 === 0 || completed === asins.length) { - console.log(` [sellability] ${completed}/${asins.length} checked`); - } + if ( + completed % SELLABILITY_PROGRESS_INTERVAL === 0 || + completed === asins.length + ) { + console.log(` [sellability] ${completed}/${asins.length} checked`); + } } } diff --git a/src/stalker.test.ts b/src/stalker.test.ts index 6c5ad7e..e86e2d5 100644 --- a/src/stalker.test.ts +++ b/src/stalker.test.ts @@ -222,7 +222,7 @@ test("runStalker fetches product offers, filters sellers, and persists storefron expect(stats.scannedAsins).toBe(1); expect(stats.sourceAsinsWithMatches).toBe(1); expect(stats.matchedSellers).toBe(1); - expect(stats.persistedInventoryAsins).toBe(2); + expect(stats.persistedInventoryAsins).toBe(0); expect(stats.failedAsins).toBe(0); expect(stats.candidateSellers).toBe(2); expect(stats.qualifyingSellers).toBe(1); @@ -253,7 +253,7 @@ test("runStalker fetches product offers, filters sellers, and persists storefron expect(run.inventory_sellability_checked_asins).toBe(0); expect(run.inventory_sellability_available_asins).toBe(0); expect(run.inventory_sellability_excluded_asins).toBe(0); - expect(run.persisted_inventory_asins).toBe(2); + expect(run.persisted_inventory_asins).toBe(0); const scan = db.query("SELECT * FROM stalker_asin_scans").get() as any; expect(scan.source_asin).toBe("B000000001"); @@ -267,7 +267,7 @@ test("runStalker fetches product offers, filters sellers, and persists storefron expect(sellers[0].seller_id).toBe("AQUALIFIED"); expect(sellers[0].rating_count).toBe(12); expect(sellers[0].storefront_asin_total).toBe(2); - expect(sellers[0].persisted_inventory_sample_count).toBe(2); + expect(sellers[0].persisted_inventory_sample_count).toBe(0); const asinSellers = db.query("SELECT * FROM stalker_asin_sellers").all() as any[]; expect(asinSellers.length).toBe(1); @@ -278,8 +278,5 @@ test("runStalker fetches product offers, filters sellers, and persists storefron const inventory = db .query("SELECT asin FROM stalker_seller_inventory ORDER BY asin") .all() as Array<{ asin: string }>; - expect(inventory.map((row) => row.asin)).toEqual([ - "B111111111", - "B222222222", - ]); + expect(inventory.map((row) => row.asin)).toEqual([]); }); diff --git a/src/stalker.ts b/src/stalker.ts index 8fba937..712a030 100644 --- a/src/stalker.ts +++ b/src/stalker.ts @@ -338,6 +338,7 @@ export async function runStalker(args: StalkerArgs): Promise { if (args.sellability && !args.dryRun) { await enrichInventorySellability(result, stats); } + applyInventoryPersistencePolicy(result, args.sellability && !args.dryRun); if (!args.dryRun && runId != null) { persistAsinResult(database, runId, result); @@ -457,6 +458,22 @@ async function scanAsin( }; } +function applyInventoryPersistencePolicy( + result: StalkerAsinResult, + requireAvailableSellability: boolean, +): void { + for (const { seller } of result.matchedSellers) { + seller.storefrontItems = seller.storefrontItems.filter((item) => { + if (!requireAvailableSellability) return false; + return ( + item.sellability?.canSell === true && + item.sellability.sellabilityStatus === "available" + ); + }); + seller.storefrontAsins = seller.storefrontItems.map((item) => item.asin); + } +} + async function enrichInventorySellability( result: StalkerAsinResult, stats: StalkerRunStats, @@ -833,6 +850,13 @@ function upsertSellerInventory( ); for (const item of seller.storefrontItems) { + if ( + item.sellability?.canSell !== true || + item.sellability.sellabilityStatus !== "available" + ) { + continue; + } + insert.run( runId, seller.sellerId, diff --git a/src/web/frontend.tsx b/src/web/frontend.tsx index f4bf2ad..dd35182 100644 --- a/src/web/frontend.tsx +++ b/src/web/frontend.tsx @@ -140,6 +140,33 @@ type StalkerResultsResponse = { totalPages: number; }; +type StalkerProductItem = { + runId: number; + started_at: string; + seller_id: string; + seller_name: string | null; + rating: number | null; + rating_count: number | null; + asin: string; + can_sell: number; + sellability_status: string; + sellability_reason: string | null; + last_seen_at: string; +}; + +type StalkerProductsResponse = { + items: StalkerProductItem[]; + summary: { + runs: number; + sellers: number; + products: number; + }; + page: number; + pageSize: number; + total: number; + totalPages: number; +}; + type SortState = { field: string; direction: SortDirection; @@ -234,10 +261,12 @@ function Dashboard({ onOpenRun, onOpenProducts, onOpenStalker, + onOpenStalkerProducts, }: { onOpenRun: (run: Run) => void; onOpenProducts: (verdict: VerdictFilter) => void; onOpenStalker: () => void; + onOpenStalkerProducts: () => void; }) { const [runs, setRuns] = useState(null); const [loading, setLoading] = useState(false); @@ -328,7 +357,10 @@ function Dashboard({

Runs Dashboard

- +
+ + +
@@ -889,9 +921,16 @@ function ProductList({ verdict, onBack }: { verdict: VerdictFilter; onBack: () = ); } -function StalkerExplorer({ onBack }: { onBack: () => void }) { +function StalkerExplorer({ + onBack, + onOpenProducts, +}: { + onBack: () => void; + onOpenProducts: () => void; +}) { const [results, setResults] = useState(null); const [loading, setLoading] = useState(false); + const [purging, setPurging] = useState(false); const [search, setSearch] = useState(""); const [sellerId, setSellerId] = useState(""); const [runId, setRunId] = useState(""); @@ -900,6 +939,7 @@ function StalkerExplorer({ onBack }: { onBack: () => void }) { const [page, setPage] = useState(1); const [pageSize, setPageSize] = useState(25); const [sort, setSort] = useState({ field: "persisted_inventory_asin_count", direction: "DESC" }); + const [refreshTick, setRefreshTick] = useState(0); useEffect(() => { let cancelled = false; @@ -928,14 +968,41 @@ function StalkerExplorer({ onBack }: { onBack: () => void }) { return () => { cancelled = true; }; - }, [search, sellerId, runId, minRatingCount, maxRatingCount, page, pageSize, sort]); + }, [search, sellerId, runId, minRatingCount, maxRatingCount, page, pageSize, sort, refreshTick]); + + async function purgeStalkerData() { + const confirmed = window.confirm("Permanently delete all Stalker runs, sellers, and sellable products from the database?"); + if (!confirmed) return; + + setPurging(true); + try { + const res = await fetch("/api/stalker/purge", { method: "DELETE" }); + if (!res.ok) { + const payload = await res.json().catch(() => null) as { error?: string } | null; + window.alert(payload?.error ?? "Failed to purge Stalker data"); + return; + } + setPage(1); + setRefreshTick((tick) => tick + 1); + } finally { + setPurging(false); + } + } return (
-

Seller Storefronts

+
+

Seller Storefronts

+
+ + +
+
@@ -1032,11 +1099,142 @@ function StalkerExplorer({ onBack }: { onBack: () => void }) { ); } +function StalkerProductsExplorer({ + onBack, + onOpenSellers, +}: { + onBack: () => void; + onOpenSellers: () => void; +}) { + const [results, setResults] = useState(null); + const [loading, setLoading] = useState(false); + const [search, setSearch] = useState(""); + const [sellerId, setSellerId] = useState(""); + const [runId, setRunId] = useState(""); + const [page, setPage] = useState(1); + const [pageSize, setPageSize] = useState(50); + const [sort, setSort] = useState({ field: "last_seen_at", direction: "DESC" }); + + useEffect(() => { + let cancelled = false; + async function load() { + setLoading(true); + const params = new URLSearchParams({ + page: String(page), + pageSize: String(pageSize), + sort: buildSortValue(sort), + }); + if (search) params.set("q", search); + if (sellerId) params.set("sellerId", sellerId); + if (runId) params.set("runId", runId); + + const res = await fetch(`/api/stalker/products?${params.toString()}`); + const payload = (await res.json()) as StalkerProductsResponse; + if (!cancelled) { + setResults(payload); + setLoading(false); + } + } + + load(); + return () => { + cancelled = true; + }; + }, [search, sellerId, runId, page, pageSize, sort]); + + return ( +
+ + +
+
+

Sellable Stalker Products

+ +
+
+ +
+
+
Runs
+
{formatNumber(results?.summary.runs)}
+
+
+
Sellers
+
{formatNumber(results?.summary.sellers)}
+
+
+
Sellable products
+
{formatNumber(results?.summary.products)}
+
+
+ +
+
+ { setPage(1); setSearch(e.target.value); }} placeholder="Search ASIN or seller" /> + { setPage(1); setSellerId(e.target.value.toUpperCase()); }} placeholder="Seller ID" /> + { setPage(1); setRunId(e.target.value); }} placeholder="Run ID" /> + +
+
+ +
+
+ + + + + + + + + + + + + + {loading ? ( + + ) : results?.items.length ? ( + results.items.map((item) => ( + + + + + + + + + + )) + ) : ( + + )} + +
Status
Loading...
{item.asin}{item.seller_id}{item.seller_name || "-"}{formatNumber(item.rating_count)}{item.sellability_status}{item.runId}{formatDate(item.last_seen_at)}
No sellable Stalker products found
+
+
+
Showing {results?.items.length ?? 0} of {results?.total ?? 0}
+
+ + Page {results?.page ?? page} / {results?.totalPages ?? 1} + +
+
+
+
+ ); +} + type AppRoute = | { kind: "dashboard" } | { kind: "run"; processType: ProcessType; runId: number } | { kind: "products"; verdict: VerdictFilter } - | { kind: "stalker" }; + | { kind: "stalker" } + | { kind: "stalker-products" }; function parseRoute(pathname: string, search: string): AppRoute { const runMatch = pathname.match(/^\/runs\/(lead_analysis|category_analysis)\/(\d+)$/); @@ -1055,6 +1253,10 @@ function parseRoute(pathname: string, search: string): AppRoute { return { kind: "stalker" }; } + if (pathname === "/stalker/products") { + return { kind: "stalker-products" }; + } + return { kind: "dashboard" }; } @@ -1084,6 +1286,11 @@ function App() { setRoute({ kind: "stalker" }); } + function openStalkerProducts() { + history.pushState({}, "", "/stalker/products"); + setRoute({ kind: "stalker-products" }); + } + function backToDashboard() { history.pushState({}, "", "/"); setRoute({ kind: "dashboard" }); @@ -1098,10 +1305,21 @@ function App() { } if (route.kind === "stalker") { - return ; + return ; } - return ; + if (route.kind === "stalker-products") { + return ; + } + + return ( + + ); } const root = document.getElementById("root"); diff --git a/src/web/styles.css b/src/web/styles.css index 205f4dd..8303a4a 100644 --- a/src/web/styles.css +++ b/src/web/styles.css @@ -48,6 +48,13 @@ p { gap: 12px; } +.button-row { + display: flex; + align-items: center; + gap: 8px; + flex-wrap: wrap; +} + .toolbar input, .toolbar select, button { @@ -63,6 +70,17 @@ button { cursor: pointer; } +button.danger { + border-color: #efb8b8; + color: #9f1c1c; + background: #fff6f6; +} + +button:disabled { + cursor: not-allowed; + opacity: 0.58; +} + .table-wrap { overflow: auto; border: 1px solid #eceef0; From 0552d183b3a47fbea7acdedad9a36add77a6439c Mon Sep 17 00:00:00 2001 From: Victor Noguera Date: Tue, 19 May 2026 19:57:53 -0400 Subject: [PATCH 4/9] feat: enhance Stalker functionality with additional product details and analysis capabilities --- src/database.ts | 19 +- src/server.ts | 48 ++++- src/stalker-analyze.ts | 329 ++++++++++++++++++++++++++++++++ src/stalker-sellability.test.ts | 46 ++++- src/stalker.test.ts | 1 + src/stalker.ts | 299 ++++++++++++++++++++++++++++- src/web/frontend.tsx | 79 ++++++-- 7 files changed, 795 insertions(+), 26 deletions(-) create mode 100644 src/stalker-analyze.ts diff --git a/src/database.ts b/src/database.ts index 35a1074..8d4413f 100644 --- a/src/database.ts +++ b/src/database.ts @@ -419,6 +419,16 @@ export function initStalkerDb(database: Database): void { can_sell INTEGER, sellability_status TEXT, sellability_reason TEXT, + product_title TEXT, + brand TEXT, + category_tree TEXT, + current_price REAL, + avg_price_90d REAL, + sales_rank INTEGER, + monthly_sold INTEGER, + seller_count INTEGER, + amazon_is_seller INTEGER, + raw_product_json TEXT, last_seen_at TEXT NOT NULL, raw_inventory_json TEXT, UNIQUE(run_id, seller_id, asin), @@ -445,6 +455,9 @@ export function initStalkerDb(database: Database): void { database.run( `CREATE INDEX IF NOT EXISTS idx_stalker_inventory_asin ON stalker_seller_inventory(asin);`, ); + database.run( + `CREATE INDEX IF NOT EXISTS idx_stalker_inventory_product_title ON stalker_seller_inventory(product_title);`, + ); } function resetLegacyStalkerSchema(database: Database): void { @@ -473,5 +486,9 @@ function inventoryColumnsHaveSellability(database: Database): boolean { const inventoryColumns = database .query("PRAGMA table_info(stalker_seller_inventory)") .all() as Array<{ name: string }>; - return inventoryColumns.some((column) => column.name === "sellability_status"); + const columnNames = new Set(inventoryColumns.map((column) => column.name)); + return ( + columnNames.has("sellability_status") && + columnNames.has("product_title") + ); } diff --git a/src/server.ts b/src/server.ts index 8d6957e..aa4213b 100644 --- a/src/server.ts +++ b/src/server.ts @@ -82,6 +82,18 @@ type StalkerProductRecord = { can_sell: number; sellability_status: string; sellability_reason: string | null; + product_title: string | null; + brand: string | null; + category_tree: string | null; + current_price: number | null; + avg_price_90d: number | null; + sales_rank: number | null; + monthly_sold: number | null; + seller_count: number | null; + amazon_is_seller: number | null; + verdict: string | null; + confidence: number | null; + reasoning: string | null; last_seen_at: string; }; @@ -840,9 +852,16 @@ function parseStalkerProductFilters(filters: URLSearchParams) { if (q) { const wildcard = `%${q}%`; conditions.push( - "(inv.asin LIKE ? OR s.seller_id LIKE ? OR s.seller_name LIKE ?)", + `( + inv.asin LIKE ? + OR inv.product_title LIKE ? + OR inv.brand LIKE ? + OR inv.category_tree LIKE ? + OR s.seller_id LIKE ? + OR s.seller_name LIKE ? + )`, ); - params.push(wildcard, wildcard, wildcard); + params.push(wildcard, wildcard, wildcard, wildcard, wildcard, wildcard); } return { @@ -860,9 +879,19 @@ function parseStalkerProductSort(sortParam: string | null): string { "rating", "rating_count", "asin", + "product_title", + "brand", + "current_price", + "avg_price_90d", + "sales_rank", + "monthly_sold", + "seller_count", + "amazon_is_seller", + "verdict", + "confidence", "last_seen_at", ]); - return parseSort(sortParam, allowedSort, "last_seen_at DESC, asin ASC"); + return parseSort(sortParam, allowedSort, "monthly_sold DESC, last_seen_at DESC, asin ASC"); } function getStalkerProducts(filters: URLSearchParams) { @@ -887,10 +916,23 @@ function getStalkerProducts(filters: URLSearchParams) { inv.can_sell, inv.sellability_status, inv.sellability_reason, + inv.product_title, + inv.brand, + inv.category_tree, + inv.current_price, + inv.avg_price_90d, + inv.sales_rank, + inv.monthly_sold, + inv.seller_count, + inv.amazon_is_seller, + analysis.verdict, + analysis.confidence, + analysis.reasoning, inv.last_seen_at FROM stalker_seller_inventory inv JOIN stalker_runs r ON r.id = inv.run_id JOIN stalker_sellers s ON s.seller_id = inv.seller_id + LEFT JOIN product_analysis_results analysis ON analysis.asin = inv.asin ${where} `; diff --git a/src/stalker-analyze.ts b/src/stalker-analyze.ts new file mode 100644 index 0000000..d93f504 --- /dev/null +++ b/src/stalker-analyze.ts @@ -0,0 +1,329 @@ +import { type Database, closeDb, getDb, initDb } from "./database.ts"; +import { analyzeProducts } from "./llm.ts"; +import { fetchSpApiPricingAndFees } from "./sp-api.ts"; +import type { + AnalysisResult, + EnrichedProduct, + KeepaData, + ProductRecord, + SellabilityInfo, +} from "./types.ts"; + +type Args = { + dbPath: string; + stalkerRunId: number; + analysisRunId: number; + asins: string[]; +}; + +type InventoryRow = { + asin: string; + product_title: string | null; + brand: string | null; + category_tree: string | null; + current_price: number | null; + avg_price_90d: number | null; + sales_rank: number | null; + monthly_sold: number | null; + seller_count: number | null; + amazon_is_seller: number | null; + can_sell: number | null; + sellability_status: SellabilityInfo["sellabilityStatus"] | null; + sellability_reason: string | null; +}; + +function readFlagValue(args: string[], flag: string): string | undefined { + const index = args.indexOf(flag); + if (index === -1) return undefined; + return args[index + 1]; +} + +function parseArgs(argv = process.argv.slice(2)): Args { + const dbPath = readFlagValue(argv, "--db"); + const stalkerRunId = Number(readFlagValue(argv, "--stalker-run-id")); + const analysisRunId = Number(readFlagValue(argv, "--analysis-run-id")); + const asins = (readFlagValue(argv, "--asins") ?? "") + .split(",") + .map((asin) => asin.trim().toUpperCase()) + .filter(Boolean); + + if (!dbPath) throw new Error("Missing --db"); + if (!Number.isInteger(stalkerRunId) || stalkerRunId <= 0) { + throw new Error("--stalker-run-id must be a positive integer"); + } + if (!Number.isInteger(analysisRunId) || analysisRunId <= 0) { + throw new Error("--analysis-run-id must be a positive integer"); + } + if (asins.length === 0) throw new Error("Missing --asins"); + + return { dbPath, stalkerRunId, analysisRunId, asins }; +} + +function parseCategoryTree(value: string | null): string[] { + if (!value) return []; + try { + const parsed = JSON.parse(value); + return Array.isArray(parsed) + ? parsed.filter((item): item is string => typeof item === "string") + : []; + } catch { + return []; + } +} + +function toProductRecord(row: InventoryRow): ProductRecord { + const categoryTree = parseCategoryTree(row.category_tree); + return { + asin: row.asin, + name: row.product_title ?? row.asin, + brand: row.brand ?? undefined, + category: categoryTree.join(" > ") || undefined, + unitCost: 0, + amazonRank: row.sales_rank ?? undefined, + sellingPriceFromSheet: row.current_price ?? undefined, + avgPrice90FromSheet: row.avg_price_90d ?? undefined, + }; +} + +function toKeepaData(row: InventoryRow): KeepaData { + return { + currentPrice: row.current_price, + avgPrice90: row.avg_price_90d, + minPrice90: null, + maxPrice90: null, + salesRank: row.sales_rank, + salesRankAvg90: null, + salesRankDrops30: null, + salesRankDrops90: null, + sellerCount: row.seller_count, + amazonIsSeller: + row.amazon_is_seller == null ? null : row.amazon_is_seller === 1, + amazonBuyboxSharePct90d: null, + buyBoxSeller: null, + buyBoxPrice: null, + buyBoxAvg90: null, + monthlySold: row.monthly_sold, + categoryTree: parseCategoryTree(row.category_tree), + }; +} + +function toSellability(row: InventoryRow): SellabilityInfo { + return { + canSell: row.can_sell == null ? null : row.can_sell === 1, + sellabilityStatus: row.sellability_status ?? "unknown", + sellabilityReason: row.sellability_reason ?? undefined, + }; +} + +function loadInventoryRows( + database: Database, + stalkerRunId: number, + asins: string[], +): InventoryRow[] { + const placeholders = asins.map(() => "?").join(","); + return database + .query( + `SELECT + asin, product_title, brand, category_tree, current_price, avg_price_90d, + sales_rank, monthly_sold, seller_count, amazon_is_seller, can_sell, + sellability_status, sellability_reason + FROM stalker_seller_inventory + WHERE run_id = ? + AND can_sell = 1 + AND sellability_status = 'available' + AND asin IN (${placeholders}) + GROUP BY asin`, + ) + .all(stalkerRunId, ...asins) as InventoryRow[]; +} + +async function buildEnrichedProducts( + rows: InventoryRow[], +): Promise { + const enriched: EnrichedProduct[] = []; + + for (const row of rows) { + const sellability = toSellability(row); + const spApi = await fetchSpApiPricingAndFees( + row.asin, + sellability, + row.current_price, + ); + + enriched.push({ + record: toProductRecord(row), + keepa: toKeepaData(row), + spApi, + fetchedAt: new Date().toISOString(), + }); + } + + return enriched; +} + +function insertProductAnalysisResults( + database: Database, + runId: number, + results: AnalysisResult[], +): void { + if (results.length === 0) return; + + const insert = database.prepare(` + INSERT INTO product_analysis_results ( + asin, run_id, name, brand, category, unit_cost, + current_price, avg_price_90d, avg_price_90d_sheet, + selling_price_sheet, sales_rank, sales_rank_avg_90d, + seller_count, amazon_is_seller, amazon_buybox_share_pct_90d, + monthly_sold, rank_drops_30d, rank_drops_90d, + fba_fee, fbm_fee, referral_percent, can_sell, + sellability_status, sellability_reason, + verdict, confidence, reasoning, fetched_at + ) VALUES ( + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ? + ) + ON CONFLICT(asin) DO UPDATE SET + run_id = excluded.run_id, + name = excluded.name, + brand = excluded.brand, + category = excluded.category, + unit_cost = excluded.unit_cost, + current_price = excluded.current_price, + avg_price_90d = excluded.avg_price_90d, + avg_price_90d_sheet = excluded.avg_price_90d_sheet, + selling_price_sheet = excluded.selling_price_sheet, + sales_rank = excluded.sales_rank, + sales_rank_avg_90d = excluded.sales_rank_avg_90d, + seller_count = excluded.seller_count, + amazon_is_seller = excluded.amazon_is_seller, + amazon_buybox_share_pct_90d = excluded.amazon_buybox_share_pct_90d, + monthly_sold = excluded.monthly_sold, + rank_drops_30d = excluded.rank_drops_30d, + rank_drops_90d = excluded.rank_drops_90d, + fba_fee = excluded.fba_fee, + fbm_fee = excluded.fbm_fee, + referral_percent = excluded.referral_percent, + can_sell = excluded.can_sell, + sellability_status = excluded.sellability_status, + sellability_reason = excluded.sellability_reason, + verdict = excluded.verdict, + confidence = excluded.confidence, + reasoning = excluded.reasoning, + fetched_at = excluded.fetched_at + `); + + database.transaction((batch: AnalysisResult[]) => { + for (const result of batch) { + const keepa = result.product.keepa; + const record = result.product.record; + const spApi = result.product.spApi; + insert.run( + record.asin, + runId, + record.name, + record.brand ?? null, + record.category ?? keepa?.categoryTree.join(" > ") ?? null, + record.unitCost ?? null, + keepa?.currentPrice ?? spApi.estimatedSalePrice ?? null, + keepa?.avgPrice90 ?? null, + record.avgPrice90FromSheet ?? null, + record.sellingPriceFromSheet ?? null, + keepa?.salesRank ?? record.amazonRank ?? null, + keepa?.salesRankAvg90 ?? null, + keepa?.sellerCount ?? null, + keepa?.amazonIsSeller == null ? null : keepa.amazonIsSeller ? 1 : 0, + keepa?.amazonBuyboxSharePct90d ?? null, + keepa?.monthlySold ?? null, + keepa?.salesRankDrops30 ?? null, + keepa?.salesRankDrops90 ?? null, + spApi.fbaFee ?? null, + spApi.fbmFee ?? null, + spApi.referralFeePercent ?? null, + spApi.canSell == null ? "unknown" : spApi.canSell ? "yes" : "no", + spApi.sellabilityStatus ?? null, + spApi.sellabilityReason ?? null, + result.verdict.verdict, + result.verdict.confidence, + result.verdict.reasoning ?? null, + result.product.fetchedAt, + ); + } + })(results); +} + +function refreshAnalysisRun(database: Database, runId: number): void { + const stats = database + .query( + `SELECT + COUNT(*) AS total, + SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba, + SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm, + SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip + FROM product_analysis_results + WHERE run_id = ?`, + ) + .get(runId) as { + total: number; + fba: number | null; + fbm: number | null; + skip: number | null; + }; + + database + .prepare( + `UPDATE category_analysis_runs + SET top_asins_checked = ?, + available_asins = ?, + fba_count = ?, + fbm_count = ?, + skip_count = ? + WHERE id = ?`, + ) + .run( + stats.total ?? 0, + stats.total ?? 0, + stats.fba ?? 0, + stats.fbm ?? 0, + stats.skip ?? 0, + runId, + ); +} + +async function main(): Promise { + const args = parseArgs(); + initDb(args.dbPath); + const database = getDb(args.dbPath); + + try { + const rows = loadInventoryRows(database, args.stalkerRunId, args.asins); + if (rows.length === 0) { + console.log("Stalker analysis: no sellable inventory rows to analyze."); + return; + } + + console.log(`Stalker analysis: analyzing ${rows.length} sellable ASIN(s).`); + const enriched = await buildEnrichedProducts(rows); + const verdicts = await analyzeProducts(enriched); + const results = enriched.map((product, index) => ({ + product, + verdict: verdicts[index] ?? { + asin: product.record.asin, + verdict: "SKIP" as const, + confidence: 0, + reasoning: "LLM analysis returned no verdict", + }, + })); + insertProductAnalysisResults(database, args.analysisRunId, results); + refreshAnalysisRun(database, args.analysisRunId); + } finally { + closeDb(); + } +} + +if (import.meta.main) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + }); +} diff --git a/src/stalker-sellability.test.ts b/src/stalker-sellability.test.ts index 33d2dbd..da5f4b8 100644 --- a/src/stalker-sellability.test.ts +++ b/src/stalker-sellability.test.ts @@ -75,6 +75,30 @@ test("sellability checks matched seller inventory, not the source ASIN", async ( const url = new URL(rawUrl); if (url.pathname === "/product") { + if (url.searchParams.get("asin") === "B111111111") { + return new Response( + JSON.stringify({ + products: [ + { + asin: "B111111111", + title: "Sellable Storefront Product", + brand: "Good Brand", + categoryTree: [{ name: "Kitchen" }, { name: "Storage" }], + monthlySold: 42, + stats: { + current: [null, null, null, 12345, null, null, null, null, null, null, null, 7], + avg: [2500], + }, + csv: [[0, 1999]], + }, + ], + tokensLeft: 10, + refillRate: 10, + }), + { status: 200 }, + ); + } + return new Response( JSON.stringify({ products: [ @@ -126,6 +150,7 @@ test("sellability checks matched seller inventory, not the source ASIN", async ( resume: true, maxSellerRequests: null, sellability: true, + analyzeSellable: false, }); expect(fetchSellabilityBatchMock.mock.calls.length).toBe(1); @@ -146,18 +171,37 @@ test("sellability checks matched seller inventory, not the source ASIN", async ( const inventory = db .query( - "SELECT asin, can_sell, sellability_status FROM stalker_seller_inventory ORDER BY asin", + `SELECT asin, can_sell, sellability_status, product_title, brand, + category_tree, current_price, avg_price_90d, sales_rank, monthly_sold, + seller_count + FROM stalker_seller_inventory ORDER BY asin`, ) .all() as Array<{ asin: string; can_sell: number | null; sellability_status: string | null; + product_title: string | null; + brand: string | null; + category_tree: string | null; + current_price: number | null; + avg_price_90d: number | null; + sales_rank: number | null; + monthly_sold: number | null; + seller_count: number | null; }>; expect(inventory).toEqual([ { asin: "B111111111", can_sell: 1, sellability_status: "available", + product_title: "Sellable Storefront Product", + brand: "Good Brand", + category_tree: JSON.stringify(["Kitchen", "Storage"]), + current_price: 19.99, + avg_price_90d: 25, + sales_rank: 12345, + monthly_sold: 42, + seller_count: 7, }, ]); }); diff --git a/src/stalker.test.ts b/src/stalker.test.ts index e86e2d5..25845c2 100644 --- a/src/stalker.test.ts +++ b/src/stalker.test.ts @@ -217,6 +217,7 @@ test("runStalker fetches product offers, filters sellers, and persists storefron resume: true, maxSellerRequests: null, sellability: false, + analyzeSellable: false, }); expect(stats.scannedAsins).toBe(1); diff --git a/src/stalker.ts b/src/stalker.ts index 712a030..306ef20 100644 --- a/src/stalker.ts +++ b/src/stalker.ts @@ -40,6 +40,7 @@ export type StalkerArgs = { resume: boolean; maxSellerRequests: number | null; sellability: boolean; + analyzeSellable: boolean; }; export type StalkerOffer = { @@ -66,6 +67,20 @@ type StalkerInventoryItem = { asin: string; rawInventory: unknown; sellability: SellabilityInfo | null; + productDetails: StalkerProductDetails | null; +}; + +type StalkerProductDetails = { + title: string | null; + brand: string | null; + categoryTree: string[]; + currentPrice: number | null; + avgPrice90: number | null; + salesRank: number | null; + monthlySold: number | null; + sellerCount: number | null; + amazonIsSeller: boolean | null; + rawProduct: Record; }; type StalkerAsinResult = { @@ -143,6 +158,11 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs { const dryRun = hasFlag(argv, "--dry-run"); const resume = !hasFlag(argv, "--no-resume"); const sellability = hasFlag(argv, "--sellability"); + const analyzeSellable = hasFlag(argv, "--analyze-sellable"); + + if (analyzeSellable && !sellability) { + printUsageAndExit("--analyze-sellable requires --sellability."); + } if (maxAsins != null && (!Number.isInteger(maxAsins) || maxAsins <= 0)) { printUsageAndExit("--max-asins must be a positive integer."); @@ -194,6 +214,7 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs { resume, maxSellerRequests, sellability, + analyzeSellable, }; } @@ -291,6 +312,10 @@ export async function runStalker(args: StalkerArgs): Promise { const runId = args.dryRun ? null : startStalkerRun(database, args.input, resumeFilteredAsins.length); + const analysisRunId = + !args.dryRun && args.analyzeSellable + ? startStalkerAnalysisRun(database, args.input) + : null; const stats: StalkerRunStats = { scannedAsins: 0, sourceAsinsWithMatches: 0, @@ -339,10 +364,23 @@ export async function runStalker(args: StalkerArgs): Promise { await enrichInventorySellability(result, stats); } applyInventoryPersistencePolicy(result, args.sellability && !args.dryRun); + if (args.sellability && !args.dryRun) { + await enrichInventoryProductDetails(result, apiKey); + } if (!args.dryRun && runId != null) { persistAsinResult(database, runId, result); } + const sellableAsins = collectPersistedInventoryAsins(result); + if ( + args.analyzeSellable && + !args.dryRun && + runId != null && + analysisRunId != null && + sellableAsins.length > 0 + ) { + await runSellableAnalysisChild(args.dbPath, runId, analysisRunId, sellableAsins); + } stats.scannedAsins += 1; stats.matchedSellers += result.matchedSellers.length; stats.persistedInventoryAsins += sumInventoryAsins(result); @@ -378,16 +416,23 @@ export async function runStalker(args: StalkerArgs): Promise { ); } logRunSummary(stats, args); + if (!args.dryRun && analysisRunId != null) { + finishStalkerAnalysisRun(database, analysisRunId, "completed"); + } return stats; } catch (error) { + const message = error instanceof Error ? error.message : String(error); if (!args.dryRun && runId != null) { finishStalkerRunWithError( database, runId, stats, - error instanceof Error ? error.message : String(error), + message, ); } + if (!args.dryRun && analysisRunId != null) { + finishStalkerAnalysisRun(database, analysisRunId, "failed", message); + } throw error; } } @@ -522,6 +567,24 @@ async function enrichInventorySellability( } } +async function enrichInventoryProductDetails( + result: StalkerAsinResult, + apiKey: string, +): Promise { + const items = result.matchedSellers.flatMap(({ seller }) => seller.storefrontItems); + const uniqueAsins = Array.from(new Set(items.map((item) => item.asin))); + if (uniqueAsins.length === 0) return; + + console.log( + `Stalker inventory details: fetching Keepa product details for ${uniqueAsins.length} sellable ASIN(s)...`, + ); + const detailsByAsin = await fetchKeepaInventoryProductDetails(apiKey, uniqueAsins); + + for (const item of items) { + item.productDetails = detailsByAsin.get(item.asin) ?? null; + } +} + async function fetchKeepaProduct( asin: string, apiKey: string, @@ -549,6 +612,39 @@ async function fetchKeepaProduct( return product; } +async function fetchKeepaInventoryProductDetails( + apiKey: string, + asins: string[], +): Promise> { + const details = new Map(); + const chunkSize = 100; + + for (let i = 0; i < asins.length; i += chunkSize) { + const chunk = asins.slice(i, i + chunkSize); + const params = new URLSearchParams({ + key: apiKey, + domain: DOMAIN_US, + asin: chunk.join(","), + stats: "30", + days: "30", + buybox: "1", + }); + + const data = await fetchKeepaWithRetries( + `${KEEPA_BASE}/product?${params.toString()}`, + `inventory product details ${i + 1}-${i + chunk.length}`, + ); + + for (const product of data.products ?? []) { + const asin = normalizeAsin(product.asin); + if (!asin) continue; + details.set(asin, parseInventoryProductDetails(product)); + } + } + + return details; +} + async function fetchSellerMetadata( sellerIds: string[], apiKey: string, @@ -839,12 +935,24 @@ function upsertSellerInventory( const insert = database.prepare( `INSERT INTO stalker_seller_inventory ( run_id, seller_id, asin, can_sell, sellability_status, - sellability_reason, last_seen_at, raw_inventory_json - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + sellability_reason, product_title, brand, category_tree, current_price, + avg_price_90d, sales_rank, monthly_sold, seller_count, amazon_is_seller, + raw_product_json, last_seen_at, raw_inventory_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(run_id, seller_id, asin) DO UPDATE SET can_sell = excluded.can_sell, sellability_status = excluded.sellability_status, sellability_reason = excluded.sellability_reason, + product_title = excluded.product_title, + brand = excluded.brand, + category_tree = excluded.category_tree, + current_price = excluded.current_price, + avg_price_90d = excluded.avg_price_90d, + sales_rank = excluded.sales_rank, + monthly_sold = excluded.monthly_sold, + seller_count = excluded.seller_count, + amazon_is_seller = excluded.amazon_is_seller, + raw_product_json = excluded.raw_product_json, last_seen_at = excluded.last_seen_at, raw_inventory_json = excluded.raw_inventory_json`, ); @@ -868,6 +976,20 @@ function upsertSellerInventory( : 0, item.sellability?.sellabilityStatus ?? null, item.sellability?.sellabilityReason ?? null, + item.productDetails?.title ?? null, + item.productDetails?.brand ?? null, + item.productDetails ? JSON.stringify(item.productDetails.categoryTree) : null, + item.productDetails?.currentPrice ?? null, + item.productDetails?.avgPrice90 ?? null, + item.productDetails?.salesRank ?? null, + item.productDetails?.monthlySold ?? null, + item.productDetails?.sellerCount ?? null, + item.productDetails?.amazonIsSeller == null + ? null + : item.productDetails.amazonIsSeller + ? 1 + : 0, + item.productDetails ? JSON.stringify(item.productDetails.rawProduct) : null, fetchedAt, JSON.stringify(item.rawInventory), ); @@ -890,6 +1012,19 @@ function startStalkerRun( return result.lastInsertRowid as number; } +function startStalkerAnalysisRun(database: Database, inputFile: string): number { + const result = database + .prepare( + `INSERT INTO category_analysis_runs ( + category_id, category_label, run_timestamp, top_asins_checked, + available_asins, fba_count, fbm_count, skip_count, status, error_message + ) VALUES (?, ?, ?, 0, 0, 0, 0, 0, 'running', NULL)`, + ) + .run(0, `Stalker: ${path.basename(inputFile)}`, new Date().toISOString()); + + return result.lastInsertRowid as number; +} + function loadPreviouslyScannedAsins(database: Database): Set { const rows = database .query(`SELECT DISTINCT source_asin FROM stalker_asin_scans`) @@ -1043,6 +1178,53 @@ function finishStalkerRunWithError( ); } +function finishStalkerAnalysisRun( + database: Database, + runId: number, + status: "completed" | "failed", + errorMessage: string | null = null, +): void { + const stats = database + .query( + `SELECT + COUNT(*) AS total, + SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba, + SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm, + SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip + FROM product_analysis_results + WHERE run_id = ?`, + ) + .get(runId) as { + total: number; + fba: number | null; + fbm: number | null; + skip: number | null; + }; + + database + .prepare( + `UPDATE category_analysis_runs + SET top_asins_checked = ?, + available_asins = ?, + fba_count = ?, + fbm_count = ?, + skip_count = ?, + status = ?, + error_message = ? + WHERE id = ?`, + ) + .run( + stats.total ?? 0, + stats.total ?? 0, + stats.fba ?? 0, + stats.fbm ?? 0, + stats.skip ?? 0, + status, + errorMessage, + runId, + ); +} + function normalizeSellerResponse( sellers: KeepaApiResponse["sellers"], ): Array<[string, Record]> { @@ -1129,7 +1311,7 @@ function collectStorefrontItems( const asin = normalizeAsin((value as Record).asin); if (asin && !seen.has(asin)) { seen.add(asin); - items.push({ asin, rawInventory: value, sellability: null }); + items.push({ asin, rawInventory: value, sellability: null, productDetails: null }); } return; } @@ -1137,7 +1319,70 @@ function collectStorefrontItems( const asin = normalizeAsin(value); if (!asin || seen.has(asin)) return; seen.add(asin); - items.push({ asin, rawInventory: { asin }, sellability: null }); + items.push({ asin, rawInventory: { asin }, sellability: null, productDetails: null }); +} + +function parseInventoryProductDetails( + product: Record, +): StalkerProductDetails { + const stats = product.stats; + const csv = product.csv; + return { + title: extractString(product.title), + brand: extractString(product.brand ?? product.manufacturer), + categoryTree: + product.categoryTree?.map((category: { name?: unknown }) => + extractString(category.name), + ).filter((name: string | null): name is string => !!name) ?? [], + currentPrice: extractCurrentPrice(csv), + avgPrice90: stats?.avg?.[0] != null ? stats.avg[0] / 100 : null, + salesRank: extractNumber(stats?.current?.[3]), + monthlySold: + extractNumber(product.monthlySold ?? stats?.monthlySold) ?? + extractNumber(product.salesRankDrops30 ?? stats?.salesRankDrops30), + sellerCount: extractNumber(stats?.current?.[11]), + amazonIsSeller: resolveAmazonIsSeller(product, stats, csv), + rawProduct: product, + }; +} + +function extractCurrentPrice(csv: unknown): number | null { + if (!Array.isArray(csv)) return null; + const amazonPrice = extractLatestPositiveKeepaPrice(csv[0]); + if (amazonPrice != null) return amazonPrice; + return extractLatestPositiveKeepaPrice(csv[1]); +} + +function extractLatestPositiveKeepaPrice(history: unknown): number | null { + if (!Array.isArray(history)) return null; + for (let i = history.length - 1; i >= 0; i--) { + const value = extractNumber(history[i]); + if (value != null && value > 0) return value / 100; + } + return null; +} + +function resolveAmazonIsSeller( + product: Record, + stats: Record | undefined, + csv: unknown, +): boolean | null { + if (typeof product.isAmazonSeller === "boolean") return product.isAmazonSeller; + if (typeof product.availabilityAmazon === "number") { + if (product.availabilityAmazon >= 0) return true; + if (product.availabilityAmazon === -1 || product.availabilityAmazon === -2) { + return false; + } + } + if (stats?.buyBoxIsAmazon === true) return true; + if (extractNumber(stats?.current?.[0]) != null) { + const currentAmazon = extractNumber(stats?.current?.[0]); + if (currentAmazon != null && currentAmazon > 0) return true; + } + const amazonHistoryPrice = Array.isArray(csv) + ? extractLatestPositiveKeepaPrice(csv[0]) + : null; + return amazonHistoryPrice == null ? null : amazonHistoryPrice > 0; } function extractSellerRatingCount(seller: Record): number | null { @@ -1173,6 +1418,48 @@ function sumInventoryAsins(result: StalkerAsinResult): number { ); } +function collectPersistedInventoryAsins(result: StalkerAsinResult): string[] { + const seen = new Set(); + for (const { seller } of result.matchedSellers) { + for (const asin of seller.storefrontAsins) { + seen.add(asin); + } + } + return Array.from(seen); +} + +async function runSellableAnalysisChild( + dbPath: string, + stalkerRunId: number, + analysisRunId: number, + asins: string[], +): Promise { + const child = Bun.spawn({ + cmd: [ + "bun", + "run", + "src/stalker-analyze.ts", + "--db", + dbPath, + "--stalker-run-id", + String(stalkerRunId), + "--analysis-run-id", + String(analysisRunId), + "--asins", + asins.join(","), + ], + stdout: "inherit", + stderr: "inherit", + }); + + const exitCode = await child.exited; + if (exitCode !== 0) { + console.warn( + `Stalker analysis child failed for ${asins.length} ASIN(s), exit=${exitCode}`, + ); + } +} + function normalizeAsin(value: unknown): string | null { const asin = String(value ?? "") .trim() @@ -1227,7 +1514,7 @@ function hasFlag(args: string[], flag: string): boolean { function printUsageAndExit(message: string): never { console.error(message); console.error( - "Usage: bun run stalker --input input/asins.xlsx [--db db/results.db] [--max-asins N] [--offer-limit 100] [--seller-limit 30] [--inventory-limit 200] [--storefront-update-hours 168] [--seller-cache-hours 168] [--max-seller-requests N] [--sellability] [--include-stock] [--dry-run] [--no-resume]", + "Usage: bun run stalker --input input/asins.xlsx [--db db/results.db] [--max-asins N] [--offer-limit 100] [--seller-limit 30] [--inventory-limit 200] [--storefront-update-hours 168] [--seller-cache-hours 168] [--max-seller-requests N] [--sellability] [--analyze-sellable] [--include-stock] [--dry-run] [--no-resume]", ); process.exit(1); } diff --git a/src/web/frontend.tsx b/src/web/frontend.tsx index dd35182..34685dd 100644 --- a/src/web/frontend.tsx +++ b/src/web/frontend.tsx @@ -151,6 +151,18 @@ type StalkerProductItem = { can_sell: number; sellability_status: string; sellability_reason: string | null; + product_title: string | null; + brand: string | null; + category_tree: string | null; + current_price: number | null; + avg_price_90d: number | null; + sales_rank: number | null; + monthly_sold: number | null; + seller_count: number | null; + amazon_is_seller: number | null; + verdict: "FBA" | "FBM" | "SKIP" | null; + confidence: number | null; + reasoning: string | null; last_seen_at: string; }; @@ -202,6 +214,18 @@ function formatBoolean(value: number | null | undefined): string { return value === 1 ? "Yes" : "No"; } +function parseStringArrayJson(value: string | null | undefined): string[] { + if (!value) return []; + try { + const parsed = JSON.parse(value); + return Array.isArray(parsed) + ? parsed.filter((item): item is string => typeof item === "string") + : []; + } catch { + return []; + } +} + function buildSortValue(sort: SortState): string { return `${sort.field}:${sort.direction}`; } @@ -1113,7 +1137,7 @@ function StalkerProductsExplorer({ const [runId, setRunId] = useState(""); const [page, setPage] = useState(1); const [pageSize, setPageSize] = useState(50); - const [sort, setSort] = useState({ field: "last_seen_at", direction: "DESC" }); + const [sort, setSort] = useState({ field: "monthly_sold", direction: "DESC" }); useEffect(() => { let cancelled = false; @@ -1170,7 +1194,7 @@ function StalkerProductsExplorer({
- { setPage(1); setSearch(e.target.value); }} placeholder="Search ASIN or seller" /> + { setPage(1); setSearch(e.target.value); }} placeholder="Search ASIN, product, brand, category, or seller" /> { setPage(1); setSellerId(e.target.value.toUpperCase()); }} placeholder="Seller ID" /> { setPage(1); setRunId(e.target.value); }} placeholder="Run ID" /> { setPage(1); setSearch(e.target.value); }} placeholder="Search ASIN, product, brand, category, or seller" /> { setPage(1); setSellerId(e.target.value.toUpperCase()); }} placeholder="Seller ID" /> { setPage(1); setRunId(e.target.value); }} placeholder="Run ID" /> + + + { setPage(1); setMinPrice(e.target.value); }} placeholder="Min price" /> + { setPage(1); setMaxPrice(e.target.value); }} placeholder="Max price" /> + { setPage(1); setMinMonthlySold(e.target.value); }} placeholder="Min monthly sold" /> + { setPage(1); setMaxMonthlySold(e.target.value); }} placeholder="Max monthly sold" /> + { setPage(1); setMinSalesRank(e.target.value); }} placeholder="Min rank" /> + { setPage(1); setMaxSalesRank(e.target.value); }} placeholder="Max rank" /> + { setPage(1); setMinSellerCount(e.target.value); }} placeholder="Min sellers" /> + { setPage(1); setMaxSellerCount(e.target.value); }} placeholder="Max sellers" /> + { setPage(1); setMinRatingCount(e.target.value); }} placeholder="Min seller rating count" /> + { setPage(1); setMaxRatingCount(e.target.value); }} placeholder="Max seller rating count" /> + { setPage(1); setMinConfidence(e.target.value); }} placeholder="Min confidence" /> + { setPage(1); setMaxConfidence(e.target.value); }} placeholder="Max confidence" /> +
From 0c2e59771cdf6cb891383781fb9dd7e0997669ad Mon Sep 17 00:00:00 2001 From: Victor Noguera Date: Tue, 19 May 2026 23:12:34 -0400 Subject: [PATCH 8/9] feat: add XLSX export functionality for Stalker products and enhance UI for export link --- src/server.ts | 132 +++++++++++++++++++++++++++++++++++++++++++ src/web/frontend.tsx | 54 ++++++++++-------- src/web/styles.css | 11 +++- 3 files changed, 174 insertions(+), 23 deletions(-) diff --git a/src/server.ts b/src/server.ts index 428570d..b177c69 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,5 +1,6 @@ import index from "./web/index.html"; import path from "node:path"; +import * as XLSX from "xlsx"; import { getDb, initDb } from "./database.ts"; import { fetchKeepaDataBatch, @@ -123,6 +124,16 @@ function csv(text: string, filename: string): Response { }); } +function xlsx(buffer: ArrayBuffer, filename: string): Response { + return new Response(buffer, { + status: 200, + headers: { + "content-type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "content-disposition": `attachment; filename="${filename}"`, + }, + }); +} + function parseIntParam(value: string | null, fallback: number): number { if (!value) return fallback; const parsed = Number.parseInt(value, 10); @@ -1025,6 +1036,123 @@ function getStalkerProducts(filters: URLSearchParams) { }; } +function getStalkerProductsForExport(filters: URLSearchParams): StalkerProductRecord[] { + const { where, params } = parseStalkerProductFilters(filters); + const orderBy = parseStalkerProductSort(filters.get("sort")); + + return db + .query( + `SELECT * FROM ( + SELECT + r.id AS runId, + r.started_at, + s.seller_id, + s.seller_name, + s.rating, + s.rating_count, + inv.asin, + inv.can_sell, + inv.sellability_status, + inv.sellability_reason, + inv.product_title, + inv.brand, + inv.category_tree, + inv.current_price, + inv.avg_price_90d, + inv.sales_rank, + inv.monthly_sold, + inv.seller_count, + inv.amazon_is_seller, + analysis.verdict, + analysis.confidence, + analysis.reasoning, + inv.last_seen_at + FROM stalker_seller_inventory inv + JOIN stalker_runs r ON r.id = inv.run_id + JOIN stalker_sellers s ON s.seller_id = inv.seller_id + LEFT JOIN product_analysis_results analysis ON analysis.asin = inv.asin + ${where} + ) stalker_products + ORDER BY ${orderBy}`, + ) + .all(...params) as StalkerProductRecord[]; +} + +function parseCategoryTreeForExport(value: string | null): string { + if (!value) return ""; + try { + const parsed = JSON.parse(value); + return Array.isArray(parsed) + ? parsed.filter((item) => typeof item === "string").join(" > ") + : ""; + } catch { + return ""; + } +} + +function exportStalkerProductsXlsx(filters: URLSearchParams): Response { + const rows = getStalkerProductsForExport(filters); + const data = rows.map((row) => ({ + ASIN: row.asin, + "Amazon URL": `https://amazon.com/dp/${row.asin}`, + Product: row.product_title ?? "", + Brand: row.brand ?? "", + Category: parseCategoryTreeForExport(row.category_tree), + "Monthly Sold": row.monthly_sold ?? null, + Sellers: row.seller_count ?? null, + "Amazon Seller": row.amazon_is_seller == null ? "" : row.amazon_is_seller === 1 ? "Yes" : "No", + "Sales Rank": row.sales_rank ?? null, + "Current Price": row.current_price ?? null, + "Avg 90d": row.avg_price_90d ?? null, + Verdict: row.verdict ?? "", + Confidence: row.confidence ?? null, + Reasoning: row.reasoning ?? "", + "Seller ID": row.seller_id, + Seller: row.seller_name ?? "", + "Seller Rating": row.rating ?? null, + "Seller Rating Count": row.rating_count ?? null, + "Sellability Status": row.sellability_status, + "Sellability Reason": row.sellability_reason ?? "", + "Run ID": row.runId, + "Last Seen": row.last_seen_at, + })); + + const workbook = XLSX.utils.book_new(); + const worksheet = XLSX.utils.json_to_sheet(data); + worksheet["!cols"] = [ + { wch: 12 }, + { wch: 32 }, + { wch: 48 }, + { wch: 20 }, + { wch: 34 }, + { wch: 14 }, + { wch: 10 }, + { wch: 14 }, + { wch: 12 }, + { wch: 12 }, + { wch: 12 }, + { wch: 10 }, + { wch: 12 }, + { wch: 60 }, + { wch: 18 }, + { wch: 24 }, + { wch: 12 }, + { wch: 20 }, + { wch: 18 }, + { wch: 40 }, + { wch: 10 }, + { wch: 24 }, + ]; + XLSX.utils.book_append_sheet(workbook, worksheet, "Sellable Products"); + + const buffer = XLSX.write(workbook, { + type: "array", + bookType: "xlsx", + }) as ArrayBuffer; + + return xlsx(buffer, "stalker-sellable-products.xlsx"); +} + function purgeStalkerData() { const counts = { inventory: (db.query("SELECT COUNT(*) AS count FROM stalker_seller_inventory").get() as { count: number }).count, @@ -1694,6 +1822,10 @@ const server = Bun.serve({ const url = new URL(req.url); return json(getStalkerProducts(url.searchParams)); }, + "/api/stalker/products/export.xlsx": (req) => { + const url = new URL(req.url); + return exportStalkerProductsXlsx(url.searchParams); + }, "/api/stalker/purge": (req) => { if (req.method !== "DELETE" && req.method !== "POST") { return json({ error: "Method not allowed" }, 405); diff --git a/src/web/frontend.tsx b/src/web/frontend.tsx index 533f3dc..43f9425 100644 --- a/src/web/frontend.tsx +++ b/src/web/frontend.tsx @@ -1153,32 +1153,39 @@ function StalkerProductsExplorer({ const [pageSize, setPageSize] = useState(50); const [sort, setSort] = useState({ field: "monthly_sold", direction: "DESC" }); + function buildStalkerProductParams(includePaging: boolean): URLSearchParams { + const params = new URLSearchParams({ + sort: buildSortValue(sort), + }); + if (includePaging) { + params.set("page", String(page)); + params.set("pageSize", String(pageSize)); + } + if (search) params.set("q", search); + if (sellerId) params.set("sellerId", sellerId); + if (runId) params.set("runId", runId); + if (verdict) params.set("verdict", verdict); + if (amazonIsSeller) params.set("amazonIsSeller", amazonIsSeller); + if (minPrice) params.set("minPrice", minPrice); + if (maxPrice) params.set("maxPrice", maxPrice); + if (minMonthlySold) params.set("minMonthlySold", minMonthlySold); + if (maxMonthlySold) params.set("maxMonthlySold", maxMonthlySold); + if (minSalesRank) params.set("minSalesRank", minSalesRank); + if (maxSalesRank) params.set("maxSalesRank", maxSalesRank); + if (minSellerCount) params.set("minSellerCount", minSellerCount); + if (maxSellerCount) params.set("maxSellerCount", maxSellerCount); + if (minRatingCount) params.set("minRatingCount", minRatingCount); + if (maxRatingCount) params.set("maxRatingCount", maxRatingCount); + if (minConfidence) params.set("minConfidence", minConfidence); + if (maxConfidence) params.set("maxConfidence", maxConfidence); + return params; + } + useEffect(() => { let cancelled = false; async function load() { setLoading(true); - const params = new URLSearchParams({ - page: String(page), - pageSize: String(pageSize), - sort: buildSortValue(sort), - }); - if (search) params.set("q", search); - if (sellerId) params.set("sellerId", sellerId); - if (runId) params.set("runId", runId); - if (verdict) params.set("verdict", verdict); - if (amazonIsSeller) params.set("amazonIsSeller", amazonIsSeller); - if (minPrice) params.set("minPrice", minPrice); - if (maxPrice) params.set("maxPrice", maxPrice); - if (minMonthlySold) params.set("minMonthlySold", minMonthlySold); - if (maxMonthlySold) params.set("maxMonthlySold", maxMonthlySold); - if (minSalesRank) params.set("minSalesRank", minSalesRank); - if (maxSalesRank) params.set("maxSalesRank", maxSalesRank); - if (minSellerCount) params.set("minSellerCount", minSellerCount); - if (maxSellerCount) params.set("maxSellerCount", maxSellerCount); - if (minRatingCount) params.set("minRatingCount", minRatingCount); - if (maxRatingCount) params.set("maxRatingCount", maxRatingCount); - if (minConfidence) params.set("minConfidence", minConfidence); - if (maxConfidence) params.set("maxConfidence", maxConfidence); + const params = buildStalkerProductParams(true); const res = await fetch(`/api/stalker/products?${params.toString()}`); const payload = (await res.json()) as StalkerProductsResponse; @@ -1236,6 +1243,8 @@ function StalkerProductsExplorer({ setPage(1); } + const exportHref = `/api/stalker/products/export.xlsx?${buildStalkerProductParams(false).toString()}`; + return (
@@ -1298,6 +1307,7 @@ function StalkerProductsExplorer({ + Export XLSX
diff --git a/src/web/styles.css b/src/web/styles.css index 8303a4a..fc36c0b 100644 --- a/src/web/styles.css +++ b/src/web/styles.css @@ -57,7 +57,8 @@ p { .toolbar input, .toolbar select, -button { +button, +.button-link { height: 36px; border-radius: 8px; border: 1px solid #d8dce0; @@ -66,6 +67,14 @@ button { font-size: 14px; } +.button-link { + display: inline-flex; + align-items: center; + justify-content: center; + color: inherit; + text-decoration: none; +} + button { cursor: pointer; } From f8bc05685e5f1dd577f91f091cda47436242fe03 Mon Sep 17 00:00:00 2001 From: Victor Noguera Date: Wed, 20 May 2026 16:18:12 -0400 Subject: [PATCH 9/9] feat: add XLSX export functionality and refactor argument parsing in main script --- src/index.ts | 59 +++++++++++++++++++++++++++++++++++++++++++++------ src/writer.ts | 19 +++++++++++++++++ 2 files changed, 72 insertions(+), 6 deletions(-) diff --git a/src/index.ts b/src/index.ts index b6b9efd..f88f424 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,10 @@ import { readProducts } from "./reader.ts"; import { connectCache, disconnectCache } from "./cache.ts"; -import { printResults, writeResultsToDb } from "./writer.ts"; +import { + printResults, + writeResultsToDb, + writeResultsWorkbook, +} from "./writer.ts"; import { initDb, closeDb } from "./database.ts"; import { chunkArray, @@ -40,14 +44,18 @@ function parseArgs(): { sellability: SellabilityFilter; } { const args = process.argv.slice(2); - const inputFile = args.find((a) => !a.startsWith("--")); - const outIdx = args.indexOf("--out"); - const outputFile = outIdx !== -1 ? args[outIdx + 1] : undefined; + const outputFile = readFlagValue(args, "--out", "--output"); + const inputFile = readInputFileArg( + args, + "--out", + "--output", + "--sellability", + ); const sellability = parseSellabilityArg(args); if (!inputFile) { console.error( - "Usage: bun run src/index.ts [--out results.csv] [--sellability available|all]", + "Usage: bun run src/index.ts [--out results.xlsx|--output results.xlsx] [--sellability available|all]", ); process.exit(1); } @@ -55,6 +63,44 @@ function parseArgs(): { return { inputFile, outputFile, sellability }; } +function readFlagValue(args: string[], ...flags: string[]): string | undefined { + for (const flag of flags) { + const equalsArg = args.find((arg) => arg.startsWith(`${flag}=`)); + if (equalsArg) { + const value = equalsArg.slice(flag.length + 1); + if (value) return value; + } + + const flagIdx = args.indexOf(flag); + if (flagIdx !== -1) { + return args[flagIdx + 1]; + } + } + + return undefined; +} + +function readInputFileArg( + args: string[], + ...flagsWithValues: string[] +): string | undefined { + for (let i = 0; i < args.length; i++) { + const arg = args[i]!; + if (flagsWithValues.includes(arg)) { + i++; + continue; + } + if (flagsWithValues.some((flag) => arg.startsWith(`${flag}=`))) { + continue; + } + if (!arg.startsWith("--")) { + return arg; + } + } + + return undefined; +} + function resolveBaseOutputPath(inputFile: string, outputFile?: string): string { if (outputFile) return outputFile; @@ -103,7 +149,8 @@ async function main() { } printResults(allResults); - writeResultsToDb(allResults, DB_PATH, inputFile, outputFile); + writeResultsWorkbook(allResults, resolvedBaseOutputPath); + writeResultsToDb(allResults, DB_PATH, inputFile, resolvedBaseOutputPath); } finally { await disconnectCache(); closeDb(); diff --git a/src/writer.ts b/src/writer.ts index 477d918..5120191 100644 --- a/src/writer.ts +++ b/src/writer.ts @@ -1,5 +1,8 @@ import { getDb } from "./database.ts"; import type { AnalysisResult, SupplierAnalysisResult } from "./types.ts"; +import { mkdirSync } from "node:fs"; +import path from "node:path"; +import * as XLSX from "xlsx"; export type RunCounts = { totalProducts: number; @@ -93,6 +96,22 @@ export function writeResultsToDb( console.log(`Results written to SQLite database for run_id: ${runId}`); } +export function writeResultsWorkbook( + results: AnalysisResult[], + outputFile: string, +): void { + const outputDir = path.dirname(outputFile); + if (outputDir && outputDir !== ".") { + mkdirSync(outputDir, { recursive: true }); + } + + const workbook = XLSX.utils.book_new(); + const worksheet = XLSX.utils.json_to_sheet(results.map(buildRow)); + XLSX.utils.book_append_sheet(workbook, worksheet, "Results"); + XLSX.writeFile(workbook, outputFile); + console.log(`Results workbook written: ${outputFile}`); +} + export function startRunInDb( dbPath: string, inputFile: string,