import * as XLSX from "xlsx"; import path from "node:path"; import { type Database, closeDb, getDb, initDb } from "./database.ts"; import { fetchSellabilityBatch } from "./sp-api.ts"; import type { SellabilityInfo } from "./types.ts"; const KEEPA_BASE = "https://api.keepa.com"; const DOMAIN_US = "1"; const AMAZON_US_SELLER_ID = "ATVPDKIKX0DER"; const ASIN_REGEX = /^B[0-9A-Z]{9}$/; const DEFAULT_DB_PATH = path.join(process.cwd(), "db", "results.db"); const DEFAULT_STOREFRONT_UPDATE_HOURS = 168; const DEFAULT_OFFER_LIMIT = 100; const DEFAULT_SELLER_LIMIT = 30; const DEFAULT_INVENTORY_LIMIT = 200; const DEFAULT_SELLER_CACHE_HOURS = 168; const MAX_SELLERS_PER_METADATA_REQUEST = 100; const MAX_KEEPA_RETRIES = 4; const KEEP_RETRY_BUFFER_MS = 250; type KeepaApiResponse = { products?: Record[]; sellers?: Record | Record[]; tokensLeft?: number; refillRate?: number; refillIn?: number; }; export type StalkerArgs = { input: string; dbPath: string; maxAsins: number | null; storefrontUpdateHours: number; offerLimit: number; sellerLimit: number; inventoryLimit: number; sellerCacheHours: number; includeStock: boolean; dryRun: boolean; resume: boolean; maxSellerRequests: number | null; sellability: boolean; analyzeSellable: boolean; useClaude: boolean; }; export type StalkerOffer = { sellerId: string; offerPrice: number | null; condition: string | null; isFba: boolean | null; stock: number | null; rawOffer: Record; }; export type StalkerSeller = { sellerId: string; sellerName: string | null; rating: number | null; ratingCount: number | null; storefrontAsins: string[]; storefrontItems: StalkerInventoryItem[]; storefrontAsinTotal: number; rawSeller: Record; }; type StalkerInventoryItem = { asin: string; rawInventory: unknown; sellability: SellabilityInfo | null; productDetails: StalkerProductDetails | null; }; type StalkerProductDetails = { title: string | null; brand: string | null; categoryTree: string[]; currentPrice: number | null; avgPrice90: number | null; salesRank: number | null; monthlySold: number | null; sellerCount: number | null; amazonIsSeller: boolean | null; rawProduct: Record; }; type StalkerAsinResult = { asin: string; title: string | null; offerCount: number; candidateSellerCount: number; matchedSellers: Array<{ seller: StalkerSeller; offer: StalkerOffer; }>; product: Record | null; error?: string; }; type StalkerRunStats = { scannedAsins: number; sourceAsinsWithMatches: number; matchedSellers: number; persistedInventoryAsins: number; failedAsins: number; skippedAsins: number; candidateSellers: number; qualifyingSellers: number; sellerMetadataRequests: number; sellerStorefrontRequests: number; inventorySellabilityCheckedAsins: number; inventorySellabilityAvailableAsins: number; inventorySellabilityExcludedAsins: number; stoppedEarly: boolean; }; type StalkerRunContext = { database: Database | null; metadataCache: Map; storefrontCache: Map; stats: StalkerRunStats; }; let tokensLeft = 1; let refillRate = 1; let lastRequestTime = 0; export function parseArgs(argv = process.argv.slice(2)): StalkerArgs { const input = readFlagValue(argv, "--input"); if (!input) { printUsageAndExit("Missing required --input file."); } const dbPath = readFlagValue(argv, "--db") ?? DEFAULT_DB_PATH; const maxAsinsRaw = readFlagValue(argv, "--max-asins"); const storefrontUpdateRaw = readFlagValue(argv, "--storefront-update-hours"); const offerLimitRaw = readFlagValue(argv, "--offer-limit"); const sellerLimitRaw = readFlagValue(argv, "--seller-limit"); const inventoryLimitRaw = readFlagValue(argv, "--inventory-limit"); const sellerCacheHoursRaw = readFlagValue(argv, "--seller-cache-hours"); const maxSellerRequestsRaw = readFlagValue(argv, "--max-seller-requests"); const maxAsins = maxAsinsRaw ? Number(maxAsinsRaw) : null; const storefrontUpdateHours = storefrontUpdateRaw ? Number(storefrontUpdateRaw) : DEFAULT_STOREFRONT_UPDATE_HOURS; const offerLimit = offerLimitRaw ? Number(offerLimitRaw) : DEFAULT_OFFER_LIMIT; const sellerLimit = sellerLimitRaw ? Number(sellerLimitRaw) : DEFAULT_SELLER_LIMIT; const inventoryLimit = inventoryLimitRaw ? Number(inventoryLimitRaw) : DEFAULT_INVENTORY_LIMIT; const sellerCacheHours = sellerCacheHoursRaw ? Number(sellerCacheHoursRaw) : DEFAULT_SELLER_CACHE_HOURS; const maxSellerRequests = maxSellerRequestsRaw ? Number(maxSellerRequestsRaw) : null; const includeStock = hasFlag(argv, "--include-stock"); const dryRun = hasFlag(argv, "--dry-run"); const resume = !hasFlag(argv, "--no-resume"); const sellability = hasFlag(argv, "--sellability"); const analyzeSellable = hasFlag(argv, "--analyze-sellable"); const useClaude = hasFlag(argv, "--claude"); if (analyzeSellable && !sellability) { printUsageAndExit("--analyze-sellable requires --sellability."); } if (maxAsins != null && (!Number.isInteger(maxAsins) || maxAsins <= 0)) { printUsageAndExit("--max-asins must be a positive integer."); } if (!Number.isInteger(storefrontUpdateHours) || storefrontUpdateHours < 0) { printUsageAndExit( "--storefront-update-hours must be a non-negative integer.", ); } if (!Number.isInteger(offerLimit) || offerLimit < 20 || offerLimit > 100) { printUsageAndExit("--offer-limit must be an integer from 20 to 100."); } if (!Number.isInteger(sellerLimit) || sellerLimit <= 0) { printUsageAndExit("--seller-limit must be a positive integer."); } if (!Number.isInteger(inventoryLimit) || inventoryLimit < 0) { printUsageAndExit("--inventory-limit must be a non-negative integer."); } if (!Number.isInteger(sellerCacheHours) || sellerCacheHours < 0) { printUsageAndExit("--seller-cache-hours must be a non-negative integer."); } if ( maxSellerRequests != null && (!Number.isInteger(maxSellerRequests) || maxSellerRequests <= 0) ) { printUsageAndExit("--max-seller-requests must be a positive integer."); } return { input, dbPath, maxAsins, storefrontUpdateHours, offerLimit, sellerLimit, inventoryLimit, sellerCacheHours, includeStock, dryRun, resume, maxSellerRequests, sellability, analyzeSellable, useClaude, }; } export function readAsinsFromXlsx(filePath: string): string[] { const workbook = XLSX.readFile(filePath); const sheetName = workbook.SheetNames[0]; if (!sheetName) throw new Error("No sheets found in file"); const sheet = workbook.Sheets[sheetName]; if (!sheet) throw new Error("First sheet is missing"); const rows = XLSX.utils.sheet_to_json>(sheet, { defval: "", }); if (rows.length === 0) throw new Error("File contains no data rows"); const headers = Object.keys(rows[0]!); const asinColumn = headers.find( (header) => normalizeHeader(header) === "asin", ); if (!asinColumn) { throw new Error( `No ASIN column found. Available columns: ${headers.join(", ")}`, ); } return extractAsinsFromRows(rows, asinColumn); } export function extractAsinsFromRows( rows: Array>, asinColumn = "asin", ): string[] { const asins: string[] = []; const seen = new Set(); for (const row of rows) { const asin = normalizeAsin(row[asinColumn]); if (!asin || seen.has(asin)) continue; seen.add(asin); asins.push(asin); } return asins; } export function isQualifyingSeller(seller: { ratingCount?: number | null; }): boolean { return ( typeof seller.ratingCount === "number" && Number.isFinite(seller.ratingCount) && seller.ratingCount >= 1 && seller.ratingCount <= 30 ); } export function extractLiveOfferSellerCandidates( product: Record, ): StalkerOffer[] { const offers = Array.isArray(product.offers) ? product.offers : []; const bySeller = new Map(); for (const offer of offers) { if (!offer || typeof offer !== "object") continue; const sellerId = normalizeSellerId( offer.sellerId ?? offer.sellerID ?? offer.seller_id, ); if (!sellerId || sellerId === AMAZON_US_SELLER_ID) continue; if (bySeller.has(sellerId)) continue; bySeller.set(sellerId, { sellerId, offerPrice: extractOfferPrice(offer), condition: extractString(offer.condition ?? offer.conditionComment), isFba: extractBoolean(offer.isFBA ?? offer.isFba ?? offer.fba), stock: extractNumber( offer.stock ?? offer.stockCount ?? offer.currentStock, ), rawOffer: offer, }); } return Array.from(bySeller.values()); } export async function runStalker(args: StalkerArgs): Promise { const apiKey = Bun.env.KEEPA_API_KEY; if (!apiKey) throw new Error("Missing required env var: KEEPA_API_KEY"); const allAsins = readAsinsFromXlsx(args.input); const cappedAsins = args.maxAsins == null ? allAsins : allAsins.slice(0, args.maxAsins); initDb(args.dbPath); const database = getDb(args.dbPath); const completedAsins = args.resume ? loadPreviouslyScannedAsins(database) : new Set(); const resumeFilteredAsins = cappedAsins.filter( (asin) => !completedAsins.has(asin), ); const runId = args.dryRun ? null : startStalkerRun(database, args.input, resumeFilteredAsins.length); const analysisRunId = !args.dryRun && args.analyzeSellable ? startStalkerAnalysisRun(database, args.input) : null; const stats: StalkerRunStats = { scannedAsins: 0, sourceAsinsWithMatches: 0, matchedSellers: 0, persistedInventoryAsins: 0, failedAsins: 0, skippedAsins: cappedAsins.length - resumeFilteredAsins.length, candidateSellers: 0, qualifyingSellers: 0, sellerMetadataRequests: 0, sellerStorefrontRequests: 0, inventorySellabilityCheckedAsins: 0, inventorySellabilityAvailableAsins: 0, inventorySellabilityExcludedAsins: 0, stoppedEarly: false, }; const context: StalkerRunContext = { database, metadataCache: new Map(), storefrontCache: new Map(), stats, }; try { if (args.dryRun) { console.log( "Stalker dry-run: product and seller metadata will be fetched, storefronts will not be fetched or persisted.", ); } if (stats.skippedAsins > 0) { console.log( `Stalker resume: skipped ${stats.skippedAsins} previously scanned ASIN(s).`, ); } for (const asin of resumeFilteredAsins) { console.log( `Stalker: scanning ${asin} (${stats.scannedAsins + 1}/${resumeFilteredAsins.length})`, ); const result = await scanAsin(asin, args, apiKey, context).catch( (error) => ({ asin, title: null, offerCount: 0, candidateSellerCount: 0, matchedSellers: [], product: null, error: error instanceof Error ? error.message : String(error), }), ); if (args.sellability && !args.dryRun) { await enrichInventorySellability(result, stats); } applyInventoryPersistencePolicy(result, args.sellability && !args.dryRun); if (args.sellability && !args.dryRun) { await enrichInventoryProductDetails(result, apiKey); } if (!args.dryRun && runId != null) { persistAsinResult(database, runId, result); } const sellableAsins = collectPersistedInventoryAsins(result); if ( args.analyzeSellable && !args.dryRun && runId != null && analysisRunId != null && sellableAsins.length > 0 ) { await runSellableAnalysisChild( args.dbPath, runId, analysisRunId, sellableAsins, args.useClaude, ); } stats.scannedAsins += 1; stats.matchedSellers += result.matchedSellers.length; stats.persistedInventoryAsins += sumInventoryAsins(result); if (result.matchedSellers.length > 0) stats.sourceAsinsWithMatches += 1; if (result.error) { stats.failedAsins += 1; console.warn(`Stalker: ${asin} failed: ${result.error}`); } if (!args.dryRun && runId != null) { refreshStalkerRun(database, runId, stats, "running"); } console.log( `Stalker: ${asin} candidates=${result.candidateSellerCount}, matched=${result.matchedSellers.length}, persisted_inventory=${sumInventoryAsins(result)}`, ); if (stats.stoppedEarly) { console.log( "Stalker: stopping early because max seller request budget was reached.", ); break; } } if (!args.dryRun && runId != null) { refreshStalkerRun( database, runId, stats, stats.stoppedEarly ? "stopped" : stats.failedAsins > 0 ? "completed_with_errors" : "completed", ); } logRunSummary(stats, args); if (!args.dryRun && analysisRunId != null) { finishStalkerAnalysisRun(database, analysisRunId, "completed"); } return stats; } catch (error) { const message = error instanceof Error ? error.message : String(error); if (!args.dryRun && runId != null) { finishStalkerRunWithError(database, runId, stats, message); } if (!args.dryRun && analysisRunId != null) { finishStalkerAnalysisRun(database, analysisRunId, "failed", message); } throw error; } } async function scanAsin( asin: string, args: StalkerArgs, apiKey: string, context: StalkerRunContext, ): Promise { const product = await fetchKeepaProduct( asin, apiKey, args.offerLimit, args.includeStock, ); const offers = extractLiveOfferSellerCandidates(product).slice( 0, args.sellerLimit, ); context.stats.candidateSellers += offers.length; const metadata = await fetchSellerMetadata( offers.map((offer) => offer.sellerId), apiKey, args, context, ); const qualifyingOffers = offers.filter((offer) => { const seller = metadata.get(offer.sellerId); return seller ? isQualifyingSeller(seller) : false; }); context.stats.qualifyingSellers += qualifyingOffers.length; if (args.dryRun) { console.log( `Stalker dry-run estimate for ${asin}: storefront requests needed=${qualifyingOffers.length}, candidate sellers=${offers.length}`, ); } const storefronts = args.dryRun ? new Map() : await fetchQualifiedSellerStorefronts( qualifyingOffers.map((offer) => offer.sellerId), apiKey, args, context, ); const matchedSellers = qualifyingOffers .map((offer) => { const seller = storefronts.get(offer.sellerId); if (!seller || !isQualifyingSeller(seller)) return null; return { seller, offer }; }) .filter( (entry): entry is { seller: StalkerSeller; offer: StalkerOffer } => entry != null, ); return { asin, title: extractString(product.title), offerCount: Array.isArray(product.offers) ? product.offers.length : 0, candidateSellerCount: offers.length, matchedSellers, product, }; } function applyInventoryPersistencePolicy( result: StalkerAsinResult, requireAvailableSellability: boolean, ): void { for (const { seller } of result.matchedSellers) { seller.storefrontItems = seller.storefrontItems.filter((item) => { if (!requireAvailableSellability) return false; return ( item.sellability?.canSell === true && item.sellability.sellabilityStatus === "available" ); }); seller.storefrontAsins = seller.storefrontItems.map((item) => item.asin); } } async function enrichInventorySellability( result: StalkerAsinResult, stats: StalkerRunStats, ): Promise { const sellers = result.matchedSellers.map(({ seller }) => seller); const items = sellers.flatMap((seller) => seller.storefrontItems); const uniqueAsins = Array.from(new Set(items.map((item) => item.asin))); if (uniqueAsins.length === 0) return; console.log( `Stalker inventory sellability: checking ${uniqueAsins.length} ASIN(s) from matched seller storefronts...`, ); const sellabilityMap = await fetchSellabilityBatch(uniqueAsins); stats.inventorySellabilityCheckedAsins += uniqueAsins.length; for (const asin of uniqueAsins) { const info = sellabilityMap.get(asin) ?? { canSell: null, sellabilityStatus: "unknown" as const, sellabilityReason: "Sellability check returned no result", }; if (info.sellabilityStatus === "available" && info.canSell === true) { stats.inventorySellabilityAvailableAsins += 1; } else { stats.inventorySellabilityExcludedAsins += 1; } } for (const item of items) { item.sellability = sellabilityMap.get(item.asin) ?? { canSell: null, sellabilityStatus: "unknown", sellabilityReason: "Sellability check returned no result", }; } for (const seller of sellers) { seller.storefrontItems = seller.storefrontItems.filter( (item) => item.sellability?.canSell === true && item.sellability.sellabilityStatus === "available", ); seller.storefrontAsins = seller.storefrontItems.map((item) => item.asin); } } async function enrichInventoryProductDetails( result: StalkerAsinResult, apiKey: string, ): Promise { const items = result.matchedSellers.flatMap( ({ seller }) => seller.storefrontItems, ); const uniqueAsins = Array.from(new Set(items.map((item) => item.asin))); if (uniqueAsins.length === 0) return; console.log( `Stalker inventory details: fetching Keepa product details for ${uniqueAsins.length} sellable ASIN(s)...`, ); const detailsByAsin = await fetchKeepaInventoryProductDetails( apiKey, uniqueAsins, ); for (const item of items) { item.productDetails = detailsByAsin.get(item.asin) ?? null; } } async function fetchKeepaProduct( asin: string, apiKey: string, offerLimit: number, includeStock: boolean, ): Promise> { const params = new URLSearchParams({ key: apiKey, domain: DOMAIN_US, asin, offers: String(offerLimit), "only-live-offers": "1", stats: "30", days: "30", }); if (includeStock) { params.set("stock", "1"); } const data = await fetchKeepaWithRetries( `${KEEPA_BASE}/product?${params.toString()}`, `product ${asin}`, ); const product = data.products?.[0]; if (!product) throw new Error("Keepa returned no product"); return product; } async function fetchKeepaInventoryProductDetails( apiKey: string, asins: string[], ): Promise> { const details = new Map(); const chunkSize = 100; for (let i = 0; i < asins.length; i += chunkSize) { const chunk = asins.slice(i, i + chunkSize); const params = new URLSearchParams({ key: apiKey, domain: DOMAIN_US, asin: chunk.join(","), stats: "30", days: "30", buybox: "1", }); const data = await fetchKeepaWithRetries( `${KEEPA_BASE}/product?${params.toString()}`, `inventory product details ${i + 1}-${i + chunk.length}`, ); for (const product of data.products ?? []) { const asin = normalizeAsin(product.asin); if (!asin) continue; details.set(asin, parseInventoryProductDetails(product)); } } return details; } async function fetchSellerMetadata( sellerIds: string[], apiKey: string, args: StalkerArgs, context: StalkerRunContext, ): Promise> { const out = new Map(); const uniqueSellerIds = Array.from(new Set(sellerIds)); const missing: string[] = []; for (const sellerId of uniqueSellerIds) { const cached = context.metadataCache.get(sellerId) ?? loadCachedSeller( context.database, sellerId, args.sellerCacheHours, false, args.inventoryLimit, ); if (cached) { context.metadataCache.set(sellerId, cached); out.set(sellerId, cached); continue; } missing.push(sellerId); } for (let i = 0; i < missing.length; i += MAX_SELLERS_PER_METADATA_REQUEST) { const chunk = missing.slice(i, i + MAX_SELLERS_PER_METADATA_REQUEST); if (!canSpendSellerRequests(context, args, 1)) break; const params = new URLSearchParams({ key: apiKey, domain: DOMAIN_US, seller: chunk.join(","), }); context.stats.sellerMetadataRequests += 1; const data = await fetchKeepaWithRetries( `${KEEPA_BASE}/seller?${params.toString()}`, `seller metadata batch ${i / MAX_SELLERS_PER_METADATA_REQUEST + 1}`, ); for (const [sellerId, seller] of normalizeSellerResponse(data.sellers)) { const parsed = parseSeller(sellerId, seller, args.inventoryLimit); context.metadataCache.set(sellerId, parsed); out.set(sellerId, parsed); } } return out; } async function fetchQualifiedSellerStorefronts( sellerIds: string[], apiKey: string, args: StalkerArgs, context: StalkerRunContext, ): Promise> { const out = new Map(); const uniqueSellerIds = Array.from(new Set(sellerIds)); // Keepa only allows a single sellerId per request when storefront=1. for (const sellerId of uniqueSellerIds) { const cached = context.storefrontCache.get(sellerId) ?? loadCachedSeller( context.database, sellerId, args.sellerCacheHours, true, args.inventoryLimit, ); if (cached) { context.storefrontCache.set(sellerId, cached); out.set(sellerId, cached); continue; } if (!canSpendSellerRequests(context, args, 1)) break; const params = new URLSearchParams({ key: apiKey, domain: DOMAIN_US, seller: sellerId, storefront: "1", update: String(args.storefrontUpdateHours), }); context.stats.sellerStorefrontRequests += 1; const data = await fetchKeepaWithRetries( `${KEEPA_BASE}/seller?${params.toString()}`, `seller ${sellerId}`, ); for (const [returnedSellerId, seller] of normalizeSellerResponse( data.sellers, )) { const parsed = parseSeller(returnedSellerId, seller, args.inventoryLimit); context.metadataCache.set(returnedSellerId, parsed); context.storefrontCache.set(returnedSellerId, parsed); out.set(returnedSellerId, parsed); } } return out; } function canSpendSellerRequests( context: StalkerRunContext, args: StalkerArgs, nextRequests: number, ): boolean { if (args.maxSellerRequests == null) return true; const spent = context.stats.sellerMetadataRequests + context.stats.sellerStorefrontRequests; if (spent + nextRequests <= args.maxSellerRequests) return true; context.stats.stoppedEarly = true; return false; } async function fetchKeepaWithRetries( url: string, operationLabel: string, ): Promise { let lastErrorMessage = "Unknown Keepa error"; for (let attempt = 1; attempt <= MAX_KEEPA_RETRIES; attempt++) { await waitForToken(); const response = await fetch(url); lastRequestTime = Date.now(); if (response.ok) { const data = (await response.json()) as KeepaApiResponse; updateTokenState(data); return data; } const text = await response.text(); const payload = parseErrorPayload(text); if (payload) updateTokenState(payload); lastErrorMessage = `Keepa API error ${response.status}: ${text}`; if (response.status !== 429 || attempt === MAX_KEEPA_RETRIES) break; const waitMs = computeWaitMsFromRefill(payload?.refillIn); tokensLeft = Math.min(tokensLeft, 0); console.warn( `Keepa throttled during ${operationLabel} (attempt ${attempt}/${MAX_KEEPA_RETRIES}). Waiting ${Math.ceil(waitMs / 1000)}s before retry...`, ); await wait(waitMs); } throw new Error(lastErrorMessage); } function persistAsinResult( database: Database, runId: number, result: StalkerAsinResult, ): void { const fetchedAt = new Date().toISOString(); database.transaction(() => { const scanId = upsertAsinScan(database, runId, result, fetchedAt); for (const { seller, offer } of result.matchedSellers) { upsertSeller(database, seller, fetchedAt); upsertAsinSeller(database, scanId, seller, offer); upsertSellerInventory(database, runId, seller, fetchedAt); } })(); } function upsertAsinScan( database: Database, runId: number, result: StalkerAsinResult, fetchedAt: string, ): number { database .prepare( `INSERT INTO stalker_asin_scans ( run_id, source_asin, title, offer_count, candidate_seller_count, matched_seller_count, fetched_at, raw_product_json ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(run_id, source_asin) DO UPDATE SET title = excluded.title, offer_count = excluded.offer_count, candidate_seller_count = excluded.candidate_seller_count, matched_seller_count = excluded.matched_seller_count, fetched_at = excluded.fetched_at, raw_product_json = excluded.raw_product_json`, ) .run( runId, result.asin, result.title, result.offerCount, result.candidateSellerCount, result.matchedSellers.length, fetchedAt, JSON.stringify(result.product ?? { error: result.error ?? null }), ); const row = database .query( `SELECT id FROM stalker_asin_scans WHERE run_id = ? AND source_asin = ?`, ) .get(runId, result.asin) as { id: number } | null; if (!row) throw new Error(`Failed to load stalker scan row for ${result.asin}`); return row.id; } function upsertSeller( database: Database, seller: StalkerSeller, fetchedAt: string, ): void { database .prepare( `INSERT INTO stalker_sellers ( seller_id, seller_name, rating, rating_count, storefront_asin_total, persisted_inventory_sample_count, last_updated_at, raw_seller_json ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(seller_id) DO UPDATE SET seller_name = excluded.seller_name, rating = excluded.rating, rating_count = excluded.rating_count, storefront_asin_total = excluded.storefront_asin_total, persisted_inventory_sample_count = excluded.persisted_inventory_sample_count, last_updated_at = excluded.last_updated_at, raw_seller_json = excluded.raw_seller_json`, ) .run( seller.sellerId, seller.sellerName, seller.rating, seller.ratingCount, seller.storefrontAsinTotal, seller.storefrontItems.length, fetchedAt, JSON.stringify(seller.rawSeller), ); } function upsertAsinSeller( database: Database, scanId: number, seller: StalkerSeller, offer: StalkerOffer, ): void { database .prepare( `INSERT INTO stalker_asin_sellers ( scan_id, seller_id, offer_price, condition, is_fba, stock, seller_rating, seller_rating_count, raw_offer_json ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(scan_id, seller_id) DO UPDATE SET offer_price = excluded.offer_price, condition = excluded.condition, is_fba = excluded.is_fba, stock = excluded.stock, seller_rating = excluded.seller_rating, seller_rating_count = excluded.seller_rating_count, raw_offer_json = excluded.raw_offer_json`, ) .run( scanId, seller.sellerId, offer.offerPrice, offer.condition, offer.isFba == null ? null : offer.isFba ? 1 : 0, offer.stock, seller.rating, seller.ratingCount, JSON.stringify(offer.rawOffer), ); } function upsertSellerInventory( database: Database, runId: number, seller: StalkerSeller, fetchedAt: string, ): void { const insert = database.prepare( `INSERT INTO stalker_seller_inventory ( run_id, seller_id, asin, can_sell, sellability_status, sellability_reason, product_title, brand, category_tree, current_price, avg_price_90d, sales_rank, monthly_sold, seller_count, amazon_is_seller, raw_product_json, last_seen_at, raw_inventory_json ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(run_id, seller_id, asin) DO UPDATE SET can_sell = excluded.can_sell, sellability_status = excluded.sellability_status, sellability_reason = excluded.sellability_reason, product_title = excluded.product_title, brand = excluded.brand, category_tree = excluded.category_tree, current_price = excluded.current_price, avg_price_90d = excluded.avg_price_90d, sales_rank = excluded.sales_rank, monthly_sold = excluded.monthly_sold, seller_count = excluded.seller_count, amazon_is_seller = excluded.amazon_is_seller, raw_product_json = excluded.raw_product_json, last_seen_at = excluded.last_seen_at, raw_inventory_json = excluded.raw_inventory_json`, ); for (const item of seller.storefrontItems) { if ( item.sellability?.canSell !== true || item.sellability.sellabilityStatus !== "available" ) { continue; } insert.run( runId, seller.sellerId, item.asin, item.sellability?.canSell == null ? null : item.sellability.canSell ? 1 : 0, item.sellability?.sellabilityStatus ?? null, item.sellability?.sellabilityReason ?? null, item.productDetails?.title ?? null, item.productDetails?.brand ?? null, item.productDetails ? JSON.stringify(item.productDetails.categoryTree) : null, item.productDetails?.currentPrice ?? null, item.productDetails?.avgPrice90 ?? null, item.productDetails?.salesRank ?? null, item.productDetails?.monthlySold ?? null, item.productDetails?.sellerCount ?? null, item.productDetails?.amazonIsSeller == null ? null : item.productDetails.amazonIsSeller ? 1 : 0, item.productDetails ? JSON.stringify(item.productDetails.rawProduct) : null, fetchedAt, JSON.stringify(item.rawInventory), ); } } function startStalkerRun( database: Database, inputFile: string, totalAsins: number, ): number { const result = database .prepare( `INSERT INTO stalker_runs ( input_file, started_at, requested_asins, status ) VALUES (?, ?, ?, ?)`, ) .run(inputFile, new Date().toISOString(), totalAsins, "running"); return result.lastInsertRowid as number; } function startStalkerAnalysisRun( database: Database, inputFile: string, ): number { const result = database .prepare( `INSERT INTO category_analysis_runs ( category_id, category_label, run_timestamp, top_asins_checked, available_asins, fba_count, fbm_count, skip_count, status, error_message ) VALUES (?, ?, ?, 0, 0, 0, 0, 0, 'running', NULL)`, ) .run(0, `Stalker: ${path.basename(inputFile)}`, new Date().toISOString()); return result.lastInsertRowid as number; } function loadPreviouslyScannedAsins(database: Database): Set { const rows = database .query(`SELECT DISTINCT source_asin FROM stalker_asin_scans`) .all() as Array<{ source_asin: string }>; return new Set(rows.map((row) => row.source_asin)); } function loadCachedSeller( database: Database | null, sellerId: string, maxAgeHours: number, requireStorefront: boolean, inventoryLimit: number, ): StalkerSeller | null { if (!database || maxAgeHours <= 0) return null; const row = database .query( `SELECT raw_seller_json, last_updated_at, storefront_asin_total FROM stalker_sellers WHERE seller_id = ?`, ) .get(sellerId) as { raw_seller_json: string | null; last_updated_at: string; storefront_asin_total: number | null; } | null; if (!row?.raw_seller_json) return null; const ageMs = Date.now() - new Date(row.last_updated_at).getTime(); if (!Number.isFinite(ageMs) || ageMs > maxAgeHours * 60 * 60 * 1000) { return null; } try { const rawSeller = JSON.parse(row.raw_seller_json) as Record; const parsed = parseSeller(sellerId, rawSeller, inventoryLimit); if (requireStorefront && parsed.storefrontAsinTotal <= 0) return null; return parsed; } catch { return null; } } function logRunSummary(stats: StalkerRunStats, args: StalkerArgs): void { const estimatedStorefrontRequestsSaved = Math.max( 0, stats.candidateSellers - stats.qualifyingSellers, ); console.log( [ "Stalker summary:", `processed=${stats.scannedAsins}`, `skipped=${stats.skippedAsins}`, `candidate_sellers=${stats.candidateSellers}`, `qualifying_sellers=${stats.qualifyingSellers}`, `metadata_requests=${stats.sellerMetadataRequests}`, `storefront_requests=${stats.sellerStorefrontRequests}`, `sellability_checked=${stats.inventorySellabilityCheckedAsins}`, `sellability_available=${stats.inventorySellabilityAvailableAsins}`, `sellability_excluded=${stats.inventorySellabilityExcludedAsins}`, `storefront_requests_saved_by_two_phase=${estimatedStorefrontRequestsSaved}`, `persisted_inventory=${stats.persistedInventoryAsins}`, `dry_run=${args.dryRun ? "yes" : "no"}`, ].join(" "), ); } function refreshStalkerRun( database: Database, runId: number, stats: StalkerRunStats, status: string, ): void { database .prepare( `UPDATE stalker_runs SET scanned_asins = ?, source_asins_with_matches = ?, candidate_sellers = ?, qualifying_sellers = ?, matched_sellers = ?, seller_metadata_requests = ?, seller_storefront_requests = ?, inventory_sellability_checked_asins = ?, inventory_sellability_available_asins = ?, inventory_sellability_excluded_asins = ?, persisted_inventory_asins = ?, status = ?, completed_at = CASE WHEN ? = 'running' THEN completed_at ELSE ? END WHERE id = ?`, ) .run( stats.scannedAsins, stats.sourceAsinsWithMatches, stats.candidateSellers, stats.qualifyingSellers, stats.matchedSellers, stats.sellerMetadataRequests, stats.sellerStorefrontRequests, stats.inventorySellabilityCheckedAsins, stats.inventorySellabilityAvailableAsins, stats.inventorySellabilityExcludedAsins, stats.persistedInventoryAsins, status, status, new Date().toISOString(), runId, ); } function finishStalkerRunWithError( database: Database, runId: number, stats: StalkerRunStats, errorMessage: string, ): void { database .prepare( `UPDATE stalker_runs SET scanned_asins = ?, source_asins_with_matches = ?, candidate_sellers = ?, qualifying_sellers = ?, matched_sellers = ?, seller_metadata_requests = ?, seller_storefront_requests = ?, inventory_sellability_checked_asins = ?, inventory_sellability_available_asins = ?, inventory_sellability_excluded_asins = ?, persisted_inventory_asins = ?, status = 'failed', error_message = ?, completed_at = ? WHERE id = ?`, ) .run( stats.scannedAsins, stats.sourceAsinsWithMatches, stats.candidateSellers, stats.qualifyingSellers, stats.matchedSellers, stats.sellerMetadataRequests, stats.sellerStorefrontRequests, stats.inventorySellabilityCheckedAsins, stats.inventorySellabilityAvailableAsins, stats.inventorySellabilityExcludedAsins, stats.persistedInventoryAsins, errorMessage, new Date().toISOString(), runId, ); } function finishStalkerAnalysisRun( database: Database, runId: number, status: "completed" | "failed", errorMessage: string | null = null, ): void { const stats = database .query( `SELECT COUNT(*) AS total, SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba, SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm, SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip FROM product_analysis_results WHERE run_id = ?`, ) .get(runId) as { total: number; fba: number | null; fbm: number | null; skip: number | null; }; database .prepare( `UPDATE category_analysis_runs SET top_asins_checked = ?, available_asins = ?, fba_count = ?, fbm_count = ?, skip_count = ?, status = ?, error_message = ? WHERE id = ?`, ) .run( stats.total ?? 0, stats.total ?? 0, stats.fba ?? 0, stats.fbm ?? 0, stats.skip ?? 0, status, errorMessage, runId, ); } function normalizeSellerResponse( sellers: KeepaApiResponse["sellers"], ): Array<[string, Record]> { if (!sellers) return []; if (Array.isArray(sellers)) { return sellers .map( (seller) => [ normalizeSellerId(seller.sellerId ?? seller.sellerID ?? seller.id), seller, ] as [string | null, Record], ) .filter((entry): entry is [string, Record] => !!entry[0]); } return Object.entries(sellers) .map( ([sellerId, seller]) => [normalizeSellerId(sellerId), seller] as [ string | null, Record, ], ) .filter((entry): entry is [string, Record] => !!entry[0]); } function parseSeller( sellerId: string, seller: Record, inventoryLimit: number, ): StalkerSeller { const allStorefrontItems = extractStorefrontItems(seller); const storefrontItems = inventoryLimit === 0 ? [] : allStorefrontItems.slice(0, inventoryLimit); const storefrontAsins = storefrontItems.map((item) => item.asin); return { sellerId, sellerName: extractString( seller.sellerName ?? seller.name ?? seller.storeName ?? seller.businessName, ), rating: extractNumber( seller.currentRating ?? seller.rating ?? seller.feedbackRating, ), ratingCount: extractSellerRatingCount(seller), storefrontAsins, storefrontItems, storefrontAsinTotal: extractNumber( seller.totalStorefrontAsinCount ?? seller.storefrontAsinCount ?? seller.asinListCount ?? seller.totalStorefrontProducts, ) ?? allStorefrontItems.length, rawSeller: seller, }; } function extractStorefrontItems( seller: Record, ): StalkerInventoryItem[] { const candidates = [ seller.asinList, seller.asins, seller.storefront, seller.storefrontAsins, seller.inventory, ]; const items: StalkerInventoryItem[] = []; const seen = new Set(); for (const candidate of candidates) { collectStorefrontItems(candidate, items, seen); } return items; } function collectStorefrontItems( value: unknown, items: StalkerInventoryItem[], seen: Set, ): void { if (Array.isArray(value)) { for (const item of value) collectStorefrontItems(item, items, seen); return; } if (value && typeof value === "object") { const asin = normalizeAsin((value as Record).asin); if (asin && !seen.has(asin)) { seen.add(asin); items.push({ asin, rawInventory: value, sellability: null, productDetails: null, }); } return; } const asin = normalizeAsin(value); if (!asin || seen.has(asin)) return; seen.add(asin); items.push({ asin, rawInventory: { asin }, sellability: null, productDetails: null, }); } function parseInventoryProductDetails( product: Record, ): StalkerProductDetails { const stats = product.stats; const csv = product.csv; return { title: extractString(product.title), brand: extractString(product.brand ?? product.manufacturer), categoryTree: product.categoryTree ?.map((category: { name?: unknown }) => extractString(category.name)) .filter((name: string | null): name is string => !!name) ?? [], currentPrice: extractCurrentPrice(csv), avgPrice90: stats?.avg?.[0] != null ? stats.avg[0] / 100 : null, salesRank: extractNumber(stats?.current?.[3]), monthlySold: extractNumber(product.monthlySold ?? stats?.monthlySold) ?? extractNumber(product.salesRankDrops30 ?? stats?.salesRankDrops30), sellerCount: extractNumber(stats?.current?.[11]), amazonIsSeller: resolveAmazonIsSeller(product, stats, csv), rawProduct: product, }; } function extractCurrentPrice(csv: unknown): number | null { if (!Array.isArray(csv)) return null; const amazonPrice = extractLatestPositiveKeepaPrice(csv[0]); if (amazonPrice != null) return amazonPrice; return extractLatestPositiveKeepaPrice(csv[1]); } function extractLatestPositiveKeepaPrice(history: unknown): number | null { if (!Array.isArray(history)) return null; // Keepa CSV histories are [time, value, time, value, ...]. Only odd indexes // are prices; even indexes are Keepa timestamps and can look like huge prices. for (let i = history.length - 1; i >= 1; i--) { if (i % 2 === 0) continue; const value = extractNumber(history[i]); if (value != null && value > 0) return value / 100; } return null; } function resolveAmazonIsSeller( product: Record, stats: Record | undefined, csv: unknown, ): boolean | null { if (typeof product.isAmazonSeller === "boolean") return product.isAmazonSeller; if (typeof product.availabilityAmazon === "number") { if (product.availabilityAmazon >= 0) return true; if ( product.availabilityAmazon === -1 || product.availabilityAmazon === -2 ) { return false; } } if (stats?.buyBoxIsAmazon === true) return true; if (extractNumber(stats?.current?.[0]) != null) { const currentAmazon = extractNumber(stats?.current?.[0]); if (currentAmazon != null && currentAmazon > 0) return true; } const amazonHistoryPrice = Array.isArray(csv) ? extractLatestPositiveKeepaPrice(csv[0]) : null; return amazonHistoryPrice == null ? null : amazonHistoryPrice > 0; } function extractSellerRatingCount(seller: Record): number | null { const direct = extractNumber( seller.currentRatingCount ?? seller.ratingCount ?? seller.ratingsCount ?? seller.feedbackCount ?? seller.reviewCount, ); if (direct != null) return direct; const ratingCountHistory = seller.ratingCountHistory ?? seller.ratingCountCSV; if (Array.isArray(ratingCountHistory) && ratingCountHistory.length > 0) { return extractNumber(ratingCountHistory[ratingCountHistory.length - 1]); } return null; } function extractOfferPrice(offer: Record): number | null { const raw = extractNumber( offer.price ?? offer.currentPrice ?? offer.offerPrice ?? offer.newPrice, ); if (raw == null) return null; return raw > 100 ? Math.round(raw) / 100 : raw; } function sumInventoryAsins(result: StalkerAsinResult): number { return result.matchedSellers.reduce( (sum, entry) => sum + entry.seller.storefrontAsins.length, 0, ); } function collectPersistedInventoryAsins(result: StalkerAsinResult): string[] { const seen = new Set(); for (const { seller } of result.matchedSellers) { for (const asin of seller.storefrontAsins) { seen.add(asin); } } return Array.from(seen); } async function runSellableAnalysisChild( dbPath: string, stalkerRunId: number, analysisRunId: number, asins: string[], useClaude: boolean, ): Promise { const cmd = [ "bun", "run", "src/stalker-analyze.ts", "--db", dbPath, "--stalker-run-id", String(stalkerRunId), "--analysis-run-id", String(analysisRunId), "--asins", asins.join(","), ]; if (useClaude) { cmd.push("--claude"); } const child = Bun.spawn({ cmd, stdout: "inherit", stderr: "inherit", }); const exitCode = await child.exited; if (exitCode !== 0) { console.warn( `Stalker analysis child failed for ${asins.length} ASIN(s), exit=${exitCode}`, ); } } function normalizeAsin(value: unknown): string | null { const asin = String(value ?? "") .trim() .toUpperCase(); return ASIN_REGEX.test(asin) ? asin : null; } function normalizeSellerId(value: unknown): string | null { const sellerId = String(value ?? "") .trim() .toUpperCase(); return sellerId.length > 0 ? sellerId : null; } function extractString(value: unknown): string | null { if (value == null) return null; const text = String(value).trim(); return text.length > 0 ? text : null; } function extractNumber(value: unknown): number | null { if (typeof value === "number" && Number.isFinite(value)) return value; if (typeof value !== "string") return null; const parsed = Number(value.trim().replace(/[$,%]/g, "").replace(/,/g, "")); return Number.isFinite(parsed) ? parsed : null; } function extractBoolean(value: unknown): boolean | null { if (typeof value === "boolean") return value; if (typeof value === "number") return value === 1 ? true : value === 0 ? false : null; if (typeof value !== "string") return null; const normalized = value.trim().toLowerCase(); if (["1", "true", "yes"].includes(normalized)) return true; if (["0", "false", "no"].includes(normalized)) return false; return null; } function normalizeHeader(value: string): string { return value .toLowerCase() .trim() .replace(/[^a-z0-9]/g, ""); } function readFlagValue(args: string[], flag: string): string | undefined { const index = args.indexOf(flag); if (index === -1) return undefined; return args[index + 1]; } function hasFlag(args: string[], flag: string): boolean { return args.includes(flag); } function printUsageAndExit(message: string): never { console.error(message); console.error( "Usage: bun run stalker --input input/asins.xlsx [--db db/results.db] [--max-asins N] [--offer-limit 100] [--seller-limit 30] [--inventory-limit 200] [--storefront-update-hours 168] [--seller-cache-hours 168] [--max-seller-requests N] [--sellability] [--analyze-sellable] [--include-stock] [--dry-run] [--no-resume] [--claude]", ); process.exit(1); } function updateTokenState(data: KeepaApiResponse): void { if (data.tokensLeft != null) tokensLeft = data.tokensLeft; if (data.refillRate != null) refillRate = data.refillRate; } async function waitForToken(): Promise { if (tokensLeft > 0) return; const elapsed = (Date.now() - lastRequestTime) / 60_000; const regenerated = Math.floor(elapsed * Math.max(1, refillRate)); if (regenerated > 0) { tokensLeft += regenerated; return; } const waitMs = Math.ceil((1 / Math.max(1, refillRate)) * 60_000) - (Date.now() - lastRequestTime); if (waitMs > 0) { console.log( `Keepa tokens exhausted. Waiting ${Math.ceil(waitMs / 1000)}s for token regeneration...`, ); await wait(waitMs); } tokensLeft = 1; } function computeWaitMsFromRefill(refillIn?: number): number { if ( typeof refillIn === "number" && Number.isFinite(refillIn) && refillIn >= 0 ) { return Math.max( Math.ceil(refillIn) + KEEP_RETRY_BUFFER_MS, KEEP_RETRY_BUFFER_MS, ); } return ( Math.ceil((1 / Math.max(1, refillRate)) * 60_000) + KEEP_RETRY_BUFFER_MS ); } function parseErrorPayload(text: string): KeepaApiResponse | null { try { const parsed = JSON.parse(text) as KeepaApiResponse; return parsed && typeof parsed === "object" ? parsed : null; } catch { return null; } } function wait(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } if (import.meta.main) { const args = parseArgs(); runStalker(args) .then((stats) => { console.log( `Stalker complete: scanned=${stats.scannedAsins}, matched_sellers=${stats.matchedSellers}, persisted_inventory_asins=${stats.persistedInventoryAsins}, failed=${stats.failedAsins}`, ); }) .catch((error) => { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; }) .finally(() => { closeDb(); }); }