95cebaa27c
- Introduced `useClaude` option in `AnalysisPipelineOptions` to toggle Claude LLM usage. - Updated `processProductChunk` and `analyzeProducts` functions to accept and handle `useClaude` parameter. - Modified argument parsing in various scripts (`bestsellers-by-category`, `mid-range-sellers-by-category`, `top-monthly-sold-by-category`, etc.) to include `--claude` flag. - Enhanced `analyzeProductsInternal` to differentiate between LLM providers and handle requests to Claude API. - Added error handling for Claude API responses and ensured proper configuration for using Claude. - Updated documentation and usage messages to reflect the new `--claude` flag.
1668 lines
48 KiB
TypeScript
1668 lines
48 KiB
TypeScript
import * as XLSX from "xlsx";
|
|
import path from "node:path";
|
|
import { type Database, closeDb, getDb, initDb } from "./database.ts";
|
|
import { fetchSellabilityBatch } from "./sp-api.ts";
|
|
import type { SellabilityInfo } from "./types.ts";
|
|
|
|
const KEEPA_BASE = "https://api.keepa.com";
|
|
const DOMAIN_US = "1";
|
|
const AMAZON_US_SELLER_ID = "ATVPDKIKX0DER";
|
|
const ASIN_REGEX = /^B[0-9A-Z]{9}$/;
|
|
const DEFAULT_DB_PATH = path.join(process.cwd(), "db", "results.db");
|
|
const DEFAULT_STOREFRONT_UPDATE_HOURS = 168;
|
|
const DEFAULT_OFFER_LIMIT = 100;
|
|
const DEFAULT_SELLER_LIMIT = 30;
|
|
const DEFAULT_INVENTORY_LIMIT = 200;
|
|
const DEFAULT_SELLER_CACHE_HOURS = 168;
|
|
const MAX_SELLERS_PER_METADATA_REQUEST = 100;
|
|
const MAX_KEEPA_RETRIES = 4;
|
|
const KEEP_RETRY_BUFFER_MS = 250;
|
|
|
|
type KeepaApiResponse = {
|
|
products?: Record<string, any>[];
|
|
sellers?: Record<string, any> | Record<string, any>[];
|
|
tokensLeft?: number;
|
|
refillRate?: number;
|
|
refillIn?: number;
|
|
};
|
|
|
|
export type StalkerArgs = {
|
|
input: string;
|
|
dbPath: string;
|
|
maxAsins: number | null;
|
|
storefrontUpdateHours: number;
|
|
offerLimit: number;
|
|
sellerLimit: number;
|
|
inventoryLimit: number;
|
|
sellerCacheHours: number;
|
|
includeStock: boolean;
|
|
dryRun: boolean;
|
|
resume: boolean;
|
|
maxSellerRequests: number | null;
|
|
sellability: boolean;
|
|
analyzeSellable: boolean;
|
|
useClaude: boolean;
|
|
};
|
|
|
|
export type StalkerOffer = {
|
|
sellerId: string;
|
|
offerPrice: number | null;
|
|
condition: string | null;
|
|
isFba: boolean | null;
|
|
stock: number | null;
|
|
rawOffer: Record<string, any>;
|
|
};
|
|
|
|
export type StalkerSeller = {
|
|
sellerId: string;
|
|
sellerName: string | null;
|
|
rating: number | null;
|
|
ratingCount: number | null;
|
|
storefrontAsins: string[];
|
|
storefrontItems: StalkerInventoryItem[];
|
|
storefrontAsinTotal: number;
|
|
rawSeller: Record<string, any>;
|
|
};
|
|
|
|
type StalkerInventoryItem = {
|
|
asin: string;
|
|
rawInventory: unknown;
|
|
sellability: SellabilityInfo | null;
|
|
productDetails: StalkerProductDetails | null;
|
|
};
|
|
|
|
type StalkerProductDetails = {
|
|
title: string | null;
|
|
brand: string | null;
|
|
categoryTree: string[];
|
|
currentPrice: number | null;
|
|
avgPrice90: number | null;
|
|
salesRank: number | null;
|
|
monthlySold: number | null;
|
|
sellerCount: number | null;
|
|
amazonIsSeller: boolean | null;
|
|
rawProduct: Record<string, any>;
|
|
};
|
|
|
|
type StalkerAsinResult = {
|
|
asin: string;
|
|
title: string | null;
|
|
offerCount: number;
|
|
candidateSellerCount: number;
|
|
matchedSellers: Array<{
|
|
seller: StalkerSeller;
|
|
offer: StalkerOffer;
|
|
}>;
|
|
product: Record<string, any> | null;
|
|
error?: string;
|
|
};
|
|
|
|
type StalkerRunStats = {
|
|
scannedAsins: number;
|
|
sourceAsinsWithMatches: number;
|
|
matchedSellers: number;
|
|
persistedInventoryAsins: number;
|
|
failedAsins: number;
|
|
skippedAsins: number;
|
|
candidateSellers: number;
|
|
qualifyingSellers: number;
|
|
sellerMetadataRequests: number;
|
|
sellerStorefrontRequests: number;
|
|
inventorySellabilityCheckedAsins: number;
|
|
inventorySellabilityAvailableAsins: number;
|
|
inventorySellabilityExcludedAsins: number;
|
|
stoppedEarly: boolean;
|
|
};
|
|
|
|
type StalkerRunContext = {
|
|
database: Database | null;
|
|
metadataCache: Map<string, StalkerSeller>;
|
|
storefrontCache: Map<string, StalkerSeller>;
|
|
stats: StalkerRunStats;
|
|
};
|
|
|
|
let tokensLeft = 1;
|
|
let refillRate = 1;
|
|
let lastRequestTime = 0;
|
|
|
|
export function parseArgs(argv = process.argv.slice(2)): StalkerArgs {
|
|
const input = readFlagValue(argv, "--input");
|
|
if (!input) {
|
|
printUsageAndExit("Missing required --input file.");
|
|
}
|
|
|
|
const dbPath = readFlagValue(argv, "--db") ?? DEFAULT_DB_PATH;
|
|
const maxAsinsRaw = readFlagValue(argv, "--max-asins");
|
|
const storefrontUpdateRaw = readFlagValue(argv, "--storefront-update-hours");
|
|
const offerLimitRaw = readFlagValue(argv, "--offer-limit");
|
|
const sellerLimitRaw = readFlagValue(argv, "--seller-limit");
|
|
const inventoryLimitRaw = readFlagValue(argv, "--inventory-limit");
|
|
const sellerCacheHoursRaw = readFlagValue(argv, "--seller-cache-hours");
|
|
const maxSellerRequestsRaw = readFlagValue(argv, "--max-seller-requests");
|
|
|
|
const maxAsins = maxAsinsRaw ? Number(maxAsinsRaw) : null;
|
|
const storefrontUpdateHours = storefrontUpdateRaw
|
|
? Number(storefrontUpdateRaw)
|
|
: DEFAULT_STOREFRONT_UPDATE_HOURS;
|
|
const offerLimit = offerLimitRaw
|
|
? Number(offerLimitRaw)
|
|
: DEFAULT_OFFER_LIMIT;
|
|
const sellerLimit = sellerLimitRaw
|
|
? Number(sellerLimitRaw)
|
|
: DEFAULT_SELLER_LIMIT;
|
|
const inventoryLimit = inventoryLimitRaw
|
|
? Number(inventoryLimitRaw)
|
|
: DEFAULT_INVENTORY_LIMIT;
|
|
const sellerCacheHours = sellerCacheHoursRaw
|
|
? Number(sellerCacheHoursRaw)
|
|
: DEFAULT_SELLER_CACHE_HOURS;
|
|
const maxSellerRequests = maxSellerRequestsRaw
|
|
? Number(maxSellerRequestsRaw)
|
|
: null;
|
|
const includeStock = hasFlag(argv, "--include-stock");
|
|
const dryRun = hasFlag(argv, "--dry-run");
|
|
const resume = !hasFlag(argv, "--no-resume");
|
|
const sellability = hasFlag(argv, "--sellability");
|
|
const analyzeSellable = hasFlag(argv, "--analyze-sellable");
|
|
const useClaude = hasFlag(argv, "--claude");
|
|
|
|
if (analyzeSellable && !sellability) {
|
|
printUsageAndExit("--analyze-sellable requires --sellability.");
|
|
}
|
|
|
|
if (maxAsins != null && (!Number.isInteger(maxAsins) || maxAsins <= 0)) {
|
|
printUsageAndExit("--max-asins must be a positive integer.");
|
|
}
|
|
|
|
if (!Number.isInteger(storefrontUpdateHours) || storefrontUpdateHours < 0) {
|
|
printUsageAndExit(
|
|
"--storefront-update-hours must be a non-negative integer.",
|
|
);
|
|
}
|
|
|
|
if (!Number.isInteger(offerLimit) || offerLimit < 20 || offerLimit > 100) {
|
|
printUsageAndExit("--offer-limit must be an integer from 20 to 100.");
|
|
}
|
|
|
|
if (!Number.isInteger(sellerLimit) || sellerLimit <= 0) {
|
|
printUsageAndExit("--seller-limit must be a positive integer.");
|
|
}
|
|
|
|
if (!Number.isInteger(inventoryLimit) || inventoryLimit < 0) {
|
|
printUsageAndExit("--inventory-limit must be a non-negative integer.");
|
|
}
|
|
|
|
if (!Number.isInteger(sellerCacheHours) || sellerCacheHours < 0) {
|
|
printUsageAndExit("--seller-cache-hours must be a non-negative integer.");
|
|
}
|
|
|
|
if (
|
|
maxSellerRequests != null &&
|
|
(!Number.isInteger(maxSellerRequests) || maxSellerRequests <= 0)
|
|
) {
|
|
printUsageAndExit("--max-seller-requests must be a positive integer.");
|
|
}
|
|
|
|
return {
|
|
input,
|
|
dbPath,
|
|
maxAsins,
|
|
storefrontUpdateHours,
|
|
offerLimit,
|
|
sellerLimit,
|
|
inventoryLimit,
|
|
sellerCacheHours,
|
|
includeStock,
|
|
dryRun,
|
|
resume,
|
|
maxSellerRequests,
|
|
sellability,
|
|
analyzeSellable,
|
|
useClaude,
|
|
};
|
|
}
|
|
|
|
export function readAsinsFromXlsx(filePath: string): string[] {
|
|
const workbook = XLSX.readFile(filePath);
|
|
const sheetName = workbook.SheetNames[0];
|
|
if (!sheetName) throw new Error("No sheets found in file");
|
|
|
|
const sheet = workbook.Sheets[sheetName];
|
|
if (!sheet) throw new Error("First sheet is missing");
|
|
|
|
const rows = XLSX.utils.sheet_to_json<Record<string, unknown>>(sheet, {
|
|
defval: "",
|
|
});
|
|
if (rows.length === 0) throw new Error("File contains no data rows");
|
|
|
|
const headers = Object.keys(rows[0]!);
|
|
const asinColumn = headers.find(
|
|
(header) => normalizeHeader(header) === "asin",
|
|
);
|
|
if (!asinColumn) {
|
|
throw new Error(
|
|
`No ASIN column found. Available columns: ${headers.join(", ")}`,
|
|
);
|
|
}
|
|
|
|
return extractAsinsFromRows(rows, asinColumn);
|
|
}
|
|
|
|
export function extractAsinsFromRows(
|
|
rows: Array<Record<string, unknown>>,
|
|
asinColumn = "asin",
|
|
): string[] {
|
|
const asins: string[] = [];
|
|
const seen = new Set<string>();
|
|
|
|
for (const row of rows) {
|
|
const asin = normalizeAsin(row[asinColumn]);
|
|
if (!asin || seen.has(asin)) continue;
|
|
seen.add(asin);
|
|
asins.push(asin);
|
|
}
|
|
|
|
return asins;
|
|
}
|
|
|
|
export function isQualifyingSeller(seller: {
|
|
ratingCount?: number | null;
|
|
}): boolean {
|
|
return (
|
|
typeof seller.ratingCount === "number" &&
|
|
Number.isFinite(seller.ratingCount) &&
|
|
seller.ratingCount >= 1 &&
|
|
seller.ratingCount <= 30
|
|
);
|
|
}
|
|
|
|
export function extractLiveOfferSellerCandidates(
|
|
product: Record<string, any>,
|
|
): StalkerOffer[] {
|
|
const offers = Array.isArray(product.offers) ? product.offers : [];
|
|
const bySeller = new Map<string, StalkerOffer>();
|
|
|
|
for (const offer of offers) {
|
|
if (!offer || typeof offer !== "object") continue;
|
|
const sellerId = normalizeSellerId(
|
|
offer.sellerId ?? offer.sellerID ?? offer.seller_id,
|
|
);
|
|
if (!sellerId || sellerId === AMAZON_US_SELLER_ID) continue;
|
|
if (bySeller.has(sellerId)) continue;
|
|
|
|
bySeller.set(sellerId, {
|
|
sellerId,
|
|
offerPrice: extractOfferPrice(offer),
|
|
condition: extractString(offer.condition ?? offer.conditionComment),
|
|
isFba: extractBoolean(offer.isFBA ?? offer.isFba ?? offer.fba),
|
|
stock: extractNumber(
|
|
offer.stock ?? offer.stockCount ?? offer.currentStock,
|
|
),
|
|
rawOffer: offer,
|
|
});
|
|
}
|
|
|
|
return Array.from(bySeller.values());
|
|
}
|
|
|
|
export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
|
|
const apiKey = Bun.env.KEEPA_API_KEY;
|
|
if (!apiKey) throw new Error("Missing required env var: KEEPA_API_KEY");
|
|
|
|
const allAsins = readAsinsFromXlsx(args.input);
|
|
const cappedAsins =
|
|
args.maxAsins == null ? allAsins : allAsins.slice(0, args.maxAsins);
|
|
|
|
initDb(args.dbPath);
|
|
const database = getDb(args.dbPath);
|
|
const completedAsins = args.resume
|
|
? loadPreviouslyScannedAsins(database)
|
|
: new Set<string>();
|
|
const resumeFilteredAsins = cappedAsins.filter(
|
|
(asin) => !completedAsins.has(asin),
|
|
);
|
|
const runId = args.dryRun
|
|
? null
|
|
: startStalkerRun(database, args.input, resumeFilteredAsins.length);
|
|
const analysisRunId =
|
|
!args.dryRun && args.analyzeSellable
|
|
? startStalkerAnalysisRun(database, args.input)
|
|
: null;
|
|
const stats: StalkerRunStats = {
|
|
scannedAsins: 0,
|
|
sourceAsinsWithMatches: 0,
|
|
matchedSellers: 0,
|
|
persistedInventoryAsins: 0,
|
|
failedAsins: 0,
|
|
skippedAsins: cappedAsins.length - resumeFilteredAsins.length,
|
|
candidateSellers: 0,
|
|
qualifyingSellers: 0,
|
|
sellerMetadataRequests: 0,
|
|
sellerStorefrontRequests: 0,
|
|
inventorySellabilityCheckedAsins: 0,
|
|
inventorySellabilityAvailableAsins: 0,
|
|
inventorySellabilityExcludedAsins: 0,
|
|
stoppedEarly: false,
|
|
};
|
|
const context: StalkerRunContext = {
|
|
database,
|
|
metadataCache: new Map(),
|
|
storefrontCache: new Map(),
|
|
stats,
|
|
};
|
|
|
|
try {
|
|
if (args.dryRun) {
|
|
console.log(
|
|
"Stalker dry-run: product and seller metadata will be fetched, storefronts will not be fetched or persisted.",
|
|
);
|
|
}
|
|
if (stats.skippedAsins > 0) {
|
|
console.log(
|
|
`Stalker resume: skipped ${stats.skippedAsins} previously scanned ASIN(s).`,
|
|
);
|
|
}
|
|
|
|
for (const asin of resumeFilteredAsins) {
|
|
console.log(
|
|
`Stalker: scanning ${asin} (${stats.scannedAsins + 1}/${resumeFilteredAsins.length})`,
|
|
);
|
|
|
|
const result = await scanAsin(asin, args, apiKey, context).catch(
|
|
(error) => ({
|
|
asin,
|
|
title: null,
|
|
offerCount: 0,
|
|
candidateSellerCount: 0,
|
|
matchedSellers: [],
|
|
product: null,
|
|
error: error instanceof Error ? error.message : String(error),
|
|
}),
|
|
);
|
|
|
|
if (args.sellability && !args.dryRun) {
|
|
await enrichInventorySellability(result, stats);
|
|
}
|
|
applyInventoryPersistencePolicy(result, args.sellability && !args.dryRun);
|
|
if (args.sellability && !args.dryRun) {
|
|
await enrichInventoryProductDetails(result, apiKey);
|
|
}
|
|
|
|
if (!args.dryRun && runId != null) {
|
|
persistAsinResult(database, runId, result);
|
|
}
|
|
const sellableAsins = collectPersistedInventoryAsins(result);
|
|
if (
|
|
args.analyzeSellable &&
|
|
!args.dryRun &&
|
|
runId != null &&
|
|
analysisRunId != null &&
|
|
sellableAsins.length > 0
|
|
) {
|
|
await runSellableAnalysisChild(
|
|
args.dbPath,
|
|
runId,
|
|
analysisRunId,
|
|
sellableAsins,
|
|
args.useClaude,
|
|
);
|
|
}
|
|
stats.scannedAsins += 1;
|
|
stats.matchedSellers += result.matchedSellers.length;
|
|
stats.persistedInventoryAsins += sumInventoryAsins(result);
|
|
if (result.matchedSellers.length > 0) stats.sourceAsinsWithMatches += 1;
|
|
if (result.error) {
|
|
stats.failedAsins += 1;
|
|
console.warn(`Stalker: ${asin} failed: ${result.error}`);
|
|
}
|
|
|
|
if (!args.dryRun && runId != null) {
|
|
refreshStalkerRun(database, runId, stats, "running");
|
|
}
|
|
console.log(
|
|
`Stalker: ${asin} candidates=${result.candidateSellerCount}, matched=${result.matchedSellers.length}, persisted_inventory=${sumInventoryAsins(result)}`,
|
|
);
|
|
|
|
if (stats.stoppedEarly) {
|
|
console.log(
|
|
"Stalker: stopping early because max seller request budget was reached.",
|
|
);
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!args.dryRun && runId != null) {
|
|
refreshStalkerRun(
|
|
database,
|
|
runId,
|
|
stats,
|
|
stats.stoppedEarly
|
|
? "stopped"
|
|
: stats.failedAsins > 0
|
|
? "completed_with_errors"
|
|
: "completed",
|
|
);
|
|
}
|
|
logRunSummary(stats, args);
|
|
if (!args.dryRun && analysisRunId != null) {
|
|
finishStalkerAnalysisRun(database, analysisRunId, "completed");
|
|
}
|
|
return stats;
|
|
} catch (error) {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
if (!args.dryRun && runId != null) {
|
|
finishStalkerRunWithError(database, runId, stats, message);
|
|
}
|
|
if (!args.dryRun && analysisRunId != null) {
|
|
finishStalkerAnalysisRun(database, analysisRunId, "failed", message);
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async function scanAsin(
|
|
asin: string,
|
|
args: StalkerArgs,
|
|
apiKey: string,
|
|
context: StalkerRunContext,
|
|
): Promise<StalkerAsinResult> {
|
|
const product = await fetchKeepaProduct(
|
|
asin,
|
|
apiKey,
|
|
args.offerLimit,
|
|
args.includeStock,
|
|
);
|
|
const offers = extractLiveOfferSellerCandidates(product).slice(
|
|
0,
|
|
args.sellerLimit,
|
|
);
|
|
context.stats.candidateSellers += offers.length;
|
|
|
|
const metadata = await fetchSellerMetadata(
|
|
offers.map((offer) => offer.sellerId),
|
|
apiKey,
|
|
args,
|
|
context,
|
|
);
|
|
const qualifyingOffers = offers.filter((offer) => {
|
|
const seller = metadata.get(offer.sellerId);
|
|
return seller ? isQualifyingSeller(seller) : false;
|
|
});
|
|
context.stats.qualifyingSellers += qualifyingOffers.length;
|
|
|
|
if (args.dryRun) {
|
|
console.log(
|
|
`Stalker dry-run estimate for ${asin}: storefront requests needed=${qualifyingOffers.length}, candidate sellers=${offers.length}`,
|
|
);
|
|
}
|
|
|
|
const storefronts = args.dryRun
|
|
? new Map<string, StalkerSeller>()
|
|
: await fetchQualifiedSellerStorefronts(
|
|
qualifyingOffers.map((offer) => offer.sellerId),
|
|
apiKey,
|
|
args,
|
|
context,
|
|
);
|
|
|
|
const matchedSellers = qualifyingOffers
|
|
.map((offer) => {
|
|
const seller = storefronts.get(offer.sellerId);
|
|
if (!seller || !isQualifyingSeller(seller)) return null;
|
|
return { seller, offer };
|
|
})
|
|
.filter(
|
|
(entry): entry is { seller: StalkerSeller; offer: StalkerOffer } =>
|
|
entry != null,
|
|
);
|
|
|
|
return {
|
|
asin,
|
|
title: extractString(product.title),
|
|
offerCount: Array.isArray(product.offers) ? product.offers.length : 0,
|
|
candidateSellerCount: offers.length,
|
|
matchedSellers,
|
|
product,
|
|
};
|
|
}
|
|
|
|
function applyInventoryPersistencePolicy(
|
|
result: StalkerAsinResult,
|
|
requireAvailableSellability: boolean,
|
|
): void {
|
|
for (const { seller } of result.matchedSellers) {
|
|
seller.storefrontItems = seller.storefrontItems.filter((item) => {
|
|
if (!requireAvailableSellability) return false;
|
|
return (
|
|
item.sellability?.canSell === true &&
|
|
item.sellability.sellabilityStatus === "available"
|
|
);
|
|
});
|
|
seller.storefrontAsins = seller.storefrontItems.map((item) => item.asin);
|
|
}
|
|
}
|
|
|
|
async function enrichInventorySellability(
|
|
result: StalkerAsinResult,
|
|
stats: StalkerRunStats,
|
|
): Promise<void> {
|
|
const sellers = result.matchedSellers.map(({ seller }) => seller);
|
|
const items = sellers.flatMap((seller) => seller.storefrontItems);
|
|
const uniqueAsins = Array.from(new Set(items.map((item) => item.asin)));
|
|
if (uniqueAsins.length === 0) return;
|
|
|
|
console.log(
|
|
`Stalker inventory sellability: checking ${uniqueAsins.length} ASIN(s) from matched seller storefronts...`,
|
|
);
|
|
const sellabilityMap = await fetchSellabilityBatch(uniqueAsins);
|
|
stats.inventorySellabilityCheckedAsins += uniqueAsins.length;
|
|
|
|
for (const asin of uniqueAsins) {
|
|
const info = sellabilityMap.get(asin) ?? {
|
|
canSell: null,
|
|
sellabilityStatus: "unknown" as const,
|
|
sellabilityReason: "Sellability check returned no result",
|
|
};
|
|
|
|
if (info.sellabilityStatus === "available" && info.canSell === true) {
|
|
stats.inventorySellabilityAvailableAsins += 1;
|
|
} else {
|
|
stats.inventorySellabilityExcludedAsins += 1;
|
|
}
|
|
}
|
|
|
|
for (const item of items) {
|
|
item.sellability = sellabilityMap.get(item.asin) ?? {
|
|
canSell: null,
|
|
sellabilityStatus: "unknown",
|
|
sellabilityReason: "Sellability check returned no result",
|
|
};
|
|
}
|
|
|
|
for (const seller of sellers) {
|
|
seller.storefrontItems = seller.storefrontItems.filter(
|
|
(item) =>
|
|
item.sellability?.canSell === true &&
|
|
item.sellability.sellabilityStatus === "available",
|
|
);
|
|
seller.storefrontAsins = seller.storefrontItems.map((item) => item.asin);
|
|
}
|
|
}
|
|
|
|
async function enrichInventoryProductDetails(
|
|
result: StalkerAsinResult,
|
|
apiKey: string,
|
|
): Promise<void> {
|
|
const items = result.matchedSellers.flatMap(
|
|
({ seller }) => seller.storefrontItems,
|
|
);
|
|
const uniqueAsins = Array.from(new Set(items.map((item) => item.asin)));
|
|
if (uniqueAsins.length === 0) return;
|
|
|
|
console.log(
|
|
`Stalker inventory details: fetching Keepa product details for ${uniqueAsins.length} sellable ASIN(s)...`,
|
|
);
|
|
const detailsByAsin = await fetchKeepaInventoryProductDetails(
|
|
apiKey,
|
|
uniqueAsins,
|
|
);
|
|
|
|
for (const item of items) {
|
|
item.productDetails = detailsByAsin.get(item.asin) ?? null;
|
|
}
|
|
}
|
|
|
|
async function fetchKeepaProduct(
|
|
asin: string,
|
|
apiKey: string,
|
|
offerLimit: number,
|
|
includeStock: boolean,
|
|
): Promise<Record<string, any>> {
|
|
const params = new URLSearchParams({
|
|
key: apiKey,
|
|
domain: DOMAIN_US,
|
|
asin,
|
|
offers: String(offerLimit),
|
|
"only-live-offers": "1",
|
|
stats: "30",
|
|
days: "30",
|
|
});
|
|
if (includeStock) {
|
|
params.set("stock", "1");
|
|
}
|
|
const data = await fetchKeepaWithRetries(
|
|
`${KEEPA_BASE}/product?${params.toString()}`,
|
|
`product ${asin}`,
|
|
);
|
|
const product = data.products?.[0];
|
|
if (!product) throw new Error("Keepa returned no product");
|
|
return product;
|
|
}
|
|
|
|
async function fetchKeepaInventoryProductDetails(
|
|
apiKey: string,
|
|
asins: string[],
|
|
): Promise<Map<string, StalkerProductDetails>> {
|
|
const details = new Map<string, StalkerProductDetails>();
|
|
const chunkSize = 100;
|
|
|
|
for (let i = 0; i < asins.length; i += chunkSize) {
|
|
const chunk = asins.slice(i, i + chunkSize);
|
|
const params = new URLSearchParams({
|
|
key: apiKey,
|
|
domain: DOMAIN_US,
|
|
asin: chunk.join(","),
|
|
stats: "30",
|
|
days: "30",
|
|
buybox: "1",
|
|
});
|
|
|
|
const data = await fetchKeepaWithRetries(
|
|
`${KEEPA_BASE}/product?${params.toString()}`,
|
|
`inventory product details ${i + 1}-${i + chunk.length}`,
|
|
);
|
|
|
|
for (const product of data.products ?? []) {
|
|
const asin = normalizeAsin(product.asin);
|
|
if (!asin) continue;
|
|
details.set(asin, parseInventoryProductDetails(product));
|
|
}
|
|
}
|
|
|
|
return details;
|
|
}
|
|
|
|
async function fetchSellerMetadata(
|
|
sellerIds: string[],
|
|
apiKey: string,
|
|
args: StalkerArgs,
|
|
context: StalkerRunContext,
|
|
): Promise<Map<string, StalkerSeller>> {
|
|
const out = new Map<string, StalkerSeller>();
|
|
const uniqueSellerIds = Array.from(new Set(sellerIds));
|
|
const missing: string[] = [];
|
|
|
|
for (const sellerId of uniqueSellerIds) {
|
|
const cached =
|
|
context.metadataCache.get(sellerId) ??
|
|
loadCachedSeller(
|
|
context.database,
|
|
sellerId,
|
|
args.sellerCacheHours,
|
|
false,
|
|
args.inventoryLimit,
|
|
);
|
|
if (cached) {
|
|
context.metadataCache.set(sellerId, cached);
|
|
out.set(sellerId, cached);
|
|
continue;
|
|
}
|
|
missing.push(sellerId);
|
|
}
|
|
|
|
for (let i = 0; i < missing.length; i += MAX_SELLERS_PER_METADATA_REQUEST) {
|
|
const chunk = missing.slice(i, i + MAX_SELLERS_PER_METADATA_REQUEST);
|
|
if (!canSpendSellerRequests(context, args, 1)) break;
|
|
|
|
const params = new URLSearchParams({
|
|
key: apiKey,
|
|
domain: DOMAIN_US,
|
|
seller: chunk.join(","),
|
|
});
|
|
|
|
context.stats.sellerMetadataRequests += 1;
|
|
const data = await fetchKeepaWithRetries(
|
|
`${KEEPA_BASE}/seller?${params.toString()}`,
|
|
`seller metadata batch ${i / MAX_SELLERS_PER_METADATA_REQUEST + 1}`,
|
|
);
|
|
|
|
for (const [sellerId, seller] of normalizeSellerResponse(data.sellers)) {
|
|
const parsed = parseSeller(sellerId, seller, args.inventoryLimit);
|
|
context.metadataCache.set(sellerId, parsed);
|
|
out.set(sellerId, parsed);
|
|
}
|
|
}
|
|
|
|
return out;
|
|
}
|
|
|
|
async function fetchQualifiedSellerStorefronts(
|
|
sellerIds: string[],
|
|
apiKey: string,
|
|
args: StalkerArgs,
|
|
context: StalkerRunContext,
|
|
): Promise<Map<string, StalkerSeller>> {
|
|
const out = new Map<string, StalkerSeller>();
|
|
const uniqueSellerIds = Array.from(new Set(sellerIds));
|
|
|
|
// Keepa only allows a single sellerId per request when storefront=1.
|
|
for (const sellerId of uniqueSellerIds) {
|
|
const cached =
|
|
context.storefrontCache.get(sellerId) ??
|
|
loadCachedSeller(
|
|
context.database,
|
|
sellerId,
|
|
args.sellerCacheHours,
|
|
true,
|
|
args.inventoryLimit,
|
|
);
|
|
if (cached) {
|
|
context.storefrontCache.set(sellerId, cached);
|
|
out.set(sellerId, cached);
|
|
continue;
|
|
}
|
|
|
|
if (!canSpendSellerRequests(context, args, 1)) break;
|
|
|
|
const params = new URLSearchParams({
|
|
key: apiKey,
|
|
domain: DOMAIN_US,
|
|
seller: sellerId,
|
|
storefront: "1",
|
|
update: String(args.storefrontUpdateHours),
|
|
});
|
|
|
|
context.stats.sellerStorefrontRequests += 1;
|
|
const data = await fetchKeepaWithRetries(
|
|
`${KEEPA_BASE}/seller?${params.toString()}`,
|
|
`seller ${sellerId}`,
|
|
);
|
|
|
|
for (const [returnedSellerId, seller] of normalizeSellerResponse(
|
|
data.sellers,
|
|
)) {
|
|
const parsed = parseSeller(returnedSellerId, seller, args.inventoryLimit);
|
|
context.metadataCache.set(returnedSellerId, parsed);
|
|
context.storefrontCache.set(returnedSellerId, parsed);
|
|
out.set(returnedSellerId, parsed);
|
|
}
|
|
}
|
|
|
|
return out;
|
|
}
|
|
|
|
function canSpendSellerRequests(
|
|
context: StalkerRunContext,
|
|
args: StalkerArgs,
|
|
nextRequests: number,
|
|
): boolean {
|
|
if (args.maxSellerRequests == null) return true;
|
|
const spent =
|
|
context.stats.sellerMetadataRequests +
|
|
context.stats.sellerStorefrontRequests;
|
|
if (spent + nextRequests <= args.maxSellerRequests) return true;
|
|
context.stats.stoppedEarly = true;
|
|
return false;
|
|
}
|
|
|
|
async function fetchKeepaWithRetries(
|
|
url: string,
|
|
operationLabel: string,
|
|
): Promise<KeepaApiResponse> {
|
|
let lastErrorMessage = "Unknown Keepa error";
|
|
|
|
for (let attempt = 1; attempt <= MAX_KEEPA_RETRIES; attempt++) {
|
|
await waitForToken();
|
|
const response = await fetch(url);
|
|
lastRequestTime = Date.now();
|
|
|
|
if (response.ok) {
|
|
const data = (await response.json()) as KeepaApiResponse;
|
|
updateTokenState(data);
|
|
return data;
|
|
}
|
|
|
|
const text = await response.text();
|
|
const payload = parseErrorPayload(text);
|
|
if (payload) updateTokenState(payload);
|
|
lastErrorMessage = `Keepa API error ${response.status}: ${text}`;
|
|
|
|
if (response.status !== 429 || attempt === MAX_KEEPA_RETRIES) break;
|
|
|
|
const waitMs = computeWaitMsFromRefill(payload?.refillIn);
|
|
tokensLeft = Math.min(tokensLeft, 0);
|
|
console.warn(
|
|
`Keepa throttled during ${operationLabel} (attempt ${attempt}/${MAX_KEEPA_RETRIES}). Waiting ${Math.ceil(waitMs / 1000)}s before retry...`,
|
|
);
|
|
await wait(waitMs);
|
|
}
|
|
|
|
throw new Error(lastErrorMessage);
|
|
}
|
|
|
|
function persistAsinResult(
|
|
database: Database,
|
|
runId: number,
|
|
result: StalkerAsinResult,
|
|
): void {
|
|
const fetchedAt = new Date().toISOString();
|
|
|
|
database.transaction(() => {
|
|
const scanId = upsertAsinScan(database, runId, result, fetchedAt);
|
|
|
|
for (const { seller, offer } of result.matchedSellers) {
|
|
upsertSeller(database, seller, fetchedAt);
|
|
upsertAsinSeller(database, scanId, seller, offer);
|
|
upsertSellerInventory(database, runId, seller, fetchedAt);
|
|
}
|
|
})();
|
|
}
|
|
|
|
function upsertAsinScan(
|
|
database: Database,
|
|
runId: number,
|
|
result: StalkerAsinResult,
|
|
fetchedAt: string,
|
|
): number {
|
|
database
|
|
.prepare(
|
|
`INSERT INTO stalker_asin_scans (
|
|
run_id, source_asin, title, offer_count, candidate_seller_count,
|
|
matched_seller_count, fetched_at, raw_product_json
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT(run_id, source_asin) DO UPDATE SET
|
|
title = excluded.title,
|
|
offer_count = excluded.offer_count,
|
|
candidate_seller_count = excluded.candidate_seller_count,
|
|
matched_seller_count = excluded.matched_seller_count,
|
|
fetched_at = excluded.fetched_at,
|
|
raw_product_json = excluded.raw_product_json`,
|
|
)
|
|
.run(
|
|
runId,
|
|
result.asin,
|
|
result.title,
|
|
result.offerCount,
|
|
result.candidateSellerCount,
|
|
result.matchedSellers.length,
|
|
fetchedAt,
|
|
JSON.stringify(result.product ?? { error: result.error ?? null }),
|
|
);
|
|
|
|
const row = database
|
|
.query(
|
|
`SELECT id FROM stalker_asin_scans WHERE run_id = ? AND source_asin = ?`,
|
|
)
|
|
.get(runId, result.asin) as { id: number } | null;
|
|
if (!row)
|
|
throw new Error(`Failed to load stalker scan row for ${result.asin}`);
|
|
return row.id;
|
|
}
|
|
|
|
function upsertSeller(
|
|
database: Database,
|
|
seller: StalkerSeller,
|
|
fetchedAt: string,
|
|
): void {
|
|
database
|
|
.prepare(
|
|
`INSERT INTO stalker_sellers (
|
|
seller_id, seller_name, rating, rating_count, storefront_asin_total,
|
|
persisted_inventory_sample_count, last_updated_at, raw_seller_json
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT(seller_id) DO UPDATE SET
|
|
seller_name = excluded.seller_name,
|
|
rating = excluded.rating,
|
|
rating_count = excluded.rating_count,
|
|
storefront_asin_total = excluded.storefront_asin_total,
|
|
persisted_inventory_sample_count = excluded.persisted_inventory_sample_count,
|
|
last_updated_at = excluded.last_updated_at,
|
|
raw_seller_json = excluded.raw_seller_json`,
|
|
)
|
|
.run(
|
|
seller.sellerId,
|
|
seller.sellerName,
|
|
seller.rating,
|
|
seller.ratingCount,
|
|
seller.storefrontAsinTotal,
|
|
seller.storefrontItems.length,
|
|
fetchedAt,
|
|
JSON.stringify(seller.rawSeller),
|
|
);
|
|
}
|
|
|
|
function upsertAsinSeller(
|
|
database: Database,
|
|
scanId: number,
|
|
seller: StalkerSeller,
|
|
offer: StalkerOffer,
|
|
): void {
|
|
database
|
|
.prepare(
|
|
`INSERT INTO stalker_asin_sellers (
|
|
scan_id, seller_id, offer_price, condition, is_fba, stock,
|
|
seller_rating, seller_rating_count, raw_offer_json
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT(scan_id, seller_id) DO UPDATE SET
|
|
offer_price = excluded.offer_price,
|
|
condition = excluded.condition,
|
|
is_fba = excluded.is_fba,
|
|
stock = excluded.stock,
|
|
seller_rating = excluded.seller_rating,
|
|
seller_rating_count = excluded.seller_rating_count,
|
|
raw_offer_json = excluded.raw_offer_json`,
|
|
)
|
|
.run(
|
|
scanId,
|
|
seller.sellerId,
|
|
offer.offerPrice,
|
|
offer.condition,
|
|
offer.isFba == null ? null : offer.isFba ? 1 : 0,
|
|
offer.stock,
|
|
seller.rating,
|
|
seller.ratingCount,
|
|
JSON.stringify(offer.rawOffer),
|
|
);
|
|
}
|
|
|
|
function upsertSellerInventory(
|
|
database: Database,
|
|
runId: number,
|
|
seller: StalkerSeller,
|
|
fetchedAt: string,
|
|
): void {
|
|
const insert = database.prepare(
|
|
`INSERT INTO stalker_seller_inventory (
|
|
run_id, seller_id, asin, can_sell, sellability_status,
|
|
sellability_reason, product_title, brand, category_tree, current_price,
|
|
avg_price_90d, sales_rank, monthly_sold, seller_count, amazon_is_seller,
|
|
raw_product_json, last_seen_at, raw_inventory_json
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT(run_id, seller_id, asin) DO UPDATE SET
|
|
can_sell = excluded.can_sell,
|
|
sellability_status = excluded.sellability_status,
|
|
sellability_reason = excluded.sellability_reason,
|
|
product_title = excluded.product_title,
|
|
brand = excluded.brand,
|
|
category_tree = excluded.category_tree,
|
|
current_price = excluded.current_price,
|
|
avg_price_90d = excluded.avg_price_90d,
|
|
sales_rank = excluded.sales_rank,
|
|
monthly_sold = excluded.monthly_sold,
|
|
seller_count = excluded.seller_count,
|
|
amazon_is_seller = excluded.amazon_is_seller,
|
|
raw_product_json = excluded.raw_product_json,
|
|
last_seen_at = excluded.last_seen_at,
|
|
raw_inventory_json = excluded.raw_inventory_json`,
|
|
);
|
|
|
|
for (const item of seller.storefrontItems) {
|
|
if (
|
|
item.sellability?.canSell !== true ||
|
|
item.sellability.sellabilityStatus !== "available"
|
|
) {
|
|
continue;
|
|
}
|
|
|
|
insert.run(
|
|
runId,
|
|
seller.sellerId,
|
|
item.asin,
|
|
item.sellability?.canSell == null
|
|
? null
|
|
: item.sellability.canSell
|
|
? 1
|
|
: 0,
|
|
item.sellability?.sellabilityStatus ?? null,
|
|
item.sellability?.sellabilityReason ?? null,
|
|
item.productDetails?.title ?? null,
|
|
item.productDetails?.brand ?? null,
|
|
item.productDetails
|
|
? JSON.stringify(item.productDetails.categoryTree)
|
|
: null,
|
|
item.productDetails?.currentPrice ?? null,
|
|
item.productDetails?.avgPrice90 ?? null,
|
|
item.productDetails?.salesRank ?? null,
|
|
item.productDetails?.monthlySold ?? null,
|
|
item.productDetails?.sellerCount ?? null,
|
|
item.productDetails?.amazonIsSeller == null
|
|
? null
|
|
: item.productDetails.amazonIsSeller
|
|
? 1
|
|
: 0,
|
|
item.productDetails
|
|
? JSON.stringify(item.productDetails.rawProduct)
|
|
: null,
|
|
fetchedAt,
|
|
JSON.stringify(item.rawInventory),
|
|
);
|
|
}
|
|
}
|
|
|
|
function startStalkerRun(
|
|
database: Database,
|
|
inputFile: string,
|
|
totalAsins: number,
|
|
): number {
|
|
const result = database
|
|
.prepare(
|
|
`INSERT INTO stalker_runs (
|
|
input_file, started_at, requested_asins, status
|
|
) VALUES (?, ?, ?, ?)`,
|
|
)
|
|
.run(inputFile, new Date().toISOString(), totalAsins, "running");
|
|
|
|
return result.lastInsertRowid as number;
|
|
}
|
|
|
|
function startStalkerAnalysisRun(
|
|
database: Database,
|
|
inputFile: string,
|
|
): number {
|
|
const result = database
|
|
.prepare(
|
|
`INSERT INTO category_analysis_runs (
|
|
category_id, category_label, run_timestamp, top_asins_checked,
|
|
available_asins, fba_count, fbm_count, skip_count, status, error_message
|
|
) VALUES (?, ?, ?, 0, 0, 0, 0, 0, 'running', NULL)`,
|
|
)
|
|
.run(0, `Stalker: ${path.basename(inputFile)}`, new Date().toISOString());
|
|
|
|
return result.lastInsertRowid as number;
|
|
}
|
|
|
|
function loadPreviouslyScannedAsins(database: Database): Set<string> {
|
|
const rows = database
|
|
.query(`SELECT DISTINCT source_asin FROM stalker_asin_scans`)
|
|
.all() as Array<{ source_asin: string }>;
|
|
return new Set(rows.map((row) => row.source_asin));
|
|
}
|
|
|
|
function loadCachedSeller(
|
|
database: Database | null,
|
|
sellerId: string,
|
|
maxAgeHours: number,
|
|
requireStorefront: boolean,
|
|
inventoryLimit: number,
|
|
): StalkerSeller | null {
|
|
if (!database || maxAgeHours <= 0) return null;
|
|
const row = database
|
|
.query(
|
|
`SELECT raw_seller_json, last_updated_at, storefront_asin_total
|
|
FROM stalker_sellers
|
|
WHERE seller_id = ?`,
|
|
)
|
|
.get(sellerId) as {
|
|
raw_seller_json: string | null;
|
|
last_updated_at: string;
|
|
storefront_asin_total: number | null;
|
|
} | null;
|
|
if (!row?.raw_seller_json) return null;
|
|
|
|
const ageMs = Date.now() - new Date(row.last_updated_at).getTime();
|
|
if (!Number.isFinite(ageMs) || ageMs > maxAgeHours * 60 * 60 * 1000) {
|
|
return null;
|
|
}
|
|
|
|
try {
|
|
const rawSeller = JSON.parse(row.raw_seller_json) as Record<string, any>;
|
|
const parsed = parseSeller(sellerId, rawSeller, inventoryLimit);
|
|
if (requireStorefront && parsed.storefrontAsinTotal <= 0) return null;
|
|
return parsed;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function logRunSummary(stats: StalkerRunStats, args: StalkerArgs): void {
|
|
const estimatedStorefrontRequestsSaved = Math.max(
|
|
0,
|
|
stats.candidateSellers - stats.qualifyingSellers,
|
|
);
|
|
console.log(
|
|
[
|
|
"Stalker summary:",
|
|
`processed=${stats.scannedAsins}`,
|
|
`skipped=${stats.skippedAsins}`,
|
|
`candidate_sellers=${stats.candidateSellers}`,
|
|
`qualifying_sellers=${stats.qualifyingSellers}`,
|
|
`metadata_requests=${stats.sellerMetadataRequests}`,
|
|
`storefront_requests=${stats.sellerStorefrontRequests}`,
|
|
`sellability_checked=${stats.inventorySellabilityCheckedAsins}`,
|
|
`sellability_available=${stats.inventorySellabilityAvailableAsins}`,
|
|
`sellability_excluded=${stats.inventorySellabilityExcludedAsins}`,
|
|
`storefront_requests_saved_by_two_phase=${estimatedStorefrontRequestsSaved}`,
|
|
`persisted_inventory=${stats.persistedInventoryAsins}`,
|
|
`dry_run=${args.dryRun ? "yes" : "no"}`,
|
|
].join(" "),
|
|
);
|
|
}
|
|
|
|
function refreshStalkerRun(
|
|
database: Database,
|
|
runId: number,
|
|
stats: StalkerRunStats,
|
|
status: string,
|
|
): void {
|
|
database
|
|
.prepare(
|
|
`UPDATE stalker_runs
|
|
SET scanned_asins = ?,
|
|
source_asins_with_matches = ?,
|
|
candidate_sellers = ?,
|
|
qualifying_sellers = ?,
|
|
matched_sellers = ?,
|
|
seller_metadata_requests = ?,
|
|
seller_storefront_requests = ?,
|
|
inventory_sellability_checked_asins = ?,
|
|
inventory_sellability_available_asins = ?,
|
|
inventory_sellability_excluded_asins = ?,
|
|
persisted_inventory_asins = ?,
|
|
status = ?,
|
|
completed_at = CASE WHEN ? = 'running' THEN completed_at ELSE ? END
|
|
WHERE id = ?`,
|
|
)
|
|
.run(
|
|
stats.scannedAsins,
|
|
stats.sourceAsinsWithMatches,
|
|
stats.candidateSellers,
|
|
stats.qualifyingSellers,
|
|
stats.matchedSellers,
|
|
stats.sellerMetadataRequests,
|
|
stats.sellerStorefrontRequests,
|
|
stats.inventorySellabilityCheckedAsins,
|
|
stats.inventorySellabilityAvailableAsins,
|
|
stats.inventorySellabilityExcludedAsins,
|
|
stats.persistedInventoryAsins,
|
|
status,
|
|
status,
|
|
new Date().toISOString(),
|
|
runId,
|
|
);
|
|
}
|
|
|
|
function finishStalkerRunWithError(
|
|
database: Database,
|
|
runId: number,
|
|
stats: StalkerRunStats,
|
|
errorMessage: string,
|
|
): void {
|
|
database
|
|
.prepare(
|
|
`UPDATE stalker_runs
|
|
SET scanned_asins = ?,
|
|
source_asins_with_matches = ?,
|
|
candidate_sellers = ?,
|
|
qualifying_sellers = ?,
|
|
matched_sellers = ?,
|
|
seller_metadata_requests = ?,
|
|
seller_storefront_requests = ?,
|
|
inventory_sellability_checked_asins = ?,
|
|
inventory_sellability_available_asins = ?,
|
|
inventory_sellability_excluded_asins = ?,
|
|
persisted_inventory_asins = ?,
|
|
status = 'failed',
|
|
error_message = ?,
|
|
completed_at = ?
|
|
WHERE id = ?`,
|
|
)
|
|
.run(
|
|
stats.scannedAsins,
|
|
stats.sourceAsinsWithMatches,
|
|
stats.candidateSellers,
|
|
stats.qualifyingSellers,
|
|
stats.matchedSellers,
|
|
stats.sellerMetadataRequests,
|
|
stats.sellerStorefrontRequests,
|
|
stats.inventorySellabilityCheckedAsins,
|
|
stats.inventorySellabilityAvailableAsins,
|
|
stats.inventorySellabilityExcludedAsins,
|
|
stats.persistedInventoryAsins,
|
|
errorMessage,
|
|
new Date().toISOString(),
|
|
runId,
|
|
);
|
|
}
|
|
|
|
function finishStalkerAnalysisRun(
|
|
database: Database,
|
|
runId: number,
|
|
status: "completed" | "failed",
|
|
errorMessage: string | null = null,
|
|
): void {
|
|
const stats = database
|
|
.query(
|
|
`SELECT
|
|
COUNT(*) AS total,
|
|
SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba,
|
|
SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm,
|
|
SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip
|
|
FROM product_analysis_results
|
|
WHERE run_id = ?`,
|
|
)
|
|
.get(runId) as {
|
|
total: number;
|
|
fba: number | null;
|
|
fbm: number | null;
|
|
skip: number | null;
|
|
};
|
|
|
|
database
|
|
.prepare(
|
|
`UPDATE category_analysis_runs
|
|
SET top_asins_checked = ?,
|
|
available_asins = ?,
|
|
fba_count = ?,
|
|
fbm_count = ?,
|
|
skip_count = ?,
|
|
status = ?,
|
|
error_message = ?
|
|
WHERE id = ?`,
|
|
)
|
|
.run(
|
|
stats.total ?? 0,
|
|
stats.total ?? 0,
|
|
stats.fba ?? 0,
|
|
stats.fbm ?? 0,
|
|
stats.skip ?? 0,
|
|
status,
|
|
errorMessage,
|
|
runId,
|
|
);
|
|
}
|
|
|
|
function normalizeSellerResponse(
|
|
sellers: KeepaApiResponse["sellers"],
|
|
): Array<[string, Record<string, any>]> {
|
|
if (!sellers) return [];
|
|
if (Array.isArray(sellers)) {
|
|
return sellers
|
|
.map(
|
|
(seller) =>
|
|
[
|
|
normalizeSellerId(seller.sellerId ?? seller.sellerID ?? seller.id),
|
|
seller,
|
|
] as [string | null, Record<string, any>],
|
|
)
|
|
.filter((entry): entry is [string, Record<string, any>] => !!entry[0]);
|
|
}
|
|
|
|
return Object.entries(sellers)
|
|
.map(
|
|
([sellerId, seller]) =>
|
|
[normalizeSellerId(sellerId), seller] as [
|
|
string | null,
|
|
Record<string, any>,
|
|
],
|
|
)
|
|
.filter((entry): entry is [string, Record<string, any>] => !!entry[0]);
|
|
}
|
|
|
|
function parseSeller(
|
|
sellerId: string,
|
|
seller: Record<string, any>,
|
|
inventoryLimit: number,
|
|
): StalkerSeller {
|
|
const allStorefrontItems = extractStorefrontItems(seller);
|
|
const storefrontItems =
|
|
inventoryLimit === 0 ? [] : allStorefrontItems.slice(0, inventoryLimit);
|
|
const storefrontAsins = storefrontItems.map((item) => item.asin);
|
|
return {
|
|
sellerId,
|
|
sellerName: extractString(
|
|
seller.sellerName ??
|
|
seller.name ??
|
|
seller.storeName ??
|
|
seller.businessName,
|
|
),
|
|
rating: extractNumber(
|
|
seller.currentRating ?? seller.rating ?? seller.feedbackRating,
|
|
),
|
|
ratingCount: extractSellerRatingCount(seller),
|
|
storefrontAsins,
|
|
storefrontItems,
|
|
storefrontAsinTotal:
|
|
extractNumber(
|
|
seller.totalStorefrontAsinCount ??
|
|
seller.storefrontAsinCount ??
|
|
seller.asinListCount ??
|
|
seller.totalStorefrontProducts,
|
|
) ?? allStorefrontItems.length,
|
|
rawSeller: seller,
|
|
};
|
|
}
|
|
|
|
function extractStorefrontItems(
|
|
seller: Record<string, any>,
|
|
): StalkerInventoryItem[] {
|
|
const candidates = [
|
|
seller.asinList,
|
|
seller.asins,
|
|
seller.storefront,
|
|
seller.storefrontAsins,
|
|
seller.inventory,
|
|
];
|
|
const items: StalkerInventoryItem[] = [];
|
|
const seen = new Set<string>();
|
|
|
|
for (const candidate of candidates) {
|
|
collectStorefrontItems(candidate, items, seen);
|
|
}
|
|
|
|
return items;
|
|
}
|
|
|
|
function collectStorefrontItems(
|
|
value: unknown,
|
|
items: StalkerInventoryItem[],
|
|
seen: Set<string>,
|
|
): void {
|
|
if (Array.isArray(value)) {
|
|
for (const item of value) collectStorefrontItems(item, items, seen);
|
|
return;
|
|
}
|
|
|
|
if (value && typeof value === "object") {
|
|
const asin = normalizeAsin((value as Record<string, unknown>).asin);
|
|
if (asin && !seen.has(asin)) {
|
|
seen.add(asin);
|
|
items.push({
|
|
asin,
|
|
rawInventory: value,
|
|
sellability: null,
|
|
productDetails: null,
|
|
});
|
|
}
|
|
return;
|
|
}
|
|
|
|
const asin = normalizeAsin(value);
|
|
if (!asin || seen.has(asin)) return;
|
|
seen.add(asin);
|
|
items.push({
|
|
asin,
|
|
rawInventory: { asin },
|
|
sellability: null,
|
|
productDetails: null,
|
|
});
|
|
}
|
|
|
|
function parseInventoryProductDetails(
|
|
product: Record<string, any>,
|
|
): StalkerProductDetails {
|
|
const stats = product.stats;
|
|
const csv = product.csv;
|
|
return {
|
|
title: extractString(product.title),
|
|
brand: extractString(product.brand ?? product.manufacturer),
|
|
categoryTree:
|
|
product.categoryTree
|
|
?.map((category: { name?: unknown }) => extractString(category.name))
|
|
.filter((name: string | null): name is string => !!name) ?? [],
|
|
currentPrice: extractCurrentPrice(csv),
|
|
avgPrice90: stats?.avg?.[0] != null ? stats.avg[0] / 100 : null,
|
|
salesRank: extractNumber(stats?.current?.[3]),
|
|
monthlySold:
|
|
extractNumber(product.monthlySold ?? stats?.monthlySold) ??
|
|
extractNumber(product.salesRankDrops30 ?? stats?.salesRankDrops30),
|
|
sellerCount: extractNumber(stats?.current?.[11]),
|
|
amazonIsSeller: resolveAmazonIsSeller(product, stats, csv),
|
|
rawProduct: product,
|
|
};
|
|
}
|
|
|
|
function extractCurrentPrice(csv: unknown): number | null {
|
|
if (!Array.isArray(csv)) return null;
|
|
const amazonPrice = extractLatestPositiveKeepaPrice(csv[0]);
|
|
if (amazonPrice != null) return amazonPrice;
|
|
return extractLatestPositiveKeepaPrice(csv[1]);
|
|
}
|
|
|
|
function extractLatestPositiveKeepaPrice(history: unknown): number | null {
|
|
if (!Array.isArray(history)) return null;
|
|
|
|
// Keepa CSV histories are [time, value, time, value, ...]. Only odd indexes
|
|
// are prices; even indexes are Keepa timestamps and can look like huge prices.
|
|
for (let i = history.length - 1; i >= 1; i--) {
|
|
if (i % 2 === 0) continue;
|
|
const value = extractNumber(history[i]);
|
|
if (value != null && value > 0) return value / 100;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function resolveAmazonIsSeller(
|
|
product: Record<string, any>,
|
|
stats: Record<string, any> | undefined,
|
|
csv: unknown,
|
|
): boolean | null {
|
|
if (typeof product.isAmazonSeller === "boolean")
|
|
return product.isAmazonSeller;
|
|
if (typeof product.availabilityAmazon === "number") {
|
|
if (product.availabilityAmazon >= 0) return true;
|
|
if (
|
|
product.availabilityAmazon === -1 ||
|
|
product.availabilityAmazon === -2
|
|
) {
|
|
return false;
|
|
}
|
|
}
|
|
if (stats?.buyBoxIsAmazon === true) return true;
|
|
if (extractNumber(stats?.current?.[0]) != null) {
|
|
const currentAmazon = extractNumber(stats?.current?.[0]);
|
|
if (currentAmazon != null && currentAmazon > 0) return true;
|
|
}
|
|
const amazonHistoryPrice = Array.isArray(csv)
|
|
? extractLatestPositiveKeepaPrice(csv[0])
|
|
: null;
|
|
return amazonHistoryPrice == null ? null : amazonHistoryPrice > 0;
|
|
}
|
|
|
|
function extractSellerRatingCount(seller: Record<string, any>): number | null {
|
|
const direct = extractNumber(
|
|
seller.currentRatingCount ??
|
|
seller.ratingCount ??
|
|
seller.ratingsCount ??
|
|
seller.feedbackCount ??
|
|
seller.reviewCount,
|
|
);
|
|
if (direct != null) return direct;
|
|
|
|
const ratingCountHistory = seller.ratingCountHistory ?? seller.ratingCountCSV;
|
|
if (Array.isArray(ratingCountHistory) && ratingCountHistory.length > 0) {
|
|
return extractNumber(ratingCountHistory[ratingCountHistory.length - 1]);
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
function extractOfferPrice(offer: Record<string, any>): number | null {
|
|
const raw = extractNumber(
|
|
offer.price ?? offer.currentPrice ?? offer.offerPrice ?? offer.newPrice,
|
|
);
|
|
if (raw == null) return null;
|
|
return raw > 100 ? Math.round(raw) / 100 : raw;
|
|
}
|
|
|
|
function sumInventoryAsins(result: StalkerAsinResult): number {
|
|
return result.matchedSellers.reduce(
|
|
(sum, entry) => sum + entry.seller.storefrontAsins.length,
|
|
0,
|
|
);
|
|
}
|
|
|
|
function collectPersistedInventoryAsins(result: StalkerAsinResult): string[] {
|
|
const seen = new Set<string>();
|
|
for (const { seller } of result.matchedSellers) {
|
|
for (const asin of seller.storefrontAsins) {
|
|
seen.add(asin);
|
|
}
|
|
}
|
|
return Array.from(seen);
|
|
}
|
|
|
|
async function runSellableAnalysisChild(
|
|
dbPath: string,
|
|
stalkerRunId: number,
|
|
analysisRunId: number,
|
|
asins: string[],
|
|
useClaude: boolean,
|
|
): Promise<void> {
|
|
const cmd = [
|
|
"bun",
|
|
"run",
|
|
"src/stalker-analyze.ts",
|
|
"--db",
|
|
dbPath,
|
|
"--stalker-run-id",
|
|
String(stalkerRunId),
|
|
"--analysis-run-id",
|
|
String(analysisRunId),
|
|
"--asins",
|
|
asins.join(","),
|
|
];
|
|
if (useClaude) {
|
|
cmd.push("--claude");
|
|
}
|
|
|
|
const child = Bun.spawn({
|
|
cmd,
|
|
stdout: "inherit",
|
|
stderr: "inherit",
|
|
});
|
|
|
|
const exitCode = await child.exited;
|
|
if (exitCode !== 0) {
|
|
console.warn(
|
|
`Stalker analysis child failed for ${asins.length} ASIN(s), exit=${exitCode}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
function normalizeAsin(value: unknown): string | null {
|
|
const asin = String(value ?? "")
|
|
.trim()
|
|
.toUpperCase();
|
|
return ASIN_REGEX.test(asin) ? asin : null;
|
|
}
|
|
|
|
function normalizeSellerId(value: unknown): string | null {
|
|
const sellerId = String(value ?? "")
|
|
.trim()
|
|
.toUpperCase();
|
|
return sellerId.length > 0 ? sellerId : null;
|
|
}
|
|
|
|
function extractString(value: unknown): string | null {
|
|
if (value == null) return null;
|
|
const text = String(value).trim();
|
|
return text.length > 0 ? text : null;
|
|
}
|
|
|
|
function extractNumber(value: unknown): number | null {
|
|
if (typeof value === "number" && Number.isFinite(value)) return value;
|
|
if (typeof value !== "string") return null;
|
|
const parsed = Number(value.trim().replace(/[$,%]/g, "").replace(/,/g, ""));
|
|
return Number.isFinite(parsed) ? parsed : null;
|
|
}
|
|
|
|
function extractBoolean(value: unknown): boolean | null {
|
|
if (typeof value === "boolean") return value;
|
|
if (typeof value === "number")
|
|
return value === 1 ? true : value === 0 ? false : null;
|
|
if (typeof value !== "string") return null;
|
|
const normalized = value.trim().toLowerCase();
|
|
if (["1", "true", "yes"].includes(normalized)) return true;
|
|
if (["0", "false", "no"].includes(normalized)) return false;
|
|
return null;
|
|
}
|
|
|
|
function normalizeHeader(value: string): string {
|
|
return value
|
|
.toLowerCase()
|
|
.trim()
|
|
.replace(/[^a-z0-9]/g, "");
|
|
}
|
|
|
|
function readFlagValue(args: string[], flag: string): string | undefined {
|
|
const index = args.indexOf(flag);
|
|
if (index === -1) return undefined;
|
|
return args[index + 1];
|
|
}
|
|
|
|
function hasFlag(args: string[], flag: string): boolean {
|
|
return args.includes(flag);
|
|
}
|
|
|
|
function printUsageAndExit(message: string): never {
|
|
console.error(message);
|
|
console.error(
|
|
"Usage: bun run stalker --input input/asins.xlsx [--db db/results.db] [--max-asins N] [--offer-limit 100] [--seller-limit 30] [--inventory-limit 200] [--storefront-update-hours 168] [--seller-cache-hours 168] [--max-seller-requests N] [--sellability] [--analyze-sellable] [--include-stock] [--dry-run] [--no-resume] [--claude]",
|
|
);
|
|
process.exit(1);
|
|
}
|
|
|
|
function updateTokenState(data: KeepaApiResponse): void {
|
|
if (data.tokensLeft != null) tokensLeft = data.tokensLeft;
|
|
if (data.refillRate != null) refillRate = data.refillRate;
|
|
}
|
|
|
|
async function waitForToken(): Promise<void> {
|
|
if (tokensLeft > 0) return;
|
|
|
|
const elapsed = (Date.now() - lastRequestTime) / 60_000;
|
|
const regenerated = Math.floor(elapsed * Math.max(1, refillRate));
|
|
if (regenerated > 0) {
|
|
tokensLeft += regenerated;
|
|
return;
|
|
}
|
|
|
|
const waitMs =
|
|
Math.ceil((1 / Math.max(1, refillRate)) * 60_000) -
|
|
(Date.now() - lastRequestTime);
|
|
if (waitMs > 0) {
|
|
console.log(
|
|
`Keepa tokens exhausted. Waiting ${Math.ceil(waitMs / 1000)}s for token regeneration...`,
|
|
);
|
|
await wait(waitMs);
|
|
}
|
|
tokensLeft = 1;
|
|
}
|
|
|
|
function computeWaitMsFromRefill(refillIn?: number): number {
|
|
if (
|
|
typeof refillIn === "number" &&
|
|
Number.isFinite(refillIn) &&
|
|
refillIn >= 0
|
|
) {
|
|
return Math.max(
|
|
Math.ceil(refillIn) + KEEP_RETRY_BUFFER_MS,
|
|
KEEP_RETRY_BUFFER_MS,
|
|
);
|
|
}
|
|
|
|
return (
|
|
Math.ceil((1 / Math.max(1, refillRate)) * 60_000) + KEEP_RETRY_BUFFER_MS
|
|
);
|
|
}
|
|
|
|
function parseErrorPayload(text: string): KeepaApiResponse | null {
|
|
try {
|
|
const parsed = JSON.parse(text) as KeepaApiResponse;
|
|
return parsed && typeof parsed === "object" ? parsed : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function wait(ms: number): Promise<void> {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
|
|
if (import.meta.main) {
|
|
const args = parseArgs();
|
|
runStalker(args)
|
|
.then((stats) => {
|
|
console.log(
|
|
`Stalker complete: scanned=${stats.scannedAsins}, matched_sellers=${stats.matchedSellers}, persisted_inventory_asins=${stats.persistedInventoryAsins}, failed=${stats.failedAsins}`,
|
|
);
|
|
})
|
|
.catch((error) => {
|
|
console.error(error instanceof Error ? error.message : String(error));
|
|
process.exitCode = 1;
|
|
})
|
|
.finally(() => {
|
|
closeDb();
|
|
});
|
|
}
|