feat: add support for Claude LLM integration across multiple modules
- Introduced `useClaude` option in `AnalysisPipelineOptions` to toggle Claude LLM usage. - Updated `processProductChunk` and `analyzeProducts` functions to accept and handle `useClaude` parameter. - Modified argument parsing in various scripts (`bestsellers-by-category`, `mid-range-sellers-by-category`, `top-monthly-sold-by-category`, etc.) to include `--claude` flag. - Enhanced `analyzeProductsInternal` to differentiate between LLM providers and handle requests to Claude API. - Added error handling for Claude API responses and ensured proper configuration for using Claude. - Updated documentation and usage messages to reflect the new `--claude` flag.
This commit is contained in:
Victor Noguera
231
src/stalker.ts
231
src/stalker.ts
@@ -41,6 +41,7 @@ export type StalkerArgs = {
|
||||
maxSellerRequests: number | null;
|
||||
sellability: boolean;
|
||||
analyzeSellable: boolean;
|
||||
useClaude: boolean;
|
||||
};
|
||||
|
||||
export type StalkerOffer = {
|
||||
@@ -143,8 +144,12 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs {
|
||||
const storefrontUpdateHours = storefrontUpdateRaw
|
||||
? Number(storefrontUpdateRaw)
|
||||
: DEFAULT_STOREFRONT_UPDATE_HOURS;
|
||||
const offerLimit = offerLimitRaw ? Number(offerLimitRaw) : DEFAULT_OFFER_LIMIT;
|
||||
const sellerLimit = sellerLimitRaw ? Number(sellerLimitRaw) : DEFAULT_SELLER_LIMIT;
|
||||
const offerLimit = offerLimitRaw
|
||||
? Number(offerLimitRaw)
|
||||
: DEFAULT_OFFER_LIMIT;
|
||||
const sellerLimit = sellerLimitRaw
|
||||
? Number(sellerLimitRaw)
|
||||
: DEFAULT_SELLER_LIMIT;
|
||||
const inventoryLimit = inventoryLimitRaw
|
||||
? Number(inventoryLimitRaw)
|
||||
: DEFAULT_INVENTORY_LIMIT;
|
||||
@@ -159,6 +164,7 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs {
|
||||
const resume = !hasFlag(argv, "--no-resume");
|
||||
const sellability = hasFlag(argv, "--sellability");
|
||||
const analyzeSellable = hasFlag(argv, "--analyze-sellable");
|
||||
const useClaude = hasFlag(argv, "--claude");
|
||||
|
||||
if (analyzeSellable && !sellability) {
|
||||
printUsageAndExit("--analyze-sellable requires --sellability.");
|
||||
@@ -168,10 +174,7 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs {
|
||||
printUsageAndExit("--max-asins must be a positive integer.");
|
||||
}
|
||||
|
||||
if (
|
||||
!Number.isInteger(storefrontUpdateHours) ||
|
||||
storefrontUpdateHours < 0
|
||||
) {
|
||||
if (!Number.isInteger(storefrontUpdateHours) || storefrontUpdateHours < 0) {
|
||||
printUsageAndExit(
|
||||
"--storefront-update-hours must be a non-negative integer.",
|
||||
);
|
||||
@@ -215,6 +218,7 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs {
|
||||
maxSellerRequests,
|
||||
sellability,
|
||||
analyzeSellable,
|
||||
useClaude,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -232,9 +236,13 @@ export function readAsinsFromXlsx(filePath: string): string[] {
|
||||
if (rows.length === 0) throw new Error("File contains no data rows");
|
||||
|
||||
const headers = Object.keys(rows[0]!);
|
||||
const asinColumn = headers.find((header) => normalizeHeader(header) === "asin");
|
||||
const asinColumn = headers.find(
|
||||
(header) => normalizeHeader(header) === "asin",
|
||||
);
|
||||
if (!asinColumn) {
|
||||
throw new Error(`No ASIN column found. Available columns: ${headers.join(", ")}`);
|
||||
throw new Error(
|
||||
`No ASIN column found. Available columns: ${headers.join(", ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
return extractAsinsFromRows(rows, asinColumn);
|
||||
@@ -287,7 +295,9 @@ export function extractLiveOfferSellerCandidates(
|
||||
offerPrice: extractOfferPrice(offer),
|
||||
condition: extractString(offer.condition ?? offer.conditionComment),
|
||||
isFba: extractBoolean(offer.isFBA ?? offer.isFba ?? offer.fba),
|
||||
stock: extractNumber(offer.stock ?? offer.stockCount ?? offer.currentStock),
|
||||
stock: extractNumber(
|
||||
offer.stock ?? offer.stockCount ?? offer.currentStock,
|
||||
),
|
||||
rawOffer: offer,
|
||||
});
|
||||
}
|
||||
@@ -305,7 +315,9 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
|
||||
|
||||
initDb(args.dbPath);
|
||||
const database = getDb(args.dbPath);
|
||||
const completedAsins = args.resume ? loadPreviouslyScannedAsins(database) : new Set<string>();
|
||||
const completedAsins = args.resume
|
||||
? loadPreviouslyScannedAsins(database)
|
||||
: new Set<string>();
|
||||
const resumeFilteredAsins = cappedAsins.filter(
|
||||
(asin) => !completedAsins.has(asin),
|
||||
);
|
||||
@@ -341,24 +353,32 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
|
||||
|
||||
try {
|
||||
if (args.dryRun) {
|
||||
console.log("Stalker dry-run: product and seller metadata will be fetched, storefronts will not be fetched or persisted.");
|
||||
console.log(
|
||||
"Stalker dry-run: product and seller metadata will be fetched, storefronts will not be fetched or persisted.",
|
||||
);
|
||||
}
|
||||
if (stats.skippedAsins > 0) {
|
||||
console.log(`Stalker resume: skipped ${stats.skippedAsins} previously scanned ASIN(s).`);
|
||||
console.log(
|
||||
`Stalker resume: skipped ${stats.skippedAsins} previously scanned ASIN(s).`,
|
||||
);
|
||||
}
|
||||
|
||||
for (const asin of resumeFilteredAsins) {
|
||||
console.log(`Stalker: scanning ${asin} (${stats.scannedAsins + 1}/${resumeFilteredAsins.length})`);
|
||||
console.log(
|
||||
`Stalker: scanning ${asin} (${stats.scannedAsins + 1}/${resumeFilteredAsins.length})`,
|
||||
);
|
||||
|
||||
const result = await scanAsin(asin, args, apiKey, context).catch((error) => ({
|
||||
asin,
|
||||
title: null,
|
||||
offerCount: 0,
|
||||
candidateSellerCount: 0,
|
||||
matchedSellers: [],
|
||||
product: null,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
}));
|
||||
const result = await scanAsin(asin, args, apiKey, context).catch(
|
||||
(error) => ({
|
||||
asin,
|
||||
title: null,
|
||||
offerCount: 0,
|
||||
candidateSellerCount: 0,
|
||||
matchedSellers: [],
|
||||
product: null,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
}),
|
||||
);
|
||||
|
||||
if (args.sellability && !args.dryRun) {
|
||||
await enrichInventorySellability(result, stats);
|
||||
@@ -379,7 +399,13 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
|
||||
analysisRunId != null &&
|
||||
sellableAsins.length > 0
|
||||
) {
|
||||
await runSellableAnalysisChild(args.dbPath, runId, analysisRunId, sellableAsins);
|
||||
await runSellableAnalysisChild(
|
||||
args.dbPath,
|
||||
runId,
|
||||
analysisRunId,
|
||||
sellableAsins,
|
||||
args.useClaude,
|
||||
);
|
||||
}
|
||||
stats.scannedAsins += 1;
|
||||
stats.matchedSellers += result.matchedSellers.length;
|
||||
@@ -398,7 +424,9 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
|
||||
);
|
||||
|
||||
if (stats.stoppedEarly) {
|
||||
console.log("Stalker: stopping early because max seller request budget was reached.");
|
||||
console.log(
|
||||
"Stalker: stopping early because max seller request budget was reached.",
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -423,12 +451,7 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
if (!args.dryRun && runId != null) {
|
||||
finishStalkerRunWithError(
|
||||
database,
|
||||
runId,
|
||||
stats,
|
||||
message,
|
||||
);
|
||||
finishStalkerRunWithError(database, runId, stats, message);
|
||||
}
|
||||
if (!args.dryRun && analysisRunId != null) {
|
||||
finishStalkerAnalysisRun(database, analysisRunId, "failed", message);
|
||||
@@ -549,12 +572,11 @@ async function enrichInventorySellability(
|
||||
}
|
||||
|
||||
for (const item of items) {
|
||||
item.sellability =
|
||||
sellabilityMap.get(item.asin) ?? {
|
||||
canSell: null,
|
||||
sellabilityStatus: "unknown",
|
||||
sellabilityReason: "Sellability check returned no result",
|
||||
};
|
||||
item.sellability = sellabilityMap.get(item.asin) ?? {
|
||||
canSell: null,
|
||||
sellabilityStatus: "unknown",
|
||||
sellabilityReason: "Sellability check returned no result",
|
||||
};
|
||||
}
|
||||
|
||||
for (const seller of sellers) {
|
||||
@@ -571,14 +593,19 @@ async function enrichInventoryProductDetails(
|
||||
result: StalkerAsinResult,
|
||||
apiKey: string,
|
||||
): Promise<void> {
|
||||
const items = result.matchedSellers.flatMap(({ seller }) => seller.storefrontItems);
|
||||
const items = result.matchedSellers.flatMap(
|
||||
({ seller }) => seller.storefrontItems,
|
||||
);
|
||||
const uniqueAsins = Array.from(new Set(items.map((item) => item.asin)));
|
||||
if (uniqueAsins.length === 0) return;
|
||||
|
||||
console.log(
|
||||
`Stalker inventory details: fetching Keepa product details for ${uniqueAsins.length} sellable ASIN(s)...`,
|
||||
);
|
||||
const detailsByAsin = await fetchKeepaInventoryProductDetails(apiKey, uniqueAsins);
|
||||
const detailsByAsin = await fetchKeepaInventoryProductDetails(
|
||||
apiKey,
|
||||
uniqueAsins,
|
||||
);
|
||||
|
||||
for (const item of items) {
|
||||
item.productDetails = detailsByAsin.get(item.asin) ?? null;
|
||||
@@ -761,7 +788,8 @@ function canSpendSellerRequests(
|
||||
): boolean {
|
||||
if (args.maxSellerRequests == null) return true;
|
||||
const spent =
|
||||
context.stats.sellerMetadataRequests + context.stats.sellerStorefrontRequests;
|
||||
context.stats.sellerMetadataRequests +
|
||||
context.stats.sellerStorefrontRequests;
|
||||
if (spent + nextRequests <= args.maxSellerRequests) return true;
|
||||
context.stats.stoppedEarly = true;
|
||||
return false;
|
||||
@@ -856,7 +884,8 @@ function upsertAsinScan(
|
||||
`SELECT id FROM stalker_asin_scans WHERE run_id = ? AND source_asin = ?`,
|
||||
)
|
||||
.get(runId, result.asin) as { id: number } | null;
|
||||
if (!row) throw new Error(`Failed to load stalker scan row for ${result.asin}`);
|
||||
if (!row)
|
||||
throw new Error(`Failed to load stalker scan row for ${result.asin}`);
|
||||
return row.id;
|
||||
}
|
||||
|
||||
@@ -978,7 +1007,9 @@ function upsertSellerInventory(
|
||||
item.sellability?.sellabilityReason ?? null,
|
||||
item.productDetails?.title ?? null,
|
||||
item.productDetails?.brand ?? null,
|
||||
item.productDetails ? JSON.stringify(item.productDetails.categoryTree) : null,
|
||||
item.productDetails
|
||||
? JSON.stringify(item.productDetails.categoryTree)
|
||||
: null,
|
||||
item.productDetails?.currentPrice ?? null,
|
||||
item.productDetails?.avgPrice90 ?? null,
|
||||
item.productDetails?.salesRank ?? null,
|
||||
@@ -989,7 +1020,9 @@ function upsertSellerInventory(
|
||||
: item.productDetails.amazonIsSeller
|
||||
? 1
|
||||
: 0,
|
||||
item.productDetails ? JSON.stringify(item.productDetails.rawProduct) : null,
|
||||
item.productDetails
|
||||
? JSON.stringify(item.productDetails.rawProduct)
|
||||
: null,
|
||||
fetchedAt,
|
||||
JSON.stringify(item.rawInventory),
|
||||
);
|
||||
@@ -1012,7 +1045,10 @@ function startStalkerRun(
|
||||
return result.lastInsertRowid as number;
|
||||
}
|
||||
|
||||
function startStalkerAnalysisRun(database: Database, inputFile: string): number {
|
||||
function startStalkerAnalysisRun(
|
||||
database: Database,
|
||||
inputFile: string,
|
||||
): number {
|
||||
const result = database
|
||||
.prepare(
|
||||
`INSERT INTO category_analysis_runs (
|
||||
@@ -1231,18 +1267,24 @@ function normalizeSellerResponse(
|
||||
if (!sellers) return [];
|
||||
if (Array.isArray(sellers)) {
|
||||
return sellers
|
||||
.map((seller) => [
|
||||
normalizeSellerId(seller.sellerId ?? seller.sellerID ?? seller.id),
|
||||
seller,
|
||||
] as [string | null, Record<string, any>])
|
||||
.map(
|
||||
(seller) =>
|
||||
[
|
||||
normalizeSellerId(seller.sellerId ?? seller.sellerID ?? seller.id),
|
||||
seller,
|
||||
] as [string | null, Record<string, any>],
|
||||
)
|
||||
.filter((entry): entry is [string, Record<string, any>] => !!entry[0]);
|
||||
}
|
||||
|
||||
return Object.entries(sellers)
|
||||
.map(([sellerId, seller]) => [
|
||||
normalizeSellerId(sellerId),
|
||||
seller,
|
||||
] as [string | null, Record<string, any>])
|
||||
.map(
|
||||
([sellerId, seller]) =>
|
||||
[normalizeSellerId(sellerId), seller] as [
|
||||
string | null,
|
||||
Record<string, any>,
|
||||
],
|
||||
)
|
||||
.filter((entry): entry is [string, Record<string, any>] => !!entry[0]);
|
||||
}
|
||||
|
||||
@@ -1253,14 +1295,15 @@ function parseSeller(
|
||||
): StalkerSeller {
|
||||
const allStorefrontItems = extractStorefrontItems(seller);
|
||||
const storefrontItems =
|
||||
inventoryLimit === 0
|
||||
? []
|
||||
: allStorefrontItems.slice(0, inventoryLimit);
|
||||
inventoryLimit === 0 ? [] : allStorefrontItems.slice(0, inventoryLimit);
|
||||
const storefrontAsins = storefrontItems.map((item) => item.asin);
|
||||
return {
|
||||
sellerId,
|
||||
sellerName: extractString(
|
||||
seller.sellerName ?? seller.name ?? seller.storeName ?? seller.businessName,
|
||||
seller.sellerName ??
|
||||
seller.name ??
|
||||
seller.storeName ??
|
||||
seller.businessName,
|
||||
),
|
||||
rating: extractNumber(
|
||||
seller.currentRating ?? seller.rating ?? seller.feedbackRating,
|
||||
@@ -1279,7 +1322,9 @@ function parseSeller(
|
||||
};
|
||||
}
|
||||
|
||||
function extractStorefrontItems(seller: Record<string, any>): StalkerInventoryItem[] {
|
||||
function extractStorefrontItems(
|
||||
seller: Record<string, any>,
|
||||
): StalkerInventoryItem[] {
|
||||
const candidates = [
|
||||
seller.asinList,
|
||||
seller.asins,
|
||||
@@ -1311,7 +1356,12 @@ function collectStorefrontItems(
|
||||
const asin = normalizeAsin((value as Record<string, unknown>).asin);
|
||||
if (asin && !seen.has(asin)) {
|
||||
seen.add(asin);
|
||||
items.push({ asin, rawInventory: value, sellability: null, productDetails: null });
|
||||
items.push({
|
||||
asin,
|
||||
rawInventory: value,
|
||||
sellability: null,
|
||||
productDetails: null,
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
@@ -1319,7 +1369,12 @@ function collectStorefrontItems(
|
||||
const asin = normalizeAsin(value);
|
||||
if (!asin || seen.has(asin)) return;
|
||||
seen.add(asin);
|
||||
items.push({ asin, rawInventory: { asin }, sellability: null, productDetails: null });
|
||||
items.push({
|
||||
asin,
|
||||
rawInventory: { asin },
|
||||
sellability: null,
|
||||
productDetails: null,
|
||||
});
|
||||
}
|
||||
|
||||
function parseInventoryProductDetails(
|
||||
@@ -1331,9 +1386,9 @@ function parseInventoryProductDetails(
|
||||
title: extractString(product.title),
|
||||
brand: extractString(product.brand ?? product.manufacturer),
|
||||
categoryTree:
|
||||
product.categoryTree?.map((category: { name?: unknown }) =>
|
||||
extractString(category.name),
|
||||
).filter((name: string | null): name is string => !!name) ?? [],
|
||||
product.categoryTree
|
||||
?.map((category: { name?: unknown }) => extractString(category.name))
|
||||
.filter((name: string | null): name is string => !!name) ?? [],
|
||||
currentPrice: extractCurrentPrice(csv),
|
||||
avgPrice90: stats?.avg?.[0] != null ? stats.avg[0] / 100 : null,
|
||||
salesRank: extractNumber(stats?.current?.[3]),
|
||||
@@ -1371,10 +1426,14 @@ function resolveAmazonIsSeller(
|
||||
stats: Record<string, any> | undefined,
|
||||
csv: unknown,
|
||||
): boolean | null {
|
||||
if (typeof product.isAmazonSeller === "boolean") return product.isAmazonSeller;
|
||||
if (typeof product.isAmazonSeller === "boolean")
|
||||
return product.isAmazonSeller;
|
||||
if (typeof product.availabilityAmazon === "number") {
|
||||
if (product.availabilityAmazon >= 0) return true;
|
||||
if (product.availabilityAmazon === -1 || product.availabilityAmazon === -2) {
|
||||
if (
|
||||
product.availabilityAmazon === -1 ||
|
||||
product.availabilityAmazon === -2
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -1437,21 +1496,27 @@ async function runSellableAnalysisChild(
|
||||
stalkerRunId: number,
|
||||
analysisRunId: number,
|
||||
asins: string[],
|
||||
useClaude: boolean,
|
||||
): Promise<void> {
|
||||
const cmd = [
|
||||
"bun",
|
||||
"run",
|
||||
"src/stalker-analyze.ts",
|
||||
"--db",
|
||||
dbPath,
|
||||
"--stalker-run-id",
|
||||
String(stalkerRunId),
|
||||
"--analysis-run-id",
|
||||
String(analysisRunId),
|
||||
"--asins",
|
||||
asins.join(","),
|
||||
];
|
||||
if (useClaude) {
|
||||
cmd.push("--claude");
|
||||
}
|
||||
|
||||
const child = Bun.spawn({
|
||||
cmd: [
|
||||
"bun",
|
||||
"run",
|
||||
"src/stalker-analyze.ts",
|
||||
"--db",
|
||||
dbPath,
|
||||
"--stalker-run-id",
|
||||
String(stalkerRunId),
|
||||
"--analysis-run-id",
|
||||
String(analysisRunId),
|
||||
"--asins",
|
||||
asins.join(","),
|
||||
],
|
||||
cmd,
|
||||
stdout: "inherit",
|
||||
stderr: "inherit",
|
||||
});
|
||||
@@ -1493,7 +1558,8 @@ function extractNumber(value: unknown): number | null {
|
||||
|
||||
function extractBoolean(value: unknown): boolean | null {
|
||||
if (typeof value === "boolean") return value;
|
||||
if (typeof value === "number") return value === 1 ? true : value === 0 ? false : null;
|
||||
if (typeof value === "number")
|
||||
return value === 1 ? true : value === 0 ? false : null;
|
||||
if (typeof value !== "string") return null;
|
||||
const normalized = value.trim().toLowerCase();
|
||||
if (["1", "true", "yes"].includes(normalized)) return true;
|
||||
@@ -1502,7 +1568,10 @@ function extractBoolean(value: unknown): boolean | null {
|
||||
}
|
||||
|
||||
function normalizeHeader(value: string): string {
|
||||
return value.toLowerCase().trim().replace(/[^a-z0-9]/g, "");
|
||||
return value
|
||||
.toLowerCase()
|
||||
.trim()
|
||||
.replace(/[^a-z0-9]/g, "");
|
||||
}
|
||||
|
||||
function readFlagValue(args: string[], flag: string): string | undefined {
|
||||
@@ -1518,7 +1587,7 @@ function hasFlag(args: string[], flag: string): boolean {
|
||||
function printUsageAndExit(message: string): never {
|
||||
console.error(message);
|
||||
console.error(
|
||||
"Usage: bun run stalker --input input/asins.xlsx [--db db/results.db] [--max-asins N] [--offer-limit 100] [--seller-limit 30] [--inventory-limit 200] [--storefront-update-hours 168] [--seller-cache-hours 168] [--max-seller-requests N] [--sellability] [--analyze-sellable] [--include-stock] [--dry-run] [--no-resume]",
|
||||
"Usage: bun run stalker --input input/asins.xlsx [--db db/results.db] [--max-asins N] [--offer-limit 100] [--seller-limit 30] [--inventory-limit 200] [--storefront-update-hours 168] [--seller-cache-hours 168] [--max-seller-requests N] [--sellability] [--analyze-sellable] [--include-stock] [--dry-run] [--no-resume] [--claude]",
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
@@ -1562,7 +1631,9 @@ function computeWaitMsFromRefill(refillIn?: number): number {
|
||||
);
|
||||
}
|
||||
|
||||
return Math.ceil((1 / Math.max(1, refillRate)) * 60_000) + KEEP_RETRY_BUFFER_MS;
|
||||
return (
|
||||
Math.ceil((1 / Math.max(1, refillRate)) * 60_000) + KEEP_RETRY_BUFFER_MS
|
||||
);
|
||||
}
|
||||
|
||||
function parseErrorPayload(text: string): KeepaApiResponse | null {
|
||||
|
||||
Reference in New Issue
Block a user