Compare commits

..

2 Commits

Author SHA1 Message Date
Victor Noguera
0e03366534 Merge branch 'claude' 2026-05-21 19:58:01 -04:00
Victor Noguera
95cebaa27c feat: add support for Claude LLM integration across multiple modules
- Introduced `useClaude` option in `AnalysisPipelineOptions` to toggle Claude LLM usage.
- Updated `processProductChunk` and `analyzeProducts` functions to accept and handle `useClaude` parameter.
- Modified argument parsing in various scripts (`bestsellers-by-category`, `mid-range-sellers-by-category`, `top-monthly-sold-by-category`, etc.) to include `--claude` flag.
- Enhanced `analyzeProductsInternal` to differentiate between LLM providers and handle requests to Claude API.
- Added error handling for Claude API responses and ensured proper configuration for using Claude.
- Updated documentation and usage messages to reflect the new `--claude` flag.
2026-05-21 19:57:46 -04:00
12 changed files with 423 additions and 144 deletions

View File

@@ -12,7 +12,10 @@ AWS_SECRET_ACCESS_KEY=your_aws_secret_access_key
REDIS_URL=redis://localhost:6379 REDIS_URL=redis://localhost:6379
LLM_URL=http://localhost:1234/v1 LLM_URL=http://localhost:1234/v1
LLM_MODEL=default LLM_MODEL=default
ANTHROPIC_API_KEY=your_anthropic_api_key
ANTHROPIC_MODEL=claude-3-5-sonnet-20241022
CACHE_TTL=86400 CACHE_TTL=86400
GOOGLE_API_KEY=your_google_api_key GOOGLE_API_KEY=your_google_api_key
GOOGLE_CSE_ID=your_google_programmable_search_engine_id GOOGLE_CSE_ID=your_google_programmable_search_engine_id
SERPAPI_API_KEY=your_serpapi_api_key_for_google_shopping SERPAPI_API_KEY=your_serpapi_api_key_for_google_shopping

View File

@@ -24,11 +24,14 @@ cp .env.example .env
bun run src/index.ts input/<input.csv|xlsx> [--out output/results.xlsx] bun run src/index.ts input/<input.csv|xlsx> [--out output/results.xlsx]
``` ```
Add `--claude` to use Anthropic Claude instead of local LM Studio for LLM analysis.
Examples: Examples:
```bash ```bash
bun run src/index.ts input/leads.xlsx bun run src/index.ts input/leads.xlsx
bun run src/index.ts input/leads.csv --out output/results.xlsx bun run src/index.ts input/leads.csv --out output/results.xlsx
bun run src/index.ts input/leads.xlsx --claude
``` ```
Large-file behavior: Large-file behavior:
@@ -55,6 +58,14 @@ bun run monthly-sold
bun run mid-range bun run mid-range
``` ```
Use Claude for category LLM analysis:
```bash
bun run bestsellers --claude
bun run monthly-sold --claude
bun run mid-range --claude
```
Mid-range process: Mid-range process:
- Script: `bun run mid-range` - Script: `bun run mid-range`
@@ -128,6 +139,12 @@ curl -X POST "http://localhost:3000/api/upc/lookup" \
-d '{"upcs":["012345678901","098765432109"]}' -d '{"upcs":["012345678901","098765432109"]}'
``` ```
Run the web server with Claude-backed LLM calls:
```bash
bun run start:web -- --claude
```
## Large UPC File Analysis (XLS/XLSX) ## Large UPC File Analysis (XLS/XLSX)
For supplier price lists that contain UPC/EAN values and unit cost, use the For supplier price lists that contain UPC/EAN values and unit cost, use the
@@ -249,7 +266,7 @@ ASIN, Name, Brand, Category, Unit Cost, Current Price, Avg Price 90d, Sales Rank
## Environment variables ## Environment variables
| Variable | Default | Description | | Variable | Default | Description |
| ----------------------- | -------------------------- | ----------------------------------------------------------------------- | | ----------------------- | ---------------------------- | ----------------------------------------------------------------------- |
| `KEEPA_API_KEY` | — | **Required.** Keepa API key | | `KEEPA_API_KEY` | — | **Required.** Keepa API key |
| `SP_API_CLIENT_ID` | — | LWA app client id from Solution Provider Portal | | `SP_API_CLIENT_ID` | — | LWA app client id from Solution Provider Portal |
| `SP_API_CLIENT_SECRET` | — | LWA app client secret from Solution Provider Portal | | `SP_API_CLIENT_SECRET` | — | LWA app client secret from Solution Provider Portal |
@@ -264,6 +281,8 @@ ASIN, Name, Brand, Category, Unit Cost, Current Price, Avg Price 90d, Sales Rank
| `REDIS_URL` | `redis://localhost:6379` | Redis connection URL | | `REDIS_URL` | `redis://localhost:6379` | Redis connection URL |
| `LLM_URL` | `http://localhost:1234/v1` | LM Studio API base URL | | `LLM_URL` | `http://localhost:1234/v1` | LM Studio API base URL |
| `LLM_MODEL` | `default` | Model name to pass to LM Studio | | `LLM_MODEL` | `default` | Model name to pass to LM Studio |
| `ANTHROPIC_API_KEY` | — | Required when running any LLM script with `--claude` |
| `ANTHROPIC_MODEL` | `claude-3-5-sonnet-20241022` | Claude model ID used with `--claude` |
| `CACHE_TTL` | `86400` | Redis cache TTL in seconds | | `CACHE_TTL` | `86400` | Redis cache TTL in seconds |
## Notes ## Notes

View File

@@ -22,6 +22,7 @@ export type AnalysisPipelineOptions = {
llmBatchDelayMs?: number; llmBatchDelayMs?: number;
llmRetryDelayMs?: number; llmRetryDelayMs?: number;
sellability?: SellabilityFilter; sellability?: SellabilityFilter;
useClaude?: boolean;
}; };
export function chunkArray<T>(items: T[], chunkSize: number): T[][] { export function chunkArray<T>(items: T[], chunkSize: number): T[][] {
@@ -60,6 +61,7 @@ export async function processProductChunk(
const llmBatchDelayMs = Math.max(0, options.llmBatchDelayMs ?? 5_000); const llmBatchDelayMs = Math.max(0, options.llmBatchDelayMs ?? 5_000);
const llmRetryDelayMs = Math.max(0, options.llmRetryDelayMs ?? 10_000); const llmRetryDelayMs = Math.max(0, options.llmRetryDelayMs ?? 10_000);
const sellabilityFilter = options.sellability ?? "available"; const sellabilityFilter = options.sellability ?? "available";
const useClaude = options.useClaude === true;
console.log(`\nChecking cache for ${products.length} products...`); console.log(`\nChecking cache for ${products.length} products...`);
const cached = new Map<string, EnrichedProduct>(); const cached = new Map<string, EnrichedProduct>();
@@ -242,6 +244,7 @@ export async function processProductChunk(
try { try {
verdicts = await analyzeProducts(batch, { verdicts = await analyzeProducts(batch, {
ignoreSellability: sellabilityFilter === "all", ignoreSellability: sellabilityFilter === "all",
useClaude,
}); });
} catch { } catch {
if (llmRetryDelayMs > 0) { if (llmRetryDelayMs > 0) {
@@ -250,6 +253,7 @@ export async function processProductChunk(
try { try {
verdicts = await analyzeProducts(batch, { verdicts = await analyzeProducts(batch, {
ignoreSellability: sellabilityFilter === "all", ignoreSellability: sellabilityFilter === "all",
useClaude,
}); });
} catch { } catch {
verdicts = null; verdicts = null;

View File

@@ -26,6 +26,7 @@ type ParsedArgs = {
categoryLimit: number; categoryLimit: number;
perCategoryTop: number; perCategoryTop: number;
blacklistFile: string; blacklistFile: string;
useClaude: boolean;
}; };
type CategoryRunSummary = { type CategoryRunSummary = {
@@ -72,6 +73,7 @@ function log(
function parseArgs(): ParsedArgs { function parseArgs(): ParsedArgs {
const args = process.argv.slice(2); const args = process.argv.slice(2);
const useClaude = hasFlag(args, "--claude");
const outputDir = const outputDir =
readFlagValue(args, "--out-dir") ?? path.join(process.cwd(), "output"); readFlagValue(args, "--out-dir") ?? path.join(process.cwd(), "output");
const blacklistFile = const blacklistFile =
@@ -100,9 +102,14 @@ function parseArgs(): ParsedArgs {
categoryLimit, categoryLimit,
perCategoryTop, perCategoryTop,
blacklistFile, blacklistFile,
useClaude,
}; };
} }
function hasFlag(args: string[], flag: string): boolean {
return args.includes(flag);
}
function readFlagValue(args: string[], flag: string): string | undefined { function readFlagValue(args: string[], flag: string): string | undefined {
const idx = args.indexOf(flag); const idx = args.indexOf(flag);
if (idx === -1) return undefined; if (idx === -1) return undefined;
@@ -118,7 +125,7 @@ function printUsageAndExit(message: string): never {
"error", "error",
[ [
"Usage:", "Usage:",
" bun run src/bestsellers-by-category.ts [--category-limit 32] [--per-category-top 100] [--out-dir output] [--blacklist-file category-blacklist.csv]", " bun run src/bestsellers-by-category.ts [--category-limit 32] [--per-category-top 100] [--out-dir output] [--blacklist-file category-blacklist.csv] [--claude]",
"", "",
"Flow:", "Flow:",
" 1) Discover categories and round-robin selection.", " 1) Discover categories and round-robin selection.",
@@ -1011,6 +1018,7 @@ export async function processCategory(
runId: number, runId: number,
category: CategoryInfo, category: CategoryInfo,
perCategoryTop: number, perCategoryTop: number,
useClaude = false,
): Promise<CategoryRunSummary> { ): Promise<CategoryRunSummary> {
log("info", `\nCategory ${category.label} (${category.id})`); log("info", `\nCategory ${category.label} (${category.id})`);
@@ -1106,7 +1114,7 @@ export async function processCategory(
let batchVerdicts: LlmVerdict[]; let batchVerdicts: LlmVerdict[];
try { try {
batchVerdicts = await analyzeProducts(batch); batchVerdicts = await analyzeProducts(batch, { useClaude });
} catch (err) { } catch (err) {
const message = err instanceof Error ? err.message : String(err); const message = err instanceof Error ? err.message : String(err);
log("warn", ` LLM batch failed: ${message}`); log("warn", ` LLM batch failed: ${message}`);
@@ -1249,6 +1257,7 @@ export async function main(): Promise<void> {
runId, runId,
category, category,
args.perCategoryTop, args.perCategoryTop,
args.useClaude,
); );
totalInsertedAsins += categorySummary.results?.length ?? 0; totalInsertedAsins += categorySummary.results?.length ?? 0;

View File

@@ -20,6 +20,8 @@ export const config = {
redisUrl: optional("REDIS_URL", "redis://localhost:6379"), redisUrl: optional("REDIS_URL", "redis://localhost:6379"),
llmUrl: optional("LLM_URL", "http://localhost:1234/v1"), llmUrl: optional("LLM_URL", "http://localhost:1234/v1"),
llmModel: optional("LLM_MODEL", "default"), llmModel: optional("LLM_MODEL", "default"),
anthropicApiKey: Bun.env.ANTHROPIC_API_KEY,
anthropicModel: Bun.env.ANTHROPIC_MODEL,
cacheTtl: parseInt(optional("CACHE_TTL", "86400"), 10), cacheTtl: parseInt(optional("CACHE_TTL", "86400"), 10),
searxngUrl: optional("SEARXNG_URL", "https://searxng.nvictor.me/"), searxngUrl: optional("SEARXNG_URL", "https://searxng.nvictor.me/"),
searxngTimeoutMs: parseInt(optional("SEARXNG_TIMEOUT_MS", "10000"), 10), searxngTimeoutMs: parseInt(optional("SEARXNG_TIMEOUT_MS", "10000"), 10),

View File

@@ -42,9 +42,11 @@ function parseArgs(): {
inputFile: string; inputFile: string;
outputFile?: string; outputFile?: string;
sellability: SellabilityFilter; sellability: SellabilityFilter;
useClaude: boolean;
} { } {
const args = process.argv.slice(2); const args = process.argv.slice(2);
const outputFile = readFlagValue(args, "--out", "--output"); const outputFile = readFlagValue(args, "--out", "--output");
const useClaude = args.includes("--claude");
const inputFile = readInputFileArg( const inputFile = readInputFileArg(
args, args,
"--out", "--out",
@@ -55,12 +57,12 @@ function parseArgs(): {
if (!inputFile) { if (!inputFile) {
console.error( console.error(
"Usage: bun run src/index.ts <input.csv|xlsx> [--out results.xlsx|--output results.xlsx] [--sellability available|all]", "Usage: bun run src/index.ts <input.csv|xlsx> [--out results.xlsx|--output results.xlsx] [--sellability available|all] [--claude]",
); );
process.exit(1); process.exit(1);
} }
return { inputFile, outputFile, sellability }; return { inputFile, outputFile, sellability, useClaude };
} }
function readFlagValue(args: string[], ...flags: string[]): string | undefined { function readFlagValue(args: string[], ...flags: string[]): string | undefined {
@@ -109,9 +111,10 @@ function resolveBaseOutputPath(inputFile: string, outputFile?: string): string {
} }
async function main() { async function main() {
const { inputFile, outputFile, sellability } = parseArgs(); const { inputFile, outputFile, sellability, useClaude } = parseArgs();
console.log(`Sellability filter: ${sellability}`); console.log(`Sellability filter: ${sellability}`);
console.log(`LLM provider: ${useClaude ? "claude" : "local"}`);
console.log("Connecting to Redis..."); console.log("Connecting to Redis...");
await connectCache(); await connectCache();
@@ -144,7 +147,10 @@ async function main() {
console.log( console.log(
`\n=== Input chunk ${chunkIndex + 1}/${productChunks.length} (${chunk.length} products) ===`, `\n=== Input chunk ${chunkIndex + 1}/${productChunks.length} (${chunk.length} products) ===`,
); );
const chunkResults = await processProductChunk(chunk, { sellability }); const chunkResults = await processProductChunk(chunk, {
sellability,
useClaude,
});
allResults.push(...chunkResults); allResults.push(...chunkResults);
} }

View File

@@ -56,6 +56,17 @@ Keep each reasoning under 100 characters to stay within output limits and mentio
type AnalyzeProductsOptions = { type AnalyzeProductsOptions = {
ignoreSellability?: boolean; ignoreSellability?: boolean;
useClaude?: boolean;
};
type LlmProvider = "lm-studio" | "claude";
type LmStudioResponse = {
choices?: { message?: { content?: string } }[];
};
type ClaudeResponse = {
content?: Array<{ type?: string; text?: string }>;
}; };
function getSystemPrompt(options: AnalyzeProductsOptions): string { function getSystemPrompt(options: AnalyzeProductsOptions): string {
@@ -72,8 +83,7 @@ export async function analyzeProducts(
try { try {
return await analyzeProductsInternal(products, options); return await analyzeProductsInternal(products, options);
} catch (err) { } catch (err) {
const msg = String(err); if (products.length > 1 && isContextOverflowError(err)) {
if (products.length > 1 && msg.includes("Context size has been exceeded")) {
console.warn( console.warn(
`LLM context exceeded for batch of ${products.length}, retrying one product at a time...`, `LLM context exceeded for batch of ${products.length}, retrying one product at a time...`,
); );
@@ -113,7 +123,43 @@ async function analyzeProductsInternal(
summarizeForLlm(p, options.ignoreSellability === true), summarizeForLlm(p, options.ignoreSellability === true),
); );
const systemPrompt = getSystemPrompt(options); const systemPrompt = getSystemPrompt(options);
const provider = options.useClaude ? "claude" : "lm-studio";
const content = await requestLlmContent(
provider,
systemPrompt,
productSummaries,
);
return parseVerdicts(content, products);
}
function isContextOverflowError(err: unknown): boolean {
const msg = String(err).toLowerCase();
return (
msg.includes("context size has been exceeded") ||
msg.includes("prompt is too long") ||
msg.includes("too many tokens") ||
msg.includes("maximum context") ||
msg.includes("context length") ||
msg.includes("max_tokens")
);
}
async function requestLlmContent(
provider: LlmProvider,
systemPrompt: string,
productSummaries: ReturnType<typeof summarizeForLlm>[],
): Promise<string> {
if (provider === "claude") {
return requestClaudeContent(systemPrompt, productSummaries);
}
return requestLmStudioContent(systemPrompt, productSummaries);
}
async function requestLmStudioContent(
systemPrompt: string,
productSummaries: ReturnType<typeof summarizeForLlm>[],
): Promise<string> {
const res = await fetch(`${config.llmUrl}/chat/completions`, { const res = await fetch(`${config.llmUrl}/chat/completions`, {
method: "POST", method: "POST",
headers: { headers: {
@@ -132,15 +178,79 @@ async function analyzeProductsInternal(
}); });
if (!res.ok) { if (!res.ok) {
throw new Error(`LLM API error ${res.status}: ${await res.text()}`); throw new Error(`LLM API error ${res.status}: ${await readErrorBody(res)}`);
} }
const data = (await res.json()) as { const data = (await res.json()) as LmStudioResponse;
choices?: { message?: { content?: string } }[]; return data.choices?.[0]?.message?.content ?? "";
}; }
const content = data.choices?.[0]?.message?.content ?? "";
return parseVerdicts(content, products); async function requestClaudeContent(
systemPrompt: string,
productSummaries: ReturnType<typeof summarizeForLlm>[],
): Promise<string> {
if (!config.anthropicApiKey) {
throw new Error(
"Missing required env var for --claude mode: ANTHROPIC_API_KEY",
);
}
const res = await fetch("https://api.anthropic.com/v1/messages", {
method: "POST",
headers: {
"Content-Type": "application/json",
"x-api-key": config.anthropicApiKey,
"anthropic-version": "2023-06-01",
},
body: JSON.stringify({
model: config.anthropicModel,
system: systemPrompt,
messages: [
{ role: "user", content: JSON.stringify(productSummaries, null, 2) },
],
temperature: 0.3,
max_tokens: 2048,
}),
});
if (!res.ok) {
throw new Error(
`Claude API error ${res.status}: ${await readErrorBody(res)}`,
);
}
const data = (await res.json()) as ClaudeResponse;
if (!Array.isArray(data.content)) {
return "";
}
return data.content
.filter((block) => block?.type === "text" && typeof block.text === "string")
.map((block) => block.text ?? "")
.join("\n");
}
async function readErrorBody(response: Response): Promise<string> {
const text = await response.text();
if (!text.trim()) return "No response body";
try {
const parsed = JSON.parse(text) as {
error?: { message?: string; type?: string };
};
const type = parsed.error?.type?.trim();
const message = parsed.error?.message?.trim();
if (type && message) {
return `${type}: ${message}`;
}
if (message) {
return message;
}
} catch {
// Response was plain text.
}
return text;
} }
function summarizeForLlm(p: EnrichedProduct, ignoreSellability: boolean) { function summarizeForLlm(p: EnrichedProduct, ignoreSellability: boolean) {

View File

@@ -34,6 +34,7 @@ type ParsedArgs = {
selectCategories: boolean; selectCategories: boolean;
categoryIds: number[]; categoryIds: number[];
sellabilityGate: "strict" | "soft" | "off"; sellabilityGate: "strict" | "soft" | "off";
useClaude: boolean;
outputDir: string; outputDir: string;
categoryLimit: number; categoryLimit: number;
perCategoryTop: number; perCategoryTop: number;
@@ -118,6 +119,7 @@ function parseArgs(): ParsedArgs {
const args = process.argv.slice(2); const args = process.argv.slice(2);
const listCategories = hasFlag(args, "--list-categories"); const listCategories = hasFlag(args, "--list-categories");
const selectCategories = hasFlag(args, "--select-categories"); const selectCategories = hasFlag(args, "--select-categories");
const useClaude = hasFlag(args, "--claude");
const categoryIdsRaw = readFlagValue(args, "--category-ids"); const categoryIdsRaw = readFlagValue(args, "--category-ids");
const sellabilityGateRaw = readFlagValue(args, "--sellability-gate"); const sellabilityGateRaw = readFlagValue(args, "--sellability-gate");
const outputDir = const outputDir =
@@ -312,6 +314,7 @@ function parseArgs(): ParsedArgs {
selectCategories, selectCategories,
categoryIds, categoryIds,
sellabilityGate, sellabilityGate,
useClaude,
outputDir, outputDir,
categoryLimit, categoryLimit,
perCategoryTop, perCategoryTop,
@@ -370,7 +373,7 @@ function printUsageAndExit(message: string): never {
"error", "error",
[ [
"Usage:", "Usage:",
" bun run src/mid-range-sellers-by-category.ts [--category-limit 32] [--list-categories] [--select-categories] [--category-ids 281053,172282] [--sellability-gate soft] [--per-category-top 100] [--category-candidate-pool 500] [--candidate-batch-size 60] [--min-monthly-sold 100] [--max-monthly-sold 1000] [--min-price 15] [--max-price 200] [--min-seller-count 3] [--max-seller-count 20] [--min-amazon-buybox-share-pct 15] [--max-amazon-buybox-share-pct 85] [--max-asins-analyzed 250] [--max-keepa-products-fetched 500] [--out-dir output] [--blacklist-file category-blacklist.csv]", " bun run src/mid-range-sellers-by-category.ts [--category-limit 32] [--list-categories] [--select-categories] [--category-ids 281053,172282] [--sellability-gate soft] [--per-category-top 100] [--category-candidate-pool 500] [--candidate-batch-size 60] [--min-monthly-sold 100] [--max-monthly-sold 1000] [--min-price 15] [--max-price 200] [--min-seller-count 3] [--max-seller-count 20] [--min-amazon-buybox-share-pct 15] [--max-amazon-buybox-share-pct 85] [--max-asins-analyzed 250] [--max-keepa-products-fetched 500] [--out-dir output] [--blacklist-file category-blacklist.csv] [--claude]",
"", "",
"Selection:", "Selection:",
" --list-categories Discover and print runnable categories, then exit.", " --list-categories Discover and print runnable categories, then exit.",
@@ -1482,6 +1485,7 @@ export async function processCategory(
minAmazonBuyboxSharePct: number, minAmazonBuyboxSharePct: number,
maxAmazonBuyboxSharePct: number, maxAmazonBuyboxSharePct: number,
sellabilityGate: "strict" | "soft" | "off", sellabilityGate: "strict" | "soft" | "off",
useClaude = false,
runtimeBudget?: RuntimeBudget, runtimeBudget?: RuntimeBudget,
candidateBatchSize = DEFAULT_CANDIDATE_BATCH_SIZE, candidateBatchSize = DEFAULT_CANDIDATE_BATCH_SIZE,
): Promise<CategoryRunSummary> { ): Promise<CategoryRunSummary> {
@@ -1739,7 +1743,7 @@ export async function processCategory(
let batchVerdicts: LlmVerdict[]; let batchVerdicts: LlmVerdict[];
try { try {
batchVerdicts = await analyzeProducts(batch); batchVerdicts = await analyzeProducts(batch, { useClaude });
} catch (err) { } catch (err) {
const message = err instanceof Error ? err.message : String(err); const message = err instanceof Error ? err.message : String(err);
log("warn", ` LLM batch failed: ${message}`); log("warn", ` LLM batch failed: ${message}`);
@@ -2014,6 +2018,7 @@ export async function main(): Promise<void> {
args.minAmazonBuyboxSharePct, args.minAmazonBuyboxSharePct,
args.maxAmazonBuyboxSharePct, args.maxAmazonBuyboxSharePct,
args.sellabilityGate, args.sellabilityGate,
args.useClaude,
runtimeBudget, runtimeBudget,
args.candidateBatchSize, args.candidateBatchSize,
); );

View File

@@ -103,6 +103,7 @@ const DEFAULT_PAGE_SIZE = 25;
const MAX_PAGE_SIZE = 200; const MAX_PAGE_SIZE = 200;
const ASIN_PATTERN = /^[A-Z0-9]{10}$/; const ASIN_PATTERN = /^[A-Z0-9]{10}$/;
const MAX_UPCS_PER_REQUEST = 1000; const MAX_UPCS_PER_REQUEST = 1000;
const USE_CLAUDE = process.argv.includes("--claude");
initDb(DB_PATH); initDb(DB_PATH);
const db = getDb(DB_PATH); const db = getDb(DB_PATH);
@@ -128,7 +129,8 @@ function xlsx(buffer: ArrayBuffer, filename: string): Response {
return new Response(buffer, { return new Response(buffer, {
status: 200, status: 200,
headers: { headers: {
"content-type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "content-type":
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"content-disposition": `attachment; filename="${filename}"`, "content-disposition": `attachment; filename="${filename}"`,
}, },
}); });
@@ -758,7 +760,10 @@ function parseStalkerSort(sortParam: string | null): string {
return parsed return parsed
.replaceAll("runId", "runId") .replaceAll("runId", "runId")
.replaceAll("rating_count", "rating_count") .replaceAll("rating_count", "rating_count")
.replaceAll("persisted_inventory_asin_count", "persisted_inventory_asin_count") .replaceAll(
"persisted_inventory_asin_count",
"persisted_inventory_asin_count",
)
.replaceAll("storefront_asin_total", "storefront_asin_total"); .replaceAll("storefront_asin_total", "storefront_asin_total");
} }
@@ -955,7 +960,11 @@ function parseStalkerProductSort(sortParam: string | null): string {
"confidence", "confidence",
"last_seen_at", "last_seen_at",
]); ]);
return parseSort(sortParam, allowedSort, "monthly_sold DESC, last_seen_at DESC, asin ASC"); return parseSort(
sortParam,
allowedSort,
"monthly_sold DESC, last_seen_at DESC, asin ASC",
);
} }
function getStalkerProducts(filters: URLSearchParams) { function getStalkerProducts(filters: URLSearchParams) {
@@ -1036,7 +1045,9 @@ function getStalkerProducts(filters: URLSearchParams) {
}; };
} }
function getStalkerProductsForExport(filters: URLSearchParams): StalkerProductRecord[] { function getStalkerProductsForExport(
filters: URLSearchParams,
): StalkerProductRecord[] {
const { where, params } = parseStalkerProductFilters(filters); const { where, params } = parseStalkerProductFilters(filters);
const orderBy = parseStalkerProductSort(filters.get("sort")); const orderBy = parseStalkerProductSort(filters.get("sort"));
@@ -1100,7 +1111,12 @@ function exportStalkerProductsXlsx(filters: URLSearchParams): Response {
Category: parseCategoryTreeForExport(row.category_tree), Category: parseCategoryTreeForExport(row.category_tree),
"Monthly Sold": row.monthly_sold ?? null, "Monthly Sold": row.monthly_sold ?? null,
Sellers: row.seller_count ?? null, Sellers: row.seller_count ?? null,
"Amazon Seller": row.amazon_is_seller == null ? "" : row.amazon_is_seller === 1 ? "Yes" : "No", "Amazon Seller":
row.amazon_is_seller == null
? ""
: row.amazon_is_seller === 1
? "Yes"
: "No",
"Sales Rank": row.sales_rank ?? null, "Sales Rank": row.sales_rank ?? null,
"Current Price": row.current_price ?? null, "Current Price": row.current_price ?? null,
"Avg 90d": row.avg_price_90d ?? null, "Avg 90d": row.avg_price_90d ?? null,
@@ -1155,11 +1171,31 @@ function exportStalkerProductsXlsx(filters: URLSearchParams): Response {
function purgeStalkerData() { function purgeStalkerData() {
const counts = { const counts = {
inventory: (db.query("SELECT COUNT(*) AS count FROM stalker_seller_inventory").get() as { count: number }).count, inventory: (
asinSellers: (db.query("SELECT COUNT(*) AS count FROM stalker_asin_sellers").get() as { count: number }).count, db
sellers: (db.query("SELECT COUNT(*) AS count FROM stalker_sellers").get() as { count: number }).count, .query("SELECT COUNT(*) AS count FROM stalker_seller_inventory")
scans: (db.query("SELECT COUNT(*) AS count FROM stalker_asin_scans").get() as { count: number }).count, .get() as { count: number }
runs: (db.query("SELECT COUNT(*) AS count FROM stalker_runs").get() as { count: number }).count, ).count,
asinSellers: (
db.query("SELECT COUNT(*) AS count FROM stalker_asin_sellers").get() as {
count: number;
}
).count,
sellers: (
db.query("SELECT COUNT(*) AS count FROM stalker_sellers").get() as {
count: number;
}
).count,
scans: (
db.query("SELECT COUNT(*) AS count FROM stalker_asin_scans").get() as {
count: number;
}
).count,
runs: (
db.query("SELECT COUNT(*) AS count FROM stalker_runs").get() as {
count: number;
}
).count,
}; };
db.transaction(() => { db.transaction(() => {
@@ -1683,7 +1719,9 @@ async function reanalyzeSingleAsin(
fetchedAt: new Date().toISOString(), fetchedAt: new Date().toISOString(),
}; };
const verdicts = await analyzeProducts([enriched]); const verdicts = await analyzeProducts([enriched], {
useClaude: USE_CLAUDE,
});
const verdict = verdicts[0] ?? { const verdict = verdicts[0] ?? {
asin, asin,
verdict: "SKIP" as const, verdict: "SKIP" as const,

View File

@@ -17,6 +17,7 @@ type Args = {
stalkerRunId: number; stalkerRunId: number;
analysisRunId: number; analysisRunId: number;
asins: string[]; asins: string[];
useClaude: boolean;
}; };
type InventoryRow = { type InventoryRow = {
@@ -45,6 +46,7 @@ function parseArgs(argv = process.argv.slice(2)): Args {
const dbPath = readFlagValue(argv, "--db"); const dbPath = readFlagValue(argv, "--db");
const stalkerRunId = Number(readFlagValue(argv, "--stalker-run-id")); const stalkerRunId = Number(readFlagValue(argv, "--stalker-run-id"));
const analysisRunId = Number(readFlagValue(argv, "--analysis-run-id")); const analysisRunId = Number(readFlagValue(argv, "--analysis-run-id"));
const useClaude = argv.includes("--claude");
const asins = (readFlagValue(argv, "--asins") ?? "") const asins = (readFlagValue(argv, "--asins") ?? "")
.split(",") .split(",")
.map((asin) => asin.trim().toUpperCase()) .map((asin) => asin.trim().toUpperCase())
@@ -59,7 +61,7 @@ function parseArgs(argv = process.argv.slice(2)): Args {
} }
if (asins.length === 0) throw new Error("Missing --asins"); if (asins.length === 0) throw new Error("Missing --asins");
return { dbPath, stalkerRunId, analysisRunId, asins }; return { dbPath, stalkerRunId, analysisRunId, asins, useClaude };
} }
function wait(ms: number): Promise<void> { function wait(ms: number): Promise<void> {
@@ -299,6 +301,7 @@ function refreshAnalysisRun(database: Database, runId: number): void {
async function analyzeInBatches( async function analyzeInBatches(
products: EnrichedProduct[], products: EnrichedProduct[],
useClaude: boolean,
): Promise<AnalysisResult[]> { ): Promise<AnalysisResult[]> {
const results: AnalysisResult[] = []; const results: AnalysisResult[] = [];
@@ -316,7 +319,7 @@ async function analyzeInBatches(
let verdicts; let verdicts;
try { try {
verdicts = await analyzeProducts(batch); verdicts = await analyzeProducts(batch, { useClaude });
} catch (error) { } catch (error) {
console.warn( console.warn(
`Stalker analysis: LLM batch ${batchNumber} failed: ${ `Stalker analysis: LLM batch ${batchNumber} failed: ${
@@ -358,7 +361,7 @@ async function main(): Promise<void> {
console.log(`Stalker analysis: analyzing ${rows.length} sellable ASIN(s).`); console.log(`Stalker analysis: analyzing ${rows.length} sellable ASIN(s).`);
const enriched = await buildEnrichedProducts(rows); const enriched = await buildEnrichedProducts(rows);
const results = await analyzeInBatches(enriched); const results = await analyzeInBatches(enriched, args.useClaude);
insertProductAnalysisResults(database, args.analysisRunId, results); insertProductAnalysisResults(database, args.analysisRunId, results);
refreshAnalysisRun(database, args.analysisRunId); refreshAnalysisRun(database, args.analysisRunId);
} finally { } finally {

View File

@@ -41,6 +41,7 @@ export type StalkerArgs = {
maxSellerRequests: number | null; maxSellerRequests: number | null;
sellability: boolean; sellability: boolean;
analyzeSellable: boolean; analyzeSellable: boolean;
useClaude: boolean;
}; };
export type StalkerOffer = { export type StalkerOffer = {
@@ -143,8 +144,12 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs {
const storefrontUpdateHours = storefrontUpdateRaw const storefrontUpdateHours = storefrontUpdateRaw
? Number(storefrontUpdateRaw) ? Number(storefrontUpdateRaw)
: DEFAULT_STOREFRONT_UPDATE_HOURS; : DEFAULT_STOREFRONT_UPDATE_HOURS;
const offerLimit = offerLimitRaw ? Number(offerLimitRaw) : DEFAULT_OFFER_LIMIT; const offerLimit = offerLimitRaw
const sellerLimit = sellerLimitRaw ? Number(sellerLimitRaw) : DEFAULT_SELLER_LIMIT; ? Number(offerLimitRaw)
: DEFAULT_OFFER_LIMIT;
const sellerLimit = sellerLimitRaw
? Number(sellerLimitRaw)
: DEFAULT_SELLER_LIMIT;
const inventoryLimit = inventoryLimitRaw const inventoryLimit = inventoryLimitRaw
? Number(inventoryLimitRaw) ? Number(inventoryLimitRaw)
: DEFAULT_INVENTORY_LIMIT; : DEFAULT_INVENTORY_LIMIT;
@@ -159,6 +164,7 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs {
const resume = !hasFlag(argv, "--no-resume"); const resume = !hasFlag(argv, "--no-resume");
const sellability = hasFlag(argv, "--sellability"); const sellability = hasFlag(argv, "--sellability");
const analyzeSellable = hasFlag(argv, "--analyze-sellable"); const analyzeSellable = hasFlag(argv, "--analyze-sellable");
const useClaude = hasFlag(argv, "--claude");
if (analyzeSellable && !sellability) { if (analyzeSellable && !sellability) {
printUsageAndExit("--analyze-sellable requires --sellability."); printUsageAndExit("--analyze-sellable requires --sellability.");
@@ -168,10 +174,7 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs {
printUsageAndExit("--max-asins must be a positive integer."); printUsageAndExit("--max-asins must be a positive integer.");
} }
if ( if (!Number.isInteger(storefrontUpdateHours) || storefrontUpdateHours < 0) {
!Number.isInteger(storefrontUpdateHours) ||
storefrontUpdateHours < 0
) {
printUsageAndExit( printUsageAndExit(
"--storefront-update-hours must be a non-negative integer.", "--storefront-update-hours must be a non-negative integer.",
); );
@@ -215,6 +218,7 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs {
maxSellerRequests, maxSellerRequests,
sellability, sellability,
analyzeSellable, analyzeSellable,
useClaude,
}; };
} }
@@ -232,9 +236,13 @@ export function readAsinsFromXlsx(filePath: string): string[] {
if (rows.length === 0) throw new Error("File contains no data rows"); if (rows.length === 0) throw new Error("File contains no data rows");
const headers = Object.keys(rows[0]!); const headers = Object.keys(rows[0]!);
const asinColumn = headers.find((header) => normalizeHeader(header) === "asin"); const asinColumn = headers.find(
(header) => normalizeHeader(header) === "asin",
);
if (!asinColumn) { if (!asinColumn) {
throw new Error(`No ASIN column found. Available columns: ${headers.join(", ")}`); throw new Error(
`No ASIN column found. Available columns: ${headers.join(", ")}`,
);
} }
return extractAsinsFromRows(rows, asinColumn); return extractAsinsFromRows(rows, asinColumn);
@@ -287,7 +295,9 @@ export function extractLiveOfferSellerCandidates(
offerPrice: extractOfferPrice(offer), offerPrice: extractOfferPrice(offer),
condition: extractString(offer.condition ?? offer.conditionComment), condition: extractString(offer.condition ?? offer.conditionComment),
isFba: extractBoolean(offer.isFBA ?? offer.isFba ?? offer.fba), isFba: extractBoolean(offer.isFBA ?? offer.isFba ?? offer.fba),
stock: extractNumber(offer.stock ?? offer.stockCount ?? offer.currentStock), stock: extractNumber(
offer.stock ?? offer.stockCount ?? offer.currentStock,
),
rawOffer: offer, rawOffer: offer,
}); });
} }
@@ -305,7 +315,9 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
initDb(args.dbPath); initDb(args.dbPath);
const database = getDb(args.dbPath); const database = getDb(args.dbPath);
const completedAsins = args.resume ? loadPreviouslyScannedAsins(database) : new Set<string>(); const completedAsins = args.resume
? loadPreviouslyScannedAsins(database)
: new Set<string>();
const resumeFilteredAsins = cappedAsins.filter( const resumeFilteredAsins = cappedAsins.filter(
(asin) => !completedAsins.has(asin), (asin) => !completedAsins.has(asin),
); );
@@ -341,16 +353,23 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
try { try {
if (args.dryRun) { if (args.dryRun) {
console.log("Stalker dry-run: product and seller metadata will be fetched, storefronts will not be fetched or persisted."); console.log(
"Stalker dry-run: product and seller metadata will be fetched, storefronts will not be fetched or persisted.",
);
} }
if (stats.skippedAsins > 0) { if (stats.skippedAsins > 0) {
console.log(`Stalker resume: skipped ${stats.skippedAsins} previously scanned ASIN(s).`); console.log(
`Stalker resume: skipped ${stats.skippedAsins} previously scanned ASIN(s).`,
);
} }
for (const asin of resumeFilteredAsins) { for (const asin of resumeFilteredAsins) {
console.log(`Stalker: scanning ${asin} (${stats.scannedAsins + 1}/${resumeFilteredAsins.length})`); console.log(
`Stalker: scanning ${asin} (${stats.scannedAsins + 1}/${resumeFilteredAsins.length})`,
);
const result = await scanAsin(asin, args, apiKey, context).catch((error) => ({ const result = await scanAsin(asin, args, apiKey, context).catch(
(error) => ({
asin, asin,
title: null, title: null,
offerCount: 0, offerCount: 0,
@@ -358,7 +377,8 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
matchedSellers: [], matchedSellers: [],
product: null, product: null,
error: error instanceof Error ? error.message : String(error), error: error instanceof Error ? error.message : String(error),
})); }),
);
if (args.sellability && !args.dryRun) { if (args.sellability && !args.dryRun) {
await enrichInventorySellability(result, stats); await enrichInventorySellability(result, stats);
@@ -379,7 +399,13 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
analysisRunId != null && analysisRunId != null &&
sellableAsins.length > 0 sellableAsins.length > 0
) { ) {
await runSellableAnalysisChild(args.dbPath, runId, analysisRunId, sellableAsins); await runSellableAnalysisChild(
args.dbPath,
runId,
analysisRunId,
sellableAsins,
args.useClaude,
);
} }
stats.scannedAsins += 1; stats.scannedAsins += 1;
stats.matchedSellers += result.matchedSellers.length; stats.matchedSellers += result.matchedSellers.length;
@@ -398,7 +424,9 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
); );
if (stats.stoppedEarly) { if (stats.stoppedEarly) {
console.log("Stalker: stopping early because max seller request budget was reached."); console.log(
"Stalker: stopping early because max seller request budget was reached.",
);
break; break;
} }
} }
@@ -423,12 +451,7 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
} catch (error) { } catch (error) {
const message = error instanceof Error ? error.message : String(error); const message = error instanceof Error ? error.message : String(error);
if (!args.dryRun && runId != null) { if (!args.dryRun && runId != null) {
finishStalkerRunWithError( finishStalkerRunWithError(database, runId, stats, message);
database,
runId,
stats,
message,
);
} }
if (!args.dryRun && analysisRunId != null) { if (!args.dryRun && analysisRunId != null) {
finishStalkerAnalysisRun(database, analysisRunId, "failed", message); finishStalkerAnalysisRun(database, analysisRunId, "failed", message);
@@ -549,8 +572,7 @@ async function enrichInventorySellability(
} }
for (const item of items) { for (const item of items) {
item.sellability = item.sellability = sellabilityMap.get(item.asin) ?? {
sellabilityMap.get(item.asin) ?? {
canSell: null, canSell: null,
sellabilityStatus: "unknown", sellabilityStatus: "unknown",
sellabilityReason: "Sellability check returned no result", sellabilityReason: "Sellability check returned no result",
@@ -571,14 +593,19 @@ async function enrichInventoryProductDetails(
result: StalkerAsinResult, result: StalkerAsinResult,
apiKey: string, apiKey: string,
): Promise<void> { ): Promise<void> {
const items = result.matchedSellers.flatMap(({ seller }) => seller.storefrontItems); const items = result.matchedSellers.flatMap(
({ seller }) => seller.storefrontItems,
);
const uniqueAsins = Array.from(new Set(items.map((item) => item.asin))); const uniqueAsins = Array.from(new Set(items.map((item) => item.asin)));
if (uniqueAsins.length === 0) return; if (uniqueAsins.length === 0) return;
console.log( console.log(
`Stalker inventory details: fetching Keepa product details for ${uniqueAsins.length} sellable ASIN(s)...`, `Stalker inventory details: fetching Keepa product details for ${uniqueAsins.length} sellable ASIN(s)...`,
); );
const detailsByAsin = await fetchKeepaInventoryProductDetails(apiKey, uniqueAsins); const detailsByAsin = await fetchKeepaInventoryProductDetails(
apiKey,
uniqueAsins,
);
for (const item of items) { for (const item of items) {
item.productDetails = detailsByAsin.get(item.asin) ?? null; item.productDetails = detailsByAsin.get(item.asin) ?? null;
@@ -761,7 +788,8 @@ function canSpendSellerRequests(
): boolean { ): boolean {
if (args.maxSellerRequests == null) return true; if (args.maxSellerRequests == null) return true;
const spent = const spent =
context.stats.sellerMetadataRequests + context.stats.sellerStorefrontRequests; context.stats.sellerMetadataRequests +
context.stats.sellerStorefrontRequests;
if (spent + nextRequests <= args.maxSellerRequests) return true; if (spent + nextRequests <= args.maxSellerRequests) return true;
context.stats.stoppedEarly = true; context.stats.stoppedEarly = true;
return false; return false;
@@ -856,7 +884,8 @@ function upsertAsinScan(
`SELECT id FROM stalker_asin_scans WHERE run_id = ? AND source_asin = ?`, `SELECT id FROM stalker_asin_scans WHERE run_id = ? AND source_asin = ?`,
) )
.get(runId, result.asin) as { id: number } | null; .get(runId, result.asin) as { id: number } | null;
if (!row) throw new Error(`Failed to load stalker scan row for ${result.asin}`); if (!row)
throw new Error(`Failed to load stalker scan row for ${result.asin}`);
return row.id; return row.id;
} }
@@ -978,7 +1007,9 @@ function upsertSellerInventory(
item.sellability?.sellabilityReason ?? null, item.sellability?.sellabilityReason ?? null,
item.productDetails?.title ?? null, item.productDetails?.title ?? null,
item.productDetails?.brand ?? null, item.productDetails?.brand ?? null,
item.productDetails ? JSON.stringify(item.productDetails.categoryTree) : null, item.productDetails
? JSON.stringify(item.productDetails.categoryTree)
: null,
item.productDetails?.currentPrice ?? null, item.productDetails?.currentPrice ?? null,
item.productDetails?.avgPrice90 ?? null, item.productDetails?.avgPrice90 ?? null,
item.productDetails?.salesRank ?? null, item.productDetails?.salesRank ?? null,
@@ -989,7 +1020,9 @@ function upsertSellerInventory(
: item.productDetails.amazonIsSeller : item.productDetails.amazonIsSeller
? 1 ? 1
: 0, : 0,
item.productDetails ? JSON.stringify(item.productDetails.rawProduct) : null, item.productDetails
? JSON.stringify(item.productDetails.rawProduct)
: null,
fetchedAt, fetchedAt,
JSON.stringify(item.rawInventory), JSON.stringify(item.rawInventory),
); );
@@ -1012,7 +1045,10 @@ function startStalkerRun(
return result.lastInsertRowid as number; return result.lastInsertRowid as number;
} }
function startStalkerAnalysisRun(database: Database, inputFile: string): number { function startStalkerAnalysisRun(
database: Database,
inputFile: string,
): number {
const result = database const result = database
.prepare( .prepare(
`INSERT INTO category_analysis_runs ( `INSERT INTO category_analysis_runs (
@@ -1231,18 +1267,24 @@ function normalizeSellerResponse(
if (!sellers) return []; if (!sellers) return [];
if (Array.isArray(sellers)) { if (Array.isArray(sellers)) {
return sellers return sellers
.map((seller) => [ .map(
(seller) =>
[
normalizeSellerId(seller.sellerId ?? seller.sellerID ?? seller.id), normalizeSellerId(seller.sellerId ?? seller.sellerID ?? seller.id),
seller, seller,
] as [string | null, Record<string, any>]) ] as [string | null, Record<string, any>],
)
.filter((entry): entry is [string, Record<string, any>] => !!entry[0]); .filter((entry): entry is [string, Record<string, any>] => !!entry[0]);
} }
return Object.entries(sellers) return Object.entries(sellers)
.map(([sellerId, seller]) => [ .map(
normalizeSellerId(sellerId), ([sellerId, seller]) =>
seller, [normalizeSellerId(sellerId), seller] as [
] as [string | null, Record<string, any>]) string | null,
Record<string, any>,
],
)
.filter((entry): entry is [string, Record<string, any>] => !!entry[0]); .filter((entry): entry is [string, Record<string, any>] => !!entry[0]);
} }
@@ -1253,14 +1295,15 @@ function parseSeller(
): StalkerSeller { ): StalkerSeller {
const allStorefrontItems = extractStorefrontItems(seller); const allStorefrontItems = extractStorefrontItems(seller);
const storefrontItems = const storefrontItems =
inventoryLimit === 0 inventoryLimit === 0 ? [] : allStorefrontItems.slice(0, inventoryLimit);
? []
: allStorefrontItems.slice(0, inventoryLimit);
const storefrontAsins = storefrontItems.map((item) => item.asin); const storefrontAsins = storefrontItems.map((item) => item.asin);
return { return {
sellerId, sellerId,
sellerName: extractString( sellerName: extractString(
seller.sellerName ?? seller.name ?? seller.storeName ?? seller.businessName, seller.sellerName ??
seller.name ??
seller.storeName ??
seller.businessName,
), ),
rating: extractNumber( rating: extractNumber(
seller.currentRating ?? seller.rating ?? seller.feedbackRating, seller.currentRating ?? seller.rating ?? seller.feedbackRating,
@@ -1279,7 +1322,9 @@ function parseSeller(
}; };
} }
function extractStorefrontItems(seller: Record<string, any>): StalkerInventoryItem[] { function extractStorefrontItems(
seller: Record<string, any>,
): StalkerInventoryItem[] {
const candidates = [ const candidates = [
seller.asinList, seller.asinList,
seller.asins, seller.asins,
@@ -1311,7 +1356,12 @@ function collectStorefrontItems(
const asin = normalizeAsin((value as Record<string, unknown>).asin); const asin = normalizeAsin((value as Record<string, unknown>).asin);
if (asin && !seen.has(asin)) { if (asin && !seen.has(asin)) {
seen.add(asin); seen.add(asin);
items.push({ asin, rawInventory: value, sellability: null, productDetails: null }); items.push({
asin,
rawInventory: value,
sellability: null,
productDetails: null,
});
} }
return; return;
} }
@@ -1319,7 +1369,12 @@ function collectStorefrontItems(
const asin = normalizeAsin(value); const asin = normalizeAsin(value);
if (!asin || seen.has(asin)) return; if (!asin || seen.has(asin)) return;
seen.add(asin); seen.add(asin);
items.push({ asin, rawInventory: { asin }, sellability: null, productDetails: null }); items.push({
asin,
rawInventory: { asin },
sellability: null,
productDetails: null,
});
} }
function parseInventoryProductDetails( function parseInventoryProductDetails(
@@ -1331,9 +1386,9 @@ function parseInventoryProductDetails(
title: extractString(product.title), title: extractString(product.title),
brand: extractString(product.brand ?? product.manufacturer), brand: extractString(product.brand ?? product.manufacturer),
categoryTree: categoryTree:
product.categoryTree?.map((category: { name?: unknown }) => product.categoryTree
extractString(category.name), ?.map((category: { name?: unknown }) => extractString(category.name))
).filter((name: string | null): name is string => !!name) ?? [], .filter((name: string | null): name is string => !!name) ?? [],
currentPrice: extractCurrentPrice(csv), currentPrice: extractCurrentPrice(csv),
avgPrice90: stats?.avg?.[0] != null ? stats.avg[0] / 100 : null, avgPrice90: stats?.avg?.[0] != null ? stats.avg[0] / 100 : null,
salesRank: extractNumber(stats?.current?.[3]), salesRank: extractNumber(stats?.current?.[3]),
@@ -1371,10 +1426,14 @@ function resolveAmazonIsSeller(
stats: Record<string, any> | undefined, stats: Record<string, any> | undefined,
csv: unknown, csv: unknown,
): boolean | null { ): boolean | null {
if (typeof product.isAmazonSeller === "boolean") return product.isAmazonSeller; if (typeof product.isAmazonSeller === "boolean")
return product.isAmazonSeller;
if (typeof product.availabilityAmazon === "number") { if (typeof product.availabilityAmazon === "number") {
if (product.availabilityAmazon >= 0) return true; if (product.availabilityAmazon >= 0) return true;
if (product.availabilityAmazon === -1 || product.availabilityAmazon === -2) { if (
product.availabilityAmazon === -1 ||
product.availabilityAmazon === -2
) {
return false; return false;
} }
} }
@@ -1437,9 +1496,9 @@ async function runSellableAnalysisChild(
stalkerRunId: number, stalkerRunId: number,
analysisRunId: number, analysisRunId: number,
asins: string[], asins: string[],
useClaude: boolean,
): Promise<void> { ): Promise<void> {
const child = Bun.spawn({ const cmd = [
cmd: [
"bun", "bun",
"run", "run",
"src/stalker-analyze.ts", "src/stalker-analyze.ts",
@@ -1451,7 +1510,13 @@ async function runSellableAnalysisChild(
String(analysisRunId), String(analysisRunId),
"--asins", "--asins",
asins.join(","), asins.join(","),
], ];
if (useClaude) {
cmd.push("--claude");
}
const child = Bun.spawn({
cmd,
stdout: "inherit", stdout: "inherit",
stderr: "inherit", stderr: "inherit",
}); });
@@ -1493,7 +1558,8 @@ function extractNumber(value: unknown): number | null {
function extractBoolean(value: unknown): boolean | null { function extractBoolean(value: unknown): boolean | null {
if (typeof value === "boolean") return value; if (typeof value === "boolean") return value;
if (typeof value === "number") return value === 1 ? true : value === 0 ? false : null; if (typeof value === "number")
return value === 1 ? true : value === 0 ? false : null;
if (typeof value !== "string") return null; if (typeof value !== "string") return null;
const normalized = value.trim().toLowerCase(); const normalized = value.trim().toLowerCase();
if (["1", "true", "yes"].includes(normalized)) return true; if (["1", "true", "yes"].includes(normalized)) return true;
@@ -1502,7 +1568,10 @@ function extractBoolean(value: unknown): boolean | null {
} }
function normalizeHeader(value: string): string { function normalizeHeader(value: string): string {
return value.toLowerCase().trim().replace(/[^a-z0-9]/g, ""); return value
.toLowerCase()
.trim()
.replace(/[^a-z0-9]/g, "");
} }
function readFlagValue(args: string[], flag: string): string | undefined { function readFlagValue(args: string[], flag: string): string | undefined {
@@ -1518,7 +1587,7 @@ function hasFlag(args: string[], flag: string): boolean {
function printUsageAndExit(message: string): never { function printUsageAndExit(message: string): never {
console.error(message); console.error(message);
console.error( console.error(
"Usage: bun run stalker --input input/asins.xlsx [--db db/results.db] [--max-asins N] [--offer-limit 100] [--seller-limit 30] [--inventory-limit 200] [--storefront-update-hours 168] [--seller-cache-hours 168] [--max-seller-requests N] [--sellability] [--analyze-sellable] [--include-stock] [--dry-run] [--no-resume]", "Usage: bun run stalker --input input/asins.xlsx [--db db/results.db] [--max-asins N] [--offer-limit 100] [--seller-limit 30] [--inventory-limit 200] [--storefront-update-hours 168] [--seller-cache-hours 168] [--max-seller-requests N] [--sellability] [--analyze-sellable] [--include-stock] [--dry-run] [--no-resume] [--claude]",
); );
process.exit(1); process.exit(1);
} }
@@ -1562,7 +1631,9 @@ function computeWaitMsFromRefill(refillIn?: number): number {
); );
} }
return Math.ceil((1 / Math.max(1, refillRate)) * 60_000) + KEEP_RETRY_BUFFER_MS; return (
Math.ceil((1 / Math.max(1, refillRate)) * 60_000) + KEEP_RETRY_BUFFER_MS
);
} }
function parseErrorPayload(text: string): KeepaApiResponse | null { function parseErrorPayload(text: string): KeepaApiResponse | null {

View File

@@ -28,6 +28,7 @@ type ParsedArgs = {
categoryCandidatePool: number; categoryCandidatePool: number;
minMonthlySold: number; minMonthlySold: number;
blacklistFile: string; blacklistFile: string;
useClaude: boolean;
}; };
type CategoryRunSummary = { type CategoryRunSummary = {
@@ -76,6 +77,7 @@ function log(
function parseArgs(): ParsedArgs { function parseArgs(): ParsedArgs {
const args = process.argv.slice(2); const args = process.argv.slice(2);
const useClaude = hasFlag(args, "--claude");
const outputDir = const outputDir =
readFlagValue(args, "--out-dir") ?? path.join(process.cwd(), "output"); readFlagValue(args, "--out-dir") ?? path.join(process.cwd(), "output");
const blacklistFile = const blacklistFile =
@@ -131,9 +133,14 @@ function parseArgs(): ParsedArgs {
categoryCandidatePool, categoryCandidatePool,
minMonthlySold, minMonthlySold,
blacklistFile, blacklistFile,
useClaude,
}; };
} }
function hasFlag(args: string[], flag: string): boolean {
return args.includes(flag);
}
function readFlagValue(args: string[], flag: string): string | undefined { function readFlagValue(args: string[], flag: string): string | undefined {
const idx = args.indexOf(flag); const idx = args.indexOf(flag);
if (idx === -1) return undefined; if (idx === -1) return undefined;
@@ -149,7 +156,7 @@ function printUsageAndExit(message: string): never {
"error", "error",
[ [
"Usage:", "Usage:",
" bun run src/top-monthly-sold-by-category.ts [--category-limit 32] [--per-category-top 100] [--category-candidate-pool 500] [--min-monthly-sold 300] [--out-dir output] [--blacklist-file category-blacklist.csv]", " bun run src/top-monthly-sold-by-category.ts [--category-limit 32] [--per-category-top 100] [--category-candidate-pool 500] [--min-monthly-sold 300] [--out-dir output] [--blacklist-file category-blacklist.csv] [--claude]",
"", "",
"Flow:", "Flow:",
" 1) Discover categories and round-robin selection.", " 1) Discover categories and round-robin selection.",
@@ -1066,6 +1073,7 @@ export async function processCategory(
perCategoryTop: number, perCategoryTop: number,
categoryCandidatePool: number, categoryCandidatePool: number,
minMonthlySold: number, minMonthlySold: number,
useClaude = false,
): Promise<CategoryRunSummary> { ): Promise<CategoryRunSummary> {
log("info", `\nCategory ${category.label} (${category.id})`); log("info", `\nCategory ${category.label} (${category.id})`);
@@ -1200,7 +1208,7 @@ export async function processCategory(
let batchVerdicts: LlmVerdict[]; let batchVerdicts: LlmVerdict[];
try { try {
batchVerdicts = await analyzeProducts(batch); batchVerdicts = await analyzeProducts(batch, { useClaude });
} catch (err) { } catch (err) {
const message = err instanceof Error ? err.message : String(err); const message = err instanceof Error ? err.message : String(err);
log("warn", ` LLM batch failed: ${message}`); log("warn", ` LLM batch failed: ${message}`);
@@ -1348,6 +1356,7 @@ export async function main(): Promise<void> {
args.perCategoryTop, args.perCategoryTop,
args.categoryCandidatePool, args.categoryCandidatePool,
args.minMonthlySold, args.minMonthlySold,
args.useClaude,
); );
totalInsertedAsins += categorySummary.results?.length ?? 0; totalInsertedAsins += categorySummary.results?.length ?? 0;