feat: implement batch processing for product analysis with delay and error handling

src/stalker-analyze.ts

@@ -9,6 +9,9 @@ import type {
   SellabilityInfo,
 } from "./types.ts";
 
+const LLM_BATCH_SIZE = 5;
+const LLM_BATCH_DELAY_MS = 5_000;
+
 type Args = {
   dbPath: string;
   stalkerRunId: number;
@@ -59,6 +62,10 @@ function parseArgs(argv = process.argv.slice(2)): Args {
   return { dbPath, stalkerRunId, analysisRunId, asins };
 }
 
+function wait(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
 function parseCategoryTree(value: string | null): string[] {
   if (!value) return [];
   try {
@@ -290,6 +297,53 @@ function refreshAnalysisRun(database: Database, runId: number): void {
     );
 }
 
+async function analyzeInBatches(
+  products: EnrichedProduct[],
+): Promise<AnalysisResult[]> {
+  const results: AnalysisResult[] = [];
+
+  for (let i = 0; i < products.length; i += LLM_BATCH_SIZE) {
+    const batch = products.slice(i, i + LLM_BATCH_SIZE);
+    const batchNumber = Math.floor(i / LLM_BATCH_SIZE) + 1;
+    const totalBatches = Math.ceil(products.length / LLM_BATCH_SIZE);
+    console.log(
+      `Stalker analysis: LLM batch ${batchNumber}/${totalBatches} (${batch.length} product(s)).`,
+    );
+
+    if (i > 0) {
+      await wait(LLM_BATCH_DELAY_MS);
+    }
+
+    let verdicts;
+    try {
+      verdicts = await analyzeProducts(batch);
+    } catch (error) {
+      console.warn(
+        `Stalker analysis: LLM batch ${batchNumber} failed: ${
+          error instanceof Error ? error.message : String(error)
+        }`,
+      );
+      verdicts = null;
+    }
+
+    for (let j = 0; j < batch.length; j++) {
+      const product = batch[j];
+      if (!product) continue;
+      results.push({
+        product,
+        verdict: verdicts?.[j] ?? {
+          asin: product.record.asin,
+          verdict: "SKIP",
+          confidence: 0,
+          reasoning: "LLM analysis failed or returned no verdict",
+        },
+      });
+    }
+  }
+
+  return results;
+}
+
 async function main(): Promise<void> {
   const args = parseArgs();
   initDb(args.dbPath);
@@ -304,16 +358,7 @@ async function main(): Promise<void> {
 
     console.log(`Stalker analysis: analyzing ${rows.length} sellable ASIN(s).`);
     const enriched = await buildEnrichedProducts(rows);
-    const verdicts = await analyzeProducts(enriched);
+    const results = await analyzeInBatches(enriched);
-    const results = enriched.map((product, index) => ({
-      product,
-      verdict: verdicts[index] ?? {
-        asin: product.record.asin,
-        verdict: "SKIP" as const,
-        confidence: 0,
-        reasoning: "LLM analysis returned no verdict",
-      },
-    }));
     insertProductAnalysisResults(database, args.analysisRunId, results);
     refreshAnalysisRun(database, args.analysisRunId);
   } finally {