feat: enhance README and improve product data handling in cache, llm, reader, and writer modules

README.md

@@ -24,6 +24,7 @@ bun run src/index.ts <input.csv|xlsx> [--out results.csv]
 ```
 
 Examples:
+
 ```bash
 bun run src/index.ts leads.xlsx
 bun run src/index.ts leads.csv --out results.xlsx
@@ -34,13 +35,13 @@ bun run src/index.ts leads.csv --out results.xlsx
 Accepts `.csv` or `.xlsx` files. Column names are matched case-insensitively. Required column:
 
 | Column | Aliases |
-|--------|---------|
+| ------ | ------- |
 | ASIN   | —       |
 
 Optional but recommended:
 
 | Column          | Aliases                      |
-|--------|---------|
+| --------------- | ---------------------------- |
 | Product Name    | Name, Title                  |
 | Unit Cost       | Cost, Price, Buy Cost        |
 | Brand           | —                            |
@@ -54,6 +55,25 @@ Optional but recommended:
 | Total Qty Avail | Qty Available                |
 | Link            | URL, Source                  |
 
+Lead-list format aliases (supported):
+
+| Column            | Aliases                                    |
+| ----------------- | ------------------------------------------ |
+| Name              | Product Name, Title, Product Title         |
+| ASIN Link         | ASIN URL, Amazon Link                      |
+| Source URL        | Source Link, Supplier URL                  |
+| 90 Day Average    | 90-day Average, Avg Price 90d, 90d Average |
+| Cost              | Unit Cost, Buy Cost, Price                 |
+| Selling Price     | Sale Price, Sell Price                     |
+| Net Profit        | Gross Profit                               |
+| ROI               | Gross Profit %, Return on Investment       |
+| Supplier          | Vendor                                     |
+| Promo/Coupon Code | Promo Code, Coupon Code                    |
+| Notes             | Note                                       |
+| Date              | Lead Date                                  |
+
+Numeric parsing accepts plain numbers as well as formatted values like `$12.50`, `1,209.60`, and `27.5%`.
+
 ## Pipeline
 
 1. **Read** — parse input file, validate ASINs
@@ -70,7 +90,7 @@ ASIN, Name, Brand, Category, Unit Cost, Current Price, Avg Price 90d, Sales Rank
 ## Environment variables
 
 | Variable        | Default                    | Description                     |
-|----------|---------|-------------|
+| --------------- | -------------------------- | ------------------------------- |
 | `KEEPA_API_KEY` | —                          | **Required.** Keepa API key     |
 | `REDIS_URL`     | `redis://localhost:6379`   | Redis connection URL            |
 | `LLM_URL`       | `http://localhost:1234/v1` | LM Studio API base URL          |

src/cache.ts

@@ -12,11 +12,20 @@ export async function connectCache(): Promise<void> {
       maxRetriesPerRequest: 1,
       connectTimeout: 3000,
       lazyConnect: true,
+      retryStrategy: () => null,
+      reconnectOnError: () => false,
+    });
+    // Swallow connection-level errors after we intentionally disable cache.
+    redis.on("error", () => {
+      // no-op
     });
     await redis.connect();
     console.log("Redis connected");
   } catch (err) {
     console.warn(`Redis unavailable, running without cache: ${err}`);
+    if (redis) {
+      redis.disconnect();
+    }
     redis = null;
     disabled = true;
   }
@@ -32,10 +41,18 @@ export async function getCache(asin: string): Promise<EnrichedProduct | null> {
   }
 }
 
-export async function setCache(asin: string, data: EnrichedProduct): Promise<void> {
+export async function setCache(
+  asin: string,
+  data: EnrichedProduct,
+): Promise<void> {
   if (!redis) return;
   try {
-    await redis.set(`asin:${asin}`, JSON.stringify(data), "EX", config.cacheTtl);
+    await redis.set(
+      `asin:${asin}`,
+      JSON.stringify(data),
+      "EX",
+      config.cacheTtl,
+    );
   } catch {
     // Non-critical, continue without caching
   }

src/llm.ts

@@ -22,6 +22,45 @@ Keep each reasoning under 100 characters to stay within output limits.`;
 
 export async function analyzeProducts(
   products: EnrichedProduct[],
+): Promise<LlmVerdict[]> {
+  try {
+    return await analyzeProductsInternal(products);
+  } catch (err) {
+    const msg = String(err);
+    if (products.length > 1 && msg.includes("Context size has been exceeded")) {
+      console.warn(
+        `LLM context exceeded for batch of ${products.length}, retrying one product at a time...`,
+      );
+
+      const fallback: LlmVerdict[] = [];
+      for (const product of products) {
+        try {
+          const single = await analyzeProductsInternal([product]);
+          fallback.push(
+            single[0] ?? {
+              asin: product.record.asin,
+              verdict: "SKIP",
+              confidence: 0,
+              reasoning: "LLM returned empty verdict",
+            },
+          );
+        } catch {
+          fallback.push({
+            asin: product.record.asin,
+            verdict: "SKIP",
+            confidence: 0,
+            reasoning: "LLM context overflow on single-item fallback",
+          });
+        }
+      }
+      return fallback;
+    }
+    throw err;
+  }
+}
+
+async function analyzeProductsInternal(
+  products: EnrichedProduct[],
 ): Promise<LlmVerdict[]> {
   const productSummaries = products.map(summarizeForLlm);
 
@@ -55,7 +94,10 @@ export async function analyzeProducts(
 }
 
 function summarizeForLlm(p: EnrichedProduct) {
-  const salePrice = p.keepa?.currentPrice ?? p.spApi.estimatedSalePrice;
+  const salePrice =
+    p.keepa?.currentPrice ??
+    p.record.sellingPriceFromSheet ??
+    p.spApi.estimatedSalePrice;
   const referralFee = salePrice * (p.spApi.referralFeePercent / 100);
   const fbaProfit =
     salePrice - p.record.unitCost - p.spApi.fbaFee - referralFee;
@@ -64,9 +106,12 @@ function summarizeForLlm(p: EnrichedProduct) {
 
   return {
     asin: p.record.asin,
-    name: p.record.name,
+    name: clampText(p.record.name, 80),
     brand: p.record.brand,
-    category: p.record.category ?? p.keepa?.categoryTree?.join(" > "),
+    category: clampText(
+      p.record.category ?? p.keepa?.categoryTree?.join(" > "),
+      60,
+    ),
     unitCost: p.record.unitCost,
     currentPrice: salePrice,
     priceRange90d: p.keepa
@@ -85,10 +130,15 @@ function summarizeForLlm(p: EnrichedProduct) {
       salesRankDrops90: p.keepa?.salesRankDrops90,
     },
     spreadsheetEstimates: {
+      avgPrice90: p.record.avgPrice90FromSheet,
+      sellingPrice: p.record.sellingPriceFromSheet,
       fbaNet: p.record.fbaNet,
       grossProfit: p.record.grossProfit,
       grossProfitPct: p.record.grossProfitPct,
+      netProfit: p.record.netProfitFromSheet,
+      roi: p.record.roiFromSheet,
     },
+    supplier: clampText(p.record.supplier, 40),
     moq: p.record.moq,
     moqCost: p.record.moqCost,
     totalQtyAvail: p.record.totalQtyAvail,
@@ -115,6 +165,13 @@ function summarizeForLlm(p: EnrichedProduct) {
   };
 }
 
+function clampText(value: unknown, maxLen: number): string | undefined {
+  if (value == null) return undefined;
+  const s = String(value).trim();
+  if (!s) return undefined;
+  return s.length > maxLen ? `${s.slice(0, maxLen - 1)}.` : s;
+}
+
 function cleanLlmJson(text: string): string {
   // Remove ```json ... ``` or ``` ... ``` wrapping
   const fenceMatch = text.match(/```(?:json)?\s*\n?([\s\S]*?)```/);
@@ -148,7 +205,9 @@ function parseVerdicts(
       reasoning: String(v.reasoning ?? "No reasoning provided"),
     }));
   } catch (err) {
-    console.warn("Failed to parse LLM response, marking all as ANALYSIS_FAILED");
+    console.warn(
+      "Failed to parse LLM response, marking all as ANALYSIS_FAILED",
+    );
     console.warn("Raw LLM content:", content.slice(0, 500));
     return products.map((p) => ({
       asin: p.record.asin,

src/reader.ts

@@ -1,6 +1,48 @@
 import * as XLSX from "xlsx";
 import type { ProductRecord } from "./types.ts";
 
+const ASIN_REGEX = /^B[0-9A-Z]{9}$/;
+
+const COLUMN_CANDIDATES = {
+  asin: ["asin"],
+  name: ["name", "product name", "title", "product title"],
+  cost: ["unit cost", "cost", "unitcost", "unit_cost", "price", "buy cost"],
+  brand: ["brand"],
+  category: ["category"],
+  amazonRank: ["amazon rank", "amazonrank", "sales rank", "bsr"],
+  avgPrice90: [
+    "90 day average",
+    "90-day average",
+    "avg price 90d",
+    "avg 90 day",
+    "90d average",
+  ],
+  sellingPrice: ["selling price", "sale price", "sell price"],
+  fbaNet: ["fba net", "fbanet", "fba_net"],
+  grossProfit: ["gross profit $", "gross profit", "grossprofit"],
+  grossProfitPct: ["gross profit %", "gross profit pct", "grossprofitpct"],
+  netProfit: ["net profit", "netprofit"],
+  roi: ["roi", "return on investment"],
+  moq: ["moq", "min order qty", "minimum order quantity"],
+  moqCost: ["moq cost", "moqcost", "moq_cost"],
+  totalQty: ["total qty avail", "totalqtyavail", "qty available", "quantity"],
+  link: ["link", "url", "source"],
+  asinLink: ["asin link", "amazon link", "asin url"],
+  sourceUrl: ["source url", "supplier url", "source link"],
+  supplier: ["supplier", "vendor"],
+  promoCouponCode: [
+    "promo/coupon code",
+    "promo coupon code",
+    "coupon code",
+    "promo code",
+  ],
+  notes: ["notes", "note"],
+  leadDate: ["date", "lead date"],
+} as const;
+
+type ColumnKey = keyof typeof COLUMN_CANDIDATES;
+type ColumnMap = Record<ColumnKey, string | undefined>;
+
 export function readProducts(filePath: string): ProductRecord[] {
   const workbook = XLSX.readFile(filePath);
   const sheetName = workbook.SheetNames[0];
@@ -12,58 +54,57 @@ export function readProducts(filePath: string): ProductRecord[] {
   if (rows.length === 0) throw new Error("File contains no data rows");
 
   const headers = Object.keys(rows[0]!);
-  const asinCol = findColumn(headers, ["asin"]);
+  const columns = detectColumns(headers);
-  const nameCol = findColumn(headers, ["name", "product name", "title", "product title"]);
+  const asinColumn = columns.asin;
-  const costCol = findColumn(headers, ["unit cost", "cost", "unitcost", "unit_cost", "price", "buy cost"]);
 
-  const brandCol = findColumn(headers, ["brand"]);
+  if (!asinColumn)
-  const categoryCol = findColumn(headers, ["category"]);
+    throw new Error(
-  const amazonRankCol = findColumn(headers, ["amazon rank", "amazonrank", "sales rank", "bsr"]);
+      `No ASIN column found. Available columns: ${headers.join(", ")}`,
-  const fbaNetCol = findColumn(headers, ["fba net", "fbanet", "fba_net"]);
+    );
-  const grossProfitCol = findColumn(headers, ["gross profit $", "gross profit", "grossprofit"]);
-  const grossProfitPctCol = findColumn(headers, ["gross profit %", "gross profit pct", "grossprofitpct"]);
-  const moqCol = findColumn(headers, ["moq", "min order qty", "minimum order quantity"]);
-  const moqCostCol = findColumn(headers, ["moq cost", "moqcost"]);
-  const totalQtyCol = findColumn(headers, ["total qty avail", "totalqtyavail", "qty available", "quantity"]);
 
-  const linkCol = findColumn(headers, ["link", "url", "source"]);
+  logColumnDetection(headers, columns);
 
-  if (!asinCol) throw new Error(`No ASIN column found. Available columns: ${headers.join(", ")}`);
+  const knownCols = getKnownColumns(columns);
-
-  const knownCols = new Set([asinCol, nameCol, costCol, brandCol, categoryCol, amazonRankCol, fbaNetCol, grossProfitCol, grossProfitPctCol, moqCol, moqCostCol, totalQtyCol, linkCol].filter(Boolean));
 
   const products: ProductRecord[] = [];
 
   for (const row of rows) {
-    const asin = String(row[asinCol] ?? "").trim().toUpperCase();
+    const asin = parseAsin(row[asinColumn]);
-    if (!asin || !/^B[0-9A-Z]{9}$/.test(asin)) {
+    if (!asin) continue;
-      console.warn(`Skipping invalid ASIN: "${asin}"`);
-      continue;
-    }
 
-    const name = nameCol ? String(row[nameCol] ?? "") : "";
+    const sourceUrl = getOptionalString(row, columns.sourceUrl);
-    const unitCost = costCol ? parseFloat(String(row[costCol] ?? "0")) : 0;
+    const asinLink = getOptionalString(row, columns.asinLink);
+    const link = sourceUrl ?? asinLink ?? getOptionalString(row, columns.link);
 
-    const extra: Record<string, unknown> = {};
+    const extra = getExtraFields(row, headers, knownCols);
-    for (const h of headers) {
+    const netProfitFromSheet = getOptionalNumber(row, columns.netProfit);
-      if (!knownCols.has(h)) extra[h] = row[h];
+    const roiFromSheet = getOptionalNumber(row, columns.roi);
-    }
 
     products.push({
       asin,
-      name,
+      name: getOptionalString(row, columns.name) ?? "",
-      unitCost,
+      unitCost: getOptionalNumber(row, columns.cost) ?? 0,
-      brand: brandCol ? String(row[brandCol] ?? "") : undefined,
+      brand: getOptionalString(row, columns.brand),
-      category: categoryCol ? String(row[categoryCol] ?? "") : undefined,
+      category: getOptionalString(row, columns.category),
-      amazonRank: amazonRankCol ? Number(row[amazonRankCol]) || undefined : undefined,
+      amazonRank: getOptionalNumber(row, columns.amazonRank),
-      fbaNet: fbaNetCol ? Number(row[fbaNetCol]) || undefined : undefined,
+      avgPrice90FromSheet: getOptionalNumber(row, columns.avgPrice90),
-      grossProfit: grossProfitCol ? Number(row[grossProfitCol]) || undefined : undefined,
+      sellingPriceFromSheet: getOptionalNumber(row, columns.sellingPrice),
-      grossProfitPct: grossProfitPctCol ? Number(row[grossProfitPctCol]) || undefined : undefined,
+      fbaNet: getOptionalNumber(row, columns.fbaNet),
-      moq: moqCol ? Number(row[moqCol]) || undefined : undefined,
+      grossProfit: getOptionalNumber(row, columns.grossProfit) ?? netProfitFromSheet,
-      moqCost: moqCostCol ? Number(row[moqCostCol]) || undefined : undefined,
+      grossProfitPct:
-      totalQtyAvail: totalQtyCol ? Number(row[totalQtyCol]) || undefined : undefined,
+        getOptionalNumber(row, columns.grossProfitPct) ?? roiFromSheet,
-
+      netProfitFromSheet,
-      link: linkCol ? String(row[linkCol] ?? "") : undefined,
+      roiFromSheet,
+      moq: getOptionalNumber(row, columns.moq),
+      moqCost: getOptionalNumber(row, columns.moqCost),
+      totalQtyAvail: getOptionalNumber(row, columns.totalQty),
+      link,
+      asinLink,
+      sourceUrl,
+      supplier: getOptionalString(row, columns.supplier),
+      promoCouponCode: getOptionalString(row, columns.promoCouponCode),
+      notes: getOptionalString(row, columns.notes),
+      leadDate: getOptionalString(row, columns.leadDate),
       ...extra,
     });
   }
@@ -72,10 +113,97 @@ export function readProducts(filePath: string): ProductRecord[] {
   return products;
 }
 
-function findColumn(headers: string[], candidates: string[]): string | undefined {
+function detectColumns(headers: string[]): ColumnMap {
-  for (const candidate of candidates) {
+  const columns = {} as ColumnMap;
-    const match = headers.find((h) => h.toLowerCase().trim() === candidate);
+  for (const key of Object.keys(COLUMN_CANDIDATES) as ColumnKey[]) {
-    if (match) return match;
+    columns[key] = findColumn(headers, [...COLUMN_CANDIDATES[key]]);
   }
+  return columns;
+}
+
+function logColumnDetection(headers: string[], columns: ColumnMap): void {
+  console.log(`Found columns: ${headers.join(", ")}`);
+  console.log(
+    `Detected columns -> ASIN: ${columns.asin ?? "n/a"}, Name: ${columns.name ?? "n/a"}, Cost: ${columns.cost ?? "n/a"}, 90d Avg: ${columns.avgPrice90 ?? "n/a"}, Selling Price: ${columns.sellingPrice ?? "n/a"}, Net Profit: ${columns.netProfit ?? columns.grossProfit ?? "n/a"}, ROI: ${columns.roi ?? columns.grossProfitPct ?? "n/a"}, Source URL: ${columns.sourceUrl ?? "n/a"}, ASIN Link: ${columns.asinLink ?? "n/a"}`,
+  );
+}
+
+function getKnownColumns(columns: ColumnMap): Set<string> {
+  return new Set(Object.values(columns).filter((column): column is string => !!column));
+}
+
+function parseAsin(value: unknown): string | undefined {
+  const asin = String(value ?? "")
+    .trim()
+    .toUpperCase();
+  if (!asin || !ASIN_REGEX.test(asin)) {
+    console.warn(`Skipping invalid ASIN: "${asin}"`);
+    return undefined;
+  }
+  return asin;
+}
+
+function getOptionalString(
+  row: Record<string, unknown>,
+  column: string | undefined,
+): string | undefined {
+  if (!column) return undefined;
+  return normalizeOptionalString(row[column]);
+}
+
+function getOptionalNumber(
+  row: Record<string, unknown>,
+  column: string | undefined,
+): number | undefined {
+  if (!column) return undefined;
+  return parseOptionalNumber(row[column]);
+}
+
+function getExtraFields(
+  row: Record<string, unknown>,
+  headers: string[],
+  knownCols: Set<string>,
+): Record<string, unknown> {
+  const extra: Record<string, unknown> = {};
+  for (const header of headers) {
+    if (!knownCols.has(header)) extra[header] = row[header];
+  }
+  return extra;
+}
+
+function findColumn(
+  headers: string[],
+  candidates: string[],
+): string | undefined {
+  const normalizedCandidates = new Set(candidates.map(normalizeHeader));
+
+  for (const header of headers) {
+    if (normalizedCandidates.has(normalizeHeader(header))) {
+      return header;
+    }
+  }
+
   return undefined;
 }
+
+function normalizeHeader(value: string): string {
+  return value
+    .toLowerCase()
+    .trim()
+    .replace(/%/g, " pct ")
+    .replace(/\$/g, " usd ")
+    .replace(/[^a-z0-9]/g, "");
+}
+
+function normalizeOptionalString(value: unknown): string | undefined {
+  if (value == null) return undefined;
+  const s = String(value).trim();
+  return s.length > 0 ? s : undefined;
+}
+
+function parseOptionalNumber(value: unknown): number | undefined {
+  if (value == null || value === "") return undefined;
+  const cleaned = String(value).trim().replace(/[$,%]/g, "").replace(/,/g, "");
+  const parsed = Number(cleaned);
+  return Number.isFinite(parsed) ? parsed : undefined;
+}

src/types.ts

@@ -5,14 +5,24 @@ export interface ProductRecord {
   brand?: string;
   category?: string;
   amazonRank?: number;
+  avgPrice90FromSheet?: number;
+  sellingPriceFromSheet?: number;
   fbaNet?: number;
   grossProfit?: number;
   grossProfitPct?: number;
+  netProfitFromSheet?: number;
+  roiFromSheet?: number;
   moq?: number;
   moqCost?: number;
   totalQtyAvail?: number;
 
   link?: string;
+  asinLink?: string;
+  sourceUrl?: string;
+  supplier?: string;
+  promoCouponCode?: string;
+  notes?: string;
+  leadDate?: string;
   [key: string]: unknown;
 }
 

src/writer.ts

@@ -2,17 +2,25 @@ import * as XLSX from "xlsx";
 import type { AnalysisResult } from "./types.ts";
 
 function buildRow(r: AnalysisResult) {
-  const price = r.product.keepa?.currentPrice ?? r.product.spApi.estimatedSalePrice;
+  const price =
+    r.product.keepa?.currentPrice ??
+    r.product.record.sellingPriceFromSheet ??
+    r.product.spApi.estimatedSalePrice;
   const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank;
 
   return {
     ASIN: r.product.record.asin,
     Name: r.product.record.name,
     Brand: r.product.record.brand ?? "",
-    Category: r.product.record.category ?? r.product.keepa?.categoryTree?.join(" > ") ?? "",
+    Category:
+      r.product.record.category ??
+      r.product.keepa?.categoryTree?.join(" > ") ??
+      "",
     "Unit Cost": r.product.record.unitCost,
     "Current Price": price ?? "",
     "Avg Price 90d": r.product.keepa?.avgPrice90 ?? "",
+    "Avg Price 90d (sheet)": r.product.record.avgPrice90FromSheet ?? "",
+    "Selling Price (sheet)": r.product.record.sellingPriceFromSheet ?? "",
     "Sales Rank": rank ?? "",
     "Rank Avg 90d": r.product.keepa?.salesRankAvg90 ?? "",
     Sellers: r.product.keepa?.sellerCount ?? "",
@@ -22,9 +30,17 @@ function buildRow(r: AnalysisResult) {
     "FBA Net (sheet)": r.product.record.fbaNet ?? "",
     "Gross Profit $": r.product.record.grossProfit ?? "",
     "Gross Profit %": r.product.record.grossProfitPct ?? "",
+    "Net Profit (sheet)": r.product.record.netProfitFromSheet ?? "",
+    "ROI (sheet)": r.product.record.roiFromSheet ?? "",
     MOQ: r.product.record.moq ?? "",
     "MOQ Cost": r.product.record.moqCost ?? "",
     "Qty Available": r.product.record.totalQtyAvail ?? "",
+    Supplier: r.product.record.supplier ?? "",
+    "Source URL": r.product.record.sourceUrl ?? "",
+    "ASIN Link": r.product.record.asinLink ?? "",
+    "Promo/Coupon Code": r.product.record.promoCouponCode ?? "",
+    Notes: r.product.record.notes ?? "",
+    "Lead Date": r.product.record.leadDate ?? "",
     "FBA Fee": r.product.spApi.fbaFee,
     "FBM Fee": r.product.spApi.fbmFee,
     "Referral %": r.product.spApi.referralFeePercent,
@@ -53,10 +69,15 @@ export function printResults(results: AnalysisResult[]): void {
     FBM: results.filter((r) => r.verdict.verdict === "FBM").length,
     SKIP: results.filter((r) => r.verdict.verdict === "SKIP").length,
   };
-  console.log(`\nSummary: ${summary.FBA} FBA | ${summary.FBM} FBM | ${summary.SKIP} SKIP out of ${results.length} products\n`);
+  console.log(
+    `\nSummary: ${summary.FBA} FBA | ${summary.FBM} FBM | ${summary.SKIP} SKIP out of ${results.length} products\n`,
+  );
 }
 
-export function writeResultsCsv(results: AnalysisResult[], outputPath: string): void {
+export function writeResultsCsv(
+  results: AnalysisResult[],
+  outputPath: string,
+): void {
   const rows = results.map(buildRow);
 
   const ws = XLSX.utils.json_to_sheet(rows);