diff --git a/README.md b/README.md index 0fefee0..e089902 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ bun run src/index.ts [--out results.csv] ``` Examples: + ```bash bun run src/index.ts leads.xlsx bun run src/index.ts leads.csv --out results.xlsx @@ -34,25 +35,44 @@ bun run src/index.ts leads.csv --out results.xlsx Accepts `.csv` or `.xlsx` files. Column names are matched case-insensitively. Required column: | Column | Aliases | -|--------|---------| -| ASIN | — | +| ------ | ------- | +| ASIN | — | Optional but recommended: -| Column | Aliases | -|--------|---------| -| Product Name | Name, Title | -| Unit Cost | Cost, Price, Buy Cost | -| Brand | — | -| Category | — | -| Amazon Rank | Amazon Rank, BSR, Sales Rank | -| FBA NET | — | -| Gross Profit $ | Gross Profit | -| Gross Profit % | — | -| MOQ | Min Order Qty | -| MOQ Cost | — | -| Total Qty Avail | Qty Available | -| Link | URL, Source | +| Column | Aliases | +| --------------- | ---------------------------- | +| Product Name | Name, Title | +| Unit Cost | Cost, Price, Buy Cost | +| Brand | — | +| Category | — | +| Amazon Rank | Amazon Rank, BSR, Sales Rank | +| FBA NET | — | +| Gross Profit $ | Gross Profit | +| Gross Profit % | — | +| MOQ | Min Order Qty | +| MOQ Cost | — | +| Total Qty Avail | Qty Available | +| Link | URL, Source | + +Lead-list format aliases (supported): + +| Column | Aliases | +| ----------------- | ------------------------------------------ | +| Name | Product Name, Title, Product Title | +| ASIN Link | ASIN URL, Amazon Link | +| Source URL | Source Link, Supplier URL | +| 90 Day Average | 90-day Average, Avg Price 90d, 90d Average | +| Cost | Unit Cost, Buy Cost, Price | +| Selling Price | Sale Price, Sell Price | +| Net Profit | Gross Profit | +| ROI | Gross Profit %, Return on Investment | +| Supplier | Vendor | +| Promo/Coupon Code | Promo Code, Coupon Code | +| Notes | Note | +| Date | Lead Date | + +Numeric parsing accepts plain numbers as well as formatted values like `$12.50`, `1,209.60`, and `27.5%`. ## Pipeline @@ -69,13 +89,13 @@ ASIN, Name, Brand, Category, Unit Cost, Current Price, Avg Price 90d, Sales Rank ## Environment variables -| Variable | Default | Description | -|----------|---------|-------------| -| `KEEPA_API_KEY` | — | **Required.** Keepa API key | -| `REDIS_URL` | `redis://localhost:6379` | Redis connection URL | -| `LLM_URL` | `http://localhost:1234/v1` | LM Studio API base URL | -| `LLM_MODEL` | `default` | Model name to pass to LM Studio | -| `CACHE_TTL` | `86400` | Redis cache TTL in seconds | +| Variable | Default | Description | +| --------------- | -------------------------- | ------------------------------- | +| `KEEPA_API_KEY` | — | **Required.** Keepa API key | +| `REDIS_URL` | `redis://localhost:6379` | Redis connection URL | +| `LLM_URL` | `http://localhost:1234/v1` | LM Studio API base URL | +| `LLM_MODEL` | `default` | Model name to pass to LM Studio | +| `CACHE_TTL` | `86400` | Redis cache TTL in seconds | ## Notes diff --git a/src/cache.ts b/src/cache.ts index e630842..93fa618 100644 --- a/src/cache.ts +++ b/src/cache.ts @@ -12,11 +12,20 @@ export async function connectCache(): Promise { maxRetriesPerRequest: 1, connectTimeout: 3000, lazyConnect: true, + retryStrategy: () => null, + reconnectOnError: () => false, + }); + // Swallow connection-level errors after we intentionally disable cache. + redis.on("error", () => { + // no-op }); await redis.connect(); console.log("Redis connected"); } catch (err) { console.warn(`Redis unavailable, running without cache: ${err}`); + if (redis) { + redis.disconnect(); + } redis = null; disabled = true; } @@ -32,10 +41,18 @@ export async function getCache(asin: string): Promise { } } -export async function setCache(asin: string, data: EnrichedProduct): Promise { +export async function setCache( + asin: string, + data: EnrichedProduct, +): Promise { if (!redis) return; try { - await redis.set(`asin:${asin}`, JSON.stringify(data), "EX", config.cacheTtl); + await redis.set( + `asin:${asin}`, + JSON.stringify(data), + "EX", + config.cacheTtl, + ); } catch { // Non-critical, continue without caching } diff --git a/src/llm.ts b/src/llm.ts index ad230ed..66d0ca0 100644 --- a/src/llm.ts +++ b/src/llm.ts @@ -22,6 +22,45 @@ Keep each reasoning under 100 characters to stay within output limits.`; export async function analyzeProducts( products: EnrichedProduct[], +): Promise { + try { + return await analyzeProductsInternal(products); + } catch (err) { + const msg = String(err); + if (products.length > 1 && msg.includes("Context size has been exceeded")) { + console.warn( + `LLM context exceeded for batch of ${products.length}, retrying one product at a time...`, + ); + + const fallback: LlmVerdict[] = []; + for (const product of products) { + try { + const single = await analyzeProductsInternal([product]); + fallback.push( + single[0] ?? { + asin: product.record.asin, + verdict: "SKIP", + confidence: 0, + reasoning: "LLM returned empty verdict", + }, + ); + } catch { + fallback.push({ + asin: product.record.asin, + verdict: "SKIP", + confidence: 0, + reasoning: "LLM context overflow on single-item fallback", + }); + } + } + return fallback; + } + throw err; + } +} + +async function analyzeProductsInternal( + products: EnrichedProduct[], ): Promise { const productSummaries = products.map(summarizeForLlm); @@ -55,7 +94,10 @@ export async function analyzeProducts( } function summarizeForLlm(p: EnrichedProduct) { - const salePrice = p.keepa?.currentPrice ?? p.spApi.estimatedSalePrice; + const salePrice = + p.keepa?.currentPrice ?? + p.record.sellingPriceFromSheet ?? + p.spApi.estimatedSalePrice; const referralFee = salePrice * (p.spApi.referralFeePercent / 100); const fbaProfit = salePrice - p.record.unitCost - p.spApi.fbaFee - referralFee; @@ -64,9 +106,12 @@ function summarizeForLlm(p: EnrichedProduct) { return { asin: p.record.asin, - name: p.record.name, + name: clampText(p.record.name, 80), brand: p.record.brand, - category: p.record.category ?? p.keepa?.categoryTree?.join(" > "), + category: clampText( + p.record.category ?? p.keepa?.categoryTree?.join(" > "), + 60, + ), unitCost: p.record.unitCost, currentPrice: salePrice, priceRange90d: p.keepa @@ -85,10 +130,15 @@ function summarizeForLlm(p: EnrichedProduct) { salesRankDrops90: p.keepa?.salesRankDrops90, }, spreadsheetEstimates: { + avgPrice90: p.record.avgPrice90FromSheet, + sellingPrice: p.record.sellingPriceFromSheet, fbaNet: p.record.fbaNet, grossProfit: p.record.grossProfit, grossProfitPct: p.record.grossProfitPct, + netProfit: p.record.netProfitFromSheet, + roi: p.record.roiFromSheet, }, + supplier: clampText(p.record.supplier, 40), moq: p.record.moq, moqCost: p.record.moqCost, totalQtyAvail: p.record.totalQtyAvail, @@ -115,6 +165,13 @@ function summarizeForLlm(p: EnrichedProduct) { }; } +function clampText(value: unknown, maxLen: number): string | undefined { + if (value == null) return undefined; + const s = String(value).trim(); + if (!s) return undefined; + return s.length > maxLen ? `${s.slice(0, maxLen - 1)}.` : s; +} + function cleanLlmJson(text: string): string { // Remove ```json ... ``` or ``` ... ``` wrapping const fenceMatch = text.match(/```(?:json)?\s*\n?([\s\S]*?)```/); @@ -148,7 +205,9 @@ function parseVerdicts( reasoning: String(v.reasoning ?? "No reasoning provided"), })); } catch (err) { - console.warn("Failed to parse LLM response, marking all as ANALYSIS_FAILED"); + console.warn( + "Failed to parse LLM response, marking all as ANALYSIS_FAILED", + ); console.warn("Raw LLM content:", content.slice(0, 500)); return products.map((p) => ({ asin: p.record.asin, diff --git a/src/reader.ts b/src/reader.ts index 9972dac..e48939c 100644 --- a/src/reader.ts +++ b/src/reader.ts @@ -1,6 +1,48 @@ import * as XLSX from "xlsx"; import type { ProductRecord } from "./types.ts"; +const ASIN_REGEX = /^B[0-9A-Z]{9}$/; + +const COLUMN_CANDIDATES = { + asin: ["asin"], + name: ["name", "product name", "title", "product title"], + cost: ["unit cost", "cost", "unitcost", "unit_cost", "price", "buy cost"], + brand: ["brand"], + category: ["category"], + amazonRank: ["amazon rank", "amazonrank", "sales rank", "bsr"], + avgPrice90: [ + "90 day average", + "90-day average", + "avg price 90d", + "avg 90 day", + "90d average", + ], + sellingPrice: ["selling price", "sale price", "sell price"], + fbaNet: ["fba net", "fbanet", "fba_net"], + grossProfit: ["gross profit $", "gross profit", "grossprofit"], + grossProfitPct: ["gross profit %", "gross profit pct", "grossprofitpct"], + netProfit: ["net profit", "netprofit"], + roi: ["roi", "return on investment"], + moq: ["moq", "min order qty", "minimum order quantity"], + moqCost: ["moq cost", "moqcost", "moq_cost"], + totalQty: ["total qty avail", "totalqtyavail", "qty available", "quantity"], + link: ["link", "url", "source"], + asinLink: ["asin link", "amazon link", "asin url"], + sourceUrl: ["source url", "supplier url", "source link"], + supplier: ["supplier", "vendor"], + promoCouponCode: [ + "promo/coupon code", + "promo coupon code", + "coupon code", + "promo code", + ], + notes: ["notes", "note"], + leadDate: ["date", "lead date"], +} as const; + +type ColumnKey = keyof typeof COLUMN_CANDIDATES; +type ColumnMap = Record; + export function readProducts(filePath: string): ProductRecord[] { const workbook = XLSX.readFile(filePath); const sheetName = workbook.SheetNames[0]; @@ -12,58 +54,57 @@ export function readProducts(filePath: string): ProductRecord[] { if (rows.length === 0) throw new Error("File contains no data rows"); const headers = Object.keys(rows[0]!); - const asinCol = findColumn(headers, ["asin"]); - const nameCol = findColumn(headers, ["name", "product name", "title", "product title"]); - const costCol = findColumn(headers, ["unit cost", "cost", "unitcost", "unit_cost", "price", "buy cost"]); + const columns = detectColumns(headers); + const asinColumn = columns.asin; - const brandCol = findColumn(headers, ["brand"]); - const categoryCol = findColumn(headers, ["category"]); - const amazonRankCol = findColumn(headers, ["amazon rank", "amazonrank", "sales rank", "bsr"]); - const fbaNetCol = findColumn(headers, ["fba net", "fbanet", "fba_net"]); - const grossProfitCol = findColumn(headers, ["gross profit $", "gross profit", "grossprofit"]); - const grossProfitPctCol = findColumn(headers, ["gross profit %", "gross profit pct", "grossprofitpct"]); - const moqCol = findColumn(headers, ["moq", "min order qty", "minimum order quantity"]); - const moqCostCol = findColumn(headers, ["moq cost", "moqcost"]); - const totalQtyCol = findColumn(headers, ["total qty avail", "totalqtyavail", "qty available", "quantity"]); + if (!asinColumn) + throw new Error( + `No ASIN column found. Available columns: ${headers.join(", ")}`, + ); - const linkCol = findColumn(headers, ["link", "url", "source"]); + logColumnDetection(headers, columns); - if (!asinCol) throw new Error(`No ASIN column found. Available columns: ${headers.join(", ")}`); - - const knownCols = new Set([asinCol, nameCol, costCol, brandCol, categoryCol, amazonRankCol, fbaNetCol, grossProfitCol, grossProfitPctCol, moqCol, moqCostCol, totalQtyCol, linkCol].filter(Boolean)); + const knownCols = getKnownColumns(columns); const products: ProductRecord[] = []; for (const row of rows) { - const asin = String(row[asinCol] ?? "").trim().toUpperCase(); - if (!asin || !/^B[0-9A-Z]{9}$/.test(asin)) { - console.warn(`Skipping invalid ASIN: "${asin}"`); - continue; - } + const asin = parseAsin(row[asinColumn]); + if (!asin) continue; - const name = nameCol ? String(row[nameCol] ?? "") : ""; - const unitCost = costCol ? parseFloat(String(row[costCol] ?? "0")) : 0; + const sourceUrl = getOptionalString(row, columns.sourceUrl); + const asinLink = getOptionalString(row, columns.asinLink); + const link = sourceUrl ?? asinLink ?? getOptionalString(row, columns.link); - const extra: Record = {}; - for (const h of headers) { - if (!knownCols.has(h)) extra[h] = row[h]; - } + const extra = getExtraFields(row, headers, knownCols); + const netProfitFromSheet = getOptionalNumber(row, columns.netProfit); + const roiFromSheet = getOptionalNumber(row, columns.roi); products.push({ asin, - name, - unitCost, - brand: brandCol ? String(row[brandCol] ?? "") : undefined, - category: categoryCol ? String(row[categoryCol] ?? "") : undefined, - amazonRank: amazonRankCol ? Number(row[amazonRankCol]) || undefined : undefined, - fbaNet: fbaNetCol ? Number(row[fbaNetCol]) || undefined : undefined, - grossProfit: grossProfitCol ? Number(row[grossProfitCol]) || undefined : undefined, - grossProfitPct: grossProfitPctCol ? Number(row[grossProfitPctCol]) || undefined : undefined, - moq: moqCol ? Number(row[moqCol]) || undefined : undefined, - moqCost: moqCostCol ? Number(row[moqCostCol]) || undefined : undefined, - totalQtyAvail: totalQtyCol ? Number(row[totalQtyCol]) || undefined : undefined, - - link: linkCol ? String(row[linkCol] ?? "") : undefined, + name: getOptionalString(row, columns.name) ?? "", + unitCost: getOptionalNumber(row, columns.cost) ?? 0, + brand: getOptionalString(row, columns.brand), + category: getOptionalString(row, columns.category), + amazonRank: getOptionalNumber(row, columns.amazonRank), + avgPrice90FromSheet: getOptionalNumber(row, columns.avgPrice90), + sellingPriceFromSheet: getOptionalNumber(row, columns.sellingPrice), + fbaNet: getOptionalNumber(row, columns.fbaNet), + grossProfit: getOptionalNumber(row, columns.grossProfit) ?? netProfitFromSheet, + grossProfitPct: + getOptionalNumber(row, columns.grossProfitPct) ?? roiFromSheet, + netProfitFromSheet, + roiFromSheet, + moq: getOptionalNumber(row, columns.moq), + moqCost: getOptionalNumber(row, columns.moqCost), + totalQtyAvail: getOptionalNumber(row, columns.totalQty), + link, + asinLink, + sourceUrl, + supplier: getOptionalString(row, columns.supplier), + promoCouponCode: getOptionalString(row, columns.promoCouponCode), + notes: getOptionalString(row, columns.notes), + leadDate: getOptionalString(row, columns.leadDate), ...extra, }); } @@ -72,10 +113,97 @@ export function readProducts(filePath: string): ProductRecord[] { return products; } -function findColumn(headers: string[], candidates: string[]): string | undefined { - for (const candidate of candidates) { - const match = headers.find((h) => h.toLowerCase().trim() === candidate); - if (match) return match; +function detectColumns(headers: string[]): ColumnMap { + const columns = {} as ColumnMap; + for (const key of Object.keys(COLUMN_CANDIDATES) as ColumnKey[]) { + columns[key] = findColumn(headers, [...COLUMN_CANDIDATES[key]]); } + return columns; +} + +function logColumnDetection(headers: string[], columns: ColumnMap): void { + console.log(`Found columns: ${headers.join(", ")}`); + console.log( + `Detected columns -> ASIN: ${columns.asin ?? "n/a"}, Name: ${columns.name ?? "n/a"}, Cost: ${columns.cost ?? "n/a"}, 90d Avg: ${columns.avgPrice90 ?? "n/a"}, Selling Price: ${columns.sellingPrice ?? "n/a"}, Net Profit: ${columns.netProfit ?? columns.grossProfit ?? "n/a"}, ROI: ${columns.roi ?? columns.grossProfitPct ?? "n/a"}, Source URL: ${columns.sourceUrl ?? "n/a"}, ASIN Link: ${columns.asinLink ?? "n/a"}`, + ); +} + +function getKnownColumns(columns: ColumnMap): Set { + return new Set(Object.values(columns).filter((column): column is string => !!column)); +} + +function parseAsin(value: unknown): string | undefined { + const asin = String(value ?? "") + .trim() + .toUpperCase(); + if (!asin || !ASIN_REGEX.test(asin)) { + console.warn(`Skipping invalid ASIN: "${asin}"`); + return undefined; + } + return asin; +} + +function getOptionalString( + row: Record, + column: string | undefined, +): string | undefined { + if (!column) return undefined; + return normalizeOptionalString(row[column]); +} + +function getOptionalNumber( + row: Record, + column: string | undefined, +): number | undefined { + if (!column) return undefined; + return parseOptionalNumber(row[column]); +} + +function getExtraFields( + row: Record, + headers: string[], + knownCols: Set, +): Record { + const extra: Record = {}; + for (const header of headers) { + if (!knownCols.has(header)) extra[header] = row[header]; + } + return extra; +} + +function findColumn( + headers: string[], + candidates: string[], +): string | undefined { + const normalizedCandidates = new Set(candidates.map(normalizeHeader)); + + for (const header of headers) { + if (normalizedCandidates.has(normalizeHeader(header))) { + return header; + } + } + return undefined; } + +function normalizeHeader(value: string): string { + return value + .toLowerCase() + .trim() + .replace(/%/g, " pct ") + .replace(/\$/g, " usd ") + .replace(/[^a-z0-9]/g, ""); +} + +function normalizeOptionalString(value: unknown): string | undefined { + if (value == null) return undefined; + const s = String(value).trim(); + return s.length > 0 ? s : undefined; +} + +function parseOptionalNumber(value: unknown): number | undefined { + if (value == null || value === "") return undefined; + const cleaned = String(value).trim().replace(/[$,%]/g, "").replace(/,/g, ""); + const parsed = Number(cleaned); + return Number.isFinite(parsed) ? parsed : undefined; +} diff --git a/src/types.ts b/src/types.ts index c290e4b..95afd4f 100644 --- a/src/types.ts +++ b/src/types.ts @@ -5,14 +5,24 @@ export interface ProductRecord { brand?: string; category?: string; amazonRank?: number; + avgPrice90FromSheet?: number; + sellingPriceFromSheet?: number; fbaNet?: number; grossProfit?: number; grossProfitPct?: number; + netProfitFromSheet?: number; + roiFromSheet?: number; moq?: number; moqCost?: number; totalQtyAvail?: number; link?: string; + asinLink?: string; + sourceUrl?: string; + supplier?: string; + promoCouponCode?: string; + notes?: string; + leadDate?: string; [key: string]: unknown; } diff --git a/src/writer.ts b/src/writer.ts index 8b41168..a7b5063 100644 --- a/src/writer.ts +++ b/src/writer.ts @@ -2,17 +2,25 @@ import * as XLSX from "xlsx"; import type { AnalysisResult } from "./types.ts"; function buildRow(r: AnalysisResult) { - const price = r.product.keepa?.currentPrice ?? r.product.spApi.estimatedSalePrice; + const price = + r.product.keepa?.currentPrice ?? + r.product.record.sellingPriceFromSheet ?? + r.product.spApi.estimatedSalePrice; const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank; return { ASIN: r.product.record.asin, Name: r.product.record.name, Brand: r.product.record.brand ?? "", - Category: r.product.record.category ?? r.product.keepa?.categoryTree?.join(" > ") ?? "", + Category: + r.product.record.category ?? + r.product.keepa?.categoryTree?.join(" > ") ?? + "", "Unit Cost": r.product.record.unitCost, "Current Price": price ?? "", "Avg Price 90d": r.product.keepa?.avgPrice90 ?? "", + "Avg Price 90d (sheet)": r.product.record.avgPrice90FromSheet ?? "", + "Selling Price (sheet)": r.product.record.sellingPriceFromSheet ?? "", "Sales Rank": rank ?? "", "Rank Avg 90d": r.product.keepa?.salesRankAvg90 ?? "", Sellers: r.product.keepa?.sellerCount ?? "", @@ -22,9 +30,17 @@ function buildRow(r: AnalysisResult) { "FBA Net (sheet)": r.product.record.fbaNet ?? "", "Gross Profit $": r.product.record.grossProfit ?? "", "Gross Profit %": r.product.record.grossProfitPct ?? "", + "Net Profit (sheet)": r.product.record.netProfitFromSheet ?? "", + "ROI (sheet)": r.product.record.roiFromSheet ?? "", MOQ: r.product.record.moq ?? "", "MOQ Cost": r.product.record.moqCost ?? "", "Qty Available": r.product.record.totalQtyAvail ?? "", + Supplier: r.product.record.supplier ?? "", + "Source URL": r.product.record.sourceUrl ?? "", + "ASIN Link": r.product.record.asinLink ?? "", + "Promo/Coupon Code": r.product.record.promoCouponCode ?? "", + Notes: r.product.record.notes ?? "", + "Lead Date": r.product.record.leadDate ?? "", "FBA Fee": r.product.spApi.fbaFee, "FBM Fee": r.product.spApi.fbmFee, "Referral %": r.product.spApi.referralFeePercent, @@ -53,10 +69,15 @@ export function printResults(results: AnalysisResult[]): void { FBM: results.filter((r) => r.verdict.verdict === "FBM").length, SKIP: results.filter((r) => r.verdict.verdict === "SKIP").length, }; - console.log(`\nSummary: ${summary.FBA} FBA | ${summary.FBM} FBM | ${summary.SKIP} SKIP out of ${results.length} products\n`); + console.log( + `\nSummary: ${summary.FBA} FBA | ${summary.FBM} FBM | ${summary.SKIP} SKIP out of ${results.length} products\n`, + ); } -export function writeResultsCsv(results: AnalysisResult[], outputPath: string): void { +export function writeResultsCsv( + results: AnalysisResult[], + outputPath: string, +): void { const rows = results.map(buildRow); const ws = XLSX.utils.json_to_sheet(rows);