Merge branches 'main' and 'main' of https://github.com/nvictorme/asin-check

.gitignore

@@ -35,3 +35,9 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
 *.xlsx
 *.csv
 
+
+results.db
+
+results.db-shm
+
+results.db-wal

README.md

@@ -96,9 +96,21 @@ Numeric parsing accepts plain numbers as well as formatted values like `$12.50`,
 3. **Sellability gate** — check all uncached ASINs against SP-API `getListingsRestrictions` (concurrency: 5 workers); immediately skip ASINs with status `not_available` and `canSell=false` (no Keepa/fees wasted)
 4. **Keepa fetch** — batch the sellable (uncached) ASINs in a single API call (up to 100 per request)
 5. **Enrich** — fetch SP-API pricing + FBA/FBM fees for sellable ASINs; combine with Keepa data and spreadsheet data
-6. **LLM analysis** — send batches of 5 available products to LM Studio for FBA/FBM/SKIP verdict
+6. **LLM analysis** — send batches of 5 sellable products to LM Studio for FBA/FBM/SKIP verdict; skipped ASINs get auto-SKIP verdict (confidence 100) and bypass LLM entirely
-7. **Chunk orchestration** — if input size is greater than 50, run phases 2-6 for each 50-item chunk sequentially
+7. **Output** — print results table to console (includes all ASINs), optionally write CSV/XLSX, and **persist results to a SQLite database**.
-8. **Output** — print results table to console (includes all ASINs); for chunked runs, always write seriated chunk files (`*_part_001`, `*_part_002`, ...); for non-chunked runs, write a single file only when `--out` is provided
+
+## Persistent Storage with SQLite
+
+Results from each run are now stored in a SQLite database named `results.db` in the project root. The SQLite implementation details are handled in `src/database.ts`. This allows you to:
+
+- Revisit past analysis results.
+- Query and analyze historical data.
+- Track product performance over time.
+
+The database will automatically be created if it doesn't exist. Two tables are created:
+
+- `runs`: Stores metadata about each analysis run (timestamp, input file, output file, and summary counts).
+- `results`: Stores detailed analysis results for each product from each run, linked to the `runs` table.
 
 ## Output columns
 

src/database.ts

@@ -0,0 +1,80 @@
+import { Database } from "bun:sqlite";
+
+let db: Database | null = null;
+
+export function getDb(dbPath: string): Database {
+  if (!db) {
+    db = new Database(dbPath);
+    db.run("PRAGMA journal_mode = WAL;"); // Enable WAL mode for better performance
+  }
+  return db;
+}
+
+export function closeDb(): void {
+  if (db) {
+    db.close();
+    db = null;
+  }
+}
+
+export function initDb(dbPath: string): void {
+  const database = getDb(dbPath);
+  database.run(`
+    CREATE TABLE IF NOT EXISTS runs (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      timestamp TEXT NOT NULL,
+      input_file TEXT NOT NULL,
+      output_file TEXT,
+      total_products INTEGER,
+      fba_count INTEGER,
+      fbm_count INTEGER,
+      skip_count INTEGER
+    );
+  `);
+  database.run(`
+    CREATE TABLE IF NOT EXISTS results (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      run_id INTEGER NOT NULL,
+      asin TEXT NOT NULL,
+      product_name TEXT,
+      brand TEXT,
+      category TEXT,
+      unit_cost REAL,
+      current_price REAL,
+      avg_price_90d REAL,
+      avg_price_90d_sheet REAL,
+      selling_price_sheet REAL,
+      sales_rank INTEGER,
+      rank_avg_90d INTEGER,
+      sellers INTEGER,
+      monthly_sold INTEGER,
+      rank_drops_30d INTEGER,
+      rank_drops_90d INTEGER,
+      fba_net_sheet REAL,
+      gross_profit_dollar REAL,
+      gross_profit_pct REAL,
+      net_profit_sheet REAL,
+      roi_sheet REAL,
+      moq INTEGER,
+      moq_cost REAL,
+      qty_available INTEGER,
+      supplier TEXT,
+      source_url TEXT,
+      asin_link TEXT,
+      promo_coupon_code TEXT,
+      notes TEXT,
+      lead_date TEXT,
+      fba_fee REAL,
+      fbm_fee REAL,
+      referral_percent REAL,
+      can_sell TEXT,
+      sellability_status TEXT,
+      sellability_reason TEXT,
+      verdict TEXT NOT NULL,
+      confidence INTEGER,
+      reasoning TEXT,
+      fetched_at TEXT NOT NULL,
+      FOREIGN KEY (run_id) REFERENCES runs(id)
+    );
+  `);
+}

src/index.ts

@@ -3,8 +3,10 @@ import { fetchKeepaDataBatch } from "./keepa.ts";
 import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts";
 import { connectCache, getCache, setCache, disconnectCache } from "./cache.ts";
 import { analyzeProducts } from "./llm.ts";
-import { printResults, writeResultsCsv } from "./writer.ts";
+import { printResults, writeResultsToDb } from "./writer.ts";
-import path from "node:path";
+import { initDb, closeDb } from "./database.ts";
+
+const DB_PATH = "./results.db";
 import type {
   EnrichedProduct,
   AnalysisResult,
@@ -32,38 +34,26 @@ function parseArgs(): { inputFile: string; outputFile?: string } {
   return { inputFile, outputFile };
 }
 
-function chunkArray<T>(items: T[], chunkSize: number): T[][] {
+async function main() {
-  const chunks: T[][] = [];
+  const { inputFile, outputFile } = parseArgs();
-  for (let i = 0; i < items.length; i += chunkSize) {
+
-    chunks.push(items.slice(i, i + chunkSize));
+  console.log("Connecting to Redis...");
+  await connectCache();
+
+  // Initialize SQLite DB
+  console.log("Initializing SQLite database...");
+  initDb(DB_PATH);
+
+  // Phase 1: Read input file
+  console.log(`\nReading ${inputFile}...`);
+  const products = readProducts(inputFile);
+
+  if (products.length === 0) {
+    console.error("No valid products found in input file.");
+    process.exit(1);
   }
-  return chunks;
-}
 
-function resolveBaseOutputPath(inputFile: string, outputFile?: string): string {
+  // Phase 2: Check cache for all ASINs
-  if (outputFile) return outputFile;
-
-  const parsedInput = path.parse(inputFile);
-  return path.join(parsedInput.dir, `${parsedInput.name}_results.xlsx`);
-}
-
-function buildChunkOutputPath(
-  baseOutputPath: string,
-  chunkIndex: number,
-): string {
-  const parsed = path.parse(baseOutputPath);
-  const extension = parsed.ext || ".xlsx";
-  const chunkSuffix = String(chunkIndex + 1).padStart(3, "0");
-  return path.join(
-    parsed.dir,
-    `${parsed.name}_part_${chunkSuffix}${extension}`,
-  );
-}
-
-async function processProductChunk(
-  products: ProductRecord[],
-): Promise<AnalysisResult[]> {
-  // Phase 2: Check cache for all ASINs in chunk
   console.log(`\nChecking cache for ${products.length} products...`);
   const cached = new Map<string, EnrichedProduct>();
   const excludedCachedAsins = new Set<string>();
@@ -331,12 +321,10 @@ async function main() {
 
     printResults(allResults);
 
-    if (!hasMultipleChunks && outputFile) {
+  writeResultsToDb(allResults, DB_PATH, inputFile, outputFile);
-      writeResultsCsv(allResults, outputFile);
+
-    }
-  } finally {
   await disconnectCache();
-  }
+  closeDb();
 }
 
 main().catch((err) => {

src/writer.ts

@@ -1,4 +1,4 @@
-import * as XLSX from "xlsx";
+import { getDb } from "./database.ts";
 import type { AnalysisResult } from "./types.ts";
 
 function buildRow(r: AnalysisResult) {
@@ -7,6 +7,12 @@ function buildRow(r: AnalysisResult) {
     r.product.record.sellingPriceFromSheet ??
     r.product.spApi.estimatedSalePrice;
   const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank;
+  const canSellStatus =
+    r.product.spApi.canSell == null
+      ? "unknown"
+      : r.product.spApi.canSell
+        ? "yes"
+        : "no";
 
   return {
     ASIN: r.product.record.asin,
@@ -44,12 +50,7 @@ function buildRow(r: AnalysisResult) {
     "FBA Fee": r.product.spApi.fbaFee,
     "FBM Fee": r.product.spApi.fbmFee,
     "Referral %": r.product.spApi.referralFeePercent,
-    "Can Sell":
+    "Can Sell": canSellStatus,
-      r.product.spApi.canSell == null
-        ? "unknown"
-        : r.product.spApi.canSell
-          ? "yes"
-          : "no",
     Sellability: r.product.spApi.sellabilityStatus,
     "Sellability Reason": r.product.spApi.sellabilityReason ?? "",
     Verdict: r.verdict.verdict,
@@ -58,6 +59,113 @@ function buildRow(r: AnalysisResult) {
   };
 }
 
+export function writeResultsToDb(
+  results: AnalysisResult[],
+  dbPath: string,
+  inputFile: string,
+  outputFile: string | undefined,
+): void {
+  const database = getDb(dbPath);
+
+  const timestamp = new Date().toISOString();
+  const fbaCount = results.filter((r) => r.verdict.verdict === "FBA").length;
+  const fbmCount = results.filter((r) => r.verdict.verdict === "FBM").length;
+  const skipCount = results.filter((r) => r.verdict.verdict === "SKIP").length;
+
+  const insertRun = database.prepare(
+    `INSERT INTO runs (
+      timestamp,
+      input_file,
+      output_file,
+      total_products,
+      fba_count,
+      fbm_count,
+      skip_count
+    ) VALUES (?, ?, ?, ?, ?, ?, ?)`,
+  );
+  const runInfo = insertRun.run(
+    timestamp,
+    inputFile,
+    outputFile ?? null,
+    results.length,
+    fbaCount,
+    fbmCount,
+    skipCount,
+  );
+  const runId =
+    (runInfo.changes as number) > 0
+      ? (runInfo.lastInsertRowid as number)
+      : null;
+
+  if (runId === null) {
+    console.error("Failed to insert run record into SQLite.");
+    return;
+  }
+
+  const insertResult = database.prepare(
+    `INSERT INTO results (
+      run_id, asin, product_name, brand, category, unit_cost, current_price,
+      avg_price_90d, avg_price_90d_sheet, selling_price_sheet, sales_rank, rank_avg_90d,
+      sellers, monthly_sold, rank_drops_30d, rank_drops_90d, fba_net_sheet,
+      gross_profit_dollar, gross_profit_pct, net_profit_sheet, roi_sheet, moq, moq_cost,
+      qty_available, supplier, source_url, asin_link, promo_coupon_code, notes, lead_date,
+      fba_fee, fbm_fee, referral_percent, can_sell, sellability_status, sellability_reason,
+      verdict, confidence, reasoning, fetched_at
+    ) VALUES (
+      ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
+      ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
+    )`,
+  );
+
+  database.transaction(() => {
+    for (const r of results) {
+      const row = buildRow(r);
+      insertResult.run(
+        runId,
+        row.ASIN,
+        row.Name,
+        row.Brand,
+        row.Category,
+        row["Unit Cost"] ?? null,
+        row["Current Price"] ?? null,
+        row["Avg Price 90d"] ?? null,
+        row["Avg Price 90d (sheet)"] ?? null,
+        row["Selling Price (sheet)"] ?? null,
+        row["Sales Rank"] ?? null,
+        row["Rank Avg 90d"] ?? null,
+        row.Sellers ?? null,
+        row["Monthly Sold"] ?? null,
+        row["Rank Drops 30d"] ?? null,
+        row["Rank Drops 90d"] ?? null,
+        row["FBA Net (sheet)"] ?? null,
+        row["Gross Profit $"] ?? null,
+        row["Gross Profit %"] ?? null,
+        row["Net Profit (sheet)"] ?? null,
+        row["ROI (sheet)"] ?? null,
+        row.MOQ ?? null,
+        row["MOQ Cost"] ?? null,
+        row["Qty Available"] ?? null,
+        row.Supplier ?? null,
+        row["Source URL"] ?? null,
+        row["ASIN Link"] ?? null,
+        row["Promo/Coupon Code"] ?? null,
+        row.Notes ?? null,
+        row["Lead Date"] ?? null,
+        row["FBA Fee"] ?? null,
+        row["FBM Fee"] ?? null,
+        row["Referral %"] ?? null,
+        row["Can Sell"],
+        row.Sellability,
+        row["Sellability Reason"] ?? null,
+        row.Verdict,
+        row.Confidence ?? null,
+        row.Reasoning,
+        r.product.fetchedAt,
+      );
+    }
+  })();
+  console.log(`Results written to SQLite database for run_id: ${runId}`);
+}
 export function printResults(results: AnalysisResult[]): void {
   const rows = results
     .filter((r) => r.verdict.verdict === "FBA" || r.verdict.verdict === "FBM")
@@ -144,16 +252,3 @@ export function printResults(results: AnalysisResult[]): void {
     `Sellability: ${summary.Available} available | ${summary.Restricted} restricted | ${summary.NotAvailable} not_available | ${summary.Unknown} unknown\n`,
   );
 }
-
-export function writeResultsCsv(
-  results: AnalysisResult[],
-  outputPath: string,
-): void {
-  const rows = results.map(buildRow);
-
-  const ws = XLSX.utils.json_to_sheet(rows);
-  const wb = XLSX.utils.book_new();
-  XLSX.utils.book_append_sheet(wb, ws, "Results");
-  XLSX.writeFile(wb, outputPath);
-  console.log(`Results written to ${outputPath}`);
-}