From 8ffbd48c4640a76887e923d1992dc82076f91ecd Mon Sep 17 00:00:00 2001 From: Victor Noguera Date: Sun, 12 Apr 2026 23:41:40 -0400 Subject: [PATCH] feat: implement SQLite persistent storage for analysis results - Implemented database initialization and connection management in `database.ts` using Bun's SQLite. - Created `runs` and `results` tables to track historical analysis metadata and detailed product performance. - Updated `writer.ts` to persist analysis results to the database within a transaction, replacing the previous CSV output logic. - Updated README and `.gitignore` to reflect the new persistent storage capability. --- .gitignore | 6 +++ README.md | 13 ++++- src/database.ts | 80 ++++++++++++++++++++++++++++ src/index.ts | 14 +++-- src/writer.ts | 135 +++++++++++++++++++++++++++++++++++++++++------- 5 files changed, 223 insertions(+), 25 deletions(-) create mode 100644 src/database.ts diff --git a/.gitignore b/.gitignore index 694ac49..c68fdaa 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,9 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json *.xlsx *.csv + +results.db + +results.db-shm + +results.db-wal diff --git a/README.md b/README.md index f5f5afe..3752f9a 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,18 @@ Numeric parsing accepts plain numbers as well as formatted values like `$12.50`, 4. **Keepa fetch** — batch the sellable (uncached) ASINs in a single API call (up to 100 per request) 5. **Enrich** — fetch SP-API pricing + FBA/FBM fees for sellable ASINs; combine with Keepa data and spreadsheet data 6. **LLM analysis** — send batches of 5 sellable products to LM Studio for FBA/FBM/SKIP verdict; skipped ASINs get auto-SKIP verdict (confidence 100) and bypass LLM entirely -7. **Output** — print results table to console (includes all ASINs), optionally write CSV/XLSX +7. **Output** — print results table to console (includes all ASINs), optionally write CSV/XLSX, and **persist results to a SQLite database**. + +## Persistent Storage with SQLite + +Results from each run are now stored in a SQLite database named `results.db` in the project root. The SQLite implementation details are handled in `src/database.ts`. This allows you to: +- Revisit past analysis results. +- Query and analyze historical data. +- Track product performance over time. + +The database will automatically be created if it doesn't exist. Two tables are created: +- `runs`: Stores metadata about each analysis run (timestamp, input file, output file, and summary counts). +- `results`: Stores detailed analysis results for each product from each run, linked to the `runs` table. ## Output columns diff --git a/src/database.ts b/src/database.ts new file mode 100644 index 0000000..fd0a0be --- /dev/null +++ b/src/database.ts @@ -0,0 +1,80 @@ +import { Database } from "bun:sqlite"; + +let db: Database | null = null; + +export function getDb(dbPath: string): Database { + if (!db) { + db = new Database(dbPath); + db.run("PRAGMA journal_mode = WAL;"); // Enable WAL mode for better performance + } + return db; +} + +export function closeDb(): void { + if (db) { + db.close(); + db = null; + } +} + +export function initDb(dbPath: string): void { + const database = getDb(dbPath); + database.run(` + CREATE TABLE IF NOT EXISTS runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp TEXT NOT NULL, + input_file TEXT NOT NULL, + output_file TEXT, + total_products INTEGER, + fba_count INTEGER, + fbm_count INTEGER, + skip_count INTEGER + ); + `); + database.run(` + CREATE TABLE IF NOT EXISTS results ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_id INTEGER NOT NULL, + asin TEXT NOT NULL, + product_name TEXT, + brand TEXT, + category TEXT, + unit_cost REAL, + current_price REAL, + avg_price_90d REAL, + avg_price_90d_sheet REAL, + selling_price_sheet REAL, + sales_rank INTEGER, + rank_avg_90d INTEGER, + sellers INTEGER, + monthly_sold INTEGER, + rank_drops_30d INTEGER, + rank_drops_90d INTEGER, + fba_net_sheet REAL, + gross_profit_dollar REAL, + gross_profit_pct REAL, + net_profit_sheet REAL, + roi_sheet REAL, + moq INTEGER, + moq_cost REAL, + qty_available INTEGER, + supplier TEXT, + source_url TEXT, + asin_link TEXT, + promo_coupon_code TEXT, + notes TEXT, + lead_date TEXT, + fba_fee REAL, + fbm_fee REAL, + referral_percent REAL, + can_sell TEXT, + sellability_status TEXT, + sellability_reason TEXT, + verdict TEXT NOT NULL, + confidence INTEGER, + reasoning TEXT, + fetched_at TEXT NOT NULL, + FOREIGN KEY (run_id) REFERENCES runs(id) + ); + `); +} diff --git a/src/index.ts b/src/index.ts index 5f7d25b..3148aac 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,7 +3,10 @@ import { fetchKeepaDataBatch } from "./keepa.ts"; import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts"; import { connectCache, getCache, setCache, disconnectCache } from "./cache.ts"; import { analyzeProducts } from "./llm.ts"; -import { printResults, writeResultsCsv } from "./writer.ts"; +import { printResults, writeResultsToDb } from "./writer.ts"; +import { initDb, closeDb } from "./database.ts"; + +const DB_PATH = "./results.db"; import type { EnrichedProduct, AnalysisResult, @@ -35,6 +38,10 @@ async function main() { console.log("Connecting to Redis..."); await connectCache(); + // Initialize SQLite DB + console.log("Initializing SQLite database..."); + initDb(DB_PATH); + // Phase 1: Read input file console.log(`\nReading ${inputFile}...`); const products = readProducts(inputFile); @@ -279,11 +286,10 @@ async function main() { printResults(allResults); - if (outputFile) { - writeResultsCsv(allResults, outputFile); - } + writeResultsToDb(allResults, DB_PATH, inputFile, outputFile); await disconnectCache(); + closeDb(); } main().catch((err) => { diff --git a/src/writer.ts b/src/writer.ts index 2172af9..667f072 100644 --- a/src/writer.ts +++ b/src/writer.ts @@ -1,4 +1,4 @@ -import * as XLSX from "xlsx"; +import { getDb } from "./database.ts"; import type { AnalysisResult } from "./types.ts"; function buildRow(r: AnalysisResult) { @@ -7,6 +7,12 @@ function buildRow(r: AnalysisResult) { r.product.record.sellingPriceFromSheet ?? r.product.spApi.estimatedSalePrice; const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank; + const canSellStatus = + r.product.spApi.canSell == null + ? "unknown" + : r.product.spApi.canSell + ? "yes" + : "no"; return { ASIN: r.product.record.asin, @@ -44,12 +50,7 @@ function buildRow(r: AnalysisResult) { "FBA Fee": r.product.spApi.fbaFee, "FBM Fee": r.product.spApi.fbmFee, "Referral %": r.product.spApi.referralFeePercent, - "Can Sell": - r.product.spApi.canSell == null - ? "unknown" - : r.product.spApi.canSell - ? "yes" - : "no", + "Can Sell": canSellStatus, Sellability: r.product.spApi.sellabilityStatus, "Sellability Reason": r.product.spApi.sellabilityReason ?? "", Verdict: r.verdict.verdict, @@ -58,6 +59,113 @@ function buildRow(r: AnalysisResult) { }; } +export function writeResultsToDb( + results: AnalysisResult[], + dbPath: string, + inputFile: string, + outputFile: string | undefined, +): void { + const database = getDb(dbPath); + + const timestamp = new Date().toISOString(); + const fbaCount = results.filter((r) => r.verdict.verdict === "FBA").length; + const fbmCount = results.filter((r) => r.verdict.verdict === "FBM").length; + const skipCount = results.filter((r) => r.verdict.verdict === "SKIP").length; + + const insertRun = database.prepare( + `INSERT INTO runs ( + timestamp, + input_file, + output_file, + total_products, + fba_count, + fbm_count, + skip_count + ) VALUES (?, ?, ?, ?, ?, ?, ?)`, + ); + const runInfo = insertRun.run( + timestamp, + inputFile, + outputFile ?? null, + results.length, + fbaCount, + fbmCount, + skipCount, + ); + const runId = + (runInfo.changes as number) > 0 + ? (runInfo.lastInsertRowid as number) + : null; + + if (runId === null) { + console.error("Failed to insert run record into SQLite."); + return; + } + + const insertResult = database.prepare( + `INSERT INTO results ( + run_id, asin, product_name, brand, category, unit_cost, current_price, + avg_price_90d, avg_price_90d_sheet, selling_price_sheet, sales_rank, rank_avg_90d, + sellers, monthly_sold, rank_drops_30d, rank_drops_90d, fba_net_sheet, + gross_profit_dollar, gross_profit_pct, net_profit_sheet, roi_sheet, moq, moq_cost, + qty_available, supplier, source_url, asin_link, promo_coupon_code, notes, lead_date, + fba_fee, fbm_fee, referral_percent, can_sell, sellability_status, sellability_reason, + verdict, confidence, reasoning, fetched_at + ) VALUES ( + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + )`, + ); + + database.transaction(() => { + for (const r of results) { + const row = buildRow(r); + insertResult.run( + runId, + row.ASIN, + row.Name, + row.Brand, + row.Category, + row["Unit Cost"] ?? null, + row["Current Price"] ?? null, + row["Avg Price 90d"] ?? null, + row["Avg Price 90d (sheet)"] ?? null, + row["Selling Price (sheet)"] ?? null, + row["Sales Rank"] ?? null, + row["Rank Avg 90d"] ?? null, + row.Sellers ?? null, + row["Monthly Sold"] ?? null, + row["Rank Drops 30d"] ?? null, + row["Rank Drops 90d"] ?? null, + row["FBA Net (sheet)"] ?? null, + row["Gross Profit $"] ?? null, + row["Gross Profit %"] ?? null, + row["Net Profit (sheet)"] ?? null, + row["ROI (sheet)"] ?? null, + row.MOQ ?? null, + row["MOQ Cost"] ?? null, + row["Qty Available"] ?? null, + row.Supplier ?? null, + row["Source URL"] ?? null, + row["ASIN Link"] ?? null, + row["Promo/Coupon Code"] ?? null, + row.Notes ?? null, + row["Lead Date"] ?? null, + row["FBA Fee"] ?? null, + row["FBM Fee"] ?? null, + row["Referral %"] ?? null, + row["Can Sell"], + row.Sellability, + row["Sellability Reason"] ?? null, + row.Verdict, + row.Confidence ?? null, + row.Reasoning, + r.product.fetchedAt, + ); + } + })(); + console.log(`Results written to SQLite database for run_id: ${runId}`); +} export function printResults(results: AnalysisResult[]): void { const rows = results .filter((r) => r.verdict.verdict === "FBA" || r.verdict.verdict === "FBM") @@ -144,16 +252,3 @@ export function printResults(results: AnalysisResult[]): void { `Sellability: ${summary.Available} available | ${summary.Restricted} restricted | ${summary.NotAvailable} not_available | ${summary.Unknown} unknown\n`, ); } - -export function writeResultsCsv( - results: AnalysisResult[], - outputPath: string, -): void { - const rows = results.map(buildRow); - - const ws = XLSX.utils.json_to_sheet(rows); - const wb = XLSX.utils.book_new(); - XLSX.utils.book_append_sheet(wb, ws, "Results"); - XLSX.writeFile(wb, outputPath); - console.log(`Results written to ${outputPath}`); -}