diff --git a/.abacusai/config.json b/.abacusai/config.json new file mode 100644 index 0000000..60b41cb --- /dev/null +++ b/.abacusai/config.json @@ -0,0 +1,16 @@ +{ + "permissions": { + "allow": [ + "Bash(bun install 2>&1 *)", + "Bash(bun run build:web *)", + "Bash(bun *)", + "Bash(curl *)", + "Bash(curl *)", + "Bash(curl *)", + "KillShell", + "Bash(bunx *)", + "Bash(git *)", + "Bash(ls *)" + ] + } +} \ No newline at end of file diff --git a/.gitignore b/.gitignore index e7a2b1f..17d7179 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,5 @@ results.db-wal output/ temp_output/ + +dist-server/ diff --git a/bun.lock b/bun.lock index 537ab31..b5784ae 100644 --- a/bun.lock +++ b/bun.lock @@ -7,10 +7,14 @@ "dependencies": { "amazon-sp-api": "^1.2.1", "ioredis": "^5.10.1", + "react": "^19.2.0", + "react-dom": "^19.2.0", "xlsx": "^0.18.5", }, "devDependencies": { "@types/bun": "latest", + "@types/react": "^19.2.14", + "@types/react-dom": "^19.2.3", }, "peerDependencies": { "typescript": "^5", @@ -24,6 +28,10 @@ "@types/node": ["@types/node@25.5.2", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-tO4ZIRKNC+MDWV4qKVZe3Ql/woTnmHDr5JD8UI5hn2pwBrHEwOEMZK7WlNb5RKB6EoJ02gwmQS9OrjuFnZYdpg=="], + "@types/react": ["@types/react@19.2.14", "", { "dependencies": { "csstype": "^3.2.2" } }, "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w=="], + + "@types/react-dom": ["@types/react-dom@19.2.3", "", { "peerDependencies": { "@types/react": "^19.2.0" } }, "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ=="], + "adler-32": ["adler-32@1.3.1", "", {}, "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A=="], "amazon-sp-api": ["amazon-sp-api@1.2.1", "", { "dependencies": { "csvtojson": "^2.0.14", "fast-xml-parser": "^5.3.1", "iconv-lite": "^0.7.0", "qs": "^6.14.0" } }, "sha512-zxX3KtoCDx0wxkkBgFM6qew49JJoL1XZQgUnztfp+8Im2HLHBAt4beSiDo/AkH00Gr8paHBAjdcJY6LC6ISU7w=="], @@ -42,6 +50,8 @@ "crc-32": ["crc-32@1.2.2", "", { "bin": { "crc32": "bin/crc32.njs" } }, "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ=="], + "csstype": ["csstype@3.2.3", "", {}, "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ=="], + "csvtojson": ["csvtojson@2.0.14", "", { "dependencies": { "lodash": "^4.17.21" }, "bin": { "csvtojson": "bin/csvtojson" } }, "sha512-F7NNvhhDyob7OsuEGRsH0FM1aqLs/WYITyza3l+hTEEmOK9sGPBlYQZwlVG0ezCojXYpE17lhS5qL6BCOZSPyA=="], "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], @@ -94,12 +104,18 @@ "qs": ["qs@6.15.1", "", { "dependencies": { "side-channel": "^1.1.0" } }, "sha512-6YHEFRL9mfgcAvql/XhwTvf5jKcOiiupt2FiJxHkiX1z4j7WL8J/jRHYLluORvc1XxB5rV20KoeK00gVJamspg=="], + "react": ["react@19.2.5", "", {}, "sha512-llUJLzz1zTUBrskt2pwZgLq59AemifIftw4aB7JxOqf1HY2FDaGDxgwpAPVzHU1kdWabH7FauP4i1oEeer2WCA=="], + + "react-dom": ["react-dom@19.2.5", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.5" } }, "sha512-J5bAZz+DXMMwW/wV3xzKke59Af6CHY7G4uYLN1OvBcKEsWOs4pQExj86BBKamxl/Ik5bx9whOrvBlSDfWzgSag=="], + "redis-errors": ["redis-errors@1.2.0", "", {}, "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w=="], "redis-parser": ["redis-parser@3.0.0", "", { "dependencies": { "redis-errors": "^1.0.0" } }, "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A=="], "safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="], + "scheduler": ["scheduler@0.27.0", "", {}, "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q=="], + "side-channel": ["side-channel@1.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3", "side-channel-list": "^1.0.0", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" } }, "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw=="], "side-channel-list": ["side-channel-list@1.0.1", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.4" } }, "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w=="], diff --git a/package.json b/package.json index 93473a1..f994757 100644 --- a/package.json +++ b/package.json @@ -1,17 +1,28 @@ -{ - "name": "asin-check", - "module": "src/index.ts", - "type": "module", - "private": true, - "devDependencies": { - "@types/bun": "latest" - }, - "peerDependencies": { - "typescript": "^5" - }, - "dependencies": { - "amazon-sp-api": "^1.2.1", - "ioredis": "^5.10.1", - "xlsx": "^0.18.5" - } -} +{ + "name": "asin-check", + "module": "src/index.ts", + "type": "module", + "private": true, + "scripts": { + "bestsellers": "bun run src/bestsellers-by-category.ts", + "start": "bun run src/index.ts", + "start:web": "bun --hot src/server.ts", + "build:web": "bun build src/web/index.html --outdir dist", + "test": "bun test" + }, + "devDependencies": { + "@types/bun": "latest", + "@types/react": "^19.2.14", + "@types/react-dom": "^19.2.3" + }, + "peerDependencies": { + "typescript": "^5" + }, + "dependencies": { + "amazon-sp-api": "^1.2.1", + "ioredis": "^5.10.1", + "react": "^19.2.0", + "react-dom": "^19.2.0", + "xlsx": "^0.18.5" + } +} diff --git a/src/bestsellers-by-category.test.ts b/src/bestsellers-by-category.test.ts index b667329..bd75b79 100644 --- a/src/bestsellers-by-category.test.ts +++ b/src/bestsellers-by-category.test.ts @@ -1,17 +1,51 @@ import { test, expect, beforeAll, afterAll, beforeEach, mock } from "bun:test"; import { Database } from "bun:sqlite"; -import { getDb, initDb, closeDb } from "./database"; +import { getDb, initDb, closeDb } from "./database.ts"; import path from "node:path"; import { rmSync, mkdirSync } from "node:fs"; -import { - main, - processCategory, - insertCategoryRunSummary, - insertProductAnalysisResults, -} from "./bestsellers-by-category"; -import * as keepaModule from "./keepa"; -import * as spApiModule from "./sp-api"; -import * as llmModule from "./llm"; + +const fetchSellabilityBatchMock = mock(async (asins: string[]) => { + return new Map( + asins.map((asin) => [ + asin, + { + canSell: true, + sellabilityStatus: "available" as const, + sellabilityReason: "ok", + }, + ]), + ); +}); + +const fetchSpApiPricingAndFeesMock = mock(async () => ({ + fbaFee: 4, + fbmFee: 2, + referralFeePercent: 15, + estimatedSalePrice: 25, + canSell: true, + sellabilityStatus: "available" as const, + sellabilityReason: "ok", +})); + +const analyzeProductsMock = mock(async (products: any[]) => { + return products.map((p, idx) => ({ + asin: p.record.asin, + verdict: idx === 0 ? "FBA" : "FBM", + confidence: 90, + reasoning: "mocked", + })); +}); + +mock.module("./sp-api.ts", () => ({ + fetchSellabilityBatch: fetchSellabilityBatchMock, + fetchSpApiPricingAndFees: fetchSpApiPricingAndFeesMock, +})); + +mock.module("./llm.ts", () => ({ + analyzeProducts: analyzeProductsMock, +})); + +const modulePromise = import("./bestsellers-by-category.ts"); const DB_TEST_PATH = path.join( process.cwd(), @@ -20,24 +54,81 @@ const DB_TEST_PATH = path.join( ); let db: Database; +let processCategory: (db: Database, runId: number, category: any, perCategoryTop: number) => Promise; +let insertCategoryRunSummary: (db: Database, summary: any, runTimestamp: string) => Promise; +let originalFetch: typeof globalThis.fetch; + +beforeAll(async () => { + const mod = await modulePromise; + processCategory = mod.processCategory; + insertCategoryRunSummary = mod.insertCategoryRunSummary; -beforeAll(() => { - // Ensure the test output directory exists and is clean rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true }); mkdirSync(path.dirname(DB_TEST_PATH), { recursive: true }); initDb(DB_TEST_PATH); db = getDb(DB_TEST_PATH); + + originalFetch = globalThis.fetch; }); afterAll(() => { + globalThis.fetch = originalFetch; closeDb(); rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true }); }); beforeEach(() => { - // Clear tables before each test if necessary, or use a fresh DB for each test - // For simplicity, we'll assume tables are clean after initDb in beforeAll - // and not clear for each test if data is not interdependent. + db.run("DELETE FROM product_analysis_results"); + db.run("DELETE FROM category_analysis_runs"); + + globalThis.fetch = mock(async (input: string | URL | Request) => { + const rawUrl = + typeof input === "string" + ? input + : input instanceof URL + ? input.toString() + : input.url; + const url = new URL(rawUrl); + + if (url.pathname === "/bestsellers") { + return new Response( + JSON.stringify({ + bestSellersList: ["B000000001", "B000000002"], + tokensLeft: 10, + refillRate: 1, + }), + { status: 200 }, + ); + } + + if (url.pathname === "/product") { + return new Response( + JSON.stringify({ + products: [ + { + asin: "B000000001", + title: "Product One", + stats: { current: [null, null, null, 1000, null, null, null, null, null, null, null, 2, null, null, null, null, null, null, 2599], avg: [2400, null, null, 1200] }, + csv: [[1, 2599]], + categoryTree: [{ name: "Category 1" }], + }, + { + asin: "B000000002", + title: "Product Two", + stats: { current: [null, null, null, 2000, null, null, null, null, null, null, null, 3, null, null, null, null, null, null, 1999], avg: [1800, null, null, 2200] }, + csv: [[1, 1999]], + categoryTree: [{ name: "Category 1" }], + }, + ], + tokensLeft: 10, + refillRate: 1, + }), + { status: 200 }, + ); + } + + return new Response("not found", { status: 404 }); + }) as unknown as typeof globalThis.fetch; }); test("processCategory function test", async () => { @@ -48,31 +139,21 @@ test("processCategory function test", async () => { childCount: 0, }; - const summary = await processCategory(db, mockCategory, 2); + const runId = await insertCategoryRunSummary(db, { + categoryId: mockCategory.id, + categoryLabel: mockCategory.label, + topAsinsChecked: 0, + availableAsins: 0, + fba: 0, + fbm: 0, + skip: 0, + status: "running", + error: "", + results: [], + }, new Date().toISOString()); + const summary = await processCategory(db, runId, mockCategory, 2); - expect(summary.status).toBe("ok"); - expect(summary.categoryId).toBe(mockCategory.id); - expect(summary.categoryLabel).toBe(mockCategory.label); - expect(summary.topAsinsChecked).toBe(2); - expect(summary.availableAsins).toBe(2); - expect(summary.fba).toBe(1); - expect(summary.fbm).toBe(1); - expect(summary.skip).toBe(0); - expect(summary.results?.length).toBe(2); - - const runId = await insertCategoryRunSummary( - db, - summary, - new Date().toISOString(), - ); - if (summary.results) { - await insertProductAnalysisResults(db, runId, summary.results); - } - - // Verify category run summary insertion - const categoryRun = db - .query("SELECT * FROM category_analysis_runs") - .all() as any[]; + const categoryRun = db.query("SELECT * FROM category_analysis_runs").all() as any[]; expect(categoryRun.length).toBe(1); expect(categoryRun[0].category_label).toBe("Category 1"); expect(categoryRun[0].top_asins_checked).toBe(2); @@ -81,10 +162,7 @@ test("processCategory function test", async () => { expect(categoryRun[0].fbm_count).toBe(1); expect(categoryRun[0].status).toBe("ok"); - // Verify product analysis results insertion - const productResults = db - .query("SELECT * FROM product_analysis_results ORDER BY asin") - .all() as any[]; + const productResults = db.query("SELECT * FROM product_analysis_results ORDER BY asin").all() as any[]; expect(productResults.length).toBe(2); expect(productResults[0].asin).toBe("B000000001"); diff --git a/src/bestsellers-by-category.ts b/src/bestsellers-by-category.ts index 4addf35..13231e9 100644 --- a/src/bestsellers-by-category.ts +++ b/src/bestsellers-by-category.ts @@ -13,6 +13,7 @@ import type { SellabilityInfo, SpApiData, } from "./types.ts"; + type CategoryInfo = { id: number; @@ -36,7 +37,7 @@ type CategoryRunSummary = { fba: number; fbm: number; skip: number; - status: "ok" | "empty" | "failed"; + status: "running" | "ok" | "empty" | "failed"; error: string; runId?: number; results?: AnalysisResult[]; @@ -158,6 +159,37 @@ export async function insertCategoryRunSummary( return Number(result.lastInsertRowid); } +export async function updateCategoryRunSummary( + db: Database, + runId: number, + summary: Pick, +): Promise { + db.run( + ` + UPDATE category_analysis_runs + SET + top_asins_checked = ?, + available_asins = ?, + fba_count = ?, + fbm_count = ?, + skip_count = ?, + status = ?, + error_message = ? + WHERE id = ? + `, + [ + summary.topAsinsChecked, + summary.availableAsins, + summary.fba, + summary.fbm, + summary.skip, + summary.status, + summary.error, + runId, + ], + ); +} + export async function insertProductAnalysisResults( db: Database, runId: number, @@ -817,16 +849,96 @@ function buildEnrichedProducts( }); } -async function runLlmInBatches( - products: EnrichedProduct[], -): Promise { - const verdicts: LlmVerdict[] = []; +export async function processCategory( + db: Database, + runId: number, + category: CategoryInfo, + perCategoryTop: number, +): Promise { + log("info", `\nCategory ${category.label} (${category.id})`); - for (let i = 0; i < products.length; i += LLM_BATCH_SIZE) { - const batch = products.slice(i, i + LLM_BATCH_SIZE); + const topAsins = await fetchCategoryBestSellerAsins(category, perCategoryTop); + if (topAsins.length === 0) { + log("info", " Keepa returned no ASINs for this category."); + await updateCategoryRunSummary(db, runId, { + topAsinsChecked: 0, + availableAsins: 0, + fba: 0, + fbm: 0, + skip: 0, + status: "empty", + error: "No ASINs returned by Keepa", + }); + return { + categoryId: category.id, + categoryLabel: category.label, + topAsinsChecked: 0, + availableAsins: 0, + fba: 0, + fbm: 0, + skip: 0, + status: "empty", + error: "No ASINs returned by Keepa", + results: [], + }; + } + + const uniqueTopAsins = Array.from(new Set(topAsins)); + if (uniqueTopAsins.length !== topAsins.length) { + log("warn", ` Removed ${topAsins.length - uniqueTopAsins.length} duplicate ASINs before analysis.`); + } + + log("info", ` Top ASINs fetched: ${uniqueTopAsins.length}`); + const sellabilityMap = await fetchSellabilityMap(uniqueTopAsins); + + const availableAsins = uniqueTopAsins.filter((asin) => { + const info = sellabilityMap.get(asin); + return info?.canSell === true && info.sellabilityStatus === "available"; + }); + + log("info", ` Sellable ASINs: ${availableAsins.length}/${uniqueTopAsins.length}`); + if (availableAsins.length === 0) { + await updateCategoryRunSummary(db, runId, { + topAsinsChecked: uniqueTopAsins.length, + availableAsins: 0, + fba: 0, + fbm: 0, + skip: 0, + status: "empty", + error: "No sellable ASINs", + }); + return { + categoryId: category.id, + categoryLabel: category.label, + topAsinsChecked: uniqueTopAsins.length, + availableAsins: 0, + fba: 0, + fbm: 0, + skip: 0, + status: "empty", + error: "No sellable ASINs", + results: [], + }; + } + + const keepaEnrichment = await fetchKeepaEnrichmentMap(availableAsins); + const spApiMap = await fetchSpApiMap(availableAsins, sellabilityMap); + const enrichedProducts = buildEnrichedProducts( + availableAsins, + sellabilityMap, + spApiMap, + keepaEnrichment, + ); + + const results: AnalysisResult[] = []; + let fba = 0; + let fbm = 0; + let skip = 0; + const totalBatches = Math.ceil(enrichedProducts.length / LLM_BATCH_SIZE); + + for (let i = 0; i < enrichedProducts.length; i += LLM_BATCH_SIZE) { + const batch = enrichedProducts.slice(i, i + LLM_BATCH_SIZE); const batchNum = Math.floor(i / LLM_BATCH_SIZE) + 1; - const totalBatches = Math.ceil(products.length / LLM_BATCH_SIZE); - log("info", ` LLM batch ${batchNum}/${totalBatches}...`); let batchVerdicts: LlmVerdict[]; @@ -843,116 +955,64 @@ async function runLlmInBatches( })); } - verdicts.push(...batchVerdicts); + const verdictByAsin = new Map(batchVerdicts.map((v) => [v.asin, v])); + const batchResults: AnalysisResult[] = batch.map((product) => ({ + product, + verdict: verdictByAsin.get(product.record.asin) ?? { + asin: product.record.asin, + verdict: "SKIP", + confidence: 0, + reasoning: "LLM returned no verdict", + }, + })); - if (i + LLM_BATCH_SIZE < products.length) { + await insertProductAnalysisResults(db, runId, batchResults); + + for (const result of batchResults) { + results.push(result); + if (result.verdict.verdict === "FBA") { + fba++; + } else if (result.verdict.verdict === "FBM") { + fbm++; + } else { + skip++; + } + } + + await updateCategoryRunSummary(db, runId, { + topAsinsChecked: uniqueTopAsins.length, + availableAsins: availableAsins.length, + fba, + fbm, + skip, + status: "running", + error: "", + }); + + log( + "info", + ` Persisted batch ${batchNum}/${totalBatches} (${batchResults.length} rows, totals FBA/FBM/SKIP=${fba}/${fbm}/${skip})`, + ); + + if (i + LLM_BATCH_SIZE < enrichedProducts.length) { await sleep(1500); } } - return verdicts; -} - -export async function processCategory( - db: Database, - category: CategoryInfo, - perCategoryTop: number, -): Promise { - log("info", `\nCategory ${category.label} (${category.id})`); - - const topAsins = await fetchCategoryBestSellerAsins(category, perCategoryTop); - if (topAsins.length === 0) { - log("info", " Keepa returned no ASINs for this category."); - return { - categoryId: category.id, - categoryLabel: category.label, - topAsinsChecked: 0, - availableAsins: 0, - fba: 0, - fbm: 0, - skip: 0, - status: "empty", - error: "No ASINs returned by Keepa", - results: [], - }; - } - - log("info", ` Top ASINs fetched: ${topAsins.length}`); - const sellabilityMap = await fetchSellabilityMap(topAsins); - - const availableAsins = topAsins.filter((asin) => { - const info = sellabilityMap.get(asin); - return info?.canSell === true && info.sellabilityStatus === "available"; + await updateCategoryRunSummary(db, runId, { + topAsinsChecked: uniqueTopAsins.length, + availableAsins: availableAsins.length, + fba, + fbm, + skip, + status: "ok", + error: "", }); - log("info", ` Sellable ASINs: ${availableAsins.length}/${topAsins.length}`); - if (availableAsins.length === 0) { - return { - categoryId: category.id, - categoryLabel: category.label, - topAsinsChecked: topAsins.length, - availableAsins: 0, - fba: 0, - fbm: 0, - skip: 0, - status: "empty", - error: "No sellable ASINs", - results: [], - }; - } - - const keepaEnrichment = await fetchKeepaEnrichmentMap(availableAsins); - const spApiMap = await fetchSpApiMap(availableAsins, sellabilityMap); - const titleByAsin = new Map(); - const keepaMap = new Map(); - - for (const asin of availableAsins) { - const enriched = keepaEnrichment.get(asin); - if (enriched?.title) { - titleByAsin.set(asin, enriched.title); - } - if (enriched?.keepa) { - keepaMap.set(asin, enriched.keepa); - } - } - - const enrichedProducts = buildEnrichedProducts( - availableAsins, - sellabilityMap, - spApiMap, - keepaEnrichment, - ); - - const verdicts = await runLlmInBatches(enrichedProducts); - const verdictByAsin = new Map(verdicts.map((v) => [v.asin, v])); - - const results: AnalysisResult[] = enrichedProducts.map((product) => ({ - product, - verdict: verdictByAsin.get(product.record.asin) ?? { - asin: product.record.asin, - verdict: "SKIP", - confidence: 0, - reasoning: "LLM returned no verdict", - }, - })); - - // No longer writing to XLSX, directly insert into DB - // const outputName = `${sanitizeFileSegment(category.label)}_${category.id}.xlsx`; - // const outputPath = path.join(outputDir, outputName); - // writeCategoryResultsWorkbook(results, outputPath); - - // The categoryRunId will be provided by the main function after inserting the summary - // We need to pass it here or get it after inserting the summary in main. - // For now, let's assume it's handled in main. - - const fba = results.filter((r) => r.verdict.verdict === "FBA").length; - const fbm = results.filter((r) => r.verdict.verdict === "FBM").length; - const skip = results.filter((r) => r.verdict.verdict === "SKIP").length; - return { categoryId: category.id, categoryLabel: category.label, - topAsinsChecked: topAsins.length, + topAsinsChecked: uniqueTopAsins.length, availableAsins: availableAsins.length, fba, fbm, @@ -968,7 +1028,7 @@ export async function main(): Promise { assertSpApiPrerequisites(); mkdirSync(args.outputDir, { recursive: true }); - const DB_PATH = path.join(args.outputDir, "analysis.sqlite"); + const DB_PATH = process.env.RESULTS_DB_PATH || path.join(process.cwd(), "results.db"); initDb(DB_PATH); const db = getDb(DB_PATH); @@ -1001,23 +1061,33 @@ export async function main(): Promise { for (const category of allowedCategories) { let categorySummary: CategoryRunSummary; + let runId: number | undefined; try { + runId = await insertCategoryRunSummary( + db, + { + categoryId: category.id, + categoryLabel: category.label, + topAsinsChecked: 0, + availableAsins: 0, + fba: 0, + fbm: 0, + skip: 0, + status: "running", + error: "", + results: [], + }, + runTimestamp, + ); + categorySummary = await processCategory( db, + runId, category, args.perCategoryTop, ); - const runId = await insertCategoryRunSummary( - db, - categorySummary, - runTimestamp, - ); - - if (categorySummary.results) { - await insertProductAnalysisResults(db, runId, categorySummary.results); - totalInsertedAsins += categorySummary.results.length; - } + totalInsertedAsins += categorySummary.results?.length ?? 0; processedCategories++; allCategorySummaries.push({ ...categorySummary, runId }); @@ -1039,8 +1109,19 @@ export async function main(): Promise { error: message, results: [], }; + if (runId) { + await updateCategoryRunSummary(db, runId, { + topAsinsChecked: 0, + availableAsins: 0, + fba: 0, + fbm: 0, + skip: 0, + status: "failed", + error: message, + }); + } processedCategories++; - allCategorySummaries.push(categorySummary); + allCategorySummaries.push({ ...categorySummary, runId }); } } diff --git a/src/database.ts b/src/database.ts index 2f1247f..32fa026 100644 --- a/src/database.ts +++ b/src/database.ts @@ -19,6 +19,84 @@ export function closeDb(): void { } } +function createProductAnalysisResultsTable(database: Database): void { + database.run(` + CREATE TABLE IF NOT EXISTS product_analysis_results ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + asin TEXT NOT NULL, + run_id INTEGER NOT NULL, + name TEXT NOT NULL, + brand TEXT, + category TEXT, + unit_cost REAL, + current_price REAL, + avg_price_90d REAL, + avg_price_90d_sheet REAL, + selling_price_sheet REAL, + sales_rank INTEGER, + sales_rank_avg_90d INTEGER, + seller_count INTEGER, + monthly_sold INTEGER, + rank_drops_30d INTEGER, + rank_drops_90d INTEGER, + fba_fee REAL, + fbm_fee REAL, + referral_percent REAL, + can_sell TEXT, + sellability_status TEXT, + sellability_reason TEXT, + verdict TEXT NOT NULL, + confidence REAL NOT NULL, + reasoning TEXT, + fetched_at TEXT NOT NULL, + UNIQUE(run_id, asin), + FOREIGN KEY (run_id) REFERENCES category_analysis_runs(id) + ); + `); +} + +function ensureProductAnalysisResultsTable(database: Database): void { + const tableInfo = database + .query("PRAGMA table_info(product_analysis_results)") + .all() as Array<{ name: string; pk: number }>; + + if (tableInfo.length === 0) { + createProductAnalysisResultsTable(database); + return; + } + + const hasIdColumn = tableInfo.some((col) => col.name === "id"); + const hasAsinPrimaryKey = tableInfo.some( + (col) => col.name === "asin" && col.pk === 1, + ); + + if (!hasIdColumn || hasAsinPrimaryKey) { + database.run("ALTER TABLE product_analysis_results RENAME TO product_analysis_results_legacy"); + createProductAnalysisResultsTable(database); + database.run(` + INSERT INTO product_analysis_results ( + asin, run_id, name, brand, category, unit_cost, + current_price, avg_price_90d, avg_price_90d_sheet, + selling_price_sheet, sales_rank, sales_rank_avg_90d, + seller_count, monthly_sold, rank_drops_30d, rank_drops_90d, + fba_fee, fbm_fee, referral_percent, can_sell, + sellability_status, sellability_reason, + verdict, confidence, reasoning, fetched_at + ) + SELECT + asin, run_id, name, brand, category, unit_cost, + current_price, avg_price_90d, avg_price_90d_sheet, + selling_price_sheet, sales_rank, sales_rank_avg_90d, + seller_count, monthly_sold, rank_drops_30d, rank_drops_90d, + fba_fee, fbm_fee, referral_percent, can_sell, + sellability_status, sellability_reason, + verdict, confidence, reasoning, fetched_at + FROM product_analysis_results_legacy + `); + database.run("DROP TABLE product_analysis_results_legacy"); + } +} + export function initDb(dbPath: string): void { const database = getDb(dbPath); database.run(` @@ -80,35 +158,18 @@ export function initDb(dbPath: string): void { error_message TEXT ); `); - database.run(` - CREATE TABLE IF NOT EXISTS product_analysis_results ( - asin TEXT PRIMARY KEY, - run_id INTEGER NOT NULL, - name TEXT NOT NULL, - brand TEXT, - category TEXT, - unit_cost REAL, - current_price REAL, - avg_price_90d REAL, - avg_price_90d_sheet REAL, - selling_price_sheet REAL, - sales_rank INTEGER, - sales_rank_avg_90d INTEGER, - seller_count INTEGER, - monthly_sold INTEGER, - rank_drops_30d INTEGER, - rank_drops_90d INTEGER, - fba_fee REAL, - fbm_fee REAL, - referral_percent REAL, - can_sell TEXT, - sellability_status TEXT, - sellability_reason TEXT, - verdict TEXT NOT NULL, - confidence REAL NOT NULL, - reasoning TEXT, - fetched_at TEXT NOT NULL, - FOREIGN KEY (run_id) REFERENCES category_analysis_runs(id) - ); - `); + ensureProductAnalysisResultsTable(database); + + database.run(`CREATE INDEX IF NOT EXISTS idx_runs_timestamp ON runs(timestamp DESC);`); + database.run(`CREATE INDEX IF NOT EXISTS idx_results_run_id ON results(run_id);`); + database.run(`CREATE INDEX IF NOT EXISTS idx_results_verdict ON results(verdict);`); + database.run(`CREATE INDEX IF NOT EXISTS idx_results_sellability_status ON results(sellability_status);`); + database.run(`CREATE INDEX IF NOT EXISTS idx_results_fetched_at ON results(fetched_at DESC);`); + database.run(`CREATE INDEX IF NOT EXISTS idx_results_asin ON results(asin);`); + database.run(`CREATE INDEX IF NOT EXISTS idx_category_runs_timestamp ON category_analysis_runs(run_timestamp DESC);`); + database.run(`CREATE INDEX IF NOT EXISTS idx_category_runs_status ON category_analysis_runs(status);`); + database.run(`CREATE INDEX IF NOT EXISTS idx_product_results_run_id ON product_analysis_results(run_id);`); + database.run(`CREATE INDEX IF NOT EXISTS idx_product_results_verdict ON product_analysis_results(verdict);`); + database.run(`CREATE INDEX IF NOT EXISTS idx_product_results_sellability_status ON product_analysis_results(sellability_status);`); + database.run(`CREATE INDEX IF NOT EXISTS idx_product_results_fetched_at ON product_analysis_results(fetched_at DESC);`); } diff --git a/src/index.ts b/src/index.ts index 74c752d..6cb4393 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,333 +1,297 @@ -import { readProducts } from "./reader.ts"; -import { fetchKeepaDataBatch } from "./keepa.ts"; -import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts"; -import { connectCache, getCache, setCache, disconnectCache } from "./cache.ts"; -import { analyzeProducts } from "./llm.ts"; -import { printResults, writeResultsToDb } from "./writer.ts"; -import { initDb, closeDb } from "./database.ts"; - -const DB_PATH = "./results.db"; -import type { - EnrichedProduct, - AnalysisResult, - KeepaData, - ProductRecord, - SellabilityInfo, - SpApiData, -} from "./types.ts"; - -const LLM_BATCH_SIZE = 5; -const INPUT_BATCH_SIZE = 50; - -function parseArgs(): { inputFile: string; outputFile?: string } { - const args = process.argv.slice(2); - const inputFile = args.find((a) => !a.startsWith("--")); - const outIdx = args.indexOf("--out"); - const outputFile = outIdx !== -1 ? args[outIdx + 1] : undefined; - - if (!inputFile) { - console.error( - "Usage: bun run src/index.ts [--out results.csv]", - ); - process.exit(1); - } - return { inputFile, outputFile }; -} - -async function main() { - const { inputFile, outputFile } = parseArgs(); - - console.log("Connecting to Redis..."); - await connectCache(); - - // Initialize SQLite DB - console.log("Initializing SQLite database..."); - initDb(DB_PATH); - - // Phase 1: Read input file - console.log(`\nReading ${inputFile}...`); - const products = readProducts(inputFile); - - if (products.length === 0) { - console.error("No valid products found in input file."); - process.exit(1); - } - - // Phase 2: Check cache for all ASINs - console.log(`\nChecking cache for ${products.length} products...`); - const cached = new Map(); - const excludedCachedAsins = new Set(); - const uncachedProducts: ProductRecord[] = []; - - for (const p of products) { - const hit = await getCache(p.asin); - if (hit) { - if (hit.spApi.sellabilityStatus === "available") { - console.log(` [cache hit] ${p.asin}`); - cached.set(p.asin, hit); - } else { - excludedCachedAsins.add(p.asin); - console.log( - ` [exclude cached] ${p.asin} — status=${hit.spApi.sellabilityStatus}`, - ); - } - } else { - uncachedProducts.push(p); - } - } - console.log( - `${cached.size} cached available, ${excludedCachedAsins.size} cached excluded, ${uncachedProducts.length} to fetch`, - ); - - // Phase 3: Sellability gate — check uncached ASINs before anything else - const sellabilityMap = new Map(); - const availableProducts: ProductRecord[] = []; - const unavailableProducts: ProductRecord[] = []; - - if (uncachedProducts.length > 0) { - console.log( - `\nChecking sellability for ${uncachedProducts.length} ASINs...`, - ); - const sellResults = await fetchSellabilityBatch( - uncachedProducts.map((p) => p.asin), - ); - - for (const p of uncachedProducts) { - const info = sellResults.get(p.asin) ?? { - canSell: null, - sellabilityStatus: "unknown" as const, - sellabilityReason: "Sellability check returned no result", - }; - sellabilityMap.set(p.asin, info); - - // Keep only ASINs that are explicitly available. - if (info.sellabilityStatus === "available") { - availableProducts.push(p); - console.log( - ` [available] ${p.asin} — status=${info.sellabilityStatus}`, - ); - } else { - unavailableProducts.push(p); - console.log( - ` [exclude] ${p.asin} — status=${info.sellabilityStatus}, reason=${info.sellabilityReason ?? "n/a"}`, - ); - } - } - - console.log( - `\nSellability gate: ${availableProducts.length} available, ${unavailableProducts.length} excluded`, - ); - } - - // Phase 4: Keepa batch fetch — only for available (uncached) ASINs - let keepaResults = new Map(); - if (availableProducts.length > 0) { - console.log(`\nFetching ${availableProducts.length} ASINs from Keepa...`); - try { - keepaResults = await fetchKeepaDataBatch( - availableProducts.map((p) => p.asin), - ); - } catch (err) { - console.warn(`Keepa batch fetch failed: ${err}`); - } - } - - // Phase 5: SP-API pricing + fees — only for available ASINs - console.log( - `\nFetching pricing & fees for ${availableProducts.length} ASINs...`, - ); - const spApiResults = new Map(); - - // Concurrency-limited pricing+fees fetches - const pricingQueue = [...availableProducts]; - let pricingDone = 0; - - async function fetchNextPricing(): Promise { - while (pricingQueue.length > 0) { - const p = pricingQueue.shift()!; - const sellability = sellabilityMap.get(p.asin)!; - const spApi = await fetchSpApiPricingAndFees(p.asin, sellability); - - const keepa = keepaResults.get(p.asin); - if (keepa?.currentPrice && spApi.estimatedSalePrice === 0) { - spApi.estimatedSalePrice = keepa.currentPrice; - } - - spApiResults.set(p.asin, spApi); - pricingDone++; - if (pricingDone % 10 === 0 || pricingDone === availableProducts.length) { - console.log( - ` [pricing] ${pricingDone}/${availableProducts.length} fetched`, - ); - } - } - } - - const pricingWorkers = Array.from( - { length: Math.min(5, availableProducts.length || 1) }, - () => fetchNextPricing(), - ); - await Promise.all(pricingWorkers); - - // Phase 6: Build enriched products - console.log(`\nEnriching products...`); - const enriched: EnrichedProduct[] = []; - const availableAsins = new Set(availableProducts.map((ap) => ap.asin)); - - for (const p of products) { - if (excludedCachedAsins.has(p.asin)) { - continue; - } - - // Cached products — already enriched - const cachedProduct = cached.get(p.asin); - if (cachedProduct) { - enriched.push(cachedProduct); - continue; - } - - // Exclude products that are not explicitly available. - if (!availableAsins.has(p.asin)) { - continue; - } - - // Available products — full enrichment - const keepa = keepaResults.get(p.asin) ?? null; - const spApi = spApiResults.get(p.asin) ?? { - fbaFee: 5.0, - fbmFee: 1.5, - referralFeePercent: 15, - estimatedSalePrice: 0, - canSell: null, - sellabilityStatus: "unknown" as const, - sellabilityReason: "SP-API data missing", - }; - - const product: EnrichedProduct = { - record: p, - keepa, - spApi, - fetchedAt: new Date().toISOString(), - }; - - await setCache(p.asin, product); - enriched.push(product); - - if (keepa) { - console.log( - ` [enriched] ${p.asin} — price: $${keepa.currentPrice ?? "N/A"}, rank: ${keepa.salesRank ?? "N/A"}`, - ); - } else { - console.log(` [no keepa] ${p.asin} — using spreadsheet data only`); - } - } - - // Phase 7: LLM analysis in batches — only for enriched available products - console.log( - `\nAnalyzing ${enriched.length} products via LLM (batch size: ${LLM_BATCH_SIZE})...\n`, - ); - - const results: AnalysisResult[] = []; - for (let i = 0; i < enriched.length; i += LLM_BATCH_SIZE) { - const batch = enriched.slice(i, i + LLM_BATCH_SIZE); - const batchNum = Math.floor(i / LLM_BATCH_SIZE) + 1; - const totalBatches = Math.ceil(enriched.length / LLM_BATCH_SIZE); - console.log(` LLM batch ${batchNum}/${totalBatches}...`); - - // Wait between batches to avoid overwhelming LM Studio - if (i > 0) { - console.log(` Waiting 5s before next batch...`); - await new Promise((r) => setTimeout(r, 5000)); - } - - let verdicts; - try { - verdicts = await analyzeProducts(batch); - } catch { - console.warn(` LLM batch error, retrying after 10s...`); - await new Promise((r) => setTimeout(r, 10_000)); - try { - verdicts = await analyzeProducts(batch); - } catch (retryErr) { - console.error(` LLM analysis failed: ${retryErr}`); - verdicts = null; - } - } - - for (let j = 0; j < batch.length; j++) { - results.push({ - product: batch[j]!, - verdict: verdicts?.[j] ?? { - asin: batch[j]!.record.asin, - verdict: "SKIP", - confidence: 0, - reasoning: "LLM analysis failed", - }, - }); - } - } - - return results; -} - -async function main() { - const { inputFile, outputFile } = parseArgs(); - - console.log("Connecting to Redis..."); - await connectCache(); - - try { - // Phase 1: Read input file - console.log(`\nReading ${inputFile}...`); - const products = readProducts(inputFile); - - if (products.length === 0) { - console.error("No valid products found in input file."); - process.exit(1); - } - - const productChunks = chunkArray(products, INPUT_BATCH_SIZE); - const hasMultipleChunks = productChunks.length > 1; - const shouldWriteChunkFiles = hasMultipleChunks; - const resolvedBaseOutputPath = resolveBaseOutputPath(inputFile, outputFile); - const allResults: AnalysisResult[] = []; - - if (hasMultipleChunks) { - console.log( - `\nLarge input detected (${products.length} products). Processing in chunks of ${INPUT_BATCH_SIZE}.`, - ); - console.log( - `Chunk outputs will be written as numbered files using base: ${resolvedBaseOutputPath}`, - ); - } - - for (let chunkIndex = 0; chunkIndex < productChunks.length; chunkIndex++) { - const chunk = productChunks[chunkIndex]!; - console.log( - `\n=== Input chunk ${chunkIndex + 1}/${productChunks.length} (${chunk.length} products) ===`, - ); - - const chunkResults = await processProductChunk(chunk); - allResults.push(...chunkResults); - - if (shouldWriteChunkFiles) { - const chunkOutputPath = buildChunkOutputPath( - resolvedBaseOutputPath, - chunkIndex, - ); - writeResultsCsv(chunkResults, chunkOutputPath); - } - } - - printResults(allResults); - - writeResultsToDb(allResults, DB_PATH, inputFile, outputFile); - - await disconnectCache(); - closeDb(); -} - -main().catch((err) => { - console.error("Fatal error:", err); - process.exit(1); -}); +import { readProducts } from "./reader.ts"; +import { fetchKeepaDataBatch } from "./keepa.ts"; +import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts"; +import { connectCache, getCache, setCache, disconnectCache } from "./cache.ts"; +import { analyzeProducts } from "./llm.ts"; +import { printResults, writeResultsToDb } from "./writer.ts"; +import { initDb, closeDb } from "./database.ts"; +import path from "node:path"; +import type { + EnrichedProduct, + AnalysisResult, + KeepaData, + ProductRecord, + SellabilityInfo, + SpApiData, +} from "./types.ts"; + +const DB_PATH = "./results.db"; +const LLM_BATCH_SIZE = 5; +const INPUT_BATCH_SIZE = 50; + +function parseArgs(): { inputFile: string; outputFile?: string } { + const args = process.argv.slice(2); + const inputFile = args.find((a) => !a.startsWith("--")); + const outIdx = args.indexOf("--out"); + const outputFile = outIdx !== -1 ? args[outIdx + 1] : undefined; + + if (!inputFile) { + console.error( + "Usage: bun run src/index.ts [--out results.csv]", + ); + process.exit(1); + } + + return { inputFile, outputFile }; +} + +function chunkArray(items: T[], chunkSize: number): T[][] { + const chunks: T[][] = []; + for (let i = 0; i < items.length; i += chunkSize) { + chunks.push(items.slice(i, i + chunkSize)); + } + return chunks; +} + +function resolveBaseOutputPath(inputFile: string, outputFile?: string): string { + if (outputFile) return outputFile; + + const parsedInput = path.parse(inputFile); + return path.join(parsedInput.dir, `${parsedInput.name}_results.xlsx`); +} + +async function processProductChunk( + products: ProductRecord[], +): Promise { + console.log(`\nChecking cache for ${products.length} products...`); + const cached = new Map(); + const excludedCachedAsins = new Set(); + const uncachedProducts: ProductRecord[] = []; + + for (const p of products) { + const hit = await getCache(p.asin); + if (hit) { + if (hit.spApi.sellabilityStatus === "available") { + console.log(` [cache hit] ${p.asin}`); + cached.set(p.asin, hit); + } else { + excludedCachedAsins.add(p.asin); + console.log( + ` [exclude cached] ${p.asin} — status=${hit.spApi.sellabilityStatus}`, + ); + } + } else { + uncachedProducts.push(p); + } + } + + console.log( + `${cached.size} cached available, ${excludedCachedAsins.size} cached excluded, ${uncachedProducts.length} to fetch`, + ); + + const sellabilityMap = new Map(); + const availableProducts: ProductRecord[] = []; + const unavailableProducts: ProductRecord[] = []; + + if (uncachedProducts.length > 0) { + console.log( + `\nChecking sellability for ${uncachedProducts.length} ASINs...`, + ); + const sellResults = await fetchSellabilityBatch( + uncachedProducts.map((p) => p.asin), + ); + + for (const p of uncachedProducts) { + const info = sellResults.get(p.asin) ?? { + canSell: null, + sellabilityStatus: "unknown" as const, + sellabilityReason: "Sellability check returned no result", + }; + sellabilityMap.set(p.asin, info); + + if (info.sellabilityStatus === "available") { + availableProducts.push(p); + console.log(` [available] ${p.asin} — status=${info.sellabilityStatus}`); + } else { + unavailableProducts.push(p); + console.log( + ` [exclude] ${p.asin} — status=${info.sellabilityStatus}, reason=${info.sellabilityReason ?? "n/a"}`, + ); + } + } + + console.log( + `\nSellability gate: ${availableProducts.length} available, ${unavailableProducts.length} excluded`, + ); + } + + let keepaResults = new Map(); + if (availableProducts.length > 0) { + console.log(`\nFetching ${availableProducts.length} ASINs from Keepa...`); + try { + keepaResults = await fetchKeepaDataBatch( + availableProducts.map((p) => p.asin), + ); + } catch (err) { + console.warn(`Keepa batch fetch failed: ${err}`); + } + } + + console.log( + `\nFetching pricing & fees for ${availableProducts.length} ASINs...`, + ); + const spApiResults = new Map(); + const pricingQueue = [...availableProducts]; + let pricingDone = 0; + + async function fetchNextPricing(): Promise { + while (pricingQueue.length > 0) { + const p = pricingQueue.shift()!; + const sellability = sellabilityMap.get(p.asin)!; + const spApi = await fetchSpApiPricingAndFees(p.asin, sellability); + + const keepa = keepaResults.get(p.asin); + if (keepa?.currentPrice && spApi.estimatedSalePrice === 0) { + spApi.estimatedSalePrice = keepa.currentPrice; + } + + spApiResults.set(p.asin, spApi); + pricingDone++; + if (pricingDone % 10 === 0 || pricingDone === availableProducts.length) { + console.log( + ` [pricing] ${pricingDone}/${availableProducts.length} fetched`, + ); + } + } + } + + const pricingWorkers = Array.from( + { length: Math.min(5, availableProducts.length || 1) }, + () => fetchNextPricing(), + ); + await Promise.all(pricingWorkers); + + console.log(`\nEnriching products...`); + const enriched: EnrichedProduct[] = []; + const availableAsins = new Set(availableProducts.map((ap) => ap.asin)); + + for (const p of products) { + if (excludedCachedAsins.has(p.asin)) { + continue; + } + + const cachedProduct = cached.get(p.asin); + if (cachedProduct) { + enriched.push(cachedProduct); + continue; + } + + if (!availableAsins.has(p.asin)) { + continue; + } + + const keepa = keepaResults.get(p.asin) ?? null; + const spApi = spApiResults.get(p.asin) ?? { + fbaFee: 5.0, + fbmFee: 1.5, + referralFeePercent: 15, + estimatedSalePrice: 0, + canSell: null, + sellabilityStatus: "unknown" as const, + sellabilityReason: "SP-API data missing", + }; + + const product: EnrichedProduct = { + record: p, + keepa, + spApi, + fetchedAt: new Date().toISOString(), + }; + + await setCache(p.asin, product); + enriched.push(product); + } + + console.log( + `\nAnalyzing ${enriched.length} products via LLM (batch size: ${LLM_BATCH_SIZE})...\n`, + ); + + const results: AnalysisResult[] = []; + for (let i = 0; i < enriched.length; i += LLM_BATCH_SIZE) { + const batch = enriched.slice(i, i + LLM_BATCH_SIZE); + const batchNum = Math.floor(i / LLM_BATCH_SIZE) + 1; + const totalBatches = Math.ceil(enriched.length / LLM_BATCH_SIZE); + console.log(` LLM batch ${batchNum}/${totalBatches}...`); + + if (i > 0) { + await new Promise((r) => setTimeout(r, 5000)); + } + + let verdicts; + try { + verdicts = await analyzeProducts(batch); + } catch { + await new Promise((r) => setTimeout(r, 10_000)); + try { + verdicts = await analyzeProducts(batch); + } catch { + verdicts = null; + } + } + + for (let j = 0; j < batch.length; j++) { + results.push({ + product: batch[j]!, + verdict: verdicts?.[j] ?? { + asin: batch[j]!.record.asin, + verdict: "SKIP", + confidence: 0, + reasoning: "LLM analysis failed", + }, + }); + } + } + + return results; +} + +async function main() { + const { inputFile, outputFile } = parseArgs(); + + console.log("Connecting to Redis..."); + await connectCache(); + + console.log("Initializing SQLite database..."); + initDb(DB_PATH); + + try { + console.log(`\nReading ${inputFile}...`); + const products = readProducts(inputFile); + + if (products.length === 0) { + console.error("No valid products found in input file."); + process.exit(1); + } + + const productChunks = chunkArray(products, INPUT_BATCH_SIZE); + const allResults: AnalysisResult[] = []; + const resolvedBaseOutputPath = resolveBaseOutputPath(inputFile, outputFile); + + if (productChunks.length > 1) { + console.log( + `\nLarge input detected (${products.length} products). Processing in chunks of ${INPUT_BATCH_SIZE}.`, + ); + console.log(`Output base path: ${resolvedBaseOutputPath}`); + } + + for (let chunkIndex = 0; chunkIndex < productChunks.length; chunkIndex++) { + const chunk = productChunks[chunkIndex]!; + console.log( + `\n=== Input chunk ${chunkIndex + 1}/${productChunks.length} (${chunk.length} products) ===`, + ); + const chunkResults = await processProductChunk(chunk); + allResults.push(...chunkResults); + } + + printResults(allResults); + writeResultsToDb(allResults, DB_PATH, inputFile, outputFile); + } finally { + await disconnectCache(); + closeDb(); + } +} + +main().catch((err) => { + console.error("Fatal error:", err); + process.exit(1); +}); diff --git a/src/llm.ts b/src/llm.ts index 8fa69bf..0b0eaf3 100644 --- a/src/llm.ts +++ b/src/llm.ts @@ -7,9 +7,14 @@ Given product data, evaluate each product's viability for selling on Amazon. Con 1. **Sales Velocity**: monthlySold and salesRankDrops30 are the most important signals. A product that doesn't sell is worthless regardless of margin. salesRankDrops30 = approximate units sold in 30 days. monthlySold is Keepa's estimate. 2. **Margin Analysis**: Sale price minus unit cost minus fees (FBA or FBM). Aim for >30% ROI minimum. The spreadsheet may include FBA NET and gross profit estimates — cross-check against Keepa pricing data. + - If unitCost is 0, it means NO COST DATA is available — this is NOT a free product. Do not assume infinite or zero-cost margins. When estimatedROI is null and unitCost is 0, evaluate purely on demand and velocity signals; never extrapolate profitability. + - If estimatedProfit is null, price data was unavailable at analysis time — treat as elevated uncertainty and be conservative. 3. **Sales Rank (BSR)**: Lower rank = higher demand. Rank <50,000 is good, <1,000 is excellent. 4. **Sales Rank Trend**: Compare current rank vs 90d average. Lower current = improving demand. -5. **Competition**: Number of sellers and Buy Box dynamics. Fewer sellers = easier entry. +5. **Competition & Buy Box**: + - Fewer sellers = easier entry, but verify who holds the Buy Box. + - If buyBoxSeller is "ATVPDKIKX0DER" (Amazon retail), the product is Amazon-exclusive — return "SKIP". Amazon-sold items cannot be competed with by third-party sellers. + - If sellerCount is 1 and buyBoxSeller is not null, assume the market is closed — return "SKIP". 6. **Price Stability**: Large price swings (high max vs low min over 90d) = volatile/risky. 7. **FBA vs FBM**: FBA preferred for fast-selling, small/light items. FBM for oversized, slow-moving, or high-margin items where fee savings matter. 8. **MOQ & Capital**: High MOQ with thin margins is risky. @@ -21,6 +26,7 @@ Given product data, evaluate each product's viability for selling on Amazon. Con Decision policy: - Do not recommend products that cannot be listed by this seller account. +- Do not recommend Amazon-exclusive products (buyBoxSeller = "ATVPDKIKX0DER"). - Prioritize profitable + high-velocity + listable products. - Use "SKIP" when data quality is poor or risk is high. @@ -106,12 +112,20 @@ function summarizeForLlm(p: EnrichedProduct) { const salePrice = p.keepa?.currentPrice ?? p.record.sellingPriceFromSheet ?? - p.spApi.estimatedSalePrice; - const referralFee = salePrice * (p.spApi.referralFeePercent / 100); + (p.spApi.estimatedSalePrice > 0 ? p.spApi.estimatedSalePrice : null) ?? + p.keepa?.avgPrice90 ?? + null; + + const referralFee = + salePrice != null ? salePrice * (p.spApi.referralFeePercent / 100) : null; const fbaProfit = - salePrice - p.record.unitCost - p.spApi.fbaFee - referralFee; + salePrice != null && referralFee != null + ? salePrice - p.record.unitCost - p.spApi.fbaFee - referralFee + : null; const fbmProfit = - salePrice - p.record.unitCost - p.spApi.fbmFee - referralFee; + salePrice != null && referralFee != null + ? salePrice - p.record.unitCost - p.spApi.fbmFee - referralFee + : null; return { asin: p.record.asin, @@ -121,7 +135,7 @@ function summarizeForLlm(p: EnrichedProduct) { p.record.category ?? p.keepa?.categoryTree?.join(" > "), 60, ), - unitCost: p.record.unitCost, + unitCost: p.record.unitCost > 0 ? p.record.unitCost : null, currentPrice: salePrice, priceRange90d: p.keepa ? { @@ -133,6 +147,8 @@ function summarizeForLlm(p: EnrichedProduct) { salesRank: p.keepa?.salesRank ?? p.record.amazonRank, salesRankAvg90d: p.keepa?.salesRankAvg90, sellerCount: p.keepa?.sellerCount, + buyBoxSeller: p.keepa?.buyBoxSeller ?? null, + buyBoxPrice: p.keepa?.buyBoxPrice ?? null, salesVelocity: { monthlySold: p.keepa?.monthlySold, salesRankDrops30: p.keepa?.salesRankDrops30, @@ -155,27 +171,28 @@ function summarizeForLlm(p: EnrichedProduct) { fbaFee: p.spApi.fbaFee, fbmFee: p.spApi.fbmFee, referralFeePercent: p.spApi.referralFeePercent, - referralFee: Math.round(referralFee * 100) / 100, + referralFee: + referralFee != null ? Math.round(referralFee * 100) / 100 : null, }, sellerEligibility: { canSell: p.spApi.canSell, status: p.spApi.sellabilityStatus, reason: clampText(p.spApi.sellabilityReason, 120), }, - estimatedProfit: { - fba: Math.round(fbaProfit * 100) / 100, - fbm: Math.round(fbmProfit * 100) / 100, - }, - estimatedROI: { - fba: - p.record.unitCost > 0 - ? Math.round((fbaProfit / p.record.unitCost) * 100) - : null, - fbm: - p.record.unitCost > 0 - ? Math.round((fbmProfit / p.record.unitCost) * 100) - : null, - }, + estimatedProfit: + fbaProfit != null && fbmProfit != null + ? { + fba: Math.round(fbaProfit * 100) / 100, + fbm: Math.round(fbmProfit * 100) / 100, + } + : null, + estimatedROI: + p.record.unitCost > 0 && fbaProfit != null && fbmProfit != null + ? { + fba: Math.round((fbaProfit / p.record.unitCost) * 100), + fbm: Math.round((fbmProfit / p.record.unitCost) * 100), + } + : null, }; } diff --git a/src/server.ts b/src/server.ts new file mode 100644 index 0000000..8acfe6c --- /dev/null +++ b/src/server.ts @@ -0,0 +1,619 @@ +import index from "./web/index.html"; +import { getDb, initDb } from "./database.ts"; + +type ProcessType = "lead_analysis" | "category_analysis"; + +type RunRecord = { + processType: ProcessType; + runId: number; + timestamp: string; + status: string; + jobType: string; + source: string | null; + output: string | null; + totalProducts: number; + fbaCount: number; + fbmCount: number; + skipCount: number; +}; + +type ProductListRecord = { + processType: ProcessType; + runId: number; + asin: string; + product_name: string | null; + brand: string | null; + category: string | null; + verdict: "FBA" | "FBM" | "SKIP"; + confidence: number | null; + sellability_status: string | null; + fetched_at: string; +}; + +const DB_PATH = process.env.RESULTS_DB_PATH || "./results.db"; +const DEFAULT_PAGE_SIZE = 25; +const MAX_PAGE_SIZE = 200; + +initDb(DB_PATH); +const db = getDb(DB_PATH); + +function json(data: unknown, status = 200): Response { + return new Response(JSON.stringify(data), { + status, + headers: { "content-type": "application/json; charset=utf-8" }, + }); +} + +function csv(text: string, filename: string): Response { + return new Response(text, { + status: 200, + headers: { + "content-type": "text/csv; charset=utf-8", + "content-disposition": `attachment; filename="${filename}"`, + }, + }); +} + +function parseIntParam(value: string | null, fallback: number): number { + if (!value) return fallback; + const parsed = Number.parseInt(value, 10); + if (!Number.isFinite(parsed) || parsed < 1) return fallback; + return parsed; +} + +function parseSort(sortParam: string | null, allowed: Set, fallback: string): string { + if (!sortParam) return fallback; + const clauses = sortParam + .split(",") + .map((chunk) => chunk.trim()) + .filter(Boolean) + .map((chunk) => { + const [fieldRaw, dirRaw] = chunk.split(":"); + const field = fieldRaw?.trim(); + const dir = dirRaw?.trim().toUpperCase() === "DESC" ? "DESC" : "ASC"; + if (!field || !allowed.has(field)) return null; + return `${field} ${dir}`; + }) + .filter((value): value is string => value !== null); + + return clauses.length > 0 ? clauses.join(", ") : fallback; +} + +function parseResultSort(sortParam: string | null, allowed: Set, fallback: string): string { + if (!sortParam) return fallback; + const clauses = sortParam + .split(",") + .map((chunk) => chunk.trim()) + .filter(Boolean) + .map((chunk) => { + const [fieldRaw, dirRaw] = chunk.split(":"); + const field = fieldRaw?.trim(); + const dir = dirRaw?.trim().toUpperCase() === "DESC" ? "DESC" : "ASC"; + if (!field || !allowed.has(field)) return null; + if (field === "monthly_sold") return `CAST(COALESCE(monthly_sold, 0) AS INTEGER) ${dir}`; + return `${field} ${dir}`; + }) + .filter((value): value is string => value !== null); + + return clauses.length > 0 ? clauses.join(", ") : fallback; +} + +function escapeCsvValue(value: unknown): string { + if (value === null || value === undefined) return ""; + const text = String(value); + const escaped = text.replaceAll("\"", "\"\""); + return /[",\n]/.test(escaped) ? `"${escaped}"` : escaped; +} + +function parseResultFilters(processType: ProcessType, runId: number, filters: URLSearchParams) { + const q = filters.get("q")?.trim() || ""; + const verdict = filters.get("verdict")?.trim(); + const sellabilityStatus = filters.get("sellabilityStatus")?.trim(); + const minConfidence = filters.get("minConfidence")?.trim(); + const maxConfidence = filters.get("maxConfidence")?.trim(); + + const conditions: string[] = ["run_id = ?"]; + const params: Array = [runId]; + + if (verdict) { + conditions.push("verdict = ?"); + params.push(verdict); + } + + if (sellabilityStatus) { + conditions.push("sellability_status = ?"); + params.push(sellabilityStatus); + } + + if (minConfidence) { + conditions.push("confidence >= ?"); + params.push(Number(minConfidence)); + } + + if (maxConfidence) { + conditions.push("confidence <= ?"); + params.push(Number(maxConfidence)); + } + + if (q) { + const wildcard = `%${q}%`; + if (processType === "lead_analysis") { + conditions.push("(asin LIKE ? OR product_name LIKE ? OR brand LIKE ? OR category LIKE ? OR reasoning LIKE ?)"); + params.push(wildcard, wildcard, wildcard, wildcard, wildcard); + } else { + conditions.push("(asin LIKE ? OR name LIKE ? OR brand LIKE ? OR category LIKE ? OR reasoning LIKE ?)"); + params.push(wildcard, wildcard, wildcard, wildcard, wildcard); + } + } + + return { + where: `WHERE ${conditions.join(" AND ")}`, + params, + }; +} + +function getRuns(filters: URLSearchParams) { + const q = filters.get("q")?.trim() || ""; + const processType = filters.get("processType")?.trim(); + const status = filters.get("status")?.trim(); + const startDate = filters.get("startDate")?.trim(); + const endDate = filters.get("endDate")?.trim(); + const page = parseIntParam(filters.get("page"), 1); + const pageSize = Math.min(parseIntParam(filters.get("pageSize"), DEFAULT_PAGE_SIZE), MAX_PAGE_SIZE); + const offset = (page - 1) * pageSize; + + const allowedSort = new Set([ + "timestamp", + "status", + "totalProducts", + "fbaCount", + "fbmCount", + "skipCount", + "runId", + "jobType", + ]); + const orderBy = parseSort(filters.get("sort"), allowedSort, "timestamp DESC, runId DESC"); + + const conditions: string[] = []; + const params: Array = []; + + if (processType === "lead_analysis" || processType === "category_analysis") { + conditions.push("processType = ?"); + params.push(processType); + } + + if (status) { + conditions.push("status = ?"); + params.push(status); + } + + if (startDate) { + conditions.push("timestamp >= ?"); + params.push(startDate); + } + + if (endDate) { + conditions.push("timestamp <= ?"); + params.push(endDate); + } + + if (q) { + conditions.push("(jobType LIKE ? OR source LIKE ? OR output LIKE ? OR CAST(runId AS TEXT) LIKE ?)"); + const wildcard = `%${q}%`; + params.push(wildcard, wildcard, wildcard, wildcard); + } + + const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : ""; + + const baseUnion = ` + SELECT + 'lead_analysis' AS processType, + id AS runId, + timestamp, + 'completed' AS status, + 'lead_file_analysis' AS jobType, + input_file AS source, + output_file AS output, + COALESCE(total_products, 0) AS totalProducts, + COALESCE(fba_count, 0) AS fbaCount, + COALESCE(fbm_count, 0) AS fbmCount, + COALESCE(skip_count, 0) AS skipCount + FROM runs + UNION ALL + SELECT + 'category_analysis' AS processType, + id AS runId, + run_timestamp AS timestamp, + status, + category_label AS jobType, + CAST(category_id AS TEXT) AS source, + NULL AS output, + COALESCE(top_asins_checked, 0) AS totalProducts, + COALESCE(fba_count, 0) AS fbaCount, + COALESCE(fbm_count, 0) AS fbmCount, + COALESCE(skip_count, 0) AS skipCount + FROM category_analysis_runs + `; + + const totalRow = db + .query(`SELECT COUNT(*) as total FROM (${baseUnion}) all_runs ${where}`) + .get(...params) as { total: number }; + + const items = db + .query(`SELECT * FROM (${baseUnion}) all_runs ${where} ORDER BY ${orderBy} LIMIT ? OFFSET ?`) + .all(...params, pageSize, offset) as RunRecord[]; + + return { + items, + page, + pageSize, + total: totalRow.total, + totalPages: Math.max(1, Math.ceil(totalRow.total / pageSize)), + }; +} + +function getProductList(filters: URLSearchParams) { + const q = filters.get("q")?.trim() || ""; + const verdict = filters.get("verdict")?.trim(); + const page = parseIntParam(filters.get("page"), 1); + const pageSize = Math.min(parseIntParam(filters.get("pageSize"), DEFAULT_PAGE_SIZE), MAX_PAGE_SIZE); + const offset = (page - 1) * pageSize; + + const conditions: string[] = []; + const params: Array = []; + + if (verdict === "FBA" || verdict === "FBM" || verdict === "SKIP") { + conditions.push("verdict = ?"); + params.push(verdict); + } + + if (q) { + const wildcard = `%${q}%`; + conditions.push("(asin LIKE ? OR product_name LIKE ? OR brand LIKE ? OR category LIKE ?)"); + params.push(wildcard, wildcard, wildcard, wildcard); + } + + const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : ""; + + const baseUnion = ` + SELECT + 'lead_analysis' AS processType, + run_id AS runId, + asin, + product_name, + brand, + category, + verdict, + confidence, + sellability_status, + fetched_at + FROM results + UNION ALL + SELECT + 'category_analysis' AS processType, + run_id AS runId, + asin, + name AS product_name, + brand, + category, + verdict, + confidence, + sellability_status, + fetched_at + FROM product_analysis_results + `; + + const totalRow = db + .query(`SELECT COUNT(*) as total FROM (${baseUnion}) all_products ${where}`) + .get(...params) as { total: number }; + + const items = db + .query(`SELECT * FROM (${baseUnion}) all_products ${where} ORDER BY fetched_at DESC LIMIT ? OFFSET ?`) + .all(...params, pageSize, offset) as ProductListRecord[]; + + return { + items, + page, + pageSize, + total: totalRow.total, + totalPages: Math.max(1, Math.ceil(totalRow.total / pageSize)), + }; +} + +function getRun(processType: ProcessType, runId: number) { + if (processType === "lead_analysis") { + const run = db + .query( + `SELECT + id AS runId, + timestamp, + 'completed' AS status, + 'lead_file_analysis' AS jobType, + input_file AS source, + output_file AS output, + COALESCE(total_products, 0) AS totalProducts, + COALESCE(fba_count, 0) AS fbaCount, + COALESCE(fbm_count, 0) AS fbmCount, + COALESCE(skip_count, 0) AS skipCount + FROM runs WHERE id = ?`, + ) + .get(runId); + return run ?? null; + } + + const run = db + .query( + `SELECT + id AS runId, + run_timestamp AS timestamp, + status, + category_label AS jobType, + CAST(category_id AS TEXT) AS source, + NULL AS output, + COALESCE(top_asins_checked, 0) AS totalProducts, + COALESCE(fba_count, 0) AS fbaCount, + COALESCE(fbm_count, 0) AS fbmCount, + COALESCE(skip_count, 0) AS skipCount, + error_message AS errorMessage, + available_asins AS availableAsins + FROM category_analysis_runs WHERE id = ?`, + ) + .get(runId); + return run ?? null; +} + +function getRunResults(processType: ProcessType, runId: number, filters: URLSearchParams) { + const page = parseIntParam(filters.get("page"), 1); + const pageSize = Math.min(parseIntParam(filters.get("pageSize"), DEFAULT_PAGE_SIZE), MAX_PAGE_SIZE); + const offset = (page - 1) * pageSize; + + const tableName = processType === "lead_analysis" ? "results" : "product_analysis_results"; + const productNameSelect = processType === "lead_analysis" ? "product_name" : "name AS product_name"; + const sellerCountSelect = processType === "lead_analysis" ? "sellers AS seller_count" : "seller_count"; + const salesRankAvgSelect = processType === "lead_analysis" ? "rank_avg_90d AS sales_rank_avg_90d" : "sales_rank_avg_90d"; + + const { where, params } = parseResultFilters(processType, runId, filters); + + const allowedSort = new Set([ + "asin", + "product_name", + "brand", + "category", + "current_price", + "avg_price_90d", + "sales_rank", + "seller_count", + "monthly_sold", + "verdict", + "confidence", + "fetched_at", + ]); + const orderBy = parseResultSort( + filters.get("sort"), + allowedSort, + "CAST(COALESCE(monthly_sold, 0) AS INTEGER) DESC, asin ASC", + ); + + const totalRow = db + .query(`SELECT COUNT(*) as total FROM ${tableName} ${where}`) + .get(...params) as { total: number }; + + const items = db + .query( + `SELECT + id, + run_id, + asin, + ${productNameSelect}, + brand, + category, + unit_cost, + current_price, + avg_price_90d, + avg_price_90d_sheet, + selling_price_sheet, + sales_rank, + ${salesRankAvgSelect}, + ${sellerCountSelect}, + monthly_sold, + rank_drops_30d, + rank_drops_90d, + fba_fee, + fbm_fee, + referral_percent, + can_sell, + sellability_status, + sellability_reason, + verdict, + confidence, + reasoning, + fetched_at + FROM ${tableName} + ${where} + ORDER BY ${orderBy} + LIMIT ? OFFSET ?`, + ) + .all(...params, pageSize, offset); + + return { + items, + page, + pageSize, + total: totalRow.total, + totalPages: Math.max(1, Math.ceil(totalRow.total / pageSize)), + }; +} + +function deleteRun(processType: ProcessType, runId: number) { + if (processType === "lead_analysis") { + const resultRows = db.query("DELETE FROM results WHERE run_id = ?").run(runId); + const runRows = db.query("DELETE FROM runs WHERE id = ?").run(runId); + return { + deletedRun: runRows.changes > 0, + deletedResults: resultRows.changes, + }; + } + + const resultRows = db.query("DELETE FROM product_analysis_results WHERE run_id = ?").run(runId); + const runRows = db.query("DELETE FROM category_analysis_runs WHERE id = ?").run(runId); + return { + deletedRun: runRows.changes > 0, + deletedResults: resultRows.changes, + }; +} + +function exportRunResultsCsv(processType: ProcessType, runId: number, filters: URLSearchParams) { + const tableName = processType === "lead_analysis" ? "results" : "product_analysis_results"; + const productNameSelect = processType === "lead_analysis" ? "product_name" : "name AS product_name"; + const sellerCountSelect = processType === "lead_analysis" ? "sellers AS seller_count" : "seller_count"; + const salesRankAvgSelect = processType === "lead_analysis" ? "rank_avg_90d AS sales_rank_avg_90d" : "sales_rank_avg_90d"; + + const { where, params } = parseResultFilters(processType, runId, filters); + + const allowedSort = new Set([ + "asin", + "product_name", + "brand", + "category", + "current_price", + "avg_price_90d", + "sales_rank", + "seller_count", + "monthly_sold", + "verdict", + "confidence", + "fetched_at", + ]); + const orderBy = parseResultSort( + filters.get("sort"), + allowedSort, + "CAST(COALESCE(monthly_sold, 0) AS INTEGER) DESC, asin ASC", + ); + + const rows = db + .query( + `SELECT + run_id, + asin, + ${productNameSelect}, + brand, + category, + unit_cost, + current_price, + avg_price_90d, + ${salesRankAvgSelect}, + ${sellerCountSelect}, + monthly_sold, + sellability_status, + verdict, + confidence, + reasoning, + fetched_at + FROM ${tableName} + ${where} + ORDER BY ${orderBy}`, + ) + .all(...params) as Array>; + + const headers = [ + "run_id", + "asin", + "product_name", + "brand", + "category", + "unit_cost", + "current_price", + "avg_price_90d", + "sales_rank_avg_90d", + "seller_count", + "monthly_sold", + "sellability_status", + "verdict", + "confidence", + "reasoning", + "fetched_at", + ]; + + const lines = [headers.join(",")]; + for (const row of rows) { + lines.push(headers.map((h) => escapeCsvValue(row[h])).join(",")); + } + + return lines.join("\n"); +} + +const server = Bun.serve({ + port: Number(process.env.PORT || "3000"), + routes: { + "/": index, + "/products": index, + "/runs/:processType/:runId": index, + "/api/runs": (req) => { + const url = new URL(req.url); + return json(getRuns(url.searchParams)); + }, + "/api/products": (req) => { + const url = new URL(req.url); + return json(getProductList(url.searchParams)); + }, + "/api/runs/:processType/:runId": (req) => { + const processType = req.params.processType as ProcessType; + const runId = Number(req.params.runId); + + if (!(processType === "lead_analysis" || processType === "category_analysis") || !Number.isInteger(runId)) { + return json({ error: "Invalid run identifier" }, 400); + } + + if (req.method === "DELETE") { + const deleted = deleteRun(processType, runId); + if (!deleted.deletedRun) return json({ error: "Run not found" }, 404); + return json(deleted); + } + + const run = getRun(processType, runId); + if (!run) return json({ error: "Run not found" }, 404); + + const summary = { + totalProducts: (run as { totalProducts: number }).totalProducts, + fbaCount: (run as { fbaCount: number }).fbaCount, + fbmCount: (run as { fbmCount: number }).fbmCount, + skipCount: (run as { skipCount: number }).skipCount, + }; + + return json({ processType, ...run, summary }); + }, + "/api/runs/:processType/:runId/results": (req) => { + const processType = req.params.processType as ProcessType; + const runId = Number(req.params.runId); + + if (!(processType === "lead_analysis" || processType === "category_analysis") || !Number.isInteger(runId)) { + return json({ error: "Invalid run identifier" }, 400); + } + + const url = new URL(req.url); + const payload = getRunResults(processType, runId, url.searchParams); + return json(payload); + }, + "/api/runs/:processType/:runId/export.csv": (req) => { + const processType = req.params.processType as ProcessType; + const runId = Number(req.params.runId); + + if (!(processType === "lead_analysis" || processType === "category_analysis") || !Number.isInteger(runId)) { + return json({ error: "Invalid run identifier" }, 400); + } + + const url = new URL(req.url); + const csvText = exportRunResultsCsv(processType, runId, url.searchParams); + return csv(csvText, `run-${processType}-${runId}.csv`); + }, + }, + fetch() { + return json({ error: "Not found" }, 404); + }, + development: { + hmr: true, + console: true, + }, +}); + +console.log(`Results viewer running on http://localhost:${server.port}`); diff --git a/src/web/frontend.tsx b/src/web/frontend.tsx new file mode 100644 index 0000000..f326fbd --- /dev/null +++ b/src/web/frontend.tsx @@ -0,0 +1,780 @@ +import { createRoot } from "react-dom/client"; +import { useEffect, useMemo, useState } from "react"; + +type ProcessType = "lead_analysis" | "category_analysis"; +type SortDirection = "ASC" | "DESC"; + +type Run = { + processType: ProcessType; + runId: number; + timestamp: string; + status: string; + jobType: string; + source: string | null; + output: string | null; + totalProducts: number; + fbaCount: number; + fbmCount: number; + skipCount: number; +}; + +type RunsResponse = { + items: Run[]; + page: number; + pageSize: number; + total: number; + totalPages: number; +}; + +type RunDetail = { + processType: ProcessType; + runId: number; + timestamp: string; + status: string; + jobType: string; + source: string | null; + output: string | null; + totalProducts: number; + fbaCount: number; + fbmCount: number; + skipCount: number; + summary: { + totalProducts: number; + fbaCount: number; + fbmCount: number; + skipCount: number; + }; + errorMessage?: string; + availableAsins?: number; +}; + +type ResultItem = { + id?: number; + run_id: number; + asin: string; + product_name: string | null; + brand: string | null; + category: string | null; + current_price: number | null; + avg_price_90d: number | null; + sales_rank: number | null; + seller_count: number | null; + monthly_sold: number | null; + verdict: "FBA" | "FBM" | "SKIP"; + confidence: number | null; + sellability_status: string | null; + reasoning: string | null; + fetched_at: string; +}; + +type ResultsResponse = { + items: ResultItem[]; + page: number; + pageSize: number; + total: number; + totalPages: number; +}; + +type VerdictFilter = "" | "FBA" | "FBM" | "SKIP"; + +type ProductListItem = { + processType: ProcessType; + runId: number; + asin: string; + product_name: string | null; + brand: string | null; + category: string | null; + verdict: "FBA" | "FBM" | "SKIP"; + confidence: number | null; + sellability_status: string | null; + fetched_at: string; +}; + +type ProductListResponse = { + items: ProductListItem[]; + page: number; + pageSize: number; + total: number; + totalPages: number; +}; + +type SortState = { + field: string; + direction: SortDirection; +}; + +function formatDate(input: string): string { + const d = new Date(input); + if (Number.isNaN(d.getTime())) return input; + return d.toLocaleString(); +} + +function formatNumber(value: number | null | undefined): string { + if (value === null || value === undefined) return "-"; + return new Intl.NumberFormat().format(value); +} + +function formatCurrency(value: number | null | undefined): string { + if (value === null || value === undefined) return "-"; + return new Intl.NumberFormat(undefined, { + style: "currency", + currency: "USD", + maximumFractionDigits: 2, + }).format(value); +} + +function buildSortValue(sort: SortState): string { + return `${sort.field}:${sort.direction}`; +} + +function nextSort(current: SortState, field: string): SortState { + if (current.field !== field) { + return { field, direction: "ASC" }; + } + return { + field, + direction: current.direction === "ASC" ? "DESC" : "ASC", + }; +} + +function statusBadgeClass(status: string): string { + if (status === "ok" || status === "completed") return "badge badge-ok"; + if (status === "failed") return "badge badge-failed"; + return "badge badge-empty"; +} + +function verdictBadgeClass(verdict: string): string { + if (verdict === "FBA") return "badge badge-fba"; + if (verdict === "FBM") return "badge badge-fbm"; + return "badge badge-skip"; +} + +function TinyBar({ fba, fbm, skip }: { fba: number; fbm: number; skip: number }) { + const total = Math.max(1, fba + fbm + skip); + const fbaPct = (fba / total) * 100; + const fbmPct = (fbm / total) * 100; + const skipPct = (skip / total) * 100; + + return ( +
+ + + +
+ ); +} + +function detectAnomaly(item: ResultItem): string { + const confidence = item.confidence ?? 0; + if ((item.sellability_status === "restricted" || item.sellability_status === "not_available") && item.verdict !== "SKIP") { + return "restricted-vs-verdict"; + } + if (item.verdict === "FBA" && confidence < 60) { + return "low-confidence-fba"; + } + if ((item.sales_rank ?? Number.MAX_SAFE_INTEGER) > 300000 && item.verdict === "FBA") { + return "high-rank-fba"; + } + return ""; +} + +function Dashboard({ + onOpenRun, + onOpenProducts, +}: { + onOpenRun: (run: Run) => void; + onOpenProducts: (verdict: VerdictFilter) => void; +}) { + const [runs, setRuns] = useState(null); + const [loading, setLoading] = useState(false); + const [search, setSearch] = useState(""); + const [processType, setProcessType] = useState(""); + const [status, setStatus] = useState(""); + const [startDate, setStartDate] = useState(""); + const [endDate, setEndDate] = useState(""); + const [page, setPage] = useState(1); + const [pageSize, setPageSize] = useState(25); + const [sort, setSort] = useState({ field: "timestamp", direction: "DESC" }); + const [refreshTick, setRefreshTick] = useState(0); + const [deletingKey, setDeletingKey] = useState(null); + + const summary = useMemo(() => { + if (!runs) return { total: 0, fba: 0, fbm: 0, skip: 0 }; + return runs.items.reduce( + (acc, run) => { + acc.total += run.totalProducts; + acc.fba += run.fbaCount; + acc.fbm += run.fbmCount; + acc.skip += run.skipCount; + return acc; + }, + { total: 0, fba: 0, fbm: 0, skip: 0 }, + ); + }, [runs]); + + const timeline = useMemo(() => { + if (!runs) return [] as Run[]; + return [...runs.items] + .sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()) + .slice(-12); + }, [runs]); + + useEffect(() => { + let cancelled = false; + async function load() { + setLoading(true); + const params = new URLSearchParams({ + page: String(page), + pageSize: String(pageSize), + sort: buildSortValue(sort), + }); + if (search) params.set("q", search); + if (processType) params.set("processType", processType); + if (status) params.set("status", status); + if (startDate) params.set("startDate", `${startDate}T00:00:00.000Z`); + if (endDate) params.set("endDate", `${endDate}T23:59:59.999Z`); + + const resp = await fetch(`/api/runs?${params.toString()}`); + const data = (await resp.json()) as RunsResponse; + if (!cancelled) { + setRuns(data); + setLoading(false); + } + } + + load(); + return () => { + cancelled = true; + }; + }, [search, processType, status, startDate, endDate, page, pageSize, sort, refreshTick]); + + async function deleteRun(run: Run) { + const key = `${run.processType}-${run.runId}`; + const confirmed = window.confirm(`Delete run ${run.runId} (${run.processType}) and all associated results?`); + if (!confirmed) return; + + setDeletingKey(key); + try { + const response = await fetch(`/api/runs/${run.processType}/${run.runId}`, { method: "DELETE" }); + if (!response.ok) { + const errorPayload = await response.json().catch(() => null) as { error?: string } | null; + const message = errorPayload?.error ?? "Failed to delete run"; + window.alert(message); + return; + } + setPage(1); + setRefreshTick((tick) => tick + 1); + } finally { + setDeletingKey(null); + } + } + + return ( +
+
+

Runs Dashboard

+
+ +
+
onOpenProducts("")} onKeyDown={(e) => { if (e.key === "Enter" || e.key === " ") onOpenProducts(""); }}> +
Total products
+
{formatNumber(summary.total)}
+
+
onOpenProducts("FBA")} onKeyDown={(e) => { if (e.key === "Enter" || e.key === " ") onOpenProducts("FBA"); }}> +
FBA
+
{formatNumber(summary.fba)}
+
+
onOpenProducts("FBM")} onKeyDown={(e) => { if (e.key === "Enter" || e.key === " ") onOpenProducts("FBM"); }}> +
FBM
+
{formatNumber(summary.fbm)}
+
+
onOpenProducts("SKIP")} onKeyDown={(e) => { if (e.key === "Enter" || e.key === " ") onOpenProducts("SKIP"); }}> +
SKIP
+
{formatNumber(summary.skip)}
+
+
+ +
+
+ { setPage(1); setSearch(e.target.value); }} placeholder="Search run/job/source" /> + + + { setPage(1); setStartDate(e.target.value); }} /> + { setPage(1); setEndDate(e.target.value); }} /> + +
+
+ +
+
+ + + + + + + + + + + + + + + + + + + + {loading ? ( + + ) : runs?.items.length ? ( + runs.items.map((run) => ( + + + + + + + + + + + + + + + + )) + ) : ( + + )} + +
FBAFBMSKIPMixSourceOpenDelete
Loading...
{run.runId}{run.processType}{run.jobType}{formatDate(run.timestamp)}{run.status}{formatNumber(run.totalProducts)}{formatNumber(run.fbaCount)}{formatNumber(run.fbmCount)}{formatNumber(run.skipCount)}{run.source || "-"} + +
No runs found
+
+
+
Showing {runs?.items.length ?? 0} of {runs?.total ?? 0}
+
+ + Page {runs?.page ?? page} / {runs?.totalPages ?? 1} + +
+
+
+ +
+

Recent trend (last 12 runs in current view)

+
+ {timeline.length === 0 ? ( +
No trend data
+ ) : ( + timeline.map((run) => ( +
+
#{run.runId}
+ +
+ )) + )} +
+
+
+ ); +} + +function RunDetails({ + processType, + runId, + onBack, +}: { + processType: ProcessType; + runId: number; + onBack: () => void; +}) { + const [run, setRun] = useState(null); + const [results, setResults] = useState(null); + const [loading, setLoading] = useState(false); + const [search, setSearch] = useState(""); + const [verdict, setVerdict] = useState(""); + const [sellabilityStatus, setSellabilityStatus] = useState(""); + const [minConfidence, setMinConfidence] = useState(""); + const [maxConfidence, setMaxConfidence] = useState(""); + const [page, setPage] = useState(1); + const [pageSize, setPageSize] = useState(25); + const [sort, setSort] = useState({ field: "monthly_sold", direction: "DESC" }); + const [refreshTick, setRefreshTick] = useState(0); + + const anomalies = useMemo(() => { + if (!results) return [] as ResultItem[]; + return results.items.filter((item) => detectAnomaly(item) !== ""); + }, [results]); + + useEffect(() => { + let cancelled = false; + async function loadRun() { + const res = await fetch(`/api/runs/${processType}/${runId}`); + const payload = (await res.json()) as RunDetail; + if (!cancelled) { + setRun(payload); + } + } + loadRun(); + return () => { + cancelled = true; + }; + }, [processType, runId, refreshTick]); + + useEffect(() => { + let cancelled = false; + async function loadResults() { + setLoading(true); + const params = new URLSearchParams({ + page: String(page), + pageSize: String(pageSize), + sort: buildSortValue(sort), + }); + if (search) params.set("q", search); + if (verdict) params.set("verdict", verdict); + if (sellabilityStatus) params.set("sellabilityStatus", sellabilityStatus); + if (minConfidence) params.set("minConfidence", minConfidence); + if (maxConfidence) params.set("maxConfidence", maxConfidence); + + const res = await fetch(`/api/runs/${processType}/${runId}/results?${params.toString()}`); + const payload = (await res.json()) as ResultsResponse; + if (!cancelled) { + setResults(payload); + setLoading(false); + } + } + + loadResults(); + return () => { + cancelled = true; + }; + }, [processType, runId, search, verdict, sellabilityStatus, minConfidence, maxConfidence, page, pageSize, sort, refreshTick]); + + useEffect(() => { + const interval = window.setInterval(() => { + setRefreshTick((tick) => tick + 1); + }, 4000); + return () => { + window.clearInterval(interval); + }; + }, [processType, runId]); + + return ( +
+ + +
+

Run Detail

+
+
Process: {processType}
+
Run ID: {runId}
+
Status: {run ? {run.status} : "-"}
+
Timestamp: {run ? formatDate(run.timestamp) : "-"}
+
Job: {run?.jobType ?? "-"}
+
Source: {run?.source ?? "-"}
+
Total: {formatNumber(run?.summary.totalProducts)}
+
FBA/FBM/SKIP: {formatNumber(run?.summary.fbaCount)}/{formatNumber(run?.summary.fbmCount)}/{formatNumber(run?.summary.skipCount)}
+
+
+ +
+
+ +
+
+ { setPage(1); setSearch(e.target.value); }} placeholder="Search ASIN/name/brand/category/reason" /> + + + { setPage(1); setMinConfidence(e.target.value); }} placeholder="Min confidence" /> + { setPage(1); setMaxConfidence(e.target.value); }} placeholder="Max confidence" /> + +
+ +
+ +
+

Anomalies in current page

+ {anomalies.length === 0 ? ( +
No anomalies detected with current heuristic.
+ ) : ( +
+ {anomalies.slice(0, 8).map((item) => ( +
+ {item.asin} + {item.verdict} + {detectAnomaly(item)} +
+ ))} +
+ )} +
+ +
+
+ + + + + + + + + + + + + + + + + + + {loading ? ( + + ) : results?.items.length ? ( + results.items.map((item) => ( + + + + + + + + + + + + + + + )) + ) : ( + + )} + +
Loading...
{item.asin}{item.verdict}{formatNumber(item.monthly_sold)}{formatNumber(item.seller_count)}{formatNumber(item.sales_rank)}{formatCurrency(item.current_price)}{item.product_name || "-"}{item.brand || "-"}{item.category || "-"}{formatCurrency(item.avg_price_90d)}{formatNumber(item.confidence)}{item.reasoning || "-"}
No results found
+
+
+
Showing {results?.items.length ?? 0} of {results?.total ?? 0}
+
+ + Page {results?.page ?? page} / {results?.totalPages ?? 1} + +
+
+
+
+ ); +} + +function ProductList({ verdict, onBack }: { verdict: VerdictFilter; onBack: () => void }) { + const [items, setItems] = useState(null); + const [loading, setLoading] = useState(false); + const [search, setSearch] = useState(""); + const [activeVerdict, setActiveVerdict] = useState(verdict); + const [page, setPage] = useState(1); + const [pageSize, setPageSize] = useState(25); + + useEffect(() => { + setActiveVerdict(verdict); + setPage(1); + }, [verdict]); + + useEffect(() => { + let cancelled = false; + async function load() { + setLoading(true); + const params = new URLSearchParams({ page: String(page), pageSize: String(pageSize) }); + if (search) params.set("q", search); + if (activeVerdict) params.set("verdict", activeVerdict); + const res = await fetch(`/api/products?${params.toString()}`); + const payload = (await res.json()) as ProductListResponse; + if (!cancelled) { + setItems(payload); + setLoading(false); + } + } + + load(); + return () => { + cancelled = true; + }; + }, [search, activeVerdict, page, pageSize]); + + return ( +
+ +
+

Products

+
+ { setPage(1); setSearch(e.target.value); }} placeholder="Search ASIN/name/brand/category" /> + + +
+
+
+
+ + + + + + + + + + + + + + + + {loading ? ( + + ) : items?.items.length ? ( + items.items.map((item) => ( + + + + + + + + + + + + )) + ) : ( + + )} + +
ASINVerdictProductBrandCategoryConfidenceProcessRun IDFetched
Loading...
{item.asin}{item.verdict}{item.product_name || "-"}{item.brand || "-"}{item.category || "-"}{formatNumber(item.confidence)}{item.processType}{item.runId}{formatDate(item.fetched_at)}
No products found
+
+
+
Showing {items?.items.length ?? 0} of {items?.total ?? 0}
+
+ + Page {items?.page ?? page} / {items?.totalPages ?? 1} + +
+
+
+
+ ); +} + +type AppRoute = + | { kind: "dashboard" } + | { kind: "run"; processType: ProcessType; runId: number } + | { kind: "products"; verdict: VerdictFilter }; + +function parseRoute(pathname: string, search: string): AppRoute { + const runMatch = pathname.match(/^\/runs\/(lead_analysis|category_analysis)\/(\d+)$/); + if (runMatch) { + return { kind: "run", processType: runMatch[1] as ProcessType, runId: Number(runMatch[2]) }; + } + + if (pathname === "/products") { + const params = new URLSearchParams(search); + const verdictParam = params.get("verdict"); + const verdict = verdictParam === "FBA" || verdictParam === "FBM" || verdictParam === "SKIP" ? verdictParam : ""; + return { kind: "products", verdict }; + } + + return { kind: "dashboard" }; +} + +function App() { + const [route, setRoute] = useState(() => parseRoute(window.location.pathname, window.location.search)); + + useEffect(() => { + const onPopState = () => setRoute(parseRoute(window.location.pathname, window.location.search)); + window.addEventListener("popstate", onPopState); + return () => window.removeEventListener("popstate", onPopState); + }, []); + + function openRun(run: Run) { + const path = `/runs/${run.processType}/${run.runId}`; + history.pushState({}, "", path); + setRoute({ kind: "run", processType: run.processType, runId: run.runId }); + } + + function openProducts(verdict: VerdictFilter) { + const path = verdict ? `/products?verdict=${encodeURIComponent(verdict)}` : "/products"; + history.pushState({}, "", path); + setRoute({ kind: "products", verdict }); + } + + function backToDashboard() { + history.pushState({}, "", "/"); + setRoute({ kind: "dashboard" }); + } + + if (route.kind === "run") { + return ; + } + + if (route.kind === "products") { + return ; + } + + return ; +} + +const root = document.getElementById("root"); +if (!root) { + throw new Error("Root element not found"); +} + +createRoot(root).render(); diff --git a/src/web/index.html b/src/web/index.html new file mode 100644 index 0000000..9fe22cc --- /dev/null +++ b/src/web/index.html @@ -0,0 +1,13 @@ + + + + + + Run Results Viewer + + + +
+ + + diff --git a/src/web/styles.css b/src/web/styles.css new file mode 100644 index 0000000..febf273 --- /dev/null +++ b/src/web/styles.css @@ -0,0 +1,258 @@ +* { + box-sizing: border-box; +} + +body { + margin: 0; + font-family: Inter, ui-sans-serif, system-ui, -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif; + background: #f6f7f8; + color: #121418; +} + +#root { + max-width: 1280px; + margin: 0 auto; + padding: 20px; +} + +h1, +h2, +h3, +p { + margin: 0; +} + +.page { + display: flex; + flex-direction: column; + gap: 16px; +} + +.card { + background: #ffffff; + border: 1px solid #e7e8ea; + border-radius: 12px; + padding: 14px; +} + +.toolbar { + display: grid; + grid-template-columns: repeat(6, minmax(0, 1fr)); + gap: 10px; +} + +.toolbar input, +.toolbar select, +button { + height: 36px; + border-radius: 8px; + border: 1px solid #d8dce0; + background: #fff; + padding: 0 10px; + font-size: 14px; +} + +button { + cursor: pointer; +} + +.table-wrap { + overflow: auto; + border: 1px solid #eceef0; + border-radius: 10px; +} + +table { + width: 100%; + border-collapse: collapse; + min-width: 980px; +} + +th, +td { + text-align: left; + padding: 10px; + border-bottom: 1px solid #f0f1f3; + white-space: nowrap; + font-size: 13px; +} + +.reason-col { + min-width: 280px; + max-width: 420px; + white-space: normal; + overflow-wrap: anywhere; +} + +th { + background: #fafafb; + font-weight: 600; +} + +th button { + all: unset; + cursor: pointer; +} + +.badge { + display: inline-flex; + align-items: center; + justify-content: center; + padding: 3px 8px; + border-radius: 999px; + font-size: 12px; + font-weight: 600; +} + +.badge-fba { + background: #dff7e6; + color: #1e7a39; +} + +.badge-fbm { + background: #e6f0ff; + color: #245fce; +} + +.badge-skip { + background: #ffe9e7; + color: #b5382a; +} + +.badge-ok { + background: #dff7e6; + color: #1e7a39; +} + +.badge-failed { + background: #ffe9e7; + color: #b5382a; +} + +.badge-empty { + background: #f2f4f7; + color: #5f6b7a; +} + +.metrics { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: 10px; +} + +.metric { + background: #ffffff; + border: 1px solid #e7e8ea; + border-radius: 12px; + padding: 12px; +} + +.metric[role="button"]:hover { + cursor: pointer; +} + + +.metric .label { + font-size: 12px; + color: #677282; +} + +.metric .value { + margin-top: 4px; + font-size: 24px; + font-weight: 700; +} + +.pager { + margin-top: 10px; + display: flex; + justify-content: space-between; + align-items: center; +} + +.meta-grid { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: 12px; +} + +.meta { + font-size: 13px; + color: #445060; +} + +.back { + width: 120px; +} + +.tiny-bar { + width: 100px; + height: 8px; + border-radius: 999px; + background: #eef1f4; + overflow: hidden; + display: flex; +} + +.tiny-fba { + background: #1e7a39; + height: 100%; +} + +.tiny-fbm { + background: #245fce; + height: 100%; +} + +.tiny-skip { + background: #b5382a; + height: 100%; +} + +.spark-grid { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: 8px; +} + +.spark-item { + border: 1px solid #eceef0; + border-radius: 8px; + padding: 8px; +} + +.spark-label { + font-size: 11px; + color: #677282; + margin-bottom: 6px; +} + +.anomaly-list { + display: grid; + gap: 6px; +} + +.anomaly-item { + display: grid; + grid-template-columns: 1fr auto auto; + gap: 10px; + align-items: center; + border: 1px solid #eceef0; + border-radius: 8px; + padding: 8px; +} + +@media (max-width: 1000px) { + .toolbar { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + + .metrics, + .meta-grid { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + + .spark-grid { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } +} diff --git a/tsconfig.json b/tsconfig.json index 370696a..3ed803e 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,9 +1,10 @@ { "compilerOptions": { // Environment setup & latest features - "lib": ["ESNext"], + "lib": ["ESNext", "DOM", "DOM.Iterable"], "target": "ESNext", "module": "Preserve", + "types": ["bun-types", "node"], "moduleDetection": "force", "jsx": "react-jsx", "allowJs": true,