diff --git a/src/server.ts b/src/server.ts index 92a3a2e..8d6957e 100644 --- a/src/server.ts +++ b/src/server.ts @@ -71,6 +71,20 @@ type StalkerResultRecord = { inventory_sample_asins: string | null; }; +type StalkerProductRecord = { + runId: number; + started_at: string; + seller_id: string; + seller_name: string | null; + rating: number | null; + rating_count: number | null; + asin: string; + can_sell: number; + sellability_status: string; + sellability_reason: string | null; + last_seen_at: string; +}; + const DB_PATH = process.env.RESULTS_DB_PATH || path.join("db", "results.db"); const DEFAULT_PAGE_SIZE = 25; const MAX_PAGE_SIZE = 200; @@ -799,6 +813,143 @@ function getStalkerResults(filters: URLSearchParams) { }; } +function parseStalkerProductFilters(filters: URLSearchParams) { + const q = filters.get("q")?.trim() || ""; + const sellerId = filters.get("sellerId")?.trim().toUpperCase() || ""; + const runIdRaw = filters.get("runId")?.trim() || ""; + + const conditions = [ + "inv.can_sell = 1", + "inv.sellability_status = 'available'", + ]; + const params: Array = []; + + if (runIdRaw) { + const runId = Number(runIdRaw); + if (Number.isInteger(runId) && runId > 0) { + conditions.push("r.id = ?"); + params.push(runId); + } + } + + if (sellerId) { + conditions.push("s.seller_id = ?"); + params.push(sellerId); + } + + if (q) { + const wildcard = `%${q}%`; + conditions.push( + "(inv.asin LIKE ? OR s.seller_id LIKE ? OR s.seller_name LIKE ?)", + ); + params.push(wildcard, wildcard, wildcard); + } + + return { + where: `WHERE ${conditions.join(" AND ")}`, + params, + }; +} + +function parseStalkerProductSort(sortParam: string | null): string { + const allowedSort = new Set([ + "runId", + "started_at", + "seller_id", + "seller_name", + "rating", + "rating_count", + "asin", + "last_seen_at", + ]); + return parseSort(sortParam, allowedSort, "last_seen_at DESC, asin ASC"); +} + +function getStalkerProducts(filters: URLSearchParams) { + const page = parseIntParam(filters.get("page"), 1); + const pageSize = Math.min( + parseIntParam(filters.get("pageSize"), DEFAULT_PAGE_SIZE), + MAX_PAGE_SIZE, + ); + const offset = (page - 1) * pageSize; + const { where, params } = parseStalkerProductFilters(filters); + const orderBy = parseStalkerProductSort(filters.get("sort")); + + const baseSelect = ` + SELECT + r.id AS runId, + r.started_at, + s.seller_id, + s.seller_name, + s.rating, + s.rating_count, + inv.asin, + inv.can_sell, + inv.sellability_status, + inv.sellability_reason, + inv.last_seen_at + FROM stalker_seller_inventory inv + JOIN stalker_runs r ON r.id = inv.run_id + JOIN stalker_sellers s ON s.seller_id = inv.seller_id + ${where} + `; + + const totalRow = db + .query(`SELECT COUNT(*) AS total FROM (${baseSelect}) stalker_products`) + .get(...params) as { total: number }; + + const summary = db + .query( + `SELECT + COUNT(DISTINCT runId) AS runs, + COUNT(DISTINCT seller_id) AS sellers, + COUNT(DISTINCT asin) AS products + FROM (${baseSelect}) stalker_products`, + ) + .get(...params) as { + runs: number; + sellers: number; + products: number; + }; + + const items = db + .query( + `SELECT * FROM (${baseSelect}) stalker_products + ORDER BY ${orderBy} + LIMIT ? OFFSET ?`, + ) + .all(...params, pageSize, offset) as StalkerProductRecord[]; + + return { + items, + summary, + page, + pageSize, + total: totalRow.total, + totalPages: Math.max(1, Math.ceil(totalRow.total / pageSize)), + }; +} + +function purgeStalkerData() { + const counts = { + inventory: (db.query("SELECT COUNT(*) AS count FROM stalker_seller_inventory").get() as { count: number }).count, + asinSellers: (db.query("SELECT COUNT(*) AS count FROM stalker_asin_sellers").get() as { count: number }).count, + sellers: (db.query("SELECT COUNT(*) AS count FROM stalker_sellers").get() as { count: number }).count, + scans: (db.query("SELECT COUNT(*) AS count FROM stalker_asin_scans").get() as { count: number }).count, + runs: (db.query("SELECT COUNT(*) AS count FROM stalker_runs").get() as { count: number }).count, + }; + + db.transaction(() => { + db.run("DELETE FROM stalker_seller_inventory"); + db.run("DELETE FROM stalker_asin_sellers"); + db.run("DELETE FROM stalker_sellers"); + db.run("DELETE FROM stalker_asin_scans"); + db.run("DELETE FROM stalker_runs"); + })(); + + return { ok: true, deleted: counts }; +} + function getRun(processType: ProcessType, runId: number) { if (processType === "lead_analysis") { const run = db @@ -1430,6 +1581,7 @@ const server = Bun.serve({ "/": index, "/products": index, "/stalker": index, + "/stalker/products": index, "/runs/:processType/:runId": index, "/api/runs": (req) => { const url = new URL(req.url); @@ -1443,6 +1595,16 @@ const server = Bun.serve({ const url = new URL(req.url); return json(getStalkerResults(url.searchParams)); }, + "/api/stalker/products": (req) => { + const url = new URL(req.url); + return json(getStalkerProducts(url.searchParams)); + }, + "/api/stalker/purge": (req) => { + if (req.method !== "DELETE" && req.method !== "POST") { + return json({ error: "Method not allowed" }, 405); + } + return json(purgeStalkerData()); + }, "/api/upc/map": async (req) => { let upcs: string[]; try { diff --git a/src/sp-api.ts b/src/sp-api.ts index eace361..eb2b9a8 100644 --- a/src/sp-api.ts +++ b/src/sp-api.ts @@ -123,7 +123,10 @@ function round2(value: number): number { return Math.round(value * 100) / 100; } -const SELLABILITY_CONCURRENCY = 5; +const LISTINGS_RESTRICTIONS_RATE_LIMIT_PER_SECOND = 5; +const LISTINGS_RESTRICTIONS_BURST_REQUESTS = 10; +const SELLABILITY_CONCURRENCY = LISTINGS_RESTRICTIONS_RATE_LIMIT_PER_SECOND; +const SELLABILITY_PROGRESS_INTERVAL = LISTINGS_RESTRICTIONS_BURST_REQUESTS; const PRICING_CONCURRENCY = 5; const UPC_PATTERN = /^\d{12,14}$/; @@ -621,8 +624,7 @@ export async function fetchSellabilityBatch( } let completed = 0; - let running = 0; - const queue = [...asins]; + const queue = [...asins]; async function next(): Promise { while (queue.length > 0) { @@ -630,9 +632,12 @@ export async function fetchSellabilityBatch( const info = await fetchSellabilityInternal(spClient!, asin); results.set(asin, info); completed++; - if (completed % 10 === 0 || completed === asins.length) { - console.log(` [sellability] ${completed}/${asins.length} checked`); - } + if ( + completed % SELLABILITY_PROGRESS_INTERVAL === 0 || + completed === asins.length + ) { + console.log(` [sellability] ${completed}/${asins.length} checked`); + } } } diff --git a/src/stalker.test.ts b/src/stalker.test.ts index 6c5ad7e..e86e2d5 100644 --- a/src/stalker.test.ts +++ b/src/stalker.test.ts @@ -222,7 +222,7 @@ test("runStalker fetches product offers, filters sellers, and persists storefron expect(stats.scannedAsins).toBe(1); expect(stats.sourceAsinsWithMatches).toBe(1); expect(stats.matchedSellers).toBe(1); - expect(stats.persistedInventoryAsins).toBe(2); + expect(stats.persistedInventoryAsins).toBe(0); expect(stats.failedAsins).toBe(0); expect(stats.candidateSellers).toBe(2); expect(stats.qualifyingSellers).toBe(1); @@ -253,7 +253,7 @@ test("runStalker fetches product offers, filters sellers, and persists storefron expect(run.inventory_sellability_checked_asins).toBe(0); expect(run.inventory_sellability_available_asins).toBe(0); expect(run.inventory_sellability_excluded_asins).toBe(0); - expect(run.persisted_inventory_asins).toBe(2); + expect(run.persisted_inventory_asins).toBe(0); const scan = db.query("SELECT * FROM stalker_asin_scans").get() as any; expect(scan.source_asin).toBe("B000000001"); @@ -267,7 +267,7 @@ test("runStalker fetches product offers, filters sellers, and persists storefron expect(sellers[0].seller_id).toBe("AQUALIFIED"); expect(sellers[0].rating_count).toBe(12); expect(sellers[0].storefront_asin_total).toBe(2); - expect(sellers[0].persisted_inventory_sample_count).toBe(2); + expect(sellers[0].persisted_inventory_sample_count).toBe(0); const asinSellers = db.query("SELECT * FROM stalker_asin_sellers").all() as any[]; expect(asinSellers.length).toBe(1); @@ -278,8 +278,5 @@ test("runStalker fetches product offers, filters sellers, and persists storefron const inventory = db .query("SELECT asin FROM stalker_seller_inventory ORDER BY asin") .all() as Array<{ asin: string }>; - expect(inventory.map((row) => row.asin)).toEqual([ - "B111111111", - "B222222222", - ]); + expect(inventory.map((row) => row.asin)).toEqual([]); }); diff --git a/src/stalker.ts b/src/stalker.ts index 8fba937..712a030 100644 --- a/src/stalker.ts +++ b/src/stalker.ts @@ -338,6 +338,7 @@ export async function runStalker(args: StalkerArgs): Promise { if (args.sellability && !args.dryRun) { await enrichInventorySellability(result, stats); } + applyInventoryPersistencePolicy(result, args.sellability && !args.dryRun); if (!args.dryRun && runId != null) { persistAsinResult(database, runId, result); @@ -457,6 +458,22 @@ async function scanAsin( }; } +function applyInventoryPersistencePolicy( + result: StalkerAsinResult, + requireAvailableSellability: boolean, +): void { + for (const { seller } of result.matchedSellers) { + seller.storefrontItems = seller.storefrontItems.filter((item) => { + if (!requireAvailableSellability) return false; + return ( + item.sellability?.canSell === true && + item.sellability.sellabilityStatus === "available" + ); + }); + seller.storefrontAsins = seller.storefrontItems.map((item) => item.asin); + } +} + async function enrichInventorySellability( result: StalkerAsinResult, stats: StalkerRunStats, @@ -833,6 +850,13 @@ function upsertSellerInventory( ); for (const item of seller.storefrontItems) { + if ( + item.sellability?.canSell !== true || + item.sellability.sellabilityStatus !== "available" + ) { + continue; + } + insert.run( runId, seller.sellerId, diff --git a/src/web/frontend.tsx b/src/web/frontend.tsx index f4bf2ad..dd35182 100644 --- a/src/web/frontend.tsx +++ b/src/web/frontend.tsx @@ -140,6 +140,33 @@ type StalkerResultsResponse = { totalPages: number; }; +type StalkerProductItem = { + runId: number; + started_at: string; + seller_id: string; + seller_name: string | null; + rating: number | null; + rating_count: number | null; + asin: string; + can_sell: number; + sellability_status: string; + sellability_reason: string | null; + last_seen_at: string; +}; + +type StalkerProductsResponse = { + items: StalkerProductItem[]; + summary: { + runs: number; + sellers: number; + products: number; + }; + page: number; + pageSize: number; + total: number; + totalPages: number; +}; + type SortState = { field: string; direction: SortDirection; @@ -234,10 +261,12 @@ function Dashboard({ onOpenRun, onOpenProducts, onOpenStalker, + onOpenStalkerProducts, }: { onOpenRun: (run: Run) => void; onOpenProducts: (verdict: VerdictFilter) => void; onOpenStalker: () => void; + onOpenStalkerProducts: () => void; }) { const [runs, setRuns] = useState(null); const [loading, setLoading] = useState(false); @@ -328,7 +357,10 @@ function Dashboard({

Runs Dashboard

- +
+ + +
@@ -889,9 +921,16 @@ function ProductList({ verdict, onBack }: { verdict: VerdictFilter; onBack: () = ); } -function StalkerExplorer({ onBack }: { onBack: () => void }) { +function StalkerExplorer({ + onBack, + onOpenProducts, +}: { + onBack: () => void; + onOpenProducts: () => void; +}) { const [results, setResults] = useState(null); const [loading, setLoading] = useState(false); + const [purging, setPurging] = useState(false); const [search, setSearch] = useState(""); const [sellerId, setSellerId] = useState(""); const [runId, setRunId] = useState(""); @@ -900,6 +939,7 @@ function StalkerExplorer({ onBack }: { onBack: () => void }) { const [page, setPage] = useState(1); const [pageSize, setPageSize] = useState(25); const [sort, setSort] = useState({ field: "persisted_inventory_asin_count", direction: "DESC" }); + const [refreshTick, setRefreshTick] = useState(0); useEffect(() => { let cancelled = false; @@ -928,14 +968,41 @@ function StalkerExplorer({ onBack }: { onBack: () => void }) { return () => { cancelled = true; }; - }, [search, sellerId, runId, minRatingCount, maxRatingCount, page, pageSize, sort]); + }, [search, sellerId, runId, minRatingCount, maxRatingCount, page, pageSize, sort, refreshTick]); + + async function purgeStalkerData() { + const confirmed = window.confirm("Permanently delete all Stalker runs, sellers, and sellable products from the database?"); + if (!confirmed) return; + + setPurging(true); + try { + const res = await fetch("/api/stalker/purge", { method: "DELETE" }); + if (!res.ok) { + const payload = await res.json().catch(() => null) as { error?: string } | null; + window.alert(payload?.error ?? "Failed to purge Stalker data"); + return; + } + setPage(1); + setRefreshTick((tick) => tick + 1); + } finally { + setPurging(false); + } + } return (
-

Seller Storefronts

+
+

Seller Storefronts

+
+ + +
+
@@ -1032,11 +1099,142 @@ function StalkerExplorer({ onBack }: { onBack: () => void }) { ); } +function StalkerProductsExplorer({ + onBack, + onOpenSellers, +}: { + onBack: () => void; + onOpenSellers: () => void; +}) { + const [results, setResults] = useState(null); + const [loading, setLoading] = useState(false); + const [search, setSearch] = useState(""); + const [sellerId, setSellerId] = useState(""); + const [runId, setRunId] = useState(""); + const [page, setPage] = useState(1); + const [pageSize, setPageSize] = useState(50); + const [sort, setSort] = useState({ field: "last_seen_at", direction: "DESC" }); + + useEffect(() => { + let cancelled = false; + async function load() { + setLoading(true); + const params = new URLSearchParams({ + page: String(page), + pageSize: String(pageSize), + sort: buildSortValue(sort), + }); + if (search) params.set("q", search); + if (sellerId) params.set("sellerId", sellerId); + if (runId) params.set("runId", runId); + + const res = await fetch(`/api/stalker/products?${params.toString()}`); + const payload = (await res.json()) as StalkerProductsResponse; + if (!cancelled) { + setResults(payload); + setLoading(false); + } + } + + load(); + return () => { + cancelled = true; + }; + }, [search, sellerId, runId, page, pageSize, sort]); + + return ( +
+ + +
+
+

Sellable Stalker Products

+ +
+
+ +
+
+
Runs
+
{formatNumber(results?.summary.runs)}
+
+
+
Sellers
+
{formatNumber(results?.summary.sellers)}
+
+
+
Sellable products
+
{formatNumber(results?.summary.products)}
+
+
+ +
+
+ { setPage(1); setSearch(e.target.value); }} placeholder="Search ASIN or seller" /> + { setPage(1); setSellerId(e.target.value.toUpperCase()); }} placeholder="Seller ID" /> + { setPage(1); setRunId(e.target.value); }} placeholder="Run ID" /> + +
+
+ +
+
+ + + + + + + + + + + + + + {loading ? ( + + ) : results?.items.length ? ( + results.items.map((item) => ( + + + + + + + + + + )) + ) : ( + + )} + +
Status
Loading...
{item.asin}{item.seller_id}{item.seller_name || "-"}{formatNumber(item.rating_count)}{item.sellability_status}{item.runId}{formatDate(item.last_seen_at)}
No sellable Stalker products found
+
+
+
Showing {results?.items.length ?? 0} of {results?.total ?? 0}
+
+ + Page {results?.page ?? page} / {results?.totalPages ?? 1} + +
+
+
+
+ ); +} + type AppRoute = | { kind: "dashboard" } | { kind: "run"; processType: ProcessType; runId: number } | { kind: "products"; verdict: VerdictFilter } - | { kind: "stalker" }; + | { kind: "stalker" } + | { kind: "stalker-products" }; function parseRoute(pathname: string, search: string): AppRoute { const runMatch = pathname.match(/^\/runs\/(lead_analysis|category_analysis)\/(\d+)$/); @@ -1055,6 +1253,10 @@ function parseRoute(pathname: string, search: string): AppRoute { return { kind: "stalker" }; } + if (pathname === "/stalker/products") { + return { kind: "stalker-products" }; + } + return { kind: "dashboard" }; } @@ -1084,6 +1286,11 @@ function App() { setRoute({ kind: "stalker" }); } + function openStalkerProducts() { + history.pushState({}, "", "/stalker/products"); + setRoute({ kind: "stalker-products" }); + } + function backToDashboard() { history.pushState({}, "", "/"); setRoute({ kind: "dashboard" }); @@ -1098,10 +1305,21 @@ function App() { } if (route.kind === "stalker") { - return ; + return ; } - return ; + if (route.kind === "stalker-products") { + return ; + } + + return ( + + ); } const root = document.getElementById("root"); diff --git a/src/web/styles.css b/src/web/styles.css index 205f4dd..8303a4a 100644 --- a/src/web/styles.css +++ b/src/web/styles.css @@ -48,6 +48,13 @@ p { gap: 12px; } +.button-row { + display: flex; + align-items: center; + gap: 8px; + flex-wrap: wrap; +} + .toolbar input, .toolbar select, button { @@ -63,6 +70,17 @@ button { cursor: pointer; } +button.danger { + border-color: #efb8b8; + color: #9f1c1c; + background: #fff6f6; +} + +button:disabled { + cursor: not-allowed; + opacity: 0.58; +} + .table-wrap { overflow: auto; border: 1px solid #eceef0;