Compare commits

...

10 Commits

Author SHA1 Message Date
Victor Noguera
1d2e92addb Merge branch 'jaime' 2026-05-20 16:24:17 -04:00
Victor Noguera
f8bc05685e feat: add XLSX export functionality and refactor argument parsing in main script 2026-05-20 16:18:12 -04:00
Victor Noguera
0c2e59771c feat: add XLSX export functionality for Stalker products and enhance UI for export link 2026-05-19 23:12:34 -04:00
Victor Noguera
90bfee8791 feat: add advanced filtering options for Stalker products including price, sales rank, and seller metrics 2026-05-19 23:01:28 -04:00
Victor Noguera
1f57900da2 feat: implement batch processing for product analysis with delay and error handling 2026-05-19 20:24:08 -04:00
Victor Noguera
7bda3710ed feat: update Keepa and Stalker functionalities with enhanced price extraction logic and test cases 2026-05-19 19:59:20 -04:00
Victor Noguera
0552d183b3 feat: enhance Stalker functionality with additional product details and analysis capabilities 2026-05-19 19:57:53 -04:00
Victor Noguera
f6178a665c feat: add Stalker products functionality with filtering, pagination, and purge option 2026-05-19 19:37:05 -04:00
Victor Noguera
aed0c11017 feat: enhance stalker functionality with inventory sellability checks and update frontend display 2026-05-19 18:35:55 -04:00
Victor Noguera
a7c0e44e3d feat: add Stalker results page with filtering and pagination
- Introduced StalkerResultItem and StalkerResultsResponse types for handling API responses.
- Implemented StalkerExplorer component for displaying Stalker results with search and filter options.
- Added sorting functionality for Stalker results table.
- Enhanced Dashboard to include a button for navigating to Stalker results.
- Updated routing to support Stalker results page.
- Improved styles for section headers and inventory columns in the results table.
2026-05-19 18:10:01 -04:00
14 changed files with 3921 additions and 38 deletions

View File

@@ -7,6 +7,7 @@
"bestsellers": "bun run src/bestsellers-by-category.ts", "bestsellers": "bun run src/bestsellers-by-category.ts",
"monthly-sold": "bun run src/top-monthly-sold-by-category.ts", "monthly-sold": "bun run src/top-monthly-sold-by-category.ts",
"mid-range": "bun run src/mid-range-sellers-by-category.ts", "mid-range": "bun run src/mid-range-sellers-by-category.ts",
"stalker": "bun run src/stalker.ts",
"upc": "bun run src/upc-lookup.ts", "upc": "bun run src/upc-lookup.ts",
"upc-file": "bun run src/upc-file-analysis.ts", "upc-file": "bun run src/upc-file-analysis.ts",
"start": "bun run src/index.ts", "start": "bun run src/index.ts",

View File

@@ -333,4 +333,162 @@ export function initDb(dbPath: string): void {
database.run( database.run(
`CREATE INDEX IF NOT EXISTS idx_product_results_fetched_at ON product_analysis_results(fetched_at DESC);`, `CREATE INDEX IF NOT EXISTS idx_product_results_fetched_at ON product_analysis_results(fetched_at DESC);`,
); );
initStalkerDb(database);
}
export function initStalkerDb(database: Database): void {
resetLegacyStalkerSchema(database);
database.run(`
CREATE TABLE IF NOT EXISTS stalker_runs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
input_file TEXT NOT NULL,
started_at TEXT NOT NULL,
completed_at TEXT,
requested_asins INTEGER NOT NULL DEFAULT 0,
skipped_asins INTEGER NOT NULL DEFAULT 0,
scanned_asins INTEGER NOT NULL DEFAULT 0,
source_asins_with_matches INTEGER NOT NULL DEFAULT 0,
candidate_sellers INTEGER NOT NULL DEFAULT 0,
qualifying_sellers INTEGER NOT NULL DEFAULT 0,
matched_sellers INTEGER NOT NULL DEFAULT 0,
seller_metadata_requests INTEGER NOT NULL DEFAULT 0,
seller_storefront_requests INTEGER NOT NULL DEFAULT 0,
inventory_sellability_checked_asins INTEGER NOT NULL DEFAULT 0,
inventory_sellability_available_asins INTEGER NOT NULL DEFAULT 0,
inventory_sellability_excluded_asins INTEGER NOT NULL DEFAULT 0,
persisted_inventory_asins INTEGER NOT NULL DEFAULT 0,
status TEXT NOT NULL,
error_message TEXT
);
`);
database.run(`
CREATE TABLE IF NOT EXISTS stalker_asin_scans (
id INTEGER PRIMARY KEY AUTOINCREMENT,
run_id INTEGER NOT NULL,
source_asin TEXT NOT NULL,
title TEXT,
offer_count INTEGER NOT NULL DEFAULT 0,
candidate_seller_count INTEGER NOT NULL DEFAULT 0,
matched_seller_count INTEGER NOT NULL DEFAULT 0,
fetched_at TEXT NOT NULL,
raw_product_json TEXT,
UNIQUE(run_id, source_asin),
FOREIGN KEY (run_id) REFERENCES stalker_runs(id) ON DELETE CASCADE
);
`);
database.run(`
CREATE TABLE IF NOT EXISTS stalker_sellers (
seller_id TEXT PRIMARY KEY,
seller_name TEXT,
rating REAL,
rating_count INTEGER,
storefront_asin_total INTEGER,
persisted_inventory_sample_count INTEGER,
last_updated_at TEXT NOT NULL,
raw_seller_json TEXT
);
`);
database.run(`
CREATE TABLE IF NOT EXISTS stalker_asin_sellers (
id INTEGER PRIMARY KEY AUTOINCREMENT,
scan_id INTEGER NOT NULL,
seller_id TEXT NOT NULL,
offer_price REAL,
condition TEXT,
is_fba INTEGER,
stock INTEGER,
seller_rating REAL,
seller_rating_count INTEGER,
raw_offer_json TEXT,
UNIQUE(scan_id, seller_id),
FOREIGN KEY (scan_id) REFERENCES stalker_asin_scans(id) ON DELETE CASCADE,
FOREIGN KEY (seller_id) REFERENCES stalker_sellers(seller_id)
);
`);
database.run(`
CREATE TABLE IF NOT EXISTS stalker_seller_inventory (
id INTEGER PRIMARY KEY AUTOINCREMENT,
run_id INTEGER NOT NULL,
seller_id TEXT NOT NULL,
asin TEXT NOT NULL,
can_sell INTEGER,
sellability_status TEXT,
sellability_reason TEXT,
product_title TEXT,
brand TEXT,
category_tree TEXT,
current_price REAL,
avg_price_90d REAL,
sales_rank INTEGER,
monthly_sold INTEGER,
seller_count INTEGER,
amazon_is_seller INTEGER,
raw_product_json TEXT,
last_seen_at TEXT NOT NULL,
raw_inventory_json TEXT,
UNIQUE(run_id, seller_id, asin),
FOREIGN KEY (run_id) REFERENCES stalker_runs(id) ON DELETE CASCADE,
FOREIGN KEY (seller_id) REFERENCES stalker_sellers(seller_id)
);
`);
database.run(
`CREATE INDEX IF NOT EXISTS idx_stalker_runs_started_at ON stalker_runs(started_at DESC);`,
);
database.run(
`CREATE INDEX IF NOT EXISTS idx_stalker_scans_run_id ON stalker_asin_scans(run_id);`,
);
database.run(
`CREATE INDEX IF NOT EXISTS idx_stalker_scans_source_asin ON stalker_asin_scans(source_asin);`,
);
database.run(
`CREATE INDEX IF NOT EXISTS idx_stalker_asin_sellers_seller_id ON stalker_asin_sellers(seller_id);`,
);
database.run(
`CREATE INDEX IF NOT EXISTS idx_stalker_inventory_seller_id ON stalker_seller_inventory(seller_id);`,
);
database.run(
`CREATE INDEX IF NOT EXISTS idx_stalker_inventory_asin ON stalker_seller_inventory(asin);`,
);
database.run(
`CREATE INDEX IF NOT EXISTS idx_stalker_inventory_product_title ON stalker_seller_inventory(product_title);`,
);
}
function resetLegacyStalkerSchema(database: Database): void {
const runColumns = database
.query("PRAGMA table_info(stalker_runs)")
.all() as Array<{ name: string }>;
if (runColumns.length === 0) return;
const columnNames = new Set(runColumns.map((column) => column.name));
if (
columnNames.has("scanned_asins") &&
columnNames.has("inventory_sellability_checked_asins") &&
inventoryColumnsHaveSellability(database)
) {
return;
}
database.run("DROP TABLE IF EXISTS stalker_seller_inventory");
database.run("DROP TABLE IF EXISTS stalker_asin_sellers");
database.run("DROP TABLE IF EXISTS stalker_sellers");
database.run("DROP TABLE IF EXISTS stalker_asin_scans");
database.run("DROP TABLE IF EXISTS stalker_runs");
}
function inventoryColumnsHaveSellability(database: Database): boolean {
const inventoryColumns = database
.query("PRAGMA table_info(stalker_seller_inventory)")
.all() as Array<{ name: string }>;
const columnNames = new Set(inventoryColumns.map((column) => column.name));
return (
columnNames.has("sellability_status") &&
columnNames.has("product_title")
);
} }

View File

@@ -1,6 +1,10 @@
import { readProducts } from "./reader.ts"; import { readProducts } from "./reader.ts";
import { connectCache, disconnectCache } from "./cache.ts"; import { connectCache, disconnectCache } from "./cache.ts";
import { printResults, writeResultsToDb } from "./writer.ts"; import {
printResults,
writeResultsToDb,
writeResultsWorkbook,
} from "./writer.ts";
import { initDb, closeDb } from "./database.ts"; import { initDb, closeDb } from "./database.ts";
import { import {
chunkArray, chunkArray,
@@ -40,14 +44,18 @@ function parseArgs(): {
sellability: SellabilityFilter; sellability: SellabilityFilter;
} { } {
const args = process.argv.slice(2); const args = process.argv.slice(2);
const inputFile = args.find((a) => !a.startsWith("--")); const outputFile = readFlagValue(args, "--out", "--output");
const outIdx = args.indexOf("--out"); const inputFile = readInputFileArg(
const outputFile = outIdx !== -1 ? args[outIdx + 1] : undefined; args,
"--out",
"--output",
"--sellability",
);
const sellability = parseSellabilityArg(args); const sellability = parseSellabilityArg(args);
if (!inputFile) { if (!inputFile) {
console.error( console.error(
"Usage: bun run src/index.ts <input.csv|xlsx> [--out results.csv] [--sellability available|all]", "Usage: bun run src/index.ts <input.csv|xlsx> [--out results.xlsx|--output results.xlsx] [--sellability available|all]",
); );
process.exit(1); process.exit(1);
} }
@@ -55,6 +63,44 @@ function parseArgs(): {
return { inputFile, outputFile, sellability }; return { inputFile, outputFile, sellability };
} }
function readFlagValue(args: string[], ...flags: string[]): string | undefined {
for (const flag of flags) {
const equalsArg = args.find((arg) => arg.startsWith(`${flag}=`));
if (equalsArg) {
const value = equalsArg.slice(flag.length + 1);
if (value) return value;
}
const flagIdx = args.indexOf(flag);
if (flagIdx !== -1) {
return args[flagIdx + 1];
}
}
return undefined;
}
function readInputFileArg(
args: string[],
...flagsWithValues: string[]
): string | undefined {
for (let i = 0; i < args.length; i++) {
const arg = args[i]!;
if (flagsWithValues.includes(arg)) {
i++;
continue;
}
if (flagsWithValues.some((flag) => arg.startsWith(`${flag}=`))) {
continue;
}
if (!arg.startsWith("--")) {
return arg;
}
}
return undefined;
}
function resolveBaseOutputPath(inputFile: string, outputFile?: string): string { function resolveBaseOutputPath(inputFile: string, outputFile?: string): string {
if (outputFile) return outputFile; if (outputFile) return outputFile;
@@ -103,7 +149,8 @@ async function main() {
} }
printResults(allResults); printResults(allResults);
writeResultsToDb(allResults, DB_PATH, inputFile, outputFile); writeResultsWorkbook(allResults, resolvedBaseOutputPath);
writeResultsToDb(allResults, DB_PATH, inputFile, resolvedBaseOutputPath);
} finally { } finally {
await disconnectCache(); await disconnectCache();
closeDb(); closeDb();

View File

@@ -48,7 +48,7 @@ test("lookupKeepaUpcs returns found, not_found, and multiple_asins outcomes", as
current: [null, null, null, 1234], current: [null, null, null, 1234],
avg: [2500, null, null, 1400], avg: [2500, null, null, 1400],
}, },
csv: [[1, 2999]], csv: [[5000000, 2999, 5000100]],
}, },
{ {
asin: "B000MULTI01", asin: "B000MULTI01",
@@ -85,6 +85,7 @@ test("lookupKeepaUpcs returns found, not_found, and multiple_asins outcomes", as
expect(details.get("012345678901")?.status).toBe("found"); expect(details.get("012345678901")?.status).toBe("found");
expect(details.get("012345678901")?.asin).toBe("B000FOUND01"); expect(details.get("012345678901")?.asin).toBe("B000FOUND01");
expect(details.get("012345678901")?.keepaData?.currentPrice).toBe(29.99); expect(details.get("012345678901")?.keepaData?.currentPrice).toBe(29.99);
expect(details.get("012345678901")?.keepaData?.currentPrice).toBe(29.99);
expect(details.get("098765432109")?.status).toBe("multiple_asins"); expect(details.get("098765432109")?.status).toBe("multiple_asins");
expect(details.get("098765432109")?.candidateAsins).toEqual([ expect(details.get("098765432109")?.candidateAsins).toEqual([

View File

@@ -529,14 +529,17 @@ function computeAmazonBuyBoxSharePctFromHistory(
return Math.round((amazonMinutes / qualifiedMinutes) * 10_000) / 100; return Math.round((amazonMinutes / qualifiedMinutes) * 10_000) / 100;
} }
function extractLatestPositivePrice(series: unknown): number | null { function extractLatestPositivePrice(series: unknown): number | null {
if (!Array.isArray(series) || series.length < 2) return null; if (!Array.isArray(series) || series.length < 2) return null;
const last = series[series.length - 1]; for (let i = series.length - 1; i >= 1; i--) {
if (typeof last !== "number" || !Number.isFinite(last) || last <= 0) { if (i % 2 === 0) continue;
return null; const value = series[i];
} if (typeof value === "number" && Number.isFinite(value) && value > 0) {
return last / 100; return value / 100;
} }
}
return null;
}
function pickKeepaNumber(...values: unknown[]): number | null { function pickKeepaNumber(...values: unknown[]): number | null {
for (const value of values) { for (const value of values) {
@@ -548,16 +551,14 @@ function pickKeepaNumber(...values: unknown[]): number | null {
return null; return null;
} }
function extractCurrentPrice(csv: number[][] | undefined): number | null { function extractCurrentPrice(csv: number[][] | undefined): number | null {
if (!csv) return null; if (!csv) return null;
// csv[0] = Amazon price history, csv[1] = Marketplace new price history // csv[0] = Amazon price history, csv[1] = Marketplace new price history
// Each is [time, price, time, price, ...] — last value is most recent // Each is [time, price, time, price, ...]. Only odd indexes are prices.
for (const series of [csv[0], csv[1]]) { for (const series of [csv[0], csv[1]]) {
if (series && series.length >= 2) { const latestPrice = extractLatestPositivePrice(series);
const lastPrice = series[series.length - 1]!; if (latestPrice != null) return latestPrice;
if (lastPrice > 0) return lastPrice / 100; }
} return null;
} }
return null;
}

View File

@@ -1,5 +1,6 @@
import index from "./web/index.html"; import index from "./web/index.html";
import path from "node:path"; import path from "node:path";
import * as XLSX from "xlsx";
import { getDb, initDb } from "./database.ts"; import { getDb, initDb } from "./database.ts";
import { import {
fetchKeepaDataBatch, fetchKeepaDataBatch,
@@ -53,6 +54,50 @@ type ProductListRecord = {
fetched_at: string; fetched_at: string;
}; };
type StalkerResultRecord = {
runId: number;
started_at: string;
status: string;
input_file: string;
seller_id: string;
seller_name: string | null;
rating: number | null;
rating_count: number | null;
storefront_asin_total: number | null;
persisted_inventory_sample_count: number | null;
discovered_from_count: number;
first_seen_at: string;
last_seen_at: string;
persisted_inventory_asin_count: number;
inventory_sample_asins: string | null;
};
type StalkerProductRecord = {
runId: number;
started_at: string;
seller_id: string;
seller_name: string | null;
rating: number | null;
rating_count: number | null;
asin: string;
can_sell: number;
sellability_status: string;
sellability_reason: string | null;
product_title: string | null;
brand: string | null;
category_tree: string | null;
current_price: number | null;
avg_price_90d: number | null;
sales_rank: number | null;
monthly_sold: number | null;
seller_count: number | null;
amazon_is_seller: number | null;
verdict: string | null;
confidence: number | null;
reasoning: string | null;
last_seen_at: string;
};
const DB_PATH = process.env.RESULTS_DB_PATH || path.join("db", "results.db"); const DB_PATH = process.env.RESULTS_DB_PATH || path.join("db", "results.db");
const DEFAULT_PAGE_SIZE = 25; const DEFAULT_PAGE_SIZE = 25;
const MAX_PAGE_SIZE = 200; const MAX_PAGE_SIZE = 200;
@@ -79,6 +124,16 @@ function csv(text: string, filename: string): Response {
}); });
} }
function xlsx(buffer: ArrayBuffer, filename: string): Response {
return new Response(buffer, {
status: 200,
headers: {
"content-type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"content-disposition": `attachment; filename="${filename}"`,
},
});
}
function parseIntParam(value: string | null, fallback: number): number { function parseIntParam(value: string | null, fallback: number): number {
if (!value) return fallback; if (!value) return fallback;
const parsed = Number.parseInt(value, 10); const parsed = Number.parseInt(value, 10);
@@ -629,6 +684,495 @@ function getProductList(filters: URLSearchParams) {
}; };
} }
function parseStalkerFilters(filters: URLSearchParams) {
const q = filters.get("q")?.trim() || "";
const sellerId = filters.get("sellerId")?.trim().toUpperCase() || "";
const runIdRaw = filters.get("runId")?.trim() || "";
const minRatingCountRaw = filters.get("minRatingCount")?.trim() || "";
const maxRatingCountRaw = filters.get("maxRatingCount")?.trim() || "";
const conditions: string[] = [];
const params: Array<string | number> = [];
if (runIdRaw) {
const runId = Number(runIdRaw);
if (Number.isInteger(runId) && runId > 0) {
conditions.push("r.id = ?");
params.push(runId);
}
}
if (sellerId) {
conditions.push("s.seller_id = ?");
params.push(sellerId);
}
if (minRatingCountRaw) {
conditions.push("s.rating_count >= ?");
params.push(Number(minRatingCountRaw));
}
if (maxRatingCountRaw) {
conditions.push("s.rating_count <= ?");
params.push(Number(maxRatingCountRaw));
}
if (q) {
const wildcard = `%${q}%`;
conditions.push(
`(s.seller_id LIKE ? OR s.seller_name LIKE ? OR EXISTS (
SELECT 1 FROM stalker_seller_inventory inv_q
WHERE inv_q.run_id = r.id
AND inv_q.seller_id = s.seller_id
AND inv_q.asin LIKE ?
))`,
);
params.push(wildcard, wildcard, wildcard);
}
return {
where: conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "",
params,
};
}
function parseStalkerSort(sortParam: string | null): string {
const allowedSort = new Set([
"runId",
"started_at",
"seller_id",
"seller_name",
"rating",
"rating_count",
"discovered_from_count",
"persisted_inventory_asin_count",
"storefront_asin_total",
"last_seen_at",
]);
const parsed = parseSort(
sortParam,
allowedSort,
"persisted_inventory_asin_count DESC, last_seen_at DESC, seller_id ASC",
);
return parsed
.replaceAll("runId", "runId")
.replaceAll("rating_count", "rating_count")
.replaceAll("persisted_inventory_asin_count", "persisted_inventory_asin_count")
.replaceAll("storefront_asin_total", "storefront_asin_total");
}
function getStalkerResults(filters: URLSearchParams) {
const page = parseIntParam(filters.get("page"), 1);
const pageSize = Math.min(
parseIntParam(filters.get("pageSize"), DEFAULT_PAGE_SIZE),
MAX_PAGE_SIZE,
);
const offset = (page - 1) * pageSize;
const { where, params } = parseStalkerFilters(filters);
const orderBy = parseStalkerSort(filters.get("sort"));
const baseSelect = `
SELECT
r.id AS runId,
r.started_at,
r.status,
r.input_file,
s.seller_id,
s.seller_name,
s.rating,
s.rating_count,
s.storefront_asin_total,
s.persisted_inventory_sample_count,
COUNT(DISTINCT sc.source_asin) AS discovered_from_count,
MIN(sc.fetched_at) AS first_seen_at,
MAX(sc.fetched_at) AS last_seen_at,
COUNT(DISTINCT inv.asin) AS persisted_inventory_asin_count,
GROUP_CONCAT(DISTINCT inv.asin) AS inventory_sample_asins
FROM stalker_asin_sellers sas
JOIN stalker_asin_scans sc ON sc.id = sas.scan_id
JOIN stalker_runs r ON r.id = sc.run_id
JOIN stalker_sellers s ON s.seller_id = sas.seller_id
LEFT JOIN stalker_seller_inventory inv
ON inv.run_id = r.id
AND inv.seller_id = s.seller_id
${where}
GROUP BY r.id, s.seller_id
`;
const totalRow = db
.query(`SELECT COUNT(*) AS total FROM (${baseSelect}) stalker_rows`)
.get(...params) as { total: number };
const summary = db
.query(
`SELECT
COUNT(DISTINCT runId) AS runs,
COUNT(DISTINCT seller_id) AS sellers,
COALESCE(SUM(persisted_inventory_asin_count), 0) AS persistedInventoryAsins
FROM (${baseSelect}) stalker_rows`,
)
.get(...params) as {
runs: number;
sellers: number;
persistedInventoryAsins: number;
};
const items = db
.query(
`SELECT * FROM (${baseSelect}) stalker_rows
ORDER BY ${orderBy}
LIMIT ? OFFSET ?`,
)
.all(...params, pageSize, offset) as StalkerResultRecord[];
return {
items,
summary,
page,
pageSize,
total: totalRow.total,
totalPages: Math.max(1, Math.ceil(totalRow.total / pageSize)),
};
}
function parseStalkerProductFilters(filters: URLSearchParams) {
const q = filters.get("q")?.trim() || "";
const sellerId = filters.get("sellerId")?.trim().toUpperCase() || "";
const runIdRaw = filters.get("runId")?.trim() || "";
const verdict = filters.get("verdict")?.trim().toUpperCase() || "";
const amazonIsSeller = filters.get("amazonIsSeller")?.trim() || "";
const minPriceRaw = filters.get("minPrice")?.trim() || "";
const maxPriceRaw = filters.get("maxPrice")?.trim() || "";
const minMonthlySoldRaw = filters.get("minMonthlySold")?.trim() || "";
const maxMonthlySoldRaw = filters.get("maxMonthlySold")?.trim() || "";
const minSalesRankRaw = filters.get("minSalesRank")?.trim() || "";
const maxSalesRankRaw = filters.get("maxSalesRank")?.trim() || "";
const minSellerCountRaw = filters.get("minSellerCount")?.trim() || "";
const maxSellerCountRaw = filters.get("maxSellerCount")?.trim() || "";
const minRatingCountRaw = filters.get("minRatingCount")?.trim() || "";
const maxRatingCountRaw = filters.get("maxRatingCount")?.trim() || "";
const minConfidenceRaw = filters.get("minConfidence")?.trim() || "";
const maxConfidenceRaw = filters.get("maxConfidence")?.trim() || "";
const conditions = [
"inv.can_sell = 1",
"inv.sellability_status = 'available'",
];
const params: Array<string | number> = [];
if (runIdRaw) {
const runId = Number(runIdRaw);
if (Number.isInteger(runId) && runId > 0) {
conditions.push("r.id = ?");
params.push(runId);
}
}
if (sellerId) {
conditions.push("s.seller_id = ?");
params.push(sellerId);
}
if (verdict === "FBA" || verdict === "FBM" || verdict === "SKIP") {
conditions.push("analysis.verdict = ?");
params.push(verdict);
} else if (verdict === "UNANALYZED") {
conditions.push("analysis.verdict IS NULL");
}
if (amazonIsSeller === "yes") {
conditions.push("inv.amazon_is_seller = 1");
} else if (amazonIsSeller === "no") {
conditions.push("inv.amazon_is_seller = 0");
} else if (amazonIsSeller === "unknown") {
conditions.push("inv.amazon_is_seller IS NULL");
}
const numericFilters: Array<[string, string, string]> = [
[minPriceRaw, "inv.current_price >= ?", "minPrice"],
[maxPriceRaw, "inv.current_price <= ?", "maxPrice"],
[minMonthlySoldRaw, "inv.monthly_sold >= ?", "minMonthlySold"],
[maxMonthlySoldRaw, "inv.monthly_sold <= ?", "maxMonthlySold"],
[minSalesRankRaw, "inv.sales_rank >= ?", "minSalesRank"],
[maxSalesRankRaw, "inv.sales_rank <= ?", "maxSalesRank"],
[minSellerCountRaw, "inv.seller_count >= ?", "minSellerCount"],
[maxSellerCountRaw, "inv.seller_count <= ?", "maxSellerCount"],
[minRatingCountRaw, "s.rating_count >= ?", "minRatingCount"],
[maxRatingCountRaw, "s.rating_count <= ?", "maxRatingCount"],
[minConfidenceRaw, "analysis.confidence >= ?", "minConfidence"],
[maxConfidenceRaw, "analysis.confidence <= ?", "maxConfidence"],
];
for (const [raw, condition] of numericFilters) {
if (!raw) continue;
const value = Number(raw);
if (Number.isFinite(value)) {
conditions.push(condition);
params.push(value);
}
}
if (q) {
const wildcard = `%${q}%`;
conditions.push(
`(
inv.asin LIKE ?
OR inv.product_title LIKE ?
OR inv.brand LIKE ?
OR inv.category_tree LIKE ?
OR s.seller_id LIKE ?
OR s.seller_name LIKE ?
)`,
);
params.push(wildcard, wildcard, wildcard, wildcard, wildcard, wildcard);
}
return {
where: `WHERE ${conditions.join(" AND ")}`,
params,
};
}
function parseStalkerProductSort(sortParam: string | null): string {
const allowedSort = new Set([
"runId",
"started_at",
"seller_id",
"seller_name",
"rating",
"rating_count",
"asin",
"product_title",
"brand",
"current_price",
"avg_price_90d",
"sales_rank",
"monthly_sold",
"seller_count",
"amazon_is_seller",
"verdict",
"confidence",
"last_seen_at",
]);
return parseSort(sortParam, allowedSort, "monthly_sold DESC, last_seen_at DESC, asin ASC");
}
function getStalkerProducts(filters: URLSearchParams) {
const page = parseIntParam(filters.get("page"), 1);
const pageSize = Math.min(
parseIntParam(filters.get("pageSize"), DEFAULT_PAGE_SIZE),
MAX_PAGE_SIZE,
);
const offset = (page - 1) * pageSize;
const { where, params } = parseStalkerProductFilters(filters);
const orderBy = parseStalkerProductSort(filters.get("sort"));
const baseSelect = `
SELECT
r.id AS runId,
r.started_at,
s.seller_id,
s.seller_name,
s.rating,
s.rating_count,
inv.asin,
inv.can_sell,
inv.sellability_status,
inv.sellability_reason,
inv.product_title,
inv.brand,
inv.category_tree,
inv.current_price,
inv.avg_price_90d,
inv.sales_rank,
inv.monthly_sold,
inv.seller_count,
inv.amazon_is_seller,
analysis.verdict,
analysis.confidence,
analysis.reasoning,
inv.last_seen_at
FROM stalker_seller_inventory inv
JOIN stalker_runs r ON r.id = inv.run_id
JOIN stalker_sellers s ON s.seller_id = inv.seller_id
LEFT JOIN product_analysis_results analysis ON analysis.asin = inv.asin
${where}
`;
const totalRow = db
.query(`SELECT COUNT(*) AS total FROM (${baseSelect}) stalker_products`)
.get(...params) as { total: number };
const summary = db
.query(
`SELECT
COUNT(DISTINCT runId) AS runs,
COUNT(DISTINCT seller_id) AS sellers,
COUNT(DISTINCT asin) AS products
FROM (${baseSelect}) stalker_products`,
)
.get(...params) as {
runs: number;
sellers: number;
products: number;
};
const items = db
.query(
`SELECT * FROM (${baseSelect}) stalker_products
ORDER BY ${orderBy}
LIMIT ? OFFSET ?`,
)
.all(...params, pageSize, offset) as StalkerProductRecord[];
return {
items,
summary,
page,
pageSize,
total: totalRow.total,
totalPages: Math.max(1, Math.ceil(totalRow.total / pageSize)),
};
}
function getStalkerProductsForExport(filters: URLSearchParams): StalkerProductRecord[] {
const { where, params } = parseStalkerProductFilters(filters);
const orderBy = parseStalkerProductSort(filters.get("sort"));
return db
.query(
`SELECT * FROM (
SELECT
r.id AS runId,
r.started_at,
s.seller_id,
s.seller_name,
s.rating,
s.rating_count,
inv.asin,
inv.can_sell,
inv.sellability_status,
inv.sellability_reason,
inv.product_title,
inv.brand,
inv.category_tree,
inv.current_price,
inv.avg_price_90d,
inv.sales_rank,
inv.monthly_sold,
inv.seller_count,
inv.amazon_is_seller,
analysis.verdict,
analysis.confidence,
analysis.reasoning,
inv.last_seen_at
FROM stalker_seller_inventory inv
JOIN stalker_runs r ON r.id = inv.run_id
JOIN stalker_sellers s ON s.seller_id = inv.seller_id
LEFT JOIN product_analysis_results analysis ON analysis.asin = inv.asin
${where}
) stalker_products
ORDER BY ${orderBy}`,
)
.all(...params) as StalkerProductRecord[];
}
function parseCategoryTreeForExport(value: string | null): string {
if (!value) return "";
try {
const parsed = JSON.parse(value);
return Array.isArray(parsed)
? parsed.filter((item) => typeof item === "string").join(" > ")
: "";
} catch {
return "";
}
}
function exportStalkerProductsXlsx(filters: URLSearchParams): Response {
const rows = getStalkerProductsForExport(filters);
const data = rows.map((row) => ({
ASIN: row.asin,
"Amazon URL": `https://amazon.com/dp/${row.asin}`,
Product: row.product_title ?? "",
Brand: row.brand ?? "",
Category: parseCategoryTreeForExport(row.category_tree),
"Monthly Sold": row.monthly_sold ?? null,
Sellers: row.seller_count ?? null,
"Amazon Seller": row.amazon_is_seller == null ? "" : row.amazon_is_seller === 1 ? "Yes" : "No",
"Sales Rank": row.sales_rank ?? null,
"Current Price": row.current_price ?? null,
"Avg 90d": row.avg_price_90d ?? null,
Verdict: row.verdict ?? "",
Confidence: row.confidence ?? null,
Reasoning: row.reasoning ?? "",
"Seller ID": row.seller_id,
Seller: row.seller_name ?? "",
"Seller Rating": row.rating ?? null,
"Seller Rating Count": row.rating_count ?? null,
"Sellability Status": row.sellability_status,
"Sellability Reason": row.sellability_reason ?? "",
"Run ID": row.runId,
"Last Seen": row.last_seen_at,
}));
const workbook = XLSX.utils.book_new();
const worksheet = XLSX.utils.json_to_sheet(data);
worksheet["!cols"] = [
{ wch: 12 },
{ wch: 32 },
{ wch: 48 },
{ wch: 20 },
{ wch: 34 },
{ wch: 14 },
{ wch: 10 },
{ wch: 14 },
{ wch: 12 },
{ wch: 12 },
{ wch: 12 },
{ wch: 10 },
{ wch: 12 },
{ wch: 60 },
{ wch: 18 },
{ wch: 24 },
{ wch: 12 },
{ wch: 20 },
{ wch: 18 },
{ wch: 40 },
{ wch: 10 },
{ wch: 24 },
];
XLSX.utils.book_append_sheet(workbook, worksheet, "Sellable Products");
const buffer = XLSX.write(workbook, {
type: "array",
bookType: "xlsx",
}) as ArrayBuffer;
return xlsx(buffer, "stalker-sellable-products.xlsx");
}
function purgeStalkerData() {
const counts = {
inventory: (db.query("SELECT COUNT(*) AS count FROM stalker_seller_inventory").get() as { count: number }).count,
asinSellers: (db.query("SELECT COUNT(*) AS count FROM stalker_asin_sellers").get() as { count: number }).count,
sellers: (db.query("SELECT COUNT(*) AS count FROM stalker_sellers").get() as { count: number }).count,
scans: (db.query("SELECT COUNT(*) AS count FROM stalker_asin_scans").get() as { count: number }).count,
runs: (db.query("SELECT COUNT(*) AS count FROM stalker_runs").get() as { count: number }).count,
};
db.transaction(() => {
db.run("DELETE FROM stalker_seller_inventory");
db.run("DELETE FROM stalker_asin_sellers");
db.run("DELETE FROM stalker_sellers");
db.run("DELETE FROM stalker_asin_scans");
db.run("DELETE FROM stalker_runs");
})();
return { ok: true, deleted: counts };
}
function getRun(processType: ProcessType, runId: number) { function getRun(processType: ProcessType, runId: number) {
if (processType === "lead_analysis") { if (processType === "lead_analysis") {
const run = db const run = db
@@ -1259,6 +1803,8 @@ const server = Bun.serve({
routes: { routes: {
"/": index, "/": index,
"/products": index, "/products": index,
"/stalker": index,
"/stalker/products": index,
"/runs/:processType/:runId": index, "/runs/:processType/:runId": index,
"/api/runs": (req) => { "/api/runs": (req) => {
const url = new URL(req.url); const url = new URL(req.url);
@@ -1268,6 +1814,24 @@ const server = Bun.serve({
const url = new URL(req.url); const url = new URL(req.url);
return json(getProductList(url.searchParams)); return json(getProductList(url.searchParams));
}, },
"/api/stalker/results": (req) => {
const url = new URL(req.url);
return json(getStalkerResults(url.searchParams));
},
"/api/stalker/products": (req) => {
const url = new URL(req.url);
return json(getStalkerProducts(url.searchParams));
},
"/api/stalker/products/export.xlsx": (req) => {
const url = new URL(req.url);
return exportStalkerProductsXlsx(url.searchParams);
},
"/api/stalker/purge": (req) => {
if (req.method !== "DELETE" && req.method !== "POST") {
return json({ error: "Method not allowed" }, 405);
}
return json(purgeStalkerData());
},
"/api/upc/map": async (req) => { "/api/upc/map": async (req) => {
let upcs: string[]; let upcs: string[];
try { try {

View File

@@ -123,7 +123,10 @@ function round2(value: number): number {
return Math.round(value * 100) / 100; return Math.round(value * 100) / 100;
} }
const SELLABILITY_CONCURRENCY = 5; const LISTINGS_RESTRICTIONS_RATE_LIMIT_PER_SECOND = 5;
const LISTINGS_RESTRICTIONS_BURST_REQUESTS = 10;
const SELLABILITY_CONCURRENCY = LISTINGS_RESTRICTIONS_RATE_LIMIT_PER_SECOND;
const SELLABILITY_PROGRESS_INTERVAL = LISTINGS_RESTRICTIONS_BURST_REQUESTS;
const PRICING_CONCURRENCY = 5; const PRICING_CONCURRENCY = 5;
const UPC_PATTERN = /^\d{12,14}$/; const UPC_PATTERN = /^\d{12,14}$/;
@@ -621,8 +624,7 @@ export async function fetchSellabilityBatch(
} }
let completed = 0; let completed = 0;
let running = 0; const queue = [...asins];
const queue = [...asins];
async function next(): Promise<void> { async function next(): Promise<void> {
while (queue.length > 0) { while (queue.length > 0) {
@@ -630,9 +632,12 @@ export async function fetchSellabilityBatch(
const info = await fetchSellabilityInternal(spClient!, asin); const info = await fetchSellabilityInternal(spClient!, asin);
results.set(asin, info); results.set(asin, info);
completed++; completed++;
if (completed % 10 === 0 || completed === asins.length) { if (
console.log(` [sellability] ${completed}/${asins.length} checked`); completed % SELLABILITY_PROGRESS_INTERVAL === 0 ||
} completed === asins.length
) {
console.log(` [sellability] ${completed}/${asins.length} checked`);
}
} }
} }

374
src/stalker-analyze.ts Normal file
View File

@@ -0,0 +1,374 @@
import { type Database, closeDb, getDb, initDb } from "./database.ts";
import { analyzeProducts } from "./llm.ts";
import { fetchSpApiPricingAndFees } from "./sp-api.ts";
import type {
AnalysisResult,
EnrichedProduct,
KeepaData,
ProductRecord,
SellabilityInfo,
} from "./types.ts";
const LLM_BATCH_SIZE = 5;
const LLM_BATCH_DELAY_MS = 5_000;
type Args = {
dbPath: string;
stalkerRunId: number;
analysisRunId: number;
asins: string[];
};
type InventoryRow = {
asin: string;
product_title: string | null;
brand: string | null;
category_tree: string | null;
current_price: number | null;
avg_price_90d: number | null;
sales_rank: number | null;
monthly_sold: number | null;
seller_count: number | null;
amazon_is_seller: number | null;
can_sell: number | null;
sellability_status: SellabilityInfo["sellabilityStatus"] | null;
sellability_reason: string | null;
};
function readFlagValue(args: string[], flag: string): string | undefined {
const index = args.indexOf(flag);
if (index === -1) return undefined;
return args[index + 1];
}
function parseArgs(argv = process.argv.slice(2)): Args {
const dbPath = readFlagValue(argv, "--db");
const stalkerRunId = Number(readFlagValue(argv, "--stalker-run-id"));
const analysisRunId = Number(readFlagValue(argv, "--analysis-run-id"));
const asins = (readFlagValue(argv, "--asins") ?? "")
.split(",")
.map((asin) => asin.trim().toUpperCase())
.filter(Boolean);
if (!dbPath) throw new Error("Missing --db");
if (!Number.isInteger(stalkerRunId) || stalkerRunId <= 0) {
throw new Error("--stalker-run-id must be a positive integer");
}
if (!Number.isInteger(analysisRunId) || analysisRunId <= 0) {
throw new Error("--analysis-run-id must be a positive integer");
}
if (asins.length === 0) throw new Error("Missing --asins");
return { dbPath, stalkerRunId, analysisRunId, asins };
}
function wait(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
function parseCategoryTree(value: string | null): string[] {
if (!value) return [];
try {
const parsed = JSON.parse(value);
return Array.isArray(parsed)
? parsed.filter((item): item is string => typeof item === "string")
: [];
} catch {
return [];
}
}
function toProductRecord(row: InventoryRow): ProductRecord {
const categoryTree = parseCategoryTree(row.category_tree);
return {
asin: row.asin,
name: row.product_title ?? row.asin,
brand: row.brand ?? undefined,
category: categoryTree.join(" > ") || undefined,
unitCost: 0,
amazonRank: row.sales_rank ?? undefined,
sellingPriceFromSheet: row.current_price ?? undefined,
avgPrice90FromSheet: row.avg_price_90d ?? undefined,
};
}
function toKeepaData(row: InventoryRow): KeepaData {
return {
currentPrice: row.current_price,
avgPrice90: row.avg_price_90d,
minPrice90: null,
maxPrice90: null,
salesRank: row.sales_rank,
salesRankAvg90: null,
salesRankDrops30: null,
salesRankDrops90: null,
sellerCount: row.seller_count,
amazonIsSeller:
row.amazon_is_seller == null ? null : row.amazon_is_seller === 1,
amazonBuyboxSharePct90d: null,
buyBoxSeller: null,
buyBoxPrice: null,
buyBoxAvg90: null,
monthlySold: row.monthly_sold,
categoryTree: parseCategoryTree(row.category_tree),
};
}
function toSellability(row: InventoryRow): SellabilityInfo {
return {
canSell: row.can_sell == null ? null : row.can_sell === 1,
sellabilityStatus: row.sellability_status ?? "unknown",
sellabilityReason: row.sellability_reason ?? undefined,
};
}
function loadInventoryRows(
database: Database,
stalkerRunId: number,
asins: string[],
): InventoryRow[] {
const placeholders = asins.map(() => "?").join(",");
return database
.query(
`SELECT
asin, product_title, brand, category_tree, current_price, avg_price_90d,
sales_rank, monthly_sold, seller_count, amazon_is_seller, can_sell,
sellability_status, sellability_reason
FROM stalker_seller_inventory
WHERE run_id = ?
AND can_sell = 1
AND sellability_status = 'available'
AND asin IN (${placeholders})
GROUP BY asin`,
)
.all(stalkerRunId, ...asins) as InventoryRow[];
}
async function buildEnrichedProducts(
rows: InventoryRow[],
): Promise<EnrichedProduct[]> {
const enriched: EnrichedProduct[] = [];
for (const row of rows) {
const sellability = toSellability(row);
const spApi = await fetchSpApiPricingAndFees(
row.asin,
sellability,
row.current_price,
);
enriched.push({
record: toProductRecord(row),
keepa: toKeepaData(row),
spApi,
fetchedAt: new Date().toISOString(),
});
}
return enriched;
}
function insertProductAnalysisResults(
database: Database,
runId: number,
results: AnalysisResult[],
): void {
if (results.length === 0) return;
const insert = database.prepare(`
INSERT INTO product_analysis_results (
asin, run_id, name, brand, category, unit_cost,
current_price, avg_price_90d, avg_price_90d_sheet,
selling_price_sheet, sales_rank, sales_rank_avg_90d,
seller_count, amazon_is_seller, amazon_buybox_share_pct_90d,
monthly_sold, rank_drops_30d, rank_drops_90d,
fba_fee, fbm_fee, referral_percent, can_sell,
sellability_status, sellability_reason,
verdict, confidence, reasoning, fetched_at
) VALUES (
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?
)
ON CONFLICT(asin) DO UPDATE SET
run_id = excluded.run_id,
name = excluded.name,
brand = excluded.brand,
category = excluded.category,
unit_cost = excluded.unit_cost,
current_price = excluded.current_price,
avg_price_90d = excluded.avg_price_90d,
avg_price_90d_sheet = excluded.avg_price_90d_sheet,
selling_price_sheet = excluded.selling_price_sheet,
sales_rank = excluded.sales_rank,
sales_rank_avg_90d = excluded.sales_rank_avg_90d,
seller_count = excluded.seller_count,
amazon_is_seller = excluded.amazon_is_seller,
amazon_buybox_share_pct_90d = excluded.amazon_buybox_share_pct_90d,
monthly_sold = excluded.monthly_sold,
rank_drops_30d = excluded.rank_drops_30d,
rank_drops_90d = excluded.rank_drops_90d,
fba_fee = excluded.fba_fee,
fbm_fee = excluded.fbm_fee,
referral_percent = excluded.referral_percent,
can_sell = excluded.can_sell,
sellability_status = excluded.sellability_status,
sellability_reason = excluded.sellability_reason,
verdict = excluded.verdict,
confidence = excluded.confidence,
reasoning = excluded.reasoning,
fetched_at = excluded.fetched_at
`);
database.transaction((batch: AnalysisResult[]) => {
for (const result of batch) {
const keepa = result.product.keepa;
const record = result.product.record;
const spApi = result.product.spApi;
insert.run(
record.asin,
runId,
record.name,
record.brand ?? null,
record.category ?? keepa?.categoryTree.join(" > ") ?? null,
record.unitCost ?? null,
keepa?.currentPrice ?? spApi.estimatedSalePrice ?? null,
keepa?.avgPrice90 ?? null,
record.avgPrice90FromSheet ?? null,
record.sellingPriceFromSheet ?? null,
keepa?.salesRank ?? record.amazonRank ?? null,
keepa?.salesRankAvg90 ?? null,
keepa?.sellerCount ?? null,
keepa?.amazonIsSeller == null ? null : keepa.amazonIsSeller ? 1 : 0,
keepa?.amazonBuyboxSharePct90d ?? null,
keepa?.monthlySold ?? null,
keepa?.salesRankDrops30 ?? null,
keepa?.salesRankDrops90 ?? null,
spApi.fbaFee ?? null,
spApi.fbmFee ?? null,
spApi.referralFeePercent ?? null,
spApi.canSell == null ? "unknown" : spApi.canSell ? "yes" : "no",
spApi.sellabilityStatus ?? null,
spApi.sellabilityReason ?? null,
result.verdict.verdict,
result.verdict.confidence,
result.verdict.reasoning ?? null,
result.product.fetchedAt,
);
}
})(results);
}
function refreshAnalysisRun(database: Database, runId: number): void {
const stats = database
.query(
`SELECT
COUNT(*) AS total,
SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba,
SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm,
SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip
FROM product_analysis_results
WHERE run_id = ?`,
)
.get(runId) as {
total: number;
fba: number | null;
fbm: number | null;
skip: number | null;
};
database
.prepare(
`UPDATE category_analysis_runs
SET top_asins_checked = ?,
available_asins = ?,
fba_count = ?,
fbm_count = ?,
skip_count = ?
WHERE id = ?`,
)
.run(
stats.total ?? 0,
stats.total ?? 0,
stats.fba ?? 0,
stats.fbm ?? 0,
stats.skip ?? 0,
runId,
);
}
async function analyzeInBatches(
products: EnrichedProduct[],
): Promise<AnalysisResult[]> {
const results: AnalysisResult[] = [];
for (let i = 0; i < products.length; i += LLM_BATCH_SIZE) {
const batch = products.slice(i, i + LLM_BATCH_SIZE);
const batchNumber = Math.floor(i / LLM_BATCH_SIZE) + 1;
const totalBatches = Math.ceil(products.length / LLM_BATCH_SIZE);
console.log(
`Stalker analysis: LLM batch ${batchNumber}/${totalBatches} (${batch.length} product(s)).`,
);
if (i > 0) {
await wait(LLM_BATCH_DELAY_MS);
}
let verdicts;
try {
verdicts = await analyzeProducts(batch);
} catch (error) {
console.warn(
`Stalker analysis: LLM batch ${batchNumber} failed: ${
error instanceof Error ? error.message : String(error)
}`,
);
verdicts = null;
}
for (let j = 0; j < batch.length; j++) {
const product = batch[j];
if (!product) continue;
results.push({
product,
verdict: verdicts?.[j] ?? {
asin: product.record.asin,
verdict: "SKIP",
confidence: 0,
reasoning: "LLM analysis failed or returned no verdict",
},
});
}
}
return results;
}
async function main(): Promise<void> {
const args = parseArgs();
initDb(args.dbPath);
const database = getDb(args.dbPath);
try {
const rows = loadInventoryRows(database, args.stalkerRunId, args.asins);
if (rows.length === 0) {
console.log("Stalker analysis: no sellable inventory rows to analyze.");
return;
}
console.log(`Stalker analysis: analyzing ${rows.length} sellable ASIN(s).`);
const enriched = await buildEnrichedProducts(rows);
const results = await analyzeInBatches(enriched);
insertProductAnalysisResults(database, args.analysisRunId, results);
refreshAnalysisRun(database, args.analysisRunId);
} finally {
closeDb();
}
}
if (import.meta.main) {
main().catch((error) => {
console.error(error instanceof Error ? error.message : String(error));
process.exit(1);
});
}

View File

@@ -0,0 +1,207 @@
import { afterAll, beforeEach, expect, mock, test } from "bun:test";
import { mkdirSync, rmSync } from "node:fs";
import path from "node:path";
import * as XLSX from "xlsx";
import { closeDb, getDb } from "./database.ts";
const TEST_DIR = path.join(process.cwd(), "test_output", "stalker-sellability");
const originalFetch = globalThis.fetch;
const originalKeepaKey = Bun.env.KEEPA_API_KEY;
const fetchSellabilityBatchMock = mock(async (asins: string[]) => {
return new Map(
asins.map((asin) => [
asin,
asin === "B111111111"
? {
canSell: true,
sellabilityStatus: "available" as const,
sellabilityReason: "No listing restrictions reported",
}
: {
canSell: false,
sellabilityStatus: "restricted" as const,
sellabilityReason: "approval required",
},
]),
);
});
mock.module("./sp-api.ts", () => ({
fetchSellabilityBatch: fetchSellabilityBatchMock,
}));
const modulePromise = import("./stalker.ts");
beforeEach(() => {
closeDb();
rmSync(TEST_DIR, { recursive: true, force: true });
mkdirSync(TEST_DIR, { recursive: true });
globalThis.fetch = originalFetch;
Bun.env.KEEPA_API_KEY = "test-keepa-key";
fetchSellabilityBatchMock.mockClear();
});
afterAll(() => {
globalThis.fetch = originalFetch;
if (originalKeepaKey == null) {
delete Bun.env.KEEPA_API_KEY;
} else {
Bun.env.KEEPA_API_KEY = originalKeepaKey;
}
closeDb();
rmSync(TEST_DIR, { recursive: true, force: true });
});
test("sellability checks matched seller inventory, not the source ASIN", async () => {
const { runStalker } = await modulePromise;
const inputPath = path.join(TEST_DIR, "input.xlsx");
const dbPath = path.join(TEST_DIR, "stalker.sqlite");
const workbook = XLSX.utils.book_new();
XLSX.utils.book_append_sheet(
workbook,
XLSX.utils.json_to_sheet([{ asin: "B000000001" }]),
"Input",
);
XLSX.writeFile(workbook, inputPath);
globalThis.fetch = mock(async (input: string | URL | Request) => {
const rawUrl =
typeof input === "string"
? input
: input instanceof URL
? input.toString()
: input.url;
const url = new URL(rawUrl);
if (url.pathname === "/product") {
if (url.searchParams.get("asin") === "B111111111") {
return new Response(
JSON.stringify({
products: [
{
asin: "B111111111",
title: "Sellable Storefront Product",
brand: "Good Brand",
categoryTree: [{ name: "Kitchen" }, { name: "Storage" }],
monthlySold: 42,
stats: {
current: [null, null, null, 12345, null, null, null, null, null, null, null, 7],
avg: [2500],
},
csv: [[5000000, 1999, 5000100]],
},
],
tokensLeft: 10,
refillRate: 10,
}),
{ status: 200 },
);
}
return new Response(
JSON.stringify({
products: [
{
asin: "B000000001",
title: "Source Product",
offers: [{ sellerId: "AQUALIFIED", price: 1999 }],
},
],
tokensLeft: 10,
refillRate: 10,
}),
{ status: 200 },
);
}
if (url.pathname === "/seller") {
const wantsStorefront = url.searchParams.get("storefront") === "1";
return new Response(
JSON.stringify({
sellers: {
AQUALIFIED: {
sellerName: "New Seller",
currentRatingCount: 12,
asinList: wantsStorefront ? ["B111111111", "B222222222"] : [],
},
},
tokensLeft: 10,
refillRate: 10,
}),
{ status: 200 },
);
}
return new Response("not found", { status: 404 });
}) as unknown as typeof globalThis.fetch;
const stats = await runStalker({
input: inputPath,
dbPath,
maxAsins: null,
storefrontUpdateHours: 168,
offerLimit: 20,
sellerLimit: 30,
inventoryLimit: 200,
sellerCacheHours: 168,
includeStock: false,
dryRun: false,
resume: true,
maxSellerRequests: null,
sellability: true,
analyzeSellable: false,
});
expect(fetchSellabilityBatchMock.mock.calls.length).toBe(1);
expect(fetchSellabilityBatchMock.mock.calls[0]?.[0]).toEqual([
"B111111111",
"B222222222",
]);
expect(stats.inventorySellabilityCheckedAsins).toBe(2);
expect(stats.inventorySellabilityAvailableAsins).toBe(1);
expect(stats.inventorySellabilityExcludedAsins).toBe(1);
expect(stats.persistedInventoryAsins).toBe(1);
const db = getDb(dbPath);
const scan = db.query("SELECT source_asin FROM stalker_asin_scans").get() as {
source_asin: string;
};
expect(scan.source_asin).toBe("B000000001");
const inventory = db
.query(
`SELECT asin, can_sell, sellability_status, product_title, brand,
category_tree, current_price, avg_price_90d, sales_rank, monthly_sold,
seller_count
FROM stalker_seller_inventory ORDER BY asin`,
)
.all() as Array<{
asin: string;
can_sell: number | null;
sellability_status: string | null;
product_title: string | null;
brand: string | null;
category_tree: string | null;
current_price: number | null;
avg_price_90d: number | null;
sales_rank: number | null;
monthly_sold: number | null;
seller_count: number | null;
}>;
expect(inventory).toEqual([
{
asin: "B111111111",
can_sell: 1,
sellability_status: "available",
product_title: "Sellable Storefront Product",
brand: "Good Brand",
category_tree: JSON.stringify(["Kitchen", "Storage"]),
current_price: 19.99,
avg_price_90d: 25,
sales_rank: 12345,
monthly_sold: 42,
seller_count: 7,
},
]);
});

283
src/stalker.test.ts Normal file
View File

@@ -0,0 +1,283 @@
import { afterAll, beforeEach, expect, mock, test } from "bun:test";
import { mkdirSync, rmSync } from "node:fs";
import path from "node:path";
import * as XLSX from "xlsx";
import { closeDb, getDb, initDb } from "./database.ts";
import {
extractLiveOfferSellerCandidates,
isQualifyingSeller,
readAsinsFromXlsx,
runStalker,
} from "./stalker.ts";
const TEST_DIR = path.join(process.cwd(), "test_output", "stalker");
const originalFetch = globalThis.fetch;
const originalKeepaKey = Bun.env.KEEPA_API_KEY;
beforeEach(() => {
closeDb();
rmSync(TEST_DIR, { recursive: true, force: true });
mkdirSync(TEST_DIR, { recursive: true });
globalThis.fetch = originalFetch;
Bun.env.KEEPA_API_KEY = "test-keepa-key";
});
afterAll(() => {
globalThis.fetch = originalFetch;
if (originalKeepaKey == null) {
delete Bun.env.KEEPA_API_KEY;
} else {
Bun.env.KEEPA_API_KEY = originalKeepaKey;
}
closeDb();
rmSync(TEST_DIR, { recursive: true, force: true });
});
test("readAsinsFromXlsx extracts valid ASINs and deduplicates in order", () => {
const filePath = path.join(TEST_DIR, "asins.xlsx");
const workbook = XLSX.utils.book_new();
const sheet = XLSX.utils.json_to_sheet([
{ ASIN: "b000000001" },
{ ASIN: "invalid" },
{ ASIN: "B000000002" },
{ ASIN: "B000000001" },
{ ASIN: "" },
]);
XLSX.utils.book_append_sheet(workbook, sheet, "Input");
XLSX.writeFile(workbook, filePath);
expect(readAsinsFromXlsx(filePath)).toEqual(["B000000001", "B000000002"]);
});
test("isQualifyingSeller accepts rating counts from 1 to 30 only", () => {
expect(isQualifyingSeller({ ratingCount: null })).toBe(false);
expect(isQualifyingSeller({ ratingCount: 0 })).toBe(false);
expect(isQualifyingSeller({ ratingCount: 1 })).toBe(true);
expect(isQualifyingSeller({ ratingCount: 30 })).toBe(true);
expect(isQualifyingSeller({ ratingCount: 31 })).toBe(false);
});
test("extractLiveOfferSellerCandidates ignores Amazon, missing seller IDs, and duplicates", () => {
const offers = extractLiveOfferSellerCandidates({
offers: [
{ sellerId: "ATVPDKIKX0DER", price: 1999 },
{ price: 1899 },
{ sellerId: "A1SELLER", price: 1599, isFBA: true, stock: 4 },
{ sellerId: "A1SELLER", price: 1499 },
{ sellerID: "A2SELLER", currentPrice: 2499, isFba: false },
],
});
expect(offers.map((offer) => offer.sellerId)).toEqual([
"A1SELLER",
"A2SELLER",
]);
expect(offers[0]?.offerPrice).toBe(15.99);
expect(offers[0]?.isFba).toBe(true);
expect(offers[0]?.stock).toBe(4);
});
test("initDb creates stalker tables and indexes", () => {
const dbPath = path.join(TEST_DIR, "schema.sqlite");
initDb(dbPath);
const db = getDb(dbPath);
const tables = db
.query(
`SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'stalker_%' ORDER BY name`,
)
.all() as Array<{ name: string }>;
expect(tables.map((row) => row.name)).toEqual([
"stalker_asin_scans",
"stalker_asin_sellers",
"stalker_runs",
"stalker_seller_inventory",
"stalker_sellers",
]);
const indexes = db
.query(
`SELECT name FROM sqlite_master WHERE type = 'index' AND name LIKE 'idx_stalker_%' ORDER BY name`,
)
.all() as Array<{ name: string }>;
expect(indexes.length).toBeGreaterThanOrEqual(6);
});
test("runStalker fetches product offers, filters sellers, and persists storefront inventory", async () => {
const inputPath = path.join(TEST_DIR, "input.xlsx");
const dbPath = path.join(TEST_DIR, "stalker.sqlite");
const workbook = XLSX.utils.book_new();
XLSX.utils.book_append_sheet(
workbook,
XLSX.utils.json_to_sheet([{ asin: "B000000001" }]),
"Input",
);
XLSX.writeFile(workbook, inputPath);
const fetchMock = mock(async (input: string | URL | Request) => {
const rawUrl =
typeof input === "string"
? input
: input instanceof URL
? input.toString()
: input.url;
const url = new URL(rawUrl);
if (url.pathname === "/product") {
expect(url.searchParams.get("asin")).toBe("B000000001");
expect(url.searchParams.get("offers")).toBe("20");
expect(url.searchParams.get("only-live-offers")).toBe("1");
expect(url.searchParams.has("stock")).toBe(false);
return new Response(
JSON.stringify({
products: [
{
asin: "B000000001",
title: "Tracked Product",
offers: [
{
sellerId: "AQUALIFIED",
price: 1999,
condition: "New",
isFBA: true,
stock: 3,
},
{
sellerId: "AOLDSELLER",
price: 2099,
},
],
},
],
tokensLeft: 10,
refillRate: 10,
}),
{ status: 200 },
);
}
if (url.pathname === "/seller") {
const wantsStorefront = url.searchParams.get("storefront") === "1";
if (wantsStorefront) {
expect(url.searchParams.get("update")).toBe("168");
}
const sellerId = url.searchParams.get("seller");
return new Response(
JSON.stringify({
sellers: {
...(!wantsStorefront && sellerId === "AQUALIFIED,AOLDSELLER"
? {
AQUALIFIED: {
sellerName: "New Seller",
currentRating: 96,
currentRatingCount: 12,
},
AOLDSELLER: {
sellerName: "Old Seller",
currentRating: 99,
currentRatingCount: 120,
},
}
: {}),
...(wantsStorefront && sellerId === "AQUALIFIED"
? {
AQUALIFIED: {
sellerName: "New Seller",
currentRating: 96,
currentRatingCount: 12,
asinList: ["B111111111", "B222222222"],
},
}
: {}),
},
tokensLeft: 10,
refillRate: 10,
}),
{ status: 200 },
);
}
return new Response("not found", { status: 404 });
});
globalThis.fetch = fetchMock as unknown as typeof globalThis.fetch;
const stats = await runStalker({
input: inputPath,
dbPath,
maxAsins: null,
storefrontUpdateHours: 168,
offerLimit: 20,
sellerLimit: 30,
inventoryLimit: 200,
sellerCacheHours: 168,
includeStock: false,
dryRun: false,
resume: true,
maxSellerRequests: null,
sellability: false,
analyzeSellable: false,
});
expect(stats.scannedAsins).toBe(1);
expect(stats.sourceAsinsWithMatches).toBe(1);
expect(stats.matchedSellers).toBe(1);
expect(stats.persistedInventoryAsins).toBe(0);
expect(stats.failedAsins).toBe(0);
expect(stats.candidateSellers).toBe(2);
expect(stats.qualifyingSellers).toBe(1);
expect(stats.sellerMetadataRequests).toBe(1);
expect(stats.sellerStorefrontRequests).toBe(1);
const sellerCalls = fetchMock.mock.calls.filter((call) => {
const rawUrl =
typeof call[0] === "string"
? call[0]
: call[0] instanceof URL
? call[0].toString()
: (call[0] as Request).url;
return new URL(rawUrl).pathname === "/seller";
});
expect(sellerCalls.length).toBe(2);
const db = getDb(dbPath);
const run = db.query("SELECT * FROM stalker_runs").get() as any;
expect(run.status).toBe("completed");
expect(run.requested_asins).toBe(1);
expect(run.scanned_asins).toBe(1);
expect(run.source_asins_with_matches).toBe(1);
expect(run.candidate_sellers).toBe(2);
expect(run.qualifying_sellers).toBe(1);
expect(run.matched_sellers).toBe(1);
expect(run.seller_metadata_requests).toBe(1);
expect(run.seller_storefront_requests).toBe(1);
expect(run.inventory_sellability_checked_asins).toBe(0);
expect(run.inventory_sellability_available_asins).toBe(0);
expect(run.inventory_sellability_excluded_asins).toBe(0);
expect(run.persisted_inventory_asins).toBe(0);
const scan = db.query("SELECT * FROM stalker_asin_scans").get() as any;
expect(scan.source_asin).toBe("B000000001");
expect(scan.title).toBe("Tracked Product");
expect(scan.offer_count).toBe(2);
expect(scan.candidate_seller_count).toBe(2);
expect(scan.matched_seller_count).toBe(1);
const sellers = db.query("SELECT * FROM stalker_sellers").all() as any[];
expect(sellers.length).toBe(1);
expect(sellers[0].seller_id).toBe("AQUALIFIED");
expect(sellers[0].rating_count).toBe(12);
expect(sellers[0].storefront_asin_total).toBe(2);
expect(sellers[0].persisted_inventory_sample_count).toBe(0);
const asinSellers = db.query("SELECT * FROM stalker_asin_sellers").all() as any[];
expect(asinSellers.length).toBe(1);
expect(asinSellers[0].offer_price).toBe(19.99);
expect(asinSellers[0].is_fba).toBe(1);
expect(asinSellers[0].stock).toBe(3);
const inventory = db
.query("SELECT asin FROM stalker_seller_inventory ORDER BY asin")
.all() as Array<{ asin: string }>;
expect(inventory.map((row) => row.asin)).toEqual([]);
});

1596
src/stalker.ts Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -109,6 +109,76 @@ type ProductListResponse = {
totalPages: number; totalPages: number;
}; };
type StalkerResultItem = {
runId: number;
started_at: string;
status: string;
input_file: string;
seller_id: string;
seller_name: string | null;
rating: number | null;
rating_count: number | null;
storefront_asin_total: number | null;
persisted_inventory_sample_count: number | null;
discovered_from_count: number;
first_seen_at: string;
last_seen_at: string;
persisted_inventory_asin_count: number;
inventory_sample_asins: string | null;
};
type StalkerResultsResponse = {
items: StalkerResultItem[];
summary: {
runs: number;
sellers: number;
persistedInventoryAsins: number;
};
page: number;
pageSize: number;
total: number;
totalPages: number;
};
type StalkerProductItem = {
runId: number;
started_at: string;
seller_id: string;
seller_name: string | null;
rating: number | null;
rating_count: number | null;
asin: string;
can_sell: number;
sellability_status: string;
sellability_reason: string | null;
product_title: string | null;
brand: string | null;
category_tree: string | null;
current_price: number | null;
avg_price_90d: number | null;
sales_rank: number | null;
monthly_sold: number | null;
seller_count: number | null;
amazon_is_seller: number | null;
verdict: "FBA" | "FBM" | "SKIP" | null;
confidence: number | null;
reasoning: string | null;
last_seen_at: string;
};
type StalkerProductsResponse = {
items: StalkerProductItem[];
summary: {
runs: number;
sellers: number;
products: number;
};
page: number;
pageSize: number;
total: number;
totalPages: number;
};
type SortState = { type SortState = {
field: string; field: string;
direction: SortDirection; direction: SortDirection;
@@ -139,6 +209,23 @@ function formatAmazonSeller(value: number | null | undefined): string {
return value === 1 ? "Yes" : "No"; return value === 1 ? "Yes" : "No";
} }
function formatBoolean(value: number | null | undefined): string {
if (value === null || value === undefined) return "-";
return value === 1 ? "Yes" : "No";
}
function parseStringArrayJson(value: string | null | undefined): string[] {
if (!value) return [];
try {
const parsed = JSON.parse(value);
return Array.isArray(parsed)
? parsed.filter((item): item is string => typeof item === "string")
: [];
} catch {
return [];
}
}
function buildSortValue(sort: SortState): string { function buildSortValue(sort: SortState): string {
return `${sort.field}:${sort.direction}`; return `${sort.field}:${sort.direction}`;
} }
@@ -197,9 +284,13 @@ function detectAnomaly(item: ResultItem): string {
function Dashboard({ function Dashboard({
onOpenRun, onOpenRun,
onOpenProducts, onOpenProducts,
onOpenStalker,
onOpenStalkerProducts,
}: { }: {
onOpenRun: (run: Run) => void; onOpenRun: (run: Run) => void;
onOpenProducts: (verdict: VerdictFilter) => void; onOpenProducts: (verdict: VerdictFilter) => void;
onOpenStalker: () => void;
onOpenStalkerProducts: () => void;
}) { }) {
const [runs, setRuns] = useState<RunsResponse | null>(null); const [runs, setRuns] = useState<RunsResponse | null>(null);
const [loading, setLoading] = useState(false); const [loading, setLoading] = useState(false);
@@ -288,7 +379,13 @@ function Dashboard({
return ( return (
<div className="page"> <div className="page">
<div className="card"> <div className="card">
<h2>Runs Dashboard</h2> <div className="section-header">
<h2>Runs Dashboard</h2>
<div className="button-row">
<button onClick={onOpenStalker}>Stalker sellers</button>
<button onClick={onOpenStalkerProducts}>Sellable products</button>
</div>
</div>
</div> </div>
<div className="metrics"> <div className="metrics">
@@ -848,10 +945,451 @@ function ProductList({ verdict, onBack }: { verdict: VerdictFilter; onBack: () =
); );
} }
function StalkerExplorer({
onBack,
onOpenProducts,
}: {
onBack: () => void;
onOpenProducts: () => void;
}) {
const [results, setResults] = useState<StalkerResultsResponse | null>(null);
const [loading, setLoading] = useState(false);
const [purging, setPurging] = useState(false);
const [search, setSearch] = useState("");
const [sellerId, setSellerId] = useState("");
const [runId, setRunId] = useState("");
const [minRatingCount, setMinRatingCount] = useState("1");
const [maxRatingCount, setMaxRatingCount] = useState("30");
const [page, setPage] = useState(1);
const [pageSize, setPageSize] = useState(25);
const [sort, setSort] = useState<SortState>({ field: "persisted_inventory_asin_count", direction: "DESC" });
const [refreshTick, setRefreshTick] = useState(0);
useEffect(() => {
let cancelled = false;
async function load() {
setLoading(true);
const params = new URLSearchParams({
page: String(page),
pageSize: String(pageSize),
sort: buildSortValue(sort),
});
if (search) params.set("q", search);
if (sellerId) params.set("sellerId", sellerId);
if (runId) params.set("runId", runId);
if (minRatingCount) params.set("minRatingCount", minRatingCount);
if (maxRatingCount) params.set("maxRatingCount", maxRatingCount);
const res = await fetch(`/api/stalker/results?${params.toString()}`);
const payload = (await res.json()) as StalkerResultsResponse;
if (!cancelled) {
setResults(payload);
setLoading(false);
}
}
load();
return () => {
cancelled = true;
};
}, [search, sellerId, runId, minRatingCount, maxRatingCount, page, pageSize, sort, refreshTick]);
async function purgeStalkerData() {
const confirmed = window.confirm("Permanently delete all Stalker runs, sellers, and sellable products from the database?");
if (!confirmed) return;
setPurging(true);
try {
const res = await fetch("/api/stalker/purge", { method: "DELETE" });
if (!res.ok) {
const payload = await res.json().catch(() => null) as { error?: string } | null;
window.alert(payload?.error ?? "Failed to purge Stalker data");
return;
}
setPage(1);
setRefreshTick((tick) => tick + 1);
} finally {
setPurging(false);
}
}
return (
<div className="page">
<button className="back" onClick={onBack}>Back</button>
<div className="card">
<div className="section-header">
<h2>Seller Storefronts</h2>
<div className="button-row">
<button onClick={onOpenProducts}>Sellable products</button>
<button className="danger" disabled={purging} onClick={purgeStalkerData}>
{purging ? "Purging..." : "Purge Stalker data"}
</button>
</div>
</div>
</div>
<div className="metrics">
<div className="metric">
<div className="label">Runs</div>
<div className="value">{formatNumber(results?.summary.runs)}</div>
</div>
<div className="metric">
<div className="label">Matched sellers</div>
<div className="value">{formatNumber(results?.summary.sellers)}</div>
</div>
<div className="metric">
<div className="label">Sellable inventory ASINs</div>
<div className="value">{formatNumber(results?.summary.persistedInventoryAsins)}</div>
</div>
</div>
<div className="card">
<div className="toolbar">
<input value={search} onChange={(e) => { setPage(1); setSearch(e.target.value); }} placeholder="Search seller or sellable ASIN" />
<input value={sellerId} onChange={(e) => { setPage(1); setSellerId(e.target.value.toUpperCase()); }} placeholder="Seller ID" />
<input value={runId} onChange={(e) => { setPage(1); setRunId(e.target.value); }} placeholder="Run ID" />
<input value={minRatingCount} onChange={(e) => { setPage(1); setMinRatingCount(e.target.value); }} placeholder="Min rating count" />
<input value={maxRatingCount} onChange={(e) => { setPage(1); setMaxRatingCount(e.target.value); }} placeholder="Max rating count" />
<select value={String(pageSize)} onChange={(e) => { setPage(1); setPageSize(Number(e.target.value)); }}>
<option value="25">25 / page</option>
<option value="50">50 / page</option>
<option value="100">100 / page</option>
</select>
</div>
</div>
<div className="card">
<div className="table-wrap">
<table className="stalker-table">
<thead>
<tr>
<th><button onClick={() => setSort(nextSort(sort, "runId"))}>Run</button></th>
<th><button onClick={() => setSort(nextSort(sort, "seller_id"))}>Seller ID</button></th>
<th><button onClick={() => setSort(nextSort(sort, "seller_name"))}>Seller</button></th>
<th><button onClick={() => setSort(nextSort(sort, "rating"))}>Rating</button></th>
<th><button onClick={() => setSort(nextSort(sort, "rating_count"))}>Rating Count</button></th>
<th><button onClick={() => setSort(nextSort(sort, "discovered_from_count"))}>Hits</button></th>
<th><button onClick={() => setSort(nextSort(sort, "storefront_asin_total"))}>Storefront total</button></th>
<th><button onClick={() => setSort(nextSort(sort, "persisted_inventory_asin_count"))}>Sellable sample</button></th>
<th><button onClick={() => setSort(nextSort(sort, "last_seen_at"))}>Last Seen</button></th>
<th className="inventory-col">Sellable inventory ASIN sample</th>
</tr>
</thead>
<tbody>
{loading ? (
<tr><td colSpan={10}>Loading...</td></tr>
) : results?.items.length ? (
results.items.map((item) => {
const inventorySample = (item.inventory_sample_asins ?? "")
.split(",")
.filter(Boolean)
.slice(0, 20);
return (
<tr key={`${item.runId}-${item.seller_id}`}>
<td>{item.runId}</td>
<td>{item.seller_id}</td>
<td>{item.seller_name || "-"}</td>
<td>{formatNumber(item.rating)}</td>
<td>{formatNumber(item.rating_count)}</td>
<td>{formatNumber(item.discovered_from_count)}</td>
<td>{formatNumber(item.storefront_asin_total)}</td>
<td>{formatNumber(item.persisted_inventory_asin_count)}</td>
<td>{formatDate(item.last_seen_at)}</td>
<td className="inventory-col">
{inventorySample.length === 0 ? "-" : inventorySample.map((asin) => (
<a key={asin} href={`http://amazon.com/dp/${asin}`} target="_blank" rel="noreferrer">{asin}</a>
))}
</td>
</tr>
);
})
) : (
<tr><td colSpan={10}>No seller storefronts found</td></tr>
)}
</tbody>
</table>
</div>
<div className="pager">
<div>Showing {results?.items.length ?? 0} of {results?.total ?? 0}</div>
<div>
<button disabled={page <= 1} onClick={() => setPage((p) => Math.max(1, p - 1))}>Previous</button>
<span style={{ padding: "0 8px" }}>Page {results?.page ?? page} / {results?.totalPages ?? 1}</span>
<button disabled={Boolean(results && page >= results.totalPages)} onClick={() => setPage((p) => p + 1)}>Next</button>
</div>
</div>
</div>
</div>
);
}
function StalkerProductsExplorer({
onBack,
onOpenSellers,
}: {
onBack: () => void;
onOpenSellers: () => void;
}) {
const [results, setResults] = useState<StalkerProductsResponse | null>(null);
const [loading, setLoading] = useState(false);
const [search, setSearch] = useState("");
const [sellerId, setSellerId] = useState("");
const [runId, setRunId] = useState("");
const [verdict, setVerdict] = useState("");
const [amazonIsSeller, setAmazonIsSeller] = useState("");
const [minPrice, setMinPrice] = useState("");
const [maxPrice, setMaxPrice] = useState("");
const [minMonthlySold, setMinMonthlySold] = useState("");
const [maxMonthlySold, setMaxMonthlySold] = useState("");
const [minSalesRank, setMinSalesRank] = useState("");
const [maxSalesRank, setMaxSalesRank] = useState("");
const [minSellerCount, setMinSellerCount] = useState("");
const [maxSellerCount, setMaxSellerCount] = useState("");
const [minRatingCount, setMinRatingCount] = useState("");
const [maxRatingCount, setMaxRatingCount] = useState("");
const [minConfidence, setMinConfidence] = useState("");
const [maxConfidence, setMaxConfidence] = useState("");
const [page, setPage] = useState(1);
const [pageSize, setPageSize] = useState(50);
const [sort, setSort] = useState<SortState>({ field: "monthly_sold", direction: "DESC" });
function buildStalkerProductParams(includePaging: boolean): URLSearchParams {
const params = new URLSearchParams({
sort: buildSortValue(sort),
});
if (includePaging) {
params.set("page", String(page));
params.set("pageSize", String(pageSize));
}
if (search) params.set("q", search);
if (sellerId) params.set("sellerId", sellerId);
if (runId) params.set("runId", runId);
if (verdict) params.set("verdict", verdict);
if (amazonIsSeller) params.set("amazonIsSeller", amazonIsSeller);
if (minPrice) params.set("minPrice", minPrice);
if (maxPrice) params.set("maxPrice", maxPrice);
if (minMonthlySold) params.set("minMonthlySold", minMonthlySold);
if (maxMonthlySold) params.set("maxMonthlySold", maxMonthlySold);
if (minSalesRank) params.set("minSalesRank", minSalesRank);
if (maxSalesRank) params.set("maxSalesRank", maxSalesRank);
if (minSellerCount) params.set("minSellerCount", minSellerCount);
if (maxSellerCount) params.set("maxSellerCount", maxSellerCount);
if (minRatingCount) params.set("minRatingCount", minRatingCount);
if (maxRatingCount) params.set("maxRatingCount", maxRatingCount);
if (minConfidence) params.set("minConfidence", minConfidence);
if (maxConfidence) params.set("maxConfidence", maxConfidence);
return params;
}
useEffect(() => {
let cancelled = false;
async function load() {
setLoading(true);
const params = buildStalkerProductParams(true);
const res = await fetch(`/api/stalker/products?${params.toString()}`);
const payload = (await res.json()) as StalkerProductsResponse;
if (!cancelled) {
setResults(payload);
setLoading(false);
}
}
load();
return () => {
cancelled = true;
};
}, [
search,
sellerId,
runId,
verdict,
amazonIsSeller,
minPrice,
maxPrice,
minMonthlySold,
maxMonthlySold,
minSalesRank,
maxSalesRank,
minSellerCount,
maxSellerCount,
minRatingCount,
maxRatingCount,
minConfidence,
maxConfidence,
page,
pageSize,
sort,
]);
function resetFilters() {
setSearch("");
setSellerId("");
setRunId("");
setVerdict("");
setAmazonIsSeller("");
setMinPrice("");
setMaxPrice("");
setMinMonthlySold("");
setMaxMonthlySold("");
setMinSalesRank("");
setMaxSalesRank("");
setMinSellerCount("");
setMaxSellerCount("");
setMinRatingCount("");
setMaxRatingCount("");
setMinConfidence("");
setMaxConfidence("");
setPage(1);
}
const exportHref = `/api/stalker/products/export.xlsx?${buildStalkerProductParams(false).toString()}`;
return (
<div className="page">
<button className="back" onClick={onBack}>Back</button>
<div className="card">
<div className="section-header">
<h2>Sellable Stalker Products</h2>
<button onClick={onOpenSellers}>Seller storefronts</button>
</div>
</div>
<div className="metrics">
<div className="metric">
<div className="label">Runs</div>
<div className="value">{formatNumber(results?.summary.runs)}</div>
</div>
<div className="metric">
<div className="label">Sellers</div>
<div className="value">{formatNumber(results?.summary.sellers)}</div>
</div>
<div className="metric">
<div className="label">Sellable products</div>
<div className="value">{formatNumber(results?.summary.products)}</div>
</div>
</div>
<div className="card">
<div className="toolbar">
<input value={search} onChange={(e) => { setPage(1); setSearch(e.target.value); }} placeholder="Search ASIN, product, brand, category, or seller" />
<input value={sellerId} onChange={(e) => { setPage(1); setSellerId(e.target.value.toUpperCase()); }} placeholder="Seller ID" />
<input value={runId} onChange={(e) => { setPage(1); setRunId(e.target.value); }} placeholder="Run ID" />
<select value={verdict} onChange={(e) => { setPage(1); setVerdict(e.target.value); }}>
<option value="">All verdicts</option>
<option value="FBA">FBA</option>
<option value="FBM">FBM</option>
<option value="SKIP">SKIP</option>
<option value="UNANALYZED">Unanalyzed</option>
</select>
<select value={amazonIsSeller} onChange={(e) => { setPage(1); setAmazonIsSeller(e.target.value); }}>
<option value="">Amazon seller: all</option>
<option value="yes">Amazon seller: yes</option>
<option value="no">Amazon seller: no</option>
<option value="unknown">Amazon seller: unknown</option>
</select>
<input value={minPrice} onChange={(e) => { setPage(1); setMinPrice(e.target.value); }} placeholder="Min price" />
<input value={maxPrice} onChange={(e) => { setPage(1); setMaxPrice(e.target.value); }} placeholder="Max price" />
<input value={minMonthlySold} onChange={(e) => { setPage(1); setMinMonthlySold(e.target.value); }} placeholder="Min monthly sold" />
<input value={maxMonthlySold} onChange={(e) => { setPage(1); setMaxMonthlySold(e.target.value); }} placeholder="Max monthly sold" />
<input value={minSalesRank} onChange={(e) => { setPage(1); setMinSalesRank(e.target.value); }} placeholder="Min rank" />
<input value={maxSalesRank} onChange={(e) => { setPage(1); setMaxSalesRank(e.target.value); }} placeholder="Max rank" />
<input value={minSellerCount} onChange={(e) => { setPage(1); setMinSellerCount(e.target.value); }} placeholder="Min sellers" />
<input value={maxSellerCount} onChange={(e) => { setPage(1); setMaxSellerCount(e.target.value); }} placeholder="Max sellers" />
<input value={minRatingCount} onChange={(e) => { setPage(1); setMinRatingCount(e.target.value); }} placeholder="Min seller rating count" />
<input value={maxRatingCount} onChange={(e) => { setPage(1); setMaxRatingCount(e.target.value); }} placeholder="Max seller rating count" />
<input value={minConfidence} onChange={(e) => { setPage(1); setMinConfidence(e.target.value); }} placeholder="Min confidence" />
<input value={maxConfidence} onChange={(e) => { setPage(1); setMaxConfidence(e.target.value); }} placeholder="Max confidence" />
<select value={String(pageSize)} onChange={(e) => { setPage(1); setPageSize(Number(e.target.value)); }}>
<option value="25">25 / page</option>
<option value="50">50 / page</option>
<option value="100">100 / page</option>
</select>
<button onClick={resetFilters}>Reset filters</button>
<a className="button-link" href={exportHref}>Export XLSX</a>
</div>
</div>
<div className="card">
<div className="table-wrap">
<table className="stalker-table">
<thead>
<tr>
<th><button onClick={() => setSort(nextSort(sort, "asin"))}>ASIN</button></th>
<th className="product-col"><button onClick={() => setSort(nextSort(sort, "product_title"))}>Product</button></th>
<th><button onClick={() => setSort(nextSort(sort, "brand"))}>Brand</button></th>
<th>Category</th>
<th><button onClick={() => setSort(nextSort(sort, "monthly_sold"))}>Monthly Sold</button></th>
<th><button onClick={() => setSort(nextSort(sort, "seller_count"))}>Sellers</button></th>
<th><button onClick={() => setSort(nextSort(sort, "amazon_is_seller"))}>Amazon Seller</button></th>
<th><button onClick={() => setSort(nextSort(sort, "sales_rank"))}>Sales Rank</button></th>
<th><button onClick={() => setSort(nextSort(sort, "current_price"))}>Current Price</button></th>
<th><button onClick={() => setSort(nextSort(sort, "avg_price_90d"))}>Avg 90d</button></th>
<th><button onClick={() => setSort(nextSort(sort, "verdict"))}>Verdict</button></th>
<th><button onClick={() => setSort(nextSort(sort, "confidence"))}>Confidence</button></th>
<th><button onClick={() => setSort(nextSort(sort, "seller_id"))}>Seller ID</button></th>
<th><button onClick={() => setSort(nextSort(sort, "seller_name"))}>Seller</button></th>
<th><button onClick={() => setSort(nextSort(sort, "rating_count"))}>Rating Count</button></th>
<th>Status</th>
<th><button onClick={() => setSort(nextSort(sort, "runId"))}>Run</button></th>
<th><button onClick={() => setSort(nextSort(sort, "last_seen_at"))}>Last Seen</button></th>
</tr>
</thead>
<tbody>
{loading ? (
<tr><td colSpan={18}>Loading...</td></tr>
) : results?.items.length ? (
results.items.map((item) => {
const categories = parseStringArrayJson(item.category_tree);
return (
<tr key={`${item.runId}-${item.seller_id}-${item.asin}`}>
<td><a href={`http://amazon.com/dp/${item.asin}`} target="_blank" rel="noreferrer">{item.asin}</a></td>
<td className="product-col" title={item.product_title || undefined}>{item.product_title || "-"}</td>
<td>{item.brand || "-"}</td>
<td>{categories.at(-1) || "-"}</td>
<td>{formatNumber(item.monthly_sold)}</td>
<td>{formatNumber(item.seller_count)}</td>
<td>{formatAmazonSeller(item.amazon_is_seller)}</td>
<td>{formatNumber(item.sales_rank)}</td>
<td>{formatCurrency(item.current_price)}</td>
<td>{formatCurrency(item.avg_price_90d)}</td>
<td>{item.verdict ? <span className={verdictBadgeClass(item.verdict)}>{item.verdict}</span> : "-"}</td>
<td title={item.reasoning || undefined}>{formatNumber(item.confidence)}</td>
<td>{item.seller_id}</td>
<td>{item.seller_name || "-"}</td>
<td>{formatNumber(item.rating_count)}</td>
<td><span className="badge badge-ok">{item.sellability_status}</span></td>
<td>{item.runId}</td>
<td>{formatDate(item.last_seen_at)}</td>
</tr>
);
})
) : (
<tr><td colSpan={18}>No sellable Stalker products found</td></tr>
)}
</tbody>
</table>
</div>
<div className="pager">
<div>Showing {results?.items.length ?? 0} of {results?.total ?? 0}</div>
<div>
<button disabled={page <= 1} onClick={() => setPage((p) => Math.max(1, p - 1))}>Previous</button>
<span style={{ padding: "0 8px" }}>Page {results?.page ?? page} / {results?.totalPages ?? 1}</span>
<button disabled={Boolean(results && page >= results.totalPages)} onClick={() => setPage((p) => p + 1)}>Next</button>
</div>
</div>
</div>
</div>
);
}
type AppRoute = type AppRoute =
| { kind: "dashboard" } | { kind: "dashboard" }
| { kind: "run"; processType: ProcessType; runId: number } | { kind: "run"; processType: ProcessType; runId: number }
| { kind: "products"; verdict: VerdictFilter }; | { kind: "products"; verdict: VerdictFilter }
| { kind: "stalker" }
| { kind: "stalker-products" };
function parseRoute(pathname: string, search: string): AppRoute { function parseRoute(pathname: string, search: string): AppRoute {
const runMatch = pathname.match(/^\/runs\/(lead_analysis|category_analysis)\/(\d+)$/); const runMatch = pathname.match(/^\/runs\/(lead_analysis|category_analysis)\/(\d+)$/);
@@ -866,6 +1404,14 @@ function parseRoute(pathname: string, search: string): AppRoute {
return { kind: "products", verdict }; return { kind: "products", verdict };
} }
if (pathname === "/stalker") {
return { kind: "stalker" };
}
if (pathname === "/stalker/products") {
return { kind: "stalker-products" };
}
return { kind: "dashboard" }; return { kind: "dashboard" };
} }
@@ -890,6 +1436,16 @@ function App() {
setRoute({ kind: "products", verdict }); setRoute({ kind: "products", verdict });
} }
function openStalker() {
history.pushState({}, "", "/stalker");
setRoute({ kind: "stalker" });
}
function openStalkerProducts() {
history.pushState({}, "", "/stalker/products");
setRoute({ kind: "stalker-products" });
}
function backToDashboard() { function backToDashboard() {
history.pushState({}, "", "/"); history.pushState({}, "", "/");
setRoute({ kind: "dashboard" }); setRoute({ kind: "dashboard" });
@@ -903,7 +1459,22 @@ function App() {
return <ProductList verdict={route.verdict} onBack={backToDashboard} />; return <ProductList verdict={route.verdict} onBack={backToDashboard} />;
} }
return <Dashboard onOpenRun={openRun} onOpenProducts={openProducts} />; if (route.kind === "stalker") {
return <StalkerExplorer onBack={backToDashboard} onOpenProducts={openStalkerProducts} />;
}
if (route.kind === "stalker-products") {
return <StalkerProductsExplorer onBack={backToDashboard} onOpenSellers={openStalker} />;
}
return (
<Dashboard
onOpenRun={openRun}
onOpenProducts={openProducts}
onOpenStalker={openStalker}
onOpenStalkerProducts={openStalkerProducts}
/>
);
} }
const root = document.getElementById("root"); const root = document.getElementById("root");

View File

@@ -41,9 +41,24 @@ p {
gap: 10px; gap: 10px;
} }
.section-header {
display: flex;
align-items: center;
justify-content: space-between;
gap: 12px;
}
.button-row {
display: flex;
align-items: center;
gap: 8px;
flex-wrap: wrap;
}
.toolbar input, .toolbar input,
.toolbar select, .toolbar select,
button { button,
.button-link {
height: 36px; height: 36px;
border-radius: 8px; border-radius: 8px;
border: 1px solid #d8dce0; border: 1px solid #d8dce0;
@@ -52,10 +67,29 @@ button {
font-size: 14px; font-size: 14px;
} }
.button-link {
display: inline-flex;
align-items: center;
justify-content: center;
color: inherit;
text-decoration: none;
}
button { button {
cursor: pointer; cursor: pointer;
} }
button.danger {
border-color: #efb8b8;
color: #9f1c1c;
background: #fff6f6;
}
button:disabled {
cursor: not-allowed;
opacity: 0.58;
}
.table-wrap { .table-wrap {
overflow: auto; overflow: auto;
border: 1px solid #eceef0; border: 1px solid #eceef0;
@@ -91,6 +125,23 @@ td {
overflow-wrap: anywhere; overflow-wrap: anywhere;
} }
.inventory-col {
min-width: 360px;
max-width: 520px;
white-space: normal;
overflow-wrap: anywhere;
}
.inventory-col a {
display: inline-block;
margin-right: 8px;
margin-bottom: 4px;
}
.stalker-table {
min-width: 1320px;
}
th { th {
background: #fafafb; background: #fafafb;
font-weight: 600; font-weight: 600;
@@ -262,4 +313,9 @@ th button {
.spark-grid { .spark-grid {
grid-template-columns: repeat(2, minmax(0, 1fr)); grid-template-columns: repeat(2, minmax(0, 1fr));
} }
.section-header {
align-items: flex-start;
flex-direction: column;
}
} }

View File

@@ -1,5 +1,8 @@
import { getDb } from "./database.ts"; import { getDb } from "./database.ts";
import type { AnalysisResult, SupplierAnalysisResult } from "./types.ts"; import type { AnalysisResult, SupplierAnalysisResult } from "./types.ts";
import { mkdirSync } from "node:fs";
import path from "node:path";
import * as XLSX from "xlsx";
export type RunCounts = { export type RunCounts = {
totalProducts: number; totalProducts: number;
@@ -93,6 +96,22 @@ export function writeResultsToDb(
console.log(`Results written to SQLite database for run_id: ${runId}`); console.log(`Results written to SQLite database for run_id: ${runId}`);
} }
export function writeResultsWorkbook(
results: AnalysisResult[],
outputFile: string,
): void {
const outputDir = path.dirname(outputFile);
if (outputDir && outputDir !== ".") {
mkdirSync(outputDir, { recursive: true });
}
const workbook = XLSX.utils.book_new();
const worksheet = XLSX.utils.json_to_sheet(results.map(buildRow));
XLSX.utils.book_append_sheet(workbook, worksheet, "Results");
XLSX.writeFile(workbook, outputFile);
console.log(`Results workbook written: ${outputFile}`);
}
export function startRunInDb( export function startRunInDb(
dbPath: string, dbPath: string,
inputFile: string, inputFile: string,