495 lines
16 KiB
TypeScript
495 lines
16 KiB
TypeScript
import { Database } from "bun:sqlite";
|
|
import { dirname } from "node:path";
|
|
import { mkdirSync } from "node:fs";
|
|
export { Database } from "bun:sqlite";
|
|
|
|
let db: Database | null = null;
|
|
|
|
export function getDb(dbPath: string): Database {
|
|
if (!db) {
|
|
const dbDir = dirname(dbPath);
|
|
if (dbDir && dbDir !== ".") {
|
|
mkdirSync(dbDir, { recursive: true });
|
|
}
|
|
db = new Database(dbPath);
|
|
db.run("PRAGMA journal_mode = WAL;"); // Enable WAL mode for better performance
|
|
db.run("PRAGMA foreign_keys = ON;"); // Enforce foreign key constraints
|
|
}
|
|
return db;
|
|
}
|
|
|
|
export function closeDb(): void {
|
|
if (db) {
|
|
db.close();
|
|
db = null;
|
|
}
|
|
}
|
|
|
|
function createProductAnalysisResultsTable(database: Database): void {
|
|
database.run(`
|
|
CREATE TABLE IF NOT EXISTS product_analysis_results (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
asin TEXT NOT NULL,
|
|
run_id INTEGER NOT NULL,
|
|
name TEXT NOT NULL,
|
|
brand TEXT,
|
|
category TEXT,
|
|
unit_cost REAL,
|
|
current_price REAL,
|
|
avg_price_90d REAL,
|
|
avg_price_90d_sheet REAL,
|
|
selling_price_sheet REAL,
|
|
sales_rank INTEGER,
|
|
sales_rank_avg_90d INTEGER,
|
|
seller_count INTEGER,
|
|
amazon_is_seller INTEGER,
|
|
amazon_buybox_share_pct_90d REAL,
|
|
monthly_sold INTEGER,
|
|
rank_drops_30d INTEGER,
|
|
rank_drops_90d INTEGER,
|
|
fba_fee REAL,
|
|
fbm_fee REAL,
|
|
referral_percent REAL,
|
|
can_sell TEXT,
|
|
sellability_status TEXT,
|
|
sellability_reason TEXT,
|
|
verdict TEXT NOT NULL,
|
|
confidence REAL NOT NULL,
|
|
reasoning TEXT,
|
|
fetched_at TEXT NOT NULL,
|
|
UNIQUE(asin),
|
|
FOREIGN KEY (run_id) REFERENCES category_analysis_runs(id)
|
|
);
|
|
`);
|
|
}
|
|
|
|
function ensureProductAnalysisResultsTable(database: Database): void {
|
|
const tableInfo = database
|
|
.query("PRAGMA table_info(product_analysis_results)")
|
|
.all() as Array<{ name: string; pk: number }>;
|
|
|
|
if (tableInfo.length === 0) {
|
|
createProductAnalysisResultsTable(database);
|
|
return;
|
|
}
|
|
|
|
const hasIdColumn = tableInfo.some((col) => col.name === "id");
|
|
const hasAsinPrimaryKey = tableInfo.some(
|
|
(col) => col.name === "asin" && col.pk === 1,
|
|
);
|
|
|
|
const indexList = database
|
|
.query("PRAGMA index_list(product_analysis_results)")
|
|
.all() as Array<{ name: string; unique: number }>;
|
|
const hasUniqueAsinConstraint = indexList.some((idx) => {
|
|
if (idx.unique !== 1) return false;
|
|
const columns = database
|
|
.query(`PRAGMA index_info(${JSON.stringify(idx.name)})`)
|
|
.all() as Array<{ name: string }>;
|
|
return columns.length === 1 && columns[0]?.name === "asin";
|
|
});
|
|
|
|
if (!hasIdColumn || hasAsinPrimaryKey || !hasUniqueAsinConstraint) {
|
|
database.run(
|
|
"ALTER TABLE product_analysis_results RENAME TO product_analysis_results_legacy",
|
|
);
|
|
createProductAnalysisResultsTable(database);
|
|
database.run(`
|
|
WITH ranked AS (
|
|
SELECT
|
|
asin, run_id, name, brand, category, unit_cost,
|
|
current_price, avg_price_90d, avg_price_90d_sheet,
|
|
selling_price_sheet, sales_rank, sales_rank_avg_90d,
|
|
seller_count, NULL AS amazon_is_seller,
|
|
NULL AS amazon_buybox_share_pct_90d,
|
|
monthly_sold, rank_drops_30d, rank_drops_90d,
|
|
fba_fee, fbm_fee, referral_percent, can_sell,
|
|
sellability_status, sellability_reason,
|
|
verdict, confidence, reasoning, fetched_at,
|
|
ROW_NUMBER() OVER (
|
|
PARTITION BY asin
|
|
ORDER BY datetime(fetched_at) DESC, run_id DESC, id DESC
|
|
) AS row_num
|
|
FROM product_analysis_results_legacy
|
|
)
|
|
INSERT INTO product_analysis_results (
|
|
asin, run_id, name, brand, category, unit_cost,
|
|
current_price, avg_price_90d, avg_price_90d_sheet,
|
|
selling_price_sheet, sales_rank, sales_rank_avg_90d,
|
|
seller_count, amazon_is_seller, amazon_buybox_share_pct_90d,
|
|
monthly_sold, rank_drops_30d, rank_drops_90d,
|
|
fba_fee, fbm_fee, referral_percent, can_sell,
|
|
sellability_status, sellability_reason,
|
|
verdict, confidence, reasoning, fetched_at
|
|
)
|
|
SELECT
|
|
asin, run_id, name, brand, category, unit_cost,
|
|
current_price, avg_price_90d, avg_price_90d_sheet,
|
|
selling_price_sheet, sales_rank, sales_rank_avg_90d,
|
|
seller_count, amazon_is_seller, amazon_buybox_share_pct_90d,
|
|
monthly_sold, rank_drops_30d, rank_drops_90d,
|
|
fba_fee, fbm_fee, referral_percent, can_sell,
|
|
sellability_status, sellability_reason,
|
|
verdict, confidence, reasoning, fetched_at
|
|
FROM ranked
|
|
WHERE row_num = 1
|
|
`);
|
|
database.run("DROP TABLE product_analysis_results_legacy");
|
|
}
|
|
}
|
|
|
|
function ensureProductAnalysisResultsColumns(database: Database): void {
|
|
const tableInfo = database
|
|
.query("PRAGMA table_info(product_analysis_results)")
|
|
.all() as Array<{ name: string }>;
|
|
|
|
if (tableInfo.length === 0) {
|
|
return;
|
|
}
|
|
|
|
const existingColumns = new Set(tableInfo.map((col) => col.name));
|
|
const requiredColumns: Array<{ name: string; type: string }> = [
|
|
{ name: "amazon_is_seller", type: "INTEGER" },
|
|
{ name: "amazon_buybox_share_pct_90d", type: "REAL" },
|
|
];
|
|
|
|
for (const column of requiredColumns) {
|
|
if (!existingColumns.has(column.name)) {
|
|
database.run(
|
|
`ALTER TABLE product_analysis_results ADD COLUMN ${column.name} ${column.type}`,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
function ensureResultsTableColumns(database: Database): void {
|
|
const tableInfo = database
|
|
.query("PRAGMA table_info(results)")
|
|
.all() as Array<{ name: string }>;
|
|
|
|
if (tableInfo.length === 0) {
|
|
return;
|
|
}
|
|
|
|
const existingColumns = new Set(tableInfo.map((col) => col.name));
|
|
const requiredColumns: Array<{ name: string; type: string }> = [
|
|
{ name: "fba_net_sheet", type: "REAL" },
|
|
{ name: "gross_profit_dollar", type: "REAL" },
|
|
{ name: "gross_profit_pct", type: "REAL" },
|
|
{ name: "net_profit_sheet", type: "REAL" },
|
|
{ name: "roi_sheet", type: "REAL" },
|
|
{ name: "moq", type: "INTEGER" },
|
|
{ name: "moq_cost", type: "REAL" },
|
|
{ name: "qty_available", type: "INTEGER" },
|
|
{ name: "supplier", type: "TEXT" },
|
|
{ name: "source_url", type: "TEXT" },
|
|
{ name: "asin_link", type: "TEXT" },
|
|
{ name: "promo_coupon_code", type: "TEXT" },
|
|
{ name: "notes", type: "TEXT" },
|
|
{ name: "lead_date", type: "TEXT" },
|
|
{ name: "amazon_is_seller", type: "INTEGER" },
|
|
{ name: "amazon_buybox_share_pct_90d", type: "REAL" },
|
|
{ name: "upc", type: "TEXT" },
|
|
{ name: "supplier_score", type: "REAL" },
|
|
{ name: "supplier_profit", type: "REAL" },
|
|
{ name: "supplier_margin", type: "REAL" },
|
|
{ name: "supplier_roi", type: "REAL" },
|
|
{ name: "supplier_reason", type: "TEXT" },
|
|
{ name: "upc_lookup_status", type: "TEXT" },
|
|
{ name: "upc_lookup_reason", type: "TEXT" },
|
|
{ name: "candidate_asins", type: "TEXT" },
|
|
];
|
|
|
|
for (const column of requiredColumns) {
|
|
if (!existingColumns.has(column.name)) {
|
|
database.run(
|
|
`ALTER TABLE results ADD COLUMN ${column.name} ${column.type}`,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
export function initDb(dbPath: string): void {
|
|
const database = getDb(dbPath);
|
|
database.run(`
|
|
CREATE TABLE IF NOT EXISTS runs (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
timestamp TEXT NOT NULL,
|
|
input_file TEXT NOT NULL,
|
|
output_file TEXT,
|
|
total_products INTEGER,
|
|
fba_count INTEGER,
|
|
fbm_count INTEGER,
|
|
skip_count INTEGER
|
|
);
|
|
`);
|
|
database.run(`
|
|
CREATE TABLE IF NOT EXISTS results (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
run_id INTEGER NOT NULL,
|
|
asin TEXT NOT NULL,
|
|
product_name TEXT,
|
|
brand TEXT,
|
|
category TEXT,
|
|
unit_cost REAL,
|
|
current_price REAL,
|
|
avg_price_90d REAL,
|
|
avg_price_90d_sheet REAL,
|
|
selling_price_sheet REAL,
|
|
sales_rank INTEGER,
|
|
rank_avg_90d INTEGER,
|
|
sellers INTEGER,
|
|
amazon_is_seller INTEGER,
|
|
amazon_buybox_share_pct_90d REAL,
|
|
monthly_sold INTEGER,
|
|
rank_drops_30d INTEGER,
|
|
rank_drops_90d INTEGER,
|
|
fba_net_sheet REAL,
|
|
gross_profit_dollar REAL,
|
|
gross_profit_pct REAL,
|
|
net_profit_sheet REAL,
|
|
roi_sheet REAL,
|
|
moq INTEGER,
|
|
moq_cost REAL,
|
|
qty_available INTEGER,
|
|
supplier TEXT,
|
|
source_url TEXT,
|
|
asin_link TEXT,
|
|
promo_coupon_code TEXT,
|
|
notes TEXT,
|
|
lead_date TEXT,
|
|
upc TEXT,
|
|
fba_fee REAL,
|
|
fbm_fee REAL,
|
|
referral_percent REAL,
|
|
supplier_score REAL,
|
|
supplier_profit REAL,
|
|
supplier_margin REAL,
|
|
supplier_roi REAL,
|
|
supplier_reason TEXT,
|
|
upc_lookup_status TEXT,
|
|
upc_lookup_reason TEXT,
|
|
candidate_asins TEXT,
|
|
can_sell TEXT,
|
|
sellability_status TEXT,
|
|
sellability_reason TEXT,
|
|
verdict TEXT NOT NULL,
|
|
confidence INTEGER,
|
|
reasoning TEXT,
|
|
fetched_at TEXT NOT NULL,
|
|
FOREIGN KEY (run_id) REFERENCES runs(id)
|
|
);
|
|
`);
|
|
ensureResultsTableColumns(database);
|
|
database.run(`
|
|
CREATE TABLE IF NOT EXISTS category_analysis_runs (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
category_id INTEGER NOT NULL,
|
|
category_label TEXT NOT NULL,
|
|
run_timestamp TEXT NOT NULL,
|
|
top_asins_checked INTEGER NOT NULL,
|
|
available_asins INTEGER NOT NULL,
|
|
fba_count INTEGER NOT NULL,
|
|
fbm_count INTEGER NOT NULL,
|
|
skip_count INTEGER NOT NULL,
|
|
status TEXT NOT NULL,
|
|
error_message TEXT
|
|
);
|
|
`);
|
|
ensureProductAnalysisResultsTable(database);
|
|
ensureProductAnalysisResultsColumns(database);
|
|
|
|
database.run(
|
|
`CREATE INDEX IF NOT EXISTS idx_runs_timestamp ON runs(timestamp DESC);`,
|
|
);
|
|
database.run(
|
|
`CREATE INDEX IF NOT EXISTS idx_results_run_id ON results(run_id);`,
|
|
);
|
|
database.run(
|
|
`CREATE INDEX IF NOT EXISTS idx_results_verdict ON results(verdict);`,
|
|
);
|
|
database.run(
|
|
`CREATE INDEX IF NOT EXISTS idx_results_sellability_status ON results(sellability_status);`,
|
|
);
|
|
database.run(
|
|
`CREATE INDEX IF NOT EXISTS idx_results_fetched_at ON results(fetched_at DESC);`,
|
|
);
|
|
database.run(`CREATE INDEX IF NOT EXISTS idx_results_asin ON results(asin);`);
|
|
database.run(
|
|
`CREATE INDEX IF NOT EXISTS idx_category_runs_timestamp ON category_analysis_runs(run_timestamp DESC);`,
|
|
);
|
|
database.run(
|
|
`CREATE INDEX IF NOT EXISTS idx_category_runs_status ON category_analysis_runs(status);`,
|
|
);
|
|
database.run(
|
|
`CREATE INDEX IF NOT EXISTS idx_product_results_run_id ON product_analysis_results(run_id);`,
|
|
);
|
|
database.run(
|
|
`CREATE INDEX IF NOT EXISTS idx_product_results_verdict ON product_analysis_results(verdict);`,
|
|
);
|
|
database.run(
|
|
`CREATE INDEX IF NOT EXISTS idx_product_results_sellability_status ON product_analysis_results(sellability_status);`,
|
|
);
|
|
database.run(
|
|
`CREATE INDEX IF NOT EXISTS idx_product_results_fetched_at ON product_analysis_results(fetched_at DESC);`,
|
|
);
|
|
initStalkerDb(database);
|
|
}
|
|
|
|
export function initStalkerDb(database: Database): void {
|
|
resetLegacyStalkerSchema(database);
|
|
|
|
database.run(`
|
|
CREATE TABLE IF NOT EXISTS stalker_runs (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
input_file TEXT NOT NULL,
|
|
started_at TEXT NOT NULL,
|
|
completed_at TEXT,
|
|
requested_asins INTEGER NOT NULL DEFAULT 0,
|
|
skipped_asins INTEGER NOT NULL DEFAULT 0,
|
|
scanned_asins INTEGER NOT NULL DEFAULT 0,
|
|
source_asins_with_matches INTEGER NOT NULL DEFAULT 0,
|
|
candidate_sellers INTEGER NOT NULL DEFAULT 0,
|
|
qualifying_sellers INTEGER NOT NULL DEFAULT 0,
|
|
matched_sellers INTEGER NOT NULL DEFAULT 0,
|
|
seller_metadata_requests INTEGER NOT NULL DEFAULT 0,
|
|
seller_storefront_requests INTEGER NOT NULL DEFAULT 0,
|
|
inventory_sellability_checked_asins INTEGER NOT NULL DEFAULT 0,
|
|
inventory_sellability_available_asins INTEGER NOT NULL DEFAULT 0,
|
|
inventory_sellability_excluded_asins INTEGER NOT NULL DEFAULT 0,
|
|
persisted_inventory_asins INTEGER NOT NULL DEFAULT 0,
|
|
status TEXT NOT NULL,
|
|
error_message TEXT
|
|
);
|
|
`);
|
|
|
|
database.run(`
|
|
CREATE TABLE IF NOT EXISTS stalker_asin_scans (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
run_id INTEGER NOT NULL,
|
|
source_asin TEXT NOT NULL,
|
|
title TEXT,
|
|
offer_count INTEGER NOT NULL DEFAULT 0,
|
|
candidate_seller_count INTEGER NOT NULL DEFAULT 0,
|
|
matched_seller_count INTEGER NOT NULL DEFAULT 0,
|
|
fetched_at TEXT NOT NULL,
|
|
raw_product_json TEXT,
|
|
UNIQUE(run_id, source_asin),
|
|
FOREIGN KEY (run_id) REFERENCES stalker_runs(id) ON DELETE CASCADE
|
|
);
|
|
`);
|
|
|
|
database.run(`
|
|
CREATE TABLE IF NOT EXISTS stalker_sellers (
|
|
seller_id TEXT PRIMARY KEY,
|
|
seller_name TEXT,
|
|
rating REAL,
|
|
rating_count INTEGER,
|
|
storefront_asin_total INTEGER,
|
|
persisted_inventory_sample_count INTEGER,
|
|
last_updated_at TEXT NOT NULL,
|
|
raw_seller_json TEXT
|
|
);
|
|
`);
|
|
|
|
database.run(`
|
|
CREATE TABLE IF NOT EXISTS stalker_asin_sellers (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
scan_id INTEGER NOT NULL,
|
|
seller_id TEXT NOT NULL,
|
|
offer_price REAL,
|
|
condition TEXT,
|
|
is_fba INTEGER,
|
|
stock INTEGER,
|
|
seller_rating REAL,
|
|
seller_rating_count INTEGER,
|
|
raw_offer_json TEXT,
|
|
UNIQUE(scan_id, seller_id),
|
|
FOREIGN KEY (scan_id) REFERENCES stalker_asin_scans(id) ON DELETE CASCADE,
|
|
FOREIGN KEY (seller_id) REFERENCES stalker_sellers(seller_id)
|
|
);
|
|
`);
|
|
|
|
database.run(`
|
|
CREATE TABLE IF NOT EXISTS stalker_seller_inventory (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
run_id INTEGER NOT NULL,
|
|
seller_id TEXT NOT NULL,
|
|
asin TEXT NOT NULL,
|
|
can_sell INTEGER,
|
|
sellability_status TEXT,
|
|
sellability_reason TEXT,
|
|
product_title TEXT,
|
|
brand TEXT,
|
|
category_tree TEXT,
|
|
current_price REAL,
|
|
avg_price_90d REAL,
|
|
sales_rank INTEGER,
|
|
monthly_sold INTEGER,
|
|
seller_count INTEGER,
|
|
amazon_is_seller INTEGER,
|
|
raw_product_json TEXT,
|
|
last_seen_at TEXT NOT NULL,
|
|
raw_inventory_json TEXT,
|
|
UNIQUE(run_id, seller_id, asin),
|
|
FOREIGN KEY (run_id) REFERENCES stalker_runs(id) ON DELETE CASCADE,
|
|
FOREIGN KEY (seller_id) REFERENCES stalker_sellers(seller_id)
|
|
);
|
|
`);
|
|
|
|
database.run(
|
|
`CREATE INDEX IF NOT EXISTS idx_stalker_runs_started_at ON stalker_runs(started_at DESC);`,
|
|
);
|
|
database.run(
|
|
`CREATE INDEX IF NOT EXISTS idx_stalker_scans_run_id ON stalker_asin_scans(run_id);`,
|
|
);
|
|
database.run(
|
|
`CREATE INDEX IF NOT EXISTS idx_stalker_scans_source_asin ON stalker_asin_scans(source_asin);`,
|
|
);
|
|
database.run(
|
|
`CREATE INDEX IF NOT EXISTS idx_stalker_asin_sellers_seller_id ON stalker_asin_sellers(seller_id);`,
|
|
);
|
|
database.run(
|
|
`CREATE INDEX IF NOT EXISTS idx_stalker_inventory_seller_id ON stalker_seller_inventory(seller_id);`,
|
|
);
|
|
database.run(
|
|
`CREATE INDEX IF NOT EXISTS idx_stalker_inventory_asin ON stalker_seller_inventory(asin);`,
|
|
);
|
|
database.run(
|
|
`CREATE INDEX IF NOT EXISTS idx_stalker_inventory_product_title ON stalker_seller_inventory(product_title);`,
|
|
);
|
|
}
|
|
|
|
function resetLegacyStalkerSchema(database: Database): void {
|
|
const runColumns = database
|
|
.query("PRAGMA table_info(stalker_runs)")
|
|
.all() as Array<{ name: string }>;
|
|
if (runColumns.length === 0) return;
|
|
|
|
const columnNames = new Set(runColumns.map((column) => column.name));
|
|
if (
|
|
columnNames.has("scanned_asins") &&
|
|
columnNames.has("inventory_sellability_checked_asins") &&
|
|
inventoryColumnsHaveSellability(database)
|
|
) {
|
|
return;
|
|
}
|
|
|
|
database.run("DROP TABLE IF EXISTS stalker_seller_inventory");
|
|
database.run("DROP TABLE IF EXISTS stalker_asin_sellers");
|
|
database.run("DROP TABLE IF EXISTS stalker_sellers");
|
|
database.run("DROP TABLE IF EXISTS stalker_asin_scans");
|
|
database.run("DROP TABLE IF EXISTS stalker_runs");
|
|
}
|
|
|
|
function inventoryColumnsHaveSellability(database: Database): boolean {
|
|
const inventoryColumns = database
|
|
.query("PRAGMA table_info(stalker_seller_inventory)")
|
|
.all() as Array<{ name: string }>;
|
|
const columnNames = new Set(inventoryColumns.map((column) => column.name));
|
|
return (
|
|
columnNames.has("sellability_status") &&
|
|
columnNames.has("product_title")
|
|
);
|
|
}
|