feat: add supplier scoring and UPC file analysis functionality
- Implemented supplier scoring logic in `supplier-scoring.ts` with functions to compute demand score, competition penalty, and overall supplier product score. - Created unit tests for supplier scoring in `supplier-scoring.test.ts` to validate scoring logic against various scenarios. - Developed UPC file analysis tool in `upc-file-analysis.ts` to process UPCs in batches, fetch product data from Keepa and SP-API, and generate supplier results. - Added UPC input reading functionality in `upc-file-reader.ts` to handle XLSX and XLS files, including validation for UPC formats. - Introduced a command-line tool in `upc-lookup.ts` for looking up UPCs and displaying detailed results or mappings to ASINs. - Enhanced error handling and logging throughout the new modules for better traceability and user feedback.
This commit is contained in:
Victor Noguera
356
src/stalker/stalker-analyze.ts
Normal file
356
src/stalker/stalker-analyze.ts
Normal file
@@ -0,0 +1,356 @@
|
||||
import { db } from "../db/index.ts";
|
||||
import { categoryProductResults, runs } from "../db/schema.ts";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { analyzeProducts } from "../integrations/llm.ts";
|
||||
import { fetchSpApiPricingAndFees } from "../integrations/sp-api.ts";
|
||||
import type {
|
||||
AnalysisResult,
|
||||
EnrichedProduct,
|
||||
KeepaData,
|
||||
ProductRecord,
|
||||
SellabilityInfo,
|
||||
} from "../types.ts";
|
||||
|
||||
const LLM_BATCH_SIZE = 5;
|
||||
const LLM_BATCH_DELAY_MS = 5_000;
|
||||
|
||||
type Args = {
|
||||
stalkerRunId: number;
|
||||
analysisRunId: number;
|
||||
asins: string[];
|
||||
useClaude: boolean;
|
||||
};
|
||||
|
||||
type InventoryRow = {
|
||||
asin: string;
|
||||
productTitle: string | null;
|
||||
brand: string | null;
|
||||
categoryTree: string | null;
|
||||
currentPrice: number | null;
|
||||
avgPrice90d: number | null;
|
||||
salesRank: number | null;
|
||||
monthlySold: number | null;
|
||||
sellerCount: number | null;
|
||||
amazonIsSeller: boolean | null;
|
||||
canSell: boolean | null;
|
||||
sellabilityStatus: SellabilityInfo["sellabilityStatus"] | null;
|
||||
sellabilityReason: string | null;
|
||||
};
|
||||
|
||||
function readFlagValue(args: string[], flag: string): string | undefined {
|
||||
const index = args.indexOf(flag);
|
||||
if (index === -1) return undefined;
|
||||
return args[index + 1];
|
||||
}
|
||||
|
||||
function parseArgs(argv = process.argv.slice(2)): Args {
|
||||
const stalkerRunId = Number(readFlagValue(argv, "--stalker-run-id"));
|
||||
const analysisRunId = Number(readFlagValue(argv, "--analysis-run-id"));
|
||||
const useClaude = argv.includes("--claude");
|
||||
const asins = (readFlagValue(argv, "--asins") ?? "")
|
||||
.split(",")
|
||||
.map((asin) => asin.trim().toUpperCase())
|
||||
.filter(Boolean);
|
||||
|
||||
if (!Number.isInteger(stalkerRunId) || stalkerRunId <= 0) {
|
||||
throw new Error("--stalker-run-id must be a positive integer");
|
||||
}
|
||||
if (!Number.isInteger(analysisRunId) || analysisRunId <= 0) {
|
||||
throw new Error("--analysis-run-id must be a positive integer");
|
||||
}
|
||||
if (asins.length === 0) throw new Error("Missing --asins");
|
||||
|
||||
return { stalkerRunId, analysisRunId, asins, useClaude };
|
||||
}
|
||||
|
||||
function wait(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function parseCategoryTree(value: string | null): string[] {
|
||||
if (!value) return [];
|
||||
try {
|
||||
const parsed = JSON.parse(value);
|
||||
return Array.isArray(parsed)
|
||||
? parsed.filter((item): item is string => typeof item === "string")
|
||||
: [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function toProductRecord(row: InventoryRow): ProductRecord {
|
||||
const categoryTree = parseCategoryTree(row.categoryTree);
|
||||
return {
|
||||
asin: row.asin,
|
||||
name: row.productTitle ?? row.asin,
|
||||
brand: row.brand ?? undefined,
|
||||
category: categoryTree.join(" > ") || undefined,
|
||||
unitCost: 0,
|
||||
amazonRank: row.salesRank ?? undefined,
|
||||
sellingPriceFromSheet: row.currentPrice ?? undefined,
|
||||
avgPrice90FromSheet: row.avgPrice90d ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function toKeepaData(row: InventoryRow): KeepaData {
|
||||
return {
|
||||
currentPrice: row.currentPrice,
|
||||
avgPrice90: row.avgPrice90d,
|
||||
minPrice90: null,
|
||||
maxPrice90: null,
|
||||
salesRank: row.salesRank,
|
||||
salesRankAvg90: null,
|
||||
salesRankDrops30: null,
|
||||
salesRankDrops90: null,
|
||||
sellerCount: row.sellerCount,
|
||||
amazonIsSeller: row.amazonIsSeller,
|
||||
amazonBuyboxSharePct90d: null,
|
||||
buyBoxSeller: null,
|
||||
buyBoxPrice: null,
|
||||
buyBoxAvg90: null,
|
||||
monthlySold: row.monthlySold,
|
||||
categoryTree: parseCategoryTree(row.categoryTree),
|
||||
};
|
||||
}
|
||||
|
||||
function toSellability(row: InventoryRow): SellabilityInfo {
|
||||
return {
|
||||
canSell: row.canSell,
|
||||
sellabilityStatus: row.sellabilityStatus ?? "unknown",
|
||||
sellabilityReason: row.sellabilityReason ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
async function loadInventoryRows(
|
||||
stalkerRunId: number,
|
||||
asins: string[],
|
||||
): Promise<InventoryRow[]> {
|
||||
if (asins.length === 0) return [];
|
||||
return db.execute(
|
||||
sql<InventoryRow>`SELECT DISTINCT ON (asin)
|
||||
asin,
|
||||
product_title AS "productTitle",
|
||||
brand,
|
||||
category_tree AS "categoryTree",
|
||||
current_price AS "currentPrice",
|
||||
avg_price_90d AS "avgPrice90d",
|
||||
sales_rank AS "salesRank",
|
||||
monthly_sold AS "monthlySold",
|
||||
seller_count AS "sellerCount",
|
||||
amazon_is_seller AS "amazonIsSeller",
|
||||
can_sell AS "canSell",
|
||||
sellability_status AS "sellabilityStatus",
|
||||
sellability_reason AS "sellabilityReason"
|
||||
FROM stalker_seller_inventory
|
||||
WHERE run_id = ${stalkerRunId}
|
||||
AND can_sell = true
|
||||
AND sellability_status = 'available'
|
||||
AND asin = ANY(${asins})`,
|
||||
);
|
||||
}
|
||||
|
||||
async function buildEnrichedProducts(
|
||||
rows: InventoryRow[],
|
||||
): Promise<EnrichedProduct[]> {
|
||||
const enriched: EnrichedProduct[] = [];
|
||||
|
||||
for (const row of rows) {
|
||||
const sellability = toSellability(row);
|
||||
const spApi = await fetchSpApiPricingAndFees(
|
||||
row.asin,
|
||||
sellability,
|
||||
row.currentPrice,
|
||||
);
|
||||
|
||||
enriched.push({
|
||||
record: toProductRecord(row),
|
||||
keepa: toKeepaData(row),
|
||||
spApi,
|
||||
fetchedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
return enriched;
|
||||
}
|
||||
|
||||
async function insertProductAnalysisResults(
|
||||
runId: number,
|
||||
results: AnalysisResult[],
|
||||
): Promise<void> {
|
||||
if (results.length === 0) return;
|
||||
|
||||
const rows = results.map((result) => {
|
||||
const keepa = result.product.keepa;
|
||||
const record = result.product.record;
|
||||
const spApi = result.product.spApi;
|
||||
const canSell =
|
||||
spApi.canSell == null ? "unknown" : spApi.canSell ? "yes" : "no";
|
||||
|
||||
return {
|
||||
asin: record.asin,
|
||||
runId,
|
||||
name: record.name,
|
||||
brand: record.brand ?? null,
|
||||
category: record.category ?? keepa?.categoryTree.join(" > ") ?? null,
|
||||
unitCost: record.unitCost ?? null,
|
||||
currentPrice: keepa?.currentPrice ?? spApi.estimatedSalePrice ?? null,
|
||||
avgPrice90d: keepa?.avgPrice90 ?? null,
|
||||
avgPrice90dSheet: record.avgPrice90FromSheet ?? null,
|
||||
sellingPriceSheet: record.sellingPriceFromSheet ?? null,
|
||||
salesRank: keepa?.salesRank ?? record.amazonRank ?? null,
|
||||
salesRankAvg90d: keepa?.salesRankAvg90 ?? null,
|
||||
sellerCount: keepa?.sellerCount ?? null,
|
||||
amazonIsSeller: keepa?.amazonIsSeller ?? null,
|
||||
amazonBuyboxSharePct90d: keepa?.amazonBuyboxSharePct90d ?? null,
|
||||
monthlySold: keepa?.monthlySold ?? null,
|
||||
rankDrops30d: keepa?.salesRankDrops30 ?? null,
|
||||
rankDrops90d: keepa?.salesRankDrops90 ?? null,
|
||||
fbaFee: spApi.fbaFee ?? null,
|
||||
fbmFee: spApi.fbmFee ?? null,
|
||||
referralPercent: spApi.referralFeePercent ?? null,
|
||||
canSell,
|
||||
sellabilityStatus: spApi.sellabilityStatus ?? null,
|
||||
sellabilityReason: spApi.sellabilityReason ?? null,
|
||||
verdict: result.verdict.verdict,
|
||||
confidence: result.verdict.confidence ?? 0,
|
||||
reasoning: result.verdict.reasoning ?? null,
|
||||
fetchedAt: new Date(result.product.fetchedAt),
|
||||
};
|
||||
});
|
||||
|
||||
await db
|
||||
.insert(categoryProductResults)
|
||||
.values(rows)
|
||||
.onConflictDoUpdate({
|
||||
target: categoryProductResults.asin,
|
||||
set: {
|
||||
runId: sql`EXCLUDED.run_id`,
|
||||
name: sql`EXCLUDED.name`,
|
||||
brand: sql`EXCLUDED.brand`,
|
||||
category: sql`EXCLUDED.category`,
|
||||
unitCost: sql`EXCLUDED.unit_cost`,
|
||||
currentPrice: sql`EXCLUDED.current_price`,
|
||||
avgPrice90d: sql`EXCLUDED.avg_price_90d`,
|
||||
avgPrice90dSheet: sql`EXCLUDED.avg_price_90d_sheet`,
|
||||
sellingPriceSheet: sql`EXCLUDED.selling_price_sheet`,
|
||||
salesRank: sql`EXCLUDED.sales_rank`,
|
||||
salesRankAvg90d: sql`EXCLUDED.sales_rank_avg_90d`,
|
||||
sellerCount: sql`EXCLUDED.seller_count`,
|
||||
amazonIsSeller: sql`EXCLUDED.amazon_is_seller`,
|
||||
amazonBuyboxSharePct90d: sql`EXCLUDED.amazon_buybox_share_pct_90d`,
|
||||
monthlySold: sql`EXCLUDED.monthly_sold`,
|
||||
rankDrops30d: sql`EXCLUDED.rank_drops_30d`,
|
||||
rankDrops90d: sql`EXCLUDED.rank_drops_90d`,
|
||||
fbaFee: sql`EXCLUDED.fba_fee`,
|
||||
fbmFee: sql`EXCLUDED.fbm_fee`,
|
||||
referralPercent: sql`EXCLUDED.referral_percent`,
|
||||
canSell: sql`EXCLUDED.can_sell`,
|
||||
sellabilityStatus: sql`EXCLUDED.sellability_status`,
|
||||
sellabilityReason: sql`EXCLUDED.sellability_reason`,
|
||||
verdict: sql`EXCLUDED.verdict`,
|
||||
confidence: sql`EXCLUDED.confidence`,
|
||||
reasoning: sql`EXCLUDED.reasoning`,
|
||||
fetchedAt: sql`EXCLUDED.fetched_at`,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function refreshAnalysisRun(runId: number): Promise<void> {
|
||||
const [stats] = await db.execute(
|
||||
sql<{
|
||||
total: string;
|
||||
fba: string | null;
|
||||
fbm: string | null;
|
||||
skip: string | null;
|
||||
}>`SELECT
|
||||
COUNT(*) AS total,
|
||||
SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba,
|
||||
SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm,
|
||||
SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip
|
||||
FROM category_product_results
|
||||
WHERE run_id = ${runId}`,
|
||||
);
|
||||
|
||||
await db
|
||||
.update(runs)
|
||||
.set({
|
||||
topAsinsChecked: Number(stats?.total ?? 0),
|
||||
availableAsins: Number(stats?.total ?? 0),
|
||||
fbaCount: Number(stats?.fba ?? 0),
|
||||
fbmCount: Number(stats?.fbm ?? 0),
|
||||
skipCount: Number(stats?.skip ?? 0),
|
||||
})
|
||||
.where(eq(runs.id, runId));
|
||||
}
|
||||
|
||||
async function analyzeInBatches(
|
||||
products: EnrichedProduct[],
|
||||
useClaude: boolean,
|
||||
): Promise<AnalysisResult[]> {
|
||||
const results: AnalysisResult[] = [];
|
||||
|
||||
for (let i = 0; i < products.length; i += LLM_BATCH_SIZE) {
|
||||
const batch = products.slice(i, i + LLM_BATCH_SIZE);
|
||||
const batchNumber = Math.floor(i / LLM_BATCH_SIZE) + 1;
|
||||
const totalBatches = Math.ceil(products.length / LLM_BATCH_SIZE);
|
||||
console.log(
|
||||
`Stalker analysis: LLM batch ${batchNumber}/${totalBatches} (${batch.length} product(s)).`,
|
||||
);
|
||||
|
||||
if (i > 0) {
|
||||
await wait(LLM_BATCH_DELAY_MS);
|
||||
}
|
||||
|
||||
let verdicts;
|
||||
try {
|
||||
verdicts = await analyzeProducts(batch, { useClaude });
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
`Stalker analysis: LLM batch ${batchNumber} failed: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`,
|
||||
);
|
||||
verdicts = null;
|
||||
}
|
||||
|
||||
for (let j = 0; j < batch.length; j++) {
|
||||
const product = batch[j];
|
||||
if (!product) continue;
|
||||
results.push({
|
||||
product,
|
||||
verdict: verdicts?.[j] ?? {
|
||||
asin: product.record.asin,
|
||||
verdict: "SKIP",
|
||||
confidence: 0,
|
||||
reasoning: "LLM analysis failed or returned no verdict",
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = parseArgs();
|
||||
|
||||
const rows = await loadInventoryRows(args.stalkerRunId, args.asins);
|
||||
if (rows.length === 0) {
|
||||
console.log("Stalker analysis: no sellable inventory rows to analyze.");
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`Stalker analysis: analyzing ${rows.length} sellable ASIN(s).`);
|
||||
const enriched = await buildEnrichedProducts(rows);
|
||||
const results = await analyzeInBatches(enriched, args.useClaude);
|
||||
await insertProductAnalysisResults(args.analysisRunId, results);
|
||||
await refreshAnalysisRun(args.analysisRunId);
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user