feat: add supplier scoring and UPC file analysis functionality
- Implemented supplier scoring logic in `supplier-scoring.ts` with functions to compute demand score, competition penalty, and overall supplier product score. - Created unit tests for supplier scoring in `supplier-scoring.test.ts` to validate scoring logic against various scenarios. - Developed UPC file analysis tool in `upc-file-analysis.ts` to process UPCs in batches, fetch product data from Keepa and SP-API, and generate supplier results. - Added UPC input reading functionality in `upc-file-reader.ts` to handle XLSX and XLS files, including validation for UPC formats. - Introduced a command-line tool in `upc-lookup.ts` for looking up UPCs and displaying detailed results or mappings to ASINs. - Enhanced error handling and logging throughout the new modules for better traceability and user feedback.
This commit is contained in:
Victor Noguera
554
src/supplier/upc-file-analysis.ts
Normal file
554
src/supplier/upc-file-analysis.ts
Normal file
@@ -0,0 +1,554 @@
|
||||
import path from "node:path";
|
||||
import { fetchKeepaDataBatch, lookupKeepaUpcs } from "../integrations/keepa.ts";
|
||||
import {
|
||||
fetchSellabilityBatch,
|
||||
fetchSpApiPricingAndFees,
|
||||
lookupSpApiUpcs,
|
||||
} from "../integrations/sp-api.ts";
|
||||
import {
|
||||
processUpcFileInBatches,
|
||||
type UpcInputRow,
|
||||
} from "./upc-file-reader.ts";
|
||||
import {
|
||||
appendSupplierResultsToRun,
|
||||
refreshRunCountsInDb,
|
||||
startRunInDb,
|
||||
type RunCounts,
|
||||
} from "../writer.ts";
|
||||
import { connectCache, disconnectCache } from "../integrations/cache.ts";
|
||||
import { scoreSupplierProduct, resolveSupplierSalePrice } from "./supplier-scoring.ts";
|
||||
import {
|
||||
writeSupplierWorkbook,
|
||||
type SupplierExportSummary,
|
||||
} from "./supplier-export.ts";
|
||||
import type {
|
||||
KeepaUpcLookupDetail,
|
||||
KeepaUpcLookupStatus,
|
||||
ProductRecord,
|
||||
SupplierAnalysisResult,
|
||||
SupplierScore,
|
||||
UpcLookupDetail,
|
||||
} from "../types.ts";
|
||||
|
||||
const DEFAULT_INPUT_BATCH_SIZE = 200;
|
||||
const DEFAULT_UPC_LOOKUP_BATCH_SIZE = 100;
|
||||
const DEFAULT_PRICING_CONCURRENCY = 5;
|
||||
|
||||
export type UpcFileAnalysisOptions = {
|
||||
inputFile: string;
|
||||
outputFile?: string;
|
||||
inputBatchSize?: number;
|
||||
upcLookupBatchSize?: number;
|
||||
maxRows?: number;
|
||||
manageResources?: boolean;
|
||||
dbPath?: string;
|
||||
};
|
||||
|
||||
export type UpcFileAnalysisSummary = {
|
||||
runId: number;
|
||||
inputFile: string;
|
||||
outputFile?: string;
|
||||
processedRows: number;
|
||||
matchedRows: number;
|
||||
unresolvedByStatus: Record<KeepaUpcLookupStatus, number>;
|
||||
runCounts: RunCounts;
|
||||
reader: {
|
||||
mode: "xlsx_stream" | "xlsx_fallback" | "xls_fallback";
|
||||
totalRowsSeen: number;
|
||||
emittedRows: number;
|
||||
skippedMissingUpc: number;
|
||||
skippedInvalidUpc: number;
|
||||
};
|
||||
};
|
||||
|
||||
function printUsage(): void {
|
||||
console.log("Usage:");
|
||||
console.log(
|
||||
" bun run src/upc-file-analysis.ts --input input/<file.xls|file.xlsx> [--out output/results.xlsx] [--input-batch-size 200] [--upc-lookup-batch-size 100] [--max-rows 1000]",
|
||||
);
|
||||
}
|
||||
|
||||
function parsePositiveInt(value: string | undefined, flagName: string): number {
|
||||
const parsed = Number.parseInt(String(value), 10);
|
||||
if (!Number.isFinite(parsed) || parsed < 1) {
|
||||
throw new Error(`Invalid value for ${flagName}: ${value}`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): UpcFileAnalysisOptions {
|
||||
let inputFile: string | undefined;
|
||||
let outputFile: string | undefined;
|
||||
let inputBatchSize: number | undefined;
|
||||
let upcLookupBatchSize: number | undefined;
|
||||
let maxRows: number | undefined;
|
||||
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const arg = argv[i]!;
|
||||
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
printUsage();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (arg === "--input") {
|
||||
const next = argv[i + 1];
|
||||
if (!next) throw new Error("Missing value after --input");
|
||||
inputFile = next;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--out") {
|
||||
const next = argv[i + 1];
|
||||
if (!next) throw new Error("Missing value after --out");
|
||||
outputFile = next;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--input-batch-size") {
|
||||
inputBatchSize = parsePositiveInt(argv[i + 1], "--input-batch-size");
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--upc-lookup-batch-size") {
|
||||
upcLookupBatchSize = parsePositiveInt(
|
||||
argv[i + 1],
|
||||
"--upc-lookup-batch-size",
|
||||
);
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--max-rows") {
|
||||
maxRows = parsePositiveInt(argv[i + 1], "--max-rows");
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg.startsWith("--")) {
|
||||
throw new Error(`Unknown flag: ${arg}`);
|
||||
}
|
||||
|
||||
if (!inputFile) {
|
||||
inputFile = arg;
|
||||
continue;
|
||||
}
|
||||
|
||||
throw new Error(`Unexpected positional argument: ${arg}`);
|
||||
}
|
||||
|
||||
if (!inputFile) {
|
||||
throw new Error("Missing --input <file.xls|file.xlsx>");
|
||||
}
|
||||
|
||||
return {
|
||||
inputFile,
|
||||
outputFile,
|
||||
inputBatchSize,
|
||||
upcLookupBatchSize,
|
||||
maxRows,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveDefaultOutputPath(inputFile: string): string {
|
||||
const parsedInput = path.parse(inputFile);
|
||||
return path.join("output", `${parsedInput.name}_upc_results.xlsx`);
|
||||
}
|
||||
|
||||
function createStatusCounter(): Record<KeepaUpcLookupStatus, number> {
|
||||
return {
|
||||
found: 0,
|
||||
invalid_upc: 0,
|
||||
not_found: 0,
|
||||
multiple_asins: 0,
|
||||
request_failed: 0,
|
||||
};
|
||||
}
|
||||
|
||||
function chunkArray<T>(items: T[], chunkSize: number): T[][] {
|
||||
const chunks: T[][] = [];
|
||||
for (let i = 0; i < items.length; i += chunkSize) {
|
||||
chunks.push(items.slice(i, i + chunkSize));
|
||||
}
|
||||
return chunks;
|
||||
}
|
||||
|
||||
function skippedScore(reason: string): SupplierScore {
|
||||
return {
|
||||
salePrice: null,
|
||||
fbaFee: null,
|
||||
profit: null,
|
||||
margin: null,
|
||||
roi: null,
|
||||
demandScore: 0,
|
||||
competitionPenalty: 1,
|
||||
score: 0,
|
||||
verdict: "SKIP",
|
||||
reason,
|
||||
};
|
||||
}
|
||||
|
||||
async function lookupUpcsWithChunking(
|
||||
rows: UpcInputRow[],
|
||||
lookupBatchSize: number,
|
||||
runCache: Map<string, KeepaUpcLookupDetail>,
|
||||
): Promise<Map<string, UpcLookupDetail>> {
|
||||
const uniqueUpcs = Array.from(new Set(rows.map((row) => row.upc)));
|
||||
const missingUpcs = uniqueUpcs.filter((upc) => !runCache.has(upc));
|
||||
const chunks = chunkArray(missingUpcs, lookupBatchSize);
|
||||
const details = new Map<string, UpcLookupDetail>();
|
||||
|
||||
const cacheHits = uniqueUpcs.length - missingUpcs.length;
|
||||
if (cacheHits > 0) {
|
||||
console.log(
|
||||
` Reusing cached UPC lookup results for ${cacheHits}/${uniqueUpcs.length} UPCs in this batch.`,
|
||||
);
|
||||
}
|
||||
|
||||
if (missingUpcs.length === 0) {
|
||||
for (const upc of uniqueUpcs) {
|
||||
const detail = runCache.get(upc);
|
||||
if (detail) details.set(upc, detail);
|
||||
}
|
||||
return details;
|
||||
}
|
||||
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
const chunk = chunks[i]!;
|
||||
console.log(
|
||||
` SP-API UPC lookup chunk ${i + 1}/${chunks.length} (${chunk.length} UPCs)...`,
|
||||
);
|
||||
|
||||
const spDetails = await lookupSpApiUpcs(chunk);
|
||||
const fallbackUpcs = Array.from(spDetails.values())
|
||||
.filter(
|
||||
(detail) =>
|
||||
detail.status === "not_found" || detail.status === "request_failed",
|
||||
)
|
||||
.map((detail) => detail.normalizedUpc);
|
||||
const fallbackDetails =
|
||||
fallbackUpcs.length > 0 ? await lookupKeepaUpcs(fallbackUpcs) : new Map();
|
||||
|
||||
const chunkDetails = new Map<string, UpcLookupDetail>();
|
||||
for (const upc of chunk) {
|
||||
const spDetail = spDetails.get(upc);
|
||||
const fallbackDetail = fallbackDetails.get(upc);
|
||||
chunkDetails.set(
|
||||
upc,
|
||||
fallbackDetail && fallbackDetail.status !== "request_failed"
|
||||
? fallbackDetail
|
||||
: spDetail!,
|
||||
);
|
||||
}
|
||||
|
||||
for (const [upc, detail] of chunkDetails.entries()) {
|
||||
runCache.set(upc, detail);
|
||||
}
|
||||
}
|
||||
|
||||
for (const upc of uniqueUpcs) {
|
||||
const detail = runCache.get(upc);
|
||||
if (detail) {
|
||||
details.set(upc, detail);
|
||||
}
|
||||
}
|
||||
|
||||
return details;
|
||||
}
|
||||
|
||||
function toProductRecord(
|
||||
row: UpcInputRow,
|
||||
detail: UpcLookupDetail,
|
||||
): ProductRecord {
|
||||
const keepaCategory = detail.keepaData?.categoryTree?.[0];
|
||||
|
||||
return {
|
||||
asin: detail.asin ?? row.upc,
|
||||
name: row.name ?? detail.asin ?? row.upc,
|
||||
unitCost: row.unitCost ?? 0,
|
||||
brand: row.brand,
|
||||
category: row.category ?? keepaCategory,
|
||||
};
|
||||
}
|
||||
|
||||
async function fetchFeesForProducts(
|
||||
products: ProductRecord[],
|
||||
keepaResults: Map<string, NonNullable<SupplierAnalysisResult["keepa"]>>,
|
||||
sellabilityMap: Awaited<ReturnType<typeof fetchSellabilityBatch>>,
|
||||
): Promise<Map<string, NonNullable<SupplierAnalysisResult["spApi"]>>> {
|
||||
const spApiResults = new Map<string, NonNullable<SupplierAnalysisResult["spApi"]>>();
|
||||
const queue = [...products];
|
||||
let completed = 0;
|
||||
|
||||
async function next(): Promise<void> {
|
||||
while (queue.length > 0) {
|
||||
const product = queue.shift();
|
||||
if (!product) return;
|
||||
const sellability =
|
||||
sellabilityMap.get(product.asin) ?? {
|
||||
canSell: null,
|
||||
sellabilityStatus: "unknown" as const,
|
||||
sellabilityReason: "Sellability check returned no result",
|
||||
};
|
||||
const price = resolveSupplierSalePrice(
|
||||
keepaResults.get(product.asin) ?? null,
|
||||
null,
|
||||
);
|
||||
const spApi = await fetchSpApiPricingAndFees(product.asin, sellability, price);
|
||||
spApiResults.set(product.asin, spApi);
|
||||
completed++;
|
||||
if (completed % 10 === 0 || completed === products.length) {
|
||||
console.log(` [fees] ${completed}/${products.length} fetched`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const workers = Array.from(
|
||||
{ length: Math.min(DEFAULT_PRICING_CONCURRENCY, products.length || 1) },
|
||||
() => next(),
|
||||
);
|
||||
await Promise.all(workers);
|
||||
return spApiResults;
|
||||
}
|
||||
|
||||
function summarizeSupplierResults(
|
||||
results: SupplierAnalysisResult[],
|
||||
unresolvedByStatus: Record<KeepaUpcLookupStatus, number>,
|
||||
): SupplierExportSummary {
|
||||
return {
|
||||
processedRows: results.length,
|
||||
resolvedRows: results.filter((result) => result.lookup.status === "found").length,
|
||||
eligibleRows: results.filter(
|
||||
(result) => result.spApi?.sellabilityStatus === "available",
|
||||
).length,
|
||||
verdictCounts: {
|
||||
BUY: results.filter((result) => result.score.verdict === "BUY").length,
|
||||
WATCH: results.filter((result) => result.score.verdict === "WATCH").length,
|
||||
SKIP: results.filter((result) => result.score.verdict === "SKIP").length,
|
||||
},
|
||||
unresolvedByStatus,
|
||||
};
|
||||
}
|
||||
|
||||
export async function runUpcFileAnalysis(
|
||||
options: UpcFileAnalysisOptions,
|
||||
): Promise<UpcFileAnalysisSummary> {
|
||||
const inputBatchSize = Math.max(
|
||||
1,
|
||||
options.inputBatchSize ?? DEFAULT_INPUT_BATCH_SIZE,
|
||||
);
|
||||
const lookupBatchSize = Math.max(
|
||||
1,
|
||||
options.upcLookupBatchSize ?? DEFAULT_UPC_LOOKUP_BATCH_SIZE,
|
||||
);
|
||||
const outputFile =
|
||||
options.outputFile ?? resolveDefaultOutputPath(options.inputFile);
|
||||
const manageResources = options.manageResources ?? true;
|
||||
|
||||
if (manageResources) {
|
||||
console.log("Connecting to Redis...");
|
||||
await connectCache();
|
||||
}
|
||||
|
||||
const unresolvedByStatus = createStatusCounter();
|
||||
const allResults: SupplierAnalysisResult[] = [];
|
||||
const upcLookupCache = new Map<string, KeepaUpcLookupDetail>();
|
||||
let processedRows = 0;
|
||||
let matchedRows = 0;
|
||||
|
||||
const runId = await startRunInDb(options.inputFile, outputFile);
|
||||
|
||||
try {
|
||||
const readerSummary = await processUpcFileInBatches(
|
||||
options.inputFile,
|
||||
async ({ batchNumber, rows }) => {
|
||||
console.log(
|
||||
`\n=== UPC input batch ${batchNumber} (${rows.length} rows) ===`,
|
||||
);
|
||||
|
||||
processedRows += rows.length;
|
||||
const detailMap = await lookupUpcsWithChunking(
|
||||
rows,
|
||||
lookupBatchSize,
|
||||
upcLookupCache,
|
||||
);
|
||||
|
||||
const matchedEntries: Array<{
|
||||
row: UpcInputRow;
|
||||
detail: UpcLookupDetail;
|
||||
product: ProductRecord;
|
||||
}> = [];
|
||||
for (const row of rows) {
|
||||
const detail = detailMap.get(row.upc);
|
||||
if (!detail) {
|
||||
unresolvedByStatus.request_failed += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
unresolvedByStatus[detail.status] += 1;
|
||||
|
||||
if (detail.status === "found" && detail.asin) {
|
||||
matchedRows += 1;
|
||||
matchedEntries.push({
|
||||
row,
|
||||
detail,
|
||||
product: toProductRecord(row, detail),
|
||||
});
|
||||
}
|
||||
}
|
||||
const matchedProducts = matchedEntries.map((entry) => entry.product);
|
||||
|
||||
console.log(
|
||||
`Batch ${batchNumber}: ${matchedProducts.length}/${rows.length} rows resolved to single ASINs`,
|
||||
);
|
||||
|
||||
const batchResults: SupplierAnalysisResult[] = [];
|
||||
for (const row of rows) {
|
||||
const detail = detailMap.get(row.upc);
|
||||
if (!detail || detail.status === "found") continue;
|
||||
|
||||
batchResults.push({
|
||||
upc: row.upc,
|
||||
rowNumber: row.rowNumber,
|
||||
record: {
|
||||
asin: detail?.asin ?? row.upc,
|
||||
name: row.name ?? row.upc,
|
||||
unitCost: row.unitCost ?? 0,
|
||||
brand: row.brand,
|
||||
category: row.category,
|
||||
},
|
||||
lookup:
|
||||
detail ??
|
||||
({
|
||||
requestedUpc: row.upc,
|
||||
normalizedUpc: row.upc,
|
||||
status: "request_failed",
|
||||
asin: null,
|
||||
candidateAsins: [],
|
||||
keepaData: null,
|
||||
reason: "UPC lookup returned no result",
|
||||
} satisfies UpcLookupDetail),
|
||||
keepa: null,
|
||||
spApi: null,
|
||||
score: skippedScore(detail?.reason ?? "UPC unresolved"),
|
||||
fetchedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
if (matchedProducts.length > 0) {
|
||||
console.log(`Fetching ${matchedProducts.length} ASINs from Keepa...`);
|
||||
const keepaResults = await fetchKeepaDataBatch(
|
||||
matchedProducts.map((product) => product.asin),
|
||||
);
|
||||
|
||||
console.log(`Checking sellability for ${matchedProducts.length} ASINs...`);
|
||||
const sellabilityMap = await fetchSellabilityBatch(
|
||||
matchedProducts.map((product) => product.asin),
|
||||
);
|
||||
|
||||
console.log(`Fetching fees for ${matchedProducts.length} ASINs...`);
|
||||
const spApiResults = await fetchFeesForProducts(
|
||||
matchedProducts,
|
||||
keepaResults,
|
||||
sellabilityMap,
|
||||
);
|
||||
|
||||
for (const entry of matchedEntries) {
|
||||
const keepa =
|
||||
keepaResults.get(entry.product.asin) ??
|
||||
entry.detail.keepaData ??
|
||||
null;
|
||||
const spApi = spApiResults.get(entry.product.asin) ?? null;
|
||||
batchResults.push({
|
||||
upc: entry.detail.normalizedUpc,
|
||||
rowNumber: entry.row.rowNumber,
|
||||
record: entry.product,
|
||||
lookup: entry.detail,
|
||||
keepa,
|
||||
spApi,
|
||||
score: scoreSupplierProduct(entry.product, keepa, spApi),
|
||||
fetchedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
await appendSupplierResultsToRun(runId, batchResults);
|
||||
allResults.push(...batchResults);
|
||||
},
|
||||
{
|
||||
batchSize: inputBatchSize,
|
||||
maxRows: options.maxRows,
|
||||
},
|
||||
);
|
||||
|
||||
const runCounts = await refreshRunCountsInDb(runId);
|
||||
|
||||
const exportSummary = summarizeSupplierResults(allResults, unresolvedByStatus);
|
||||
await writeSupplierWorkbook(outputFile, allResults, exportSummary);
|
||||
|
||||
if (allResults.length > 0) {
|
||||
const ranked = allResults
|
||||
.filter((result) => result.score.verdict !== "SKIP")
|
||||
.sort((a, b) => b.score.score - a.score.score)
|
||||
.slice(0, 25)
|
||||
.map((result) => ({
|
||||
UPC: result.upc,
|
||||
ASIN: result.lookup.asin ?? "",
|
||||
Name: result.record.name.slice(0, 40),
|
||||
Cost: result.record.unitCost,
|
||||
Price: result.score.salePrice ?? "",
|
||||
Profit: result.score.profit ?? "",
|
||||
ROI: result.score.roi == null ? "" : `${Math.round(result.score.roi * 100)}%`,
|
||||
Score: result.score.score,
|
||||
Verdict: result.score.verdict,
|
||||
Reason: result.score.reason,
|
||||
}));
|
||||
console.log("\n=== Top Supplier Leads ===\n");
|
||||
console.table(ranked);
|
||||
} else {
|
||||
console.log("No supplier rows were analyzed.");
|
||||
}
|
||||
|
||||
console.log(`Ranked workbook written: ${outputFile}`);
|
||||
|
||||
return {
|
||||
runId,
|
||||
inputFile: options.inputFile,
|
||||
outputFile,
|
||||
processedRows,
|
||||
matchedRows,
|
||||
unresolvedByStatus,
|
||||
runCounts,
|
||||
reader: {
|
||||
mode: readerSummary.mode,
|
||||
totalRowsSeen: readerSummary.totalRowsSeen,
|
||||
emittedRows: readerSummary.emittedRows,
|
||||
skippedMissingUpc: readerSummary.skippedMissingUpc,
|
||||
skippedInvalidUpc: readerSummary.skippedInvalidUpc,
|
||||
},
|
||||
};
|
||||
} finally {
|
||||
if (manageResources) {
|
||||
await disconnectCache();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const parsed = parseArgs(process.argv.slice(2));
|
||||
const summary = await runUpcFileAnalysis(parsed);
|
||||
|
||||
console.log("\n=== UPC file analysis summary ===");
|
||||
console.log(JSON.stringify(summary, null, 2));
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
main().catch((err) => {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.error(`UPC file analysis failed: ${message}`);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user