Compare commits

...

2 Commits

10 changed files with 78 additions and 29 deletions

View File

@@ -10,7 +10,16 @@
"KillShell", "KillShell",
"Bash(bunx *)", "Bash(bunx *)",
"Bash(git *)", "Bash(git *)",
"Bash(ls *)" "Bash(ls *)",
"Bash(bun run build:web 2>&1 || true)",
"Bash(bun run build:web 2>&1 || true)",
"Bash(bun run build:web 2>&1 || true)",
"Bash(bun run build:web 2>&1 || true)",
"Bash(bun run build:web 2>&1 || true)",
"Bash(bun run build:web 2>&1 || true)"
] ]
} },
"additionalDirectories": [
"/Users/nvictor/.abacusai/tmp/codellm-prompt-djc6Bc"
]
} }

View File

@@ -510,7 +510,7 @@ async function discoverCategories(
maxCategories: number, maxCategories: number,
): Promise<CategoryInfo[]> { ): Promise<CategoryInfo[]> {
const data = await keepaGetJson( const data = await keepaGetJson(
`/category?key=${encodeURIComponent(config.keepaApiKey)}&domain=${DOMAIN_US}&category=0`, `/category?key=${encodeURIComponent(config.keepaApiKey)}&domain=${DOMAIN_US}&category=0&parents=0`,
); );
const categories = normalizeCategoryList(data); const categories = normalizeCategoryList(data);

View File

@@ -845,7 +845,7 @@ async function discoverCategories(
maxCategories: number, maxCategories: number,
): Promise<CategoryInfo[]> { ): Promise<CategoryInfo[]> {
const data = await keepaGetJson( const data = await keepaGetJson(
`/category?key=${encodeURIComponent(config.keepaApiKey)}&domain=${DOMAIN_US}&category=0`, `/category?key=${encodeURIComponent(config.keepaApiKey)}&domain=${DOMAIN_US}&category=0&parents=0`,
); );
const categories = normalizeCategoryList(data); const categories = normalizeCategoryList(data);

View File

@@ -542,7 +542,7 @@ async function discoverCategories(
maxCategories: number, maxCategories: number,
): Promise<CategoryInfo[]> { ): Promise<CategoryInfo[]> {
const data = await keepaGetJson( const data = await keepaGetJson(
`/category?key=${encodeURIComponent(config.keepaApiKey)}&domain=${DOMAIN_US}&category=0`, `/category?key=${encodeURIComponent(config.keepaApiKey)}&domain=${DOMAIN_US}&category=0&parents=0`,
); );
const categories = normalizeCategoryList(data); const categories = normalizeCategoryList(data);

View File

@@ -1,5 +1,5 @@
import { afterAll, beforeEach, expect, mock, test } from "bun:test"; import { afterAll, beforeEach, expect, mock, test } from "bun:test";
import { lookupKeepaUpcs, mapUpcsToAsins } from "./keepa.ts"; import { fetchKeepaDataBatch, lookupKeepaUpcs, mapUpcsToAsins } from "./keepa.ts";
const originalFetch = globalThis.fetch; const originalFetch = globalThis.fetch;
@@ -215,6 +215,7 @@ test("lookupKeepaUpcs uses lightweight query params for code mapping", async ()
expect(url.searchParams.has("stats")).toBe(false); expect(url.searchParams.has("stats")).toBe(false);
expect(url.searchParams.has("buybox")).toBe(false); expect(url.searchParams.has("buybox")).toBe(false);
expect(url.searchParams.has("days")).toBe(false); expect(url.searchParams.has("days")).toBe(false);
expect(url.searchParams.get("history")).toBe("0");
return new Response( return new Response(
JSON.stringify({ JSON.stringify({
@@ -240,3 +241,49 @@ test("lookupKeepaUpcs uses lightweight query params for code mapping", async ()
expect(details.get(targetUpc)?.status).toBe("found"); expect(details.get(targetUpc)?.status).toBe("found");
expect(details.get(targetUpc)?.asin).toBe("B000LGT001"); expect(details.get(targetUpc)?.asin).toBe("B000LGT001");
}); });
test("fetchKeepaDataBatch uses token-efficient params", async () => {
const targetAsin = "B000EFF001";
const fetchMock = mock(async (input: string | URL | Request) => {
const rawUrl =
typeof input === "string"
? input
: input instanceof URL
? input.toString()
: input.url;
const url = new URL(rawUrl);
expect(url.searchParams.get("asin")).toBe(targetAsin);
expect(url.searchParams.get("stats")).toBe("90");
expect(url.searchParams.get("days")).toBe("90");
expect(url.searchParams.get("history")).toBe("0");
expect(url.searchParams.has("buybox")).toBe(false);
return new Response(
JSON.stringify({
products: [
{
asin: targetAsin,
stats: {
current: [1999, null, null, 1234, null, null, null, null, null, null, null, 8],
avg: [2099, null, null, 1300],
min: [1799],
max: [2299],
},
csv: [[1, 1999]],
},
],
tokensLeft: 9,
refillRate: 21,
}),
{ status: 200 },
);
});
globalThis.fetch = fetchMock as unknown as typeof globalThis.fetch;
const details = await fetchKeepaDataBatch([targetAsin]);
expect(fetchMock.mock.calls.length).toBe(1);
expect(details.get(targetAsin)?.currentPrice).toBe(19.99);
});

View File

@@ -57,11 +57,13 @@ function buildProductUrl(
options?: { options?: {
includeStats?: boolean; includeStats?: boolean;
includeBuybox?: boolean; includeBuybox?: boolean;
includeHistory?: boolean;
days?: number; days?: number;
}, },
): string { ): string {
const includeStats = options?.includeStats ?? true; const includeStats = options?.includeStats ?? true;
const includeBuybox = options?.includeBuybox ?? true; const includeBuybox = options?.includeBuybox ?? true;
const includeHistory = options?.includeHistory ?? true;
const days = options?.days ?? 90; const days = options?.days ?? 90;
const params = new URLSearchParams({ const params = new URLSearchParams({
@@ -78,6 +80,10 @@ function buildProductUrl(
params.set("buybox", "1"); params.set("buybox", "1");
} }
if (!includeHistory) {
params.set("history", "0");
}
params.set(queryParam, values.join(",")); params.set(queryParam, values.join(","));
return `${KEEPA_BASE}/product?${params.toString()}`; return `${KEEPA_BASE}/product?${params.toString()}`;
} }
@@ -242,7 +248,8 @@ export async function fetchKeepaDataBatch(
const chunk = canonicalAsins.slice(i, i + MAX_ASINS_PER_REQUEST); const chunk = canonicalAsins.slice(i, i + MAX_ASINS_PER_REQUEST);
const url = buildProductUrl("asin", chunk, { const url = buildProductUrl("asin", chunk, {
includeStats: true, includeStats: true,
includeBuybox: true, includeBuybox: false,
includeHistory: false,
days: 90, days: 90,
}); });
@@ -302,6 +309,7 @@ export async function lookupKeepaUpcs(
const url = buildProductUrl("code", chunk, { const url = buildProductUrl("code", chunk, {
includeStats: false, includeStats: false,
includeBuybox: false, includeBuybox: false,
includeHistory: false,
}); });
console.log( console.log(

View File

@@ -143,7 +143,7 @@ async function loadInventoryRows(
WHERE inventory.run_id = ${stalkerRunId} WHERE inventory.run_id = ${stalkerRunId}
AND observation.can_sell = true AND observation.can_sell = true
AND observation.sellability_status = 'available' AND observation.sellability_status = 'available'
AND inventory.product_asin = ANY(${asins}) AND inventory.product_asin = ANY(ARRAY[${sql.join(asins.map((asin) => sql`${asin}`), sql`, `)}])
ORDER BY inventory.product_asin, observation.fetched_at DESC`, ORDER BY inventory.product_asin, observation.fetched_at DESC`,
); );
} }

View File

@@ -194,7 +194,6 @@ test("sellability checks matched seller inventory, not the source ASIN", async (
{ {
input: inputPath, input: inputPath,
maxAsins: null, maxAsins: null,
storefrontUpdateHours: 168,
offerLimit: 20, offerLimit: 20,
sellerLimit: 30, sellerLimit: 30,
inventoryLimit: 200, inventoryLimit: 200,

View File

@@ -198,7 +198,7 @@ test("runStalker fetches product offers, filters sellers, and tracks stats", asy
if (url.pathname === "/seller") { if (url.pathname === "/seller") {
const wantsStorefront = url.searchParams.get("storefront") === "1"; const wantsStorefront = url.searchParams.get("storefront") === "1";
if (wantsStorefront) { if (wantsStorefront) {
expect(url.searchParams.get("update")).toBe("168"); expect(url.searchParams.has("update")).toBeFalse();
} }
const sellerId = url.searchParams.get("seller"); const sellerId = url.searchParams.get("seller");
@@ -244,7 +244,6 @@ test("runStalker fetches product offers, filters sellers, and tracks stats", asy
const stats = await runStalker({ const stats = await runStalker({
input: inputPath, input: inputPath,
maxAsins: null, maxAsins: null,
storefrontUpdateHours: 168,
offerLimit: 20, offerLimit: 20,
sellerLimit: 30, sellerLimit: 30,
inventoryLimit: 200, inventoryLimit: 200,

View File

@@ -20,7 +20,6 @@ import type { SellabilityInfo } from "../types.ts";
const KEEPA_BASE = "https://api.keepa.com"; const KEEPA_BASE = "https://api.keepa.com";
const DOMAIN_US = "1"; const DOMAIN_US = "1";
const AMAZON_US_SELLER_ID = "ATVPDKIKX0DER"; const AMAZON_US_SELLER_ID = "ATVPDKIKX0DER";
const DEFAULT_STOREFRONT_UPDATE_HOURS = 168;
const DEFAULT_OFFER_LIMIT = 100; const DEFAULT_OFFER_LIMIT = 100;
const DEFAULT_SELLER_LIMIT = 30; const DEFAULT_SELLER_LIMIT = 30;
const DEFAULT_INVENTORY_LIMIT = 200; const DEFAULT_INVENTORY_LIMIT = 200;
@@ -41,7 +40,6 @@ export type StalkerArgs = {
input: string; input: string;
dbPath?: string; dbPath?: string;
maxAsins: number | null; maxAsins: number | null;
storefrontUpdateHours: number;
offerLimit: number; offerLimit: number;
sellerLimit: number; sellerLimit: number;
inventoryLimit: number; inventoryLimit: number;
@@ -142,7 +140,6 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs {
} }
const maxAsinsRaw = readFlagValue(argv, "--max-asins"); const maxAsinsRaw = readFlagValue(argv, "--max-asins");
const storefrontUpdateRaw = readFlagValue(argv, "--storefront-update-hours");
const offerLimitRaw = readFlagValue(argv, "--offer-limit"); const offerLimitRaw = readFlagValue(argv, "--offer-limit");
const sellerLimitRaw = readFlagValue(argv, "--seller-limit"); const sellerLimitRaw = readFlagValue(argv, "--seller-limit");
const inventoryLimitRaw = readFlagValue(argv, "--inventory-limit"); const inventoryLimitRaw = readFlagValue(argv, "--inventory-limit");
@@ -150,9 +147,6 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs {
const maxSellerRequestsRaw = readFlagValue(argv, "--max-seller-requests"); const maxSellerRequestsRaw = readFlagValue(argv, "--max-seller-requests");
const maxAsins = maxAsinsRaw ? Number(maxAsinsRaw) : null; const maxAsins = maxAsinsRaw ? Number(maxAsinsRaw) : null;
const storefrontUpdateHours = storefrontUpdateRaw
? Number(storefrontUpdateRaw)
: DEFAULT_STOREFRONT_UPDATE_HOURS;
const offerLimit = offerLimitRaw const offerLimit = offerLimitRaw
? Number(offerLimitRaw) ? Number(offerLimitRaw)
: DEFAULT_OFFER_LIMIT; : DEFAULT_OFFER_LIMIT;
@@ -183,12 +177,6 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs {
printUsageAndExit("--max-asins must be a positive integer."); printUsageAndExit("--max-asins must be a positive integer.");
} }
if (!Number.isInteger(storefrontUpdateHours) || storefrontUpdateHours < 0) {
printUsageAndExit(
"--storefront-update-hours must be a non-negative integer.",
);
}
if (!Number.isInteger(offerLimit) || offerLimit < 20 || offerLimit > 100) { if (!Number.isInteger(offerLimit) || offerLimit < 20 || offerLimit > 100) {
printUsageAndExit("--offer-limit must be an integer from 20 to 100."); printUsageAndExit("--offer-limit must be an integer from 20 to 100.");
} }
@@ -215,7 +203,6 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs {
return { return {
input, input,
maxAsins, maxAsins,
storefrontUpdateHours,
offerLimit, offerLimit,
sellerLimit, sellerLimit,
inventoryLimit, inventoryLimit,
@@ -662,7 +649,6 @@ async function fetchKeepaInventoryProductDetails(
asin: chunk.join(","), asin: chunk.join(","),
stats: "30", stats: "30",
days: "30", days: "30",
buybox: "1",
}); });
const data = await fetchKeepaWithRetries( const data = await fetchKeepaWithRetries(
@@ -746,13 +732,15 @@ async function fetchQualifiedSellerStorefronts(
for (const sellerId of uniqueSellerIds) { for (const sellerId of uniqueSellerIds) {
const cached = const cached =
context.storefrontCache.get(sellerId) ?? context.storefrontCache.get(sellerId) ??
context.metadataCache.get(sellerId) ??
(await loadCachedSeller( (await loadCachedSeller(
sellerId, sellerId,
args.sellerCacheHours, args.sellerCacheHours,
true, true,
args.inventoryLimit, args.inventoryLimit,
)); ));
if (cached) { if (cached && cached.storefrontAsinTotal > 0) {
context.metadataCache.set(sellerId, cached);
context.storefrontCache.set(sellerId, cached); context.storefrontCache.set(sellerId, cached);
out.set(sellerId, cached); out.set(sellerId, cached);
continue; continue;
@@ -765,7 +753,6 @@ async function fetchQualifiedSellerStorefronts(
domain: DOMAIN_US, domain: DOMAIN_US,
seller: sellerId, seller: sellerId,
storefront: "1", storefront: "1",
update: String(args.storefrontUpdateHours),
}); });
context.stats.sellerStorefrontRequests += 1; context.stats.sellerStorefrontRequests += 1;
@@ -1489,7 +1476,7 @@ async function runSellableAnalysisChild(
const cmd = [ const cmd = [
"bun", "bun",
"run", "run",
"src/stalker-analyze.ts", "src/stalker/stalker-analyze.ts",
"--stalker-run-id", "--stalker-run-id",
String(stalkerRunId), String(stalkerRunId),
"--analysis-run-id", "--analysis-run-id",
@@ -1566,7 +1553,7 @@ function hasFlag(args: string[], flag: string): boolean {
function printUsageAndExit(message: string): never { function printUsageAndExit(message: string): never {
console.error(message); console.error(message);
console.error( console.error(
"Usage: bun run stalker --input input/asins.xlsx [--db db/results.db] [--max-asins N] [--offer-limit 100] [--seller-limit 30] [--inventory-limit 200] [--storefront-update-hours 168] [--seller-cache-hours 168] [--max-seller-requests N] [--sellability] [--analyze-sellable] [--include-stock] [--dry-run] [--no-resume] [--claude]", "Usage: bun run stalker --input input/asins.xlsx [--db db/results.db] [--max-asins N] [--offer-limit 100] [--seller-limit 30] [--inventory-limit 200] [--seller-cache-hours 168] [--max-seller-requests N] [--sellability] [--analyze-sellable] [--include-stock] [--dry-run] [--no-resume] [--claude]",
); );
process.exit(1); process.exit(1);
} }