Refactor SP-API test script and improve type definitions
- Updated `sp-test.ts` to enhance argument parsing and error handling for sellability checks. - Refactored `types.ts` to maintain consistent formatting and improve readability. - Improved `writer.ts` for better result handling and CSV writing, ensuring clarity in output. - Adjusted `tsconfig.json` formatting for consistency and readability.
This commit is contained in:
Victor Noguera
282
src/keepa.ts
282
src/keepa.ts
@@ -1,141 +1,141 @@
|
||||
import { config } from "./config.ts";
|
||||
import type { KeepaData } from "./types.ts";
|
||||
|
||||
const KEEPA_BASE = "https://api.keepa.com";
|
||||
const MAX_ASINS_PER_REQUEST = 100;
|
||||
|
||||
// Token-based rate limiting: Keepa Pro = 1 token/min regeneration.
|
||||
// Each product request costs 1 token regardless of ASIN count (up to 100).
|
||||
// The API response includes tokensLeft and refillRate — we use those to pace.
|
||||
let tokensLeft = 1; // Conservative start; updated from API response
|
||||
let refillRate = 1; // tokens per minute, updated from API response
|
||||
let lastRequestTime = 0;
|
||||
|
||||
async function waitForToken(): Promise<void> {
|
||||
if (tokensLeft > 0) return;
|
||||
|
||||
const elapsed = (Date.now() - lastRequestTime) / 60_000; // minutes
|
||||
const regenerated = Math.floor(elapsed * refillRate);
|
||||
if (regenerated > 0) {
|
||||
tokensLeft += regenerated;
|
||||
return;
|
||||
}
|
||||
|
||||
// Wait until we regenerate at least 1 token
|
||||
const waitMs =
|
||||
Math.ceil((1 / refillRate) * 60_000) - (Date.now() - lastRequestTime);
|
||||
if (waitMs > 0) {
|
||||
console.log(
|
||||
`Keepa tokens exhausted. Waiting ${Math.ceil(waitMs / 1000)}s for token regeneration...`,
|
||||
);
|
||||
await new Promise((r) => setTimeout(r, waitMs));
|
||||
}
|
||||
tokensLeft = 1;
|
||||
}
|
||||
|
||||
export async function fetchKeepaDataBatch(
|
||||
asins: string[],
|
||||
): Promise<Map<string, KeepaData>> {
|
||||
const results = new Map<string, KeepaData>();
|
||||
|
||||
// Split into chunks of MAX_ASINS_PER_REQUEST
|
||||
for (let i = 0; i < asins.length; i += MAX_ASINS_PER_REQUEST) {
|
||||
const chunk = asins.slice(i, i + MAX_ASINS_PER_REQUEST);
|
||||
await waitForToken();
|
||||
|
||||
const asinParam = chunk.join(",");
|
||||
const url = `${KEEPA_BASE}/product?key=${config.keepaApiKey}&domain=1&asin=${asinParam}&stats=90`;
|
||||
|
||||
console.log(
|
||||
`Keepa: fetching ${chunk.length} ASINs (tokens left: ${tokensLeft})...`,
|
||||
);
|
||||
|
||||
const res = await fetch(url);
|
||||
lastRequestTime = Date.now();
|
||||
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
throw new Error(`Keepa API error ${res.status}: ${text}`);
|
||||
}
|
||||
|
||||
const data = (await res.json()) as {
|
||||
products?: Record<string, any>[];
|
||||
tokensLeft?: number;
|
||||
refillRate?: number;
|
||||
};
|
||||
|
||||
// Update token state from API response
|
||||
if (data.tokensLeft != null) tokensLeft = data.tokensLeft;
|
||||
if (data.refillRate != null) refillRate = data.refillRate;
|
||||
|
||||
console.log(
|
||||
`Keepa: ${data.products?.length ?? 0} products returned, ${tokensLeft} tokens remaining (refill: ${refillRate}/min)`,
|
||||
);
|
||||
|
||||
if (data.products) {
|
||||
for (const product of data.products) {
|
||||
const asin = product.asin;
|
||||
if (!asin) continue;
|
||||
results.set(asin, parseKeepaProduct(product));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
function parseKeepaProduct(product: Record<string, any>): KeepaData {
|
||||
const stats = product.stats;
|
||||
const csv = product.csv;
|
||||
const salesRankDrops30 = pickKeepaNumber(
|
||||
product.salesRankDrops30,
|
||||
stats?.salesRankDrops30,
|
||||
);
|
||||
const salesRankDrops90 =
|
||||
pickKeepaNumber(product.salesRankDrops90, stats?.salesRankDrops90) ??
|
||||
(salesRankDrops30 != null ? salesRankDrops30 * 3 : null);
|
||||
const monthlySold =
|
||||
pickKeepaNumber(product.monthlySold, stats?.monthlySold) ??
|
||||
salesRankDrops30;
|
||||
|
||||
return {
|
||||
currentPrice: extractCurrentPrice(csv),
|
||||
avgPrice90: stats?.avg?.[0] != null ? stats.avg[0] / 100 : null,
|
||||
minPrice90: stats?.min?.[0] != null ? stats.min[0] / 100 : null,
|
||||
maxPrice90: stats?.max?.[0] != null ? stats.max[0] / 100 : null,
|
||||
salesRank: stats?.current?.[3] ?? null,
|
||||
salesRankAvg90: stats?.avg?.[3] ?? null,
|
||||
salesRankDrops30,
|
||||
salesRankDrops90,
|
||||
sellerCount: stats?.current?.[11] ?? null,
|
||||
buyBoxSeller: product.buyBoxSellerId ?? null,
|
||||
buyBoxPrice: stats?.current?.[18] != null ? stats.current[18] / 100 : null,
|
||||
monthlySold,
|
||||
categoryTree:
|
||||
product.categoryTree?.map((c: { name: string }) => c.name) ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
function pickKeepaNumber(...values: unknown[]): number | null {
|
||||
for (const value of values) {
|
||||
if (typeof value !== "number" || !Number.isFinite(value)) continue;
|
||||
// Keepa often uses -1 as "not available".
|
||||
if (value < 0) continue;
|
||||
return value;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function extractCurrentPrice(csv: number[][] | undefined): number | null {
|
||||
if (!csv) return null;
|
||||
|
||||
// csv[0] = Amazon price history, csv[1] = Marketplace new price history
|
||||
// Each is [time, price, time, price, ...] — last value is most recent
|
||||
for (const series of [csv[0], csv[1]]) {
|
||||
if (series && series.length >= 2) {
|
||||
const lastPrice = series[series.length - 1]!;
|
||||
if (lastPrice > 0) return lastPrice / 100;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
import { config } from "./config.ts";
|
||||
import type { KeepaData } from "./types.ts";
|
||||
|
||||
const KEEPA_BASE = "https://api.keepa.com";
|
||||
const MAX_ASINS_PER_REQUEST = 100;
|
||||
|
||||
// Token-based rate limiting: Keepa Pro = 1 token/min regeneration.
|
||||
// Each product request costs 1 token regardless of ASIN count (up to 100).
|
||||
// The API response includes tokensLeft and refillRate — we use those to pace.
|
||||
let tokensLeft = 1; // Conservative start; updated from API response
|
||||
let refillRate = 1; // tokens per minute, updated from API response
|
||||
let lastRequestTime = 0;
|
||||
|
||||
async function waitForToken(): Promise<void> {
|
||||
if (tokensLeft > 0) return;
|
||||
|
||||
const elapsed = (Date.now() - lastRequestTime) / 60_000; // minutes
|
||||
const regenerated = Math.floor(elapsed * refillRate);
|
||||
if (regenerated > 0) {
|
||||
tokensLeft += regenerated;
|
||||
return;
|
||||
}
|
||||
|
||||
// Wait until we regenerate at least 1 token
|
||||
const waitMs =
|
||||
Math.ceil((1 / refillRate) * 60_000) - (Date.now() - lastRequestTime);
|
||||
if (waitMs > 0) {
|
||||
console.log(
|
||||
`Keepa tokens exhausted. Waiting ${Math.ceil(waitMs / 1000)}s for token regeneration...`,
|
||||
);
|
||||
await new Promise((r) => setTimeout(r, waitMs));
|
||||
}
|
||||
tokensLeft = 1;
|
||||
}
|
||||
|
||||
export async function fetchKeepaDataBatch(
|
||||
asins: string[],
|
||||
): Promise<Map<string, KeepaData>> {
|
||||
const results = new Map<string, KeepaData>();
|
||||
|
||||
// Split into chunks of MAX_ASINS_PER_REQUEST
|
||||
for (let i = 0; i < asins.length; i += MAX_ASINS_PER_REQUEST) {
|
||||
const chunk = asins.slice(i, i + MAX_ASINS_PER_REQUEST);
|
||||
await waitForToken();
|
||||
|
||||
const asinParam = chunk.join(",");
|
||||
const url = `${KEEPA_BASE}/product?key=${config.keepaApiKey}&domain=1&asin=${asinParam}&stats=90`;
|
||||
|
||||
console.log(
|
||||
`Keepa: fetching ${chunk.length} ASINs (tokens left: ${tokensLeft})...`,
|
||||
);
|
||||
|
||||
const res = await fetch(url);
|
||||
lastRequestTime = Date.now();
|
||||
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
throw new Error(`Keepa API error ${res.status}: ${text}`);
|
||||
}
|
||||
|
||||
const data = (await res.json()) as {
|
||||
products?: Record<string, any>[];
|
||||
tokensLeft?: number;
|
||||
refillRate?: number;
|
||||
};
|
||||
|
||||
// Update token state from API response
|
||||
if (data.tokensLeft != null) tokensLeft = data.tokensLeft;
|
||||
if (data.refillRate != null) refillRate = data.refillRate;
|
||||
|
||||
console.log(
|
||||
`Keepa: ${data.products?.length ?? 0} products returned, ${tokensLeft} tokens remaining (refill: ${refillRate}/min)`,
|
||||
);
|
||||
|
||||
if (data.products) {
|
||||
for (const product of data.products) {
|
||||
const asin = product.asin;
|
||||
if (!asin) continue;
|
||||
results.set(asin, parseKeepaProduct(product));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
function parseKeepaProduct(product: Record<string, any>): KeepaData {
|
||||
const stats = product.stats;
|
||||
const csv = product.csv;
|
||||
const salesRankDrops30 = pickKeepaNumber(
|
||||
product.salesRankDrops30,
|
||||
stats?.salesRankDrops30,
|
||||
);
|
||||
const salesRankDrops90 =
|
||||
pickKeepaNumber(product.salesRankDrops90, stats?.salesRankDrops90) ??
|
||||
(salesRankDrops30 != null ? salesRankDrops30 * 3 : null);
|
||||
const monthlySold =
|
||||
pickKeepaNumber(product.monthlySold, stats?.monthlySold) ??
|
||||
salesRankDrops30;
|
||||
|
||||
return {
|
||||
currentPrice: extractCurrentPrice(csv),
|
||||
avgPrice90: stats?.avg?.[0] != null ? stats.avg[0] / 100 : null,
|
||||
minPrice90: stats?.min?.[0] != null ? stats.min[0] / 100 : null,
|
||||
maxPrice90: stats?.max?.[0] != null ? stats.max[0] / 100 : null,
|
||||
salesRank: stats?.current?.[3] ?? null,
|
||||
salesRankAvg90: stats?.avg?.[3] ?? null,
|
||||
salesRankDrops30,
|
||||
salesRankDrops90,
|
||||
sellerCount: stats?.current?.[11] ?? null,
|
||||
buyBoxSeller: product.buyBoxSellerId ?? null,
|
||||
buyBoxPrice: stats?.current?.[18] != null ? stats.current[18] / 100 : null,
|
||||
monthlySold,
|
||||
categoryTree:
|
||||
product.categoryTree?.map((c: { name: string }) => c.name) ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
function pickKeepaNumber(...values: unknown[]): number | null {
|
||||
for (const value of values) {
|
||||
if (typeof value !== "number" || !Number.isFinite(value)) continue;
|
||||
// Keepa often uses -1 as "not available".
|
||||
if (value < 0) continue;
|
||||
return value;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function extractCurrentPrice(csv: number[][] | undefined): number | null {
|
||||
if (!csv) return null;
|
||||
|
||||
// csv[0] = Amazon price history, csv[1] = Marketplace new price history
|
||||
// Each is [time, price, time, price, ...] — last value is most recent
|
||||
for (const series of [csv[0], csv[1]]) {
|
||||
if (series && series.length >= 2) {
|
||||
const lastPrice = series[series.length - 1]!;
|
||||
if (lastPrice > 0) return lastPrice / 100;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user