f2c8a9728d
This new pipeline identifies products meeting specific monthly sold, price, seller count, and Amazon buy box share criteria across categories. It fetches comprehensive product data from Keepa and SP-API, analyzes it using an LLM, and persists the results. A key enhancement is the introduction of a dedicated Redis cache for Keepa and SP-API responses. This reduces API token consumption and improves performance for subsequent runs by caching enriched ASIN data with a 12-hour TTL. Products are saved regardless of their sellability status to provide a complete view.
107 lines
2.4 KiB
TypeScript
107 lines
2.4 KiB
TypeScript
import Redis from "ioredis";
|
|
import { config } from "./config.ts";
|
|
import type { EnrichedProduct, KeepaData, SpApiData } from "./types.ts";
|
|
|
|
let redis: Redis | null = null;
|
|
let disabled = false;
|
|
|
|
export type ApiCacheEntry = {
|
|
title: string;
|
|
keepa: KeepaData | null;
|
|
spApi: SpApiData;
|
|
fetchedAt: string;
|
|
};
|
|
|
|
function getApiCacheKey(asin: string): string {
|
|
return `api:asin:${asin}`;
|
|
}
|
|
|
|
export async function connectCache(): Promise<void> {
|
|
if (disabled) return;
|
|
try {
|
|
redis = new Redis(config.redisUrl, {
|
|
maxRetriesPerRequest: 1,
|
|
connectTimeout: 3000,
|
|
lazyConnect: true,
|
|
retryStrategy: () => null,
|
|
reconnectOnError: () => false,
|
|
});
|
|
// Swallow connection-level errors after we intentionally disable cache.
|
|
redis.on("error", () => {
|
|
// no-op
|
|
});
|
|
await redis.connect();
|
|
console.log("Redis connected");
|
|
} catch (err) {
|
|
console.warn(`Redis unavailable, running without cache: ${err}`);
|
|
if (redis) {
|
|
redis.disconnect();
|
|
}
|
|
redis = null;
|
|
disabled = true;
|
|
}
|
|
}
|
|
|
|
export async function getCache(asin: string): Promise<EnrichedProduct | null> {
|
|
if (!redis) return null;
|
|
try {
|
|
const data = await redis.get(`asin:${asin}`);
|
|
return data ? JSON.parse(data) : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
export async function setCache(
|
|
asin: string,
|
|
data: EnrichedProduct,
|
|
): Promise<void> {
|
|
if (!redis) return;
|
|
try {
|
|
await redis.set(
|
|
`asin:${asin}`,
|
|
JSON.stringify(data),
|
|
"EX",
|
|
config.cacheTtl,
|
|
);
|
|
} catch {
|
|
// Non-critical, continue without caching
|
|
}
|
|
}
|
|
|
|
export async function getApiCache(asin: string): Promise<ApiCacheEntry | null> {
|
|
if (!redis) return null;
|
|
try {
|
|
const raw = await redis.get(getApiCacheKey(asin));
|
|
if (!raw) return null;
|
|
return JSON.parse(raw) as ApiCacheEntry;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
export async function setApiCache(
|
|
asin: string,
|
|
data: ApiCacheEntry,
|
|
ttlSeconds: number,
|
|
): Promise<void> {
|
|
if (!redis) return;
|
|
try {
|
|
await redis.set(
|
|
getApiCacheKey(asin),
|
|
JSON.stringify(data),
|
|
"EX",
|
|
ttlSeconds,
|
|
);
|
|
} catch {
|
|
// Non-critical, continue without caching
|
|
}
|
|
}
|
|
|
|
export async function disconnectCache(): Promise<void> {
|
|
if (redis) {
|
|
await redis.quit();
|
|
redis = null;
|
|
}
|
|
}
|