Compare commits
2 Commits
0f256be2be
...
0e03366534
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0e03366534 | ||
|
|
95cebaa27c |
@@ -12,7 +12,10 @@ AWS_SECRET_ACCESS_KEY=your_aws_secret_access_key
|
|||||||
REDIS_URL=redis://localhost:6379
|
REDIS_URL=redis://localhost:6379
|
||||||
LLM_URL=http://localhost:1234/v1
|
LLM_URL=http://localhost:1234/v1
|
||||||
LLM_MODEL=default
|
LLM_MODEL=default
|
||||||
|
ANTHROPIC_API_KEY=your_anthropic_api_key
|
||||||
|
ANTHROPIC_MODEL=claude-3-5-sonnet-20241022
|
||||||
CACHE_TTL=86400
|
CACHE_TTL=86400
|
||||||
GOOGLE_API_KEY=your_google_api_key
|
GOOGLE_API_KEY=your_google_api_key
|
||||||
GOOGLE_CSE_ID=your_google_programmable_search_engine_id
|
GOOGLE_CSE_ID=your_google_programmable_search_engine_id
|
||||||
SERPAPI_API_KEY=your_serpapi_api_key_for_google_shopping
|
SERPAPI_API_KEY=your_serpapi_api_key_for_google_shopping
|
||||||
|
|
||||||
|
|||||||
53
README.md
53
README.md
@@ -24,11 +24,14 @@ cp .env.example .env
|
|||||||
bun run src/index.ts input/<input.csv|xlsx> [--out output/results.xlsx]
|
bun run src/index.ts input/<input.csv|xlsx> [--out output/results.xlsx]
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Add `--claude` to use Anthropic Claude instead of local LM Studio for LLM analysis.
|
||||||
|
|
||||||
Examples:
|
Examples:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
bun run src/index.ts input/leads.xlsx
|
bun run src/index.ts input/leads.xlsx
|
||||||
bun run src/index.ts input/leads.csv --out output/results.xlsx
|
bun run src/index.ts input/leads.csv --out output/results.xlsx
|
||||||
|
bun run src/index.ts input/leads.xlsx --claude
|
||||||
```
|
```
|
||||||
|
|
||||||
Large-file behavior:
|
Large-file behavior:
|
||||||
@@ -55,6 +58,14 @@ bun run monthly-sold
|
|||||||
bun run mid-range
|
bun run mid-range
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Use Claude for category LLM analysis:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bun run bestsellers --claude
|
||||||
|
bun run monthly-sold --claude
|
||||||
|
bun run mid-range --claude
|
||||||
|
```
|
||||||
|
|
||||||
Mid-range process:
|
Mid-range process:
|
||||||
|
|
||||||
- Script: `bun run mid-range`
|
- Script: `bun run mid-range`
|
||||||
@@ -128,6 +139,12 @@ curl -X POST "http://localhost:3000/api/upc/lookup" \
|
|||||||
-d '{"upcs":["012345678901","098765432109"]}'
|
-d '{"upcs":["012345678901","098765432109"]}'
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Run the web server with Claude-backed LLM calls:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bun run start:web -- --claude
|
||||||
|
```
|
||||||
|
|
||||||
## Large UPC File Analysis (XLS/XLSX)
|
## Large UPC File Analysis (XLS/XLSX)
|
||||||
|
|
||||||
For supplier price lists that contain UPC/EAN values and unit cost, use the
|
For supplier price lists that contain UPC/EAN values and unit cost, use the
|
||||||
@@ -248,23 +265,25 @@ ASIN, Name, Brand, Category, Unit Cost, Current Price, Avg Price 90d, Sales Rank
|
|||||||
|
|
||||||
## Environment variables
|
## Environment variables
|
||||||
|
|
||||||
| Variable | Default | Description |
|
| Variable | Default | Description |
|
||||||
| ----------------------- | -------------------------- | ----------------------------------------------------------------------- |
|
| ----------------------- | ---------------------------- | ----------------------------------------------------------------------- |
|
||||||
| `KEEPA_API_KEY` | — | **Required.** Keepa API key |
|
| `KEEPA_API_KEY` | — | **Required.** Keepa API key |
|
||||||
| `SP_API_CLIENT_ID` | — | LWA app client id from Solution Provider Portal |
|
| `SP_API_CLIENT_ID` | — | LWA app client id from Solution Provider Portal |
|
||||||
| `SP_API_CLIENT_SECRET` | — | LWA app client secret from Solution Provider Portal |
|
| `SP_API_CLIENT_SECRET` | — | LWA app client secret from Solution Provider Portal |
|
||||||
| `SP_API_REFRESH_TOKEN` | — | Refresh token from self-authorization |
|
| `SP_API_REFRESH_TOKEN` | — | Refresh token from self-authorization |
|
||||||
| `SP_API_REGION` | `na` | SP-API endpoint region (`na`, `eu`, `fe`; `us` is accepted as `na`) |
|
| `SP_API_REGION` | `na` | SP-API endpoint region (`na`, `eu`, `fe`; `us` is accepted as `na`) |
|
||||||
| `SP_API_MARKETPLACE_ID` | `ATVPDKIKX0DER` | Marketplace id used for pricing and fee calls (default: US) |
|
| `SP_API_MARKETPLACE_ID` | `ATVPDKIKX0DER` | Marketplace id used for pricing and fee calls (default: US) |
|
||||||
| `SP_API_SELLER_ID` | — | Seller ID used for listing restrictions eligibility checks |
|
| `SP_API_SELLER_ID` | — | Seller ID used for listing restrictions eligibility checks |
|
||||||
| `SP_API_USE_SANDBOX` | `false` | Enable SP-API sandbox mode (`true`/`false`) |
|
| `SP_API_USE_SANDBOX` | `false` | Enable SP-API sandbox mode (`true`/`false`) |
|
||||||
| `AWS_ACCESS_KEY_ID` | — | AWS credentials for SigV4 signing (required in most private app setups) |
|
| `AWS_ACCESS_KEY_ID` | — | AWS credentials for SigV4 signing (required in most private app setups) |
|
||||||
| `AWS_SECRET_ACCESS_KEY` | — | AWS credentials for SigV4 signing |
|
| `AWS_SECRET_ACCESS_KEY` | — | AWS credentials for SigV4 signing |
|
||||||
| `AWS_SESSION_TOKEN` | — | Optional session token when using STS credentials |
|
| `AWS_SESSION_TOKEN` | — | Optional session token when using STS credentials |
|
||||||
| `REDIS_URL` | `redis://localhost:6379` | Redis connection URL |
|
| `REDIS_URL` | `redis://localhost:6379` | Redis connection URL |
|
||||||
| `LLM_URL` | `http://localhost:1234/v1` | LM Studio API base URL |
|
| `LLM_URL` | `http://localhost:1234/v1` | LM Studio API base URL |
|
||||||
| `LLM_MODEL` | `default` | Model name to pass to LM Studio |
|
| `LLM_MODEL` | `default` | Model name to pass to LM Studio |
|
||||||
| `CACHE_TTL` | `86400` | Redis cache TTL in seconds |
|
| `ANTHROPIC_API_KEY` | — | Required when running any LLM script with `--claude` |
|
||||||
|
| `ANTHROPIC_MODEL` | `claude-3-5-sonnet-20241022` | Claude model ID used with `--claude` |
|
||||||
|
| `CACHE_TTL` | `86400` | Redis cache TTL in seconds |
|
||||||
|
|
||||||
## Notes
|
## Notes
|
||||||
|
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ export type AnalysisPipelineOptions = {
|
|||||||
llmBatchDelayMs?: number;
|
llmBatchDelayMs?: number;
|
||||||
llmRetryDelayMs?: number;
|
llmRetryDelayMs?: number;
|
||||||
sellability?: SellabilityFilter;
|
sellability?: SellabilityFilter;
|
||||||
|
useClaude?: boolean;
|
||||||
};
|
};
|
||||||
|
|
||||||
export function chunkArray<T>(items: T[], chunkSize: number): T[][] {
|
export function chunkArray<T>(items: T[], chunkSize: number): T[][] {
|
||||||
@@ -60,6 +61,7 @@ export async function processProductChunk(
|
|||||||
const llmBatchDelayMs = Math.max(0, options.llmBatchDelayMs ?? 5_000);
|
const llmBatchDelayMs = Math.max(0, options.llmBatchDelayMs ?? 5_000);
|
||||||
const llmRetryDelayMs = Math.max(0, options.llmRetryDelayMs ?? 10_000);
|
const llmRetryDelayMs = Math.max(0, options.llmRetryDelayMs ?? 10_000);
|
||||||
const sellabilityFilter = options.sellability ?? "available";
|
const sellabilityFilter = options.sellability ?? "available";
|
||||||
|
const useClaude = options.useClaude === true;
|
||||||
|
|
||||||
console.log(`\nChecking cache for ${products.length} products...`);
|
console.log(`\nChecking cache for ${products.length} products...`);
|
||||||
const cached = new Map<string, EnrichedProduct>();
|
const cached = new Map<string, EnrichedProduct>();
|
||||||
@@ -242,6 +244,7 @@ export async function processProductChunk(
|
|||||||
try {
|
try {
|
||||||
verdicts = await analyzeProducts(batch, {
|
verdicts = await analyzeProducts(batch, {
|
||||||
ignoreSellability: sellabilityFilter === "all",
|
ignoreSellability: sellabilityFilter === "all",
|
||||||
|
useClaude,
|
||||||
});
|
});
|
||||||
} catch {
|
} catch {
|
||||||
if (llmRetryDelayMs > 0) {
|
if (llmRetryDelayMs > 0) {
|
||||||
@@ -250,6 +253,7 @@ export async function processProductChunk(
|
|||||||
try {
|
try {
|
||||||
verdicts = await analyzeProducts(batch, {
|
verdicts = await analyzeProducts(batch, {
|
||||||
ignoreSellability: sellabilityFilter === "all",
|
ignoreSellability: sellabilityFilter === "all",
|
||||||
|
useClaude,
|
||||||
});
|
});
|
||||||
} catch {
|
} catch {
|
||||||
verdicts = null;
|
verdicts = null;
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ type ParsedArgs = {
|
|||||||
categoryLimit: number;
|
categoryLimit: number;
|
||||||
perCategoryTop: number;
|
perCategoryTop: number;
|
||||||
blacklistFile: string;
|
blacklistFile: string;
|
||||||
|
useClaude: boolean;
|
||||||
};
|
};
|
||||||
|
|
||||||
type CategoryRunSummary = {
|
type CategoryRunSummary = {
|
||||||
@@ -72,6 +73,7 @@ function log(
|
|||||||
|
|
||||||
function parseArgs(): ParsedArgs {
|
function parseArgs(): ParsedArgs {
|
||||||
const args = process.argv.slice(2);
|
const args = process.argv.slice(2);
|
||||||
|
const useClaude = hasFlag(args, "--claude");
|
||||||
const outputDir =
|
const outputDir =
|
||||||
readFlagValue(args, "--out-dir") ?? path.join(process.cwd(), "output");
|
readFlagValue(args, "--out-dir") ?? path.join(process.cwd(), "output");
|
||||||
const blacklistFile =
|
const blacklistFile =
|
||||||
@@ -100,9 +102,14 @@ function parseArgs(): ParsedArgs {
|
|||||||
categoryLimit,
|
categoryLimit,
|
||||||
perCategoryTop,
|
perCategoryTop,
|
||||||
blacklistFile,
|
blacklistFile,
|
||||||
|
useClaude,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function hasFlag(args: string[], flag: string): boolean {
|
||||||
|
return args.includes(flag);
|
||||||
|
}
|
||||||
|
|
||||||
function readFlagValue(args: string[], flag: string): string | undefined {
|
function readFlagValue(args: string[], flag: string): string | undefined {
|
||||||
const idx = args.indexOf(flag);
|
const idx = args.indexOf(flag);
|
||||||
if (idx === -1) return undefined;
|
if (idx === -1) return undefined;
|
||||||
@@ -118,7 +125,7 @@ function printUsageAndExit(message: string): never {
|
|||||||
"error",
|
"error",
|
||||||
[
|
[
|
||||||
"Usage:",
|
"Usage:",
|
||||||
" bun run src/bestsellers-by-category.ts [--category-limit 32] [--per-category-top 100] [--out-dir output] [--blacklist-file category-blacklist.csv]",
|
" bun run src/bestsellers-by-category.ts [--category-limit 32] [--per-category-top 100] [--out-dir output] [--blacklist-file category-blacklist.csv] [--claude]",
|
||||||
"",
|
"",
|
||||||
"Flow:",
|
"Flow:",
|
||||||
" 1) Discover categories and round-robin selection.",
|
" 1) Discover categories and round-robin selection.",
|
||||||
@@ -1011,6 +1018,7 @@ export async function processCategory(
|
|||||||
runId: number,
|
runId: number,
|
||||||
category: CategoryInfo,
|
category: CategoryInfo,
|
||||||
perCategoryTop: number,
|
perCategoryTop: number,
|
||||||
|
useClaude = false,
|
||||||
): Promise<CategoryRunSummary> {
|
): Promise<CategoryRunSummary> {
|
||||||
log("info", `\nCategory ${category.label} (${category.id})`);
|
log("info", `\nCategory ${category.label} (${category.id})`);
|
||||||
|
|
||||||
@@ -1106,7 +1114,7 @@ export async function processCategory(
|
|||||||
|
|
||||||
let batchVerdicts: LlmVerdict[];
|
let batchVerdicts: LlmVerdict[];
|
||||||
try {
|
try {
|
||||||
batchVerdicts = await analyzeProducts(batch);
|
batchVerdicts = await analyzeProducts(batch, { useClaude });
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const message = err instanceof Error ? err.message : String(err);
|
const message = err instanceof Error ? err.message : String(err);
|
||||||
log("warn", ` LLM batch failed: ${message}`);
|
log("warn", ` LLM batch failed: ${message}`);
|
||||||
@@ -1249,6 +1257,7 @@ export async function main(): Promise<void> {
|
|||||||
runId,
|
runId,
|
||||||
category,
|
category,
|
||||||
args.perCategoryTop,
|
args.perCategoryTop,
|
||||||
|
args.useClaude,
|
||||||
);
|
);
|
||||||
|
|
||||||
totalInsertedAsins += categorySummary.results?.length ?? 0;
|
totalInsertedAsins += categorySummary.results?.length ?? 0;
|
||||||
|
|||||||
@@ -20,6 +20,8 @@ export const config = {
|
|||||||
redisUrl: optional("REDIS_URL", "redis://localhost:6379"),
|
redisUrl: optional("REDIS_URL", "redis://localhost:6379"),
|
||||||
llmUrl: optional("LLM_URL", "http://localhost:1234/v1"),
|
llmUrl: optional("LLM_URL", "http://localhost:1234/v1"),
|
||||||
llmModel: optional("LLM_MODEL", "default"),
|
llmModel: optional("LLM_MODEL", "default"),
|
||||||
|
anthropicApiKey: Bun.env.ANTHROPIC_API_KEY,
|
||||||
|
anthropicModel: Bun.env.ANTHROPIC_MODEL,
|
||||||
cacheTtl: parseInt(optional("CACHE_TTL", "86400"), 10),
|
cacheTtl: parseInt(optional("CACHE_TTL", "86400"), 10),
|
||||||
searxngUrl: optional("SEARXNG_URL", "https://searxng.nvictor.me/"),
|
searxngUrl: optional("SEARXNG_URL", "https://searxng.nvictor.me/"),
|
||||||
searxngTimeoutMs: parseInt(optional("SEARXNG_TIMEOUT_MS", "10000"), 10),
|
searxngTimeoutMs: parseInt(optional("SEARXNG_TIMEOUT_MS", "10000"), 10),
|
||||||
|
|||||||
14
src/index.ts
14
src/index.ts
@@ -42,9 +42,11 @@ function parseArgs(): {
|
|||||||
inputFile: string;
|
inputFile: string;
|
||||||
outputFile?: string;
|
outputFile?: string;
|
||||||
sellability: SellabilityFilter;
|
sellability: SellabilityFilter;
|
||||||
|
useClaude: boolean;
|
||||||
} {
|
} {
|
||||||
const args = process.argv.slice(2);
|
const args = process.argv.slice(2);
|
||||||
const outputFile = readFlagValue(args, "--out", "--output");
|
const outputFile = readFlagValue(args, "--out", "--output");
|
||||||
|
const useClaude = args.includes("--claude");
|
||||||
const inputFile = readInputFileArg(
|
const inputFile = readInputFileArg(
|
||||||
args,
|
args,
|
||||||
"--out",
|
"--out",
|
||||||
@@ -55,12 +57,12 @@ function parseArgs(): {
|
|||||||
|
|
||||||
if (!inputFile) {
|
if (!inputFile) {
|
||||||
console.error(
|
console.error(
|
||||||
"Usage: bun run src/index.ts <input.csv|xlsx> [--out results.xlsx|--output results.xlsx] [--sellability available|all]",
|
"Usage: bun run src/index.ts <input.csv|xlsx> [--out results.xlsx|--output results.xlsx] [--sellability available|all] [--claude]",
|
||||||
);
|
);
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
return { inputFile, outputFile, sellability };
|
return { inputFile, outputFile, sellability, useClaude };
|
||||||
}
|
}
|
||||||
|
|
||||||
function readFlagValue(args: string[], ...flags: string[]): string | undefined {
|
function readFlagValue(args: string[], ...flags: string[]): string | undefined {
|
||||||
@@ -109,9 +111,10 @@ function resolveBaseOutputPath(inputFile: string, outputFile?: string): string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function main() {
|
async function main() {
|
||||||
const { inputFile, outputFile, sellability } = parseArgs();
|
const { inputFile, outputFile, sellability, useClaude } = parseArgs();
|
||||||
|
|
||||||
console.log(`Sellability filter: ${sellability}`);
|
console.log(`Sellability filter: ${sellability}`);
|
||||||
|
console.log(`LLM provider: ${useClaude ? "claude" : "local"}`);
|
||||||
|
|
||||||
console.log("Connecting to Redis...");
|
console.log("Connecting to Redis...");
|
||||||
await connectCache();
|
await connectCache();
|
||||||
@@ -144,7 +147,10 @@ async function main() {
|
|||||||
console.log(
|
console.log(
|
||||||
`\n=== Input chunk ${chunkIndex + 1}/${productChunks.length} (${chunk.length} products) ===`,
|
`\n=== Input chunk ${chunkIndex + 1}/${productChunks.length} (${chunk.length} products) ===`,
|
||||||
);
|
);
|
||||||
const chunkResults = await processProductChunk(chunk, { sellability });
|
const chunkResults = await processProductChunk(chunk, {
|
||||||
|
sellability,
|
||||||
|
useClaude,
|
||||||
|
});
|
||||||
allResults.push(...chunkResults);
|
allResults.push(...chunkResults);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
126
src/llm.ts
126
src/llm.ts
@@ -56,6 +56,17 @@ Keep each reasoning under 100 characters to stay within output limits and mentio
|
|||||||
|
|
||||||
type AnalyzeProductsOptions = {
|
type AnalyzeProductsOptions = {
|
||||||
ignoreSellability?: boolean;
|
ignoreSellability?: boolean;
|
||||||
|
useClaude?: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
type LlmProvider = "lm-studio" | "claude";
|
||||||
|
|
||||||
|
type LmStudioResponse = {
|
||||||
|
choices?: { message?: { content?: string } }[];
|
||||||
|
};
|
||||||
|
|
||||||
|
type ClaudeResponse = {
|
||||||
|
content?: Array<{ type?: string; text?: string }>;
|
||||||
};
|
};
|
||||||
|
|
||||||
function getSystemPrompt(options: AnalyzeProductsOptions): string {
|
function getSystemPrompt(options: AnalyzeProductsOptions): string {
|
||||||
@@ -72,8 +83,7 @@ export async function analyzeProducts(
|
|||||||
try {
|
try {
|
||||||
return await analyzeProductsInternal(products, options);
|
return await analyzeProductsInternal(products, options);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const msg = String(err);
|
if (products.length > 1 && isContextOverflowError(err)) {
|
||||||
if (products.length > 1 && msg.includes("Context size has been exceeded")) {
|
|
||||||
console.warn(
|
console.warn(
|
||||||
`LLM context exceeded for batch of ${products.length}, retrying one product at a time...`,
|
`LLM context exceeded for batch of ${products.length}, retrying one product at a time...`,
|
||||||
);
|
);
|
||||||
@@ -113,7 +123,43 @@ async function analyzeProductsInternal(
|
|||||||
summarizeForLlm(p, options.ignoreSellability === true),
|
summarizeForLlm(p, options.ignoreSellability === true),
|
||||||
);
|
);
|
||||||
const systemPrompt = getSystemPrompt(options);
|
const systemPrompt = getSystemPrompt(options);
|
||||||
|
const provider = options.useClaude ? "claude" : "lm-studio";
|
||||||
|
const content = await requestLlmContent(
|
||||||
|
provider,
|
||||||
|
systemPrompt,
|
||||||
|
productSummaries,
|
||||||
|
);
|
||||||
|
|
||||||
|
return parseVerdicts(content, products);
|
||||||
|
}
|
||||||
|
|
||||||
|
function isContextOverflowError(err: unknown): boolean {
|
||||||
|
const msg = String(err).toLowerCase();
|
||||||
|
return (
|
||||||
|
msg.includes("context size has been exceeded") ||
|
||||||
|
msg.includes("prompt is too long") ||
|
||||||
|
msg.includes("too many tokens") ||
|
||||||
|
msg.includes("maximum context") ||
|
||||||
|
msg.includes("context length") ||
|
||||||
|
msg.includes("max_tokens")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function requestLlmContent(
|
||||||
|
provider: LlmProvider,
|
||||||
|
systemPrompt: string,
|
||||||
|
productSummaries: ReturnType<typeof summarizeForLlm>[],
|
||||||
|
): Promise<string> {
|
||||||
|
if (provider === "claude") {
|
||||||
|
return requestClaudeContent(systemPrompt, productSummaries);
|
||||||
|
}
|
||||||
|
return requestLmStudioContent(systemPrompt, productSummaries);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function requestLmStudioContent(
|
||||||
|
systemPrompt: string,
|
||||||
|
productSummaries: ReturnType<typeof summarizeForLlm>[],
|
||||||
|
): Promise<string> {
|
||||||
const res = await fetch(`${config.llmUrl}/chat/completions`, {
|
const res = await fetch(`${config.llmUrl}/chat/completions`, {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
headers: {
|
headers: {
|
||||||
@@ -132,15 +178,79 @@ async function analyzeProductsInternal(
|
|||||||
});
|
});
|
||||||
|
|
||||||
if (!res.ok) {
|
if (!res.ok) {
|
||||||
throw new Error(`LLM API error ${res.status}: ${await res.text()}`);
|
throw new Error(`LLM API error ${res.status}: ${await readErrorBody(res)}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const data = (await res.json()) as {
|
const data = (await res.json()) as LmStudioResponse;
|
||||||
choices?: { message?: { content?: string } }[];
|
return data.choices?.[0]?.message?.content ?? "";
|
||||||
};
|
}
|
||||||
const content = data.choices?.[0]?.message?.content ?? "";
|
|
||||||
|
|
||||||
return parseVerdicts(content, products);
|
async function requestClaudeContent(
|
||||||
|
systemPrompt: string,
|
||||||
|
productSummaries: ReturnType<typeof summarizeForLlm>[],
|
||||||
|
): Promise<string> {
|
||||||
|
if (!config.anthropicApiKey) {
|
||||||
|
throw new Error(
|
||||||
|
"Missing required env var for --claude mode: ANTHROPIC_API_KEY",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const res = await fetch("https://api.anthropic.com/v1/messages", {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"x-api-key": config.anthropicApiKey,
|
||||||
|
"anthropic-version": "2023-06-01",
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
model: config.anthropicModel,
|
||||||
|
system: systemPrompt,
|
||||||
|
messages: [
|
||||||
|
{ role: "user", content: JSON.stringify(productSummaries, null, 2) },
|
||||||
|
],
|
||||||
|
temperature: 0.3,
|
||||||
|
max_tokens: 2048,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!res.ok) {
|
||||||
|
throw new Error(
|
||||||
|
`Claude API error ${res.status}: ${await readErrorBody(res)}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const data = (await res.json()) as ClaudeResponse;
|
||||||
|
if (!Array.isArray(data.content)) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
return data.content
|
||||||
|
.filter((block) => block?.type === "text" && typeof block.text === "string")
|
||||||
|
.map((block) => block.text ?? "")
|
||||||
|
.join("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readErrorBody(response: Response): Promise<string> {
|
||||||
|
const text = await response.text();
|
||||||
|
if (!text.trim()) return "No response body";
|
||||||
|
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(text) as {
|
||||||
|
error?: { message?: string; type?: string };
|
||||||
|
};
|
||||||
|
const type = parsed.error?.type?.trim();
|
||||||
|
const message = parsed.error?.message?.trim();
|
||||||
|
if (type && message) {
|
||||||
|
return `${type}: ${message}`;
|
||||||
|
}
|
||||||
|
if (message) {
|
||||||
|
return message;
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Response was plain text.
|
||||||
|
}
|
||||||
|
|
||||||
|
return text;
|
||||||
}
|
}
|
||||||
|
|
||||||
function summarizeForLlm(p: EnrichedProduct, ignoreSellability: boolean) {
|
function summarizeForLlm(p: EnrichedProduct, ignoreSellability: boolean) {
|
||||||
|
|||||||
@@ -34,6 +34,7 @@ type ParsedArgs = {
|
|||||||
selectCategories: boolean;
|
selectCategories: boolean;
|
||||||
categoryIds: number[];
|
categoryIds: number[];
|
||||||
sellabilityGate: "strict" | "soft" | "off";
|
sellabilityGate: "strict" | "soft" | "off";
|
||||||
|
useClaude: boolean;
|
||||||
outputDir: string;
|
outputDir: string;
|
||||||
categoryLimit: number;
|
categoryLimit: number;
|
||||||
perCategoryTop: number;
|
perCategoryTop: number;
|
||||||
@@ -118,6 +119,7 @@ function parseArgs(): ParsedArgs {
|
|||||||
const args = process.argv.slice(2);
|
const args = process.argv.slice(2);
|
||||||
const listCategories = hasFlag(args, "--list-categories");
|
const listCategories = hasFlag(args, "--list-categories");
|
||||||
const selectCategories = hasFlag(args, "--select-categories");
|
const selectCategories = hasFlag(args, "--select-categories");
|
||||||
|
const useClaude = hasFlag(args, "--claude");
|
||||||
const categoryIdsRaw = readFlagValue(args, "--category-ids");
|
const categoryIdsRaw = readFlagValue(args, "--category-ids");
|
||||||
const sellabilityGateRaw = readFlagValue(args, "--sellability-gate");
|
const sellabilityGateRaw = readFlagValue(args, "--sellability-gate");
|
||||||
const outputDir =
|
const outputDir =
|
||||||
@@ -312,6 +314,7 @@ function parseArgs(): ParsedArgs {
|
|||||||
selectCategories,
|
selectCategories,
|
||||||
categoryIds,
|
categoryIds,
|
||||||
sellabilityGate,
|
sellabilityGate,
|
||||||
|
useClaude,
|
||||||
outputDir,
|
outputDir,
|
||||||
categoryLimit,
|
categoryLimit,
|
||||||
perCategoryTop,
|
perCategoryTop,
|
||||||
@@ -370,7 +373,7 @@ function printUsageAndExit(message: string): never {
|
|||||||
"error",
|
"error",
|
||||||
[
|
[
|
||||||
"Usage:",
|
"Usage:",
|
||||||
" bun run src/mid-range-sellers-by-category.ts [--category-limit 32] [--list-categories] [--select-categories] [--category-ids 281053,172282] [--sellability-gate soft] [--per-category-top 100] [--category-candidate-pool 500] [--candidate-batch-size 60] [--min-monthly-sold 100] [--max-monthly-sold 1000] [--min-price 15] [--max-price 200] [--min-seller-count 3] [--max-seller-count 20] [--min-amazon-buybox-share-pct 15] [--max-amazon-buybox-share-pct 85] [--max-asins-analyzed 250] [--max-keepa-products-fetched 500] [--out-dir output] [--blacklist-file category-blacklist.csv]",
|
" bun run src/mid-range-sellers-by-category.ts [--category-limit 32] [--list-categories] [--select-categories] [--category-ids 281053,172282] [--sellability-gate soft] [--per-category-top 100] [--category-candidate-pool 500] [--candidate-batch-size 60] [--min-monthly-sold 100] [--max-monthly-sold 1000] [--min-price 15] [--max-price 200] [--min-seller-count 3] [--max-seller-count 20] [--min-amazon-buybox-share-pct 15] [--max-amazon-buybox-share-pct 85] [--max-asins-analyzed 250] [--max-keepa-products-fetched 500] [--out-dir output] [--blacklist-file category-blacklist.csv] [--claude]",
|
||||||
"",
|
"",
|
||||||
"Selection:",
|
"Selection:",
|
||||||
" --list-categories Discover and print runnable categories, then exit.",
|
" --list-categories Discover and print runnable categories, then exit.",
|
||||||
@@ -1482,6 +1485,7 @@ export async function processCategory(
|
|||||||
minAmazonBuyboxSharePct: number,
|
minAmazonBuyboxSharePct: number,
|
||||||
maxAmazonBuyboxSharePct: number,
|
maxAmazonBuyboxSharePct: number,
|
||||||
sellabilityGate: "strict" | "soft" | "off",
|
sellabilityGate: "strict" | "soft" | "off",
|
||||||
|
useClaude = false,
|
||||||
runtimeBudget?: RuntimeBudget,
|
runtimeBudget?: RuntimeBudget,
|
||||||
candidateBatchSize = DEFAULT_CANDIDATE_BATCH_SIZE,
|
candidateBatchSize = DEFAULT_CANDIDATE_BATCH_SIZE,
|
||||||
): Promise<CategoryRunSummary> {
|
): Promise<CategoryRunSummary> {
|
||||||
@@ -1739,7 +1743,7 @@ export async function processCategory(
|
|||||||
|
|
||||||
let batchVerdicts: LlmVerdict[];
|
let batchVerdicts: LlmVerdict[];
|
||||||
try {
|
try {
|
||||||
batchVerdicts = await analyzeProducts(batch);
|
batchVerdicts = await analyzeProducts(batch, { useClaude });
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const message = err instanceof Error ? err.message : String(err);
|
const message = err instanceof Error ? err.message : String(err);
|
||||||
log("warn", ` LLM batch failed: ${message}`);
|
log("warn", ` LLM batch failed: ${message}`);
|
||||||
@@ -2014,6 +2018,7 @@ export async function main(): Promise<void> {
|
|||||||
args.minAmazonBuyboxSharePct,
|
args.minAmazonBuyboxSharePct,
|
||||||
args.maxAmazonBuyboxSharePct,
|
args.maxAmazonBuyboxSharePct,
|
||||||
args.sellabilityGate,
|
args.sellabilityGate,
|
||||||
|
args.useClaude,
|
||||||
runtimeBudget,
|
runtimeBudget,
|
||||||
args.candidateBatchSize,
|
args.candidateBatchSize,
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -103,6 +103,7 @@ const DEFAULT_PAGE_SIZE = 25;
|
|||||||
const MAX_PAGE_SIZE = 200;
|
const MAX_PAGE_SIZE = 200;
|
||||||
const ASIN_PATTERN = /^[A-Z0-9]{10}$/;
|
const ASIN_PATTERN = /^[A-Z0-9]{10}$/;
|
||||||
const MAX_UPCS_PER_REQUEST = 1000;
|
const MAX_UPCS_PER_REQUEST = 1000;
|
||||||
|
const USE_CLAUDE = process.argv.includes("--claude");
|
||||||
|
|
||||||
initDb(DB_PATH);
|
initDb(DB_PATH);
|
||||||
const db = getDb(DB_PATH);
|
const db = getDb(DB_PATH);
|
||||||
@@ -128,7 +129,8 @@ function xlsx(buffer: ArrayBuffer, filename: string): Response {
|
|||||||
return new Response(buffer, {
|
return new Response(buffer, {
|
||||||
status: 200,
|
status: 200,
|
||||||
headers: {
|
headers: {
|
||||||
"content-type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
"content-type":
|
||||||
|
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||||
"content-disposition": `attachment; filename="${filename}"`,
|
"content-disposition": `attachment; filename="${filename}"`,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
@@ -758,7 +760,10 @@ function parseStalkerSort(sortParam: string | null): string {
|
|||||||
return parsed
|
return parsed
|
||||||
.replaceAll("runId", "runId")
|
.replaceAll("runId", "runId")
|
||||||
.replaceAll("rating_count", "rating_count")
|
.replaceAll("rating_count", "rating_count")
|
||||||
.replaceAll("persisted_inventory_asin_count", "persisted_inventory_asin_count")
|
.replaceAll(
|
||||||
|
"persisted_inventory_asin_count",
|
||||||
|
"persisted_inventory_asin_count",
|
||||||
|
)
|
||||||
.replaceAll("storefront_asin_total", "storefront_asin_total");
|
.replaceAll("storefront_asin_total", "storefront_asin_total");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -955,7 +960,11 @@ function parseStalkerProductSort(sortParam: string | null): string {
|
|||||||
"confidence",
|
"confidence",
|
||||||
"last_seen_at",
|
"last_seen_at",
|
||||||
]);
|
]);
|
||||||
return parseSort(sortParam, allowedSort, "monthly_sold DESC, last_seen_at DESC, asin ASC");
|
return parseSort(
|
||||||
|
sortParam,
|
||||||
|
allowedSort,
|
||||||
|
"monthly_sold DESC, last_seen_at DESC, asin ASC",
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function getStalkerProducts(filters: URLSearchParams) {
|
function getStalkerProducts(filters: URLSearchParams) {
|
||||||
@@ -1036,7 +1045,9 @@ function getStalkerProducts(filters: URLSearchParams) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function getStalkerProductsForExport(filters: URLSearchParams): StalkerProductRecord[] {
|
function getStalkerProductsForExport(
|
||||||
|
filters: URLSearchParams,
|
||||||
|
): StalkerProductRecord[] {
|
||||||
const { where, params } = parseStalkerProductFilters(filters);
|
const { where, params } = parseStalkerProductFilters(filters);
|
||||||
const orderBy = parseStalkerProductSort(filters.get("sort"));
|
const orderBy = parseStalkerProductSort(filters.get("sort"));
|
||||||
|
|
||||||
@@ -1100,7 +1111,12 @@ function exportStalkerProductsXlsx(filters: URLSearchParams): Response {
|
|||||||
Category: parseCategoryTreeForExport(row.category_tree),
|
Category: parseCategoryTreeForExport(row.category_tree),
|
||||||
"Monthly Sold": row.monthly_sold ?? null,
|
"Monthly Sold": row.monthly_sold ?? null,
|
||||||
Sellers: row.seller_count ?? null,
|
Sellers: row.seller_count ?? null,
|
||||||
"Amazon Seller": row.amazon_is_seller == null ? "" : row.amazon_is_seller === 1 ? "Yes" : "No",
|
"Amazon Seller":
|
||||||
|
row.amazon_is_seller == null
|
||||||
|
? ""
|
||||||
|
: row.amazon_is_seller === 1
|
||||||
|
? "Yes"
|
||||||
|
: "No",
|
||||||
"Sales Rank": row.sales_rank ?? null,
|
"Sales Rank": row.sales_rank ?? null,
|
||||||
"Current Price": row.current_price ?? null,
|
"Current Price": row.current_price ?? null,
|
||||||
"Avg 90d": row.avg_price_90d ?? null,
|
"Avg 90d": row.avg_price_90d ?? null,
|
||||||
@@ -1155,11 +1171,31 @@ function exportStalkerProductsXlsx(filters: URLSearchParams): Response {
|
|||||||
|
|
||||||
function purgeStalkerData() {
|
function purgeStalkerData() {
|
||||||
const counts = {
|
const counts = {
|
||||||
inventory: (db.query("SELECT COUNT(*) AS count FROM stalker_seller_inventory").get() as { count: number }).count,
|
inventory: (
|
||||||
asinSellers: (db.query("SELECT COUNT(*) AS count FROM stalker_asin_sellers").get() as { count: number }).count,
|
db
|
||||||
sellers: (db.query("SELECT COUNT(*) AS count FROM stalker_sellers").get() as { count: number }).count,
|
.query("SELECT COUNT(*) AS count FROM stalker_seller_inventory")
|
||||||
scans: (db.query("SELECT COUNT(*) AS count FROM stalker_asin_scans").get() as { count: number }).count,
|
.get() as { count: number }
|
||||||
runs: (db.query("SELECT COUNT(*) AS count FROM stalker_runs").get() as { count: number }).count,
|
).count,
|
||||||
|
asinSellers: (
|
||||||
|
db.query("SELECT COUNT(*) AS count FROM stalker_asin_sellers").get() as {
|
||||||
|
count: number;
|
||||||
|
}
|
||||||
|
).count,
|
||||||
|
sellers: (
|
||||||
|
db.query("SELECT COUNT(*) AS count FROM stalker_sellers").get() as {
|
||||||
|
count: number;
|
||||||
|
}
|
||||||
|
).count,
|
||||||
|
scans: (
|
||||||
|
db.query("SELECT COUNT(*) AS count FROM stalker_asin_scans").get() as {
|
||||||
|
count: number;
|
||||||
|
}
|
||||||
|
).count,
|
||||||
|
runs: (
|
||||||
|
db.query("SELECT COUNT(*) AS count FROM stalker_runs").get() as {
|
||||||
|
count: number;
|
||||||
|
}
|
||||||
|
).count,
|
||||||
};
|
};
|
||||||
|
|
||||||
db.transaction(() => {
|
db.transaction(() => {
|
||||||
@@ -1683,7 +1719,9 @@ async function reanalyzeSingleAsin(
|
|||||||
fetchedAt: new Date().toISOString(),
|
fetchedAt: new Date().toISOString(),
|
||||||
};
|
};
|
||||||
|
|
||||||
const verdicts = await analyzeProducts([enriched]);
|
const verdicts = await analyzeProducts([enriched], {
|
||||||
|
useClaude: USE_CLAUDE,
|
||||||
|
});
|
||||||
const verdict = verdicts[0] ?? {
|
const verdict = verdicts[0] ?? {
|
||||||
asin,
|
asin,
|
||||||
verdict: "SKIP" as const,
|
verdict: "SKIP" as const,
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ type Args = {
|
|||||||
stalkerRunId: number;
|
stalkerRunId: number;
|
||||||
analysisRunId: number;
|
analysisRunId: number;
|
||||||
asins: string[];
|
asins: string[];
|
||||||
|
useClaude: boolean;
|
||||||
};
|
};
|
||||||
|
|
||||||
type InventoryRow = {
|
type InventoryRow = {
|
||||||
@@ -45,6 +46,7 @@ function parseArgs(argv = process.argv.slice(2)): Args {
|
|||||||
const dbPath = readFlagValue(argv, "--db");
|
const dbPath = readFlagValue(argv, "--db");
|
||||||
const stalkerRunId = Number(readFlagValue(argv, "--stalker-run-id"));
|
const stalkerRunId = Number(readFlagValue(argv, "--stalker-run-id"));
|
||||||
const analysisRunId = Number(readFlagValue(argv, "--analysis-run-id"));
|
const analysisRunId = Number(readFlagValue(argv, "--analysis-run-id"));
|
||||||
|
const useClaude = argv.includes("--claude");
|
||||||
const asins = (readFlagValue(argv, "--asins") ?? "")
|
const asins = (readFlagValue(argv, "--asins") ?? "")
|
||||||
.split(",")
|
.split(",")
|
||||||
.map((asin) => asin.trim().toUpperCase())
|
.map((asin) => asin.trim().toUpperCase())
|
||||||
@@ -59,7 +61,7 @@ function parseArgs(argv = process.argv.slice(2)): Args {
|
|||||||
}
|
}
|
||||||
if (asins.length === 0) throw new Error("Missing --asins");
|
if (asins.length === 0) throw new Error("Missing --asins");
|
||||||
|
|
||||||
return { dbPath, stalkerRunId, analysisRunId, asins };
|
return { dbPath, stalkerRunId, analysisRunId, asins, useClaude };
|
||||||
}
|
}
|
||||||
|
|
||||||
function wait(ms: number): Promise<void> {
|
function wait(ms: number): Promise<void> {
|
||||||
@@ -299,6 +301,7 @@ function refreshAnalysisRun(database: Database, runId: number): void {
|
|||||||
|
|
||||||
async function analyzeInBatches(
|
async function analyzeInBatches(
|
||||||
products: EnrichedProduct[],
|
products: EnrichedProduct[],
|
||||||
|
useClaude: boolean,
|
||||||
): Promise<AnalysisResult[]> {
|
): Promise<AnalysisResult[]> {
|
||||||
const results: AnalysisResult[] = [];
|
const results: AnalysisResult[] = [];
|
||||||
|
|
||||||
@@ -316,7 +319,7 @@ async function analyzeInBatches(
|
|||||||
|
|
||||||
let verdicts;
|
let verdicts;
|
||||||
try {
|
try {
|
||||||
verdicts = await analyzeProducts(batch);
|
verdicts = await analyzeProducts(batch, { useClaude });
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.warn(
|
console.warn(
|
||||||
`Stalker analysis: LLM batch ${batchNumber} failed: ${
|
`Stalker analysis: LLM batch ${batchNumber} failed: ${
|
||||||
@@ -358,7 +361,7 @@ async function main(): Promise<void> {
|
|||||||
|
|
||||||
console.log(`Stalker analysis: analyzing ${rows.length} sellable ASIN(s).`);
|
console.log(`Stalker analysis: analyzing ${rows.length} sellable ASIN(s).`);
|
||||||
const enriched = await buildEnrichedProducts(rows);
|
const enriched = await buildEnrichedProducts(rows);
|
||||||
const results = await analyzeInBatches(enriched);
|
const results = await analyzeInBatches(enriched, args.useClaude);
|
||||||
insertProductAnalysisResults(database, args.analysisRunId, results);
|
insertProductAnalysisResults(database, args.analysisRunId, results);
|
||||||
refreshAnalysisRun(database, args.analysisRunId);
|
refreshAnalysisRun(database, args.analysisRunId);
|
||||||
} finally {
|
} finally {
|
||||||
|
|||||||
231
src/stalker.ts
231
src/stalker.ts
@@ -41,6 +41,7 @@ export type StalkerArgs = {
|
|||||||
maxSellerRequests: number | null;
|
maxSellerRequests: number | null;
|
||||||
sellability: boolean;
|
sellability: boolean;
|
||||||
analyzeSellable: boolean;
|
analyzeSellable: boolean;
|
||||||
|
useClaude: boolean;
|
||||||
};
|
};
|
||||||
|
|
||||||
export type StalkerOffer = {
|
export type StalkerOffer = {
|
||||||
@@ -143,8 +144,12 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs {
|
|||||||
const storefrontUpdateHours = storefrontUpdateRaw
|
const storefrontUpdateHours = storefrontUpdateRaw
|
||||||
? Number(storefrontUpdateRaw)
|
? Number(storefrontUpdateRaw)
|
||||||
: DEFAULT_STOREFRONT_UPDATE_HOURS;
|
: DEFAULT_STOREFRONT_UPDATE_HOURS;
|
||||||
const offerLimit = offerLimitRaw ? Number(offerLimitRaw) : DEFAULT_OFFER_LIMIT;
|
const offerLimit = offerLimitRaw
|
||||||
const sellerLimit = sellerLimitRaw ? Number(sellerLimitRaw) : DEFAULT_SELLER_LIMIT;
|
? Number(offerLimitRaw)
|
||||||
|
: DEFAULT_OFFER_LIMIT;
|
||||||
|
const sellerLimit = sellerLimitRaw
|
||||||
|
? Number(sellerLimitRaw)
|
||||||
|
: DEFAULT_SELLER_LIMIT;
|
||||||
const inventoryLimit = inventoryLimitRaw
|
const inventoryLimit = inventoryLimitRaw
|
||||||
? Number(inventoryLimitRaw)
|
? Number(inventoryLimitRaw)
|
||||||
: DEFAULT_INVENTORY_LIMIT;
|
: DEFAULT_INVENTORY_LIMIT;
|
||||||
@@ -159,6 +164,7 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs {
|
|||||||
const resume = !hasFlag(argv, "--no-resume");
|
const resume = !hasFlag(argv, "--no-resume");
|
||||||
const sellability = hasFlag(argv, "--sellability");
|
const sellability = hasFlag(argv, "--sellability");
|
||||||
const analyzeSellable = hasFlag(argv, "--analyze-sellable");
|
const analyzeSellable = hasFlag(argv, "--analyze-sellable");
|
||||||
|
const useClaude = hasFlag(argv, "--claude");
|
||||||
|
|
||||||
if (analyzeSellable && !sellability) {
|
if (analyzeSellable && !sellability) {
|
||||||
printUsageAndExit("--analyze-sellable requires --sellability.");
|
printUsageAndExit("--analyze-sellable requires --sellability.");
|
||||||
@@ -168,10 +174,7 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs {
|
|||||||
printUsageAndExit("--max-asins must be a positive integer.");
|
printUsageAndExit("--max-asins must be a positive integer.");
|
||||||
}
|
}
|
||||||
|
|
||||||
if (
|
if (!Number.isInteger(storefrontUpdateHours) || storefrontUpdateHours < 0) {
|
||||||
!Number.isInteger(storefrontUpdateHours) ||
|
|
||||||
storefrontUpdateHours < 0
|
|
||||||
) {
|
|
||||||
printUsageAndExit(
|
printUsageAndExit(
|
||||||
"--storefront-update-hours must be a non-negative integer.",
|
"--storefront-update-hours must be a non-negative integer.",
|
||||||
);
|
);
|
||||||
@@ -215,6 +218,7 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs {
|
|||||||
maxSellerRequests,
|
maxSellerRequests,
|
||||||
sellability,
|
sellability,
|
||||||
analyzeSellable,
|
analyzeSellable,
|
||||||
|
useClaude,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -232,9 +236,13 @@ export function readAsinsFromXlsx(filePath: string): string[] {
|
|||||||
if (rows.length === 0) throw new Error("File contains no data rows");
|
if (rows.length === 0) throw new Error("File contains no data rows");
|
||||||
|
|
||||||
const headers = Object.keys(rows[0]!);
|
const headers = Object.keys(rows[0]!);
|
||||||
const asinColumn = headers.find((header) => normalizeHeader(header) === "asin");
|
const asinColumn = headers.find(
|
||||||
|
(header) => normalizeHeader(header) === "asin",
|
||||||
|
);
|
||||||
if (!asinColumn) {
|
if (!asinColumn) {
|
||||||
throw new Error(`No ASIN column found. Available columns: ${headers.join(", ")}`);
|
throw new Error(
|
||||||
|
`No ASIN column found. Available columns: ${headers.join(", ")}`,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
return extractAsinsFromRows(rows, asinColumn);
|
return extractAsinsFromRows(rows, asinColumn);
|
||||||
@@ -287,7 +295,9 @@ export function extractLiveOfferSellerCandidates(
|
|||||||
offerPrice: extractOfferPrice(offer),
|
offerPrice: extractOfferPrice(offer),
|
||||||
condition: extractString(offer.condition ?? offer.conditionComment),
|
condition: extractString(offer.condition ?? offer.conditionComment),
|
||||||
isFba: extractBoolean(offer.isFBA ?? offer.isFba ?? offer.fba),
|
isFba: extractBoolean(offer.isFBA ?? offer.isFba ?? offer.fba),
|
||||||
stock: extractNumber(offer.stock ?? offer.stockCount ?? offer.currentStock),
|
stock: extractNumber(
|
||||||
|
offer.stock ?? offer.stockCount ?? offer.currentStock,
|
||||||
|
),
|
||||||
rawOffer: offer,
|
rawOffer: offer,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -305,7 +315,9 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
|
|||||||
|
|
||||||
initDb(args.dbPath);
|
initDb(args.dbPath);
|
||||||
const database = getDb(args.dbPath);
|
const database = getDb(args.dbPath);
|
||||||
const completedAsins = args.resume ? loadPreviouslyScannedAsins(database) : new Set<string>();
|
const completedAsins = args.resume
|
||||||
|
? loadPreviouslyScannedAsins(database)
|
||||||
|
: new Set<string>();
|
||||||
const resumeFilteredAsins = cappedAsins.filter(
|
const resumeFilteredAsins = cappedAsins.filter(
|
||||||
(asin) => !completedAsins.has(asin),
|
(asin) => !completedAsins.has(asin),
|
||||||
);
|
);
|
||||||
@@ -341,24 +353,32 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
if (args.dryRun) {
|
if (args.dryRun) {
|
||||||
console.log("Stalker dry-run: product and seller metadata will be fetched, storefronts will not be fetched or persisted.");
|
console.log(
|
||||||
|
"Stalker dry-run: product and seller metadata will be fetched, storefronts will not be fetched or persisted.",
|
||||||
|
);
|
||||||
}
|
}
|
||||||
if (stats.skippedAsins > 0) {
|
if (stats.skippedAsins > 0) {
|
||||||
console.log(`Stalker resume: skipped ${stats.skippedAsins} previously scanned ASIN(s).`);
|
console.log(
|
||||||
|
`Stalker resume: skipped ${stats.skippedAsins} previously scanned ASIN(s).`,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const asin of resumeFilteredAsins) {
|
for (const asin of resumeFilteredAsins) {
|
||||||
console.log(`Stalker: scanning ${asin} (${stats.scannedAsins + 1}/${resumeFilteredAsins.length})`);
|
console.log(
|
||||||
|
`Stalker: scanning ${asin} (${stats.scannedAsins + 1}/${resumeFilteredAsins.length})`,
|
||||||
|
);
|
||||||
|
|
||||||
const result = await scanAsin(asin, args, apiKey, context).catch((error) => ({
|
const result = await scanAsin(asin, args, apiKey, context).catch(
|
||||||
asin,
|
(error) => ({
|
||||||
title: null,
|
asin,
|
||||||
offerCount: 0,
|
title: null,
|
||||||
candidateSellerCount: 0,
|
offerCount: 0,
|
||||||
matchedSellers: [],
|
candidateSellerCount: 0,
|
||||||
product: null,
|
matchedSellers: [],
|
||||||
error: error instanceof Error ? error.message : String(error),
|
product: null,
|
||||||
}));
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
if (args.sellability && !args.dryRun) {
|
if (args.sellability && !args.dryRun) {
|
||||||
await enrichInventorySellability(result, stats);
|
await enrichInventorySellability(result, stats);
|
||||||
@@ -379,7 +399,13 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
|
|||||||
analysisRunId != null &&
|
analysisRunId != null &&
|
||||||
sellableAsins.length > 0
|
sellableAsins.length > 0
|
||||||
) {
|
) {
|
||||||
await runSellableAnalysisChild(args.dbPath, runId, analysisRunId, sellableAsins);
|
await runSellableAnalysisChild(
|
||||||
|
args.dbPath,
|
||||||
|
runId,
|
||||||
|
analysisRunId,
|
||||||
|
sellableAsins,
|
||||||
|
args.useClaude,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
stats.scannedAsins += 1;
|
stats.scannedAsins += 1;
|
||||||
stats.matchedSellers += result.matchedSellers.length;
|
stats.matchedSellers += result.matchedSellers.length;
|
||||||
@@ -398,7 +424,9 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
|
|||||||
);
|
);
|
||||||
|
|
||||||
if (stats.stoppedEarly) {
|
if (stats.stoppedEarly) {
|
||||||
console.log("Stalker: stopping early because max seller request budget was reached.");
|
console.log(
|
||||||
|
"Stalker: stopping early because max seller request budget was reached.",
|
||||||
|
);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -423,12 +451,7 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
|
|||||||
} catch (error) {
|
} catch (error) {
|
||||||
const message = error instanceof Error ? error.message : String(error);
|
const message = error instanceof Error ? error.message : String(error);
|
||||||
if (!args.dryRun && runId != null) {
|
if (!args.dryRun && runId != null) {
|
||||||
finishStalkerRunWithError(
|
finishStalkerRunWithError(database, runId, stats, message);
|
||||||
database,
|
|
||||||
runId,
|
|
||||||
stats,
|
|
||||||
message,
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
if (!args.dryRun && analysisRunId != null) {
|
if (!args.dryRun && analysisRunId != null) {
|
||||||
finishStalkerAnalysisRun(database, analysisRunId, "failed", message);
|
finishStalkerAnalysisRun(database, analysisRunId, "failed", message);
|
||||||
@@ -549,12 +572,11 @@ async function enrichInventorySellability(
|
|||||||
}
|
}
|
||||||
|
|
||||||
for (const item of items) {
|
for (const item of items) {
|
||||||
item.sellability =
|
item.sellability = sellabilityMap.get(item.asin) ?? {
|
||||||
sellabilityMap.get(item.asin) ?? {
|
canSell: null,
|
||||||
canSell: null,
|
sellabilityStatus: "unknown",
|
||||||
sellabilityStatus: "unknown",
|
sellabilityReason: "Sellability check returned no result",
|
||||||
sellabilityReason: "Sellability check returned no result",
|
};
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const seller of sellers) {
|
for (const seller of sellers) {
|
||||||
@@ -571,14 +593,19 @@ async function enrichInventoryProductDetails(
|
|||||||
result: StalkerAsinResult,
|
result: StalkerAsinResult,
|
||||||
apiKey: string,
|
apiKey: string,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const items = result.matchedSellers.flatMap(({ seller }) => seller.storefrontItems);
|
const items = result.matchedSellers.flatMap(
|
||||||
|
({ seller }) => seller.storefrontItems,
|
||||||
|
);
|
||||||
const uniqueAsins = Array.from(new Set(items.map((item) => item.asin)));
|
const uniqueAsins = Array.from(new Set(items.map((item) => item.asin)));
|
||||||
if (uniqueAsins.length === 0) return;
|
if (uniqueAsins.length === 0) return;
|
||||||
|
|
||||||
console.log(
|
console.log(
|
||||||
`Stalker inventory details: fetching Keepa product details for ${uniqueAsins.length} sellable ASIN(s)...`,
|
`Stalker inventory details: fetching Keepa product details for ${uniqueAsins.length} sellable ASIN(s)...`,
|
||||||
);
|
);
|
||||||
const detailsByAsin = await fetchKeepaInventoryProductDetails(apiKey, uniqueAsins);
|
const detailsByAsin = await fetchKeepaInventoryProductDetails(
|
||||||
|
apiKey,
|
||||||
|
uniqueAsins,
|
||||||
|
);
|
||||||
|
|
||||||
for (const item of items) {
|
for (const item of items) {
|
||||||
item.productDetails = detailsByAsin.get(item.asin) ?? null;
|
item.productDetails = detailsByAsin.get(item.asin) ?? null;
|
||||||
@@ -761,7 +788,8 @@ function canSpendSellerRequests(
|
|||||||
): boolean {
|
): boolean {
|
||||||
if (args.maxSellerRequests == null) return true;
|
if (args.maxSellerRequests == null) return true;
|
||||||
const spent =
|
const spent =
|
||||||
context.stats.sellerMetadataRequests + context.stats.sellerStorefrontRequests;
|
context.stats.sellerMetadataRequests +
|
||||||
|
context.stats.sellerStorefrontRequests;
|
||||||
if (spent + nextRequests <= args.maxSellerRequests) return true;
|
if (spent + nextRequests <= args.maxSellerRequests) return true;
|
||||||
context.stats.stoppedEarly = true;
|
context.stats.stoppedEarly = true;
|
||||||
return false;
|
return false;
|
||||||
@@ -856,7 +884,8 @@ function upsertAsinScan(
|
|||||||
`SELECT id FROM stalker_asin_scans WHERE run_id = ? AND source_asin = ?`,
|
`SELECT id FROM stalker_asin_scans WHERE run_id = ? AND source_asin = ?`,
|
||||||
)
|
)
|
||||||
.get(runId, result.asin) as { id: number } | null;
|
.get(runId, result.asin) as { id: number } | null;
|
||||||
if (!row) throw new Error(`Failed to load stalker scan row for ${result.asin}`);
|
if (!row)
|
||||||
|
throw new Error(`Failed to load stalker scan row for ${result.asin}`);
|
||||||
return row.id;
|
return row.id;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -978,7 +1007,9 @@ function upsertSellerInventory(
|
|||||||
item.sellability?.sellabilityReason ?? null,
|
item.sellability?.sellabilityReason ?? null,
|
||||||
item.productDetails?.title ?? null,
|
item.productDetails?.title ?? null,
|
||||||
item.productDetails?.brand ?? null,
|
item.productDetails?.brand ?? null,
|
||||||
item.productDetails ? JSON.stringify(item.productDetails.categoryTree) : null,
|
item.productDetails
|
||||||
|
? JSON.stringify(item.productDetails.categoryTree)
|
||||||
|
: null,
|
||||||
item.productDetails?.currentPrice ?? null,
|
item.productDetails?.currentPrice ?? null,
|
||||||
item.productDetails?.avgPrice90 ?? null,
|
item.productDetails?.avgPrice90 ?? null,
|
||||||
item.productDetails?.salesRank ?? null,
|
item.productDetails?.salesRank ?? null,
|
||||||
@@ -989,7 +1020,9 @@ function upsertSellerInventory(
|
|||||||
: item.productDetails.amazonIsSeller
|
: item.productDetails.amazonIsSeller
|
||||||
? 1
|
? 1
|
||||||
: 0,
|
: 0,
|
||||||
item.productDetails ? JSON.stringify(item.productDetails.rawProduct) : null,
|
item.productDetails
|
||||||
|
? JSON.stringify(item.productDetails.rawProduct)
|
||||||
|
: null,
|
||||||
fetchedAt,
|
fetchedAt,
|
||||||
JSON.stringify(item.rawInventory),
|
JSON.stringify(item.rawInventory),
|
||||||
);
|
);
|
||||||
@@ -1012,7 +1045,10 @@ function startStalkerRun(
|
|||||||
return result.lastInsertRowid as number;
|
return result.lastInsertRowid as number;
|
||||||
}
|
}
|
||||||
|
|
||||||
function startStalkerAnalysisRun(database: Database, inputFile: string): number {
|
function startStalkerAnalysisRun(
|
||||||
|
database: Database,
|
||||||
|
inputFile: string,
|
||||||
|
): number {
|
||||||
const result = database
|
const result = database
|
||||||
.prepare(
|
.prepare(
|
||||||
`INSERT INTO category_analysis_runs (
|
`INSERT INTO category_analysis_runs (
|
||||||
@@ -1231,18 +1267,24 @@ function normalizeSellerResponse(
|
|||||||
if (!sellers) return [];
|
if (!sellers) return [];
|
||||||
if (Array.isArray(sellers)) {
|
if (Array.isArray(sellers)) {
|
||||||
return sellers
|
return sellers
|
||||||
.map((seller) => [
|
.map(
|
||||||
normalizeSellerId(seller.sellerId ?? seller.sellerID ?? seller.id),
|
(seller) =>
|
||||||
seller,
|
[
|
||||||
] as [string | null, Record<string, any>])
|
normalizeSellerId(seller.sellerId ?? seller.sellerID ?? seller.id),
|
||||||
|
seller,
|
||||||
|
] as [string | null, Record<string, any>],
|
||||||
|
)
|
||||||
.filter((entry): entry is [string, Record<string, any>] => !!entry[0]);
|
.filter((entry): entry is [string, Record<string, any>] => !!entry[0]);
|
||||||
}
|
}
|
||||||
|
|
||||||
return Object.entries(sellers)
|
return Object.entries(sellers)
|
||||||
.map(([sellerId, seller]) => [
|
.map(
|
||||||
normalizeSellerId(sellerId),
|
([sellerId, seller]) =>
|
||||||
seller,
|
[normalizeSellerId(sellerId), seller] as [
|
||||||
] as [string | null, Record<string, any>])
|
string | null,
|
||||||
|
Record<string, any>,
|
||||||
|
],
|
||||||
|
)
|
||||||
.filter((entry): entry is [string, Record<string, any>] => !!entry[0]);
|
.filter((entry): entry is [string, Record<string, any>] => !!entry[0]);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1253,14 +1295,15 @@ function parseSeller(
|
|||||||
): StalkerSeller {
|
): StalkerSeller {
|
||||||
const allStorefrontItems = extractStorefrontItems(seller);
|
const allStorefrontItems = extractStorefrontItems(seller);
|
||||||
const storefrontItems =
|
const storefrontItems =
|
||||||
inventoryLimit === 0
|
inventoryLimit === 0 ? [] : allStorefrontItems.slice(0, inventoryLimit);
|
||||||
? []
|
|
||||||
: allStorefrontItems.slice(0, inventoryLimit);
|
|
||||||
const storefrontAsins = storefrontItems.map((item) => item.asin);
|
const storefrontAsins = storefrontItems.map((item) => item.asin);
|
||||||
return {
|
return {
|
||||||
sellerId,
|
sellerId,
|
||||||
sellerName: extractString(
|
sellerName: extractString(
|
||||||
seller.sellerName ?? seller.name ?? seller.storeName ?? seller.businessName,
|
seller.sellerName ??
|
||||||
|
seller.name ??
|
||||||
|
seller.storeName ??
|
||||||
|
seller.businessName,
|
||||||
),
|
),
|
||||||
rating: extractNumber(
|
rating: extractNumber(
|
||||||
seller.currentRating ?? seller.rating ?? seller.feedbackRating,
|
seller.currentRating ?? seller.rating ?? seller.feedbackRating,
|
||||||
@@ -1279,7 +1322,9 @@ function parseSeller(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function extractStorefrontItems(seller: Record<string, any>): StalkerInventoryItem[] {
|
function extractStorefrontItems(
|
||||||
|
seller: Record<string, any>,
|
||||||
|
): StalkerInventoryItem[] {
|
||||||
const candidates = [
|
const candidates = [
|
||||||
seller.asinList,
|
seller.asinList,
|
||||||
seller.asins,
|
seller.asins,
|
||||||
@@ -1311,7 +1356,12 @@ function collectStorefrontItems(
|
|||||||
const asin = normalizeAsin((value as Record<string, unknown>).asin);
|
const asin = normalizeAsin((value as Record<string, unknown>).asin);
|
||||||
if (asin && !seen.has(asin)) {
|
if (asin && !seen.has(asin)) {
|
||||||
seen.add(asin);
|
seen.add(asin);
|
||||||
items.push({ asin, rawInventory: value, sellability: null, productDetails: null });
|
items.push({
|
||||||
|
asin,
|
||||||
|
rawInventory: value,
|
||||||
|
sellability: null,
|
||||||
|
productDetails: null,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -1319,7 +1369,12 @@ function collectStorefrontItems(
|
|||||||
const asin = normalizeAsin(value);
|
const asin = normalizeAsin(value);
|
||||||
if (!asin || seen.has(asin)) return;
|
if (!asin || seen.has(asin)) return;
|
||||||
seen.add(asin);
|
seen.add(asin);
|
||||||
items.push({ asin, rawInventory: { asin }, sellability: null, productDetails: null });
|
items.push({
|
||||||
|
asin,
|
||||||
|
rawInventory: { asin },
|
||||||
|
sellability: null,
|
||||||
|
productDetails: null,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function parseInventoryProductDetails(
|
function parseInventoryProductDetails(
|
||||||
@@ -1331,9 +1386,9 @@ function parseInventoryProductDetails(
|
|||||||
title: extractString(product.title),
|
title: extractString(product.title),
|
||||||
brand: extractString(product.brand ?? product.manufacturer),
|
brand: extractString(product.brand ?? product.manufacturer),
|
||||||
categoryTree:
|
categoryTree:
|
||||||
product.categoryTree?.map((category: { name?: unknown }) =>
|
product.categoryTree
|
||||||
extractString(category.name),
|
?.map((category: { name?: unknown }) => extractString(category.name))
|
||||||
).filter((name: string | null): name is string => !!name) ?? [],
|
.filter((name: string | null): name is string => !!name) ?? [],
|
||||||
currentPrice: extractCurrentPrice(csv),
|
currentPrice: extractCurrentPrice(csv),
|
||||||
avgPrice90: stats?.avg?.[0] != null ? stats.avg[0] / 100 : null,
|
avgPrice90: stats?.avg?.[0] != null ? stats.avg[0] / 100 : null,
|
||||||
salesRank: extractNumber(stats?.current?.[3]),
|
salesRank: extractNumber(stats?.current?.[3]),
|
||||||
@@ -1371,10 +1426,14 @@ function resolveAmazonIsSeller(
|
|||||||
stats: Record<string, any> | undefined,
|
stats: Record<string, any> | undefined,
|
||||||
csv: unknown,
|
csv: unknown,
|
||||||
): boolean | null {
|
): boolean | null {
|
||||||
if (typeof product.isAmazonSeller === "boolean") return product.isAmazonSeller;
|
if (typeof product.isAmazonSeller === "boolean")
|
||||||
|
return product.isAmazonSeller;
|
||||||
if (typeof product.availabilityAmazon === "number") {
|
if (typeof product.availabilityAmazon === "number") {
|
||||||
if (product.availabilityAmazon >= 0) return true;
|
if (product.availabilityAmazon >= 0) return true;
|
||||||
if (product.availabilityAmazon === -1 || product.availabilityAmazon === -2) {
|
if (
|
||||||
|
product.availabilityAmazon === -1 ||
|
||||||
|
product.availabilityAmazon === -2
|
||||||
|
) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1437,21 +1496,27 @@ async function runSellableAnalysisChild(
|
|||||||
stalkerRunId: number,
|
stalkerRunId: number,
|
||||||
analysisRunId: number,
|
analysisRunId: number,
|
||||||
asins: string[],
|
asins: string[],
|
||||||
|
useClaude: boolean,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
|
const cmd = [
|
||||||
|
"bun",
|
||||||
|
"run",
|
||||||
|
"src/stalker-analyze.ts",
|
||||||
|
"--db",
|
||||||
|
dbPath,
|
||||||
|
"--stalker-run-id",
|
||||||
|
String(stalkerRunId),
|
||||||
|
"--analysis-run-id",
|
||||||
|
String(analysisRunId),
|
||||||
|
"--asins",
|
||||||
|
asins.join(","),
|
||||||
|
];
|
||||||
|
if (useClaude) {
|
||||||
|
cmd.push("--claude");
|
||||||
|
}
|
||||||
|
|
||||||
const child = Bun.spawn({
|
const child = Bun.spawn({
|
||||||
cmd: [
|
cmd,
|
||||||
"bun",
|
|
||||||
"run",
|
|
||||||
"src/stalker-analyze.ts",
|
|
||||||
"--db",
|
|
||||||
dbPath,
|
|
||||||
"--stalker-run-id",
|
|
||||||
String(stalkerRunId),
|
|
||||||
"--analysis-run-id",
|
|
||||||
String(analysisRunId),
|
|
||||||
"--asins",
|
|
||||||
asins.join(","),
|
|
||||||
],
|
|
||||||
stdout: "inherit",
|
stdout: "inherit",
|
||||||
stderr: "inherit",
|
stderr: "inherit",
|
||||||
});
|
});
|
||||||
@@ -1493,7 +1558,8 @@ function extractNumber(value: unknown): number | null {
|
|||||||
|
|
||||||
function extractBoolean(value: unknown): boolean | null {
|
function extractBoolean(value: unknown): boolean | null {
|
||||||
if (typeof value === "boolean") return value;
|
if (typeof value === "boolean") return value;
|
||||||
if (typeof value === "number") return value === 1 ? true : value === 0 ? false : null;
|
if (typeof value === "number")
|
||||||
|
return value === 1 ? true : value === 0 ? false : null;
|
||||||
if (typeof value !== "string") return null;
|
if (typeof value !== "string") return null;
|
||||||
const normalized = value.trim().toLowerCase();
|
const normalized = value.trim().toLowerCase();
|
||||||
if (["1", "true", "yes"].includes(normalized)) return true;
|
if (["1", "true", "yes"].includes(normalized)) return true;
|
||||||
@@ -1502,7 +1568,10 @@ function extractBoolean(value: unknown): boolean | null {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function normalizeHeader(value: string): string {
|
function normalizeHeader(value: string): string {
|
||||||
return value.toLowerCase().trim().replace(/[^a-z0-9]/g, "");
|
return value
|
||||||
|
.toLowerCase()
|
||||||
|
.trim()
|
||||||
|
.replace(/[^a-z0-9]/g, "");
|
||||||
}
|
}
|
||||||
|
|
||||||
function readFlagValue(args: string[], flag: string): string | undefined {
|
function readFlagValue(args: string[], flag: string): string | undefined {
|
||||||
@@ -1518,7 +1587,7 @@ function hasFlag(args: string[], flag: string): boolean {
|
|||||||
function printUsageAndExit(message: string): never {
|
function printUsageAndExit(message: string): never {
|
||||||
console.error(message);
|
console.error(message);
|
||||||
console.error(
|
console.error(
|
||||||
"Usage: bun run stalker --input input/asins.xlsx [--db db/results.db] [--max-asins N] [--offer-limit 100] [--seller-limit 30] [--inventory-limit 200] [--storefront-update-hours 168] [--seller-cache-hours 168] [--max-seller-requests N] [--sellability] [--analyze-sellable] [--include-stock] [--dry-run] [--no-resume]",
|
"Usage: bun run stalker --input input/asins.xlsx [--db db/results.db] [--max-asins N] [--offer-limit 100] [--seller-limit 30] [--inventory-limit 200] [--storefront-update-hours 168] [--seller-cache-hours 168] [--max-seller-requests N] [--sellability] [--analyze-sellable] [--include-stock] [--dry-run] [--no-resume] [--claude]",
|
||||||
);
|
);
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
@@ -1562,7 +1631,9 @@ function computeWaitMsFromRefill(refillIn?: number): number {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
return Math.ceil((1 / Math.max(1, refillRate)) * 60_000) + KEEP_RETRY_BUFFER_MS;
|
return (
|
||||||
|
Math.ceil((1 / Math.max(1, refillRate)) * 60_000) + KEEP_RETRY_BUFFER_MS
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function parseErrorPayload(text: string): KeepaApiResponse | null {
|
function parseErrorPayload(text: string): KeepaApiResponse | null {
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ type ParsedArgs = {
|
|||||||
categoryCandidatePool: number;
|
categoryCandidatePool: number;
|
||||||
minMonthlySold: number;
|
minMonthlySold: number;
|
||||||
blacklistFile: string;
|
blacklistFile: string;
|
||||||
|
useClaude: boolean;
|
||||||
};
|
};
|
||||||
|
|
||||||
type CategoryRunSummary = {
|
type CategoryRunSummary = {
|
||||||
@@ -76,6 +77,7 @@ function log(
|
|||||||
|
|
||||||
function parseArgs(): ParsedArgs {
|
function parseArgs(): ParsedArgs {
|
||||||
const args = process.argv.slice(2);
|
const args = process.argv.slice(2);
|
||||||
|
const useClaude = hasFlag(args, "--claude");
|
||||||
const outputDir =
|
const outputDir =
|
||||||
readFlagValue(args, "--out-dir") ?? path.join(process.cwd(), "output");
|
readFlagValue(args, "--out-dir") ?? path.join(process.cwd(), "output");
|
||||||
const blacklistFile =
|
const blacklistFile =
|
||||||
@@ -131,9 +133,14 @@ function parseArgs(): ParsedArgs {
|
|||||||
categoryCandidatePool,
|
categoryCandidatePool,
|
||||||
minMonthlySold,
|
minMonthlySold,
|
||||||
blacklistFile,
|
blacklistFile,
|
||||||
|
useClaude,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function hasFlag(args: string[], flag: string): boolean {
|
||||||
|
return args.includes(flag);
|
||||||
|
}
|
||||||
|
|
||||||
function readFlagValue(args: string[], flag: string): string | undefined {
|
function readFlagValue(args: string[], flag: string): string | undefined {
|
||||||
const idx = args.indexOf(flag);
|
const idx = args.indexOf(flag);
|
||||||
if (idx === -1) return undefined;
|
if (idx === -1) return undefined;
|
||||||
@@ -149,7 +156,7 @@ function printUsageAndExit(message: string): never {
|
|||||||
"error",
|
"error",
|
||||||
[
|
[
|
||||||
"Usage:",
|
"Usage:",
|
||||||
" bun run src/top-monthly-sold-by-category.ts [--category-limit 32] [--per-category-top 100] [--category-candidate-pool 500] [--min-monthly-sold 300] [--out-dir output] [--blacklist-file category-blacklist.csv]",
|
" bun run src/top-monthly-sold-by-category.ts [--category-limit 32] [--per-category-top 100] [--category-candidate-pool 500] [--min-monthly-sold 300] [--out-dir output] [--blacklist-file category-blacklist.csv] [--claude]",
|
||||||
"",
|
"",
|
||||||
"Flow:",
|
"Flow:",
|
||||||
" 1) Discover categories and round-robin selection.",
|
" 1) Discover categories and round-robin selection.",
|
||||||
@@ -1066,6 +1073,7 @@ export async function processCategory(
|
|||||||
perCategoryTop: number,
|
perCategoryTop: number,
|
||||||
categoryCandidatePool: number,
|
categoryCandidatePool: number,
|
||||||
minMonthlySold: number,
|
minMonthlySold: number,
|
||||||
|
useClaude = false,
|
||||||
): Promise<CategoryRunSummary> {
|
): Promise<CategoryRunSummary> {
|
||||||
log("info", `\nCategory ${category.label} (${category.id})`);
|
log("info", `\nCategory ${category.label} (${category.id})`);
|
||||||
|
|
||||||
@@ -1200,7 +1208,7 @@ export async function processCategory(
|
|||||||
|
|
||||||
let batchVerdicts: LlmVerdict[];
|
let batchVerdicts: LlmVerdict[];
|
||||||
try {
|
try {
|
||||||
batchVerdicts = await analyzeProducts(batch);
|
batchVerdicts = await analyzeProducts(batch, { useClaude });
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const message = err instanceof Error ? err.message : String(err);
|
const message = err instanceof Error ? err.message : String(err);
|
||||||
log("warn", ` LLM batch failed: ${message}`);
|
log("warn", ` LLM batch failed: ${message}`);
|
||||||
@@ -1348,6 +1356,7 @@ export async function main(): Promise<void> {
|
|||||||
args.perCategoryTop,
|
args.perCategoryTop,
|
||||||
args.categoryCandidatePool,
|
args.categoryCandidatePool,
|
||||||
args.minMonthlySold,
|
args.minMonthlySold,
|
||||||
|
args.useClaude,
|
||||||
);
|
);
|
||||||
|
|
||||||
totalInsertedAsins += categorySummary.results?.length ?? 0;
|
totalInsertedAsins += categorySummary.results?.length ?? 0;
|
||||||
|
|||||||
Reference in New Issue
Block a user