Merge branches 'main' and 'main' of https://github.com/nvictorme/asin-check
This commit is contained in:
80
.gitignore
vendored
80
.gitignore
vendored
@@ -1,37 +1,43 @@
|
||||
# dependencies (bun install)
|
||||
node_modules
|
||||
|
||||
# output
|
||||
out
|
||||
dist
|
||||
*.tgz
|
||||
|
||||
# code coverage
|
||||
coverage
|
||||
*.lcov
|
||||
|
||||
# logs
|
||||
logs
|
||||
_.log
|
||||
report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
|
||||
|
||||
# dotenv environment variable files
|
||||
.env
|
||||
.env.development.local
|
||||
.env.test.local
|
||||
.env.production.local
|
||||
.env.local
|
||||
|
||||
# caches
|
||||
.eslintcache
|
||||
.cache
|
||||
*.tsbuildinfo
|
||||
|
||||
# IntelliJ based IDEs
|
||||
.idea
|
||||
|
||||
# Finder (MacOS) folder config
|
||||
.DS_Store
|
||||
*.xlsx
|
||||
*.csv
|
||||
|
||||
# dependencies (bun install)
|
||||
node_modules
|
||||
|
||||
# output
|
||||
out
|
||||
dist
|
||||
*.tgz
|
||||
|
||||
# code coverage
|
||||
coverage
|
||||
*.lcov
|
||||
|
||||
# logs
|
||||
logs
|
||||
_.log
|
||||
report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
|
||||
|
||||
# dotenv environment variable files
|
||||
.env
|
||||
.env.development.local
|
||||
.env.test.local
|
||||
.env.production.local
|
||||
.env.local
|
||||
|
||||
# caches
|
||||
.eslintcache
|
||||
.cache
|
||||
*.tsbuildinfo
|
||||
|
||||
# IntelliJ based IDEs
|
||||
.idea
|
||||
|
||||
# Finder (MacOS) folder config
|
||||
.DS_Store
|
||||
*.xlsx
|
||||
*.csv
|
||||
|
||||
|
||||
results.db
|
||||
|
||||
results.db-shm
|
||||
|
||||
results.db-wal
|
||||
|
||||
282
README.md
282
README.md
@@ -1,135 +1,147 @@
|
||||
# asin-check
|
||||
|
||||
Amazon product analysis and lead finder agent. Reads product leads from a CSV/XLSX file, enriches them with Keepa pricing and sales data, caches results in Redis, and runs each product through a local LLM to get an FBA/FBM/SKIP verdict.
|
||||
|
||||
## Requirements
|
||||
|
||||
- [Bun](https://bun.com) runtime
|
||||
- Redis (local or Docker)
|
||||
- [LM Studio](https://lmstudio.ai) running locally with a model loaded
|
||||
- Keepa API key ([keepa.com](https://keepa.com))
|
||||
- Amazon SP-API private app credentials (LWA + refresh token + IAM)
|
||||
|
||||
## Setup
|
||||
|
||||
```bash
|
||||
bun install
|
||||
cp .env.example .env
|
||||
# Edit .env and set your KEEPA_API_KEY and SP-API credentials
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
bun run src/index.ts <input.csv|xlsx> [--out results.csv]
|
||||
```
|
||||
|
||||
Examples:
|
||||
|
||||
```bash
|
||||
bun run src/index.ts leads.xlsx
|
||||
bun run src/index.ts leads.csv --out results.xlsx
|
||||
```
|
||||
|
||||
Large-file behavior:
|
||||
|
||||
- If the input has more than 50 products, processing is done in chunks of 50.
|
||||
- Each chunk is analyzed and written to a numbered output file, for example: `results_part_001.xlsx`, `results_part_002.xlsx`, ...
|
||||
- If `--out` is omitted for large files, the base output name defaults to `<input>_results.xlsx` and chunk files are still written with numbered suffixes.
|
||||
|
||||
Quick SP-API connectivity tests:
|
||||
|
||||
```bash
|
||||
bun run src/sp-test.ts # Auth + sellers endpoint
|
||||
bun run src/sp-test.ts B07SN9BHVV # Auth + sellers endpoint + pricing offer check
|
||||
bun run src/sp-test.ts --sellability B07SN9BHVV # Standalone sellability check
|
||||
```
|
||||
|
||||
## Input file format
|
||||
|
||||
Accepts `.csv` or `.xlsx` files. Column names are matched case-insensitively. Required column:
|
||||
|
||||
| Column | Aliases |
|
||||
| ------ | ------- |
|
||||
| ASIN | — |
|
||||
|
||||
Optional but recommended:
|
||||
|
||||
| Column | Aliases |
|
||||
| --------------- | ---------------------------- |
|
||||
| Product Name | Name, Title |
|
||||
| Unit Cost | Cost, Price, Buy Cost |
|
||||
| Brand | — |
|
||||
| Category | — |
|
||||
| Amazon Rank | Amazon Rank, BSR, Sales Rank |
|
||||
| FBA NET | — |
|
||||
| Gross Profit $ | Gross Profit |
|
||||
| Gross Profit % | — |
|
||||
| MOQ | Min Order Qty |
|
||||
| MOQ Cost | — |
|
||||
| Total Qty Avail | Qty Available |
|
||||
| Link | URL, Source |
|
||||
|
||||
Lead-list format aliases (supported):
|
||||
|
||||
| Column | Aliases |
|
||||
| ----------------- | ------------------------------------------ |
|
||||
| Name | Product Name, Title, Product Title |
|
||||
| ASIN Link | ASIN URL, Amazon Link |
|
||||
| Source URL | Source Link, Supplier URL |
|
||||
| 90 Day Average | 90-day Average, Avg Price 90d, 90d Average |
|
||||
| Cost | Unit Cost, Buy Cost, Price |
|
||||
| Selling Price | Sale Price, Sell Price |
|
||||
| Net Profit | Gross Profit |
|
||||
| ROI | Gross Profit %, Return on Investment |
|
||||
| Supplier | Vendor |
|
||||
| Promo/Coupon Code | Promo Code, Coupon Code |
|
||||
| Notes | Note |
|
||||
| Date | Lead Date |
|
||||
|
||||
Numeric parsing accepts plain numbers as well as formatted values like `$12.50`, `1,209.60`, and `27.5%`.
|
||||
|
||||
## Pipeline
|
||||
|
||||
1. **Read** — parse input file, validate ASINs
|
||||
2. **Cache check** — look up each ASIN in Redis (24h TTL by default)
|
||||
3. **Sellability gate** — check all uncached ASINs against SP-API `getListingsRestrictions` (concurrency: 5 workers); immediately skip ASINs with status `not_available` and `canSell=false` (no Keepa/fees wasted)
|
||||
4. **Keepa fetch** — batch the sellable (uncached) ASINs in a single API call (up to 100 per request)
|
||||
5. **Enrich** — fetch SP-API pricing + FBA/FBM fees for sellable ASINs; combine with Keepa data and spreadsheet data
|
||||
6. **LLM analysis** — send batches of 5 available products to LM Studio for FBA/FBM/SKIP verdict
|
||||
7. **Chunk orchestration** — if input size is greater than 50, run phases 2-6 for each 50-item chunk sequentially
|
||||
8. **Output** — print results table to console (includes all ASINs); for chunked runs, always write seriated chunk files (`*_part_001`, `*_part_002`, ...); for non-chunked runs, write a single file only when `--out` is provided
|
||||
|
||||
## Output columns
|
||||
|
||||
ASIN, Name, Brand, Category, Unit Cost, Current Price, Avg Price 90d, Sales Rank, Rank Avg 90d, Sellers, Monthly Sold, Rank Drops 30d, Rank Drops 90d, FBA Net (sheet), Gross Profit $, Gross Profit %, MOQ, MOQ Cost, Qty Available, FBA Fee, FBM Fee, Referral %, Verdict, Confidence, Reasoning
|
||||
|
||||
## Environment variables
|
||||
|
||||
| Variable | Default | Description |
|
||||
| ----------------------- | -------------------------- | ----------------------------------------------------------------------- |
|
||||
| `KEEPA_API_KEY` | — | **Required.** Keepa API key |
|
||||
| `SP_API_CLIENT_ID` | — | LWA app client id from Solution Provider Portal |
|
||||
| `SP_API_CLIENT_SECRET` | — | LWA app client secret from Solution Provider Portal |
|
||||
| `SP_API_REFRESH_TOKEN` | — | Refresh token from self-authorization |
|
||||
| `SP_API_REGION` | `na` | SP-API endpoint region (`na`, `eu`, `fe`; `us` is accepted as `na`) |
|
||||
| `SP_API_MARKETPLACE_ID` | `ATVPDKIKX0DER` | Marketplace id used for pricing and fee calls (default: US) |
|
||||
| `SP_API_SELLER_ID` | — | Seller ID used for listing restrictions eligibility checks |
|
||||
| `SP_API_USE_SANDBOX` | `false` | Enable SP-API sandbox mode (`true`/`false`) |
|
||||
| `AWS_ACCESS_KEY_ID` | — | AWS credentials for SigV4 signing (required in most private app setups) |
|
||||
| `AWS_SECRET_ACCESS_KEY` | — | AWS credentials for SigV4 signing |
|
||||
| `AWS_SESSION_TOKEN` | — | Optional session token when using STS credentials |
|
||||
| `REDIS_URL` | `redis://localhost:6379` | Redis connection URL |
|
||||
| `LLM_URL` | `http://localhost:1234/v1` | LM Studio API base URL |
|
||||
| `LLM_MODEL` | `default` | Model name to pass to LM Studio |
|
||||
| `CACHE_TTL` | `86400` | Redis cache TTL in seconds |
|
||||
|
||||
## Notes
|
||||
|
||||
- **Available-only processing**: SP-API `getListingsRestrictions` is checked first and only ASINs with `sellabilityStatus=available` are enriched, analyzed, and included in outputs. Restricted, not_available, and unknown items are excluded.
|
||||
- **SP-API concurrency**: `fetchSellabilityBatch` limits concurrent requests to 5 workers to avoid 429 throttling. Pricing+fees fetches also use 5 concurrent workers.
|
||||
- **No batch endpoint**: Amazon SP-API does not provide batch endpoints for `getListingsRestrictions` or `getMyFeesEstimate*`. Concurrency limiting with the library's built-in `auto_request_throttled` safety net prevents overwhelming the API.
|
||||
- **Keepa rate limiting**: The client reads `tokensLeft` and `refillRate` from each API response and waits automatically when tokens are exhausted. With a Pro subscription (1 token/min), all 100 ASINs in a batch cost 1 token.
|
||||
- **Redis is optional**: If Redis is unavailable the tool runs without caching — every run re-fetches from Keepa.
|
||||
- **SP-API**: `src/sp-api.ts` provides `fetchSellability`, `fetchSellabilityBatch`, and `fetchSpApiPricingAndFees` functions. If SP-API credentials are missing or a call fails, the tool falls back to conservative fee defaults and keeps processing.
|
||||
- **Sandbox vs production**: When `SP_API_USE_SANDBOX=true`, production ASIN calls can be denied. Use sandbox-compatible test data or set it to `false` for live marketplace connectivity.
|
||||
# asin-check
|
||||
|
||||
Amazon product analysis and lead finder agent. Reads product leads from a CSV/XLSX file, enriches them with Keepa pricing and sales data, caches results in Redis, and runs each product through a local LLM to get an FBA/FBM/SKIP verdict.
|
||||
|
||||
## Requirements
|
||||
|
||||
- [Bun](https://bun.com) runtime
|
||||
- Redis (local or Docker)
|
||||
- [LM Studio](https://lmstudio.ai) running locally with a model loaded
|
||||
- Keepa API key ([keepa.com](https://keepa.com))
|
||||
- Amazon SP-API private app credentials (LWA + refresh token + IAM)
|
||||
|
||||
## Setup
|
||||
|
||||
```bash
|
||||
bun install
|
||||
cp .env.example .env
|
||||
# Edit .env and set your KEEPA_API_KEY and SP-API credentials
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
bun run src/index.ts <input.csv|xlsx> [--out results.csv]
|
||||
```
|
||||
|
||||
Examples:
|
||||
|
||||
```bash
|
||||
bun run src/index.ts leads.xlsx
|
||||
bun run src/index.ts leads.csv --out results.xlsx
|
||||
```
|
||||
|
||||
Large-file behavior:
|
||||
|
||||
- If the input has more than 50 products, processing is done in chunks of 50.
|
||||
- Each chunk is analyzed and written to a numbered output file, for example: `results_part_001.xlsx`, `results_part_002.xlsx`, ...
|
||||
- If `--out` is omitted for large files, the base output name defaults to `<input>_results.xlsx` and chunk files are still written with numbered suffixes.
|
||||
|
||||
Quick SP-API connectivity tests:
|
||||
|
||||
```bash
|
||||
bun run src/sp-test.ts # Auth + sellers endpoint
|
||||
bun run src/sp-test.ts B07SN9BHVV # Auth + sellers endpoint + pricing offer check
|
||||
bun run src/sp-test.ts --sellability B07SN9BHVV # Standalone sellability check
|
||||
```
|
||||
|
||||
## Input file format
|
||||
|
||||
Accepts `.csv` or `.xlsx` files. Column names are matched case-insensitively. Required column:
|
||||
|
||||
| Column | Aliases |
|
||||
| ------ | ------- |
|
||||
| ASIN | — |
|
||||
|
||||
Optional but recommended:
|
||||
|
||||
| Column | Aliases |
|
||||
| --------------- | ---------------------------- |
|
||||
| Product Name | Name, Title |
|
||||
| Unit Cost | Cost, Price, Buy Cost |
|
||||
| Brand | — |
|
||||
| Category | — |
|
||||
| Amazon Rank | Amazon Rank, BSR, Sales Rank |
|
||||
| FBA NET | — |
|
||||
| Gross Profit $ | Gross Profit |
|
||||
| Gross Profit % | — |
|
||||
| MOQ | Min Order Qty |
|
||||
| MOQ Cost | — |
|
||||
| Total Qty Avail | Qty Available |
|
||||
| Link | URL, Source |
|
||||
|
||||
Lead-list format aliases (supported):
|
||||
|
||||
| Column | Aliases |
|
||||
| ----------------- | ------------------------------------------ |
|
||||
| Name | Product Name, Title, Product Title |
|
||||
| ASIN Link | ASIN URL, Amazon Link |
|
||||
| Source URL | Source Link, Supplier URL |
|
||||
| 90 Day Average | 90-day Average, Avg Price 90d, 90d Average |
|
||||
| Cost | Unit Cost, Buy Cost, Price |
|
||||
| Selling Price | Sale Price, Sell Price |
|
||||
| Net Profit | Gross Profit |
|
||||
| ROI | Gross Profit %, Return on Investment |
|
||||
| Supplier | Vendor |
|
||||
| Promo/Coupon Code | Promo Code, Coupon Code |
|
||||
| Notes | Note |
|
||||
| Date | Lead Date |
|
||||
|
||||
Numeric parsing accepts plain numbers as well as formatted values like `$12.50`, `1,209.60`, and `27.5%`.
|
||||
|
||||
## Pipeline
|
||||
|
||||
1. **Read** — parse input file, validate ASINs
|
||||
2. **Cache check** — look up each ASIN in Redis (24h TTL by default)
|
||||
3. **Sellability gate** — check all uncached ASINs against SP-API `getListingsRestrictions` (concurrency: 5 workers); immediately skip ASINs with status `not_available` and `canSell=false` (no Keepa/fees wasted)
|
||||
4. **Keepa fetch** — batch the sellable (uncached) ASINs in a single API call (up to 100 per request)
|
||||
5. **Enrich** — fetch SP-API pricing + FBA/FBM fees for sellable ASINs; combine with Keepa data and spreadsheet data
|
||||
6. **LLM analysis** — send batches of 5 sellable products to LM Studio for FBA/FBM/SKIP verdict; skipped ASINs get auto-SKIP verdict (confidence 100) and bypass LLM entirely
|
||||
7. **Output** — print results table to console (includes all ASINs), optionally write CSV/XLSX, and **persist results to a SQLite database**.
|
||||
|
||||
## Persistent Storage with SQLite
|
||||
|
||||
Results from each run are now stored in a SQLite database named `results.db` in the project root. The SQLite implementation details are handled in `src/database.ts`. This allows you to:
|
||||
|
||||
- Revisit past analysis results.
|
||||
- Query and analyze historical data.
|
||||
- Track product performance over time.
|
||||
|
||||
The database will automatically be created if it doesn't exist. Two tables are created:
|
||||
|
||||
- `runs`: Stores metadata about each analysis run (timestamp, input file, output file, and summary counts).
|
||||
- `results`: Stores detailed analysis results for each product from each run, linked to the `runs` table.
|
||||
|
||||
## Output columns
|
||||
|
||||
ASIN, Name, Brand, Category, Unit Cost, Current Price, Avg Price 90d, Sales Rank, Rank Avg 90d, Sellers, Monthly Sold, Rank Drops 30d, Rank Drops 90d, FBA Net (sheet), Gross Profit $, Gross Profit %, MOQ, MOQ Cost, Qty Available, FBA Fee, FBM Fee, Referral %, Verdict, Confidence, Reasoning
|
||||
|
||||
## Environment variables
|
||||
|
||||
| Variable | Default | Description |
|
||||
| ----------------------- | -------------------------- | ----------------------------------------------------------------------- |
|
||||
| `KEEPA_API_KEY` | — | **Required.** Keepa API key |
|
||||
| `SP_API_CLIENT_ID` | — | LWA app client id from Solution Provider Portal |
|
||||
| `SP_API_CLIENT_SECRET` | — | LWA app client secret from Solution Provider Portal |
|
||||
| `SP_API_REFRESH_TOKEN` | — | Refresh token from self-authorization |
|
||||
| `SP_API_REGION` | `na` | SP-API endpoint region (`na`, `eu`, `fe`; `us` is accepted as `na`) |
|
||||
| `SP_API_MARKETPLACE_ID` | `ATVPDKIKX0DER` | Marketplace id used for pricing and fee calls (default: US) |
|
||||
| `SP_API_SELLER_ID` | — | Seller ID used for listing restrictions eligibility checks |
|
||||
| `SP_API_USE_SANDBOX` | `false` | Enable SP-API sandbox mode (`true`/`false`) |
|
||||
| `AWS_ACCESS_KEY_ID` | — | AWS credentials for SigV4 signing (required in most private app setups) |
|
||||
| `AWS_SECRET_ACCESS_KEY` | — | AWS credentials for SigV4 signing |
|
||||
| `AWS_SESSION_TOKEN` | — | Optional session token when using STS credentials |
|
||||
| `REDIS_URL` | `redis://localhost:6379` | Redis connection URL |
|
||||
| `LLM_URL` | `http://localhost:1234/v1` | LM Studio API base URL |
|
||||
| `LLM_MODEL` | `default` | Model name to pass to LM Studio |
|
||||
| `CACHE_TTL` | `86400` | Redis cache TTL in seconds |
|
||||
|
||||
## Notes
|
||||
|
||||
- **Available-only processing**: SP-API `getListingsRestrictions` is checked first and only ASINs with `sellabilityStatus=available` are enriched, analyzed, and included in outputs. Restricted, not_available, and unknown items are excluded.
|
||||
- **SP-API concurrency**: `fetchSellabilityBatch` limits concurrent requests to 5 workers to avoid 429 throttling. Pricing+fees fetches also use 5 concurrent workers.
|
||||
- **No batch endpoint**: Amazon SP-API does not provide batch endpoints for `getListingsRestrictions` or `getMyFeesEstimate*`. Concurrency limiting with the library's built-in `auto_request_throttled` safety net prevents overwhelming the API.
|
||||
- **Keepa rate limiting**: The client reads `tokensLeft` and `refillRate` from each API response and waits automatically when tokens are exhausted. With a Pro subscription (1 token/min), all 100 ASINs in a batch cost 1 token.
|
||||
- **Redis is optional**: If Redis is unavailable the tool runs without caching — every run re-fetches from Keepa.
|
||||
- **SP-API**: `src/sp-api.ts` provides `fetchSellability`, `fetchSellabilityBatch`, and `fetchSpApiPricingAndFees` functions. If SP-API credentials are missing or a call fails, the tool falls back to conservative fee defaults and keeps processing.
|
||||
- **Sandbox vs production**: When `SP_API_USE_SANDBOX=true`, production ASIN calls can be denied. Use sandbox-compatible test data or set it to `false` for live marketplace connectivity.
|
||||
|
||||
80
src/database.ts
Normal file
80
src/database.ts
Normal file
@@ -0,0 +1,80 @@
|
||||
import { Database } from "bun:sqlite";
|
||||
|
||||
let db: Database | null = null;
|
||||
|
||||
export function getDb(dbPath: string): Database {
|
||||
if (!db) {
|
||||
db = new Database(dbPath);
|
||||
db.run("PRAGMA journal_mode = WAL;"); // Enable WAL mode for better performance
|
||||
}
|
||||
return db;
|
||||
}
|
||||
|
||||
export function closeDb(): void {
|
||||
if (db) {
|
||||
db.close();
|
||||
db = null;
|
||||
}
|
||||
}
|
||||
|
||||
export function initDb(dbPath: string): void {
|
||||
const database = getDb(dbPath);
|
||||
database.run(`
|
||||
CREATE TABLE IF NOT EXISTS runs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
timestamp TEXT NOT NULL,
|
||||
input_file TEXT NOT NULL,
|
||||
output_file TEXT,
|
||||
total_products INTEGER,
|
||||
fba_count INTEGER,
|
||||
fbm_count INTEGER,
|
||||
skip_count INTEGER
|
||||
);
|
||||
`);
|
||||
database.run(`
|
||||
CREATE TABLE IF NOT EXISTS results (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
run_id INTEGER NOT NULL,
|
||||
asin TEXT NOT NULL,
|
||||
product_name TEXT,
|
||||
brand TEXT,
|
||||
category TEXT,
|
||||
unit_cost REAL,
|
||||
current_price REAL,
|
||||
avg_price_90d REAL,
|
||||
avg_price_90d_sheet REAL,
|
||||
selling_price_sheet REAL,
|
||||
sales_rank INTEGER,
|
||||
rank_avg_90d INTEGER,
|
||||
sellers INTEGER,
|
||||
monthly_sold INTEGER,
|
||||
rank_drops_30d INTEGER,
|
||||
rank_drops_90d INTEGER,
|
||||
fba_net_sheet REAL,
|
||||
gross_profit_dollar REAL,
|
||||
gross_profit_pct REAL,
|
||||
net_profit_sheet REAL,
|
||||
roi_sheet REAL,
|
||||
moq INTEGER,
|
||||
moq_cost REAL,
|
||||
qty_available INTEGER,
|
||||
supplier TEXT,
|
||||
source_url TEXT,
|
||||
asin_link TEXT,
|
||||
promo_coupon_code TEXT,
|
||||
notes TEXT,
|
||||
lead_date TEXT,
|
||||
fba_fee REAL,
|
||||
fbm_fee REAL,
|
||||
referral_percent REAL,
|
||||
can_sell TEXT,
|
||||
sellability_status TEXT,
|
||||
sellability_reason TEXT,
|
||||
verdict TEXT NOT NULL,
|
||||
confidence INTEGER,
|
||||
reasoning TEXT,
|
||||
fetched_at TEXT NOT NULL,
|
||||
FOREIGN KEY (run_id) REFERENCES runs(id)
|
||||
);
|
||||
`);
|
||||
}
|
||||
64
src/index.ts
64
src/index.ts
@@ -3,8 +3,10 @@ import { fetchKeepaDataBatch } from "./keepa.ts";
|
||||
import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts";
|
||||
import { connectCache, getCache, setCache, disconnectCache } from "./cache.ts";
|
||||
import { analyzeProducts } from "./llm.ts";
|
||||
import { printResults, writeResultsCsv } from "./writer.ts";
|
||||
import path from "node:path";
|
||||
import { printResults, writeResultsToDb } from "./writer.ts";
|
||||
import { initDb, closeDb } from "./database.ts";
|
||||
|
||||
const DB_PATH = "./results.db";
|
||||
import type {
|
||||
EnrichedProduct,
|
||||
AnalysisResult,
|
||||
@@ -32,38 +34,26 @@ function parseArgs(): { inputFile: string; outputFile?: string } {
|
||||
return { inputFile, outputFile };
|
||||
}
|
||||
|
||||
function chunkArray<T>(items: T[], chunkSize: number): T[][] {
|
||||
const chunks: T[][] = [];
|
||||
for (let i = 0; i < items.length; i += chunkSize) {
|
||||
chunks.push(items.slice(i, i + chunkSize));
|
||||
async function main() {
|
||||
const { inputFile, outputFile } = parseArgs();
|
||||
|
||||
console.log("Connecting to Redis...");
|
||||
await connectCache();
|
||||
|
||||
// Initialize SQLite DB
|
||||
console.log("Initializing SQLite database...");
|
||||
initDb(DB_PATH);
|
||||
|
||||
// Phase 1: Read input file
|
||||
console.log(`\nReading ${inputFile}...`);
|
||||
const products = readProducts(inputFile);
|
||||
|
||||
if (products.length === 0) {
|
||||
console.error("No valid products found in input file.");
|
||||
process.exit(1);
|
||||
}
|
||||
return chunks;
|
||||
}
|
||||
|
||||
function resolveBaseOutputPath(inputFile: string, outputFile?: string): string {
|
||||
if (outputFile) return outputFile;
|
||||
|
||||
const parsedInput = path.parse(inputFile);
|
||||
return path.join(parsedInput.dir, `${parsedInput.name}_results.xlsx`);
|
||||
}
|
||||
|
||||
function buildChunkOutputPath(
|
||||
baseOutputPath: string,
|
||||
chunkIndex: number,
|
||||
): string {
|
||||
const parsed = path.parse(baseOutputPath);
|
||||
const extension = parsed.ext || ".xlsx";
|
||||
const chunkSuffix = String(chunkIndex + 1).padStart(3, "0");
|
||||
return path.join(
|
||||
parsed.dir,
|
||||
`${parsed.name}_part_${chunkSuffix}${extension}`,
|
||||
);
|
||||
}
|
||||
|
||||
async function processProductChunk(
|
||||
products: ProductRecord[],
|
||||
): Promise<AnalysisResult[]> {
|
||||
// Phase 2: Check cache for all ASINs in chunk
|
||||
// Phase 2: Check cache for all ASINs
|
||||
console.log(`\nChecking cache for ${products.length} products...`);
|
||||
const cached = new Map<string, EnrichedProduct>();
|
||||
const excludedCachedAsins = new Set<string>();
|
||||
@@ -331,12 +321,10 @@ async function main() {
|
||||
|
||||
printResults(allResults);
|
||||
|
||||
if (!hasMultipleChunks && outputFile) {
|
||||
writeResultsCsv(allResults, outputFile);
|
||||
}
|
||||
} finally {
|
||||
await disconnectCache();
|
||||
}
|
||||
writeResultsToDb(allResults, DB_PATH, inputFile, outputFile);
|
||||
|
||||
await disconnectCache();
|
||||
closeDb();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
|
||||
413
src/writer.ts
413
src/writer.ts
@@ -1,159 +1,254 @@
|
||||
import * as XLSX from "xlsx";
|
||||
import type { AnalysisResult } from "./types.ts";
|
||||
|
||||
function buildRow(r: AnalysisResult) {
|
||||
const price =
|
||||
r.product.keepa?.currentPrice ??
|
||||
r.product.record.sellingPriceFromSheet ??
|
||||
r.product.spApi.estimatedSalePrice;
|
||||
const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank;
|
||||
|
||||
return {
|
||||
ASIN: r.product.record.asin,
|
||||
Name: r.product.record.name,
|
||||
Brand: r.product.record.brand ?? "",
|
||||
Category:
|
||||
r.product.record.category ??
|
||||
r.product.keepa?.categoryTree?.join(" > ") ??
|
||||
"",
|
||||
"Unit Cost": r.product.record.unitCost,
|
||||
"Current Price": price ?? "",
|
||||
"Avg Price 90d": r.product.keepa?.avgPrice90 ?? "",
|
||||
"Avg Price 90d (sheet)": r.product.record.avgPrice90FromSheet ?? "",
|
||||
"Selling Price (sheet)": r.product.record.sellingPriceFromSheet ?? "",
|
||||
"Sales Rank": rank ?? "",
|
||||
"Rank Avg 90d": r.product.keepa?.salesRankAvg90 ?? "",
|
||||
Sellers: r.product.keepa?.sellerCount ?? "",
|
||||
"Monthly Sold": r.product.keepa?.monthlySold ?? "",
|
||||
"Rank Drops 30d": r.product.keepa?.salesRankDrops30 ?? "",
|
||||
"Rank Drops 90d": r.product.keepa?.salesRankDrops90 ?? "",
|
||||
"FBA Net (sheet)": r.product.record.fbaNet ?? "",
|
||||
"Gross Profit $": r.product.record.grossProfit ?? "",
|
||||
"Gross Profit %": r.product.record.grossProfitPct ?? "",
|
||||
"Net Profit (sheet)": r.product.record.netProfitFromSheet ?? "",
|
||||
"ROI (sheet)": r.product.record.roiFromSheet ?? "",
|
||||
MOQ: r.product.record.moq ?? "",
|
||||
"MOQ Cost": r.product.record.moqCost ?? "",
|
||||
"Qty Available": r.product.record.totalQtyAvail ?? "",
|
||||
Supplier: r.product.record.supplier ?? "",
|
||||
"Source URL": r.product.record.sourceUrl ?? "",
|
||||
"ASIN Link": r.product.record.asinLink ?? "",
|
||||
"Promo/Coupon Code": r.product.record.promoCouponCode ?? "",
|
||||
Notes: r.product.record.notes ?? "",
|
||||
"Lead Date": r.product.record.leadDate ?? "",
|
||||
"FBA Fee": r.product.spApi.fbaFee,
|
||||
"FBM Fee": r.product.spApi.fbmFee,
|
||||
"Referral %": r.product.spApi.referralFeePercent,
|
||||
"Can Sell":
|
||||
r.product.spApi.canSell == null
|
||||
? "unknown"
|
||||
: r.product.spApi.canSell
|
||||
? "yes"
|
||||
: "no",
|
||||
Sellability: r.product.spApi.sellabilityStatus,
|
||||
"Sellability Reason": r.product.spApi.sellabilityReason ?? "",
|
||||
Verdict: r.verdict.verdict,
|
||||
Confidence: r.verdict.confidence,
|
||||
Reasoning: r.verdict.reasoning,
|
||||
};
|
||||
}
|
||||
|
||||
export function printResults(results: AnalysisResult[]): void {
|
||||
const rows = results
|
||||
.filter((r) => r.verdict.verdict === "FBA" || r.verdict.verdict === "FBM")
|
||||
.map((r) => {
|
||||
const sellingPrice =
|
||||
r.product.keepa?.currentPrice ??
|
||||
r.product.record.sellingPriceFromSheet ??
|
||||
r.product.spApi.estimatedSalePrice;
|
||||
const referralFee =
|
||||
sellingPrice != null
|
||||
? sellingPrice * (r.product.spApi.referralFeePercent / 100)
|
||||
: null;
|
||||
const fulfillmentFee =
|
||||
r.verdict.verdict === "FBA"
|
||||
? r.product.spApi.fbaFee
|
||||
: r.product.spApi.fbmFee;
|
||||
const netProfit =
|
||||
sellingPrice != null
|
||||
? Math.round(
|
||||
(sellingPrice -
|
||||
r.product.record.unitCost -
|
||||
fulfillmentFee -
|
||||
(referralFee ?? 0)) *
|
||||
100,
|
||||
) / 100
|
||||
: "";
|
||||
|
||||
return {
|
||||
ASIN: r.product.record.asin,
|
||||
Name: r.product.record.name.slice(0, 40),
|
||||
Category: String(
|
||||
r.product.record.category ??
|
||||
r.product.keepa?.categoryTree?.join(" > ") ??
|
||||
"",
|
||||
).slice(0, 20),
|
||||
"Unit Cost": r.product.record.unitCost,
|
||||
"Selling Price": sellingPrice ?? "",
|
||||
"Net Profit": netProfit,
|
||||
"Monthly Sold": r.product.keepa?.monthlySold ?? "",
|
||||
"Sold 90 Day": r.product.keepa?.salesRankDrops90 ?? "",
|
||||
"Can Sell":
|
||||
r.product.spApi.canSell == null
|
||||
? "unknown"
|
||||
: r.product.spApi.canSell
|
||||
? "yes"
|
||||
: "no",
|
||||
Sellability: r.product.spApi.sellabilityStatus,
|
||||
"Sellability Reason": String(
|
||||
r.product.spApi.sellabilityReason ?? "",
|
||||
).slice(0, 60),
|
||||
Confidence: r.verdict.confidence,
|
||||
Reasoning: r.verdict.reasoning.slice(0, 60),
|
||||
};
|
||||
});
|
||||
|
||||
console.log("\n=== Analysis Results ===\n");
|
||||
if (rows.length === 0) {
|
||||
console.log("No FBA/FBM leads found.");
|
||||
} else {
|
||||
console.table(rows);
|
||||
}
|
||||
|
||||
const summary = {
|
||||
FBA: results.filter((r) => r.verdict.verdict === "FBA").length,
|
||||
FBM: results.filter((r) => r.verdict.verdict === "FBM").length,
|
||||
SKIP: results.filter((r) => r.verdict.verdict === "SKIP").length,
|
||||
Available: results.filter(
|
||||
(r) => r.product.spApi.sellabilityStatus === "available",
|
||||
).length,
|
||||
Restricted: results.filter(
|
||||
(r) => r.product.spApi.sellabilityStatus === "restricted",
|
||||
).length,
|
||||
NotAvailable: results.filter(
|
||||
(r) => r.product.spApi.sellabilityStatus === "not_available",
|
||||
).length,
|
||||
Unknown: results.filter(
|
||||
(r) => r.product.spApi.sellabilityStatus === "unknown",
|
||||
).length,
|
||||
};
|
||||
console.log(
|
||||
`\nSummary: ${summary.FBA} FBA | ${summary.FBM} FBM | ${summary.SKIP} SKIP out of ${results.length} products\n`,
|
||||
);
|
||||
console.log(
|
||||
`Sellability: ${summary.Available} available | ${summary.Restricted} restricted | ${summary.NotAvailable} not_available | ${summary.Unknown} unknown\n`,
|
||||
);
|
||||
}
|
||||
|
||||
export function writeResultsCsv(
|
||||
results: AnalysisResult[],
|
||||
outputPath: string,
|
||||
): void {
|
||||
const rows = results.map(buildRow);
|
||||
|
||||
const ws = XLSX.utils.json_to_sheet(rows);
|
||||
const wb = XLSX.utils.book_new();
|
||||
XLSX.utils.book_append_sheet(wb, ws, "Results");
|
||||
XLSX.writeFile(wb, outputPath);
|
||||
console.log(`Results written to ${outputPath}`);
|
||||
}
|
||||
import { getDb } from "./database.ts";
|
||||
import type { AnalysisResult } from "./types.ts";
|
||||
|
||||
function buildRow(r: AnalysisResult) {
|
||||
const price =
|
||||
r.product.keepa?.currentPrice ??
|
||||
r.product.record.sellingPriceFromSheet ??
|
||||
r.product.spApi.estimatedSalePrice;
|
||||
const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank;
|
||||
const canSellStatus =
|
||||
r.product.spApi.canSell == null
|
||||
? "unknown"
|
||||
: r.product.spApi.canSell
|
||||
? "yes"
|
||||
: "no";
|
||||
|
||||
return {
|
||||
ASIN: r.product.record.asin,
|
||||
Name: r.product.record.name,
|
||||
Brand: r.product.record.brand ?? "",
|
||||
Category:
|
||||
r.product.record.category ??
|
||||
r.product.keepa?.categoryTree?.join(" > ") ??
|
||||
"",
|
||||
"Unit Cost": r.product.record.unitCost,
|
||||
"Current Price": price ?? "",
|
||||
"Avg Price 90d": r.product.keepa?.avgPrice90 ?? "",
|
||||
"Avg Price 90d (sheet)": r.product.record.avgPrice90FromSheet ?? "",
|
||||
"Selling Price (sheet)": r.product.record.sellingPriceFromSheet ?? "",
|
||||
"Sales Rank": rank ?? "",
|
||||
"Rank Avg 90d": r.product.keepa?.salesRankAvg90 ?? "",
|
||||
Sellers: r.product.keepa?.sellerCount ?? "",
|
||||
"Monthly Sold": r.product.keepa?.monthlySold ?? "",
|
||||
"Rank Drops 30d": r.product.keepa?.salesRankDrops30 ?? "",
|
||||
"Rank Drops 90d": r.product.keepa?.salesRankDrops90 ?? "",
|
||||
"FBA Net (sheet)": r.product.record.fbaNet ?? "",
|
||||
"Gross Profit $": r.product.record.grossProfit ?? "",
|
||||
"Gross Profit %": r.product.record.grossProfitPct ?? "",
|
||||
"Net Profit (sheet)": r.product.record.netProfitFromSheet ?? "",
|
||||
"ROI (sheet)": r.product.record.roiFromSheet ?? "",
|
||||
MOQ: r.product.record.moq ?? "",
|
||||
"MOQ Cost": r.product.record.moqCost ?? "",
|
||||
"Qty Available": r.product.record.totalQtyAvail ?? "",
|
||||
Supplier: r.product.record.supplier ?? "",
|
||||
"Source URL": r.product.record.sourceUrl ?? "",
|
||||
"ASIN Link": r.product.record.asinLink ?? "",
|
||||
"Promo/Coupon Code": r.product.record.promoCouponCode ?? "",
|
||||
Notes: r.product.record.notes ?? "",
|
||||
"Lead Date": r.product.record.leadDate ?? "",
|
||||
"FBA Fee": r.product.spApi.fbaFee,
|
||||
"FBM Fee": r.product.spApi.fbmFee,
|
||||
"Referral %": r.product.spApi.referralFeePercent,
|
||||
"Can Sell": canSellStatus,
|
||||
Sellability: r.product.spApi.sellabilityStatus,
|
||||
"Sellability Reason": r.product.spApi.sellabilityReason ?? "",
|
||||
Verdict: r.verdict.verdict,
|
||||
Confidence: r.verdict.confidence,
|
||||
Reasoning: r.verdict.reasoning,
|
||||
};
|
||||
}
|
||||
|
||||
export function writeResultsToDb(
|
||||
results: AnalysisResult[],
|
||||
dbPath: string,
|
||||
inputFile: string,
|
||||
outputFile: string | undefined,
|
||||
): void {
|
||||
const database = getDb(dbPath);
|
||||
|
||||
const timestamp = new Date().toISOString();
|
||||
const fbaCount = results.filter((r) => r.verdict.verdict === "FBA").length;
|
||||
const fbmCount = results.filter((r) => r.verdict.verdict === "FBM").length;
|
||||
const skipCount = results.filter((r) => r.verdict.verdict === "SKIP").length;
|
||||
|
||||
const insertRun = database.prepare(
|
||||
`INSERT INTO runs (
|
||||
timestamp,
|
||||
input_file,
|
||||
output_file,
|
||||
total_products,
|
||||
fba_count,
|
||||
fbm_count,
|
||||
skip_count
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
||||
);
|
||||
const runInfo = insertRun.run(
|
||||
timestamp,
|
||||
inputFile,
|
||||
outputFile ?? null,
|
||||
results.length,
|
||||
fbaCount,
|
||||
fbmCount,
|
||||
skipCount,
|
||||
);
|
||||
const runId =
|
||||
(runInfo.changes as number) > 0
|
||||
? (runInfo.lastInsertRowid as number)
|
||||
: null;
|
||||
|
||||
if (runId === null) {
|
||||
console.error("Failed to insert run record into SQLite.");
|
||||
return;
|
||||
}
|
||||
|
||||
const insertResult = database.prepare(
|
||||
`INSERT INTO results (
|
||||
run_id, asin, product_name, brand, category, unit_cost, current_price,
|
||||
avg_price_90d, avg_price_90d_sheet, selling_price_sheet, sales_rank, rank_avg_90d,
|
||||
sellers, monthly_sold, rank_drops_30d, rank_drops_90d, fba_net_sheet,
|
||||
gross_profit_dollar, gross_profit_pct, net_profit_sheet, roi_sheet, moq, moq_cost,
|
||||
qty_available, supplier, source_url, asin_link, promo_coupon_code, notes, lead_date,
|
||||
fba_fee, fbm_fee, referral_percent, can_sell, sellability_status, sellability_reason,
|
||||
verdict, confidence, reasoning, fetched_at
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
|
||||
)`,
|
||||
);
|
||||
|
||||
database.transaction(() => {
|
||||
for (const r of results) {
|
||||
const row = buildRow(r);
|
||||
insertResult.run(
|
||||
runId,
|
||||
row.ASIN,
|
||||
row.Name,
|
||||
row.Brand,
|
||||
row.Category,
|
||||
row["Unit Cost"] ?? null,
|
||||
row["Current Price"] ?? null,
|
||||
row["Avg Price 90d"] ?? null,
|
||||
row["Avg Price 90d (sheet)"] ?? null,
|
||||
row["Selling Price (sheet)"] ?? null,
|
||||
row["Sales Rank"] ?? null,
|
||||
row["Rank Avg 90d"] ?? null,
|
||||
row.Sellers ?? null,
|
||||
row["Monthly Sold"] ?? null,
|
||||
row["Rank Drops 30d"] ?? null,
|
||||
row["Rank Drops 90d"] ?? null,
|
||||
row["FBA Net (sheet)"] ?? null,
|
||||
row["Gross Profit $"] ?? null,
|
||||
row["Gross Profit %"] ?? null,
|
||||
row["Net Profit (sheet)"] ?? null,
|
||||
row["ROI (sheet)"] ?? null,
|
||||
row.MOQ ?? null,
|
||||
row["MOQ Cost"] ?? null,
|
||||
row["Qty Available"] ?? null,
|
||||
row.Supplier ?? null,
|
||||
row["Source URL"] ?? null,
|
||||
row["ASIN Link"] ?? null,
|
||||
row["Promo/Coupon Code"] ?? null,
|
||||
row.Notes ?? null,
|
||||
row["Lead Date"] ?? null,
|
||||
row["FBA Fee"] ?? null,
|
||||
row["FBM Fee"] ?? null,
|
||||
row["Referral %"] ?? null,
|
||||
row["Can Sell"],
|
||||
row.Sellability,
|
||||
row["Sellability Reason"] ?? null,
|
||||
row.Verdict,
|
||||
row.Confidence ?? null,
|
||||
row.Reasoning,
|
||||
r.product.fetchedAt,
|
||||
);
|
||||
}
|
||||
})();
|
||||
console.log(`Results written to SQLite database for run_id: ${runId}`);
|
||||
}
|
||||
export function printResults(results: AnalysisResult[]): void {
|
||||
const rows = results
|
||||
.filter((r) => r.verdict.verdict === "FBA" || r.verdict.verdict === "FBM")
|
||||
.map((r) => {
|
||||
const sellingPrice =
|
||||
r.product.keepa?.currentPrice ??
|
||||
r.product.record.sellingPriceFromSheet ??
|
||||
r.product.spApi.estimatedSalePrice;
|
||||
const referralFee =
|
||||
sellingPrice != null
|
||||
? sellingPrice * (r.product.spApi.referralFeePercent / 100)
|
||||
: null;
|
||||
const fulfillmentFee =
|
||||
r.verdict.verdict === "FBA"
|
||||
? r.product.spApi.fbaFee
|
||||
: r.product.spApi.fbmFee;
|
||||
const netProfit =
|
||||
sellingPrice != null
|
||||
? Math.round(
|
||||
(sellingPrice -
|
||||
r.product.record.unitCost -
|
||||
fulfillmentFee -
|
||||
(referralFee ?? 0)) *
|
||||
100,
|
||||
) / 100
|
||||
: "";
|
||||
|
||||
return {
|
||||
ASIN: r.product.record.asin,
|
||||
Name: r.product.record.name.slice(0, 40),
|
||||
Category: String(
|
||||
r.product.record.category ??
|
||||
r.product.keepa?.categoryTree?.join(" > ") ??
|
||||
"",
|
||||
).slice(0, 20),
|
||||
"Unit Cost": r.product.record.unitCost,
|
||||
"Selling Price": sellingPrice ?? "",
|
||||
"Net Profit": netProfit,
|
||||
"Monthly Sold": r.product.keepa?.monthlySold ?? "",
|
||||
"Sold 90 Day": r.product.keepa?.salesRankDrops90 ?? "",
|
||||
"Can Sell":
|
||||
r.product.spApi.canSell == null
|
||||
? "unknown"
|
||||
: r.product.spApi.canSell
|
||||
? "yes"
|
||||
: "no",
|
||||
Sellability: r.product.spApi.sellabilityStatus,
|
||||
"Sellability Reason": String(
|
||||
r.product.spApi.sellabilityReason ?? "",
|
||||
).slice(0, 60),
|
||||
Confidence: r.verdict.confidence,
|
||||
Reasoning: r.verdict.reasoning.slice(0, 60),
|
||||
};
|
||||
});
|
||||
|
||||
console.log("\n=== Analysis Results ===\n");
|
||||
if (rows.length === 0) {
|
||||
console.log("No FBA/FBM leads found.");
|
||||
} else {
|
||||
console.table(rows);
|
||||
}
|
||||
|
||||
const summary = {
|
||||
FBA: results.filter((r) => r.verdict.verdict === "FBA").length,
|
||||
FBM: results.filter((r) => r.verdict.verdict === "FBM").length,
|
||||
SKIP: results.filter((r) => r.verdict.verdict === "SKIP").length,
|
||||
Available: results.filter(
|
||||
(r) => r.product.spApi.sellabilityStatus === "available",
|
||||
).length,
|
||||
Restricted: results.filter(
|
||||
(r) => r.product.spApi.sellabilityStatus === "restricted",
|
||||
).length,
|
||||
NotAvailable: results.filter(
|
||||
(r) => r.product.spApi.sellabilityStatus === "not_available",
|
||||
).length,
|
||||
Unknown: results.filter(
|
||||
(r) => r.product.spApi.sellabilityStatus === "unknown",
|
||||
).length,
|
||||
};
|
||||
console.log(
|
||||
`\nSummary: ${summary.FBA} FBA | ${summary.FBM} FBM | ${summary.SKIP} SKIP out of ${results.length} products\n`,
|
||||
);
|
||||
console.log(
|
||||
`Sellability: ${summary.Available} available | ${summary.Restricted} restricted | ${summary.NotAvailable} not_available | ${summary.Unknown} unknown\n`,
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user