Merge branches 'main' and 'main' of https://github.com/nvictorme/asin-check
This commit is contained in:
80
.gitignore
vendored
80
.gitignore
vendored
@@ -1,37 +1,43 @@
|
|||||||
# dependencies (bun install)
|
# dependencies (bun install)
|
||||||
node_modules
|
node_modules
|
||||||
|
|
||||||
# output
|
# output
|
||||||
out
|
out
|
||||||
dist
|
dist
|
||||||
*.tgz
|
*.tgz
|
||||||
|
|
||||||
# code coverage
|
# code coverage
|
||||||
coverage
|
coverage
|
||||||
*.lcov
|
*.lcov
|
||||||
|
|
||||||
# logs
|
# logs
|
||||||
logs
|
logs
|
||||||
_.log
|
_.log
|
||||||
report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
|
report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
|
||||||
|
|
||||||
# dotenv environment variable files
|
# dotenv environment variable files
|
||||||
.env
|
.env
|
||||||
.env.development.local
|
.env.development.local
|
||||||
.env.test.local
|
.env.test.local
|
||||||
.env.production.local
|
.env.production.local
|
||||||
.env.local
|
.env.local
|
||||||
|
|
||||||
# caches
|
# caches
|
||||||
.eslintcache
|
.eslintcache
|
||||||
.cache
|
.cache
|
||||||
*.tsbuildinfo
|
*.tsbuildinfo
|
||||||
|
|
||||||
# IntelliJ based IDEs
|
# IntelliJ based IDEs
|
||||||
.idea
|
.idea
|
||||||
|
|
||||||
# Finder (MacOS) folder config
|
# Finder (MacOS) folder config
|
||||||
.DS_Store
|
.DS_Store
|
||||||
*.xlsx
|
*.xlsx
|
||||||
*.csv
|
*.csv
|
||||||
|
|
||||||
|
|
||||||
|
results.db
|
||||||
|
|
||||||
|
results.db-shm
|
||||||
|
|
||||||
|
results.db-wal
|
||||||
|
|||||||
282
README.md
282
README.md
@@ -1,135 +1,147 @@
|
|||||||
# asin-check
|
# asin-check
|
||||||
|
|
||||||
Amazon product analysis and lead finder agent. Reads product leads from a CSV/XLSX file, enriches them with Keepa pricing and sales data, caches results in Redis, and runs each product through a local LLM to get an FBA/FBM/SKIP verdict.
|
Amazon product analysis and lead finder agent. Reads product leads from a CSV/XLSX file, enriches them with Keepa pricing and sales data, caches results in Redis, and runs each product through a local LLM to get an FBA/FBM/SKIP verdict.
|
||||||
|
|
||||||
## Requirements
|
## Requirements
|
||||||
|
|
||||||
- [Bun](https://bun.com) runtime
|
- [Bun](https://bun.com) runtime
|
||||||
- Redis (local or Docker)
|
- Redis (local or Docker)
|
||||||
- [LM Studio](https://lmstudio.ai) running locally with a model loaded
|
- [LM Studio](https://lmstudio.ai) running locally with a model loaded
|
||||||
- Keepa API key ([keepa.com](https://keepa.com))
|
- Keepa API key ([keepa.com](https://keepa.com))
|
||||||
- Amazon SP-API private app credentials (LWA + refresh token + IAM)
|
- Amazon SP-API private app credentials (LWA + refresh token + IAM)
|
||||||
|
|
||||||
## Setup
|
## Setup
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
bun install
|
bun install
|
||||||
cp .env.example .env
|
cp .env.example .env
|
||||||
# Edit .env and set your KEEPA_API_KEY and SP-API credentials
|
# Edit .env and set your KEEPA_API_KEY and SP-API credentials
|
||||||
```
|
```
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
bun run src/index.ts <input.csv|xlsx> [--out results.csv]
|
bun run src/index.ts <input.csv|xlsx> [--out results.csv]
|
||||||
```
|
```
|
||||||
|
|
||||||
Examples:
|
Examples:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
bun run src/index.ts leads.xlsx
|
bun run src/index.ts leads.xlsx
|
||||||
bun run src/index.ts leads.csv --out results.xlsx
|
bun run src/index.ts leads.csv --out results.xlsx
|
||||||
```
|
```
|
||||||
|
|
||||||
Large-file behavior:
|
Large-file behavior:
|
||||||
|
|
||||||
- If the input has more than 50 products, processing is done in chunks of 50.
|
- If the input has more than 50 products, processing is done in chunks of 50.
|
||||||
- Each chunk is analyzed and written to a numbered output file, for example: `results_part_001.xlsx`, `results_part_002.xlsx`, ...
|
- Each chunk is analyzed and written to a numbered output file, for example: `results_part_001.xlsx`, `results_part_002.xlsx`, ...
|
||||||
- If `--out` is omitted for large files, the base output name defaults to `<input>_results.xlsx` and chunk files are still written with numbered suffixes.
|
- If `--out` is omitted for large files, the base output name defaults to `<input>_results.xlsx` and chunk files are still written with numbered suffixes.
|
||||||
|
|
||||||
Quick SP-API connectivity tests:
|
Quick SP-API connectivity tests:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
bun run src/sp-test.ts # Auth + sellers endpoint
|
bun run src/sp-test.ts # Auth + sellers endpoint
|
||||||
bun run src/sp-test.ts B07SN9BHVV # Auth + sellers endpoint + pricing offer check
|
bun run src/sp-test.ts B07SN9BHVV # Auth + sellers endpoint + pricing offer check
|
||||||
bun run src/sp-test.ts --sellability B07SN9BHVV # Standalone sellability check
|
bun run src/sp-test.ts --sellability B07SN9BHVV # Standalone sellability check
|
||||||
```
|
```
|
||||||
|
|
||||||
## Input file format
|
## Input file format
|
||||||
|
|
||||||
Accepts `.csv` or `.xlsx` files. Column names are matched case-insensitively. Required column:
|
Accepts `.csv` or `.xlsx` files. Column names are matched case-insensitively. Required column:
|
||||||
|
|
||||||
| Column | Aliases |
|
| Column | Aliases |
|
||||||
| ------ | ------- |
|
| ------ | ------- |
|
||||||
| ASIN | — |
|
| ASIN | — |
|
||||||
|
|
||||||
Optional but recommended:
|
Optional but recommended:
|
||||||
|
|
||||||
| Column | Aliases |
|
| Column | Aliases |
|
||||||
| --------------- | ---------------------------- |
|
| --------------- | ---------------------------- |
|
||||||
| Product Name | Name, Title |
|
| Product Name | Name, Title |
|
||||||
| Unit Cost | Cost, Price, Buy Cost |
|
| Unit Cost | Cost, Price, Buy Cost |
|
||||||
| Brand | — |
|
| Brand | — |
|
||||||
| Category | — |
|
| Category | — |
|
||||||
| Amazon Rank | Amazon Rank, BSR, Sales Rank |
|
| Amazon Rank | Amazon Rank, BSR, Sales Rank |
|
||||||
| FBA NET | — |
|
| FBA NET | — |
|
||||||
| Gross Profit $ | Gross Profit |
|
| Gross Profit $ | Gross Profit |
|
||||||
| Gross Profit % | — |
|
| Gross Profit % | — |
|
||||||
| MOQ | Min Order Qty |
|
| MOQ | Min Order Qty |
|
||||||
| MOQ Cost | — |
|
| MOQ Cost | — |
|
||||||
| Total Qty Avail | Qty Available |
|
| Total Qty Avail | Qty Available |
|
||||||
| Link | URL, Source |
|
| Link | URL, Source |
|
||||||
|
|
||||||
Lead-list format aliases (supported):
|
Lead-list format aliases (supported):
|
||||||
|
|
||||||
| Column | Aliases |
|
| Column | Aliases |
|
||||||
| ----------------- | ------------------------------------------ |
|
| ----------------- | ------------------------------------------ |
|
||||||
| Name | Product Name, Title, Product Title |
|
| Name | Product Name, Title, Product Title |
|
||||||
| ASIN Link | ASIN URL, Amazon Link |
|
| ASIN Link | ASIN URL, Amazon Link |
|
||||||
| Source URL | Source Link, Supplier URL |
|
| Source URL | Source Link, Supplier URL |
|
||||||
| 90 Day Average | 90-day Average, Avg Price 90d, 90d Average |
|
| 90 Day Average | 90-day Average, Avg Price 90d, 90d Average |
|
||||||
| Cost | Unit Cost, Buy Cost, Price |
|
| Cost | Unit Cost, Buy Cost, Price |
|
||||||
| Selling Price | Sale Price, Sell Price |
|
| Selling Price | Sale Price, Sell Price |
|
||||||
| Net Profit | Gross Profit |
|
| Net Profit | Gross Profit |
|
||||||
| ROI | Gross Profit %, Return on Investment |
|
| ROI | Gross Profit %, Return on Investment |
|
||||||
| Supplier | Vendor |
|
| Supplier | Vendor |
|
||||||
| Promo/Coupon Code | Promo Code, Coupon Code |
|
| Promo/Coupon Code | Promo Code, Coupon Code |
|
||||||
| Notes | Note |
|
| Notes | Note |
|
||||||
| Date | Lead Date |
|
| Date | Lead Date |
|
||||||
|
|
||||||
Numeric parsing accepts plain numbers as well as formatted values like `$12.50`, `1,209.60`, and `27.5%`.
|
Numeric parsing accepts plain numbers as well as formatted values like `$12.50`, `1,209.60`, and `27.5%`.
|
||||||
|
|
||||||
## Pipeline
|
## Pipeline
|
||||||
|
|
||||||
1. **Read** — parse input file, validate ASINs
|
1. **Read** — parse input file, validate ASINs
|
||||||
2. **Cache check** — look up each ASIN in Redis (24h TTL by default)
|
2. **Cache check** — look up each ASIN in Redis (24h TTL by default)
|
||||||
3. **Sellability gate** — check all uncached ASINs against SP-API `getListingsRestrictions` (concurrency: 5 workers); immediately skip ASINs with status `not_available` and `canSell=false` (no Keepa/fees wasted)
|
3. **Sellability gate** — check all uncached ASINs against SP-API `getListingsRestrictions` (concurrency: 5 workers); immediately skip ASINs with status `not_available` and `canSell=false` (no Keepa/fees wasted)
|
||||||
4. **Keepa fetch** — batch the sellable (uncached) ASINs in a single API call (up to 100 per request)
|
4. **Keepa fetch** — batch the sellable (uncached) ASINs in a single API call (up to 100 per request)
|
||||||
5. **Enrich** — fetch SP-API pricing + FBA/FBM fees for sellable ASINs; combine with Keepa data and spreadsheet data
|
5. **Enrich** — fetch SP-API pricing + FBA/FBM fees for sellable ASINs; combine with Keepa data and spreadsheet data
|
||||||
6. **LLM analysis** — send batches of 5 available products to LM Studio for FBA/FBM/SKIP verdict
|
6. **LLM analysis** — send batches of 5 sellable products to LM Studio for FBA/FBM/SKIP verdict; skipped ASINs get auto-SKIP verdict (confidence 100) and bypass LLM entirely
|
||||||
7. **Chunk orchestration** — if input size is greater than 50, run phases 2-6 for each 50-item chunk sequentially
|
7. **Output** — print results table to console (includes all ASINs), optionally write CSV/XLSX, and **persist results to a SQLite database**.
|
||||||
8. **Output** — print results table to console (includes all ASINs); for chunked runs, always write seriated chunk files (`*_part_001`, `*_part_002`, ...); for non-chunked runs, write a single file only when `--out` is provided
|
|
||||||
|
## Persistent Storage with SQLite
|
||||||
## Output columns
|
|
||||||
|
Results from each run are now stored in a SQLite database named `results.db` in the project root. The SQLite implementation details are handled in `src/database.ts`. This allows you to:
|
||||||
ASIN, Name, Brand, Category, Unit Cost, Current Price, Avg Price 90d, Sales Rank, Rank Avg 90d, Sellers, Monthly Sold, Rank Drops 30d, Rank Drops 90d, FBA Net (sheet), Gross Profit $, Gross Profit %, MOQ, MOQ Cost, Qty Available, FBA Fee, FBM Fee, Referral %, Verdict, Confidence, Reasoning
|
|
||||||
|
- Revisit past analysis results.
|
||||||
## Environment variables
|
- Query and analyze historical data.
|
||||||
|
- Track product performance over time.
|
||||||
| Variable | Default | Description |
|
|
||||||
| ----------------------- | -------------------------- | ----------------------------------------------------------------------- |
|
The database will automatically be created if it doesn't exist. Two tables are created:
|
||||||
| `KEEPA_API_KEY` | — | **Required.** Keepa API key |
|
|
||||||
| `SP_API_CLIENT_ID` | — | LWA app client id from Solution Provider Portal |
|
- `runs`: Stores metadata about each analysis run (timestamp, input file, output file, and summary counts).
|
||||||
| `SP_API_CLIENT_SECRET` | — | LWA app client secret from Solution Provider Portal |
|
- `results`: Stores detailed analysis results for each product from each run, linked to the `runs` table.
|
||||||
| `SP_API_REFRESH_TOKEN` | — | Refresh token from self-authorization |
|
|
||||||
| `SP_API_REGION` | `na` | SP-API endpoint region (`na`, `eu`, `fe`; `us` is accepted as `na`) |
|
## Output columns
|
||||||
| `SP_API_MARKETPLACE_ID` | `ATVPDKIKX0DER` | Marketplace id used for pricing and fee calls (default: US) |
|
|
||||||
| `SP_API_SELLER_ID` | — | Seller ID used for listing restrictions eligibility checks |
|
ASIN, Name, Brand, Category, Unit Cost, Current Price, Avg Price 90d, Sales Rank, Rank Avg 90d, Sellers, Monthly Sold, Rank Drops 30d, Rank Drops 90d, FBA Net (sheet), Gross Profit $, Gross Profit %, MOQ, MOQ Cost, Qty Available, FBA Fee, FBM Fee, Referral %, Verdict, Confidence, Reasoning
|
||||||
| `SP_API_USE_SANDBOX` | `false` | Enable SP-API sandbox mode (`true`/`false`) |
|
|
||||||
| `AWS_ACCESS_KEY_ID` | — | AWS credentials for SigV4 signing (required in most private app setups) |
|
## Environment variables
|
||||||
| `AWS_SECRET_ACCESS_KEY` | — | AWS credentials for SigV4 signing |
|
|
||||||
| `AWS_SESSION_TOKEN` | — | Optional session token when using STS credentials |
|
| Variable | Default | Description |
|
||||||
| `REDIS_URL` | `redis://localhost:6379` | Redis connection URL |
|
| ----------------------- | -------------------------- | ----------------------------------------------------------------------- |
|
||||||
| `LLM_URL` | `http://localhost:1234/v1` | LM Studio API base URL |
|
| `KEEPA_API_KEY` | — | **Required.** Keepa API key |
|
||||||
| `LLM_MODEL` | `default` | Model name to pass to LM Studio |
|
| `SP_API_CLIENT_ID` | — | LWA app client id from Solution Provider Portal |
|
||||||
| `CACHE_TTL` | `86400` | Redis cache TTL in seconds |
|
| `SP_API_CLIENT_SECRET` | — | LWA app client secret from Solution Provider Portal |
|
||||||
|
| `SP_API_REFRESH_TOKEN` | — | Refresh token from self-authorization |
|
||||||
## Notes
|
| `SP_API_REGION` | `na` | SP-API endpoint region (`na`, `eu`, `fe`; `us` is accepted as `na`) |
|
||||||
|
| `SP_API_MARKETPLACE_ID` | `ATVPDKIKX0DER` | Marketplace id used for pricing and fee calls (default: US) |
|
||||||
- **Available-only processing**: SP-API `getListingsRestrictions` is checked first and only ASINs with `sellabilityStatus=available` are enriched, analyzed, and included in outputs. Restricted, not_available, and unknown items are excluded.
|
| `SP_API_SELLER_ID` | — | Seller ID used for listing restrictions eligibility checks |
|
||||||
- **SP-API concurrency**: `fetchSellabilityBatch` limits concurrent requests to 5 workers to avoid 429 throttling. Pricing+fees fetches also use 5 concurrent workers.
|
| `SP_API_USE_SANDBOX` | `false` | Enable SP-API sandbox mode (`true`/`false`) |
|
||||||
- **No batch endpoint**: Amazon SP-API does not provide batch endpoints for `getListingsRestrictions` or `getMyFeesEstimate*`. Concurrency limiting with the library's built-in `auto_request_throttled` safety net prevents overwhelming the API.
|
| `AWS_ACCESS_KEY_ID` | — | AWS credentials for SigV4 signing (required in most private app setups) |
|
||||||
- **Keepa rate limiting**: The client reads `tokensLeft` and `refillRate` from each API response and waits automatically when tokens are exhausted. With a Pro subscription (1 token/min), all 100 ASINs in a batch cost 1 token.
|
| `AWS_SECRET_ACCESS_KEY` | — | AWS credentials for SigV4 signing |
|
||||||
- **Redis is optional**: If Redis is unavailable the tool runs without caching — every run re-fetches from Keepa.
|
| `AWS_SESSION_TOKEN` | — | Optional session token when using STS credentials |
|
||||||
- **SP-API**: `src/sp-api.ts` provides `fetchSellability`, `fetchSellabilityBatch`, and `fetchSpApiPricingAndFees` functions. If SP-API credentials are missing or a call fails, the tool falls back to conservative fee defaults and keeps processing.
|
| `REDIS_URL` | `redis://localhost:6379` | Redis connection URL |
|
||||||
- **Sandbox vs production**: When `SP_API_USE_SANDBOX=true`, production ASIN calls can be denied. Use sandbox-compatible test data or set it to `false` for live marketplace connectivity.
|
| `LLM_URL` | `http://localhost:1234/v1` | LM Studio API base URL |
|
||||||
|
| `LLM_MODEL` | `default` | Model name to pass to LM Studio |
|
||||||
|
| `CACHE_TTL` | `86400` | Redis cache TTL in seconds |
|
||||||
|
|
||||||
|
## Notes
|
||||||
|
|
||||||
|
- **Available-only processing**: SP-API `getListingsRestrictions` is checked first and only ASINs with `sellabilityStatus=available` are enriched, analyzed, and included in outputs. Restricted, not_available, and unknown items are excluded.
|
||||||
|
- **SP-API concurrency**: `fetchSellabilityBatch` limits concurrent requests to 5 workers to avoid 429 throttling. Pricing+fees fetches also use 5 concurrent workers.
|
||||||
|
- **No batch endpoint**: Amazon SP-API does not provide batch endpoints for `getListingsRestrictions` or `getMyFeesEstimate*`. Concurrency limiting with the library's built-in `auto_request_throttled` safety net prevents overwhelming the API.
|
||||||
|
- **Keepa rate limiting**: The client reads `tokensLeft` and `refillRate` from each API response and waits automatically when tokens are exhausted. With a Pro subscription (1 token/min), all 100 ASINs in a batch cost 1 token.
|
||||||
|
- **Redis is optional**: If Redis is unavailable the tool runs without caching — every run re-fetches from Keepa.
|
||||||
|
- **SP-API**: `src/sp-api.ts` provides `fetchSellability`, `fetchSellabilityBatch`, and `fetchSpApiPricingAndFees` functions. If SP-API credentials are missing or a call fails, the tool falls back to conservative fee defaults and keeps processing.
|
||||||
|
- **Sandbox vs production**: When `SP_API_USE_SANDBOX=true`, production ASIN calls can be denied. Use sandbox-compatible test data or set it to `false` for live marketplace connectivity.
|
||||||
|
|||||||
80
src/database.ts
Normal file
80
src/database.ts
Normal file
@@ -0,0 +1,80 @@
|
|||||||
|
import { Database } from "bun:sqlite";
|
||||||
|
|
||||||
|
let db: Database | null = null;
|
||||||
|
|
||||||
|
export function getDb(dbPath: string): Database {
|
||||||
|
if (!db) {
|
||||||
|
db = new Database(dbPath);
|
||||||
|
db.run("PRAGMA journal_mode = WAL;"); // Enable WAL mode for better performance
|
||||||
|
}
|
||||||
|
return db;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function closeDb(): void {
|
||||||
|
if (db) {
|
||||||
|
db.close();
|
||||||
|
db = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function initDb(dbPath: string): void {
|
||||||
|
const database = getDb(dbPath);
|
||||||
|
database.run(`
|
||||||
|
CREATE TABLE IF NOT EXISTS runs (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
timestamp TEXT NOT NULL,
|
||||||
|
input_file TEXT NOT NULL,
|
||||||
|
output_file TEXT,
|
||||||
|
total_products INTEGER,
|
||||||
|
fba_count INTEGER,
|
||||||
|
fbm_count INTEGER,
|
||||||
|
skip_count INTEGER
|
||||||
|
);
|
||||||
|
`);
|
||||||
|
database.run(`
|
||||||
|
CREATE TABLE IF NOT EXISTS results (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
run_id INTEGER NOT NULL,
|
||||||
|
asin TEXT NOT NULL,
|
||||||
|
product_name TEXT,
|
||||||
|
brand TEXT,
|
||||||
|
category TEXT,
|
||||||
|
unit_cost REAL,
|
||||||
|
current_price REAL,
|
||||||
|
avg_price_90d REAL,
|
||||||
|
avg_price_90d_sheet REAL,
|
||||||
|
selling_price_sheet REAL,
|
||||||
|
sales_rank INTEGER,
|
||||||
|
rank_avg_90d INTEGER,
|
||||||
|
sellers INTEGER,
|
||||||
|
monthly_sold INTEGER,
|
||||||
|
rank_drops_30d INTEGER,
|
||||||
|
rank_drops_90d INTEGER,
|
||||||
|
fba_net_sheet REAL,
|
||||||
|
gross_profit_dollar REAL,
|
||||||
|
gross_profit_pct REAL,
|
||||||
|
net_profit_sheet REAL,
|
||||||
|
roi_sheet REAL,
|
||||||
|
moq INTEGER,
|
||||||
|
moq_cost REAL,
|
||||||
|
qty_available INTEGER,
|
||||||
|
supplier TEXT,
|
||||||
|
source_url TEXT,
|
||||||
|
asin_link TEXT,
|
||||||
|
promo_coupon_code TEXT,
|
||||||
|
notes TEXT,
|
||||||
|
lead_date TEXT,
|
||||||
|
fba_fee REAL,
|
||||||
|
fbm_fee REAL,
|
||||||
|
referral_percent REAL,
|
||||||
|
can_sell TEXT,
|
||||||
|
sellability_status TEXT,
|
||||||
|
sellability_reason TEXT,
|
||||||
|
verdict TEXT NOT NULL,
|
||||||
|
confidence INTEGER,
|
||||||
|
reasoning TEXT,
|
||||||
|
fetched_at TEXT NOT NULL,
|
||||||
|
FOREIGN KEY (run_id) REFERENCES runs(id)
|
||||||
|
);
|
||||||
|
`);
|
||||||
|
}
|
||||||
64
src/index.ts
64
src/index.ts
@@ -3,8 +3,10 @@ import { fetchKeepaDataBatch } from "./keepa.ts";
|
|||||||
import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts";
|
import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts";
|
||||||
import { connectCache, getCache, setCache, disconnectCache } from "./cache.ts";
|
import { connectCache, getCache, setCache, disconnectCache } from "./cache.ts";
|
||||||
import { analyzeProducts } from "./llm.ts";
|
import { analyzeProducts } from "./llm.ts";
|
||||||
import { printResults, writeResultsCsv } from "./writer.ts";
|
import { printResults, writeResultsToDb } from "./writer.ts";
|
||||||
import path from "node:path";
|
import { initDb, closeDb } from "./database.ts";
|
||||||
|
|
||||||
|
const DB_PATH = "./results.db";
|
||||||
import type {
|
import type {
|
||||||
EnrichedProduct,
|
EnrichedProduct,
|
||||||
AnalysisResult,
|
AnalysisResult,
|
||||||
@@ -32,38 +34,26 @@ function parseArgs(): { inputFile: string; outputFile?: string } {
|
|||||||
return { inputFile, outputFile };
|
return { inputFile, outputFile };
|
||||||
}
|
}
|
||||||
|
|
||||||
function chunkArray<T>(items: T[], chunkSize: number): T[][] {
|
async function main() {
|
||||||
const chunks: T[][] = [];
|
const { inputFile, outputFile } = parseArgs();
|
||||||
for (let i = 0; i < items.length; i += chunkSize) {
|
|
||||||
chunks.push(items.slice(i, i + chunkSize));
|
console.log("Connecting to Redis...");
|
||||||
|
await connectCache();
|
||||||
|
|
||||||
|
// Initialize SQLite DB
|
||||||
|
console.log("Initializing SQLite database...");
|
||||||
|
initDb(DB_PATH);
|
||||||
|
|
||||||
|
// Phase 1: Read input file
|
||||||
|
console.log(`\nReading ${inputFile}...`);
|
||||||
|
const products = readProducts(inputFile);
|
||||||
|
|
||||||
|
if (products.length === 0) {
|
||||||
|
console.error("No valid products found in input file.");
|
||||||
|
process.exit(1);
|
||||||
}
|
}
|
||||||
return chunks;
|
|
||||||
}
|
|
||||||
|
|
||||||
function resolveBaseOutputPath(inputFile: string, outputFile?: string): string {
|
// Phase 2: Check cache for all ASINs
|
||||||
if (outputFile) return outputFile;
|
|
||||||
|
|
||||||
const parsedInput = path.parse(inputFile);
|
|
||||||
return path.join(parsedInput.dir, `${parsedInput.name}_results.xlsx`);
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildChunkOutputPath(
|
|
||||||
baseOutputPath: string,
|
|
||||||
chunkIndex: number,
|
|
||||||
): string {
|
|
||||||
const parsed = path.parse(baseOutputPath);
|
|
||||||
const extension = parsed.ext || ".xlsx";
|
|
||||||
const chunkSuffix = String(chunkIndex + 1).padStart(3, "0");
|
|
||||||
return path.join(
|
|
||||||
parsed.dir,
|
|
||||||
`${parsed.name}_part_${chunkSuffix}${extension}`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function processProductChunk(
|
|
||||||
products: ProductRecord[],
|
|
||||||
): Promise<AnalysisResult[]> {
|
|
||||||
// Phase 2: Check cache for all ASINs in chunk
|
|
||||||
console.log(`\nChecking cache for ${products.length} products...`);
|
console.log(`\nChecking cache for ${products.length} products...`);
|
||||||
const cached = new Map<string, EnrichedProduct>();
|
const cached = new Map<string, EnrichedProduct>();
|
||||||
const excludedCachedAsins = new Set<string>();
|
const excludedCachedAsins = new Set<string>();
|
||||||
@@ -331,12 +321,10 @@ async function main() {
|
|||||||
|
|
||||||
printResults(allResults);
|
printResults(allResults);
|
||||||
|
|
||||||
if (!hasMultipleChunks && outputFile) {
|
writeResultsToDb(allResults, DB_PATH, inputFile, outputFile);
|
||||||
writeResultsCsv(allResults, outputFile);
|
|
||||||
}
|
await disconnectCache();
|
||||||
} finally {
|
closeDb();
|
||||||
await disconnectCache();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
main().catch((err) => {
|
main().catch((err) => {
|
||||||
|
|||||||
413
src/writer.ts
413
src/writer.ts
@@ -1,159 +1,254 @@
|
|||||||
import * as XLSX from "xlsx";
|
import { getDb } from "./database.ts";
|
||||||
import type { AnalysisResult } from "./types.ts";
|
import type { AnalysisResult } from "./types.ts";
|
||||||
|
|
||||||
function buildRow(r: AnalysisResult) {
|
function buildRow(r: AnalysisResult) {
|
||||||
const price =
|
const price =
|
||||||
r.product.keepa?.currentPrice ??
|
r.product.keepa?.currentPrice ??
|
||||||
r.product.record.sellingPriceFromSheet ??
|
r.product.record.sellingPriceFromSheet ??
|
||||||
r.product.spApi.estimatedSalePrice;
|
r.product.spApi.estimatedSalePrice;
|
||||||
const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank;
|
const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank;
|
||||||
|
const canSellStatus =
|
||||||
return {
|
r.product.spApi.canSell == null
|
||||||
ASIN: r.product.record.asin,
|
? "unknown"
|
||||||
Name: r.product.record.name,
|
: r.product.spApi.canSell
|
||||||
Brand: r.product.record.brand ?? "",
|
? "yes"
|
||||||
Category:
|
: "no";
|
||||||
r.product.record.category ??
|
|
||||||
r.product.keepa?.categoryTree?.join(" > ") ??
|
return {
|
||||||
"",
|
ASIN: r.product.record.asin,
|
||||||
"Unit Cost": r.product.record.unitCost,
|
Name: r.product.record.name,
|
||||||
"Current Price": price ?? "",
|
Brand: r.product.record.brand ?? "",
|
||||||
"Avg Price 90d": r.product.keepa?.avgPrice90 ?? "",
|
Category:
|
||||||
"Avg Price 90d (sheet)": r.product.record.avgPrice90FromSheet ?? "",
|
r.product.record.category ??
|
||||||
"Selling Price (sheet)": r.product.record.sellingPriceFromSheet ?? "",
|
r.product.keepa?.categoryTree?.join(" > ") ??
|
||||||
"Sales Rank": rank ?? "",
|
"",
|
||||||
"Rank Avg 90d": r.product.keepa?.salesRankAvg90 ?? "",
|
"Unit Cost": r.product.record.unitCost,
|
||||||
Sellers: r.product.keepa?.sellerCount ?? "",
|
"Current Price": price ?? "",
|
||||||
"Monthly Sold": r.product.keepa?.monthlySold ?? "",
|
"Avg Price 90d": r.product.keepa?.avgPrice90 ?? "",
|
||||||
"Rank Drops 30d": r.product.keepa?.salesRankDrops30 ?? "",
|
"Avg Price 90d (sheet)": r.product.record.avgPrice90FromSheet ?? "",
|
||||||
"Rank Drops 90d": r.product.keepa?.salesRankDrops90 ?? "",
|
"Selling Price (sheet)": r.product.record.sellingPriceFromSheet ?? "",
|
||||||
"FBA Net (sheet)": r.product.record.fbaNet ?? "",
|
"Sales Rank": rank ?? "",
|
||||||
"Gross Profit $": r.product.record.grossProfit ?? "",
|
"Rank Avg 90d": r.product.keepa?.salesRankAvg90 ?? "",
|
||||||
"Gross Profit %": r.product.record.grossProfitPct ?? "",
|
Sellers: r.product.keepa?.sellerCount ?? "",
|
||||||
"Net Profit (sheet)": r.product.record.netProfitFromSheet ?? "",
|
"Monthly Sold": r.product.keepa?.monthlySold ?? "",
|
||||||
"ROI (sheet)": r.product.record.roiFromSheet ?? "",
|
"Rank Drops 30d": r.product.keepa?.salesRankDrops30 ?? "",
|
||||||
MOQ: r.product.record.moq ?? "",
|
"Rank Drops 90d": r.product.keepa?.salesRankDrops90 ?? "",
|
||||||
"MOQ Cost": r.product.record.moqCost ?? "",
|
"FBA Net (sheet)": r.product.record.fbaNet ?? "",
|
||||||
"Qty Available": r.product.record.totalQtyAvail ?? "",
|
"Gross Profit $": r.product.record.grossProfit ?? "",
|
||||||
Supplier: r.product.record.supplier ?? "",
|
"Gross Profit %": r.product.record.grossProfitPct ?? "",
|
||||||
"Source URL": r.product.record.sourceUrl ?? "",
|
"Net Profit (sheet)": r.product.record.netProfitFromSheet ?? "",
|
||||||
"ASIN Link": r.product.record.asinLink ?? "",
|
"ROI (sheet)": r.product.record.roiFromSheet ?? "",
|
||||||
"Promo/Coupon Code": r.product.record.promoCouponCode ?? "",
|
MOQ: r.product.record.moq ?? "",
|
||||||
Notes: r.product.record.notes ?? "",
|
"MOQ Cost": r.product.record.moqCost ?? "",
|
||||||
"Lead Date": r.product.record.leadDate ?? "",
|
"Qty Available": r.product.record.totalQtyAvail ?? "",
|
||||||
"FBA Fee": r.product.spApi.fbaFee,
|
Supplier: r.product.record.supplier ?? "",
|
||||||
"FBM Fee": r.product.spApi.fbmFee,
|
"Source URL": r.product.record.sourceUrl ?? "",
|
||||||
"Referral %": r.product.spApi.referralFeePercent,
|
"ASIN Link": r.product.record.asinLink ?? "",
|
||||||
"Can Sell":
|
"Promo/Coupon Code": r.product.record.promoCouponCode ?? "",
|
||||||
r.product.spApi.canSell == null
|
Notes: r.product.record.notes ?? "",
|
||||||
? "unknown"
|
"Lead Date": r.product.record.leadDate ?? "",
|
||||||
: r.product.spApi.canSell
|
"FBA Fee": r.product.spApi.fbaFee,
|
||||||
? "yes"
|
"FBM Fee": r.product.spApi.fbmFee,
|
||||||
: "no",
|
"Referral %": r.product.spApi.referralFeePercent,
|
||||||
Sellability: r.product.spApi.sellabilityStatus,
|
"Can Sell": canSellStatus,
|
||||||
"Sellability Reason": r.product.spApi.sellabilityReason ?? "",
|
Sellability: r.product.spApi.sellabilityStatus,
|
||||||
Verdict: r.verdict.verdict,
|
"Sellability Reason": r.product.spApi.sellabilityReason ?? "",
|
||||||
Confidence: r.verdict.confidence,
|
Verdict: r.verdict.verdict,
|
||||||
Reasoning: r.verdict.reasoning,
|
Confidence: r.verdict.confidence,
|
||||||
};
|
Reasoning: r.verdict.reasoning,
|
||||||
}
|
};
|
||||||
|
}
|
||||||
export function printResults(results: AnalysisResult[]): void {
|
|
||||||
const rows = results
|
export function writeResultsToDb(
|
||||||
.filter((r) => r.verdict.verdict === "FBA" || r.verdict.verdict === "FBM")
|
results: AnalysisResult[],
|
||||||
.map((r) => {
|
dbPath: string,
|
||||||
const sellingPrice =
|
inputFile: string,
|
||||||
r.product.keepa?.currentPrice ??
|
outputFile: string | undefined,
|
||||||
r.product.record.sellingPriceFromSheet ??
|
): void {
|
||||||
r.product.spApi.estimatedSalePrice;
|
const database = getDb(dbPath);
|
||||||
const referralFee =
|
|
||||||
sellingPrice != null
|
const timestamp = new Date().toISOString();
|
||||||
? sellingPrice * (r.product.spApi.referralFeePercent / 100)
|
const fbaCount = results.filter((r) => r.verdict.verdict === "FBA").length;
|
||||||
: null;
|
const fbmCount = results.filter((r) => r.verdict.verdict === "FBM").length;
|
||||||
const fulfillmentFee =
|
const skipCount = results.filter((r) => r.verdict.verdict === "SKIP").length;
|
||||||
r.verdict.verdict === "FBA"
|
|
||||||
? r.product.spApi.fbaFee
|
const insertRun = database.prepare(
|
||||||
: r.product.spApi.fbmFee;
|
`INSERT INTO runs (
|
||||||
const netProfit =
|
timestamp,
|
||||||
sellingPrice != null
|
input_file,
|
||||||
? Math.round(
|
output_file,
|
||||||
(sellingPrice -
|
total_products,
|
||||||
r.product.record.unitCost -
|
fba_count,
|
||||||
fulfillmentFee -
|
fbm_count,
|
||||||
(referralFee ?? 0)) *
|
skip_count
|
||||||
100,
|
) VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
||||||
) / 100
|
);
|
||||||
: "";
|
const runInfo = insertRun.run(
|
||||||
|
timestamp,
|
||||||
return {
|
inputFile,
|
||||||
ASIN: r.product.record.asin,
|
outputFile ?? null,
|
||||||
Name: r.product.record.name.slice(0, 40),
|
results.length,
|
||||||
Category: String(
|
fbaCount,
|
||||||
r.product.record.category ??
|
fbmCount,
|
||||||
r.product.keepa?.categoryTree?.join(" > ") ??
|
skipCount,
|
||||||
"",
|
);
|
||||||
).slice(0, 20),
|
const runId =
|
||||||
"Unit Cost": r.product.record.unitCost,
|
(runInfo.changes as number) > 0
|
||||||
"Selling Price": sellingPrice ?? "",
|
? (runInfo.lastInsertRowid as number)
|
||||||
"Net Profit": netProfit,
|
: null;
|
||||||
"Monthly Sold": r.product.keepa?.monthlySold ?? "",
|
|
||||||
"Sold 90 Day": r.product.keepa?.salesRankDrops90 ?? "",
|
if (runId === null) {
|
||||||
"Can Sell":
|
console.error("Failed to insert run record into SQLite.");
|
||||||
r.product.spApi.canSell == null
|
return;
|
||||||
? "unknown"
|
}
|
||||||
: r.product.spApi.canSell
|
|
||||||
? "yes"
|
const insertResult = database.prepare(
|
||||||
: "no",
|
`INSERT INTO results (
|
||||||
Sellability: r.product.spApi.sellabilityStatus,
|
run_id, asin, product_name, brand, category, unit_cost, current_price,
|
||||||
"Sellability Reason": String(
|
avg_price_90d, avg_price_90d_sheet, selling_price_sheet, sales_rank, rank_avg_90d,
|
||||||
r.product.spApi.sellabilityReason ?? "",
|
sellers, monthly_sold, rank_drops_30d, rank_drops_90d, fba_net_sheet,
|
||||||
).slice(0, 60),
|
gross_profit_dollar, gross_profit_pct, net_profit_sheet, roi_sheet, moq, moq_cost,
|
||||||
Confidence: r.verdict.confidence,
|
qty_available, supplier, source_url, asin_link, promo_coupon_code, notes, lead_date,
|
||||||
Reasoning: r.verdict.reasoning.slice(0, 60),
|
fba_fee, fbm_fee, referral_percent, can_sell, sellability_status, sellability_reason,
|
||||||
};
|
verdict, confidence, reasoning, fetched_at
|
||||||
});
|
) VALUES (
|
||||||
|
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||||
console.log("\n=== Analysis Results ===\n");
|
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
|
||||||
if (rows.length === 0) {
|
)`,
|
||||||
console.log("No FBA/FBM leads found.");
|
);
|
||||||
} else {
|
|
||||||
console.table(rows);
|
database.transaction(() => {
|
||||||
}
|
for (const r of results) {
|
||||||
|
const row = buildRow(r);
|
||||||
const summary = {
|
insertResult.run(
|
||||||
FBA: results.filter((r) => r.verdict.verdict === "FBA").length,
|
runId,
|
||||||
FBM: results.filter((r) => r.verdict.verdict === "FBM").length,
|
row.ASIN,
|
||||||
SKIP: results.filter((r) => r.verdict.verdict === "SKIP").length,
|
row.Name,
|
||||||
Available: results.filter(
|
row.Brand,
|
||||||
(r) => r.product.spApi.sellabilityStatus === "available",
|
row.Category,
|
||||||
).length,
|
row["Unit Cost"] ?? null,
|
||||||
Restricted: results.filter(
|
row["Current Price"] ?? null,
|
||||||
(r) => r.product.spApi.sellabilityStatus === "restricted",
|
row["Avg Price 90d"] ?? null,
|
||||||
).length,
|
row["Avg Price 90d (sheet)"] ?? null,
|
||||||
NotAvailable: results.filter(
|
row["Selling Price (sheet)"] ?? null,
|
||||||
(r) => r.product.spApi.sellabilityStatus === "not_available",
|
row["Sales Rank"] ?? null,
|
||||||
).length,
|
row["Rank Avg 90d"] ?? null,
|
||||||
Unknown: results.filter(
|
row.Sellers ?? null,
|
||||||
(r) => r.product.spApi.sellabilityStatus === "unknown",
|
row["Monthly Sold"] ?? null,
|
||||||
).length,
|
row["Rank Drops 30d"] ?? null,
|
||||||
};
|
row["Rank Drops 90d"] ?? null,
|
||||||
console.log(
|
row["FBA Net (sheet)"] ?? null,
|
||||||
`\nSummary: ${summary.FBA} FBA | ${summary.FBM} FBM | ${summary.SKIP} SKIP out of ${results.length} products\n`,
|
row["Gross Profit $"] ?? null,
|
||||||
);
|
row["Gross Profit %"] ?? null,
|
||||||
console.log(
|
row["Net Profit (sheet)"] ?? null,
|
||||||
`Sellability: ${summary.Available} available | ${summary.Restricted} restricted | ${summary.NotAvailable} not_available | ${summary.Unknown} unknown\n`,
|
row["ROI (sheet)"] ?? null,
|
||||||
);
|
row.MOQ ?? null,
|
||||||
}
|
row["MOQ Cost"] ?? null,
|
||||||
|
row["Qty Available"] ?? null,
|
||||||
export function writeResultsCsv(
|
row.Supplier ?? null,
|
||||||
results: AnalysisResult[],
|
row["Source URL"] ?? null,
|
||||||
outputPath: string,
|
row["ASIN Link"] ?? null,
|
||||||
): void {
|
row["Promo/Coupon Code"] ?? null,
|
||||||
const rows = results.map(buildRow);
|
row.Notes ?? null,
|
||||||
|
row["Lead Date"] ?? null,
|
||||||
const ws = XLSX.utils.json_to_sheet(rows);
|
row["FBA Fee"] ?? null,
|
||||||
const wb = XLSX.utils.book_new();
|
row["FBM Fee"] ?? null,
|
||||||
XLSX.utils.book_append_sheet(wb, ws, "Results");
|
row["Referral %"] ?? null,
|
||||||
XLSX.writeFile(wb, outputPath);
|
row["Can Sell"],
|
||||||
console.log(`Results written to ${outputPath}`);
|
row.Sellability,
|
||||||
}
|
row["Sellability Reason"] ?? null,
|
||||||
|
row.Verdict,
|
||||||
|
row.Confidence ?? null,
|
||||||
|
row.Reasoning,
|
||||||
|
r.product.fetchedAt,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
console.log(`Results written to SQLite database for run_id: ${runId}`);
|
||||||
|
}
|
||||||
|
export function printResults(results: AnalysisResult[]): void {
|
||||||
|
const rows = results
|
||||||
|
.filter((r) => r.verdict.verdict === "FBA" || r.verdict.verdict === "FBM")
|
||||||
|
.map((r) => {
|
||||||
|
const sellingPrice =
|
||||||
|
r.product.keepa?.currentPrice ??
|
||||||
|
r.product.record.sellingPriceFromSheet ??
|
||||||
|
r.product.spApi.estimatedSalePrice;
|
||||||
|
const referralFee =
|
||||||
|
sellingPrice != null
|
||||||
|
? sellingPrice * (r.product.spApi.referralFeePercent / 100)
|
||||||
|
: null;
|
||||||
|
const fulfillmentFee =
|
||||||
|
r.verdict.verdict === "FBA"
|
||||||
|
? r.product.spApi.fbaFee
|
||||||
|
: r.product.spApi.fbmFee;
|
||||||
|
const netProfit =
|
||||||
|
sellingPrice != null
|
||||||
|
? Math.round(
|
||||||
|
(sellingPrice -
|
||||||
|
r.product.record.unitCost -
|
||||||
|
fulfillmentFee -
|
||||||
|
(referralFee ?? 0)) *
|
||||||
|
100,
|
||||||
|
) / 100
|
||||||
|
: "";
|
||||||
|
|
||||||
|
return {
|
||||||
|
ASIN: r.product.record.asin,
|
||||||
|
Name: r.product.record.name.slice(0, 40),
|
||||||
|
Category: String(
|
||||||
|
r.product.record.category ??
|
||||||
|
r.product.keepa?.categoryTree?.join(" > ") ??
|
||||||
|
"",
|
||||||
|
).slice(0, 20),
|
||||||
|
"Unit Cost": r.product.record.unitCost,
|
||||||
|
"Selling Price": sellingPrice ?? "",
|
||||||
|
"Net Profit": netProfit,
|
||||||
|
"Monthly Sold": r.product.keepa?.monthlySold ?? "",
|
||||||
|
"Sold 90 Day": r.product.keepa?.salesRankDrops90 ?? "",
|
||||||
|
"Can Sell":
|
||||||
|
r.product.spApi.canSell == null
|
||||||
|
? "unknown"
|
||||||
|
: r.product.spApi.canSell
|
||||||
|
? "yes"
|
||||||
|
: "no",
|
||||||
|
Sellability: r.product.spApi.sellabilityStatus,
|
||||||
|
"Sellability Reason": String(
|
||||||
|
r.product.spApi.sellabilityReason ?? "",
|
||||||
|
).slice(0, 60),
|
||||||
|
Confidence: r.verdict.confidence,
|
||||||
|
Reasoning: r.verdict.reasoning.slice(0, 60),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log("\n=== Analysis Results ===\n");
|
||||||
|
if (rows.length === 0) {
|
||||||
|
console.log("No FBA/FBM leads found.");
|
||||||
|
} else {
|
||||||
|
console.table(rows);
|
||||||
|
}
|
||||||
|
|
||||||
|
const summary = {
|
||||||
|
FBA: results.filter((r) => r.verdict.verdict === "FBA").length,
|
||||||
|
FBM: results.filter((r) => r.verdict.verdict === "FBM").length,
|
||||||
|
SKIP: results.filter((r) => r.verdict.verdict === "SKIP").length,
|
||||||
|
Available: results.filter(
|
||||||
|
(r) => r.product.spApi.sellabilityStatus === "available",
|
||||||
|
).length,
|
||||||
|
Restricted: results.filter(
|
||||||
|
(r) => r.product.spApi.sellabilityStatus === "restricted",
|
||||||
|
).length,
|
||||||
|
NotAvailable: results.filter(
|
||||||
|
(r) => r.product.spApi.sellabilityStatus === "not_available",
|
||||||
|
).length,
|
||||||
|
Unknown: results.filter(
|
||||||
|
(r) => r.product.spApi.sellabilityStatus === "unknown",
|
||||||
|
).length,
|
||||||
|
};
|
||||||
|
console.log(
|
||||||
|
`\nSummary: ${summary.FBA} FBA | ${summary.FBM} FBM | ${summary.SKIP} SKIP out of ${results.length} products\n`,
|
||||||
|
);
|
||||||
|
console.log(
|
||||||
|
`Sellability: ${summary.Available} available | ${summary.Restricted} restricted | ${summary.NotAvailable} not_available | ${summary.Unknown} unknown\n`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user