Merge branch 'upc-to-asin'
This commit is contained in:
2
.gitignore
vendored
2
.gitignore
vendored
@@ -45,3 +45,5 @@ output/
|
||||
temp_output/
|
||||
|
||||
dist-server/
|
||||
|
||||
*.xls
|
||||
|
||||
89
README.md
89
README.md
@@ -45,6 +45,95 @@ bun run src/sp-test.ts B07SN9BHVV # Auth + sellers endpoint + pricing offer c
|
||||
bun run src/sp-test.ts --sellability B07SN9BHVV # Standalone sellability check
|
||||
```
|
||||
|
||||
## UPC to ASIN Mapping
|
||||
|
||||
You can map UPCs to ASINs directly through the Keepa integration in `src/keepa.ts`.
|
||||
|
||||
```ts
|
||||
import { mapUpcsToAsins, lookupKeepaUpcs } from "./src/keepa.ts";
|
||||
|
||||
const upcs = ["012345678901", "098765432109", "112233445566"];
|
||||
|
||||
// Simple map output (UPC -> ASIN) for clean one-to-one matches only.
|
||||
const asinMap = await mapUpcsToAsins(upcs);
|
||||
for (const [upc, asin] of asinMap.entries()) {
|
||||
console.log(`UPC ${upc} -> ASIN ${asin}`);
|
||||
}
|
||||
|
||||
// Rich output includes status for every UPC (invalid, not found, collisions, etc.).
|
||||
const details = await lookupKeepaUpcs(upcs);
|
||||
for (const [upc, detail] of details.entries()) {
|
||||
console.log(upc, detail.status, detail.asin, detail.reason ?? "");
|
||||
}
|
||||
```
|
||||
|
||||
Behavior:
|
||||
|
||||
- Strict validation accepts only 12, 13, or 14 digit UPC values.
|
||||
- If a UPC resolves to multiple ASINs, it is excluded from the simple map.
|
||||
- The rich lookup returns all candidate ASINs and status per UPC.
|
||||
|
||||
CLI usage:
|
||||
|
||||
```bash
|
||||
bun run upc 012345678901 098765432109
|
||||
bun run upc 012345678901,098765432109 --detailed
|
||||
bun run upc --file upcs.txt --detailed --json
|
||||
```
|
||||
|
||||
API usage (when `bun run start:web` is running):
|
||||
|
||||
```bash
|
||||
# Simple one-to-one mapping (GET)
|
||||
curl "http://localhost:3000/api/upc/map?upc=012345678901&upc=098765432109"
|
||||
|
||||
# Detailed lookup with statuses (GET)
|
||||
curl "http://localhost:3000/api/upc/lookup?upcs=012345678901,098765432109"
|
||||
|
||||
# Detailed lookup (POST JSON)
|
||||
curl -X POST "http://localhost:3000/api/upc/lookup" \
|
||||
-H "content-type: application/json" \
|
||||
-d '{"upcs":["012345678901","098765432109"]}'
|
||||
```
|
||||
|
||||
## Large UPC File Analysis (XLS/XLSX)
|
||||
|
||||
For very large Excel files that contain UPC values, use the dedicated UPC-file process. It runs in batches:
|
||||
|
||||
1. Reads UPC rows in batches (`.xlsx` uses streaming reader, `.xls` uses fallback row-window parsing).
|
||||
2. Resolves UPCs to ASINs with Keepa.
|
||||
3. Runs the same sellability + Keepa/SP-API enrichment + LLM verdict pipeline as lead analysis.
|
||||
4. Persists output into existing `runs` + `results` tables, so it appears in current reporting APIs/UI.
|
||||
|
||||
CLI usage:
|
||||
|
||||
```bash
|
||||
bun run upc-file --input huge-upcs.xlsx
|
||||
bun run upc-file --input huge-upcs.xls --input-batch-size 500 --upc-lookup-batch-size 100 --max-rows 10000
|
||||
```
|
||||
|
||||
API usage (when `bun run start:web` is running):
|
||||
|
||||
```bash
|
||||
curl -X POST "http://localhost:3000/api/process/upc-file" \
|
||||
-H "content-type: application/json" \
|
||||
-d '{
|
||||
"inputFile": "/absolute/path/to/huge-upcs.xlsx",
|
||||
"inputBatchSize": 300,
|
||||
"upcLookupBatchSize": 100
|
||||
}'
|
||||
```
|
||||
|
||||
Request body fields:
|
||||
|
||||
- `inputFile` (required): server-local path to `.xls` or `.xlsx` file.
|
||||
- `outputFile` (optional): stored in run metadata.
|
||||
- `inputBatchSize` (optional): number of input rows per processing batch (default `200`).
|
||||
- `upcLookupBatchSize` (optional): UPC chunk size per Keepa lookup call (default `100`).
|
||||
- `maxRows` (optional): cap processed valid UPC rows for dry runs.
|
||||
|
||||
Response includes run metadata and status counts, including unresolved UPC reasons and lead verdict totals.
|
||||
|
||||
## Input file format
|
||||
|
||||
Accepts `.csv` or `.xlsx` files. Column names are matched case-insensitively. Required column:
|
||||
|
||||
183
bun.lock
183
bun.lock
@@ -6,6 +6,7 @@
|
||||
"name": "asin-check",
|
||||
"dependencies": {
|
||||
"amazon-sp-api": "^1.2.1",
|
||||
"exceljs": "^4.4.0",
|
||||
"ioredis": "^5.10.1",
|
||||
"react": "^19.2.0",
|
||||
"react-dom": "^19.2.0",
|
||||
@@ -22,6 +23,10 @@
|
||||
},
|
||||
},
|
||||
"packages": {
|
||||
"@fast-csv/format": ["@fast-csv/format@4.3.5", "", { "dependencies": { "@types/node": "^14.0.1", "lodash.escaperegexp": "^4.1.2", "lodash.isboolean": "^3.0.3", "lodash.isequal": "^4.5.0", "lodash.isfunction": "^3.0.9", "lodash.isnil": "^4.0.0" } }, "sha512-8iRn6QF3I8Ak78lNAa+Gdl5MJJBM5vRHivFtMRUWINdevNo00K7OXxS2PshawLKTejVwieIlPmK5YlLu6w4u8A=="],
|
||||
|
||||
"@fast-csv/parse": ["@fast-csv/parse@4.3.6", "", { "dependencies": { "@types/node": "^14.0.1", "lodash.escaperegexp": "^4.1.2", "lodash.groupby": "^4.6.0", "lodash.isfunction": "^3.0.9", "lodash.isnil": "^4.0.0", "lodash.isundefined": "^3.0.1", "lodash.uniq": "^4.5.0" } }, "sha512-uRsLYksqpbDmWaSmzvJcuApSEe38+6NQZBUsuAyMZKqHxH0g1wcJgsKUvN3WC8tewaqFjBMMGrkHmC+T7k8LvA=="],
|
||||
|
||||
"@ioredis/commands": ["@ioredis/commands@1.5.1", "", {}, "sha512-JH8ZL/ywcJyR9MmJ5BNqZllXNZQqQbnVZOqpPQqE1vHiFgAw4NHbvE0FOduNU8IX9babitBT46571OnPTT0Zcw=="],
|
||||
|
||||
"@types/bun": ["@types/bun@1.3.11", "", { "dependencies": { "bun-types": "1.3.11" } }, "sha512-5vPne5QvtpjGpsGYXiFyycfpDF2ECyPcTSsFBMa0fraoxiQyMJ3SmuQIGhzPg2WJuWxVBoxWJ2kClYTcw/4fAg=="],
|
||||
@@ -36,6 +41,34 @@
|
||||
|
||||
"amazon-sp-api": ["amazon-sp-api@1.2.1", "", { "dependencies": { "csvtojson": "^2.0.14", "fast-xml-parser": "^5.3.1", "iconv-lite": "^0.7.0", "qs": "^6.14.0" } }, "sha512-zxX3KtoCDx0wxkkBgFM6qew49JJoL1XZQgUnztfp+8Im2HLHBAt4beSiDo/AkH00Gr8paHBAjdcJY6LC6ISU7w=="],
|
||||
|
||||
"archiver": ["archiver@5.3.2", "", { "dependencies": { "archiver-utils": "^2.1.0", "async": "^3.2.4", "buffer-crc32": "^0.2.1", "readable-stream": "^3.6.0", "readdir-glob": "^1.1.2", "tar-stream": "^2.2.0", "zip-stream": "^4.1.0" } }, "sha512-+25nxyyznAXF7Nef3y0EbBeqmGZgeN/BxHX29Rs39djAfaFalmQ89SE6CWyDCHzGL0yt/ycBtNOmGTW0FyGWNw=="],
|
||||
|
||||
"archiver-utils": ["archiver-utils@2.1.0", "", { "dependencies": { "glob": "^7.1.4", "graceful-fs": "^4.2.0", "lazystream": "^1.0.0", "lodash.defaults": "^4.2.0", "lodash.difference": "^4.5.0", "lodash.flatten": "^4.4.0", "lodash.isplainobject": "^4.0.6", "lodash.union": "^4.6.0", "normalize-path": "^3.0.0", "readable-stream": "^2.0.0" } }, "sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw=="],
|
||||
|
||||
"async": ["async@3.2.6", "", {}, "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA=="],
|
||||
|
||||
"balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="],
|
||||
|
||||
"base64-js": ["base64-js@1.5.1", "", {}, "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="],
|
||||
|
||||
"big-integer": ["big-integer@1.6.52", "", {}, "sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg=="],
|
||||
|
||||
"binary": ["binary@0.3.0", "", { "dependencies": { "buffers": "~0.1.1", "chainsaw": "~0.1.0" } }, "sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg=="],
|
||||
|
||||
"bl": ["bl@4.1.0", "", { "dependencies": { "buffer": "^5.5.0", "inherits": "^2.0.4", "readable-stream": "^3.4.0" } }, "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w=="],
|
||||
|
||||
"bluebird": ["bluebird@3.4.7", "", {}, "sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA=="],
|
||||
|
||||
"brace-expansion": ["brace-expansion@2.1.0", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w=="],
|
||||
|
||||
"buffer": ["buffer@5.7.1", "", { "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.1.13" } }, "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ=="],
|
||||
|
||||
"buffer-crc32": ["buffer-crc32@0.2.13", "", {}, "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ=="],
|
||||
|
||||
"buffer-indexof-polyfill": ["buffer-indexof-polyfill@1.0.2", "", {}, "sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A=="],
|
||||
|
||||
"buffers": ["buffers@0.1.1", "", {}, "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ=="],
|
||||
|
||||
"bun-types": ["bun-types@1.3.11", "", { "dependencies": { "@types/node": "*" } }, "sha512-1KGPpoxQWl9f6wcZh57LvrPIInQMn2TQ7jsgxqpRzg+l0QPOFvJVH7HmvHo/AiPgwXy+/Thf6Ov3EdVn1vOabg=="],
|
||||
|
||||
"call-bind-apply-helpers": ["call-bind-apply-helpers@1.0.2", "", { "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2" } }, "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ=="],
|
||||
@@ -44,78 +77,176 @@
|
||||
|
||||
"cfb": ["cfb@1.2.2", "", { "dependencies": { "adler-32": "~1.3.0", "crc-32": "~1.2.0" } }, "sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA=="],
|
||||
|
||||
"chainsaw": ["chainsaw@0.1.0", "", { "dependencies": { "traverse": ">=0.3.0 <0.4" } }, "sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ=="],
|
||||
|
||||
"cluster-key-slot": ["cluster-key-slot@1.1.2", "", {}, "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA=="],
|
||||
|
||||
"codepage": ["codepage@1.15.0", "", {}, "sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA=="],
|
||||
|
||||
"compress-commons": ["compress-commons@4.1.2", "", { "dependencies": { "buffer-crc32": "^0.2.13", "crc32-stream": "^4.0.2", "normalize-path": "^3.0.0", "readable-stream": "^3.6.0" } }, "sha512-D3uMHtGc/fcO1Gt1/L7i1e33VOvD4A9hfQLP+6ewd+BvG/gQ84Yh4oftEhAdjSMgBgwGL+jsppT7JYNpo6MHHg=="],
|
||||
|
||||
"concat-map": ["concat-map@0.0.1", "", {}, "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg=="],
|
||||
|
||||
"core-util-is": ["core-util-is@1.0.3", "", {}, "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ=="],
|
||||
|
||||
"crc-32": ["crc-32@1.2.2", "", { "bin": { "crc32": "bin/crc32.njs" } }, "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ=="],
|
||||
|
||||
"crc32-stream": ["crc32-stream@4.0.3", "", { "dependencies": { "crc-32": "^1.2.0", "readable-stream": "^3.4.0" } }, "sha512-NT7w2JVU7DFroFdYkeq8cywxrgjPHWkdX1wjpRQXPX5Asews3tA+Ght6lddQO5Mkumffp3X7GEqku3epj2toIw=="],
|
||||
|
||||
"csstype": ["csstype@3.2.3", "", {}, "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ=="],
|
||||
|
||||
"csvtojson": ["csvtojson@2.0.14", "", { "dependencies": { "lodash": "^4.17.21" }, "bin": { "csvtojson": "bin/csvtojson" } }, "sha512-F7NNvhhDyob7OsuEGRsH0FM1aqLs/WYITyza3l+hTEEmOK9sGPBlYQZwlVG0ezCojXYpE17lhS5qL6BCOZSPyA=="],
|
||||
|
||||
"dayjs": ["dayjs@1.11.20", "", {}, "sha512-YbwwqR/uYpeoP4pu043q+LTDLFBLApUP6VxRihdfNTqu4ubqMlGDLd6ErXhEgsyvY0K6nCs7nggYumAN+9uEuQ=="],
|
||||
|
||||
"debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="],
|
||||
|
||||
"denque": ["denque@2.1.0", "", {}, "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw=="],
|
||||
|
||||
"dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="],
|
||||
|
||||
"duplexer2": ["duplexer2@0.1.4", "", { "dependencies": { "readable-stream": "^2.0.2" } }, "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA=="],
|
||||
|
||||
"end-of-stream": ["end-of-stream@1.4.5", "", { "dependencies": { "once": "^1.4.0" } }, "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg=="],
|
||||
|
||||
"es-define-property": ["es-define-property@1.0.1", "", {}, "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g=="],
|
||||
|
||||
"es-errors": ["es-errors@1.3.0", "", {}, "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw=="],
|
||||
|
||||
"es-object-atoms": ["es-object-atoms@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0" } }, "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA=="],
|
||||
|
||||
"exceljs": ["exceljs@4.4.0", "", { "dependencies": { "archiver": "^5.0.0", "dayjs": "^1.8.34", "fast-csv": "^4.3.1", "jszip": "^3.10.1", "readable-stream": "^3.6.0", "saxes": "^5.0.1", "tmp": "^0.2.0", "unzipper": "^0.10.11", "uuid": "^8.3.0" } }, "sha512-XctvKaEMaj1Ii9oDOqbW/6e1gXknSY4g/aLCDicOXqBE4M0nRWkUu0PTp++UPNzoFY12BNHMfs/VadKIS6llvg=="],
|
||||
|
||||
"fast-csv": ["fast-csv@4.3.6", "", { "dependencies": { "@fast-csv/format": "4.3.5", "@fast-csv/parse": "4.3.6" } }, "sha512-2RNSpuwwsJGP0frGsOmTb9oUF+VkFSM4SyLTDgwf2ciHWTarN0lQTC+F2f/t5J9QjW+c65VFIAAu85GsvMIusw=="],
|
||||
|
||||
"fast-xml-builder": ["fast-xml-builder@1.1.4", "", { "dependencies": { "path-expression-matcher": "^1.1.3" } }, "sha512-f2jhpN4Eccy0/Uz9csxh3Nu6q4ErKxf0XIsasomfOihuSUa3/xw6w8dnOtCDgEItQFJG8KyXPzQXzcODDrrbOg=="],
|
||||
|
||||
"fast-xml-parser": ["fast-xml-parser@5.5.11", "", { "dependencies": { "fast-xml-builder": "^1.1.4", "path-expression-matcher": "^1.4.0", "strnum": "^2.2.3" }, "bin": { "fxparser": "src/cli/cli.js" } }, "sha512-QL0eb0YbSTVWF6tTf1+LEMSgtCEjBYPpnAjoLC8SscESlAjXEIRJ7cHtLG0pLeDFaZLa4VKZLArtA/60ZS7vyA=="],
|
||||
|
||||
"frac": ["frac@1.1.2", "", {}, "sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA=="],
|
||||
|
||||
"fs-constants": ["fs-constants@1.0.0", "", {}, "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="],
|
||||
|
||||
"fs.realpath": ["fs.realpath@1.0.0", "", {}, "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw=="],
|
||||
|
||||
"fstream": ["fstream@1.0.12", "", { "dependencies": { "graceful-fs": "^4.1.2", "inherits": "~2.0.0", "mkdirp": ">=0.5 0", "rimraf": "2" } }, "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg=="],
|
||||
|
||||
"function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="],
|
||||
|
||||
"get-intrinsic": ["get-intrinsic@1.3.0", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "es-define-property": "^1.0.1", "es-errors": "^1.3.0", "es-object-atoms": "^1.1.1", "function-bind": "^1.1.2", "get-proto": "^1.0.1", "gopd": "^1.2.0", "has-symbols": "^1.1.0", "hasown": "^2.0.2", "math-intrinsics": "^1.1.0" } }, "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ=="],
|
||||
|
||||
"get-proto": ["get-proto@1.0.1", "", { "dependencies": { "dunder-proto": "^1.0.1", "es-object-atoms": "^1.0.0" } }, "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g=="],
|
||||
|
||||
"glob": ["glob@7.2.3", "", { "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", "minimatch": "^3.1.1", "once": "^1.3.0", "path-is-absolute": "^1.0.0" } }, "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q=="],
|
||||
|
||||
"gopd": ["gopd@1.2.0", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="],
|
||||
|
||||
"graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="],
|
||||
|
||||
"has-symbols": ["has-symbols@1.1.0", "", {}, "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ=="],
|
||||
|
||||
"hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="],
|
||||
|
||||
"iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="],
|
||||
|
||||
"ieee754": ["ieee754@1.2.1", "", {}, "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="],
|
||||
|
||||
"immediate": ["immediate@3.0.6", "", {}, "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ=="],
|
||||
|
||||
"inflight": ["inflight@1.0.6", "", { "dependencies": { "once": "^1.3.0", "wrappy": "1" } }, "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA=="],
|
||||
|
||||
"inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="],
|
||||
|
||||
"ioredis": ["ioredis@5.10.1", "", { "dependencies": { "@ioredis/commands": "1.5.1", "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", "denque": "^2.1.0", "lodash.defaults": "^4.2.0", "lodash.isarguments": "^3.1.0", "redis-errors": "^1.2.0", "redis-parser": "^3.0.0", "standard-as-callback": "^2.1.0" } }, "sha512-HuEDBTI70aYdx1v6U97SbNx9F1+svQKBDo30o0b9fw055LMepzpOOd0Ccg9Q6tbqmBSJaMuY0fB7yw9/vjBYCA=="],
|
||||
|
||||
"isarray": ["isarray@1.0.0", "", {}, "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="],
|
||||
|
||||
"jszip": ["jszip@3.10.1", "", { "dependencies": { "lie": "~3.3.0", "pako": "~1.0.2", "readable-stream": "~2.3.6", "setimmediate": "^1.0.5" } }, "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g=="],
|
||||
|
||||
"lazystream": ["lazystream@1.0.1", "", { "dependencies": { "readable-stream": "^2.0.5" } }, "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw=="],
|
||||
|
||||
"lie": ["lie@3.3.0", "", { "dependencies": { "immediate": "~3.0.5" } }, "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ=="],
|
||||
|
||||
"listenercount": ["listenercount@1.0.1", "", {}, "sha512-3mk/Zag0+IJxeDrxSgaDPy4zZ3w05PRZeJNnlWhzFz5OkX49J4krc+A8X2d2M69vGMBEX0uyl8M+W+8gH+kBqQ=="],
|
||||
|
||||
"lodash": ["lodash@4.18.1", "", {}, "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q=="],
|
||||
|
||||
"lodash.defaults": ["lodash.defaults@4.2.0", "", {}, "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ=="],
|
||||
|
||||
"lodash.difference": ["lodash.difference@4.5.0", "", {}, "sha512-dS2j+W26TQ7taQBGN8Lbbq04ssV3emRw4NY58WErlTO29pIqS0HmoT5aJ9+TUQ1N3G+JOZSji4eugsWwGp9yPA=="],
|
||||
|
||||
"lodash.escaperegexp": ["lodash.escaperegexp@4.1.2", "", {}, "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw=="],
|
||||
|
||||
"lodash.flatten": ["lodash.flatten@4.4.0", "", {}, "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g=="],
|
||||
|
||||
"lodash.groupby": ["lodash.groupby@4.6.0", "", {}, "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw=="],
|
||||
|
||||
"lodash.isarguments": ["lodash.isarguments@3.1.0", "", {}, "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg=="],
|
||||
|
||||
"lodash.isboolean": ["lodash.isboolean@3.0.3", "", {}, "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg=="],
|
||||
|
||||
"lodash.isequal": ["lodash.isequal@4.5.0", "", {}, "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ=="],
|
||||
|
||||
"lodash.isfunction": ["lodash.isfunction@3.0.9", "", {}, "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw=="],
|
||||
|
||||
"lodash.isnil": ["lodash.isnil@4.0.0", "", {}, "sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng=="],
|
||||
|
||||
"lodash.isplainobject": ["lodash.isplainobject@4.0.6", "", {}, "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA=="],
|
||||
|
||||
"lodash.isundefined": ["lodash.isundefined@3.0.1", "", {}, "sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA=="],
|
||||
|
||||
"lodash.union": ["lodash.union@4.6.0", "", {}, "sha512-c4pB2CdGrGdjMKYLA+XiRDO7Y0PRQbm/Gzg8qMj+QH+pFVAoTp5sBpO0odL3FjoPCGjK96p6qsP+yQoiLoOBcw=="],
|
||||
|
||||
"lodash.uniq": ["lodash.uniq@4.5.0", "", {}, "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ=="],
|
||||
|
||||
"math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="],
|
||||
|
||||
"minimatch": ["minimatch@5.1.9", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw=="],
|
||||
|
||||
"mkdirp": ["mkdirp@3.0.1", "", { "bin": { "mkdirp": "dist/cjs/src/bin.js" } }, "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg=="],
|
||||
|
||||
"ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
|
||||
|
||||
"normalize-path": ["normalize-path@3.0.0", "", {}, "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA=="],
|
||||
|
||||
"object-inspect": ["object-inspect@1.13.4", "", {}, "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew=="],
|
||||
|
||||
"once": ["once@1.4.0", "", { "dependencies": { "wrappy": "1" } }, "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w=="],
|
||||
|
||||
"pako": ["pako@1.0.11", "", {}, "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw=="],
|
||||
|
||||
"path-expression-matcher": ["path-expression-matcher@1.4.0", "", {}, "sha512-s4DQMxIdhj3jLFWd9LxHOplj4p9yQ4ffMGowFf3cpEgrrJjEhN0V5nxw4Ye1EViAGDoL4/1AeO6qHpqYPOzE4Q=="],
|
||||
|
||||
"path-is-absolute": ["path-is-absolute@1.0.1", "", {}, "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg=="],
|
||||
|
||||
"process-nextick-args": ["process-nextick-args@2.0.1", "", {}, "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="],
|
||||
|
||||
"qs": ["qs@6.15.1", "", { "dependencies": { "side-channel": "^1.1.0" } }, "sha512-6YHEFRL9mfgcAvql/XhwTvf5jKcOiiupt2FiJxHkiX1z4j7WL8J/jRHYLluORvc1XxB5rV20KoeK00gVJamspg=="],
|
||||
|
||||
"react": ["react@19.2.5", "", {}, "sha512-llUJLzz1zTUBrskt2pwZgLq59AemifIftw4aB7JxOqf1HY2FDaGDxgwpAPVzHU1kdWabH7FauP4i1oEeer2WCA=="],
|
||||
|
||||
"react-dom": ["react-dom@19.2.5", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.5" } }, "sha512-J5bAZz+DXMMwW/wV3xzKke59Af6CHY7G4uYLN1OvBcKEsWOs4pQExj86BBKamxl/Ik5bx9whOrvBlSDfWzgSag=="],
|
||||
|
||||
"readable-stream": ["readable-stream@3.6.2", "", { "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } }, "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA=="],
|
||||
|
||||
"readdir-glob": ["readdir-glob@1.1.3", "", { "dependencies": { "minimatch": "^5.1.0" } }, "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA=="],
|
||||
|
||||
"redis-errors": ["redis-errors@1.2.0", "", {}, "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w=="],
|
||||
|
||||
"redis-parser": ["redis-parser@3.0.0", "", { "dependencies": { "redis-errors": "^1.0.0" } }, "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A=="],
|
||||
|
||||
"rimraf": ["rimraf@2.7.1", "", { "dependencies": { "glob": "^7.1.3" }, "bin": { "rimraf": "./bin.js" } }, "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w=="],
|
||||
|
||||
"safe-buffer": ["safe-buffer@5.1.2", "", {}, "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="],
|
||||
|
||||
"safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="],
|
||||
|
||||
"saxes": ["saxes@5.0.1", "", { "dependencies": { "xmlchars": "^2.2.0" } }, "sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw=="],
|
||||
|
||||
"scheduler": ["scheduler@0.27.0", "", {}, "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q=="],
|
||||
|
||||
"setimmediate": ["setimmediate@1.0.5", "", {}, "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA=="],
|
||||
|
||||
"side-channel": ["side-channel@1.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3", "side-channel-list": "^1.0.0", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" } }, "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw=="],
|
||||
|
||||
"side-channel-list": ["side-channel-list@1.0.1", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.4" } }, "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w=="],
|
||||
@@ -128,16 +259,68 @@
|
||||
|
||||
"standard-as-callback": ["standard-as-callback@2.1.0", "", {}, "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A=="],
|
||||
|
||||
"string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="],
|
||||
|
||||
"strnum": ["strnum@2.2.3", "", {}, "sha512-oKx6RUCuHfT3oyVjtnrmn19H1SiCqgJSg+54XqURKp5aCMbrXrhLjRN9TjuwMjiYstZ0MzDrHqkGZ5dFTKd+zg=="],
|
||||
|
||||
"tar-stream": ["tar-stream@2.2.0", "", { "dependencies": { "bl": "^4.0.3", "end-of-stream": "^1.4.1", "fs-constants": "^1.0.0", "inherits": "^2.0.3", "readable-stream": "^3.1.1" } }, "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ=="],
|
||||
|
||||
"tmp": ["tmp@0.2.5", "", {}, "sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow=="],
|
||||
|
||||
"traverse": ["traverse@0.3.9", "", {}, "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ=="],
|
||||
|
||||
"typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
|
||||
|
||||
"undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="],
|
||||
|
||||
"unzipper": ["unzipper@0.10.14", "", { "dependencies": { "big-integer": "^1.6.17", "binary": "~0.3.0", "bluebird": "~3.4.1", "buffer-indexof-polyfill": "~1.0.0", "duplexer2": "~0.1.4", "fstream": "^1.0.12", "graceful-fs": "^4.2.2", "listenercount": "~1.0.1", "readable-stream": "~2.3.6", "setimmediate": "~1.0.4" } }, "sha512-ti4wZj+0bQTiX2KmKWuwj7lhV+2n//uXEotUmGuQqrbVZSEGFMbI68+c6JCQ8aAmUWYvtHEz2A8K6wXvueR/6g=="],
|
||||
|
||||
"util-deprecate": ["util-deprecate@1.0.2", "", {}, "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="],
|
||||
|
||||
"uuid": ["uuid@8.3.2", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg=="],
|
||||
|
||||
"wmf": ["wmf@1.0.2", "", {}, "sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw=="],
|
||||
|
||||
"word": ["word@0.3.0", "", {}, "sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA=="],
|
||||
|
||||
"wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="],
|
||||
|
||||
"xlsx": ["xlsx@0.18.5", "", { "dependencies": { "adler-32": "~1.3.0", "cfb": "~1.2.1", "codepage": "~1.15.0", "crc-32": "~1.2.1", "ssf": "~0.11.2", "wmf": "~1.0.1", "word": "~0.3.0" }, "bin": { "xlsx": "bin/xlsx.njs" } }, "sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ=="],
|
||||
|
||||
"xmlchars": ["xmlchars@2.2.0", "", {}, "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw=="],
|
||||
|
||||
"zip-stream": ["zip-stream@4.1.1", "", { "dependencies": { "archiver-utils": "^3.0.4", "compress-commons": "^4.1.2", "readable-stream": "^3.6.0" } }, "sha512-9qv4rlDiopXg4E69k+vMHjNN63YFMe9sZMrdlvKnCjlCRWeCBswPPMPUfx+ipsAWq1LXHe70RcbaHdJJpS6hyQ=="],
|
||||
|
||||
"@fast-csv/format/@types/node": ["@types/node@14.18.63", "", {}, "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ=="],
|
||||
|
||||
"@fast-csv/parse/@types/node": ["@types/node@14.18.63", "", {}, "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ=="],
|
||||
|
||||
"archiver-utils/readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="],
|
||||
|
||||
"duplexer2/readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="],
|
||||
|
||||
"glob/minimatch": ["minimatch@3.1.5", "", { "dependencies": { "brace-expansion": "^1.1.7" } }, "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w=="],
|
||||
|
||||
"jszip/readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="],
|
||||
|
||||
"lazystream/readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="],
|
||||
|
||||
"string_decoder/safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="],
|
||||
|
||||
"unzipper/readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="],
|
||||
|
||||
"zip-stream/archiver-utils": ["archiver-utils@3.0.4", "", { "dependencies": { "glob": "^7.2.3", "graceful-fs": "^4.2.0", "lazystream": "^1.0.0", "lodash.defaults": "^4.2.0", "lodash.difference": "^4.5.0", "lodash.flatten": "^4.4.0", "lodash.isplainobject": "^4.0.6", "lodash.union": "^4.6.0", "normalize-path": "^3.0.0", "readable-stream": "^3.6.0" } }, "sha512-KVgf4XQVrTjhyWmx6cte4RxonPLR9onExufI1jhvw/MQ4BB6IsZD5gT8Lq+u/+pRkWna/6JoHpiQioaqFP5Rzw=="],
|
||||
|
||||
"archiver-utils/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="],
|
||||
|
||||
"duplexer2/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="],
|
||||
|
||||
"glob/minimatch/brace-expansion": ["brace-expansion@1.1.14", "", { "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g=="],
|
||||
|
||||
"jszip/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="],
|
||||
|
||||
"lazystream/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="],
|
||||
|
||||
"unzipper/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="],
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,8 @@
|
||||
"scripts": {
|
||||
"bestsellers": "bun run src/bestsellers-by-category.ts",
|
||||
"monthly-sold": "bun run src/top-monthly-sold-by-category.ts",
|
||||
"upc": "bun run src/upc-lookup.ts",
|
||||
"upc-file": "bun run src/upc-file-analysis.ts",
|
||||
"start": "bun run src/index.ts",
|
||||
"start:web": "bun --hot src/server.ts",
|
||||
"build:web": "bun build src/web/index.html --outdir dist",
|
||||
@@ -21,6 +23,7 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"amazon-sp-api": "^1.2.1",
|
||||
"exceljs": "^4.4.0",
|
||||
"ioredis": "^5.10.1",
|
||||
"react": "^19.2.0",
|
||||
"react-dom": "^19.2.0",
|
||||
|
||||
256
src/analysis-pipeline.ts
Normal file
256
src/analysis-pipeline.ts
Normal file
@@ -0,0 +1,256 @@
|
||||
import { fetchKeepaDataBatch } from "./keepa.ts";
|
||||
import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts";
|
||||
import { getCache, setCache } from "./cache.ts";
|
||||
import { analyzeProducts } from "./llm.ts";
|
||||
import type {
|
||||
AnalysisResult,
|
||||
EnrichedProduct,
|
||||
KeepaData,
|
||||
ProductRecord,
|
||||
SellabilityInfo,
|
||||
SpApiData,
|
||||
} from "./types.ts";
|
||||
|
||||
export const DEFAULT_LLM_BATCH_SIZE = 5;
|
||||
export const DEFAULT_PRICING_CONCURRENCY = 5;
|
||||
|
||||
export type AnalysisPipelineOptions = {
|
||||
llmBatchSize?: number;
|
||||
pricingConcurrency?: number;
|
||||
llmBatchDelayMs?: number;
|
||||
llmRetryDelayMs?: number;
|
||||
};
|
||||
|
||||
export function chunkArray<T>(items: T[], chunkSize: number): T[][] {
|
||||
const chunks: T[][] = [];
|
||||
for (let i = 0; i < items.length; i += chunkSize) {
|
||||
chunks.push(items.slice(i, i + chunkSize));
|
||||
}
|
||||
return chunks;
|
||||
}
|
||||
|
||||
function wait(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function unknownSpApiData(reason: string): SpApiData {
|
||||
return {
|
||||
fbaFee: 5.0,
|
||||
fbmFee: 1.5,
|
||||
referralFeePercent: 15,
|
||||
estimatedSalePrice: 0,
|
||||
canSell: null,
|
||||
sellabilityStatus: "unknown",
|
||||
sellabilityReason: reason,
|
||||
};
|
||||
}
|
||||
|
||||
export async function processProductChunk(
|
||||
products: ProductRecord[],
|
||||
options: AnalysisPipelineOptions = {},
|
||||
): Promise<AnalysisResult[]> {
|
||||
const llmBatchSize = options.llmBatchSize ?? DEFAULT_LLM_BATCH_SIZE;
|
||||
const pricingConcurrency = Math.max(
|
||||
1,
|
||||
options.pricingConcurrency ?? DEFAULT_PRICING_CONCURRENCY,
|
||||
);
|
||||
const llmBatchDelayMs = Math.max(0, options.llmBatchDelayMs ?? 5_000);
|
||||
const llmRetryDelayMs = Math.max(0, options.llmRetryDelayMs ?? 10_000);
|
||||
|
||||
console.log(`\nChecking cache for ${products.length} products...`);
|
||||
const cached = new Map<string, EnrichedProduct>();
|
||||
const excludedCachedAsins = new Set<string>();
|
||||
const uncachedProducts: ProductRecord[] = [];
|
||||
|
||||
for (const p of products) {
|
||||
const hit = await getCache(p.asin);
|
||||
if (hit) {
|
||||
if (hit.spApi.sellabilityStatus === "available") {
|
||||
console.log(` [cache hit] ${p.asin}`);
|
||||
cached.set(p.asin, hit);
|
||||
} else {
|
||||
excludedCachedAsins.add(p.asin);
|
||||
console.log(
|
||||
` [exclude cached] ${p.asin} - status=${hit.spApi.sellabilityStatus}`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
uncachedProducts.push(p);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(
|
||||
`${cached.size} cached available, ${excludedCachedAsins.size} cached excluded, ${uncachedProducts.length} to fetch`,
|
||||
);
|
||||
|
||||
const sellabilityMap = new Map<string, SellabilityInfo>();
|
||||
const availableProducts: ProductRecord[] = [];
|
||||
const unavailableProducts: ProductRecord[] = [];
|
||||
|
||||
if (uncachedProducts.length > 0) {
|
||||
console.log(
|
||||
`\nChecking sellability for ${uncachedProducts.length} ASINs...`,
|
||||
);
|
||||
const sellResults = await fetchSellabilityBatch(
|
||||
uncachedProducts.map((p) => p.asin),
|
||||
);
|
||||
|
||||
for (const p of uncachedProducts) {
|
||||
const info = sellResults.get(p.asin) ?? {
|
||||
canSell: null,
|
||||
sellabilityStatus: "unknown" as const,
|
||||
sellabilityReason: "Sellability check returned no result",
|
||||
};
|
||||
sellabilityMap.set(p.asin, info);
|
||||
|
||||
if (info.sellabilityStatus === "available") {
|
||||
availableProducts.push(p);
|
||||
console.log(
|
||||
` [available] ${p.asin} - status=${info.sellabilityStatus}`,
|
||||
);
|
||||
} else {
|
||||
unavailableProducts.push(p);
|
||||
console.log(
|
||||
` [exclude] ${p.asin} - status=${info.sellabilityStatus}, reason=${info.sellabilityReason ?? "n/a"}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(
|
||||
`\nSellability gate: ${availableProducts.length} available, ${unavailableProducts.length} excluded`,
|
||||
);
|
||||
}
|
||||
|
||||
let keepaResults = new Map<string, KeepaData>();
|
||||
if (availableProducts.length > 0) {
|
||||
console.log(`\nFetching ${availableProducts.length} ASINs from Keepa...`);
|
||||
try {
|
||||
keepaResults = await fetchKeepaDataBatch(
|
||||
availableProducts.map((p) => p.asin),
|
||||
);
|
||||
} catch (err) {
|
||||
console.warn(`Keepa batch fetch failed: ${err}`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(
|
||||
`\nFetching pricing & fees for ${availableProducts.length} ASINs...`,
|
||||
);
|
||||
const spApiResults = new Map<string, SpApiData>();
|
||||
const pricingQueue = [...availableProducts];
|
||||
let pricingDone = 0;
|
||||
|
||||
async function fetchNextPricing(): Promise<void> {
|
||||
while (pricingQueue.length > 0) {
|
||||
const p = pricingQueue.shift();
|
||||
if (!p) return;
|
||||
|
||||
const sellability = sellabilityMap.get(p.asin) ?? {
|
||||
canSell: null,
|
||||
sellabilityStatus: "unknown" as const,
|
||||
sellabilityReason: "Sellability check returned no result",
|
||||
};
|
||||
const spApi = await fetchSpApiPricingAndFees(p.asin, sellability);
|
||||
|
||||
const keepa = keepaResults.get(p.asin);
|
||||
if (keepa?.currentPrice && spApi.estimatedSalePrice === 0) {
|
||||
spApi.estimatedSalePrice = keepa.currentPrice;
|
||||
}
|
||||
|
||||
spApiResults.set(p.asin, spApi);
|
||||
pricingDone++;
|
||||
if (pricingDone % 10 === 0 || pricingDone === availableProducts.length) {
|
||||
console.log(
|
||||
` [pricing] ${pricingDone}/${availableProducts.length} fetched`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const pricingWorkers = Array.from(
|
||||
{ length: Math.min(pricingConcurrency, availableProducts.length || 1) },
|
||||
() => fetchNextPricing(),
|
||||
);
|
||||
await Promise.all(pricingWorkers);
|
||||
|
||||
console.log(`\nEnriching products...`);
|
||||
const enriched: EnrichedProduct[] = [];
|
||||
const availableAsins = new Set(availableProducts.map((ap) => ap.asin));
|
||||
|
||||
for (const p of products) {
|
||||
if (excludedCachedAsins.has(p.asin)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const cachedProduct = cached.get(p.asin);
|
||||
if (cachedProduct) {
|
||||
enriched.push(cachedProduct);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!availableAsins.has(p.asin)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const keepa = keepaResults.get(p.asin) ?? null;
|
||||
const spApi =
|
||||
spApiResults.get(p.asin) ?? unknownSpApiData("SP-API data missing");
|
||||
|
||||
const product: EnrichedProduct = {
|
||||
record: p,
|
||||
keepa,
|
||||
spApi,
|
||||
fetchedAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
await setCache(p.asin, product);
|
||||
enriched.push(product);
|
||||
}
|
||||
|
||||
console.log(
|
||||
`\nAnalyzing ${enriched.length} products via LLM (batch size: ${llmBatchSize})...\n`,
|
||||
);
|
||||
|
||||
const results: AnalysisResult[] = [];
|
||||
for (let i = 0; i < enriched.length; i += llmBatchSize) {
|
||||
const batch = enriched.slice(i, i + llmBatchSize);
|
||||
const batchNum = Math.floor(i / llmBatchSize) + 1;
|
||||
const totalBatches = Math.ceil(enriched.length / llmBatchSize);
|
||||
console.log(` LLM batch ${batchNum}/${totalBatches}...`);
|
||||
|
||||
if (i > 0 && llmBatchDelayMs > 0) {
|
||||
await wait(llmBatchDelayMs);
|
||||
}
|
||||
|
||||
let verdicts;
|
||||
try {
|
||||
verdicts = await analyzeProducts(batch);
|
||||
} catch {
|
||||
if (llmRetryDelayMs > 0) {
|
||||
await wait(llmRetryDelayMs);
|
||||
}
|
||||
try {
|
||||
verdicts = await analyzeProducts(batch);
|
||||
} catch {
|
||||
verdicts = null;
|
||||
}
|
||||
}
|
||||
|
||||
for (let j = 0; j < batch.length; j++) {
|
||||
const enrichedProduct = batch[j];
|
||||
if (!enrichedProduct) continue;
|
||||
|
||||
results.push({
|
||||
product: enrichedProduct,
|
||||
verdict: verdicts?.[j] ?? {
|
||||
asin: enrichedProduct.record.asin,
|
||||
verdict: "SKIP",
|
||||
confidence: 0,
|
||||
reasoning: "LLM analysis failed",
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
219
src/index.ts
219
src/index.ts
@@ -1,22 +1,12 @@
|
||||
import { readProducts } from "./reader.ts";
|
||||
import { fetchKeepaDataBatch } from "./keepa.ts";
|
||||
import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts";
|
||||
import { connectCache, getCache, setCache, disconnectCache } from "./cache.ts";
|
||||
import { analyzeProducts } from "./llm.ts";
|
||||
import { connectCache, disconnectCache } from "./cache.ts";
|
||||
import { printResults, writeResultsToDb } from "./writer.ts";
|
||||
import { initDb, closeDb } from "./database.ts";
|
||||
import { chunkArray, processProductChunk } from "./analysis-pipeline.ts";
|
||||
import path from "node:path";
|
||||
import type {
|
||||
EnrichedProduct,
|
||||
AnalysisResult,
|
||||
KeepaData,
|
||||
ProductRecord,
|
||||
SellabilityInfo,
|
||||
SpApiData,
|
||||
} from "./types.ts";
|
||||
import type { AnalysisResult } from "./types.ts";
|
||||
|
||||
const DB_PATH = "./results.db";
|
||||
const LLM_BATCH_SIZE = 5;
|
||||
const INPUT_BATCH_SIZE = 50;
|
||||
|
||||
function parseArgs(): { inputFile: string; outputFile?: string } {
|
||||
@@ -35,14 +25,6 @@ function parseArgs(): { inputFile: string; outputFile?: string } {
|
||||
return { inputFile, outputFile };
|
||||
}
|
||||
|
||||
function chunkArray<T>(items: T[], chunkSize: number): T[][] {
|
||||
const chunks: T[][] = [];
|
||||
for (let i = 0; i < items.length; i += chunkSize) {
|
||||
chunks.push(items.slice(i, i + chunkSize));
|
||||
}
|
||||
return chunks;
|
||||
}
|
||||
|
||||
function resolveBaseOutputPath(inputFile: string, outputFile?: string): string {
|
||||
if (outputFile) return outputFile;
|
||||
|
||||
@@ -50,201 +32,6 @@ function resolveBaseOutputPath(inputFile: string, outputFile?: string): string {
|
||||
return path.join(parsedInput.dir, `${parsedInput.name}_results.xlsx`);
|
||||
}
|
||||
|
||||
async function processProductChunk(
|
||||
products: ProductRecord[],
|
||||
): Promise<AnalysisResult[]> {
|
||||
console.log(`\nChecking cache for ${products.length} products...`);
|
||||
const cached = new Map<string, EnrichedProduct>();
|
||||
const excludedCachedAsins = new Set<string>();
|
||||
const uncachedProducts: ProductRecord[] = [];
|
||||
|
||||
for (const p of products) {
|
||||
const hit = await getCache(p.asin);
|
||||
if (hit) {
|
||||
if (hit.spApi.sellabilityStatus === "available") {
|
||||
console.log(` [cache hit] ${p.asin}`);
|
||||
cached.set(p.asin, hit);
|
||||
} else {
|
||||
excludedCachedAsins.add(p.asin);
|
||||
console.log(
|
||||
` [exclude cached] ${p.asin} — status=${hit.spApi.sellabilityStatus}`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
uncachedProducts.push(p);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(
|
||||
`${cached.size} cached available, ${excludedCachedAsins.size} cached excluded, ${uncachedProducts.length} to fetch`,
|
||||
);
|
||||
|
||||
const sellabilityMap = new Map<string, SellabilityInfo>();
|
||||
const availableProducts: ProductRecord[] = [];
|
||||
const unavailableProducts: ProductRecord[] = [];
|
||||
|
||||
if (uncachedProducts.length > 0) {
|
||||
console.log(
|
||||
`\nChecking sellability for ${uncachedProducts.length} ASINs...`,
|
||||
);
|
||||
const sellResults = await fetchSellabilityBatch(
|
||||
uncachedProducts.map((p) => p.asin),
|
||||
);
|
||||
|
||||
for (const p of uncachedProducts) {
|
||||
const info = sellResults.get(p.asin) ?? {
|
||||
canSell: null,
|
||||
sellabilityStatus: "unknown" as const,
|
||||
sellabilityReason: "Sellability check returned no result",
|
||||
};
|
||||
sellabilityMap.set(p.asin, info);
|
||||
|
||||
if (info.sellabilityStatus === "available") {
|
||||
availableProducts.push(p);
|
||||
console.log(` [available] ${p.asin} — status=${info.sellabilityStatus}`);
|
||||
} else {
|
||||
unavailableProducts.push(p);
|
||||
console.log(
|
||||
` [exclude] ${p.asin} — status=${info.sellabilityStatus}, reason=${info.sellabilityReason ?? "n/a"}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(
|
||||
`\nSellability gate: ${availableProducts.length} available, ${unavailableProducts.length} excluded`,
|
||||
);
|
||||
}
|
||||
|
||||
let keepaResults = new Map<string, KeepaData>();
|
||||
if (availableProducts.length > 0) {
|
||||
console.log(`\nFetching ${availableProducts.length} ASINs from Keepa...`);
|
||||
try {
|
||||
keepaResults = await fetchKeepaDataBatch(
|
||||
availableProducts.map((p) => p.asin),
|
||||
);
|
||||
} catch (err) {
|
||||
console.warn(`Keepa batch fetch failed: ${err}`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(
|
||||
`\nFetching pricing & fees for ${availableProducts.length} ASINs...`,
|
||||
);
|
||||
const spApiResults = new Map<string, SpApiData>();
|
||||
const pricingQueue = [...availableProducts];
|
||||
let pricingDone = 0;
|
||||
|
||||
async function fetchNextPricing(): Promise<void> {
|
||||
while (pricingQueue.length > 0) {
|
||||
const p = pricingQueue.shift()!;
|
||||
const sellability = sellabilityMap.get(p.asin)!;
|
||||
const spApi = await fetchSpApiPricingAndFees(p.asin, sellability);
|
||||
|
||||
const keepa = keepaResults.get(p.asin);
|
||||
if (keepa?.currentPrice && spApi.estimatedSalePrice === 0) {
|
||||
spApi.estimatedSalePrice = keepa.currentPrice;
|
||||
}
|
||||
|
||||
spApiResults.set(p.asin, spApi);
|
||||
pricingDone++;
|
||||
if (pricingDone % 10 === 0 || pricingDone === availableProducts.length) {
|
||||
console.log(
|
||||
` [pricing] ${pricingDone}/${availableProducts.length} fetched`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const pricingWorkers = Array.from(
|
||||
{ length: Math.min(5, availableProducts.length || 1) },
|
||||
() => fetchNextPricing(),
|
||||
);
|
||||
await Promise.all(pricingWorkers);
|
||||
|
||||
console.log(`\nEnriching products...`);
|
||||
const enriched: EnrichedProduct[] = [];
|
||||
const availableAsins = new Set(availableProducts.map((ap) => ap.asin));
|
||||
|
||||
for (const p of products) {
|
||||
if (excludedCachedAsins.has(p.asin)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const cachedProduct = cached.get(p.asin);
|
||||
if (cachedProduct) {
|
||||
enriched.push(cachedProduct);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!availableAsins.has(p.asin)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const keepa = keepaResults.get(p.asin) ?? null;
|
||||
const spApi = spApiResults.get(p.asin) ?? {
|
||||
fbaFee: 5.0,
|
||||
fbmFee: 1.5,
|
||||
referralFeePercent: 15,
|
||||
estimatedSalePrice: 0,
|
||||
canSell: null,
|
||||
sellabilityStatus: "unknown" as const,
|
||||
sellabilityReason: "SP-API data missing",
|
||||
};
|
||||
|
||||
const product: EnrichedProduct = {
|
||||
record: p,
|
||||
keepa,
|
||||
spApi,
|
||||
fetchedAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
await setCache(p.asin, product);
|
||||
enriched.push(product);
|
||||
}
|
||||
|
||||
console.log(
|
||||
`\nAnalyzing ${enriched.length} products via LLM (batch size: ${LLM_BATCH_SIZE})...\n`,
|
||||
);
|
||||
|
||||
const results: AnalysisResult[] = [];
|
||||
for (let i = 0; i < enriched.length; i += LLM_BATCH_SIZE) {
|
||||
const batch = enriched.slice(i, i + LLM_BATCH_SIZE);
|
||||
const batchNum = Math.floor(i / LLM_BATCH_SIZE) + 1;
|
||||
const totalBatches = Math.ceil(enriched.length / LLM_BATCH_SIZE);
|
||||
console.log(` LLM batch ${batchNum}/${totalBatches}...`);
|
||||
|
||||
if (i > 0) {
|
||||
await new Promise((r) => setTimeout(r, 5000));
|
||||
}
|
||||
|
||||
let verdicts;
|
||||
try {
|
||||
verdicts = await analyzeProducts(batch);
|
||||
} catch {
|
||||
await new Promise((r) => setTimeout(r, 10_000));
|
||||
try {
|
||||
verdicts = await analyzeProducts(batch);
|
||||
} catch {
|
||||
verdicts = null;
|
||||
}
|
||||
}
|
||||
|
||||
for (let j = 0; j < batch.length; j++) {
|
||||
results.push({
|
||||
product: batch[j]!,
|
||||
verdict: verdicts?.[j] ?? {
|
||||
asin: batch[j]!.record.asin,
|
||||
verdict: "SKIP",
|
||||
confidence: 0,
|
||||
reasoning: "LLM analysis failed",
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const { inputFile, outputFile } = parseArgs();
|
||||
|
||||
|
||||
200
src/keepa.test.ts
Normal file
200
src/keepa.test.ts
Normal file
@@ -0,0 +1,200 @@
|
||||
import { afterAll, beforeEach, expect, mock, test } from "bun:test";
|
||||
import { lookupKeepaUpcs, mapUpcsToAsins } from "./keepa.ts";
|
||||
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
function makeUpc(index: number): string {
|
||||
return String(index).padStart(12, "0");
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
});
|
||||
|
||||
test("lookupKeepaUpcs marks invalid UPCs and skips API calls", async () => {
|
||||
const fetchMock = mock(async () => {
|
||||
return new Response("should not be called", { status: 500 });
|
||||
});
|
||||
globalThis.fetch = fetchMock as unknown as typeof globalThis.fetch;
|
||||
|
||||
const details = await lookupKeepaUpcs([
|
||||
"",
|
||||
"abc",
|
||||
"12345678901",
|
||||
"123456789012345",
|
||||
]);
|
||||
|
||||
expect(fetchMock.mock.calls.length).toBe(0);
|
||||
expect(details.size).toBe(4);
|
||||
expect(details.get("")?.status).toBe("invalid_upc");
|
||||
expect(details.get("abc")?.status).toBe("invalid_upc");
|
||||
expect(details.get("12345678901")?.status).toBe("invalid_upc");
|
||||
expect(details.get("123456789012345")?.status).toBe("invalid_upc");
|
||||
});
|
||||
|
||||
test("lookupKeepaUpcs returns found, not_found, and multiple_asins outcomes", async () => {
|
||||
globalThis.fetch = mock(async () => {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
products: [
|
||||
{
|
||||
asin: "B000FOUND01",
|
||||
upcList: ["012345678901"],
|
||||
stats: {
|
||||
current: [null, null, null, 1234],
|
||||
avg: [2500, null, null, 1400],
|
||||
},
|
||||
csv: [[1, 2999]],
|
||||
},
|
||||
{
|
||||
asin: "B000MULTI01",
|
||||
upcList: ["098765432109"],
|
||||
stats: {
|
||||
current: [null, null, null, 2000],
|
||||
avg: [1800, null, null, 2200],
|
||||
},
|
||||
csv: [[1, 1999]],
|
||||
},
|
||||
{
|
||||
asin: "B000MULTI02",
|
||||
upcList: ["098765432109"],
|
||||
stats: {
|
||||
current: [null, null, null, 2100],
|
||||
avg: [1850, null, null, 2250],
|
||||
},
|
||||
csv: [[1, 2099]],
|
||||
},
|
||||
],
|
||||
tokensLeft: 10,
|
||||
refillRate: 1,
|
||||
}),
|
||||
{ status: 200 },
|
||||
);
|
||||
}) as unknown as typeof globalThis.fetch;
|
||||
|
||||
const details = await lookupKeepaUpcs([
|
||||
"012345678901",
|
||||
"098765432109",
|
||||
"111111111111",
|
||||
]);
|
||||
|
||||
expect(details.get("012345678901")?.status).toBe("found");
|
||||
expect(details.get("012345678901")?.asin).toBe("B000FOUND01");
|
||||
expect(details.get("012345678901")?.keepaData?.currentPrice).toBe(29.99);
|
||||
|
||||
expect(details.get("098765432109")?.status).toBe("multiple_asins");
|
||||
expect(details.get("098765432109")?.candidateAsins).toEqual([
|
||||
"B000MULTI01",
|
||||
"B000MULTI02",
|
||||
]);
|
||||
|
||||
expect(details.get("111111111111")?.status).toBe("not_found");
|
||||
|
||||
const simpleMap = await mapUpcsToAsins([
|
||||
"012345678901",
|
||||
"098765432109",
|
||||
"111111111111",
|
||||
]);
|
||||
expect(simpleMap.get("012345678901")).toBe("B000FOUND01");
|
||||
expect(simpleMap.has("098765432109")).toBe(false);
|
||||
expect(simpleMap.has("111111111111")).toBe(false);
|
||||
});
|
||||
|
||||
test("lookupKeepaUpcs keeps partial success when one chunk fails", async () => {
|
||||
const upcs = Array.from({ length: 101 }, (_, i) => makeUpc(700000000000 + i));
|
||||
const firstChunkFirstUpc = upcs[0]!;
|
||||
const secondChunkUpc = upcs[100]!;
|
||||
|
||||
globalThis.fetch = mock(async (input: string | URL | Request) => {
|
||||
const rawUrl =
|
||||
typeof input === "string"
|
||||
? input
|
||||
: input instanceof URL
|
||||
? input.toString()
|
||||
: input.url;
|
||||
const url = new URL(rawUrl);
|
||||
const codes = (url.searchParams.get("code") ?? "").split(",");
|
||||
|
||||
if (codes.includes(firstChunkFirstUpc)) {
|
||||
return new Response("first chunk failed", { status: 500 });
|
||||
}
|
||||
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
products: [
|
||||
{
|
||||
asin: "B000LAST001",
|
||||
upcList: [secondChunkUpc],
|
||||
stats: {
|
||||
current: [null, null, null, 1000],
|
||||
avg: [1500, null, null, 1200],
|
||||
},
|
||||
csv: [[1, 1599]],
|
||||
},
|
||||
],
|
||||
tokensLeft: 10,
|
||||
refillRate: 1,
|
||||
}),
|
||||
{ status: 200 },
|
||||
);
|
||||
}) as unknown as typeof globalThis.fetch;
|
||||
|
||||
const details = await lookupKeepaUpcs(upcs);
|
||||
|
||||
expect(details.get(firstChunkFirstUpc)?.status).toBe("request_failed");
|
||||
expect(details.get(secondChunkUpc)?.status).toBe("found");
|
||||
expect(details.get(secondChunkUpc)?.asin).toBe("B000LAST001");
|
||||
|
||||
const simpleMap = await mapUpcsToAsins(upcs);
|
||||
expect(simpleMap.has(firstChunkFirstUpc)).toBe(false);
|
||||
expect(simpleMap.get(secondChunkUpc)).toBe("B000LAST001");
|
||||
});
|
||||
|
||||
test("lookupKeepaUpcs retries on 429 and succeeds after refill wait", async () => {
|
||||
const targetUpc = "123456789012";
|
||||
const fetchMock = mock(async () => {
|
||||
const callNumber = fetchMock.mock.calls.length;
|
||||
|
||||
if (callNumber === 1) {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
refillIn: 0,
|
||||
refillRate: 21,
|
||||
tokensLeft: -1,
|
||||
}),
|
||||
{ status: 429 },
|
||||
);
|
||||
}
|
||||
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
products: [
|
||||
{
|
||||
asin: "B000RETRY01",
|
||||
upcList: [targetUpc],
|
||||
stats: {
|
||||
current: [null, null, null, 1111],
|
||||
avg: [1299, null, null, 1234],
|
||||
},
|
||||
csv: [[1, 1399]],
|
||||
},
|
||||
],
|
||||
tokensLeft: 10,
|
||||
refillRate: 21,
|
||||
}),
|
||||
{ status: 200 },
|
||||
);
|
||||
});
|
||||
|
||||
globalThis.fetch = fetchMock as unknown as typeof globalThis.fetch;
|
||||
|
||||
const details = await lookupKeepaUpcs([targetUpc]);
|
||||
|
||||
expect(fetchMock.mock.calls.length).toBe(2);
|
||||
expect(details.get(targetUpc)?.status).toBe("found");
|
||||
expect(details.get(targetUpc)?.asin).toBe("B000RETRY01");
|
||||
});
|
||||
325
src/keepa.ts
325
src/keepa.ts
@@ -1,10 +1,21 @@
|
||||
import { config } from "./config.ts";
|
||||
import type { KeepaData } from "./types.ts";
|
||||
import type { KeepaData, KeepaUpcLookupDetail } from "./types.ts";
|
||||
|
||||
const KEEPA_BASE = "https://api.keepa.com";
|
||||
const MAX_ASINS_PER_REQUEST = 100;
|
||||
const MAX_CODES_PER_REQUEST = MAX_ASINS_PER_REQUEST;
|
||||
const MAX_KEEPA_RETRIES = 4;
|
||||
const KEEP_RETRY_BUFFER_MS = 250;
|
||||
const AMAZON_US_SELLER_ID = "ATVPDKIKX0DER";
|
||||
const KEEPA_MINUTES_OFFSET = 21_564_000;
|
||||
const UPC_PATTERN = /^\d{12,14}$/;
|
||||
|
||||
type KeepaApiResponse = {
|
||||
products?: Record<string, any>[];
|
||||
tokensLeft?: number;
|
||||
refillRate?: number;
|
||||
refillIn?: number;
|
||||
};
|
||||
|
||||
// Token-based rate limiting: Keepa Pro = 1 token/min regeneration.
|
||||
// Each product request costs 1 token regardless of ASIN count (up to 100).
|
||||
@@ -35,6 +46,168 @@ async function waitForToken(): Promise<void> {
|
||||
tokensLeft = 1;
|
||||
}
|
||||
|
||||
function wait(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function buildProductUrl(
|
||||
queryParam: "asin" | "code",
|
||||
values: string[],
|
||||
): string {
|
||||
const params = new URLSearchParams({
|
||||
key: config.keepaApiKey,
|
||||
domain: "1",
|
||||
stats: "90",
|
||||
buybox: "1",
|
||||
days: "90",
|
||||
});
|
||||
params.set(queryParam, values.join(","));
|
||||
return `${KEEPA_BASE}/product?${params.toString()}`;
|
||||
}
|
||||
|
||||
function updateTokenState(data: KeepaApiResponse): void {
|
||||
if (data.tokensLeft != null) tokensLeft = data.tokensLeft;
|
||||
if (data.refillRate != null) refillRate = data.refillRate;
|
||||
}
|
||||
|
||||
function computeWaitMsFromRefill(refillIn?: number): number {
|
||||
if (
|
||||
typeof refillIn === "number" &&
|
||||
Number.isFinite(refillIn) &&
|
||||
refillIn >= 0
|
||||
) {
|
||||
return Math.max(
|
||||
Math.ceil(refillIn) + KEEP_RETRY_BUFFER_MS,
|
||||
KEEP_RETRY_BUFFER_MS,
|
||||
);
|
||||
}
|
||||
|
||||
const safeRefillRate = Math.max(1, refillRate);
|
||||
return Math.ceil((1 / safeRefillRate) * 60_000) + KEEP_RETRY_BUFFER_MS;
|
||||
}
|
||||
|
||||
function parseErrorPayload(text: string): KeepaApiResponse | null {
|
||||
try {
|
||||
const parsed = JSON.parse(text) as KeepaApiResponse;
|
||||
return parsed && typeof parsed === "object" ? parsed : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchKeepaWithRetries(
|
||||
url: string,
|
||||
operationLabel: string,
|
||||
): Promise<KeepaApiResponse> {
|
||||
let lastErrorMessage = "Unknown Keepa error";
|
||||
|
||||
for (let attempt = 1; attempt <= MAX_KEEPA_RETRIES; attempt++) {
|
||||
await waitForToken();
|
||||
|
||||
const res = await fetch(url);
|
||||
lastRequestTime = Date.now();
|
||||
|
||||
if (res.ok) {
|
||||
const data = (await res.json()) as KeepaApiResponse;
|
||||
updateTokenState(data);
|
||||
return data;
|
||||
}
|
||||
|
||||
const text = await res.text();
|
||||
const payload = parseErrorPayload(text);
|
||||
if (payload) {
|
||||
updateTokenState(payload);
|
||||
}
|
||||
|
||||
lastErrorMessage = `Keepa API error ${res.status}: ${text}`;
|
||||
|
||||
if (res.status !== 429 || attempt === MAX_KEEPA_RETRIES) {
|
||||
break;
|
||||
}
|
||||
|
||||
const waitMs = computeWaitMsFromRefill(payload?.refillIn);
|
||||
tokensLeft = Math.min(tokensLeft, 0);
|
||||
console.warn(
|
||||
`Keepa throttled during ${operationLabel} (attempt ${attempt}/${MAX_KEEPA_RETRIES}). Waiting ${Math.ceil(waitMs / 1000)}s before retry...`,
|
||||
);
|
||||
await wait(waitMs);
|
||||
}
|
||||
|
||||
throw new Error(lastErrorMessage);
|
||||
}
|
||||
|
||||
function normalizeUpc(input: string): string {
|
||||
return input.trim();
|
||||
}
|
||||
|
||||
function isValidUpc(value: string): boolean {
|
||||
return UPC_PATTERN.test(value);
|
||||
}
|
||||
|
||||
function normalizeCodeFromKeepa(value: string): string {
|
||||
return value.replace(/\D/g, "");
|
||||
}
|
||||
|
||||
function collectCodes(value: unknown, target: Set<string>): void {
|
||||
if (Array.isArray(value)) {
|
||||
for (const item of value) {
|
||||
collectCodes(item, target);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (typeof value === "number" && Number.isFinite(value)) {
|
||||
const normalized = normalizeCodeFromKeepa(String(Math.trunc(value)));
|
||||
if (isValidUpc(normalized)) target.add(normalized);
|
||||
return;
|
||||
}
|
||||
|
||||
if (typeof value !== "string") {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const rawPart of value.split(/[\s,;|]+/)) {
|
||||
if (!rawPart) continue;
|
||||
const normalized = normalizeCodeFromKeepa(rawPart);
|
||||
if (isValidUpc(normalized)) target.add(normalized);
|
||||
}
|
||||
}
|
||||
|
||||
function extractUpcsFromProduct(product: Record<string, any>): string[] {
|
||||
const codes = new Set<string>();
|
||||
const candidates: unknown[] = [
|
||||
product.upcList,
|
||||
product.upc,
|
||||
product.eanList,
|
||||
product.ean,
|
||||
product.gtinList,
|
||||
product.gtin,
|
||||
];
|
||||
|
||||
for (const candidate of candidates) {
|
||||
collectCodes(candidate, codes);
|
||||
}
|
||||
|
||||
return Array.from(codes);
|
||||
}
|
||||
|
||||
function buildFailureDetail(
|
||||
upc: string,
|
||||
status: "invalid_upc" | "not_found" | "multiple_asins" | "request_failed",
|
||||
reason: string,
|
||||
candidateAsins: string[] = [],
|
||||
): KeepaUpcLookupDetail {
|
||||
return {
|
||||
requestedUpc: upc,
|
||||
normalizedUpc: upc,
|
||||
status,
|
||||
asin: null,
|
||||
candidateAsins,
|
||||
keepaData: null,
|
||||
reason,
|
||||
};
|
||||
}
|
||||
|
||||
export async function fetchKeepaDataBatch(
|
||||
asins: string[],
|
||||
): Promise<Map<string, KeepaData>> {
|
||||
@@ -43,32 +216,13 @@ export async function fetchKeepaDataBatch(
|
||||
// Split into chunks of MAX_ASINS_PER_REQUEST
|
||||
for (let i = 0; i < asins.length; i += MAX_ASINS_PER_REQUEST) {
|
||||
const chunk = asins.slice(i, i + MAX_ASINS_PER_REQUEST);
|
||||
await waitForToken();
|
||||
|
||||
const asinParam = chunk.join(",");
|
||||
const url = `${KEEPA_BASE}/product?key=${config.keepaApiKey}&domain=1&asin=${asinParam}&stats=90&buybox=1&days=90`;
|
||||
const url = buildProductUrl("asin", chunk);
|
||||
|
||||
console.log(
|
||||
`Keepa: fetching ${chunk.length} ASINs (tokens left: ${tokensLeft})...`,
|
||||
);
|
||||
|
||||
const res = await fetch(url);
|
||||
lastRequestTime = Date.now();
|
||||
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
throw new Error(`Keepa API error ${res.status}: ${text}`);
|
||||
}
|
||||
|
||||
const data = (await res.json()) as {
|
||||
products?: Record<string, any>[];
|
||||
tokensLeft?: number;
|
||||
refillRate?: number;
|
||||
};
|
||||
|
||||
// Update token state from API response
|
||||
if (data.tokensLeft != null) tokensLeft = data.tokensLeft;
|
||||
if (data.refillRate != null) refillRate = data.refillRate;
|
||||
const data = await fetchKeepaWithRetries(url, "ASIN batch fetch");
|
||||
|
||||
console.log(
|
||||
`Keepa: ${data.products?.length ?? 0} products returned, ${tokensLeft} tokens remaining (refill: ${refillRate}/min)`,
|
||||
@@ -86,6 +240,133 @@ export async function fetchKeepaDataBatch(
|
||||
return results;
|
||||
}
|
||||
|
||||
export async function lookupKeepaUpcs(
|
||||
upcs: string[],
|
||||
): Promise<Map<string, KeepaUpcLookupDetail>> {
|
||||
const details = new Map<string, KeepaUpcLookupDetail>();
|
||||
const validUpcs: string[] = [];
|
||||
const seenValid = new Set<string>();
|
||||
|
||||
for (const rawUpc of upcs) {
|
||||
const normalized = normalizeUpc(rawUpc);
|
||||
if (!isValidUpc(normalized)) {
|
||||
if (!details.has(normalized)) {
|
||||
details.set(
|
||||
normalized,
|
||||
buildFailureDetail(
|
||||
normalized,
|
||||
"invalid_upc",
|
||||
"UPC must be 12, 13, or 14 digits",
|
||||
),
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (seenValid.has(normalized)) continue;
|
||||
seenValid.add(normalized);
|
||||
validUpcs.push(normalized);
|
||||
}
|
||||
|
||||
for (let i = 0; i < validUpcs.length; i += MAX_CODES_PER_REQUEST) {
|
||||
const chunk = validUpcs.slice(i, i + MAX_CODES_PER_REQUEST);
|
||||
const chunkSet = new Set(chunk);
|
||||
const url = buildProductUrl("code", chunk);
|
||||
|
||||
console.log(
|
||||
`Keepa: mapping ${chunk.length} UPCs to ASINs (tokens left: ${tokensLeft})...`,
|
||||
);
|
||||
|
||||
try {
|
||||
const data = await fetchKeepaWithRetries(url, "UPC code lookup");
|
||||
|
||||
console.log(
|
||||
`Keepa: ${data.products?.length ?? 0} products returned for UPC query, ${tokensLeft} tokens remaining (refill: ${refillRate}/min)`,
|
||||
);
|
||||
|
||||
const byUpc = new Map<string, Map<string, KeepaData>>();
|
||||
for (const product of data.products ?? []) {
|
||||
const asin = String(product.asin ?? "").trim();
|
||||
if (!asin) continue;
|
||||
|
||||
const keepaData = parseKeepaProduct(product);
|
||||
const productUpcs = extractUpcsFromProduct(product);
|
||||
|
||||
for (const upc of productUpcs) {
|
||||
if (!chunkSet.has(upc)) continue;
|
||||
if (!byUpc.has(upc)) byUpc.set(upc, new Map());
|
||||
byUpc.get(upc)!.set(asin, keepaData);
|
||||
}
|
||||
}
|
||||
|
||||
for (const upc of chunk) {
|
||||
const asinMap = byUpc.get(upc);
|
||||
if (!asinMap || asinMap.size === 0) {
|
||||
details.set(
|
||||
upc,
|
||||
buildFailureDetail(
|
||||
upc,
|
||||
"not_found",
|
||||
"No Keepa product matched this UPC",
|
||||
),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const candidateAsins = Array.from(asinMap.keys());
|
||||
if (candidateAsins.length > 1) {
|
||||
details.set(
|
||||
upc,
|
||||
buildFailureDetail(
|
||||
upc,
|
||||
"multiple_asins",
|
||||
`UPC matched multiple ASINs (${candidateAsins.length})`,
|
||||
candidateAsins,
|
||||
),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const asin = candidateAsins[0]!;
|
||||
details.set(upc, {
|
||||
requestedUpc: upc,
|
||||
normalizedUpc: upc,
|
||||
status: "found",
|
||||
asin,
|
||||
candidateAsins: [asin],
|
||||
keepaData: asinMap.get(asin) ?? null,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
const reason = error instanceof Error ? error.message : String(error);
|
||||
console.warn(
|
||||
`Keepa UPC chunk failed (offset ${i}, size ${chunk.length}): ${reason}`,
|
||||
);
|
||||
|
||||
for (const upc of chunk) {
|
||||
details.set(upc, buildFailureDetail(upc, "request_failed", reason));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return details;
|
||||
}
|
||||
|
||||
export async function mapUpcsToAsins(
|
||||
upcs: string[],
|
||||
): Promise<Map<string, string>> {
|
||||
const details = await lookupKeepaUpcs(upcs);
|
||||
const mapping = new Map<string, string>();
|
||||
|
||||
for (const [upc, detail] of details.entries()) {
|
||||
if (detail.status === "found" && detail.asin) {
|
||||
mapping.set(upc, detail.asin);
|
||||
}
|
||||
}
|
||||
|
||||
return mapping;
|
||||
}
|
||||
|
||||
function parseKeepaProduct(product: Record<string, any>): KeepaData {
|
||||
const stats = product.stats;
|
||||
const csv = product.csv;
|
||||
|
||||
288
src/server.ts
288
src/server.ts
@@ -1,9 +1,19 @@
|
||||
import index from "./web/index.html";
|
||||
import { getDb, initDb } from "./database.ts";
|
||||
import { fetchKeepaDataBatch } from "./keepa.ts";
|
||||
import {
|
||||
fetchKeepaDataBatch,
|
||||
lookupKeepaUpcs,
|
||||
mapUpcsToAsins,
|
||||
} from "./keepa.ts";
|
||||
import { runUpcFileAnalysis } from "./upc-file-analysis.ts";
|
||||
import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts";
|
||||
import { analyzeProducts } from "./llm.ts";
|
||||
import type { EnrichedProduct, ProductRecord, SpApiData } from "./types.ts";
|
||||
import type {
|
||||
EnrichedProduct,
|
||||
KeepaUpcLookupDetail,
|
||||
ProductRecord,
|
||||
SpApiData,
|
||||
} from "./types.ts";
|
||||
|
||||
type ProcessType = "lead_analysis" | "category_analysis";
|
||||
|
||||
@@ -46,6 +56,7 @@ const DB_PATH = process.env.RESULTS_DB_PATH || "./results.db";
|
||||
const DEFAULT_PAGE_SIZE = 25;
|
||||
const MAX_PAGE_SIZE = 200;
|
||||
const ASIN_PATTERN = /^[A-Z0-9]{10}$/;
|
||||
const MAX_UPCS_PER_REQUEST = 1000;
|
||||
|
||||
initDb(DB_PATH);
|
||||
const db = getDb(DB_PATH);
|
||||
@@ -82,6 +93,188 @@ function isValidAsin(value: string): boolean {
|
||||
return ASIN_PATTERN.test(value);
|
||||
}
|
||||
|
||||
function splitRawUpcValues(input: string): string[] {
|
||||
return input
|
||||
.split(/[\s,;|]+/)
|
||||
.map((chunk) => chunk.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function collectUpcsFromUnknown(value: unknown, target: string[]): void {
|
||||
if (typeof value === "string") {
|
||||
target.push(...splitRawUpcValues(value));
|
||||
return;
|
||||
}
|
||||
|
||||
if (typeof value === "number" && Number.isFinite(value)) {
|
||||
target.push(String(Math.trunc(value)));
|
||||
return;
|
||||
}
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
for (const item of value) {
|
||||
collectUpcsFromUnknown(item, target);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeAndDedupeUpcs(values: string[]): string[] {
|
||||
const seen = new Set<string>();
|
||||
const normalized: string[] = [];
|
||||
|
||||
for (const value of values) {
|
||||
const upc = value.trim();
|
||||
if (!upc || seen.has(upc)) continue;
|
||||
seen.add(upc);
|
||||
normalized.push(upc);
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function parseUpcsFromSearchParams(params: URLSearchParams): string[] {
|
||||
const parsed: string[] = [];
|
||||
for (const value of params.getAll("upc")) {
|
||||
collectUpcsFromUnknown(value, parsed);
|
||||
}
|
||||
|
||||
const upcsValue = params.get("upcs");
|
||||
if (upcsValue) {
|
||||
collectUpcsFromUnknown(upcsValue, parsed);
|
||||
}
|
||||
|
||||
return normalizeAndDedupeUpcs(parsed);
|
||||
}
|
||||
|
||||
async function parseUpcsFromRequest(req: Request): Promise<string[]> {
|
||||
if (req.method === "GET") {
|
||||
const url = new URL(req.url);
|
||||
return parseUpcsFromSearchParams(url.searchParams);
|
||||
}
|
||||
|
||||
if (req.method !== "POST") {
|
||||
throw new Error("Method not allowed");
|
||||
}
|
||||
|
||||
let body: unknown;
|
||||
try {
|
||||
body = await req.json();
|
||||
} catch {
|
||||
throw new Error("Invalid JSON body");
|
||||
}
|
||||
|
||||
const parsed: string[] = [];
|
||||
if (body && typeof body === "object" && "upcs" in body) {
|
||||
collectUpcsFromUnknown((body as { upcs?: unknown }).upcs, parsed);
|
||||
} else {
|
||||
collectUpcsFromUnknown(body, parsed);
|
||||
}
|
||||
|
||||
return normalizeAndDedupeUpcs(parsed);
|
||||
}
|
||||
|
||||
function validateUpcRequest(upcs: string[]): string | null {
|
||||
if (upcs.length === 0) {
|
||||
return "Provide at least one UPC via query (?upc=...) or JSON body { upcs: [...] }";
|
||||
}
|
||||
|
||||
if (upcs.length > MAX_UPCS_PER_REQUEST) {
|
||||
return `Too many UPCs. Maximum allowed per request is ${MAX_UPCS_PER_REQUEST}.`;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function summarizeLookupStatuses(
|
||||
details: KeepaUpcLookupDetail[],
|
||||
): Record<string, number> {
|
||||
const counts: Record<string, number> = {};
|
||||
for (const detail of details) {
|
||||
counts[detail.status] = (counts[detail.status] ?? 0) + 1;
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
function parsePositiveIntField(
|
||||
value: unknown,
|
||||
fieldName: string,
|
||||
): number | undefined {
|
||||
if (value == null) return undefined;
|
||||
if (typeof value === "number") {
|
||||
if (!Number.isInteger(value) || value < 1) {
|
||||
throw new Error(`${fieldName} must be a positive integer`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
if (typeof value === "string" && value.trim().length > 0) {
|
||||
const parsed = Number.parseInt(value, 10);
|
||||
if (!Number.isFinite(parsed) || parsed < 1) {
|
||||
throw new Error(`${fieldName} must be a positive integer`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
throw new Error(`${fieldName} must be a positive integer`);
|
||||
}
|
||||
|
||||
type UpcFileProcessRequest = {
|
||||
inputFile: string;
|
||||
outputFile?: string;
|
||||
inputBatchSize?: number;
|
||||
upcLookupBatchSize?: number;
|
||||
maxRows?: number;
|
||||
};
|
||||
|
||||
async function parseUpcFileProcessRequest(
|
||||
req: Request,
|
||||
): Promise<UpcFileProcessRequest> {
|
||||
if (req.method !== "POST") {
|
||||
throw new Error("Method not allowed");
|
||||
}
|
||||
|
||||
let body: unknown;
|
||||
try {
|
||||
body = await req.json();
|
||||
} catch {
|
||||
throw new Error("Invalid JSON body");
|
||||
}
|
||||
|
||||
if (!body || typeof body !== "object") {
|
||||
throw new Error("Request body must be an object");
|
||||
}
|
||||
|
||||
const parsedBody = body as Record<string, unknown>;
|
||||
const inputFileValue = parsedBody.inputFile;
|
||||
if (
|
||||
typeof inputFileValue !== "string" ||
|
||||
inputFileValue.trim().length === 0
|
||||
) {
|
||||
throw new Error("inputFile is required and must be a non-empty string");
|
||||
}
|
||||
|
||||
const outputFileValue = parsedBody.outputFile;
|
||||
if (
|
||||
outputFileValue != null &&
|
||||
(typeof outputFileValue !== "string" || outputFileValue.trim().length === 0)
|
||||
) {
|
||||
throw new Error("outputFile must be a non-empty string when provided");
|
||||
}
|
||||
|
||||
return {
|
||||
inputFile: inputFileValue.trim(),
|
||||
outputFile:
|
||||
typeof outputFileValue === "string" ? outputFileValue.trim() : undefined,
|
||||
inputBatchSize: parsePositiveIntField(
|
||||
parsedBody.inputBatchSize,
|
||||
"inputBatchSize",
|
||||
),
|
||||
upcLookupBatchSize: parsePositiveIntField(
|
||||
parsedBody.upcLookupBatchSize,
|
||||
"upcLookupBatchSize",
|
||||
),
|
||||
maxRows: parsePositiveIntField(parsedBody.maxRows, "maxRows"),
|
||||
};
|
||||
}
|
||||
|
||||
function parseSort(
|
||||
sortParam: string | null,
|
||||
allowed: Set<string>,
|
||||
@@ -1074,6 +1267,97 @@ const server = Bun.serve({
|
||||
const url = new URL(req.url);
|
||||
return json(getProductList(url.searchParams));
|
||||
},
|
||||
"/api/upc/map": async (req) => {
|
||||
let upcs: string[];
|
||||
try {
|
||||
upcs = await parseUpcsFromRequest(req);
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
const status = message === "Method not allowed" ? 405 : 400;
|
||||
return json({ error: message }, status);
|
||||
}
|
||||
|
||||
const validationError = validateUpcRequest(upcs);
|
||||
if (validationError) {
|
||||
return json({ error: validationError }, 400);
|
||||
}
|
||||
|
||||
try {
|
||||
const mapping = await mapUpcsToAsins(upcs);
|
||||
const items = Array.from(mapping.entries()).map(([upc, asin]) => ({
|
||||
upc,
|
||||
asin,
|
||||
}));
|
||||
|
||||
return json({
|
||||
requested: upcs.length,
|
||||
matched: items.length,
|
||||
items,
|
||||
});
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
return json({ error: message }, 500);
|
||||
}
|
||||
},
|
||||
"/api/upc/lookup": async (req) => {
|
||||
let upcs: string[];
|
||||
try {
|
||||
upcs = await parseUpcsFromRequest(req);
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
const status = message === "Method not allowed" ? 405 : 400;
|
||||
return json({ error: message }, status);
|
||||
}
|
||||
|
||||
const validationError = validateUpcRequest(upcs);
|
||||
if (validationError) {
|
||||
return json({ error: validationError }, 400);
|
||||
}
|
||||
|
||||
try {
|
||||
const detailMap = await lookupKeepaUpcs(upcs);
|
||||
const items = Array.from(detailMap.values());
|
||||
return json({
|
||||
requested: upcs.length,
|
||||
statusCounts: summarizeLookupStatuses(items),
|
||||
items,
|
||||
});
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
return json({ error: message }, 500);
|
||||
}
|
||||
},
|
||||
"/api/process/upc-file": async (req) => {
|
||||
let parsed: UpcFileProcessRequest;
|
||||
try {
|
||||
parsed = await parseUpcFileProcessRequest(req);
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
const status =
|
||||
message === "Method not allowed"
|
||||
? 405
|
||||
: message === "Invalid JSON body"
|
||||
? 400
|
||||
: 400;
|
||||
return json({ error: message }, status);
|
||||
}
|
||||
|
||||
try {
|
||||
const summary = await runUpcFileAnalysis({
|
||||
inputFile: parsed.inputFile,
|
||||
outputFile: parsed.outputFile,
|
||||
inputBatchSize: parsed.inputBatchSize,
|
||||
upcLookupBatchSize: parsed.upcLookupBatchSize,
|
||||
maxRows: parsed.maxRows,
|
||||
dbPath: DB_PATH,
|
||||
manageResources: false,
|
||||
});
|
||||
return json(summary);
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
return json({ error: message }, 500);
|
||||
}
|
||||
},
|
||||
"/api/runs/:processType/:runId": (req) => {
|
||||
const processType = req.params.processType as ProcessType;
|
||||
const runId = Number(req.params.runId);
|
||||
|
||||
17
src/types.ts
17
src/types.ts
@@ -44,6 +44,23 @@ export interface KeepaData {
|
||||
categoryTree: string[];
|
||||
}
|
||||
|
||||
export type KeepaUpcLookupStatus =
|
||||
| "found"
|
||||
| "invalid_upc"
|
||||
| "not_found"
|
||||
| "multiple_asins"
|
||||
| "request_failed";
|
||||
|
||||
export interface KeepaUpcLookupDetail {
|
||||
requestedUpc: string;
|
||||
normalizedUpc: string;
|
||||
status: KeepaUpcLookupStatus;
|
||||
asin: string | null;
|
||||
candidateAsins: string[];
|
||||
keepaData: KeepaData | null;
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
export type SellabilityInfo = {
|
||||
canSell: boolean | null;
|
||||
sellabilityStatus: "available" | "restricted" | "not_available" | "unknown";
|
||||
|
||||
331
src/upc-file-analysis.ts
Normal file
331
src/upc-file-analysis.ts
Normal file
@@ -0,0 +1,331 @@
|
||||
import path from "node:path";
|
||||
import { lookupKeepaUpcs } from "./keepa.ts";
|
||||
import { processProductChunk, chunkArray } from "./analysis-pipeline.ts";
|
||||
import {
|
||||
processUpcFileInBatches,
|
||||
type UpcInputRow,
|
||||
} from "./upc-file-reader.ts";
|
||||
import {
|
||||
appendResultsToRun,
|
||||
printResults,
|
||||
refreshRunCountsInDb,
|
||||
startRunInDb,
|
||||
type RunCounts,
|
||||
} from "./writer.ts";
|
||||
import { initDb, closeDb } from "./database.ts";
|
||||
import { connectCache, disconnectCache } from "./cache.ts";
|
||||
import type {
|
||||
KeepaUpcLookupDetail,
|
||||
KeepaUpcLookupStatus,
|
||||
ProductRecord,
|
||||
} from "./types.ts";
|
||||
|
||||
const DB_PATH = process.env.RESULTS_DB_PATH || "./results.db";
|
||||
const DEFAULT_INPUT_BATCH_SIZE = 200;
|
||||
const DEFAULT_UPC_LOOKUP_BATCH_SIZE = 100;
|
||||
|
||||
export type UpcFileAnalysisOptions = {
|
||||
inputFile: string;
|
||||
outputFile?: string;
|
||||
inputBatchSize?: number;
|
||||
upcLookupBatchSize?: number;
|
||||
maxRows?: number;
|
||||
manageResources?: boolean;
|
||||
dbPath?: string;
|
||||
};
|
||||
|
||||
export type UpcFileAnalysisSummary = {
|
||||
runId: number;
|
||||
dbPath: string;
|
||||
inputFile: string;
|
||||
outputFile?: string;
|
||||
processedRows: number;
|
||||
matchedRows: number;
|
||||
unresolvedByStatus: Record<KeepaUpcLookupStatus, number>;
|
||||
runCounts: RunCounts;
|
||||
reader: {
|
||||
mode: "xlsx_stream" | "xlsx_fallback" | "xls_fallback";
|
||||
totalRowsSeen: number;
|
||||
emittedRows: number;
|
||||
skippedMissingUpc: number;
|
||||
skippedInvalidUpc: number;
|
||||
};
|
||||
};
|
||||
|
||||
function printUsage(): void {
|
||||
console.log("Usage:");
|
||||
console.log(
|
||||
" bun run src/upc-file-analysis.ts --input <file.xls|file.xlsx> [--out output.xlsx] [--input-batch-size 200] [--upc-lookup-batch-size 100] [--max-rows 1000]",
|
||||
);
|
||||
}
|
||||
|
||||
function parsePositiveInt(value: string | undefined, flagName: string): number {
|
||||
const parsed = Number.parseInt(String(value), 10);
|
||||
if (!Number.isFinite(parsed) || parsed < 1) {
|
||||
throw new Error(`Invalid value for ${flagName}: ${value}`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): UpcFileAnalysisOptions {
|
||||
let inputFile: string | undefined;
|
||||
let outputFile: string | undefined;
|
||||
let inputBatchSize: number | undefined;
|
||||
let upcLookupBatchSize: number | undefined;
|
||||
let maxRows: number | undefined;
|
||||
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const arg = argv[i]!;
|
||||
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
printUsage();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (arg === "--input") {
|
||||
const next = argv[i + 1];
|
||||
if (!next) throw new Error("Missing value after --input");
|
||||
inputFile = next;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--out") {
|
||||
const next = argv[i + 1];
|
||||
if (!next) throw new Error("Missing value after --out");
|
||||
outputFile = next;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--input-batch-size") {
|
||||
inputBatchSize = parsePositiveInt(argv[i + 1], "--input-batch-size");
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--upc-lookup-batch-size") {
|
||||
upcLookupBatchSize = parsePositiveInt(
|
||||
argv[i + 1],
|
||||
"--upc-lookup-batch-size",
|
||||
);
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--max-rows") {
|
||||
maxRows = parsePositiveInt(argv[i + 1], "--max-rows");
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg.startsWith("--")) {
|
||||
throw new Error(`Unknown flag: ${arg}`);
|
||||
}
|
||||
|
||||
if (!inputFile) {
|
||||
inputFile = arg;
|
||||
continue;
|
||||
}
|
||||
|
||||
throw new Error(`Unexpected positional argument: ${arg}`);
|
||||
}
|
||||
|
||||
if (!inputFile) {
|
||||
throw new Error("Missing --input <file.xls|file.xlsx>");
|
||||
}
|
||||
|
||||
return {
|
||||
inputFile,
|
||||
outputFile,
|
||||
inputBatchSize,
|
||||
upcLookupBatchSize,
|
||||
maxRows,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveDefaultOutputPath(inputFile: string): string {
|
||||
const parsedInput = path.parse(inputFile);
|
||||
return path.join(parsedInput.dir, `${parsedInput.name}_upc_results.xlsx`);
|
||||
}
|
||||
|
||||
function createStatusCounter(): Record<KeepaUpcLookupStatus, number> {
|
||||
return {
|
||||
found: 0,
|
||||
invalid_upc: 0,
|
||||
not_found: 0,
|
||||
multiple_asins: 0,
|
||||
request_failed: 0,
|
||||
};
|
||||
}
|
||||
|
||||
async function lookupUpcsWithChunking(
|
||||
rows: UpcInputRow[],
|
||||
lookupBatchSize: number,
|
||||
): Promise<Map<string, KeepaUpcLookupDetail>> {
|
||||
const uniqueUpcs = Array.from(new Set(rows.map((row) => row.upc)));
|
||||
const chunks = chunkArray(uniqueUpcs, lookupBatchSize);
|
||||
const details = new Map<string, KeepaUpcLookupDetail>();
|
||||
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
const chunk = chunks[i]!;
|
||||
console.log(
|
||||
` Keepa UPC lookup chunk ${i + 1}/${chunks.length} (${chunk.length} UPCs)...`,
|
||||
);
|
||||
|
||||
const chunkDetails = await lookupKeepaUpcs(chunk);
|
||||
for (const [upc, detail] of chunkDetails.entries()) {
|
||||
details.set(upc, detail);
|
||||
}
|
||||
}
|
||||
|
||||
return details;
|
||||
}
|
||||
|
||||
function toProductRecord(
|
||||
row: UpcInputRow,
|
||||
detail: KeepaUpcLookupDetail,
|
||||
): ProductRecord {
|
||||
const keepaCategory = detail.keepaData?.categoryTree?.[0];
|
||||
|
||||
return {
|
||||
asin: detail.asin ?? row.upc,
|
||||
name: row.name ?? detail.asin ?? row.upc,
|
||||
unitCost: row.unitCost ?? 0,
|
||||
brand: row.brand,
|
||||
category: row.category ?? keepaCategory,
|
||||
};
|
||||
}
|
||||
|
||||
export async function runUpcFileAnalysis(
|
||||
options: UpcFileAnalysisOptions,
|
||||
): Promise<UpcFileAnalysisSummary> {
|
||||
const dbPath = options.dbPath ?? DB_PATH;
|
||||
const inputBatchSize = Math.max(
|
||||
1,
|
||||
options.inputBatchSize ?? DEFAULT_INPUT_BATCH_SIZE,
|
||||
);
|
||||
const lookupBatchSize = Math.max(
|
||||
1,
|
||||
options.upcLookupBatchSize ?? DEFAULT_UPC_LOOKUP_BATCH_SIZE,
|
||||
);
|
||||
const outputFile =
|
||||
options.outputFile ?? resolveDefaultOutputPath(options.inputFile);
|
||||
const manageResources = options.manageResources ?? true;
|
||||
|
||||
if (manageResources) {
|
||||
console.log("Connecting to Redis...");
|
||||
await connectCache();
|
||||
console.log("Initializing SQLite database...");
|
||||
initDb(dbPath);
|
||||
}
|
||||
|
||||
const unresolvedByStatus = createStatusCounter();
|
||||
const printableSample = [];
|
||||
let processedRows = 0;
|
||||
let matchedRows = 0;
|
||||
|
||||
const runId = startRunInDb(dbPath, options.inputFile, outputFile);
|
||||
|
||||
try {
|
||||
const readerSummary = await processUpcFileInBatches(
|
||||
options.inputFile,
|
||||
async ({ batchNumber, rows }) => {
|
||||
console.log(
|
||||
`\n=== UPC input batch ${batchNumber} (${rows.length} rows) ===`,
|
||||
);
|
||||
|
||||
processedRows += rows.length;
|
||||
const detailMap = await lookupUpcsWithChunking(rows, lookupBatchSize);
|
||||
|
||||
const matchedProducts: ProductRecord[] = [];
|
||||
for (const row of rows) {
|
||||
const detail = detailMap.get(row.upc);
|
||||
if (!detail) {
|
||||
unresolvedByStatus.request_failed += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
unresolvedByStatus[detail.status] += 1;
|
||||
|
||||
if (detail.status === "found" && detail.asin) {
|
||||
matchedRows += 1;
|
||||
matchedProducts.push(toProductRecord(row, detail));
|
||||
}
|
||||
}
|
||||
|
||||
console.log(
|
||||
`Batch ${batchNumber}: ${matchedProducts.length}/${rows.length} rows resolved to single ASINs`,
|
||||
);
|
||||
|
||||
if (matchedProducts.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const analyzed = await processProductChunk(matchedProducts);
|
||||
appendResultsToRun(dbPath, runId, analyzed);
|
||||
|
||||
if (printableSample.length < 200) {
|
||||
const remaining = 200 - printableSample.length;
|
||||
printableSample.push(...analyzed.slice(0, remaining));
|
||||
}
|
||||
},
|
||||
{
|
||||
batchSize: inputBatchSize,
|
||||
maxRows: options.maxRows,
|
||||
},
|
||||
);
|
||||
|
||||
const runCounts = refreshRunCountsInDb(dbPath, runId);
|
||||
|
||||
if (printableSample.length > 0) {
|
||||
printResults(printableSample);
|
||||
if (runCounts.totalProducts > printableSample.length) {
|
||||
console.log(
|
||||
`Printed ${printableSample.length} sampled results out of ${runCounts.totalProducts} analyzed products.`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
console.log("No products were eligible for analysis after UPC mapping.");
|
||||
}
|
||||
|
||||
return {
|
||||
runId,
|
||||
dbPath,
|
||||
inputFile: options.inputFile,
|
||||
outputFile,
|
||||
processedRows,
|
||||
matchedRows,
|
||||
unresolvedByStatus,
|
||||
runCounts,
|
||||
reader: {
|
||||
mode: readerSummary.mode,
|
||||
totalRowsSeen: readerSummary.totalRowsSeen,
|
||||
emittedRows: readerSummary.emittedRows,
|
||||
skippedMissingUpc: readerSummary.skippedMissingUpc,
|
||||
skippedInvalidUpc: readerSummary.skippedInvalidUpc,
|
||||
},
|
||||
};
|
||||
} finally {
|
||||
if (manageResources) {
|
||||
await disconnectCache();
|
||||
closeDb();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const parsed = parseArgs(process.argv.slice(2));
|
||||
const summary = await runUpcFileAnalysis(parsed);
|
||||
|
||||
console.log("\n=== UPC file analysis summary ===");
|
||||
console.log(JSON.stringify(summary, null, 2));
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
main().catch((err) => {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.error(`UPC file analysis failed: ${message}`);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
360
src/upc-file-reader.ts
Normal file
360
src/upc-file-reader.ts
Normal file
@@ -0,0 +1,360 @@
|
||||
import ExcelJS from "exceljs";
|
||||
import * as XLSX from "xlsx";
|
||||
import path from "node:path";
|
||||
|
||||
const UPC_PATTERN = /^\d{12,14}$/;
|
||||
|
||||
const COLUMN_CANDIDATES = {
|
||||
upc: ["upc", "upc code", "upc/ean", "ean", "gtin", "barcode", "product code"],
|
||||
name: ["name", "product name", "title", "product title"],
|
||||
unitCost: ["unit cost", "cost", "price", "buy cost", "unit_cost", "unitcost"],
|
||||
brand: ["brand"],
|
||||
category: ["category"],
|
||||
} as const;
|
||||
|
||||
type ColumnKey = keyof typeof COLUMN_CANDIDATES;
|
||||
type ColumnMap = Record<ColumnKey, number | undefined>;
|
||||
|
||||
export type UpcInputRow = {
|
||||
rowNumber: number;
|
||||
upc: string;
|
||||
name?: string;
|
||||
unitCost?: number;
|
||||
brand?: string;
|
||||
category?: string;
|
||||
};
|
||||
|
||||
export type UpcInputBatch = {
|
||||
batchNumber: number;
|
||||
rows: UpcInputRow[];
|
||||
};
|
||||
|
||||
export type UpcReaderSummary = {
|
||||
filePath: string;
|
||||
mode: "xlsx_stream" | "xlsx_fallback" | "xls_fallback";
|
||||
totalRowsSeen: number;
|
||||
emittedRows: number;
|
||||
skippedMissingUpc: number;
|
||||
skippedInvalidUpc: number;
|
||||
};
|
||||
|
||||
export type UpcReaderOptions = {
|
||||
batchSize?: number;
|
||||
maxRows?: number;
|
||||
};
|
||||
|
||||
export async function processUpcFileInBatches(
|
||||
filePath: string,
|
||||
onBatch: (batch: UpcInputBatch) => Promise<void>,
|
||||
options: UpcReaderOptions = {},
|
||||
): Promise<UpcReaderSummary> {
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
|
||||
if (ext === ".xlsx") {
|
||||
try {
|
||||
return await processXlsxStreaming(filePath, onBatch, options);
|
||||
} catch (err) {
|
||||
console.warn(
|
||||
`XLSX streaming reader failed, falling back to in-memory parser: ${err}`,
|
||||
);
|
||||
return processXlsLikeFallback(
|
||||
filePath,
|
||||
onBatch,
|
||||
options,
|
||||
"xlsx_fallback",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (ext === ".xls") {
|
||||
return processXlsLikeFallback(filePath, onBatch, options, "xls_fallback");
|
||||
}
|
||||
|
||||
throw new Error(`Unsupported file extension: ${ext}. Expected .xls or .xlsx`);
|
||||
}
|
||||
|
||||
async function processXlsxStreaming(
|
||||
filePath: string,
|
||||
onBatch: (batch: UpcInputBatch) => Promise<void>,
|
||||
options: UpcReaderOptions,
|
||||
): Promise<UpcReaderSummary> {
|
||||
const batchSize = Math.max(1, options.batchSize ?? 200);
|
||||
const maxRows =
|
||||
options.maxRows && options.maxRows > 0 ? options.maxRows : null;
|
||||
|
||||
let headerDetected = false;
|
||||
let columns: ColumnMap | null = null;
|
||||
let seenRows = 0;
|
||||
let emittedRows = 0;
|
||||
let skippedMissingUpc = 0;
|
||||
let skippedInvalidUpc = 0;
|
||||
let batchNumber = 1;
|
||||
let currentBatch: UpcInputRow[] = [];
|
||||
let stop = false;
|
||||
|
||||
const flush = async () => {
|
||||
if (currentBatch.length === 0) return;
|
||||
await onBatch({ batchNumber, rows: currentBatch });
|
||||
batchNumber += 1;
|
||||
currentBatch = [];
|
||||
};
|
||||
|
||||
const workbookReader = new ExcelJS.stream.xlsx.WorkbookReader(filePath, {
|
||||
worksheets: "emit",
|
||||
sharedStrings: "cache",
|
||||
hyperlinks: "ignore",
|
||||
styles: "ignore",
|
||||
});
|
||||
|
||||
for await (const worksheet of workbookReader) {
|
||||
if (stop) break;
|
||||
|
||||
for await (const row of worksheet) {
|
||||
const values = normalizeExcelJsRow(row.values as unknown[]);
|
||||
if (!headerDetected) {
|
||||
columns = detectColumns(values);
|
||||
if (columns.upc == null) {
|
||||
throw new Error(
|
||||
`No UPC column found in header row. Header row values: ${values.join(", ")}`,
|
||||
);
|
||||
}
|
||||
headerDetected = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
seenRows += 1;
|
||||
const parsed = parseUpcInputRow(values, columns, row.number);
|
||||
if (!parsed) {
|
||||
skippedMissingUpc += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!isValidUpc(parsed.upc)) {
|
||||
skippedInvalidUpc += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
currentBatch.push(parsed);
|
||||
emittedRows += 1;
|
||||
|
||||
if (currentBatch.length >= batchSize) {
|
||||
await flush();
|
||||
}
|
||||
|
||||
if (maxRows != null && emittedRows >= maxRows) {
|
||||
stop = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Process only the first worksheet.
|
||||
break;
|
||||
}
|
||||
|
||||
await flush();
|
||||
|
||||
if (!headerDetected) {
|
||||
throw new Error("No rows found in the first worksheet.");
|
||||
}
|
||||
|
||||
return {
|
||||
filePath,
|
||||
mode: "xlsx_stream",
|
||||
totalRowsSeen: seenRows,
|
||||
emittedRows,
|
||||
skippedMissingUpc,
|
||||
skippedInvalidUpc,
|
||||
};
|
||||
}
|
||||
|
||||
function processXlsLikeFallback(
|
||||
filePath: string,
|
||||
onBatch: (batch: UpcInputBatch) => Promise<void>,
|
||||
options: UpcReaderOptions,
|
||||
mode: "xlsx_fallback" | "xls_fallback",
|
||||
): Promise<UpcReaderSummary> {
|
||||
return new Promise<UpcReaderSummary>(async (resolve, reject) => {
|
||||
try {
|
||||
const batchSize = Math.max(1, options.batchSize ?? 200);
|
||||
const maxRows =
|
||||
options.maxRows && options.maxRows > 0 ? options.maxRows : null;
|
||||
|
||||
const workbook = XLSX.readFile(filePath, { raw: true });
|
||||
const sheetName = workbook.SheetNames[0];
|
||||
if (!sheetName) throw new Error("No sheets found in file");
|
||||
const sheet = workbook.Sheets[sheetName];
|
||||
if (!sheet || !sheet["!ref"]) throw new Error("Sheet has no data");
|
||||
|
||||
const range = XLSX.utils.decode_range(sheet["!ref"]);
|
||||
const headerValues: string[] = [];
|
||||
for (let c = range.s.c; c <= range.e.c; c++) {
|
||||
const cellAddress = XLSX.utils.encode_cell({ r: range.s.r, c });
|
||||
const value = sheet[cellAddress]?.v;
|
||||
headerValues.push(normalizeOptionalString(value) ?? "");
|
||||
}
|
||||
|
||||
const columns = detectColumns(headerValues);
|
||||
if (columns.upc == null) {
|
||||
throw new Error(
|
||||
`No UPC column found in header row. Header row values: ${headerValues.join(", ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
let seenRows = 0;
|
||||
let emittedRows = 0;
|
||||
let skippedMissingUpc = 0;
|
||||
let skippedInvalidUpc = 0;
|
||||
let batchNumber = 1;
|
||||
let currentBatch: UpcInputRow[] = [];
|
||||
|
||||
const flush = async () => {
|
||||
if (currentBatch.length === 0) return;
|
||||
await onBatch({ batchNumber, rows: currentBatch });
|
||||
batchNumber += 1;
|
||||
currentBatch = [];
|
||||
};
|
||||
|
||||
for (let r = range.s.r + 1; r <= range.e.r; r++) {
|
||||
seenRows += 1;
|
||||
const rowValues: string[] = [];
|
||||
for (let c = range.s.c; c <= range.e.c; c++) {
|
||||
const cellAddress = XLSX.utils.encode_cell({ r, c });
|
||||
rowValues.push(normalizeOptionalString(sheet[cellAddress]?.v) ?? "");
|
||||
}
|
||||
|
||||
const parsed = parseUpcInputRow(rowValues, columns, r + 1);
|
||||
if (!parsed) {
|
||||
skippedMissingUpc += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!isValidUpc(parsed.upc)) {
|
||||
skippedInvalidUpc += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
currentBatch.push(parsed);
|
||||
emittedRows += 1;
|
||||
|
||||
if (currentBatch.length >= batchSize) {
|
||||
await flush();
|
||||
}
|
||||
|
||||
if (maxRows != null && emittedRows >= maxRows) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
await flush();
|
||||
|
||||
resolve({
|
||||
filePath,
|
||||
mode,
|
||||
totalRowsSeen: seenRows,
|
||||
emittedRows,
|
||||
skippedMissingUpc,
|
||||
skippedInvalidUpc,
|
||||
});
|
||||
} catch (err) {
|
||||
reject(err);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function detectColumns(headers: string[]): ColumnMap {
|
||||
const columns = {} as ColumnMap;
|
||||
for (const key of Object.keys(COLUMN_CANDIDATES) as ColumnKey[]) {
|
||||
columns[key] = findColumnIndex(headers, [...COLUMN_CANDIDATES[key]]);
|
||||
}
|
||||
return columns;
|
||||
}
|
||||
|
||||
function findColumnIndex(
|
||||
headers: string[],
|
||||
candidates: string[],
|
||||
): number | undefined {
|
||||
const normalizedCandidates = new Set(candidates.map(normalizeHeader));
|
||||
for (let i = 0; i < headers.length; i++) {
|
||||
if (normalizedCandidates.has(normalizeHeader(headers[i] ?? ""))) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function parseUpcInputRow(
|
||||
rowValues: string[],
|
||||
columns: ColumnMap,
|
||||
rowNumber: number,
|
||||
): UpcInputRow | null {
|
||||
if (columns.upc == null) return null;
|
||||
|
||||
const rawUpc = rowValues[columns.upc] ?? "";
|
||||
const upc = rawUpc.replace(/\D/g, "").trim();
|
||||
if (!upc) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
rowNumber,
|
||||
upc,
|
||||
name: getRowString(rowValues, columns.name),
|
||||
unitCost: parseOptionalNumber(rowValues[columns.unitCost ?? -1]),
|
||||
brand: getRowString(rowValues, columns.brand),
|
||||
category: getRowString(rowValues, columns.category),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeExcelJsRow(values: unknown[]): string[] {
|
||||
// ExcelJS row.values is 1-indexed with values[0] intentionally empty.
|
||||
const normalized: string[] = [];
|
||||
for (let i = 1; i < values.length; i++) {
|
||||
normalized.push(normalizeOptionalString(values[i]) ?? "");
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function getRowString(
|
||||
values: string[],
|
||||
index: number | undefined,
|
||||
): string | undefined {
|
||||
if (index == null || index < 0) return undefined;
|
||||
const value = values[index];
|
||||
return value?.trim() ? value.trim() : undefined;
|
||||
}
|
||||
|
||||
function normalizeHeader(value: string): string {
|
||||
return value
|
||||
.toLowerCase()
|
||||
.trim()
|
||||
.replace(/%/g, " pct ")
|
||||
.replace(/\$/g, " usd ")
|
||||
.replace(/[^a-z0-9]/g, "");
|
||||
}
|
||||
|
||||
function normalizeOptionalString(value: unknown): string | undefined {
|
||||
if (value == null) return undefined;
|
||||
|
||||
if (typeof value === "object") {
|
||||
if ("text" in (value as Record<string, unknown>)) {
|
||||
return normalizeOptionalString((value as { text?: unknown }).text);
|
||||
}
|
||||
if ("result" in (value as Record<string, unknown>)) {
|
||||
return normalizeOptionalString((value as { result?: unknown }).result);
|
||||
}
|
||||
}
|
||||
|
||||
const text = String(value).trim();
|
||||
return text.length > 0 ? text : undefined;
|
||||
}
|
||||
|
||||
function parseOptionalNumber(value: unknown): number | undefined {
|
||||
if (value == null || value === "") return undefined;
|
||||
const cleaned = String(value).trim().replace(/[$,%]/g, "").replace(/,/g, "");
|
||||
const parsed = Number(cleaned);
|
||||
return Number.isFinite(parsed) ? parsed : undefined;
|
||||
}
|
||||
|
||||
function isValidUpc(value: string): boolean {
|
||||
return UPC_PATTERN.test(value);
|
||||
}
|
||||
147
src/upc-lookup.ts
Normal file
147
src/upc-lookup.ts
Normal file
@@ -0,0 +1,147 @@
|
||||
import { lookupKeepaUpcs, mapUpcsToAsins } from "./keepa.ts";
|
||||
|
||||
function printUsage(): void {
|
||||
console.log("Usage:");
|
||||
console.log(
|
||||
" bun run src/upc-lookup.ts <upc...> [--detailed] [--json] [--file path]",
|
||||
);
|
||||
console.log("");
|
||||
console.log("Examples:");
|
||||
console.log(" bun run src/upc-lookup.ts 012345678901 098765432109");
|
||||
console.log(
|
||||
" bun run src/upc-lookup.ts 012345678901,098765432109 --detailed",
|
||||
);
|
||||
console.log(" bun run src/upc-lookup.ts --file upcs.txt --detailed --json");
|
||||
}
|
||||
|
||||
function splitRawUpcValues(input: string): string[] {
|
||||
return input
|
||||
.split(/[\s,;|]+/)
|
||||
.map((chunk) => chunk.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
async function readUpcsFromFile(path: string): Promise<string[]> {
|
||||
const file = Bun.file(path);
|
||||
if (!(await file.exists())) {
|
||||
throw new Error(`UPC file not found: ${path}`);
|
||||
}
|
||||
return splitRawUpcValues(await file.text());
|
||||
}
|
||||
|
||||
function parseArgs(args: string[]): {
|
||||
upcs: string[];
|
||||
filePaths: string[];
|
||||
detailed: boolean;
|
||||
asJson: boolean;
|
||||
} {
|
||||
let detailed = false;
|
||||
let asJson = false;
|
||||
const collected: string[] = [];
|
||||
const filePaths: string[] = [];
|
||||
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const arg = args[i]!;
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
printUsage();
|
||||
process.exit(0);
|
||||
}
|
||||
if (arg === "--detailed") {
|
||||
detailed = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--json") {
|
||||
asJson = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--file") {
|
||||
const next = args[i + 1];
|
||||
if (!next) {
|
||||
throw new Error("Missing file path after --file");
|
||||
}
|
||||
filePaths.push(next);
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg.startsWith("--")) {
|
||||
throw new Error(`Unknown flag: ${arg}`);
|
||||
}
|
||||
|
||||
collected.push(...splitRawUpcValues(arg));
|
||||
}
|
||||
|
||||
return {
|
||||
upcs: collected,
|
||||
filePaths,
|
||||
detailed,
|
||||
asJson,
|
||||
};
|
||||
}
|
||||
|
||||
function dedupeUpcs(upcs: string[]): string[] {
|
||||
return Array.from(new Set(upcs.map((upc) => upc.trim()).filter(Boolean)));
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = process.argv.slice(2);
|
||||
const parsed = parseArgs(args);
|
||||
|
||||
const fileUpcs: string[] = [];
|
||||
for (const path of parsed.filePaths) {
|
||||
fileUpcs.push(...(await readUpcsFromFile(path)));
|
||||
}
|
||||
|
||||
const upcs = dedupeUpcs([...parsed.upcs, ...fileUpcs]);
|
||||
if (upcs.length === 0) {
|
||||
printUsage();
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (parsed.detailed) {
|
||||
const details = await lookupKeepaUpcs(upcs);
|
||||
const items = Array.from(details.values());
|
||||
|
||||
if (parsed.asJson) {
|
||||
console.log(JSON.stringify(items, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
console.table(
|
||||
items.map((item) => ({
|
||||
upc: item.normalizedUpc,
|
||||
status: item.status,
|
||||
asin: item.asin ?? "",
|
||||
candidates: item.candidateAsins.join("|"),
|
||||
reason: item.reason ?? "",
|
||||
})),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const mapping = await mapUpcsToAsins(upcs);
|
||||
const items = Array.from(mapping.entries()).map(([upc, asin]) => ({
|
||||
upc,
|
||||
asin,
|
||||
}));
|
||||
|
||||
if (parsed.asJson) {
|
||||
console.log(JSON.stringify(items, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
if (items.length === 0) {
|
||||
console.log(
|
||||
"No one-to-one UPC to ASIN matches found. Run with --detailed for per-UPC status.",
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
console.table(items);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.error(`UPC lookup failed: ${message}`);
|
||||
process.exit(1);
|
||||
});
|
||||
108
src/writer.ts
108
src/writer.ts
@@ -1,6 +1,25 @@
|
||||
import { getDb } from "./database.ts";
|
||||
import type { AnalysisResult } from "./types.ts";
|
||||
|
||||
export type RunCounts = {
|
||||
totalProducts: number;
|
||||
fbaCount: number;
|
||||
fbmCount: number;
|
||||
skipCount: number;
|
||||
};
|
||||
|
||||
function computeRunCountsFromResults(results: AnalysisResult[]): RunCounts {
|
||||
const fbaCount = results.filter((r) => r.verdict.verdict === "FBA").length;
|
||||
const fbmCount = results.filter((r) => r.verdict.verdict === "FBM").length;
|
||||
const skipCount = results.filter((r) => r.verdict.verdict === "SKIP").length;
|
||||
return {
|
||||
totalProducts: results.length,
|
||||
fbaCount,
|
||||
fbmCount,
|
||||
skipCount,
|
||||
};
|
||||
}
|
||||
|
||||
function buildRow(r: AnalysisResult) {
|
||||
const price =
|
||||
r.product.keepa?.currentPrice ??
|
||||
@@ -68,12 +87,25 @@ export function writeResultsToDb(
|
||||
inputFile: string,
|
||||
outputFile: string | undefined,
|
||||
): void {
|
||||
const database = getDb(dbPath);
|
||||
const runCounts = computeRunCountsFromResults(results);
|
||||
const runId = startRunInDb(dbPath, inputFile, outputFile, runCounts);
|
||||
appendResultsToRun(dbPath, runId, results);
|
||||
console.log(`Results written to SQLite database for run_id: ${runId}`);
|
||||
}
|
||||
|
||||
export function startRunInDb(
|
||||
dbPath: string,
|
||||
inputFile: string,
|
||||
outputFile: string | undefined,
|
||||
counts: RunCounts = {
|
||||
totalProducts: 0,
|
||||
fbaCount: 0,
|
||||
fbmCount: 0,
|
||||
skipCount: 0,
|
||||
},
|
||||
): number {
|
||||
const database = getDb(dbPath);
|
||||
const timestamp = new Date().toISOString();
|
||||
const fbaCount = results.filter((r) => r.verdict.verdict === "FBA").length;
|
||||
const fbmCount = results.filter((r) => r.verdict.verdict === "FBM").length;
|
||||
const skipCount = results.filter((r) => r.verdict.verdict === "SKIP").length;
|
||||
|
||||
const insertRun = database.prepare(
|
||||
`INSERT INTO runs (
|
||||
@@ -86,25 +118,39 @@ export function writeResultsToDb(
|
||||
skip_count
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
||||
);
|
||||
|
||||
const runInfo = insertRun.run(
|
||||
timestamp,
|
||||
inputFile,
|
||||
outputFile ?? null,
|
||||
results.length,
|
||||
fbaCount,
|
||||
fbmCount,
|
||||
skipCount,
|
||||
counts.totalProducts,
|
||||
counts.fbaCount,
|
||||
counts.fbmCount,
|
||||
counts.skipCount,
|
||||
);
|
||||
|
||||
const runId =
|
||||
(runInfo.changes as number) > 0
|
||||
? (runInfo.lastInsertRowid as number)
|
||||
: null;
|
||||
|
||||
if (runId === null) {
|
||||
console.error("Failed to insert run record into SQLite.");
|
||||
throw new Error("Failed to insert run record into SQLite.");
|
||||
}
|
||||
|
||||
return runId;
|
||||
}
|
||||
|
||||
export function appendResultsToRun(
|
||||
dbPath: string,
|
||||
runId: number,
|
||||
results: AnalysisResult[],
|
||||
): void {
|
||||
if (results.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const database = getDb(dbPath);
|
||||
const insertResult = database.prepare(
|
||||
`INSERT INTO results (
|
||||
run_id, asin, product_name, brand, category, unit_cost, current_price,
|
||||
@@ -174,7 +220,49 @@ export function writeResultsToDb(
|
||||
);
|
||||
}
|
||||
})();
|
||||
console.log(`Results written to SQLite database for run_id: ${runId}`);
|
||||
}
|
||||
|
||||
export function refreshRunCountsInDb(dbPath: string, runId: number): RunCounts {
|
||||
const database = getDb(dbPath);
|
||||
const stats = database
|
||||
.query(
|
||||
`SELECT
|
||||
COUNT(*) AS total,
|
||||
SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba,
|
||||
SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm,
|
||||
SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip
|
||||
FROM results
|
||||
WHERE run_id = ?`,
|
||||
)
|
||||
.get(runId) as {
|
||||
total: number;
|
||||
fba: number | null;
|
||||
fbm: number | null;
|
||||
skip: number | null;
|
||||
};
|
||||
|
||||
const counts: RunCounts = {
|
||||
totalProducts: stats.total ?? 0,
|
||||
fbaCount: stats.fba ?? 0,
|
||||
fbmCount: stats.fbm ?? 0,
|
||||
skipCount: stats.skip ?? 0,
|
||||
};
|
||||
|
||||
database
|
||||
.query(
|
||||
`UPDATE runs
|
||||
SET total_products = ?, fba_count = ?, fbm_count = ?, skip_count = ?
|
||||
WHERE id = ?`,
|
||||
)
|
||||
.run(
|
||||
counts.totalProducts,
|
||||
counts.fbaCount,
|
||||
counts.fbmCount,
|
||||
counts.skipCount,
|
||||
runId,
|
||||
);
|
||||
|
||||
return counts;
|
||||
}
|
||||
export function printResults(results: AnalysisResult[]): void {
|
||||
const rows = results
|
||||
|
||||
Reference in New Issue
Block a user