diff --git a/.gitignore b/.gitignore index 17d7179..6daa5a1 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,5 @@ output/ temp_output/ dist-server/ + +*.xls diff --git a/README.md b/README.md index c5d8a4e..e0ec131 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,95 @@ bun run src/sp-test.ts B07SN9BHVV # Auth + sellers endpoint + pricing offer c bun run src/sp-test.ts --sellability B07SN9BHVV # Standalone sellability check ``` +## UPC to ASIN Mapping + +You can map UPCs to ASINs directly through the Keepa integration in `src/keepa.ts`. + +```ts +import { mapUpcsToAsins, lookupKeepaUpcs } from "./src/keepa.ts"; + +const upcs = ["012345678901", "098765432109", "112233445566"]; + +// Simple map output (UPC -> ASIN) for clean one-to-one matches only. +const asinMap = await mapUpcsToAsins(upcs); +for (const [upc, asin] of asinMap.entries()) { + console.log(`UPC ${upc} -> ASIN ${asin}`); +} + +// Rich output includes status for every UPC (invalid, not found, collisions, etc.). +const details = await lookupKeepaUpcs(upcs); +for (const [upc, detail] of details.entries()) { + console.log(upc, detail.status, detail.asin, detail.reason ?? ""); +} +``` + +Behavior: + +- Strict validation accepts only 12, 13, or 14 digit UPC values. +- If a UPC resolves to multiple ASINs, it is excluded from the simple map. +- The rich lookup returns all candidate ASINs and status per UPC. + +CLI usage: + +```bash +bun run upc 012345678901 098765432109 +bun run upc 012345678901,098765432109 --detailed +bun run upc --file upcs.txt --detailed --json +``` + +API usage (when `bun run start:web` is running): + +```bash +# Simple one-to-one mapping (GET) +curl "http://localhost:3000/api/upc/map?upc=012345678901&upc=098765432109" + +# Detailed lookup with statuses (GET) +curl "http://localhost:3000/api/upc/lookup?upcs=012345678901,098765432109" + +# Detailed lookup (POST JSON) +curl -X POST "http://localhost:3000/api/upc/lookup" \ + -H "content-type: application/json" \ + -d '{"upcs":["012345678901","098765432109"]}' +``` + +## Large UPC File Analysis (XLS/XLSX) + +For very large Excel files that contain UPC values, use the dedicated UPC-file process. It runs in batches: + +1. Reads UPC rows in batches (`.xlsx` uses streaming reader, `.xls` uses fallback row-window parsing). +2. Resolves UPCs to ASINs with Keepa. +3. Runs the same sellability + Keepa/SP-API enrichment + LLM verdict pipeline as lead analysis. +4. Persists output into existing `runs` + `results` tables, so it appears in current reporting APIs/UI. + +CLI usage: + +```bash +bun run upc-file --input huge-upcs.xlsx +bun run upc-file --input huge-upcs.xls --input-batch-size 500 --upc-lookup-batch-size 100 --max-rows 10000 +``` + +API usage (when `bun run start:web` is running): + +```bash +curl -X POST "http://localhost:3000/api/process/upc-file" \ + -H "content-type: application/json" \ + -d '{ + "inputFile": "/absolute/path/to/huge-upcs.xlsx", + "inputBatchSize": 300, + "upcLookupBatchSize": 100 + }' +``` + +Request body fields: + +- `inputFile` (required): server-local path to `.xls` or `.xlsx` file. +- `outputFile` (optional): stored in run metadata. +- `inputBatchSize` (optional): number of input rows per processing batch (default `200`). +- `upcLookupBatchSize` (optional): UPC chunk size per Keepa lookup call (default `100`). +- `maxRows` (optional): cap processed valid UPC rows for dry runs. + +Response includes run metadata and status counts, including unresolved UPC reasons and lead verdict totals. + ## Input file format Accepts `.csv` or `.xlsx` files. Column names are matched case-insensitively. Required column: diff --git a/bun.lock b/bun.lock index b5784ae..1c4de3f 100644 --- a/bun.lock +++ b/bun.lock @@ -6,6 +6,7 @@ "name": "asin-check", "dependencies": { "amazon-sp-api": "^1.2.1", + "exceljs": "^4.4.0", "ioredis": "^5.10.1", "react": "^19.2.0", "react-dom": "^19.2.0", @@ -22,6 +23,10 @@ }, }, "packages": { + "@fast-csv/format": ["@fast-csv/format@4.3.5", "", { "dependencies": { "@types/node": "^14.0.1", "lodash.escaperegexp": "^4.1.2", "lodash.isboolean": "^3.0.3", "lodash.isequal": "^4.5.0", "lodash.isfunction": "^3.0.9", "lodash.isnil": "^4.0.0" } }, "sha512-8iRn6QF3I8Ak78lNAa+Gdl5MJJBM5vRHivFtMRUWINdevNo00K7OXxS2PshawLKTejVwieIlPmK5YlLu6w4u8A=="], + + "@fast-csv/parse": ["@fast-csv/parse@4.3.6", "", { "dependencies": { "@types/node": "^14.0.1", "lodash.escaperegexp": "^4.1.2", "lodash.groupby": "^4.6.0", "lodash.isfunction": "^3.0.9", "lodash.isnil": "^4.0.0", "lodash.isundefined": "^3.0.1", "lodash.uniq": "^4.5.0" } }, "sha512-uRsLYksqpbDmWaSmzvJcuApSEe38+6NQZBUsuAyMZKqHxH0g1wcJgsKUvN3WC8tewaqFjBMMGrkHmC+T7k8LvA=="], + "@ioredis/commands": ["@ioredis/commands@1.5.1", "", {}, "sha512-JH8ZL/ywcJyR9MmJ5BNqZllXNZQqQbnVZOqpPQqE1vHiFgAw4NHbvE0FOduNU8IX9babitBT46571OnPTT0Zcw=="], "@types/bun": ["@types/bun@1.3.11", "", { "dependencies": { "bun-types": "1.3.11" } }, "sha512-5vPne5QvtpjGpsGYXiFyycfpDF2ECyPcTSsFBMa0fraoxiQyMJ3SmuQIGhzPg2WJuWxVBoxWJ2kClYTcw/4fAg=="], @@ -36,6 +41,34 @@ "amazon-sp-api": ["amazon-sp-api@1.2.1", "", { "dependencies": { "csvtojson": "^2.0.14", "fast-xml-parser": "^5.3.1", "iconv-lite": "^0.7.0", "qs": "^6.14.0" } }, "sha512-zxX3KtoCDx0wxkkBgFM6qew49JJoL1XZQgUnztfp+8Im2HLHBAt4beSiDo/AkH00Gr8paHBAjdcJY6LC6ISU7w=="], + "archiver": ["archiver@5.3.2", "", { "dependencies": { "archiver-utils": "^2.1.0", "async": "^3.2.4", "buffer-crc32": "^0.2.1", "readable-stream": "^3.6.0", "readdir-glob": "^1.1.2", "tar-stream": "^2.2.0", "zip-stream": "^4.1.0" } }, "sha512-+25nxyyznAXF7Nef3y0EbBeqmGZgeN/BxHX29Rs39djAfaFalmQ89SE6CWyDCHzGL0yt/ycBtNOmGTW0FyGWNw=="], + + "archiver-utils": ["archiver-utils@2.1.0", "", { "dependencies": { "glob": "^7.1.4", "graceful-fs": "^4.2.0", "lazystream": "^1.0.0", "lodash.defaults": "^4.2.0", "lodash.difference": "^4.5.0", "lodash.flatten": "^4.4.0", "lodash.isplainobject": "^4.0.6", "lodash.union": "^4.6.0", "normalize-path": "^3.0.0", "readable-stream": "^2.0.0" } }, "sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw=="], + + "async": ["async@3.2.6", "", {}, "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA=="], + + "balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="], + + "base64-js": ["base64-js@1.5.1", "", {}, "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="], + + "big-integer": ["big-integer@1.6.52", "", {}, "sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg=="], + + "binary": ["binary@0.3.0", "", { "dependencies": { "buffers": "~0.1.1", "chainsaw": "~0.1.0" } }, "sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg=="], + + "bl": ["bl@4.1.0", "", { "dependencies": { "buffer": "^5.5.0", "inherits": "^2.0.4", "readable-stream": "^3.4.0" } }, "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w=="], + + "bluebird": ["bluebird@3.4.7", "", {}, "sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA=="], + + "brace-expansion": ["brace-expansion@2.1.0", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w=="], + + "buffer": ["buffer@5.7.1", "", { "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.1.13" } }, "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ=="], + + "buffer-crc32": ["buffer-crc32@0.2.13", "", {}, "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ=="], + + "buffer-indexof-polyfill": ["buffer-indexof-polyfill@1.0.2", "", {}, "sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A=="], + + "buffers": ["buffers@0.1.1", "", {}, "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ=="], + "bun-types": ["bun-types@1.3.11", "", { "dependencies": { "@types/node": "*" } }, "sha512-1KGPpoxQWl9f6wcZh57LvrPIInQMn2TQ7jsgxqpRzg+l0QPOFvJVH7HmvHo/AiPgwXy+/Thf6Ov3EdVn1vOabg=="], "call-bind-apply-helpers": ["call-bind-apply-helpers@1.0.2", "", { "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2" } }, "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ=="], @@ -44,78 +77,176 @@ "cfb": ["cfb@1.2.2", "", { "dependencies": { "adler-32": "~1.3.0", "crc-32": "~1.2.0" } }, "sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA=="], + "chainsaw": ["chainsaw@0.1.0", "", { "dependencies": { "traverse": ">=0.3.0 <0.4" } }, "sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ=="], + "cluster-key-slot": ["cluster-key-slot@1.1.2", "", {}, "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA=="], "codepage": ["codepage@1.15.0", "", {}, "sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA=="], + "compress-commons": ["compress-commons@4.1.2", "", { "dependencies": { "buffer-crc32": "^0.2.13", "crc32-stream": "^4.0.2", "normalize-path": "^3.0.0", "readable-stream": "^3.6.0" } }, "sha512-D3uMHtGc/fcO1Gt1/L7i1e33VOvD4A9hfQLP+6ewd+BvG/gQ84Yh4oftEhAdjSMgBgwGL+jsppT7JYNpo6MHHg=="], + + "concat-map": ["concat-map@0.0.1", "", {}, "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg=="], + + "core-util-is": ["core-util-is@1.0.3", "", {}, "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ=="], + "crc-32": ["crc-32@1.2.2", "", { "bin": { "crc32": "bin/crc32.njs" } }, "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ=="], + "crc32-stream": ["crc32-stream@4.0.3", "", { "dependencies": { "crc-32": "^1.2.0", "readable-stream": "^3.4.0" } }, "sha512-NT7w2JVU7DFroFdYkeq8cywxrgjPHWkdX1wjpRQXPX5Asews3tA+Ght6lddQO5Mkumffp3X7GEqku3epj2toIw=="], + "csstype": ["csstype@3.2.3", "", {}, "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ=="], "csvtojson": ["csvtojson@2.0.14", "", { "dependencies": { "lodash": "^4.17.21" }, "bin": { "csvtojson": "bin/csvtojson" } }, "sha512-F7NNvhhDyob7OsuEGRsH0FM1aqLs/WYITyza3l+hTEEmOK9sGPBlYQZwlVG0ezCojXYpE17lhS5qL6BCOZSPyA=="], + "dayjs": ["dayjs@1.11.20", "", {}, "sha512-YbwwqR/uYpeoP4pu043q+LTDLFBLApUP6VxRihdfNTqu4ubqMlGDLd6ErXhEgsyvY0K6nCs7nggYumAN+9uEuQ=="], + "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], "denque": ["denque@2.1.0", "", {}, "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw=="], "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="], + "duplexer2": ["duplexer2@0.1.4", "", { "dependencies": { "readable-stream": "^2.0.2" } }, "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA=="], + + "end-of-stream": ["end-of-stream@1.4.5", "", { "dependencies": { "once": "^1.4.0" } }, "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg=="], + "es-define-property": ["es-define-property@1.0.1", "", {}, "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g=="], "es-errors": ["es-errors@1.3.0", "", {}, "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw=="], "es-object-atoms": ["es-object-atoms@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0" } }, "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA=="], + "exceljs": ["exceljs@4.4.0", "", { "dependencies": { "archiver": "^5.0.0", "dayjs": "^1.8.34", "fast-csv": "^4.3.1", "jszip": "^3.10.1", "readable-stream": "^3.6.0", "saxes": "^5.0.1", "tmp": "^0.2.0", "unzipper": "^0.10.11", "uuid": "^8.3.0" } }, "sha512-XctvKaEMaj1Ii9oDOqbW/6e1gXknSY4g/aLCDicOXqBE4M0nRWkUu0PTp++UPNzoFY12BNHMfs/VadKIS6llvg=="], + + "fast-csv": ["fast-csv@4.3.6", "", { "dependencies": { "@fast-csv/format": "4.3.5", "@fast-csv/parse": "4.3.6" } }, "sha512-2RNSpuwwsJGP0frGsOmTb9oUF+VkFSM4SyLTDgwf2ciHWTarN0lQTC+F2f/t5J9QjW+c65VFIAAu85GsvMIusw=="], + "fast-xml-builder": ["fast-xml-builder@1.1.4", "", { "dependencies": { "path-expression-matcher": "^1.1.3" } }, "sha512-f2jhpN4Eccy0/Uz9csxh3Nu6q4ErKxf0XIsasomfOihuSUa3/xw6w8dnOtCDgEItQFJG8KyXPzQXzcODDrrbOg=="], "fast-xml-parser": ["fast-xml-parser@5.5.11", "", { "dependencies": { "fast-xml-builder": "^1.1.4", "path-expression-matcher": "^1.4.0", "strnum": "^2.2.3" }, "bin": { "fxparser": "src/cli/cli.js" } }, "sha512-QL0eb0YbSTVWF6tTf1+LEMSgtCEjBYPpnAjoLC8SscESlAjXEIRJ7cHtLG0pLeDFaZLa4VKZLArtA/60ZS7vyA=="], "frac": ["frac@1.1.2", "", {}, "sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA=="], + "fs-constants": ["fs-constants@1.0.0", "", {}, "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="], + + "fs.realpath": ["fs.realpath@1.0.0", "", {}, "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw=="], + + "fstream": ["fstream@1.0.12", "", { "dependencies": { "graceful-fs": "^4.1.2", "inherits": "~2.0.0", "mkdirp": ">=0.5 0", "rimraf": "2" } }, "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg=="], + "function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="], "get-intrinsic": ["get-intrinsic@1.3.0", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "es-define-property": "^1.0.1", "es-errors": "^1.3.0", "es-object-atoms": "^1.1.1", "function-bind": "^1.1.2", "get-proto": "^1.0.1", "gopd": "^1.2.0", "has-symbols": "^1.1.0", "hasown": "^2.0.2", "math-intrinsics": "^1.1.0" } }, "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ=="], "get-proto": ["get-proto@1.0.1", "", { "dependencies": { "dunder-proto": "^1.0.1", "es-object-atoms": "^1.0.0" } }, "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g=="], + "glob": ["glob@7.2.3", "", { "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", "minimatch": "^3.1.1", "once": "^1.3.0", "path-is-absolute": "^1.0.0" } }, "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q=="], + "gopd": ["gopd@1.2.0", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="], + "graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="], + "has-symbols": ["has-symbols@1.1.0", "", {}, "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ=="], "hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="], "iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="], + "ieee754": ["ieee754@1.2.1", "", {}, "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="], + + "immediate": ["immediate@3.0.6", "", {}, "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ=="], + + "inflight": ["inflight@1.0.6", "", { "dependencies": { "once": "^1.3.0", "wrappy": "1" } }, "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA=="], + + "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], + "ioredis": ["ioredis@5.10.1", "", { "dependencies": { "@ioredis/commands": "1.5.1", "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", "denque": "^2.1.0", "lodash.defaults": "^4.2.0", "lodash.isarguments": "^3.1.0", "redis-errors": "^1.2.0", "redis-parser": "^3.0.0", "standard-as-callback": "^2.1.0" } }, "sha512-HuEDBTI70aYdx1v6U97SbNx9F1+svQKBDo30o0b9fw055LMepzpOOd0Ccg9Q6tbqmBSJaMuY0fB7yw9/vjBYCA=="], + "isarray": ["isarray@1.0.0", "", {}, "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="], + + "jszip": ["jszip@3.10.1", "", { "dependencies": { "lie": "~3.3.0", "pako": "~1.0.2", "readable-stream": "~2.3.6", "setimmediate": "^1.0.5" } }, "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g=="], + + "lazystream": ["lazystream@1.0.1", "", { "dependencies": { "readable-stream": "^2.0.5" } }, "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw=="], + + "lie": ["lie@3.3.0", "", { "dependencies": { "immediate": "~3.0.5" } }, "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ=="], + + "listenercount": ["listenercount@1.0.1", "", {}, "sha512-3mk/Zag0+IJxeDrxSgaDPy4zZ3w05PRZeJNnlWhzFz5OkX49J4krc+A8X2d2M69vGMBEX0uyl8M+W+8gH+kBqQ=="], + "lodash": ["lodash@4.18.1", "", {}, "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q=="], "lodash.defaults": ["lodash.defaults@4.2.0", "", {}, "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ=="], + "lodash.difference": ["lodash.difference@4.5.0", "", {}, "sha512-dS2j+W26TQ7taQBGN8Lbbq04ssV3emRw4NY58WErlTO29pIqS0HmoT5aJ9+TUQ1N3G+JOZSji4eugsWwGp9yPA=="], + + "lodash.escaperegexp": ["lodash.escaperegexp@4.1.2", "", {}, "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw=="], + + "lodash.flatten": ["lodash.flatten@4.4.0", "", {}, "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g=="], + + "lodash.groupby": ["lodash.groupby@4.6.0", "", {}, "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw=="], + "lodash.isarguments": ["lodash.isarguments@3.1.0", "", {}, "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg=="], + "lodash.isboolean": ["lodash.isboolean@3.0.3", "", {}, "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg=="], + + "lodash.isequal": ["lodash.isequal@4.5.0", "", {}, "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ=="], + + "lodash.isfunction": ["lodash.isfunction@3.0.9", "", {}, "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw=="], + + "lodash.isnil": ["lodash.isnil@4.0.0", "", {}, "sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng=="], + + "lodash.isplainobject": ["lodash.isplainobject@4.0.6", "", {}, "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA=="], + + "lodash.isundefined": ["lodash.isundefined@3.0.1", "", {}, "sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA=="], + + "lodash.union": ["lodash.union@4.6.0", "", {}, "sha512-c4pB2CdGrGdjMKYLA+XiRDO7Y0PRQbm/Gzg8qMj+QH+pFVAoTp5sBpO0odL3FjoPCGjK96p6qsP+yQoiLoOBcw=="], + + "lodash.uniq": ["lodash.uniq@4.5.0", "", {}, "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ=="], + "math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="], + "minimatch": ["minimatch@5.1.9", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw=="], + + "mkdirp": ["mkdirp@3.0.1", "", { "bin": { "mkdirp": "dist/cjs/src/bin.js" } }, "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg=="], + "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + "normalize-path": ["normalize-path@3.0.0", "", {}, "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA=="], + "object-inspect": ["object-inspect@1.13.4", "", {}, "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew=="], + "once": ["once@1.4.0", "", { "dependencies": { "wrappy": "1" } }, "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w=="], + + "pako": ["pako@1.0.11", "", {}, "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw=="], + "path-expression-matcher": ["path-expression-matcher@1.4.0", "", {}, "sha512-s4DQMxIdhj3jLFWd9LxHOplj4p9yQ4ffMGowFf3cpEgrrJjEhN0V5nxw4Ye1EViAGDoL4/1AeO6qHpqYPOzE4Q=="], + "path-is-absolute": ["path-is-absolute@1.0.1", "", {}, "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg=="], + + "process-nextick-args": ["process-nextick-args@2.0.1", "", {}, "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="], + "qs": ["qs@6.15.1", "", { "dependencies": { "side-channel": "^1.1.0" } }, "sha512-6YHEFRL9mfgcAvql/XhwTvf5jKcOiiupt2FiJxHkiX1z4j7WL8J/jRHYLluORvc1XxB5rV20KoeK00gVJamspg=="], "react": ["react@19.2.5", "", {}, "sha512-llUJLzz1zTUBrskt2pwZgLq59AemifIftw4aB7JxOqf1HY2FDaGDxgwpAPVzHU1kdWabH7FauP4i1oEeer2WCA=="], "react-dom": ["react-dom@19.2.5", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.5" } }, "sha512-J5bAZz+DXMMwW/wV3xzKke59Af6CHY7G4uYLN1OvBcKEsWOs4pQExj86BBKamxl/Ik5bx9whOrvBlSDfWzgSag=="], + "readable-stream": ["readable-stream@3.6.2", "", { "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } }, "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA=="], + + "readdir-glob": ["readdir-glob@1.1.3", "", { "dependencies": { "minimatch": "^5.1.0" } }, "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA=="], + "redis-errors": ["redis-errors@1.2.0", "", {}, "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w=="], "redis-parser": ["redis-parser@3.0.0", "", { "dependencies": { "redis-errors": "^1.0.0" } }, "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A=="], + "rimraf": ["rimraf@2.7.1", "", { "dependencies": { "glob": "^7.1.3" }, "bin": { "rimraf": "./bin.js" } }, "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w=="], + + "safe-buffer": ["safe-buffer@5.1.2", "", {}, "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="], + "safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="], + "saxes": ["saxes@5.0.1", "", { "dependencies": { "xmlchars": "^2.2.0" } }, "sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw=="], + "scheduler": ["scheduler@0.27.0", "", {}, "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q=="], + "setimmediate": ["setimmediate@1.0.5", "", {}, "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA=="], + "side-channel": ["side-channel@1.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3", "side-channel-list": "^1.0.0", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" } }, "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw=="], "side-channel-list": ["side-channel-list@1.0.1", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.4" } }, "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w=="], @@ -128,16 +259,68 @@ "standard-as-callback": ["standard-as-callback@2.1.0", "", {}, "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A=="], + "string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], + "strnum": ["strnum@2.2.3", "", {}, "sha512-oKx6RUCuHfT3oyVjtnrmn19H1SiCqgJSg+54XqURKp5aCMbrXrhLjRN9TjuwMjiYstZ0MzDrHqkGZ5dFTKd+zg=="], + "tar-stream": ["tar-stream@2.2.0", "", { "dependencies": { "bl": "^4.0.3", "end-of-stream": "^1.4.1", "fs-constants": "^1.0.0", "inherits": "^2.0.3", "readable-stream": "^3.1.1" } }, "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ=="], + + "tmp": ["tmp@0.2.5", "", {}, "sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow=="], + + "traverse": ["traverse@0.3.9", "", {}, "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ=="], + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], "undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="], + "unzipper": ["unzipper@0.10.14", "", { "dependencies": { "big-integer": "^1.6.17", "binary": "~0.3.0", "bluebird": "~3.4.1", "buffer-indexof-polyfill": "~1.0.0", "duplexer2": "~0.1.4", "fstream": "^1.0.12", "graceful-fs": "^4.2.2", "listenercount": "~1.0.1", "readable-stream": "~2.3.6", "setimmediate": "~1.0.4" } }, "sha512-ti4wZj+0bQTiX2KmKWuwj7lhV+2n//uXEotUmGuQqrbVZSEGFMbI68+c6JCQ8aAmUWYvtHEz2A8K6wXvueR/6g=="], + + "util-deprecate": ["util-deprecate@1.0.2", "", {}, "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="], + + "uuid": ["uuid@8.3.2", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg=="], + "wmf": ["wmf@1.0.2", "", {}, "sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw=="], "word": ["word@0.3.0", "", {}, "sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA=="], + "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], + "xlsx": ["xlsx@0.18.5", "", { "dependencies": { "adler-32": "~1.3.0", "cfb": "~1.2.1", "codepage": "~1.15.0", "crc-32": "~1.2.1", "ssf": "~0.11.2", "wmf": "~1.0.1", "word": "~0.3.0" }, "bin": { "xlsx": "bin/xlsx.njs" } }, "sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ=="], + + "xmlchars": ["xmlchars@2.2.0", "", {}, "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw=="], + + "zip-stream": ["zip-stream@4.1.1", "", { "dependencies": { "archiver-utils": "^3.0.4", "compress-commons": "^4.1.2", "readable-stream": "^3.6.0" } }, "sha512-9qv4rlDiopXg4E69k+vMHjNN63YFMe9sZMrdlvKnCjlCRWeCBswPPMPUfx+ipsAWq1LXHe70RcbaHdJJpS6hyQ=="], + + "@fast-csv/format/@types/node": ["@types/node@14.18.63", "", {}, "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ=="], + + "@fast-csv/parse/@types/node": ["@types/node@14.18.63", "", {}, "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ=="], + + "archiver-utils/readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="], + + "duplexer2/readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="], + + "glob/minimatch": ["minimatch@3.1.5", "", { "dependencies": { "brace-expansion": "^1.1.7" } }, "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w=="], + + "jszip/readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="], + + "lazystream/readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="], + + "string_decoder/safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="], + + "unzipper/readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="], + + "zip-stream/archiver-utils": ["archiver-utils@3.0.4", "", { "dependencies": { "glob": "^7.2.3", "graceful-fs": "^4.2.0", "lazystream": "^1.0.0", "lodash.defaults": "^4.2.0", "lodash.difference": "^4.5.0", "lodash.flatten": "^4.4.0", "lodash.isplainobject": "^4.0.6", "lodash.union": "^4.6.0", "normalize-path": "^3.0.0", "readable-stream": "^3.6.0" } }, "sha512-KVgf4XQVrTjhyWmx6cte4RxonPLR9onExufI1jhvw/MQ4BB6IsZD5gT8Lq+u/+pRkWna/6JoHpiQioaqFP5Rzw=="], + + "archiver-utils/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], + + "duplexer2/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], + + "glob/minimatch/brace-expansion": ["brace-expansion@1.1.14", "", { "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g=="], + + "jszip/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], + + "lazystream/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], + + "unzipper/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], } } diff --git a/package.json b/package.json index efd40dd..9b09c6d 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,8 @@ "scripts": { "bestsellers": "bun run src/bestsellers-by-category.ts", "monthly-sold": "bun run src/top-monthly-sold-by-category.ts", + "upc": "bun run src/upc-lookup.ts", + "upc-file": "bun run src/upc-file-analysis.ts", "start": "bun run src/index.ts", "start:web": "bun --hot src/server.ts", "build:web": "bun build src/web/index.html --outdir dist", @@ -21,6 +23,7 @@ }, "dependencies": { "amazon-sp-api": "^1.2.1", + "exceljs": "^4.4.0", "ioredis": "^5.10.1", "react": "^19.2.0", "react-dom": "^19.2.0", diff --git a/src/analysis-pipeline.ts b/src/analysis-pipeline.ts new file mode 100644 index 0000000..5337f94 --- /dev/null +++ b/src/analysis-pipeline.ts @@ -0,0 +1,256 @@ +import { fetchKeepaDataBatch } from "./keepa.ts"; +import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts"; +import { getCache, setCache } from "./cache.ts"; +import { analyzeProducts } from "./llm.ts"; +import type { + AnalysisResult, + EnrichedProduct, + KeepaData, + ProductRecord, + SellabilityInfo, + SpApiData, +} from "./types.ts"; + +export const DEFAULT_LLM_BATCH_SIZE = 5; +export const DEFAULT_PRICING_CONCURRENCY = 5; + +export type AnalysisPipelineOptions = { + llmBatchSize?: number; + pricingConcurrency?: number; + llmBatchDelayMs?: number; + llmRetryDelayMs?: number; +}; + +export function chunkArray(items: T[], chunkSize: number): T[][] { + const chunks: T[][] = []; + for (let i = 0; i < items.length; i += chunkSize) { + chunks.push(items.slice(i, i + chunkSize)); + } + return chunks; +} + +function wait(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function unknownSpApiData(reason: string): SpApiData { + return { + fbaFee: 5.0, + fbmFee: 1.5, + referralFeePercent: 15, + estimatedSalePrice: 0, + canSell: null, + sellabilityStatus: "unknown", + sellabilityReason: reason, + }; +} + +export async function processProductChunk( + products: ProductRecord[], + options: AnalysisPipelineOptions = {}, +): Promise { + const llmBatchSize = options.llmBatchSize ?? DEFAULT_LLM_BATCH_SIZE; + const pricingConcurrency = Math.max( + 1, + options.pricingConcurrency ?? DEFAULT_PRICING_CONCURRENCY, + ); + const llmBatchDelayMs = Math.max(0, options.llmBatchDelayMs ?? 5_000); + const llmRetryDelayMs = Math.max(0, options.llmRetryDelayMs ?? 10_000); + + console.log(`\nChecking cache for ${products.length} products...`); + const cached = new Map(); + const excludedCachedAsins = new Set(); + const uncachedProducts: ProductRecord[] = []; + + for (const p of products) { + const hit = await getCache(p.asin); + if (hit) { + if (hit.spApi.sellabilityStatus === "available") { + console.log(` [cache hit] ${p.asin}`); + cached.set(p.asin, hit); + } else { + excludedCachedAsins.add(p.asin); + console.log( + ` [exclude cached] ${p.asin} - status=${hit.spApi.sellabilityStatus}`, + ); + } + } else { + uncachedProducts.push(p); + } + } + + console.log( + `${cached.size} cached available, ${excludedCachedAsins.size} cached excluded, ${uncachedProducts.length} to fetch`, + ); + + const sellabilityMap = new Map(); + const availableProducts: ProductRecord[] = []; + const unavailableProducts: ProductRecord[] = []; + + if (uncachedProducts.length > 0) { + console.log( + `\nChecking sellability for ${uncachedProducts.length} ASINs...`, + ); + const sellResults = await fetchSellabilityBatch( + uncachedProducts.map((p) => p.asin), + ); + + for (const p of uncachedProducts) { + const info = sellResults.get(p.asin) ?? { + canSell: null, + sellabilityStatus: "unknown" as const, + sellabilityReason: "Sellability check returned no result", + }; + sellabilityMap.set(p.asin, info); + + if (info.sellabilityStatus === "available") { + availableProducts.push(p); + console.log( + ` [available] ${p.asin} - status=${info.sellabilityStatus}`, + ); + } else { + unavailableProducts.push(p); + console.log( + ` [exclude] ${p.asin} - status=${info.sellabilityStatus}, reason=${info.sellabilityReason ?? "n/a"}`, + ); + } + } + + console.log( + `\nSellability gate: ${availableProducts.length} available, ${unavailableProducts.length} excluded`, + ); + } + + let keepaResults = new Map(); + if (availableProducts.length > 0) { + console.log(`\nFetching ${availableProducts.length} ASINs from Keepa...`); + try { + keepaResults = await fetchKeepaDataBatch( + availableProducts.map((p) => p.asin), + ); + } catch (err) { + console.warn(`Keepa batch fetch failed: ${err}`); + } + } + + console.log( + `\nFetching pricing & fees for ${availableProducts.length} ASINs...`, + ); + const spApiResults = new Map(); + const pricingQueue = [...availableProducts]; + let pricingDone = 0; + + async function fetchNextPricing(): Promise { + while (pricingQueue.length > 0) { + const p = pricingQueue.shift(); + if (!p) return; + + const sellability = sellabilityMap.get(p.asin) ?? { + canSell: null, + sellabilityStatus: "unknown" as const, + sellabilityReason: "Sellability check returned no result", + }; + const spApi = await fetchSpApiPricingAndFees(p.asin, sellability); + + const keepa = keepaResults.get(p.asin); + if (keepa?.currentPrice && spApi.estimatedSalePrice === 0) { + spApi.estimatedSalePrice = keepa.currentPrice; + } + + spApiResults.set(p.asin, spApi); + pricingDone++; + if (pricingDone % 10 === 0 || pricingDone === availableProducts.length) { + console.log( + ` [pricing] ${pricingDone}/${availableProducts.length} fetched`, + ); + } + } + } + + const pricingWorkers = Array.from( + { length: Math.min(pricingConcurrency, availableProducts.length || 1) }, + () => fetchNextPricing(), + ); + await Promise.all(pricingWorkers); + + console.log(`\nEnriching products...`); + const enriched: EnrichedProduct[] = []; + const availableAsins = new Set(availableProducts.map((ap) => ap.asin)); + + for (const p of products) { + if (excludedCachedAsins.has(p.asin)) { + continue; + } + + const cachedProduct = cached.get(p.asin); + if (cachedProduct) { + enriched.push(cachedProduct); + continue; + } + + if (!availableAsins.has(p.asin)) { + continue; + } + + const keepa = keepaResults.get(p.asin) ?? null; + const spApi = + spApiResults.get(p.asin) ?? unknownSpApiData("SP-API data missing"); + + const product: EnrichedProduct = { + record: p, + keepa, + spApi, + fetchedAt: new Date().toISOString(), + }; + + await setCache(p.asin, product); + enriched.push(product); + } + + console.log( + `\nAnalyzing ${enriched.length} products via LLM (batch size: ${llmBatchSize})...\n`, + ); + + const results: AnalysisResult[] = []; + for (let i = 0; i < enriched.length; i += llmBatchSize) { + const batch = enriched.slice(i, i + llmBatchSize); + const batchNum = Math.floor(i / llmBatchSize) + 1; + const totalBatches = Math.ceil(enriched.length / llmBatchSize); + console.log(` LLM batch ${batchNum}/${totalBatches}...`); + + if (i > 0 && llmBatchDelayMs > 0) { + await wait(llmBatchDelayMs); + } + + let verdicts; + try { + verdicts = await analyzeProducts(batch); + } catch { + if (llmRetryDelayMs > 0) { + await wait(llmRetryDelayMs); + } + try { + verdicts = await analyzeProducts(batch); + } catch { + verdicts = null; + } + } + + for (let j = 0; j < batch.length; j++) { + const enrichedProduct = batch[j]; + if (!enrichedProduct) continue; + + results.push({ + product: enrichedProduct, + verdict: verdicts?.[j] ?? { + asin: enrichedProduct.record.asin, + verdict: "SKIP", + confidence: 0, + reasoning: "LLM analysis failed", + }, + }); + } + } + + return results; +} diff --git a/src/index.ts b/src/index.ts index 6cb4393..87ea880 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,22 +1,12 @@ import { readProducts } from "./reader.ts"; -import { fetchKeepaDataBatch } from "./keepa.ts"; -import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts"; -import { connectCache, getCache, setCache, disconnectCache } from "./cache.ts"; -import { analyzeProducts } from "./llm.ts"; +import { connectCache, disconnectCache } from "./cache.ts"; import { printResults, writeResultsToDb } from "./writer.ts"; import { initDb, closeDb } from "./database.ts"; +import { chunkArray, processProductChunk } from "./analysis-pipeline.ts"; import path from "node:path"; -import type { - EnrichedProduct, - AnalysisResult, - KeepaData, - ProductRecord, - SellabilityInfo, - SpApiData, -} from "./types.ts"; +import type { AnalysisResult } from "./types.ts"; const DB_PATH = "./results.db"; -const LLM_BATCH_SIZE = 5; const INPUT_BATCH_SIZE = 50; function parseArgs(): { inputFile: string; outputFile?: string } { @@ -35,14 +25,6 @@ function parseArgs(): { inputFile: string; outputFile?: string } { return { inputFile, outputFile }; } -function chunkArray(items: T[], chunkSize: number): T[][] { - const chunks: T[][] = []; - for (let i = 0; i < items.length; i += chunkSize) { - chunks.push(items.slice(i, i + chunkSize)); - } - return chunks; -} - function resolveBaseOutputPath(inputFile: string, outputFile?: string): string { if (outputFile) return outputFile; @@ -50,201 +32,6 @@ function resolveBaseOutputPath(inputFile: string, outputFile?: string): string { return path.join(parsedInput.dir, `${parsedInput.name}_results.xlsx`); } -async function processProductChunk( - products: ProductRecord[], -): Promise { - console.log(`\nChecking cache for ${products.length} products...`); - const cached = new Map(); - const excludedCachedAsins = new Set(); - const uncachedProducts: ProductRecord[] = []; - - for (const p of products) { - const hit = await getCache(p.asin); - if (hit) { - if (hit.spApi.sellabilityStatus === "available") { - console.log(` [cache hit] ${p.asin}`); - cached.set(p.asin, hit); - } else { - excludedCachedAsins.add(p.asin); - console.log( - ` [exclude cached] ${p.asin} — status=${hit.spApi.sellabilityStatus}`, - ); - } - } else { - uncachedProducts.push(p); - } - } - - console.log( - `${cached.size} cached available, ${excludedCachedAsins.size} cached excluded, ${uncachedProducts.length} to fetch`, - ); - - const sellabilityMap = new Map(); - const availableProducts: ProductRecord[] = []; - const unavailableProducts: ProductRecord[] = []; - - if (uncachedProducts.length > 0) { - console.log( - `\nChecking sellability for ${uncachedProducts.length} ASINs...`, - ); - const sellResults = await fetchSellabilityBatch( - uncachedProducts.map((p) => p.asin), - ); - - for (const p of uncachedProducts) { - const info = sellResults.get(p.asin) ?? { - canSell: null, - sellabilityStatus: "unknown" as const, - sellabilityReason: "Sellability check returned no result", - }; - sellabilityMap.set(p.asin, info); - - if (info.sellabilityStatus === "available") { - availableProducts.push(p); - console.log(` [available] ${p.asin} — status=${info.sellabilityStatus}`); - } else { - unavailableProducts.push(p); - console.log( - ` [exclude] ${p.asin} — status=${info.sellabilityStatus}, reason=${info.sellabilityReason ?? "n/a"}`, - ); - } - } - - console.log( - `\nSellability gate: ${availableProducts.length} available, ${unavailableProducts.length} excluded`, - ); - } - - let keepaResults = new Map(); - if (availableProducts.length > 0) { - console.log(`\nFetching ${availableProducts.length} ASINs from Keepa...`); - try { - keepaResults = await fetchKeepaDataBatch( - availableProducts.map((p) => p.asin), - ); - } catch (err) { - console.warn(`Keepa batch fetch failed: ${err}`); - } - } - - console.log( - `\nFetching pricing & fees for ${availableProducts.length} ASINs...`, - ); - const spApiResults = new Map(); - const pricingQueue = [...availableProducts]; - let pricingDone = 0; - - async function fetchNextPricing(): Promise { - while (pricingQueue.length > 0) { - const p = pricingQueue.shift()!; - const sellability = sellabilityMap.get(p.asin)!; - const spApi = await fetchSpApiPricingAndFees(p.asin, sellability); - - const keepa = keepaResults.get(p.asin); - if (keepa?.currentPrice && spApi.estimatedSalePrice === 0) { - spApi.estimatedSalePrice = keepa.currentPrice; - } - - spApiResults.set(p.asin, spApi); - pricingDone++; - if (pricingDone % 10 === 0 || pricingDone === availableProducts.length) { - console.log( - ` [pricing] ${pricingDone}/${availableProducts.length} fetched`, - ); - } - } - } - - const pricingWorkers = Array.from( - { length: Math.min(5, availableProducts.length || 1) }, - () => fetchNextPricing(), - ); - await Promise.all(pricingWorkers); - - console.log(`\nEnriching products...`); - const enriched: EnrichedProduct[] = []; - const availableAsins = new Set(availableProducts.map((ap) => ap.asin)); - - for (const p of products) { - if (excludedCachedAsins.has(p.asin)) { - continue; - } - - const cachedProduct = cached.get(p.asin); - if (cachedProduct) { - enriched.push(cachedProduct); - continue; - } - - if (!availableAsins.has(p.asin)) { - continue; - } - - const keepa = keepaResults.get(p.asin) ?? null; - const spApi = spApiResults.get(p.asin) ?? { - fbaFee: 5.0, - fbmFee: 1.5, - referralFeePercent: 15, - estimatedSalePrice: 0, - canSell: null, - sellabilityStatus: "unknown" as const, - sellabilityReason: "SP-API data missing", - }; - - const product: EnrichedProduct = { - record: p, - keepa, - spApi, - fetchedAt: new Date().toISOString(), - }; - - await setCache(p.asin, product); - enriched.push(product); - } - - console.log( - `\nAnalyzing ${enriched.length} products via LLM (batch size: ${LLM_BATCH_SIZE})...\n`, - ); - - const results: AnalysisResult[] = []; - for (let i = 0; i < enriched.length; i += LLM_BATCH_SIZE) { - const batch = enriched.slice(i, i + LLM_BATCH_SIZE); - const batchNum = Math.floor(i / LLM_BATCH_SIZE) + 1; - const totalBatches = Math.ceil(enriched.length / LLM_BATCH_SIZE); - console.log(` LLM batch ${batchNum}/${totalBatches}...`); - - if (i > 0) { - await new Promise((r) => setTimeout(r, 5000)); - } - - let verdicts; - try { - verdicts = await analyzeProducts(batch); - } catch { - await new Promise((r) => setTimeout(r, 10_000)); - try { - verdicts = await analyzeProducts(batch); - } catch { - verdicts = null; - } - } - - for (let j = 0; j < batch.length; j++) { - results.push({ - product: batch[j]!, - verdict: verdicts?.[j] ?? { - asin: batch[j]!.record.asin, - verdict: "SKIP", - confidence: 0, - reasoning: "LLM analysis failed", - }, - }); - } - } - - return results; -} - async function main() { const { inputFile, outputFile } = parseArgs(); diff --git a/src/keepa.test.ts b/src/keepa.test.ts new file mode 100644 index 0000000..821839e --- /dev/null +++ b/src/keepa.test.ts @@ -0,0 +1,200 @@ +import { afterAll, beforeEach, expect, mock, test } from "bun:test"; +import { lookupKeepaUpcs, mapUpcsToAsins } from "./keepa.ts"; + +const originalFetch = globalThis.fetch; + +function makeUpc(index: number): string { + return String(index).padStart(12, "0"); +} + +beforeEach(() => { + globalThis.fetch = originalFetch; +}); + +afterAll(() => { + globalThis.fetch = originalFetch; +}); + +test("lookupKeepaUpcs marks invalid UPCs and skips API calls", async () => { + const fetchMock = mock(async () => { + return new Response("should not be called", { status: 500 }); + }); + globalThis.fetch = fetchMock as unknown as typeof globalThis.fetch; + + const details = await lookupKeepaUpcs([ + "", + "abc", + "12345678901", + "123456789012345", + ]); + + expect(fetchMock.mock.calls.length).toBe(0); + expect(details.size).toBe(4); + expect(details.get("")?.status).toBe("invalid_upc"); + expect(details.get("abc")?.status).toBe("invalid_upc"); + expect(details.get("12345678901")?.status).toBe("invalid_upc"); + expect(details.get("123456789012345")?.status).toBe("invalid_upc"); +}); + +test("lookupKeepaUpcs returns found, not_found, and multiple_asins outcomes", async () => { + globalThis.fetch = mock(async () => { + return new Response( + JSON.stringify({ + products: [ + { + asin: "B000FOUND01", + upcList: ["012345678901"], + stats: { + current: [null, null, null, 1234], + avg: [2500, null, null, 1400], + }, + csv: [[1, 2999]], + }, + { + asin: "B000MULTI01", + upcList: ["098765432109"], + stats: { + current: [null, null, null, 2000], + avg: [1800, null, null, 2200], + }, + csv: [[1, 1999]], + }, + { + asin: "B000MULTI02", + upcList: ["098765432109"], + stats: { + current: [null, null, null, 2100], + avg: [1850, null, null, 2250], + }, + csv: [[1, 2099]], + }, + ], + tokensLeft: 10, + refillRate: 1, + }), + { status: 200 }, + ); + }) as unknown as typeof globalThis.fetch; + + const details = await lookupKeepaUpcs([ + "012345678901", + "098765432109", + "111111111111", + ]); + + expect(details.get("012345678901")?.status).toBe("found"); + expect(details.get("012345678901")?.asin).toBe("B000FOUND01"); + expect(details.get("012345678901")?.keepaData?.currentPrice).toBe(29.99); + + expect(details.get("098765432109")?.status).toBe("multiple_asins"); + expect(details.get("098765432109")?.candidateAsins).toEqual([ + "B000MULTI01", + "B000MULTI02", + ]); + + expect(details.get("111111111111")?.status).toBe("not_found"); + + const simpleMap = await mapUpcsToAsins([ + "012345678901", + "098765432109", + "111111111111", + ]); + expect(simpleMap.get("012345678901")).toBe("B000FOUND01"); + expect(simpleMap.has("098765432109")).toBe(false); + expect(simpleMap.has("111111111111")).toBe(false); +}); + +test("lookupKeepaUpcs keeps partial success when one chunk fails", async () => { + const upcs = Array.from({ length: 101 }, (_, i) => makeUpc(700000000000 + i)); + const firstChunkFirstUpc = upcs[0]!; + const secondChunkUpc = upcs[100]!; + + globalThis.fetch = mock(async (input: string | URL | Request) => { + const rawUrl = + typeof input === "string" + ? input + : input instanceof URL + ? input.toString() + : input.url; + const url = new URL(rawUrl); + const codes = (url.searchParams.get("code") ?? "").split(","); + + if (codes.includes(firstChunkFirstUpc)) { + return new Response("first chunk failed", { status: 500 }); + } + + return new Response( + JSON.stringify({ + products: [ + { + asin: "B000LAST001", + upcList: [secondChunkUpc], + stats: { + current: [null, null, null, 1000], + avg: [1500, null, null, 1200], + }, + csv: [[1, 1599]], + }, + ], + tokensLeft: 10, + refillRate: 1, + }), + { status: 200 }, + ); + }) as unknown as typeof globalThis.fetch; + + const details = await lookupKeepaUpcs(upcs); + + expect(details.get(firstChunkFirstUpc)?.status).toBe("request_failed"); + expect(details.get(secondChunkUpc)?.status).toBe("found"); + expect(details.get(secondChunkUpc)?.asin).toBe("B000LAST001"); + + const simpleMap = await mapUpcsToAsins(upcs); + expect(simpleMap.has(firstChunkFirstUpc)).toBe(false); + expect(simpleMap.get(secondChunkUpc)).toBe("B000LAST001"); +}); + +test("lookupKeepaUpcs retries on 429 and succeeds after refill wait", async () => { + const targetUpc = "123456789012"; + const fetchMock = mock(async () => { + const callNumber = fetchMock.mock.calls.length; + + if (callNumber === 1) { + return new Response( + JSON.stringify({ + refillIn: 0, + refillRate: 21, + tokensLeft: -1, + }), + { status: 429 }, + ); + } + + return new Response( + JSON.stringify({ + products: [ + { + asin: "B000RETRY01", + upcList: [targetUpc], + stats: { + current: [null, null, null, 1111], + avg: [1299, null, null, 1234], + }, + csv: [[1, 1399]], + }, + ], + tokensLeft: 10, + refillRate: 21, + }), + { status: 200 }, + ); + }); + + globalThis.fetch = fetchMock as unknown as typeof globalThis.fetch; + + const details = await lookupKeepaUpcs([targetUpc]); + + expect(fetchMock.mock.calls.length).toBe(2); + expect(details.get(targetUpc)?.status).toBe("found"); + expect(details.get(targetUpc)?.asin).toBe("B000RETRY01"); +}); diff --git a/src/keepa.ts b/src/keepa.ts index 3478e88..eb7bd58 100644 --- a/src/keepa.ts +++ b/src/keepa.ts @@ -1,10 +1,21 @@ import { config } from "./config.ts"; -import type { KeepaData } from "./types.ts"; +import type { KeepaData, KeepaUpcLookupDetail } from "./types.ts"; const KEEPA_BASE = "https://api.keepa.com"; const MAX_ASINS_PER_REQUEST = 100; +const MAX_CODES_PER_REQUEST = MAX_ASINS_PER_REQUEST; +const MAX_KEEPA_RETRIES = 4; +const KEEP_RETRY_BUFFER_MS = 250; const AMAZON_US_SELLER_ID = "ATVPDKIKX0DER"; const KEEPA_MINUTES_OFFSET = 21_564_000; +const UPC_PATTERN = /^\d{12,14}$/; + +type KeepaApiResponse = { + products?: Record[]; + tokensLeft?: number; + refillRate?: number; + refillIn?: number; +}; // Token-based rate limiting: Keepa Pro = 1 token/min regeneration. // Each product request costs 1 token regardless of ASIN count (up to 100). @@ -35,6 +46,168 @@ async function waitForToken(): Promise { tokensLeft = 1; } +function wait(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function buildProductUrl( + queryParam: "asin" | "code", + values: string[], +): string { + const params = new URLSearchParams({ + key: config.keepaApiKey, + domain: "1", + stats: "90", + buybox: "1", + days: "90", + }); + params.set(queryParam, values.join(",")); + return `${KEEPA_BASE}/product?${params.toString()}`; +} + +function updateTokenState(data: KeepaApiResponse): void { + if (data.tokensLeft != null) tokensLeft = data.tokensLeft; + if (data.refillRate != null) refillRate = data.refillRate; +} + +function computeWaitMsFromRefill(refillIn?: number): number { + if ( + typeof refillIn === "number" && + Number.isFinite(refillIn) && + refillIn >= 0 + ) { + return Math.max( + Math.ceil(refillIn) + KEEP_RETRY_BUFFER_MS, + KEEP_RETRY_BUFFER_MS, + ); + } + + const safeRefillRate = Math.max(1, refillRate); + return Math.ceil((1 / safeRefillRate) * 60_000) + KEEP_RETRY_BUFFER_MS; +} + +function parseErrorPayload(text: string): KeepaApiResponse | null { + try { + const parsed = JSON.parse(text) as KeepaApiResponse; + return parsed && typeof parsed === "object" ? parsed : null; + } catch { + return null; + } +} + +async function fetchKeepaWithRetries( + url: string, + operationLabel: string, +): Promise { + let lastErrorMessage = "Unknown Keepa error"; + + for (let attempt = 1; attempt <= MAX_KEEPA_RETRIES; attempt++) { + await waitForToken(); + + const res = await fetch(url); + lastRequestTime = Date.now(); + + if (res.ok) { + const data = (await res.json()) as KeepaApiResponse; + updateTokenState(data); + return data; + } + + const text = await res.text(); + const payload = parseErrorPayload(text); + if (payload) { + updateTokenState(payload); + } + + lastErrorMessage = `Keepa API error ${res.status}: ${text}`; + + if (res.status !== 429 || attempt === MAX_KEEPA_RETRIES) { + break; + } + + const waitMs = computeWaitMsFromRefill(payload?.refillIn); + tokensLeft = Math.min(tokensLeft, 0); + console.warn( + `Keepa throttled during ${operationLabel} (attempt ${attempt}/${MAX_KEEPA_RETRIES}). Waiting ${Math.ceil(waitMs / 1000)}s before retry...`, + ); + await wait(waitMs); + } + + throw new Error(lastErrorMessage); +} + +function normalizeUpc(input: string): string { + return input.trim(); +} + +function isValidUpc(value: string): boolean { + return UPC_PATTERN.test(value); +} + +function normalizeCodeFromKeepa(value: string): string { + return value.replace(/\D/g, ""); +} + +function collectCodes(value: unknown, target: Set): void { + if (Array.isArray(value)) { + for (const item of value) { + collectCodes(item, target); + } + return; + } + + if (typeof value === "number" && Number.isFinite(value)) { + const normalized = normalizeCodeFromKeepa(String(Math.trunc(value))); + if (isValidUpc(normalized)) target.add(normalized); + return; + } + + if (typeof value !== "string") { + return; + } + + for (const rawPart of value.split(/[\s,;|]+/)) { + if (!rawPart) continue; + const normalized = normalizeCodeFromKeepa(rawPart); + if (isValidUpc(normalized)) target.add(normalized); + } +} + +function extractUpcsFromProduct(product: Record): string[] { + const codes = new Set(); + const candidates: unknown[] = [ + product.upcList, + product.upc, + product.eanList, + product.ean, + product.gtinList, + product.gtin, + ]; + + for (const candidate of candidates) { + collectCodes(candidate, codes); + } + + return Array.from(codes); +} + +function buildFailureDetail( + upc: string, + status: "invalid_upc" | "not_found" | "multiple_asins" | "request_failed", + reason: string, + candidateAsins: string[] = [], +): KeepaUpcLookupDetail { + return { + requestedUpc: upc, + normalizedUpc: upc, + status, + asin: null, + candidateAsins, + keepaData: null, + reason, + }; +} + export async function fetchKeepaDataBatch( asins: string[], ): Promise> { @@ -43,32 +216,13 @@ export async function fetchKeepaDataBatch( // Split into chunks of MAX_ASINS_PER_REQUEST for (let i = 0; i < asins.length; i += MAX_ASINS_PER_REQUEST) { const chunk = asins.slice(i, i + MAX_ASINS_PER_REQUEST); - await waitForToken(); - - const asinParam = chunk.join(","); - const url = `${KEEPA_BASE}/product?key=${config.keepaApiKey}&domain=1&asin=${asinParam}&stats=90&buybox=1&days=90`; + const url = buildProductUrl("asin", chunk); console.log( `Keepa: fetching ${chunk.length} ASINs (tokens left: ${tokensLeft})...`, ); - const res = await fetch(url); - lastRequestTime = Date.now(); - - if (!res.ok) { - const text = await res.text(); - throw new Error(`Keepa API error ${res.status}: ${text}`); - } - - const data = (await res.json()) as { - products?: Record[]; - tokensLeft?: number; - refillRate?: number; - }; - - // Update token state from API response - if (data.tokensLeft != null) tokensLeft = data.tokensLeft; - if (data.refillRate != null) refillRate = data.refillRate; + const data = await fetchKeepaWithRetries(url, "ASIN batch fetch"); console.log( `Keepa: ${data.products?.length ?? 0} products returned, ${tokensLeft} tokens remaining (refill: ${refillRate}/min)`, @@ -86,6 +240,133 @@ export async function fetchKeepaDataBatch( return results; } +export async function lookupKeepaUpcs( + upcs: string[], +): Promise> { + const details = new Map(); + const validUpcs: string[] = []; + const seenValid = new Set(); + + for (const rawUpc of upcs) { + const normalized = normalizeUpc(rawUpc); + if (!isValidUpc(normalized)) { + if (!details.has(normalized)) { + details.set( + normalized, + buildFailureDetail( + normalized, + "invalid_upc", + "UPC must be 12, 13, or 14 digits", + ), + ); + } + continue; + } + + if (seenValid.has(normalized)) continue; + seenValid.add(normalized); + validUpcs.push(normalized); + } + + for (let i = 0; i < validUpcs.length; i += MAX_CODES_PER_REQUEST) { + const chunk = validUpcs.slice(i, i + MAX_CODES_PER_REQUEST); + const chunkSet = new Set(chunk); + const url = buildProductUrl("code", chunk); + + console.log( + `Keepa: mapping ${chunk.length} UPCs to ASINs (tokens left: ${tokensLeft})...`, + ); + + try { + const data = await fetchKeepaWithRetries(url, "UPC code lookup"); + + console.log( + `Keepa: ${data.products?.length ?? 0} products returned for UPC query, ${tokensLeft} tokens remaining (refill: ${refillRate}/min)`, + ); + + const byUpc = new Map>(); + for (const product of data.products ?? []) { + const asin = String(product.asin ?? "").trim(); + if (!asin) continue; + + const keepaData = parseKeepaProduct(product); + const productUpcs = extractUpcsFromProduct(product); + + for (const upc of productUpcs) { + if (!chunkSet.has(upc)) continue; + if (!byUpc.has(upc)) byUpc.set(upc, new Map()); + byUpc.get(upc)!.set(asin, keepaData); + } + } + + for (const upc of chunk) { + const asinMap = byUpc.get(upc); + if (!asinMap || asinMap.size === 0) { + details.set( + upc, + buildFailureDetail( + upc, + "not_found", + "No Keepa product matched this UPC", + ), + ); + continue; + } + + const candidateAsins = Array.from(asinMap.keys()); + if (candidateAsins.length > 1) { + details.set( + upc, + buildFailureDetail( + upc, + "multiple_asins", + `UPC matched multiple ASINs (${candidateAsins.length})`, + candidateAsins, + ), + ); + continue; + } + + const asin = candidateAsins[0]!; + details.set(upc, { + requestedUpc: upc, + normalizedUpc: upc, + status: "found", + asin, + candidateAsins: [asin], + keepaData: asinMap.get(asin) ?? null, + }); + } + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + console.warn( + `Keepa UPC chunk failed (offset ${i}, size ${chunk.length}): ${reason}`, + ); + + for (const upc of chunk) { + details.set(upc, buildFailureDetail(upc, "request_failed", reason)); + } + } + } + + return details; +} + +export async function mapUpcsToAsins( + upcs: string[], +): Promise> { + const details = await lookupKeepaUpcs(upcs); + const mapping = new Map(); + + for (const [upc, detail] of details.entries()) { + if (detail.status === "found" && detail.asin) { + mapping.set(upc, detail.asin); + } + } + + return mapping; +} + function parseKeepaProduct(product: Record): KeepaData { const stats = product.stats; const csv = product.csv; diff --git a/src/server.ts b/src/server.ts index adc708f..29fbb3a 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,9 +1,19 @@ import index from "./web/index.html"; import { getDb, initDb } from "./database.ts"; -import { fetchKeepaDataBatch } from "./keepa.ts"; +import { + fetchKeepaDataBatch, + lookupKeepaUpcs, + mapUpcsToAsins, +} from "./keepa.ts"; +import { runUpcFileAnalysis } from "./upc-file-analysis.ts"; import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts"; import { analyzeProducts } from "./llm.ts"; -import type { EnrichedProduct, ProductRecord, SpApiData } from "./types.ts"; +import type { + EnrichedProduct, + KeepaUpcLookupDetail, + ProductRecord, + SpApiData, +} from "./types.ts"; type ProcessType = "lead_analysis" | "category_analysis"; @@ -46,6 +56,7 @@ const DB_PATH = process.env.RESULTS_DB_PATH || "./results.db"; const DEFAULT_PAGE_SIZE = 25; const MAX_PAGE_SIZE = 200; const ASIN_PATTERN = /^[A-Z0-9]{10}$/; +const MAX_UPCS_PER_REQUEST = 1000; initDb(DB_PATH); const db = getDb(DB_PATH); @@ -82,6 +93,188 @@ function isValidAsin(value: string): boolean { return ASIN_PATTERN.test(value); } +function splitRawUpcValues(input: string): string[] { + return input + .split(/[\s,;|]+/) + .map((chunk) => chunk.trim()) + .filter(Boolean); +} + +function collectUpcsFromUnknown(value: unknown, target: string[]): void { + if (typeof value === "string") { + target.push(...splitRawUpcValues(value)); + return; + } + + if (typeof value === "number" && Number.isFinite(value)) { + target.push(String(Math.trunc(value))); + return; + } + + if (Array.isArray(value)) { + for (const item of value) { + collectUpcsFromUnknown(item, target); + } + } +} + +function normalizeAndDedupeUpcs(values: string[]): string[] { + const seen = new Set(); + const normalized: string[] = []; + + for (const value of values) { + const upc = value.trim(); + if (!upc || seen.has(upc)) continue; + seen.add(upc); + normalized.push(upc); + } + + return normalized; +} + +function parseUpcsFromSearchParams(params: URLSearchParams): string[] { + const parsed: string[] = []; + for (const value of params.getAll("upc")) { + collectUpcsFromUnknown(value, parsed); + } + + const upcsValue = params.get("upcs"); + if (upcsValue) { + collectUpcsFromUnknown(upcsValue, parsed); + } + + return normalizeAndDedupeUpcs(parsed); +} + +async function parseUpcsFromRequest(req: Request): Promise { + if (req.method === "GET") { + const url = new URL(req.url); + return parseUpcsFromSearchParams(url.searchParams); + } + + if (req.method !== "POST") { + throw new Error("Method not allowed"); + } + + let body: unknown; + try { + body = await req.json(); + } catch { + throw new Error("Invalid JSON body"); + } + + const parsed: string[] = []; + if (body && typeof body === "object" && "upcs" in body) { + collectUpcsFromUnknown((body as { upcs?: unknown }).upcs, parsed); + } else { + collectUpcsFromUnknown(body, parsed); + } + + return normalizeAndDedupeUpcs(parsed); +} + +function validateUpcRequest(upcs: string[]): string | null { + if (upcs.length === 0) { + return "Provide at least one UPC via query (?upc=...) or JSON body { upcs: [...] }"; + } + + if (upcs.length > MAX_UPCS_PER_REQUEST) { + return `Too many UPCs. Maximum allowed per request is ${MAX_UPCS_PER_REQUEST}.`; + } + + return null; +} + +function summarizeLookupStatuses( + details: KeepaUpcLookupDetail[], +): Record { + const counts: Record = {}; + for (const detail of details) { + counts[detail.status] = (counts[detail.status] ?? 0) + 1; + } + return counts; +} + +function parsePositiveIntField( + value: unknown, + fieldName: string, +): number | undefined { + if (value == null) return undefined; + if (typeof value === "number") { + if (!Number.isInteger(value) || value < 1) { + throw new Error(`${fieldName} must be a positive integer`); + } + return value; + } + if (typeof value === "string" && value.trim().length > 0) { + const parsed = Number.parseInt(value, 10); + if (!Number.isFinite(parsed) || parsed < 1) { + throw new Error(`${fieldName} must be a positive integer`); + } + return parsed; + } + throw new Error(`${fieldName} must be a positive integer`); +} + +type UpcFileProcessRequest = { + inputFile: string; + outputFile?: string; + inputBatchSize?: number; + upcLookupBatchSize?: number; + maxRows?: number; +}; + +async function parseUpcFileProcessRequest( + req: Request, +): Promise { + if (req.method !== "POST") { + throw new Error("Method not allowed"); + } + + let body: unknown; + try { + body = await req.json(); + } catch { + throw new Error("Invalid JSON body"); + } + + if (!body || typeof body !== "object") { + throw new Error("Request body must be an object"); + } + + const parsedBody = body as Record; + const inputFileValue = parsedBody.inputFile; + if ( + typeof inputFileValue !== "string" || + inputFileValue.trim().length === 0 + ) { + throw new Error("inputFile is required and must be a non-empty string"); + } + + const outputFileValue = parsedBody.outputFile; + if ( + outputFileValue != null && + (typeof outputFileValue !== "string" || outputFileValue.trim().length === 0) + ) { + throw new Error("outputFile must be a non-empty string when provided"); + } + + return { + inputFile: inputFileValue.trim(), + outputFile: + typeof outputFileValue === "string" ? outputFileValue.trim() : undefined, + inputBatchSize: parsePositiveIntField( + parsedBody.inputBatchSize, + "inputBatchSize", + ), + upcLookupBatchSize: parsePositiveIntField( + parsedBody.upcLookupBatchSize, + "upcLookupBatchSize", + ), + maxRows: parsePositiveIntField(parsedBody.maxRows, "maxRows"), + }; +} + function parseSort( sortParam: string | null, allowed: Set, @@ -1074,6 +1267,97 @@ const server = Bun.serve({ const url = new URL(req.url); return json(getProductList(url.searchParams)); }, + "/api/upc/map": async (req) => { + let upcs: string[]; + try { + upcs = await parseUpcsFromRequest(req); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + const status = message === "Method not allowed" ? 405 : 400; + return json({ error: message }, status); + } + + const validationError = validateUpcRequest(upcs); + if (validationError) { + return json({ error: validationError }, 400); + } + + try { + const mapping = await mapUpcsToAsins(upcs); + const items = Array.from(mapping.entries()).map(([upc, asin]) => ({ + upc, + asin, + })); + + return json({ + requested: upcs.length, + matched: items.length, + items, + }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return json({ error: message }, 500); + } + }, + "/api/upc/lookup": async (req) => { + let upcs: string[]; + try { + upcs = await parseUpcsFromRequest(req); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + const status = message === "Method not allowed" ? 405 : 400; + return json({ error: message }, status); + } + + const validationError = validateUpcRequest(upcs); + if (validationError) { + return json({ error: validationError }, 400); + } + + try { + const detailMap = await lookupKeepaUpcs(upcs); + const items = Array.from(detailMap.values()); + return json({ + requested: upcs.length, + statusCounts: summarizeLookupStatuses(items), + items, + }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return json({ error: message }, 500); + } + }, + "/api/process/upc-file": async (req) => { + let parsed: UpcFileProcessRequest; + try { + parsed = await parseUpcFileProcessRequest(req); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + const status = + message === "Method not allowed" + ? 405 + : message === "Invalid JSON body" + ? 400 + : 400; + return json({ error: message }, status); + } + + try { + const summary = await runUpcFileAnalysis({ + inputFile: parsed.inputFile, + outputFile: parsed.outputFile, + inputBatchSize: parsed.inputBatchSize, + upcLookupBatchSize: parsed.upcLookupBatchSize, + maxRows: parsed.maxRows, + dbPath: DB_PATH, + manageResources: false, + }); + return json(summary); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return json({ error: message }, 500); + } + }, "/api/runs/:processType/:runId": (req) => { const processType = req.params.processType as ProcessType; const runId = Number(req.params.runId); diff --git a/src/types.ts b/src/types.ts index 48cb0e4..8be360c 100644 --- a/src/types.ts +++ b/src/types.ts @@ -44,6 +44,23 @@ export interface KeepaData { categoryTree: string[]; } +export type KeepaUpcLookupStatus = + | "found" + | "invalid_upc" + | "not_found" + | "multiple_asins" + | "request_failed"; + +export interface KeepaUpcLookupDetail { + requestedUpc: string; + normalizedUpc: string; + status: KeepaUpcLookupStatus; + asin: string | null; + candidateAsins: string[]; + keepaData: KeepaData | null; + reason?: string; +} + export type SellabilityInfo = { canSell: boolean | null; sellabilityStatus: "available" | "restricted" | "not_available" | "unknown"; diff --git a/src/upc-file-analysis.ts b/src/upc-file-analysis.ts new file mode 100644 index 0000000..8a11d8e --- /dev/null +++ b/src/upc-file-analysis.ts @@ -0,0 +1,331 @@ +import path from "node:path"; +import { lookupKeepaUpcs } from "./keepa.ts"; +import { processProductChunk, chunkArray } from "./analysis-pipeline.ts"; +import { + processUpcFileInBatches, + type UpcInputRow, +} from "./upc-file-reader.ts"; +import { + appendResultsToRun, + printResults, + refreshRunCountsInDb, + startRunInDb, + type RunCounts, +} from "./writer.ts"; +import { initDb, closeDb } from "./database.ts"; +import { connectCache, disconnectCache } from "./cache.ts"; +import type { + KeepaUpcLookupDetail, + KeepaUpcLookupStatus, + ProductRecord, +} from "./types.ts"; + +const DB_PATH = process.env.RESULTS_DB_PATH || "./results.db"; +const DEFAULT_INPUT_BATCH_SIZE = 200; +const DEFAULT_UPC_LOOKUP_BATCH_SIZE = 100; + +export type UpcFileAnalysisOptions = { + inputFile: string; + outputFile?: string; + inputBatchSize?: number; + upcLookupBatchSize?: number; + maxRows?: number; + manageResources?: boolean; + dbPath?: string; +}; + +export type UpcFileAnalysisSummary = { + runId: number; + dbPath: string; + inputFile: string; + outputFile?: string; + processedRows: number; + matchedRows: number; + unresolvedByStatus: Record; + runCounts: RunCounts; + reader: { + mode: "xlsx_stream" | "xlsx_fallback" | "xls_fallback"; + totalRowsSeen: number; + emittedRows: number; + skippedMissingUpc: number; + skippedInvalidUpc: number; + }; +}; + +function printUsage(): void { + console.log("Usage:"); + console.log( + " bun run src/upc-file-analysis.ts --input [--out output.xlsx] [--input-batch-size 200] [--upc-lookup-batch-size 100] [--max-rows 1000]", + ); +} + +function parsePositiveInt(value: string | undefined, flagName: string): number { + const parsed = Number.parseInt(String(value), 10); + if (!Number.isFinite(parsed) || parsed < 1) { + throw new Error(`Invalid value for ${flagName}: ${value}`); + } + return parsed; +} + +function parseArgs(argv: string[]): UpcFileAnalysisOptions { + let inputFile: string | undefined; + let outputFile: string | undefined; + let inputBatchSize: number | undefined; + let upcLookupBatchSize: number | undefined; + let maxRows: number | undefined; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]!; + + if (arg === "--help" || arg === "-h") { + printUsage(); + process.exit(0); + } + + if (arg === "--input") { + const next = argv[i + 1]; + if (!next) throw new Error("Missing value after --input"); + inputFile = next; + i++; + continue; + } + + if (arg === "--out") { + const next = argv[i + 1]; + if (!next) throw new Error("Missing value after --out"); + outputFile = next; + i++; + continue; + } + + if (arg === "--input-batch-size") { + inputBatchSize = parsePositiveInt(argv[i + 1], "--input-batch-size"); + i++; + continue; + } + + if (arg === "--upc-lookup-batch-size") { + upcLookupBatchSize = parsePositiveInt( + argv[i + 1], + "--upc-lookup-batch-size", + ); + i++; + continue; + } + + if (arg === "--max-rows") { + maxRows = parsePositiveInt(argv[i + 1], "--max-rows"); + i++; + continue; + } + + if (arg.startsWith("--")) { + throw new Error(`Unknown flag: ${arg}`); + } + + if (!inputFile) { + inputFile = arg; + continue; + } + + throw new Error(`Unexpected positional argument: ${arg}`); + } + + if (!inputFile) { + throw new Error("Missing --input "); + } + + return { + inputFile, + outputFile, + inputBatchSize, + upcLookupBatchSize, + maxRows, + }; +} + +function resolveDefaultOutputPath(inputFile: string): string { + const parsedInput = path.parse(inputFile); + return path.join(parsedInput.dir, `${parsedInput.name}_upc_results.xlsx`); +} + +function createStatusCounter(): Record { + return { + found: 0, + invalid_upc: 0, + not_found: 0, + multiple_asins: 0, + request_failed: 0, + }; +} + +async function lookupUpcsWithChunking( + rows: UpcInputRow[], + lookupBatchSize: number, +): Promise> { + const uniqueUpcs = Array.from(new Set(rows.map((row) => row.upc))); + const chunks = chunkArray(uniqueUpcs, lookupBatchSize); + const details = new Map(); + + for (let i = 0; i < chunks.length; i++) { + const chunk = chunks[i]!; + console.log( + ` Keepa UPC lookup chunk ${i + 1}/${chunks.length} (${chunk.length} UPCs)...`, + ); + + const chunkDetails = await lookupKeepaUpcs(chunk); + for (const [upc, detail] of chunkDetails.entries()) { + details.set(upc, detail); + } + } + + return details; +} + +function toProductRecord( + row: UpcInputRow, + detail: KeepaUpcLookupDetail, +): ProductRecord { + const keepaCategory = detail.keepaData?.categoryTree?.[0]; + + return { + asin: detail.asin ?? row.upc, + name: row.name ?? detail.asin ?? row.upc, + unitCost: row.unitCost ?? 0, + brand: row.brand, + category: row.category ?? keepaCategory, + }; +} + +export async function runUpcFileAnalysis( + options: UpcFileAnalysisOptions, +): Promise { + const dbPath = options.dbPath ?? DB_PATH; + const inputBatchSize = Math.max( + 1, + options.inputBatchSize ?? DEFAULT_INPUT_BATCH_SIZE, + ); + const lookupBatchSize = Math.max( + 1, + options.upcLookupBatchSize ?? DEFAULT_UPC_LOOKUP_BATCH_SIZE, + ); + const outputFile = + options.outputFile ?? resolveDefaultOutputPath(options.inputFile); + const manageResources = options.manageResources ?? true; + + if (manageResources) { + console.log("Connecting to Redis..."); + await connectCache(); + console.log("Initializing SQLite database..."); + initDb(dbPath); + } + + const unresolvedByStatus = createStatusCounter(); + const printableSample = []; + let processedRows = 0; + let matchedRows = 0; + + const runId = startRunInDb(dbPath, options.inputFile, outputFile); + + try { + const readerSummary = await processUpcFileInBatches( + options.inputFile, + async ({ batchNumber, rows }) => { + console.log( + `\n=== UPC input batch ${batchNumber} (${rows.length} rows) ===`, + ); + + processedRows += rows.length; + const detailMap = await lookupUpcsWithChunking(rows, lookupBatchSize); + + const matchedProducts: ProductRecord[] = []; + for (const row of rows) { + const detail = detailMap.get(row.upc); + if (!detail) { + unresolvedByStatus.request_failed += 1; + continue; + } + + unresolvedByStatus[detail.status] += 1; + + if (detail.status === "found" && detail.asin) { + matchedRows += 1; + matchedProducts.push(toProductRecord(row, detail)); + } + } + + console.log( + `Batch ${batchNumber}: ${matchedProducts.length}/${rows.length} rows resolved to single ASINs`, + ); + + if (matchedProducts.length === 0) { + return; + } + + const analyzed = await processProductChunk(matchedProducts); + appendResultsToRun(dbPath, runId, analyzed); + + if (printableSample.length < 200) { + const remaining = 200 - printableSample.length; + printableSample.push(...analyzed.slice(0, remaining)); + } + }, + { + batchSize: inputBatchSize, + maxRows: options.maxRows, + }, + ); + + const runCounts = refreshRunCountsInDb(dbPath, runId); + + if (printableSample.length > 0) { + printResults(printableSample); + if (runCounts.totalProducts > printableSample.length) { + console.log( + `Printed ${printableSample.length} sampled results out of ${runCounts.totalProducts} analyzed products.`, + ); + } + } else { + console.log("No products were eligible for analysis after UPC mapping."); + } + + return { + runId, + dbPath, + inputFile: options.inputFile, + outputFile, + processedRows, + matchedRows, + unresolvedByStatus, + runCounts, + reader: { + mode: readerSummary.mode, + totalRowsSeen: readerSummary.totalRowsSeen, + emittedRows: readerSummary.emittedRows, + skippedMissingUpc: readerSummary.skippedMissingUpc, + skippedInvalidUpc: readerSummary.skippedInvalidUpc, + }, + }; + } finally { + if (manageResources) { + await disconnectCache(); + closeDb(); + } + } +} + +async function main(): Promise { + const parsed = parseArgs(process.argv.slice(2)); + const summary = await runUpcFileAnalysis(parsed); + + console.log("\n=== UPC file analysis summary ==="); + console.log(JSON.stringify(summary, null, 2)); +} + +if (import.meta.main) { + main().catch((err) => { + const message = err instanceof Error ? err.message : String(err); + console.error(`UPC file analysis failed: ${message}`); + process.exit(1); + }); +} diff --git a/src/upc-file-reader.ts b/src/upc-file-reader.ts new file mode 100644 index 0000000..bca05a5 --- /dev/null +++ b/src/upc-file-reader.ts @@ -0,0 +1,360 @@ +import ExcelJS from "exceljs"; +import * as XLSX from "xlsx"; +import path from "node:path"; + +const UPC_PATTERN = /^\d{12,14}$/; + +const COLUMN_CANDIDATES = { + upc: ["upc", "upc code", "upc/ean", "ean", "gtin", "barcode", "product code"], + name: ["name", "product name", "title", "product title"], + unitCost: ["unit cost", "cost", "price", "buy cost", "unit_cost", "unitcost"], + brand: ["brand"], + category: ["category"], +} as const; + +type ColumnKey = keyof typeof COLUMN_CANDIDATES; +type ColumnMap = Record; + +export type UpcInputRow = { + rowNumber: number; + upc: string; + name?: string; + unitCost?: number; + brand?: string; + category?: string; +}; + +export type UpcInputBatch = { + batchNumber: number; + rows: UpcInputRow[]; +}; + +export type UpcReaderSummary = { + filePath: string; + mode: "xlsx_stream" | "xlsx_fallback" | "xls_fallback"; + totalRowsSeen: number; + emittedRows: number; + skippedMissingUpc: number; + skippedInvalidUpc: number; +}; + +export type UpcReaderOptions = { + batchSize?: number; + maxRows?: number; +}; + +export async function processUpcFileInBatches( + filePath: string, + onBatch: (batch: UpcInputBatch) => Promise, + options: UpcReaderOptions = {}, +): Promise { + const ext = path.extname(filePath).toLowerCase(); + + if (ext === ".xlsx") { + try { + return await processXlsxStreaming(filePath, onBatch, options); + } catch (err) { + console.warn( + `XLSX streaming reader failed, falling back to in-memory parser: ${err}`, + ); + return processXlsLikeFallback( + filePath, + onBatch, + options, + "xlsx_fallback", + ); + } + } + + if (ext === ".xls") { + return processXlsLikeFallback(filePath, onBatch, options, "xls_fallback"); + } + + throw new Error(`Unsupported file extension: ${ext}. Expected .xls or .xlsx`); +} + +async function processXlsxStreaming( + filePath: string, + onBatch: (batch: UpcInputBatch) => Promise, + options: UpcReaderOptions, +): Promise { + const batchSize = Math.max(1, options.batchSize ?? 200); + const maxRows = + options.maxRows && options.maxRows > 0 ? options.maxRows : null; + + let headerDetected = false; + let columns: ColumnMap | null = null; + let seenRows = 0; + let emittedRows = 0; + let skippedMissingUpc = 0; + let skippedInvalidUpc = 0; + let batchNumber = 1; + let currentBatch: UpcInputRow[] = []; + let stop = false; + + const flush = async () => { + if (currentBatch.length === 0) return; + await onBatch({ batchNumber, rows: currentBatch }); + batchNumber += 1; + currentBatch = []; + }; + + const workbookReader = new ExcelJS.stream.xlsx.WorkbookReader(filePath, { + worksheets: "emit", + sharedStrings: "cache", + hyperlinks: "ignore", + styles: "ignore", + }); + + for await (const worksheet of workbookReader) { + if (stop) break; + + for await (const row of worksheet) { + const values = normalizeExcelJsRow(row.values as unknown[]); + if (!headerDetected) { + columns = detectColumns(values); + if (columns.upc == null) { + throw new Error( + `No UPC column found in header row. Header row values: ${values.join(", ")}`, + ); + } + headerDetected = true; + continue; + } + + seenRows += 1; + const parsed = parseUpcInputRow(values, columns, row.number); + if (!parsed) { + skippedMissingUpc += 1; + continue; + } + + if (!isValidUpc(parsed.upc)) { + skippedInvalidUpc += 1; + continue; + } + + currentBatch.push(parsed); + emittedRows += 1; + + if (currentBatch.length >= batchSize) { + await flush(); + } + + if (maxRows != null && emittedRows >= maxRows) { + stop = true; + break; + } + } + + // Process only the first worksheet. + break; + } + + await flush(); + + if (!headerDetected) { + throw new Error("No rows found in the first worksheet."); + } + + return { + filePath, + mode: "xlsx_stream", + totalRowsSeen: seenRows, + emittedRows, + skippedMissingUpc, + skippedInvalidUpc, + }; +} + +function processXlsLikeFallback( + filePath: string, + onBatch: (batch: UpcInputBatch) => Promise, + options: UpcReaderOptions, + mode: "xlsx_fallback" | "xls_fallback", +): Promise { + return new Promise(async (resolve, reject) => { + try { + const batchSize = Math.max(1, options.batchSize ?? 200); + const maxRows = + options.maxRows && options.maxRows > 0 ? options.maxRows : null; + + const workbook = XLSX.readFile(filePath, { raw: true }); + const sheetName = workbook.SheetNames[0]; + if (!sheetName) throw new Error("No sheets found in file"); + const sheet = workbook.Sheets[sheetName]; + if (!sheet || !sheet["!ref"]) throw new Error("Sheet has no data"); + + const range = XLSX.utils.decode_range(sheet["!ref"]); + const headerValues: string[] = []; + for (let c = range.s.c; c <= range.e.c; c++) { + const cellAddress = XLSX.utils.encode_cell({ r: range.s.r, c }); + const value = sheet[cellAddress]?.v; + headerValues.push(normalizeOptionalString(value) ?? ""); + } + + const columns = detectColumns(headerValues); + if (columns.upc == null) { + throw new Error( + `No UPC column found in header row. Header row values: ${headerValues.join(", ")}`, + ); + } + + let seenRows = 0; + let emittedRows = 0; + let skippedMissingUpc = 0; + let skippedInvalidUpc = 0; + let batchNumber = 1; + let currentBatch: UpcInputRow[] = []; + + const flush = async () => { + if (currentBatch.length === 0) return; + await onBatch({ batchNumber, rows: currentBatch }); + batchNumber += 1; + currentBatch = []; + }; + + for (let r = range.s.r + 1; r <= range.e.r; r++) { + seenRows += 1; + const rowValues: string[] = []; + for (let c = range.s.c; c <= range.e.c; c++) { + const cellAddress = XLSX.utils.encode_cell({ r, c }); + rowValues.push(normalizeOptionalString(sheet[cellAddress]?.v) ?? ""); + } + + const parsed = parseUpcInputRow(rowValues, columns, r + 1); + if (!parsed) { + skippedMissingUpc += 1; + continue; + } + + if (!isValidUpc(parsed.upc)) { + skippedInvalidUpc += 1; + continue; + } + + currentBatch.push(parsed); + emittedRows += 1; + + if (currentBatch.length >= batchSize) { + await flush(); + } + + if (maxRows != null && emittedRows >= maxRows) { + break; + } + } + + await flush(); + + resolve({ + filePath, + mode, + totalRowsSeen: seenRows, + emittedRows, + skippedMissingUpc, + skippedInvalidUpc, + }); + } catch (err) { + reject(err); + } + }); +} + +function detectColumns(headers: string[]): ColumnMap { + const columns = {} as ColumnMap; + for (const key of Object.keys(COLUMN_CANDIDATES) as ColumnKey[]) { + columns[key] = findColumnIndex(headers, [...COLUMN_CANDIDATES[key]]); + } + return columns; +} + +function findColumnIndex( + headers: string[], + candidates: string[], +): number | undefined { + const normalizedCandidates = new Set(candidates.map(normalizeHeader)); + for (let i = 0; i < headers.length; i++) { + if (normalizedCandidates.has(normalizeHeader(headers[i] ?? ""))) { + return i; + } + } + return undefined; +} + +function parseUpcInputRow( + rowValues: string[], + columns: ColumnMap, + rowNumber: number, +): UpcInputRow | null { + if (columns.upc == null) return null; + + const rawUpc = rowValues[columns.upc] ?? ""; + const upc = rawUpc.replace(/\D/g, "").trim(); + if (!upc) { + return null; + } + + return { + rowNumber, + upc, + name: getRowString(rowValues, columns.name), + unitCost: parseOptionalNumber(rowValues[columns.unitCost ?? -1]), + brand: getRowString(rowValues, columns.brand), + category: getRowString(rowValues, columns.category), + }; +} + +function normalizeExcelJsRow(values: unknown[]): string[] { + // ExcelJS row.values is 1-indexed with values[0] intentionally empty. + const normalized: string[] = []; + for (let i = 1; i < values.length; i++) { + normalized.push(normalizeOptionalString(values[i]) ?? ""); + } + return normalized; +} + +function getRowString( + values: string[], + index: number | undefined, +): string | undefined { + if (index == null || index < 0) return undefined; + const value = values[index]; + return value?.trim() ? value.trim() : undefined; +} + +function normalizeHeader(value: string): string { + return value + .toLowerCase() + .trim() + .replace(/%/g, " pct ") + .replace(/\$/g, " usd ") + .replace(/[^a-z0-9]/g, ""); +} + +function normalizeOptionalString(value: unknown): string | undefined { + if (value == null) return undefined; + + if (typeof value === "object") { + if ("text" in (value as Record)) { + return normalizeOptionalString((value as { text?: unknown }).text); + } + if ("result" in (value as Record)) { + return normalizeOptionalString((value as { result?: unknown }).result); + } + } + + const text = String(value).trim(); + return text.length > 0 ? text : undefined; +} + +function parseOptionalNumber(value: unknown): number | undefined { + if (value == null || value === "") return undefined; + const cleaned = String(value).trim().replace(/[$,%]/g, "").replace(/,/g, ""); + const parsed = Number(cleaned); + return Number.isFinite(parsed) ? parsed : undefined; +} + +function isValidUpc(value: string): boolean { + return UPC_PATTERN.test(value); +} diff --git a/src/upc-lookup.ts b/src/upc-lookup.ts new file mode 100644 index 0000000..afeaa60 --- /dev/null +++ b/src/upc-lookup.ts @@ -0,0 +1,147 @@ +import { lookupKeepaUpcs, mapUpcsToAsins } from "./keepa.ts"; + +function printUsage(): void { + console.log("Usage:"); + console.log( + " bun run src/upc-lookup.ts [--detailed] [--json] [--file path]", + ); + console.log(""); + console.log("Examples:"); + console.log(" bun run src/upc-lookup.ts 012345678901 098765432109"); + console.log( + " bun run src/upc-lookup.ts 012345678901,098765432109 --detailed", + ); + console.log(" bun run src/upc-lookup.ts --file upcs.txt --detailed --json"); +} + +function splitRawUpcValues(input: string): string[] { + return input + .split(/[\s,;|]+/) + .map((chunk) => chunk.trim()) + .filter(Boolean); +} + +async function readUpcsFromFile(path: string): Promise { + const file = Bun.file(path); + if (!(await file.exists())) { + throw new Error(`UPC file not found: ${path}`); + } + return splitRawUpcValues(await file.text()); +} + +function parseArgs(args: string[]): { + upcs: string[]; + filePaths: string[]; + detailed: boolean; + asJson: boolean; +} { + let detailed = false; + let asJson = false; + const collected: string[] = []; + const filePaths: string[] = []; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]!; + if (arg === "--help" || arg === "-h") { + printUsage(); + process.exit(0); + } + if (arg === "--detailed") { + detailed = true; + continue; + } + if (arg === "--json") { + asJson = true; + continue; + } + if (arg === "--file") { + const next = args[i + 1]; + if (!next) { + throw new Error("Missing file path after --file"); + } + filePaths.push(next); + i++; + continue; + } + + if (arg.startsWith("--")) { + throw new Error(`Unknown flag: ${arg}`); + } + + collected.push(...splitRawUpcValues(arg)); + } + + return { + upcs: collected, + filePaths, + detailed, + asJson, + }; +} + +function dedupeUpcs(upcs: string[]): string[] { + return Array.from(new Set(upcs.map((upc) => upc.trim()).filter(Boolean))); +} + +async function main(): Promise { + const args = process.argv.slice(2); + const parsed = parseArgs(args); + + const fileUpcs: string[] = []; + for (const path of parsed.filePaths) { + fileUpcs.push(...(await readUpcsFromFile(path))); + } + + const upcs = dedupeUpcs([...parsed.upcs, ...fileUpcs]); + if (upcs.length === 0) { + printUsage(); + process.exit(1); + } + + if (parsed.detailed) { + const details = await lookupKeepaUpcs(upcs); + const items = Array.from(details.values()); + + if (parsed.asJson) { + console.log(JSON.stringify(items, null, 2)); + return; + } + + console.table( + items.map((item) => ({ + upc: item.normalizedUpc, + status: item.status, + asin: item.asin ?? "", + candidates: item.candidateAsins.join("|"), + reason: item.reason ?? "", + })), + ); + return; + } + + const mapping = await mapUpcsToAsins(upcs); + const items = Array.from(mapping.entries()).map(([upc, asin]) => ({ + upc, + asin, + })); + + if (parsed.asJson) { + console.log(JSON.stringify(items, null, 2)); + return; + } + + if (items.length === 0) { + console.log( + "No one-to-one UPC to ASIN matches found. Run with --detailed for per-UPC status.", + ); + return; + } + + console.table(items); +} + +main().catch((err) => { + const message = err instanceof Error ? err.message : String(err); + console.error(`UPC lookup failed: ${message}`); + process.exit(1); +}); diff --git a/src/writer.ts b/src/writer.ts index 88d6416..21d22e2 100644 --- a/src/writer.ts +++ b/src/writer.ts @@ -1,6 +1,25 @@ import { getDb } from "./database.ts"; import type { AnalysisResult } from "./types.ts"; +export type RunCounts = { + totalProducts: number; + fbaCount: number; + fbmCount: number; + skipCount: number; +}; + +function computeRunCountsFromResults(results: AnalysisResult[]): RunCounts { + const fbaCount = results.filter((r) => r.verdict.verdict === "FBA").length; + const fbmCount = results.filter((r) => r.verdict.verdict === "FBM").length; + const skipCount = results.filter((r) => r.verdict.verdict === "SKIP").length; + return { + totalProducts: results.length, + fbaCount, + fbmCount, + skipCount, + }; +} + function buildRow(r: AnalysisResult) { const price = r.product.keepa?.currentPrice ?? @@ -68,12 +87,25 @@ export function writeResultsToDb( inputFile: string, outputFile: string | undefined, ): void { - const database = getDb(dbPath); + const runCounts = computeRunCountsFromResults(results); + const runId = startRunInDb(dbPath, inputFile, outputFile, runCounts); + appendResultsToRun(dbPath, runId, results); + console.log(`Results written to SQLite database for run_id: ${runId}`); +} +export function startRunInDb( + dbPath: string, + inputFile: string, + outputFile: string | undefined, + counts: RunCounts = { + totalProducts: 0, + fbaCount: 0, + fbmCount: 0, + skipCount: 0, + }, +): number { + const database = getDb(dbPath); const timestamp = new Date().toISOString(); - const fbaCount = results.filter((r) => r.verdict.verdict === "FBA").length; - const fbmCount = results.filter((r) => r.verdict.verdict === "FBM").length; - const skipCount = results.filter((r) => r.verdict.verdict === "SKIP").length; const insertRun = database.prepare( `INSERT INTO runs ( @@ -86,25 +118,39 @@ export function writeResultsToDb( skip_count ) VALUES (?, ?, ?, ?, ?, ?, ?)`, ); + const runInfo = insertRun.run( timestamp, inputFile, outputFile ?? null, - results.length, - fbaCount, - fbmCount, - skipCount, + counts.totalProducts, + counts.fbaCount, + counts.fbmCount, + counts.skipCount, ); + const runId = (runInfo.changes as number) > 0 ? (runInfo.lastInsertRowid as number) : null; if (runId === null) { - console.error("Failed to insert run record into SQLite."); + throw new Error("Failed to insert run record into SQLite."); + } + + return runId; +} + +export function appendResultsToRun( + dbPath: string, + runId: number, + results: AnalysisResult[], +): void { + if (results.length === 0) { return; } + const database = getDb(dbPath); const insertResult = database.prepare( `INSERT INTO results ( run_id, asin, product_name, brand, category, unit_cost, current_price, @@ -174,7 +220,49 @@ export function writeResultsToDb( ); } })(); - console.log(`Results written to SQLite database for run_id: ${runId}`); +} + +export function refreshRunCountsInDb(dbPath: string, runId: number): RunCounts { + const database = getDb(dbPath); + const stats = database + .query( + `SELECT + COUNT(*) AS total, + SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba, + SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm, + SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip + FROM results + WHERE run_id = ?`, + ) + .get(runId) as { + total: number; + fba: number | null; + fbm: number | null; + skip: number | null; + }; + + const counts: RunCounts = { + totalProducts: stats.total ?? 0, + fbaCount: stats.fba ?? 0, + fbmCount: stats.fbm ?? 0, + skipCount: stats.skip ?? 0, + }; + + database + .query( + `UPDATE runs + SET total_products = ?, fba_count = ?, fbm_count = ?, skip_count = ? + WHERE id = ?`, + ) + .run( + counts.totalProducts, + counts.fbaCount, + counts.fbmCount, + counts.skipCount, + runId, + ); + + return counts; } export function printResults(results: AnalysisResult[]): void { const rows = results