Refactor database interactions to use Drizzle ORM

- Replaced direct SQLite database calls with Drizzle ORM methods in `top-monthly-sold-by-category.ts`, `writer.ts`, and `upc-file-analysis.ts`.
- Updated test cases in `top-monthly-sold-by-category.test.ts` to mock the new database interactions.
- Removed unnecessary database initialization and cleanup code.
- Improved code readability and maintainability by using ORM features for inserting and updating records.
This commit is contained in:
Victor Noguera
2026-05-25 00:08:30 -04:00
parent 70e0e8a535
commit b982edd160
22 changed files with 2456 additions and 2766 deletions

View File

@@ -19,3 +19,4 @@ GOOGLE_API_KEY=your_google_api_key
GOOGLE_CSE_ID=your_google_programmable_search_engine_id
SERPAPI_API_KEY=your_serpapi_api_key_for_google_shopping
DB_CONNECTION_STRING=your_database_connection_string

183
bun.lock
View File

@@ -6,8 +6,10 @@
"name": "asin-check",
"dependencies": {
"amazon-sp-api": "^1.2.1",
"drizzle-orm": "^0.45.2",
"exceljs": "^4.4.0",
"ioredis": "^5.10.1",
"postgres": "^3.4.9",
"react": "^19.2.0",
"react-dom": "^19.2.0",
"xlsx": "^0.18.5",
@@ -16,11 +18,70 @@
"@types/bun": "latest",
"@types/react": "^19.2.14",
"@types/react-dom": "^19.2.3",
"drizzle-kit": "^0.31.10",
"typescript": "^6.0.3",
},
},
},
"packages": {
"@drizzle-team/brocli": ["@drizzle-team/brocli@0.10.2", "", {}, "sha512-z33Il7l5dKjUgGULTqBsQBQwckHh5AbIuxhdsIxDDiZAzBOrZO6q9ogcWC65kU382AfynTfgNumVcNIjuIua6w=="],
"@esbuild-kit/core-utils": ["@esbuild-kit/core-utils@3.3.2", "", { "dependencies": { "esbuild": "~0.18.20", "source-map-support": "^0.5.21" } }, "sha512-sPRAnw9CdSsRmEtnsl2WXWdyquogVpB3yZ3dgwJfe8zrOzTsV7cJvmwrKVa+0ma5BoiGJ+BoqkMvawbayKUsqQ=="],
"@esbuild-kit/esm-loader": ["@esbuild-kit/esm-loader@2.6.5", "", { "dependencies": { "@esbuild-kit/core-utils": "^3.3.2", "get-tsconfig": "^4.7.0" } }, "sha512-FxEMIkJKnodyA1OaCUoEvbYRkoZlLZ4d/eXFu9Fh8CbBBgP5EmZxrfTRyN0qpXZ4vOvqnE5YdRdcrmUUXuU+dA=="],
"@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.25.12", "", { "os": "aix", "cpu": "ppc64" }, "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA=="],
"@esbuild/android-arm": ["@esbuild/android-arm@0.25.12", "", { "os": "android", "cpu": "arm" }, "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg=="],
"@esbuild/android-arm64": ["@esbuild/android-arm64@0.25.12", "", { "os": "android", "cpu": "arm64" }, "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg=="],
"@esbuild/android-x64": ["@esbuild/android-x64@0.25.12", "", { "os": "android", "cpu": "x64" }, "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg=="],
"@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.25.12", "", { "os": "darwin", "cpu": "arm64" }, "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg=="],
"@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.25.12", "", { "os": "darwin", "cpu": "x64" }, "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA=="],
"@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.25.12", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg=="],
"@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.25.12", "", { "os": "freebsd", "cpu": "x64" }, "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ=="],
"@esbuild/linux-arm": ["@esbuild/linux-arm@0.25.12", "", { "os": "linux", "cpu": "arm" }, "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw=="],
"@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.25.12", "", { "os": "linux", "cpu": "arm64" }, "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ=="],
"@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.25.12", "", { "os": "linux", "cpu": "ia32" }, "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA=="],
"@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng=="],
"@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw=="],
"@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.25.12", "", { "os": "linux", "cpu": "ppc64" }, "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA=="],
"@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w=="],
"@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.25.12", "", { "os": "linux", "cpu": "s390x" }, "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg=="],
"@esbuild/linux-x64": ["@esbuild/linux-x64@0.25.12", "", { "os": "linux", "cpu": "x64" }, "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw=="],
"@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.25.12", "", { "os": "none", "cpu": "arm64" }, "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg=="],
"@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.25.12", "", { "os": "none", "cpu": "x64" }, "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ=="],
"@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.25.12", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A=="],
"@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.25.12", "", { "os": "openbsd", "cpu": "x64" }, "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw=="],
"@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.25.12", "", { "os": "none", "cpu": "arm64" }, "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg=="],
"@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.25.12", "", { "os": "sunos", "cpu": "x64" }, "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w=="],
"@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.25.12", "", { "os": "win32", "cpu": "arm64" }, "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg=="],
"@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.25.12", "", { "os": "win32", "cpu": "ia32" }, "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ=="],
"@esbuild/win32-x64": ["@esbuild/win32-x64@0.25.12", "", { "os": "win32", "cpu": "x64" }, "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA=="],
"@fast-csv/format": ["@fast-csv/format@4.3.5", "", { "dependencies": { "@types/node": "^14.0.1", "lodash.escaperegexp": "^4.1.2", "lodash.isboolean": "^3.0.3", "lodash.isequal": "^4.5.0", "lodash.isfunction": "^3.0.9", "lodash.isnil": "^4.0.0" } }, "sha512-8iRn6QF3I8Ak78lNAa+Gdl5MJJBM5vRHivFtMRUWINdevNo00K7OXxS2PshawLKTejVwieIlPmK5YlLu6w4u8A=="],
"@fast-csv/parse": ["@fast-csv/parse@4.3.6", "", { "dependencies": { "@types/node": "^14.0.1", "lodash.escaperegexp": "^4.1.2", "lodash.groupby": "^4.6.0", "lodash.isfunction": "^3.0.9", "lodash.isnil": "^4.0.0", "lodash.isundefined": "^3.0.1", "lodash.uniq": "^4.5.0" } }, "sha512-uRsLYksqpbDmWaSmzvJcuApSEe38+6NQZBUsuAyMZKqHxH0g1wcJgsKUvN3WC8tewaqFjBMMGrkHmC+T7k8LvA=="],
@@ -63,6 +124,8 @@
"buffer-crc32": ["buffer-crc32@0.2.13", "", {}, "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ=="],
"buffer-from": ["buffer-from@1.1.2", "", {}, "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="],
"buffer-indexof-polyfill": ["buffer-indexof-polyfill@1.0.2", "", {}, "sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A=="],
"buffers": ["buffers@0.1.1", "", {}, "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ=="],
@@ -101,6 +164,10 @@
"denque": ["denque@2.1.0", "", {}, "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw=="],
"drizzle-kit": ["drizzle-kit@0.31.10", "", { "dependencies": { "@drizzle-team/brocli": "^0.10.2", "@esbuild-kit/esm-loader": "^2.5.5", "esbuild": "^0.25.4", "tsx": "^4.21.0" }, "bin": { "drizzle-kit": "bin.cjs" } }, "sha512-7OZcmQUrdGI+DUNNsKBn1aW8qSoKuTH7d0mYgSP8bAzdFzKoovxEFnoGQp2dVs82EOJeYycqRtciopszwUf8bw=="],
"drizzle-orm": ["drizzle-orm@0.45.2", "", { "peerDependencies": { "@aws-sdk/client-rds-data": ">=3", "@cloudflare/workers-types": ">=4", "@electric-sql/pglite": ">=0.2.0", "@libsql/client": ">=0.10.0", "@libsql/client-wasm": ">=0.10.0", "@neondatabase/serverless": ">=0.10.0", "@op-engineering/op-sqlite": ">=2", "@opentelemetry/api": "^1.4.1", "@planetscale/database": ">=1.13", "@prisma/client": "*", "@tidbcloud/serverless": "*", "@types/better-sqlite3": "*", "@types/pg": "*", "@types/sql.js": "*", "@upstash/redis": ">=1.34.7", "@vercel/postgres": ">=0.8.0", "@xata.io/client": "*", "better-sqlite3": ">=7", "bun-types": "*", "expo-sqlite": ">=14.0.0", "gel": ">=2", "knex": "*", "kysely": "*", "mysql2": ">=2", "pg": ">=8", "postgres": ">=3", "prisma": "*", "sql.js": ">=1", "sqlite3": ">=5" }, "optionalPeers": ["@aws-sdk/client-rds-data", "@cloudflare/workers-types", "@electric-sql/pglite", "@libsql/client", "@libsql/client-wasm", "@neondatabase/serverless", "@op-engineering/op-sqlite", "@opentelemetry/api", "@planetscale/database", "@prisma/client", "@tidbcloud/serverless", "@types/better-sqlite3", "@types/pg", "@types/sql.js", "@upstash/redis", "@vercel/postgres", "@xata.io/client", "better-sqlite3", "bun-types", "expo-sqlite", "gel", "knex", "kysely", "mysql2", "pg", "postgres", "prisma", "sql.js", "sqlite3"] }, "sha512-kY0BSaTNYWnoDMVoyY8uxmyHjpJW1geOmBMdSSicKo9CIIWkSxMIj2rkeSR51b8KAPB7m+qysjuHme5nKP+E5Q=="],
"dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="],
"duplexer2": ["duplexer2@0.1.4", "", { "dependencies": { "readable-stream": "^2.0.2" } }, "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA=="],
@@ -113,6 +180,8 @@
"es-object-atoms": ["es-object-atoms@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0" } }, "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA=="],
"esbuild": ["esbuild@0.25.12", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.25.12", "@esbuild/android-arm": "0.25.12", "@esbuild/android-arm64": "0.25.12", "@esbuild/android-x64": "0.25.12", "@esbuild/darwin-arm64": "0.25.12", "@esbuild/darwin-x64": "0.25.12", "@esbuild/freebsd-arm64": "0.25.12", "@esbuild/freebsd-x64": "0.25.12", "@esbuild/linux-arm": "0.25.12", "@esbuild/linux-arm64": "0.25.12", "@esbuild/linux-ia32": "0.25.12", "@esbuild/linux-loong64": "0.25.12", "@esbuild/linux-mips64el": "0.25.12", "@esbuild/linux-ppc64": "0.25.12", "@esbuild/linux-riscv64": "0.25.12", "@esbuild/linux-s390x": "0.25.12", "@esbuild/linux-x64": "0.25.12", "@esbuild/netbsd-arm64": "0.25.12", "@esbuild/netbsd-x64": "0.25.12", "@esbuild/openbsd-arm64": "0.25.12", "@esbuild/openbsd-x64": "0.25.12", "@esbuild/openharmony-arm64": "0.25.12", "@esbuild/sunos-x64": "0.25.12", "@esbuild/win32-arm64": "0.25.12", "@esbuild/win32-ia32": "0.25.12", "@esbuild/win32-x64": "0.25.12" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg=="],
"exceljs": ["exceljs@4.4.0", "", { "dependencies": { "archiver": "^5.0.0", "dayjs": "^1.8.34", "fast-csv": "^4.3.1", "jszip": "^3.10.1", "readable-stream": "^3.6.0", "saxes": "^5.0.1", "tmp": "^0.2.0", "unzipper": "^0.10.11", "uuid": "^8.3.0" } }, "sha512-XctvKaEMaj1Ii9oDOqbW/6e1gXknSY4g/aLCDicOXqBE4M0nRWkUu0PTp++UPNzoFY12BNHMfs/VadKIS6llvg=="],
"fast-csv": ["fast-csv@4.3.6", "", { "dependencies": { "@fast-csv/format": "4.3.5", "@fast-csv/parse": "4.3.6" } }, "sha512-2RNSpuwwsJGP0frGsOmTb9oUF+VkFSM4SyLTDgwf2ciHWTarN0lQTC+F2f/t5J9QjW+c65VFIAAu85GsvMIusw=="],
@@ -127,6 +196,8 @@
"fs.realpath": ["fs.realpath@1.0.0", "", {}, "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw=="],
"fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="],
"fstream": ["fstream@1.0.12", "", { "dependencies": { "graceful-fs": "^4.1.2", "inherits": "~2.0.0", "mkdirp": ">=0.5 0", "rimraf": "2" } }, "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg=="],
"function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="],
@@ -135,6 +206,8 @@
"get-proto": ["get-proto@1.0.1", "", { "dependencies": { "dunder-proto": "^1.0.1", "es-object-atoms": "^1.0.0" } }, "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g=="],
"get-tsconfig": ["get-tsconfig@4.14.0", "", { "dependencies": { "resolve-pkg-maps": "^1.0.0" } }, "sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA=="],
"glob": ["glob@7.2.3", "", { "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", "minimatch": "^3.1.1", "once": "^1.3.0", "path-is-absolute": "^1.0.0" } }, "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q=="],
"gopd": ["gopd@1.2.0", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="],
@@ -217,6 +290,8 @@
"path-is-absolute": ["path-is-absolute@1.0.1", "", {}, "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg=="],
"postgres": ["postgres@3.4.9", "", {}, "sha512-GD3qdB0x1z9xgFI6cdRD6xu2Sp2WCOEoe3mtnyB5Ee0XrrL5Pe+e4CCnJrRMnL1zYtRDZmQQVbvOttLnKDLnaw=="],
"process-nextick-args": ["process-nextick-args@2.0.1", "", {}, "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="],
"qs": ["qs@6.15.1", "", { "dependencies": { "side-channel": "^1.1.0" } }, "sha512-6YHEFRL9mfgcAvql/XhwTvf5jKcOiiupt2FiJxHkiX1z4j7WL8J/jRHYLluORvc1XxB5rV20KoeK00gVJamspg=="],
@@ -233,6 +308,8 @@
"redis-parser": ["redis-parser@3.0.0", "", { "dependencies": { "redis-errors": "^1.0.0" } }, "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A=="],
"resolve-pkg-maps": ["resolve-pkg-maps@1.0.0", "", {}, "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw=="],
"rimraf": ["rimraf@2.7.1", "", { "dependencies": { "glob": "^7.1.3" }, "bin": { "rimraf": "./bin.js" } }, "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w=="],
"safe-buffer": ["safe-buffer@5.1.2", "", {}, "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="],
@@ -253,6 +330,10 @@
"side-channel-weakmap": ["side-channel-weakmap@1.0.2", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3", "side-channel-map": "^1.0.1" } }, "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A=="],
"source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="],
"source-map-support": ["source-map-support@0.5.21", "", { "dependencies": { "buffer-from": "^1.0.0", "source-map": "^0.6.0" } }, "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w=="],
"ssf": ["ssf@0.11.2", "", { "dependencies": { "frac": "~1.1.2" } }, "sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g=="],
"standard-as-callback": ["standard-as-callback@2.1.0", "", {}, "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A=="],
@@ -267,6 +348,8 @@
"traverse": ["traverse@0.3.9", "", {}, "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ=="],
"tsx": ["tsx@4.22.3", "", { "dependencies": { "esbuild": "~0.28.0" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "bin": { "tsx": "dist/cli.mjs" } }, "sha512-mdoNxBC/cSQObGGVQ5Bpn5i+yv7j68gk3Nfm3wFjcJg3Z0Mix9jzAFfP12prmm5eVGmDKtp0yyArrs0Q+8gZHg=="],
"typescript": ["typescript@6.0.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw=="],
"undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="],
@@ -289,6 +372,8 @@
"zip-stream": ["zip-stream@4.1.1", "", { "dependencies": { "archiver-utils": "^3.0.4", "compress-commons": "^4.1.2", "readable-stream": "^3.6.0" } }, "sha512-9qv4rlDiopXg4E69k+vMHjNN63YFMe9sZMrdlvKnCjlCRWeCBswPPMPUfx+ipsAWq1LXHe70RcbaHdJJpS6hyQ=="],
"@esbuild-kit/core-utils/esbuild": ["esbuild@0.18.20", "", { "optionalDependencies": { "@esbuild/android-arm": "0.18.20", "@esbuild/android-arm64": "0.18.20", "@esbuild/android-x64": "0.18.20", "@esbuild/darwin-arm64": "0.18.20", "@esbuild/darwin-x64": "0.18.20", "@esbuild/freebsd-arm64": "0.18.20", "@esbuild/freebsd-x64": "0.18.20", "@esbuild/linux-arm": "0.18.20", "@esbuild/linux-arm64": "0.18.20", "@esbuild/linux-ia32": "0.18.20", "@esbuild/linux-loong64": "0.18.20", "@esbuild/linux-mips64el": "0.18.20", "@esbuild/linux-ppc64": "0.18.20", "@esbuild/linux-riscv64": "0.18.20", "@esbuild/linux-s390x": "0.18.20", "@esbuild/linux-x64": "0.18.20", "@esbuild/netbsd-x64": "0.18.20", "@esbuild/openbsd-x64": "0.18.20", "@esbuild/sunos-x64": "0.18.20", "@esbuild/win32-arm64": "0.18.20", "@esbuild/win32-ia32": "0.18.20", "@esbuild/win32-x64": "0.18.20" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA=="],
"@fast-csv/format/@types/node": ["@types/node@14.18.63", "", {}, "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ=="],
"@fast-csv/parse/@types/node": ["@types/node@14.18.63", "", {}, "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ=="],
@@ -305,10 +390,56 @@
"string_decoder/safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="],
"tsx/esbuild": ["esbuild@0.28.0", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.28.0", "@esbuild/android-arm": "0.28.0", "@esbuild/android-arm64": "0.28.0", "@esbuild/android-x64": "0.28.0", "@esbuild/darwin-arm64": "0.28.0", "@esbuild/darwin-x64": "0.28.0", "@esbuild/freebsd-arm64": "0.28.0", "@esbuild/freebsd-x64": "0.28.0", "@esbuild/linux-arm": "0.28.0", "@esbuild/linux-arm64": "0.28.0", "@esbuild/linux-ia32": "0.28.0", "@esbuild/linux-loong64": "0.28.0", "@esbuild/linux-mips64el": "0.28.0", "@esbuild/linux-ppc64": "0.28.0", "@esbuild/linux-riscv64": "0.28.0", "@esbuild/linux-s390x": "0.28.0", "@esbuild/linux-x64": "0.28.0", "@esbuild/netbsd-arm64": "0.28.0", "@esbuild/netbsd-x64": "0.28.0", "@esbuild/openbsd-arm64": "0.28.0", "@esbuild/openbsd-x64": "0.28.0", "@esbuild/openharmony-arm64": "0.28.0", "@esbuild/sunos-x64": "0.28.0", "@esbuild/win32-arm64": "0.28.0", "@esbuild/win32-ia32": "0.28.0", "@esbuild/win32-x64": "0.28.0" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-sNR9MHpXSUV/XB4zmsFKN+QgVG82Cc7+/aaxJ8Adi8hyOac+EXptIp45QBPaVyX3N70664wRbTcLTOemCAnyqw=="],
"unzipper/readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="],
"zip-stream/archiver-utils": ["archiver-utils@3.0.4", "", { "dependencies": { "glob": "^7.2.3", "graceful-fs": "^4.2.0", "lazystream": "^1.0.0", "lodash.defaults": "^4.2.0", "lodash.difference": "^4.5.0", "lodash.flatten": "^4.4.0", "lodash.isplainobject": "^4.0.6", "lodash.union": "^4.6.0", "normalize-path": "^3.0.0", "readable-stream": "^3.6.0" } }, "sha512-KVgf4XQVrTjhyWmx6cte4RxonPLR9onExufI1jhvw/MQ4BB6IsZD5gT8Lq+u/+pRkWna/6JoHpiQioaqFP5Rzw=="],
"@esbuild-kit/core-utils/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.18.20", "", { "os": "android", "cpu": "arm" }, "sha512-fyi7TDI/ijKKNZTUJAQqiG5T7YjJXgnzkURqmGj13C6dCqckZBLdl4h7bkhHt/t0WP+zO9/zwroDvANaOqO5Sw=="],
"@esbuild-kit/core-utils/esbuild/@esbuild/android-arm64": ["@esbuild/android-arm64@0.18.20", "", { "os": "android", "cpu": "arm64" }, "sha512-Nz4rJcchGDtENV0eMKUNa6L12zz2zBDXuhj/Vjh18zGqB44Bi7MBMSXjgunJgjRhCmKOjnPuZp4Mb6OKqtMHLQ=="],
"@esbuild-kit/core-utils/esbuild/@esbuild/android-x64": ["@esbuild/android-x64@0.18.20", "", { "os": "android", "cpu": "x64" }, "sha512-8GDdlePJA8D6zlZYJV/jnrRAi6rOiNaCC/JclcXpB+KIuvfBN4owLtgzY2bsxnx666XjJx2kDPUmnTtR8qKQUg=="],
"@esbuild-kit/core-utils/esbuild/@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.18.20", "", { "os": "darwin", "cpu": "arm64" }, "sha512-bxRHW5kHU38zS2lPTPOyuyTm+S+eobPUnTNkdJEfAddYgEcll4xkT8DB9d2008DtTbl7uJag2HuE5NZAZgnNEA=="],
"@esbuild-kit/core-utils/esbuild/@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.18.20", "", { "os": "darwin", "cpu": "x64" }, "sha512-pc5gxlMDxzm513qPGbCbDukOdsGtKhfxD1zJKXjCCcU7ju50O7MeAZ8c4krSJcOIJGFR+qx21yMMVYwiQvyTyQ=="],
"@esbuild-kit/core-utils/esbuild/@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.18.20", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-yqDQHy4QHevpMAaxhhIwYPMv1NECwOvIpGCZkECn8w2WFHXjEwrBn3CeNIYsibZ/iZEUemj++M26W3cNR5h+Tw=="],
"@esbuild-kit/core-utils/esbuild/@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.18.20", "", { "os": "freebsd", "cpu": "x64" }, "sha512-tgWRPPuQsd3RmBZwarGVHZQvtzfEBOreNuxEMKFcd5DaDn2PbBxfwLcj4+aenoh7ctXcbXmOQIn8HI6mCSw5MQ=="],
"@esbuild-kit/core-utils/esbuild/@esbuild/linux-arm": ["@esbuild/linux-arm@0.18.20", "", { "os": "linux", "cpu": "arm" }, "sha512-/5bHkMWnq1EgKr1V+Ybz3s1hWXok7mDFUMQ4cG10AfW3wL02PSZi5kFpYKrptDsgb2WAJIvRcDm+qIvXf/apvg=="],
"@esbuild-kit/core-utils/esbuild/@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.18.20", "", { "os": "linux", "cpu": "arm64" }, "sha512-2YbscF+UL7SQAVIpnWvYwM+3LskyDmPhe31pE7/aoTMFKKzIc9lLbyGUpmmb8a8AixOL61sQ/mFh3jEjHYFvdA=="],
"@esbuild-kit/core-utils/esbuild/@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.18.20", "", { "os": "linux", "cpu": "ia32" }, "sha512-P4etWwq6IsReT0E1KHU40bOnzMHoH73aXp96Fs8TIT6z9Hu8G6+0SHSw9i2isWrD2nbx2qo5yUqACgdfVGx7TA=="],
"@esbuild-kit/core-utils/esbuild/@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.18.20", "", { "os": "linux", "cpu": "none" }, "sha512-nXW8nqBTrOpDLPgPY9uV+/1DjxoQ7DoB2N8eocyq8I9XuqJ7BiAMDMf9n1xZM9TgW0J8zrquIb/A7s3BJv7rjg=="],
"@esbuild-kit/core-utils/esbuild/@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.18.20", "", { "os": "linux", "cpu": "none" }, "sha512-d5NeaXZcHp8PzYy5VnXV3VSd2D328Zb+9dEq5HE6bw6+N86JVPExrA6O68OPwobntbNJ0pzCpUFZTo3w0GyetQ=="],
"@esbuild-kit/core-utils/esbuild/@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.18.20", "", { "os": "linux", "cpu": "ppc64" }, "sha512-WHPyeScRNcmANnLQkq6AfyXRFr5D6N2sKgkFo2FqguP44Nw2eyDlbTdZwd9GYk98DZG9QItIiTlFLHJHjxP3FA=="],
"@esbuild-kit/core-utils/esbuild/@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.18.20", "", { "os": "linux", "cpu": "none" }, "sha512-WSxo6h5ecI5XH34KC7w5veNnKkju3zBRLEQNY7mv5mtBmrP/MjNBCAlsM2u5hDBlS3NGcTQpoBvRzqBcRtpq1A=="],
"@esbuild-kit/core-utils/esbuild/@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.18.20", "", { "os": "linux", "cpu": "s390x" }, "sha512-+8231GMs3mAEth6Ja1iK0a1sQ3ohfcpzpRLH8uuc5/KVDFneH6jtAJLFGafpzpMRO6DzJ6AvXKze9LfFMrIHVQ=="],
"@esbuild-kit/core-utils/esbuild/@esbuild/linux-x64": ["@esbuild/linux-x64@0.18.20", "", { "os": "linux", "cpu": "x64" }, "sha512-UYqiqemphJcNsFEskc73jQ7B9jgwjWrSayxawS6UVFZGWrAAtkzjxSqnoclCXxWtfwLdzU+vTpcNYhpn43uP1w=="],
"@esbuild-kit/core-utils/esbuild/@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.18.20", "", { "os": "none", "cpu": "x64" }, "sha512-iO1c++VP6xUBUmltHZoMtCUdPlnPGdBom6IrO4gyKPFFVBKioIImVooR5I83nTew5UOYrk3gIJhbZh8X44y06A=="],
"@esbuild-kit/core-utils/esbuild/@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.18.20", "", { "os": "openbsd", "cpu": "x64" }, "sha512-e5e4YSsuQfX4cxcygw/UCPIEP6wbIL+se3sxPdCiMbFLBWu0eiZOJ7WoD+ptCLrmjZBK1Wk7I6D/I3NglUGOxg=="],
"@esbuild-kit/core-utils/esbuild/@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.18.20", "", { "os": "sunos", "cpu": "x64" }, "sha512-kDbFRFp0YpTQVVrqUd5FTYmWo45zGaXe0X8E1G/LKFC0v8x0vWrhOWSLITcCn63lmZIxfOMXtCfti/RxN/0wnQ=="],
"@esbuild-kit/core-utils/esbuild/@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.18.20", "", { "os": "win32", "cpu": "arm64" }, "sha512-ddYFR6ItYgoaq4v4JmQQaAI5s7npztfV4Ag6NrhiaW0RrnOXqBkgwZLofVTlq1daVTQNhtI5oieTvkRPfZrePg=="],
"@esbuild-kit/core-utils/esbuild/@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.18.20", "", { "os": "win32", "cpu": "ia32" }, "sha512-Wv7QBi3ID/rROT08SABTS7eV4hX26sVduqDOTe1MvGMjNd3EjOz4b7zeexIR62GTIEKrfJXKL9LFxTYgkyeu7g=="],
"@esbuild-kit/core-utils/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.18.20", "", { "os": "win32", "cpu": "x64" }, "sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ=="],
"archiver-utils/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="],
"duplexer2/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="],
@@ -319,6 +450,58 @@
"lazystream/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="],
"tsx/esbuild/@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.28.0", "", { "os": "aix", "cpu": "ppc64" }, "sha512-lhRUCeuOyJQURhTxl4WkpFTjIsbDayJHih5kZC1giwE+MhIzAb7mEsQMqMf18rHLsrb5qI1tafG20mLxEWcWlA=="],
"tsx/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.28.0", "", { "os": "android", "cpu": "arm" }, "sha512-wqh0ByljabXLKHeWXYLqoJ5jKC4XBaw6Hk08OfMrCRd2nP2ZQ5eleDZC41XHyCNgktBGYMbqnrJKq/K/lzPMSQ=="],
"tsx/esbuild/@esbuild/android-arm64": ["@esbuild/android-arm64@0.28.0", "", { "os": "android", "cpu": "arm64" }, "sha512-+WzIXQOSaGs33tLEgYPYe/yQHf0WTU0X42Jca3y8NWMbUVhp7rUnw+vAsRC/QiDrdD31IszMrZy+qwPOPjd+rw=="],
"tsx/esbuild/@esbuild/android-x64": ["@esbuild/android-x64@0.28.0", "", { "os": "android", "cpu": "x64" }, "sha512-+VJggoaKhk2VNNqVL7f6S189UzShHC/mR9EE8rDdSkdpN0KflSwWY/gWjDrNxxisg8Fp1ZCD9jLMo4m0OUfeUA=="],
"tsx/esbuild/@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.28.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-0T+A9WZm+bZ84nZBtk1ckYsOvyA3x7e2Acj1KdVfV4/2tdG4fzUp91YHx+GArWLtwqp77pBXVCPn2We7Letr0Q=="],
"tsx/esbuild/@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.28.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-fyzLm/DLDl/84OCfp2f/XQ4flmORsjU7VKt8HLjvIXChJoFFOIL6pLJPH4Yhd1n1gGFF9mPwtlN5Wf82DZs+LQ=="],
"tsx/esbuild/@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.28.0", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-l9GeW5UZBT9k9brBYI+0WDffcRxgHQD8ShN2Ur4xWq/NFzUKm3k5lsH4PdaRgb2w7mI9u61nr2gI2mLI27Nh3Q=="],
"tsx/esbuild/@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.28.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-BXoQai/A0wPO6Es3yFJ7APCiKGc1tdAEOgeTNy3SsB491S3aHn4S4r3e976eUnPdU+NbdtmBuLncYir2tMU9Nw=="],
"tsx/esbuild/@esbuild/linux-arm": ["@esbuild/linux-arm@0.28.0", "", { "os": "linux", "cpu": "arm" }, "sha512-CjaaREJagqJp7iTaNQjjidaNbCKYcd4IDkzbwwxtSvjI7NZm79qiHc8HqciMddQ6CKvJT6aBd8lO9kN/ZudLlw=="],
"tsx/esbuild/@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.28.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-RVyzfb3FWsGA55n6WY0MEIEPURL1FcbhFE6BffZEMEekfCzCIMtB5yyDcFnVbTnwk+CLAgTujmV/Lgvih56W+A=="],
"tsx/esbuild/@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.28.0", "", { "os": "linux", "cpu": "ia32" }, "sha512-KBnSTt1kxl9x70q+ydterVdl+Cn0H18ngRMRCEQfrbqdUuntQQ0LoMZv47uB97NljZFzY6HcfqEZ2SAyIUTQBQ=="],
"tsx/esbuild/@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.28.0", "", { "os": "linux", "cpu": "none" }, "sha512-zpSlUce1mnxzgBADvxKXX5sl8aYQHo2ezvMNI8I0lbblJtp8V4odlm3Yzlj7gPyt3T8ReksE6bK+pT3WD+aJRg=="],
"tsx/esbuild/@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.28.0", "", { "os": "linux", "cpu": "none" }, "sha512-2jIfP6mmjkdmeTlsX/9vmdmhBmKADrWqN7zcdtHIeNSCH1SqIoNI63cYsjQR8J+wGa4Y5izRcSHSm8K3QWmk3w=="],
"tsx/esbuild/@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.28.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-bc0FE9wWeC0WBm49IQMPSPILRocGTQt3j5KPCA8os6VprfuJ7KD+5PzESSrJ6GmPIPJK965ZJHTUlSA6GNYEhg=="],
"tsx/esbuild/@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.28.0", "", { "os": "linux", "cpu": "none" }, "sha512-SQPZOwoTTT/HXFXQJG/vBX8sOFagGqvZyXcgLA3NhIqcBv1BJU1d46c0rGcrij2B56Z2rNiSLaZOYW5cUk7yLQ=="],
"tsx/esbuild/@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.28.0", "", { "os": "linux", "cpu": "s390x" }, "sha512-SCfR0HN8CEEjnYnySJTd2cw0k9OHB/YFzt5zgJEwa+wL/T/raGWYMBqwDNAC6dqFKmJYZoQBRfHjgwLHGSrn3Q=="],
"tsx/esbuild/@esbuild/linux-x64": ["@esbuild/linux-x64@0.28.0", "", { "os": "linux", "cpu": "x64" }, "sha512-us0dSb9iFxIi8srnpl931Nvs65it/Jd2a2K3qs7fz2WfGPHqzfzZTfec7oxZJRNPXPnNYZtanmRc4AL/JwVzHQ=="],
"tsx/esbuild/@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.28.0", "", { "os": "none", "cpu": "arm64" }, "sha512-CR/RYotgtCKwtftMwJlUU7xCVNg3lMYZ0RzTmAHSfLCXw3NtZtNpswLEj/Kkf6kEL3Gw+BpOekRX0BYCtklhUw=="],
"tsx/esbuild/@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.28.0", "", { "os": "none", "cpu": "x64" }, "sha512-nU1yhmYutL+fQ71Kxnhg8uEOdC0pwEW9entHykTgEbna2pw2dkbFSMeqjjyHZoCmt8SBkOSvV+yNmm94aUrrqw=="],
"tsx/esbuild/@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.28.0", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-cXb5vApOsRsxsEl4mcZ1XY3D4DzcoMxR/nnc4IyqYs0rTI8ZKmW6kyyg+11Z8yvgMfAEldKzP7AdP64HnSC/6g=="],
"tsx/esbuild/@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.28.0", "", { "os": "openbsd", "cpu": "x64" }, "sha512-8wZM2qqtv9UP3mzy7HiGYNH/zjTA355mpeuA+859TyR+e+Tc08IHYpLJuMsfpDJwoLo1ikIJI8jC3GFjnRClzA=="],
"tsx/esbuild/@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.28.0", "", { "os": "none", "cpu": "arm64" }, "sha512-FLGfyizszcef5C3YtoyQDACyg95+dndv79i2EekILBofh5wpCa1KuBqOWKrEHZg3zrL3t5ouE5jgr94vA+Wb2w=="],
"tsx/esbuild/@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.28.0", "", { "os": "sunos", "cpu": "x64" }, "sha512-1ZgjUoEdHZZl/YlV76TSCz9Hqj9h9YmMGAgAPYd+q4SicWNX3G5GCyx9uhQWSLcbvPW8Ni7lj4gDa1T40akdlw=="],
"tsx/esbuild/@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.28.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-Q9StnDmQ/enxnpxCCLSg0oo4+34B9TdXpuyPeTedN/6+iXBJ4J+zwfQI28u/Jl40nOYAxGoNi7mFP40RUtkmUA=="],
"tsx/esbuild/@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.28.0", "", { "os": "win32", "cpu": "ia32" }, "sha512-zF3ag/gfiCe6U2iczcRzSYJKH1DCI+ByzSENHlM2FcDbEeo5Zd2C86Aq0tKUYAJJ1obRP84ymxIAksZUcdztHA=="],
"tsx/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.28.0", "", { "os": "win32", "cpu": "x64" }, "sha512-pEl1bO9mfAmIC+tW5btTmrKaujg3zGtUmWNdCw/xs70FBjwAL3o9OEKNHvNmnyylD6ubxUERiEhdsL0xBQ9efw=="],
"unzipper/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="],
}
}

10
drizzle.config.ts Normal file
View File

@@ -0,0 +1,10 @@
import { defineConfig } from "drizzle-kit";
export default defineConfig({
schema: "./src/db/schema.ts",
out: "./drizzle",
dialect: "postgresql",
dbCredentials: {
url: process.env.DB_CONNECTION_STRING!,
},
});

View File

@@ -14,18 +14,23 @@
"start": "bun run src/index.ts",
"start:web": "bun --hot src/server.ts",
"build:web": "bun build src/web/index.html --outdir dist",
"test": "bun test"
"test": "bun test",
"db:generate": "drizzle-kit generate",
"db:migrate": "drizzle-kit migrate"
},
"devDependencies": {
"@types/bun": "latest",
"@types/react": "^19.2.14",
"@types/react-dom": "^19.2.3",
"drizzle-kit": "^0.31.10",
"typescript": "^6.0.3"
},
"dependencies": {
"amazon-sp-api": "^1.2.1",
"drizzle-orm": "^0.45.2",
"exceljs": "^4.4.0",
"ioredis": "^5.10.1",
"postgres": "^3.4.9",
"react": "^19.2.0",
"react-dom": "^19.2.0",
"xlsx": "^0.18.5"

View File

@@ -1,8 +1,41 @@
import { test, expect, beforeAll, afterAll, beforeEach, mock } from "bun:test";
import { Database } from "bun:sqlite";
import { getDb, initDb, closeDb } from "./database.ts";
import path from "node:path";
import { rmSync, mkdirSync } from "node:fs";
import { test, expect, beforeEach, mock } from "bun:test";
let nextId = 0;
function chainable(resolveWith: any[] = []): any {
const p: any = Promise.resolve(resolveWith);
p.limit = (_n: any) => chainable(resolveWith);
p.where = (_cond: any) => chainable(resolveWith);
p.from = (_table: any) => chainable(resolveWith);
return p;
}
const makeMockDb = (): any => ({
insert: (_table: any) => ({
values: (_vals: any) => ({
returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]),
onConflictDoUpdate: (_conf: any) => Promise.resolve([]),
}),
}),
update: (_table: any) => ({
set: (_vals: any) => ({
where: (_cond: any) => Promise.resolve([]),
}),
}),
select: (_sel?: any) => ({
from: (_table: any) => ({
where: (_cond: any) => chainable(),
limit: (_n: any) => chainable(),
}),
}),
selectDistinct: (_sel: any) => ({
from: (_table: any) => chainable(),
}),
execute: (_query: any) => Promise.resolve([]),
transaction: async (fn: (tx: any) => Promise<any>) => fn(makeMockDb()),
});
mock.module("./db/index.ts", () => ({ db: makeMockDb(), client: {} }));
const fetchSellabilityBatchMock = mock(async (asins: string[]) => {
return new Map(
@@ -47,40 +80,17 @@ mock.module("./llm.ts", () => ({
const modulePromise = import("./bestsellers-by-category.ts");
const DB_TEST_PATH = path.join(
process.cwd(),
"test_output",
"test_analysis.sqlite",
);
let db: Database;
let processCategory: (db: Database, runId: number, category: any, perCategoryTop: number) => Promise<any>;
let insertCategoryRunSummary: (db: Database, summary: any, runTimestamp: string) => Promise<number>;
let processCategory: (runId: number, category: any, perCategoryTop: number) => Promise<any>;
let insertCategoryRunSummary: (summary: any, runTimestamp: string) => Promise<number>;
let originalFetch: typeof globalThis.fetch;
beforeAll(async () => {
const mod = await modulePromise;
processCategory = mod.processCategory;
insertCategoryRunSummary = mod.insertCategoryRunSummary;
rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true });
mkdirSync(path.dirname(DB_TEST_PATH), { recursive: true });
initDb(DB_TEST_PATH);
db = getDb(DB_TEST_PATH);
originalFetch = globalThis.fetch;
});
afterAll(() => {
globalThis.fetch = originalFetch;
closeDb();
rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true });
});
const mod = await modulePromise;
processCategory = mod.processCategory;
insertCategoryRunSummary = mod.insertCategoryRunSummary;
originalFetch = globalThis.fetch;
beforeEach(() => {
db.run("DELETE FROM product_analysis_results");
db.run("DELETE FROM category_analysis_runs");
nextId = 0;
globalThis.fetch = mock(async (input: string | URL | Request) => {
const rawUrl =
typeof input === "string"
@@ -139,39 +149,34 @@ test("processCategory function test", async () => {
childCount: 0,
};
const runId = await insertCategoryRunSummary(db, {
categoryId: mockCategory.id,
categoryLabel: mockCategory.label,
topAsinsChecked: 0,
availableAsins: 0,
fba: 0,
fbm: 0,
skip: 0,
status: "running",
error: "",
results: [],
}, new Date().toISOString());
const summary = await processCategory(db, runId, mockCategory, 2);
const runId = await insertCategoryRunSummary(
{
categoryId: mockCategory.id,
categoryLabel: mockCategory.label,
topAsinsChecked: 0,
availableAsins: 0,
fba: 0,
fbm: 0,
skip: 0,
status: "running",
error: "",
results: [],
},
new Date().toISOString(),
);
const categoryRun = db.query("SELECT * FROM category_analysis_runs").all() as any[];
expect(categoryRun.length).toBe(1);
expect(categoryRun[0].category_label).toBe("Category 1");
expect(categoryRun[0].top_asins_checked).toBe(2);
expect(categoryRun[0].available_asins).toBe(2);
expect(categoryRun[0].fba_count).toBe(1);
expect(categoryRun[0].fbm_count).toBe(1);
expect(categoryRun[0].status).toBe("ok");
const summary = await processCategory(runId, mockCategory, 2);
const productResults = db.query("SELECT * FROM product_analysis_results ORDER BY asin").all() as any[];
expect(productResults.length).toBe(2);
expect(summary.status).toBe("ok");
expect(summary.topAsinsChecked).toBe(2);
expect(summary.availableAsins).toBe(2);
expect(summary.fba).toBe(1);
expect(summary.fbm).toBe(1);
expect(summary.results?.length).toBe(2);
expect(summary.results?.[0]?.product.record.asin).toBe("B000000001");
expect(summary.results?.[0]?.verdict.verdict).toBe("FBA");
expect(summary.results?.[1]?.product.record.asin).toBe("B000000002");
expect(summary.results?.[1]?.verdict.verdict).toBe("FBM");
expect(productResults[0].asin).toBe("B000000001");
expect(productResults[0].name).toBe("Product One");
expect(productResults[0].verdict).toBe("FBA");
expect(productResults[0].run_id).toBe(categoryRun[0].id);
expect(productResults[1].asin).toBe("B000000002");
expect(productResults[1].name).toBe("Product Two");
expect(productResults[1].verdict).toBe("FBM");
expect(productResults[1].run_id).toBe(categoryRun[0].id);
globalThis.fetch = originalFetch;
});

View File

@@ -1,6 +1,8 @@
import { existsSync, mkdirSync, readFileSync } from "node:fs";
import path from "node:path";
import { type Database, getDb, initDb } from "./database.ts";
import { db } from "./db/index.ts";
import { runs, categoryProductResults } from "./db/schema.ts";
import { eq, sql } from "drizzle-orm";
import { config } from "./config.ts";
import { analyzeProducts } from "./llm.ts";
import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts";
@@ -139,36 +141,32 @@ function printUsageAndExit(message: string): never {
}
export async function insertCategoryRunSummary(
db: Database,
summary: CategoryRunSummary,
runTimestamp: string,
): Promise<number> {
const query = `
INSERT INTO category_analysis_runs (
category_id, category_label, run_timestamp,
top_asins_checked, available_asins,
fba_count, fbm_count, skip_count,
status, error_message
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
`;
const result = db.run(query, [
summary.categoryId,
summary.categoryLabel,
runTimestamp,
summary.topAsinsChecked,
summary.availableAsins,
summary.fba,
summary.fbm,
summary.skip,
summary.status,
summary.error,
]);
// Bun's SQLite client returns { changes: number, lastInsertRowid: number | bigint }
return Number(result.lastInsertRowid);
const [row] = await db
.insert(runs)
.values({
type: "category_analysis",
status: (summary.status as typeof runs.$inferInsert.status) ?? "running",
categoryId: summary.categoryId,
categoryLabel: summary.categoryLabel,
topAsinsChecked: summary.topAsinsChecked,
availableAsins: summary.availableAsins,
totalProducts: summary.topAsinsChecked,
fbaCount: summary.fba,
fbmCount: summary.fbm,
skipCount: summary.skip,
errorMessage: summary.error || null,
startedAt: new Date(runTimestamp),
})
.returning({ id: runs.id });
if (!row) throw new Error("Failed to insert category run.");
return row.id;
}
export async function updateCategoryRunSummary(
db: Database,
runId: number,
summary: Pick<
CategoryRunSummary,
@@ -181,136 +179,110 @@ export async function updateCategoryRunSummary(
| "error"
>,
): Promise<void> {
db.run(
`
UPDATE category_analysis_runs
SET
top_asins_checked = ?,
available_asins = ?,
fba_count = ?,
fbm_count = ?,
skip_count = ?,
status = ?,
error_message = ?
WHERE id = ?
`,
[
summary.topAsinsChecked,
summary.availableAsins,
summary.fba,
summary.fbm,
summary.skip,
summary.status,
summary.error,
runId,
],
);
await db
.update(runs)
.set({
topAsinsChecked: summary.topAsinsChecked,
availableAsins: summary.availableAsins,
totalProducts: summary.topAsinsChecked,
fbaCount: summary.fba,
fbmCount: summary.fbm,
skipCount: summary.skip,
status: summary.status as typeof runs.$inferInsert.status,
errorMessage: summary.error || null,
...(summary.status !== "running" ? { completedAt: new Date() } : {}),
})
.where(eq(runs.id, runId));
}
export async function insertProductAnalysisResults(
db: Database,
runId: number,
results: AnalysisResult[],
): Promise<void> {
if (results.length === 0) {
return;
}
if (results.length === 0) return;
const insertStmt = db.prepare(`
INSERT INTO product_analysis_results (
asin, run_id, name, brand, category, unit_cost,
current_price, avg_price_90d, avg_price_90d_sheet,
selling_price_sheet, sales_rank, sales_rank_avg_90d,
seller_count, amazon_is_seller, amazon_buybox_share_pct_90d,
monthly_sold, rank_drops_30d, rank_drops_90d,
fba_fee, fbm_fee, referral_percent, can_sell,
sellability_status, sellability_reason,
verdict, confidence, reasoning, fetched_at
) VALUES (
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?
)
ON CONFLICT(asin) DO UPDATE SET
run_id = excluded.run_id,
name = excluded.name,
brand = excluded.brand,
category = excluded.category,
unit_cost = excluded.unit_cost,
current_price = excluded.current_price,
avg_price_90d = excluded.avg_price_90d,
avg_price_90d_sheet = excluded.avg_price_90d_sheet,
selling_price_sheet = excluded.selling_price_sheet,
sales_rank = excluded.sales_rank,
sales_rank_avg_90d = excluded.sales_rank_avg_90d,
seller_count = excluded.seller_count,
amazon_is_seller = excluded.amazon_is_seller,
amazon_buybox_share_pct_90d = excluded.amazon_buybox_share_pct_90d,
monthly_sold = excluded.monthly_sold,
rank_drops_30d = excluded.rank_drops_30d,
rank_drops_90d = excluded.rank_drops_90d,
fba_fee = excluded.fba_fee,
fbm_fee = excluded.fbm_fee,
referral_percent = excluded.referral_percent,
can_sell = excluded.can_sell,
sellability_status = excluded.sellability_status,
sellability_reason = excluded.sellability_reason,
verdict = excluded.verdict,
confidence = excluded.confidence,
reasoning = excluded.reasoning,
fetched_at = excluded.fetched_at;
`);
const rows = results.map((r) => {
const price =
r.product.keepa?.currentPrice ??
r.product.record.sellingPriceFromSheet ??
r.product.spApi.estimatedSalePrice;
const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank;
db.transaction((resultsBatch: AnalysisResult[]) => {
for (const r of resultsBatch) {
const price =
r.product.keepa?.currentPrice ??
r.product.record.sellingPriceFromSheet ??
r.product.spApi.estimatedSalePrice;
const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank;
insertStmt.run(
r.product.record.asin,
runId,
r.product.record.name,
r.product.record.brand ?? null,
return {
asin: r.product.record.asin,
runId,
name: r.product.record.name,
brand: r.product.record.brand ?? null,
category:
r.product.record.category ??
r.product.keepa?.categoryTree?.join(" > ") ??
null,
r.product.record.unitCost ?? null,
price ?? null,
r.product.keepa?.avgPrice90 ?? null,
r.product.record.avgPrice90FromSheet ?? null,
r.product.record.sellingPriceFromSheet ?? null,
rank ?? null,
r.product.keepa?.salesRankAvg90 ?? null,
r.product.keepa?.sellerCount ?? null,
r.product.keepa?.amazonIsSeller == null
? null
: r.product.keepa.amazonIsSeller
? 1
: 0,
r.product.keepa?.amazonBuyboxSharePct90d ?? null,
r.product.keepa?.monthlySold ?? null,
r.product.keepa?.salesRankDrops30 ?? null,
r.product.keepa?.salesRankDrops90 ?? null,
r.product.spApi.fbaFee ?? null,
r.product.spApi.fbmFee ?? null,
r.product.spApi.referralFeePercent ?? null,
r.product.keepa?.categoryTree?.join(" > ") ??
null,
unitCost: r.product.record.unitCost ?? null,
currentPrice: price ?? null,
avgPrice90d: r.product.keepa?.avgPrice90 ?? null,
avgPrice90dSheet: r.product.record.avgPrice90FromSheet ?? null,
sellingPriceSheet: r.product.record.sellingPriceFromSheet ?? null,
salesRank: rank ?? null,
salesRankAvg90d: r.product.keepa?.salesRankAvg90 ?? null,
sellerCount: r.product.keepa?.sellerCount ?? null,
amazonIsSeller: r.product.keepa?.amazonIsSeller ?? null,
amazonBuyboxSharePct90d: r.product.keepa?.amazonBuyboxSharePct90d ?? null,
monthlySold: r.product.keepa?.monthlySold ?? null,
rankDrops30d: r.product.keepa?.salesRankDrops30 ?? null,
rankDrops90d: r.product.keepa?.salesRankDrops90 ?? null,
fbaFee: r.product.spApi.fbaFee ?? null,
fbmFee: r.product.spApi.fbmFee ?? null,
referralPercent: r.product.spApi.referralFeePercent ?? null,
canSell:
r.product.spApi.canSell == null
? "unknown"
: r.product.spApi.canSell
? "yes"
: "no",
r.product.spApi.sellabilityStatus ?? null,
r.product.spApi.sellabilityReason ?? null,
r.verdict.verdict,
r.verdict.confidence,
r.verdict.reasoning ?? null,
r.product.fetchedAt,
);
}
})(results); // Execute the transaction with the results batch
sellabilityStatus: r.product.spApi.sellabilityStatus ?? null,
sellabilityReason: r.product.spApi.sellabilityReason ?? null,
verdict: r.verdict.verdict,
confidence: r.verdict.confidence,
reasoning: r.verdict.reasoning ?? null,
fetchedAt: new Date(r.product.fetchedAt),
};
});
await db
.insert(categoryProductResults)
.values(rows)
.onConflictDoUpdate({
target: categoryProductResults.asin,
set: {
runId: sql`EXCLUDED.run_id`,
name: sql`EXCLUDED.name`,
brand: sql`EXCLUDED.brand`,
category: sql`EXCLUDED.category`,
unitCost: sql`EXCLUDED.unit_cost`,
currentPrice: sql`EXCLUDED.current_price`,
avgPrice90d: sql`EXCLUDED.avg_price_90d`,
avgPrice90dSheet: sql`EXCLUDED.avg_price_90d_sheet`,
sellingPriceSheet: sql`EXCLUDED.selling_price_sheet`,
salesRank: sql`EXCLUDED.sales_rank`,
salesRankAvg90d: sql`EXCLUDED.sales_rank_avg_90d`,
sellerCount: sql`EXCLUDED.seller_count`,
amazonIsSeller: sql`EXCLUDED.amazon_is_seller`,
amazonBuyboxSharePct90d: sql`EXCLUDED.amazon_buybox_share_pct_90d`,
monthlySold: sql`EXCLUDED.monthly_sold`,
rankDrops30d: sql`EXCLUDED.rank_drops_30d`,
rankDrops90d: sql`EXCLUDED.rank_drops_90d`,
fbaFee: sql`EXCLUDED.fba_fee`,
fbmFee: sql`EXCLUDED.fbm_fee`,
referralPercent: sql`EXCLUDED.referral_percent`,
canSell: sql`EXCLUDED.can_sell`,
sellabilityStatus: sql`EXCLUDED.sellability_status`,
sellabilityReason: sql`EXCLUDED.sellability_reason`,
verdict: sql`EXCLUDED.verdict`,
confidence: sql`EXCLUDED.confidence`,
reasoning: sql`EXCLUDED.reasoning`,
fetchedAt: sql`EXCLUDED.fetched_at`,
},
});
}
function loadCategoryBlacklist(filePath: string): Set<number> {
@@ -1014,7 +986,6 @@ function buildEnrichedProducts(
}
export async function processCategory(
db: Database,
runId: number,
category: CategoryInfo,
perCategoryTop: number,
@@ -1025,7 +996,7 @@ export async function processCategory(
const topAsins = await fetchCategoryBestSellerAsins(category, perCategoryTop);
if (topAsins.length === 0) {
log("info", " Keepa returned no ASINs for this category.");
await updateCategoryRunSummary(db, runId, {
await updateCategoryRunSummary(runId, {
topAsinsChecked: 0,
availableAsins: 0,
fba: 0,
@@ -1069,7 +1040,7 @@ export async function processCategory(
` Sellable ASINs: ${availableAsins.length}/${uniqueTopAsins.length}`,
);
if (availableAsins.length === 0) {
await updateCategoryRunSummary(db, runId, {
await updateCategoryRunSummary(runId, {
topAsinsChecked: uniqueTopAsins.length,
availableAsins: 0,
fba: 0,
@@ -1137,7 +1108,7 @@ export async function processCategory(
},
}));
await insertProductAnalysisResults(db, runId, batchResults);
await insertProductAnalysisResults(runId, batchResults);
for (const result of batchResults) {
results.push(result);
@@ -1150,7 +1121,7 @@ export async function processCategory(
}
}
await updateCategoryRunSummary(db, runId, {
await updateCategoryRunSummary(runId, {
topAsinsChecked: uniqueTopAsins.length,
availableAsins: availableAsins.length,
fba,
@@ -1170,7 +1141,7 @@ export async function processCategory(
}
}
await updateCategoryRunSummary(db, runId, {
await updateCategoryRunSummary(runId, {
topAsinsChecked: uniqueTopAsins.length,
availableAsins: availableAsins.length,
fba,
@@ -1199,10 +1170,6 @@ export async function main(): Promise<void> {
assertSpApiPrerequisites();
mkdirSync(args.outputDir, { recursive: true });
const DB_PATH =
process.env.RESULTS_DB_PATH || path.join(process.cwd(), "db", "results.db");
initDb(DB_PATH);
const db = getDb(DB_PATH);
log("info", "Starting per-category bestseller pipeline");
log("info", `Marketplace: ${config.spApiMarketplaceId}`);
@@ -1236,7 +1203,6 @@ export async function main(): Promise<void> {
let runId: number | undefined;
try {
runId = await insertCategoryRunSummary(
db,
{
categoryId: category.id,
categoryLabel: category.label,
@@ -1253,7 +1219,6 @@ export async function main(): Promise<void> {
);
categorySummary = await processCategory(
db,
runId,
category,
args.perCategoryTop,
@@ -1283,7 +1248,7 @@ export async function main(): Promise<void> {
results: [],
};
if (runId) {
await updateCategoryRunSummary(db, runId, {
await updateCategoryRunSummary(runId, {
topAsinsChecked: 0,
availableAsins: 0,
fba: 0,

View File

@@ -1,21 +1,16 @@
import { getDb } from "./database.ts";
import path from "node:path";
import { db } from "./db/index.ts";
import { runs } from "./db/schema.ts";
import { eq } from "drizzle-orm";
async function checkDb() {
const DB_PATH =
process.env.RESULTS_DB_PATH || path.join(process.cwd(), "db", "results.db");
const db = getDb(DB_PATH);
try {
const query = db.query(
"SELECT * FROM category_analysis_runs WHERE category_id = ?",
);
const result = query.all(19419898011);
const result = await db
.select()
.from(runs)
.where(eq(runs.type, "category_analysis"));
console.log(JSON.stringify(result, null, 2));
} catch (error) {
console.error("Database query failed:", error);
} finally {
db.close();
}
}

View File

@@ -1,494 +1,3 @@
import { Database } from "bun:sqlite";
import { dirname } from "node:path";
import { mkdirSync } from "node:fs";
export { Database } from "bun:sqlite";
let db: Database | null = null;
export function getDb(dbPath: string): Database {
if (!db) {
const dbDir = dirname(dbPath);
if (dbDir && dbDir !== ".") {
mkdirSync(dbDir, { recursive: true });
}
db = new Database(dbPath);
db.run("PRAGMA journal_mode = WAL;"); // Enable WAL mode for better performance
db.run("PRAGMA foreign_keys = ON;"); // Enforce foreign key constraints
}
return db;
}
export function closeDb(): void {
if (db) {
db.close();
db = null;
}
}
function createProductAnalysisResultsTable(database: Database): void {
database.run(`
CREATE TABLE IF NOT EXISTS product_analysis_results (
id INTEGER PRIMARY KEY AUTOINCREMENT,
asin TEXT NOT NULL,
run_id INTEGER NOT NULL,
name TEXT NOT NULL,
brand TEXT,
category TEXT,
unit_cost REAL,
current_price REAL,
avg_price_90d REAL,
avg_price_90d_sheet REAL,
selling_price_sheet REAL,
sales_rank INTEGER,
sales_rank_avg_90d INTEGER,
seller_count INTEGER,
amazon_is_seller INTEGER,
amazon_buybox_share_pct_90d REAL,
monthly_sold INTEGER,
rank_drops_30d INTEGER,
rank_drops_90d INTEGER,
fba_fee REAL,
fbm_fee REAL,
referral_percent REAL,
can_sell TEXT,
sellability_status TEXT,
sellability_reason TEXT,
verdict TEXT NOT NULL,
confidence REAL NOT NULL,
reasoning TEXT,
fetched_at TEXT NOT NULL,
UNIQUE(asin),
FOREIGN KEY (run_id) REFERENCES category_analysis_runs(id)
);
`);
}
function ensureProductAnalysisResultsTable(database: Database): void {
const tableInfo = database
.query("PRAGMA table_info(product_analysis_results)")
.all() as Array<{ name: string; pk: number }>;
if (tableInfo.length === 0) {
createProductAnalysisResultsTable(database);
return;
}
const hasIdColumn = tableInfo.some((col) => col.name === "id");
const hasAsinPrimaryKey = tableInfo.some(
(col) => col.name === "asin" && col.pk === 1,
);
const indexList = database
.query("PRAGMA index_list(product_analysis_results)")
.all() as Array<{ name: string; unique: number }>;
const hasUniqueAsinConstraint = indexList.some((idx) => {
if (idx.unique !== 1) return false;
const columns = database
.query(`PRAGMA index_info(${JSON.stringify(idx.name)})`)
.all() as Array<{ name: string }>;
return columns.length === 1 && columns[0]?.name === "asin";
});
if (!hasIdColumn || hasAsinPrimaryKey || !hasUniqueAsinConstraint) {
database.run(
"ALTER TABLE product_analysis_results RENAME TO product_analysis_results_legacy",
);
createProductAnalysisResultsTable(database);
database.run(`
WITH ranked AS (
SELECT
asin, run_id, name, brand, category, unit_cost,
current_price, avg_price_90d, avg_price_90d_sheet,
selling_price_sheet, sales_rank, sales_rank_avg_90d,
seller_count, NULL AS amazon_is_seller,
NULL AS amazon_buybox_share_pct_90d,
monthly_sold, rank_drops_30d, rank_drops_90d,
fba_fee, fbm_fee, referral_percent, can_sell,
sellability_status, sellability_reason,
verdict, confidence, reasoning, fetched_at,
ROW_NUMBER() OVER (
PARTITION BY asin
ORDER BY datetime(fetched_at) DESC, run_id DESC, id DESC
) AS row_num
FROM product_analysis_results_legacy
)
INSERT INTO product_analysis_results (
asin, run_id, name, brand, category, unit_cost,
current_price, avg_price_90d, avg_price_90d_sheet,
selling_price_sheet, sales_rank, sales_rank_avg_90d,
seller_count, amazon_is_seller, amazon_buybox_share_pct_90d,
monthly_sold, rank_drops_30d, rank_drops_90d,
fba_fee, fbm_fee, referral_percent, can_sell,
sellability_status, sellability_reason,
verdict, confidence, reasoning, fetched_at
)
SELECT
asin, run_id, name, brand, category, unit_cost,
current_price, avg_price_90d, avg_price_90d_sheet,
selling_price_sheet, sales_rank, sales_rank_avg_90d,
seller_count, amazon_is_seller, amazon_buybox_share_pct_90d,
monthly_sold, rank_drops_30d, rank_drops_90d,
fba_fee, fbm_fee, referral_percent, can_sell,
sellability_status, sellability_reason,
verdict, confidence, reasoning, fetched_at
FROM ranked
WHERE row_num = 1
`);
database.run("DROP TABLE product_analysis_results_legacy");
}
}
function ensureProductAnalysisResultsColumns(database: Database): void {
const tableInfo = database
.query("PRAGMA table_info(product_analysis_results)")
.all() as Array<{ name: string }>;
if (tableInfo.length === 0) {
return;
}
const existingColumns = new Set(tableInfo.map((col) => col.name));
const requiredColumns: Array<{ name: string; type: string }> = [
{ name: "amazon_is_seller", type: "INTEGER" },
{ name: "amazon_buybox_share_pct_90d", type: "REAL" },
];
for (const column of requiredColumns) {
if (!existingColumns.has(column.name)) {
database.run(
`ALTER TABLE product_analysis_results ADD COLUMN ${column.name} ${column.type}`,
);
}
}
}
function ensureResultsTableColumns(database: Database): void {
const tableInfo = database
.query("PRAGMA table_info(results)")
.all() as Array<{ name: string }>;
if (tableInfo.length === 0) {
return;
}
const existingColumns = new Set(tableInfo.map((col) => col.name));
const requiredColumns: Array<{ name: string; type: string }> = [
{ name: "fba_net_sheet", type: "REAL" },
{ name: "gross_profit_dollar", type: "REAL" },
{ name: "gross_profit_pct", type: "REAL" },
{ name: "net_profit_sheet", type: "REAL" },
{ name: "roi_sheet", type: "REAL" },
{ name: "moq", type: "INTEGER" },
{ name: "moq_cost", type: "REAL" },
{ name: "qty_available", type: "INTEGER" },
{ name: "supplier", type: "TEXT" },
{ name: "source_url", type: "TEXT" },
{ name: "asin_link", type: "TEXT" },
{ name: "promo_coupon_code", type: "TEXT" },
{ name: "notes", type: "TEXT" },
{ name: "lead_date", type: "TEXT" },
{ name: "amazon_is_seller", type: "INTEGER" },
{ name: "amazon_buybox_share_pct_90d", type: "REAL" },
{ name: "upc", type: "TEXT" },
{ name: "supplier_score", type: "REAL" },
{ name: "supplier_profit", type: "REAL" },
{ name: "supplier_margin", type: "REAL" },
{ name: "supplier_roi", type: "REAL" },
{ name: "supplier_reason", type: "TEXT" },
{ name: "upc_lookup_status", type: "TEXT" },
{ name: "upc_lookup_reason", type: "TEXT" },
{ name: "candidate_asins", type: "TEXT" },
];
for (const column of requiredColumns) {
if (!existingColumns.has(column.name)) {
database.run(
`ALTER TABLE results ADD COLUMN ${column.name} ${column.type}`,
);
}
}
}
export function initDb(dbPath: string): void {
const database = getDb(dbPath);
database.run(`
CREATE TABLE IF NOT EXISTS runs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp TEXT NOT NULL,
input_file TEXT NOT NULL,
output_file TEXT,
total_products INTEGER,
fba_count INTEGER,
fbm_count INTEGER,
skip_count INTEGER
);
`);
database.run(`
CREATE TABLE IF NOT EXISTS results (
id INTEGER PRIMARY KEY AUTOINCREMENT,
run_id INTEGER NOT NULL,
asin TEXT NOT NULL,
product_name TEXT,
brand TEXT,
category TEXT,
unit_cost REAL,
current_price REAL,
avg_price_90d REAL,
avg_price_90d_sheet REAL,
selling_price_sheet REAL,
sales_rank INTEGER,
rank_avg_90d INTEGER,
sellers INTEGER,
amazon_is_seller INTEGER,
amazon_buybox_share_pct_90d REAL,
monthly_sold INTEGER,
rank_drops_30d INTEGER,
rank_drops_90d INTEGER,
fba_net_sheet REAL,
gross_profit_dollar REAL,
gross_profit_pct REAL,
net_profit_sheet REAL,
roi_sheet REAL,
moq INTEGER,
moq_cost REAL,
qty_available INTEGER,
supplier TEXT,
source_url TEXT,
asin_link TEXT,
promo_coupon_code TEXT,
notes TEXT,
lead_date TEXT,
upc TEXT,
fba_fee REAL,
fbm_fee REAL,
referral_percent REAL,
supplier_score REAL,
supplier_profit REAL,
supplier_margin REAL,
supplier_roi REAL,
supplier_reason TEXT,
upc_lookup_status TEXT,
upc_lookup_reason TEXT,
candidate_asins TEXT,
can_sell TEXT,
sellability_status TEXT,
sellability_reason TEXT,
verdict TEXT NOT NULL,
confidence INTEGER,
reasoning TEXT,
fetched_at TEXT NOT NULL,
FOREIGN KEY (run_id) REFERENCES runs(id)
);
`);
ensureResultsTableColumns(database);
database.run(`
CREATE TABLE IF NOT EXISTS category_analysis_runs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
category_id INTEGER NOT NULL,
category_label TEXT NOT NULL,
run_timestamp TEXT NOT NULL,
top_asins_checked INTEGER NOT NULL,
available_asins INTEGER NOT NULL,
fba_count INTEGER NOT NULL,
fbm_count INTEGER NOT NULL,
skip_count INTEGER NOT NULL,
status TEXT NOT NULL,
error_message TEXT
);
`);
ensureProductAnalysisResultsTable(database);
ensureProductAnalysisResultsColumns(database);
database.run(
`CREATE INDEX IF NOT EXISTS idx_runs_timestamp ON runs(timestamp DESC);`,
);
database.run(
`CREATE INDEX IF NOT EXISTS idx_results_run_id ON results(run_id);`,
);
database.run(
`CREATE INDEX IF NOT EXISTS idx_results_verdict ON results(verdict);`,
);
database.run(
`CREATE INDEX IF NOT EXISTS idx_results_sellability_status ON results(sellability_status);`,
);
database.run(
`CREATE INDEX IF NOT EXISTS idx_results_fetched_at ON results(fetched_at DESC);`,
);
database.run(`CREATE INDEX IF NOT EXISTS idx_results_asin ON results(asin);`);
database.run(
`CREATE INDEX IF NOT EXISTS idx_category_runs_timestamp ON category_analysis_runs(run_timestamp DESC);`,
);
database.run(
`CREATE INDEX IF NOT EXISTS idx_category_runs_status ON category_analysis_runs(status);`,
);
database.run(
`CREATE INDEX IF NOT EXISTS idx_product_results_run_id ON product_analysis_results(run_id);`,
);
database.run(
`CREATE INDEX IF NOT EXISTS idx_product_results_verdict ON product_analysis_results(verdict);`,
);
database.run(
`CREATE INDEX IF NOT EXISTS idx_product_results_sellability_status ON product_analysis_results(sellability_status);`,
);
database.run(
`CREATE INDEX IF NOT EXISTS idx_product_results_fetched_at ON product_analysis_results(fetched_at DESC);`,
);
initStalkerDb(database);
}
export function initStalkerDb(database: Database): void {
resetLegacyStalkerSchema(database);
database.run(`
CREATE TABLE IF NOT EXISTS stalker_runs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
input_file TEXT NOT NULL,
started_at TEXT NOT NULL,
completed_at TEXT,
requested_asins INTEGER NOT NULL DEFAULT 0,
skipped_asins INTEGER NOT NULL DEFAULT 0,
scanned_asins INTEGER NOT NULL DEFAULT 0,
source_asins_with_matches INTEGER NOT NULL DEFAULT 0,
candidate_sellers INTEGER NOT NULL DEFAULT 0,
qualifying_sellers INTEGER NOT NULL DEFAULT 0,
matched_sellers INTEGER NOT NULL DEFAULT 0,
seller_metadata_requests INTEGER NOT NULL DEFAULT 0,
seller_storefront_requests INTEGER NOT NULL DEFAULT 0,
inventory_sellability_checked_asins INTEGER NOT NULL DEFAULT 0,
inventory_sellability_available_asins INTEGER NOT NULL DEFAULT 0,
inventory_sellability_excluded_asins INTEGER NOT NULL DEFAULT 0,
persisted_inventory_asins INTEGER NOT NULL DEFAULT 0,
status TEXT NOT NULL,
error_message TEXT
);
`);
database.run(`
CREATE TABLE IF NOT EXISTS stalker_asin_scans (
id INTEGER PRIMARY KEY AUTOINCREMENT,
run_id INTEGER NOT NULL,
source_asin TEXT NOT NULL,
title TEXT,
offer_count INTEGER NOT NULL DEFAULT 0,
candidate_seller_count INTEGER NOT NULL DEFAULT 0,
matched_seller_count INTEGER NOT NULL DEFAULT 0,
fetched_at TEXT NOT NULL,
raw_product_json TEXT,
UNIQUE(run_id, source_asin),
FOREIGN KEY (run_id) REFERENCES stalker_runs(id) ON DELETE CASCADE
);
`);
database.run(`
CREATE TABLE IF NOT EXISTS stalker_sellers (
seller_id TEXT PRIMARY KEY,
seller_name TEXT,
rating REAL,
rating_count INTEGER,
storefront_asin_total INTEGER,
persisted_inventory_sample_count INTEGER,
last_updated_at TEXT NOT NULL,
raw_seller_json TEXT
);
`);
database.run(`
CREATE TABLE IF NOT EXISTS stalker_asin_sellers (
id INTEGER PRIMARY KEY AUTOINCREMENT,
scan_id INTEGER NOT NULL,
seller_id TEXT NOT NULL,
offer_price REAL,
condition TEXT,
is_fba INTEGER,
stock INTEGER,
seller_rating REAL,
seller_rating_count INTEGER,
raw_offer_json TEXT,
UNIQUE(scan_id, seller_id),
FOREIGN KEY (scan_id) REFERENCES stalker_asin_scans(id) ON DELETE CASCADE,
FOREIGN KEY (seller_id) REFERENCES stalker_sellers(seller_id)
);
`);
database.run(`
CREATE TABLE IF NOT EXISTS stalker_seller_inventory (
id INTEGER PRIMARY KEY AUTOINCREMENT,
run_id INTEGER NOT NULL,
seller_id TEXT NOT NULL,
asin TEXT NOT NULL,
can_sell INTEGER,
sellability_status TEXT,
sellability_reason TEXT,
product_title TEXT,
brand TEXT,
category_tree TEXT,
current_price REAL,
avg_price_90d REAL,
sales_rank INTEGER,
monthly_sold INTEGER,
seller_count INTEGER,
amazon_is_seller INTEGER,
raw_product_json TEXT,
last_seen_at TEXT NOT NULL,
raw_inventory_json TEXT,
UNIQUE(run_id, seller_id, asin),
FOREIGN KEY (run_id) REFERENCES stalker_runs(id) ON DELETE CASCADE,
FOREIGN KEY (seller_id) REFERENCES stalker_sellers(seller_id)
);
`);
database.run(
`CREATE INDEX IF NOT EXISTS idx_stalker_runs_started_at ON stalker_runs(started_at DESC);`,
);
database.run(
`CREATE INDEX IF NOT EXISTS idx_stalker_scans_run_id ON stalker_asin_scans(run_id);`,
);
database.run(
`CREATE INDEX IF NOT EXISTS idx_stalker_scans_source_asin ON stalker_asin_scans(source_asin);`,
);
database.run(
`CREATE INDEX IF NOT EXISTS idx_stalker_asin_sellers_seller_id ON stalker_asin_sellers(seller_id);`,
);
database.run(
`CREATE INDEX IF NOT EXISTS idx_stalker_inventory_seller_id ON stalker_seller_inventory(seller_id);`,
);
database.run(
`CREATE INDEX IF NOT EXISTS idx_stalker_inventory_asin ON stalker_seller_inventory(asin);`,
);
database.run(
`CREATE INDEX IF NOT EXISTS idx_stalker_inventory_product_title ON stalker_seller_inventory(product_title);`,
);
}
function resetLegacyStalkerSchema(database: Database): void {
const runColumns = database
.query("PRAGMA table_info(stalker_runs)")
.all() as Array<{ name: string }>;
if (runColumns.length === 0) return;
const columnNames = new Set(runColumns.map((column) => column.name));
if (
columnNames.has("scanned_asins") &&
columnNames.has("inventory_sellability_checked_asins") &&
inventoryColumnsHaveSellability(database)
) {
return;
}
database.run("DROP TABLE IF EXISTS stalker_seller_inventory");
database.run("DROP TABLE IF EXISTS stalker_asin_sellers");
database.run("DROP TABLE IF EXISTS stalker_sellers");
database.run("DROP TABLE IF EXISTS stalker_asin_scans");
database.run("DROP TABLE IF EXISTS stalker_runs");
}
function inventoryColumnsHaveSellability(database: Database): boolean {
const inventoryColumns = database
.query("PRAGMA table_info(stalker_seller_inventory)")
.all() as Array<{ name: string }>;
const columnNames = new Set(inventoryColumns.map((column) => column.name));
return (
columnNames.has("sellability_status") &&
columnNames.has("product_title")
);
}
// Central re-export so existing `import { db } from "./database.ts"` keeps working.
export { db, type Db } from "./db/index.ts";
export * as schema from "./db/schema.ts";

15
src/db/index.ts Normal file
View File

@@ -0,0 +1,15 @@
import { drizzle } from "drizzle-orm/postgres-js";
import postgres from "postgres";
import * as schema from "./schema.ts";
const url = Bun.env.DB_CONNECTION_STRING;
if (!url) {
throw new Error("Missing required env var: DB_CONNECTION_STRING");
}
// Shared connection pool — imported once and reused across the process.
export const client = postgres(url);
export const db = drizzle(client, { schema });
export type Db = typeof db;

343
src/db/schema.ts Normal file
View File

@@ -0,0 +1,343 @@
import {
boolean,
index,
integer,
pgEnum,
pgTable,
real,
serial,
text,
timestamp,
unique,
} from "drizzle-orm/pg-core";
// ─── Enums ───────────────────────────────────────────────────────────────────
export const runTypeEnum = pgEnum("run_type", [
"lead_analysis",
"category_analysis",
"supplier_upc",
"stalker",
]);
export const runStatusEnum = pgEnum("run_status", [
"running",
"ok",
"empty",
"failed",
"completed",
]);
// ─── Runs ─────────────────────────────────────────────────────────────────────
// Unified run log; replaces the old `runs` and `category_analysis_runs` tables.
// Category-specific columns (categoryId, categoryLabel, …) are null for
// lead_analysis / supplier_upc runs.
export const runs = pgTable(
"runs",
{
id: serial("id").primaryKey(),
type: runTypeEnum("type").notNull(),
inputFile: text("input_file"),
outputFile: text("output_file"),
status: runStatusEnum("status").notNull().default("running"),
errorMessage: text("error_message"),
totalProducts: integer("total_products"),
fbaCount: integer("fba_count"),
fbmCount: integer("fbm_count"),
skipCount: integer("skip_count"),
// Category-pipeline only
categoryId: integer("category_id"),
categoryLabel: text("category_label"),
topAsinsChecked: integer("top_asins_checked"),
availableAsins: integer("available_asins"),
startedAt: timestamp("started_at", { withTimezone: true })
.notNull()
.defaultNow(),
completedAt: timestamp("completed_at", { withTimezone: true }),
},
(t) => [
index("idx_runs_started_at").on(t.startedAt),
index("idx_runs_type").on(t.type),
index("idx_runs_status").on(t.status),
],
);
// ─── Analysis results ─────────────────────────────────────────────────────────
// Archival table: one row per product per run for the lead-list and supplier
// UPC pipelines. Multiple rows for the same ASIN across different runs is fine.
export const analysisResults = pgTable(
"analysis_results",
{
id: serial("id").primaryKey(),
runId: integer("run_id")
.notNull()
.references(() => runs.id),
asin: text("asin").notNull(),
// Product identity
productName: text("product_name"),
brand: text("brand"),
category: text("category"),
upc: text("upc"),
// Supplier sheet data (lead_analysis only)
unitCost: real("unit_cost"),
avgPrice90dSheet: real("avg_price_90d_sheet"),
sellingPriceSheet: real("selling_price_sheet"),
fbaNetSheet: real("fba_net_sheet"),
grossProfitDollar: real("gross_profit_dollar"),
grossProfitPct: real("gross_profit_pct"),
netProfitSheet: real("net_profit_sheet"),
roiSheet: real("roi_sheet"),
moq: integer("moq"),
moqCost: real("moq_cost"),
qtyAvailable: integer("qty_available"),
supplier: text("supplier"),
sourceUrl: text("source_url"),
asinLink: text("asin_link"),
promoCouponCode: text("promo_coupon_code"),
notes: text("notes"),
leadDate: text("lead_date"),
// Market data
currentPrice: real("current_price"),
avgPrice90d: real("avg_price_90d"),
salesRank: integer("sales_rank"),
rankAvg90d: integer("rank_avg_90d"),
monthlySold: integer("monthly_sold"),
rankDrops30d: integer("rank_drops_30d"),
rankDrops90d: integer("rank_drops_90d"),
sellerCount: integer("seller_count"),
amazonIsSeller: boolean("amazon_is_seller"),
amazonBuyboxSharePct90d: real("amazon_buybox_share_pct_90d"),
// Fees
fbaFee: real("fba_fee"),
fbmFee: real("fbm_fee"),
referralPercent: real("referral_percent"),
// Sellability
canSell: text("can_sell"),
sellabilityStatus: text("sellability_status"),
sellabilityReason: text("sellability_reason"),
// Supplier-UPC scoring
supplierScore: real("supplier_score"),
supplierProfit: real("supplier_profit"),
supplierMargin: real("supplier_margin"),
supplierRoi: real("supplier_roi"),
supplierReason: text("supplier_reason"),
upcLookupStatus: text("upc_lookup_status"),
upcLookupReason: text("upc_lookup_reason"),
candidateAsins: text("candidate_asins"),
// Verdict
verdict: text("verdict").notNull(),
confidence: real("confidence"),
reasoning: text("reasoning"),
fetchedAt: timestamp("fetched_at", { withTimezone: true }).notNull(),
},
(t) => [
index("idx_analysis_results_run_id").on(t.runId),
index("idx_analysis_results_asin").on(t.asin),
index("idx_analysis_results_verdict").on(t.verdict),
index("idx_analysis_results_sellability_status").on(t.sellabilityStatus),
index("idx_analysis_results_fetched_at").on(t.fetchedAt),
],
);
// ─── Category product results ──────────────────────────────────────────────────
// Latest-per-ASIN snapshot for the category pipelines (bestsellers, monthly-sold,
// mid-range, stalker analysis). Upserted on conflict so each ASIN has one row.
export const categoryProductResults = pgTable(
"category_product_results",
{
id: serial("id").primaryKey(),
asin: text("asin").notNull().unique(),
runId: integer("run_id")
.notNull()
.references(() => runs.id),
name: text("name").notNull(),
brand: text("brand"),
category: text("category"),
unitCost: real("unit_cost"),
currentPrice: real("current_price"),
avgPrice90d: real("avg_price_90d"),
avgPrice90dSheet: real("avg_price_90d_sheet"),
sellingPriceSheet: real("selling_price_sheet"),
salesRank: integer("sales_rank"),
salesRankAvg90d: integer("sales_rank_avg_90d"),
sellerCount: integer("seller_count"),
amazonIsSeller: boolean("amazon_is_seller"),
amazonBuyboxSharePct90d: real("amazon_buybox_share_pct_90d"),
monthlySold: integer("monthly_sold"),
rankDrops30d: integer("rank_drops_30d"),
rankDrops90d: integer("rank_drops_90d"),
fbaFee: real("fba_fee"),
fbmFee: real("fbm_fee"),
referralPercent: real("referral_percent"),
canSell: text("can_sell"),
sellabilityStatus: text("sellability_status"),
sellabilityReason: text("sellability_reason"),
verdict: text("verdict").notNull(),
confidence: real("confidence").notNull(),
reasoning: text("reasoning"),
fetchedAt: timestamp("fetched_at", { withTimezone: true }).notNull(),
},
(t) => [
index("idx_category_results_run_id").on(t.runId),
index("idx_category_results_verdict").on(t.verdict),
index("idx_category_results_sellability_status").on(t.sellabilityStatus),
index("idx_category_results_fetched_at").on(t.fetchedAt),
],
);
// ─── Stalker runs ─────────────────────────────────────────────────────────────
export const stalkerRuns = pgTable(
"stalker_runs",
{
id: serial("id").primaryKey(),
inputFile: text("input_file").notNull(),
startedAt: timestamp("started_at", { withTimezone: true }).notNull(),
completedAt: timestamp("completed_at", { withTimezone: true }),
requestedAsins: integer("requested_asins").notNull().default(0),
skippedAsins: integer("skipped_asins").notNull().default(0),
scannedAsins: integer("scanned_asins").notNull().default(0),
sourceAsinsWithMatches: integer("source_asins_with_matches")
.notNull()
.default(0),
candidateSellers: integer("candidate_sellers").notNull().default(0),
qualifyingSellers: integer("qualifying_sellers").notNull().default(0),
matchedSellers: integer("matched_sellers").notNull().default(0),
sellerMetadataRequests: integer("seller_metadata_requests")
.notNull()
.default(0),
sellerStorefrontRequests: integer("seller_storefront_requests")
.notNull()
.default(0),
inventorySellabilityCheckedAsins: integer(
"inventory_sellability_checked_asins",
)
.notNull()
.default(0),
inventorySellabilityAvailableAsins: integer(
"inventory_sellability_available_asins",
)
.notNull()
.default(0),
inventorySellabilityExcludedAsins: integer(
"inventory_sellability_excluded_asins",
)
.notNull()
.default(0),
persistedInventoryAsins: integer("persisted_inventory_asins")
.notNull()
.default(0),
status: text("status").notNull(),
errorMessage: text("error_message"),
},
(t) => [index("idx_stalker_runs_started_at").on(t.startedAt)],
);
// ─── Stalker ASIN scans ───────────────────────────────────────────────────────
export const stalkerAsinScans = pgTable(
"stalker_asin_scans",
{
id: serial("id").primaryKey(),
runId: integer("run_id")
.notNull()
.references(() => stalkerRuns.id, { onDelete: "cascade" }),
sourceAsin: text("source_asin").notNull(),
title: text("title"),
offerCount: integer("offer_count").notNull().default(0),
candidateSellerCount: integer("candidate_seller_count")
.notNull()
.default(0),
matchedSellerCount: integer("matched_seller_count").notNull().default(0),
fetchedAt: timestamp("fetched_at", { withTimezone: true }).notNull(),
rawProductJson: text("raw_product_json"),
},
(t) => [
unique("uq_stalker_scans_run_asin").on(t.runId, t.sourceAsin),
index("idx_stalker_scans_run_id").on(t.runId),
index("idx_stalker_scans_source_asin").on(t.sourceAsin),
],
);
// ─── Sellers ──────────────────────────────────────────────────────────────────
// General seller registry (was stalker_sellers).
export const sellers = pgTable("sellers", {
sellerId: text("seller_id").primaryKey(),
sellerName: text("seller_name"),
rating: real("rating"),
ratingCount: integer("rating_count"),
storefrontAsinTotal: integer("storefront_asin_total"),
persistedInventorySampleCount: integer("persisted_inventory_sample_count"),
lastUpdatedAt: timestamp("last_updated_at", { withTimezone: true }).notNull(),
rawSellerJson: text("raw_seller_json"),
});
// ─── Stalker ASIN sellers ─────────────────────────────────────────────────────
export const stalkerAsinSellers = pgTable(
"stalker_asin_sellers",
{
id: serial("id").primaryKey(),
scanId: integer("scan_id")
.notNull()
.references(() => stalkerAsinScans.id, { onDelete: "cascade" }),
sellerId: text("seller_id")
.notNull()
.references(() => sellers.sellerId),
offerPrice: real("offer_price"),
condition: text("condition"),
isFba: boolean("is_fba"),
stock: integer("stock"),
sellerRating: real("seller_rating"),
sellerRatingCount: integer("seller_rating_count"),
rawOfferJson: text("raw_offer_json"),
},
(t) => [
unique("uq_stalker_asin_sellers_scan_seller").on(t.scanId, t.sellerId),
],
);
// ─── Stalker seller inventory ─────────────────────────────────────────────────
export const stalkerSellerInventory = pgTable(
"stalker_seller_inventory",
{
id: serial("id").primaryKey(),
runId: integer("run_id")
.notNull()
.references(() => stalkerRuns.id, { onDelete: "cascade" }),
sellerId: text("seller_id")
.notNull()
.references(() => sellers.sellerId),
asin: text("asin").notNull(),
canSell: boolean("can_sell"),
sellabilityStatus: text("sellability_status"),
sellabilityReason: text("sellability_reason"),
productTitle: text("product_title"),
brand: text("brand"),
categoryTree: text("category_tree"),
currentPrice: real("current_price"),
avgPrice90d: real("avg_price_90d"),
salesRank: integer("sales_rank"),
monthlySold: integer("monthly_sold"),
sellerCount: integer("seller_count"),
amazonIsSeller: boolean("amazon_is_seller"),
rawProductJson: text("raw_product_json"),
lastSeenAt: timestamp("last_seen_at", { withTimezone: true }).notNull(),
rawInventoryJson: text("raw_inventory_json"),
},
(t) => [
unique("uq_stalker_inventory_run_seller_asin").on(
t.runId,
t.sellerId,
t.asin,
),
index("idx_stalker_inventory_seller_id").on(t.sellerId),
index("idx_stalker_inventory_asin").on(t.asin),
index("idx_stalker_inventory_product_title").on(t.productTitle),
],
);

View File

@@ -5,7 +5,6 @@ import {
writeResultsToDb,
writeResultsWorkbook,
} from "./writer.ts";
import { initDb, closeDb } from "./database.ts";
import {
chunkArray,
processProductChunk,
@@ -14,7 +13,6 @@ import {
import path from "node:path";
import type { AnalysisResult } from "./types.ts";
const DB_PATH = process.env.RESULTS_DB_PATH || path.join("db", "results.db");
const INPUT_BATCH_SIZE = 50;
function parseSellabilityArg(args: string[]): SellabilityFilter {
@@ -119,9 +117,6 @@ async function main() {
console.log("Connecting to Redis...");
await connectCache();
console.log("Initializing SQLite database...");
initDb(DB_PATH);
try {
console.log(`\nReading ${inputFile}...`);
const products = readProducts(inputFile);
@@ -156,10 +151,9 @@ async function main() {
printResults(allResults);
writeResultsWorkbook(allResults, resolvedBaseOutputPath);
writeResultsToDb(allResults, DB_PATH, inputFile, resolvedBaseOutputPath);
await writeResultsToDb(allResults, inputFile, resolvedBaseOutputPath);
} finally {
await disconnectCache();
closeDb();
}
}

View File

@@ -1,8 +1,41 @@
import { test, expect, beforeAll, afterAll, beforeEach, mock } from "bun:test";
import { Database } from "bun:sqlite";
import { getDb, initDb, closeDb } from "./database.ts";
import path from "node:path";
import { rmSync, mkdirSync } from "node:fs";
import { test, expect, beforeEach, mock } from "bun:test";
let nextId = 0;
function chainable(resolveWith: any[] = []): any {
const p: any = Promise.resolve(resolveWith);
p.limit = (_n: any) => chainable(resolveWith);
p.where = (_cond: any) => chainable(resolveWith);
p.from = (_table: any) => chainable(resolveWith);
return p;
}
const makeMockDb = (): any => ({
insert: (_table: any) => ({
values: (_vals: any) => ({
returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]),
onConflictDoUpdate: (_conf: any) => Promise.resolve([]),
}),
}),
update: (_table: any) => ({
set: (_vals: any) => ({
where: (_cond: any) => Promise.resolve([]),
}),
}),
select: (_sel?: any) => ({
from: (_table: any) => ({
where: (_cond: any) => chainable(),
limit: (_n: any) => chainable(),
}),
}),
selectDistinct: (_sel: any) => ({
from: (_table: any) => chainable(),
}),
execute: (_query: any) => Promise.resolve([]),
transaction: async (fn: (tx: any) => Promise<any>) => fn(makeMockDb()),
});
mock.module("./db/index.ts", () => ({ db: makeMockDb(), client: {} }));
const fetchSellabilityBatchMock = mock(async (asins: string[]) => {
return new Map<string, any>(
@@ -62,44 +95,17 @@ mock.module("./llm.ts", () => ({
const modulePromise = import("./mid-range-sellers-by-category.ts");
const DB_TEST_PATH = path.join(
process.cwd(),
"test_output",
"test_mid_range_analysis.sqlite",
);
let db: Database;
let processCategory: any;
let insertCategoryRunSummary: (
db: Database,
summary: any,
runTimestamp: string,
) => Promise<number>;
let insertCategoryRunSummary: (summary: any, runTimestamp: string) => Promise<number>;
let originalFetch: typeof globalThis.fetch;
beforeAll(async () => {
const mod = await modulePromise;
processCategory = mod.processCategory;
insertCategoryRunSummary = mod.insertCategoryRunSummary;
rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true });
mkdirSync(path.dirname(DB_TEST_PATH), { recursive: true });
initDb(DB_TEST_PATH);
db = getDb(DB_TEST_PATH);
originalFetch = globalThis.fetch;
});
afterAll(() => {
globalThis.fetch = originalFetch;
closeDb();
rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true });
});
const mod = await modulePromise;
processCategory = mod.processCategory;
insertCategoryRunSummary = mod.insertCategoryRunSummary;
originalFetch = globalThis.fetch;
beforeEach(() => {
db.run("DELETE FROM product_analysis_results");
db.run("DELETE FROM category_analysis_runs");
nextId = 0;
globalThis.fetch = mock(async (input: string | URL | Request) => {
const rawUrl =
typeof input === "string"
@@ -138,25 +144,8 @@ beforeEach(() => {
buyBoxStatsAmazon90: 40,
stats: {
current: [
null,
null,
null,
1000,
null,
null,
null,
null,
null,
null,
null,
5,
null,
null,
null,
null,
null,
null,
2599,
null, null, null, 1000, null, null, null, null, null, null, null, 5,
null, null, null, null, null, null, 2599,
],
avg: [2400, null, null, 1200],
},
@@ -171,25 +160,8 @@ beforeEach(() => {
buyBoxStatsAmazon90: 50,
stats: {
current: [
null,
null,
null,
2000,
null,
null,
null,
null,
null,
null,
null,
3,
null,
null,
null,
null,
null,
null,
1999,
null, null, null, 2000, null, null, null, null, null, null, null, 3,
null, null, null, null, null, null, 1999,
],
avg: [1800, null, null, 2200],
},
@@ -204,25 +176,8 @@ beforeEach(() => {
buyBoxStatsAmazon90: 50,
stats: {
current: [
null,
null,
null,
1500,
null,
null,
null,
null,
null,
null,
null,
4,
null,
null,
null,
null,
null,
null,
2099,
null, null, null, 1500, null, null, null, null, null, null, null, 4,
null, null, null, null, null, null, 2099,
],
avg: [2000, null, null, 1800],
},
@@ -237,25 +192,8 @@ beforeEach(() => {
buyBoxStatsAmazon90: 95,
stats: {
current: [
null,
null,
null,
3000,
null,
null,
null,
null,
null,
null,
null,
4,
null,
null,
null,
null,
null,
null,
2899,
null, null, null, 3000, null, null, null, null, null, null, null, 4,
null, null, null, null, null, null, 2899,
],
avg: [2600, null, null, 2800],
},
@@ -269,25 +207,8 @@ beforeEach(() => {
isAmazonSeller: false,
stats: {
current: [
null,
null,
null,
3200,
null,
null,
null,
null,
null,
null,
null,
25,
null,
null,
null,
null,
null,
null,
3500,
null, null, null, 3200, null, null, null, null, null, null, null, 25,
null, null, null, null, null, null, 3500,
],
avg: [3200, null, null, 3200],
},
@@ -315,7 +236,6 @@ test("processCategory only analyzes sellable mid-range matches", async () => {
};
const runId = await insertCategoryRunSummary(
db,
{
categoryId: mockCategory.id,
categoryLabel: mockCategory.label,
@@ -332,7 +252,6 @@ test("processCategory only analyzes sellable mid-range matches", async () => {
);
const summary = await processCategory(
db,
runId,
mockCategory,
3,
@@ -345,6 +264,7 @@ test("processCategory only analyzes sellable mid-range matches", async () => {
20,
15,
85,
"strict",
);
expect(summary.status).toBe("ok");
@@ -352,23 +272,7 @@ test("processCategory only analyzes sellable mid-range matches", async () => {
expect(summary.availableAsins).toBe(1);
expect(summary.results?.length).toBe(1);
const productResults = db
.query(
"SELECT asin, monthly_sold, can_sell, sellability_status FROM product_analysis_results ORDER BY monthly_sold DESC",
)
.all() as Array<{
asin: string;
monthly_sold: number;
can_sell: string;
sellability_status: string;
}>;
expect(productResults.length).toBe(1);
expect(productResults.map((row) => row.asin)).toEqual(["B000000001"]);
const sellable = productResults.find((row) => row.asin === "B000000001");
expect(sellable?.can_sell).toBe("yes");
expect(sellable?.sellability_status).toBe("available");
globalThis.fetch = originalFetch;
});
test("processCategory returns empty when no products match mid-range criteria", async () => {
@@ -380,7 +284,6 @@ test("processCategory returns empty when no products match mid-range criteria",
};
const runId = await insertCategoryRunSummary(
db,
{
categoryId: mockCategory.id,
categoryLabel: mockCategory.label,
@@ -397,7 +300,6 @@ test("processCategory returns empty when no products match mid-range criteria",
);
const summary = await processCategory(
db,
runId,
mockCategory,
3,
@@ -410,6 +312,7 @@ test("processCategory returns empty when no products match mid-range criteria",
20,
15,
85,
"strict",
);
expect(summary.status).toBe("empty");
@@ -417,8 +320,5 @@ test("processCategory returns empty when no products match mid-range criteria",
expect(summary.availableAsins).toBe(0);
expect(summary.results?.length).toBe(0);
const rows = db
.query("SELECT COUNT(*) as c FROM product_analysis_results")
.all() as Array<{ c: number }>;
expect(rows[0]?.c).toBe(0);
globalThis.fetch = originalFetch;
});

View File

@@ -2,7 +2,9 @@ import { existsSync, mkdirSync, readFileSync } from "node:fs";
import path from "node:path";
import { createInterface } from "node:readline/promises";
import { stdin as input, stdout as output } from "node:process";
import { type Database, getDb, initDb } from "./database.ts";
import { db } from "./db/index.ts";
import { runs, categoryProductResults } from "./db/schema.ts";
import { eq, sql } from "drizzle-orm";
import { config } from "./config.ts";
import {
connectCache,
@@ -474,36 +476,32 @@ async function promptCategoryIds(
}
export async function insertCategoryRunSummary(
db: Database,
summary: CategoryRunSummary,
runTimestamp: string,
): Promise<number> {
const query = `
INSERT INTO category_analysis_runs (
category_id, category_label, run_timestamp,
top_asins_checked, available_asins,
fba_count, fbm_count, skip_count,
status, error_message
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
`;
const result = db.run(query, [
summary.categoryId,
summary.categoryLabel,
runTimestamp,
summary.topAsinsChecked,
summary.availableAsins,
summary.fba,
summary.fbm,
summary.skip,
summary.status,
summary.error,
]);
// Bun's SQLite client returns { changes: number, lastInsertRowid: number | bigint }
return Number(result.lastInsertRowid);
const [row] = await db
.insert(runs)
.values({
type: "category_analysis",
status: (summary.status as typeof runs.$inferInsert.status) ?? "running",
categoryId: summary.categoryId,
categoryLabel: summary.categoryLabel,
topAsinsChecked: summary.topAsinsChecked,
availableAsins: summary.availableAsins,
totalProducts: summary.topAsinsChecked,
fbaCount: summary.fba,
fbmCount: summary.fbm,
skipCount: summary.skip,
errorMessage: summary.error || null,
startedAt: new Date(runTimestamp),
})
.returning({ id: runs.id });
if (!row) throw new Error("Failed to insert category run.");
return row.id;
}
export async function updateCategoryRunSummary(
db: Database,
runId: number,
summary: Pick<
CategoryRunSummary,
@@ -516,136 +514,110 @@ export async function updateCategoryRunSummary(
| "error"
>,
): Promise<void> {
db.run(
`
UPDATE category_analysis_runs
SET
top_asins_checked = ?,
available_asins = ?,
fba_count = ?,
fbm_count = ?,
skip_count = ?,
status = ?,
error_message = ?
WHERE id = ?
`,
[
summary.topAsinsChecked,
summary.availableAsins,
summary.fba,
summary.fbm,
summary.skip,
summary.status,
summary.error,
runId,
],
);
await db
.update(runs)
.set({
topAsinsChecked: summary.topAsinsChecked,
availableAsins: summary.availableAsins,
totalProducts: summary.topAsinsChecked,
fbaCount: summary.fba,
fbmCount: summary.fbm,
skipCount: summary.skip,
status: summary.status as typeof runs.$inferInsert.status,
errorMessage: summary.error || null,
...(summary.status !== "running" ? { completedAt: new Date() } : {}),
})
.where(eq(runs.id, runId));
}
export async function insertProductAnalysisResults(
db: Database,
runId: number,
results: AnalysisResult[],
): Promise<void> {
if (results.length === 0) {
return;
}
if (results.length === 0) return;
const insertStmt = db.prepare(`
INSERT INTO product_analysis_results (
asin, run_id, name, brand, category, unit_cost,
current_price, avg_price_90d, avg_price_90d_sheet,
selling_price_sheet, sales_rank, sales_rank_avg_90d,
seller_count, amazon_is_seller, amazon_buybox_share_pct_90d,
monthly_sold, rank_drops_30d, rank_drops_90d,
fba_fee, fbm_fee, referral_percent, can_sell,
sellability_status, sellability_reason,
verdict, confidence, reasoning, fetched_at
) VALUES (
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?
)
ON CONFLICT(asin) DO UPDATE SET
run_id = excluded.run_id,
name = excluded.name,
brand = excluded.brand,
category = excluded.category,
unit_cost = excluded.unit_cost,
current_price = excluded.current_price,
avg_price_90d = excluded.avg_price_90d,
avg_price_90d_sheet = excluded.avg_price_90d_sheet,
selling_price_sheet = excluded.selling_price_sheet,
sales_rank = excluded.sales_rank,
sales_rank_avg_90d = excluded.sales_rank_avg_90d,
seller_count = excluded.seller_count,
amazon_is_seller = excluded.amazon_is_seller,
amazon_buybox_share_pct_90d = excluded.amazon_buybox_share_pct_90d,
monthly_sold = excluded.monthly_sold,
rank_drops_30d = excluded.rank_drops_30d,
rank_drops_90d = excluded.rank_drops_90d,
fba_fee = excluded.fba_fee,
fbm_fee = excluded.fbm_fee,
referral_percent = excluded.referral_percent,
can_sell = excluded.can_sell,
sellability_status = excluded.sellability_status,
sellability_reason = excluded.sellability_reason,
verdict = excluded.verdict,
confidence = excluded.confidence,
reasoning = excluded.reasoning,
fetched_at = excluded.fetched_at;
`);
const rows = results.map((r) => {
const price =
r.product.keepa?.currentPrice ??
r.product.record.sellingPriceFromSheet ??
r.product.spApi.estimatedSalePrice;
const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank;
db.transaction((resultsBatch: AnalysisResult[]) => {
for (const r of resultsBatch) {
const price =
r.product.keepa?.currentPrice ??
r.product.record.sellingPriceFromSheet ??
r.product.spApi.estimatedSalePrice;
const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank;
insertStmt.run(
r.product.record.asin,
runId,
r.product.record.name,
r.product.record.brand ?? null,
return {
asin: r.product.record.asin,
runId,
name: r.product.record.name,
brand: r.product.record.brand ?? null,
category:
r.product.record.category ??
r.product.keepa?.categoryTree?.join(" > ") ??
null,
r.product.record.unitCost ?? null,
price ?? null,
r.product.keepa?.avgPrice90 ?? null,
r.product.record.avgPrice90FromSheet ?? null,
r.product.record.sellingPriceFromSheet ?? null,
rank ?? null,
r.product.keepa?.salesRankAvg90 ?? null,
r.product.keepa?.sellerCount ?? null,
r.product.keepa?.amazonIsSeller == null
? null
: r.product.keepa.amazonIsSeller
? 1
: 0,
r.product.keepa?.amazonBuyboxSharePct90d ?? null,
r.product.keepa?.monthlySold ?? null,
r.product.keepa?.salesRankDrops30 ?? null,
r.product.keepa?.salesRankDrops90 ?? null,
r.product.spApi.fbaFee ?? null,
r.product.spApi.fbmFee ?? null,
r.product.spApi.referralFeePercent ?? null,
r.product.keepa?.categoryTree?.join(" > ") ??
null,
unitCost: r.product.record.unitCost ?? null,
currentPrice: price ?? null,
avgPrice90d: r.product.keepa?.avgPrice90 ?? null,
avgPrice90dSheet: r.product.record.avgPrice90FromSheet ?? null,
sellingPriceSheet: r.product.record.sellingPriceFromSheet ?? null,
salesRank: rank ?? null,
salesRankAvg90d: r.product.keepa?.salesRankAvg90 ?? null,
sellerCount: r.product.keepa?.sellerCount ?? null,
amazonIsSeller: r.product.keepa?.amazonIsSeller ?? null,
amazonBuyboxSharePct90d: r.product.keepa?.amazonBuyboxSharePct90d ?? null,
monthlySold: r.product.keepa?.monthlySold ?? null,
rankDrops30d: r.product.keepa?.salesRankDrops30 ?? null,
rankDrops90d: r.product.keepa?.salesRankDrops90 ?? null,
fbaFee: r.product.spApi.fbaFee ?? null,
fbmFee: r.product.spApi.fbmFee ?? null,
referralPercent: r.product.spApi.referralFeePercent ?? null,
canSell:
r.product.spApi.canSell == null
? "unknown"
: r.product.spApi.canSell
? "yes"
: "no",
r.product.spApi.sellabilityStatus ?? null,
r.product.spApi.sellabilityReason ?? null,
r.verdict.verdict,
r.verdict.confidence,
r.verdict.reasoning ?? null,
r.product.fetchedAt,
);
}
})(results); // Execute the transaction with the results batch
sellabilityStatus: r.product.spApi.sellabilityStatus ?? null,
sellabilityReason: r.product.spApi.sellabilityReason ?? null,
verdict: r.verdict.verdict,
confidence: r.verdict.confidence,
reasoning: r.verdict.reasoning ?? null,
fetchedAt: new Date(r.product.fetchedAt),
};
});
await db
.insert(categoryProductResults)
.values(rows)
.onConflictDoUpdate({
target: categoryProductResults.asin,
set: {
runId: sql`EXCLUDED.run_id`,
name: sql`EXCLUDED.name`,
brand: sql`EXCLUDED.brand`,
category: sql`EXCLUDED.category`,
unitCost: sql`EXCLUDED.unit_cost`,
currentPrice: sql`EXCLUDED.current_price`,
avgPrice90d: sql`EXCLUDED.avg_price_90d`,
avgPrice90dSheet: sql`EXCLUDED.avg_price_90d_sheet`,
sellingPriceSheet: sql`EXCLUDED.selling_price_sheet`,
salesRank: sql`EXCLUDED.sales_rank`,
salesRankAvg90d: sql`EXCLUDED.sales_rank_avg_90d`,
sellerCount: sql`EXCLUDED.seller_count`,
amazonIsSeller: sql`EXCLUDED.amazon_is_seller`,
amazonBuyboxSharePct90d: sql`EXCLUDED.amazon_buybox_share_pct_90d`,
monthlySold: sql`EXCLUDED.monthly_sold`,
rankDrops30d: sql`EXCLUDED.rank_drops_30d`,
rankDrops90d: sql`EXCLUDED.rank_drops_90d`,
fbaFee: sql`EXCLUDED.fba_fee`,
fbmFee: sql`EXCLUDED.fbm_fee`,
referralPercent: sql`EXCLUDED.referral_percent`,
canSell: sql`EXCLUDED.can_sell`,
sellabilityStatus: sql`EXCLUDED.sellability_status`,
sellabilityReason: sql`EXCLUDED.sellability_reason`,
verdict: sql`EXCLUDED.verdict`,
confidence: sql`EXCLUDED.confidence`,
reasoning: sql`EXCLUDED.reasoning`,
fetchedAt: sql`EXCLUDED.fetched_at`,
},
});
}
function loadCategoryBlacklist(filePath: string): Set<number> {
@@ -1471,7 +1443,6 @@ function shouldKeepCandidateBySellability(
}
export async function processCategory(
db: Database,
runId: number,
category: CategoryInfo,
perCategoryTop: number,
@@ -1505,7 +1476,7 @@ export async function processCategory(
);
if (topAsins.length === 0) {
log("info", " Keepa returned no ASINs for this category.");
await updateCategoryRunSummary(db, runId, {
await updateCategoryRunSummary(runId, {
topAsinsChecked: 0,
availableAsins: 0,
fba: 0,
@@ -1766,7 +1737,7 @@ export async function processCategory(
},
}));
await insertProductAnalysisResults(db, runId, batchResults);
await insertProductAnalysisResults(runId, batchResults);
for (const result of batchResults) {
if (result.verdict.verdict === "FBA") {
@@ -1781,7 +1752,7 @@ export async function processCategory(
budget.analyzedAsins += batchResults.length;
results.push(...batchResults);
await updateCategoryRunSummary(db, runId, {
await updateCategoryRunSummary(runId, {
topAsinsChecked: checkedAsins,
availableAsins: results.length,
fba,
@@ -1802,7 +1773,7 @@ export async function processCategory(
const emptyReason =
budget.stopReason ||
"No sellable ASINs matched the configured mid-range criteria";
await updateCategoryRunSummary(db, runId, {
await updateCategoryRunSummary(runId, {
topAsinsChecked: checkedAsins,
availableAsins: 0,
fba,
@@ -1830,7 +1801,7 @@ export async function processCategory(
` Category stream totals: checked=${checkedAsins}, sellable=${sellableAsins}, keepaEnriched=${keepaEnrichedAsins}, analyzed=${results.length}`,
);
await updateCategoryRunSummary(db, runId, {
await updateCategoryRunSummary(runId, {
topAsinsChecked: checkedAsins,
availableAsins: results.length,
fba,
@@ -1923,11 +1894,6 @@ export async function main(): Promise<void> {
await connectCache();
try {
mkdirSync(args.outputDir, { recursive: true });
const DB_PATH =
process.env.RESULTS_DB_PATH ||
path.join(process.cwd(), "db", "results.db");
initDb(DB_PATH);
const db = getDb(DB_PATH);
log("info", "Starting per-category mid-range pipeline");
log("info", `Marketplace: ${config.spApiMarketplaceId}`);
@@ -1987,7 +1953,6 @@ export async function main(): Promise<void> {
let runId: number | undefined;
try {
runId = await insertCategoryRunSummary(
db,
{
categoryId: category.id,
categoryLabel: category.label,
@@ -2004,7 +1969,6 @@ export async function main(): Promise<void> {
);
categorySummary = await processCategory(
db,
runId,
category,
args.perCategoryTop,
@@ -2046,7 +2010,7 @@ export async function main(): Promise<void> {
results: [],
};
if (runId) {
await updateCategoryRunSummary(db, runId, {
await updateCategoryRunSummary(runId, {
topAsinsChecked: 0,
availableAsins: 0,
fba: 0,

File diff suppressed because it is too large Load Diff

View File

@@ -1,4 +1,6 @@
import { type Database, closeDb, getDb, initDb } from "./database.ts";
import { db } from "./db/index.ts";
import { categoryProductResults, runs } from "./db/schema.ts";
import { eq, sql } from "drizzle-orm";
import { analyzeProducts } from "./llm.ts";
import { fetchSpApiPricingAndFees } from "./sp-api.ts";
import type {
@@ -13,7 +15,6 @@ const LLM_BATCH_SIZE = 5;
const LLM_BATCH_DELAY_MS = 5_000;
type Args = {
dbPath: string;
stalkerRunId: number;
analysisRunId: number;
asins: string[];
@@ -22,18 +23,18 @@ type Args = {
type InventoryRow = {
asin: string;
product_title: string | null;
productTitle: string | null;
brand: string | null;
category_tree: string | null;
current_price: number | null;
avg_price_90d: number | null;
sales_rank: number | null;
monthly_sold: number | null;
seller_count: number | null;
amazon_is_seller: number | null;
can_sell: number | null;
sellability_status: SellabilityInfo["sellabilityStatus"] | null;
sellability_reason: string | null;
categoryTree: string | null;
currentPrice: number | null;
avgPrice90d: number | null;
salesRank: number | null;
monthlySold: number | null;
sellerCount: number | null;
amazonIsSeller: boolean | null;
canSell: boolean | null;
sellabilityStatus: SellabilityInfo["sellabilityStatus"] | null;
sellabilityReason: string | null;
};
function readFlagValue(args: string[], flag: string): string | undefined {
@@ -43,7 +44,6 @@ function readFlagValue(args: string[], flag: string): string | undefined {
}
function parseArgs(argv = process.argv.slice(2)): Args {
const dbPath = readFlagValue(argv, "--db");
const stalkerRunId = Number(readFlagValue(argv, "--stalker-run-id"));
const analysisRunId = Number(readFlagValue(argv, "--analysis-run-id"));
const useClaude = argv.includes("--claude");
@@ -52,7 +52,6 @@ function parseArgs(argv = process.argv.slice(2)): Args {
.map((asin) => asin.trim().toUpperCase())
.filter(Boolean);
if (!dbPath) throw new Error("Missing --db");
if (!Number.isInteger(stalkerRunId) || stalkerRunId <= 0) {
throw new Error("--stalker-run-id must be a positive integer");
}
@@ -61,7 +60,7 @@ function parseArgs(argv = process.argv.slice(2)): Args {
}
if (asins.length === 0) throw new Error("Missing --asins");
return { dbPath, stalkerRunId, analysisRunId, asins, useClaude };
return { stalkerRunId, analysisRunId, asins, useClaude };
}
function wait(ms: number): Promise<void> {
@@ -81,69 +80,74 @@ function parseCategoryTree(value: string | null): string[] {
}
function toProductRecord(row: InventoryRow): ProductRecord {
const categoryTree = parseCategoryTree(row.category_tree);
const categoryTree = parseCategoryTree(row.categoryTree);
return {
asin: row.asin,
name: row.product_title ?? row.asin,
name: row.productTitle ?? row.asin,
brand: row.brand ?? undefined,
category: categoryTree.join(" > ") || undefined,
unitCost: 0,
amazonRank: row.sales_rank ?? undefined,
sellingPriceFromSheet: row.current_price ?? undefined,
avgPrice90FromSheet: row.avg_price_90d ?? undefined,
amazonRank: row.salesRank ?? undefined,
sellingPriceFromSheet: row.currentPrice ?? undefined,
avgPrice90FromSheet: row.avgPrice90d ?? undefined,
};
}
function toKeepaData(row: InventoryRow): KeepaData {
return {
currentPrice: row.current_price,
avgPrice90: row.avg_price_90d,
currentPrice: row.currentPrice,
avgPrice90: row.avgPrice90d,
minPrice90: null,
maxPrice90: null,
salesRank: row.sales_rank,
salesRank: row.salesRank,
salesRankAvg90: null,
salesRankDrops30: null,
salesRankDrops90: null,
sellerCount: row.seller_count,
amazonIsSeller:
row.amazon_is_seller == null ? null : row.amazon_is_seller === 1,
sellerCount: row.sellerCount,
amazonIsSeller: row.amazonIsSeller,
amazonBuyboxSharePct90d: null,
buyBoxSeller: null,
buyBoxPrice: null,
buyBoxAvg90: null,
monthlySold: row.monthly_sold,
categoryTree: parseCategoryTree(row.category_tree),
monthlySold: row.monthlySold,
categoryTree: parseCategoryTree(row.categoryTree),
};
}
function toSellability(row: InventoryRow): SellabilityInfo {
return {
canSell: row.can_sell == null ? null : row.can_sell === 1,
sellabilityStatus: row.sellability_status ?? "unknown",
sellabilityReason: row.sellability_reason ?? undefined,
canSell: row.canSell,
sellabilityStatus: row.sellabilityStatus ?? "unknown",
sellabilityReason: row.sellabilityReason ?? undefined,
};
}
function loadInventoryRows(
database: Database,
async function loadInventoryRows(
stalkerRunId: number,
asins: string[],
): InventoryRow[] {
const placeholders = asins.map(() => "?").join(",");
return database
.query(
`SELECT
asin, product_title, brand, category_tree, current_price, avg_price_90d,
sales_rank, monthly_sold, seller_count, amazon_is_seller, can_sell,
sellability_status, sellability_reason
FROM stalker_seller_inventory
WHERE run_id = ?
AND can_sell = 1
AND sellability_status = 'available'
AND asin IN (${placeholders})
GROUP BY asin`,
)
.all(stalkerRunId, ...asins) as InventoryRow[];
): Promise<InventoryRow[]> {
if (asins.length === 0) return [];
return db.execute(
sql<InventoryRow>`SELECT DISTINCT ON (asin)
asin,
product_title AS "productTitle",
brand,
category_tree AS "categoryTree",
current_price AS "currentPrice",
avg_price_90d AS "avgPrice90d",
sales_rank AS "salesRank",
monthly_sold AS "monthlySold",
seller_count AS "sellerCount",
amazon_is_seller AS "amazonIsSeller",
can_sell AS "canSell",
sellability_status AS "sellabilityStatus",
sellability_reason AS "sellabilityReason"
FROM stalker_seller_inventory
WHERE run_id = ${stalkerRunId}
AND can_sell = true
AND sellability_status = 'available'
AND asin = ANY(${asins})`,
);
}
async function buildEnrichedProducts(
@@ -156,7 +160,7 @@ async function buildEnrichedProducts(
const spApi = await fetchSpApiPricingAndFees(
row.asin,
sellability,
row.current_price,
row.currentPrice,
);
enriched.push({
@@ -170,133 +174,114 @@ async function buildEnrichedProducts(
return enriched;
}
function insertProductAnalysisResults(
database: Database,
async function insertProductAnalysisResults(
runId: number,
results: AnalysisResult[],
): void {
): Promise<void> {
if (results.length === 0) return;
const insert = database.prepare(`
INSERT INTO product_analysis_results (
asin, run_id, name, brand, category, unit_cost,
current_price, avg_price_90d, avg_price_90d_sheet,
selling_price_sheet, sales_rank, sales_rank_avg_90d,
seller_count, amazon_is_seller, amazon_buybox_share_pct_90d,
monthly_sold, rank_drops_30d, rank_drops_90d,
fba_fee, fbm_fee, referral_percent, can_sell,
sellability_status, sellability_reason,
verdict, confidence, reasoning, fetched_at
) VALUES (
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?
)
ON CONFLICT(asin) DO UPDATE SET
run_id = excluded.run_id,
name = excluded.name,
brand = excluded.brand,
category = excluded.category,
unit_cost = excluded.unit_cost,
current_price = excluded.current_price,
avg_price_90d = excluded.avg_price_90d,
avg_price_90d_sheet = excluded.avg_price_90d_sheet,
selling_price_sheet = excluded.selling_price_sheet,
sales_rank = excluded.sales_rank,
sales_rank_avg_90d = excluded.sales_rank_avg_90d,
seller_count = excluded.seller_count,
amazon_is_seller = excluded.amazon_is_seller,
amazon_buybox_share_pct_90d = excluded.amazon_buybox_share_pct_90d,
monthly_sold = excluded.monthly_sold,
rank_drops_30d = excluded.rank_drops_30d,
rank_drops_90d = excluded.rank_drops_90d,
fba_fee = excluded.fba_fee,
fbm_fee = excluded.fbm_fee,
referral_percent = excluded.referral_percent,
can_sell = excluded.can_sell,
sellability_status = excluded.sellability_status,
sellability_reason = excluded.sellability_reason,
verdict = excluded.verdict,
confidence = excluded.confidence,
reasoning = excluded.reasoning,
fetched_at = excluded.fetched_at
`);
const rows = results.map((result) => {
const keepa = result.product.keepa;
const record = result.product.record;
const spApi = result.product.spApi;
const canSell =
spApi.canSell == null ? "unknown" : spApi.canSell ? "yes" : "no";
database.transaction((batch: AnalysisResult[]) => {
for (const result of batch) {
const keepa = result.product.keepa;
const record = result.product.record;
const spApi = result.product.spApi;
insert.run(
record.asin,
runId,
record.name,
record.brand ?? null,
record.category ?? keepa?.categoryTree.join(" > ") ?? null,
record.unitCost ?? null,
keepa?.currentPrice ?? spApi.estimatedSalePrice ?? null,
keepa?.avgPrice90 ?? null,
record.avgPrice90FromSheet ?? null,
record.sellingPriceFromSheet ?? null,
keepa?.salesRank ?? record.amazonRank ?? null,
keepa?.salesRankAvg90 ?? null,
keepa?.sellerCount ?? null,
keepa?.amazonIsSeller == null ? null : keepa.amazonIsSeller ? 1 : 0,
keepa?.amazonBuyboxSharePct90d ?? null,
keepa?.monthlySold ?? null,
keepa?.salesRankDrops30 ?? null,
keepa?.salesRankDrops90 ?? null,
spApi.fbaFee ?? null,
spApi.fbmFee ?? null,
spApi.referralFeePercent ?? null,
spApi.canSell == null ? "unknown" : spApi.canSell ? "yes" : "no",
spApi.sellabilityStatus ?? null,
spApi.sellabilityReason ?? null,
result.verdict.verdict,
result.verdict.confidence,
result.verdict.reasoning ?? null,
result.product.fetchedAt,
);
}
})(results);
return {
asin: record.asin,
runId,
name: record.name,
brand: record.brand ?? null,
category: record.category ?? keepa?.categoryTree.join(" > ") ?? null,
unitCost: record.unitCost ?? null,
currentPrice: keepa?.currentPrice ?? spApi.estimatedSalePrice ?? null,
avgPrice90d: keepa?.avgPrice90 ?? null,
avgPrice90dSheet: record.avgPrice90FromSheet ?? null,
sellingPriceSheet: record.sellingPriceFromSheet ?? null,
salesRank: keepa?.salesRank ?? record.amazonRank ?? null,
salesRankAvg90d: keepa?.salesRankAvg90 ?? null,
sellerCount: keepa?.sellerCount ?? null,
amazonIsSeller: keepa?.amazonIsSeller ?? null,
amazonBuyboxSharePct90d: keepa?.amazonBuyboxSharePct90d ?? null,
monthlySold: keepa?.monthlySold ?? null,
rankDrops30d: keepa?.salesRankDrops30 ?? null,
rankDrops90d: keepa?.salesRankDrops90 ?? null,
fbaFee: spApi.fbaFee ?? null,
fbmFee: spApi.fbmFee ?? null,
referralPercent: spApi.referralFeePercent ?? null,
canSell,
sellabilityStatus: spApi.sellabilityStatus ?? null,
sellabilityReason: spApi.sellabilityReason ?? null,
verdict: result.verdict.verdict,
confidence: result.verdict.confidence ?? 0,
reasoning: result.verdict.reasoning ?? null,
fetchedAt: new Date(result.product.fetchedAt),
};
});
await db
.insert(categoryProductResults)
.values(rows)
.onConflictDoUpdate({
target: categoryProductResults.asin,
set: {
runId: sql`EXCLUDED.run_id`,
name: sql`EXCLUDED.name`,
brand: sql`EXCLUDED.brand`,
category: sql`EXCLUDED.category`,
unitCost: sql`EXCLUDED.unit_cost`,
currentPrice: sql`EXCLUDED.current_price`,
avgPrice90d: sql`EXCLUDED.avg_price_90d`,
avgPrice90dSheet: sql`EXCLUDED.avg_price_90d_sheet`,
sellingPriceSheet: sql`EXCLUDED.selling_price_sheet`,
salesRank: sql`EXCLUDED.sales_rank`,
salesRankAvg90d: sql`EXCLUDED.sales_rank_avg_90d`,
sellerCount: sql`EXCLUDED.seller_count`,
amazonIsSeller: sql`EXCLUDED.amazon_is_seller`,
amazonBuyboxSharePct90d: sql`EXCLUDED.amazon_buybox_share_pct_90d`,
monthlySold: sql`EXCLUDED.monthly_sold`,
rankDrops30d: sql`EXCLUDED.rank_drops_30d`,
rankDrops90d: sql`EXCLUDED.rank_drops_90d`,
fbaFee: sql`EXCLUDED.fba_fee`,
fbmFee: sql`EXCLUDED.fbm_fee`,
referralPercent: sql`EXCLUDED.referral_percent`,
canSell: sql`EXCLUDED.can_sell`,
sellabilityStatus: sql`EXCLUDED.sellability_status`,
sellabilityReason: sql`EXCLUDED.sellability_reason`,
verdict: sql`EXCLUDED.verdict`,
confidence: sql`EXCLUDED.confidence`,
reasoning: sql`EXCLUDED.reasoning`,
fetchedAt: sql`EXCLUDED.fetched_at`,
},
});
}
function refreshAnalysisRun(database: Database, runId: number): void {
const stats = database
.query(
`SELECT
async function refreshAnalysisRun(runId: number): Promise<void> {
const [stats] = await db.execute(
sql<{
total: string;
fba: string | null;
fbm: string | null;
skip: string | null;
}>`SELECT
COUNT(*) AS total,
SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba,
SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm,
SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip
FROM product_analysis_results
WHERE run_id = ?`,
)
.get(runId) as {
total: number;
fba: number | null;
fbm: number | null;
skip: number | null;
};
FROM category_product_results
WHERE run_id = ${runId}`,
);
database
.prepare(
`UPDATE category_analysis_runs
SET top_asins_checked = ?,
available_asins = ?,
fba_count = ?,
fbm_count = ?,
skip_count = ?
WHERE id = ?`,
)
.run(
stats.total ?? 0,
stats.total ?? 0,
stats.fba ?? 0,
stats.fbm ?? 0,
stats.skip ?? 0,
runId,
);
await db
.update(runs)
.set({
topAsinsChecked: Number(stats?.total ?? 0),
availableAsins: Number(stats?.total ?? 0),
fbaCount: Number(stats?.fba ?? 0),
fbmCount: Number(stats?.fbm ?? 0),
skipCount: Number(stats?.skip ?? 0),
})
.where(eq(runs.id, runId));
}
async function analyzeInBatches(
@@ -349,24 +334,18 @@ async function analyzeInBatches(
async function main(): Promise<void> {
const args = parseArgs();
initDb(args.dbPath);
const database = getDb(args.dbPath);
try {
const rows = loadInventoryRows(database, args.stalkerRunId, args.asins);
if (rows.length === 0) {
console.log("Stalker analysis: no sellable inventory rows to analyze.");
return;
}
console.log(`Stalker analysis: analyzing ${rows.length} sellable ASIN(s).`);
const enriched = await buildEnrichedProducts(rows);
const results = await analyzeInBatches(enriched, args.useClaude);
insertProductAnalysisResults(database, args.analysisRunId, results);
refreshAnalysisRun(database, args.analysisRunId);
} finally {
closeDb();
const rows = await loadInventoryRows(args.stalkerRunId, args.asins);
if (rows.length === 0) {
console.log("Stalker analysis: no sellable inventory rows to analyze.");
return;
}
console.log(`Stalker analysis: analyzing ${rows.length} sellable ASIN(s).`);
const enriched = await buildEnrichedProducts(rows);
const results = await analyzeInBatches(enriched, args.useClaude);
await insertProductAnalysisResults(args.analysisRunId, results);
await refreshAnalysisRun(args.analysisRunId);
}
if (import.meta.main) {

View File

@@ -2,7 +2,67 @@ import { afterAll, beforeEach, expect, mock, test } from "bun:test";
import { mkdirSync, rmSync } from "node:fs";
import path from "node:path";
import * as XLSX from "xlsx";
import { closeDb, getDb } from "./database.ts";
let nextId = 0;
function chainable(resolveWith: any[] = []): any {
const p: any = Promise.resolve(resolveWith);
p.limit = (_n: any) => chainable(resolveWith);
p.where = (_cond: any) => chainable(resolveWith);
p.from = (_table: any) => chainable(resolveWith);
return p;
}
const makeMockTx = (): any => ({
insert: (_table: any) => ({
values: (_vals: any) => ({
returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]),
onConflictDoUpdate: (_conf: any) => Promise.resolve([]),
}),
}),
update: (_table: any) => ({
set: (_vals: any) => ({
where: (_cond: any) => Promise.resolve([]),
}),
}),
select: (_sel?: any) => ({
from: (_table: any) => ({
where: (_cond: any) => chainable([{ id: ++nextId }]),
limit: (_n: any) => chainable([{ id: nextId }]),
}),
}),
selectDistinct: (_sel: any) => ({
from: (_table: any) => chainable([]),
}),
execute: (_query: any) => Promise.resolve([]),
});
const makeMockDb = (): any => ({
insert: (_table: any) => ({
values: (_vals: any) => ({
returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]),
onConflictDoUpdate: (_conf: any) => Promise.resolve([]),
}),
}),
update: (_table: any) => ({
set: (_vals: any) => ({
where: (_cond: any) => Promise.resolve([]),
}),
}),
select: (_sel?: any) => ({
from: (_table: any) => ({
where: (_cond: any) => chainable(),
limit: (_n: any) => chainable(),
}),
}),
selectDistinct: (_sel: any) => ({
from: (_table: any) => chainable(),
}),
execute: (_query: any) => Promise.resolve([]),
transaction: async (fn: (tx: any) => Promise<any>) => fn(makeMockTx()),
});
mock.module("./db/index.ts", () => ({ db: makeMockDb(), client: {} }));
const TEST_DIR = path.join(process.cwd(), "test_output", "stalker-sellability");
const originalFetch = globalThis.fetch;
@@ -34,7 +94,7 @@ mock.module("./sp-api.ts", () => ({
const modulePromise = import("./stalker.ts");
beforeEach(() => {
closeDb();
nextId = 0;
rmSync(TEST_DIR, { recursive: true, force: true });
mkdirSync(TEST_DIR, { recursive: true });
globalThis.fetch = originalFetch;
@@ -49,14 +109,12 @@ afterAll(() => {
} else {
Bun.env.KEEPA_API_KEY = originalKeepaKey;
}
closeDb();
rmSync(TEST_DIR, { recursive: true, force: true });
});
test("sellability checks matched seller inventory, not the source ASIN", async () => {
const { runStalker } = await modulePromise;
const inputPath = path.join(TEST_DIR, "input.xlsx");
const dbPath = path.join(TEST_DIR, "stalker.sqlite");
const workbook = XLSX.utils.book_new();
XLSX.utils.book_append_sheet(
workbook,
@@ -138,7 +196,6 @@ test("sellability checks matched seller inventory, not the source ASIN", async (
const stats = await runStalker({
input: inputPath,
dbPath,
maxAsins: null,
storefrontUpdateHours: 168,
offerLimit: 20,
@@ -151,6 +208,7 @@ test("sellability checks matched seller inventory, not the source ASIN", async (
maxSellerRequests: null,
sellability: true,
analyzeSellable: false,
useClaude: false,
});
expect(fetchSellabilityBatchMock.mock.calls.length).toBe(1);
@@ -162,46 +220,4 @@ test("sellability checks matched seller inventory, not the source ASIN", async (
expect(stats.inventorySellabilityAvailableAsins).toBe(1);
expect(stats.inventorySellabilityExcludedAsins).toBe(1);
expect(stats.persistedInventoryAsins).toBe(1);
const db = getDb(dbPath);
const scan = db.query("SELECT source_asin FROM stalker_asin_scans").get() as {
source_asin: string;
};
expect(scan.source_asin).toBe("B000000001");
const inventory = db
.query(
`SELECT asin, can_sell, sellability_status, product_title, brand,
category_tree, current_price, avg_price_90d, sales_rank, monthly_sold,
seller_count
FROM stalker_seller_inventory ORDER BY asin`,
)
.all() as Array<{
asin: string;
can_sell: number | null;
sellability_status: string | null;
product_title: string | null;
brand: string | null;
category_tree: string | null;
current_price: number | null;
avg_price_90d: number | null;
sales_rank: number | null;
monthly_sold: number | null;
seller_count: number | null;
}>;
expect(inventory).toEqual([
{
asin: "B111111111",
can_sell: 1,
sellability_status: "available",
product_title: "Sellable Storefront Product",
brand: "Good Brand",
category_tree: JSON.stringify(["Kitchen", "Storage"]),
current_price: 19.99,
avg_price_90d: 25,
sales_rank: 12345,
monthly_sold: 42,
seller_count: 7,
},
]);
});

View File

@@ -2,7 +2,6 @@ import { afterAll, beforeEach, expect, mock, test } from "bun:test";
import { mkdirSync, rmSync } from "node:fs";
import path from "node:path";
import * as XLSX from "xlsx";
import { closeDb, getDb, initDb } from "./database.ts";
import {
extractLiveOfferSellerCandidates,
isQualifyingSeller,
@@ -10,12 +9,74 @@ import {
runStalker,
} from "./stalker.ts";
let nextId = 0;
function chainable(resolveWith: any[] = []): any {
const p: any = Promise.resolve(resolveWith);
p.limit = (_n: any) => chainable(resolveWith);
p.where = (_cond: any) => chainable(resolveWith);
p.from = (_table: any) => chainable(resolveWith);
return p;
}
// Transaction mock returns rows for selects (needed for upsert-then-select patterns).
const makeMockTx = (): any => ({
insert: (_table: any) => ({
values: (_vals: any) => ({
returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]),
onConflictDoUpdate: (_conf: any) => Promise.resolve([]),
}),
}),
update: (_table: any) => ({
set: (_vals: any) => ({
where: (_cond: any) => Promise.resolve([]),
}),
}),
select: (_sel?: any) => ({
from: (_table: any) => ({
where: (_cond: any) => chainable([{ id: ++nextId }]),
limit: (_n: any) => chainable([{ id: nextId }]),
}),
}),
selectDistinct: (_sel: any) => ({
from: (_table: any) => chainable([]),
}),
execute: (_query: any) => Promise.resolve([]),
});
const makeMockDb = (): any => ({
insert: (_table: any) => ({
values: (_vals: any) => ({
returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]),
onConflictDoUpdate: (_conf: any) => Promise.resolve([]),
}),
}),
update: (_table: any) => ({
set: (_vals: any) => ({
where: (_cond: any) => Promise.resolve([]),
}),
}),
select: (_sel?: any) => ({
from: (_table: any) => ({
where: (_cond: any) => chainable(),
limit: (_n: any) => chainable(),
}),
}),
selectDistinct: (_sel: any) => ({
from: (_table: any) => chainable(),
}),
execute: (_query: any) => Promise.resolve([]),
transaction: async (fn: (tx: any) => Promise<any>) => fn(makeMockTx()),
});
mock.module("./db/index.ts", () => ({ db: makeMockDb(), client: {} }));
const TEST_DIR = path.join(process.cwd(), "test_output", "stalker");
const originalFetch = globalThis.fetch;
const originalKeepaKey = Bun.env.KEEPA_API_KEY;
beforeEach(() => {
closeDb();
nextId = 0;
rmSync(TEST_DIR, { recursive: true, force: true });
mkdirSync(TEST_DIR, { recursive: true });
globalThis.fetch = originalFetch;
@@ -29,7 +90,6 @@ afterAll(() => {
} else {
Bun.env.KEEPA_API_KEY = originalKeepaKey;
}
closeDb();
rmSync(TEST_DIR, { recursive: true, force: true });
});
@@ -77,35 +137,8 @@ test("extractLiveOfferSellerCandidates ignores Amazon, missing seller IDs, and d
expect(offers[0]?.stock).toBe(4);
});
test("initDb creates stalker tables and indexes", () => {
const dbPath = path.join(TEST_DIR, "schema.sqlite");
initDb(dbPath);
const db = getDb(dbPath);
const tables = db
.query(
`SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'stalker_%' ORDER BY name`,
)
.all() as Array<{ name: string }>;
expect(tables.map((row) => row.name)).toEqual([
"stalker_asin_scans",
"stalker_asin_sellers",
"stalker_runs",
"stalker_seller_inventory",
"stalker_sellers",
]);
const indexes = db
.query(
`SELECT name FROM sqlite_master WHERE type = 'index' AND name LIKE 'idx_stalker_%' ORDER BY name`,
)
.all() as Array<{ name: string }>;
expect(indexes.length).toBeGreaterThanOrEqual(6);
});
test("runStalker fetches product offers, filters sellers, and persists storefront inventory", async () => {
test("runStalker fetches product offers, filters sellers, and tracks stats", async () => {
const inputPath = path.join(TEST_DIR, "input.xlsx");
const dbPath = path.join(TEST_DIR, "stalker.sqlite");
const workbook = XLSX.utils.book_new();
XLSX.utils.book_append_sheet(
workbook,
@@ -205,7 +238,6 @@ test("runStalker fetches product offers, filters sellers, and persists storefron
const stats = await runStalker({
input: inputPath,
dbPath,
maxAsins: null,
storefrontUpdateHours: 168,
offerLimit: 20,
@@ -218,6 +250,7 @@ test("runStalker fetches product offers, filters sellers, and persists storefron
maxSellerRequests: null,
sellability: false,
analyzeSellable: false,
useClaude: false,
});
expect(stats.scannedAsins).toBe(1);
@@ -229,6 +262,7 @@ test("runStalker fetches product offers, filters sellers, and persists storefron
expect(stats.qualifyingSellers).toBe(1);
expect(stats.sellerMetadataRequests).toBe(1);
expect(stats.sellerStorefrontRequests).toBe(1);
const sellerCalls = fetchMock.mock.calls.filter((call) => {
const rawUrl =
typeof call[0] === "string"
@@ -239,45 +273,4 @@ test("runStalker fetches product offers, filters sellers, and persists storefron
return new URL(rawUrl).pathname === "/seller";
});
expect(sellerCalls.length).toBe(2);
const db = getDb(dbPath);
const run = db.query("SELECT * FROM stalker_runs").get() as any;
expect(run.status).toBe("completed");
expect(run.requested_asins).toBe(1);
expect(run.scanned_asins).toBe(1);
expect(run.source_asins_with_matches).toBe(1);
expect(run.candidate_sellers).toBe(2);
expect(run.qualifying_sellers).toBe(1);
expect(run.matched_sellers).toBe(1);
expect(run.seller_metadata_requests).toBe(1);
expect(run.seller_storefront_requests).toBe(1);
expect(run.inventory_sellability_checked_asins).toBe(0);
expect(run.inventory_sellability_available_asins).toBe(0);
expect(run.inventory_sellability_excluded_asins).toBe(0);
expect(run.persisted_inventory_asins).toBe(0);
const scan = db.query("SELECT * FROM stalker_asin_scans").get() as any;
expect(scan.source_asin).toBe("B000000001");
expect(scan.title).toBe("Tracked Product");
expect(scan.offer_count).toBe(2);
expect(scan.candidate_seller_count).toBe(2);
expect(scan.matched_seller_count).toBe(1);
const sellers = db.query("SELECT * FROM stalker_sellers").all() as any[];
expect(sellers.length).toBe(1);
expect(sellers[0].seller_id).toBe("AQUALIFIED");
expect(sellers[0].rating_count).toBe(12);
expect(sellers[0].storefront_asin_total).toBe(2);
expect(sellers[0].persisted_inventory_sample_count).toBe(0);
const asinSellers = db.query("SELECT * FROM stalker_asin_sellers").all() as any[];
expect(asinSellers.length).toBe(1);
expect(asinSellers[0].offer_price).toBe(19.99);
expect(asinSellers[0].is_fba).toBe(1);
expect(asinSellers[0].stock).toBe(3);
const inventory = db
.query("SELECT asin FROM stalker_seller_inventory ORDER BY asin")
.all() as Array<{ asin: string }>;
expect(inventory.map((row) => row.asin)).toEqual([]);
});

View File

@@ -1,6 +1,15 @@
import * as XLSX from "xlsx";
import path from "node:path";
import { type Database, closeDb, getDb, initDb } from "./database.ts";
import { db } from "./db/index.ts";
import {
runs,
stalkerRuns,
stalkerAsinScans,
sellers,
stalkerAsinSellers,
stalkerSellerInventory,
} from "./db/schema.ts";
import { eq, sql } from "drizzle-orm";
import { fetchSellabilityBatch } from "./sp-api.ts";
import type { SellabilityInfo } from "./types.ts";
@@ -8,7 +17,6 @@ const KEEPA_BASE = "https://api.keepa.com";
const DOMAIN_US = "1";
const AMAZON_US_SELLER_ID = "ATVPDKIKX0DER";
const ASIN_REGEX = /^B[0-9A-Z]{9}$/;
const DEFAULT_DB_PATH = path.join(process.cwd(), "db", "results.db");
const DEFAULT_STOREFRONT_UPDATE_HOURS = 168;
const DEFAULT_OFFER_LIMIT = 100;
const DEFAULT_SELLER_LIMIT = 30;
@@ -28,7 +36,7 @@ type KeepaApiResponse = {
export type StalkerArgs = {
input: string;
dbPath: string;
dbPath?: string;
maxAsins: number | null;
storefrontUpdateHours: number;
offerLimit: number;
@@ -115,7 +123,6 @@ type StalkerRunStats = {
};
type StalkerRunContext = {
database: Database | null;
metadataCache: Map<string, StalkerSeller>;
storefrontCache: Map<string, StalkerSeller>;
stats: StalkerRunStats;
@@ -131,7 +138,6 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs {
printUsageAndExit("Missing required --input file.");
}
const dbPath = readFlagValue(argv, "--db") ?? DEFAULT_DB_PATH;
const maxAsinsRaw = readFlagValue(argv, "--max-asins");
const storefrontUpdateRaw = readFlagValue(argv, "--storefront-update-hours");
const offerLimitRaw = readFlagValue(argv, "--offer-limit");
@@ -205,7 +211,6 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs {
return {
input,
dbPath,
maxAsins,
storefrontUpdateHours,
offerLimit,
@@ -313,20 +318,18 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
const cappedAsins =
args.maxAsins == null ? allAsins : allAsins.slice(0, args.maxAsins);
initDb(args.dbPath);
const database = getDb(args.dbPath);
const completedAsins = args.resume
? loadPreviouslyScannedAsins(database)
? await loadPreviouslyScannedAsins()
: new Set<string>();
const resumeFilteredAsins = cappedAsins.filter(
(asin) => !completedAsins.has(asin),
);
const runId = args.dryRun
? null
: startStalkerRun(database, args.input, resumeFilteredAsins.length);
: await startStalkerRun(args.input, resumeFilteredAsins.length);
const analysisRunId =
!args.dryRun && args.analyzeSellable
? startStalkerAnalysisRun(database, args.input)
? await startStalkerAnalysisRun(args.input)
: null;
const stats: StalkerRunStats = {
scannedAsins: 0,
@@ -345,7 +348,6 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
stoppedEarly: false,
};
const context: StalkerRunContext = {
database,
metadataCache: new Map(),
storefrontCache: new Map(),
stats,
@@ -389,7 +391,7 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
}
if (!args.dryRun && runId != null) {
persistAsinResult(database, runId, result);
await persistAsinResult(runId, result);
}
const sellableAsins = collectPersistedInventoryAsins(result);
if (
@@ -400,7 +402,6 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
sellableAsins.length > 0
) {
await runSellableAnalysisChild(
args.dbPath,
runId,
analysisRunId,
sellableAsins,
@@ -417,7 +418,7 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
}
if (!args.dryRun && runId != null) {
refreshStalkerRun(database, runId, stats, "running");
await refreshStalkerRun(runId, stats, "running");
}
console.log(
`Stalker: ${asin} candidates=${result.candidateSellerCount}, matched=${result.matchedSellers.length}, persisted_inventory=${sumInventoryAsins(result)}`,
@@ -432,8 +433,7 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
}
if (!args.dryRun && runId != null) {
refreshStalkerRun(
database,
await refreshStalkerRun(
runId,
stats,
stats.stoppedEarly
@@ -445,16 +445,16 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
}
logRunSummary(stats, args);
if (!args.dryRun && analysisRunId != null) {
finishStalkerAnalysisRun(database, analysisRunId, "completed");
await finishStalkerAnalysisRun(analysisRunId, "completed");
}
return stats;
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
if (!args.dryRun && runId != null) {
finishStalkerRunWithError(database, runId, stats, message);
await finishStalkerRunWithError(runId, stats, message);
}
if (!args.dryRun && analysisRunId != null) {
finishStalkerAnalysisRun(database, analysisRunId, "failed", message);
await finishStalkerAnalysisRun(analysisRunId, "failed", message);
}
throw error;
}
@@ -685,13 +685,12 @@ async function fetchSellerMetadata(
for (const sellerId of uniqueSellerIds) {
const cached =
context.metadataCache.get(sellerId) ??
loadCachedSeller(
context.database,
(await loadCachedSeller(
sellerId,
args.sellerCacheHours,
false,
args.inventoryLimit,
);
));
if (cached) {
context.metadataCache.set(sellerId, cached);
out.set(sellerId, cached);
@@ -739,13 +738,12 @@ async function fetchQualifiedSellerStorefronts(
for (const sellerId of uniqueSellerIds) {
const cached =
context.storefrontCache.get(sellerId) ??
loadCachedSeller(
context.database,
(await loadCachedSeller(
sellerId,
args.sellerCacheHours,
true,
args.inventoryLimit,
);
));
if (cached) {
context.storefrontCache.set(sellerId, cached);
out.set(sellerId, cached);
@@ -830,272 +828,268 @@ async function fetchKeepaWithRetries(
throw new Error(lastErrorMessage);
}
function persistAsinResult(
database: Database,
async function persistAsinResult(
runId: number,
result: StalkerAsinResult,
): void {
const fetchedAt = new Date().toISOString();
): Promise<void> {
const fetchedAt = new Date();
database.transaction(() => {
const scanId = upsertAsinScan(database, runId, result, fetchedAt);
await db.transaction(async (tx) => {
const scanId = await upsertAsinScan(tx, runId, result, fetchedAt);
for (const { seller, offer } of result.matchedSellers) {
upsertSeller(database, seller, fetchedAt);
upsertAsinSeller(database, scanId, seller, offer);
upsertSellerInventory(database, runId, seller, fetchedAt);
await upsertSeller(tx, seller, fetchedAt);
await upsertAsinSeller(tx, scanId, seller, offer);
await upsertSellerInventory(tx, runId, seller, fetchedAt);
}
})();
});
}
function upsertAsinScan(
database: Database,
async function upsertAsinScan(
tx: Parameters<Parameters<typeof db.transaction>[0]>[0],
runId: number,
result: StalkerAsinResult,
fetchedAt: string,
): number {
database
.prepare(
`INSERT INTO stalker_asin_scans (
run_id, source_asin, title, offer_count, candidate_seller_count,
matched_seller_count, fetched_at, raw_product_json
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(run_id, source_asin) DO UPDATE SET
title = excluded.title,
offer_count = excluded.offer_count,
candidate_seller_count = excluded.candidate_seller_count,
matched_seller_count = excluded.matched_seller_count,
fetched_at = excluded.fetched_at,
raw_product_json = excluded.raw_product_json`,
)
.run(
fetchedAt: Date,
): Promise<number> {
await tx
.insert(stalkerAsinScans)
.values({
runId,
result.asin,
result.title,
result.offerCount,
result.candidateSellerCount,
result.matchedSellers.length,
sourceAsin: result.asin,
title: result.title,
offerCount: result.offerCount,
candidateSellerCount: result.candidateSellerCount,
matchedSellerCount: result.matchedSellers.length,
fetchedAt,
JSON.stringify(result.product ?? { error: result.error ?? null }),
);
rawProductJson: JSON.stringify(
result.product ?? { error: result.error ?? null },
),
})
.onConflictDoUpdate({
target: [stalkerAsinScans.runId, stalkerAsinScans.sourceAsin],
set: {
title: sql`EXCLUDED.title`,
offerCount: sql`EXCLUDED.offer_count`,
candidateSellerCount: sql`EXCLUDED.candidate_seller_count`,
matchedSellerCount: sql`EXCLUDED.matched_seller_count`,
fetchedAt: sql`EXCLUDED.fetched_at`,
rawProductJson: sql`EXCLUDED.raw_product_json`,
},
});
const row = database
.query(
`SELECT id FROM stalker_asin_scans WHERE run_id = ? AND source_asin = ?`,
)
.get(runId, result.asin) as { id: number } | null;
const [row] = await tx
.select({ id: stalkerAsinScans.id })
.from(stalkerAsinScans)
.where(
sql`${stalkerAsinScans.runId} = ${runId} AND ${stalkerAsinScans.sourceAsin} = ${result.asin}`,
);
if (!row)
throw new Error(`Failed to load stalker scan row for ${result.asin}`);
return row.id;
}
function upsertSeller(
database: Database,
async function upsertSeller(
tx: Parameters<Parameters<typeof db.transaction>[0]>[0],
seller: StalkerSeller,
fetchedAt: string,
): void {
database
.prepare(
`INSERT INTO stalker_sellers (
seller_id, seller_name, rating, rating_count, storefront_asin_total,
persisted_inventory_sample_count, last_updated_at, raw_seller_json
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(seller_id) DO UPDATE SET
seller_name = excluded.seller_name,
rating = excluded.rating,
rating_count = excluded.rating_count,
storefront_asin_total = excluded.storefront_asin_total,
persisted_inventory_sample_count = excluded.persisted_inventory_sample_count,
last_updated_at = excluded.last_updated_at,
raw_seller_json = excluded.raw_seller_json`,
)
.run(
seller.sellerId,
seller.sellerName,
seller.rating,
seller.ratingCount,
seller.storefrontAsinTotal,
seller.storefrontItems.length,
fetchedAt,
JSON.stringify(seller.rawSeller),
);
fetchedAt: Date,
): Promise<void> {
await tx
.insert(sellers)
.values({
sellerId: seller.sellerId,
sellerName: seller.sellerName,
rating: seller.rating,
ratingCount: seller.ratingCount,
storefrontAsinTotal: seller.storefrontAsinTotal,
persistedInventorySampleCount: seller.storefrontItems.length,
lastUpdatedAt: fetchedAt,
rawSellerJson: JSON.stringify(seller.rawSeller),
})
.onConflictDoUpdate({
target: sellers.sellerId,
set: {
sellerName: sql`EXCLUDED.seller_name`,
rating: sql`EXCLUDED.rating`,
ratingCount: sql`EXCLUDED.rating_count`,
storefrontAsinTotal: sql`EXCLUDED.storefront_asin_total`,
persistedInventorySampleCount: sql`EXCLUDED.persisted_inventory_sample_count`,
lastUpdatedAt: sql`EXCLUDED.last_updated_at`,
rawSellerJson: sql`EXCLUDED.raw_seller_json`,
},
});
}
function upsertAsinSeller(
database: Database,
async function upsertAsinSeller(
tx: Parameters<Parameters<typeof db.transaction>[0]>[0],
scanId: number,
seller: StalkerSeller,
offer: StalkerOffer,
): void {
database
.prepare(
`INSERT INTO stalker_asin_sellers (
scan_id, seller_id, offer_price, condition, is_fba, stock,
seller_rating, seller_rating_count, raw_offer_json
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(scan_id, seller_id) DO UPDATE SET
offer_price = excluded.offer_price,
condition = excluded.condition,
is_fba = excluded.is_fba,
stock = excluded.stock,
seller_rating = excluded.seller_rating,
seller_rating_count = excluded.seller_rating_count,
raw_offer_json = excluded.raw_offer_json`,
)
.run(
): Promise<void> {
await tx
.insert(stalkerAsinSellers)
.values({
scanId,
seller.sellerId,
offer.offerPrice,
offer.condition,
offer.isFba == null ? null : offer.isFba ? 1 : 0,
offer.stock,
seller.rating,
seller.ratingCount,
JSON.stringify(offer.rawOffer),
);
sellerId: seller.sellerId,
offerPrice: offer.offerPrice,
condition: offer.condition,
isFba: offer.isFba,
stock: offer.stock,
sellerRating: seller.rating,
sellerRatingCount: seller.ratingCount,
rawOfferJson: JSON.stringify(offer.rawOffer),
})
.onConflictDoUpdate({
target: [stalkerAsinSellers.scanId, stalkerAsinSellers.sellerId],
set: {
offerPrice: sql`EXCLUDED.offer_price`,
condition: sql`EXCLUDED.condition`,
isFba: sql`EXCLUDED.is_fba`,
stock: sql`EXCLUDED.stock`,
sellerRating: sql`EXCLUDED.seller_rating`,
sellerRatingCount: sql`EXCLUDED.seller_rating_count`,
rawOfferJson: sql`EXCLUDED.raw_offer_json`,
},
});
}
function upsertSellerInventory(
database: Database,
async function upsertSellerInventory(
tx: Parameters<Parameters<typeof db.transaction>[0]>[0],
runId: number,
seller: StalkerSeller,
fetchedAt: string,
): void {
const insert = database.prepare(
`INSERT INTO stalker_seller_inventory (
run_id, seller_id, asin, can_sell, sellability_status,
sellability_reason, product_title, brand, category_tree, current_price,
avg_price_90d, sales_rank, monthly_sold, seller_count, amazon_is_seller,
raw_product_json, last_seen_at, raw_inventory_json
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(run_id, seller_id, asin) DO UPDATE SET
can_sell = excluded.can_sell,
sellability_status = excluded.sellability_status,
sellability_reason = excluded.sellability_reason,
product_title = excluded.product_title,
brand = excluded.brand,
category_tree = excluded.category_tree,
current_price = excluded.current_price,
avg_price_90d = excluded.avg_price_90d,
sales_rank = excluded.sales_rank,
monthly_sold = excluded.monthly_sold,
seller_count = excluded.seller_count,
amazon_is_seller = excluded.amazon_is_seller,
raw_product_json = excluded.raw_product_json,
last_seen_at = excluded.last_seen_at,
raw_inventory_json = excluded.raw_inventory_json`,
fetchedAt: Date,
): Promise<void> {
const items = seller.storefrontItems.filter(
(item) =>
item.sellability?.canSell === true &&
item.sellability.sellabilityStatus === "available",
);
for (const item of seller.storefrontItems) {
if (
item.sellability?.canSell !== true ||
item.sellability.sellabilityStatus !== "available"
) {
continue;
}
if (items.length === 0) return;
insert.run(
runId,
seller.sellerId,
item.asin,
item.sellability?.canSell == null
? null
: item.sellability.canSell
? 1
: 0,
item.sellability?.sellabilityStatus ?? null,
item.sellability?.sellabilityReason ?? null,
item.productDetails?.title ?? null,
item.productDetails?.brand ?? null,
item.productDetails
? JSON.stringify(item.productDetails.categoryTree)
: null,
item.productDetails?.currentPrice ?? null,
item.productDetails?.avgPrice90 ?? null,
item.productDetails?.salesRank ?? null,
item.productDetails?.monthlySold ?? null,
item.productDetails?.sellerCount ?? null,
item.productDetails?.amazonIsSeller == null
? null
: item.productDetails.amazonIsSeller
? 1
: 0,
item.productDetails
? JSON.stringify(item.productDetails.rawProduct)
: null,
fetchedAt,
JSON.stringify(item.rawInventory),
);
}
await tx
.insert(stalkerSellerInventory)
.values(
items.map((item) => ({
runId,
sellerId: seller.sellerId,
asin: item.asin,
canSell: item.sellability?.canSell ?? null,
sellabilityStatus: item.sellability?.sellabilityStatus ?? null,
sellabilityReason: item.sellability?.sellabilityReason ?? null,
productTitle: item.productDetails?.title ?? null,
brand: item.productDetails?.brand ?? null,
categoryTree: item.productDetails
? JSON.stringify(item.productDetails.categoryTree)
: null,
currentPrice: item.productDetails?.currentPrice ?? null,
avgPrice90d: item.productDetails?.avgPrice90 ?? null,
salesRank: item.productDetails?.salesRank ?? null,
monthlySold: item.productDetails?.monthlySold ?? null,
sellerCount: item.productDetails?.sellerCount ?? null,
amazonIsSeller: item.productDetails?.amazonIsSeller ?? null,
rawProductJson: item.productDetails
? JSON.stringify(item.productDetails.rawProduct)
: null,
lastSeenAt: fetchedAt,
rawInventoryJson: JSON.stringify(item.rawInventory),
})),
)
.onConflictDoUpdate({
target: [
stalkerSellerInventory.runId,
stalkerSellerInventory.sellerId,
stalkerSellerInventory.asin,
],
set: {
canSell: sql`EXCLUDED.can_sell`,
sellabilityStatus: sql`EXCLUDED.sellability_status`,
sellabilityReason: sql`EXCLUDED.sellability_reason`,
productTitle: sql`EXCLUDED.product_title`,
brand: sql`EXCLUDED.brand`,
categoryTree: sql`EXCLUDED.category_tree`,
currentPrice: sql`EXCLUDED.current_price`,
avgPrice90d: sql`EXCLUDED.avg_price_90d`,
salesRank: sql`EXCLUDED.sales_rank`,
monthlySold: sql`EXCLUDED.monthly_sold`,
sellerCount: sql`EXCLUDED.seller_count`,
amazonIsSeller: sql`EXCLUDED.amazon_is_seller`,
rawProductJson: sql`EXCLUDED.raw_product_json`,
lastSeenAt: sql`EXCLUDED.last_seen_at`,
rawInventoryJson: sql`EXCLUDED.raw_inventory_json`,
},
});
}
function startStalkerRun(
database: Database,
async function startStalkerRun(
inputFile: string,
totalAsins: number,
): number {
const result = database
.prepare(
`INSERT INTO stalker_runs (
input_file, started_at, requested_asins, status
) VALUES (?, ?, ?, ?)`,
)
.run(inputFile, new Date().toISOString(), totalAsins, "running");
return result.lastInsertRowid as number;
): Promise<number> {
const [row] = await db
.insert(stalkerRuns)
.values({
inputFile,
startedAt: new Date(),
requestedAsins: totalAsins,
status: "running",
})
.returning({ id: stalkerRuns.id });
if (!row) throw new Error("Failed to insert stalker run record.");
return row.id;
}
function startStalkerAnalysisRun(
database: Database,
inputFile: string,
): number {
const result = database
.prepare(
`INSERT INTO category_analysis_runs (
category_id, category_label, run_timestamp, top_asins_checked,
available_asins, fba_count, fbm_count, skip_count, status, error_message
) VALUES (?, ?, ?, 0, 0, 0, 0, 0, 'running', NULL)`,
)
.run(0, `Stalker: ${path.basename(inputFile)}`, new Date().toISOString());
return result.lastInsertRowid as number;
async function startStalkerAnalysisRun(inputFile: string): Promise<number> {
const [row] = await db
.insert(runs)
.values({
type: "category_analysis",
categoryId: 0,
categoryLabel: `Stalker: ${path.basename(inputFile)}`,
topAsinsChecked: 0,
availableAsins: 0,
fbaCount: 0,
fbmCount: 0,
skipCount: 0,
status: "running",
startedAt: new Date(),
})
.returning({ id: runs.id });
if (!row) throw new Error("Failed to insert stalker analysis run record.");
return row.id;
}
function loadPreviouslyScannedAsins(database: Database): Set<string> {
const rows = database
.query(`SELECT DISTINCT source_asin FROM stalker_asin_scans`)
.all() as Array<{ source_asin: string }>;
return new Set(rows.map((row) => row.source_asin));
async function loadPreviouslyScannedAsins(): Promise<Set<string>> {
const rows = await db
.selectDistinct({ sourceAsin: stalkerAsinScans.sourceAsin })
.from(stalkerAsinScans);
return new Set(rows.map((row) => row.sourceAsin));
}
function loadCachedSeller(
database: Database | null,
async function loadCachedSeller(
sellerId: string,
maxAgeHours: number,
requireStorefront: boolean,
inventoryLimit: number,
): StalkerSeller | null {
if (!database || maxAgeHours <= 0) return null;
const row = database
.query(
`SELECT raw_seller_json, last_updated_at, storefront_asin_total
FROM stalker_sellers
WHERE seller_id = ?`,
)
.get(sellerId) as {
raw_seller_json: string | null;
last_updated_at: string;
storefront_asin_total: number | null;
} | null;
if (!row?.raw_seller_json) return null;
): Promise<StalkerSeller | null> {
if (maxAgeHours <= 0) return null;
const ageMs = Date.now() - new Date(row.last_updated_at).getTime();
const [row] = await db
.select({
rawSellerJson: sellers.rawSellerJson,
lastUpdatedAt: sellers.lastUpdatedAt,
})
.from(sellers)
.where(eq(sellers.sellerId, sellerId))
.limit(1);
if (!row?.rawSellerJson) return null;
const ageMs = Date.now() - new Date(row.lastUpdatedAt).getTime();
if (!Number.isFinite(ageMs) || ageMs > maxAgeHours * 60 * 60 * 1000) {
return null;
}
try {
const rawSeller = JSON.parse(row.raw_seller_json) as Record<string, any>;
const rawSeller = JSON.parse(row.rawSellerJson) as Record<string, any>;
const parsed = parseSeller(sellerId, rawSeller, inventoryLimit);
if (requireStorefront && parsed.storefrontAsinTotal <= 0) return null;
return parsed;
@@ -1128,137 +1122,92 @@ function logRunSummary(stats: StalkerRunStats, args: StalkerArgs): void {
);
}
function refreshStalkerRun(
database: Database,
async function refreshStalkerRun(
runId: number,
stats: StalkerRunStats,
status: string,
): void {
database
.prepare(
`UPDATE stalker_runs
SET scanned_asins = ?,
source_asins_with_matches = ?,
candidate_sellers = ?,
qualifying_sellers = ?,
matched_sellers = ?,
seller_metadata_requests = ?,
seller_storefront_requests = ?,
inventory_sellability_checked_asins = ?,
inventory_sellability_available_asins = ?,
inventory_sellability_excluded_asins = ?,
persisted_inventory_asins = ?,
status = ?,
completed_at = CASE WHEN ? = 'running' THEN completed_at ELSE ? END
WHERE id = ?`,
)
.run(
stats.scannedAsins,
stats.sourceAsinsWithMatches,
stats.candidateSellers,
stats.qualifyingSellers,
stats.matchedSellers,
stats.sellerMetadataRequests,
stats.sellerStorefrontRequests,
stats.inventorySellabilityCheckedAsins,
stats.inventorySellabilityAvailableAsins,
stats.inventorySellabilityExcludedAsins,
stats.persistedInventoryAsins,
): Promise<void> {
await db
.update(stalkerRuns)
.set({
scannedAsins: stats.scannedAsins,
sourceAsinsWithMatches: stats.sourceAsinsWithMatches,
candidateSellers: stats.candidateSellers,
qualifyingSellers: stats.qualifyingSellers,
matchedSellers: stats.matchedSellers,
sellerMetadataRequests: stats.sellerMetadataRequests,
sellerStorefrontRequests: stats.sellerStorefrontRequests,
inventorySellabilityCheckedAsins: stats.inventorySellabilityCheckedAsins,
inventorySellabilityAvailableAsins:
stats.inventorySellabilityAvailableAsins,
inventorySellabilityExcludedAsins: stats.inventorySellabilityExcludedAsins,
persistedInventoryAsins: stats.persistedInventoryAsins,
status,
status,
new Date().toISOString(),
runId,
);
...(status !== "running" ? { completedAt: new Date() } : {}),
})
.where(eq(stalkerRuns.id, runId));
}
function finishStalkerRunWithError(
database: Database,
async function finishStalkerRunWithError(
runId: number,
stats: StalkerRunStats,
errorMessage: string,
): void {
database
.prepare(
`UPDATE stalker_runs
SET scanned_asins = ?,
source_asins_with_matches = ?,
candidate_sellers = ?,
qualifying_sellers = ?,
matched_sellers = ?,
seller_metadata_requests = ?,
seller_storefront_requests = ?,
inventory_sellability_checked_asins = ?,
inventory_sellability_available_asins = ?,
inventory_sellability_excluded_asins = ?,
persisted_inventory_asins = ?,
status = 'failed',
error_message = ?,
completed_at = ?
WHERE id = ?`,
)
.run(
stats.scannedAsins,
stats.sourceAsinsWithMatches,
stats.candidateSellers,
stats.qualifyingSellers,
stats.matchedSellers,
stats.sellerMetadataRequests,
stats.sellerStorefrontRequests,
stats.inventorySellabilityCheckedAsins,
stats.inventorySellabilityAvailableAsins,
stats.inventorySellabilityExcludedAsins,
stats.persistedInventoryAsins,
): Promise<void> {
await db
.update(stalkerRuns)
.set({
scannedAsins: stats.scannedAsins,
sourceAsinsWithMatches: stats.sourceAsinsWithMatches,
candidateSellers: stats.candidateSellers,
qualifyingSellers: stats.qualifyingSellers,
matchedSellers: stats.matchedSellers,
sellerMetadataRequests: stats.sellerMetadataRequests,
sellerStorefrontRequests: stats.sellerStorefrontRequests,
inventorySellabilityCheckedAsins: stats.inventorySellabilityCheckedAsins,
inventorySellabilityAvailableAsins:
stats.inventorySellabilityAvailableAsins,
inventorySellabilityExcludedAsins: stats.inventorySellabilityExcludedAsins,
persistedInventoryAsins: stats.persistedInventoryAsins,
status: "failed",
errorMessage,
new Date().toISOString(),
runId,
);
completedAt: new Date(),
})
.where(eq(stalkerRuns.id, runId));
}
function finishStalkerAnalysisRun(
database: Database,
async function finishStalkerAnalysisRun(
runId: number,
status: "completed" | "failed",
errorMessage: string | null = null,
): void {
const stats = database
.query(
`SELECT
): Promise<void> {
const [stats] = await db.execute(
sql<{
total: string;
fba: string | null;
fbm: string | null;
skip: string | null;
}>`SELECT
COUNT(*) AS total,
SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba,
SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm,
SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip
FROM product_analysis_results
WHERE run_id = ?`,
)
.get(runId) as {
total: number;
fba: number | null;
fbm: number | null;
skip: number | null;
};
FROM category_product_results
WHERE run_id = ${runId}`,
);
database
.prepare(
`UPDATE category_analysis_runs
SET top_asins_checked = ?,
available_asins = ?,
fba_count = ?,
fbm_count = ?,
skip_count = ?,
status = ?,
error_message = ?
WHERE id = ?`,
)
.run(
stats.total ?? 0,
stats.total ?? 0,
stats.fba ?? 0,
stats.fbm ?? 0,
stats.skip ?? 0,
await db
.update(runs)
.set({
topAsinsChecked: Number(stats?.total ?? 0),
availableAsins: Number(stats?.total ?? 0),
fbaCount: Number(stats?.fba ?? 0),
fbmCount: Number(stats?.fbm ?? 0),
skipCount: Number(stats?.skip ?? 0),
status,
errorMessage,
runId,
);
completedAt: new Date(),
})
.where(eq(runs.id, runId));
}
function normalizeSellerResponse(
@@ -1492,7 +1441,6 @@ function collectPersistedInventoryAsins(result: StalkerAsinResult): string[] {
}
async function runSellableAnalysisChild(
dbPath: string,
stalkerRunId: number,
analysisRunId: number,
asins: string[],
@@ -1502,8 +1450,6 @@ async function runSellableAnalysisChild(
"bun",
"run",
"src/stalker-analyze.ts",
"--db",
dbPath,
"--stalker-run-id",
String(stalkerRunId),
"--analysis-run-id",
@@ -1660,8 +1606,5 @@ if (import.meta.main) {
.catch((error) => {
console.error(error instanceof Error ? error.message : String(error));
process.exitCode = 1;
})
.finally(() => {
closeDb();
});
}

View File

@@ -1,8 +1,41 @@
import { test, expect, beforeAll, afterAll, beforeEach, mock } from "bun:test";
import { Database } from "bun:sqlite";
import { getDb, initDb, closeDb } from "./database.ts";
import path from "node:path";
import { rmSync, mkdirSync } from "node:fs";
import { test, expect, beforeEach, mock } from "bun:test";
let nextId = 0;
function chainable(resolveWith: any[] = []): any {
const p: any = Promise.resolve(resolveWith);
p.limit = (_n: any) => chainable(resolveWith);
p.where = (_cond: any) => chainable(resolveWith);
p.from = (_table: any) => chainable(resolveWith);
return p;
}
const makeMockDb = (): any => ({
insert: (_table: any) => ({
values: (_vals: any) => ({
returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]),
onConflictDoUpdate: (_conf: any) => Promise.resolve([]),
}),
}),
update: (_table: any) => ({
set: (_vals: any) => ({
where: (_cond: any) => Promise.resolve([]),
}),
}),
select: (_sel?: any) => ({
from: (_table: any) => ({
where: (_cond: any) => chainable(),
limit: (_n: any) => chainable(),
}),
}),
selectDistinct: (_sel: any) => ({
from: (_table: any) => chainable(),
}),
execute: (_query: any) => Promise.resolve([]),
transaction: async (fn: (tx: any) => Promise<any>) => fn(makeMockDb()),
});
mock.module("./db/index.ts", () => ({ db: makeMockDb(), client: {} }));
const fetchSellabilityBatchMock = mock(async (asins: string[]) => {
return new Map<string, any>(
@@ -60,51 +93,23 @@ mock.module("./llm.ts", () => ({
const modulePromise = import("./top-monthly-sold-by-category.ts");
const DB_TEST_PATH = path.join(
process.cwd(),
"test_output",
"test_monthly_sold_analysis.sqlite",
);
let db: Database;
let processCategory: (
db: Database,
runId: number,
category: any,
perCategoryTop: number,
categoryCandidatePool: number,
minMonthlySold: number,
) => Promise<any>;
let insertCategoryRunSummary: (
db: Database,
summary: any,
runTimestamp: string,
) => Promise<number>;
let insertCategoryRunSummary: (summary: any, runTimestamp: string) => Promise<number>;
let originalFetch: typeof globalThis.fetch;
beforeAll(async () => {
const mod = await modulePromise;
processCategory = mod.processCategory;
insertCategoryRunSummary = mod.insertCategoryRunSummary;
rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true });
mkdirSync(path.dirname(DB_TEST_PATH), { recursive: true });
initDb(DB_TEST_PATH);
db = getDb(DB_TEST_PATH);
originalFetch = globalThis.fetch;
});
afterAll(() => {
globalThis.fetch = originalFetch;
closeDb();
rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true });
});
const mod = await modulePromise;
processCategory = mod.processCategory;
insertCategoryRunSummary = mod.insertCategoryRunSummary;
originalFetch = globalThis.fetch;
beforeEach(() => {
db.run("DELETE FROM product_analysis_results");
db.run("DELETE FROM category_analysis_runs");
nextId = 0;
globalThis.fetch = mock(async (input: string | URL | Request) => {
const rawUrl =
typeof input === "string"
@@ -140,25 +145,8 @@ beforeEach(() => {
monthlySold: 600,
stats: {
current: [
null,
null,
null,
1000,
null,
null,
null,
null,
null,
null,
null,
2,
null,
null,
null,
null,
null,
null,
2599,
null, null, null, 1000, null, null, null, null, null, null, null, 2,
null, null, null, null, null, null, 2599,
],
avg: [2400, null, null, 1200],
},
@@ -171,25 +159,8 @@ beforeEach(() => {
monthlySold: 250,
stats: {
current: [
null,
null,
null,
2000,
null,
null,
null,
null,
null,
null,
null,
3,
null,
null,
null,
null,
null,
null,
1999,
null, null, null, 2000, null, null, null, null, null, null, null, 3,
null, null, null, null, null, null, 1999,
],
avg: [1800, null, null, 2200],
},
@@ -202,25 +173,8 @@ beforeEach(() => {
monthlySold: 800,
stats: {
current: [
null,
null,
null,
1500,
null,
null,
null,
null,
null,
null,
null,
1,
null,
null,
null,
null,
null,
null,
2099,
null, null, null, 1500, null, null, null, null, null, null, null, 1,
null, null, null, null, null, null, 2099,
],
avg: [2000, null, null, 1800],
},
@@ -233,25 +187,8 @@ beforeEach(() => {
monthlySold: 400,
stats: {
current: [
null,
null,
null,
3000,
null,
null,
null,
null,
null,
null,
null,
4,
null,
null,
null,
null,
null,
null,
2899,
null, null, null, 3000, null, null, null, null, null, null, null, 4,
null, null, null, null, null, null, 2899,
],
avg: [2600, null, null, 2800],
},
@@ -279,7 +216,6 @@ test("processCategory filters to sellable ASINs with monthly sold >= threshold a
};
const runId = await insertCategoryRunSummary(
db,
{
categoryId: mockCategory.id,
categoryLabel: mockCategory.label,
@@ -295,22 +231,16 @@ test("processCategory filters to sellable ASINs with monthly sold >= threshold a
new Date().toISOString(),
);
const summary = await processCategory(db, runId, mockCategory, 2, 4, 300);
const summary = await processCategory(runId, mockCategory, 2, 4, 300);
expect(summary.status).toBe("ok");
expect(summary.topAsinsChecked).toBe(4);
expect(summary.availableAsins).toBe(2);
expect(summary.results?.length).toBe(2);
const productResults = db
.query(
"SELECT asin, monthly_sold FROM product_analysis_results ORDER BY monthly_sold DESC",
)
.all() as Array<{ asin: string; monthly_sold: number }>;
const asins = summary.results?.map((r: any) => r.product.record.asin) ?? [];
expect(asins).toContain("B000000001");
expect(asins).toContain("B000000004");
expect(productResults.length).toBe(2);
expect(productResults[0]?.asin).toBe("B000000001");
expect(productResults[0]?.monthly_sold).toBe(600);
expect(productResults[1]?.asin).toBe("B000000004");
expect(productResults[1]?.monthly_sold).toBe(400);
globalThis.fetch = originalFetch;
});

View File

@@ -1,6 +1,8 @@
import { existsSync, mkdirSync, readFileSync } from "node:fs";
import path from "node:path";
import { type Database, getDb, initDb } from "./database.ts";
import { db } from "./db/index.ts";
import { runs, categoryProductResults } from "./db/schema.ts";
import { eq, sql } from "drizzle-orm";
import { config } from "./config.ts";
import { analyzeProducts } from "./llm.ts";
import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts";
@@ -171,36 +173,32 @@ function printUsageAndExit(message: string): never {
}
export async function insertCategoryRunSummary(
db: Database,
summary: CategoryRunSummary,
runTimestamp: string,
): Promise<number> {
const query = `
INSERT INTO category_analysis_runs (
category_id, category_label, run_timestamp,
top_asins_checked, available_asins,
fba_count, fbm_count, skip_count,
status, error_message
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
`;
const result = db.run(query, [
summary.categoryId,
summary.categoryLabel,
runTimestamp,
summary.topAsinsChecked,
summary.availableAsins,
summary.fba,
summary.fbm,
summary.skip,
summary.status,
summary.error,
]);
// Bun's SQLite client returns { changes: number, lastInsertRowid: number | bigint }
return Number(result.lastInsertRowid);
const [row] = await db
.insert(runs)
.values({
type: "category_analysis",
status: (summary.status as typeof runs.$inferInsert.status) ?? "running",
categoryId: summary.categoryId,
categoryLabel: summary.categoryLabel,
topAsinsChecked: summary.topAsinsChecked,
availableAsins: summary.availableAsins,
totalProducts: summary.topAsinsChecked,
fbaCount: summary.fba,
fbmCount: summary.fbm,
skipCount: summary.skip,
errorMessage: summary.error || null,
startedAt: new Date(runTimestamp),
})
.returning({ id: runs.id });
if (!row) throw new Error("Failed to insert category run.");
return row.id;
}
export async function updateCategoryRunSummary(
db: Database,
runId: number,
summary: Pick<
CategoryRunSummary,
@@ -213,136 +211,110 @@ export async function updateCategoryRunSummary(
| "error"
>,
): Promise<void> {
db.run(
`
UPDATE category_analysis_runs
SET
top_asins_checked = ?,
available_asins = ?,
fba_count = ?,
fbm_count = ?,
skip_count = ?,
status = ?,
error_message = ?
WHERE id = ?
`,
[
summary.topAsinsChecked,
summary.availableAsins,
summary.fba,
summary.fbm,
summary.skip,
summary.status,
summary.error,
runId,
],
);
await db
.update(runs)
.set({
topAsinsChecked: summary.topAsinsChecked,
availableAsins: summary.availableAsins,
totalProducts: summary.topAsinsChecked,
fbaCount: summary.fba,
fbmCount: summary.fbm,
skipCount: summary.skip,
status: summary.status as typeof runs.$inferInsert.status,
errorMessage: summary.error || null,
...(summary.status !== "running" ? { completedAt: new Date() } : {}),
})
.where(eq(runs.id, runId));
}
export async function insertProductAnalysisResults(
db: Database,
runId: number,
results: AnalysisResult[],
): Promise<void> {
if (results.length === 0) {
return;
}
if (results.length === 0) return;
const insertStmt = db.prepare(`
INSERT INTO product_analysis_results (
asin, run_id, name, brand, category, unit_cost,
current_price, avg_price_90d, avg_price_90d_sheet,
selling_price_sheet, sales_rank, sales_rank_avg_90d,
seller_count, amazon_is_seller, amazon_buybox_share_pct_90d,
monthly_sold, rank_drops_30d, rank_drops_90d,
fba_fee, fbm_fee, referral_percent, can_sell,
sellability_status, sellability_reason,
verdict, confidence, reasoning, fetched_at
) VALUES (
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?
)
ON CONFLICT(asin) DO UPDATE SET
run_id = excluded.run_id,
name = excluded.name,
brand = excluded.brand,
category = excluded.category,
unit_cost = excluded.unit_cost,
current_price = excluded.current_price,
avg_price_90d = excluded.avg_price_90d,
avg_price_90d_sheet = excluded.avg_price_90d_sheet,
selling_price_sheet = excluded.selling_price_sheet,
sales_rank = excluded.sales_rank,
sales_rank_avg_90d = excluded.sales_rank_avg_90d,
seller_count = excluded.seller_count,
amazon_is_seller = excluded.amazon_is_seller,
amazon_buybox_share_pct_90d = excluded.amazon_buybox_share_pct_90d,
monthly_sold = excluded.monthly_sold,
rank_drops_30d = excluded.rank_drops_30d,
rank_drops_90d = excluded.rank_drops_90d,
fba_fee = excluded.fba_fee,
fbm_fee = excluded.fbm_fee,
referral_percent = excluded.referral_percent,
can_sell = excluded.can_sell,
sellability_status = excluded.sellability_status,
sellability_reason = excluded.sellability_reason,
verdict = excluded.verdict,
confidence = excluded.confidence,
reasoning = excluded.reasoning,
fetched_at = excluded.fetched_at;
`);
const rows = results.map((r) => {
const price =
r.product.keepa?.currentPrice ??
r.product.record.sellingPriceFromSheet ??
r.product.spApi.estimatedSalePrice;
const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank;
db.transaction((resultsBatch: AnalysisResult[]) => {
for (const r of resultsBatch) {
const price =
r.product.keepa?.currentPrice ??
r.product.record.sellingPriceFromSheet ??
r.product.spApi.estimatedSalePrice;
const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank;
insertStmt.run(
r.product.record.asin,
runId,
r.product.record.name,
r.product.record.brand ?? null,
return {
asin: r.product.record.asin,
runId,
name: r.product.record.name,
brand: r.product.record.brand ?? null,
category:
r.product.record.category ??
r.product.keepa?.categoryTree?.join(" > ") ??
null,
r.product.record.unitCost ?? null,
price ?? null,
r.product.keepa?.avgPrice90 ?? null,
r.product.record.avgPrice90FromSheet ?? null,
r.product.record.sellingPriceFromSheet ?? null,
rank ?? null,
r.product.keepa?.salesRankAvg90 ?? null,
r.product.keepa?.sellerCount ?? null,
r.product.keepa?.amazonIsSeller == null
? null
: r.product.keepa.amazonIsSeller
? 1
: 0,
r.product.keepa?.amazonBuyboxSharePct90d ?? null,
r.product.keepa?.monthlySold ?? null,
r.product.keepa?.salesRankDrops30 ?? null,
r.product.keepa?.salesRankDrops90 ?? null,
r.product.spApi.fbaFee ?? null,
r.product.spApi.fbmFee ?? null,
r.product.spApi.referralFeePercent ?? null,
r.product.keepa?.categoryTree?.join(" > ") ??
null,
unitCost: r.product.record.unitCost ?? null,
currentPrice: price ?? null,
avgPrice90d: r.product.keepa?.avgPrice90 ?? null,
avgPrice90dSheet: r.product.record.avgPrice90FromSheet ?? null,
sellingPriceSheet: r.product.record.sellingPriceFromSheet ?? null,
salesRank: rank ?? null,
salesRankAvg90d: r.product.keepa?.salesRankAvg90 ?? null,
sellerCount: r.product.keepa?.sellerCount ?? null,
amazonIsSeller: r.product.keepa?.amazonIsSeller ?? null,
amazonBuyboxSharePct90d: r.product.keepa?.amazonBuyboxSharePct90d ?? null,
monthlySold: r.product.keepa?.monthlySold ?? null,
rankDrops30d: r.product.keepa?.salesRankDrops30 ?? null,
rankDrops90d: r.product.keepa?.salesRankDrops90 ?? null,
fbaFee: r.product.spApi.fbaFee ?? null,
fbmFee: r.product.spApi.fbmFee ?? null,
referralPercent: r.product.spApi.referralFeePercent ?? null,
canSell:
r.product.spApi.canSell == null
? "unknown"
: r.product.spApi.canSell
? "yes"
: "no",
r.product.spApi.sellabilityStatus ?? null,
r.product.spApi.sellabilityReason ?? null,
r.verdict.verdict,
r.verdict.confidence,
r.verdict.reasoning ?? null,
r.product.fetchedAt,
);
}
})(results); // Execute the transaction with the results batch
sellabilityStatus: r.product.spApi.sellabilityStatus ?? null,
sellabilityReason: r.product.spApi.sellabilityReason ?? null,
verdict: r.verdict.verdict,
confidence: r.verdict.confidence,
reasoning: r.verdict.reasoning ?? null,
fetchedAt: new Date(r.product.fetchedAt),
};
});
await db
.insert(categoryProductResults)
.values(rows)
.onConflictDoUpdate({
target: categoryProductResults.asin,
set: {
runId: sql`EXCLUDED.run_id`,
name: sql`EXCLUDED.name`,
brand: sql`EXCLUDED.brand`,
category: sql`EXCLUDED.category`,
unitCost: sql`EXCLUDED.unit_cost`,
currentPrice: sql`EXCLUDED.current_price`,
avgPrice90d: sql`EXCLUDED.avg_price_90d`,
avgPrice90dSheet: sql`EXCLUDED.avg_price_90d_sheet`,
sellingPriceSheet: sql`EXCLUDED.selling_price_sheet`,
salesRank: sql`EXCLUDED.sales_rank`,
salesRankAvg90d: sql`EXCLUDED.sales_rank_avg_90d`,
sellerCount: sql`EXCLUDED.seller_count`,
amazonIsSeller: sql`EXCLUDED.amazon_is_seller`,
amazonBuyboxSharePct90d: sql`EXCLUDED.amazon_buybox_share_pct_90d`,
monthlySold: sql`EXCLUDED.monthly_sold`,
rankDrops30d: sql`EXCLUDED.rank_drops_30d`,
rankDrops90d: sql`EXCLUDED.rank_drops_90d`,
fbaFee: sql`EXCLUDED.fba_fee`,
fbmFee: sql`EXCLUDED.fbm_fee`,
referralPercent: sql`EXCLUDED.referral_percent`,
canSell: sql`EXCLUDED.can_sell`,
sellabilityStatus: sql`EXCLUDED.sellability_status`,
sellabilityReason: sql`EXCLUDED.sellability_reason`,
verdict: sql`EXCLUDED.verdict`,
confidence: sql`EXCLUDED.confidence`,
reasoning: sql`EXCLUDED.reasoning`,
fetchedAt: sql`EXCLUDED.fetched_at`,
},
});
}
function loadCategoryBlacklist(filePath: string): Set<number> {
@@ -1067,7 +1039,6 @@ function buildEnrichedProducts(
}
export async function processCategory(
db: Database,
runId: number,
category: CategoryInfo,
perCategoryTop: number,
@@ -1083,7 +1054,7 @@ export async function processCategory(
);
if (topAsins.length === 0) {
log("info", " Keepa returned no ASINs for this category.");
await updateCategoryRunSummary(db, runId, {
await updateCategoryRunSummary(runId, {
topAsinsChecked: 0,
availableAsins: 0,
fba: 0,
@@ -1127,7 +1098,7 @@ export async function processCategory(
` Sellable ASINs: ${availableAsins.length}/${uniqueTopAsins.length}`,
);
if (availableAsins.length === 0) {
await updateCategoryRunSummary(db, runId, {
await updateCategoryRunSummary(runId, {
topAsinsChecked: uniqueTopAsins.length,
availableAsins: 0,
fba: 0,
@@ -1164,7 +1135,7 @@ export async function processCategory(
);
if (selectedAsins.length === 0) {
await updateCategoryRunSummary(db, runId, {
await updateCategoryRunSummary(runId, {
topAsinsChecked: uniqueTopAsins.length,
availableAsins: 0,
fba: 0,
@@ -1231,7 +1202,7 @@ export async function processCategory(
},
}));
await insertProductAnalysisResults(db, runId, batchResults);
await insertProductAnalysisResults(runId, batchResults);
for (const result of batchResults) {
results.push(result);
@@ -1244,7 +1215,7 @@ export async function processCategory(
}
}
await updateCategoryRunSummary(db, runId, {
await updateCategoryRunSummary(runId, {
topAsinsChecked: uniqueTopAsins.length,
availableAsins: selectedAsins.length,
fba,
@@ -1264,7 +1235,7 @@ export async function processCategory(
}
}
await updateCategoryRunSummary(db, runId, {
await updateCategoryRunSummary(runId, {
topAsinsChecked: uniqueTopAsins.length,
availableAsins: selectedAsins.length,
fba,
@@ -1293,10 +1264,6 @@ export async function main(): Promise<void> {
assertSpApiPrerequisites();
mkdirSync(args.outputDir, { recursive: true });
const DB_PATH =
process.env.RESULTS_DB_PATH || path.join(process.cwd(), "db", "results.db");
initDb(DB_PATH);
const db = getDb(DB_PATH);
log("info", "Starting per-category monthly-sold pipeline");
log("info", `Marketplace: ${config.spApiMarketplaceId}`);
@@ -1333,7 +1300,6 @@ export async function main(): Promise<void> {
let runId: number | undefined;
try {
runId = await insertCategoryRunSummary(
db,
{
categoryId: category.id,
categoryLabel: category.label,
@@ -1350,7 +1316,6 @@ export async function main(): Promise<void> {
);
categorySummary = await processCategory(
db,
runId,
category,
args.perCategoryTop,
@@ -1382,7 +1347,7 @@ export async function main(): Promise<void> {
results: [],
};
if (runId) {
await updateCategoryRunSummary(db, runId, {
await updateCategoryRunSummary(runId, {
topAsinsChecked: 0,
availableAsins: 0,
fba: 0,

View File

@@ -15,7 +15,6 @@ import {
startRunInDb,
type RunCounts,
} from "./writer.ts";
import { initDb, closeDb } from "./database.ts";
import { connectCache, disconnectCache } from "./cache.ts";
import { scoreSupplierProduct, resolveSupplierSalePrice } from "./supplier-scoring.ts";
import {
@@ -31,7 +30,6 @@ import type {
UpcLookupDetail,
} from "./types.ts";
const DB_PATH = process.env.RESULTS_DB_PATH || path.join("db", "results.db");
const DEFAULT_INPUT_BATCH_SIZE = 200;
const DEFAULT_UPC_LOOKUP_BATCH_SIZE = 100;
const DEFAULT_PRICING_CONCURRENCY = 5;
@@ -48,7 +46,6 @@ export type UpcFileAnalysisOptions = {
export type UpcFileAnalysisSummary = {
runId: number;
dbPath: string;
inputFile: string;
outputFile?: string;
processedRows: number;
@@ -339,7 +336,6 @@ function summarizeSupplierResults(
export async function runUpcFileAnalysis(
options: UpcFileAnalysisOptions,
): Promise<UpcFileAnalysisSummary> {
const dbPath = options.dbPath ?? DB_PATH;
const inputBatchSize = Math.max(
1,
options.inputBatchSize ?? DEFAULT_INPUT_BATCH_SIZE,
@@ -355,8 +351,6 @@ export async function runUpcFileAnalysis(
if (manageResources) {
console.log("Connecting to Redis...");
await connectCache();
console.log("Initializing SQLite database...");
initDb(dbPath);
}
const unresolvedByStatus = createStatusCounter();
@@ -365,7 +359,7 @@ export async function runUpcFileAnalysis(
let processedRows = 0;
let matchedRows = 0;
const runId = startRunInDb(dbPath, options.inputFile, outputFile);
const runId = await startRunInDb(options.inputFile, outputFile);
try {
const readerSummary = await processUpcFileInBatches(
@@ -481,7 +475,7 @@ export async function runUpcFileAnalysis(
}
}
appendSupplierResultsToRun(dbPath, runId, batchResults);
await appendSupplierResultsToRun(runId, batchResults);
allResults.push(...batchResults);
},
{
@@ -490,7 +484,7 @@ export async function runUpcFileAnalysis(
},
);
const runCounts = refreshRunCountsInDb(dbPath, runId);
const runCounts = await refreshRunCountsInDb(runId);
const exportSummary = summarizeSupplierResults(allResults, unresolvedByStatus);
await writeSupplierWorkbook(outputFile, allResults, exportSummary);
@@ -522,7 +516,6 @@ export async function runUpcFileAnalysis(
return {
runId,
dbPath,
inputFile: options.inputFile,
outputFile,
processedRows,
@@ -540,7 +533,6 @@ export async function runUpcFileAnalysis(
} finally {
if (manageResources) {
await disconnectCache();
closeDb();
}
}
}

View File

@@ -1,4 +1,6 @@
import { getDb } from "./database.ts";
import { db } from "./db/index.ts";
import { runs, analysisResults } from "./db/schema.ts";
import { eq, sql } from "drizzle-orm";
import type { AnalysisResult, SupplierAnalysisResult } from "./types.ts";
import { mkdirSync } from "node:fs";
import path from "node:path";
@@ -84,16 +86,15 @@ function buildRow(r: AnalysisResult) {
};
}
export function writeResultsToDb(
export async function writeResultsToDb(
results: AnalysisResult[],
dbPath: string,
inputFile: string,
outputFile: string | undefined,
): void {
): Promise<void> {
const runCounts = computeRunCountsFromResults(results);
const runId = startRunInDb(dbPath, inputFile, outputFile, runCounts);
appendResultsToRun(dbPath, runId, results);
console.log(`Results written to SQLite database for run_id: ${runId}`);
const runId = await startRunInDb(inputFile, outputFile, runCounts);
await appendResultsToRun(runId, results);
console.log(`Results written to database for run_id: ${runId}`);
}
export function writeResultsWorkbook(
@@ -112,8 +113,7 @@ export function writeResultsWorkbook(
console.log(`Results workbook written: ${outputFile}`);
}
export function startRunInDb(
dbPath: string,
export async function startRunInDb(
inputFile: string,
outputFile: string | undefined,
counts: RunCounts = {
@@ -122,244 +122,181 @@ export function startRunInDb(
fbmCount: 0,
skipCount: 0,
},
): number {
const database = getDb(dbPath);
const timestamp = new Date().toISOString();
): Promise<number> {
const [row] = await db
.insert(runs)
.values({
type: "lead_analysis",
inputFile,
outputFile: outputFile ?? null,
status: "ok",
totalProducts: counts.totalProducts,
fbaCount: counts.fbaCount,
fbmCount: counts.fbmCount,
skipCount: counts.skipCount,
startedAt: new Date(),
completedAt: new Date(),
})
.returning({ id: runs.id });
const insertRun = database.prepare(
`INSERT INTO runs (
timestamp,
input_file,
output_file,
total_products,
fba_count,
fbm_count,
skip_count
) VALUES (?, ?, ?, ?, ?, ?, ?)`,
);
const runInfo = insertRun.run(
timestamp,
inputFile,
outputFile ?? null,
counts.totalProducts,
counts.fbaCount,
counts.fbmCount,
counts.skipCount,
);
const runId =
(runInfo.changes as number) > 0
? (runInfo.lastInsertRowid as number)
: null;
if (runId === null) {
throw new Error("Failed to insert run record into SQLite.");
}
return runId;
if (!row) throw new Error("Failed to insert run record.");
return row.id;
}
export function appendResultsToRun(
dbPath: string,
export async function appendResultsToRun(
runId: number,
results: AnalysisResult[],
): void {
if (results.length === 0) {
return;
}
): Promise<void> {
if (results.length === 0) return;
const database = getDb(dbPath);
const insertResult = database.prepare(
`INSERT INTO results (
run_id, asin, product_name, brand, category, unit_cost, current_price,
avg_price_90d, avg_price_90d_sheet, selling_price_sheet, sales_rank, rank_avg_90d,
sellers, amazon_is_seller, amazon_buybox_share_pct_90d,
monthly_sold, rank_drops_30d, rank_drops_90d, fba_net_sheet,
gross_profit_dollar, gross_profit_pct, net_profit_sheet, roi_sheet, moq, moq_cost,
qty_available, supplier, source_url, asin_link, promo_coupon_code, notes, lead_date,
fba_fee, fbm_fee, referral_percent, can_sell, sellability_status, sellability_reason,
verdict, confidence, reasoning, fetched_at
) VALUES (
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
)`,
);
database.transaction(() => {
for (const r of results) {
const row = buildRow(r);
insertResult.run(
runId,
row.ASIN,
row.Name,
row.Brand,
row.Category,
row["Unit Cost"] ?? null,
row["Current Price"] ?? null,
row["Avg Price 90d"] ?? null,
row["Avg Price 90d (sheet)"] ?? null,
row["Selling Price (sheet)"] ?? null,
row["Sales Rank"] ?? null,
row["Rank Avg 90d"] ?? null,
row.Sellers ?? null,
const rows = results.map((r) => {
const row = buildRow(r);
return {
runId,
asin: row.ASIN,
productName: row.Name || null,
brand: row.Brand || null,
category: row.Category || null,
unitCost: (row["Unit Cost"] as number) ?? null,
currentPrice: (row["Current Price"] as number) || null,
avgPrice90d: (row["Avg Price 90d"] as number) || null,
avgPrice90dSheet: (row["Avg Price 90d (sheet)"] as number) || null,
sellingPriceSheet: (row["Selling Price (sheet)"] as number) || null,
salesRank: (row["Sales Rank"] as number) || null,
rankAvg90d: (row["Rank Avg 90d"] as number) || null,
sellerCount: (row.Sellers as number) || null,
amazonIsSeller:
row["Amazon Is Seller"] == null
? null
: row["Amazon Is Seller"]
? 1
: 0,
row["Amazon Buy Box Share 90d %"] ?? null,
row["Monthly Sold"] ?? null,
row["Rank Drops 30d"] ?? null,
row["Rank Drops 90d"] ?? null,
row["FBA Net (sheet)"] ?? null,
row["Gross Profit $"] ?? null,
row["Gross Profit %"] ?? null,
row["Net Profit (sheet)"] ?? null,
row["ROI (sheet)"] ?? null,
row.MOQ ?? null,
row["MOQ Cost"] ?? null,
row["Qty Available"] ?? null,
row.Supplier ?? null,
row["Source URL"] ?? null,
row["ASIN Link"] ?? null,
row["Promo/Coupon Code"] ?? null,
row.Notes ?? null,
row["Lead Date"] ?? null,
row["FBA Fee"] ?? null,
row["FBM Fee"] ?? null,
row["Referral %"] ?? null,
row["Can Sell"],
row.Sellability,
row["Sellability Reason"] ?? null,
row.Verdict,
row.Confidence ?? null,
row.Reasoning,
r.product.fetchedAt,
);
}
})();
: Boolean(row["Amazon Is Seller"]),
amazonBuyboxSharePct90d:
(row["Amazon Buy Box Share 90d %"] as number) || null,
monthlySold: (row["Monthly Sold"] as number) || null,
rankDrops30d: (row["Rank Drops 30d"] as number) || null,
rankDrops90d: (row["Rank Drops 90d"] as number) || null,
fbaNetSheet: (row["FBA Net (sheet)"] as number) || null,
grossProfitDollar: (row["Gross Profit $"] as number) || null,
grossProfitPct: (row["Gross Profit %"] as number) || null,
netProfitSheet: (row["Net Profit (sheet)"] as number) || null,
roiSheet: (row["ROI (sheet)"] as number) || null,
moq: (row.MOQ as number) || null,
moqCost: (row["MOQ Cost"] as number) || null,
qtyAvailable: (row["Qty Available"] as number) || null,
supplier: row.Supplier || null,
sourceUrl: row["Source URL"] || null,
asinLink: row["ASIN Link"] || null,
promoCouponCode: row["Promo/Coupon Code"] || null,
notes: row.Notes || null,
leadDate: row["Lead Date"] || null,
fbaFee: row["FBA Fee"] ?? null,
fbmFee: row["FBM Fee"] ?? null,
referralPercent: row["Referral %"] ?? null,
canSell: row["Can Sell"],
sellabilityStatus: row.Sellability,
sellabilityReason: row["Sellability Reason"] || null,
verdict: row.Verdict,
confidence: row.Confidence ?? null,
reasoning: row.Reasoning,
fetchedAt: new Date(r.product.fetchedAt),
};
});
await db.insert(analysisResults).values(rows);
}
export function appendSupplierResultsToRun(
dbPath: string,
export async function appendSupplierResultsToRun(
runId: number,
results: SupplierAnalysisResult[],
): void {
if (results.length === 0) {
return;
}
): Promise<void> {
if (results.length === 0) return;
const database = getDb(dbPath);
const insertResult = database.prepare(
`INSERT INTO results (
run_id, asin, product_name, brand, category, unit_cost, current_price,
avg_price_90d, sales_rank, rank_avg_90d, sellers,
amazon_is_seller, amazon_buybox_share_pct_90d, monthly_sold,
rank_drops_30d, rank_drops_90d, upc, fba_fee, fbm_fee,
referral_percent, supplier_score, supplier_profit, supplier_margin,
supplier_roi, supplier_reason, upc_lookup_status, upc_lookup_reason,
candidate_asins, can_sell, sellability_status, sellability_reason,
verdict, confidence, reasoning, fetched_at
) VALUES (
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
)`,
);
const rows = results.map((result) => {
const keepa = result.keepa;
const spApi = result.spApi;
const asin = result.lookup.asin ?? result.record.asin ?? result.upc;
const category =
result.record.category ?? keepa?.categoryTree?.join(" > ") ?? null;
const canSell =
spApi?.canSell == null ? null : spApi.canSell ? "yes" : "no";
database.transaction(() => {
for (const result of results) {
const keepa = result.keepa;
const spApi = result.spApi;
const asin = result.lookup.asin ?? result.record.asin ?? result.upc;
const category =
result.record.category ?? keepa?.categoryTree?.join(" > ") ?? null;
const canSell =
spApi?.canSell == null ? null : spApi.canSell ? "yes" : "no";
return {
runId,
asin,
productName: result.record.name,
brand: result.record.brand ?? null,
category,
unitCost: result.record.unitCost || null,
currentPrice: result.score.salePrice,
avgPrice90d: keepa?.avgPrice90 ?? null,
salesRank: keepa?.salesRank ?? null,
rankAvg90d: keepa?.salesRankAvg90 ?? null,
sellerCount: keepa?.sellerCount ?? null,
amazonIsSeller: keepa?.amazonIsSeller ?? null,
amazonBuyboxSharePct90d: keepa?.amazonBuyboxSharePct90d ?? null,
monthlySold: keepa?.monthlySold ?? null,
rankDrops30d: keepa?.salesRankDrops30 ?? null,
rankDrops90d: keepa?.salesRankDrops90 ?? null,
upc: result.upc,
fbaFee: result.score.fbaFee,
fbmFee: spApi?.fbmFee ?? null,
referralPercent: spApi?.referralFeePercent ?? null,
supplierScore: result.score.score,
supplierProfit: result.score.profit,
supplierMargin: result.score.margin,
supplierRoi: result.score.roi,
supplierReason: result.score.reason,
upcLookupStatus: result.lookup.status,
upcLookupReason: result.lookup.reason ?? null,
candidateAsins: result.lookup.candidateAsins.join(","),
canSell,
sellabilityStatus: spApi?.sellabilityStatus ?? null,
sellabilityReason: spApi?.sellabilityReason ?? null,
verdict: result.score.verdict,
confidence: result.score.score,
reasoning: result.score.reason,
fetchedAt: new Date(result.fetchedAt),
};
});
insertResult.run(
runId,
asin,
result.record.name,
result.record.brand ?? null,
category,
result.record.unitCost || null,
result.score.salePrice,
keepa?.avgPrice90 ?? null,
keepa?.salesRank ?? null,
keepa?.salesRankAvg90 ?? null,
keepa?.sellerCount ?? null,
keepa?.amazonIsSeller == null ? null : keepa.amazonIsSeller ? 1 : 0,
keepa?.amazonBuyboxSharePct90d ?? null,
keepa?.monthlySold ?? null,
keepa?.salesRankDrops30 ?? null,
keepa?.salesRankDrops90 ?? null,
result.upc,
result.score.fbaFee,
spApi?.fbmFee ?? null,
spApi?.referralFeePercent ?? null,
result.score.score,
result.score.profit,
result.score.margin,
result.score.roi,
result.score.reason,
result.lookup.status,
result.lookup.reason ?? null,
result.lookup.candidateAsins.join(","),
canSell,
spApi?.sellabilityStatus ?? null,
spApi?.sellabilityReason ?? null,
result.score.verdict,
Math.round(result.score.score),
result.score.reason,
result.fetchedAt,
);
}
})();
await db.insert(analysisResults).values(rows);
}
export function refreshRunCountsInDb(dbPath: string, runId: number): RunCounts {
const database = getDb(dbPath);
const stats = database
.query(
`SELECT
export async function refreshRunCountsInDb(runId: number): Promise<RunCounts> {
const [stats] = await db.execute(
sql<{
total: string;
fba: string | null;
fbm: string | null;
skip: string | null;
}>`SELECT
COUNT(*) AS total,
SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba,
SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm,
SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip
FROM results
WHERE run_id = ?`,
)
.get(runId) as {
total: number;
fba: number | null;
fbm: number | null;
skip: number | null;
};
FROM analysis_results
WHERE run_id = ${runId}`,
);
const counts: RunCounts = {
totalProducts: stats.total ?? 0,
fbaCount: stats.fba ?? 0,
fbmCount: stats.fbm ?? 0,
skipCount: stats.skip ?? 0,
totalProducts: Number(stats?.total ?? 0),
fbaCount: Number(stats?.fba ?? 0),
fbmCount: Number(stats?.fbm ?? 0),
skipCount: Number(stats?.skip ?? 0),
};
database
.query(
`UPDATE runs
SET total_products = ?, fba_count = ?, fbm_count = ?, skip_count = ?
WHERE id = ?`,
)
.run(
counts.totalProducts,
counts.fbaCount,
counts.fbmCount,
counts.skipCount,
runId,
);
await db
.update(runs)
.set({
totalProducts: counts.totalProducts,
fbaCount: counts.fbaCount,
fbmCount: counts.fbmCount,
skipCount: counts.skipCount,
})
.where(eq(runs.id, runId));
return counts;
}
export function printResults(results: AnalysisResult[]): void {
const rows = results
.filter((r) => r.verdict.verdict === "FBA" || r.verdict.verdict === "FBM")