Refactor database interactions to use Drizzle ORM
- Replaced direct SQLite database calls with Drizzle ORM methods in `top-monthly-sold-by-category.ts`, `writer.ts`, and `upc-file-analysis.ts`. - Updated test cases in `top-monthly-sold-by-category.test.ts` to mock the new database interactions. - Removed unnecessary database initialization and cleanup code. - Improved code readability and maintainability by using ORM features for inserting and updating records.
This commit is contained in:
Victor Noguera
@@ -19,3 +19,4 @@ GOOGLE_API_KEY=your_google_api_key
|
||||
GOOGLE_CSE_ID=your_google_programmable_search_engine_id
|
||||
SERPAPI_API_KEY=your_serpapi_api_key_for_google_shopping
|
||||
|
||||
DB_CONNECTION_STRING=your_database_connection_string
|
||||
183
bun.lock
183
bun.lock
@@ -6,8 +6,10 @@
|
||||
"name": "asin-check",
|
||||
"dependencies": {
|
||||
"amazon-sp-api": "^1.2.1",
|
||||
"drizzle-orm": "^0.45.2",
|
||||
"exceljs": "^4.4.0",
|
||||
"ioredis": "^5.10.1",
|
||||
"postgres": "^3.4.9",
|
||||
"react": "^19.2.0",
|
||||
"react-dom": "^19.2.0",
|
||||
"xlsx": "^0.18.5",
|
||||
@@ -16,11 +18,70 @@
|
||||
"@types/bun": "latest",
|
||||
"@types/react": "^19.2.14",
|
||||
"@types/react-dom": "^19.2.3",
|
||||
"drizzle-kit": "^0.31.10",
|
||||
"typescript": "^6.0.3",
|
||||
},
|
||||
},
|
||||
},
|
||||
"packages": {
|
||||
"@drizzle-team/brocli": ["@drizzle-team/brocli@0.10.2", "", {}, "sha512-z33Il7l5dKjUgGULTqBsQBQwckHh5AbIuxhdsIxDDiZAzBOrZO6q9ogcWC65kU382AfynTfgNumVcNIjuIua6w=="],
|
||||
|
||||
"@esbuild-kit/core-utils": ["@esbuild-kit/core-utils@3.3.2", "", { "dependencies": { "esbuild": "~0.18.20", "source-map-support": "^0.5.21" } }, "sha512-sPRAnw9CdSsRmEtnsl2WXWdyquogVpB3yZ3dgwJfe8zrOzTsV7cJvmwrKVa+0ma5BoiGJ+BoqkMvawbayKUsqQ=="],
|
||||
|
||||
"@esbuild-kit/esm-loader": ["@esbuild-kit/esm-loader@2.6.5", "", { "dependencies": { "@esbuild-kit/core-utils": "^3.3.2", "get-tsconfig": "^4.7.0" } }, "sha512-FxEMIkJKnodyA1OaCUoEvbYRkoZlLZ4d/eXFu9Fh8CbBBgP5EmZxrfTRyN0qpXZ4vOvqnE5YdRdcrmUUXuU+dA=="],
|
||||
|
||||
"@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.25.12", "", { "os": "aix", "cpu": "ppc64" }, "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA=="],
|
||||
|
||||
"@esbuild/android-arm": ["@esbuild/android-arm@0.25.12", "", { "os": "android", "cpu": "arm" }, "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg=="],
|
||||
|
||||
"@esbuild/android-arm64": ["@esbuild/android-arm64@0.25.12", "", { "os": "android", "cpu": "arm64" }, "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg=="],
|
||||
|
||||
"@esbuild/android-x64": ["@esbuild/android-x64@0.25.12", "", { "os": "android", "cpu": "x64" }, "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg=="],
|
||||
|
||||
"@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.25.12", "", { "os": "darwin", "cpu": "arm64" }, "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg=="],
|
||||
|
||||
"@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.25.12", "", { "os": "darwin", "cpu": "x64" }, "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA=="],
|
||||
|
||||
"@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.25.12", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg=="],
|
||||
|
||||
"@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.25.12", "", { "os": "freebsd", "cpu": "x64" }, "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ=="],
|
||||
|
||||
"@esbuild/linux-arm": ["@esbuild/linux-arm@0.25.12", "", { "os": "linux", "cpu": "arm" }, "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw=="],
|
||||
|
||||
"@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.25.12", "", { "os": "linux", "cpu": "arm64" }, "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ=="],
|
||||
|
||||
"@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.25.12", "", { "os": "linux", "cpu": "ia32" }, "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA=="],
|
||||
|
||||
"@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng=="],
|
||||
|
||||
"@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw=="],
|
||||
|
||||
"@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.25.12", "", { "os": "linux", "cpu": "ppc64" }, "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA=="],
|
||||
|
||||
"@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w=="],
|
||||
|
||||
"@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.25.12", "", { "os": "linux", "cpu": "s390x" }, "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg=="],
|
||||
|
||||
"@esbuild/linux-x64": ["@esbuild/linux-x64@0.25.12", "", { "os": "linux", "cpu": "x64" }, "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw=="],
|
||||
|
||||
"@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.25.12", "", { "os": "none", "cpu": "arm64" }, "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg=="],
|
||||
|
||||
"@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.25.12", "", { "os": "none", "cpu": "x64" }, "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ=="],
|
||||
|
||||
"@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.25.12", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A=="],
|
||||
|
||||
"@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.25.12", "", { "os": "openbsd", "cpu": "x64" }, "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw=="],
|
||||
|
||||
"@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.25.12", "", { "os": "none", "cpu": "arm64" }, "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg=="],
|
||||
|
||||
"@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.25.12", "", { "os": "sunos", "cpu": "x64" }, "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w=="],
|
||||
|
||||
"@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.25.12", "", { "os": "win32", "cpu": "arm64" }, "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg=="],
|
||||
|
||||
"@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.25.12", "", { "os": "win32", "cpu": "ia32" }, "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ=="],
|
||||
|
||||
"@esbuild/win32-x64": ["@esbuild/win32-x64@0.25.12", "", { "os": "win32", "cpu": "x64" }, "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA=="],
|
||||
|
||||
"@fast-csv/format": ["@fast-csv/format@4.3.5", "", { "dependencies": { "@types/node": "^14.0.1", "lodash.escaperegexp": "^4.1.2", "lodash.isboolean": "^3.0.3", "lodash.isequal": "^4.5.0", "lodash.isfunction": "^3.0.9", "lodash.isnil": "^4.0.0" } }, "sha512-8iRn6QF3I8Ak78lNAa+Gdl5MJJBM5vRHivFtMRUWINdevNo00K7OXxS2PshawLKTejVwieIlPmK5YlLu6w4u8A=="],
|
||||
|
||||
"@fast-csv/parse": ["@fast-csv/parse@4.3.6", "", { "dependencies": { "@types/node": "^14.0.1", "lodash.escaperegexp": "^4.1.2", "lodash.groupby": "^4.6.0", "lodash.isfunction": "^3.0.9", "lodash.isnil": "^4.0.0", "lodash.isundefined": "^3.0.1", "lodash.uniq": "^4.5.0" } }, "sha512-uRsLYksqpbDmWaSmzvJcuApSEe38+6NQZBUsuAyMZKqHxH0g1wcJgsKUvN3WC8tewaqFjBMMGrkHmC+T7k8LvA=="],
|
||||
@@ -63,6 +124,8 @@
|
||||
|
||||
"buffer-crc32": ["buffer-crc32@0.2.13", "", {}, "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ=="],
|
||||
|
||||
"buffer-from": ["buffer-from@1.1.2", "", {}, "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="],
|
||||
|
||||
"buffer-indexof-polyfill": ["buffer-indexof-polyfill@1.0.2", "", {}, "sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A=="],
|
||||
|
||||
"buffers": ["buffers@0.1.1", "", {}, "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ=="],
|
||||
@@ -101,6 +164,10 @@
|
||||
|
||||
"denque": ["denque@2.1.0", "", {}, "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw=="],
|
||||
|
||||
"drizzle-kit": ["drizzle-kit@0.31.10", "", { "dependencies": { "@drizzle-team/brocli": "^0.10.2", "@esbuild-kit/esm-loader": "^2.5.5", "esbuild": "^0.25.4", "tsx": "^4.21.0" }, "bin": { "drizzle-kit": "bin.cjs" } }, "sha512-7OZcmQUrdGI+DUNNsKBn1aW8qSoKuTH7d0mYgSP8bAzdFzKoovxEFnoGQp2dVs82EOJeYycqRtciopszwUf8bw=="],
|
||||
|
||||
"drizzle-orm": ["drizzle-orm@0.45.2", "", { "peerDependencies": { "@aws-sdk/client-rds-data": ">=3", "@cloudflare/workers-types": ">=4", "@electric-sql/pglite": ">=0.2.0", "@libsql/client": ">=0.10.0", "@libsql/client-wasm": ">=0.10.0", "@neondatabase/serverless": ">=0.10.0", "@op-engineering/op-sqlite": ">=2", "@opentelemetry/api": "^1.4.1", "@planetscale/database": ">=1.13", "@prisma/client": "*", "@tidbcloud/serverless": "*", "@types/better-sqlite3": "*", "@types/pg": "*", "@types/sql.js": "*", "@upstash/redis": ">=1.34.7", "@vercel/postgres": ">=0.8.0", "@xata.io/client": "*", "better-sqlite3": ">=7", "bun-types": "*", "expo-sqlite": ">=14.0.0", "gel": ">=2", "knex": "*", "kysely": "*", "mysql2": ">=2", "pg": ">=8", "postgres": ">=3", "prisma": "*", "sql.js": ">=1", "sqlite3": ">=5" }, "optionalPeers": ["@aws-sdk/client-rds-data", "@cloudflare/workers-types", "@electric-sql/pglite", "@libsql/client", "@libsql/client-wasm", "@neondatabase/serverless", "@op-engineering/op-sqlite", "@opentelemetry/api", "@planetscale/database", "@prisma/client", "@tidbcloud/serverless", "@types/better-sqlite3", "@types/pg", "@types/sql.js", "@upstash/redis", "@vercel/postgres", "@xata.io/client", "better-sqlite3", "bun-types", "expo-sqlite", "gel", "knex", "kysely", "mysql2", "pg", "postgres", "prisma", "sql.js", "sqlite3"] }, "sha512-kY0BSaTNYWnoDMVoyY8uxmyHjpJW1geOmBMdSSicKo9CIIWkSxMIj2rkeSR51b8KAPB7m+qysjuHme5nKP+E5Q=="],
|
||||
|
||||
"dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="],
|
||||
|
||||
"duplexer2": ["duplexer2@0.1.4", "", { "dependencies": { "readable-stream": "^2.0.2" } }, "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA=="],
|
||||
@@ -113,6 +180,8 @@
|
||||
|
||||
"es-object-atoms": ["es-object-atoms@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0" } }, "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA=="],
|
||||
|
||||
"esbuild": ["esbuild@0.25.12", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.25.12", "@esbuild/android-arm": "0.25.12", "@esbuild/android-arm64": "0.25.12", "@esbuild/android-x64": "0.25.12", "@esbuild/darwin-arm64": "0.25.12", "@esbuild/darwin-x64": "0.25.12", "@esbuild/freebsd-arm64": "0.25.12", "@esbuild/freebsd-x64": "0.25.12", "@esbuild/linux-arm": "0.25.12", "@esbuild/linux-arm64": "0.25.12", "@esbuild/linux-ia32": "0.25.12", "@esbuild/linux-loong64": "0.25.12", "@esbuild/linux-mips64el": "0.25.12", "@esbuild/linux-ppc64": "0.25.12", "@esbuild/linux-riscv64": "0.25.12", "@esbuild/linux-s390x": "0.25.12", "@esbuild/linux-x64": "0.25.12", "@esbuild/netbsd-arm64": "0.25.12", "@esbuild/netbsd-x64": "0.25.12", "@esbuild/openbsd-arm64": "0.25.12", "@esbuild/openbsd-x64": "0.25.12", "@esbuild/openharmony-arm64": "0.25.12", "@esbuild/sunos-x64": "0.25.12", "@esbuild/win32-arm64": "0.25.12", "@esbuild/win32-ia32": "0.25.12", "@esbuild/win32-x64": "0.25.12" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg=="],
|
||||
|
||||
"exceljs": ["exceljs@4.4.0", "", { "dependencies": { "archiver": "^5.0.0", "dayjs": "^1.8.34", "fast-csv": "^4.3.1", "jszip": "^3.10.1", "readable-stream": "^3.6.0", "saxes": "^5.0.1", "tmp": "^0.2.0", "unzipper": "^0.10.11", "uuid": "^8.3.0" } }, "sha512-XctvKaEMaj1Ii9oDOqbW/6e1gXknSY4g/aLCDicOXqBE4M0nRWkUu0PTp++UPNzoFY12BNHMfs/VadKIS6llvg=="],
|
||||
|
||||
"fast-csv": ["fast-csv@4.3.6", "", { "dependencies": { "@fast-csv/format": "4.3.5", "@fast-csv/parse": "4.3.6" } }, "sha512-2RNSpuwwsJGP0frGsOmTb9oUF+VkFSM4SyLTDgwf2ciHWTarN0lQTC+F2f/t5J9QjW+c65VFIAAu85GsvMIusw=="],
|
||||
@@ -127,6 +196,8 @@
|
||||
|
||||
"fs.realpath": ["fs.realpath@1.0.0", "", {}, "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw=="],
|
||||
|
||||
"fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="],
|
||||
|
||||
"fstream": ["fstream@1.0.12", "", { "dependencies": { "graceful-fs": "^4.1.2", "inherits": "~2.0.0", "mkdirp": ">=0.5 0", "rimraf": "2" } }, "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg=="],
|
||||
|
||||
"function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="],
|
||||
@@ -135,6 +206,8 @@
|
||||
|
||||
"get-proto": ["get-proto@1.0.1", "", { "dependencies": { "dunder-proto": "^1.0.1", "es-object-atoms": "^1.0.0" } }, "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g=="],
|
||||
|
||||
"get-tsconfig": ["get-tsconfig@4.14.0", "", { "dependencies": { "resolve-pkg-maps": "^1.0.0" } }, "sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA=="],
|
||||
|
||||
"glob": ["glob@7.2.3", "", { "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", "minimatch": "^3.1.1", "once": "^1.3.0", "path-is-absolute": "^1.0.0" } }, "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q=="],
|
||||
|
||||
"gopd": ["gopd@1.2.0", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="],
|
||||
@@ -217,6 +290,8 @@
|
||||
|
||||
"path-is-absolute": ["path-is-absolute@1.0.1", "", {}, "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg=="],
|
||||
|
||||
"postgres": ["postgres@3.4.9", "", {}, "sha512-GD3qdB0x1z9xgFI6cdRD6xu2Sp2WCOEoe3mtnyB5Ee0XrrL5Pe+e4CCnJrRMnL1zYtRDZmQQVbvOttLnKDLnaw=="],
|
||||
|
||||
"process-nextick-args": ["process-nextick-args@2.0.1", "", {}, "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="],
|
||||
|
||||
"qs": ["qs@6.15.1", "", { "dependencies": { "side-channel": "^1.1.0" } }, "sha512-6YHEFRL9mfgcAvql/XhwTvf5jKcOiiupt2FiJxHkiX1z4j7WL8J/jRHYLluORvc1XxB5rV20KoeK00gVJamspg=="],
|
||||
@@ -233,6 +308,8 @@
|
||||
|
||||
"redis-parser": ["redis-parser@3.0.0", "", { "dependencies": { "redis-errors": "^1.0.0" } }, "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A=="],
|
||||
|
||||
"resolve-pkg-maps": ["resolve-pkg-maps@1.0.0", "", {}, "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw=="],
|
||||
|
||||
"rimraf": ["rimraf@2.7.1", "", { "dependencies": { "glob": "^7.1.3" }, "bin": { "rimraf": "./bin.js" } }, "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w=="],
|
||||
|
||||
"safe-buffer": ["safe-buffer@5.1.2", "", {}, "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="],
|
||||
@@ -253,6 +330,10 @@
|
||||
|
||||
"side-channel-weakmap": ["side-channel-weakmap@1.0.2", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3", "side-channel-map": "^1.0.1" } }, "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A=="],
|
||||
|
||||
"source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="],
|
||||
|
||||
"source-map-support": ["source-map-support@0.5.21", "", { "dependencies": { "buffer-from": "^1.0.0", "source-map": "^0.6.0" } }, "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w=="],
|
||||
|
||||
"ssf": ["ssf@0.11.2", "", { "dependencies": { "frac": "~1.1.2" } }, "sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g=="],
|
||||
|
||||
"standard-as-callback": ["standard-as-callback@2.1.0", "", {}, "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A=="],
|
||||
@@ -267,6 +348,8 @@
|
||||
|
||||
"traverse": ["traverse@0.3.9", "", {}, "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ=="],
|
||||
|
||||
"tsx": ["tsx@4.22.3", "", { "dependencies": { "esbuild": "~0.28.0" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "bin": { "tsx": "dist/cli.mjs" } }, "sha512-mdoNxBC/cSQObGGVQ5Bpn5i+yv7j68gk3Nfm3wFjcJg3Z0Mix9jzAFfP12prmm5eVGmDKtp0yyArrs0Q+8gZHg=="],
|
||||
|
||||
"typescript": ["typescript@6.0.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw=="],
|
||||
|
||||
"undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="],
|
||||
@@ -289,6 +372,8 @@
|
||||
|
||||
"zip-stream": ["zip-stream@4.1.1", "", { "dependencies": { "archiver-utils": "^3.0.4", "compress-commons": "^4.1.2", "readable-stream": "^3.6.0" } }, "sha512-9qv4rlDiopXg4E69k+vMHjNN63YFMe9sZMrdlvKnCjlCRWeCBswPPMPUfx+ipsAWq1LXHe70RcbaHdJJpS6hyQ=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild": ["esbuild@0.18.20", "", { "optionalDependencies": { "@esbuild/android-arm": "0.18.20", "@esbuild/android-arm64": "0.18.20", "@esbuild/android-x64": "0.18.20", "@esbuild/darwin-arm64": "0.18.20", "@esbuild/darwin-x64": "0.18.20", "@esbuild/freebsd-arm64": "0.18.20", "@esbuild/freebsd-x64": "0.18.20", "@esbuild/linux-arm": "0.18.20", "@esbuild/linux-arm64": "0.18.20", "@esbuild/linux-ia32": "0.18.20", "@esbuild/linux-loong64": "0.18.20", "@esbuild/linux-mips64el": "0.18.20", "@esbuild/linux-ppc64": "0.18.20", "@esbuild/linux-riscv64": "0.18.20", "@esbuild/linux-s390x": "0.18.20", "@esbuild/linux-x64": "0.18.20", "@esbuild/netbsd-x64": "0.18.20", "@esbuild/openbsd-x64": "0.18.20", "@esbuild/sunos-x64": "0.18.20", "@esbuild/win32-arm64": "0.18.20", "@esbuild/win32-ia32": "0.18.20", "@esbuild/win32-x64": "0.18.20" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA=="],
|
||||
|
||||
"@fast-csv/format/@types/node": ["@types/node@14.18.63", "", {}, "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ=="],
|
||||
|
||||
"@fast-csv/parse/@types/node": ["@types/node@14.18.63", "", {}, "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ=="],
|
||||
@@ -305,10 +390,56 @@
|
||||
|
||||
"string_decoder/safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="],
|
||||
|
||||
"tsx/esbuild": ["esbuild@0.28.0", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.28.0", "@esbuild/android-arm": "0.28.0", "@esbuild/android-arm64": "0.28.0", "@esbuild/android-x64": "0.28.0", "@esbuild/darwin-arm64": "0.28.0", "@esbuild/darwin-x64": "0.28.0", "@esbuild/freebsd-arm64": "0.28.0", "@esbuild/freebsd-x64": "0.28.0", "@esbuild/linux-arm": "0.28.0", "@esbuild/linux-arm64": "0.28.0", "@esbuild/linux-ia32": "0.28.0", "@esbuild/linux-loong64": "0.28.0", "@esbuild/linux-mips64el": "0.28.0", "@esbuild/linux-ppc64": "0.28.0", "@esbuild/linux-riscv64": "0.28.0", "@esbuild/linux-s390x": "0.28.0", "@esbuild/linux-x64": "0.28.0", "@esbuild/netbsd-arm64": "0.28.0", "@esbuild/netbsd-x64": "0.28.0", "@esbuild/openbsd-arm64": "0.28.0", "@esbuild/openbsd-x64": "0.28.0", "@esbuild/openharmony-arm64": "0.28.0", "@esbuild/sunos-x64": "0.28.0", "@esbuild/win32-arm64": "0.28.0", "@esbuild/win32-ia32": "0.28.0", "@esbuild/win32-x64": "0.28.0" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-sNR9MHpXSUV/XB4zmsFKN+QgVG82Cc7+/aaxJ8Adi8hyOac+EXptIp45QBPaVyX3N70664wRbTcLTOemCAnyqw=="],
|
||||
|
||||
"unzipper/readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="],
|
||||
|
||||
"zip-stream/archiver-utils": ["archiver-utils@3.0.4", "", { "dependencies": { "glob": "^7.2.3", "graceful-fs": "^4.2.0", "lazystream": "^1.0.0", "lodash.defaults": "^4.2.0", "lodash.difference": "^4.5.0", "lodash.flatten": "^4.4.0", "lodash.isplainobject": "^4.0.6", "lodash.union": "^4.6.0", "normalize-path": "^3.0.0", "readable-stream": "^3.6.0" } }, "sha512-KVgf4XQVrTjhyWmx6cte4RxonPLR9onExufI1jhvw/MQ4BB6IsZD5gT8Lq+u/+pRkWna/6JoHpiQioaqFP5Rzw=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.18.20", "", { "os": "android", "cpu": "arm" }, "sha512-fyi7TDI/ijKKNZTUJAQqiG5T7YjJXgnzkURqmGj13C6dCqckZBLdl4h7bkhHt/t0WP+zO9/zwroDvANaOqO5Sw=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/android-arm64": ["@esbuild/android-arm64@0.18.20", "", { "os": "android", "cpu": "arm64" }, "sha512-Nz4rJcchGDtENV0eMKUNa6L12zz2zBDXuhj/Vjh18zGqB44Bi7MBMSXjgunJgjRhCmKOjnPuZp4Mb6OKqtMHLQ=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/android-x64": ["@esbuild/android-x64@0.18.20", "", { "os": "android", "cpu": "x64" }, "sha512-8GDdlePJA8D6zlZYJV/jnrRAi6rOiNaCC/JclcXpB+KIuvfBN4owLtgzY2bsxnx666XjJx2kDPUmnTtR8qKQUg=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.18.20", "", { "os": "darwin", "cpu": "arm64" }, "sha512-bxRHW5kHU38zS2lPTPOyuyTm+S+eobPUnTNkdJEfAddYgEcll4xkT8DB9d2008DtTbl7uJag2HuE5NZAZgnNEA=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.18.20", "", { "os": "darwin", "cpu": "x64" }, "sha512-pc5gxlMDxzm513qPGbCbDukOdsGtKhfxD1zJKXjCCcU7ju50O7MeAZ8c4krSJcOIJGFR+qx21yMMVYwiQvyTyQ=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.18.20", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-yqDQHy4QHevpMAaxhhIwYPMv1NECwOvIpGCZkECn8w2WFHXjEwrBn3CeNIYsibZ/iZEUemj++M26W3cNR5h+Tw=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.18.20", "", { "os": "freebsd", "cpu": "x64" }, "sha512-tgWRPPuQsd3RmBZwarGVHZQvtzfEBOreNuxEMKFcd5DaDn2PbBxfwLcj4+aenoh7ctXcbXmOQIn8HI6mCSw5MQ=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/linux-arm": ["@esbuild/linux-arm@0.18.20", "", { "os": "linux", "cpu": "arm" }, "sha512-/5bHkMWnq1EgKr1V+Ybz3s1hWXok7mDFUMQ4cG10AfW3wL02PSZi5kFpYKrptDsgb2WAJIvRcDm+qIvXf/apvg=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.18.20", "", { "os": "linux", "cpu": "arm64" }, "sha512-2YbscF+UL7SQAVIpnWvYwM+3LskyDmPhe31pE7/aoTMFKKzIc9lLbyGUpmmb8a8AixOL61sQ/mFh3jEjHYFvdA=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.18.20", "", { "os": "linux", "cpu": "ia32" }, "sha512-P4etWwq6IsReT0E1KHU40bOnzMHoH73aXp96Fs8TIT6z9Hu8G6+0SHSw9i2isWrD2nbx2qo5yUqACgdfVGx7TA=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.18.20", "", { "os": "linux", "cpu": "none" }, "sha512-nXW8nqBTrOpDLPgPY9uV+/1DjxoQ7DoB2N8eocyq8I9XuqJ7BiAMDMf9n1xZM9TgW0J8zrquIb/A7s3BJv7rjg=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.18.20", "", { "os": "linux", "cpu": "none" }, "sha512-d5NeaXZcHp8PzYy5VnXV3VSd2D328Zb+9dEq5HE6bw6+N86JVPExrA6O68OPwobntbNJ0pzCpUFZTo3w0GyetQ=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.18.20", "", { "os": "linux", "cpu": "ppc64" }, "sha512-WHPyeScRNcmANnLQkq6AfyXRFr5D6N2sKgkFo2FqguP44Nw2eyDlbTdZwd9GYk98DZG9QItIiTlFLHJHjxP3FA=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.18.20", "", { "os": "linux", "cpu": "none" }, "sha512-WSxo6h5ecI5XH34KC7w5veNnKkju3zBRLEQNY7mv5mtBmrP/MjNBCAlsM2u5hDBlS3NGcTQpoBvRzqBcRtpq1A=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.18.20", "", { "os": "linux", "cpu": "s390x" }, "sha512-+8231GMs3mAEth6Ja1iK0a1sQ3ohfcpzpRLH8uuc5/KVDFneH6jtAJLFGafpzpMRO6DzJ6AvXKze9LfFMrIHVQ=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/linux-x64": ["@esbuild/linux-x64@0.18.20", "", { "os": "linux", "cpu": "x64" }, "sha512-UYqiqemphJcNsFEskc73jQ7B9jgwjWrSayxawS6UVFZGWrAAtkzjxSqnoclCXxWtfwLdzU+vTpcNYhpn43uP1w=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.18.20", "", { "os": "none", "cpu": "x64" }, "sha512-iO1c++VP6xUBUmltHZoMtCUdPlnPGdBom6IrO4gyKPFFVBKioIImVooR5I83nTew5UOYrk3gIJhbZh8X44y06A=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.18.20", "", { "os": "openbsd", "cpu": "x64" }, "sha512-e5e4YSsuQfX4cxcygw/UCPIEP6wbIL+se3sxPdCiMbFLBWu0eiZOJ7WoD+ptCLrmjZBK1Wk7I6D/I3NglUGOxg=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.18.20", "", { "os": "sunos", "cpu": "x64" }, "sha512-kDbFRFp0YpTQVVrqUd5FTYmWo45zGaXe0X8E1G/LKFC0v8x0vWrhOWSLITcCn63lmZIxfOMXtCfti/RxN/0wnQ=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.18.20", "", { "os": "win32", "cpu": "arm64" }, "sha512-ddYFR6ItYgoaq4v4JmQQaAI5s7npztfV4Ag6NrhiaW0RrnOXqBkgwZLofVTlq1daVTQNhtI5oieTvkRPfZrePg=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.18.20", "", { "os": "win32", "cpu": "ia32" }, "sha512-Wv7QBi3ID/rROT08SABTS7eV4hX26sVduqDOTe1MvGMjNd3EjOz4b7zeexIR62GTIEKrfJXKL9LFxTYgkyeu7g=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.18.20", "", { "os": "win32", "cpu": "x64" }, "sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ=="],
|
||||
|
||||
"archiver-utils/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="],
|
||||
|
||||
"duplexer2/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="],
|
||||
@@ -319,6 +450,58 @@
|
||||
|
||||
"lazystream/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.28.0", "", { "os": "aix", "cpu": "ppc64" }, "sha512-lhRUCeuOyJQURhTxl4WkpFTjIsbDayJHih5kZC1giwE+MhIzAb7mEsQMqMf18rHLsrb5qI1tafG20mLxEWcWlA=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.28.0", "", { "os": "android", "cpu": "arm" }, "sha512-wqh0ByljabXLKHeWXYLqoJ5jKC4XBaw6Hk08OfMrCRd2nP2ZQ5eleDZC41XHyCNgktBGYMbqnrJKq/K/lzPMSQ=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/android-arm64": ["@esbuild/android-arm64@0.28.0", "", { "os": "android", "cpu": "arm64" }, "sha512-+WzIXQOSaGs33tLEgYPYe/yQHf0WTU0X42Jca3y8NWMbUVhp7rUnw+vAsRC/QiDrdD31IszMrZy+qwPOPjd+rw=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/android-x64": ["@esbuild/android-x64@0.28.0", "", { "os": "android", "cpu": "x64" }, "sha512-+VJggoaKhk2VNNqVL7f6S189UzShHC/mR9EE8rDdSkdpN0KflSwWY/gWjDrNxxisg8Fp1ZCD9jLMo4m0OUfeUA=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.28.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-0T+A9WZm+bZ84nZBtk1ckYsOvyA3x7e2Acj1KdVfV4/2tdG4fzUp91YHx+GArWLtwqp77pBXVCPn2We7Letr0Q=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.28.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-fyzLm/DLDl/84OCfp2f/XQ4flmORsjU7VKt8HLjvIXChJoFFOIL6pLJPH4Yhd1n1gGFF9mPwtlN5Wf82DZs+LQ=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.28.0", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-l9GeW5UZBT9k9brBYI+0WDffcRxgHQD8ShN2Ur4xWq/NFzUKm3k5lsH4PdaRgb2w7mI9u61nr2gI2mLI27Nh3Q=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.28.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-BXoQai/A0wPO6Es3yFJ7APCiKGc1tdAEOgeTNy3SsB491S3aHn4S4r3e976eUnPdU+NbdtmBuLncYir2tMU9Nw=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/linux-arm": ["@esbuild/linux-arm@0.28.0", "", { "os": "linux", "cpu": "arm" }, "sha512-CjaaREJagqJp7iTaNQjjidaNbCKYcd4IDkzbwwxtSvjI7NZm79qiHc8HqciMddQ6CKvJT6aBd8lO9kN/ZudLlw=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.28.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-RVyzfb3FWsGA55n6WY0MEIEPURL1FcbhFE6BffZEMEekfCzCIMtB5yyDcFnVbTnwk+CLAgTujmV/Lgvih56W+A=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.28.0", "", { "os": "linux", "cpu": "ia32" }, "sha512-KBnSTt1kxl9x70q+ydterVdl+Cn0H18ngRMRCEQfrbqdUuntQQ0LoMZv47uB97NljZFzY6HcfqEZ2SAyIUTQBQ=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.28.0", "", { "os": "linux", "cpu": "none" }, "sha512-zpSlUce1mnxzgBADvxKXX5sl8aYQHo2ezvMNI8I0lbblJtp8V4odlm3Yzlj7gPyt3T8ReksE6bK+pT3WD+aJRg=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.28.0", "", { "os": "linux", "cpu": "none" }, "sha512-2jIfP6mmjkdmeTlsX/9vmdmhBmKADrWqN7zcdtHIeNSCH1SqIoNI63cYsjQR8J+wGa4Y5izRcSHSm8K3QWmk3w=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.28.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-bc0FE9wWeC0WBm49IQMPSPILRocGTQt3j5KPCA8os6VprfuJ7KD+5PzESSrJ6GmPIPJK965ZJHTUlSA6GNYEhg=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.28.0", "", { "os": "linux", "cpu": "none" }, "sha512-SQPZOwoTTT/HXFXQJG/vBX8sOFagGqvZyXcgLA3NhIqcBv1BJU1d46c0rGcrij2B56Z2rNiSLaZOYW5cUk7yLQ=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.28.0", "", { "os": "linux", "cpu": "s390x" }, "sha512-SCfR0HN8CEEjnYnySJTd2cw0k9OHB/YFzt5zgJEwa+wL/T/raGWYMBqwDNAC6dqFKmJYZoQBRfHjgwLHGSrn3Q=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/linux-x64": ["@esbuild/linux-x64@0.28.0", "", { "os": "linux", "cpu": "x64" }, "sha512-us0dSb9iFxIi8srnpl931Nvs65it/Jd2a2K3qs7fz2WfGPHqzfzZTfec7oxZJRNPXPnNYZtanmRc4AL/JwVzHQ=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.28.0", "", { "os": "none", "cpu": "arm64" }, "sha512-CR/RYotgtCKwtftMwJlUU7xCVNg3lMYZ0RzTmAHSfLCXw3NtZtNpswLEj/Kkf6kEL3Gw+BpOekRX0BYCtklhUw=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.28.0", "", { "os": "none", "cpu": "x64" }, "sha512-nU1yhmYutL+fQ71Kxnhg8uEOdC0pwEW9entHykTgEbna2pw2dkbFSMeqjjyHZoCmt8SBkOSvV+yNmm94aUrrqw=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.28.0", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-cXb5vApOsRsxsEl4mcZ1XY3D4DzcoMxR/nnc4IyqYs0rTI8ZKmW6kyyg+11Z8yvgMfAEldKzP7AdP64HnSC/6g=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.28.0", "", { "os": "openbsd", "cpu": "x64" }, "sha512-8wZM2qqtv9UP3mzy7HiGYNH/zjTA355mpeuA+859TyR+e+Tc08IHYpLJuMsfpDJwoLo1ikIJI8jC3GFjnRClzA=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.28.0", "", { "os": "none", "cpu": "arm64" }, "sha512-FLGfyizszcef5C3YtoyQDACyg95+dndv79i2EekILBofh5wpCa1KuBqOWKrEHZg3zrL3t5ouE5jgr94vA+Wb2w=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.28.0", "", { "os": "sunos", "cpu": "x64" }, "sha512-1ZgjUoEdHZZl/YlV76TSCz9Hqj9h9YmMGAgAPYd+q4SicWNX3G5GCyx9uhQWSLcbvPW8Ni7lj4gDa1T40akdlw=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.28.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-Q9StnDmQ/enxnpxCCLSg0oo4+34B9TdXpuyPeTedN/6+iXBJ4J+zwfQI28u/Jl40nOYAxGoNi7mFP40RUtkmUA=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.28.0", "", { "os": "win32", "cpu": "ia32" }, "sha512-zF3ag/gfiCe6U2iczcRzSYJKH1DCI+ByzSENHlM2FcDbEeo5Zd2C86Aq0tKUYAJJ1obRP84ymxIAksZUcdztHA=="],
|
||||
|
||||
"tsx/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.28.0", "", { "os": "win32", "cpu": "x64" }, "sha512-pEl1bO9mfAmIC+tW5btTmrKaujg3zGtUmWNdCw/xs70FBjwAL3o9OEKNHvNmnyylD6ubxUERiEhdsL0xBQ9efw=="],
|
||||
|
||||
"unzipper/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="],
|
||||
}
|
||||
}
|
||||
|
||||
10
drizzle.config.ts
Normal file
10
drizzle.config.ts
Normal file
@@ -0,0 +1,10 @@
|
||||
import { defineConfig } from "drizzle-kit";
|
||||
|
||||
export default defineConfig({
|
||||
schema: "./src/db/schema.ts",
|
||||
out: "./drizzle",
|
||||
dialect: "postgresql",
|
||||
dbCredentials: {
|
||||
url: process.env.DB_CONNECTION_STRING!,
|
||||
},
|
||||
});
|
||||
@@ -14,18 +14,23 @@
|
||||
"start": "bun run src/index.ts",
|
||||
"start:web": "bun --hot src/server.ts",
|
||||
"build:web": "bun build src/web/index.html --outdir dist",
|
||||
"test": "bun test"
|
||||
"test": "bun test",
|
||||
"db:generate": "drizzle-kit generate",
|
||||
"db:migrate": "drizzle-kit migrate"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "latest",
|
||||
"@types/react": "^19.2.14",
|
||||
"@types/react-dom": "^19.2.3",
|
||||
"drizzle-kit": "^0.31.10",
|
||||
"typescript": "^6.0.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"amazon-sp-api": "^1.2.1",
|
||||
"drizzle-orm": "^0.45.2",
|
||||
"exceljs": "^4.4.0",
|
||||
"ioredis": "^5.10.1",
|
||||
"postgres": "^3.4.9",
|
||||
"react": "^19.2.0",
|
||||
"react-dom": "^19.2.0",
|
||||
"xlsx": "^0.18.5"
|
||||
|
||||
@@ -1,8 +1,41 @@
|
||||
import { test, expect, beforeAll, afterAll, beforeEach, mock } from "bun:test";
|
||||
import { Database } from "bun:sqlite";
|
||||
import { getDb, initDb, closeDb } from "./database.ts";
|
||||
import path from "node:path";
|
||||
import { rmSync, mkdirSync } from "node:fs";
|
||||
import { test, expect, beforeEach, mock } from "bun:test";
|
||||
|
||||
let nextId = 0;
|
||||
|
||||
function chainable(resolveWith: any[] = []): any {
|
||||
const p: any = Promise.resolve(resolveWith);
|
||||
p.limit = (_n: any) => chainable(resolveWith);
|
||||
p.where = (_cond: any) => chainable(resolveWith);
|
||||
p.from = (_table: any) => chainable(resolveWith);
|
||||
return p;
|
||||
}
|
||||
|
||||
const makeMockDb = (): any => ({
|
||||
insert: (_table: any) => ({
|
||||
values: (_vals: any) => ({
|
||||
returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]),
|
||||
onConflictDoUpdate: (_conf: any) => Promise.resolve([]),
|
||||
}),
|
||||
}),
|
||||
update: (_table: any) => ({
|
||||
set: (_vals: any) => ({
|
||||
where: (_cond: any) => Promise.resolve([]),
|
||||
}),
|
||||
}),
|
||||
select: (_sel?: any) => ({
|
||||
from: (_table: any) => ({
|
||||
where: (_cond: any) => chainable(),
|
||||
limit: (_n: any) => chainable(),
|
||||
}),
|
||||
}),
|
||||
selectDistinct: (_sel: any) => ({
|
||||
from: (_table: any) => chainable(),
|
||||
}),
|
||||
execute: (_query: any) => Promise.resolve([]),
|
||||
transaction: async (fn: (tx: any) => Promise<any>) => fn(makeMockDb()),
|
||||
});
|
||||
|
||||
mock.module("./db/index.ts", () => ({ db: makeMockDb(), client: {} }));
|
||||
|
||||
const fetchSellabilityBatchMock = mock(async (asins: string[]) => {
|
||||
return new Map(
|
||||
@@ -47,40 +80,17 @@ mock.module("./llm.ts", () => ({
|
||||
|
||||
const modulePromise = import("./bestsellers-by-category.ts");
|
||||
|
||||
const DB_TEST_PATH = path.join(
|
||||
process.cwd(),
|
||||
"test_output",
|
||||
"test_analysis.sqlite",
|
||||
);
|
||||
|
||||
let db: Database;
|
||||
let processCategory: (db: Database, runId: number, category: any, perCategoryTop: number) => Promise<any>;
|
||||
let insertCategoryRunSummary: (db: Database, summary: any, runTimestamp: string) => Promise<number>;
|
||||
let processCategory: (runId: number, category: any, perCategoryTop: number) => Promise<any>;
|
||||
let insertCategoryRunSummary: (summary: any, runTimestamp: string) => Promise<number>;
|
||||
let originalFetch: typeof globalThis.fetch;
|
||||
|
||||
beforeAll(async () => {
|
||||
const mod = await modulePromise;
|
||||
processCategory = mod.processCategory;
|
||||
insertCategoryRunSummary = mod.insertCategoryRunSummary;
|
||||
|
||||
rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true });
|
||||
mkdirSync(path.dirname(DB_TEST_PATH), { recursive: true });
|
||||
initDb(DB_TEST_PATH);
|
||||
db = getDb(DB_TEST_PATH);
|
||||
|
||||
originalFetch = globalThis.fetch;
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
closeDb();
|
||||
rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true });
|
||||
});
|
||||
const mod = await modulePromise;
|
||||
processCategory = mod.processCategory;
|
||||
insertCategoryRunSummary = mod.insertCategoryRunSummary;
|
||||
originalFetch = globalThis.fetch;
|
||||
|
||||
beforeEach(() => {
|
||||
db.run("DELETE FROM product_analysis_results");
|
||||
db.run("DELETE FROM category_analysis_runs");
|
||||
|
||||
nextId = 0;
|
||||
globalThis.fetch = mock(async (input: string | URL | Request) => {
|
||||
const rawUrl =
|
||||
typeof input === "string"
|
||||
@@ -139,7 +149,8 @@ test("processCategory function test", async () => {
|
||||
childCount: 0,
|
||||
};
|
||||
|
||||
const runId = await insertCategoryRunSummary(db, {
|
||||
const runId = await insertCategoryRunSummary(
|
||||
{
|
||||
categoryId: mockCategory.id,
|
||||
categoryLabel: mockCategory.label,
|
||||
topAsinsChecked: 0,
|
||||
@@ -150,28 +161,22 @@ test("processCategory function test", async () => {
|
||||
status: "running",
|
||||
error: "",
|
||||
results: [],
|
||||
}, new Date().toISOString());
|
||||
const summary = await processCategory(db, runId, mockCategory, 2);
|
||||
},
|
||||
new Date().toISOString(),
|
||||
);
|
||||
|
||||
const categoryRun = db.query("SELECT * FROM category_analysis_runs").all() as any[];
|
||||
expect(categoryRun.length).toBe(1);
|
||||
expect(categoryRun[0].category_label).toBe("Category 1");
|
||||
expect(categoryRun[0].top_asins_checked).toBe(2);
|
||||
expect(categoryRun[0].available_asins).toBe(2);
|
||||
expect(categoryRun[0].fba_count).toBe(1);
|
||||
expect(categoryRun[0].fbm_count).toBe(1);
|
||||
expect(categoryRun[0].status).toBe("ok");
|
||||
const summary = await processCategory(runId, mockCategory, 2);
|
||||
|
||||
const productResults = db.query("SELECT * FROM product_analysis_results ORDER BY asin").all() as any[];
|
||||
expect(productResults.length).toBe(2);
|
||||
expect(summary.status).toBe("ok");
|
||||
expect(summary.topAsinsChecked).toBe(2);
|
||||
expect(summary.availableAsins).toBe(2);
|
||||
expect(summary.fba).toBe(1);
|
||||
expect(summary.fbm).toBe(1);
|
||||
expect(summary.results?.length).toBe(2);
|
||||
expect(summary.results?.[0]?.product.record.asin).toBe("B000000001");
|
||||
expect(summary.results?.[0]?.verdict.verdict).toBe("FBA");
|
||||
expect(summary.results?.[1]?.product.record.asin).toBe("B000000002");
|
||||
expect(summary.results?.[1]?.verdict.verdict).toBe("FBM");
|
||||
|
||||
expect(productResults[0].asin).toBe("B000000001");
|
||||
expect(productResults[0].name).toBe("Product One");
|
||||
expect(productResults[0].verdict).toBe("FBA");
|
||||
expect(productResults[0].run_id).toBe(categoryRun[0].id);
|
||||
|
||||
expect(productResults[1].asin).toBe("B000000002");
|
||||
expect(productResults[1].name).toBe("Product Two");
|
||||
expect(productResults[1].verdict).toBe("FBM");
|
||||
expect(productResults[1].run_id).toBe(categoryRun[0].id);
|
||||
globalThis.fetch = originalFetch;
|
||||
});
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { existsSync, mkdirSync, readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { type Database, getDb, initDb } from "./database.ts";
|
||||
import { db } from "./db/index.ts";
|
||||
import { runs, categoryProductResults } from "./db/schema.ts";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { config } from "./config.ts";
|
||||
import { analyzeProducts } from "./llm.ts";
|
||||
import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts";
|
||||
@@ -139,36 +141,32 @@ function printUsageAndExit(message: string): never {
|
||||
}
|
||||
|
||||
export async function insertCategoryRunSummary(
|
||||
db: Database,
|
||||
summary: CategoryRunSummary,
|
||||
runTimestamp: string,
|
||||
): Promise<number> {
|
||||
const query = `
|
||||
INSERT INTO category_analysis_runs (
|
||||
category_id, category_label, run_timestamp,
|
||||
top_asins_checked, available_asins,
|
||||
fba_count, fbm_count, skip_count,
|
||||
status, error_message
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
|
||||
`;
|
||||
const result = db.run(query, [
|
||||
summary.categoryId,
|
||||
summary.categoryLabel,
|
||||
runTimestamp,
|
||||
summary.topAsinsChecked,
|
||||
summary.availableAsins,
|
||||
summary.fba,
|
||||
summary.fbm,
|
||||
summary.skip,
|
||||
summary.status,
|
||||
summary.error,
|
||||
]);
|
||||
// Bun's SQLite client returns { changes: number, lastInsertRowid: number | bigint }
|
||||
return Number(result.lastInsertRowid);
|
||||
const [row] = await db
|
||||
.insert(runs)
|
||||
.values({
|
||||
type: "category_analysis",
|
||||
status: (summary.status as typeof runs.$inferInsert.status) ?? "running",
|
||||
categoryId: summary.categoryId,
|
||||
categoryLabel: summary.categoryLabel,
|
||||
topAsinsChecked: summary.topAsinsChecked,
|
||||
availableAsins: summary.availableAsins,
|
||||
totalProducts: summary.topAsinsChecked,
|
||||
fbaCount: summary.fba,
|
||||
fbmCount: summary.fbm,
|
||||
skipCount: summary.skip,
|
||||
errorMessage: summary.error || null,
|
||||
startedAt: new Date(runTimestamp),
|
||||
})
|
||||
.returning({ id: runs.id });
|
||||
|
||||
if (!row) throw new Error("Failed to insert category run.");
|
||||
return row.id;
|
||||
}
|
||||
|
||||
export async function updateCategoryRunSummary(
|
||||
db: Database,
|
||||
runId: number,
|
||||
summary: Pick<
|
||||
CategoryRunSummary,
|
||||
@@ -181,136 +179,110 @@ export async function updateCategoryRunSummary(
|
||||
| "error"
|
||||
>,
|
||||
): Promise<void> {
|
||||
db.run(
|
||||
`
|
||||
UPDATE category_analysis_runs
|
||||
SET
|
||||
top_asins_checked = ?,
|
||||
available_asins = ?,
|
||||
fba_count = ?,
|
||||
fbm_count = ?,
|
||||
skip_count = ?,
|
||||
status = ?,
|
||||
error_message = ?
|
||||
WHERE id = ?
|
||||
`,
|
||||
[
|
||||
summary.topAsinsChecked,
|
||||
summary.availableAsins,
|
||||
summary.fba,
|
||||
summary.fbm,
|
||||
summary.skip,
|
||||
summary.status,
|
||||
summary.error,
|
||||
runId,
|
||||
],
|
||||
);
|
||||
await db
|
||||
.update(runs)
|
||||
.set({
|
||||
topAsinsChecked: summary.topAsinsChecked,
|
||||
availableAsins: summary.availableAsins,
|
||||
totalProducts: summary.topAsinsChecked,
|
||||
fbaCount: summary.fba,
|
||||
fbmCount: summary.fbm,
|
||||
skipCount: summary.skip,
|
||||
status: summary.status as typeof runs.$inferInsert.status,
|
||||
errorMessage: summary.error || null,
|
||||
...(summary.status !== "running" ? { completedAt: new Date() } : {}),
|
||||
})
|
||||
.where(eq(runs.id, runId));
|
||||
}
|
||||
|
||||
export async function insertProductAnalysisResults(
|
||||
db: Database,
|
||||
runId: number,
|
||||
results: AnalysisResult[],
|
||||
): Promise<void> {
|
||||
if (results.length === 0) {
|
||||
return;
|
||||
}
|
||||
if (results.length === 0) return;
|
||||
|
||||
const insertStmt = db.prepare(`
|
||||
INSERT INTO product_analysis_results (
|
||||
asin, run_id, name, brand, category, unit_cost,
|
||||
current_price, avg_price_90d, avg_price_90d_sheet,
|
||||
selling_price_sheet, sales_rank, sales_rank_avg_90d,
|
||||
seller_count, amazon_is_seller, amazon_buybox_share_pct_90d,
|
||||
monthly_sold, rank_drops_30d, rank_drops_90d,
|
||||
fba_fee, fbm_fee, referral_percent, can_sell,
|
||||
sellability_status, sellability_reason,
|
||||
verdict, confidence, reasoning, fetched_at
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?
|
||||
)
|
||||
ON CONFLICT(asin) DO UPDATE SET
|
||||
run_id = excluded.run_id,
|
||||
name = excluded.name,
|
||||
brand = excluded.brand,
|
||||
category = excluded.category,
|
||||
unit_cost = excluded.unit_cost,
|
||||
current_price = excluded.current_price,
|
||||
avg_price_90d = excluded.avg_price_90d,
|
||||
avg_price_90d_sheet = excluded.avg_price_90d_sheet,
|
||||
selling_price_sheet = excluded.selling_price_sheet,
|
||||
sales_rank = excluded.sales_rank,
|
||||
sales_rank_avg_90d = excluded.sales_rank_avg_90d,
|
||||
seller_count = excluded.seller_count,
|
||||
amazon_is_seller = excluded.amazon_is_seller,
|
||||
amazon_buybox_share_pct_90d = excluded.amazon_buybox_share_pct_90d,
|
||||
monthly_sold = excluded.monthly_sold,
|
||||
rank_drops_30d = excluded.rank_drops_30d,
|
||||
rank_drops_90d = excluded.rank_drops_90d,
|
||||
fba_fee = excluded.fba_fee,
|
||||
fbm_fee = excluded.fbm_fee,
|
||||
referral_percent = excluded.referral_percent,
|
||||
can_sell = excluded.can_sell,
|
||||
sellability_status = excluded.sellability_status,
|
||||
sellability_reason = excluded.sellability_reason,
|
||||
verdict = excluded.verdict,
|
||||
confidence = excluded.confidence,
|
||||
reasoning = excluded.reasoning,
|
||||
fetched_at = excluded.fetched_at;
|
||||
`);
|
||||
|
||||
db.transaction((resultsBatch: AnalysisResult[]) => {
|
||||
for (const r of resultsBatch) {
|
||||
const rows = results.map((r) => {
|
||||
const price =
|
||||
r.product.keepa?.currentPrice ??
|
||||
r.product.record.sellingPriceFromSheet ??
|
||||
r.product.spApi.estimatedSalePrice;
|
||||
const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank;
|
||||
|
||||
insertStmt.run(
|
||||
r.product.record.asin,
|
||||
return {
|
||||
asin: r.product.record.asin,
|
||||
runId,
|
||||
r.product.record.name,
|
||||
r.product.record.brand ?? null,
|
||||
name: r.product.record.name,
|
||||
brand: r.product.record.brand ?? null,
|
||||
category:
|
||||
r.product.record.category ??
|
||||
r.product.keepa?.categoryTree?.join(" > ") ??
|
||||
null,
|
||||
r.product.record.unitCost ?? null,
|
||||
price ?? null,
|
||||
r.product.keepa?.avgPrice90 ?? null,
|
||||
r.product.record.avgPrice90FromSheet ?? null,
|
||||
r.product.record.sellingPriceFromSheet ?? null,
|
||||
rank ?? null,
|
||||
r.product.keepa?.salesRankAvg90 ?? null,
|
||||
r.product.keepa?.sellerCount ?? null,
|
||||
r.product.keepa?.amazonIsSeller == null
|
||||
? null
|
||||
: r.product.keepa.amazonIsSeller
|
||||
? 1
|
||||
: 0,
|
||||
r.product.keepa?.amazonBuyboxSharePct90d ?? null,
|
||||
r.product.keepa?.monthlySold ?? null,
|
||||
r.product.keepa?.salesRankDrops30 ?? null,
|
||||
r.product.keepa?.salesRankDrops90 ?? null,
|
||||
r.product.spApi.fbaFee ?? null,
|
||||
r.product.spApi.fbmFee ?? null,
|
||||
r.product.spApi.referralFeePercent ?? null,
|
||||
unitCost: r.product.record.unitCost ?? null,
|
||||
currentPrice: price ?? null,
|
||||
avgPrice90d: r.product.keepa?.avgPrice90 ?? null,
|
||||
avgPrice90dSheet: r.product.record.avgPrice90FromSheet ?? null,
|
||||
sellingPriceSheet: r.product.record.sellingPriceFromSheet ?? null,
|
||||
salesRank: rank ?? null,
|
||||
salesRankAvg90d: r.product.keepa?.salesRankAvg90 ?? null,
|
||||
sellerCount: r.product.keepa?.sellerCount ?? null,
|
||||
amazonIsSeller: r.product.keepa?.amazonIsSeller ?? null,
|
||||
amazonBuyboxSharePct90d: r.product.keepa?.amazonBuyboxSharePct90d ?? null,
|
||||
monthlySold: r.product.keepa?.monthlySold ?? null,
|
||||
rankDrops30d: r.product.keepa?.salesRankDrops30 ?? null,
|
||||
rankDrops90d: r.product.keepa?.salesRankDrops90 ?? null,
|
||||
fbaFee: r.product.spApi.fbaFee ?? null,
|
||||
fbmFee: r.product.spApi.fbmFee ?? null,
|
||||
referralPercent: r.product.spApi.referralFeePercent ?? null,
|
||||
canSell:
|
||||
r.product.spApi.canSell == null
|
||||
? "unknown"
|
||||
: r.product.spApi.canSell
|
||||
? "yes"
|
||||
: "no",
|
||||
r.product.spApi.sellabilityStatus ?? null,
|
||||
r.product.spApi.sellabilityReason ?? null,
|
||||
r.verdict.verdict,
|
||||
r.verdict.confidence,
|
||||
r.verdict.reasoning ?? null,
|
||||
r.product.fetchedAt,
|
||||
);
|
||||
}
|
||||
})(results); // Execute the transaction with the results batch
|
||||
sellabilityStatus: r.product.spApi.sellabilityStatus ?? null,
|
||||
sellabilityReason: r.product.spApi.sellabilityReason ?? null,
|
||||
verdict: r.verdict.verdict,
|
||||
confidence: r.verdict.confidence,
|
||||
reasoning: r.verdict.reasoning ?? null,
|
||||
fetchedAt: new Date(r.product.fetchedAt),
|
||||
};
|
||||
});
|
||||
|
||||
await db
|
||||
.insert(categoryProductResults)
|
||||
.values(rows)
|
||||
.onConflictDoUpdate({
|
||||
target: categoryProductResults.asin,
|
||||
set: {
|
||||
runId: sql`EXCLUDED.run_id`,
|
||||
name: sql`EXCLUDED.name`,
|
||||
brand: sql`EXCLUDED.brand`,
|
||||
category: sql`EXCLUDED.category`,
|
||||
unitCost: sql`EXCLUDED.unit_cost`,
|
||||
currentPrice: sql`EXCLUDED.current_price`,
|
||||
avgPrice90d: sql`EXCLUDED.avg_price_90d`,
|
||||
avgPrice90dSheet: sql`EXCLUDED.avg_price_90d_sheet`,
|
||||
sellingPriceSheet: sql`EXCLUDED.selling_price_sheet`,
|
||||
salesRank: sql`EXCLUDED.sales_rank`,
|
||||
salesRankAvg90d: sql`EXCLUDED.sales_rank_avg_90d`,
|
||||
sellerCount: sql`EXCLUDED.seller_count`,
|
||||
amazonIsSeller: sql`EXCLUDED.amazon_is_seller`,
|
||||
amazonBuyboxSharePct90d: sql`EXCLUDED.amazon_buybox_share_pct_90d`,
|
||||
monthlySold: sql`EXCLUDED.monthly_sold`,
|
||||
rankDrops30d: sql`EXCLUDED.rank_drops_30d`,
|
||||
rankDrops90d: sql`EXCLUDED.rank_drops_90d`,
|
||||
fbaFee: sql`EXCLUDED.fba_fee`,
|
||||
fbmFee: sql`EXCLUDED.fbm_fee`,
|
||||
referralPercent: sql`EXCLUDED.referral_percent`,
|
||||
canSell: sql`EXCLUDED.can_sell`,
|
||||
sellabilityStatus: sql`EXCLUDED.sellability_status`,
|
||||
sellabilityReason: sql`EXCLUDED.sellability_reason`,
|
||||
verdict: sql`EXCLUDED.verdict`,
|
||||
confidence: sql`EXCLUDED.confidence`,
|
||||
reasoning: sql`EXCLUDED.reasoning`,
|
||||
fetchedAt: sql`EXCLUDED.fetched_at`,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function loadCategoryBlacklist(filePath: string): Set<number> {
|
||||
@@ -1014,7 +986,6 @@ function buildEnrichedProducts(
|
||||
}
|
||||
|
||||
export async function processCategory(
|
||||
db: Database,
|
||||
runId: number,
|
||||
category: CategoryInfo,
|
||||
perCategoryTop: number,
|
||||
@@ -1025,7 +996,7 @@ export async function processCategory(
|
||||
const topAsins = await fetchCategoryBestSellerAsins(category, perCategoryTop);
|
||||
if (topAsins.length === 0) {
|
||||
log("info", " Keepa returned no ASINs for this category.");
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: 0,
|
||||
availableAsins: 0,
|
||||
fba: 0,
|
||||
@@ -1069,7 +1040,7 @@ export async function processCategory(
|
||||
` Sellable ASINs: ${availableAsins.length}/${uniqueTopAsins.length}`,
|
||||
);
|
||||
if (availableAsins.length === 0) {
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: uniqueTopAsins.length,
|
||||
availableAsins: 0,
|
||||
fba: 0,
|
||||
@@ -1137,7 +1108,7 @@ export async function processCategory(
|
||||
},
|
||||
}));
|
||||
|
||||
await insertProductAnalysisResults(db, runId, batchResults);
|
||||
await insertProductAnalysisResults(runId, batchResults);
|
||||
|
||||
for (const result of batchResults) {
|
||||
results.push(result);
|
||||
@@ -1150,7 +1121,7 @@ export async function processCategory(
|
||||
}
|
||||
}
|
||||
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: uniqueTopAsins.length,
|
||||
availableAsins: availableAsins.length,
|
||||
fba,
|
||||
@@ -1170,7 +1141,7 @@ export async function processCategory(
|
||||
}
|
||||
}
|
||||
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: uniqueTopAsins.length,
|
||||
availableAsins: availableAsins.length,
|
||||
fba,
|
||||
@@ -1199,10 +1170,6 @@ export async function main(): Promise<void> {
|
||||
assertSpApiPrerequisites();
|
||||
|
||||
mkdirSync(args.outputDir, { recursive: true });
|
||||
const DB_PATH =
|
||||
process.env.RESULTS_DB_PATH || path.join(process.cwd(), "db", "results.db");
|
||||
initDb(DB_PATH);
|
||||
const db = getDb(DB_PATH);
|
||||
|
||||
log("info", "Starting per-category bestseller pipeline");
|
||||
log("info", `Marketplace: ${config.spApiMarketplaceId}`);
|
||||
@@ -1236,7 +1203,6 @@ export async function main(): Promise<void> {
|
||||
let runId: number | undefined;
|
||||
try {
|
||||
runId = await insertCategoryRunSummary(
|
||||
db,
|
||||
{
|
||||
categoryId: category.id,
|
||||
categoryLabel: category.label,
|
||||
@@ -1253,7 +1219,6 @@ export async function main(): Promise<void> {
|
||||
);
|
||||
|
||||
categorySummary = await processCategory(
|
||||
db,
|
||||
runId,
|
||||
category,
|
||||
args.perCategoryTop,
|
||||
@@ -1283,7 +1248,7 @@ export async function main(): Promise<void> {
|
||||
results: [],
|
||||
};
|
||||
if (runId) {
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: 0,
|
||||
availableAsins: 0,
|
||||
fba: 0,
|
||||
|
||||
@@ -1,21 +1,16 @@
|
||||
import { getDb } from "./database.ts";
|
||||
import path from "node:path";
|
||||
import { db } from "./db/index.ts";
|
||||
import { runs } from "./db/schema.ts";
|
||||
import { eq } from "drizzle-orm";
|
||||
|
||||
async function checkDb() {
|
||||
const DB_PATH =
|
||||
process.env.RESULTS_DB_PATH || path.join(process.cwd(), "db", "results.db");
|
||||
const db = getDb(DB_PATH);
|
||||
|
||||
try {
|
||||
const query = db.query(
|
||||
"SELECT * FROM category_analysis_runs WHERE category_id = ?",
|
||||
);
|
||||
const result = query.all(19419898011);
|
||||
const result = await db
|
||||
.select()
|
||||
.from(runs)
|
||||
.where(eq(runs.type, "category_analysis"));
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
} catch (error) {
|
||||
console.error("Database query failed:", error);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
497
src/database.ts
497
src/database.ts
@@ -1,494 +1,3 @@
|
||||
import { Database } from "bun:sqlite";
|
||||
import { dirname } from "node:path";
|
||||
import { mkdirSync } from "node:fs";
|
||||
export { Database } from "bun:sqlite";
|
||||
|
||||
let db: Database | null = null;
|
||||
|
||||
export function getDb(dbPath: string): Database {
|
||||
if (!db) {
|
||||
const dbDir = dirname(dbPath);
|
||||
if (dbDir && dbDir !== ".") {
|
||||
mkdirSync(dbDir, { recursive: true });
|
||||
}
|
||||
db = new Database(dbPath);
|
||||
db.run("PRAGMA journal_mode = WAL;"); // Enable WAL mode for better performance
|
||||
db.run("PRAGMA foreign_keys = ON;"); // Enforce foreign key constraints
|
||||
}
|
||||
return db;
|
||||
}
|
||||
|
||||
export function closeDb(): void {
|
||||
if (db) {
|
||||
db.close();
|
||||
db = null;
|
||||
}
|
||||
}
|
||||
|
||||
function createProductAnalysisResultsTable(database: Database): void {
|
||||
database.run(`
|
||||
CREATE TABLE IF NOT EXISTS product_analysis_results (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
asin TEXT NOT NULL,
|
||||
run_id INTEGER NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
brand TEXT,
|
||||
category TEXT,
|
||||
unit_cost REAL,
|
||||
current_price REAL,
|
||||
avg_price_90d REAL,
|
||||
avg_price_90d_sheet REAL,
|
||||
selling_price_sheet REAL,
|
||||
sales_rank INTEGER,
|
||||
sales_rank_avg_90d INTEGER,
|
||||
seller_count INTEGER,
|
||||
amazon_is_seller INTEGER,
|
||||
amazon_buybox_share_pct_90d REAL,
|
||||
monthly_sold INTEGER,
|
||||
rank_drops_30d INTEGER,
|
||||
rank_drops_90d INTEGER,
|
||||
fba_fee REAL,
|
||||
fbm_fee REAL,
|
||||
referral_percent REAL,
|
||||
can_sell TEXT,
|
||||
sellability_status TEXT,
|
||||
sellability_reason TEXT,
|
||||
verdict TEXT NOT NULL,
|
||||
confidence REAL NOT NULL,
|
||||
reasoning TEXT,
|
||||
fetched_at TEXT NOT NULL,
|
||||
UNIQUE(asin),
|
||||
FOREIGN KEY (run_id) REFERENCES category_analysis_runs(id)
|
||||
);
|
||||
`);
|
||||
}
|
||||
|
||||
function ensureProductAnalysisResultsTable(database: Database): void {
|
||||
const tableInfo = database
|
||||
.query("PRAGMA table_info(product_analysis_results)")
|
||||
.all() as Array<{ name: string; pk: number }>;
|
||||
|
||||
if (tableInfo.length === 0) {
|
||||
createProductAnalysisResultsTable(database);
|
||||
return;
|
||||
}
|
||||
|
||||
const hasIdColumn = tableInfo.some((col) => col.name === "id");
|
||||
const hasAsinPrimaryKey = tableInfo.some(
|
||||
(col) => col.name === "asin" && col.pk === 1,
|
||||
);
|
||||
|
||||
const indexList = database
|
||||
.query("PRAGMA index_list(product_analysis_results)")
|
||||
.all() as Array<{ name: string; unique: number }>;
|
||||
const hasUniqueAsinConstraint = indexList.some((idx) => {
|
||||
if (idx.unique !== 1) return false;
|
||||
const columns = database
|
||||
.query(`PRAGMA index_info(${JSON.stringify(idx.name)})`)
|
||||
.all() as Array<{ name: string }>;
|
||||
return columns.length === 1 && columns[0]?.name === "asin";
|
||||
});
|
||||
|
||||
if (!hasIdColumn || hasAsinPrimaryKey || !hasUniqueAsinConstraint) {
|
||||
database.run(
|
||||
"ALTER TABLE product_analysis_results RENAME TO product_analysis_results_legacy",
|
||||
);
|
||||
createProductAnalysisResultsTable(database);
|
||||
database.run(`
|
||||
WITH ranked AS (
|
||||
SELECT
|
||||
asin, run_id, name, brand, category, unit_cost,
|
||||
current_price, avg_price_90d, avg_price_90d_sheet,
|
||||
selling_price_sheet, sales_rank, sales_rank_avg_90d,
|
||||
seller_count, NULL AS amazon_is_seller,
|
||||
NULL AS amazon_buybox_share_pct_90d,
|
||||
monthly_sold, rank_drops_30d, rank_drops_90d,
|
||||
fba_fee, fbm_fee, referral_percent, can_sell,
|
||||
sellability_status, sellability_reason,
|
||||
verdict, confidence, reasoning, fetched_at,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY asin
|
||||
ORDER BY datetime(fetched_at) DESC, run_id DESC, id DESC
|
||||
) AS row_num
|
||||
FROM product_analysis_results_legacy
|
||||
)
|
||||
INSERT INTO product_analysis_results (
|
||||
asin, run_id, name, brand, category, unit_cost,
|
||||
current_price, avg_price_90d, avg_price_90d_sheet,
|
||||
selling_price_sheet, sales_rank, sales_rank_avg_90d,
|
||||
seller_count, amazon_is_seller, amazon_buybox_share_pct_90d,
|
||||
monthly_sold, rank_drops_30d, rank_drops_90d,
|
||||
fba_fee, fbm_fee, referral_percent, can_sell,
|
||||
sellability_status, sellability_reason,
|
||||
verdict, confidence, reasoning, fetched_at
|
||||
)
|
||||
SELECT
|
||||
asin, run_id, name, brand, category, unit_cost,
|
||||
current_price, avg_price_90d, avg_price_90d_sheet,
|
||||
selling_price_sheet, sales_rank, sales_rank_avg_90d,
|
||||
seller_count, amazon_is_seller, amazon_buybox_share_pct_90d,
|
||||
monthly_sold, rank_drops_30d, rank_drops_90d,
|
||||
fba_fee, fbm_fee, referral_percent, can_sell,
|
||||
sellability_status, sellability_reason,
|
||||
verdict, confidence, reasoning, fetched_at
|
||||
FROM ranked
|
||||
WHERE row_num = 1
|
||||
`);
|
||||
database.run("DROP TABLE product_analysis_results_legacy");
|
||||
}
|
||||
}
|
||||
|
||||
function ensureProductAnalysisResultsColumns(database: Database): void {
|
||||
const tableInfo = database
|
||||
.query("PRAGMA table_info(product_analysis_results)")
|
||||
.all() as Array<{ name: string }>;
|
||||
|
||||
if (tableInfo.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const existingColumns = new Set(tableInfo.map((col) => col.name));
|
||||
const requiredColumns: Array<{ name: string; type: string }> = [
|
||||
{ name: "amazon_is_seller", type: "INTEGER" },
|
||||
{ name: "amazon_buybox_share_pct_90d", type: "REAL" },
|
||||
];
|
||||
|
||||
for (const column of requiredColumns) {
|
||||
if (!existingColumns.has(column.name)) {
|
||||
database.run(
|
||||
`ALTER TABLE product_analysis_results ADD COLUMN ${column.name} ${column.type}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function ensureResultsTableColumns(database: Database): void {
|
||||
const tableInfo = database
|
||||
.query("PRAGMA table_info(results)")
|
||||
.all() as Array<{ name: string }>;
|
||||
|
||||
if (tableInfo.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const existingColumns = new Set(tableInfo.map((col) => col.name));
|
||||
const requiredColumns: Array<{ name: string; type: string }> = [
|
||||
{ name: "fba_net_sheet", type: "REAL" },
|
||||
{ name: "gross_profit_dollar", type: "REAL" },
|
||||
{ name: "gross_profit_pct", type: "REAL" },
|
||||
{ name: "net_profit_sheet", type: "REAL" },
|
||||
{ name: "roi_sheet", type: "REAL" },
|
||||
{ name: "moq", type: "INTEGER" },
|
||||
{ name: "moq_cost", type: "REAL" },
|
||||
{ name: "qty_available", type: "INTEGER" },
|
||||
{ name: "supplier", type: "TEXT" },
|
||||
{ name: "source_url", type: "TEXT" },
|
||||
{ name: "asin_link", type: "TEXT" },
|
||||
{ name: "promo_coupon_code", type: "TEXT" },
|
||||
{ name: "notes", type: "TEXT" },
|
||||
{ name: "lead_date", type: "TEXT" },
|
||||
{ name: "amazon_is_seller", type: "INTEGER" },
|
||||
{ name: "amazon_buybox_share_pct_90d", type: "REAL" },
|
||||
{ name: "upc", type: "TEXT" },
|
||||
{ name: "supplier_score", type: "REAL" },
|
||||
{ name: "supplier_profit", type: "REAL" },
|
||||
{ name: "supplier_margin", type: "REAL" },
|
||||
{ name: "supplier_roi", type: "REAL" },
|
||||
{ name: "supplier_reason", type: "TEXT" },
|
||||
{ name: "upc_lookup_status", type: "TEXT" },
|
||||
{ name: "upc_lookup_reason", type: "TEXT" },
|
||||
{ name: "candidate_asins", type: "TEXT" },
|
||||
];
|
||||
|
||||
for (const column of requiredColumns) {
|
||||
if (!existingColumns.has(column.name)) {
|
||||
database.run(
|
||||
`ALTER TABLE results ADD COLUMN ${column.name} ${column.type}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function initDb(dbPath: string): void {
|
||||
const database = getDb(dbPath);
|
||||
database.run(`
|
||||
CREATE TABLE IF NOT EXISTS runs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
timestamp TEXT NOT NULL,
|
||||
input_file TEXT NOT NULL,
|
||||
output_file TEXT,
|
||||
total_products INTEGER,
|
||||
fba_count INTEGER,
|
||||
fbm_count INTEGER,
|
||||
skip_count INTEGER
|
||||
);
|
||||
`);
|
||||
database.run(`
|
||||
CREATE TABLE IF NOT EXISTS results (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
run_id INTEGER NOT NULL,
|
||||
asin TEXT NOT NULL,
|
||||
product_name TEXT,
|
||||
brand TEXT,
|
||||
category TEXT,
|
||||
unit_cost REAL,
|
||||
current_price REAL,
|
||||
avg_price_90d REAL,
|
||||
avg_price_90d_sheet REAL,
|
||||
selling_price_sheet REAL,
|
||||
sales_rank INTEGER,
|
||||
rank_avg_90d INTEGER,
|
||||
sellers INTEGER,
|
||||
amazon_is_seller INTEGER,
|
||||
amazon_buybox_share_pct_90d REAL,
|
||||
monthly_sold INTEGER,
|
||||
rank_drops_30d INTEGER,
|
||||
rank_drops_90d INTEGER,
|
||||
fba_net_sheet REAL,
|
||||
gross_profit_dollar REAL,
|
||||
gross_profit_pct REAL,
|
||||
net_profit_sheet REAL,
|
||||
roi_sheet REAL,
|
||||
moq INTEGER,
|
||||
moq_cost REAL,
|
||||
qty_available INTEGER,
|
||||
supplier TEXT,
|
||||
source_url TEXT,
|
||||
asin_link TEXT,
|
||||
promo_coupon_code TEXT,
|
||||
notes TEXT,
|
||||
lead_date TEXT,
|
||||
upc TEXT,
|
||||
fba_fee REAL,
|
||||
fbm_fee REAL,
|
||||
referral_percent REAL,
|
||||
supplier_score REAL,
|
||||
supplier_profit REAL,
|
||||
supplier_margin REAL,
|
||||
supplier_roi REAL,
|
||||
supplier_reason TEXT,
|
||||
upc_lookup_status TEXT,
|
||||
upc_lookup_reason TEXT,
|
||||
candidate_asins TEXT,
|
||||
can_sell TEXT,
|
||||
sellability_status TEXT,
|
||||
sellability_reason TEXT,
|
||||
verdict TEXT NOT NULL,
|
||||
confidence INTEGER,
|
||||
reasoning TEXT,
|
||||
fetched_at TEXT NOT NULL,
|
||||
FOREIGN KEY (run_id) REFERENCES runs(id)
|
||||
);
|
||||
`);
|
||||
ensureResultsTableColumns(database);
|
||||
database.run(`
|
||||
CREATE TABLE IF NOT EXISTS category_analysis_runs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
category_id INTEGER NOT NULL,
|
||||
category_label TEXT NOT NULL,
|
||||
run_timestamp TEXT NOT NULL,
|
||||
top_asins_checked INTEGER NOT NULL,
|
||||
available_asins INTEGER NOT NULL,
|
||||
fba_count INTEGER NOT NULL,
|
||||
fbm_count INTEGER NOT NULL,
|
||||
skip_count INTEGER NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
error_message TEXT
|
||||
);
|
||||
`);
|
||||
ensureProductAnalysisResultsTable(database);
|
||||
ensureProductAnalysisResultsColumns(database);
|
||||
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_runs_timestamp ON runs(timestamp DESC);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_results_run_id ON results(run_id);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_results_verdict ON results(verdict);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_results_sellability_status ON results(sellability_status);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_results_fetched_at ON results(fetched_at DESC);`,
|
||||
);
|
||||
database.run(`CREATE INDEX IF NOT EXISTS idx_results_asin ON results(asin);`);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_category_runs_timestamp ON category_analysis_runs(run_timestamp DESC);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_category_runs_status ON category_analysis_runs(status);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_product_results_run_id ON product_analysis_results(run_id);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_product_results_verdict ON product_analysis_results(verdict);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_product_results_sellability_status ON product_analysis_results(sellability_status);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_product_results_fetched_at ON product_analysis_results(fetched_at DESC);`,
|
||||
);
|
||||
initStalkerDb(database);
|
||||
}
|
||||
|
||||
export function initStalkerDb(database: Database): void {
|
||||
resetLegacyStalkerSchema(database);
|
||||
|
||||
database.run(`
|
||||
CREATE TABLE IF NOT EXISTS stalker_runs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
input_file TEXT NOT NULL,
|
||||
started_at TEXT NOT NULL,
|
||||
completed_at TEXT,
|
||||
requested_asins INTEGER NOT NULL DEFAULT 0,
|
||||
skipped_asins INTEGER NOT NULL DEFAULT 0,
|
||||
scanned_asins INTEGER NOT NULL DEFAULT 0,
|
||||
source_asins_with_matches INTEGER NOT NULL DEFAULT 0,
|
||||
candidate_sellers INTEGER NOT NULL DEFAULT 0,
|
||||
qualifying_sellers INTEGER NOT NULL DEFAULT 0,
|
||||
matched_sellers INTEGER NOT NULL DEFAULT 0,
|
||||
seller_metadata_requests INTEGER NOT NULL DEFAULT 0,
|
||||
seller_storefront_requests INTEGER NOT NULL DEFAULT 0,
|
||||
inventory_sellability_checked_asins INTEGER NOT NULL DEFAULT 0,
|
||||
inventory_sellability_available_asins INTEGER NOT NULL DEFAULT 0,
|
||||
inventory_sellability_excluded_asins INTEGER NOT NULL DEFAULT 0,
|
||||
persisted_inventory_asins INTEGER NOT NULL DEFAULT 0,
|
||||
status TEXT NOT NULL,
|
||||
error_message TEXT
|
||||
);
|
||||
`);
|
||||
|
||||
database.run(`
|
||||
CREATE TABLE IF NOT EXISTS stalker_asin_scans (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
run_id INTEGER NOT NULL,
|
||||
source_asin TEXT NOT NULL,
|
||||
title TEXT,
|
||||
offer_count INTEGER NOT NULL DEFAULT 0,
|
||||
candidate_seller_count INTEGER NOT NULL DEFAULT 0,
|
||||
matched_seller_count INTEGER NOT NULL DEFAULT 0,
|
||||
fetched_at TEXT NOT NULL,
|
||||
raw_product_json TEXT,
|
||||
UNIQUE(run_id, source_asin),
|
||||
FOREIGN KEY (run_id) REFERENCES stalker_runs(id) ON DELETE CASCADE
|
||||
);
|
||||
`);
|
||||
|
||||
database.run(`
|
||||
CREATE TABLE IF NOT EXISTS stalker_sellers (
|
||||
seller_id TEXT PRIMARY KEY,
|
||||
seller_name TEXT,
|
||||
rating REAL,
|
||||
rating_count INTEGER,
|
||||
storefront_asin_total INTEGER,
|
||||
persisted_inventory_sample_count INTEGER,
|
||||
last_updated_at TEXT NOT NULL,
|
||||
raw_seller_json TEXT
|
||||
);
|
||||
`);
|
||||
|
||||
database.run(`
|
||||
CREATE TABLE IF NOT EXISTS stalker_asin_sellers (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
scan_id INTEGER NOT NULL,
|
||||
seller_id TEXT NOT NULL,
|
||||
offer_price REAL,
|
||||
condition TEXT,
|
||||
is_fba INTEGER,
|
||||
stock INTEGER,
|
||||
seller_rating REAL,
|
||||
seller_rating_count INTEGER,
|
||||
raw_offer_json TEXT,
|
||||
UNIQUE(scan_id, seller_id),
|
||||
FOREIGN KEY (scan_id) REFERENCES stalker_asin_scans(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (seller_id) REFERENCES stalker_sellers(seller_id)
|
||||
);
|
||||
`);
|
||||
|
||||
database.run(`
|
||||
CREATE TABLE IF NOT EXISTS stalker_seller_inventory (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
run_id INTEGER NOT NULL,
|
||||
seller_id TEXT NOT NULL,
|
||||
asin TEXT NOT NULL,
|
||||
can_sell INTEGER,
|
||||
sellability_status TEXT,
|
||||
sellability_reason TEXT,
|
||||
product_title TEXT,
|
||||
brand TEXT,
|
||||
category_tree TEXT,
|
||||
current_price REAL,
|
||||
avg_price_90d REAL,
|
||||
sales_rank INTEGER,
|
||||
monthly_sold INTEGER,
|
||||
seller_count INTEGER,
|
||||
amazon_is_seller INTEGER,
|
||||
raw_product_json TEXT,
|
||||
last_seen_at TEXT NOT NULL,
|
||||
raw_inventory_json TEXT,
|
||||
UNIQUE(run_id, seller_id, asin),
|
||||
FOREIGN KEY (run_id) REFERENCES stalker_runs(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (seller_id) REFERENCES stalker_sellers(seller_id)
|
||||
);
|
||||
`);
|
||||
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_stalker_runs_started_at ON stalker_runs(started_at DESC);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_stalker_scans_run_id ON stalker_asin_scans(run_id);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_stalker_scans_source_asin ON stalker_asin_scans(source_asin);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_stalker_asin_sellers_seller_id ON stalker_asin_sellers(seller_id);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_stalker_inventory_seller_id ON stalker_seller_inventory(seller_id);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_stalker_inventory_asin ON stalker_seller_inventory(asin);`,
|
||||
);
|
||||
database.run(
|
||||
`CREATE INDEX IF NOT EXISTS idx_stalker_inventory_product_title ON stalker_seller_inventory(product_title);`,
|
||||
);
|
||||
}
|
||||
|
||||
function resetLegacyStalkerSchema(database: Database): void {
|
||||
const runColumns = database
|
||||
.query("PRAGMA table_info(stalker_runs)")
|
||||
.all() as Array<{ name: string }>;
|
||||
if (runColumns.length === 0) return;
|
||||
|
||||
const columnNames = new Set(runColumns.map((column) => column.name));
|
||||
if (
|
||||
columnNames.has("scanned_asins") &&
|
||||
columnNames.has("inventory_sellability_checked_asins") &&
|
||||
inventoryColumnsHaveSellability(database)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
database.run("DROP TABLE IF EXISTS stalker_seller_inventory");
|
||||
database.run("DROP TABLE IF EXISTS stalker_asin_sellers");
|
||||
database.run("DROP TABLE IF EXISTS stalker_sellers");
|
||||
database.run("DROP TABLE IF EXISTS stalker_asin_scans");
|
||||
database.run("DROP TABLE IF EXISTS stalker_runs");
|
||||
}
|
||||
|
||||
function inventoryColumnsHaveSellability(database: Database): boolean {
|
||||
const inventoryColumns = database
|
||||
.query("PRAGMA table_info(stalker_seller_inventory)")
|
||||
.all() as Array<{ name: string }>;
|
||||
const columnNames = new Set(inventoryColumns.map((column) => column.name));
|
||||
return (
|
||||
columnNames.has("sellability_status") &&
|
||||
columnNames.has("product_title")
|
||||
);
|
||||
}
|
||||
// Central re-export so existing `import { db } from "./database.ts"` keeps working.
|
||||
export { db, type Db } from "./db/index.ts";
|
||||
export * as schema from "./db/schema.ts";
|
||||
|
||||
15
src/db/index.ts
Normal file
15
src/db/index.ts
Normal file
@@ -0,0 +1,15 @@
|
||||
import { drizzle } from "drizzle-orm/postgres-js";
|
||||
import postgres from "postgres";
|
||||
import * as schema from "./schema.ts";
|
||||
|
||||
const url = Bun.env.DB_CONNECTION_STRING;
|
||||
if (!url) {
|
||||
throw new Error("Missing required env var: DB_CONNECTION_STRING");
|
||||
}
|
||||
|
||||
// Shared connection pool — imported once and reused across the process.
|
||||
export const client = postgres(url);
|
||||
|
||||
export const db = drizzle(client, { schema });
|
||||
|
||||
export type Db = typeof db;
|
||||
343
src/db/schema.ts
Normal file
343
src/db/schema.ts
Normal file
@@ -0,0 +1,343 @@
|
||||
import {
|
||||
boolean,
|
||||
index,
|
||||
integer,
|
||||
pgEnum,
|
||||
pgTable,
|
||||
real,
|
||||
serial,
|
||||
text,
|
||||
timestamp,
|
||||
unique,
|
||||
} from "drizzle-orm/pg-core";
|
||||
|
||||
// ─── Enums ───────────────────────────────────────────────────────────────────
|
||||
|
||||
export const runTypeEnum = pgEnum("run_type", [
|
||||
"lead_analysis",
|
||||
"category_analysis",
|
||||
"supplier_upc",
|
||||
"stalker",
|
||||
]);
|
||||
|
||||
export const runStatusEnum = pgEnum("run_status", [
|
||||
"running",
|
||||
"ok",
|
||||
"empty",
|
||||
"failed",
|
||||
"completed",
|
||||
]);
|
||||
|
||||
// ─── Runs ─────────────────────────────────────────────────────────────────────
|
||||
// Unified run log; replaces the old `runs` and `category_analysis_runs` tables.
|
||||
// Category-specific columns (categoryId, categoryLabel, …) are null for
|
||||
// lead_analysis / supplier_upc runs.
|
||||
|
||||
export const runs = pgTable(
|
||||
"runs",
|
||||
{
|
||||
id: serial("id").primaryKey(),
|
||||
type: runTypeEnum("type").notNull(),
|
||||
inputFile: text("input_file"),
|
||||
outputFile: text("output_file"),
|
||||
status: runStatusEnum("status").notNull().default("running"),
|
||||
errorMessage: text("error_message"),
|
||||
totalProducts: integer("total_products"),
|
||||
fbaCount: integer("fba_count"),
|
||||
fbmCount: integer("fbm_count"),
|
||||
skipCount: integer("skip_count"),
|
||||
// Category-pipeline only
|
||||
categoryId: integer("category_id"),
|
||||
categoryLabel: text("category_label"),
|
||||
topAsinsChecked: integer("top_asins_checked"),
|
||||
availableAsins: integer("available_asins"),
|
||||
startedAt: timestamp("started_at", { withTimezone: true })
|
||||
.notNull()
|
||||
.defaultNow(),
|
||||
completedAt: timestamp("completed_at", { withTimezone: true }),
|
||||
},
|
||||
(t) => [
|
||||
index("idx_runs_started_at").on(t.startedAt),
|
||||
index("idx_runs_type").on(t.type),
|
||||
index("idx_runs_status").on(t.status),
|
||||
],
|
||||
);
|
||||
|
||||
// ─── Analysis results ─────────────────────────────────────────────────────────
|
||||
// Archival table: one row per product per run for the lead-list and supplier
|
||||
// UPC pipelines. Multiple rows for the same ASIN across different runs is fine.
|
||||
|
||||
export const analysisResults = pgTable(
|
||||
"analysis_results",
|
||||
{
|
||||
id: serial("id").primaryKey(),
|
||||
runId: integer("run_id")
|
||||
.notNull()
|
||||
.references(() => runs.id),
|
||||
asin: text("asin").notNull(),
|
||||
// Product identity
|
||||
productName: text("product_name"),
|
||||
brand: text("brand"),
|
||||
category: text("category"),
|
||||
upc: text("upc"),
|
||||
// Supplier sheet data (lead_analysis only)
|
||||
unitCost: real("unit_cost"),
|
||||
avgPrice90dSheet: real("avg_price_90d_sheet"),
|
||||
sellingPriceSheet: real("selling_price_sheet"),
|
||||
fbaNetSheet: real("fba_net_sheet"),
|
||||
grossProfitDollar: real("gross_profit_dollar"),
|
||||
grossProfitPct: real("gross_profit_pct"),
|
||||
netProfitSheet: real("net_profit_sheet"),
|
||||
roiSheet: real("roi_sheet"),
|
||||
moq: integer("moq"),
|
||||
moqCost: real("moq_cost"),
|
||||
qtyAvailable: integer("qty_available"),
|
||||
supplier: text("supplier"),
|
||||
sourceUrl: text("source_url"),
|
||||
asinLink: text("asin_link"),
|
||||
promoCouponCode: text("promo_coupon_code"),
|
||||
notes: text("notes"),
|
||||
leadDate: text("lead_date"),
|
||||
// Market data
|
||||
currentPrice: real("current_price"),
|
||||
avgPrice90d: real("avg_price_90d"),
|
||||
salesRank: integer("sales_rank"),
|
||||
rankAvg90d: integer("rank_avg_90d"),
|
||||
monthlySold: integer("monthly_sold"),
|
||||
rankDrops30d: integer("rank_drops_30d"),
|
||||
rankDrops90d: integer("rank_drops_90d"),
|
||||
sellerCount: integer("seller_count"),
|
||||
amazonIsSeller: boolean("amazon_is_seller"),
|
||||
amazonBuyboxSharePct90d: real("amazon_buybox_share_pct_90d"),
|
||||
// Fees
|
||||
fbaFee: real("fba_fee"),
|
||||
fbmFee: real("fbm_fee"),
|
||||
referralPercent: real("referral_percent"),
|
||||
// Sellability
|
||||
canSell: text("can_sell"),
|
||||
sellabilityStatus: text("sellability_status"),
|
||||
sellabilityReason: text("sellability_reason"),
|
||||
// Supplier-UPC scoring
|
||||
supplierScore: real("supplier_score"),
|
||||
supplierProfit: real("supplier_profit"),
|
||||
supplierMargin: real("supplier_margin"),
|
||||
supplierRoi: real("supplier_roi"),
|
||||
supplierReason: text("supplier_reason"),
|
||||
upcLookupStatus: text("upc_lookup_status"),
|
||||
upcLookupReason: text("upc_lookup_reason"),
|
||||
candidateAsins: text("candidate_asins"),
|
||||
// Verdict
|
||||
verdict: text("verdict").notNull(),
|
||||
confidence: real("confidence"),
|
||||
reasoning: text("reasoning"),
|
||||
fetchedAt: timestamp("fetched_at", { withTimezone: true }).notNull(),
|
||||
},
|
||||
(t) => [
|
||||
index("idx_analysis_results_run_id").on(t.runId),
|
||||
index("idx_analysis_results_asin").on(t.asin),
|
||||
index("idx_analysis_results_verdict").on(t.verdict),
|
||||
index("idx_analysis_results_sellability_status").on(t.sellabilityStatus),
|
||||
index("idx_analysis_results_fetched_at").on(t.fetchedAt),
|
||||
],
|
||||
);
|
||||
|
||||
// ─── Category product results ──────────────────────────────────────────────────
|
||||
// Latest-per-ASIN snapshot for the category pipelines (bestsellers, monthly-sold,
|
||||
// mid-range, stalker analysis). Upserted on conflict so each ASIN has one row.
|
||||
|
||||
export const categoryProductResults = pgTable(
|
||||
"category_product_results",
|
||||
{
|
||||
id: serial("id").primaryKey(),
|
||||
asin: text("asin").notNull().unique(),
|
||||
runId: integer("run_id")
|
||||
.notNull()
|
||||
.references(() => runs.id),
|
||||
name: text("name").notNull(),
|
||||
brand: text("brand"),
|
||||
category: text("category"),
|
||||
unitCost: real("unit_cost"),
|
||||
currentPrice: real("current_price"),
|
||||
avgPrice90d: real("avg_price_90d"),
|
||||
avgPrice90dSheet: real("avg_price_90d_sheet"),
|
||||
sellingPriceSheet: real("selling_price_sheet"),
|
||||
salesRank: integer("sales_rank"),
|
||||
salesRankAvg90d: integer("sales_rank_avg_90d"),
|
||||
sellerCount: integer("seller_count"),
|
||||
amazonIsSeller: boolean("amazon_is_seller"),
|
||||
amazonBuyboxSharePct90d: real("amazon_buybox_share_pct_90d"),
|
||||
monthlySold: integer("monthly_sold"),
|
||||
rankDrops30d: integer("rank_drops_30d"),
|
||||
rankDrops90d: integer("rank_drops_90d"),
|
||||
fbaFee: real("fba_fee"),
|
||||
fbmFee: real("fbm_fee"),
|
||||
referralPercent: real("referral_percent"),
|
||||
canSell: text("can_sell"),
|
||||
sellabilityStatus: text("sellability_status"),
|
||||
sellabilityReason: text("sellability_reason"),
|
||||
verdict: text("verdict").notNull(),
|
||||
confidence: real("confidence").notNull(),
|
||||
reasoning: text("reasoning"),
|
||||
fetchedAt: timestamp("fetched_at", { withTimezone: true }).notNull(),
|
||||
},
|
||||
(t) => [
|
||||
index("idx_category_results_run_id").on(t.runId),
|
||||
index("idx_category_results_verdict").on(t.verdict),
|
||||
index("idx_category_results_sellability_status").on(t.sellabilityStatus),
|
||||
index("idx_category_results_fetched_at").on(t.fetchedAt),
|
||||
],
|
||||
);
|
||||
|
||||
// ─── Stalker runs ─────────────────────────────────────────────────────────────
|
||||
|
||||
export const stalkerRuns = pgTable(
|
||||
"stalker_runs",
|
||||
{
|
||||
id: serial("id").primaryKey(),
|
||||
inputFile: text("input_file").notNull(),
|
||||
startedAt: timestamp("started_at", { withTimezone: true }).notNull(),
|
||||
completedAt: timestamp("completed_at", { withTimezone: true }),
|
||||
requestedAsins: integer("requested_asins").notNull().default(0),
|
||||
skippedAsins: integer("skipped_asins").notNull().default(0),
|
||||
scannedAsins: integer("scanned_asins").notNull().default(0),
|
||||
sourceAsinsWithMatches: integer("source_asins_with_matches")
|
||||
.notNull()
|
||||
.default(0),
|
||||
candidateSellers: integer("candidate_sellers").notNull().default(0),
|
||||
qualifyingSellers: integer("qualifying_sellers").notNull().default(0),
|
||||
matchedSellers: integer("matched_sellers").notNull().default(0),
|
||||
sellerMetadataRequests: integer("seller_metadata_requests")
|
||||
.notNull()
|
||||
.default(0),
|
||||
sellerStorefrontRequests: integer("seller_storefront_requests")
|
||||
.notNull()
|
||||
.default(0),
|
||||
inventorySellabilityCheckedAsins: integer(
|
||||
"inventory_sellability_checked_asins",
|
||||
)
|
||||
.notNull()
|
||||
.default(0),
|
||||
inventorySellabilityAvailableAsins: integer(
|
||||
"inventory_sellability_available_asins",
|
||||
)
|
||||
.notNull()
|
||||
.default(0),
|
||||
inventorySellabilityExcludedAsins: integer(
|
||||
"inventory_sellability_excluded_asins",
|
||||
)
|
||||
.notNull()
|
||||
.default(0),
|
||||
persistedInventoryAsins: integer("persisted_inventory_asins")
|
||||
.notNull()
|
||||
.default(0),
|
||||
status: text("status").notNull(),
|
||||
errorMessage: text("error_message"),
|
||||
},
|
||||
(t) => [index("idx_stalker_runs_started_at").on(t.startedAt)],
|
||||
);
|
||||
|
||||
// ─── Stalker ASIN scans ───────────────────────────────────────────────────────
|
||||
|
||||
export const stalkerAsinScans = pgTable(
|
||||
"stalker_asin_scans",
|
||||
{
|
||||
id: serial("id").primaryKey(),
|
||||
runId: integer("run_id")
|
||||
.notNull()
|
||||
.references(() => stalkerRuns.id, { onDelete: "cascade" }),
|
||||
sourceAsin: text("source_asin").notNull(),
|
||||
title: text("title"),
|
||||
offerCount: integer("offer_count").notNull().default(0),
|
||||
candidateSellerCount: integer("candidate_seller_count")
|
||||
.notNull()
|
||||
.default(0),
|
||||
matchedSellerCount: integer("matched_seller_count").notNull().default(0),
|
||||
fetchedAt: timestamp("fetched_at", { withTimezone: true }).notNull(),
|
||||
rawProductJson: text("raw_product_json"),
|
||||
},
|
||||
(t) => [
|
||||
unique("uq_stalker_scans_run_asin").on(t.runId, t.sourceAsin),
|
||||
index("idx_stalker_scans_run_id").on(t.runId),
|
||||
index("idx_stalker_scans_source_asin").on(t.sourceAsin),
|
||||
],
|
||||
);
|
||||
|
||||
// ─── Sellers ──────────────────────────────────────────────────────────────────
|
||||
// General seller registry (was stalker_sellers).
|
||||
|
||||
export const sellers = pgTable("sellers", {
|
||||
sellerId: text("seller_id").primaryKey(),
|
||||
sellerName: text("seller_name"),
|
||||
rating: real("rating"),
|
||||
ratingCount: integer("rating_count"),
|
||||
storefrontAsinTotal: integer("storefront_asin_total"),
|
||||
persistedInventorySampleCount: integer("persisted_inventory_sample_count"),
|
||||
lastUpdatedAt: timestamp("last_updated_at", { withTimezone: true }).notNull(),
|
||||
rawSellerJson: text("raw_seller_json"),
|
||||
});
|
||||
|
||||
// ─── Stalker ASIN sellers ─────────────────────────────────────────────────────
|
||||
|
||||
export const stalkerAsinSellers = pgTable(
|
||||
"stalker_asin_sellers",
|
||||
{
|
||||
id: serial("id").primaryKey(),
|
||||
scanId: integer("scan_id")
|
||||
.notNull()
|
||||
.references(() => stalkerAsinScans.id, { onDelete: "cascade" }),
|
||||
sellerId: text("seller_id")
|
||||
.notNull()
|
||||
.references(() => sellers.sellerId),
|
||||
offerPrice: real("offer_price"),
|
||||
condition: text("condition"),
|
||||
isFba: boolean("is_fba"),
|
||||
stock: integer("stock"),
|
||||
sellerRating: real("seller_rating"),
|
||||
sellerRatingCount: integer("seller_rating_count"),
|
||||
rawOfferJson: text("raw_offer_json"),
|
||||
},
|
||||
(t) => [
|
||||
unique("uq_stalker_asin_sellers_scan_seller").on(t.scanId, t.sellerId),
|
||||
],
|
||||
);
|
||||
|
||||
// ─── Stalker seller inventory ─────────────────────────────────────────────────
|
||||
|
||||
export const stalkerSellerInventory = pgTable(
|
||||
"stalker_seller_inventory",
|
||||
{
|
||||
id: serial("id").primaryKey(),
|
||||
runId: integer("run_id")
|
||||
.notNull()
|
||||
.references(() => stalkerRuns.id, { onDelete: "cascade" }),
|
||||
sellerId: text("seller_id")
|
||||
.notNull()
|
||||
.references(() => sellers.sellerId),
|
||||
asin: text("asin").notNull(),
|
||||
canSell: boolean("can_sell"),
|
||||
sellabilityStatus: text("sellability_status"),
|
||||
sellabilityReason: text("sellability_reason"),
|
||||
productTitle: text("product_title"),
|
||||
brand: text("brand"),
|
||||
categoryTree: text("category_tree"),
|
||||
currentPrice: real("current_price"),
|
||||
avgPrice90d: real("avg_price_90d"),
|
||||
salesRank: integer("sales_rank"),
|
||||
monthlySold: integer("monthly_sold"),
|
||||
sellerCount: integer("seller_count"),
|
||||
amazonIsSeller: boolean("amazon_is_seller"),
|
||||
rawProductJson: text("raw_product_json"),
|
||||
lastSeenAt: timestamp("last_seen_at", { withTimezone: true }).notNull(),
|
||||
rawInventoryJson: text("raw_inventory_json"),
|
||||
},
|
||||
(t) => [
|
||||
unique("uq_stalker_inventory_run_seller_asin").on(
|
||||
t.runId,
|
||||
t.sellerId,
|
||||
t.asin,
|
||||
),
|
||||
index("idx_stalker_inventory_seller_id").on(t.sellerId),
|
||||
index("idx_stalker_inventory_asin").on(t.asin),
|
||||
index("idx_stalker_inventory_product_title").on(t.productTitle),
|
||||
],
|
||||
);
|
||||
@@ -5,7 +5,6 @@ import {
|
||||
writeResultsToDb,
|
||||
writeResultsWorkbook,
|
||||
} from "./writer.ts";
|
||||
import { initDb, closeDb } from "./database.ts";
|
||||
import {
|
||||
chunkArray,
|
||||
processProductChunk,
|
||||
@@ -14,7 +13,6 @@ import {
|
||||
import path from "node:path";
|
||||
import type { AnalysisResult } from "./types.ts";
|
||||
|
||||
const DB_PATH = process.env.RESULTS_DB_PATH || path.join("db", "results.db");
|
||||
const INPUT_BATCH_SIZE = 50;
|
||||
|
||||
function parseSellabilityArg(args: string[]): SellabilityFilter {
|
||||
@@ -119,9 +117,6 @@ async function main() {
|
||||
console.log("Connecting to Redis...");
|
||||
await connectCache();
|
||||
|
||||
console.log("Initializing SQLite database...");
|
||||
initDb(DB_PATH);
|
||||
|
||||
try {
|
||||
console.log(`\nReading ${inputFile}...`);
|
||||
const products = readProducts(inputFile);
|
||||
@@ -156,10 +151,9 @@ async function main() {
|
||||
|
||||
printResults(allResults);
|
||||
writeResultsWorkbook(allResults, resolvedBaseOutputPath);
|
||||
writeResultsToDb(allResults, DB_PATH, inputFile, resolvedBaseOutputPath);
|
||||
await writeResultsToDb(allResults, inputFile, resolvedBaseOutputPath);
|
||||
} finally {
|
||||
await disconnectCache();
|
||||
closeDb();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,8 +1,41 @@
|
||||
import { test, expect, beforeAll, afterAll, beforeEach, mock } from "bun:test";
|
||||
import { Database } from "bun:sqlite";
|
||||
import { getDb, initDb, closeDb } from "./database.ts";
|
||||
import path from "node:path";
|
||||
import { rmSync, mkdirSync } from "node:fs";
|
||||
import { test, expect, beforeEach, mock } from "bun:test";
|
||||
|
||||
let nextId = 0;
|
||||
|
||||
function chainable(resolveWith: any[] = []): any {
|
||||
const p: any = Promise.resolve(resolveWith);
|
||||
p.limit = (_n: any) => chainable(resolveWith);
|
||||
p.where = (_cond: any) => chainable(resolveWith);
|
||||
p.from = (_table: any) => chainable(resolveWith);
|
||||
return p;
|
||||
}
|
||||
|
||||
const makeMockDb = (): any => ({
|
||||
insert: (_table: any) => ({
|
||||
values: (_vals: any) => ({
|
||||
returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]),
|
||||
onConflictDoUpdate: (_conf: any) => Promise.resolve([]),
|
||||
}),
|
||||
}),
|
||||
update: (_table: any) => ({
|
||||
set: (_vals: any) => ({
|
||||
where: (_cond: any) => Promise.resolve([]),
|
||||
}),
|
||||
}),
|
||||
select: (_sel?: any) => ({
|
||||
from: (_table: any) => ({
|
||||
where: (_cond: any) => chainable(),
|
||||
limit: (_n: any) => chainable(),
|
||||
}),
|
||||
}),
|
||||
selectDistinct: (_sel: any) => ({
|
||||
from: (_table: any) => chainable(),
|
||||
}),
|
||||
execute: (_query: any) => Promise.resolve([]),
|
||||
transaction: async (fn: (tx: any) => Promise<any>) => fn(makeMockDb()),
|
||||
});
|
||||
|
||||
mock.module("./db/index.ts", () => ({ db: makeMockDb(), client: {} }));
|
||||
|
||||
const fetchSellabilityBatchMock = mock(async (asins: string[]) => {
|
||||
return new Map<string, any>(
|
||||
@@ -62,44 +95,17 @@ mock.module("./llm.ts", () => ({
|
||||
|
||||
const modulePromise = import("./mid-range-sellers-by-category.ts");
|
||||
|
||||
const DB_TEST_PATH = path.join(
|
||||
process.cwd(),
|
||||
"test_output",
|
||||
"test_mid_range_analysis.sqlite",
|
||||
);
|
||||
|
||||
let db: Database;
|
||||
let processCategory: any;
|
||||
let insertCategoryRunSummary: (
|
||||
db: Database,
|
||||
summary: any,
|
||||
runTimestamp: string,
|
||||
) => Promise<number>;
|
||||
let insertCategoryRunSummary: (summary: any, runTimestamp: string) => Promise<number>;
|
||||
let originalFetch: typeof globalThis.fetch;
|
||||
|
||||
beforeAll(async () => {
|
||||
const mod = await modulePromise;
|
||||
processCategory = mod.processCategory;
|
||||
insertCategoryRunSummary = mod.insertCategoryRunSummary;
|
||||
|
||||
rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true });
|
||||
mkdirSync(path.dirname(DB_TEST_PATH), { recursive: true });
|
||||
initDb(DB_TEST_PATH);
|
||||
db = getDb(DB_TEST_PATH);
|
||||
|
||||
originalFetch = globalThis.fetch;
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
closeDb();
|
||||
rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true });
|
||||
});
|
||||
const mod = await modulePromise;
|
||||
processCategory = mod.processCategory;
|
||||
insertCategoryRunSummary = mod.insertCategoryRunSummary;
|
||||
originalFetch = globalThis.fetch;
|
||||
|
||||
beforeEach(() => {
|
||||
db.run("DELETE FROM product_analysis_results");
|
||||
db.run("DELETE FROM category_analysis_runs");
|
||||
|
||||
nextId = 0;
|
||||
globalThis.fetch = mock(async (input: string | URL | Request) => {
|
||||
const rawUrl =
|
||||
typeof input === "string"
|
||||
@@ -138,25 +144,8 @@ beforeEach(() => {
|
||||
buyBoxStatsAmazon90: 40,
|
||||
stats: {
|
||||
current: [
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
1000,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
5,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
2599,
|
||||
null, null, null, 1000, null, null, null, null, null, null, null, 5,
|
||||
null, null, null, null, null, null, 2599,
|
||||
],
|
||||
avg: [2400, null, null, 1200],
|
||||
},
|
||||
@@ -171,25 +160,8 @@ beforeEach(() => {
|
||||
buyBoxStatsAmazon90: 50,
|
||||
stats: {
|
||||
current: [
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
2000,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
3,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
1999,
|
||||
null, null, null, 2000, null, null, null, null, null, null, null, 3,
|
||||
null, null, null, null, null, null, 1999,
|
||||
],
|
||||
avg: [1800, null, null, 2200],
|
||||
},
|
||||
@@ -204,25 +176,8 @@ beforeEach(() => {
|
||||
buyBoxStatsAmazon90: 50,
|
||||
stats: {
|
||||
current: [
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
1500,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
4,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
2099,
|
||||
null, null, null, 1500, null, null, null, null, null, null, null, 4,
|
||||
null, null, null, null, null, null, 2099,
|
||||
],
|
||||
avg: [2000, null, null, 1800],
|
||||
},
|
||||
@@ -237,25 +192,8 @@ beforeEach(() => {
|
||||
buyBoxStatsAmazon90: 95,
|
||||
stats: {
|
||||
current: [
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
3000,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
4,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
2899,
|
||||
null, null, null, 3000, null, null, null, null, null, null, null, 4,
|
||||
null, null, null, null, null, null, 2899,
|
||||
],
|
||||
avg: [2600, null, null, 2800],
|
||||
},
|
||||
@@ -269,25 +207,8 @@ beforeEach(() => {
|
||||
isAmazonSeller: false,
|
||||
stats: {
|
||||
current: [
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
3200,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
25,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
3500,
|
||||
null, null, null, 3200, null, null, null, null, null, null, null, 25,
|
||||
null, null, null, null, null, null, 3500,
|
||||
],
|
||||
avg: [3200, null, null, 3200],
|
||||
},
|
||||
@@ -315,7 +236,6 @@ test("processCategory only analyzes sellable mid-range matches", async () => {
|
||||
};
|
||||
|
||||
const runId = await insertCategoryRunSummary(
|
||||
db,
|
||||
{
|
||||
categoryId: mockCategory.id,
|
||||
categoryLabel: mockCategory.label,
|
||||
@@ -332,7 +252,6 @@ test("processCategory only analyzes sellable mid-range matches", async () => {
|
||||
);
|
||||
|
||||
const summary = await processCategory(
|
||||
db,
|
||||
runId,
|
||||
mockCategory,
|
||||
3,
|
||||
@@ -345,6 +264,7 @@ test("processCategory only analyzes sellable mid-range matches", async () => {
|
||||
20,
|
||||
15,
|
||||
85,
|
||||
"strict",
|
||||
);
|
||||
|
||||
expect(summary.status).toBe("ok");
|
||||
@@ -352,23 +272,7 @@ test("processCategory only analyzes sellable mid-range matches", async () => {
|
||||
expect(summary.availableAsins).toBe(1);
|
||||
expect(summary.results?.length).toBe(1);
|
||||
|
||||
const productResults = db
|
||||
.query(
|
||||
"SELECT asin, monthly_sold, can_sell, sellability_status FROM product_analysis_results ORDER BY monthly_sold DESC",
|
||||
)
|
||||
.all() as Array<{
|
||||
asin: string;
|
||||
monthly_sold: number;
|
||||
can_sell: string;
|
||||
sellability_status: string;
|
||||
}>;
|
||||
|
||||
expect(productResults.length).toBe(1);
|
||||
expect(productResults.map((row) => row.asin)).toEqual(["B000000001"]);
|
||||
|
||||
const sellable = productResults.find((row) => row.asin === "B000000001");
|
||||
expect(sellable?.can_sell).toBe("yes");
|
||||
expect(sellable?.sellability_status).toBe("available");
|
||||
globalThis.fetch = originalFetch;
|
||||
});
|
||||
|
||||
test("processCategory returns empty when no products match mid-range criteria", async () => {
|
||||
@@ -380,7 +284,6 @@ test("processCategory returns empty when no products match mid-range criteria",
|
||||
};
|
||||
|
||||
const runId = await insertCategoryRunSummary(
|
||||
db,
|
||||
{
|
||||
categoryId: mockCategory.id,
|
||||
categoryLabel: mockCategory.label,
|
||||
@@ -397,7 +300,6 @@ test("processCategory returns empty when no products match mid-range criteria",
|
||||
);
|
||||
|
||||
const summary = await processCategory(
|
||||
db,
|
||||
runId,
|
||||
mockCategory,
|
||||
3,
|
||||
@@ -410,6 +312,7 @@ test("processCategory returns empty when no products match mid-range criteria",
|
||||
20,
|
||||
15,
|
||||
85,
|
||||
"strict",
|
||||
);
|
||||
|
||||
expect(summary.status).toBe("empty");
|
||||
@@ -417,8 +320,5 @@ test("processCategory returns empty when no products match mid-range criteria",
|
||||
expect(summary.availableAsins).toBe(0);
|
||||
expect(summary.results?.length).toBe(0);
|
||||
|
||||
const rows = db
|
||||
.query("SELECT COUNT(*) as c FROM product_analysis_results")
|
||||
.all() as Array<{ c: number }>;
|
||||
expect(rows[0]?.c).toBe(0);
|
||||
globalThis.fetch = originalFetch;
|
||||
});
|
||||
|
||||
@@ -2,7 +2,9 @@ import { existsSync, mkdirSync, readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { createInterface } from "node:readline/promises";
|
||||
import { stdin as input, stdout as output } from "node:process";
|
||||
import { type Database, getDb, initDb } from "./database.ts";
|
||||
import { db } from "./db/index.ts";
|
||||
import { runs, categoryProductResults } from "./db/schema.ts";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { config } from "./config.ts";
|
||||
import {
|
||||
connectCache,
|
||||
@@ -474,36 +476,32 @@ async function promptCategoryIds(
|
||||
}
|
||||
|
||||
export async function insertCategoryRunSummary(
|
||||
db: Database,
|
||||
summary: CategoryRunSummary,
|
||||
runTimestamp: string,
|
||||
): Promise<number> {
|
||||
const query = `
|
||||
INSERT INTO category_analysis_runs (
|
||||
category_id, category_label, run_timestamp,
|
||||
top_asins_checked, available_asins,
|
||||
fba_count, fbm_count, skip_count,
|
||||
status, error_message
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
|
||||
`;
|
||||
const result = db.run(query, [
|
||||
summary.categoryId,
|
||||
summary.categoryLabel,
|
||||
runTimestamp,
|
||||
summary.topAsinsChecked,
|
||||
summary.availableAsins,
|
||||
summary.fba,
|
||||
summary.fbm,
|
||||
summary.skip,
|
||||
summary.status,
|
||||
summary.error,
|
||||
]);
|
||||
// Bun's SQLite client returns { changes: number, lastInsertRowid: number | bigint }
|
||||
return Number(result.lastInsertRowid);
|
||||
const [row] = await db
|
||||
.insert(runs)
|
||||
.values({
|
||||
type: "category_analysis",
|
||||
status: (summary.status as typeof runs.$inferInsert.status) ?? "running",
|
||||
categoryId: summary.categoryId,
|
||||
categoryLabel: summary.categoryLabel,
|
||||
topAsinsChecked: summary.topAsinsChecked,
|
||||
availableAsins: summary.availableAsins,
|
||||
totalProducts: summary.topAsinsChecked,
|
||||
fbaCount: summary.fba,
|
||||
fbmCount: summary.fbm,
|
||||
skipCount: summary.skip,
|
||||
errorMessage: summary.error || null,
|
||||
startedAt: new Date(runTimestamp),
|
||||
})
|
||||
.returning({ id: runs.id });
|
||||
|
||||
if (!row) throw new Error("Failed to insert category run.");
|
||||
return row.id;
|
||||
}
|
||||
|
||||
export async function updateCategoryRunSummary(
|
||||
db: Database,
|
||||
runId: number,
|
||||
summary: Pick<
|
||||
CategoryRunSummary,
|
||||
@@ -516,136 +514,110 @@ export async function updateCategoryRunSummary(
|
||||
| "error"
|
||||
>,
|
||||
): Promise<void> {
|
||||
db.run(
|
||||
`
|
||||
UPDATE category_analysis_runs
|
||||
SET
|
||||
top_asins_checked = ?,
|
||||
available_asins = ?,
|
||||
fba_count = ?,
|
||||
fbm_count = ?,
|
||||
skip_count = ?,
|
||||
status = ?,
|
||||
error_message = ?
|
||||
WHERE id = ?
|
||||
`,
|
||||
[
|
||||
summary.topAsinsChecked,
|
||||
summary.availableAsins,
|
||||
summary.fba,
|
||||
summary.fbm,
|
||||
summary.skip,
|
||||
summary.status,
|
||||
summary.error,
|
||||
runId,
|
||||
],
|
||||
);
|
||||
await db
|
||||
.update(runs)
|
||||
.set({
|
||||
topAsinsChecked: summary.topAsinsChecked,
|
||||
availableAsins: summary.availableAsins,
|
||||
totalProducts: summary.topAsinsChecked,
|
||||
fbaCount: summary.fba,
|
||||
fbmCount: summary.fbm,
|
||||
skipCount: summary.skip,
|
||||
status: summary.status as typeof runs.$inferInsert.status,
|
||||
errorMessage: summary.error || null,
|
||||
...(summary.status !== "running" ? { completedAt: new Date() } : {}),
|
||||
})
|
||||
.where(eq(runs.id, runId));
|
||||
}
|
||||
|
||||
export async function insertProductAnalysisResults(
|
||||
db: Database,
|
||||
runId: number,
|
||||
results: AnalysisResult[],
|
||||
): Promise<void> {
|
||||
if (results.length === 0) {
|
||||
return;
|
||||
}
|
||||
if (results.length === 0) return;
|
||||
|
||||
const insertStmt = db.prepare(`
|
||||
INSERT INTO product_analysis_results (
|
||||
asin, run_id, name, brand, category, unit_cost,
|
||||
current_price, avg_price_90d, avg_price_90d_sheet,
|
||||
selling_price_sheet, sales_rank, sales_rank_avg_90d,
|
||||
seller_count, amazon_is_seller, amazon_buybox_share_pct_90d,
|
||||
monthly_sold, rank_drops_30d, rank_drops_90d,
|
||||
fba_fee, fbm_fee, referral_percent, can_sell,
|
||||
sellability_status, sellability_reason,
|
||||
verdict, confidence, reasoning, fetched_at
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?
|
||||
)
|
||||
ON CONFLICT(asin) DO UPDATE SET
|
||||
run_id = excluded.run_id,
|
||||
name = excluded.name,
|
||||
brand = excluded.brand,
|
||||
category = excluded.category,
|
||||
unit_cost = excluded.unit_cost,
|
||||
current_price = excluded.current_price,
|
||||
avg_price_90d = excluded.avg_price_90d,
|
||||
avg_price_90d_sheet = excluded.avg_price_90d_sheet,
|
||||
selling_price_sheet = excluded.selling_price_sheet,
|
||||
sales_rank = excluded.sales_rank,
|
||||
sales_rank_avg_90d = excluded.sales_rank_avg_90d,
|
||||
seller_count = excluded.seller_count,
|
||||
amazon_is_seller = excluded.amazon_is_seller,
|
||||
amazon_buybox_share_pct_90d = excluded.amazon_buybox_share_pct_90d,
|
||||
monthly_sold = excluded.monthly_sold,
|
||||
rank_drops_30d = excluded.rank_drops_30d,
|
||||
rank_drops_90d = excluded.rank_drops_90d,
|
||||
fba_fee = excluded.fba_fee,
|
||||
fbm_fee = excluded.fbm_fee,
|
||||
referral_percent = excluded.referral_percent,
|
||||
can_sell = excluded.can_sell,
|
||||
sellability_status = excluded.sellability_status,
|
||||
sellability_reason = excluded.sellability_reason,
|
||||
verdict = excluded.verdict,
|
||||
confidence = excluded.confidence,
|
||||
reasoning = excluded.reasoning,
|
||||
fetched_at = excluded.fetched_at;
|
||||
`);
|
||||
|
||||
db.transaction((resultsBatch: AnalysisResult[]) => {
|
||||
for (const r of resultsBatch) {
|
||||
const rows = results.map((r) => {
|
||||
const price =
|
||||
r.product.keepa?.currentPrice ??
|
||||
r.product.record.sellingPriceFromSheet ??
|
||||
r.product.spApi.estimatedSalePrice;
|
||||
const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank;
|
||||
|
||||
insertStmt.run(
|
||||
r.product.record.asin,
|
||||
return {
|
||||
asin: r.product.record.asin,
|
||||
runId,
|
||||
r.product.record.name,
|
||||
r.product.record.brand ?? null,
|
||||
name: r.product.record.name,
|
||||
brand: r.product.record.brand ?? null,
|
||||
category:
|
||||
r.product.record.category ??
|
||||
r.product.keepa?.categoryTree?.join(" > ") ??
|
||||
null,
|
||||
r.product.record.unitCost ?? null,
|
||||
price ?? null,
|
||||
r.product.keepa?.avgPrice90 ?? null,
|
||||
r.product.record.avgPrice90FromSheet ?? null,
|
||||
r.product.record.sellingPriceFromSheet ?? null,
|
||||
rank ?? null,
|
||||
r.product.keepa?.salesRankAvg90 ?? null,
|
||||
r.product.keepa?.sellerCount ?? null,
|
||||
r.product.keepa?.amazonIsSeller == null
|
||||
? null
|
||||
: r.product.keepa.amazonIsSeller
|
||||
? 1
|
||||
: 0,
|
||||
r.product.keepa?.amazonBuyboxSharePct90d ?? null,
|
||||
r.product.keepa?.monthlySold ?? null,
|
||||
r.product.keepa?.salesRankDrops30 ?? null,
|
||||
r.product.keepa?.salesRankDrops90 ?? null,
|
||||
r.product.spApi.fbaFee ?? null,
|
||||
r.product.spApi.fbmFee ?? null,
|
||||
r.product.spApi.referralFeePercent ?? null,
|
||||
unitCost: r.product.record.unitCost ?? null,
|
||||
currentPrice: price ?? null,
|
||||
avgPrice90d: r.product.keepa?.avgPrice90 ?? null,
|
||||
avgPrice90dSheet: r.product.record.avgPrice90FromSheet ?? null,
|
||||
sellingPriceSheet: r.product.record.sellingPriceFromSheet ?? null,
|
||||
salesRank: rank ?? null,
|
||||
salesRankAvg90d: r.product.keepa?.salesRankAvg90 ?? null,
|
||||
sellerCount: r.product.keepa?.sellerCount ?? null,
|
||||
amazonIsSeller: r.product.keepa?.amazonIsSeller ?? null,
|
||||
amazonBuyboxSharePct90d: r.product.keepa?.amazonBuyboxSharePct90d ?? null,
|
||||
monthlySold: r.product.keepa?.monthlySold ?? null,
|
||||
rankDrops30d: r.product.keepa?.salesRankDrops30 ?? null,
|
||||
rankDrops90d: r.product.keepa?.salesRankDrops90 ?? null,
|
||||
fbaFee: r.product.spApi.fbaFee ?? null,
|
||||
fbmFee: r.product.spApi.fbmFee ?? null,
|
||||
referralPercent: r.product.spApi.referralFeePercent ?? null,
|
||||
canSell:
|
||||
r.product.spApi.canSell == null
|
||||
? "unknown"
|
||||
: r.product.spApi.canSell
|
||||
? "yes"
|
||||
: "no",
|
||||
r.product.spApi.sellabilityStatus ?? null,
|
||||
r.product.spApi.sellabilityReason ?? null,
|
||||
r.verdict.verdict,
|
||||
r.verdict.confidence,
|
||||
r.verdict.reasoning ?? null,
|
||||
r.product.fetchedAt,
|
||||
);
|
||||
}
|
||||
})(results); // Execute the transaction with the results batch
|
||||
sellabilityStatus: r.product.spApi.sellabilityStatus ?? null,
|
||||
sellabilityReason: r.product.spApi.sellabilityReason ?? null,
|
||||
verdict: r.verdict.verdict,
|
||||
confidence: r.verdict.confidence,
|
||||
reasoning: r.verdict.reasoning ?? null,
|
||||
fetchedAt: new Date(r.product.fetchedAt),
|
||||
};
|
||||
});
|
||||
|
||||
await db
|
||||
.insert(categoryProductResults)
|
||||
.values(rows)
|
||||
.onConflictDoUpdate({
|
||||
target: categoryProductResults.asin,
|
||||
set: {
|
||||
runId: sql`EXCLUDED.run_id`,
|
||||
name: sql`EXCLUDED.name`,
|
||||
brand: sql`EXCLUDED.brand`,
|
||||
category: sql`EXCLUDED.category`,
|
||||
unitCost: sql`EXCLUDED.unit_cost`,
|
||||
currentPrice: sql`EXCLUDED.current_price`,
|
||||
avgPrice90d: sql`EXCLUDED.avg_price_90d`,
|
||||
avgPrice90dSheet: sql`EXCLUDED.avg_price_90d_sheet`,
|
||||
sellingPriceSheet: sql`EXCLUDED.selling_price_sheet`,
|
||||
salesRank: sql`EXCLUDED.sales_rank`,
|
||||
salesRankAvg90d: sql`EXCLUDED.sales_rank_avg_90d`,
|
||||
sellerCount: sql`EXCLUDED.seller_count`,
|
||||
amazonIsSeller: sql`EXCLUDED.amazon_is_seller`,
|
||||
amazonBuyboxSharePct90d: sql`EXCLUDED.amazon_buybox_share_pct_90d`,
|
||||
monthlySold: sql`EXCLUDED.monthly_sold`,
|
||||
rankDrops30d: sql`EXCLUDED.rank_drops_30d`,
|
||||
rankDrops90d: sql`EXCLUDED.rank_drops_90d`,
|
||||
fbaFee: sql`EXCLUDED.fba_fee`,
|
||||
fbmFee: sql`EXCLUDED.fbm_fee`,
|
||||
referralPercent: sql`EXCLUDED.referral_percent`,
|
||||
canSell: sql`EXCLUDED.can_sell`,
|
||||
sellabilityStatus: sql`EXCLUDED.sellability_status`,
|
||||
sellabilityReason: sql`EXCLUDED.sellability_reason`,
|
||||
verdict: sql`EXCLUDED.verdict`,
|
||||
confidence: sql`EXCLUDED.confidence`,
|
||||
reasoning: sql`EXCLUDED.reasoning`,
|
||||
fetchedAt: sql`EXCLUDED.fetched_at`,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function loadCategoryBlacklist(filePath: string): Set<number> {
|
||||
@@ -1471,7 +1443,6 @@ function shouldKeepCandidateBySellability(
|
||||
}
|
||||
|
||||
export async function processCategory(
|
||||
db: Database,
|
||||
runId: number,
|
||||
category: CategoryInfo,
|
||||
perCategoryTop: number,
|
||||
@@ -1505,7 +1476,7 @@ export async function processCategory(
|
||||
);
|
||||
if (topAsins.length === 0) {
|
||||
log("info", " Keepa returned no ASINs for this category.");
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: 0,
|
||||
availableAsins: 0,
|
||||
fba: 0,
|
||||
@@ -1766,7 +1737,7 @@ export async function processCategory(
|
||||
},
|
||||
}));
|
||||
|
||||
await insertProductAnalysisResults(db, runId, batchResults);
|
||||
await insertProductAnalysisResults(runId, batchResults);
|
||||
|
||||
for (const result of batchResults) {
|
||||
if (result.verdict.verdict === "FBA") {
|
||||
@@ -1781,7 +1752,7 @@ export async function processCategory(
|
||||
budget.analyzedAsins += batchResults.length;
|
||||
results.push(...batchResults);
|
||||
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: checkedAsins,
|
||||
availableAsins: results.length,
|
||||
fba,
|
||||
@@ -1802,7 +1773,7 @@ export async function processCategory(
|
||||
const emptyReason =
|
||||
budget.stopReason ||
|
||||
"No sellable ASINs matched the configured mid-range criteria";
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: checkedAsins,
|
||||
availableAsins: 0,
|
||||
fba,
|
||||
@@ -1830,7 +1801,7 @@ export async function processCategory(
|
||||
` Category stream totals: checked=${checkedAsins}, sellable=${sellableAsins}, keepaEnriched=${keepaEnrichedAsins}, analyzed=${results.length}`,
|
||||
);
|
||||
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: checkedAsins,
|
||||
availableAsins: results.length,
|
||||
fba,
|
||||
@@ -1923,11 +1894,6 @@ export async function main(): Promise<void> {
|
||||
await connectCache();
|
||||
try {
|
||||
mkdirSync(args.outputDir, { recursive: true });
|
||||
const DB_PATH =
|
||||
process.env.RESULTS_DB_PATH ||
|
||||
path.join(process.cwd(), "db", "results.db");
|
||||
initDb(DB_PATH);
|
||||
const db = getDb(DB_PATH);
|
||||
|
||||
log("info", "Starting per-category mid-range pipeline");
|
||||
log("info", `Marketplace: ${config.spApiMarketplaceId}`);
|
||||
@@ -1987,7 +1953,6 @@ export async function main(): Promise<void> {
|
||||
let runId: number | undefined;
|
||||
try {
|
||||
runId = await insertCategoryRunSummary(
|
||||
db,
|
||||
{
|
||||
categoryId: category.id,
|
||||
categoryLabel: category.label,
|
||||
@@ -2004,7 +1969,6 @@ export async function main(): Promise<void> {
|
||||
);
|
||||
|
||||
categorySummary = await processCategory(
|
||||
db,
|
||||
runId,
|
||||
category,
|
||||
args.perCategoryTop,
|
||||
@@ -2046,7 +2010,7 @@ export async function main(): Promise<void> {
|
||||
results: [],
|
||||
};
|
||||
if (runId) {
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: 0,
|
||||
availableAsins: 0,
|
||||
fba: 0,
|
||||
|
||||
732
src/server.ts
732
src/server.ts
File diff suppressed because it is too large
Load Diff
@@ -1,4 +1,6 @@
|
||||
import { type Database, closeDb, getDb, initDb } from "./database.ts";
|
||||
import { db } from "./db/index.ts";
|
||||
import { categoryProductResults, runs } from "./db/schema.ts";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { analyzeProducts } from "./llm.ts";
|
||||
import { fetchSpApiPricingAndFees } from "./sp-api.ts";
|
||||
import type {
|
||||
@@ -13,7 +15,6 @@ const LLM_BATCH_SIZE = 5;
|
||||
const LLM_BATCH_DELAY_MS = 5_000;
|
||||
|
||||
type Args = {
|
||||
dbPath: string;
|
||||
stalkerRunId: number;
|
||||
analysisRunId: number;
|
||||
asins: string[];
|
||||
@@ -22,18 +23,18 @@ type Args = {
|
||||
|
||||
type InventoryRow = {
|
||||
asin: string;
|
||||
product_title: string | null;
|
||||
productTitle: string | null;
|
||||
brand: string | null;
|
||||
category_tree: string | null;
|
||||
current_price: number | null;
|
||||
avg_price_90d: number | null;
|
||||
sales_rank: number | null;
|
||||
monthly_sold: number | null;
|
||||
seller_count: number | null;
|
||||
amazon_is_seller: number | null;
|
||||
can_sell: number | null;
|
||||
sellability_status: SellabilityInfo["sellabilityStatus"] | null;
|
||||
sellability_reason: string | null;
|
||||
categoryTree: string | null;
|
||||
currentPrice: number | null;
|
||||
avgPrice90d: number | null;
|
||||
salesRank: number | null;
|
||||
monthlySold: number | null;
|
||||
sellerCount: number | null;
|
||||
amazonIsSeller: boolean | null;
|
||||
canSell: boolean | null;
|
||||
sellabilityStatus: SellabilityInfo["sellabilityStatus"] | null;
|
||||
sellabilityReason: string | null;
|
||||
};
|
||||
|
||||
function readFlagValue(args: string[], flag: string): string | undefined {
|
||||
@@ -43,7 +44,6 @@ function readFlagValue(args: string[], flag: string): string | undefined {
|
||||
}
|
||||
|
||||
function parseArgs(argv = process.argv.slice(2)): Args {
|
||||
const dbPath = readFlagValue(argv, "--db");
|
||||
const stalkerRunId = Number(readFlagValue(argv, "--stalker-run-id"));
|
||||
const analysisRunId = Number(readFlagValue(argv, "--analysis-run-id"));
|
||||
const useClaude = argv.includes("--claude");
|
||||
@@ -52,7 +52,6 @@ function parseArgs(argv = process.argv.slice(2)): Args {
|
||||
.map((asin) => asin.trim().toUpperCase())
|
||||
.filter(Boolean);
|
||||
|
||||
if (!dbPath) throw new Error("Missing --db");
|
||||
if (!Number.isInteger(stalkerRunId) || stalkerRunId <= 0) {
|
||||
throw new Error("--stalker-run-id must be a positive integer");
|
||||
}
|
||||
@@ -61,7 +60,7 @@ function parseArgs(argv = process.argv.slice(2)): Args {
|
||||
}
|
||||
if (asins.length === 0) throw new Error("Missing --asins");
|
||||
|
||||
return { dbPath, stalkerRunId, analysisRunId, asins, useClaude };
|
||||
return { stalkerRunId, analysisRunId, asins, useClaude };
|
||||
}
|
||||
|
||||
function wait(ms: number): Promise<void> {
|
||||
@@ -81,69 +80,74 @@ function parseCategoryTree(value: string | null): string[] {
|
||||
}
|
||||
|
||||
function toProductRecord(row: InventoryRow): ProductRecord {
|
||||
const categoryTree = parseCategoryTree(row.category_tree);
|
||||
const categoryTree = parseCategoryTree(row.categoryTree);
|
||||
return {
|
||||
asin: row.asin,
|
||||
name: row.product_title ?? row.asin,
|
||||
name: row.productTitle ?? row.asin,
|
||||
brand: row.brand ?? undefined,
|
||||
category: categoryTree.join(" > ") || undefined,
|
||||
unitCost: 0,
|
||||
amazonRank: row.sales_rank ?? undefined,
|
||||
sellingPriceFromSheet: row.current_price ?? undefined,
|
||||
avgPrice90FromSheet: row.avg_price_90d ?? undefined,
|
||||
amazonRank: row.salesRank ?? undefined,
|
||||
sellingPriceFromSheet: row.currentPrice ?? undefined,
|
||||
avgPrice90FromSheet: row.avgPrice90d ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function toKeepaData(row: InventoryRow): KeepaData {
|
||||
return {
|
||||
currentPrice: row.current_price,
|
||||
avgPrice90: row.avg_price_90d,
|
||||
currentPrice: row.currentPrice,
|
||||
avgPrice90: row.avgPrice90d,
|
||||
minPrice90: null,
|
||||
maxPrice90: null,
|
||||
salesRank: row.sales_rank,
|
||||
salesRank: row.salesRank,
|
||||
salesRankAvg90: null,
|
||||
salesRankDrops30: null,
|
||||
salesRankDrops90: null,
|
||||
sellerCount: row.seller_count,
|
||||
amazonIsSeller:
|
||||
row.amazon_is_seller == null ? null : row.amazon_is_seller === 1,
|
||||
sellerCount: row.sellerCount,
|
||||
amazonIsSeller: row.amazonIsSeller,
|
||||
amazonBuyboxSharePct90d: null,
|
||||
buyBoxSeller: null,
|
||||
buyBoxPrice: null,
|
||||
buyBoxAvg90: null,
|
||||
monthlySold: row.monthly_sold,
|
||||
categoryTree: parseCategoryTree(row.category_tree),
|
||||
monthlySold: row.monthlySold,
|
||||
categoryTree: parseCategoryTree(row.categoryTree),
|
||||
};
|
||||
}
|
||||
|
||||
function toSellability(row: InventoryRow): SellabilityInfo {
|
||||
return {
|
||||
canSell: row.can_sell == null ? null : row.can_sell === 1,
|
||||
sellabilityStatus: row.sellability_status ?? "unknown",
|
||||
sellabilityReason: row.sellability_reason ?? undefined,
|
||||
canSell: row.canSell,
|
||||
sellabilityStatus: row.sellabilityStatus ?? "unknown",
|
||||
sellabilityReason: row.sellabilityReason ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function loadInventoryRows(
|
||||
database: Database,
|
||||
async function loadInventoryRows(
|
||||
stalkerRunId: number,
|
||||
asins: string[],
|
||||
): InventoryRow[] {
|
||||
const placeholders = asins.map(() => "?").join(",");
|
||||
return database
|
||||
.query(
|
||||
`SELECT
|
||||
asin, product_title, brand, category_tree, current_price, avg_price_90d,
|
||||
sales_rank, monthly_sold, seller_count, amazon_is_seller, can_sell,
|
||||
sellability_status, sellability_reason
|
||||
): Promise<InventoryRow[]> {
|
||||
if (asins.length === 0) return [];
|
||||
return db.execute(
|
||||
sql<InventoryRow>`SELECT DISTINCT ON (asin)
|
||||
asin,
|
||||
product_title AS "productTitle",
|
||||
brand,
|
||||
category_tree AS "categoryTree",
|
||||
current_price AS "currentPrice",
|
||||
avg_price_90d AS "avgPrice90d",
|
||||
sales_rank AS "salesRank",
|
||||
monthly_sold AS "monthlySold",
|
||||
seller_count AS "sellerCount",
|
||||
amazon_is_seller AS "amazonIsSeller",
|
||||
can_sell AS "canSell",
|
||||
sellability_status AS "sellabilityStatus",
|
||||
sellability_reason AS "sellabilityReason"
|
||||
FROM stalker_seller_inventory
|
||||
WHERE run_id = ?
|
||||
AND can_sell = 1
|
||||
WHERE run_id = ${stalkerRunId}
|
||||
AND can_sell = true
|
||||
AND sellability_status = 'available'
|
||||
AND asin IN (${placeholders})
|
||||
GROUP BY asin`,
|
||||
)
|
||||
.all(stalkerRunId, ...asins) as InventoryRow[];
|
||||
AND asin = ANY(${asins})`,
|
||||
);
|
||||
}
|
||||
|
||||
async function buildEnrichedProducts(
|
||||
@@ -156,7 +160,7 @@ async function buildEnrichedProducts(
|
||||
const spApi = await fetchSpApiPricingAndFees(
|
||||
row.asin,
|
||||
sellability,
|
||||
row.current_price,
|
||||
row.currentPrice,
|
||||
);
|
||||
|
||||
enriched.push({
|
||||
@@ -170,133 +174,114 @@ async function buildEnrichedProducts(
|
||||
return enriched;
|
||||
}
|
||||
|
||||
function insertProductAnalysisResults(
|
||||
database: Database,
|
||||
async function insertProductAnalysisResults(
|
||||
runId: number,
|
||||
results: AnalysisResult[],
|
||||
): void {
|
||||
): Promise<void> {
|
||||
if (results.length === 0) return;
|
||||
|
||||
const insert = database.prepare(`
|
||||
INSERT INTO product_analysis_results (
|
||||
asin, run_id, name, brand, category, unit_cost,
|
||||
current_price, avg_price_90d, avg_price_90d_sheet,
|
||||
selling_price_sheet, sales_rank, sales_rank_avg_90d,
|
||||
seller_count, amazon_is_seller, amazon_buybox_share_pct_90d,
|
||||
monthly_sold, rank_drops_30d, rank_drops_90d,
|
||||
fba_fee, fbm_fee, referral_percent, can_sell,
|
||||
sellability_status, sellability_reason,
|
||||
verdict, confidence, reasoning, fetched_at
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?
|
||||
)
|
||||
ON CONFLICT(asin) DO UPDATE SET
|
||||
run_id = excluded.run_id,
|
||||
name = excluded.name,
|
||||
brand = excluded.brand,
|
||||
category = excluded.category,
|
||||
unit_cost = excluded.unit_cost,
|
||||
current_price = excluded.current_price,
|
||||
avg_price_90d = excluded.avg_price_90d,
|
||||
avg_price_90d_sheet = excluded.avg_price_90d_sheet,
|
||||
selling_price_sheet = excluded.selling_price_sheet,
|
||||
sales_rank = excluded.sales_rank,
|
||||
sales_rank_avg_90d = excluded.sales_rank_avg_90d,
|
||||
seller_count = excluded.seller_count,
|
||||
amazon_is_seller = excluded.amazon_is_seller,
|
||||
amazon_buybox_share_pct_90d = excluded.amazon_buybox_share_pct_90d,
|
||||
monthly_sold = excluded.monthly_sold,
|
||||
rank_drops_30d = excluded.rank_drops_30d,
|
||||
rank_drops_90d = excluded.rank_drops_90d,
|
||||
fba_fee = excluded.fba_fee,
|
||||
fbm_fee = excluded.fbm_fee,
|
||||
referral_percent = excluded.referral_percent,
|
||||
can_sell = excluded.can_sell,
|
||||
sellability_status = excluded.sellability_status,
|
||||
sellability_reason = excluded.sellability_reason,
|
||||
verdict = excluded.verdict,
|
||||
confidence = excluded.confidence,
|
||||
reasoning = excluded.reasoning,
|
||||
fetched_at = excluded.fetched_at
|
||||
`);
|
||||
|
||||
database.transaction((batch: AnalysisResult[]) => {
|
||||
for (const result of batch) {
|
||||
const rows = results.map((result) => {
|
||||
const keepa = result.product.keepa;
|
||||
const record = result.product.record;
|
||||
const spApi = result.product.spApi;
|
||||
insert.run(
|
||||
record.asin,
|
||||
const canSell =
|
||||
spApi.canSell == null ? "unknown" : spApi.canSell ? "yes" : "no";
|
||||
|
||||
return {
|
||||
asin: record.asin,
|
||||
runId,
|
||||
record.name,
|
||||
record.brand ?? null,
|
||||
record.category ?? keepa?.categoryTree.join(" > ") ?? null,
|
||||
record.unitCost ?? null,
|
||||
keepa?.currentPrice ?? spApi.estimatedSalePrice ?? null,
|
||||
keepa?.avgPrice90 ?? null,
|
||||
record.avgPrice90FromSheet ?? null,
|
||||
record.sellingPriceFromSheet ?? null,
|
||||
keepa?.salesRank ?? record.amazonRank ?? null,
|
||||
keepa?.salesRankAvg90 ?? null,
|
||||
keepa?.sellerCount ?? null,
|
||||
keepa?.amazonIsSeller == null ? null : keepa.amazonIsSeller ? 1 : 0,
|
||||
keepa?.amazonBuyboxSharePct90d ?? null,
|
||||
keepa?.monthlySold ?? null,
|
||||
keepa?.salesRankDrops30 ?? null,
|
||||
keepa?.salesRankDrops90 ?? null,
|
||||
spApi.fbaFee ?? null,
|
||||
spApi.fbmFee ?? null,
|
||||
spApi.referralFeePercent ?? null,
|
||||
spApi.canSell == null ? "unknown" : spApi.canSell ? "yes" : "no",
|
||||
spApi.sellabilityStatus ?? null,
|
||||
spApi.sellabilityReason ?? null,
|
||||
result.verdict.verdict,
|
||||
result.verdict.confidence,
|
||||
result.verdict.reasoning ?? null,
|
||||
result.product.fetchedAt,
|
||||
);
|
||||
}
|
||||
})(results);
|
||||
name: record.name,
|
||||
brand: record.brand ?? null,
|
||||
category: record.category ?? keepa?.categoryTree.join(" > ") ?? null,
|
||||
unitCost: record.unitCost ?? null,
|
||||
currentPrice: keepa?.currentPrice ?? spApi.estimatedSalePrice ?? null,
|
||||
avgPrice90d: keepa?.avgPrice90 ?? null,
|
||||
avgPrice90dSheet: record.avgPrice90FromSheet ?? null,
|
||||
sellingPriceSheet: record.sellingPriceFromSheet ?? null,
|
||||
salesRank: keepa?.salesRank ?? record.amazonRank ?? null,
|
||||
salesRankAvg90d: keepa?.salesRankAvg90 ?? null,
|
||||
sellerCount: keepa?.sellerCount ?? null,
|
||||
amazonIsSeller: keepa?.amazonIsSeller ?? null,
|
||||
amazonBuyboxSharePct90d: keepa?.amazonBuyboxSharePct90d ?? null,
|
||||
monthlySold: keepa?.monthlySold ?? null,
|
||||
rankDrops30d: keepa?.salesRankDrops30 ?? null,
|
||||
rankDrops90d: keepa?.salesRankDrops90 ?? null,
|
||||
fbaFee: spApi.fbaFee ?? null,
|
||||
fbmFee: spApi.fbmFee ?? null,
|
||||
referralPercent: spApi.referralFeePercent ?? null,
|
||||
canSell,
|
||||
sellabilityStatus: spApi.sellabilityStatus ?? null,
|
||||
sellabilityReason: spApi.sellabilityReason ?? null,
|
||||
verdict: result.verdict.verdict,
|
||||
confidence: result.verdict.confidence ?? 0,
|
||||
reasoning: result.verdict.reasoning ?? null,
|
||||
fetchedAt: new Date(result.product.fetchedAt),
|
||||
};
|
||||
});
|
||||
|
||||
await db
|
||||
.insert(categoryProductResults)
|
||||
.values(rows)
|
||||
.onConflictDoUpdate({
|
||||
target: categoryProductResults.asin,
|
||||
set: {
|
||||
runId: sql`EXCLUDED.run_id`,
|
||||
name: sql`EXCLUDED.name`,
|
||||
brand: sql`EXCLUDED.brand`,
|
||||
category: sql`EXCLUDED.category`,
|
||||
unitCost: sql`EXCLUDED.unit_cost`,
|
||||
currentPrice: sql`EXCLUDED.current_price`,
|
||||
avgPrice90d: sql`EXCLUDED.avg_price_90d`,
|
||||
avgPrice90dSheet: sql`EXCLUDED.avg_price_90d_sheet`,
|
||||
sellingPriceSheet: sql`EXCLUDED.selling_price_sheet`,
|
||||
salesRank: sql`EXCLUDED.sales_rank`,
|
||||
salesRankAvg90d: sql`EXCLUDED.sales_rank_avg_90d`,
|
||||
sellerCount: sql`EXCLUDED.seller_count`,
|
||||
amazonIsSeller: sql`EXCLUDED.amazon_is_seller`,
|
||||
amazonBuyboxSharePct90d: sql`EXCLUDED.amazon_buybox_share_pct_90d`,
|
||||
monthlySold: sql`EXCLUDED.monthly_sold`,
|
||||
rankDrops30d: sql`EXCLUDED.rank_drops_30d`,
|
||||
rankDrops90d: sql`EXCLUDED.rank_drops_90d`,
|
||||
fbaFee: sql`EXCLUDED.fba_fee`,
|
||||
fbmFee: sql`EXCLUDED.fbm_fee`,
|
||||
referralPercent: sql`EXCLUDED.referral_percent`,
|
||||
canSell: sql`EXCLUDED.can_sell`,
|
||||
sellabilityStatus: sql`EXCLUDED.sellability_status`,
|
||||
sellabilityReason: sql`EXCLUDED.sellability_reason`,
|
||||
verdict: sql`EXCLUDED.verdict`,
|
||||
confidence: sql`EXCLUDED.confidence`,
|
||||
reasoning: sql`EXCLUDED.reasoning`,
|
||||
fetchedAt: sql`EXCLUDED.fetched_at`,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function refreshAnalysisRun(database: Database, runId: number): void {
|
||||
const stats = database
|
||||
.query(
|
||||
`SELECT
|
||||
async function refreshAnalysisRun(runId: number): Promise<void> {
|
||||
const [stats] = await db.execute(
|
||||
sql<{
|
||||
total: string;
|
||||
fba: string | null;
|
||||
fbm: string | null;
|
||||
skip: string | null;
|
||||
}>`SELECT
|
||||
COUNT(*) AS total,
|
||||
SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba,
|
||||
SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm,
|
||||
SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip
|
||||
FROM product_analysis_results
|
||||
WHERE run_id = ?`,
|
||||
)
|
||||
.get(runId) as {
|
||||
total: number;
|
||||
fba: number | null;
|
||||
fbm: number | null;
|
||||
skip: number | null;
|
||||
};
|
||||
|
||||
database
|
||||
.prepare(
|
||||
`UPDATE category_analysis_runs
|
||||
SET top_asins_checked = ?,
|
||||
available_asins = ?,
|
||||
fba_count = ?,
|
||||
fbm_count = ?,
|
||||
skip_count = ?
|
||||
WHERE id = ?`,
|
||||
)
|
||||
.run(
|
||||
stats.total ?? 0,
|
||||
stats.total ?? 0,
|
||||
stats.fba ?? 0,
|
||||
stats.fbm ?? 0,
|
||||
stats.skip ?? 0,
|
||||
runId,
|
||||
FROM category_product_results
|
||||
WHERE run_id = ${runId}`,
|
||||
);
|
||||
|
||||
await db
|
||||
.update(runs)
|
||||
.set({
|
||||
topAsinsChecked: Number(stats?.total ?? 0),
|
||||
availableAsins: Number(stats?.total ?? 0),
|
||||
fbaCount: Number(stats?.fba ?? 0),
|
||||
fbmCount: Number(stats?.fbm ?? 0),
|
||||
skipCount: Number(stats?.skip ?? 0),
|
||||
})
|
||||
.where(eq(runs.id, runId));
|
||||
}
|
||||
|
||||
async function analyzeInBatches(
|
||||
@@ -349,11 +334,8 @@ async function analyzeInBatches(
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = parseArgs();
|
||||
initDb(args.dbPath);
|
||||
const database = getDb(args.dbPath);
|
||||
|
||||
try {
|
||||
const rows = loadInventoryRows(database, args.stalkerRunId, args.asins);
|
||||
const rows = await loadInventoryRows(args.stalkerRunId, args.asins);
|
||||
if (rows.length === 0) {
|
||||
console.log("Stalker analysis: no sellable inventory rows to analyze.");
|
||||
return;
|
||||
@@ -362,11 +344,8 @@ async function main(): Promise<void> {
|
||||
console.log(`Stalker analysis: analyzing ${rows.length} sellable ASIN(s).`);
|
||||
const enriched = await buildEnrichedProducts(rows);
|
||||
const results = await analyzeInBatches(enriched, args.useClaude);
|
||||
insertProductAnalysisResults(database, args.analysisRunId, results);
|
||||
refreshAnalysisRun(database, args.analysisRunId);
|
||||
} finally {
|
||||
closeDb();
|
||||
}
|
||||
await insertProductAnalysisResults(args.analysisRunId, results);
|
||||
await refreshAnalysisRun(args.analysisRunId);
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
|
||||
@@ -2,7 +2,67 @@ import { afterAll, beforeEach, expect, mock, test } from "bun:test";
|
||||
import { mkdirSync, rmSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import * as XLSX from "xlsx";
|
||||
import { closeDb, getDb } from "./database.ts";
|
||||
|
||||
let nextId = 0;
|
||||
|
||||
function chainable(resolveWith: any[] = []): any {
|
||||
const p: any = Promise.resolve(resolveWith);
|
||||
p.limit = (_n: any) => chainable(resolveWith);
|
||||
p.where = (_cond: any) => chainable(resolveWith);
|
||||
p.from = (_table: any) => chainable(resolveWith);
|
||||
return p;
|
||||
}
|
||||
|
||||
const makeMockTx = (): any => ({
|
||||
insert: (_table: any) => ({
|
||||
values: (_vals: any) => ({
|
||||
returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]),
|
||||
onConflictDoUpdate: (_conf: any) => Promise.resolve([]),
|
||||
}),
|
||||
}),
|
||||
update: (_table: any) => ({
|
||||
set: (_vals: any) => ({
|
||||
where: (_cond: any) => Promise.resolve([]),
|
||||
}),
|
||||
}),
|
||||
select: (_sel?: any) => ({
|
||||
from: (_table: any) => ({
|
||||
where: (_cond: any) => chainable([{ id: ++nextId }]),
|
||||
limit: (_n: any) => chainable([{ id: nextId }]),
|
||||
}),
|
||||
}),
|
||||
selectDistinct: (_sel: any) => ({
|
||||
from: (_table: any) => chainable([]),
|
||||
}),
|
||||
execute: (_query: any) => Promise.resolve([]),
|
||||
});
|
||||
|
||||
const makeMockDb = (): any => ({
|
||||
insert: (_table: any) => ({
|
||||
values: (_vals: any) => ({
|
||||
returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]),
|
||||
onConflictDoUpdate: (_conf: any) => Promise.resolve([]),
|
||||
}),
|
||||
}),
|
||||
update: (_table: any) => ({
|
||||
set: (_vals: any) => ({
|
||||
where: (_cond: any) => Promise.resolve([]),
|
||||
}),
|
||||
}),
|
||||
select: (_sel?: any) => ({
|
||||
from: (_table: any) => ({
|
||||
where: (_cond: any) => chainable(),
|
||||
limit: (_n: any) => chainable(),
|
||||
}),
|
||||
}),
|
||||
selectDistinct: (_sel: any) => ({
|
||||
from: (_table: any) => chainable(),
|
||||
}),
|
||||
execute: (_query: any) => Promise.resolve([]),
|
||||
transaction: async (fn: (tx: any) => Promise<any>) => fn(makeMockTx()),
|
||||
});
|
||||
|
||||
mock.module("./db/index.ts", () => ({ db: makeMockDb(), client: {} }));
|
||||
|
||||
const TEST_DIR = path.join(process.cwd(), "test_output", "stalker-sellability");
|
||||
const originalFetch = globalThis.fetch;
|
||||
@@ -34,7 +94,7 @@ mock.module("./sp-api.ts", () => ({
|
||||
const modulePromise = import("./stalker.ts");
|
||||
|
||||
beforeEach(() => {
|
||||
closeDb();
|
||||
nextId = 0;
|
||||
rmSync(TEST_DIR, { recursive: true, force: true });
|
||||
mkdirSync(TEST_DIR, { recursive: true });
|
||||
globalThis.fetch = originalFetch;
|
||||
@@ -49,14 +109,12 @@ afterAll(() => {
|
||||
} else {
|
||||
Bun.env.KEEPA_API_KEY = originalKeepaKey;
|
||||
}
|
||||
closeDb();
|
||||
rmSync(TEST_DIR, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("sellability checks matched seller inventory, not the source ASIN", async () => {
|
||||
const { runStalker } = await modulePromise;
|
||||
const inputPath = path.join(TEST_DIR, "input.xlsx");
|
||||
const dbPath = path.join(TEST_DIR, "stalker.sqlite");
|
||||
const workbook = XLSX.utils.book_new();
|
||||
XLSX.utils.book_append_sheet(
|
||||
workbook,
|
||||
@@ -138,7 +196,6 @@ test("sellability checks matched seller inventory, not the source ASIN", async (
|
||||
|
||||
const stats = await runStalker({
|
||||
input: inputPath,
|
||||
dbPath,
|
||||
maxAsins: null,
|
||||
storefrontUpdateHours: 168,
|
||||
offerLimit: 20,
|
||||
@@ -151,6 +208,7 @@ test("sellability checks matched seller inventory, not the source ASIN", async (
|
||||
maxSellerRequests: null,
|
||||
sellability: true,
|
||||
analyzeSellable: false,
|
||||
useClaude: false,
|
||||
});
|
||||
|
||||
expect(fetchSellabilityBatchMock.mock.calls.length).toBe(1);
|
||||
@@ -162,46 +220,4 @@ test("sellability checks matched seller inventory, not the source ASIN", async (
|
||||
expect(stats.inventorySellabilityAvailableAsins).toBe(1);
|
||||
expect(stats.inventorySellabilityExcludedAsins).toBe(1);
|
||||
expect(stats.persistedInventoryAsins).toBe(1);
|
||||
|
||||
const db = getDb(dbPath);
|
||||
const scan = db.query("SELECT source_asin FROM stalker_asin_scans").get() as {
|
||||
source_asin: string;
|
||||
};
|
||||
expect(scan.source_asin).toBe("B000000001");
|
||||
|
||||
const inventory = db
|
||||
.query(
|
||||
`SELECT asin, can_sell, sellability_status, product_title, brand,
|
||||
category_tree, current_price, avg_price_90d, sales_rank, monthly_sold,
|
||||
seller_count
|
||||
FROM stalker_seller_inventory ORDER BY asin`,
|
||||
)
|
||||
.all() as Array<{
|
||||
asin: string;
|
||||
can_sell: number | null;
|
||||
sellability_status: string | null;
|
||||
product_title: string | null;
|
||||
brand: string | null;
|
||||
category_tree: string | null;
|
||||
current_price: number | null;
|
||||
avg_price_90d: number | null;
|
||||
sales_rank: number | null;
|
||||
monthly_sold: number | null;
|
||||
seller_count: number | null;
|
||||
}>;
|
||||
expect(inventory).toEqual([
|
||||
{
|
||||
asin: "B111111111",
|
||||
can_sell: 1,
|
||||
sellability_status: "available",
|
||||
product_title: "Sellable Storefront Product",
|
||||
brand: "Good Brand",
|
||||
category_tree: JSON.stringify(["Kitchen", "Storage"]),
|
||||
current_price: 19.99,
|
||||
avg_price_90d: 25,
|
||||
sales_rank: 12345,
|
||||
monthly_sold: 42,
|
||||
seller_count: 7,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -2,7 +2,6 @@ import { afterAll, beforeEach, expect, mock, test } from "bun:test";
|
||||
import { mkdirSync, rmSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import * as XLSX from "xlsx";
|
||||
import { closeDb, getDb, initDb } from "./database.ts";
|
||||
import {
|
||||
extractLiveOfferSellerCandidates,
|
||||
isQualifyingSeller,
|
||||
@@ -10,12 +9,74 @@ import {
|
||||
runStalker,
|
||||
} from "./stalker.ts";
|
||||
|
||||
let nextId = 0;
|
||||
|
||||
function chainable(resolveWith: any[] = []): any {
|
||||
const p: any = Promise.resolve(resolveWith);
|
||||
p.limit = (_n: any) => chainable(resolveWith);
|
||||
p.where = (_cond: any) => chainable(resolveWith);
|
||||
p.from = (_table: any) => chainable(resolveWith);
|
||||
return p;
|
||||
}
|
||||
|
||||
// Transaction mock returns rows for selects (needed for upsert-then-select patterns).
|
||||
const makeMockTx = (): any => ({
|
||||
insert: (_table: any) => ({
|
||||
values: (_vals: any) => ({
|
||||
returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]),
|
||||
onConflictDoUpdate: (_conf: any) => Promise.resolve([]),
|
||||
}),
|
||||
}),
|
||||
update: (_table: any) => ({
|
||||
set: (_vals: any) => ({
|
||||
where: (_cond: any) => Promise.resolve([]),
|
||||
}),
|
||||
}),
|
||||
select: (_sel?: any) => ({
|
||||
from: (_table: any) => ({
|
||||
where: (_cond: any) => chainable([{ id: ++nextId }]),
|
||||
limit: (_n: any) => chainable([{ id: nextId }]),
|
||||
}),
|
||||
}),
|
||||
selectDistinct: (_sel: any) => ({
|
||||
from: (_table: any) => chainable([]),
|
||||
}),
|
||||
execute: (_query: any) => Promise.resolve([]),
|
||||
});
|
||||
|
||||
const makeMockDb = (): any => ({
|
||||
insert: (_table: any) => ({
|
||||
values: (_vals: any) => ({
|
||||
returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]),
|
||||
onConflictDoUpdate: (_conf: any) => Promise.resolve([]),
|
||||
}),
|
||||
}),
|
||||
update: (_table: any) => ({
|
||||
set: (_vals: any) => ({
|
||||
where: (_cond: any) => Promise.resolve([]),
|
||||
}),
|
||||
}),
|
||||
select: (_sel?: any) => ({
|
||||
from: (_table: any) => ({
|
||||
where: (_cond: any) => chainable(),
|
||||
limit: (_n: any) => chainable(),
|
||||
}),
|
||||
}),
|
||||
selectDistinct: (_sel: any) => ({
|
||||
from: (_table: any) => chainable(),
|
||||
}),
|
||||
execute: (_query: any) => Promise.resolve([]),
|
||||
transaction: async (fn: (tx: any) => Promise<any>) => fn(makeMockTx()),
|
||||
});
|
||||
|
||||
mock.module("./db/index.ts", () => ({ db: makeMockDb(), client: {} }));
|
||||
|
||||
const TEST_DIR = path.join(process.cwd(), "test_output", "stalker");
|
||||
const originalFetch = globalThis.fetch;
|
||||
const originalKeepaKey = Bun.env.KEEPA_API_KEY;
|
||||
|
||||
beforeEach(() => {
|
||||
closeDb();
|
||||
nextId = 0;
|
||||
rmSync(TEST_DIR, { recursive: true, force: true });
|
||||
mkdirSync(TEST_DIR, { recursive: true });
|
||||
globalThis.fetch = originalFetch;
|
||||
@@ -29,7 +90,6 @@ afterAll(() => {
|
||||
} else {
|
||||
Bun.env.KEEPA_API_KEY = originalKeepaKey;
|
||||
}
|
||||
closeDb();
|
||||
rmSync(TEST_DIR, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
@@ -77,35 +137,8 @@ test("extractLiveOfferSellerCandidates ignores Amazon, missing seller IDs, and d
|
||||
expect(offers[0]?.stock).toBe(4);
|
||||
});
|
||||
|
||||
test("initDb creates stalker tables and indexes", () => {
|
||||
const dbPath = path.join(TEST_DIR, "schema.sqlite");
|
||||
initDb(dbPath);
|
||||
const db = getDb(dbPath);
|
||||
|
||||
const tables = db
|
||||
.query(
|
||||
`SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'stalker_%' ORDER BY name`,
|
||||
)
|
||||
.all() as Array<{ name: string }>;
|
||||
expect(tables.map((row) => row.name)).toEqual([
|
||||
"stalker_asin_scans",
|
||||
"stalker_asin_sellers",
|
||||
"stalker_runs",
|
||||
"stalker_seller_inventory",
|
||||
"stalker_sellers",
|
||||
]);
|
||||
|
||||
const indexes = db
|
||||
.query(
|
||||
`SELECT name FROM sqlite_master WHERE type = 'index' AND name LIKE 'idx_stalker_%' ORDER BY name`,
|
||||
)
|
||||
.all() as Array<{ name: string }>;
|
||||
expect(indexes.length).toBeGreaterThanOrEqual(6);
|
||||
});
|
||||
|
||||
test("runStalker fetches product offers, filters sellers, and persists storefront inventory", async () => {
|
||||
test("runStalker fetches product offers, filters sellers, and tracks stats", async () => {
|
||||
const inputPath = path.join(TEST_DIR, "input.xlsx");
|
||||
const dbPath = path.join(TEST_DIR, "stalker.sqlite");
|
||||
const workbook = XLSX.utils.book_new();
|
||||
XLSX.utils.book_append_sheet(
|
||||
workbook,
|
||||
@@ -205,7 +238,6 @@ test("runStalker fetches product offers, filters sellers, and persists storefron
|
||||
|
||||
const stats = await runStalker({
|
||||
input: inputPath,
|
||||
dbPath,
|
||||
maxAsins: null,
|
||||
storefrontUpdateHours: 168,
|
||||
offerLimit: 20,
|
||||
@@ -218,6 +250,7 @@ test("runStalker fetches product offers, filters sellers, and persists storefron
|
||||
maxSellerRequests: null,
|
||||
sellability: false,
|
||||
analyzeSellable: false,
|
||||
useClaude: false,
|
||||
});
|
||||
|
||||
expect(stats.scannedAsins).toBe(1);
|
||||
@@ -229,6 +262,7 @@ test("runStalker fetches product offers, filters sellers, and persists storefron
|
||||
expect(stats.qualifyingSellers).toBe(1);
|
||||
expect(stats.sellerMetadataRequests).toBe(1);
|
||||
expect(stats.sellerStorefrontRequests).toBe(1);
|
||||
|
||||
const sellerCalls = fetchMock.mock.calls.filter((call) => {
|
||||
const rawUrl =
|
||||
typeof call[0] === "string"
|
||||
@@ -239,45 +273,4 @@ test("runStalker fetches product offers, filters sellers, and persists storefron
|
||||
return new URL(rawUrl).pathname === "/seller";
|
||||
});
|
||||
expect(sellerCalls.length).toBe(2);
|
||||
|
||||
const db = getDb(dbPath);
|
||||
const run = db.query("SELECT * FROM stalker_runs").get() as any;
|
||||
expect(run.status).toBe("completed");
|
||||
expect(run.requested_asins).toBe(1);
|
||||
expect(run.scanned_asins).toBe(1);
|
||||
expect(run.source_asins_with_matches).toBe(1);
|
||||
expect(run.candidate_sellers).toBe(2);
|
||||
expect(run.qualifying_sellers).toBe(1);
|
||||
expect(run.matched_sellers).toBe(1);
|
||||
expect(run.seller_metadata_requests).toBe(1);
|
||||
expect(run.seller_storefront_requests).toBe(1);
|
||||
expect(run.inventory_sellability_checked_asins).toBe(0);
|
||||
expect(run.inventory_sellability_available_asins).toBe(0);
|
||||
expect(run.inventory_sellability_excluded_asins).toBe(0);
|
||||
expect(run.persisted_inventory_asins).toBe(0);
|
||||
|
||||
const scan = db.query("SELECT * FROM stalker_asin_scans").get() as any;
|
||||
expect(scan.source_asin).toBe("B000000001");
|
||||
expect(scan.title).toBe("Tracked Product");
|
||||
expect(scan.offer_count).toBe(2);
|
||||
expect(scan.candidate_seller_count).toBe(2);
|
||||
expect(scan.matched_seller_count).toBe(1);
|
||||
|
||||
const sellers = db.query("SELECT * FROM stalker_sellers").all() as any[];
|
||||
expect(sellers.length).toBe(1);
|
||||
expect(sellers[0].seller_id).toBe("AQUALIFIED");
|
||||
expect(sellers[0].rating_count).toBe(12);
|
||||
expect(sellers[0].storefront_asin_total).toBe(2);
|
||||
expect(sellers[0].persisted_inventory_sample_count).toBe(0);
|
||||
|
||||
const asinSellers = db.query("SELECT * FROM stalker_asin_sellers").all() as any[];
|
||||
expect(asinSellers.length).toBe(1);
|
||||
expect(asinSellers[0].offer_price).toBe(19.99);
|
||||
expect(asinSellers[0].is_fba).toBe(1);
|
||||
expect(asinSellers[0].stock).toBe(3);
|
||||
|
||||
const inventory = db
|
||||
.query("SELECT asin FROM stalker_seller_inventory ORDER BY asin")
|
||||
.all() as Array<{ asin: string }>;
|
||||
expect(inventory.map((row) => row.asin)).toEqual([]);
|
||||
});
|
||||
|
||||
639
src/stalker.ts
639
src/stalker.ts
@@ -1,6 +1,15 @@
|
||||
import * as XLSX from "xlsx";
|
||||
import path from "node:path";
|
||||
import { type Database, closeDb, getDb, initDb } from "./database.ts";
|
||||
import { db } from "./db/index.ts";
|
||||
import {
|
||||
runs,
|
||||
stalkerRuns,
|
||||
stalkerAsinScans,
|
||||
sellers,
|
||||
stalkerAsinSellers,
|
||||
stalkerSellerInventory,
|
||||
} from "./db/schema.ts";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { fetchSellabilityBatch } from "./sp-api.ts";
|
||||
import type { SellabilityInfo } from "./types.ts";
|
||||
|
||||
@@ -8,7 +17,6 @@ const KEEPA_BASE = "https://api.keepa.com";
|
||||
const DOMAIN_US = "1";
|
||||
const AMAZON_US_SELLER_ID = "ATVPDKIKX0DER";
|
||||
const ASIN_REGEX = /^B[0-9A-Z]{9}$/;
|
||||
const DEFAULT_DB_PATH = path.join(process.cwd(), "db", "results.db");
|
||||
const DEFAULT_STOREFRONT_UPDATE_HOURS = 168;
|
||||
const DEFAULT_OFFER_LIMIT = 100;
|
||||
const DEFAULT_SELLER_LIMIT = 30;
|
||||
@@ -28,7 +36,7 @@ type KeepaApiResponse = {
|
||||
|
||||
export type StalkerArgs = {
|
||||
input: string;
|
||||
dbPath: string;
|
||||
dbPath?: string;
|
||||
maxAsins: number | null;
|
||||
storefrontUpdateHours: number;
|
||||
offerLimit: number;
|
||||
@@ -115,7 +123,6 @@ type StalkerRunStats = {
|
||||
};
|
||||
|
||||
type StalkerRunContext = {
|
||||
database: Database | null;
|
||||
metadataCache: Map<string, StalkerSeller>;
|
||||
storefrontCache: Map<string, StalkerSeller>;
|
||||
stats: StalkerRunStats;
|
||||
@@ -131,7 +138,6 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs {
|
||||
printUsageAndExit("Missing required --input file.");
|
||||
}
|
||||
|
||||
const dbPath = readFlagValue(argv, "--db") ?? DEFAULT_DB_PATH;
|
||||
const maxAsinsRaw = readFlagValue(argv, "--max-asins");
|
||||
const storefrontUpdateRaw = readFlagValue(argv, "--storefront-update-hours");
|
||||
const offerLimitRaw = readFlagValue(argv, "--offer-limit");
|
||||
@@ -205,7 +211,6 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs {
|
||||
|
||||
return {
|
||||
input,
|
||||
dbPath,
|
||||
maxAsins,
|
||||
storefrontUpdateHours,
|
||||
offerLimit,
|
||||
@@ -313,20 +318,18 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
|
||||
const cappedAsins =
|
||||
args.maxAsins == null ? allAsins : allAsins.slice(0, args.maxAsins);
|
||||
|
||||
initDb(args.dbPath);
|
||||
const database = getDb(args.dbPath);
|
||||
const completedAsins = args.resume
|
||||
? loadPreviouslyScannedAsins(database)
|
||||
? await loadPreviouslyScannedAsins()
|
||||
: new Set<string>();
|
||||
const resumeFilteredAsins = cappedAsins.filter(
|
||||
(asin) => !completedAsins.has(asin),
|
||||
);
|
||||
const runId = args.dryRun
|
||||
? null
|
||||
: startStalkerRun(database, args.input, resumeFilteredAsins.length);
|
||||
: await startStalkerRun(args.input, resumeFilteredAsins.length);
|
||||
const analysisRunId =
|
||||
!args.dryRun && args.analyzeSellable
|
||||
? startStalkerAnalysisRun(database, args.input)
|
||||
? await startStalkerAnalysisRun(args.input)
|
||||
: null;
|
||||
const stats: StalkerRunStats = {
|
||||
scannedAsins: 0,
|
||||
@@ -345,7 +348,6 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
|
||||
stoppedEarly: false,
|
||||
};
|
||||
const context: StalkerRunContext = {
|
||||
database,
|
||||
metadataCache: new Map(),
|
||||
storefrontCache: new Map(),
|
||||
stats,
|
||||
@@ -389,7 +391,7 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
|
||||
}
|
||||
|
||||
if (!args.dryRun && runId != null) {
|
||||
persistAsinResult(database, runId, result);
|
||||
await persistAsinResult(runId, result);
|
||||
}
|
||||
const sellableAsins = collectPersistedInventoryAsins(result);
|
||||
if (
|
||||
@@ -400,7 +402,6 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
|
||||
sellableAsins.length > 0
|
||||
) {
|
||||
await runSellableAnalysisChild(
|
||||
args.dbPath,
|
||||
runId,
|
||||
analysisRunId,
|
||||
sellableAsins,
|
||||
@@ -417,7 +418,7 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
|
||||
}
|
||||
|
||||
if (!args.dryRun && runId != null) {
|
||||
refreshStalkerRun(database, runId, stats, "running");
|
||||
await refreshStalkerRun(runId, stats, "running");
|
||||
}
|
||||
console.log(
|
||||
`Stalker: ${asin} candidates=${result.candidateSellerCount}, matched=${result.matchedSellers.length}, persisted_inventory=${sumInventoryAsins(result)}`,
|
||||
@@ -432,8 +433,7 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
|
||||
}
|
||||
|
||||
if (!args.dryRun && runId != null) {
|
||||
refreshStalkerRun(
|
||||
database,
|
||||
await refreshStalkerRun(
|
||||
runId,
|
||||
stats,
|
||||
stats.stoppedEarly
|
||||
@@ -445,16 +445,16 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
|
||||
}
|
||||
logRunSummary(stats, args);
|
||||
if (!args.dryRun && analysisRunId != null) {
|
||||
finishStalkerAnalysisRun(database, analysisRunId, "completed");
|
||||
await finishStalkerAnalysisRun(analysisRunId, "completed");
|
||||
}
|
||||
return stats;
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
if (!args.dryRun && runId != null) {
|
||||
finishStalkerRunWithError(database, runId, stats, message);
|
||||
await finishStalkerRunWithError(runId, stats, message);
|
||||
}
|
||||
if (!args.dryRun && analysisRunId != null) {
|
||||
finishStalkerAnalysisRun(database, analysisRunId, "failed", message);
|
||||
await finishStalkerAnalysisRun(analysisRunId, "failed", message);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
@@ -685,13 +685,12 @@ async function fetchSellerMetadata(
|
||||
for (const sellerId of uniqueSellerIds) {
|
||||
const cached =
|
||||
context.metadataCache.get(sellerId) ??
|
||||
loadCachedSeller(
|
||||
context.database,
|
||||
(await loadCachedSeller(
|
||||
sellerId,
|
||||
args.sellerCacheHours,
|
||||
false,
|
||||
args.inventoryLimit,
|
||||
);
|
||||
));
|
||||
if (cached) {
|
||||
context.metadataCache.set(sellerId, cached);
|
||||
out.set(sellerId, cached);
|
||||
@@ -739,13 +738,12 @@ async function fetchQualifiedSellerStorefronts(
|
||||
for (const sellerId of uniqueSellerIds) {
|
||||
const cached =
|
||||
context.storefrontCache.get(sellerId) ??
|
||||
loadCachedSeller(
|
||||
context.database,
|
||||
(await loadCachedSeller(
|
||||
sellerId,
|
||||
args.sellerCacheHours,
|
||||
true,
|
||||
args.inventoryLimit,
|
||||
);
|
||||
));
|
||||
if (cached) {
|
||||
context.storefrontCache.set(sellerId, cached);
|
||||
out.set(sellerId, cached);
|
||||
@@ -830,272 +828,268 @@ async function fetchKeepaWithRetries(
|
||||
throw new Error(lastErrorMessage);
|
||||
}
|
||||
|
||||
function persistAsinResult(
|
||||
database: Database,
|
||||
async function persistAsinResult(
|
||||
runId: number,
|
||||
result: StalkerAsinResult,
|
||||
): void {
|
||||
const fetchedAt = new Date().toISOString();
|
||||
): Promise<void> {
|
||||
const fetchedAt = new Date();
|
||||
|
||||
database.transaction(() => {
|
||||
const scanId = upsertAsinScan(database, runId, result, fetchedAt);
|
||||
await db.transaction(async (tx) => {
|
||||
const scanId = await upsertAsinScan(tx, runId, result, fetchedAt);
|
||||
|
||||
for (const { seller, offer } of result.matchedSellers) {
|
||||
upsertSeller(database, seller, fetchedAt);
|
||||
upsertAsinSeller(database, scanId, seller, offer);
|
||||
upsertSellerInventory(database, runId, seller, fetchedAt);
|
||||
await upsertSeller(tx, seller, fetchedAt);
|
||||
await upsertAsinSeller(tx, scanId, seller, offer);
|
||||
await upsertSellerInventory(tx, runId, seller, fetchedAt);
|
||||
}
|
||||
})();
|
||||
});
|
||||
}
|
||||
|
||||
function upsertAsinScan(
|
||||
database: Database,
|
||||
async function upsertAsinScan(
|
||||
tx: Parameters<Parameters<typeof db.transaction>[0]>[0],
|
||||
runId: number,
|
||||
result: StalkerAsinResult,
|
||||
fetchedAt: string,
|
||||
): number {
|
||||
database
|
||||
.prepare(
|
||||
`INSERT INTO stalker_asin_scans (
|
||||
run_id, source_asin, title, offer_count, candidate_seller_count,
|
||||
matched_seller_count, fetched_at, raw_product_json
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(run_id, source_asin) DO UPDATE SET
|
||||
title = excluded.title,
|
||||
offer_count = excluded.offer_count,
|
||||
candidate_seller_count = excluded.candidate_seller_count,
|
||||
matched_seller_count = excluded.matched_seller_count,
|
||||
fetched_at = excluded.fetched_at,
|
||||
raw_product_json = excluded.raw_product_json`,
|
||||
)
|
||||
.run(
|
||||
fetchedAt: Date,
|
||||
): Promise<number> {
|
||||
await tx
|
||||
.insert(stalkerAsinScans)
|
||||
.values({
|
||||
runId,
|
||||
result.asin,
|
||||
result.title,
|
||||
result.offerCount,
|
||||
result.candidateSellerCount,
|
||||
result.matchedSellers.length,
|
||||
sourceAsin: result.asin,
|
||||
title: result.title,
|
||||
offerCount: result.offerCount,
|
||||
candidateSellerCount: result.candidateSellerCount,
|
||||
matchedSellerCount: result.matchedSellers.length,
|
||||
fetchedAt,
|
||||
JSON.stringify(result.product ?? { error: result.error ?? null }),
|
||||
);
|
||||
rawProductJson: JSON.stringify(
|
||||
result.product ?? { error: result.error ?? null },
|
||||
),
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: [stalkerAsinScans.runId, stalkerAsinScans.sourceAsin],
|
||||
set: {
|
||||
title: sql`EXCLUDED.title`,
|
||||
offerCount: sql`EXCLUDED.offer_count`,
|
||||
candidateSellerCount: sql`EXCLUDED.candidate_seller_count`,
|
||||
matchedSellerCount: sql`EXCLUDED.matched_seller_count`,
|
||||
fetchedAt: sql`EXCLUDED.fetched_at`,
|
||||
rawProductJson: sql`EXCLUDED.raw_product_json`,
|
||||
},
|
||||
});
|
||||
|
||||
const row = database
|
||||
.query(
|
||||
`SELECT id FROM stalker_asin_scans WHERE run_id = ? AND source_asin = ?`,
|
||||
)
|
||||
.get(runId, result.asin) as { id: number } | null;
|
||||
const [row] = await tx
|
||||
.select({ id: stalkerAsinScans.id })
|
||||
.from(stalkerAsinScans)
|
||||
.where(
|
||||
sql`${stalkerAsinScans.runId} = ${runId} AND ${stalkerAsinScans.sourceAsin} = ${result.asin}`,
|
||||
);
|
||||
if (!row)
|
||||
throw new Error(`Failed to load stalker scan row for ${result.asin}`);
|
||||
return row.id;
|
||||
}
|
||||
|
||||
function upsertSeller(
|
||||
database: Database,
|
||||
async function upsertSeller(
|
||||
tx: Parameters<Parameters<typeof db.transaction>[0]>[0],
|
||||
seller: StalkerSeller,
|
||||
fetchedAt: string,
|
||||
): void {
|
||||
database
|
||||
.prepare(
|
||||
`INSERT INTO stalker_sellers (
|
||||
seller_id, seller_name, rating, rating_count, storefront_asin_total,
|
||||
persisted_inventory_sample_count, last_updated_at, raw_seller_json
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(seller_id) DO UPDATE SET
|
||||
seller_name = excluded.seller_name,
|
||||
rating = excluded.rating,
|
||||
rating_count = excluded.rating_count,
|
||||
storefront_asin_total = excluded.storefront_asin_total,
|
||||
persisted_inventory_sample_count = excluded.persisted_inventory_sample_count,
|
||||
last_updated_at = excluded.last_updated_at,
|
||||
raw_seller_json = excluded.raw_seller_json`,
|
||||
)
|
||||
.run(
|
||||
seller.sellerId,
|
||||
seller.sellerName,
|
||||
seller.rating,
|
||||
seller.ratingCount,
|
||||
seller.storefrontAsinTotal,
|
||||
seller.storefrontItems.length,
|
||||
fetchedAt,
|
||||
JSON.stringify(seller.rawSeller),
|
||||
);
|
||||
fetchedAt: Date,
|
||||
): Promise<void> {
|
||||
await tx
|
||||
.insert(sellers)
|
||||
.values({
|
||||
sellerId: seller.sellerId,
|
||||
sellerName: seller.sellerName,
|
||||
rating: seller.rating,
|
||||
ratingCount: seller.ratingCount,
|
||||
storefrontAsinTotal: seller.storefrontAsinTotal,
|
||||
persistedInventorySampleCount: seller.storefrontItems.length,
|
||||
lastUpdatedAt: fetchedAt,
|
||||
rawSellerJson: JSON.stringify(seller.rawSeller),
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: sellers.sellerId,
|
||||
set: {
|
||||
sellerName: sql`EXCLUDED.seller_name`,
|
||||
rating: sql`EXCLUDED.rating`,
|
||||
ratingCount: sql`EXCLUDED.rating_count`,
|
||||
storefrontAsinTotal: sql`EXCLUDED.storefront_asin_total`,
|
||||
persistedInventorySampleCount: sql`EXCLUDED.persisted_inventory_sample_count`,
|
||||
lastUpdatedAt: sql`EXCLUDED.last_updated_at`,
|
||||
rawSellerJson: sql`EXCLUDED.raw_seller_json`,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function upsertAsinSeller(
|
||||
database: Database,
|
||||
async function upsertAsinSeller(
|
||||
tx: Parameters<Parameters<typeof db.transaction>[0]>[0],
|
||||
scanId: number,
|
||||
seller: StalkerSeller,
|
||||
offer: StalkerOffer,
|
||||
): void {
|
||||
database
|
||||
.prepare(
|
||||
`INSERT INTO stalker_asin_sellers (
|
||||
scan_id, seller_id, offer_price, condition, is_fba, stock,
|
||||
seller_rating, seller_rating_count, raw_offer_json
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(scan_id, seller_id) DO UPDATE SET
|
||||
offer_price = excluded.offer_price,
|
||||
condition = excluded.condition,
|
||||
is_fba = excluded.is_fba,
|
||||
stock = excluded.stock,
|
||||
seller_rating = excluded.seller_rating,
|
||||
seller_rating_count = excluded.seller_rating_count,
|
||||
raw_offer_json = excluded.raw_offer_json`,
|
||||
)
|
||||
.run(
|
||||
): Promise<void> {
|
||||
await tx
|
||||
.insert(stalkerAsinSellers)
|
||||
.values({
|
||||
scanId,
|
||||
seller.sellerId,
|
||||
offer.offerPrice,
|
||||
offer.condition,
|
||||
offer.isFba == null ? null : offer.isFba ? 1 : 0,
|
||||
offer.stock,
|
||||
seller.rating,
|
||||
seller.ratingCount,
|
||||
JSON.stringify(offer.rawOffer),
|
||||
);
|
||||
sellerId: seller.sellerId,
|
||||
offerPrice: offer.offerPrice,
|
||||
condition: offer.condition,
|
||||
isFba: offer.isFba,
|
||||
stock: offer.stock,
|
||||
sellerRating: seller.rating,
|
||||
sellerRatingCount: seller.ratingCount,
|
||||
rawOfferJson: JSON.stringify(offer.rawOffer),
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: [stalkerAsinSellers.scanId, stalkerAsinSellers.sellerId],
|
||||
set: {
|
||||
offerPrice: sql`EXCLUDED.offer_price`,
|
||||
condition: sql`EXCLUDED.condition`,
|
||||
isFba: sql`EXCLUDED.is_fba`,
|
||||
stock: sql`EXCLUDED.stock`,
|
||||
sellerRating: sql`EXCLUDED.seller_rating`,
|
||||
sellerRatingCount: sql`EXCLUDED.seller_rating_count`,
|
||||
rawOfferJson: sql`EXCLUDED.raw_offer_json`,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function upsertSellerInventory(
|
||||
database: Database,
|
||||
async function upsertSellerInventory(
|
||||
tx: Parameters<Parameters<typeof db.transaction>[0]>[0],
|
||||
runId: number,
|
||||
seller: StalkerSeller,
|
||||
fetchedAt: string,
|
||||
): void {
|
||||
const insert = database.prepare(
|
||||
`INSERT INTO stalker_seller_inventory (
|
||||
run_id, seller_id, asin, can_sell, sellability_status,
|
||||
sellability_reason, product_title, brand, category_tree, current_price,
|
||||
avg_price_90d, sales_rank, monthly_sold, seller_count, amazon_is_seller,
|
||||
raw_product_json, last_seen_at, raw_inventory_json
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(run_id, seller_id, asin) DO UPDATE SET
|
||||
can_sell = excluded.can_sell,
|
||||
sellability_status = excluded.sellability_status,
|
||||
sellability_reason = excluded.sellability_reason,
|
||||
product_title = excluded.product_title,
|
||||
brand = excluded.brand,
|
||||
category_tree = excluded.category_tree,
|
||||
current_price = excluded.current_price,
|
||||
avg_price_90d = excluded.avg_price_90d,
|
||||
sales_rank = excluded.sales_rank,
|
||||
monthly_sold = excluded.monthly_sold,
|
||||
seller_count = excluded.seller_count,
|
||||
amazon_is_seller = excluded.amazon_is_seller,
|
||||
raw_product_json = excluded.raw_product_json,
|
||||
last_seen_at = excluded.last_seen_at,
|
||||
raw_inventory_json = excluded.raw_inventory_json`,
|
||||
fetchedAt: Date,
|
||||
): Promise<void> {
|
||||
const items = seller.storefrontItems.filter(
|
||||
(item) =>
|
||||
item.sellability?.canSell === true &&
|
||||
item.sellability.sellabilityStatus === "available",
|
||||
);
|
||||
|
||||
for (const item of seller.storefrontItems) {
|
||||
if (
|
||||
item.sellability?.canSell !== true ||
|
||||
item.sellability.sellabilityStatus !== "available"
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
if (items.length === 0) return;
|
||||
|
||||
insert.run(
|
||||
await tx
|
||||
.insert(stalkerSellerInventory)
|
||||
.values(
|
||||
items.map((item) => ({
|
||||
runId,
|
||||
seller.sellerId,
|
||||
item.asin,
|
||||
item.sellability?.canSell == null
|
||||
? null
|
||||
: item.sellability.canSell
|
||||
? 1
|
||||
: 0,
|
||||
item.sellability?.sellabilityStatus ?? null,
|
||||
item.sellability?.sellabilityReason ?? null,
|
||||
item.productDetails?.title ?? null,
|
||||
item.productDetails?.brand ?? null,
|
||||
item.productDetails
|
||||
sellerId: seller.sellerId,
|
||||
asin: item.asin,
|
||||
canSell: item.sellability?.canSell ?? null,
|
||||
sellabilityStatus: item.sellability?.sellabilityStatus ?? null,
|
||||
sellabilityReason: item.sellability?.sellabilityReason ?? null,
|
||||
productTitle: item.productDetails?.title ?? null,
|
||||
brand: item.productDetails?.brand ?? null,
|
||||
categoryTree: item.productDetails
|
||||
? JSON.stringify(item.productDetails.categoryTree)
|
||||
: null,
|
||||
item.productDetails?.currentPrice ?? null,
|
||||
item.productDetails?.avgPrice90 ?? null,
|
||||
item.productDetails?.salesRank ?? null,
|
||||
item.productDetails?.monthlySold ?? null,
|
||||
item.productDetails?.sellerCount ?? null,
|
||||
item.productDetails?.amazonIsSeller == null
|
||||
? null
|
||||
: item.productDetails.amazonIsSeller
|
||||
? 1
|
||||
: 0,
|
||||
item.productDetails
|
||||
currentPrice: item.productDetails?.currentPrice ?? null,
|
||||
avgPrice90d: item.productDetails?.avgPrice90 ?? null,
|
||||
salesRank: item.productDetails?.salesRank ?? null,
|
||||
monthlySold: item.productDetails?.monthlySold ?? null,
|
||||
sellerCount: item.productDetails?.sellerCount ?? null,
|
||||
amazonIsSeller: item.productDetails?.amazonIsSeller ?? null,
|
||||
rawProductJson: item.productDetails
|
||||
? JSON.stringify(item.productDetails.rawProduct)
|
||||
: null,
|
||||
fetchedAt,
|
||||
JSON.stringify(item.rawInventory),
|
||||
);
|
||||
}
|
||||
lastSeenAt: fetchedAt,
|
||||
rawInventoryJson: JSON.stringify(item.rawInventory),
|
||||
})),
|
||||
)
|
||||
.onConflictDoUpdate({
|
||||
target: [
|
||||
stalkerSellerInventory.runId,
|
||||
stalkerSellerInventory.sellerId,
|
||||
stalkerSellerInventory.asin,
|
||||
],
|
||||
set: {
|
||||
canSell: sql`EXCLUDED.can_sell`,
|
||||
sellabilityStatus: sql`EXCLUDED.sellability_status`,
|
||||
sellabilityReason: sql`EXCLUDED.sellability_reason`,
|
||||
productTitle: sql`EXCLUDED.product_title`,
|
||||
brand: sql`EXCLUDED.brand`,
|
||||
categoryTree: sql`EXCLUDED.category_tree`,
|
||||
currentPrice: sql`EXCLUDED.current_price`,
|
||||
avgPrice90d: sql`EXCLUDED.avg_price_90d`,
|
||||
salesRank: sql`EXCLUDED.sales_rank`,
|
||||
monthlySold: sql`EXCLUDED.monthly_sold`,
|
||||
sellerCount: sql`EXCLUDED.seller_count`,
|
||||
amazonIsSeller: sql`EXCLUDED.amazon_is_seller`,
|
||||
rawProductJson: sql`EXCLUDED.raw_product_json`,
|
||||
lastSeenAt: sql`EXCLUDED.last_seen_at`,
|
||||
rawInventoryJson: sql`EXCLUDED.raw_inventory_json`,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function startStalkerRun(
|
||||
database: Database,
|
||||
async function startStalkerRun(
|
||||
inputFile: string,
|
||||
totalAsins: number,
|
||||
): number {
|
||||
const result = database
|
||||
.prepare(
|
||||
`INSERT INTO stalker_runs (
|
||||
input_file, started_at, requested_asins, status
|
||||
) VALUES (?, ?, ?, ?)`,
|
||||
)
|
||||
.run(inputFile, new Date().toISOString(), totalAsins, "running");
|
||||
|
||||
return result.lastInsertRowid as number;
|
||||
): Promise<number> {
|
||||
const [row] = await db
|
||||
.insert(stalkerRuns)
|
||||
.values({
|
||||
inputFile,
|
||||
startedAt: new Date(),
|
||||
requestedAsins: totalAsins,
|
||||
status: "running",
|
||||
})
|
||||
.returning({ id: stalkerRuns.id });
|
||||
if (!row) throw new Error("Failed to insert stalker run record.");
|
||||
return row.id;
|
||||
}
|
||||
|
||||
function startStalkerAnalysisRun(
|
||||
database: Database,
|
||||
inputFile: string,
|
||||
): number {
|
||||
const result = database
|
||||
.prepare(
|
||||
`INSERT INTO category_analysis_runs (
|
||||
category_id, category_label, run_timestamp, top_asins_checked,
|
||||
available_asins, fba_count, fbm_count, skip_count, status, error_message
|
||||
) VALUES (?, ?, ?, 0, 0, 0, 0, 0, 'running', NULL)`,
|
||||
)
|
||||
.run(0, `Stalker: ${path.basename(inputFile)}`, new Date().toISOString());
|
||||
|
||||
return result.lastInsertRowid as number;
|
||||
async function startStalkerAnalysisRun(inputFile: string): Promise<number> {
|
||||
const [row] = await db
|
||||
.insert(runs)
|
||||
.values({
|
||||
type: "category_analysis",
|
||||
categoryId: 0,
|
||||
categoryLabel: `Stalker: ${path.basename(inputFile)}`,
|
||||
topAsinsChecked: 0,
|
||||
availableAsins: 0,
|
||||
fbaCount: 0,
|
||||
fbmCount: 0,
|
||||
skipCount: 0,
|
||||
status: "running",
|
||||
startedAt: new Date(),
|
||||
})
|
||||
.returning({ id: runs.id });
|
||||
if (!row) throw new Error("Failed to insert stalker analysis run record.");
|
||||
return row.id;
|
||||
}
|
||||
|
||||
function loadPreviouslyScannedAsins(database: Database): Set<string> {
|
||||
const rows = database
|
||||
.query(`SELECT DISTINCT source_asin FROM stalker_asin_scans`)
|
||||
.all() as Array<{ source_asin: string }>;
|
||||
return new Set(rows.map((row) => row.source_asin));
|
||||
async function loadPreviouslyScannedAsins(): Promise<Set<string>> {
|
||||
const rows = await db
|
||||
.selectDistinct({ sourceAsin: stalkerAsinScans.sourceAsin })
|
||||
.from(stalkerAsinScans);
|
||||
return new Set(rows.map((row) => row.sourceAsin));
|
||||
}
|
||||
|
||||
function loadCachedSeller(
|
||||
database: Database | null,
|
||||
async function loadCachedSeller(
|
||||
sellerId: string,
|
||||
maxAgeHours: number,
|
||||
requireStorefront: boolean,
|
||||
inventoryLimit: number,
|
||||
): StalkerSeller | null {
|
||||
if (!database || maxAgeHours <= 0) return null;
|
||||
const row = database
|
||||
.query(
|
||||
`SELECT raw_seller_json, last_updated_at, storefront_asin_total
|
||||
FROM stalker_sellers
|
||||
WHERE seller_id = ?`,
|
||||
)
|
||||
.get(sellerId) as {
|
||||
raw_seller_json: string | null;
|
||||
last_updated_at: string;
|
||||
storefront_asin_total: number | null;
|
||||
} | null;
|
||||
if (!row?.raw_seller_json) return null;
|
||||
): Promise<StalkerSeller | null> {
|
||||
if (maxAgeHours <= 0) return null;
|
||||
|
||||
const ageMs = Date.now() - new Date(row.last_updated_at).getTime();
|
||||
const [row] = await db
|
||||
.select({
|
||||
rawSellerJson: sellers.rawSellerJson,
|
||||
lastUpdatedAt: sellers.lastUpdatedAt,
|
||||
})
|
||||
.from(sellers)
|
||||
.where(eq(sellers.sellerId, sellerId))
|
||||
.limit(1);
|
||||
|
||||
if (!row?.rawSellerJson) return null;
|
||||
|
||||
const ageMs = Date.now() - new Date(row.lastUpdatedAt).getTime();
|
||||
if (!Number.isFinite(ageMs) || ageMs > maxAgeHours * 60 * 60 * 1000) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const rawSeller = JSON.parse(row.raw_seller_json) as Record<string, any>;
|
||||
const rawSeller = JSON.parse(row.rawSellerJson) as Record<string, any>;
|
||||
const parsed = parseSeller(sellerId, rawSeller, inventoryLimit);
|
||||
if (requireStorefront && parsed.storefrontAsinTotal <= 0) return null;
|
||||
return parsed;
|
||||
@@ -1128,137 +1122,92 @@ function logRunSummary(stats: StalkerRunStats, args: StalkerArgs): void {
|
||||
);
|
||||
}
|
||||
|
||||
function refreshStalkerRun(
|
||||
database: Database,
|
||||
async function refreshStalkerRun(
|
||||
runId: number,
|
||||
stats: StalkerRunStats,
|
||||
status: string,
|
||||
): void {
|
||||
database
|
||||
.prepare(
|
||||
`UPDATE stalker_runs
|
||||
SET scanned_asins = ?,
|
||||
source_asins_with_matches = ?,
|
||||
candidate_sellers = ?,
|
||||
qualifying_sellers = ?,
|
||||
matched_sellers = ?,
|
||||
seller_metadata_requests = ?,
|
||||
seller_storefront_requests = ?,
|
||||
inventory_sellability_checked_asins = ?,
|
||||
inventory_sellability_available_asins = ?,
|
||||
inventory_sellability_excluded_asins = ?,
|
||||
persisted_inventory_asins = ?,
|
||||
status = ?,
|
||||
completed_at = CASE WHEN ? = 'running' THEN completed_at ELSE ? END
|
||||
WHERE id = ?`,
|
||||
)
|
||||
.run(
|
||||
stats.scannedAsins,
|
||||
stats.sourceAsinsWithMatches,
|
||||
stats.candidateSellers,
|
||||
stats.qualifyingSellers,
|
||||
stats.matchedSellers,
|
||||
stats.sellerMetadataRequests,
|
||||
stats.sellerStorefrontRequests,
|
||||
stats.inventorySellabilityCheckedAsins,
|
||||
): Promise<void> {
|
||||
await db
|
||||
.update(stalkerRuns)
|
||||
.set({
|
||||
scannedAsins: stats.scannedAsins,
|
||||
sourceAsinsWithMatches: stats.sourceAsinsWithMatches,
|
||||
candidateSellers: stats.candidateSellers,
|
||||
qualifyingSellers: stats.qualifyingSellers,
|
||||
matchedSellers: stats.matchedSellers,
|
||||
sellerMetadataRequests: stats.sellerMetadataRequests,
|
||||
sellerStorefrontRequests: stats.sellerStorefrontRequests,
|
||||
inventorySellabilityCheckedAsins: stats.inventorySellabilityCheckedAsins,
|
||||
inventorySellabilityAvailableAsins:
|
||||
stats.inventorySellabilityAvailableAsins,
|
||||
stats.inventorySellabilityExcludedAsins,
|
||||
stats.persistedInventoryAsins,
|
||||
inventorySellabilityExcludedAsins: stats.inventorySellabilityExcludedAsins,
|
||||
persistedInventoryAsins: stats.persistedInventoryAsins,
|
||||
status,
|
||||
status,
|
||||
new Date().toISOString(),
|
||||
runId,
|
||||
);
|
||||
...(status !== "running" ? { completedAt: new Date() } : {}),
|
||||
})
|
||||
.where(eq(stalkerRuns.id, runId));
|
||||
}
|
||||
|
||||
function finishStalkerRunWithError(
|
||||
database: Database,
|
||||
async function finishStalkerRunWithError(
|
||||
runId: number,
|
||||
stats: StalkerRunStats,
|
||||
errorMessage: string,
|
||||
): void {
|
||||
database
|
||||
.prepare(
|
||||
`UPDATE stalker_runs
|
||||
SET scanned_asins = ?,
|
||||
source_asins_with_matches = ?,
|
||||
candidate_sellers = ?,
|
||||
qualifying_sellers = ?,
|
||||
matched_sellers = ?,
|
||||
seller_metadata_requests = ?,
|
||||
seller_storefront_requests = ?,
|
||||
inventory_sellability_checked_asins = ?,
|
||||
inventory_sellability_available_asins = ?,
|
||||
inventory_sellability_excluded_asins = ?,
|
||||
persisted_inventory_asins = ?,
|
||||
status = 'failed',
|
||||
error_message = ?,
|
||||
completed_at = ?
|
||||
WHERE id = ?`,
|
||||
)
|
||||
.run(
|
||||
stats.scannedAsins,
|
||||
stats.sourceAsinsWithMatches,
|
||||
stats.candidateSellers,
|
||||
stats.qualifyingSellers,
|
||||
stats.matchedSellers,
|
||||
stats.sellerMetadataRequests,
|
||||
stats.sellerStorefrontRequests,
|
||||
stats.inventorySellabilityCheckedAsins,
|
||||
): Promise<void> {
|
||||
await db
|
||||
.update(stalkerRuns)
|
||||
.set({
|
||||
scannedAsins: stats.scannedAsins,
|
||||
sourceAsinsWithMatches: stats.sourceAsinsWithMatches,
|
||||
candidateSellers: stats.candidateSellers,
|
||||
qualifyingSellers: stats.qualifyingSellers,
|
||||
matchedSellers: stats.matchedSellers,
|
||||
sellerMetadataRequests: stats.sellerMetadataRequests,
|
||||
sellerStorefrontRequests: stats.sellerStorefrontRequests,
|
||||
inventorySellabilityCheckedAsins: stats.inventorySellabilityCheckedAsins,
|
||||
inventorySellabilityAvailableAsins:
|
||||
stats.inventorySellabilityAvailableAsins,
|
||||
stats.inventorySellabilityExcludedAsins,
|
||||
stats.persistedInventoryAsins,
|
||||
inventorySellabilityExcludedAsins: stats.inventorySellabilityExcludedAsins,
|
||||
persistedInventoryAsins: stats.persistedInventoryAsins,
|
||||
status: "failed",
|
||||
errorMessage,
|
||||
new Date().toISOString(),
|
||||
runId,
|
||||
);
|
||||
completedAt: new Date(),
|
||||
})
|
||||
.where(eq(stalkerRuns.id, runId));
|
||||
}
|
||||
|
||||
function finishStalkerAnalysisRun(
|
||||
database: Database,
|
||||
async function finishStalkerAnalysisRun(
|
||||
runId: number,
|
||||
status: "completed" | "failed",
|
||||
errorMessage: string | null = null,
|
||||
): void {
|
||||
const stats = database
|
||||
.query(
|
||||
`SELECT
|
||||
): Promise<void> {
|
||||
const [stats] = await db.execute(
|
||||
sql<{
|
||||
total: string;
|
||||
fba: string | null;
|
||||
fbm: string | null;
|
||||
skip: string | null;
|
||||
}>`SELECT
|
||||
COUNT(*) AS total,
|
||||
SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba,
|
||||
SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm,
|
||||
SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip
|
||||
FROM product_analysis_results
|
||||
WHERE run_id = ?`,
|
||||
)
|
||||
.get(runId) as {
|
||||
total: number;
|
||||
fba: number | null;
|
||||
fbm: number | null;
|
||||
skip: number | null;
|
||||
};
|
||||
FROM category_product_results
|
||||
WHERE run_id = ${runId}`,
|
||||
);
|
||||
|
||||
database
|
||||
.prepare(
|
||||
`UPDATE category_analysis_runs
|
||||
SET top_asins_checked = ?,
|
||||
available_asins = ?,
|
||||
fba_count = ?,
|
||||
fbm_count = ?,
|
||||
skip_count = ?,
|
||||
status = ?,
|
||||
error_message = ?
|
||||
WHERE id = ?`,
|
||||
)
|
||||
.run(
|
||||
stats.total ?? 0,
|
||||
stats.total ?? 0,
|
||||
stats.fba ?? 0,
|
||||
stats.fbm ?? 0,
|
||||
stats.skip ?? 0,
|
||||
await db
|
||||
.update(runs)
|
||||
.set({
|
||||
topAsinsChecked: Number(stats?.total ?? 0),
|
||||
availableAsins: Number(stats?.total ?? 0),
|
||||
fbaCount: Number(stats?.fba ?? 0),
|
||||
fbmCount: Number(stats?.fbm ?? 0),
|
||||
skipCount: Number(stats?.skip ?? 0),
|
||||
status,
|
||||
errorMessage,
|
||||
runId,
|
||||
);
|
||||
completedAt: new Date(),
|
||||
})
|
||||
.where(eq(runs.id, runId));
|
||||
}
|
||||
|
||||
function normalizeSellerResponse(
|
||||
@@ -1492,7 +1441,6 @@ function collectPersistedInventoryAsins(result: StalkerAsinResult): string[] {
|
||||
}
|
||||
|
||||
async function runSellableAnalysisChild(
|
||||
dbPath: string,
|
||||
stalkerRunId: number,
|
||||
analysisRunId: number,
|
||||
asins: string[],
|
||||
@@ -1502,8 +1450,6 @@ async function runSellableAnalysisChild(
|
||||
"bun",
|
||||
"run",
|
||||
"src/stalker-analyze.ts",
|
||||
"--db",
|
||||
dbPath,
|
||||
"--stalker-run-id",
|
||||
String(stalkerRunId),
|
||||
"--analysis-run-id",
|
||||
@@ -1660,8 +1606,5 @@ if (import.meta.main) {
|
||||
.catch((error) => {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exitCode = 1;
|
||||
})
|
||||
.finally(() => {
|
||||
closeDb();
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,8 +1,41 @@
|
||||
import { test, expect, beforeAll, afterAll, beforeEach, mock } from "bun:test";
|
||||
import { Database } from "bun:sqlite";
|
||||
import { getDb, initDb, closeDb } from "./database.ts";
|
||||
import path from "node:path";
|
||||
import { rmSync, mkdirSync } from "node:fs";
|
||||
import { test, expect, beforeEach, mock } from "bun:test";
|
||||
|
||||
let nextId = 0;
|
||||
|
||||
function chainable(resolveWith: any[] = []): any {
|
||||
const p: any = Promise.resolve(resolveWith);
|
||||
p.limit = (_n: any) => chainable(resolveWith);
|
||||
p.where = (_cond: any) => chainable(resolveWith);
|
||||
p.from = (_table: any) => chainable(resolveWith);
|
||||
return p;
|
||||
}
|
||||
|
||||
const makeMockDb = (): any => ({
|
||||
insert: (_table: any) => ({
|
||||
values: (_vals: any) => ({
|
||||
returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]),
|
||||
onConflictDoUpdate: (_conf: any) => Promise.resolve([]),
|
||||
}),
|
||||
}),
|
||||
update: (_table: any) => ({
|
||||
set: (_vals: any) => ({
|
||||
where: (_cond: any) => Promise.resolve([]),
|
||||
}),
|
||||
}),
|
||||
select: (_sel?: any) => ({
|
||||
from: (_table: any) => ({
|
||||
where: (_cond: any) => chainable(),
|
||||
limit: (_n: any) => chainable(),
|
||||
}),
|
||||
}),
|
||||
selectDistinct: (_sel: any) => ({
|
||||
from: (_table: any) => chainable(),
|
||||
}),
|
||||
execute: (_query: any) => Promise.resolve([]),
|
||||
transaction: async (fn: (tx: any) => Promise<any>) => fn(makeMockDb()),
|
||||
});
|
||||
|
||||
mock.module("./db/index.ts", () => ({ db: makeMockDb(), client: {} }));
|
||||
|
||||
const fetchSellabilityBatchMock = mock(async (asins: string[]) => {
|
||||
return new Map<string, any>(
|
||||
@@ -60,51 +93,23 @@ mock.module("./llm.ts", () => ({
|
||||
|
||||
const modulePromise = import("./top-monthly-sold-by-category.ts");
|
||||
|
||||
const DB_TEST_PATH = path.join(
|
||||
process.cwd(),
|
||||
"test_output",
|
||||
"test_monthly_sold_analysis.sqlite",
|
||||
);
|
||||
|
||||
let db: Database;
|
||||
let processCategory: (
|
||||
db: Database,
|
||||
runId: number,
|
||||
category: any,
|
||||
perCategoryTop: number,
|
||||
categoryCandidatePool: number,
|
||||
minMonthlySold: number,
|
||||
) => Promise<any>;
|
||||
let insertCategoryRunSummary: (
|
||||
db: Database,
|
||||
summary: any,
|
||||
runTimestamp: string,
|
||||
) => Promise<number>;
|
||||
let insertCategoryRunSummary: (summary: any, runTimestamp: string) => Promise<number>;
|
||||
let originalFetch: typeof globalThis.fetch;
|
||||
|
||||
beforeAll(async () => {
|
||||
const mod = await modulePromise;
|
||||
processCategory = mod.processCategory;
|
||||
insertCategoryRunSummary = mod.insertCategoryRunSummary;
|
||||
|
||||
rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true });
|
||||
mkdirSync(path.dirname(DB_TEST_PATH), { recursive: true });
|
||||
initDb(DB_TEST_PATH);
|
||||
db = getDb(DB_TEST_PATH);
|
||||
|
||||
originalFetch = globalThis.fetch;
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
closeDb();
|
||||
rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true });
|
||||
});
|
||||
const mod = await modulePromise;
|
||||
processCategory = mod.processCategory;
|
||||
insertCategoryRunSummary = mod.insertCategoryRunSummary;
|
||||
originalFetch = globalThis.fetch;
|
||||
|
||||
beforeEach(() => {
|
||||
db.run("DELETE FROM product_analysis_results");
|
||||
db.run("DELETE FROM category_analysis_runs");
|
||||
|
||||
nextId = 0;
|
||||
globalThis.fetch = mock(async (input: string | URL | Request) => {
|
||||
const rawUrl =
|
||||
typeof input === "string"
|
||||
@@ -140,25 +145,8 @@ beforeEach(() => {
|
||||
monthlySold: 600,
|
||||
stats: {
|
||||
current: [
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
1000,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
2,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
2599,
|
||||
null, null, null, 1000, null, null, null, null, null, null, null, 2,
|
||||
null, null, null, null, null, null, 2599,
|
||||
],
|
||||
avg: [2400, null, null, 1200],
|
||||
},
|
||||
@@ -171,25 +159,8 @@ beforeEach(() => {
|
||||
monthlySold: 250,
|
||||
stats: {
|
||||
current: [
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
2000,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
3,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
1999,
|
||||
null, null, null, 2000, null, null, null, null, null, null, null, 3,
|
||||
null, null, null, null, null, null, 1999,
|
||||
],
|
||||
avg: [1800, null, null, 2200],
|
||||
},
|
||||
@@ -202,25 +173,8 @@ beforeEach(() => {
|
||||
monthlySold: 800,
|
||||
stats: {
|
||||
current: [
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
1500,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
1,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
2099,
|
||||
null, null, null, 1500, null, null, null, null, null, null, null, 1,
|
||||
null, null, null, null, null, null, 2099,
|
||||
],
|
||||
avg: [2000, null, null, 1800],
|
||||
},
|
||||
@@ -233,25 +187,8 @@ beforeEach(() => {
|
||||
monthlySold: 400,
|
||||
stats: {
|
||||
current: [
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
3000,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
4,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
2899,
|
||||
null, null, null, 3000, null, null, null, null, null, null, null, 4,
|
||||
null, null, null, null, null, null, 2899,
|
||||
],
|
||||
avg: [2600, null, null, 2800],
|
||||
},
|
||||
@@ -279,7 +216,6 @@ test("processCategory filters to sellable ASINs with monthly sold >= threshold a
|
||||
};
|
||||
|
||||
const runId = await insertCategoryRunSummary(
|
||||
db,
|
||||
{
|
||||
categoryId: mockCategory.id,
|
||||
categoryLabel: mockCategory.label,
|
||||
@@ -295,22 +231,16 @@ test("processCategory filters to sellable ASINs with monthly sold >= threshold a
|
||||
new Date().toISOString(),
|
||||
);
|
||||
|
||||
const summary = await processCategory(db, runId, mockCategory, 2, 4, 300);
|
||||
const summary = await processCategory(runId, mockCategory, 2, 4, 300);
|
||||
|
||||
expect(summary.status).toBe("ok");
|
||||
expect(summary.topAsinsChecked).toBe(4);
|
||||
expect(summary.availableAsins).toBe(2);
|
||||
expect(summary.results?.length).toBe(2);
|
||||
|
||||
const productResults = db
|
||||
.query(
|
||||
"SELECT asin, monthly_sold FROM product_analysis_results ORDER BY monthly_sold DESC",
|
||||
)
|
||||
.all() as Array<{ asin: string; monthly_sold: number }>;
|
||||
const asins = summary.results?.map((r: any) => r.product.record.asin) ?? [];
|
||||
expect(asins).toContain("B000000001");
|
||||
expect(asins).toContain("B000000004");
|
||||
|
||||
expect(productResults.length).toBe(2);
|
||||
expect(productResults[0]?.asin).toBe("B000000001");
|
||||
expect(productResults[0]?.monthly_sold).toBe(600);
|
||||
expect(productResults[1]?.asin).toBe("B000000004");
|
||||
expect(productResults[1]?.monthly_sold).toBe(400);
|
||||
globalThis.fetch = originalFetch;
|
||||
});
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { existsSync, mkdirSync, readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { type Database, getDb, initDb } from "./database.ts";
|
||||
import { db } from "./db/index.ts";
|
||||
import { runs, categoryProductResults } from "./db/schema.ts";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { config } from "./config.ts";
|
||||
import { analyzeProducts } from "./llm.ts";
|
||||
import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts";
|
||||
@@ -171,36 +173,32 @@ function printUsageAndExit(message: string): never {
|
||||
}
|
||||
|
||||
export async function insertCategoryRunSummary(
|
||||
db: Database,
|
||||
summary: CategoryRunSummary,
|
||||
runTimestamp: string,
|
||||
): Promise<number> {
|
||||
const query = `
|
||||
INSERT INTO category_analysis_runs (
|
||||
category_id, category_label, run_timestamp,
|
||||
top_asins_checked, available_asins,
|
||||
fba_count, fbm_count, skip_count,
|
||||
status, error_message
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
|
||||
`;
|
||||
const result = db.run(query, [
|
||||
summary.categoryId,
|
||||
summary.categoryLabel,
|
||||
runTimestamp,
|
||||
summary.topAsinsChecked,
|
||||
summary.availableAsins,
|
||||
summary.fba,
|
||||
summary.fbm,
|
||||
summary.skip,
|
||||
summary.status,
|
||||
summary.error,
|
||||
]);
|
||||
// Bun's SQLite client returns { changes: number, lastInsertRowid: number | bigint }
|
||||
return Number(result.lastInsertRowid);
|
||||
const [row] = await db
|
||||
.insert(runs)
|
||||
.values({
|
||||
type: "category_analysis",
|
||||
status: (summary.status as typeof runs.$inferInsert.status) ?? "running",
|
||||
categoryId: summary.categoryId,
|
||||
categoryLabel: summary.categoryLabel,
|
||||
topAsinsChecked: summary.topAsinsChecked,
|
||||
availableAsins: summary.availableAsins,
|
||||
totalProducts: summary.topAsinsChecked,
|
||||
fbaCount: summary.fba,
|
||||
fbmCount: summary.fbm,
|
||||
skipCount: summary.skip,
|
||||
errorMessage: summary.error || null,
|
||||
startedAt: new Date(runTimestamp),
|
||||
})
|
||||
.returning({ id: runs.id });
|
||||
|
||||
if (!row) throw new Error("Failed to insert category run.");
|
||||
return row.id;
|
||||
}
|
||||
|
||||
export async function updateCategoryRunSummary(
|
||||
db: Database,
|
||||
runId: number,
|
||||
summary: Pick<
|
||||
CategoryRunSummary,
|
||||
@@ -213,136 +211,110 @@ export async function updateCategoryRunSummary(
|
||||
| "error"
|
||||
>,
|
||||
): Promise<void> {
|
||||
db.run(
|
||||
`
|
||||
UPDATE category_analysis_runs
|
||||
SET
|
||||
top_asins_checked = ?,
|
||||
available_asins = ?,
|
||||
fba_count = ?,
|
||||
fbm_count = ?,
|
||||
skip_count = ?,
|
||||
status = ?,
|
||||
error_message = ?
|
||||
WHERE id = ?
|
||||
`,
|
||||
[
|
||||
summary.topAsinsChecked,
|
||||
summary.availableAsins,
|
||||
summary.fba,
|
||||
summary.fbm,
|
||||
summary.skip,
|
||||
summary.status,
|
||||
summary.error,
|
||||
runId,
|
||||
],
|
||||
);
|
||||
await db
|
||||
.update(runs)
|
||||
.set({
|
||||
topAsinsChecked: summary.topAsinsChecked,
|
||||
availableAsins: summary.availableAsins,
|
||||
totalProducts: summary.topAsinsChecked,
|
||||
fbaCount: summary.fba,
|
||||
fbmCount: summary.fbm,
|
||||
skipCount: summary.skip,
|
||||
status: summary.status as typeof runs.$inferInsert.status,
|
||||
errorMessage: summary.error || null,
|
||||
...(summary.status !== "running" ? { completedAt: new Date() } : {}),
|
||||
})
|
||||
.where(eq(runs.id, runId));
|
||||
}
|
||||
|
||||
export async function insertProductAnalysisResults(
|
||||
db: Database,
|
||||
runId: number,
|
||||
results: AnalysisResult[],
|
||||
): Promise<void> {
|
||||
if (results.length === 0) {
|
||||
return;
|
||||
}
|
||||
if (results.length === 0) return;
|
||||
|
||||
const insertStmt = db.prepare(`
|
||||
INSERT INTO product_analysis_results (
|
||||
asin, run_id, name, brand, category, unit_cost,
|
||||
current_price, avg_price_90d, avg_price_90d_sheet,
|
||||
selling_price_sheet, sales_rank, sales_rank_avg_90d,
|
||||
seller_count, amazon_is_seller, amazon_buybox_share_pct_90d,
|
||||
monthly_sold, rank_drops_30d, rank_drops_90d,
|
||||
fba_fee, fbm_fee, referral_percent, can_sell,
|
||||
sellability_status, sellability_reason,
|
||||
verdict, confidence, reasoning, fetched_at
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?
|
||||
)
|
||||
ON CONFLICT(asin) DO UPDATE SET
|
||||
run_id = excluded.run_id,
|
||||
name = excluded.name,
|
||||
brand = excluded.brand,
|
||||
category = excluded.category,
|
||||
unit_cost = excluded.unit_cost,
|
||||
current_price = excluded.current_price,
|
||||
avg_price_90d = excluded.avg_price_90d,
|
||||
avg_price_90d_sheet = excluded.avg_price_90d_sheet,
|
||||
selling_price_sheet = excluded.selling_price_sheet,
|
||||
sales_rank = excluded.sales_rank,
|
||||
sales_rank_avg_90d = excluded.sales_rank_avg_90d,
|
||||
seller_count = excluded.seller_count,
|
||||
amazon_is_seller = excluded.amazon_is_seller,
|
||||
amazon_buybox_share_pct_90d = excluded.amazon_buybox_share_pct_90d,
|
||||
monthly_sold = excluded.monthly_sold,
|
||||
rank_drops_30d = excluded.rank_drops_30d,
|
||||
rank_drops_90d = excluded.rank_drops_90d,
|
||||
fba_fee = excluded.fba_fee,
|
||||
fbm_fee = excluded.fbm_fee,
|
||||
referral_percent = excluded.referral_percent,
|
||||
can_sell = excluded.can_sell,
|
||||
sellability_status = excluded.sellability_status,
|
||||
sellability_reason = excluded.sellability_reason,
|
||||
verdict = excluded.verdict,
|
||||
confidence = excluded.confidence,
|
||||
reasoning = excluded.reasoning,
|
||||
fetched_at = excluded.fetched_at;
|
||||
`);
|
||||
|
||||
db.transaction((resultsBatch: AnalysisResult[]) => {
|
||||
for (const r of resultsBatch) {
|
||||
const rows = results.map((r) => {
|
||||
const price =
|
||||
r.product.keepa?.currentPrice ??
|
||||
r.product.record.sellingPriceFromSheet ??
|
||||
r.product.spApi.estimatedSalePrice;
|
||||
const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank;
|
||||
|
||||
insertStmt.run(
|
||||
r.product.record.asin,
|
||||
return {
|
||||
asin: r.product.record.asin,
|
||||
runId,
|
||||
r.product.record.name,
|
||||
r.product.record.brand ?? null,
|
||||
name: r.product.record.name,
|
||||
brand: r.product.record.brand ?? null,
|
||||
category:
|
||||
r.product.record.category ??
|
||||
r.product.keepa?.categoryTree?.join(" > ") ??
|
||||
null,
|
||||
r.product.record.unitCost ?? null,
|
||||
price ?? null,
|
||||
r.product.keepa?.avgPrice90 ?? null,
|
||||
r.product.record.avgPrice90FromSheet ?? null,
|
||||
r.product.record.sellingPriceFromSheet ?? null,
|
||||
rank ?? null,
|
||||
r.product.keepa?.salesRankAvg90 ?? null,
|
||||
r.product.keepa?.sellerCount ?? null,
|
||||
r.product.keepa?.amazonIsSeller == null
|
||||
? null
|
||||
: r.product.keepa.amazonIsSeller
|
||||
? 1
|
||||
: 0,
|
||||
r.product.keepa?.amazonBuyboxSharePct90d ?? null,
|
||||
r.product.keepa?.monthlySold ?? null,
|
||||
r.product.keepa?.salesRankDrops30 ?? null,
|
||||
r.product.keepa?.salesRankDrops90 ?? null,
|
||||
r.product.spApi.fbaFee ?? null,
|
||||
r.product.spApi.fbmFee ?? null,
|
||||
r.product.spApi.referralFeePercent ?? null,
|
||||
unitCost: r.product.record.unitCost ?? null,
|
||||
currentPrice: price ?? null,
|
||||
avgPrice90d: r.product.keepa?.avgPrice90 ?? null,
|
||||
avgPrice90dSheet: r.product.record.avgPrice90FromSheet ?? null,
|
||||
sellingPriceSheet: r.product.record.sellingPriceFromSheet ?? null,
|
||||
salesRank: rank ?? null,
|
||||
salesRankAvg90d: r.product.keepa?.salesRankAvg90 ?? null,
|
||||
sellerCount: r.product.keepa?.sellerCount ?? null,
|
||||
amazonIsSeller: r.product.keepa?.amazonIsSeller ?? null,
|
||||
amazonBuyboxSharePct90d: r.product.keepa?.amazonBuyboxSharePct90d ?? null,
|
||||
monthlySold: r.product.keepa?.monthlySold ?? null,
|
||||
rankDrops30d: r.product.keepa?.salesRankDrops30 ?? null,
|
||||
rankDrops90d: r.product.keepa?.salesRankDrops90 ?? null,
|
||||
fbaFee: r.product.spApi.fbaFee ?? null,
|
||||
fbmFee: r.product.spApi.fbmFee ?? null,
|
||||
referralPercent: r.product.spApi.referralFeePercent ?? null,
|
||||
canSell:
|
||||
r.product.spApi.canSell == null
|
||||
? "unknown"
|
||||
: r.product.spApi.canSell
|
||||
? "yes"
|
||||
: "no",
|
||||
r.product.spApi.sellabilityStatus ?? null,
|
||||
r.product.spApi.sellabilityReason ?? null,
|
||||
r.verdict.verdict,
|
||||
r.verdict.confidence,
|
||||
r.verdict.reasoning ?? null,
|
||||
r.product.fetchedAt,
|
||||
);
|
||||
}
|
||||
})(results); // Execute the transaction with the results batch
|
||||
sellabilityStatus: r.product.spApi.sellabilityStatus ?? null,
|
||||
sellabilityReason: r.product.spApi.sellabilityReason ?? null,
|
||||
verdict: r.verdict.verdict,
|
||||
confidence: r.verdict.confidence,
|
||||
reasoning: r.verdict.reasoning ?? null,
|
||||
fetchedAt: new Date(r.product.fetchedAt),
|
||||
};
|
||||
});
|
||||
|
||||
await db
|
||||
.insert(categoryProductResults)
|
||||
.values(rows)
|
||||
.onConflictDoUpdate({
|
||||
target: categoryProductResults.asin,
|
||||
set: {
|
||||
runId: sql`EXCLUDED.run_id`,
|
||||
name: sql`EXCLUDED.name`,
|
||||
brand: sql`EXCLUDED.brand`,
|
||||
category: sql`EXCLUDED.category`,
|
||||
unitCost: sql`EXCLUDED.unit_cost`,
|
||||
currentPrice: sql`EXCLUDED.current_price`,
|
||||
avgPrice90d: sql`EXCLUDED.avg_price_90d`,
|
||||
avgPrice90dSheet: sql`EXCLUDED.avg_price_90d_sheet`,
|
||||
sellingPriceSheet: sql`EXCLUDED.selling_price_sheet`,
|
||||
salesRank: sql`EXCLUDED.sales_rank`,
|
||||
salesRankAvg90d: sql`EXCLUDED.sales_rank_avg_90d`,
|
||||
sellerCount: sql`EXCLUDED.seller_count`,
|
||||
amazonIsSeller: sql`EXCLUDED.amazon_is_seller`,
|
||||
amazonBuyboxSharePct90d: sql`EXCLUDED.amazon_buybox_share_pct_90d`,
|
||||
monthlySold: sql`EXCLUDED.monthly_sold`,
|
||||
rankDrops30d: sql`EXCLUDED.rank_drops_30d`,
|
||||
rankDrops90d: sql`EXCLUDED.rank_drops_90d`,
|
||||
fbaFee: sql`EXCLUDED.fba_fee`,
|
||||
fbmFee: sql`EXCLUDED.fbm_fee`,
|
||||
referralPercent: sql`EXCLUDED.referral_percent`,
|
||||
canSell: sql`EXCLUDED.can_sell`,
|
||||
sellabilityStatus: sql`EXCLUDED.sellability_status`,
|
||||
sellabilityReason: sql`EXCLUDED.sellability_reason`,
|
||||
verdict: sql`EXCLUDED.verdict`,
|
||||
confidence: sql`EXCLUDED.confidence`,
|
||||
reasoning: sql`EXCLUDED.reasoning`,
|
||||
fetchedAt: sql`EXCLUDED.fetched_at`,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function loadCategoryBlacklist(filePath: string): Set<number> {
|
||||
@@ -1067,7 +1039,6 @@ function buildEnrichedProducts(
|
||||
}
|
||||
|
||||
export async function processCategory(
|
||||
db: Database,
|
||||
runId: number,
|
||||
category: CategoryInfo,
|
||||
perCategoryTop: number,
|
||||
@@ -1083,7 +1054,7 @@ export async function processCategory(
|
||||
);
|
||||
if (topAsins.length === 0) {
|
||||
log("info", " Keepa returned no ASINs for this category.");
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: 0,
|
||||
availableAsins: 0,
|
||||
fba: 0,
|
||||
@@ -1127,7 +1098,7 @@ export async function processCategory(
|
||||
` Sellable ASINs: ${availableAsins.length}/${uniqueTopAsins.length}`,
|
||||
);
|
||||
if (availableAsins.length === 0) {
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: uniqueTopAsins.length,
|
||||
availableAsins: 0,
|
||||
fba: 0,
|
||||
@@ -1164,7 +1135,7 @@ export async function processCategory(
|
||||
);
|
||||
|
||||
if (selectedAsins.length === 0) {
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: uniqueTopAsins.length,
|
||||
availableAsins: 0,
|
||||
fba: 0,
|
||||
@@ -1231,7 +1202,7 @@ export async function processCategory(
|
||||
},
|
||||
}));
|
||||
|
||||
await insertProductAnalysisResults(db, runId, batchResults);
|
||||
await insertProductAnalysisResults(runId, batchResults);
|
||||
|
||||
for (const result of batchResults) {
|
||||
results.push(result);
|
||||
@@ -1244,7 +1215,7 @@ export async function processCategory(
|
||||
}
|
||||
}
|
||||
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: uniqueTopAsins.length,
|
||||
availableAsins: selectedAsins.length,
|
||||
fba,
|
||||
@@ -1264,7 +1235,7 @@ export async function processCategory(
|
||||
}
|
||||
}
|
||||
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: uniqueTopAsins.length,
|
||||
availableAsins: selectedAsins.length,
|
||||
fba,
|
||||
@@ -1293,10 +1264,6 @@ export async function main(): Promise<void> {
|
||||
assertSpApiPrerequisites();
|
||||
|
||||
mkdirSync(args.outputDir, { recursive: true });
|
||||
const DB_PATH =
|
||||
process.env.RESULTS_DB_PATH || path.join(process.cwd(), "db", "results.db");
|
||||
initDb(DB_PATH);
|
||||
const db = getDb(DB_PATH);
|
||||
|
||||
log("info", "Starting per-category monthly-sold pipeline");
|
||||
log("info", `Marketplace: ${config.spApiMarketplaceId}`);
|
||||
@@ -1333,7 +1300,6 @@ export async function main(): Promise<void> {
|
||||
let runId: number | undefined;
|
||||
try {
|
||||
runId = await insertCategoryRunSummary(
|
||||
db,
|
||||
{
|
||||
categoryId: category.id,
|
||||
categoryLabel: category.label,
|
||||
@@ -1350,7 +1316,6 @@ export async function main(): Promise<void> {
|
||||
);
|
||||
|
||||
categorySummary = await processCategory(
|
||||
db,
|
||||
runId,
|
||||
category,
|
||||
args.perCategoryTop,
|
||||
@@ -1382,7 +1347,7 @@ export async function main(): Promise<void> {
|
||||
results: [],
|
||||
};
|
||||
if (runId) {
|
||||
await updateCategoryRunSummary(db, runId, {
|
||||
await updateCategoryRunSummary(runId, {
|
||||
topAsinsChecked: 0,
|
||||
availableAsins: 0,
|
||||
fba: 0,
|
||||
|
||||
@@ -15,7 +15,6 @@ import {
|
||||
startRunInDb,
|
||||
type RunCounts,
|
||||
} from "./writer.ts";
|
||||
import { initDb, closeDb } from "./database.ts";
|
||||
import { connectCache, disconnectCache } from "./cache.ts";
|
||||
import { scoreSupplierProduct, resolveSupplierSalePrice } from "./supplier-scoring.ts";
|
||||
import {
|
||||
@@ -31,7 +30,6 @@ import type {
|
||||
UpcLookupDetail,
|
||||
} from "./types.ts";
|
||||
|
||||
const DB_PATH = process.env.RESULTS_DB_PATH || path.join("db", "results.db");
|
||||
const DEFAULT_INPUT_BATCH_SIZE = 200;
|
||||
const DEFAULT_UPC_LOOKUP_BATCH_SIZE = 100;
|
||||
const DEFAULT_PRICING_CONCURRENCY = 5;
|
||||
@@ -48,7 +46,6 @@ export type UpcFileAnalysisOptions = {
|
||||
|
||||
export type UpcFileAnalysisSummary = {
|
||||
runId: number;
|
||||
dbPath: string;
|
||||
inputFile: string;
|
||||
outputFile?: string;
|
||||
processedRows: number;
|
||||
@@ -339,7 +336,6 @@ function summarizeSupplierResults(
|
||||
export async function runUpcFileAnalysis(
|
||||
options: UpcFileAnalysisOptions,
|
||||
): Promise<UpcFileAnalysisSummary> {
|
||||
const dbPath = options.dbPath ?? DB_PATH;
|
||||
const inputBatchSize = Math.max(
|
||||
1,
|
||||
options.inputBatchSize ?? DEFAULT_INPUT_BATCH_SIZE,
|
||||
@@ -355,8 +351,6 @@ export async function runUpcFileAnalysis(
|
||||
if (manageResources) {
|
||||
console.log("Connecting to Redis...");
|
||||
await connectCache();
|
||||
console.log("Initializing SQLite database...");
|
||||
initDb(dbPath);
|
||||
}
|
||||
|
||||
const unresolvedByStatus = createStatusCounter();
|
||||
@@ -365,7 +359,7 @@ export async function runUpcFileAnalysis(
|
||||
let processedRows = 0;
|
||||
let matchedRows = 0;
|
||||
|
||||
const runId = startRunInDb(dbPath, options.inputFile, outputFile);
|
||||
const runId = await startRunInDb(options.inputFile, outputFile);
|
||||
|
||||
try {
|
||||
const readerSummary = await processUpcFileInBatches(
|
||||
@@ -481,7 +475,7 @@ export async function runUpcFileAnalysis(
|
||||
}
|
||||
}
|
||||
|
||||
appendSupplierResultsToRun(dbPath, runId, batchResults);
|
||||
await appendSupplierResultsToRun(runId, batchResults);
|
||||
allResults.push(...batchResults);
|
||||
},
|
||||
{
|
||||
@@ -490,7 +484,7 @@ export async function runUpcFileAnalysis(
|
||||
},
|
||||
);
|
||||
|
||||
const runCounts = refreshRunCountsInDb(dbPath, runId);
|
||||
const runCounts = await refreshRunCountsInDb(runId);
|
||||
|
||||
const exportSummary = summarizeSupplierResults(allResults, unresolvedByStatus);
|
||||
await writeSupplierWorkbook(outputFile, allResults, exportSummary);
|
||||
@@ -522,7 +516,6 @@ export async function runUpcFileAnalysis(
|
||||
|
||||
return {
|
||||
runId,
|
||||
dbPath,
|
||||
inputFile: options.inputFile,
|
||||
outputFile,
|
||||
processedRows,
|
||||
@@ -540,7 +533,6 @@ export async function runUpcFileAnalysis(
|
||||
} finally {
|
||||
if (manageResources) {
|
||||
await disconnectCache();
|
||||
closeDb();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
349
src/writer.ts
349
src/writer.ts
@@ -1,4 +1,6 @@
|
||||
import { getDb } from "./database.ts";
|
||||
import { db } from "./db/index.ts";
|
||||
import { runs, analysisResults } from "./db/schema.ts";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import type { AnalysisResult, SupplierAnalysisResult } from "./types.ts";
|
||||
import { mkdirSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
@@ -84,16 +86,15 @@ function buildRow(r: AnalysisResult) {
|
||||
};
|
||||
}
|
||||
|
||||
export function writeResultsToDb(
|
||||
export async function writeResultsToDb(
|
||||
results: AnalysisResult[],
|
||||
dbPath: string,
|
||||
inputFile: string,
|
||||
outputFile: string | undefined,
|
||||
): void {
|
||||
): Promise<void> {
|
||||
const runCounts = computeRunCountsFromResults(results);
|
||||
const runId = startRunInDb(dbPath, inputFile, outputFile, runCounts);
|
||||
appendResultsToRun(dbPath, runId, results);
|
||||
console.log(`Results written to SQLite database for run_id: ${runId}`);
|
||||
const runId = await startRunInDb(inputFile, outputFile, runCounts);
|
||||
await appendResultsToRun(runId, results);
|
||||
console.log(`Results written to database for run_id: ${runId}`);
|
||||
}
|
||||
|
||||
export function writeResultsWorkbook(
|
||||
@@ -112,8 +113,7 @@ export function writeResultsWorkbook(
|
||||
console.log(`Results workbook written: ${outputFile}`);
|
||||
}
|
||||
|
||||
export function startRunInDb(
|
||||
dbPath: string,
|
||||
export async function startRunInDb(
|
||||
inputFile: string,
|
||||
outputFile: string | undefined,
|
||||
counts: RunCounts = {
|
||||
@@ -122,153 +122,95 @@ export function startRunInDb(
|
||||
fbmCount: 0,
|
||||
skipCount: 0,
|
||||
},
|
||||
): number {
|
||||
const database = getDb(dbPath);
|
||||
const timestamp = new Date().toISOString();
|
||||
|
||||
const insertRun = database.prepare(
|
||||
`INSERT INTO runs (
|
||||
timestamp,
|
||||
input_file,
|
||||
output_file,
|
||||
total_products,
|
||||
fba_count,
|
||||
fbm_count,
|
||||
skip_count
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
||||
);
|
||||
|
||||
const runInfo = insertRun.run(
|
||||
timestamp,
|
||||
): Promise<number> {
|
||||
const [row] = await db
|
||||
.insert(runs)
|
||||
.values({
|
||||
type: "lead_analysis",
|
||||
inputFile,
|
||||
outputFile ?? null,
|
||||
counts.totalProducts,
|
||||
counts.fbaCount,
|
||||
counts.fbmCount,
|
||||
counts.skipCount,
|
||||
);
|
||||
outputFile: outputFile ?? null,
|
||||
status: "ok",
|
||||
totalProducts: counts.totalProducts,
|
||||
fbaCount: counts.fbaCount,
|
||||
fbmCount: counts.fbmCount,
|
||||
skipCount: counts.skipCount,
|
||||
startedAt: new Date(),
|
||||
completedAt: new Date(),
|
||||
})
|
||||
.returning({ id: runs.id });
|
||||
|
||||
const runId =
|
||||
(runInfo.changes as number) > 0
|
||||
? (runInfo.lastInsertRowid as number)
|
||||
: null;
|
||||
|
||||
if (runId === null) {
|
||||
throw new Error("Failed to insert run record into SQLite.");
|
||||
}
|
||||
|
||||
return runId;
|
||||
if (!row) throw new Error("Failed to insert run record.");
|
||||
return row.id;
|
||||
}
|
||||
|
||||
export function appendResultsToRun(
|
||||
dbPath: string,
|
||||
export async function appendResultsToRun(
|
||||
runId: number,
|
||||
results: AnalysisResult[],
|
||||
): void {
|
||||
if (results.length === 0) {
|
||||
return;
|
||||
}
|
||||
): Promise<void> {
|
||||
if (results.length === 0) return;
|
||||
|
||||
const database = getDb(dbPath);
|
||||
const insertResult = database.prepare(
|
||||
`INSERT INTO results (
|
||||
run_id, asin, product_name, brand, category, unit_cost, current_price,
|
||||
avg_price_90d, avg_price_90d_sheet, selling_price_sheet, sales_rank, rank_avg_90d,
|
||||
sellers, amazon_is_seller, amazon_buybox_share_pct_90d,
|
||||
monthly_sold, rank_drops_30d, rank_drops_90d, fba_net_sheet,
|
||||
gross_profit_dollar, gross_profit_pct, net_profit_sheet, roi_sheet, moq, moq_cost,
|
||||
qty_available, supplier, source_url, asin_link, promo_coupon_code, notes, lead_date,
|
||||
fba_fee, fbm_fee, referral_percent, can_sell, sellability_status, sellability_reason,
|
||||
verdict, confidence, reasoning, fetched_at
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
|
||||
)`,
|
||||
);
|
||||
|
||||
database.transaction(() => {
|
||||
for (const r of results) {
|
||||
const rows = results.map((r) => {
|
||||
const row = buildRow(r);
|
||||
insertResult.run(
|
||||
return {
|
||||
runId,
|
||||
row.ASIN,
|
||||
row.Name,
|
||||
row.Brand,
|
||||
row.Category,
|
||||
row["Unit Cost"] ?? null,
|
||||
row["Current Price"] ?? null,
|
||||
row["Avg Price 90d"] ?? null,
|
||||
row["Avg Price 90d (sheet)"] ?? null,
|
||||
row["Selling Price (sheet)"] ?? null,
|
||||
row["Sales Rank"] ?? null,
|
||||
row["Rank Avg 90d"] ?? null,
|
||||
row.Sellers ?? null,
|
||||
asin: row.ASIN,
|
||||
productName: row.Name || null,
|
||||
brand: row.Brand || null,
|
||||
category: row.Category || null,
|
||||
unitCost: (row["Unit Cost"] as number) ?? null,
|
||||
currentPrice: (row["Current Price"] as number) || null,
|
||||
avgPrice90d: (row["Avg Price 90d"] as number) || null,
|
||||
avgPrice90dSheet: (row["Avg Price 90d (sheet)"] as number) || null,
|
||||
sellingPriceSheet: (row["Selling Price (sheet)"] as number) || null,
|
||||
salesRank: (row["Sales Rank"] as number) || null,
|
||||
rankAvg90d: (row["Rank Avg 90d"] as number) || null,
|
||||
sellerCount: (row.Sellers as number) || null,
|
||||
amazonIsSeller:
|
||||
row["Amazon Is Seller"] == null
|
||||
? null
|
||||
: row["Amazon Is Seller"]
|
||||
? 1
|
||||
: 0,
|
||||
row["Amazon Buy Box Share 90d %"] ?? null,
|
||||
row["Monthly Sold"] ?? null,
|
||||
row["Rank Drops 30d"] ?? null,
|
||||
row["Rank Drops 90d"] ?? null,
|
||||
row["FBA Net (sheet)"] ?? null,
|
||||
row["Gross Profit $"] ?? null,
|
||||
row["Gross Profit %"] ?? null,
|
||||
row["Net Profit (sheet)"] ?? null,
|
||||
row["ROI (sheet)"] ?? null,
|
||||
row.MOQ ?? null,
|
||||
row["MOQ Cost"] ?? null,
|
||||
row["Qty Available"] ?? null,
|
||||
row.Supplier ?? null,
|
||||
row["Source URL"] ?? null,
|
||||
row["ASIN Link"] ?? null,
|
||||
row["Promo/Coupon Code"] ?? null,
|
||||
row.Notes ?? null,
|
||||
row["Lead Date"] ?? null,
|
||||
row["FBA Fee"] ?? null,
|
||||
row["FBM Fee"] ?? null,
|
||||
row["Referral %"] ?? null,
|
||||
row["Can Sell"],
|
||||
row.Sellability,
|
||||
row["Sellability Reason"] ?? null,
|
||||
row.Verdict,
|
||||
row.Confidence ?? null,
|
||||
row.Reasoning,
|
||||
r.product.fetchedAt,
|
||||
);
|
||||
}
|
||||
})();
|
||||
: Boolean(row["Amazon Is Seller"]),
|
||||
amazonBuyboxSharePct90d:
|
||||
(row["Amazon Buy Box Share 90d %"] as number) || null,
|
||||
monthlySold: (row["Monthly Sold"] as number) || null,
|
||||
rankDrops30d: (row["Rank Drops 30d"] as number) || null,
|
||||
rankDrops90d: (row["Rank Drops 90d"] as number) || null,
|
||||
fbaNetSheet: (row["FBA Net (sheet)"] as number) || null,
|
||||
grossProfitDollar: (row["Gross Profit $"] as number) || null,
|
||||
grossProfitPct: (row["Gross Profit %"] as number) || null,
|
||||
netProfitSheet: (row["Net Profit (sheet)"] as number) || null,
|
||||
roiSheet: (row["ROI (sheet)"] as number) || null,
|
||||
moq: (row.MOQ as number) || null,
|
||||
moqCost: (row["MOQ Cost"] as number) || null,
|
||||
qtyAvailable: (row["Qty Available"] as number) || null,
|
||||
supplier: row.Supplier || null,
|
||||
sourceUrl: row["Source URL"] || null,
|
||||
asinLink: row["ASIN Link"] || null,
|
||||
promoCouponCode: row["Promo/Coupon Code"] || null,
|
||||
notes: row.Notes || null,
|
||||
leadDate: row["Lead Date"] || null,
|
||||
fbaFee: row["FBA Fee"] ?? null,
|
||||
fbmFee: row["FBM Fee"] ?? null,
|
||||
referralPercent: row["Referral %"] ?? null,
|
||||
canSell: row["Can Sell"],
|
||||
sellabilityStatus: row.Sellability,
|
||||
sellabilityReason: row["Sellability Reason"] || null,
|
||||
verdict: row.Verdict,
|
||||
confidence: row.Confidence ?? null,
|
||||
reasoning: row.Reasoning,
|
||||
fetchedAt: new Date(r.product.fetchedAt),
|
||||
};
|
||||
});
|
||||
|
||||
await db.insert(analysisResults).values(rows);
|
||||
}
|
||||
|
||||
export function appendSupplierResultsToRun(
|
||||
dbPath: string,
|
||||
export async function appendSupplierResultsToRun(
|
||||
runId: number,
|
||||
results: SupplierAnalysisResult[],
|
||||
): void {
|
||||
if (results.length === 0) {
|
||||
return;
|
||||
}
|
||||
): Promise<void> {
|
||||
if (results.length === 0) return;
|
||||
|
||||
const database = getDb(dbPath);
|
||||
const insertResult = database.prepare(
|
||||
`INSERT INTO results (
|
||||
run_id, asin, product_name, brand, category, unit_cost, current_price,
|
||||
avg_price_90d, sales_rank, rank_avg_90d, sellers,
|
||||
amazon_is_seller, amazon_buybox_share_pct_90d, monthly_sold,
|
||||
rank_drops_30d, rank_drops_90d, upc, fba_fee, fbm_fee,
|
||||
referral_percent, supplier_score, supplier_profit, supplier_margin,
|
||||
supplier_roi, supplier_reason, upc_lookup_status, upc_lookup_reason,
|
||||
candidate_asins, can_sell, sellability_status, sellability_reason,
|
||||
verdict, confidence, reasoning, fetched_at
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
|
||||
)`,
|
||||
);
|
||||
|
||||
database.transaction(() => {
|
||||
for (const result of results) {
|
||||
const rows = results.map((result) => {
|
||||
const keepa = result.keepa;
|
||||
const spApi = result.spApi;
|
||||
const asin = result.lookup.asin ?? result.record.asin ?? result.upc;
|
||||
@@ -277,89 +219,84 @@ export function appendSupplierResultsToRun(
|
||||
const canSell =
|
||||
spApi?.canSell == null ? null : spApi.canSell ? "yes" : "no";
|
||||
|
||||
insertResult.run(
|
||||
return {
|
||||
runId,
|
||||
asin,
|
||||
result.record.name,
|
||||
result.record.brand ?? null,
|
||||
productName: result.record.name,
|
||||
brand: result.record.brand ?? null,
|
||||
category,
|
||||
result.record.unitCost || null,
|
||||
result.score.salePrice,
|
||||
keepa?.avgPrice90 ?? null,
|
||||
keepa?.salesRank ?? null,
|
||||
keepa?.salesRankAvg90 ?? null,
|
||||
keepa?.sellerCount ?? null,
|
||||
keepa?.amazonIsSeller == null ? null : keepa.amazonIsSeller ? 1 : 0,
|
||||
keepa?.amazonBuyboxSharePct90d ?? null,
|
||||
keepa?.monthlySold ?? null,
|
||||
keepa?.salesRankDrops30 ?? null,
|
||||
keepa?.salesRankDrops90 ?? null,
|
||||
result.upc,
|
||||
result.score.fbaFee,
|
||||
spApi?.fbmFee ?? null,
|
||||
spApi?.referralFeePercent ?? null,
|
||||
result.score.score,
|
||||
result.score.profit,
|
||||
result.score.margin,
|
||||
result.score.roi,
|
||||
result.score.reason,
|
||||
result.lookup.status,
|
||||
result.lookup.reason ?? null,
|
||||
result.lookup.candidateAsins.join(","),
|
||||
unitCost: result.record.unitCost || null,
|
||||
currentPrice: result.score.salePrice,
|
||||
avgPrice90d: keepa?.avgPrice90 ?? null,
|
||||
salesRank: keepa?.salesRank ?? null,
|
||||
rankAvg90d: keepa?.salesRankAvg90 ?? null,
|
||||
sellerCount: keepa?.sellerCount ?? null,
|
||||
amazonIsSeller: keepa?.amazonIsSeller ?? null,
|
||||
amazonBuyboxSharePct90d: keepa?.amazonBuyboxSharePct90d ?? null,
|
||||
monthlySold: keepa?.monthlySold ?? null,
|
||||
rankDrops30d: keepa?.salesRankDrops30 ?? null,
|
||||
rankDrops90d: keepa?.salesRankDrops90 ?? null,
|
||||
upc: result.upc,
|
||||
fbaFee: result.score.fbaFee,
|
||||
fbmFee: spApi?.fbmFee ?? null,
|
||||
referralPercent: spApi?.referralFeePercent ?? null,
|
||||
supplierScore: result.score.score,
|
||||
supplierProfit: result.score.profit,
|
||||
supplierMargin: result.score.margin,
|
||||
supplierRoi: result.score.roi,
|
||||
supplierReason: result.score.reason,
|
||||
upcLookupStatus: result.lookup.status,
|
||||
upcLookupReason: result.lookup.reason ?? null,
|
||||
candidateAsins: result.lookup.candidateAsins.join(","),
|
||||
canSell,
|
||||
spApi?.sellabilityStatus ?? null,
|
||||
spApi?.sellabilityReason ?? null,
|
||||
result.score.verdict,
|
||||
Math.round(result.score.score),
|
||||
result.score.reason,
|
||||
result.fetchedAt,
|
||||
);
|
||||
}
|
||||
})();
|
||||
sellabilityStatus: spApi?.sellabilityStatus ?? null,
|
||||
sellabilityReason: spApi?.sellabilityReason ?? null,
|
||||
verdict: result.score.verdict,
|
||||
confidence: result.score.score,
|
||||
reasoning: result.score.reason,
|
||||
fetchedAt: new Date(result.fetchedAt),
|
||||
};
|
||||
});
|
||||
|
||||
await db.insert(analysisResults).values(rows);
|
||||
}
|
||||
|
||||
export function refreshRunCountsInDb(dbPath: string, runId: number): RunCounts {
|
||||
const database = getDb(dbPath);
|
||||
const stats = database
|
||||
.query(
|
||||
`SELECT
|
||||
export async function refreshRunCountsInDb(runId: number): Promise<RunCounts> {
|
||||
const [stats] = await db.execute(
|
||||
sql<{
|
||||
total: string;
|
||||
fba: string | null;
|
||||
fbm: string | null;
|
||||
skip: string | null;
|
||||
}>`SELECT
|
||||
COUNT(*) AS total,
|
||||
SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba,
|
||||
SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm,
|
||||
SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip
|
||||
FROM results
|
||||
WHERE run_id = ?`,
|
||||
)
|
||||
.get(runId) as {
|
||||
total: number;
|
||||
fba: number | null;
|
||||
fbm: number | null;
|
||||
skip: number | null;
|
||||
};
|
||||
FROM analysis_results
|
||||
WHERE run_id = ${runId}`,
|
||||
);
|
||||
|
||||
const counts: RunCounts = {
|
||||
totalProducts: stats.total ?? 0,
|
||||
fbaCount: stats.fba ?? 0,
|
||||
fbmCount: stats.fbm ?? 0,
|
||||
skipCount: stats.skip ?? 0,
|
||||
totalProducts: Number(stats?.total ?? 0),
|
||||
fbaCount: Number(stats?.fba ?? 0),
|
||||
fbmCount: Number(stats?.fbm ?? 0),
|
||||
skipCount: Number(stats?.skip ?? 0),
|
||||
};
|
||||
|
||||
database
|
||||
.query(
|
||||
`UPDATE runs
|
||||
SET total_products = ?, fba_count = ?, fbm_count = ?, skip_count = ?
|
||||
WHERE id = ?`,
|
||||
)
|
||||
.run(
|
||||
counts.totalProducts,
|
||||
counts.fbaCount,
|
||||
counts.fbmCount,
|
||||
counts.skipCount,
|
||||
runId,
|
||||
);
|
||||
await db
|
||||
.update(runs)
|
||||
.set({
|
||||
totalProducts: counts.totalProducts,
|
||||
fbaCount: counts.fbaCount,
|
||||
fbmCount: counts.fbmCount,
|
||||
skipCount: counts.skipCount,
|
||||
})
|
||||
.where(eq(runs.id, runId));
|
||||
|
||||
return counts;
|
||||
}
|
||||
|
||||
export function printResults(results: AnalysisResult[]): void {
|
||||
const rows = results
|
||||
.filter((r) => r.verdict.verdict === "FBA" || r.verdict.verdict === "FBM")
|
||||
|
||||
Reference in New Issue
Block a user