From b982edd1608a5cc7b231f051f81ecb07a4314507 Mon Sep 17 00:00:00 2001 From: Victor Noguera Date: Mon, 25 May 2026 00:08:30 -0400 Subject: [PATCH] Refactor database interactions to use Drizzle ORM - Replaced direct SQLite database calls with Drizzle ORM methods in `top-monthly-sold-by-category.ts`, `writer.ts`, and `upc-file-analysis.ts`. - Updated test cases in `top-monthly-sold-by-category.test.ts` to mock the new database interactions. - Removed unnecessary database initialization and cleanup code. - Improved code readability and maintainability by using ORM features for inserting and updating records. --- .env.example | 1 + bun.lock | 183 ++++ drizzle.config.ts | 10 + package.json | 7 +- src/bestsellers-by-category.test.ts | 139 +-- src/bestsellers-by-category.ts | 273 +++-- src/check_db.ts | 19 +- src/database.ts | 497 +-------- src/db/index.ts | 15 + src/db/schema.ts | 343 ++++++ src/index.ts | 8 +- src/mid-range-sellers-by-category.test.ts | 216 ++-- src/mid-range-sellers-by-category.ts | 274 +++-- src/server.ts | 1144 +++++++++++---------- src/stalker-analyze.ts | 343 +++--- src/stalker-sellability.test.ts | 110 +- src/stalker.test.ts | 139 ++- src/stalker.ts | 653 ++++++------ src/top-monthly-sold-by-category.test.ts | 184 +--- src/top-monthly-sold-by-category.ts | 275 +++-- src/upc-file-analysis.ts | 14 +- src/writer.ts | 375 +++---- 22 files changed, 2456 insertions(+), 2766 deletions(-) create mode 100644 drizzle.config.ts create mode 100644 src/db/index.ts create mode 100644 src/db/schema.ts diff --git a/.env.example b/.env.example index 246c10e..83f04a1 100644 --- a/.env.example +++ b/.env.example @@ -19,3 +19,4 @@ GOOGLE_API_KEY=your_google_api_key GOOGLE_CSE_ID=your_google_programmable_search_engine_id SERPAPI_API_KEY=your_serpapi_api_key_for_google_shopping +DB_CONNECTION_STRING=your_database_connection_string \ No newline at end of file diff --git a/bun.lock b/bun.lock index af03c41..42de449 100644 --- a/bun.lock +++ b/bun.lock @@ -6,8 +6,10 @@ "name": "asin-check", "dependencies": { "amazon-sp-api": "^1.2.1", + "drizzle-orm": "^0.45.2", "exceljs": "^4.4.0", "ioredis": "^5.10.1", + "postgres": "^3.4.9", "react": "^19.2.0", "react-dom": "^19.2.0", "xlsx": "^0.18.5", @@ -16,11 +18,70 @@ "@types/bun": "latest", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", + "drizzle-kit": "^0.31.10", "typescript": "^6.0.3", }, }, }, "packages": { + "@drizzle-team/brocli": ["@drizzle-team/brocli@0.10.2", "", {}, "sha512-z33Il7l5dKjUgGULTqBsQBQwckHh5AbIuxhdsIxDDiZAzBOrZO6q9ogcWC65kU382AfynTfgNumVcNIjuIua6w=="], + + "@esbuild-kit/core-utils": ["@esbuild-kit/core-utils@3.3.2", "", { "dependencies": { "esbuild": "~0.18.20", "source-map-support": "^0.5.21" } }, "sha512-sPRAnw9CdSsRmEtnsl2WXWdyquogVpB3yZ3dgwJfe8zrOzTsV7cJvmwrKVa+0ma5BoiGJ+BoqkMvawbayKUsqQ=="], + + "@esbuild-kit/esm-loader": ["@esbuild-kit/esm-loader@2.6.5", "", { "dependencies": { "@esbuild-kit/core-utils": "^3.3.2", "get-tsconfig": "^4.7.0" } }, "sha512-FxEMIkJKnodyA1OaCUoEvbYRkoZlLZ4d/eXFu9Fh8CbBBgP5EmZxrfTRyN0qpXZ4vOvqnE5YdRdcrmUUXuU+dA=="], + + "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.25.12", "", { "os": "aix", "cpu": "ppc64" }, "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA=="], + + "@esbuild/android-arm": ["@esbuild/android-arm@0.25.12", "", { "os": "android", "cpu": "arm" }, "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg=="], + + "@esbuild/android-arm64": ["@esbuild/android-arm64@0.25.12", "", { "os": "android", "cpu": "arm64" }, "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg=="], + + "@esbuild/android-x64": ["@esbuild/android-x64@0.25.12", "", { "os": "android", "cpu": "x64" }, "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg=="], + + "@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.25.12", "", { "os": "darwin", "cpu": "arm64" }, "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg=="], + + "@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.25.12", "", { "os": "darwin", "cpu": "x64" }, "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA=="], + + "@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.25.12", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg=="], + + "@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.25.12", "", { "os": "freebsd", "cpu": "x64" }, "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ=="], + + "@esbuild/linux-arm": ["@esbuild/linux-arm@0.25.12", "", { "os": "linux", "cpu": "arm" }, "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw=="], + + "@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.25.12", "", { "os": "linux", "cpu": "arm64" }, "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ=="], + + "@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.25.12", "", { "os": "linux", "cpu": "ia32" }, "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA=="], + + "@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng=="], + + "@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw=="], + + "@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.25.12", "", { "os": "linux", "cpu": "ppc64" }, "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA=="], + + "@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w=="], + + "@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.25.12", "", { "os": "linux", "cpu": "s390x" }, "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg=="], + + "@esbuild/linux-x64": ["@esbuild/linux-x64@0.25.12", "", { "os": "linux", "cpu": "x64" }, "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw=="], + + "@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.25.12", "", { "os": "none", "cpu": "arm64" }, "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg=="], + + "@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.25.12", "", { "os": "none", "cpu": "x64" }, "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ=="], + + "@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.25.12", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A=="], + + "@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.25.12", "", { "os": "openbsd", "cpu": "x64" }, "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw=="], + + "@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.25.12", "", { "os": "none", "cpu": "arm64" }, "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg=="], + + "@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.25.12", "", { "os": "sunos", "cpu": "x64" }, "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w=="], + + "@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.25.12", "", { "os": "win32", "cpu": "arm64" }, "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg=="], + + "@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.25.12", "", { "os": "win32", "cpu": "ia32" }, "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ=="], + + "@esbuild/win32-x64": ["@esbuild/win32-x64@0.25.12", "", { "os": "win32", "cpu": "x64" }, "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA=="], + "@fast-csv/format": ["@fast-csv/format@4.3.5", "", { "dependencies": { "@types/node": "^14.0.1", "lodash.escaperegexp": "^4.1.2", "lodash.isboolean": "^3.0.3", "lodash.isequal": "^4.5.0", "lodash.isfunction": "^3.0.9", "lodash.isnil": "^4.0.0" } }, "sha512-8iRn6QF3I8Ak78lNAa+Gdl5MJJBM5vRHivFtMRUWINdevNo00K7OXxS2PshawLKTejVwieIlPmK5YlLu6w4u8A=="], "@fast-csv/parse": ["@fast-csv/parse@4.3.6", "", { "dependencies": { "@types/node": "^14.0.1", "lodash.escaperegexp": "^4.1.2", "lodash.groupby": "^4.6.0", "lodash.isfunction": "^3.0.9", "lodash.isnil": "^4.0.0", "lodash.isundefined": "^3.0.1", "lodash.uniq": "^4.5.0" } }, "sha512-uRsLYksqpbDmWaSmzvJcuApSEe38+6NQZBUsuAyMZKqHxH0g1wcJgsKUvN3WC8tewaqFjBMMGrkHmC+T7k8LvA=="], @@ -63,6 +124,8 @@ "buffer-crc32": ["buffer-crc32@0.2.13", "", {}, "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ=="], + "buffer-from": ["buffer-from@1.1.2", "", {}, "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="], + "buffer-indexof-polyfill": ["buffer-indexof-polyfill@1.0.2", "", {}, "sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A=="], "buffers": ["buffers@0.1.1", "", {}, "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ=="], @@ -101,6 +164,10 @@ "denque": ["denque@2.1.0", "", {}, "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw=="], + "drizzle-kit": ["drizzle-kit@0.31.10", "", { "dependencies": { "@drizzle-team/brocli": "^0.10.2", "@esbuild-kit/esm-loader": "^2.5.5", "esbuild": "^0.25.4", "tsx": "^4.21.0" }, "bin": { "drizzle-kit": "bin.cjs" } }, "sha512-7OZcmQUrdGI+DUNNsKBn1aW8qSoKuTH7d0mYgSP8bAzdFzKoovxEFnoGQp2dVs82EOJeYycqRtciopszwUf8bw=="], + + "drizzle-orm": ["drizzle-orm@0.45.2", "", { "peerDependencies": { "@aws-sdk/client-rds-data": ">=3", "@cloudflare/workers-types": ">=4", "@electric-sql/pglite": ">=0.2.0", "@libsql/client": ">=0.10.0", "@libsql/client-wasm": ">=0.10.0", "@neondatabase/serverless": ">=0.10.0", "@op-engineering/op-sqlite": ">=2", "@opentelemetry/api": "^1.4.1", "@planetscale/database": ">=1.13", "@prisma/client": "*", "@tidbcloud/serverless": "*", "@types/better-sqlite3": "*", "@types/pg": "*", "@types/sql.js": "*", "@upstash/redis": ">=1.34.7", "@vercel/postgres": ">=0.8.0", "@xata.io/client": "*", "better-sqlite3": ">=7", "bun-types": "*", "expo-sqlite": ">=14.0.0", "gel": ">=2", "knex": "*", "kysely": "*", "mysql2": ">=2", "pg": ">=8", "postgres": ">=3", "prisma": "*", "sql.js": ">=1", "sqlite3": ">=5" }, "optionalPeers": ["@aws-sdk/client-rds-data", "@cloudflare/workers-types", "@electric-sql/pglite", "@libsql/client", "@libsql/client-wasm", "@neondatabase/serverless", "@op-engineering/op-sqlite", "@opentelemetry/api", "@planetscale/database", "@prisma/client", "@tidbcloud/serverless", "@types/better-sqlite3", "@types/pg", "@types/sql.js", "@upstash/redis", "@vercel/postgres", "@xata.io/client", "better-sqlite3", "bun-types", "expo-sqlite", "gel", "knex", "kysely", "mysql2", "pg", "postgres", "prisma", "sql.js", "sqlite3"] }, "sha512-kY0BSaTNYWnoDMVoyY8uxmyHjpJW1geOmBMdSSicKo9CIIWkSxMIj2rkeSR51b8KAPB7m+qysjuHme5nKP+E5Q=="], + "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="], "duplexer2": ["duplexer2@0.1.4", "", { "dependencies": { "readable-stream": "^2.0.2" } }, "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA=="], @@ -113,6 +180,8 @@ "es-object-atoms": ["es-object-atoms@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0" } }, "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA=="], + "esbuild": ["esbuild@0.25.12", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.25.12", "@esbuild/android-arm": "0.25.12", "@esbuild/android-arm64": "0.25.12", "@esbuild/android-x64": "0.25.12", "@esbuild/darwin-arm64": "0.25.12", "@esbuild/darwin-x64": "0.25.12", "@esbuild/freebsd-arm64": "0.25.12", "@esbuild/freebsd-x64": "0.25.12", "@esbuild/linux-arm": "0.25.12", "@esbuild/linux-arm64": "0.25.12", "@esbuild/linux-ia32": "0.25.12", "@esbuild/linux-loong64": "0.25.12", "@esbuild/linux-mips64el": "0.25.12", "@esbuild/linux-ppc64": "0.25.12", "@esbuild/linux-riscv64": "0.25.12", "@esbuild/linux-s390x": "0.25.12", "@esbuild/linux-x64": "0.25.12", "@esbuild/netbsd-arm64": "0.25.12", "@esbuild/netbsd-x64": "0.25.12", "@esbuild/openbsd-arm64": "0.25.12", "@esbuild/openbsd-x64": "0.25.12", "@esbuild/openharmony-arm64": "0.25.12", "@esbuild/sunos-x64": "0.25.12", "@esbuild/win32-arm64": "0.25.12", "@esbuild/win32-ia32": "0.25.12", "@esbuild/win32-x64": "0.25.12" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg=="], + "exceljs": ["exceljs@4.4.0", "", { "dependencies": { "archiver": "^5.0.0", "dayjs": "^1.8.34", "fast-csv": "^4.3.1", "jszip": "^3.10.1", "readable-stream": "^3.6.0", "saxes": "^5.0.1", "tmp": "^0.2.0", "unzipper": "^0.10.11", "uuid": "^8.3.0" } }, "sha512-XctvKaEMaj1Ii9oDOqbW/6e1gXknSY4g/aLCDicOXqBE4M0nRWkUu0PTp++UPNzoFY12BNHMfs/VadKIS6llvg=="], "fast-csv": ["fast-csv@4.3.6", "", { "dependencies": { "@fast-csv/format": "4.3.5", "@fast-csv/parse": "4.3.6" } }, "sha512-2RNSpuwwsJGP0frGsOmTb9oUF+VkFSM4SyLTDgwf2ciHWTarN0lQTC+F2f/t5J9QjW+c65VFIAAu85GsvMIusw=="], @@ -127,6 +196,8 @@ "fs.realpath": ["fs.realpath@1.0.0", "", {}, "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw=="], + "fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], + "fstream": ["fstream@1.0.12", "", { "dependencies": { "graceful-fs": "^4.1.2", "inherits": "~2.0.0", "mkdirp": ">=0.5 0", "rimraf": "2" } }, "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg=="], "function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="], @@ -135,6 +206,8 @@ "get-proto": ["get-proto@1.0.1", "", { "dependencies": { "dunder-proto": "^1.0.1", "es-object-atoms": "^1.0.0" } }, "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g=="], + "get-tsconfig": ["get-tsconfig@4.14.0", "", { "dependencies": { "resolve-pkg-maps": "^1.0.0" } }, "sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA=="], + "glob": ["glob@7.2.3", "", { "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", "minimatch": "^3.1.1", "once": "^1.3.0", "path-is-absolute": "^1.0.0" } }, "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q=="], "gopd": ["gopd@1.2.0", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="], @@ -217,6 +290,8 @@ "path-is-absolute": ["path-is-absolute@1.0.1", "", {}, "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg=="], + "postgres": ["postgres@3.4.9", "", {}, "sha512-GD3qdB0x1z9xgFI6cdRD6xu2Sp2WCOEoe3mtnyB5Ee0XrrL5Pe+e4CCnJrRMnL1zYtRDZmQQVbvOttLnKDLnaw=="], + "process-nextick-args": ["process-nextick-args@2.0.1", "", {}, "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="], "qs": ["qs@6.15.1", "", { "dependencies": { "side-channel": "^1.1.0" } }, "sha512-6YHEFRL9mfgcAvql/XhwTvf5jKcOiiupt2FiJxHkiX1z4j7WL8J/jRHYLluORvc1XxB5rV20KoeK00gVJamspg=="], @@ -233,6 +308,8 @@ "redis-parser": ["redis-parser@3.0.0", "", { "dependencies": { "redis-errors": "^1.0.0" } }, "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A=="], + "resolve-pkg-maps": ["resolve-pkg-maps@1.0.0", "", {}, "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw=="], + "rimraf": ["rimraf@2.7.1", "", { "dependencies": { "glob": "^7.1.3" }, "bin": { "rimraf": "./bin.js" } }, "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w=="], "safe-buffer": ["safe-buffer@5.1.2", "", {}, "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="], @@ -253,6 +330,10 @@ "side-channel-weakmap": ["side-channel-weakmap@1.0.2", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3", "side-channel-map": "^1.0.1" } }, "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A=="], + "source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="], + + "source-map-support": ["source-map-support@0.5.21", "", { "dependencies": { "buffer-from": "^1.0.0", "source-map": "^0.6.0" } }, "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w=="], + "ssf": ["ssf@0.11.2", "", { "dependencies": { "frac": "~1.1.2" } }, "sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g=="], "standard-as-callback": ["standard-as-callback@2.1.0", "", {}, "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A=="], @@ -267,6 +348,8 @@ "traverse": ["traverse@0.3.9", "", {}, "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ=="], + "tsx": ["tsx@4.22.3", "", { "dependencies": { "esbuild": "~0.28.0" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "bin": { "tsx": "dist/cli.mjs" } }, "sha512-mdoNxBC/cSQObGGVQ5Bpn5i+yv7j68gk3Nfm3wFjcJg3Z0Mix9jzAFfP12prmm5eVGmDKtp0yyArrs0Q+8gZHg=="], + "typescript": ["typescript@6.0.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw=="], "undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="], @@ -289,6 +372,8 @@ "zip-stream": ["zip-stream@4.1.1", "", { "dependencies": { "archiver-utils": "^3.0.4", "compress-commons": "^4.1.2", "readable-stream": "^3.6.0" } }, "sha512-9qv4rlDiopXg4E69k+vMHjNN63YFMe9sZMrdlvKnCjlCRWeCBswPPMPUfx+ipsAWq1LXHe70RcbaHdJJpS6hyQ=="], + "@esbuild-kit/core-utils/esbuild": ["esbuild@0.18.20", "", { "optionalDependencies": { "@esbuild/android-arm": "0.18.20", "@esbuild/android-arm64": "0.18.20", "@esbuild/android-x64": "0.18.20", "@esbuild/darwin-arm64": "0.18.20", "@esbuild/darwin-x64": "0.18.20", "@esbuild/freebsd-arm64": "0.18.20", "@esbuild/freebsd-x64": "0.18.20", "@esbuild/linux-arm": "0.18.20", "@esbuild/linux-arm64": "0.18.20", "@esbuild/linux-ia32": "0.18.20", "@esbuild/linux-loong64": "0.18.20", "@esbuild/linux-mips64el": "0.18.20", "@esbuild/linux-ppc64": "0.18.20", "@esbuild/linux-riscv64": "0.18.20", "@esbuild/linux-s390x": "0.18.20", "@esbuild/linux-x64": "0.18.20", "@esbuild/netbsd-x64": "0.18.20", "@esbuild/openbsd-x64": "0.18.20", "@esbuild/sunos-x64": "0.18.20", "@esbuild/win32-arm64": "0.18.20", "@esbuild/win32-ia32": "0.18.20", "@esbuild/win32-x64": "0.18.20" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA=="], + "@fast-csv/format/@types/node": ["@types/node@14.18.63", "", {}, "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ=="], "@fast-csv/parse/@types/node": ["@types/node@14.18.63", "", {}, "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ=="], @@ -305,10 +390,56 @@ "string_decoder/safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="], + "tsx/esbuild": ["esbuild@0.28.0", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.28.0", "@esbuild/android-arm": "0.28.0", "@esbuild/android-arm64": "0.28.0", "@esbuild/android-x64": "0.28.0", "@esbuild/darwin-arm64": "0.28.0", "@esbuild/darwin-x64": "0.28.0", "@esbuild/freebsd-arm64": "0.28.0", "@esbuild/freebsd-x64": "0.28.0", "@esbuild/linux-arm": "0.28.0", "@esbuild/linux-arm64": "0.28.0", "@esbuild/linux-ia32": "0.28.0", "@esbuild/linux-loong64": "0.28.0", "@esbuild/linux-mips64el": "0.28.0", "@esbuild/linux-ppc64": "0.28.0", "@esbuild/linux-riscv64": "0.28.0", "@esbuild/linux-s390x": "0.28.0", "@esbuild/linux-x64": "0.28.0", "@esbuild/netbsd-arm64": "0.28.0", "@esbuild/netbsd-x64": "0.28.0", "@esbuild/openbsd-arm64": "0.28.0", "@esbuild/openbsd-x64": "0.28.0", "@esbuild/openharmony-arm64": "0.28.0", "@esbuild/sunos-x64": "0.28.0", "@esbuild/win32-arm64": "0.28.0", "@esbuild/win32-ia32": "0.28.0", "@esbuild/win32-x64": "0.28.0" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-sNR9MHpXSUV/XB4zmsFKN+QgVG82Cc7+/aaxJ8Adi8hyOac+EXptIp45QBPaVyX3N70664wRbTcLTOemCAnyqw=="], + "unzipper/readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="], "zip-stream/archiver-utils": ["archiver-utils@3.0.4", "", { "dependencies": { "glob": "^7.2.3", "graceful-fs": "^4.2.0", "lazystream": "^1.0.0", "lodash.defaults": "^4.2.0", "lodash.difference": "^4.5.0", "lodash.flatten": "^4.4.0", "lodash.isplainobject": "^4.0.6", "lodash.union": "^4.6.0", "normalize-path": "^3.0.0", "readable-stream": "^3.6.0" } }, "sha512-KVgf4XQVrTjhyWmx6cte4RxonPLR9onExufI1jhvw/MQ4BB6IsZD5gT8Lq+u/+pRkWna/6JoHpiQioaqFP5Rzw=="], + "@esbuild-kit/core-utils/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.18.20", "", { "os": "android", "cpu": "arm" }, "sha512-fyi7TDI/ijKKNZTUJAQqiG5T7YjJXgnzkURqmGj13C6dCqckZBLdl4h7bkhHt/t0WP+zO9/zwroDvANaOqO5Sw=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/android-arm64": ["@esbuild/android-arm64@0.18.20", "", { "os": "android", "cpu": "arm64" }, "sha512-Nz4rJcchGDtENV0eMKUNa6L12zz2zBDXuhj/Vjh18zGqB44Bi7MBMSXjgunJgjRhCmKOjnPuZp4Mb6OKqtMHLQ=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/android-x64": ["@esbuild/android-x64@0.18.20", "", { "os": "android", "cpu": "x64" }, "sha512-8GDdlePJA8D6zlZYJV/jnrRAi6rOiNaCC/JclcXpB+KIuvfBN4owLtgzY2bsxnx666XjJx2kDPUmnTtR8qKQUg=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.18.20", "", { "os": "darwin", "cpu": "arm64" }, "sha512-bxRHW5kHU38zS2lPTPOyuyTm+S+eobPUnTNkdJEfAddYgEcll4xkT8DB9d2008DtTbl7uJag2HuE5NZAZgnNEA=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.18.20", "", { "os": "darwin", "cpu": "x64" }, "sha512-pc5gxlMDxzm513qPGbCbDukOdsGtKhfxD1zJKXjCCcU7ju50O7MeAZ8c4krSJcOIJGFR+qx21yMMVYwiQvyTyQ=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.18.20", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-yqDQHy4QHevpMAaxhhIwYPMv1NECwOvIpGCZkECn8w2WFHXjEwrBn3CeNIYsibZ/iZEUemj++M26W3cNR5h+Tw=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.18.20", "", { "os": "freebsd", "cpu": "x64" }, "sha512-tgWRPPuQsd3RmBZwarGVHZQvtzfEBOreNuxEMKFcd5DaDn2PbBxfwLcj4+aenoh7ctXcbXmOQIn8HI6mCSw5MQ=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/linux-arm": ["@esbuild/linux-arm@0.18.20", "", { "os": "linux", "cpu": "arm" }, "sha512-/5bHkMWnq1EgKr1V+Ybz3s1hWXok7mDFUMQ4cG10AfW3wL02PSZi5kFpYKrptDsgb2WAJIvRcDm+qIvXf/apvg=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.18.20", "", { "os": "linux", "cpu": "arm64" }, "sha512-2YbscF+UL7SQAVIpnWvYwM+3LskyDmPhe31pE7/aoTMFKKzIc9lLbyGUpmmb8a8AixOL61sQ/mFh3jEjHYFvdA=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.18.20", "", { "os": "linux", "cpu": "ia32" }, "sha512-P4etWwq6IsReT0E1KHU40bOnzMHoH73aXp96Fs8TIT6z9Hu8G6+0SHSw9i2isWrD2nbx2qo5yUqACgdfVGx7TA=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.18.20", "", { "os": "linux", "cpu": "none" }, "sha512-nXW8nqBTrOpDLPgPY9uV+/1DjxoQ7DoB2N8eocyq8I9XuqJ7BiAMDMf9n1xZM9TgW0J8zrquIb/A7s3BJv7rjg=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.18.20", "", { "os": "linux", "cpu": "none" }, "sha512-d5NeaXZcHp8PzYy5VnXV3VSd2D328Zb+9dEq5HE6bw6+N86JVPExrA6O68OPwobntbNJ0pzCpUFZTo3w0GyetQ=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.18.20", "", { "os": "linux", "cpu": "ppc64" }, "sha512-WHPyeScRNcmANnLQkq6AfyXRFr5D6N2sKgkFo2FqguP44Nw2eyDlbTdZwd9GYk98DZG9QItIiTlFLHJHjxP3FA=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.18.20", "", { "os": "linux", "cpu": "none" }, "sha512-WSxo6h5ecI5XH34KC7w5veNnKkju3zBRLEQNY7mv5mtBmrP/MjNBCAlsM2u5hDBlS3NGcTQpoBvRzqBcRtpq1A=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.18.20", "", { "os": "linux", "cpu": "s390x" }, "sha512-+8231GMs3mAEth6Ja1iK0a1sQ3ohfcpzpRLH8uuc5/KVDFneH6jtAJLFGafpzpMRO6DzJ6AvXKze9LfFMrIHVQ=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/linux-x64": ["@esbuild/linux-x64@0.18.20", "", { "os": "linux", "cpu": "x64" }, "sha512-UYqiqemphJcNsFEskc73jQ7B9jgwjWrSayxawS6UVFZGWrAAtkzjxSqnoclCXxWtfwLdzU+vTpcNYhpn43uP1w=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.18.20", "", { "os": "none", "cpu": "x64" }, "sha512-iO1c++VP6xUBUmltHZoMtCUdPlnPGdBom6IrO4gyKPFFVBKioIImVooR5I83nTew5UOYrk3gIJhbZh8X44y06A=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.18.20", "", { "os": "openbsd", "cpu": "x64" }, "sha512-e5e4YSsuQfX4cxcygw/UCPIEP6wbIL+se3sxPdCiMbFLBWu0eiZOJ7WoD+ptCLrmjZBK1Wk7I6D/I3NglUGOxg=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.18.20", "", { "os": "sunos", "cpu": "x64" }, "sha512-kDbFRFp0YpTQVVrqUd5FTYmWo45zGaXe0X8E1G/LKFC0v8x0vWrhOWSLITcCn63lmZIxfOMXtCfti/RxN/0wnQ=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.18.20", "", { "os": "win32", "cpu": "arm64" }, "sha512-ddYFR6ItYgoaq4v4JmQQaAI5s7npztfV4Ag6NrhiaW0RrnOXqBkgwZLofVTlq1daVTQNhtI5oieTvkRPfZrePg=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.18.20", "", { "os": "win32", "cpu": "ia32" }, "sha512-Wv7QBi3ID/rROT08SABTS7eV4hX26sVduqDOTe1MvGMjNd3EjOz4b7zeexIR62GTIEKrfJXKL9LFxTYgkyeu7g=="], + + "@esbuild-kit/core-utils/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.18.20", "", { "os": "win32", "cpu": "x64" }, "sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ=="], + "archiver-utils/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], "duplexer2/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], @@ -319,6 +450,58 @@ "lazystream/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], + "tsx/esbuild/@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.28.0", "", { "os": "aix", "cpu": "ppc64" }, "sha512-lhRUCeuOyJQURhTxl4WkpFTjIsbDayJHih5kZC1giwE+MhIzAb7mEsQMqMf18rHLsrb5qI1tafG20mLxEWcWlA=="], + + "tsx/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.28.0", "", { "os": "android", "cpu": "arm" }, "sha512-wqh0ByljabXLKHeWXYLqoJ5jKC4XBaw6Hk08OfMrCRd2nP2ZQ5eleDZC41XHyCNgktBGYMbqnrJKq/K/lzPMSQ=="], + + "tsx/esbuild/@esbuild/android-arm64": ["@esbuild/android-arm64@0.28.0", "", { "os": "android", "cpu": "arm64" }, "sha512-+WzIXQOSaGs33tLEgYPYe/yQHf0WTU0X42Jca3y8NWMbUVhp7rUnw+vAsRC/QiDrdD31IszMrZy+qwPOPjd+rw=="], + + "tsx/esbuild/@esbuild/android-x64": ["@esbuild/android-x64@0.28.0", "", { "os": "android", "cpu": "x64" }, "sha512-+VJggoaKhk2VNNqVL7f6S189UzShHC/mR9EE8rDdSkdpN0KflSwWY/gWjDrNxxisg8Fp1ZCD9jLMo4m0OUfeUA=="], + + "tsx/esbuild/@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.28.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-0T+A9WZm+bZ84nZBtk1ckYsOvyA3x7e2Acj1KdVfV4/2tdG4fzUp91YHx+GArWLtwqp77pBXVCPn2We7Letr0Q=="], + + "tsx/esbuild/@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.28.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-fyzLm/DLDl/84OCfp2f/XQ4flmORsjU7VKt8HLjvIXChJoFFOIL6pLJPH4Yhd1n1gGFF9mPwtlN5Wf82DZs+LQ=="], + + "tsx/esbuild/@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.28.0", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-l9GeW5UZBT9k9brBYI+0WDffcRxgHQD8ShN2Ur4xWq/NFzUKm3k5lsH4PdaRgb2w7mI9u61nr2gI2mLI27Nh3Q=="], + + "tsx/esbuild/@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.28.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-BXoQai/A0wPO6Es3yFJ7APCiKGc1tdAEOgeTNy3SsB491S3aHn4S4r3e976eUnPdU+NbdtmBuLncYir2tMU9Nw=="], + + "tsx/esbuild/@esbuild/linux-arm": ["@esbuild/linux-arm@0.28.0", "", { "os": "linux", "cpu": "arm" }, "sha512-CjaaREJagqJp7iTaNQjjidaNbCKYcd4IDkzbwwxtSvjI7NZm79qiHc8HqciMddQ6CKvJT6aBd8lO9kN/ZudLlw=="], + + "tsx/esbuild/@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.28.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-RVyzfb3FWsGA55n6WY0MEIEPURL1FcbhFE6BffZEMEekfCzCIMtB5yyDcFnVbTnwk+CLAgTujmV/Lgvih56W+A=="], + + "tsx/esbuild/@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.28.0", "", { "os": "linux", "cpu": "ia32" }, "sha512-KBnSTt1kxl9x70q+ydterVdl+Cn0H18ngRMRCEQfrbqdUuntQQ0LoMZv47uB97NljZFzY6HcfqEZ2SAyIUTQBQ=="], + + "tsx/esbuild/@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.28.0", "", { "os": "linux", "cpu": "none" }, "sha512-zpSlUce1mnxzgBADvxKXX5sl8aYQHo2ezvMNI8I0lbblJtp8V4odlm3Yzlj7gPyt3T8ReksE6bK+pT3WD+aJRg=="], + + "tsx/esbuild/@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.28.0", "", { "os": "linux", "cpu": "none" }, "sha512-2jIfP6mmjkdmeTlsX/9vmdmhBmKADrWqN7zcdtHIeNSCH1SqIoNI63cYsjQR8J+wGa4Y5izRcSHSm8K3QWmk3w=="], + + "tsx/esbuild/@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.28.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-bc0FE9wWeC0WBm49IQMPSPILRocGTQt3j5KPCA8os6VprfuJ7KD+5PzESSrJ6GmPIPJK965ZJHTUlSA6GNYEhg=="], + + "tsx/esbuild/@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.28.0", "", { "os": "linux", "cpu": "none" }, "sha512-SQPZOwoTTT/HXFXQJG/vBX8sOFagGqvZyXcgLA3NhIqcBv1BJU1d46c0rGcrij2B56Z2rNiSLaZOYW5cUk7yLQ=="], + + "tsx/esbuild/@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.28.0", "", { "os": "linux", "cpu": "s390x" }, "sha512-SCfR0HN8CEEjnYnySJTd2cw0k9OHB/YFzt5zgJEwa+wL/T/raGWYMBqwDNAC6dqFKmJYZoQBRfHjgwLHGSrn3Q=="], + + "tsx/esbuild/@esbuild/linux-x64": ["@esbuild/linux-x64@0.28.0", "", { "os": "linux", "cpu": "x64" }, "sha512-us0dSb9iFxIi8srnpl931Nvs65it/Jd2a2K3qs7fz2WfGPHqzfzZTfec7oxZJRNPXPnNYZtanmRc4AL/JwVzHQ=="], + + "tsx/esbuild/@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.28.0", "", { "os": "none", "cpu": "arm64" }, "sha512-CR/RYotgtCKwtftMwJlUU7xCVNg3lMYZ0RzTmAHSfLCXw3NtZtNpswLEj/Kkf6kEL3Gw+BpOekRX0BYCtklhUw=="], + + "tsx/esbuild/@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.28.0", "", { "os": "none", "cpu": "x64" }, "sha512-nU1yhmYutL+fQ71Kxnhg8uEOdC0pwEW9entHykTgEbna2pw2dkbFSMeqjjyHZoCmt8SBkOSvV+yNmm94aUrrqw=="], + + "tsx/esbuild/@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.28.0", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-cXb5vApOsRsxsEl4mcZ1XY3D4DzcoMxR/nnc4IyqYs0rTI8ZKmW6kyyg+11Z8yvgMfAEldKzP7AdP64HnSC/6g=="], + + "tsx/esbuild/@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.28.0", "", { "os": "openbsd", "cpu": "x64" }, "sha512-8wZM2qqtv9UP3mzy7HiGYNH/zjTA355mpeuA+859TyR+e+Tc08IHYpLJuMsfpDJwoLo1ikIJI8jC3GFjnRClzA=="], + + "tsx/esbuild/@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.28.0", "", { "os": "none", "cpu": "arm64" }, "sha512-FLGfyizszcef5C3YtoyQDACyg95+dndv79i2EekILBofh5wpCa1KuBqOWKrEHZg3zrL3t5ouE5jgr94vA+Wb2w=="], + + "tsx/esbuild/@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.28.0", "", { "os": "sunos", "cpu": "x64" }, "sha512-1ZgjUoEdHZZl/YlV76TSCz9Hqj9h9YmMGAgAPYd+q4SicWNX3G5GCyx9uhQWSLcbvPW8Ni7lj4gDa1T40akdlw=="], + + "tsx/esbuild/@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.28.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-Q9StnDmQ/enxnpxCCLSg0oo4+34B9TdXpuyPeTedN/6+iXBJ4J+zwfQI28u/Jl40nOYAxGoNi7mFP40RUtkmUA=="], + + "tsx/esbuild/@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.28.0", "", { "os": "win32", "cpu": "ia32" }, "sha512-zF3ag/gfiCe6U2iczcRzSYJKH1DCI+ByzSENHlM2FcDbEeo5Zd2C86Aq0tKUYAJJ1obRP84ymxIAksZUcdztHA=="], + + "tsx/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.28.0", "", { "os": "win32", "cpu": "x64" }, "sha512-pEl1bO9mfAmIC+tW5btTmrKaujg3zGtUmWNdCw/xs70FBjwAL3o9OEKNHvNmnyylD6ubxUERiEhdsL0xBQ9efw=="], + "unzipper/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], } } diff --git a/drizzle.config.ts b/drizzle.config.ts new file mode 100644 index 0000000..bcbd67a --- /dev/null +++ b/drizzle.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from "drizzle-kit"; + +export default defineConfig({ + schema: "./src/db/schema.ts", + out: "./drizzle", + dialect: "postgresql", + dbCredentials: { + url: process.env.DB_CONNECTION_STRING!, + }, +}); diff --git a/package.json b/package.json index 3e898aa..a433e51 100644 --- a/package.json +++ b/package.json @@ -14,18 +14,23 @@ "start": "bun run src/index.ts", "start:web": "bun --hot src/server.ts", "build:web": "bun build src/web/index.html --outdir dist", - "test": "bun test" + "test": "bun test", + "db:generate": "drizzle-kit generate", + "db:migrate": "drizzle-kit migrate" }, "devDependencies": { "@types/bun": "latest", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", + "drizzle-kit": "^0.31.10", "typescript": "^6.0.3" }, "dependencies": { "amazon-sp-api": "^1.2.1", + "drizzle-orm": "^0.45.2", "exceljs": "^4.4.0", "ioredis": "^5.10.1", + "postgres": "^3.4.9", "react": "^19.2.0", "react-dom": "^19.2.0", "xlsx": "^0.18.5" diff --git a/src/bestsellers-by-category.test.ts b/src/bestsellers-by-category.test.ts index bd75b79..41ed507 100644 --- a/src/bestsellers-by-category.test.ts +++ b/src/bestsellers-by-category.test.ts @@ -1,8 +1,41 @@ -import { test, expect, beforeAll, afterAll, beforeEach, mock } from "bun:test"; -import { Database } from "bun:sqlite"; -import { getDb, initDb, closeDb } from "./database.ts"; -import path from "node:path"; -import { rmSync, mkdirSync } from "node:fs"; +import { test, expect, beforeEach, mock } from "bun:test"; + +let nextId = 0; + +function chainable(resolveWith: any[] = []): any { + const p: any = Promise.resolve(resolveWith); + p.limit = (_n: any) => chainable(resolveWith); + p.where = (_cond: any) => chainable(resolveWith); + p.from = (_table: any) => chainable(resolveWith); + return p; +} + +const makeMockDb = (): any => ({ + insert: (_table: any) => ({ + values: (_vals: any) => ({ + returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]), + onConflictDoUpdate: (_conf: any) => Promise.resolve([]), + }), + }), + update: (_table: any) => ({ + set: (_vals: any) => ({ + where: (_cond: any) => Promise.resolve([]), + }), + }), + select: (_sel?: any) => ({ + from: (_table: any) => ({ + where: (_cond: any) => chainable(), + limit: (_n: any) => chainable(), + }), + }), + selectDistinct: (_sel: any) => ({ + from: (_table: any) => chainable(), + }), + execute: (_query: any) => Promise.resolve([]), + transaction: async (fn: (tx: any) => Promise) => fn(makeMockDb()), +}); + +mock.module("./db/index.ts", () => ({ db: makeMockDb(), client: {} })); const fetchSellabilityBatchMock = mock(async (asins: string[]) => { return new Map( @@ -47,40 +80,17 @@ mock.module("./llm.ts", () => ({ const modulePromise = import("./bestsellers-by-category.ts"); -const DB_TEST_PATH = path.join( - process.cwd(), - "test_output", - "test_analysis.sqlite", -); - -let db: Database; -let processCategory: (db: Database, runId: number, category: any, perCategoryTop: number) => Promise; -let insertCategoryRunSummary: (db: Database, summary: any, runTimestamp: string) => Promise; +let processCategory: (runId: number, category: any, perCategoryTop: number) => Promise; +let insertCategoryRunSummary: (summary: any, runTimestamp: string) => Promise; let originalFetch: typeof globalThis.fetch; -beforeAll(async () => { - const mod = await modulePromise; - processCategory = mod.processCategory; - insertCategoryRunSummary = mod.insertCategoryRunSummary; - - rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true }); - mkdirSync(path.dirname(DB_TEST_PATH), { recursive: true }); - initDb(DB_TEST_PATH); - db = getDb(DB_TEST_PATH); - - originalFetch = globalThis.fetch; -}); - -afterAll(() => { - globalThis.fetch = originalFetch; - closeDb(); - rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true }); -}); +const mod = await modulePromise; +processCategory = mod.processCategory; +insertCategoryRunSummary = mod.insertCategoryRunSummary; +originalFetch = globalThis.fetch; beforeEach(() => { - db.run("DELETE FROM product_analysis_results"); - db.run("DELETE FROM category_analysis_runs"); - + nextId = 0; globalThis.fetch = mock(async (input: string | URL | Request) => { const rawUrl = typeof input === "string" @@ -139,39 +149,34 @@ test("processCategory function test", async () => { childCount: 0, }; - const runId = await insertCategoryRunSummary(db, { - categoryId: mockCategory.id, - categoryLabel: mockCategory.label, - topAsinsChecked: 0, - availableAsins: 0, - fba: 0, - fbm: 0, - skip: 0, - status: "running", - error: "", - results: [], - }, new Date().toISOString()); - const summary = await processCategory(db, runId, mockCategory, 2); + const runId = await insertCategoryRunSummary( + { + categoryId: mockCategory.id, + categoryLabel: mockCategory.label, + topAsinsChecked: 0, + availableAsins: 0, + fba: 0, + fbm: 0, + skip: 0, + status: "running", + error: "", + results: [], + }, + new Date().toISOString(), + ); - const categoryRun = db.query("SELECT * FROM category_analysis_runs").all() as any[]; - expect(categoryRun.length).toBe(1); - expect(categoryRun[0].category_label).toBe("Category 1"); - expect(categoryRun[0].top_asins_checked).toBe(2); - expect(categoryRun[0].available_asins).toBe(2); - expect(categoryRun[0].fba_count).toBe(1); - expect(categoryRun[0].fbm_count).toBe(1); - expect(categoryRun[0].status).toBe("ok"); + const summary = await processCategory(runId, mockCategory, 2); - const productResults = db.query("SELECT * FROM product_analysis_results ORDER BY asin").all() as any[]; - expect(productResults.length).toBe(2); + expect(summary.status).toBe("ok"); + expect(summary.topAsinsChecked).toBe(2); + expect(summary.availableAsins).toBe(2); + expect(summary.fba).toBe(1); + expect(summary.fbm).toBe(1); + expect(summary.results?.length).toBe(2); + expect(summary.results?.[0]?.product.record.asin).toBe("B000000001"); + expect(summary.results?.[0]?.verdict.verdict).toBe("FBA"); + expect(summary.results?.[1]?.product.record.asin).toBe("B000000002"); + expect(summary.results?.[1]?.verdict.verdict).toBe("FBM"); - expect(productResults[0].asin).toBe("B000000001"); - expect(productResults[0].name).toBe("Product One"); - expect(productResults[0].verdict).toBe("FBA"); - expect(productResults[0].run_id).toBe(categoryRun[0].id); - - expect(productResults[1].asin).toBe("B000000002"); - expect(productResults[1].name).toBe("Product Two"); - expect(productResults[1].verdict).toBe("FBM"); - expect(productResults[1].run_id).toBe(categoryRun[0].id); + globalThis.fetch = originalFetch; }); diff --git a/src/bestsellers-by-category.ts b/src/bestsellers-by-category.ts index ed0721c..fbad857 100644 --- a/src/bestsellers-by-category.ts +++ b/src/bestsellers-by-category.ts @@ -1,6 +1,8 @@ import { existsSync, mkdirSync, readFileSync } from "node:fs"; import path from "node:path"; -import { type Database, getDb, initDb } from "./database.ts"; +import { db } from "./db/index.ts"; +import { runs, categoryProductResults } from "./db/schema.ts"; +import { eq, sql } from "drizzle-orm"; import { config } from "./config.ts"; import { analyzeProducts } from "./llm.ts"; import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts"; @@ -139,36 +141,32 @@ function printUsageAndExit(message: string): never { } export async function insertCategoryRunSummary( - db: Database, summary: CategoryRunSummary, runTimestamp: string, ): Promise { - const query = ` - INSERT INTO category_analysis_runs ( - category_id, category_label, run_timestamp, - top_asins_checked, available_asins, - fba_count, fbm_count, skip_count, - status, error_message - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); - `; - const result = db.run(query, [ - summary.categoryId, - summary.categoryLabel, - runTimestamp, - summary.topAsinsChecked, - summary.availableAsins, - summary.fba, - summary.fbm, - summary.skip, - summary.status, - summary.error, - ]); - // Bun's SQLite client returns { changes: number, lastInsertRowid: number | bigint } - return Number(result.lastInsertRowid); + const [row] = await db + .insert(runs) + .values({ + type: "category_analysis", + status: (summary.status as typeof runs.$inferInsert.status) ?? "running", + categoryId: summary.categoryId, + categoryLabel: summary.categoryLabel, + topAsinsChecked: summary.topAsinsChecked, + availableAsins: summary.availableAsins, + totalProducts: summary.topAsinsChecked, + fbaCount: summary.fba, + fbmCount: summary.fbm, + skipCount: summary.skip, + errorMessage: summary.error || null, + startedAt: new Date(runTimestamp), + }) + .returning({ id: runs.id }); + + if (!row) throw new Error("Failed to insert category run."); + return row.id; } export async function updateCategoryRunSummary( - db: Database, runId: number, summary: Pick< CategoryRunSummary, @@ -181,136 +179,110 @@ export async function updateCategoryRunSummary( | "error" >, ): Promise { - db.run( - ` - UPDATE category_analysis_runs - SET - top_asins_checked = ?, - available_asins = ?, - fba_count = ?, - fbm_count = ?, - skip_count = ?, - status = ?, - error_message = ? - WHERE id = ? - `, - [ - summary.topAsinsChecked, - summary.availableAsins, - summary.fba, - summary.fbm, - summary.skip, - summary.status, - summary.error, - runId, - ], - ); + await db + .update(runs) + .set({ + topAsinsChecked: summary.topAsinsChecked, + availableAsins: summary.availableAsins, + totalProducts: summary.topAsinsChecked, + fbaCount: summary.fba, + fbmCount: summary.fbm, + skipCount: summary.skip, + status: summary.status as typeof runs.$inferInsert.status, + errorMessage: summary.error || null, + ...(summary.status !== "running" ? { completedAt: new Date() } : {}), + }) + .where(eq(runs.id, runId)); } export async function insertProductAnalysisResults( - db: Database, runId: number, results: AnalysisResult[], ): Promise { - if (results.length === 0) { - return; - } + if (results.length === 0) return; - const insertStmt = db.prepare(` - INSERT INTO product_analysis_results ( - asin, run_id, name, brand, category, unit_cost, - current_price, avg_price_90d, avg_price_90d_sheet, - selling_price_sheet, sales_rank, sales_rank_avg_90d, - seller_count, amazon_is_seller, amazon_buybox_share_pct_90d, - monthly_sold, rank_drops_30d, rank_drops_90d, - fba_fee, fbm_fee, referral_percent, can_sell, - sellability_status, sellability_reason, - verdict, confidence, reasoning, fetched_at - ) VALUES ( - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ?, ?, ?, ? - ) - ON CONFLICT(asin) DO UPDATE SET - run_id = excluded.run_id, - name = excluded.name, - brand = excluded.brand, - category = excluded.category, - unit_cost = excluded.unit_cost, - current_price = excluded.current_price, - avg_price_90d = excluded.avg_price_90d, - avg_price_90d_sheet = excluded.avg_price_90d_sheet, - selling_price_sheet = excluded.selling_price_sheet, - sales_rank = excluded.sales_rank, - sales_rank_avg_90d = excluded.sales_rank_avg_90d, - seller_count = excluded.seller_count, - amazon_is_seller = excluded.amazon_is_seller, - amazon_buybox_share_pct_90d = excluded.amazon_buybox_share_pct_90d, - monthly_sold = excluded.monthly_sold, - rank_drops_30d = excluded.rank_drops_30d, - rank_drops_90d = excluded.rank_drops_90d, - fba_fee = excluded.fba_fee, - fbm_fee = excluded.fbm_fee, - referral_percent = excluded.referral_percent, - can_sell = excluded.can_sell, - sellability_status = excluded.sellability_status, - sellability_reason = excluded.sellability_reason, - verdict = excluded.verdict, - confidence = excluded.confidence, - reasoning = excluded.reasoning, - fetched_at = excluded.fetched_at; - `); + const rows = results.map((r) => { + const price = + r.product.keepa?.currentPrice ?? + r.product.record.sellingPriceFromSheet ?? + r.product.spApi.estimatedSalePrice; + const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank; - db.transaction((resultsBatch: AnalysisResult[]) => { - for (const r of resultsBatch) { - const price = - r.product.keepa?.currentPrice ?? - r.product.record.sellingPriceFromSheet ?? - r.product.spApi.estimatedSalePrice; - const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank; - - insertStmt.run( - r.product.record.asin, - runId, - r.product.record.name, - r.product.record.brand ?? null, + return { + asin: r.product.record.asin, + runId, + name: r.product.record.name, + brand: r.product.record.brand ?? null, + category: r.product.record.category ?? - r.product.keepa?.categoryTree?.join(" > ") ?? - null, - r.product.record.unitCost ?? null, - price ?? null, - r.product.keepa?.avgPrice90 ?? null, - r.product.record.avgPrice90FromSheet ?? null, - r.product.record.sellingPriceFromSheet ?? null, - rank ?? null, - r.product.keepa?.salesRankAvg90 ?? null, - r.product.keepa?.sellerCount ?? null, - r.product.keepa?.amazonIsSeller == null - ? null - : r.product.keepa.amazonIsSeller - ? 1 - : 0, - r.product.keepa?.amazonBuyboxSharePct90d ?? null, - r.product.keepa?.monthlySold ?? null, - r.product.keepa?.salesRankDrops30 ?? null, - r.product.keepa?.salesRankDrops90 ?? null, - r.product.spApi.fbaFee ?? null, - r.product.spApi.fbmFee ?? null, - r.product.spApi.referralFeePercent ?? null, + r.product.keepa?.categoryTree?.join(" > ") ?? + null, + unitCost: r.product.record.unitCost ?? null, + currentPrice: price ?? null, + avgPrice90d: r.product.keepa?.avgPrice90 ?? null, + avgPrice90dSheet: r.product.record.avgPrice90FromSheet ?? null, + sellingPriceSheet: r.product.record.sellingPriceFromSheet ?? null, + salesRank: rank ?? null, + salesRankAvg90d: r.product.keepa?.salesRankAvg90 ?? null, + sellerCount: r.product.keepa?.sellerCount ?? null, + amazonIsSeller: r.product.keepa?.amazonIsSeller ?? null, + amazonBuyboxSharePct90d: r.product.keepa?.amazonBuyboxSharePct90d ?? null, + monthlySold: r.product.keepa?.monthlySold ?? null, + rankDrops30d: r.product.keepa?.salesRankDrops30 ?? null, + rankDrops90d: r.product.keepa?.salesRankDrops90 ?? null, + fbaFee: r.product.spApi.fbaFee ?? null, + fbmFee: r.product.spApi.fbmFee ?? null, + referralPercent: r.product.spApi.referralFeePercent ?? null, + canSell: r.product.spApi.canSell == null ? "unknown" : r.product.spApi.canSell ? "yes" : "no", - r.product.spApi.sellabilityStatus ?? null, - r.product.spApi.sellabilityReason ?? null, - r.verdict.verdict, - r.verdict.confidence, - r.verdict.reasoning ?? null, - r.product.fetchedAt, - ); - } - })(results); // Execute the transaction with the results batch + sellabilityStatus: r.product.spApi.sellabilityStatus ?? null, + sellabilityReason: r.product.spApi.sellabilityReason ?? null, + verdict: r.verdict.verdict, + confidence: r.verdict.confidence, + reasoning: r.verdict.reasoning ?? null, + fetchedAt: new Date(r.product.fetchedAt), + }; + }); + + await db + .insert(categoryProductResults) + .values(rows) + .onConflictDoUpdate({ + target: categoryProductResults.asin, + set: { + runId: sql`EXCLUDED.run_id`, + name: sql`EXCLUDED.name`, + brand: sql`EXCLUDED.brand`, + category: sql`EXCLUDED.category`, + unitCost: sql`EXCLUDED.unit_cost`, + currentPrice: sql`EXCLUDED.current_price`, + avgPrice90d: sql`EXCLUDED.avg_price_90d`, + avgPrice90dSheet: sql`EXCLUDED.avg_price_90d_sheet`, + sellingPriceSheet: sql`EXCLUDED.selling_price_sheet`, + salesRank: sql`EXCLUDED.sales_rank`, + salesRankAvg90d: sql`EXCLUDED.sales_rank_avg_90d`, + sellerCount: sql`EXCLUDED.seller_count`, + amazonIsSeller: sql`EXCLUDED.amazon_is_seller`, + amazonBuyboxSharePct90d: sql`EXCLUDED.amazon_buybox_share_pct_90d`, + monthlySold: sql`EXCLUDED.monthly_sold`, + rankDrops30d: sql`EXCLUDED.rank_drops_30d`, + rankDrops90d: sql`EXCLUDED.rank_drops_90d`, + fbaFee: sql`EXCLUDED.fba_fee`, + fbmFee: sql`EXCLUDED.fbm_fee`, + referralPercent: sql`EXCLUDED.referral_percent`, + canSell: sql`EXCLUDED.can_sell`, + sellabilityStatus: sql`EXCLUDED.sellability_status`, + sellabilityReason: sql`EXCLUDED.sellability_reason`, + verdict: sql`EXCLUDED.verdict`, + confidence: sql`EXCLUDED.confidence`, + reasoning: sql`EXCLUDED.reasoning`, + fetchedAt: sql`EXCLUDED.fetched_at`, + }, + }); } function loadCategoryBlacklist(filePath: string): Set { @@ -1014,7 +986,6 @@ function buildEnrichedProducts( } export async function processCategory( - db: Database, runId: number, category: CategoryInfo, perCategoryTop: number, @@ -1025,7 +996,7 @@ export async function processCategory( const topAsins = await fetchCategoryBestSellerAsins(category, perCategoryTop); if (topAsins.length === 0) { log("info", " Keepa returned no ASINs for this category."); - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: 0, availableAsins: 0, fba: 0, @@ -1069,7 +1040,7 @@ export async function processCategory( ` Sellable ASINs: ${availableAsins.length}/${uniqueTopAsins.length}`, ); if (availableAsins.length === 0) { - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: uniqueTopAsins.length, availableAsins: 0, fba: 0, @@ -1137,7 +1108,7 @@ export async function processCategory( }, })); - await insertProductAnalysisResults(db, runId, batchResults); + await insertProductAnalysisResults(runId, batchResults); for (const result of batchResults) { results.push(result); @@ -1150,7 +1121,7 @@ export async function processCategory( } } - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: uniqueTopAsins.length, availableAsins: availableAsins.length, fba, @@ -1170,7 +1141,7 @@ export async function processCategory( } } - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: uniqueTopAsins.length, availableAsins: availableAsins.length, fba, @@ -1199,10 +1170,6 @@ export async function main(): Promise { assertSpApiPrerequisites(); mkdirSync(args.outputDir, { recursive: true }); - const DB_PATH = - process.env.RESULTS_DB_PATH || path.join(process.cwd(), "db", "results.db"); - initDb(DB_PATH); - const db = getDb(DB_PATH); log("info", "Starting per-category bestseller pipeline"); log("info", `Marketplace: ${config.spApiMarketplaceId}`); @@ -1236,7 +1203,6 @@ export async function main(): Promise { let runId: number | undefined; try { runId = await insertCategoryRunSummary( - db, { categoryId: category.id, categoryLabel: category.label, @@ -1253,7 +1219,6 @@ export async function main(): Promise { ); categorySummary = await processCategory( - db, runId, category, args.perCategoryTop, @@ -1283,7 +1248,7 @@ export async function main(): Promise { results: [], }; if (runId) { - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: 0, availableAsins: 0, fba: 0, diff --git a/src/check_db.ts b/src/check_db.ts index fa086c8..3b2636c 100644 --- a/src/check_db.ts +++ b/src/check_db.ts @@ -1,21 +1,16 @@ -import { getDb } from "./database.ts"; -import path from "node:path"; +import { db } from "./db/index.ts"; +import { runs } from "./db/schema.ts"; +import { eq } from "drizzle-orm"; async function checkDb() { - const DB_PATH = - process.env.RESULTS_DB_PATH || path.join(process.cwd(), "db", "results.db"); - const db = getDb(DB_PATH); - try { - const query = db.query( - "SELECT * FROM category_analysis_runs WHERE category_id = ?", - ); - const result = query.all(19419898011); + const result = await db + .select() + .from(runs) + .where(eq(runs.type, "category_analysis")); console.log(JSON.stringify(result, null, 2)); } catch (error) { console.error("Database query failed:", error); - } finally { - db.close(); } } diff --git a/src/database.ts b/src/database.ts index 8d4413f..8b0b708 100644 --- a/src/database.ts +++ b/src/database.ts @@ -1,494 +1,3 @@ -import { Database } from "bun:sqlite"; -import { dirname } from "node:path"; -import { mkdirSync } from "node:fs"; -export { Database } from "bun:sqlite"; - -let db: Database | null = null; - -export function getDb(dbPath: string): Database { - if (!db) { - const dbDir = dirname(dbPath); - if (dbDir && dbDir !== ".") { - mkdirSync(dbDir, { recursive: true }); - } - db = new Database(dbPath); - db.run("PRAGMA journal_mode = WAL;"); // Enable WAL mode for better performance - db.run("PRAGMA foreign_keys = ON;"); // Enforce foreign key constraints - } - return db; -} - -export function closeDb(): void { - if (db) { - db.close(); - db = null; - } -} - -function createProductAnalysisResultsTable(database: Database): void { - database.run(` - CREATE TABLE IF NOT EXISTS product_analysis_results ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - asin TEXT NOT NULL, - run_id INTEGER NOT NULL, - name TEXT NOT NULL, - brand TEXT, - category TEXT, - unit_cost REAL, - current_price REAL, - avg_price_90d REAL, - avg_price_90d_sheet REAL, - selling_price_sheet REAL, - sales_rank INTEGER, - sales_rank_avg_90d INTEGER, - seller_count INTEGER, - amazon_is_seller INTEGER, - amazon_buybox_share_pct_90d REAL, - monthly_sold INTEGER, - rank_drops_30d INTEGER, - rank_drops_90d INTEGER, - fba_fee REAL, - fbm_fee REAL, - referral_percent REAL, - can_sell TEXT, - sellability_status TEXT, - sellability_reason TEXT, - verdict TEXT NOT NULL, - confidence REAL NOT NULL, - reasoning TEXT, - fetched_at TEXT NOT NULL, - UNIQUE(asin), - FOREIGN KEY (run_id) REFERENCES category_analysis_runs(id) - ); - `); -} - -function ensureProductAnalysisResultsTable(database: Database): void { - const tableInfo = database - .query("PRAGMA table_info(product_analysis_results)") - .all() as Array<{ name: string; pk: number }>; - - if (tableInfo.length === 0) { - createProductAnalysisResultsTable(database); - return; - } - - const hasIdColumn = tableInfo.some((col) => col.name === "id"); - const hasAsinPrimaryKey = tableInfo.some( - (col) => col.name === "asin" && col.pk === 1, - ); - - const indexList = database - .query("PRAGMA index_list(product_analysis_results)") - .all() as Array<{ name: string; unique: number }>; - const hasUniqueAsinConstraint = indexList.some((idx) => { - if (idx.unique !== 1) return false; - const columns = database - .query(`PRAGMA index_info(${JSON.stringify(idx.name)})`) - .all() as Array<{ name: string }>; - return columns.length === 1 && columns[0]?.name === "asin"; - }); - - if (!hasIdColumn || hasAsinPrimaryKey || !hasUniqueAsinConstraint) { - database.run( - "ALTER TABLE product_analysis_results RENAME TO product_analysis_results_legacy", - ); - createProductAnalysisResultsTable(database); - database.run(` - WITH ranked AS ( - SELECT - asin, run_id, name, brand, category, unit_cost, - current_price, avg_price_90d, avg_price_90d_sheet, - selling_price_sheet, sales_rank, sales_rank_avg_90d, - seller_count, NULL AS amazon_is_seller, - NULL AS amazon_buybox_share_pct_90d, - monthly_sold, rank_drops_30d, rank_drops_90d, - fba_fee, fbm_fee, referral_percent, can_sell, - sellability_status, sellability_reason, - verdict, confidence, reasoning, fetched_at, - ROW_NUMBER() OVER ( - PARTITION BY asin - ORDER BY datetime(fetched_at) DESC, run_id DESC, id DESC - ) AS row_num - FROM product_analysis_results_legacy - ) - INSERT INTO product_analysis_results ( - asin, run_id, name, brand, category, unit_cost, - current_price, avg_price_90d, avg_price_90d_sheet, - selling_price_sheet, sales_rank, sales_rank_avg_90d, - seller_count, amazon_is_seller, amazon_buybox_share_pct_90d, - monthly_sold, rank_drops_30d, rank_drops_90d, - fba_fee, fbm_fee, referral_percent, can_sell, - sellability_status, sellability_reason, - verdict, confidence, reasoning, fetched_at - ) - SELECT - asin, run_id, name, brand, category, unit_cost, - current_price, avg_price_90d, avg_price_90d_sheet, - selling_price_sheet, sales_rank, sales_rank_avg_90d, - seller_count, amazon_is_seller, amazon_buybox_share_pct_90d, - monthly_sold, rank_drops_30d, rank_drops_90d, - fba_fee, fbm_fee, referral_percent, can_sell, - sellability_status, sellability_reason, - verdict, confidence, reasoning, fetched_at - FROM ranked - WHERE row_num = 1 - `); - database.run("DROP TABLE product_analysis_results_legacy"); - } -} - -function ensureProductAnalysisResultsColumns(database: Database): void { - const tableInfo = database - .query("PRAGMA table_info(product_analysis_results)") - .all() as Array<{ name: string }>; - - if (tableInfo.length === 0) { - return; - } - - const existingColumns = new Set(tableInfo.map((col) => col.name)); - const requiredColumns: Array<{ name: string; type: string }> = [ - { name: "amazon_is_seller", type: "INTEGER" }, - { name: "amazon_buybox_share_pct_90d", type: "REAL" }, - ]; - - for (const column of requiredColumns) { - if (!existingColumns.has(column.name)) { - database.run( - `ALTER TABLE product_analysis_results ADD COLUMN ${column.name} ${column.type}`, - ); - } - } -} - -function ensureResultsTableColumns(database: Database): void { - const tableInfo = database - .query("PRAGMA table_info(results)") - .all() as Array<{ name: string }>; - - if (tableInfo.length === 0) { - return; - } - - const existingColumns = new Set(tableInfo.map((col) => col.name)); - const requiredColumns: Array<{ name: string; type: string }> = [ - { name: "fba_net_sheet", type: "REAL" }, - { name: "gross_profit_dollar", type: "REAL" }, - { name: "gross_profit_pct", type: "REAL" }, - { name: "net_profit_sheet", type: "REAL" }, - { name: "roi_sheet", type: "REAL" }, - { name: "moq", type: "INTEGER" }, - { name: "moq_cost", type: "REAL" }, - { name: "qty_available", type: "INTEGER" }, - { name: "supplier", type: "TEXT" }, - { name: "source_url", type: "TEXT" }, - { name: "asin_link", type: "TEXT" }, - { name: "promo_coupon_code", type: "TEXT" }, - { name: "notes", type: "TEXT" }, - { name: "lead_date", type: "TEXT" }, - { name: "amazon_is_seller", type: "INTEGER" }, - { name: "amazon_buybox_share_pct_90d", type: "REAL" }, - { name: "upc", type: "TEXT" }, - { name: "supplier_score", type: "REAL" }, - { name: "supplier_profit", type: "REAL" }, - { name: "supplier_margin", type: "REAL" }, - { name: "supplier_roi", type: "REAL" }, - { name: "supplier_reason", type: "TEXT" }, - { name: "upc_lookup_status", type: "TEXT" }, - { name: "upc_lookup_reason", type: "TEXT" }, - { name: "candidate_asins", type: "TEXT" }, - ]; - - for (const column of requiredColumns) { - if (!existingColumns.has(column.name)) { - database.run( - `ALTER TABLE results ADD COLUMN ${column.name} ${column.type}`, - ); - } - } -} - -export function initDb(dbPath: string): void { - const database = getDb(dbPath); - database.run(` - CREATE TABLE IF NOT EXISTS runs ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - timestamp TEXT NOT NULL, - input_file TEXT NOT NULL, - output_file TEXT, - total_products INTEGER, - fba_count INTEGER, - fbm_count INTEGER, - skip_count INTEGER - ); - `); - database.run(` - CREATE TABLE IF NOT EXISTS results ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - run_id INTEGER NOT NULL, - asin TEXT NOT NULL, - product_name TEXT, - brand TEXT, - category TEXT, - unit_cost REAL, - current_price REAL, - avg_price_90d REAL, - avg_price_90d_sheet REAL, - selling_price_sheet REAL, - sales_rank INTEGER, - rank_avg_90d INTEGER, - sellers INTEGER, - amazon_is_seller INTEGER, - amazon_buybox_share_pct_90d REAL, - monthly_sold INTEGER, - rank_drops_30d INTEGER, - rank_drops_90d INTEGER, - fba_net_sheet REAL, - gross_profit_dollar REAL, - gross_profit_pct REAL, - net_profit_sheet REAL, - roi_sheet REAL, - moq INTEGER, - moq_cost REAL, - qty_available INTEGER, - supplier TEXT, - source_url TEXT, - asin_link TEXT, - promo_coupon_code TEXT, - notes TEXT, - lead_date TEXT, - upc TEXT, - fba_fee REAL, - fbm_fee REAL, - referral_percent REAL, - supplier_score REAL, - supplier_profit REAL, - supplier_margin REAL, - supplier_roi REAL, - supplier_reason TEXT, - upc_lookup_status TEXT, - upc_lookup_reason TEXT, - candidate_asins TEXT, - can_sell TEXT, - sellability_status TEXT, - sellability_reason TEXT, - verdict TEXT NOT NULL, - confidence INTEGER, - reasoning TEXT, - fetched_at TEXT NOT NULL, - FOREIGN KEY (run_id) REFERENCES runs(id) - ); - `); - ensureResultsTableColumns(database); - database.run(` - CREATE TABLE IF NOT EXISTS category_analysis_runs ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - category_id INTEGER NOT NULL, - category_label TEXT NOT NULL, - run_timestamp TEXT NOT NULL, - top_asins_checked INTEGER NOT NULL, - available_asins INTEGER NOT NULL, - fba_count INTEGER NOT NULL, - fbm_count INTEGER NOT NULL, - skip_count INTEGER NOT NULL, - status TEXT NOT NULL, - error_message TEXT - ); - `); - ensureProductAnalysisResultsTable(database); - ensureProductAnalysisResultsColumns(database); - - database.run( - `CREATE INDEX IF NOT EXISTS idx_runs_timestamp ON runs(timestamp DESC);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_results_run_id ON results(run_id);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_results_verdict ON results(verdict);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_results_sellability_status ON results(sellability_status);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_results_fetched_at ON results(fetched_at DESC);`, - ); - database.run(`CREATE INDEX IF NOT EXISTS idx_results_asin ON results(asin);`); - database.run( - `CREATE INDEX IF NOT EXISTS idx_category_runs_timestamp ON category_analysis_runs(run_timestamp DESC);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_category_runs_status ON category_analysis_runs(status);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_product_results_run_id ON product_analysis_results(run_id);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_product_results_verdict ON product_analysis_results(verdict);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_product_results_sellability_status ON product_analysis_results(sellability_status);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_product_results_fetched_at ON product_analysis_results(fetched_at DESC);`, - ); - initStalkerDb(database); -} - -export function initStalkerDb(database: Database): void { - resetLegacyStalkerSchema(database); - - database.run(` - CREATE TABLE IF NOT EXISTS stalker_runs ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - input_file TEXT NOT NULL, - started_at TEXT NOT NULL, - completed_at TEXT, - requested_asins INTEGER NOT NULL DEFAULT 0, - skipped_asins INTEGER NOT NULL DEFAULT 0, - scanned_asins INTEGER NOT NULL DEFAULT 0, - source_asins_with_matches INTEGER NOT NULL DEFAULT 0, - candidate_sellers INTEGER NOT NULL DEFAULT 0, - qualifying_sellers INTEGER NOT NULL DEFAULT 0, - matched_sellers INTEGER NOT NULL DEFAULT 0, - seller_metadata_requests INTEGER NOT NULL DEFAULT 0, - seller_storefront_requests INTEGER NOT NULL DEFAULT 0, - inventory_sellability_checked_asins INTEGER NOT NULL DEFAULT 0, - inventory_sellability_available_asins INTEGER NOT NULL DEFAULT 0, - inventory_sellability_excluded_asins INTEGER NOT NULL DEFAULT 0, - persisted_inventory_asins INTEGER NOT NULL DEFAULT 0, - status TEXT NOT NULL, - error_message TEXT - ); - `); - - database.run(` - CREATE TABLE IF NOT EXISTS stalker_asin_scans ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - run_id INTEGER NOT NULL, - source_asin TEXT NOT NULL, - title TEXT, - offer_count INTEGER NOT NULL DEFAULT 0, - candidate_seller_count INTEGER NOT NULL DEFAULT 0, - matched_seller_count INTEGER NOT NULL DEFAULT 0, - fetched_at TEXT NOT NULL, - raw_product_json TEXT, - UNIQUE(run_id, source_asin), - FOREIGN KEY (run_id) REFERENCES stalker_runs(id) ON DELETE CASCADE - ); - `); - - database.run(` - CREATE TABLE IF NOT EXISTS stalker_sellers ( - seller_id TEXT PRIMARY KEY, - seller_name TEXT, - rating REAL, - rating_count INTEGER, - storefront_asin_total INTEGER, - persisted_inventory_sample_count INTEGER, - last_updated_at TEXT NOT NULL, - raw_seller_json TEXT - ); - `); - - database.run(` - CREATE TABLE IF NOT EXISTS stalker_asin_sellers ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - scan_id INTEGER NOT NULL, - seller_id TEXT NOT NULL, - offer_price REAL, - condition TEXT, - is_fba INTEGER, - stock INTEGER, - seller_rating REAL, - seller_rating_count INTEGER, - raw_offer_json TEXT, - UNIQUE(scan_id, seller_id), - FOREIGN KEY (scan_id) REFERENCES stalker_asin_scans(id) ON DELETE CASCADE, - FOREIGN KEY (seller_id) REFERENCES stalker_sellers(seller_id) - ); - `); - - database.run(` - CREATE TABLE IF NOT EXISTS stalker_seller_inventory ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - run_id INTEGER NOT NULL, - seller_id TEXT NOT NULL, - asin TEXT NOT NULL, - can_sell INTEGER, - sellability_status TEXT, - sellability_reason TEXT, - product_title TEXT, - brand TEXT, - category_tree TEXT, - current_price REAL, - avg_price_90d REAL, - sales_rank INTEGER, - monthly_sold INTEGER, - seller_count INTEGER, - amazon_is_seller INTEGER, - raw_product_json TEXT, - last_seen_at TEXT NOT NULL, - raw_inventory_json TEXT, - UNIQUE(run_id, seller_id, asin), - FOREIGN KEY (run_id) REFERENCES stalker_runs(id) ON DELETE CASCADE, - FOREIGN KEY (seller_id) REFERENCES stalker_sellers(seller_id) - ); - `); - - database.run( - `CREATE INDEX IF NOT EXISTS idx_stalker_runs_started_at ON stalker_runs(started_at DESC);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_stalker_scans_run_id ON stalker_asin_scans(run_id);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_stalker_scans_source_asin ON stalker_asin_scans(source_asin);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_stalker_asin_sellers_seller_id ON stalker_asin_sellers(seller_id);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_stalker_inventory_seller_id ON stalker_seller_inventory(seller_id);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_stalker_inventory_asin ON stalker_seller_inventory(asin);`, - ); - database.run( - `CREATE INDEX IF NOT EXISTS idx_stalker_inventory_product_title ON stalker_seller_inventory(product_title);`, - ); -} - -function resetLegacyStalkerSchema(database: Database): void { - const runColumns = database - .query("PRAGMA table_info(stalker_runs)") - .all() as Array<{ name: string }>; - if (runColumns.length === 0) return; - - const columnNames = new Set(runColumns.map((column) => column.name)); - if ( - columnNames.has("scanned_asins") && - columnNames.has("inventory_sellability_checked_asins") && - inventoryColumnsHaveSellability(database) - ) { - return; - } - - database.run("DROP TABLE IF EXISTS stalker_seller_inventory"); - database.run("DROP TABLE IF EXISTS stalker_asin_sellers"); - database.run("DROP TABLE IF EXISTS stalker_sellers"); - database.run("DROP TABLE IF EXISTS stalker_asin_scans"); - database.run("DROP TABLE IF EXISTS stalker_runs"); -} - -function inventoryColumnsHaveSellability(database: Database): boolean { - const inventoryColumns = database - .query("PRAGMA table_info(stalker_seller_inventory)") - .all() as Array<{ name: string }>; - const columnNames = new Set(inventoryColumns.map((column) => column.name)); - return ( - columnNames.has("sellability_status") && - columnNames.has("product_title") - ); -} +// Central re-export so existing `import { db } from "./database.ts"` keeps working. +export { db, type Db } from "./db/index.ts"; +export * as schema from "./db/schema.ts"; diff --git a/src/db/index.ts b/src/db/index.ts new file mode 100644 index 0000000..f1b2e9a --- /dev/null +++ b/src/db/index.ts @@ -0,0 +1,15 @@ +import { drizzle } from "drizzle-orm/postgres-js"; +import postgres from "postgres"; +import * as schema from "./schema.ts"; + +const url = Bun.env.DB_CONNECTION_STRING; +if (!url) { + throw new Error("Missing required env var: DB_CONNECTION_STRING"); +} + +// Shared connection pool — imported once and reused across the process. +export const client = postgres(url); + +export const db = drizzle(client, { schema }); + +export type Db = typeof db; diff --git a/src/db/schema.ts b/src/db/schema.ts new file mode 100644 index 0000000..59b77bb --- /dev/null +++ b/src/db/schema.ts @@ -0,0 +1,343 @@ +import { + boolean, + index, + integer, + pgEnum, + pgTable, + real, + serial, + text, + timestamp, + unique, +} from "drizzle-orm/pg-core"; + +// ─── Enums ─────────────────────────────────────────────────────────────────── + +export const runTypeEnum = pgEnum("run_type", [ + "lead_analysis", + "category_analysis", + "supplier_upc", + "stalker", +]); + +export const runStatusEnum = pgEnum("run_status", [ + "running", + "ok", + "empty", + "failed", + "completed", +]); + +// ─── Runs ───────────────────────────────────────────────────────────────────── +// Unified run log; replaces the old `runs` and `category_analysis_runs` tables. +// Category-specific columns (categoryId, categoryLabel, …) are null for +// lead_analysis / supplier_upc runs. + +export const runs = pgTable( + "runs", + { + id: serial("id").primaryKey(), + type: runTypeEnum("type").notNull(), + inputFile: text("input_file"), + outputFile: text("output_file"), + status: runStatusEnum("status").notNull().default("running"), + errorMessage: text("error_message"), + totalProducts: integer("total_products"), + fbaCount: integer("fba_count"), + fbmCount: integer("fbm_count"), + skipCount: integer("skip_count"), + // Category-pipeline only + categoryId: integer("category_id"), + categoryLabel: text("category_label"), + topAsinsChecked: integer("top_asins_checked"), + availableAsins: integer("available_asins"), + startedAt: timestamp("started_at", { withTimezone: true }) + .notNull() + .defaultNow(), + completedAt: timestamp("completed_at", { withTimezone: true }), + }, + (t) => [ + index("idx_runs_started_at").on(t.startedAt), + index("idx_runs_type").on(t.type), + index("idx_runs_status").on(t.status), + ], +); + +// ─── Analysis results ───────────────────────────────────────────────────────── +// Archival table: one row per product per run for the lead-list and supplier +// UPC pipelines. Multiple rows for the same ASIN across different runs is fine. + +export const analysisResults = pgTable( + "analysis_results", + { + id: serial("id").primaryKey(), + runId: integer("run_id") + .notNull() + .references(() => runs.id), + asin: text("asin").notNull(), + // Product identity + productName: text("product_name"), + brand: text("brand"), + category: text("category"), + upc: text("upc"), + // Supplier sheet data (lead_analysis only) + unitCost: real("unit_cost"), + avgPrice90dSheet: real("avg_price_90d_sheet"), + sellingPriceSheet: real("selling_price_sheet"), + fbaNetSheet: real("fba_net_sheet"), + grossProfitDollar: real("gross_profit_dollar"), + grossProfitPct: real("gross_profit_pct"), + netProfitSheet: real("net_profit_sheet"), + roiSheet: real("roi_sheet"), + moq: integer("moq"), + moqCost: real("moq_cost"), + qtyAvailable: integer("qty_available"), + supplier: text("supplier"), + sourceUrl: text("source_url"), + asinLink: text("asin_link"), + promoCouponCode: text("promo_coupon_code"), + notes: text("notes"), + leadDate: text("lead_date"), + // Market data + currentPrice: real("current_price"), + avgPrice90d: real("avg_price_90d"), + salesRank: integer("sales_rank"), + rankAvg90d: integer("rank_avg_90d"), + monthlySold: integer("monthly_sold"), + rankDrops30d: integer("rank_drops_30d"), + rankDrops90d: integer("rank_drops_90d"), + sellerCount: integer("seller_count"), + amazonIsSeller: boolean("amazon_is_seller"), + amazonBuyboxSharePct90d: real("amazon_buybox_share_pct_90d"), + // Fees + fbaFee: real("fba_fee"), + fbmFee: real("fbm_fee"), + referralPercent: real("referral_percent"), + // Sellability + canSell: text("can_sell"), + sellabilityStatus: text("sellability_status"), + sellabilityReason: text("sellability_reason"), + // Supplier-UPC scoring + supplierScore: real("supplier_score"), + supplierProfit: real("supplier_profit"), + supplierMargin: real("supplier_margin"), + supplierRoi: real("supplier_roi"), + supplierReason: text("supplier_reason"), + upcLookupStatus: text("upc_lookup_status"), + upcLookupReason: text("upc_lookup_reason"), + candidateAsins: text("candidate_asins"), + // Verdict + verdict: text("verdict").notNull(), + confidence: real("confidence"), + reasoning: text("reasoning"), + fetchedAt: timestamp("fetched_at", { withTimezone: true }).notNull(), + }, + (t) => [ + index("idx_analysis_results_run_id").on(t.runId), + index("idx_analysis_results_asin").on(t.asin), + index("idx_analysis_results_verdict").on(t.verdict), + index("idx_analysis_results_sellability_status").on(t.sellabilityStatus), + index("idx_analysis_results_fetched_at").on(t.fetchedAt), + ], +); + +// ─── Category product results ────────────────────────────────────────────────── +// Latest-per-ASIN snapshot for the category pipelines (bestsellers, monthly-sold, +// mid-range, stalker analysis). Upserted on conflict so each ASIN has one row. + +export const categoryProductResults = pgTable( + "category_product_results", + { + id: serial("id").primaryKey(), + asin: text("asin").notNull().unique(), + runId: integer("run_id") + .notNull() + .references(() => runs.id), + name: text("name").notNull(), + brand: text("brand"), + category: text("category"), + unitCost: real("unit_cost"), + currentPrice: real("current_price"), + avgPrice90d: real("avg_price_90d"), + avgPrice90dSheet: real("avg_price_90d_sheet"), + sellingPriceSheet: real("selling_price_sheet"), + salesRank: integer("sales_rank"), + salesRankAvg90d: integer("sales_rank_avg_90d"), + sellerCount: integer("seller_count"), + amazonIsSeller: boolean("amazon_is_seller"), + amazonBuyboxSharePct90d: real("amazon_buybox_share_pct_90d"), + monthlySold: integer("monthly_sold"), + rankDrops30d: integer("rank_drops_30d"), + rankDrops90d: integer("rank_drops_90d"), + fbaFee: real("fba_fee"), + fbmFee: real("fbm_fee"), + referralPercent: real("referral_percent"), + canSell: text("can_sell"), + sellabilityStatus: text("sellability_status"), + sellabilityReason: text("sellability_reason"), + verdict: text("verdict").notNull(), + confidence: real("confidence").notNull(), + reasoning: text("reasoning"), + fetchedAt: timestamp("fetched_at", { withTimezone: true }).notNull(), + }, + (t) => [ + index("idx_category_results_run_id").on(t.runId), + index("idx_category_results_verdict").on(t.verdict), + index("idx_category_results_sellability_status").on(t.sellabilityStatus), + index("idx_category_results_fetched_at").on(t.fetchedAt), + ], +); + +// ─── Stalker runs ───────────────────────────────────────────────────────────── + +export const stalkerRuns = pgTable( + "stalker_runs", + { + id: serial("id").primaryKey(), + inputFile: text("input_file").notNull(), + startedAt: timestamp("started_at", { withTimezone: true }).notNull(), + completedAt: timestamp("completed_at", { withTimezone: true }), + requestedAsins: integer("requested_asins").notNull().default(0), + skippedAsins: integer("skipped_asins").notNull().default(0), + scannedAsins: integer("scanned_asins").notNull().default(0), + sourceAsinsWithMatches: integer("source_asins_with_matches") + .notNull() + .default(0), + candidateSellers: integer("candidate_sellers").notNull().default(0), + qualifyingSellers: integer("qualifying_sellers").notNull().default(0), + matchedSellers: integer("matched_sellers").notNull().default(0), + sellerMetadataRequests: integer("seller_metadata_requests") + .notNull() + .default(0), + sellerStorefrontRequests: integer("seller_storefront_requests") + .notNull() + .default(0), + inventorySellabilityCheckedAsins: integer( + "inventory_sellability_checked_asins", + ) + .notNull() + .default(0), + inventorySellabilityAvailableAsins: integer( + "inventory_sellability_available_asins", + ) + .notNull() + .default(0), + inventorySellabilityExcludedAsins: integer( + "inventory_sellability_excluded_asins", + ) + .notNull() + .default(0), + persistedInventoryAsins: integer("persisted_inventory_asins") + .notNull() + .default(0), + status: text("status").notNull(), + errorMessage: text("error_message"), + }, + (t) => [index("idx_stalker_runs_started_at").on(t.startedAt)], +); + +// ─── Stalker ASIN scans ─────────────────────────────────────────────────────── + +export const stalkerAsinScans = pgTable( + "stalker_asin_scans", + { + id: serial("id").primaryKey(), + runId: integer("run_id") + .notNull() + .references(() => stalkerRuns.id, { onDelete: "cascade" }), + sourceAsin: text("source_asin").notNull(), + title: text("title"), + offerCount: integer("offer_count").notNull().default(0), + candidateSellerCount: integer("candidate_seller_count") + .notNull() + .default(0), + matchedSellerCount: integer("matched_seller_count").notNull().default(0), + fetchedAt: timestamp("fetched_at", { withTimezone: true }).notNull(), + rawProductJson: text("raw_product_json"), + }, + (t) => [ + unique("uq_stalker_scans_run_asin").on(t.runId, t.sourceAsin), + index("idx_stalker_scans_run_id").on(t.runId), + index("idx_stalker_scans_source_asin").on(t.sourceAsin), + ], +); + +// ─── Sellers ────────────────────────────────────────────────────────────────── +// General seller registry (was stalker_sellers). + +export const sellers = pgTable("sellers", { + sellerId: text("seller_id").primaryKey(), + sellerName: text("seller_name"), + rating: real("rating"), + ratingCount: integer("rating_count"), + storefrontAsinTotal: integer("storefront_asin_total"), + persistedInventorySampleCount: integer("persisted_inventory_sample_count"), + lastUpdatedAt: timestamp("last_updated_at", { withTimezone: true }).notNull(), + rawSellerJson: text("raw_seller_json"), +}); + +// ─── Stalker ASIN sellers ───────────────────────────────────────────────────── + +export const stalkerAsinSellers = pgTable( + "stalker_asin_sellers", + { + id: serial("id").primaryKey(), + scanId: integer("scan_id") + .notNull() + .references(() => stalkerAsinScans.id, { onDelete: "cascade" }), + sellerId: text("seller_id") + .notNull() + .references(() => sellers.sellerId), + offerPrice: real("offer_price"), + condition: text("condition"), + isFba: boolean("is_fba"), + stock: integer("stock"), + sellerRating: real("seller_rating"), + sellerRatingCount: integer("seller_rating_count"), + rawOfferJson: text("raw_offer_json"), + }, + (t) => [ + unique("uq_stalker_asin_sellers_scan_seller").on(t.scanId, t.sellerId), + ], +); + +// ─── Stalker seller inventory ───────────────────────────────────────────────── + +export const stalkerSellerInventory = pgTable( + "stalker_seller_inventory", + { + id: serial("id").primaryKey(), + runId: integer("run_id") + .notNull() + .references(() => stalkerRuns.id, { onDelete: "cascade" }), + sellerId: text("seller_id") + .notNull() + .references(() => sellers.sellerId), + asin: text("asin").notNull(), + canSell: boolean("can_sell"), + sellabilityStatus: text("sellability_status"), + sellabilityReason: text("sellability_reason"), + productTitle: text("product_title"), + brand: text("brand"), + categoryTree: text("category_tree"), + currentPrice: real("current_price"), + avgPrice90d: real("avg_price_90d"), + salesRank: integer("sales_rank"), + monthlySold: integer("monthly_sold"), + sellerCount: integer("seller_count"), + amazonIsSeller: boolean("amazon_is_seller"), + rawProductJson: text("raw_product_json"), + lastSeenAt: timestamp("last_seen_at", { withTimezone: true }).notNull(), + rawInventoryJson: text("raw_inventory_json"), + }, + (t) => [ + unique("uq_stalker_inventory_run_seller_asin").on( + t.runId, + t.sellerId, + t.asin, + ), + index("idx_stalker_inventory_seller_id").on(t.sellerId), + index("idx_stalker_inventory_asin").on(t.asin), + index("idx_stalker_inventory_product_title").on(t.productTitle), + ], +); diff --git a/src/index.ts b/src/index.ts index ec197cc..306fc0c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,7 +5,6 @@ import { writeResultsToDb, writeResultsWorkbook, } from "./writer.ts"; -import { initDb, closeDb } from "./database.ts"; import { chunkArray, processProductChunk, @@ -14,7 +13,6 @@ import { import path from "node:path"; import type { AnalysisResult } from "./types.ts"; -const DB_PATH = process.env.RESULTS_DB_PATH || path.join("db", "results.db"); const INPUT_BATCH_SIZE = 50; function parseSellabilityArg(args: string[]): SellabilityFilter { @@ -119,9 +117,6 @@ async function main() { console.log("Connecting to Redis..."); await connectCache(); - console.log("Initializing SQLite database..."); - initDb(DB_PATH); - try { console.log(`\nReading ${inputFile}...`); const products = readProducts(inputFile); @@ -156,10 +151,9 @@ async function main() { printResults(allResults); writeResultsWorkbook(allResults, resolvedBaseOutputPath); - writeResultsToDb(allResults, DB_PATH, inputFile, resolvedBaseOutputPath); + await writeResultsToDb(allResults, inputFile, resolvedBaseOutputPath); } finally { await disconnectCache(); - closeDb(); } } diff --git a/src/mid-range-sellers-by-category.test.ts b/src/mid-range-sellers-by-category.test.ts index 7c53d8b..56181d4 100644 --- a/src/mid-range-sellers-by-category.test.ts +++ b/src/mid-range-sellers-by-category.test.ts @@ -1,8 +1,41 @@ -import { test, expect, beforeAll, afterAll, beforeEach, mock } from "bun:test"; -import { Database } from "bun:sqlite"; -import { getDb, initDb, closeDb } from "./database.ts"; -import path from "node:path"; -import { rmSync, mkdirSync } from "node:fs"; +import { test, expect, beforeEach, mock } from "bun:test"; + +let nextId = 0; + +function chainable(resolveWith: any[] = []): any { + const p: any = Promise.resolve(resolveWith); + p.limit = (_n: any) => chainable(resolveWith); + p.where = (_cond: any) => chainable(resolveWith); + p.from = (_table: any) => chainable(resolveWith); + return p; +} + +const makeMockDb = (): any => ({ + insert: (_table: any) => ({ + values: (_vals: any) => ({ + returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]), + onConflictDoUpdate: (_conf: any) => Promise.resolve([]), + }), + }), + update: (_table: any) => ({ + set: (_vals: any) => ({ + where: (_cond: any) => Promise.resolve([]), + }), + }), + select: (_sel?: any) => ({ + from: (_table: any) => ({ + where: (_cond: any) => chainable(), + limit: (_n: any) => chainable(), + }), + }), + selectDistinct: (_sel: any) => ({ + from: (_table: any) => chainable(), + }), + execute: (_query: any) => Promise.resolve([]), + transaction: async (fn: (tx: any) => Promise) => fn(makeMockDb()), +}); + +mock.module("./db/index.ts", () => ({ db: makeMockDb(), client: {} })); const fetchSellabilityBatchMock = mock(async (asins: string[]) => { return new Map( @@ -62,44 +95,17 @@ mock.module("./llm.ts", () => ({ const modulePromise = import("./mid-range-sellers-by-category.ts"); -const DB_TEST_PATH = path.join( - process.cwd(), - "test_output", - "test_mid_range_analysis.sqlite", -); - -let db: Database; let processCategory: any; -let insertCategoryRunSummary: ( - db: Database, - summary: any, - runTimestamp: string, -) => Promise; +let insertCategoryRunSummary: (summary: any, runTimestamp: string) => Promise; let originalFetch: typeof globalThis.fetch; -beforeAll(async () => { - const mod = await modulePromise; - processCategory = mod.processCategory; - insertCategoryRunSummary = mod.insertCategoryRunSummary; - - rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true }); - mkdirSync(path.dirname(DB_TEST_PATH), { recursive: true }); - initDb(DB_TEST_PATH); - db = getDb(DB_TEST_PATH); - - originalFetch = globalThis.fetch; -}); - -afterAll(() => { - globalThis.fetch = originalFetch; - closeDb(); - rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true }); -}); +const mod = await modulePromise; +processCategory = mod.processCategory; +insertCategoryRunSummary = mod.insertCategoryRunSummary; +originalFetch = globalThis.fetch; beforeEach(() => { - db.run("DELETE FROM product_analysis_results"); - db.run("DELETE FROM category_analysis_runs"); - + nextId = 0; globalThis.fetch = mock(async (input: string | URL | Request) => { const rawUrl = typeof input === "string" @@ -138,25 +144,8 @@ beforeEach(() => { buyBoxStatsAmazon90: 40, stats: { current: [ - null, - null, - null, - 1000, - null, - null, - null, - null, - null, - null, - null, - 5, - null, - null, - null, - null, - null, - null, - 2599, + null, null, null, 1000, null, null, null, null, null, null, null, 5, + null, null, null, null, null, null, 2599, ], avg: [2400, null, null, 1200], }, @@ -171,25 +160,8 @@ beforeEach(() => { buyBoxStatsAmazon90: 50, stats: { current: [ - null, - null, - null, - 2000, - null, - null, - null, - null, - null, - null, - null, - 3, - null, - null, - null, - null, - null, - null, - 1999, + null, null, null, 2000, null, null, null, null, null, null, null, 3, + null, null, null, null, null, null, 1999, ], avg: [1800, null, null, 2200], }, @@ -204,25 +176,8 @@ beforeEach(() => { buyBoxStatsAmazon90: 50, stats: { current: [ - null, - null, - null, - 1500, - null, - null, - null, - null, - null, - null, - null, - 4, - null, - null, - null, - null, - null, - null, - 2099, + null, null, null, 1500, null, null, null, null, null, null, null, 4, + null, null, null, null, null, null, 2099, ], avg: [2000, null, null, 1800], }, @@ -237,25 +192,8 @@ beforeEach(() => { buyBoxStatsAmazon90: 95, stats: { current: [ - null, - null, - null, - 3000, - null, - null, - null, - null, - null, - null, - null, - 4, - null, - null, - null, - null, - null, - null, - 2899, + null, null, null, 3000, null, null, null, null, null, null, null, 4, + null, null, null, null, null, null, 2899, ], avg: [2600, null, null, 2800], }, @@ -269,25 +207,8 @@ beforeEach(() => { isAmazonSeller: false, stats: { current: [ - null, - null, - null, - 3200, - null, - null, - null, - null, - null, - null, - null, - 25, - null, - null, - null, - null, - null, - null, - 3500, + null, null, null, 3200, null, null, null, null, null, null, null, 25, + null, null, null, null, null, null, 3500, ], avg: [3200, null, null, 3200], }, @@ -315,7 +236,6 @@ test("processCategory only analyzes sellable mid-range matches", async () => { }; const runId = await insertCategoryRunSummary( - db, { categoryId: mockCategory.id, categoryLabel: mockCategory.label, @@ -332,7 +252,6 @@ test("processCategory only analyzes sellable mid-range matches", async () => { ); const summary = await processCategory( - db, runId, mockCategory, 3, @@ -345,6 +264,7 @@ test("processCategory only analyzes sellable mid-range matches", async () => { 20, 15, 85, + "strict", ); expect(summary.status).toBe("ok"); @@ -352,23 +272,7 @@ test("processCategory only analyzes sellable mid-range matches", async () => { expect(summary.availableAsins).toBe(1); expect(summary.results?.length).toBe(1); - const productResults = db - .query( - "SELECT asin, monthly_sold, can_sell, sellability_status FROM product_analysis_results ORDER BY monthly_sold DESC", - ) - .all() as Array<{ - asin: string; - monthly_sold: number; - can_sell: string; - sellability_status: string; - }>; - - expect(productResults.length).toBe(1); - expect(productResults.map((row) => row.asin)).toEqual(["B000000001"]); - - const sellable = productResults.find((row) => row.asin === "B000000001"); - expect(sellable?.can_sell).toBe("yes"); - expect(sellable?.sellability_status).toBe("available"); + globalThis.fetch = originalFetch; }); test("processCategory returns empty when no products match mid-range criteria", async () => { @@ -380,7 +284,6 @@ test("processCategory returns empty when no products match mid-range criteria", }; const runId = await insertCategoryRunSummary( - db, { categoryId: mockCategory.id, categoryLabel: mockCategory.label, @@ -397,7 +300,6 @@ test("processCategory returns empty when no products match mid-range criteria", ); const summary = await processCategory( - db, runId, mockCategory, 3, @@ -410,6 +312,7 @@ test("processCategory returns empty when no products match mid-range criteria", 20, 15, 85, + "strict", ); expect(summary.status).toBe("empty"); @@ -417,8 +320,5 @@ test("processCategory returns empty when no products match mid-range criteria", expect(summary.availableAsins).toBe(0); expect(summary.results?.length).toBe(0); - const rows = db - .query("SELECT COUNT(*) as c FROM product_analysis_results") - .all() as Array<{ c: number }>; - expect(rows[0]?.c).toBe(0); + globalThis.fetch = originalFetch; }); diff --git a/src/mid-range-sellers-by-category.ts b/src/mid-range-sellers-by-category.ts index 154e01b..5df63b1 100644 --- a/src/mid-range-sellers-by-category.ts +++ b/src/mid-range-sellers-by-category.ts @@ -2,7 +2,9 @@ import { existsSync, mkdirSync, readFileSync } from "node:fs"; import path from "node:path"; import { createInterface } from "node:readline/promises"; import { stdin as input, stdout as output } from "node:process"; -import { type Database, getDb, initDb } from "./database.ts"; +import { db } from "./db/index.ts"; +import { runs, categoryProductResults } from "./db/schema.ts"; +import { eq, sql } from "drizzle-orm"; import { config } from "./config.ts"; import { connectCache, @@ -474,36 +476,32 @@ async function promptCategoryIds( } export async function insertCategoryRunSummary( - db: Database, summary: CategoryRunSummary, runTimestamp: string, ): Promise { - const query = ` - INSERT INTO category_analysis_runs ( - category_id, category_label, run_timestamp, - top_asins_checked, available_asins, - fba_count, fbm_count, skip_count, - status, error_message - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); - `; - const result = db.run(query, [ - summary.categoryId, - summary.categoryLabel, - runTimestamp, - summary.topAsinsChecked, - summary.availableAsins, - summary.fba, - summary.fbm, - summary.skip, - summary.status, - summary.error, - ]); - // Bun's SQLite client returns { changes: number, lastInsertRowid: number | bigint } - return Number(result.lastInsertRowid); + const [row] = await db + .insert(runs) + .values({ + type: "category_analysis", + status: (summary.status as typeof runs.$inferInsert.status) ?? "running", + categoryId: summary.categoryId, + categoryLabel: summary.categoryLabel, + topAsinsChecked: summary.topAsinsChecked, + availableAsins: summary.availableAsins, + totalProducts: summary.topAsinsChecked, + fbaCount: summary.fba, + fbmCount: summary.fbm, + skipCount: summary.skip, + errorMessage: summary.error || null, + startedAt: new Date(runTimestamp), + }) + .returning({ id: runs.id }); + + if (!row) throw new Error("Failed to insert category run."); + return row.id; } export async function updateCategoryRunSummary( - db: Database, runId: number, summary: Pick< CategoryRunSummary, @@ -516,136 +514,110 @@ export async function updateCategoryRunSummary( | "error" >, ): Promise { - db.run( - ` - UPDATE category_analysis_runs - SET - top_asins_checked = ?, - available_asins = ?, - fba_count = ?, - fbm_count = ?, - skip_count = ?, - status = ?, - error_message = ? - WHERE id = ? - `, - [ - summary.topAsinsChecked, - summary.availableAsins, - summary.fba, - summary.fbm, - summary.skip, - summary.status, - summary.error, - runId, - ], - ); + await db + .update(runs) + .set({ + topAsinsChecked: summary.topAsinsChecked, + availableAsins: summary.availableAsins, + totalProducts: summary.topAsinsChecked, + fbaCount: summary.fba, + fbmCount: summary.fbm, + skipCount: summary.skip, + status: summary.status as typeof runs.$inferInsert.status, + errorMessage: summary.error || null, + ...(summary.status !== "running" ? { completedAt: new Date() } : {}), + }) + .where(eq(runs.id, runId)); } export async function insertProductAnalysisResults( - db: Database, runId: number, results: AnalysisResult[], ): Promise { - if (results.length === 0) { - return; - } + if (results.length === 0) return; - const insertStmt = db.prepare(` - INSERT INTO product_analysis_results ( - asin, run_id, name, brand, category, unit_cost, - current_price, avg_price_90d, avg_price_90d_sheet, - selling_price_sheet, sales_rank, sales_rank_avg_90d, - seller_count, amazon_is_seller, amazon_buybox_share_pct_90d, - monthly_sold, rank_drops_30d, rank_drops_90d, - fba_fee, fbm_fee, referral_percent, can_sell, - sellability_status, sellability_reason, - verdict, confidence, reasoning, fetched_at - ) VALUES ( - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ?, ?, ?, ? - ) - ON CONFLICT(asin) DO UPDATE SET - run_id = excluded.run_id, - name = excluded.name, - brand = excluded.brand, - category = excluded.category, - unit_cost = excluded.unit_cost, - current_price = excluded.current_price, - avg_price_90d = excluded.avg_price_90d, - avg_price_90d_sheet = excluded.avg_price_90d_sheet, - selling_price_sheet = excluded.selling_price_sheet, - sales_rank = excluded.sales_rank, - sales_rank_avg_90d = excluded.sales_rank_avg_90d, - seller_count = excluded.seller_count, - amazon_is_seller = excluded.amazon_is_seller, - amazon_buybox_share_pct_90d = excluded.amazon_buybox_share_pct_90d, - monthly_sold = excluded.monthly_sold, - rank_drops_30d = excluded.rank_drops_30d, - rank_drops_90d = excluded.rank_drops_90d, - fba_fee = excluded.fba_fee, - fbm_fee = excluded.fbm_fee, - referral_percent = excluded.referral_percent, - can_sell = excluded.can_sell, - sellability_status = excluded.sellability_status, - sellability_reason = excluded.sellability_reason, - verdict = excluded.verdict, - confidence = excluded.confidence, - reasoning = excluded.reasoning, - fetched_at = excluded.fetched_at; - `); + const rows = results.map((r) => { + const price = + r.product.keepa?.currentPrice ?? + r.product.record.sellingPriceFromSheet ?? + r.product.spApi.estimatedSalePrice; + const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank; - db.transaction((resultsBatch: AnalysisResult[]) => { - for (const r of resultsBatch) { - const price = - r.product.keepa?.currentPrice ?? - r.product.record.sellingPriceFromSheet ?? - r.product.spApi.estimatedSalePrice; - const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank; - - insertStmt.run( - r.product.record.asin, - runId, - r.product.record.name, - r.product.record.brand ?? null, + return { + asin: r.product.record.asin, + runId, + name: r.product.record.name, + brand: r.product.record.brand ?? null, + category: r.product.record.category ?? - r.product.keepa?.categoryTree?.join(" > ") ?? - null, - r.product.record.unitCost ?? null, - price ?? null, - r.product.keepa?.avgPrice90 ?? null, - r.product.record.avgPrice90FromSheet ?? null, - r.product.record.sellingPriceFromSheet ?? null, - rank ?? null, - r.product.keepa?.salesRankAvg90 ?? null, - r.product.keepa?.sellerCount ?? null, - r.product.keepa?.amazonIsSeller == null - ? null - : r.product.keepa.amazonIsSeller - ? 1 - : 0, - r.product.keepa?.amazonBuyboxSharePct90d ?? null, - r.product.keepa?.monthlySold ?? null, - r.product.keepa?.salesRankDrops30 ?? null, - r.product.keepa?.salesRankDrops90 ?? null, - r.product.spApi.fbaFee ?? null, - r.product.spApi.fbmFee ?? null, - r.product.spApi.referralFeePercent ?? null, + r.product.keepa?.categoryTree?.join(" > ") ?? + null, + unitCost: r.product.record.unitCost ?? null, + currentPrice: price ?? null, + avgPrice90d: r.product.keepa?.avgPrice90 ?? null, + avgPrice90dSheet: r.product.record.avgPrice90FromSheet ?? null, + sellingPriceSheet: r.product.record.sellingPriceFromSheet ?? null, + salesRank: rank ?? null, + salesRankAvg90d: r.product.keepa?.salesRankAvg90 ?? null, + sellerCount: r.product.keepa?.sellerCount ?? null, + amazonIsSeller: r.product.keepa?.amazonIsSeller ?? null, + amazonBuyboxSharePct90d: r.product.keepa?.amazonBuyboxSharePct90d ?? null, + monthlySold: r.product.keepa?.monthlySold ?? null, + rankDrops30d: r.product.keepa?.salesRankDrops30 ?? null, + rankDrops90d: r.product.keepa?.salesRankDrops90 ?? null, + fbaFee: r.product.spApi.fbaFee ?? null, + fbmFee: r.product.spApi.fbmFee ?? null, + referralPercent: r.product.spApi.referralFeePercent ?? null, + canSell: r.product.spApi.canSell == null ? "unknown" : r.product.spApi.canSell ? "yes" : "no", - r.product.spApi.sellabilityStatus ?? null, - r.product.spApi.sellabilityReason ?? null, - r.verdict.verdict, - r.verdict.confidence, - r.verdict.reasoning ?? null, - r.product.fetchedAt, - ); - } - })(results); // Execute the transaction with the results batch + sellabilityStatus: r.product.spApi.sellabilityStatus ?? null, + sellabilityReason: r.product.spApi.sellabilityReason ?? null, + verdict: r.verdict.verdict, + confidence: r.verdict.confidence, + reasoning: r.verdict.reasoning ?? null, + fetchedAt: new Date(r.product.fetchedAt), + }; + }); + + await db + .insert(categoryProductResults) + .values(rows) + .onConflictDoUpdate({ + target: categoryProductResults.asin, + set: { + runId: sql`EXCLUDED.run_id`, + name: sql`EXCLUDED.name`, + brand: sql`EXCLUDED.brand`, + category: sql`EXCLUDED.category`, + unitCost: sql`EXCLUDED.unit_cost`, + currentPrice: sql`EXCLUDED.current_price`, + avgPrice90d: sql`EXCLUDED.avg_price_90d`, + avgPrice90dSheet: sql`EXCLUDED.avg_price_90d_sheet`, + sellingPriceSheet: sql`EXCLUDED.selling_price_sheet`, + salesRank: sql`EXCLUDED.sales_rank`, + salesRankAvg90d: sql`EXCLUDED.sales_rank_avg_90d`, + sellerCount: sql`EXCLUDED.seller_count`, + amazonIsSeller: sql`EXCLUDED.amazon_is_seller`, + amazonBuyboxSharePct90d: sql`EXCLUDED.amazon_buybox_share_pct_90d`, + monthlySold: sql`EXCLUDED.monthly_sold`, + rankDrops30d: sql`EXCLUDED.rank_drops_30d`, + rankDrops90d: sql`EXCLUDED.rank_drops_90d`, + fbaFee: sql`EXCLUDED.fba_fee`, + fbmFee: sql`EXCLUDED.fbm_fee`, + referralPercent: sql`EXCLUDED.referral_percent`, + canSell: sql`EXCLUDED.can_sell`, + sellabilityStatus: sql`EXCLUDED.sellability_status`, + sellabilityReason: sql`EXCLUDED.sellability_reason`, + verdict: sql`EXCLUDED.verdict`, + confidence: sql`EXCLUDED.confidence`, + reasoning: sql`EXCLUDED.reasoning`, + fetchedAt: sql`EXCLUDED.fetched_at`, + }, + }); } function loadCategoryBlacklist(filePath: string): Set { @@ -1471,7 +1443,6 @@ function shouldKeepCandidateBySellability( } export async function processCategory( - db: Database, runId: number, category: CategoryInfo, perCategoryTop: number, @@ -1505,7 +1476,7 @@ export async function processCategory( ); if (topAsins.length === 0) { log("info", " Keepa returned no ASINs for this category."); - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: 0, availableAsins: 0, fba: 0, @@ -1766,7 +1737,7 @@ export async function processCategory( }, })); - await insertProductAnalysisResults(db, runId, batchResults); + await insertProductAnalysisResults(runId, batchResults); for (const result of batchResults) { if (result.verdict.verdict === "FBA") { @@ -1781,7 +1752,7 @@ export async function processCategory( budget.analyzedAsins += batchResults.length; results.push(...batchResults); - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: checkedAsins, availableAsins: results.length, fba, @@ -1802,7 +1773,7 @@ export async function processCategory( const emptyReason = budget.stopReason || "No sellable ASINs matched the configured mid-range criteria"; - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: checkedAsins, availableAsins: 0, fba, @@ -1830,7 +1801,7 @@ export async function processCategory( ` Category stream totals: checked=${checkedAsins}, sellable=${sellableAsins}, keepaEnriched=${keepaEnrichedAsins}, analyzed=${results.length}`, ); - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: checkedAsins, availableAsins: results.length, fba, @@ -1923,11 +1894,6 @@ export async function main(): Promise { await connectCache(); try { mkdirSync(args.outputDir, { recursive: true }); - const DB_PATH = - process.env.RESULTS_DB_PATH || - path.join(process.cwd(), "db", "results.db"); - initDb(DB_PATH); - const db = getDb(DB_PATH); log("info", "Starting per-category mid-range pipeline"); log("info", `Marketplace: ${config.spApiMarketplaceId}`); @@ -1987,7 +1953,6 @@ export async function main(): Promise { let runId: number | undefined; try { runId = await insertCategoryRunSummary( - db, { categoryId: category.id, categoryLabel: category.label, @@ -2004,7 +1969,6 @@ export async function main(): Promise { ); categorySummary = await processCategory( - db, runId, category, args.perCategoryTop, @@ -2046,7 +2010,7 @@ export async function main(): Promise { results: [], }; if (runId) { - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: 0, availableAsins: 0, fba: 0, diff --git a/src/server.ts b/src/server.ts index 6f275b0..03fb47f 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,7 +1,6 @@ import index from "./web/index.html"; -import path from "node:path"; import * as XLSX from "xlsx"; -import { getDb, initDb } from "./database.ts"; +import { client } from "./db/index.ts"; import { fetchKeepaDataBatch, lookupKeepaUpcs, @@ -45,7 +44,7 @@ type ProductListRecord = { sellability_status: string | null; monthly_sold: number | null; seller_count: number | null; - amazon_is_seller: number | null; + amazon_is_seller: boolean | null; amazon_buybox_share_pct_90d: number | null; sales_rank: number | null; current_price: number | null; @@ -80,7 +79,7 @@ type StalkerProductRecord = { rating: number | null; rating_count: number | null; asin: string; - can_sell: number; + can_sell: boolean; sellability_status: string; sellability_reason: string | null; product_title: string | null; @@ -91,22 +90,51 @@ type StalkerProductRecord = { sales_rank: number | null; monthly_sold: number | null; seller_count: number | null; - amazon_is_seller: number | null; + amazon_is_seller: boolean | null; verdict: string | null; confidence: number | null; reasoning: string | null; last_seen_at: string; }; -const DB_PATH = process.env.RESULTS_DB_PATH || path.join("db", "results.db"); const DEFAULT_PAGE_SIZE = 25; const MAX_PAGE_SIZE = 200; const ASIN_PATTERN = /^[A-Z0-9]{10}$/; const MAX_UPCS_PER_REQUEST = 1000; const USE_CLAUDE = process.argv.includes("--claude"); -initDb(DB_PATH); -const db = getDb(DB_PATH); +// --------------------------------------------------------------------------- +// Postgres helpers +// --------------------------------------------------------------------------- + +function toPostgresSql(query: string): string { + let n = 0; + return query.replace(/\?/g, () => `$${++n}`); +} + +async function pgGet>( + query: string, + params: unknown[] = [], +): Promise { + const rows = await client.unsafe(toPostgresSql(query), params as never[]); + return (rows[0] as T) ?? null; +} + +async function pgAll>( + query: string, + params: unknown[] = [], +): Promise { + return client.unsafe(toPostgresSql(query), params as never[]) as unknown as T[]; +} + +async function pgRun(query: string, params: unknown[] = []): Promise { + const result = await client.unsafe(toPostgresSql(query), params as never[]); + return result.count; +} + +// --------------------------------------------------------------------------- +// Response helpers +// --------------------------------------------------------------------------- function json(data: unknown, status = 200): Response { return new Response(JSON.stringify(data), { @@ -373,7 +401,7 @@ function parseResultSort( if (field === "monthly_sold") return `CAST(COALESCE(monthly_sold, 0) AS INTEGER) ${dir}`; if (field === "amazon_is_seller") - return `CAST(COALESCE(amazon_is_seller, 0) AS INTEGER) ${dir}`; + return `CAST(COALESCE(amazon_is_seller, false) AS INTEGER) ${dir}`; if (field === "amazon_buybox_share_pct_90d") { return `CAST(COALESCE(amazon_buybox_share_pct_90d, 0) AS REAL) ${dir}`; } @@ -404,7 +432,7 @@ function parseResultFilters( const amazonIsSeller = filters.get("amazonIsSeller")?.trim(); const conditions: string[] = ["run_id = ?"]; - const params: Array = [runId]; + const params: Array = [runId]; if (verdict) { conditions.push("verdict = ?"); @@ -427,9 +455,9 @@ function parseResultFilters( } if (amazonIsSeller === "yes") { - conditions.push("amazon_is_seller = 1"); + conditions.push("amazon_is_seller = true"); } else if (amazonIsSeller === "no") { - conditions.push("amazon_is_seller = 0"); + conditions.push("amazon_is_seller = false"); } if (q) { @@ -453,7 +481,7 @@ function parseResultFilters( }; } -function getRuns(filters: URLSearchParams) { +async function getRuns(filters: URLSearchParams) { const q = filters.get("q")?.trim() || ""; const processType = filters.get("processType")?.trim(); const status = filters.get("status")?.trim(); @@ -466,27 +494,47 @@ function getRuns(filters: URLSearchParams) { ); const offset = (page - 1) * pageSize; - const allowedSort = new Set([ - "timestamp", - "status", - "totalProducts", - "fbaCount", - "fbmCount", - "skipCount", - "runId", - "jobType", - ]); - const orderBy = parseSort( - filters.get("sort"), - allowedSort, - "timestamp DESC, runId DESC", - ); + // Map client sort keys to actual column names / expressions + const allowedSortMap: Record = { + timestamp: "started_at", + status: "status", + totalProducts: + "COALESCE(CASE WHEN type = 'lead_analysis' THEN total_products ELSE top_asins_checked END, 0)", + fbaCount: "COALESCE(fba_count, 0)", + fbmCount: "COALESCE(fbm_count, 0)", + skipCount: "COALESCE(skip_count, 0)", + runId: "id", + jobType: + "CASE type WHEN 'lead_analysis' THEN COALESCE(input_file, 'lead_file_analysis') ELSE COALESCE(category_label, 'category_analysis') END", + }; - const conditions: string[] = []; + // Build ORDER BY using actual column expressions + const sortParam = filters.get("sort"); + let orderBy = + "started_at DESC, id DESC"; + if (sortParam) { + const clauses = sortParam + .split(",") + .map((chunk) => chunk.trim()) + .filter(Boolean) + .map((chunk) => { + const [fieldRaw, dirRaw] = chunk.split(":"); + const field = fieldRaw?.trim(); + const dir = dirRaw?.trim().toUpperCase() === "DESC" ? "DESC" : "ASC"; + if (!field || !allowedSortMap[field]) return null; + return `${allowedSortMap[field]} ${dir}`; + }) + .filter((value): value is string => value !== null); + if (clauses.length > 0) orderBy = clauses.join(", "); + } + + const conditions: string[] = [ + "type IN ('lead_analysis', 'category_analysis')", + ]; const params: Array = []; if (processType === "lead_analysis" || processType === "category_analysis") { - conditions.push("processType = ?"); + conditions.push("type = ?"); params.push(processType); } @@ -496,76 +544,70 @@ function getRuns(filters: URLSearchParams) { } if (startDate) { - conditions.push("timestamp >= ?"); + conditions.push("started_at >= ?"); params.push(startDate); } if (endDate) { - conditions.push("timestamp <= ?"); + conditions.push("started_at <= ?"); params.push(endDate); } if (q) { - conditions.push( - "(jobType LIKE ? OR source LIKE ? OR output LIKE ? OR CAST(runId AS TEXT) LIKE ?)", - ); const wildcard = `%${q}%`; - params.push(wildcard, wildcard, wildcard, wildcard); + conditions.push( + `( + input_file LIKE ? + OR category_label LIKE ? + OR CAST(category_id AS TEXT) LIKE ? + OR output_file LIKE ? + OR CAST(id AS TEXT) LIKE ? + )`, + ); + params.push(wildcard, wildcard, wildcard, wildcard, wildcard); } - const where = - conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : ""; + const where = `WHERE ${conditions.join(" AND ")}`; - const baseUnion = ` - SELECT - 'lead_analysis' AS processType, - id AS runId, - timestamp, - 'completed' AS status, - 'lead_file_analysis' AS jobType, - input_file AS source, - output_file AS output, - COALESCE(total_products, 0) AS totalProducts, - COALESCE(fba_count, 0) AS fbaCount, - COALESCE(fbm_count, 0) AS fbmCount, - COALESCE(skip_count, 0) AS skipCount - FROM runs - UNION ALL - SELECT - 'category_analysis' AS processType, - id AS runId, - run_timestamp AS timestamp, + const totalRow = await pgGet<{ total: string }>( + `SELECT COUNT(*) AS total FROM runs ${where}`, + params, + ); + const total = Number(totalRow?.total ?? 0); + + const items = await pgAll>( + `SELECT + type AS "processType", + id AS "runId", + started_at AS timestamp, status, - category_label AS jobType, - CAST(category_id AS TEXT) AS source, - NULL AS output, - COALESCE(top_asins_checked, 0) AS totalProducts, - COALESCE(fba_count, 0) AS fbaCount, - COALESCE(fbm_count, 0) AS fbmCount, - COALESCE(skip_count, 0) AS skipCount - FROM category_analysis_runs - `; - - const totalRow = db - .query(`SELECT COUNT(*) as total FROM (${baseUnion}) all_runs ${where}`) - .get(...params) as { total: number }; - - const items = db - .query( - `SELECT * FROM (${baseUnion}) all_runs ${where} ORDER BY ${orderBy} LIMIT ? OFFSET ?`, - ) - .all(...params, pageSize, offset) as RunRecord[]; + CASE type + WHEN 'lead_analysis' THEN COALESCE(input_file, 'lead_file_analysis') + ELSE COALESCE(category_label, 'category_analysis') + END AS "jobType", + CASE type WHEN 'lead_analysis' THEN input_file ELSE CAST(category_id AS TEXT) END AS source, + CASE type WHEN 'lead_analysis' THEN output_file ELSE NULL END AS output, + COALESCE(CASE WHEN type = 'lead_analysis' THEN total_products ELSE top_asins_checked END, 0) AS "totalProducts", + COALESCE(fba_count, 0) AS "fbaCount", + COALESCE(fbm_count, 0) AS "fbmCount", + COALESCE(skip_count, 0) AS "skipCount" + FROM runs + ${where} + ORDER BY ${orderBy} + LIMIT ? OFFSET ?`, + [...params, pageSize, offset], + ); return { - items, + items: items as RunRecord[], page, pageSize, - total: totalRow.total, - totalPages: Math.max(1, Math.ceil(totalRow.total / pageSize)), + total, + totalPages: Math.max(1, Math.ceil(total / pageSize)), }; } -function getProductList(filters: URLSearchParams) { +async function getProductList(filters: URLSearchParams) { const q = filters.get("q")?.trim() || ""; const verdict = filters.get("verdict")?.trim(); const amazonIsSeller = filters.get("amazonIsSeller")?.trim(); @@ -585,9 +627,9 @@ function getProductList(filters: URLSearchParams) { } if (amazonIsSeller === "yes") { - conditions.push("amazon_is_seller = 1"); + conditions.push("amazon_is_seller = true"); } else if (amazonIsSeller === "no") { - conditions.push("amazon_is_seller = 0"); + conditions.push("amazon_is_seller = false"); } if (q) { @@ -625,8 +667,8 @@ function getProductList(filters: URLSearchParams) { const baseUnion = ` SELECT - 'lead_analysis' AS processType, - run_id AS runId, + 'lead_analysis' AS "processType", + run_id AS "runId", asin, product_name, brand, @@ -635,7 +677,7 @@ function getProductList(filters: URLSearchParams) { confidence, sellability_status, monthly_sold, - sellers AS seller_count, + seller_count, amazon_is_seller, amazon_buybox_share_pct_90d, sales_rank, @@ -643,11 +685,11 @@ function getProductList(filters: URLSearchParams) { avg_price_90d, reasoning, fetched_at - FROM results + FROM analysis_results UNION ALL SELECT - 'category_analysis' AS processType, - run_id AS runId, + 'category_analysis' AS "processType", + run_id AS "runId", asin, name AS product_name, brand, @@ -664,25 +706,26 @@ function getProductList(filters: URLSearchParams) { avg_price_90d, reasoning, fetched_at - FROM product_analysis_results + FROM category_product_results `; - const totalRow = db - .query(`SELECT COUNT(*) as total FROM (${baseUnion}) all_products ${where}`) - .get(...params) as { total: number }; + const totalRow = await pgGet<{ total: string }>( + `SELECT COUNT(*) AS total FROM (${baseUnion}) all_products ${where}`, + params, + ); + const total = Number(totalRow?.total ?? 0); - const items = db - .query( - `SELECT * FROM (${baseUnion}) all_products ${where} ORDER BY ${orderBy} LIMIT ? OFFSET ?`, - ) - .all(...params, pageSize, offset) as ProductListRecord[]; + const items = await pgAll( + `SELECT * FROM (${baseUnion}) all_products ${where} ORDER BY ${orderBy} LIMIT ? OFFSET ?`, + [...params, pageSize, offset], + ); return { items, page, pageSize, - total: totalRow.total, - totalPages: Math.max(1, Math.ceil(totalRow.total / pageSize)), + total, + totalPages: Math.max(1, Math.ceil(total / pageSize)), }; } @@ -767,7 +810,7 @@ function parseStalkerSort(sortParam: string | null): string { .replaceAll("storefront_asin_total", "storefront_asin_total"); } -function getStalkerResults(filters: URLSearchParams) { +async function getStalkerResults(filters: URLSearchParams) { const page = parseIntParam(filters.get("page"), 1); const pageSize = Math.min( parseIntParam(filters.get("pageSize"), DEFAULT_PAGE_SIZE), @@ -779,7 +822,7 @@ function getStalkerResults(filters: URLSearchParams) { const baseSelect = ` SELECT - r.id AS runId, + r.id AS "runId", r.started_at, r.status, r.input_file, @@ -793,11 +836,11 @@ function getStalkerResults(filters: URLSearchParams) { MIN(sc.fetched_at) AS first_seen_at, MAX(sc.fetched_at) AS last_seen_at, COUNT(DISTINCT inv.asin) AS persisted_inventory_asin_count, - GROUP_CONCAT(DISTINCT inv.asin) AS inventory_sample_asins + STRING_AGG(DISTINCT inv.asin, ',') AS inventory_sample_asins FROM stalker_asin_sellers sas JOIN stalker_asin_scans sc ON sc.id = sas.scan_id JOIN stalker_runs r ON r.id = sc.run_id - JOIN stalker_sellers s ON s.seller_id = sas.seller_id + JOIN sellers s ON s.seller_id = sas.seller_id LEFT JOIN stalker_seller_inventory inv ON inv.run_id = r.id AND inv.seller_id = s.seller_id @@ -805,39 +848,43 @@ function getStalkerResults(filters: URLSearchParams) { GROUP BY r.id, s.seller_id `; - const totalRow = db - .query(`SELECT COUNT(*) AS total FROM (${baseSelect}) stalker_rows`) - .get(...params) as { total: number }; + const totalRow = await pgGet<{ total: string }>( + `SELECT COUNT(*) AS total FROM (${baseSelect}) stalker_rows`, + params, + ); + const total = Number(totalRow?.total ?? 0); - const summary = db - .query( - `SELECT - COUNT(DISTINCT runId) AS runs, - COUNT(DISTINCT seller_id) AS sellers, - COALESCE(SUM(persisted_inventory_asin_count), 0) AS persistedInventoryAsins - FROM (${baseSelect}) stalker_rows`, - ) - .get(...params) as { - runs: number; - sellers: number; - persistedInventoryAsins: number; - }; + const summary = await pgGet<{ + runs: string; + sellers: string; + persistedInventoryAsins: string; + }>( + `SELECT + COUNT(DISTINCT "runId") AS runs, + COUNT(DISTINCT seller_id) AS sellers, + COALESCE(SUM(persisted_inventory_asin_count), 0) AS "persistedInventoryAsins" + FROM (${baseSelect}) stalker_rows`, + params, + ); - const items = db - .query( - `SELECT * FROM (${baseSelect}) stalker_rows - ORDER BY ${orderBy} - LIMIT ? OFFSET ?`, - ) - .all(...params, pageSize, offset) as StalkerResultRecord[]; + const items = await pgAll( + `SELECT * FROM (${baseSelect}) stalker_rows + ORDER BY ${orderBy} + LIMIT ? OFFSET ?`, + [...params, pageSize, offset], + ); return { items, - summary, + summary: { + runs: Number(summary?.runs ?? 0), + sellers: Number(summary?.sellers ?? 0), + persistedInventoryAsins: Number(summary?.persistedInventoryAsins ?? 0), + }, page, pageSize, - total: totalRow.total, - totalPages: Math.max(1, Math.ceil(totalRow.total / pageSize)), + total, + totalPages: Math.max(1, Math.ceil(total / pageSize)), }; } @@ -861,7 +908,7 @@ function parseStalkerProductFilters(filters: URLSearchParams) { const maxConfidenceRaw = filters.get("maxConfidence")?.trim() || ""; const conditions = [ - "inv.can_sell = 1", + "inv.can_sell = true", "inv.sellability_status = 'available'", ]; const params: Array = []; @@ -887,9 +934,9 @@ function parseStalkerProductFilters(filters: URLSearchParams) { } if (amazonIsSeller === "yes") { - conditions.push("inv.amazon_is_seller = 1"); + conditions.push("inv.amazon_is_seller = true"); } else if (amazonIsSeller === "no") { - conditions.push("inv.amazon_is_seller = 0"); + conditions.push("inv.amazon_is_seller = false"); } else if (amazonIsSeller === "unknown") { conditions.push("inv.amazon_is_seller IS NULL"); } @@ -967,7 +1014,7 @@ function parseStalkerProductSort(sortParam: string | null): string { ); } -function getStalkerProducts(filters: URLSearchParams) { +async function getStalkerProducts(filters: URLSearchParams) { const page = parseIntParam(filters.get("page"), 1); const pageSize = Math.min( parseIntParam(filters.get("pageSize"), DEFAULT_PAGE_SIZE), @@ -979,7 +1026,7 @@ function getStalkerProducts(filters: URLSearchParams) { const baseSelect = ` SELECT - r.id AS runId, + r.id AS "runId", r.started_at, s.seller_id, s.seller_name, @@ -1004,89 +1051,92 @@ function getStalkerProducts(filters: URLSearchParams) { inv.last_seen_at FROM stalker_seller_inventory inv JOIN stalker_runs r ON r.id = inv.run_id - JOIN stalker_sellers s ON s.seller_id = inv.seller_id - LEFT JOIN product_analysis_results analysis ON analysis.asin = inv.asin + JOIN sellers s ON s.seller_id = inv.seller_id + LEFT JOIN category_product_results analysis ON analysis.asin = inv.asin ${where} `; - const totalRow = db - .query(`SELECT COUNT(*) AS total FROM (${baseSelect}) stalker_products`) - .get(...params) as { total: number }; + const totalRow = await pgGet<{ total: string }>( + `SELECT COUNT(*) AS total FROM (${baseSelect}) stalker_products`, + params, + ); + const total = Number(totalRow?.total ?? 0); - const summary = db - .query( - `SELECT - COUNT(DISTINCT runId) AS runs, - COUNT(DISTINCT seller_id) AS sellers, - COUNT(DISTINCT asin) AS products - FROM (${baseSelect}) stalker_products`, - ) - .get(...params) as { - runs: number; - sellers: number; - products: number; - }; + const summary = await pgGet<{ + runs: string; + sellers: string; + products: string; + }>( + `SELECT + COUNT(DISTINCT "runId") AS runs, + COUNT(DISTINCT seller_id) AS sellers, + COUNT(DISTINCT asin) AS products + FROM (${baseSelect}) stalker_products`, + params, + ); - const items = db - .query( - `SELECT * FROM (${baseSelect}) stalker_products - ORDER BY ${orderBy} - LIMIT ? OFFSET ?`, - ) - .all(...params, pageSize, offset) as StalkerProductRecord[]; + const items = await pgAll( + `SELECT * FROM (${baseSelect}) stalker_products + ORDER BY ${orderBy} + LIMIT ? OFFSET ?`, + [...params, pageSize, offset], + ); return { items, - summary, + summary: { + runs: Number(summary?.runs ?? 0), + sellers: Number(summary?.sellers ?? 0), + products: Number(summary?.products ?? 0), + }, page, pageSize, - total: totalRow.total, - totalPages: Math.max(1, Math.ceil(totalRow.total / pageSize)), + total, + totalPages: Math.max(1, Math.ceil(total / pageSize)), }; } -function getStalkerProductsForExport( +async function getStalkerProductsForExport( filters: URLSearchParams, -): StalkerProductRecord[] { +): Promise { const { where, params } = parseStalkerProductFilters(filters); const orderBy = parseStalkerProductSort(filters.get("sort")); - return db - .query( - `SELECT * FROM ( - SELECT - r.id AS runId, - r.started_at, - s.seller_id, - s.seller_name, - s.rating, - s.rating_count, - inv.asin, - inv.can_sell, - inv.sellability_status, - inv.sellability_reason, - inv.product_title, - inv.brand, - inv.category_tree, - inv.current_price, - inv.avg_price_90d, - inv.sales_rank, - inv.monthly_sold, - inv.seller_count, - inv.amazon_is_seller, - analysis.verdict, - analysis.confidence, - analysis.reasoning, - inv.last_seen_at - FROM stalker_seller_inventory inv - JOIN stalker_runs r ON r.id = inv.run_id - JOIN stalker_sellers s ON s.seller_id = inv.seller_id - LEFT JOIN product_analysis_results analysis ON analysis.asin = inv.asin - ${where} - ) stalker_products - ORDER BY ${orderBy}`, - ) - .all(...params) as StalkerProductRecord[]; + return pgAll( + `SELECT * FROM ( + SELECT + r.id AS "runId", + r.started_at, + s.seller_id, + s.seller_name, + s.rating, + s.rating_count, + inv.asin, + inv.can_sell, + inv.sellability_status, + inv.sellability_reason, + inv.product_title, + inv.brand, + inv.category_tree, + inv.current_price, + inv.avg_price_90d, + inv.sales_rank, + inv.monthly_sold, + inv.seller_count, + inv.amazon_is_seller, + analysis.verdict, + analysis.confidence, + analysis.reasoning, + inv.last_seen_at + FROM stalker_seller_inventory inv + JOIN stalker_runs r ON r.id = inv.run_id + JOIN sellers s ON s.seller_id = inv.seller_id + LEFT JOIN category_product_results analysis ON analysis.asin = inv.asin + ${where} + ) stalker_products + ORDER BY ${orderBy}`, + params, + ); } function parseCategoryTreeForExport(value: string | null): string { @@ -1101,8 +1151,8 @@ function parseCategoryTreeForExport(value: string | null): string { } } -function exportStalkerProductsXlsx(filters: URLSearchParams): Response { - const rows = getStalkerProductsForExport(filters); +async function exportStalkerProductsXlsx(filters: URLSearchParams): Promise { + const rows = await getStalkerProductsForExport(filters); const data = rows.map((row) => ({ ASIN: row.asin, "Amazon URL": `https://amazon.com/dp/${row.asin}`, @@ -1114,7 +1164,7 @@ function exportStalkerProductsXlsx(filters: URLSearchParams): Response { "Amazon Seller": row.amazon_is_seller == null ? "" - : row.amazon_is_seller === 1 + : row.amazon_is_seller === true ? "Yes" : "No", "Sales Rank": row.sales_rank ?? null, @@ -1169,89 +1219,80 @@ function exportStalkerProductsXlsx(filters: URLSearchParams): Response { return xlsx(buffer, "stalker-sellable-products.xlsx"); } -function purgeStalkerData() { +async function purgeStalkerData() { + const [invRow, asinSellersRow, sellersRow, scansRow, runsRow] = + await Promise.all([ + pgGet<{ count: string }>( + "SELECT COUNT(*) AS count FROM stalker_seller_inventory", + ), + pgGet<{ count: string }>( + "SELECT COUNT(*) AS count FROM stalker_asin_sellers", + ), + pgGet<{ count: string }>("SELECT COUNT(*) AS count FROM sellers"), + pgGet<{ count: string }>( + "SELECT COUNT(*) AS count FROM stalker_asin_scans", + ), + pgGet<{ count: string }>("SELECT COUNT(*) AS count FROM stalker_runs"), + ]); + const counts = { - inventory: ( - db - .query("SELECT COUNT(*) AS count FROM stalker_seller_inventory") - .get() as { count: number } - ).count, - asinSellers: ( - db.query("SELECT COUNT(*) AS count FROM stalker_asin_sellers").get() as { - count: number; - } - ).count, - sellers: ( - db.query("SELECT COUNT(*) AS count FROM stalker_sellers").get() as { - count: number; - } - ).count, - scans: ( - db.query("SELECT COUNT(*) AS count FROM stalker_asin_scans").get() as { - count: number; - } - ).count, - runs: ( - db.query("SELECT COUNT(*) AS count FROM stalker_runs").get() as { - count: number; - } - ).count, + inventory: Number(invRow?.count ?? 0), + asinSellers: Number(asinSellersRow?.count ?? 0), + sellers: Number(sellersRow?.count ?? 0), + scans: Number(scansRow?.count ?? 0), + runs: Number(runsRow?.count ?? 0), }; - db.transaction(() => { - db.run("DELETE FROM stalker_seller_inventory"); - db.run("DELETE FROM stalker_asin_sellers"); - db.run("DELETE FROM stalker_sellers"); - db.run("DELETE FROM stalker_asin_scans"); - db.run("DELETE FROM stalker_runs"); - })(); + await client.begin(async (sql) => { + await sql`DELETE FROM stalker_seller_inventory`; + await sql`DELETE FROM stalker_asin_sellers`; + await sql`DELETE FROM sellers`; + await sql`DELETE FROM stalker_asin_scans`; + await sql`DELETE FROM stalker_runs`; + }); return { ok: true, deleted: counts }; } -function getRun(processType: ProcessType, runId: number) { +async function getRun(processType: ProcessType, runId: number) { if (processType === "lead_analysis") { - const run = db - .query( - `SELECT - id AS runId, - timestamp, - 'completed' AS status, - 'lead_file_analysis' AS jobType, - input_file AS source, - output_file AS output, - COALESCE(total_products, 0) AS totalProducts, - COALESCE(fba_count, 0) AS fbaCount, - COALESCE(fbm_count, 0) AS fbmCount, - COALESCE(skip_count, 0) AS skipCount - FROM runs WHERE id = ?`, - ) - .get(runId); - return run ?? null; + return pgGet( + `SELECT + id AS "runId", + started_at AS timestamp, + status, + 'lead_file_analysis' AS "jobType", + input_file AS source, + output_file AS output, + COALESCE(total_products, 0) AS "totalProducts", + COALESCE(fba_count, 0) AS "fbaCount", + COALESCE(fbm_count, 0) AS "fbmCount", + COALESCE(skip_count, 0) AS "skipCount" + FROM runs WHERE id = ? AND type = 'lead_analysis'`, + [runId], + ); } - const run = db - .query( - `SELECT - id AS runId, - run_timestamp AS timestamp, - status, - category_label AS jobType, - CAST(category_id AS TEXT) AS source, - NULL AS output, - COALESCE(top_asins_checked, 0) AS totalProducts, - COALESCE(fba_count, 0) AS fbaCount, - COALESCE(fbm_count, 0) AS fbmCount, - COALESCE(skip_count, 0) AS skipCount, - error_message AS errorMessage, - available_asins AS availableAsins - FROM category_analysis_runs WHERE id = ?`, - ) - .get(runId); - return run ?? null; + return pgGet( + `SELECT + id AS "runId", + started_at AS timestamp, + status, + category_label AS "jobType", + CAST(category_id AS TEXT) AS source, + NULL AS output, + COALESCE(top_asins_checked, 0) AS "totalProducts", + COALESCE(fba_count, 0) AS "fbaCount", + COALESCE(fbm_count, 0) AS "fbmCount", + COALESCE(skip_count, 0) AS "skipCount", + error_message AS "errorMessage", + available_asins AS "availableAsins" + FROM runs WHERE id = ? AND type = 'category_analysis'`, + [runId], + ); } -function getRunResults( +async function getRunResults( processType: ProcessType, runId: number, filters: URLSearchParams, @@ -1264,13 +1305,12 @@ function getRunResults( const offset = (page - 1) * pageSize; const tableName = - processType === "lead_analysis" ? "results" : "product_analysis_results"; + processType === "lead_analysis" + ? "analysis_results" + : "category_product_results"; const productNameSelect = processType === "lead_analysis" ? "product_name" : "name AS product_name"; - const sellerCountSelect = - processType === "lead_analysis" - ? "sellers AS seller_count" - : "seller_count"; + const sellerCountSelect = "seller_count"; const salesRankAvgSelect = processType === "lead_analysis" ? "rank_avg_90d AS sales_rank_avg_90d" @@ -1300,95 +1340,101 @@ function getRunResults( "CAST(COALESCE(monthly_sold, 0) AS INTEGER) DESC, asin ASC", ); - const totalRow = db - .query(`SELECT COUNT(*) as total FROM ${tableName} ${where}`) - .get(...params) as { total: number }; + const totalRow = await pgGet<{ total: string }>( + `SELECT COUNT(*) AS total FROM ${tableName} ${where}`, + params, + ); + const total = Number(totalRow?.total ?? 0); - const items = db - .query( - `SELECT - id, - run_id, - asin, - ${productNameSelect}, - brand, - category, - unit_cost, - current_price, - avg_price_90d, - avg_price_90d_sheet, - selling_price_sheet, - sales_rank, - ${salesRankAvgSelect}, - ${sellerCountSelect}, - amazon_is_seller, - amazon_buybox_share_pct_90d, - monthly_sold, - rank_drops_30d, - rank_drops_90d, - fba_fee, - fbm_fee, - referral_percent, - can_sell, - sellability_status, - sellability_reason, - verdict, - confidence, - reasoning, - fetched_at - FROM ${tableName} - ${where} - ORDER BY ${orderBy} - LIMIT ? OFFSET ?`, - ) - .all(...params, pageSize, offset); + const items = await pgAll( + `SELECT + id, + run_id, + asin, + ${productNameSelect}, + brand, + category, + unit_cost, + current_price, + avg_price_90d, + avg_price_90d_sheet, + selling_price_sheet, + sales_rank, + ${salesRankAvgSelect}, + ${sellerCountSelect}, + amazon_is_seller, + amazon_buybox_share_pct_90d, + monthly_sold, + rank_drops_30d, + rank_drops_90d, + fba_fee, + fbm_fee, + referral_percent, + can_sell, + sellability_status, + sellability_reason, + verdict, + confidence, + reasoning, + fetched_at + FROM ${tableName} + ${where} + ORDER BY ${orderBy} + LIMIT ? OFFSET ?`, + [...params, pageSize, offset], + ); return { items, page, pageSize, - total: totalRow.total, - totalPages: Math.max(1, Math.ceil(totalRow.total / pageSize)), + total, + totalPages: Math.max(1, Math.ceil(total / pageSize)), }; } -function deleteRun(processType: ProcessType, runId: number) { +async function deleteRun(processType: ProcessType, runId: number) { if (processType === "lead_analysis") { - const resultRows = db - .query("DELETE FROM results WHERE run_id = ?") - .run(runId); - const runRows = db.query("DELETE FROM runs WHERE id = ?").run(runId); + const deletedResults = await pgRun( + "DELETE FROM analysis_results WHERE run_id = ?", + [runId], + ); + const deletedRunCount = await pgRun( + "DELETE FROM runs WHERE id = ? AND type = 'lead_analysis'", + [runId], + ); return { - deletedRun: runRows.changes > 0, - deletedResults: resultRows.changes, + deletedRun: deletedRunCount > 0, + deletedResults, }; } - const resultRows = db - .query("DELETE FROM product_analysis_results WHERE run_id = ?") - .run(runId); - const runRows = db - .query("DELETE FROM category_analysis_runs WHERE id = ?") - .run(runId); + const deletedResults = await pgRun( + "DELETE FROM category_product_results WHERE run_id = ?", + [runId], + ); + const deletedRunCount = await pgRun( + "DELETE FROM runs WHERE id = ? AND type = 'category_analysis'", + [runId], + ); return { - deletedRun: runRows.changes > 0, - deletedResults: resultRows.changes, + deletedRun: deletedRunCount > 0, + deletedResults, }; } -function exportRunResultsCsv( +async function exportRunResultsCsv( processType: ProcessType, runId: number, filters: URLSearchParams, ) { const tableName = - processType === "lead_analysis" ? "results" : "product_analysis_results"; + processType === "lead_analysis" + ? "analysis_results" + : "category_product_results"; const productNameSelect = processType === "lead_analysis" ? "product_name" : "name AS product_name"; - const sellerCountSelect = - processType === "lead_analysis" - ? "sellers AS seller_count" - : "seller_count"; + const sellerCountSelect = "seller_count"; const salesRankAvgSelect = processType === "lead_analysis" ? "rank_avg_90d AS sales_rank_avg_90d" @@ -1418,32 +1464,31 @@ function exportRunResultsCsv( "CAST(COALESCE(monthly_sold, 0) AS INTEGER) DESC, asin ASC", ); - const rows = db - .query( - `SELECT - run_id, - asin, - ${productNameSelect}, - brand, - category, - unit_cost, - current_price, - avg_price_90d, - ${salesRankAvgSelect}, - ${sellerCountSelect}, - amazon_is_seller, - amazon_buybox_share_pct_90d, - monthly_sold, - sellability_status, - verdict, - confidence, - reasoning, - fetched_at - FROM ${tableName} - ${where} - ORDER BY ${orderBy}`, - ) - .all(...params) as Array>; + const rows = await pgAll>( + `SELECT + run_id, + asin, + ${productNameSelect}, + brand, + category, + unit_cost, + current_price, + avg_price_90d, + ${salesRankAvgSelect}, + ${sellerCountSelect}, + amazon_is_seller, + amazon_buybox_share_pct_90d, + monthly_sold, + sellability_status, + verdict, + confidence, + reasoning, + fetched_at + FROM ${tableName} + ${where} + ORDER BY ${orderBy}`, + params, + ); const headers = [ "run_id", @@ -1499,77 +1544,71 @@ type ReanalyzeSourceRow = { lead_date: string | null; }; -function getReanalyzeSourceRow( +async function getReanalyzeSourceRow( processType: ProcessType, runId: number, asin: string, -): ReanalyzeSourceRow | null { +): Promise { if (processType === "lead_analysis") { - return ( - (db - .query( - `SELECT - asin, - product_name, - brand, - category, - unit_cost, - sales_rank, - avg_price_90d_sheet, - selling_price_sheet, - fba_net_sheet, - gross_profit_dollar, - gross_profit_pct, - net_profit_sheet, - roi_sheet, - moq, - moq_cost, - qty_available, - supplier, - source_url, - asin_link, - promo_coupon_code, - notes, - lead_date - FROM results - WHERE run_id = ? AND asin = ? - LIMIT 1`, - ) - .get(runId, asin) as ReanalyzeSourceRow | null) ?? null - ); - } - - return ( - (db - .query( - `SELECT + return pgGet( + `SELECT asin, - name AS product_name, + product_name, brand, category, unit_cost, sales_rank, avg_price_90d_sheet, selling_price_sheet, - NULL AS fba_net_sheet, - NULL AS gross_profit_dollar, - NULL AS gross_profit_pct, - NULL AS net_profit_sheet, - NULL AS roi_sheet, - NULL AS moq, - NULL AS moq_cost, - NULL AS qty_available, - NULL AS supplier, - NULL AS source_url, - NULL AS asin_link, - NULL AS promo_coupon_code, - NULL AS notes, - NULL AS lead_date - FROM product_analysis_results + fba_net_sheet, + gross_profit_dollar, + gross_profit_pct, + net_profit_sheet, + roi_sheet, + moq, + moq_cost, + qty_available, + supplier, + source_url, + asin_link, + promo_coupon_code, + notes, + lead_date + FROM analysis_results WHERE run_id = ? AND asin = ? LIMIT 1`, - ) - .get(runId, asin) as ReanalyzeSourceRow | null) ?? null + [runId, asin], + ); + } + + return pgGet( + `SELECT + asin, + name AS product_name, + brand, + category, + unit_cost, + sales_rank, + avg_price_90d_sheet, + selling_price_sheet, + NULL AS fba_net_sheet, + NULL AS gross_profit_dollar, + NULL AS gross_profit_pct, + NULL AS net_profit_sheet, + NULL AS roi_sheet, + NULL AS moq, + NULL AS moq_cost, + NULL AS qty_available, + NULL AS supplier, + NULL AS source_url, + NULL AS asin_link, + NULL AS promo_coupon_code, + NULL AS notes, + NULL AS lead_date + FROM category_product_results + WHERE run_id = ? AND asin = ? + LIMIT 1`, + [runId, asin], ); } @@ -1612,66 +1651,69 @@ function unknownSpApiData(): SpApiData { }; } -function refreshRunCounts(processType: ProcessType, runId: number): void { +async function refreshRunCounts( + processType: ProcessType, + runId: number, +): Promise { if (processType === "lead_analysis") { - const stats = db - .query( - `SELECT - COUNT(*) AS total, - SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba, - SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm, - SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip - FROM results - WHERE run_id = ?`, - ) - .get(runId) as { - total: number; - fba: number | null; - fbm: number | null; - skip: number | null; - }; + const stats = await pgGet<{ + total: string; + fba: string | null; + fbm: string | null; + skip: string | null; + }>( + `SELECT + COUNT(*) AS total, + SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba, + SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm, + SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip + FROM analysis_results + WHERE run_id = ?`, + [runId], + ); - db.query( + await pgRun( `UPDATE runs SET total_products = ?, fba_count = ?, fbm_count = ?, skip_count = ? WHERE id = ?`, - ).run( - stats.total ?? 0, - stats.fba ?? 0, - stats.fbm ?? 0, - stats.skip ?? 0, - runId, + [ + Number(stats?.total ?? 0), + Number(stats?.fba ?? 0), + Number(stats?.fbm ?? 0), + Number(stats?.skip ?? 0), + runId, + ], ); return; } - const stats = db - .query( - `SELECT - COUNT(*) AS available, - SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba, - SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm, - SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip - FROM product_analysis_results - WHERE run_id = ?`, - ) - .get(runId) as { - available: number; - fba: number | null; - fbm: number | null; - skip: number | null; - }; + const stats = await pgGet<{ + available: string; + fba: string | null; + fbm: string | null; + skip: string | null; + }>( + `SELECT + COUNT(*) AS available, + SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba, + SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm, + SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip + FROM category_product_results + WHERE run_id = ?`, + [runId], + ); - db.query( - `UPDATE category_analysis_runs + await pgRun( + `UPDATE runs SET available_asins = ?, fba_count = ?, fbm_count = ?, skip_count = ? WHERE id = ?`, - ).run( - stats.available ?? 0, - stats.fba ?? 0, - stats.fbm ?? 0, - stats.skip ?? 0, - runId, + [ + Number(stats?.available ?? 0), + Number(stats?.fba ?? 0), + Number(stats?.fbm ?? 0), + Number(stats?.skip ?? 0), + runId, + ], ); } @@ -1685,7 +1727,7 @@ async function reanalyzeSingleAsin( processType: ProcessType; fetchedAt: string; }> { - const row = getReanalyzeSourceRow(processType, runId, asin); + const row = await getReanalyzeSourceRow(processType, runId, asin); if (!row) { throw new Error("Result row not found"); } @@ -1729,18 +1771,17 @@ async function reanalyzeSingleAsin( reasoning: "LLM analysis returned no verdict", }; - const amazonIsSeller = - keepa?.amazonIsSeller == null ? null : keepa.amazonIsSeller ? 1 : 0; + const amazonIsSeller = keepa?.amazonIsSeller ?? null; const fetchedAt = enriched.fetchedAt; if (processType === "lead_analysis") { - db.query( - `UPDATE results SET + await pgRun( + `UPDATE analysis_results SET current_price = ?, avg_price_90d = ?, sales_rank = ?, rank_avg_90d = ?, - sellers = ?, + seller_count = ?, amazon_is_seller = ?, amazon_buybox_share_pct_90d = ?, monthly_sold = ?, @@ -1757,33 +1798,34 @@ async function reanalyzeSingleAsin( reasoning = ?, fetched_at = ? WHERE run_id = ? AND asin = ?`, - ).run( - keepa?.currentPrice ?? null, - keepa?.avgPrice90 ?? null, - keepa?.salesRank ?? row.sales_rank ?? null, - keepa?.salesRankAvg90 ?? null, - keepa?.sellerCount ?? null, - amazonIsSeller, - keepa?.amazonBuyboxSharePct90d ?? null, - keepa?.monthlySold ?? null, - keepa?.salesRankDrops30 ?? null, - keepa?.salesRankDrops90 ?? null, - spApi.fbaFee, - spApi.fbmFee, - spApi.referralFeePercent, - spApi.canSell == null ? null : spApi.canSell ? 1 : 0, - spApi.sellabilityStatus, - spApi.sellabilityReason ?? null, - verdict.verdict, - verdict.confidence, - verdict.reasoning, - fetchedAt, - runId, - asin, + [ + keepa?.currentPrice ?? null, + keepa?.avgPrice90 ?? null, + keepa?.salesRank ?? row.sales_rank ?? null, + keepa?.salesRankAvg90 ?? null, + keepa?.sellerCount ?? null, + amazonIsSeller, + keepa?.amazonBuyboxSharePct90d ?? null, + keepa?.monthlySold ?? null, + keepa?.salesRankDrops30 ?? null, + keepa?.salesRankDrops90 ?? null, + spApi.fbaFee, + spApi.fbmFee, + spApi.referralFeePercent, + spApi.canSell == null ? null : spApi.canSell ? "yes" : "no", + spApi.sellabilityStatus, + spApi.sellabilityReason ?? null, + verdict.verdict, + verdict.confidence, + verdict.reasoning, + fetchedAt, + runId, + asin, + ], ); } else { - db.query( - `UPDATE product_analysis_results SET + await pgRun( + `UPDATE category_product_results SET current_price = ?, avg_price_90d = ?, sales_rank = ?, @@ -1805,33 +1847,34 @@ async function reanalyzeSingleAsin( reasoning = ?, fetched_at = ? WHERE run_id = ? AND asin = ?`, - ).run( - keepa?.currentPrice ?? null, - keepa?.avgPrice90 ?? null, - keepa?.salesRank ?? row.sales_rank ?? null, - keepa?.salesRankAvg90 ?? null, - keepa?.sellerCount ?? null, - amazonIsSeller, - keepa?.amazonBuyboxSharePct90d ?? null, - keepa?.monthlySold ?? null, - keepa?.salesRankDrops30 ?? null, - keepa?.salesRankDrops90 ?? null, - spApi.fbaFee, - spApi.fbmFee, - spApi.referralFeePercent, - spApi.canSell == null ? null : spApi.canSell ? 1 : 0, - spApi.sellabilityStatus, - spApi.sellabilityReason ?? null, - verdict.verdict, - verdict.confidence, - verdict.reasoning, - fetchedAt, - runId, - asin, + [ + keepa?.currentPrice ?? null, + keepa?.avgPrice90 ?? null, + keepa?.salesRank ?? row.sales_rank ?? null, + keepa?.salesRankAvg90 ?? null, + keepa?.sellerCount ?? null, + amazonIsSeller, + keepa?.amazonBuyboxSharePct90d ?? null, + keepa?.monthlySold ?? null, + keepa?.salesRankDrops30 ?? null, + keepa?.salesRankDrops90 ?? null, + spApi.fbaFee, + spApi.fbmFee, + spApi.referralFeePercent, + spApi.canSell == null ? null : spApi.canSell, + spApi.sellabilityStatus, + spApi.sellabilityReason ?? null, + verdict.verdict, + verdict.confidence, + verdict.reasoning, + fetchedAt, + runId, + asin, + ], ); } - refreshRunCounts(processType, runId); + await refreshRunCounts(processType, runId); return { asin, runId, processType, fetchedAt }; } @@ -1844,31 +1887,31 @@ const server = Bun.serve({ "/stalker": index, "/stalker/products": index, "/runs/:processType/:runId": index, - "/api/runs": (req) => { + "/api/runs": async (req) => { const url = new URL(req.url); - return json(getRuns(url.searchParams)); + return json(await getRuns(url.searchParams)); }, - "/api/products": (req) => { + "/api/products": async (req) => { const url = new URL(req.url); - return json(getProductList(url.searchParams)); + return json(await getProductList(url.searchParams)); }, - "/api/stalker/results": (req) => { + "/api/stalker/results": async (req) => { const url = new URL(req.url); - return json(getStalkerResults(url.searchParams)); + return json(await getStalkerResults(url.searchParams)); }, - "/api/stalker/products": (req) => { + "/api/stalker/products": async (req) => { const url = new URL(req.url); - return json(getStalkerProducts(url.searchParams)); + return json(await getStalkerProducts(url.searchParams)); }, - "/api/stalker/products/export.xlsx": (req) => { + "/api/stalker/products/export.xlsx": async (req) => { const url = new URL(req.url); return exportStalkerProductsXlsx(url.searchParams); }, - "/api/stalker/purge": (req) => { + "/api/stalker/purge": async (req) => { if (req.method !== "DELETE" && req.method !== "POST") { return json({ error: "Method not allowed" }, 405); } - return json(purgeStalkerData()); + return json(await purgeStalkerData()); }, "/api/upc/map": async (req) => { let upcs: string[]; @@ -1952,7 +1995,6 @@ const server = Bun.serve({ inputBatchSize: parsed.inputBatchSize, upcLookupBatchSize: parsed.upcLookupBatchSize, maxRows: parsed.maxRows, - dbPath: DB_PATH, manageResources: false, }); return json(summary); @@ -1961,7 +2003,7 @@ const server = Bun.serve({ return json({ error: message }, 500); } }, - "/api/runs/:processType/:runId": (req) => { + "/api/runs/:processType/:runId": async (req) => { const processType = req.params.processType as ProcessType; const runId = Number(req.params.runId); @@ -1975,12 +2017,12 @@ const server = Bun.serve({ } if (req.method === "DELETE") { - const deleted = deleteRun(processType, runId); + const deleted = await deleteRun(processType, runId); if (!deleted.deletedRun) return json({ error: "Run not found" }, 404); return json(deleted); } - const run = getRun(processType, runId); + const run = await getRun(processType, runId); if (!run) return json({ error: "Run not found" }, 404); const summary = { @@ -1992,7 +2034,7 @@ const server = Bun.serve({ return json({ processType, ...run, summary }); }, - "/api/runs/:processType/:runId/results": (req) => { + "/api/runs/:processType/:runId/results": async (req) => { const processType = req.params.processType as ProcessType; const runId = Number(req.params.runId); @@ -2006,7 +2048,7 @@ const server = Bun.serve({ } const url = new URL(req.url); - const payload = getRunResults(processType, runId, url.searchParams); + const payload = await getRunResults(processType, runId, url.searchParams); return json(payload); }, "/api/runs/:processType/:runId/asins/:asin/reanalyze": async (req) => { @@ -2042,7 +2084,7 @@ const server = Bun.serve({ return json({ error: message }, 500); } }, - "/api/runs/:processType/:runId/export.csv": (req) => { + "/api/runs/:processType/:runId/export.csv": async (req) => { const processType = req.params.processType as ProcessType; const runId = Number(req.params.runId); @@ -2056,7 +2098,11 @@ const server = Bun.serve({ } const url = new URL(req.url); - const csvText = exportRunResultsCsv(processType, runId, url.searchParams); + const csvText = await exportRunResultsCsv( + processType, + runId, + url.searchParams, + ); return csv(csvText, `run-${processType}-${runId}.csv`); }, }, diff --git a/src/stalker-analyze.ts b/src/stalker-analyze.ts index 45b50dd..d0664a5 100644 --- a/src/stalker-analyze.ts +++ b/src/stalker-analyze.ts @@ -1,4 +1,6 @@ -import { type Database, closeDb, getDb, initDb } from "./database.ts"; +import { db } from "./db/index.ts"; +import { categoryProductResults, runs } from "./db/schema.ts"; +import { eq, sql } from "drizzle-orm"; import { analyzeProducts } from "./llm.ts"; import { fetchSpApiPricingAndFees } from "./sp-api.ts"; import type { @@ -13,7 +15,6 @@ const LLM_BATCH_SIZE = 5; const LLM_BATCH_DELAY_MS = 5_000; type Args = { - dbPath: string; stalkerRunId: number; analysisRunId: number; asins: string[]; @@ -22,18 +23,18 @@ type Args = { type InventoryRow = { asin: string; - product_title: string | null; + productTitle: string | null; brand: string | null; - category_tree: string | null; - current_price: number | null; - avg_price_90d: number | null; - sales_rank: number | null; - monthly_sold: number | null; - seller_count: number | null; - amazon_is_seller: number | null; - can_sell: number | null; - sellability_status: SellabilityInfo["sellabilityStatus"] | null; - sellability_reason: string | null; + categoryTree: string | null; + currentPrice: number | null; + avgPrice90d: number | null; + salesRank: number | null; + monthlySold: number | null; + sellerCount: number | null; + amazonIsSeller: boolean | null; + canSell: boolean | null; + sellabilityStatus: SellabilityInfo["sellabilityStatus"] | null; + sellabilityReason: string | null; }; function readFlagValue(args: string[], flag: string): string | undefined { @@ -43,7 +44,6 @@ function readFlagValue(args: string[], flag: string): string | undefined { } function parseArgs(argv = process.argv.slice(2)): Args { - const dbPath = readFlagValue(argv, "--db"); const stalkerRunId = Number(readFlagValue(argv, "--stalker-run-id")); const analysisRunId = Number(readFlagValue(argv, "--analysis-run-id")); const useClaude = argv.includes("--claude"); @@ -52,7 +52,6 @@ function parseArgs(argv = process.argv.slice(2)): Args { .map((asin) => asin.trim().toUpperCase()) .filter(Boolean); - if (!dbPath) throw new Error("Missing --db"); if (!Number.isInteger(stalkerRunId) || stalkerRunId <= 0) { throw new Error("--stalker-run-id must be a positive integer"); } @@ -61,7 +60,7 @@ function parseArgs(argv = process.argv.slice(2)): Args { } if (asins.length === 0) throw new Error("Missing --asins"); - return { dbPath, stalkerRunId, analysisRunId, asins, useClaude }; + return { stalkerRunId, analysisRunId, asins, useClaude }; } function wait(ms: number): Promise { @@ -81,69 +80,74 @@ function parseCategoryTree(value: string | null): string[] { } function toProductRecord(row: InventoryRow): ProductRecord { - const categoryTree = parseCategoryTree(row.category_tree); + const categoryTree = parseCategoryTree(row.categoryTree); return { asin: row.asin, - name: row.product_title ?? row.asin, + name: row.productTitle ?? row.asin, brand: row.brand ?? undefined, category: categoryTree.join(" > ") || undefined, unitCost: 0, - amazonRank: row.sales_rank ?? undefined, - sellingPriceFromSheet: row.current_price ?? undefined, - avgPrice90FromSheet: row.avg_price_90d ?? undefined, + amazonRank: row.salesRank ?? undefined, + sellingPriceFromSheet: row.currentPrice ?? undefined, + avgPrice90FromSheet: row.avgPrice90d ?? undefined, }; } function toKeepaData(row: InventoryRow): KeepaData { return { - currentPrice: row.current_price, - avgPrice90: row.avg_price_90d, + currentPrice: row.currentPrice, + avgPrice90: row.avgPrice90d, minPrice90: null, maxPrice90: null, - salesRank: row.sales_rank, + salesRank: row.salesRank, salesRankAvg90: null, salesRankDrops30: null, salesRankDrops90: null, - sellerCount: row.seller_count, - amazonIsSeller: - row.amazon_is_seller == null ? null : row.amazon_is_seller === 1, + sellerCount: row.sellerCount, + amazonIsSeller: row.amazonIsSeller, amazonBuyboxSharePct90d: null, buyBoxSeller: null, buyBoxPrice: null, buyBoxAvg90: null, - monthlySold: row.monthly_sold, - categoryTree: parseCategoryTree(row.category_tree), + monthlySold: row.monthlySold, + categoryTree: parseCategoryTree(row.categoryTree), }; } function toSellability(row: InventoryRow): SellabilityInfo { return { - canSell: row.can_sell == null ? null : row.can_sell === 1, - sellabilityStatus: row.sellability_status ?? "unknown", - sellabilityReason: row.sellability_reason ?? undefined, + canSell: row.canSell, + sellabilityStatus: row.sellabilityStatus ?? "unknown", + sellabilityReason: row.sellabilityReason ?? undefined, }; } -function loadInventoryRows( - database: Database, +async function loadInventoryRows( stalkerRunId: number, asins: string[], -): InventoryRow[] { - const placeholders = asins.map(() => "?").join(","); - return database - .query( - `SELECT - asin, product_title, brand, category_tree, current_price, avg_price_90d, - sales_rank, monthly_sold, seller_count, amazon_is_seller, can_sell, - sellability_status, sellability_reason - FROM stalker_seller_inventory - WHERE run_id = ? - AND can_sell = 1 - AND sellability_status = 'available' - AND asin IN (${placeholders}) - GROUP BY asin`, - ) - .all(stalkerRunId, ...asins) as InventoryRow[]; +): Promise { + if (asins.length === 0) return []; + return db.execute( + sql`SELECT DISTINCT ON (asin) + asin, + product_title AS "productTitle", + brand, + category_tree AS "categoryTree", + current_price AS "currentPrice", + avg_price_90d AS "avgPrice90d", + sales_rank AS "salesRank", + monthly_sold AS "monthlySold", + seller_count AS "sellerCount", + amazon_is_seller AS "amazonIsSeller", + can_sell AS "canSell", + sellability_status AS "sellabilityStatus", + sellability_reason AS "sellabilityReason" + FROM stalker_seller_inventory + WHERE run_id = ${stalkerRunId} + AND can_sell = true + AND sellability_status = 'available' + AND asin = ANY(${asins})`, + ); } async function buildEnrichedProducts( @@ -156,7 +160,7 @@ async function buildEnrichedProducts( const spApi = await fetchSpApiPricingAndFees( row.asin, sellability, - row.current_price, + row.currentPrice, ); enriched.push({ @@ -170,133 +174,114 @@ async function buildEnrichedProducts( return enriched; } -function insertProductAnalysisResults( - database: Database, +async function insertProductAnalysisResults( runId: number, results: AnalysisResult[], -): void { +): Promise { if (results.length === 0) return; - const insert = database.prepare(` - INSERT INTO product_analysis_results ( - asin, run_id, name, brand, category, unit_cost, - current_price, avg_price_90d, avg_price_90d_sheet, - selling_price_sheet, sales_rank, sales_rank_avg_90d, - seller_count, amazon_is_seller, amazon_buybox_share_pct_90d, - monthly_sold, rank_drops_30d, rank_drops_90d, - fba_fee, fbm_fee, referral_percent, can_sell, - sellability_status, sellability_reason, - verdict, confidence, reasoning, fetched_at - ) VALUES ( - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ?, ?, ?, ? - ) - ON CONFLICT(asin) DO UPDATE SET - run_id = excluded.run_id, - name = excluded.name, - brand = excluded.brand, - category = excluded.category, - unit_cost = excluded.unit_cost, - current_price = excluded.current_price, - avg_price_90d = excluded.avg_price_90d, - avg_price_90d_sheet = excluded.avg_price_90d_sheet, - selling_price_sheet = excluded.selling_price_sheet, - sales_rank = excluded.sales_rank, - sales_rank_avg_90d = excluded.sales_rank_avg_90d, - seller_count = excluded.seller_count, - amazon_is_seller = excluded.amazon_is_seller, - amazon_buybox_share_pct_90d = excluded.amazon_buybox_share_pct_90d, - monthly_sold = excluded.monthly_sold, - rank_drops_30d = excluded.rank_drops_30d, - rank_drops_90d = excluded.rank_drops_90d, - fba_fee = excluded.fba_fee, - fbm_fee = excluded.fbm_fee, - referral_percent = excluded.referral_percent, - can_sell = excluded.can_sell, - sellability_status = excluded.sellability_status, - sellability_reason = excluded.sellability_reason, - verdict = excluded.verdict, - confidence = excluded.confidence, - reasoning = excluded.reasoning, - fetched_at = excluded.fetched_at - `); + const rows = results.map((result) => { + const keepa = result.product.keepa; + const record = result.product.record; + const spApi = result.product.spApi; + const canSell = + spApi.canSell == null ? "unknown" : spApi.canSell ? "yes" : "no"; - database.transaction((batch: AnalysisResult[]) => { - for (const result of batch) { - const keepa = result.product.keepa; - const record = result.product.record; - const spApi = result.product.spApi; - insert.run( - record.asin, - runId, - record.name, - record.brand ?? null, - record.category ?? keepa?.categoryTree.join(" > ") ?? null, - record.unitCost ?? null, - keepa?.currentPrice ?? spApi.estimatedSalePrice ?? null, - keepa?.avgPrice90 ?? null, - record.avgPrice90FromSheet ?? null, - record.sellingPriceFromSheet ?? null, - keepa?.salesRank ?? record.amazonRank ?? null, - keepa?.salesRankAvg90 ?? null, - keepa?.sellerCount ?? null, - keepa?.amazonIsSeller == null ? null : keepa.amazonIsSeller ? 1 : 0, - keepa?.amazonBuyboxSharePct90d ?? null, - keepa?.monthlySold ?? null, - keepa?.salesRankDrops30 ?? null, - keepa?.salesRankDrops90 ?? null, - spApi.fbaFee ?? null, - spApi.fbmFee ?? null, - spApi.referralFeePercent ?? null, - spApi.canSell == null ? "unknown" : spApi.canSell ? "yes" : "no", - spApi.sellabilityStatus ?? null, - spApi.sellabilityReason ?? null, - result.verdict.verdict, - result.verdict.confidence, - result.verdict.reasoning ?? null, - result.product.fetchedAt, - ); - } - })(results); + return { + asin: record.asin, + runId, + name: record.name, + brand: record.brand ?? null, + category: record.category ?? keepa?.categoryTree.join(" > ") ?? null, + unitCost: record.unitCost ?? null, + currentPrice: keepa?.currentPrice ?? spApi.estimatedSalePrice ?? null, + avgPrice90d: keepa?.avgPrice90 ?? null, + avgPrice90dSheet: record.avgPrice90FromSheet ?? null, + sellingPriceSheet: record.sellingPriceFromSheet ?? null, + salesRank: keepa?.salesRank ?? record.amazonRank ?? null, + salesRankAvg90d: keepa?.salesRankAvg90 ?? null, + sellerCount: keepa?.sellerCount ?? null, + amazonIsSeller: keepa?.amazonIsSeller ?? null, + amazonBuyboxSharePct90d: keepa?.amazonBuyboxSharePct90d ?? null, + monthlySold: keepa?.monthlySold ?? null, + rankDrops30d: keepa?.salesRankDrops30 ?? null, + rankDrops90d: keepa?.salesRankDrops90 ?? null, + fbaFee: spApi.fbaFee ?? null, + fbmFee: spApi.fbmFee ?? null, + referralPercent: spApi.referralFeePercent ?? null, + canSell, + sellabilityStatus: spApi.sellabilityStatus ?? null, + sellabilityReason: spApi.sellabilityReason ?? null, + verdict: result.verdict.verdict, + confidence: result.verdict.confidence ?? 0, + reasoning: result.verdict.reasoning ?? null, + fetchedAt: new Date(result.product.fetchedAt), + }; + }); + + await db + .insert(categoryProductResults) + .values(rows) + .onConflictDoUpdate({ + target: categoryProductResults.asin, + set: { + runId: sql`EXCLUDED.run_id`, + name: sql`EXCLUDED.name`, + brand: sql`EXCLUDED.brand`, + category: sql`EXCLUDED.category`, + unitCost: sql`EXCLUDED.unit_cost`, + currentPrice: sql`EXCLUDED.current_price`, + avgPrice90d: sql`EXCLUDED.avg_price_90d`, + avgPrice90dSheet: sql`EXCLUDED.avg_price_90d_sheet`, + sellingPriceSheet: sql`EXCLUDED.selling_price_sheet`, + salesRank: sql`EXCLUDED.sales_rank`, + salesRankAvg90d: sql`EXCLUDED.sales_rank_avg_90d`, + sellerCount: sql`EXCLUDED.seller_count`, + amazonIsSeller: sql`EXCLUDED.amazon_is_seller`, + amazonBuyboxSharePct90d: sql`EXCLUDED.amazon_buybox_share_pct_90d`, + monthlySold: sql`EXCLUDED.monthly_sold`, + rankDrops30d: sql`EXCLUDED.rank_drops_30d`, + rankDrops90d: sql`EXCLUDED.rank_drops_90d`, + fbaFee: sql`EXCLUDED.fba_fee`, + fbmFee: sql`EXCLUDED.fbm_fee`, + referralPercent: sql`EXCLUDED.referral_percent`, + canSell: sql`EXCLUDED.can_sell`, + sellabilityStatus: sql`EXCLUDED.sellability_status`, + sellabilityReason: sql`EXCLUDED.sellability_reason`, + verdict: sql`EXCLUDED.verdict`, + confidence: sql`EXCLUDED.confidence`, + reasoning: sql`EXCLUDED.reasoning`, + fetchedAt: sql`EXCLUDED.fetched_at`, + }, + }); } -function refreshAnalysisRun(database: Database, runId: number): void { - const stats = database - .query( - `SELECT +async function refreshAnalysisRun(runId: number): Promise { + const [stats] = await db.execute( + sql<{ + total: string; + fba: string | null; + fbm: string | null; + skip: string | null; + }>`SELECT COUNT(*) AS total, SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba, SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm, SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip - FROM product_analysis_results - WHERE run_id = ?`, - ) - .get(runId) as { - total: number; - fba: number | null; - fbm: number | null; - skip: number | null; - }; + FROM category_product_results + WHERE run_id = ${runId}`, + ); - database - .prepare( - `UPDATE category_analysis_runs - SET top_asins_checked = ?, - available_asins = ?, - fba_count = ?, - fbm_count = ?, - skip_count = ? - WHERE id = ?`, - ) - .run( - stats.total ?? 0, - stats.total ?? 0, - stats.fba ?? 0, - stats.fbm ?? 0, - stats.skip ?? 0, - runId, - ); + await db + .update(runs) + .set({ + topAsinsChecked: Number(stats?.total ?? 0), + availableAsins: Number(stats?.total ?? 0), + fbaCount: Number(stats?.fba ?? 0), + fbmCount: Number(stats?.fbm ?? 0), + skipCount: Number(stats?.skip ?? 0), + }) + .where(eq(runs.id, runId)); } async function analyzeInBatches( @@ -349,24 +334,18 @@ async function analyzeInBatches( async function main(): Promise { const args = parseArgs(); - initDb(args.dbPath); - const database = getDb(args.dbPath); - try { - const rows = loadInventoryRows(database, args.stalkerRunId, args.asins); - if (rows.length === 0) { - console.log("Stalker analysis: no sellable inventory rows to analyze."); - return; - } - - console.log(`Stalker analysis: analyzing ${rows.length} sellable ASIN(s).`); - const enriched = await buildEnrichedProducts(rows); - const results = await analyzeInBatches(enriched, args.useClaude); - insertProductAnalysisResults(database, args.analysisRunId, results); - refreshAnalysisRun(database, args.analysisRunId); - } finally { - closeDb(); + const rows = await loadInventoryRows(args.stalkerRunId, args.asins); + if (rows.length === 0) { + console.log("Stalker analysis: no sellable inventory rows to analyze."); + return; } + + console.log(`Stalker analysis: analyzing ${rows.length} sellable ASIN(s).`); + const enriched = await buildEnrichedProducts(rows); + const results = await analyzeInBatches(enriched, args.useClaude); + await insertProductAnalysisResults(args.analysisRunId, results); + await refreshAnalysisRun(args.analysisRunId); } if (import.meta.main) { diff --git a/src/stalker-sellability.test.ts b/src/stalker-sellability.test.ts index 5cdbc4a..322e3f8 100644 --- a/src/stalker-sellability.test.ts +++ b/src/stalker-sellability.test.ts @@ -2,7 +2,67 @@ import { afterAll, beforeEach, expect, mock, test } from "bun:test"; import { mkdirSync, rmSync } from "node:fs"; import path from "node:path"; import * as XLSX from "xlsx"; -import { closeDb, getDb } from "./database.ts"; + +let nextId = 0; + +function chainable(resolveWith: any[] = []): any { + const p: any = Promise.resolve(resolveWith); + p.limit = (_n: any) => chainable(resolveWith); + p.where = (_cond: any) => chainable(resolveWith); + p.from = (_table: any) => chainable(resolveWith); + return p; +} + +const makeMockTx = (): any => ({ + insert: (_table: any) => ({ + values: (_vals: any) => ({ + returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]), + onConflictDoUpdate: (_conf: any) => Promise.resolve([]), + }), + }), + update: (_table: any) => ({ + set: (_vals: any) => ({ + where: (_cond: any) => Promise.resolve([]), + }), + }), + select: (_sel?: any) => ({ + from: (_table: any) => ({ + where: (_cond: any) => chainable([{ id: ++nextId }]), + limit: (_n: any) => chainable([{ id: nextId }]), + }), + }), + selectDistinct: (_sel: any) => ({ + from: (_table: any) => chainable([]), + }), + execute: (_query: any) => Promise.resolve([]), +}); + +const makeMockDb = (): any => ({ + insert: (_table: any) => ({ + values: (_vals: any) => ({ + returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]), + onConflictDoUpdate: (_conf: any) => Promise.resolve([]), + }), + }), + update: (_table: any) => ({ + set: (_vals: any) => ({ + where: (_cond: any) => Promise.resolve([]), + }), + }), + select: (_sel?: any) => ({ + from: (_table: any) => ({ + where: (_cond: any) => chainable(), + limit: (_n: any) => chainable(), + }), + }), + selectDistinct: (_sel: any) => ({ + from: (_table: any) => chainable(), + }), + execute: (_query: any) => Promise.resolve([]), + transaction: async (fn: (tx: any) => Promise) => fn(makeMockTx()), +}); + +mock.module("./db/index.ts", () => ({ db: makeMockDb(), client: {} })); const TEST_DIR = path.join(process.cwd(), "test_output", "stalker-sellability"); const originalFetch = globalThis.fetch; @@ -34,7 +94,7 @@ mock.module("./sp-api.ts", () => ({ const modulePromise = import("./stalker.ts"); beforeEach(() => { - closeDb(); + nextId = 0; rmSync(TEST_DIR, { recursive: true, force: true }); mkdirSync(TEST_DIR, { recursive: true }); globalThis.fetch = originalFetch; @@ -49,14 +109,12 @@ afterAll(() => { } else { Bun.env.KEEPA_API_KEY = originalKeepaKey; } - closeDb(); rmSync(TEST_DIR, { recursive: true, force: true }); }); test("sellability checks matched seller inventory, not the source ASIN", async () => { const { runStalker } = await modulePromise; const inputPath = path.join(TEST_DIR, "input.xlsx"); - const dbPath = path.join(TEST_DIR, "stalker.sqlite"); const workbook = XLSX.utils.book_new(); XLSX.utils.book_append_sheet( workbook, @@ -138,7 +196,6 @@ test("sellability checks matched seller inventory, not the source ASIN", async ( const stats = await runStalker({ input: inputPath, - dbPath, maxAsins: null, storefrontUpdateHours: 168, offerLimit: 20, @@ -151,6 +208,7 @@ test("sellability checks matched seller inventory, not the source ASIN", async ( maxSellerRequests: null, sellability: true, analyzeSellable: false, + useClaude: false, }); expect(fetchSellabilityBatchMock.mock.calls.length).toBe(1); @@ -162,46 +220,4 @@ test("sellability checks matched seller inventory, not the source ASIN", async ( expect(stats.inventorySellabilityAvailableAsins).toBe(1); expect(stats.inventorySellabilityExcludedAsins).toBe(1); expect(stats.persistedInventoryAsins).toBe(1); - - const db = getDb(dbPath); - const scan = db.query("SELECT source_asin FROM stalker_asin_scans").get() as { - source_asin: string; - }; - expect(scan.source_asin).toBe("B000000001"); - - const inventory = db - .query( - `SELECT asin, can_sell, sellability_status, product_title, brand, - category_tree, current_price, avg_price_90d, sales_rank, monthly_sold, - seller_count - FROM stalker_seller_inventory ORDER BY asin`, - ) - .all() as Array<{ - asin: string; - can_sell: number | null; - sellability_status: string | null; - product_title: string | null; - brand: string | null; - category_tree: string | null; - current_price: number | null; - avg_price_90d: number | null; - sales_rank: number | null; - monthly_sold: number | null; - seller_count: number | null; - }>; - expect(inventory).toEqual([ - { - asin: "B111111111", - can_sell: 1, - sellability_status: "available", - product_title: "Sellable Storefront Product", - brand: "Good Brand", - category_tree: JSON.stringify(["Kitchen", "Storage"]), - current_price: 19.99, - avg_price_90d: 25, - sales_rank: 12345, - monthly_sold: 42, - seller_count: 7, - }, - ]); }); diff --git a/src/stalker.test.ts b/src/stalker.test.ts index 25845c2..d324441 100644 --- a/src/stalker.test.ts +++ b/src/stalker.test.ts @@ -2,7 +2,6 @@ import { afterAll, beforeEach, expect, mock, test } from "bun:test"; import { mkdirSync, rmSync } from "node:fs"; import path from "node:path"; import * as XLSX from "xlsx"; -import { closeDb, getDb, initDb } from "./database.ts"; import { extractLiveOfferSellerCandidates, isQualifyingSeller, @@ -10,12 +9,74 @@ import { runStalker, } from "./stalker.ts"; +let nextId = 0; + +function chainable(resolveWith: any[] = []): any { + const p: any = Promise.resolve(resolveWith); + p.limit = (_n: any) => chainable(resolveWith); + p.where = (_cond: any) => chainable(resolveWith); + p.from = (_table: any) => chainable(resolveWith); + return p; +} + +// Transaction mock returns rows for selects (needed for upsert-then-select patterns). +const makeMockTx = (): any => ({ + insert: (_table: any) => ({ + values: (_vals: any) => ({ + returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]), + onConflictDoUpdate: (_conf: any) => Promise.resolve([]), + }), + }), + update: (_table: any) => ({ + set: (_vals: any) => ({ + where: (_cond: any) => Promise.resolve([]), + }), + }), + select: (_sel?: any) => ({ + from: (_table: any) => ({ + where: (_cond: any) => chainable([{ id: ++nextId }]), + limit: (_n: any) => chainable([{ id: nextId }]), + }), + }), + selectDistinct: (_sel: any) => ({ + from: (_table: any) => chainable([]), + }), + execute: (_query: any) => Promise.resolve([]), +}); + +const makeMockDb = (): any => ({ + insert: (_table: any) => ({ + values: (_vals: any) => ({ + returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]), + onConflictDoUpdate: (_conf: any) => Promise.resolve([]), + }), + }), + update: (_table: any) => ({ + set: (_vals: any) => ({ + where: (_cond: any) => Promise.resolve([]), + }), + }), + select: (_sel?: any) => ({ + from: (_table: any) => ({ + where: (_cond: any) => chainable(), + limit: (_n: any) => chainable(), + }), + }), + selectDistinct: (_sel: any) => ({ + from: (_table: any) => chainable(), + }), + execute: (_query: any) => Promise.resolve([]), + transaction: async (fn: (tx: any) => Promise) => fn(makeMockTx()), +}); + +mock.module("./db/index.ts", () => ({ db: makeMockDb(), client: {} })); + const TEST_DIR = path.join(process.cwd(), "test_output", "stalker"); const originalFetch = globalThis.fetch; const originalKeepaKey = Bun.env.KEEPA_API_KEY; beforeEach(() => { - closeDb(); + nextId = 0; rmSync(TEST_DIR, { recursive: true, force: true }); mkdirSync(TEST_DIR, { recursive: true }); globalThis.fetch = originalFetch; @@ -29,7 +90,6 @@ afterAll(() => { } else { Bun.env.KEEPA_API_KEY = originalKeepaKey; } - closeDb(); rmSync(TEST_DIR, { recursive: true, force: true }); }); @@ -77,35 +137,8 @@ test("extractLiveOfferSellerCandidates ignores Amazon, missing seller IDs, and d expect(offers[0]?.stock).toBe(4); }); -test("initDb creates stalker tables and indexes", () => { - const dbPath = path.join(TEST_DIR, "schema.sqlite"); - initDb(dbPath); - const db = getDb(dbPath); - - const tables = db - .query( - `SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'stalker_%' ORDER BY name`, - ) - .all() as Array<{ name: string }>; - expect(tables.map((row) => row.name)).toEqual([ - "stalker_asin_scans", - "stalker_asin_sellers", - "stalker_runs", - "stalker_seller_inventory", - "stalker_sellers", - ]); - - const indexes = db - .query( - `SELECT name FROM sqlite_master WHERE type = 'index' AND name LIKE 'idx_stalker_%' ORDER BY name`, - ) - .all() as Array<{ name: string }>; - expect(indexes.length).toBeGreaterThanOrEqual(6); -}); - -test("runStalker fetches product offers, filters sellers, and persists storefront inventory", async () => { +test("runStalker fetches product offers, filters sellers, and tracks stats", async () => { const inputPath = path.join(TEST_DIR, "input.xlsx"); - const dbPath = path.join(TEST_DIR, "stalker.sqlite"); const workbook = XLSX.utils.book_new(); XLSX.utils.book_append_sheet( workbook, @@ -205,7 +238,6 @@ test("runStalker fetches product offers, filters sellers, and persists storefron const stats = await runStalker({ input: inputPath, - dbPath, maxAsins: null, storefrontUpdateHours: 168, offerLimit: 20, @@ -218,6 +250,7 @@ test("runStalker fetches product offers, filters sellers, and persists storefron maxSellerRequests: null, sellability: false, analyzeSellable: false, + useClaude: false, }); expect(stats.scannedAsins).toBe(1); @@ -229,6 +262,7 @@ test("runStalker fetches product offers, filters sellers, and persists storefron expect(stats.qualifyingSellers).toBe(1); expect(stats.sellerMetadataRequests).toBe(1); expect(stats.sellerStorefrontRequests).toBe(1); + const sellerCalls = fetchMock.mock.calls.filter((call) => { const rawUrl = typeof call[0] === "string" @@ -239,45 +273,4 @@ test("runStalker fetches product offers, filters sellers, and persists storefron return new URL(rawUrl).pathname === "/seller"; }); expect(sellerCalls.length).toBe(2); - - const db = getDb(dbPath); - const run = db.query("SELECT * FROM stalker_runs").get() as any; - expect(run.status).toBe("completed"); - expect(run.requested_asins).toBe(1); - expect(run.scanned_asins).toBe(1); - expect(run.source_asins_with_matches).toBe(1); - expect(run.candidate_sellers).toBe(2); - expect(run.qualifying_sellers).toBe(1); - expect(run.matched_sellers).toBe(1); - expect(run.seller_metadata_requests).toBe(1); - expect(run.seller_storefront_requests).toBe(1); - expect(run.inventory_sellability_checked_asins).toBe(0); - expect(run.inventory_sellability_available_asins).toBe(0); - expect(run.inventory_sellability_excluded_asins).toBe(0); - expect(run.persisted_inventory_asins).toBe(0); - - const scan = db.query("SELECT * FROM stalker_asin_scans").get() as any; - expect(scan.source_asin).toBe("B000000001"); - expect(scan.title).toBe("Tracked Product"); - expect(scan.offer_count).toBe(2); - expect(scan.candidate_seller_count).toBe(2); - expect(scan.matched_seller_count).toBe(1); - - const sellers = db.query("SELECT * FROM stalker_sellers").all() as any[]; - expect(sellers.length).toBe(1); - expect(sellers[0].seller_id).toBe("AQUALIFIED"); - expect(sellers[0].rating_count).toBe(12); - expect(sellers[0].storefront_asin_total).toBe(2); - expect(sellers[0].persisted_inventory_sample_count).toBe(0); - - const asinSellers = db.query("SELECT * FROM stalker_asin_sellers").all() as any[]; - expect(asinSellers.length).toBe(1); - expect(asinSellers[0].offer_price).toBe(19.99); - expect(asinSellers[0].is_fba).toBe(1); - expect(asinSellers[0].stock).toBe(3); - - const inventory = db - .query("SELECT asin FROM stalker_seller_inventory ORDER BY asin") - .all() as Array<{ asin: string }>; - expect(inventory.map((row) => row.asin)).toEqual([]); }); diff --git a/src/stalker.ts b/src/stalker.ts index ac3abc8..629b961 100644 --- a/src/stalker.ts +++ b/src/stalker.ts @@ -1,6 +1,15 @@ import * as XLSX from "xlsx"; import path from "node:path"; -import { type Database, closeDb, getDb, initDb } from "./database.ts"; +import { db } from "./db/index.ts"; +import { + runs, + stalkerRuns, + stalkerAsinScans, + sellers, + stalkerAsinSellers, + stalkerSellerInventory, +} from "./db/schema.ts"; +import { eq, sql } from "drizzle-orm"; import { fetchSellabilityBatch } from "./sp-api.ts"; import type { SellabilityInfo } from "./types.ts"; @@ -8,7 +17,6 @@ const KEEPA_BASE = "https://api.keepa.com"; const DOMAIN_US = "1"; const AMAZON_US_SELLER_ID = "ATVPDKIKX0DER"; const ASIN_REGEX = /^B[0-9A-Z]{9}$/; -const DEFAULT_DB_PATH = path.join(process.cwd(), "db", "results.db"); const DEFAULT_STOREFRONT_UPDATE_HOURS = 168; const DEFAULT_OFFER_LIMIT = 100; const DEFAULT_SELLER_LIMIT = 30; @@ -28,7 +36,7 @@ type KeepaApiResponse = { export type StalkerArgs = { input: string; - dbPath: string; + dbPath?: string; maxAsins: number | null; storefrontUpdateHours: number; offerLimit: number; @@ -115,7 +123,6 @@ type StalkerRunStats = { }; type StalkerRunContext = { - database: Database | null; metadataCache: Map; storefrontCache: Map; stats: StalkerRunStats; @@ -131,7 +138,6 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs { printUsageAndExit("Missing required --input file."); } - const dbPath = readFlagValue(argv, "--db") ?? DEFAULT_DB_PATH; const maxAsinsRaw = readFlagValue(argv, "--max-asins"); const storefrontUpdateRaw = readFlagValue(argv, "--storefront-update-hours"); const offerLimitRaw = readFlagValue(argv, "--offer-limit"); @@ -205,7 +211,6 @@ export function parseArgs(argv = process.argv.slice(2)): StalkerArgs { return { input, - dbPath, maxAsins, storefrontUpdateHours, offerLimit, @@ -313,20 +318,18 @@ export async function runStalker(args: StalkerArgs): Promise { const cappedAsins = args.maxAsins == null ? allAsins : allAsins.slice(0, args.maxAsins); - initDb(args.dbPath); - const database = getDb(args.dbPath); const completedAsins = args.resume - ? loadPreviouslyScannedAsins(database) + ? await loadPreviouslyScannedAsins() : new Set(); const resumeFilteredAsins = cappedAsins.filter( (asin) => !completedAsins.has(asin), ); const runId = args.dryRun ? null - : startStalkerRun(database, args.input, resumeFilteredAsins.length); + : await startStalkerRun(args.input, resumeFilteredAsins.length); const analysisRunId = !args.dryRun && args.analyzeSellable - ? startStalkerAnalysisRun(database, args.input) + ? await startStalkerAnalysisRun(args.input) : null; const stats: StalkerRunStats = { scannedAsins: 0, @@ -345,7 +348,6 @@ export async function runStalker(args: StalkerArgs): Promise { stoppedEarly: false, }; const context: StalkerRunContext = { - database, metadataCache: new Map(), storefrontCache: new Map(), stats, @@ -389,7 +391,7 @@ export async function runStalker(args: StalkerArgs): Promise { } if (!args.dryRun && runId != null) { - persistAsinResult(database, runId, result); + await persistAsinResult(runId, result); } const sellableAsins = collectPersistedInventoryAsins(result); if ( @@ -400,7 +402,6 @@ export async function runStalker(args: StalkerArgs): Promise { sellableAsins.length > 0 ) { await runSellableAnalysisChild( - args.dbPath, runId, analysisRunId, sellableAsins, @@ -417,7 +418,7 @@ export async function runStalker(args: StalkerArgs): Promise { } if (!args.dryRun && runId != null) { - refreshStalkerRun(database, runId, stats, "running"); + await refreshStalkerRun(runId, stats, "running"); } console.log( `Stalker: ${asin} candidates=${result.candidateSellerCount}, matched=${result.matchedSellers.length}, persisted_inventory=${sumInventoryAsins(result)}`, @@ -432,8 +433,7 @@ export async function runStalker(args: StalkerArgs): Promise { } if (!args.dryRun && runId != null) { - refreshStalkerRun( - database, + await refreshStalkerRun( runId, stats, stats.stoppedEarly @@ -445,16 +445,16 @@ export async function runStalker(args: StalkerArgs): Promise { } logRunSummary(stats, args); if (!args.dryRun && analysisRunId != null) { - finishStalkerAnalysisRun(database, analysisRunId, "completed"); + await finishStalkerAnalysisRun(analysisRunId, "completed"); } return stats; } catch (error) { const message = error instanceof Error ? error.message : String(error); if (!args.dryRun && runId != null) { - finishStalkerRunWithError(database, runId, stats, message); + await finishStalkerRunWithError(runId, stats, message); } if (!args.dryRun && analysisRunId != null) { - finishStalkerAnalysisRun(database, analysisRunId, "failed", message); + await finishStalkerAnalysisRun(analysisRunId, "failed", message); } throw error; } @@ -685,13 +685,12 @@ async function fetchSellerMetadata( for (const sellerId of uniqueSellerIds) { const cached = context.metadataCache.get(sellerId) ?? - loadCachedSeller( - context.database, + (await loadCachedSeller( sellerId, args.sellerCacheHours, false, args.inventoryLimit, - ); + )); if (cached) { context.metadataCache.set(sellerId, cached); out.set(sellerId, cached); @@ -739,13 +738,12 @@ async function fetchQualifiedSellerStorefronts( for (const sellerId of uniqueSellerIds) { const cached = context.storefrontCache.get(sellerId) ?? - loadCachedSeller( - context.database, + (await loadCachedSeller( sellerId, args.sellerCacheHours, true, args.inventoryLimit, - ); + )); if (cached) { context.storefrontCache.set(sellerId, cached); out.set(sellerId, cached); @@ -830,272 +828,268 @@ async function fetchKeepaWithRetries( throw new Error(lastErrorMessage); } -function persistAsinResult( - database: Database, +async function persistAsinResult( runId: number, result: StalkerAsinResult, -): void { - const fetchedAt = new Date().toISOString(); +): Promise { + const fetchedAt = new Date(); - database.transaction(() => { - const scanId = upsertAsinScan(database, runId, result, fetchedAt); + await db.transaction(async (tx) => { + const scanId = await upsertAsinScan(tx, runId, result, fetchedAt); for (const { seller, offer } of result.matchedSellers) { - upsertSeller(database, seller, fetchedAt); - upsertAsinSeller(database, scanId, seller, offer); - upsertSellerInventory(database, runId, seller, fetchedAt); + await upsertSeller(tx, seller, fetchedAt); + await upsertAsinSeller(tx, scanId, seller, offer); + await upsertSellerInventory(tx, runId, seller, fetchedAt); } - })(); + }); } -function upsertAsinScan( - database: Database, +async function upsertAsinScan( + tx: Parameters[0]>[0], runId: number, result: StalkerAsinResult, - fetchedAt: string, -): number { - database - .prepare( - `INSERT INTO stalker_asin_scans ( - run_id, source_asin, title, offer_count, candidate_seller_count, - matched_seller_count, fetched_at, raw_product_json - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(run_id, source_asin) DO UPDATE SET - title = excluded.title, - offer_count = excluded.offer_count, - candidate_seller_count = excluded.candidate_seller_count, - matched_seller_count = excluded.matched_seller_count, - fetched_at = excluded.fetched_at, - raw_product_json = excluded.raw_product_json`, - ) - .run( + fetchedAt: Date, +): Promise { + await tx + .insert(stalkerAsinScans) + .values({ runId, - result.asin, - result.title, - result.offerCount, - result.candidateSellerCount, - result.matchedSellers.length, + sourceAsin: result.asin, + title: result.title, + offerCount: result.offerCount, + candidateSellerCount: result.candidateSellerCount, + matchedSellerCount: result.matchedSellers.length, fetchedAt, - JSON.stringify(result.product ?? { error: result.error ?? null }), - ); + rawProductJson: JSON.stringify( + result.product ?? { error: result.error ?? null }, + ), + }) + .onConflictDoUpdate({ + target: [stalkerAsinScans.runId, stalkerAsinScans.sourceAsin], + set: { + title: sql`EXCLUDED.title`, + offerCount: sql`EXCLUDED.offer_count`, + candidateSellerCount: sql`EXCLUDED.candidate_seller_count`, + matchedSellerCount: sql`EXCLUDED.matched_seller_count`, + fetchedAt: sql`EXCLUDED.fetched_at`, + rawProductJson: sql`EXCLUDED.raw_product_json`, + }, + }); - const row = database - .query( - `SELECT id FROM stalker_asin_scans WHERE run_id = ? AND source_asin = ?`, - ) - .get(runId, result.asin) as { id: number } | null; + const [row] = await tx + .select({ id: stalkerAsinScans.id }) + .from(stalkerAsinScans) + .where( + sql`${stalkerAsinScans.runId} = ${runId} AND ${stalkerAsinScans.sourceAsin} = ${result.asin}`, + ); if (!row) throw new Error(`Failed to load stalker scan row for ${result.asin}`); return row.id; } -function upsertSeller( - database: Database, +async function upsertSeller( + tx: Parameters[0]>[0], seller: StalkerSeller, - fetchedAt: string, -): void { - database - .prepare( - `INSERT INTO stalker_sellers ( - seller_id, seller_name, rating, rating_count, storefront_asin_total, - persisted_inventory_sample_count, last_updated_at, raw_seller_json - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(seller_id) DO UPDATE SET - seller_name = excluded.seller_name, - rating = excluded.rating, - rating_count = excluded.rating_count, - storefront_asin_total = excluded.storefront_asin_total, - persisted_inventory_sample_count = excluded.persisted_inventory_sample_count, - last_updated_at = excluded.last_updated_at, - raw_seller_json = excluded.raw_seller_json`, - ) - .run( - seller.sellerId, - seller.sellerName, - seller.rating, - seller.ratingCount, - seller.storefrontAsinTotal, - seller.storefrontItems.length, - fetchedAt, - JSON.stringify(seller.rawSeller), - ); + fetchedAt: Date, +): Promise { + await tx + .insert(sellers) + .values({ + sellerId: seller.sellerId, + sellerName: seller.sellerName, + rating: seller.rating, + ratingCount: seller.ratingCount, + storefrontAsinTotal: seller.storefrontAsinTotal, + persistedInventorySampleCount: seller.storefrontItems.length, + lastUpdatedAt: fetchedAt, + rawSellerJson: JSON.stringify(seller.rawSeller), + }) + .onConflictDoUpdate({ + target: sellers.sellerId, + set: { + sellerName: sql`EXCLUDED.seller_name`, + rating: sql`EXCLUDED.rating`, + ratingCount: sql`EXCLUDED.rating_count`, + storefrontAsinTotal: sql`EXCLUDED.storefront_asin_total`, + persistedInventorySampleCount: sql`EXCLUDED.persisted_inventory_sample_count`, + lastUpdatedAt: sql`EXCLUDED.last_updated_at`, + rawSellerJson: sql`EXCLUDED.raw_seller_json`, + }, + }); } -function upsertAsinSeller( - database: Database, +async function upsertAsinSeller( + tx: Parameters[0]>[0], scanId: number, seller: StalkerSeller, offer: StalkerOffer, -): void { - database - .prepare( - `INSERT INTO stalker_asin_sellers ( - scan_id, seller_id, offer_price, condition, is_fba, stock, - seller_rating, seller_rating_count, raw_offer_json - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(scan_id, seller_id) DO UPDATE SET - offer_price = excluded.offer_price, - condition = excluded.condition, - is_fba = excluded.is_fba, - stock = excluded.stock, - seller_rating = excluded.seller_rating, - seller_rating_count = excluded.seller_rating_count, - raw_offer_json = excluded.raw_offer_json`, - ) - .run( +): Promise { + await tx + .insert(stalkerAsinSellers) + .values({ scanId, - seller.sellerId, - offer.offerPrice, - offer.condition, - offer.isFba == null ? null : offer.isFba ? 1 : 0, - offer.stock, - seller.rating, - seller.ratingCount, - JSON.stringify(offer.rawOffer), - ); + sellerId: seller.sellerId, + offerPrice: offer.offerPrice, + condition: offer.condition, + isFba: offer.isFba, + stock: offer.stock, + sellerRating: seller.rating, + sellerRatingCount: seller.ratingCount, + rawOfferJson: JSON.stringify(offer.rawOffer), + }) + .onConflictDoUpdate({ + target: [stalkerAsinSellers.scanId, stalkerAsinSellers.sellerId], + set: { + offerPrice: sql`EXCLUDED.offer_price`, + condition: sql`EXCLUDED.condition`, + isFba: sql`EXCLUDED.is_fba`, + stock: sql`EXCLUDED.stock`, + sellerRating: sql`EXCLUDED.seller_rating`, + sellerRatingCount: sql`EXCLUDED.seller_rating_count`, + rawOfferJson: sql`EXCLUDED.raw_offer_json`, + }, + }); } -function upsertSellerInventory( - database: Database, +async function upsertSellerInventory( + tx: Parameters[0]>[0], runId: number, seller: StalkerSeller, - fetchedAt: string, -): void { - const insert = database.prepare( - `INSERT INTO stalker_seller_inventory ( - run_id, seller_id, asin, can_sell, sellability_status, - sellability_reason, product_title, brand, category_tree, current_price, - avg_price_90d, sales_rank, monthly_sold, seller_count, amazon_is_seller, - raw_product_json, last_seen_at, raw_inventory_json - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(run_id, seller_id, asin) DO UPDATE SET - can_sell = excluded.can_sell, - sellability_status = excluded.sellability_status, - sellability_reason = excluded.sellability_reason, - product_title = excluded.product_title, - brand = excluded.brand, - category_tree = excluded.category_tree, - current_price = excluded.current_price, - avg_price_90d = excluded.avg_price_90d, - sales_rank = excluded.sales_rank, - monthly_sold = excluded.monthly_sold, - seller_count = excluded.seller_count, - amazon_is_seller = excluded.amazon_is_seller, - raw_product_json = excluded.raw_product_json, - last_seen_at = excluded.last_seen_at, - raw_inventory_json = excluded.raw_inventory_json`, + fetchedAt: Date, +): Promise { + const items = seller.storefrontItems.filter( + (item) => + item.sellability?.canSell === true && + item.sellability.sellabilityStatus === "available", ); - for (const item of seller.storefrontItems) { - if ( - item.sellability?.canSell !== true || - item.sellability.sellabilityStatus !== "available" - ) { - continue; - } + if (items.length === 0) return; - insert.run( - runId, - seller.sellerId, - item.asin, - item.sellability?.canSell == null - ? null - : item.sellability.canSell - ? 1 - : 0, - item.sellability?.sellabilityStatus ?? null, - item.sellability?.sellabilityReason ?? null, - item.productDetails?.title ?? null, - item.productDetails?.brand ?? null, - item.productDetails - ? JSON.stringify(item.productDetails.categoryTree) - : null, - item.productDetails?.currentPrice ?? null, - item.productDetails?.avgPrice90 ?? null, - item.productDetails?.salesRank ?? null, - item.productDetails?.monthlySold ?? null, - item.productDetails?.sellerCount ?? null, - item.productDetails?.amazonIsSeller == null - ? null - : item.productDetails.amazonIsSeller - ? 1 - : 0, - item.productDetails - ? JSON.stringify(item.productDetails.rawProduct) - : null, - fetchedAt, - JSON.stringify(item.rawInventory), - ); - } + await tx + .insert(stalkerSellerInventory) + .values( + items.map((item) => ({ + runId, + sellerId: seller.sellerId, + asin: item.asin, + canSell: item.sellability?.canSell ?? null, + sellabilityStatus: item.sellability?.sellabilityStatus ?? null, + sellabilityReason: item.sellability?.sellabilityReason ?? null, + productTitle: item.productDetails?.title ?? null, + brand: item.productDetails?.brand ?? null, + categoryTree: item.productDetails + ? JSON.stringify(item.productDetails.categoryTree) + : null, + currentPrice: item.productDetails?.currentPrice ?? null, + avgPrice90d: item.productDetails?.avgPrice90 ?? null, + salesRank: item.productDetails?.salesRank ?? null, + monthlySold: item.productDetails?.monthlySold ?? null, + sellerCount: item.productDetails?.sellerCount ?? null, + amazonIsSeller: item.productDetails?.amazonIsSeller ?? null, + rawProductJson: item.productDetails + ? JSON.stringify(item.productDetails.rawProduct) + : null, + lastSeenAt: fetchedAt, + rawInventoryJson: JSON.stringify(item.rawInventory), + })), + ) + .onConflictDoUpdate({ + target: [ + stalkerSellerInventory.runId, + stalkerSellerInventory.sellerId, + stalkerSellerInventory.asin, + ], + set: { + canSell: sql`EXCLUDED.can_sell`, + sellabilityStatus: sql`EXCLUDED.sellability_status`, + sellabilityReason: sql`EXCLUDED.sellability_reason`, + productTitle: sql`EXCLUDED.product_title`, + brand: sql`EXCLUDED.brand`, + categoryTree: sql`EXCLUDED.category_tree`, + currentPrice: sql`EXCLUDED.current_price`, + avgPrice90d: sql`EXCLUDED.avg_price_90d`, + salesRank: sql`EXCLUDED.sales_rank`, + monthlySold: sql`EXCLUDED.monthly_sold`, + sellerCount: sql`EXCLUDED.seller_count`, + amazonIsSeller: sql`EXCLUDED.amazon_is_seller`, + rawProductJson: sql`EXCLUDED.raw_product_json`, + lastSeenAt: sql`EXCLUDED.last_seen_at`, + rawInventoryJson: sql`EXCLUDED.raw_inventory_json`, + }, + }); } -function startStalkerRun( - database: Database, +async function startStalkerRun( inputFile: string, totalAsins: number, -): number { - const result = database - .prepare( - `INSERT INTO stalker_runs ( - input_file, started_at, requested_asins, status - ) VALUES (?, ?, ?, ?)`, - ) - .run(inputFile, new Date().toISOString(), totalAsins, "running"); - - return result.lastInsertRowid as number; +): Promise { + const [row] = await db + .insert(stalkerRuns) + .values({ + inputFile, + startedAt: new Date(), + requestedAsins: totalAsins, + status: "running", + }) + .returning({ id: stalkerRuns.id }); + if (!row) throw new Error("Failed to insert stalker run record."); + return row.id; } -function startStalkerAnalysisRun( - database: Database, - inputFile: string, -): number { - const result = database - .prepare( - `INSERT INTO category_analysis_runs ( - category_id, category_label, run_timestamp, top_asins_checked, - available_asins, fba_count, fbm_count, skip_count, status, error_message - ) VALUES (?, ?, ?, 0, 0, 0, 0, 0, 'running', NULL)`, - ) - .run(0, `Stalker: ${path.basename(inputFile)}`, new Date().toISOString()); - - return result.lastInsertRowid as number; +async function startStalkerAnalysisRun(inputFile: string): Promise { + const [row] = await db + .insert(runs) + .values({ + type: "category_analysis", + categoryId: 0, + categoryLabel: `Stalker: ${path.basename(inputFile)}`, + topAsinsChecked: 0, + availableAsins: 0, + fbaCount: 0, + fbmCount: 0, + skipCount: 0, + status: "running", + startedAt: new Date(), + }) + .returning({ id: runs.id }); + if (!row) throw new Error("Failed to insert stalker analysis run record."); + return row.id; } -function loadPreviouslyScannedAsins(database: Database): Set { - const rows = database - .query(`SELECT DISTINCT source_asin FROM stalker_asin_scans`) - .all() as Array<{ source_asin: string }>; - return new Set(rows.map((row) => row.source_asin)); +async function loadPreviouslyScannedAsins(): Promise> { + const rows = await db + .selectDistinct({ sourceAsin: stalkerAsinScans.sourceAsin }) + .from(stalkerAsinScans); + return new Set(rows.map((row) => row.sourceAsin)); } -function loadCachedSeller( - database: Database | null, +async function loadCachedSeller( sellerId: string, maxAgeHours: number, requireStorefront: boolean, inventoryLimit: number, -): StalkerSeller | null { - if (!database || maxAgeHours <= 0) return null; - const row = database - .query( - `SELECT raw_seller_json, last_updated_at, storefront_asin_total - FROM stalker_sellers - WHERE seller_id = ?`, - ) - .get(sellerId) as { - raw_seller_json: string | null; - last_updated_at: string; - storefront_asin_total: number | null; - } | null; - if (!row?.raw_seller_json) return null; +): Promise { + if (maxAgeHours <= 0) return null; - const ageMs = Date.now() - new Date(row.last_updated_at).getTime(); + const [row] = await db + .select({ + rawSellerJson: sellers.rawSellerJson, + lastUpdatedAt: sellers.lastUpdatedAt, + }) + .from(sellers) + .where(eq(sellers.sellerId, sellerId)) + .limit(1); + + if (!row?.rawSellerJson) return null; + + const ageMs = Date.now() - new Date(row.lastUpdatedAt).getTime(); if (!Number.isFinite(ageMs) || ageMs > maxAgeHours * 60 * 60 * 1000) { return null; } try { - const rawSeller = JSON.parse(row.raw_seller_json) as Record; + const rawSeller = JSON.parse(row.rawSellerJson) as Record; const parsed = parseSeller(sellerId, rawSeller, inventoryLimit); if (requireStorefront && parsed.storefrontAsinTotal <= 0) return null; return parsed; @@ -1128,137 +1122,92 @@ function logRunSummary(stats: StalkerRunStats, args: StalkerArgs): void { ); } -function refreshStalkerRun( - database: Database, +async function refreshStalkerRun( runId: number, stats: StalkerRunStats, status: string, -): void { - database - .prepare( - `UPDATE stalker_runs - SET scanned_asins = ?, - source_asins_with_matches = ?, - candidate_sellers = ?, - qualifying_sellers = ?, - matched_sellers = ?, - seller_metadata_requests = ?, - seller_storefront_requests = ?, - inventory_sellability_checked_asins = ?, - inventory_sellability_available_asins = ?, - inventory_sellability_excluded_asins = ?, - persisted_inventory_asins = ?, - status = ?, - completed_at = CASE WHEN ? = 'running' THEN completed_at ELSE ? END - WHERE id = ?`, - ) - .run( - stats.scannedAsins, - stats.sourceAsinsWithMatches, - stats.candidateSellers, - stats.qualifyingSellers, - stats.matchedSellers, - stats.sellerMetadataRequests, - stats.sellerStorefrontRequests, - stats.inventorySellabilityCheckedAsins, - stats.inventorySellabilityAvailableAsins, - stats.inventorySellabilityExcludedAsins, - stats.persistedInventoryAsins, +): Promise { + await db + .update(stalkerRuns) + .set({ + scannedAsins: stats.scannedAsins, + sourceAsinsWithMatches: stats.sourceAsinsWithMatches, + candidateSellers: stats.candidateSellers, + qualifyingSellers: stats.qualifyingSellers, + matchedSellers: stats.matchedSellers, + sellerMetadataRequests: stats.sellerMetadataRequests, + sellerStorefrontRequests: stats.sellerStorefrontRequests, + inventorySellabilityCheckedAsins: stats.inventorySellabilityCheckedAsins, + inventorySellabilityAvailableAsins: + stats.inventorySellabilityAvailableAsins, + inventorySellabilityExcludedAsins: stats.inventorySellabilityExcludedAsins, + persistedInventoryAsins: stats.persistedInventoryAsins, status, - status, - new Date().toISOString(), - runId, - ); + ...(status !== "running" ? { completedAt: new Date() } : {}), + }) + .where(eq(stalkerRuns.id, runId)); } -function finishStalkerRunWithError( - database: Database, +async function finishStalkerRunWithError( runId: number, stats: StalkerRunStats, errorMessage: string, -): void { - database - .prepare( - `UPDATE stalker_runs - SET scanned_asins = ?, - source_asins_with_matches = ?, - candidate_sellers = ?, - qualifying_sellers = ?, - matched_sellers = ?, - seller_metadata_requests = ?, - seller_storefront_requests = ?, - inventory_sellability_checked_asins = ?, - inventory_sellability_available_asins = ?, - inventory_sellability_excluded_asins = ?, - persisted_inventory_asins = ?, - status = 'failed', - error_message = ?, - completed_at = ? - WHERE id = ?`, - ) - .run( - stats.scannedAsins, - stats.sourceAsinsWithMatches, - stats.candidateSellers, - stats.qualifyingSellers, - stats.matchedSellers, - stats.sellerMetadataRequests, - stats.sellerStorefrontRequests, - stats.inventorySellabilityCheckedAsins, - stats.inventorySellabilityAvailableAsins, - stats.inventorySellabilityExcludedAsins, - stats.persistedInventoryAsins, +): Promise { + await db + .update(stalkerRuns) + .set({ + scannedAsins: stats.scannedAsins, + sourceAsinsWithMatches: stats.sourceAsinsWithMatches, + candidateSellers: stats.candidateSellers, + qualifyingSellers: stats.qualifyingSellers, + matchedSellers: stats.matchedSellers, + sellerMetadataRequests: stats.sellerMetadataRequests, + sellerStorefrontRequests: stats.sellerStorefrontRequests, + inventorySellabilityCheckedAsins: stats.inventorySellabilityCheckedAsins, + inventorySellabilityAvailableAsins: + stats.inventorySellabilityAvailableAsins, + inventorySellabilityExcludedAsins: stats.inventorySellabilityExcludedAsins, + persistedInventoryAsins: stats.persistedInventoryAsins, + status: "failed", errorMessage, - new Date().toISOString(), - runId, - ); + completedAt: new Date(), + }) + .where(eq(stalkerRuns.id, runId)); } -function finishStalkerAnalysisRun( - database: Database, +async function finishStalkerAnalysisRun( runId: number, status: "completed" | "failed", errorMessage: string | null = null, -): void { - const stats = database - .query( - `SELECT +): Promise { + const [stats] = await db.execute( + sql<{ + total: string; + fba: string | null; + fbm: string | null; + skip: string | null; + }>`SELECT COUNT(*) AS total, SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba, SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm, SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip - FROM product_analysis_results - WHERE run_id = ?`, - ) - .get(runId) as { - total: number; - fba: number | null; - fbm: number | null; - skip: number | null; - }; + FROM category_product_results + WHERE run_id = ${runId}`, + ); - database - .prepare( - `UPDATE category_analysis_runs - SET top_asins_checked = ?, - available_asins = ?, - fba_count = ?, - fbm_count = ?, - skip_count = ?, - status = ?, - error_message = ? - WHERE id = ?`, - ) - .run( - stats.total ?? 0, - stats.total ?? 0, - stats.fba ?? 0, - stats.fbm ?? 0, - stats.skip ?? 0, + await db + .update(runs) + .set({ + topAsinsChecked: Number(stats?.total ?? 0), + availableAsins: Number(stats?.total ?? 0), + fbaCount: Number(stats?.fba ?? 0), + fbmCount: Number(stats?.fbm ?? 0), + skipCount: Number(stats?.skip ?? 0), status, errorMessage, - runId, - ); + completedAt: new Date(), + }) + .where(eq(runs.id, runId)); } function normalizeSellerResponse( @@ -1492,7 +1441,6 @@ function collectPersistedInventoryAsins(result: StalkerAsinResult): string[] { } async function runSellableAnalysisChild( - dbPath: string, stalkerRunId: number, analysisRunId: number, asins: string[], @@ -1502,8 +1450,6 @@ async function runSellableAnalysisChild( "bun", "run", "src/stalker-analyze.ts", - "--db", - dbPath, "--stalker-run-id", String(stalkerRunId), "--analysis-run-id", @@ -1660,8 +1606,5 @@ if (import.meta.main) { .catch((error) => { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; - }) - .finally(() => { - closeDb(); }); } diff --git a/src/top-monthly-sold-by-category.test.ts b/src/top-monthly-sold-by-category.test.ts index fce48c5..b832b6c 100644 --- a/src/top-monthly-sold-by-category.test.ts +++ b/src/top-monthly-sold-by-category.test.ts @@ -1,8 +1,41 @@ -import { test, expect, beforeAll, afterAll, beforeEach, mock } from "bun:test"; -import { Database } from "bun:sqlite"; -import { getDb, initDb, closeDb } from "./database.ts"; -import path from "node:path"; -import { rmSync, mkdirSync } from "node:fs"; +import { test, expect, beforeEach, mock } from "bun:test"; + +let nextId = 0; + +function chainable(resolveWith: any[] = []): any { + const p: any = Promise.resolve(resolveWith); + p.limit = (_n: any) => chainable(resolveWith); + p.where = (_cond: any) => chainable(resolveWith); + p.from = (_table: any) => chainable(resolveWith); + return p; +} + +const makeMockDb = (): any => ({ + insert: (_table: any) => ({ + values: (_vals: any) => ({ + returning: (_sel: any) => Promise.resolve([{ id: ++nextId }]), + onConflictDoUpdate: (_conf: any) => Promise.resolve([]), + }), + }), + update: (_table: any) => ({ + set: (_vals: any) => ({ + where: (_cond: any) => Promise.resolve([]), + }), + }), + select: (_sel?: any) => ({ + from: (_table: any) => ({ + where: (_cond: any) => chainable(), + limit: (_n: any) => chainable(), + }), + }), + selectDistinct: (_sel: any) => ({ + from: (_table: any) => chainable(), + }), + execute: (_query: any) => Promise.resolve([]), + transaction: async (fn: (tx: any) => Promise) => fn(makeMockDb()), +}); + +mock.module("./db/index.ts", () => ({ db: makeMockDb(), client: {} })); const fetchSellabilityBatchMock = mock(async (asins: string[]) => { return new Map( @@ -60,51 +93,23 @@ mock.module("./llm.ts", () => ({ const modulePromise = import("./top-monthly-sold-by-category.ts"); -const DB_TEST_PATH = path.join( - process.cwd(), - "test_output", - "test_monthly_sold_analysis.sqlite", -); - -let db: Database; let processCategory: ( - db: Database, runId: number, category: any, perCategoryTop: number, categoryCandidatePool: number, minMonthlySold: number, ) => Promise; -let insertCategoryRunSummary: ( - db: Database, - summary: any, - runTimestamp: string, -) => Promise; +let insertCategoryRunSummary: (summary: any, runTimestamp: string) => Promise; let originalFetch: typeof globalThis.fetch; -beforeAll(async () => { - const mod = await modulePromise; - processCategory = mod.processCategory; - insertCategoryRunSummary = mod.insertCategoryRunSummary; - - rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true }); - mkdirSync(path.dirname(DB_TEST_PATH), { recursive: true }); - initDb(DB_TEST_PATH); - db = getDb(DB_TEST_PATH); - - originalFetch = globalThis.fetch; -}); - -afterAll(() => { - globalThis.fetch = originalFetch; - closeDb(); - rmSync(path.dirname(DB_TEST_PATH), { recursive: true, force: true }); -}); +const mod = await modulePromise; +processCategory = mod.processCategory; +insertCategoryRunSummary = mod.insertCategoryRunSummary; +originalFetch = globalThis.fetch; beforeEach(() => { - db.run("DELETE FROM product_analysis_results"); - db.run("DELETE FROM category_analysis_runs"); - + nextId = 0; globalThis.fetch = mock(async (input: string | URL | Request) => { const rawUrl = typeof input === "string" @@ -140,25 +145,8 @@ beforeEach(() => { monthlySold: 600, stats: { current: [ - null, - null, - null, - 1000, - null, - null, - null, - null, - null, - null, - null, - 2, - null, - null, - null, - null, - null, - null, - 2599, + null, null, null, 1000, null, null, null, null, null, null, null, 2, + null, null, null, null, null, null, 2599, ], avg: [2400, null, null, 1200], }, @@ -171,25 +159,8 @@ beforeEach(() => { monthlySold: 250, stats: { current: [ - null, - null, - null, - 2000, - null, - null, - null, - null, - null, - null, - null, - 3, - null, - null, - null, - null, - null, - null, - 1999, + null, null, null, 2000, null, null, null, null, null, null, null, 3, + null, null, null, null, null, null, 1999, ], avg: [1800, null, null, 2200], }, @@ -202,25 +173,8 @@ beforeEach(() => { monthlySold: 800, stats: { current: [ - null, - null, - null, - 1500, - null, - null, - null, - null, - null, - null, - null, - 1, - null, - null, - null, - null, - null, - null, - 2099, + null, null, null, 1500, null, null, null, null, null, null, null, 1, + null, null, null, null, null, null, 2099, ], avg: [2000, null, null, 1800], }, @@ -233,25 +187,8 @@ beforeEach(() => { monthlySold: 400, stats: { current: [ - null, - null, - null, - 3000, - null, - null, - null, - null, - null, - null, - null, - 4, - null, - null, - null, - null, - null, - null, - 2899, + null, null, null, 3000, null, null, null, null, null, null, null, 4, + null, null, null, null, null, null, 2899, ], avg: [2600, null, null, 2800], }, @@ -279,7 +216,6 @@ test("processCategory filters to sellable ASINs with monthly sold >= threshold a }; const runId = await insertCategoryRunSummary( - db, { categoryId: mockCategory.id, categoryLabel: mockCategory.label, @@ -295,22 +231,16 @@ test("processCategory filters to sellable ASINs with monthly sold >= threshold a new Date().toISOString(), ); - const summary = await processCategory(db, runId, mockCategory, 2, 4, 300); + const summary = await processCategory(runId, mockCategory, 2, 4, 300); expect(summary.status).toBe("ok"); expect(summary.topAsinsChecked).toBe(4); expect(summary.availableAsins).toBe(2); expect(summary.results?.length).toBe(2); - const productResults = db - .query( - "SELECT asin, monthly_sold FROM product_analysis_results ORDER BY monthly_sold DESC", - ) - .all() as Array<{ asin: string; monthly_sold: number }>; + const asins = summary.results?.map((r: any) => r.product.record.asin) ?? []; + expect(asins).toContain("B000000001"); + expect(asins).toContain("B000000004"); - expect(productResults.length).toBe(2); - expect(productResults[0]?.asin).toBe("B000000001"); - expect(productResults[0]?.monthly_sold).toBe(600); - expect(productResults[1]?.asin).toBe("B000000004"); - expect(productResults[1]?.monthly_sold).toBe(400); + globalThis.fetch = originalFetch; }); diff --git a/src/top-monthly-sold-by-category.ts b/src/top-monthly-sold-by-category.ts index 547aa5c..4ce10c8 100644 --- a/src/top-monthly-sold-by-category.ts +++ b/src/top-monthly-sold-by-category.ts @@ -1,6 +1,8 @@ import { existsSync, mkdirSync, readFileSync } from "node:fs"; import path from "node:path"; -import { type Database, getDb, initDb } from "./database.ts"; +import { db } from "./db/index.ts"; +import { runs, categoryProductResults } from "./db/schema.ts"; +import { eq, sql } from "drizzle-orm"; import { config } from "./config.ts"; import { analyzeProducts } from "./llm.ts"; import { fetchSellabilityBatch, fetchSpApiPricingAndFees } from "./sp-api.ts"; @@ -171,36 +173,32 @@ function printUsageAndExit(message: string): never { } export async function insertCategoryRunSummary( - db: Database, summary: CategoryRunSummary, runTimestamp: string, ): Promise { - const query = ` - INSERT INTO category_analysis_runs ( - category_id, category_label, run_timestamp, - top_asins_checked, available_asins, - fba_count, fbm_count, skip_count, - status, error_message - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); - `; - const result = db.run(query, [ - summary.categoryId, - summary.categoryLabel, - runTimestamp, - summary.topAsinsChecked, - summary.availableAsins, - summary.fba, - summary.fbm, - summary.skip, - summary.status, - summary.error, - ]); - // Bun's SQLite client returns { changes: number, lastInsertRowid: number | bigint } - return Number(result.lastInsertRowid); + const [row] = await db + .insert(runs) + .values({ + type: "category_analysis", + status: (summary.status as typeof runs.$inferInsert.status) ?? "running", + categoryId: summary.categoryId, + categoryLabel: summary.categoryLabel, + topAsinsChecked: summary.topAsinsChecked, + availableAsins: summary.availableAsins, + totalProducts: summary.topAsinsChecked, + fbaCount: summary.fba, + fbmCount: summary.fbm, + skipCount: summary.skip, + errorMessage: summary.error || null, + startedAt: new Date(runTimestamp), + }) + .returning({ id: runs.id }); + + if (!row) throw new Error("Failed to insert category run."); + return row.id; } export async function updateCategoryRunSummary( - db: Database, runId: number, summary: Pick< CategoryRunSummary, @@ -213,136 +211,110 @@ export async function updateCategoryRunSummary( | "error" >, ): Promise { - db.run( - ` - UPDATE category_analysis_runs - SET - top_asins_checked = ?, - available_asins = ?, - fba_count = ?, - fbm_count = ?, - skip_count = ?, - status = ?, - error_message = ? - WHERE id = ? - `, - [ - summary.topAsinsChecked, - summary.availableAsins, - summary.fba, - summary.fbm, - summary.skip, - summary.status, - summary.error, - runId, - ], - ); + await db + .update(runs) + .set({ + topAsinsChecked: summary.topAsinsChecked, + availableAsins: summary.availableAsins, + totalProducts: summary.topAsinsChecked, + fbaCount: summary.fba, + fbmCount: summary.fbm, + skipCount: summary.skip, + status: summary.status as typeof runs.$inferInsert.status, + errorMessage: summary.error || null, + ...(summary.status !== "running" ? { completedAt: new Date() } : {}), + }) + .where(eq(runs.id, runId)); } export async function insertProductAnalysisResults( - db: Database, runId: number, results: AnalysisResult[], ): Promise { - if (results.length === 0) { - return; - } + if (results.length === 0) return; - const insertStmt = db.prepare(` - INSERT INTO product_analysis_results ( - asin, run_id, name, brand, category, unit_cost, - current_price, avg_price_90d, avg_price_90d_sheet, - selling_price_sheet, sales_rank, sales_rank_avg_90d, - seller_count, amazon_is_seller, amazon_buybox_share_pct_90d, - monthly_sold, rank_drops_30d, rank_drops_90d, - fba_fee, fbm_fee, referral_percent, can_sell, - sellability_status, sellability_reason, - verdict, confidence, reasoning, fetched_at - ) VALUES ( - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ?, ?, ?, ? - ) - ON CONFLICT(asin) DO UPDATE SET - run_id = excluded.run_id, - name = excluded.name, - brand = excluded.brand, - category = excluded.category, - unit_cost = excluded.unit_cost, - current_price = excluded.current_price, - avg_price_90d = excluded.avg_price_90d, - avg_price_90d_sheet = excluded.avg_price_90d_sheet, - selling_price_sheet = excluded.selling_price_sheet, - sales_rank = excluded.sales_rank, - sales_rank_avg_90d = excluded.sales_rank_avg_90d, - seller_count = excluded.seller_count, - amazon_is_seller = excluded.amazon_is_seller, - amazon_buybox_share_pct_90d = excluded.amazon_buybox_share_pct_90d, - monthly_sold = excluded.monthly_sold, - rank_drops_30d = excluded.rank_drops_30d, - rank_drops_90d = excluded.rank_drops_90d, - fba_fee = excluded.fba_fee, - fbm_fee = excluded.fbm_fee, - referral_percent = excluded.referral_percent, - can_sell = excluded.can_sell, - sellability_status = excluded.sellability_status, - sellability_reason = excluded.sellability_reason, - verdict = excluded.verdict, - confidence = excluded.confidence, - reasoning = excluded.reasoning, - fetched_at = excluded.fetched_at; - `); + const rows = results.map((r) => { + const price = + r.product.keepa?.currentPrice ?? + r.product.record.sellingPriceFromSheet ?? + r.product.spApi.estimatedSalePrice; + const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank; - db.transaction((resultsBatch: AnalysisResult[]) => { - for (const r of resultsBatch) { - const price = - r.product.keepa?.currentPrice ?? - r.product.record.sellingPriceFromSheet ?? - r.product.spApi.estimatedSalePrice; - const rank = r.product.keepa?.salesRank ?? r.product.record.amazonRank; - - insertStmt.run( - r.product.record.asin, - runId, - r.product.record.name, - r.product.record.brand ?? null, + return { + asin: r.product.record.asin, + runId, + name: r.product.record.name, + brand: r.product.record.brand ?? null, + category: r.product.record.category ?? - r.product.keepa?.categoryTree?.join(" > ") ?? - null, - r.product.record.unitCost ?? null, - price ?? null, - r.product.keepa?.avgPrice90 ?? null, - r.product.record.avgPrice90FromSheet ?? null, - r.product.record.sellingPriceFromSheet ?? null, - rank ?? null, - r.product.keepa?.salesRankAvg90 ?? null, - r.product.keepa?.sellerCount ?? null, - r.product.keepa?.amazonIsSeller == null - ? null - : r.product.keepa.amazonIsSeller - ? 1 - : 0, - r.product.keepa?.amazonBuyboxSharePct90d ?? null, - r.product.keepa?.monthlySold ?? null, - r.product.keepa?.salesRankDrops30 ?? null, - r.product.keepa?.salesRankDrops90 ?? null, - r.product.spApi.fbaFee ?? null, - r.product.spApi.fbmFee ?? null, - r.product.spApi.referralFeePercent ?? null, + r.product.keepa?.categoryTree?.join(" > ") ?? + null, + unitCost: r.product.record.unitCost ?? null, + currentPrice: price ?? null, + avgPrice90d: r.product.keepa?.avgPrice90 ?? null, + avgPrice90dSheet: r.product.record.avgPrice90FromSheet ?? null, + sellingPriceSheet: r.product.record.sellingPriceFromSheet ?? null, + salesRank: rank ?? null, + salesRankAvg90d: r.product.keepa?.salesRankAvg90 ?? null, + sellerCount: r.product.keepa?.sellerCount ?? null, + amazonIsSeller: r.product.keepa?.amazonIsSeller ?? null, + amazonBuyboxSharePct90d: r.product.keepa?.amazonBuyboxSharePct90d ?? null, + monthlySold: r.product.keepa?.monthlySold ?? null, + rankDrops30d: r.product.keepa?.salesRankDrops30 ?? null, + rankDrops90d: r.product.keepa?.salesRankDrops90 ?? null, + fbaFee: r.product.spApi.fbaFee ?? null, + fbmFee: r.product.spApi.fbmFee ?? null, + referralPercent: r.product.spApi.referralFeePercent ?? null, + canSell: r.product.spApi.canSell == null ? "unknown" : r.product.spApi.canSell ? "yes" : "no", - r.product.spApi.sellabilityStatus ?? null, - r.product.spApi.sellabilityReason ?? null, - r.verdict.verdict, - r.verdict.confidence, - r.verdict.reasoning ?? null, - r.product.fetchedAt, - ); - } - })(results); // Execute the transaction with the results batch + sellabilityStatus: r.product.spApi.sellabilityStatus ?? null, + sellabilityReason: r.product.spApi.sellabilityReason ?? null, + verdict: r.verdict.verdict, + confidence: r.verdict.confidence, + reasoning: r.verdict.reasoning ?? null, + fetchedAt: new Date(r.product.fetchedAt), + }; + }); + + await db + .insert(categoryProductResults) + .values(rows) + .onConflictDoUpdate({ + target: categoryProductResults.asin, + set: { + runId: sql`EXCLUDED.run_id`, + name: sql`EXCLUDED.name`, + brand: sql`EXCLUDED.brand`, + category: sql`EXCLUDED.category`, + unitCost: sql`EXCLUDED.unit_cost`, + currentPrice: sql`EXCLUDED.current_price`, + avgPrice90d: sql`EXCLUDED.avg_price_90d`, + avgPrice90dSheet: sql`EXCLUDED.avg_price_90d_sheet`, + sellingPriceSheet: sql`EXCLUDED.selling_price_sheet`, + salesRank: sql`EXCLUDED.sales_rank`, + salesRankAvg90d: sql`EXCLUDED.sales_rank_avg_90d`, + sellerCount: sql`EXCLUDED.seller_count`, + amazonIsSeller: sql`EXCLUDED.amazon_is_seller`, + amazonBuyboxSharePct90d: sql`EXCLUDED.amazon_buybox_share_pct_90d`, + monthlySold: sql`EXCLUDED.monthly_sold`, + rankDrops30d: sql`EXCLUDED.rank_drops_30d`, + rankDrops90d: sql`EXCLUDED.rank_drops_90d`, + fbaFee: sql`EXCLUDED.fba_fee`, + fbmFee: sql`EXCLUDED.fbm_fee`, + referralPercent: sql`EXCLUDED.referral_percent`, + canSell: sql`EXCLUDED.can_sell`, + sellabilityStatus: sql`EXCLUDED.sellability_status`, + sellabilityReason: sql`EXCLUDED.sellability_reason`, + verdict: sql`EXCLUDED.verdict`, + confidence: sql`EXCLUDED.confidence`, + reasoning: sql`EXCLUDED.reasoning`, + fetchedAt: sql`EXCLUDED.fetched_at`, + }, + }); } function loadCategoryBlacklist(filePath: string): Set { @@ -1067,7 +1039,6 @@ function buildEnrichedProducts( } export async function processCategory( - db: Database, runId: number, category: CategoryInfo, perCategoryTop: number, @@ -1083,7 +1054,7 @@ export async function processCategory( ); if (topAsins.length === 0) { log("info", " Keepa returned no ASINs for this category."); - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: 0, availableAsins: 0, fba: 0, @@ -1127,7 +1098,7 @@ export async function processCategory( ` Sellable ASINs: ${availableAsins.length}/${uniqueTopAsins.length}`, ); if (availableAsins.length === 0) { - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: uniqueTopAsins.length, availableAsins: 0, fba: 0, @@ -1164,7 +1135,7 @@ export async function processCategory( ); if (selectedAsins.length === 0) { - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: uniqueTopAsins.length, availableAsins: 0, fba: 0, @@ -1231,7 +1202,7 @@ export async function processCategory( }, })); - await insertProductAnalysisResults(db, runId, batchResults); + await insertProductAnalysisResults(runId, batchResults); for (const result of batchResults) { results.push(result); @@ -1244,7 +1215,7 @@ export async function processCategory( } } - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: uniqueTopAsins.length, availableAsins: selectedAsins.length, fba, @@ -1264,7 +1235,7 @@ export async function processCategory( } } - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: uniqueTopAsins.length, availableAsins: selectedAsins.length, fba, @@ -1293,10 +1264,6 @@ export async function main(): Promise { assertSpApiPrerequisites(); mkdirSync(args.outputDir, { recursive: true }); - const DB_PATH = - process.env.RESULTS_DB_PATH || path.join(process.cwd(), "db", "results.db"); - initDb(DB_PATH); - const db = getDb(DB_PATH); log("info", "Starting per-category monthly-sold pipeline"); log("info", `Marketplace: ${config.spApiMarketplaceId}`); @@ -1333,7 +1300,6 @@ export async function main(): Promise { let runId: number | undefined; try { runId = await insertCategoryRunSummary( - db, { categoryId: category.id, categoryLabel: category.label, @@ -1350,7 +1316,6 @@ export async function main(): Promise { ); categorySummary = await processCategory( - db, runId, category, args.perCategoryTop, @@ -1382,7 +1347,7 @@ export async function main(): Promise { results: [], }; if (runId) { - await updateCategoryRunSummary(db, runId, { + await updateCategoryRunSummary(runId, { topAsinsChecked: 0, availableAsins: 0, fba: 0, diff --git a/src/upc-file-analysis.ts b/src/upc-file-analysis.ts index 6d6903e..dd5e3fd 100644 --- a/src/upc-file-analysis.ts +++ b/src/upc-file-analysis.ts @@ -15,7 +15,6 @@ import { startRunInDb, type RunCounts, } from "./writer.ts"; -import { initDb, closeDb } from "./database.ts"; import { connectCache, disconnectCache } from "./cache.ts"; import { scoreSupplierProduct, resolveSupplierSalePrice } from "./supplier-scoring.ts"; import { @@ -31,7 +30,6 @@ import type { UpcLookupDetail, } from "./types.ts"; -const DB_PATH = process.env.RESULTS_DB_PATH || path.join("db", "results.db"); const DEFAULT_INPUT_BATCH_SIZE = 200; const DEFAULT_UPC_LOOKUP_BATCH_SIZE = 100; const DEFAULT_PRICING_CONCURRENCY = 5; @@ -48,7 +46,6 @@ export type UpcFileAnalysisOptions = { export type UpcFileAnalysisSummary = { runId: number; - dbPath: string; inputFile: string; outputFile?: string; processedRows: number; @@ -339,7 +336,6 @@ function summarizeSupplierResults( export async function runUpcFileAnalysis( options: UpcFileAnalysisOptions, ): Promise { - const dbPath = options.dbPath ?? DB_PATH; const inputBatchSize = Math.max( 1, options.inputBatchSize ?? DEFAULT_INPUT_BATCH_SIZE, @@ -355,8 +351,6 @@ export async function runUpcFileAnalysis( if (manageResources) { console.log("Connecting to Redis..."); await connectCache(); - console.log("Initializing SQLite database..."); - initDb(dbPath); } const unresolvedByStatus = createStatusCounter(); @@ -365,7 +359,7 @@ export async function runUpcFileAnalysis( let processedRows = 0; let matchedRows = 0; - const runId = startRunInDb(dbPath, options.inputFile, outputFile); + const runId = await startRunInDb(options.inputFile, outputFile); try { const readerSummary = await processUpcFileInBatches( @@ -481,7 +475,7 @@ export async function runUpcFileAnalysis( } } - appendSupplierResultsToRun(dbPath, runId, batchResults); + await appendSupplierResultsToRun(runId, batchResults); allResults.push(...batchResults); }, { @@ -490,7 +484,7 @@ export async function runUpcFileAnalysis( }, ); - const runCounts = refreshRunCountsInDb(dbPath, runId); + const runCounts = await refreshRunCountsInDb(runId); const exportSummary = summarizeSupplierResults(allResults, unresolvedByStatus); await writeSupplierWorkbook(outputFile, allResults, exportSummary); @@ -522,7 +516,6 @@ export async function runUpcFileAnalysis( return { runId, - dbPath, inputFile: options.inputFile, outputFile, processedRows, @@ -540,7 +533,6 @@ export async function runUpcFileAnalysis( } finally { if (manageResources) { await disconnectCache(); - closeDb(); } } } diff --git a/src/writer.ts b/src/writer.ts index 5120191..8332864 100644 --- a/src/writer.ts +++ b/src/writer.ts @@ -1,4 +1,6 @@ -import { getDb } from "./database.ts"; +import { db } from "./db/index.ts"; +import { runs, analysisResults } from "./db/schema.ts"; +import { eq, sql } from "drizzle-orm"; import type { AnalysisResult, SupplierAnalysisResult } from "./types.ts"; import { mkdirSync } from "node:fs"; import path from "node:path"; @@ -84,16 +86,15 @@ function buildRow(r: AnalysisResult) { }; } -export function writeResultsToDb( +export async function writeResultsToDb( results: AnalysisResult[], - dbPath: string, inputFile: string, outputFile: string | undefined, -): void { +): Promise { const runCounts = computeRunCountsFromResults(results); - const runId = startRunInDb(dbPath, inputFile, outputFile, runCounts); - appendResultsToRun(dbPath, runId, results); - console.log(`Results written to SQLite database for run_id: ${runId}`); + const runId = await startRunInDb(inputFile, outputFile, runCounts); + await appendResultsToRun(runId, results); + console.log(`Results written to database for run_id: ${runId}`); } export function writeResultsWorkbook( @@ -112,8 +113,7 @@ export function writeResultsWorkbook( console.log(`Results workbook written: ${outputFile}`); } -export function startRunInDb( - dbPath: string, +export async function startRunInDb( inputFile: string, outputFile: string | undefined, counts: RunCounts = { @@ -122,244 +122,181 @@ export function startRunInDb( fbmCount: 0, skipCount: 0, }, -): number { - const database = getDb(dbPath); - const timestamp = new Date().toISOString(); +): Promise { + const [row] = await db + .insert(runs) + .values({ + type: "lead_analysis", + inputFile, + outputFile: outputFile ?? null, + status: "ok", + totalProducts: counts.totalProducts, + fbaCount: counts.fbaCount, + fbmCount: counts.fbmCount, + skipCount: counts.skipCount, + startedAt: new Date(), + completedAt: new Date(), + }) + .returning({ id: runs.id }); - const insertRun = database.prepare( - `INSERT INTO runs ( - timestamp, - input_file, - output_file, - total_products, - fba_count, - fbm_count, - skip_count - ) VALUES (?, ?, ?, ?, ?, ?, ?)`, - ); - - const runInfo = insertRun.run( - timestamp, - inputFile, - outputFile ?? null, - counts.totalProducts, - counts.fbaCount, - counts.fbmCount, - counts.skipCount, - ); - - const runId = - (runInfo.changes as number) > 0 - ? (runInfo.lastInsertRowid as number) - : null; - - if (runId === null) { - throw new Error("Failed to insert run record into SQLite."); - } - - return runId; + if (!row) throw new Error("Failed to insert run record."); + return row.id; } -export function appendResultsToRun( - dbPath: string, +export async function appendResultsToRun( runId: number, results: AnalysisResult[], -): void { - if (results.length === 0) { - return; - } +): Promise { + if (results.length === 0) return; - const database = getDb(dbPath); - const insertResult = database.prepare( - `INSERT INTO results ( - run_id, asin, product_name, brand, category, unit_cost, current_price, - avg_price_90d, avg_price_90d_sheet, selling_price_sheet, sales_rank, rank_avg_90d, - sellers, amazon_is_seller, amazon_buybox_share_pct_90d, - monthly_sold, rank_drops_30d, rank_drops_90d, fba_net_sheet, - gross_profit_dollar, gross_profit_pct, net_profit_sheet, roi_sheet, moq, moq_cost, - qty_available, supplier, source_url, asin_link, promo_coupon_code, notes, lead_date, - fba_fee, fbm_fee, referral_percent, can_sell, sellability_status, sellability_reason, - verdict, confidence, reasoning, fetched_at - ) VALUES ( - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? - )`, - ); - - database.transaction(() => { - for (const r of results) { - const row = buildRow(r); - insertResult.run( - runId, - row.ASIN, - row.Name, - row.Brand, - row.Category, - row["Unit Cost"] ?? null, - row["Current Price"] ?? null, - row["Avg Price 90d"] ?? null, - row["Avg Price 90d (sheet)"] ?? null, - row["Selling Price (sheet)"] ?? null, - row["Sales Rank"] ?? null, - row["Rank Avg 90d"] ?? null, - row.Sellers ?? null, + const rows = results.map((r) => { + const row = buildRow(r); + return { + runId, + asin: row.ASIN, + productName: row.Name || null, + brand: row.Brand || null, + category: row.Category || null, + unitCost: (row["Unit Cost"] as number) ?? null, + currentPrice: (row["Current Price"] as number) || null, + avgPrice90d: (row["Avg Price 90d"] as number) || null, + avgPrice90dSheet: (row["Avg Price 90d (sheet)"] as number) || null, + sellingPriceSheet: (row["Selling Price (sheet)"] as number) || null, + salesRank: (row["Sales Rank"] as number) || null, + rankAvg90d: (row["Rank Avg 90d"] as number) || null, + sellerCount: (row.Sellers as number) || null, + amazonIsSeller: row["Amazon Is Seller"] == null ? null - : row["Amazon Is Seller"] - ? 1 - : 0, - row["Amazon Buy Box Share 90d %"] ?? null, - row["Monthly Sold"] ?? null, - row["Rank Drops 30d"] ?? null, - row["Rank Drops 90d"] ?? null, - row["FBA Net (sheet)"] ?? null, - row["Gross Profit $"] ?? null, - row["Gross Profit %"] ?? null, - row["Net Profit (sheet)"] ?? null, - row["ROI (sheet)"] ?? null, - row.MOQ ?? null, - row["MOQ Cost"] ?? null, - row["Qty Available"] ?? null, - row.Supplier ?? null, - row["Source URL"] ?? null, - row["ASIN Link"] ?? null, - row["Promo/Coupon Code"] ?? null, - row.Notes ?? null, - row["Lead Date"] ?? null, - row["FBA Fee"] ?? null, - row["FBM Fee"] ?? null, - row["Referral %"] ?? null, - row["Can Sell"], - row.Sellability, - row["Sellability Reason"] ?? null, - row.Verdict, - row.Confidence ?? null, - row.Reasoning, - r.product.fetchedAt, - ); - } - })(); + : Boolean(row["Amazon Is Seller"]), + amazonBuyboxSharePct90d: + (row["Amazon Buy Box Share 90d %"] as number) || null, + monthlySold: (row["Monthly Sold"] as number) || null, + rankDrops30d: (row["Rank Drops 30d"] as number) || null, + rankDrops90d: (row["Rank Drops 90d"] as number) || null, + fbaNetSheet: (row["FBA Net (sheet)"] as number) || null, + grossProfitDollar: (row["Gross Profit $"] as number) || null, + grossProfitPct: (row["Gross Profit %"] as number) || null, + netProfitSheet: (row["Net Profit (sheet)"] as number) || null, + roiSheet: (row["ROI (sheet)"] as number) || null, + moq: (row.MOQ as number) || null, + moqCost: (row["MOQ Cost"] as number) || null, + qtyAvailable: (row["Qty Available"] as number) || null, + supplier: row.Supplier || null, + sourceUrl: row["Source URL"] || null, + asinLink: row["ASIN Link"] || null, + promoCouponCode: row["Promo/Coupon Code"] || null, + notes: row.Notes || null, + leadDate: row["Lead Date"] || null, + fbaFee: row["FBA Fee"] ?? null, + fbmFee: row["FBM Fee"] ?? null, + referralPercent: row["Referral %"] ?? null, + canSell: row["Can Sell"], + sellabilityStatus: row.Sellability, + sellabilityReason: row["Sellability Reason"] || null, + verdict: row.Verdict, + confidence: row.Confidence ?? null, + reasoning: row.Reasoning, + fetchedAt: new Date(r.product.fetchedAt), + }; + }); + + await db.insert(analysisResults).values(rows); } -export function appendSupplierResultsToRun( - dbPath: string, +export async function appendSupplierResultsToRun( runId: number, results: SupplierAnalysisResult[], -): void { - if (results.length === 0) { - return; - } +): Promise { + if (results.length === 0) return; - const database = getDb(dbPath); - const insertResult = database.prepare( - `INSERT INTO results ( - run_id, asin, product_name, brand, category, unit_cost, current_price, - avg_price_90d, sales_rank, rank_avg_90d, sellers, - amazon_is_seller, amazon_buybox_share_pct_90d, monthly_sold, - rank_drops_30d, rank_drops_90d, upc, fba_fee, fbm_fee, - referral_percent, supplier_score, supplier_profit, supplier_margin, - supplier_roi, supplier_reason, upc_lookup_status, upc_lookup_reason, - candidate_asins, can_sell, sellability_status, sellability_reason, - verdict, confidence, reasoning, fetched_at - ) VALUES ( - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? - )`, - ); + const rows = results.map((result) => { + const keepa = result.keepa; + const spApi = result.spApi; + const asin = result.lookup.asin ?? result.record.asin ?? result.upc; + const category = + result.record.category ?? keepa?.categoryTree?.join(" > ") ?? null; + const canSell = + spApi?.canSell == null ? null : spApi.canSell ? "yes" : "no"; - database.transaction(() => { - for (const result of results) { - const keepa = result.keepa; - const spApi = result.spApi; - const asin = result.lookup.asin ?? result.record.asin ?? result.upc; - const category = - result.record.category ?? keepa?.categoryTree?.join(" > ") ?? null; - const canSell = - spApi?.canSell == null ? null : spApi.canSell ? "yes" : "no"; + return { + runId, + asin, + productName: result.record.name, + brand: result.record.brand ?? null, + category, + unitCost: result.record.unitCost || null, + currentPrice: result.score.salePrice, + avgPrice90d: keepa?.avgPrice90 ?? null, + salesRank: keepa?.salesRank ?? null, + rankAvg90d: keepa?.salesRankAvg90 ?? null, + sellerCount: keepa?.sellerCount ?? null, + amazonIsSeller: keepa?.amazonIsSeller ?? null, + amazonBuyboxSharePct90d: keepa?.amazonBuyboxSharePct90d ?? null, + monthlySold: keepa?.monthlySold ?? null, + rankDrops30d: keepa?.salesRankDrops30 ?? null, + rankDrops90d: keepa?.salesRankDrops90 ?? null, + upc: result.upc, + fbaFee: result.score.fbaFee, + fbmFee: spApi?.fbmFee ?? null, + referralPercent: spApi?.referralFeePercent ?? null, + supplierScore: result.score.score, + supplierProfit: result.score.profit, + supplierMargin: result.score.margin, + supplierRoi: result.score.roi, + supplierReason: result.score.reason, + upcLookupStatus: result.lookup.status, + upcLookupReason: result.lookup.reason ?? null, + candidateAsins: result.lookup.candidateAsins.join(","), + canSell, + sellabilityStatus: spApi?.sellabilityStatus ?? null, + sellabilityReason: spApi?.sellabilityReason ?? null, + verdict: result.score.verdict, + confidence: result.score.score, + reasoning: result.score.reason, + fetchedAt: new Date(result.fetchedAt), + }; + }); - insertResult.run( - runId, - asin, - result.record.name, - result.record.brand ?? null, - category, - result.record.unitCost || null, - result.score.salePrice, - keepa?.avgPrice90 ?? null, - keepa?.salesRank ?? null, - keepa?.salesRankAvg90 ?? null, - keepa?.sellerCount ?? null, - keepa?.amazonIsSeller == null ? null : keepa.amazonIsSeller ? 1 : 0, - keepa?.amazonBuyboxSharePct90d ?? null, - keepa?.monthlySold ?? null, - keepa?.salesRankDrops30 ?? null, - keepa?.salesRankDrops90 ?? null, - result.upc, - result.score.fbaFee, - spApi?.fbmFee ?? null, - spApi?.referralFeePercent ?? null, - result.score.score, - result.score.profit, - result.score.margin, - result.score.roi, - result.score.reason, - result.lookup.status, - result.lookup.reason ?? null, - result.lookup.candidateAsins.join(","), - canSell, - spApi?.sellabilityStatus ?? null, - spApi?.sellabilityReason ?? null, - result.score.verdict, - Math.round(result.score.score), - result.score.reason, - result.fetchedAt, - ); - } - })(); + await db.insert(analysisResults).values(rows); } -export function refreshRunCountsInDb(dbPath: string, runId: number): RunCounts { - const database = getDb(dbPath); - const stats = database - .query( - `SELECT +export async function refreshRunCountsInDb(runId: number): Promise { + const [stats] = await db.execute( + sql<{ + total: string; + fba: string | null; + fbm: string | null; + skip: string | null; + }>`SELECT COUNT(*) AS total, SUM(CASE WHEN verdict = 'FBA' THEN 1 ELSE 0 END) AS fba, SUM(CASE WHEN verdict = 'FBM' THEN 1 ELSE 0 END) AS fbm, SUM(CASE WHEN verdict = 'SKIP' THEN 1 ELSE 0 END) AS skip - FROM results - WHERE run_id = ?`, - ) - .get(runId) as { - total: number; - fba: number | null; - fbm: number | null; - skip: number | null; - }; + FROM analysis_results + WHERE run_id = ${runId}`, + ); const counts: RunCounts = { - totalProducts: stats.total ?? 0, - fbaCount: stats.fba ?? 0, - fbmCount: stats.fbm ?? 0, - skipCount: stats.skip ?? 0, + totalProducts: Number(stats?.total ?? 0), + fbaCount: Number(stats?.fba ?? 0), + fbmCount: Number(stats?.fbm ?? 0), + skipCount: Number(stats?.skip ?? 0), }; - database - .query( - `UPDATE runs - SET total_products = ?, fba_count = ?, fbm_count = ?, skip_count = ? - WHERE id = ?`, - ) - .run( - counts.totalProducts, - counts.fbaCount, - counts.fbmCount, - counts.skipCount, - runId, - ); + await db + .update(runs) + .set({ + totalProducts: counts.totalProducts, + fbaCount: counts.fbaCount, + fbmCount: counts.fbmCount, + skipCount: counts.skipCount, + }) + .where(eq(runs.id, runId)); return counts; } + export function printResults(results: AnalysisResult[]): void { const rows = results .filter((r) => r.verdict.verdict === "FBA" || r.verdict.verdict === "FBM")