feat: add Stalker products functionality with filtering, pagination, and purge option

2026-05-19 19:37:05 -04:00
parent aed0c11017
commit f6178a665c
6 changed files with 444 additions and 20 deletions
--- a/src/server.ts
+++ b/src/server.ts
@@ -71,6 +71,20 @@ type StalkerResultRecord = {
  inventory_sample_asins: string | null;
 };

+type StalkerProductRecord = {
+  runId: number;
+  started_at: string;
+  seller_id: string;
+  seller_name: string | null;
+  rating: number | null;
+  rating_count: number | null;
+  asin: string;
+  can_sell: number;
+  sellability_status: string;
+  sellability_reason: string | null;
+  last_seen_at: string;
+};
+
 const DB_PATH = process.env.RESULTS_DB_PATH || path.join("db", "results.db");
 const DEFAULT_PAGE_SIZE = 25;
 const MAX_PAGE_SIZE = 200;
@@ -799,6 +813,143 @@ function getStalkerResults(filters: URLSearchParams) {
  };
 }

+function parseStalkerProductFilters(filters: URLSearchParams) {
+  const q = filters.get("q")?.trim() || "";
+  const sellerId = filters.get("sellerId")?.trim().toUpperCase() || "";
+  const runIdRaw = filters.get("runId")?.trim() || "";
+
+  const conditions = [
+    "inv.can_sell = 1",
+    "inv.sellability_status = 'available'",
+  ];
+  const params: Array<string | number> = [];
+
+  if (runIdRaw) {
+    const runId = Number(runIdRaw);
+    if (Number.isInteger(runId) && runId > 0) {
+      conditions.push("r.id = ?");
+      params.push(runId);
+    }
+  }
+
+  if (sellerId) {
+    conditions.push("s.seller_id = ?");
+    params.push(sellerId);
+  }
+
+  if (q) {
+    const wildcard = `%${q}%`;
+    conditions.push(
+      "(inv.asin LIKE ? OR s.seller_id LIKE ? OR s.seller_name LIKE ?)",
+    );
+    params.push(wildcard, wildcard, wildcard);
+  }
+
+  return {
+    where: `WHERE ${conditions.join(" AND ")}`,
+    params,
+  };
+}
+
+function parseStalkerProductSort(sortParam: string | null): string {
+  const allowedSort = new Set([
+    "runId",
+    "started_at",
+    "seller_id",
+    "seller_name",
+    "rating",
+    "rating_count",
+    "asin",
+    "last_seen_at",
+  ]);
+  return parseSort(sortParam, allowedSort, "last_seen_at DESC, asin ASC");
+}
+
+function getStalkerProducts(filters: URLSearchParams) {
+  const page = parseIntParam(filters.get("page"), 1);
+  const pageSize = Math.min(
+    parseIntParam(filters.get("pageSize"), DEFAULT_PAGE_SIZE),
+    MAX_PAGE_SIZE,
+  );
+  const offset = (page - 1) * pageSize;
+  const { where, params } = parseStalkerProductFilters(filters);
+  const orderBy = parseStalkerProductSort(filters.get("sort"));
+
+  const baseSelect = `
+    SELECT
+      r.id AS runId,
+      r.started_at,
+      s.seller_id,
+      s.seller_name,
+      s.rating,
+      s.rating_count,
+      inv.asin,
+      inv.can_sell,
+      inv.sellability_status,
+      inv.sellability_reason,
+      inv.last_seen_at
+    FROM stalker_seller_inventory inv
+    JOIN stalker_runs r ON r.id = inv.run_id
+    JOIN stalker_sellers s ON s.seller_id = inv.seller_id
+    ${where}
+  `;
+
+  const totalRow = db
+    .query(`SELECT COUNT(*) AS total FROM (${baseSelect}) stalker_products`)
+    .get(...params) as { total: number };
+
+  const summary = db
+    .query(
+      `SELECT
+        COUNT(DISTINCT runId) AS runs,
+        COUNT(DISTINCT seller_id) AS sellers,
+        COUNT(DISTINCT asin) AS products
+      FROM (${baseSelect}) stalker_products`,
+    )
+    .get(...params) as {
+    runs: number;
+    sellers: number;
+    products: number;
+  };
+
+  const items = db
+    .query(
+      `SELECT * FROM (${baseSelect}) stalker_products
+       ORDER BY ${orderBy}
+       LIMIT ? OFFSET ?`,
+    )
+    .all(...params, pageSize, offset) as StalkerProductRecord[];
+
+  return {
+    items,
+    summary,
+    page,
+    pageSize,
+    total: totalRow.total,
+    totalPages: Math.max(1, Math.ceil(totalRow.total / pageSize)),
+  };
+}
+
+function purgeStalkerData() {
+  const counts = {
+    inventory: (db.query("SELECT COUNT(*) AS count FROM stalker_seller_inventory").get() as { count: number }).count,
+    asinSellers: (db.query("SELECT COUNT(*) AS count FROM stalker_asin_sellers").get() as { count: number }).count,
+    sellers: (db.query("SELECT COUNT(*) AS count FROM stalker_sellers").get() as { count: number }).count,
+    scans: (db.query("SELECT COUNT(*) AS count FROM stalker_asin_scans").get() as { count: number }).count,
+    runs: (db.query("SELECT COUNT(*) AS count FROM stalker_runs").get() as { count: number }).count,
+  };
+
+  db.transaction(() => {
+    db.run("DELETE FROM stalker_seller_inventory");
+    db.run("DELETE FROM stalker_asin_sellers");
+    db.run("DELETE FROM stalker_sellers");
+    db.run("DELETE FROM stalker_asin_scans");
+    db.run("DELETE FROM stalker_runs");
+  })();
+
+  return { ok: true, deleted: counts };
+}
+
 function getRun(processType: ProcessType, runId: number) {
  if (processType === "lead_analysis") {
    const run = db
@@ -1430,6 +1581,7 @@ const server = Bun.serve({
    "/": index,
    "/products": index,
    "/stalker": index,
+    "/stalker/products": index,
    "/runs/:processType/:runId": index,
    "/api/runs": (req) => {
      const url = new URL(req.url);
@@ -1443,6 +1595,16 @@ const server = Bun.serve({
      const url = new URL(req.url);
      return json(getStalkerResults(url.searchParams));
    },
+    "/api/stalker/products": (req) => {
+      const url = new URL(req.url);
+      return json(getStalkerProducts(url.searchParams));
+    },
+    "/api/stalker/purge": (req) => {
+      if (req.method !== "DELETE" && req.method !== "POST") {
+        return json({ error: "Method not allowed" }, 405);
+      }
+      return json(purgeStalkerData());
+    },
    "/api/upc/map": async (req) => {
      let upcs: string[];
      try {
--- a/src/sp-api.ts
+++ b/src/sp-api.ts
@@ -123,7 +123,10 @@ function round2(value: number): number {
  return Math.round(value * 100) / 100;
 }

-const SELLABILITY_CONCURRENCY = 5;
+const LISTINGS_RESTRICTIONS_RATE_LIMIT_PER_SECOND = 5;
+const LISTINGS_RESTRICTIONS_BURST_REQUESTS = 10;
+const SELLABILITY_CONCURRENCY = LISTINGS_RESTRICTIONS_RATE_LIMIT_PER_SECOND;
+const SELLABILITY_PROGRESS_INTERVAL = LISTINGS_RESTRICTIONS_BURST_REQUESTS;
 const PRICING_CONCURRENCY = 5;
 const UPC_PATTERN = /^\d{12,14}$/;

@@ -621,7 +624,6 @@ export async function fetchSellabilityBatch(
  }

  let completed = 0;
-  let running = 0;
  const queue = [...asins];

  async function next(): Promise<void> {
@@ -630,7 +632,10 @@ export async function fetchSellabilityBatch(
      const info = await fetchSellabilityInternal(spClient!, asin);
      results.set(asin, info);
      completed++;
-      if (completed % 10 === 0 || completed === asins.length) {
+      if (
+        completed % SELLABILITY_PROGRESS_INTERVAL === 0 ||
+        completed === asins.length
+      ) {
        console.log(`  [sellability] ${completed}/${asins.length} checked`);
      }
    }
--- a/src/stalker.test.ts
+++ b/src/stalker.test.ts
@@ -222,7 +222,7 @@ test("runStalker fetches product offers, filters sellers, and persists storefron
  expect(stats.scannedAsins).toBe(1);
  expect(stats.sourceAsinsWithMatches).toBe(1);
  expect(stats.matchedSellers).toBe(1);
-  expect(stats.persistedInventoryAsins).toBe(2);
+  expect(stats.persistedInventoryAsins).toBe(0);
  expect(stats.failedAsins).toBe(0);
  expect(stats.candidateSellers).toBe(2);
  expect(stats.qualifyingSellers).toBe(1);
@@ -253,7 +253,7 @@ test("runStalker fetches product offers, filters sellers, and persists storefron
  expect(run.inventory_sellability_checked_asins).toBe(0);
  expect(run.inventory_sellability_available_asins).toBe(0);
  expect(run.inventory_sellability_excluded_asins).toBe(0);
-  expect(run.persisted_inventory_asins).toBe(2);
+  expect(run.persisted_inventory_asins).toBe(0);

  const scan = db.query("SELECT * FROM stalker_asin_scans").get() as any;
  expect(scan.source_asin).toBe("B000000001");
@@ -267,7 +267,7 @@ test("runStalker fetches product offers, filters sellers, and persists storefron
  expect(sellers[0].seller_id).toBe("AQUALIFIED");
  expect(sellers[0].rating_count).toBe(12);
  expect(sellers[0].storefront_asin_total).toBe(2);
-  expect(sellers[0].persisted_inventory_sample_count).toBe(2);
+  expect(sellers[0].persisted_inventory_sample_count).toBe(0);

  const asinSellers = db.query("SELECT * FROM stalker_asin_sellers").all() as any[];
  expect(asinSellers.length).toBe(1);
@@ -278,8 +278,5 @@ test("runStalker fetches product offers, filters sellers, and persists storefron
  const inventory = db
    .query("SELECT asin FROM stalker_seller_inventory ORDER BY asin")
    .all() as Array<{ asin: string }>;
-  expect(inventory.map((row) => row.asin)).toEqual([
-    "B111111111",
-    "B222222222",
-  ]);
+  expect(inventory.map((row) => row.asin)).toEqual([]);
 });
--- a/src/stalker.ts
+++ b/src/stalker.ts
@@ -338,6 +338,7 @@ export async function runStalker(args: StalkerArgs): Promise<StalkerRunStats> {
      if (args.sellability && !args.dryRun) {
        await enrichInventorySellability(result, stats);
      }
+      applyInventoryPersistencePolicy(result, args.sellability && !args.dryRun);

      if (!args.dryRun && runId != null) {
        persistAsinResult(database, runId, result);
@@ -457,6 +458,22 @@ async function scanAsin(
  };
 }

+function applyInventoryPersistencePolicy(
+  result: StalkerAsinResult,
+  requireAvailableSellability: boolean,
+): void {
+  for (const { seller } of result.matchedSellers) {
+    seller.storefrontItems = seller.storefrontItems.filter((item) => {
+      if (!requireAvailableSellability) return false;
+      return (
+        item.sellability?.canSell === true &&
+        item.sellability.sellabilityStatus === "available"
+      );
+    });
+    seller.storefrontAsins = seller.storefrontItems.map((item) => item.asin);
+  }
+}
+
 async function enrichInventorySellability(
  result: StalkerAsinResult,
  stats: StalkerRunStats,
@@ -833,6 +850,13 @@ function upsertSellerInventory(
  );

  for (const item of seller.storefrontItems) {
+    if (
+      item.sellability?.canSell !== true ||
+      item.sellability.sellabilityStatus !== "available"
+    ) {
+      continue;
+    }
+
    insert.run(
      runId,
      seller.sellerId,
--- a/src/web/frontend.tsx
+++ b/src/web/frontend.tsx
@@ -140,6 +140,33 @@ type StalkerResultsResponse = {
  totalPages: number;
 };

+type StalkerProductItem = {
+  runId: number;
+  started_at: string;
+  seller_id: string;
+  seller_name: string | null;
+  rating: number | null;
+  rating_count: number | null;
+  asin: string;
+  can_sell: number;
+  sellability_status: string;
+  sellability_reason: string | null;
+  last_seen_at: string;
+};
+
+type StalkerProductsResponse = {
+  items: StalkerProductItem[];
+  summary: {
+    runs: number;
+    sellers: number;
+    products: number;
+  };
+  page: number;
+  pageSize: number;
+  total: number;
+  totalPages: number;
+};
+
 type SortState = {
  field: string;
  direction: SortDirection;
@@ -234,10 +261,12 @@ function Dashboard({
  onOpenRun,
  onOpenProducts,
  onOpenStalker,
+  onOpenStalkerProducts,
 }: {
  onOpenRun: (run: Run) => void;
  onOpenProducts: (verdict: VerdictFilter) => void;
  onOpenStalker: () => void;
+  onOpenStalkerProducts: () => void;
 }) {
  const [runs, setRuns] = useState<RunsResponse | null>(null);
  const [loading, setLoading] = useState(false);
@@ -328,7 +357,10 @@ function Dashboard({
      <div className="card">
        <div className="section-header">
          <h2>Runs Dashboard</h2>
-          <button onClick={onOpenStalker}>Stalker results</button>
+          <div className="button-row">
+            <button onClick={onOpenStalker}>Stalker sellers</button>
+            <button onClick={onOpenStalkerProducts}>Sellable products</button>
+          </div>
        </div>
      </div>

@@ -889,9 +921,16 @@ function ProductList({ verdict, onBack }: { verdict: VerdictFilter; onBack: () =
  );
 }

-function StalkerExplorer({ onBack }: { onBack: () => void }) {
+function StalkerExplorer({
+  onBack,
+  onOpenProducts,
+}: {
+  onBack: () => void;
+  onOpenProducts: () => void;
+}) {
  const [results, setResults] = useState<StalkerResultsResponse | null>(null);
  const [loading, setLoading] = useState(false);
+  const [purging, setPurging] = useState(false);
  const [search, setSearch] = useState("");
  const [sellerId, setSellerId] = useState("");
  const [runId, setRunId] = useState("");
@@ -900,6 +939,7 @@ function StalkerExplorer({ onBack }: { onBack: () => void }) {
  const [page, setPage] = useState(1);
  const [pageSize, setPageSize] = useState(25);
  const [sort, setSort] = useState<SortState>({ field: "persisted_inventory_asin_count", direction: "DESC" });
+  const [refreshTick, setRefreshTick] = useState(0);

  useEffect(() => {
    let cancelled = false;
@@ -928,14 +968,41 @@ function StalkerExplorer({ onBack }: { onBack: () => void }) {
    return () => {
      cancelled = true;
    };
-  }, [search, sellerId, runId, minRatingCount, maxRatingCount, page, pageSize, sort]);
+  }, [search, sellerId, runId, minRatingCount, maxRatingCount, page, pageSize, sort, refreshTick]);
+
+  async function purgeStalkerData() {
+    const confirmed = window.confirm("Permanently delete all Stalker runs, sellers, and sellable products from the database?");
+    if (!confirmed) return;
+
+    setPurging(true);
+    try {
+      const res = await fetch("/api/stalker/purge", { method: "DELETE" });
+      if (!res.ok) {
+        const payload = await res.json().catch(() => null) as { error?: string } | null;
+        window.alert(payload?.error ?? "Failed to purge Stalker data");
+        return;
+      }
+      setPage(1);
+      setRefreshTick((tick) => tick + 1);
+    } finally {
+      setPurging(false);
+    }
+  }

  return (
    <div className="page">
      <button className="back" onClick={onBack}>Back</button>

      <div className="card">
-        <h2>Seller Storefronts</h2>
+        <div className="section-header">
+          <h2>Seller Storefronts</h2>
+          <div className="button-row">
+            <button onClick={onOpenProducts}>Sellable products</button>
+            <button className="danger" disabled={purging} onClick={purgeStalkerData}>
+              {purging ? "Purging..." : "Purge Stalker data"}
+            </button>
+          </div>
+        </div>
      </div>

      <div className="metrics">
@@ -1032,11 +1099,142 @@ function StalkerExplorer({ onBack }: { onBack: () => void }) {
  );
 }

+function StalkerProductsExplorer({
+  onBack,
+  onOpenSellers,
+}: {
+  onBack: () => void;
+  onOpenSellers: () => void;
+}) {
+  const [results, setResults] = useState<StalkerProductsResponse | null>(null);
+  const [loading, setLoading] = useState(false);
+  const [search, setSearch] = useState("");
+  const [sellerId, setSellerId] = useState("");
+  const [runId, setRunId] = useState("");
+  const [page, setPage] = useState(1);
+  const [pageSize, setPageSize] = useState(50);
+  const [sort, setSort] = useState<SortState>({ field: "last_seen_at", direction: "DESC" });
+
+  useEffect(() => {
+    let cancelled = false;
+    async function load() {
+      setLoading(true);
+      const params = new URLSearchParams({
+        page: String(page),
+        pageSize: String(pageSize),
+        sort: buildSortValue(sort),
+      });
+      if (search) params.set("q", search);
+      if (sellerId) params.set("sellerId", sellerId);
+      if (runId) params.set("runId", runId);
+
+      const res = await fetch(`/api/stalker/products?${params.toString()}`);
+      const payload = (await res.json()) as StalkerProductsResponse;
+      if (!cancelled) {
+        setResults(payload);
+        setLoading(false);
+      }
+    }
+
+    load();
+    return () => {
+      cancelled = true;
+    };
+  }, [search, sellerId, runId, page, pageSize, sort]);
+
+  return (
+    <div className="page">
+      <button className="back" onClick={onBack}>Back</button>
+
+      <div className="card">
+        <div className="section-header">
+          <h2>Sellable Stalker Products</h2>
+          <button onClick={onOpenSellers}>Seller storefronts</button>
+        </div>
+      </div>
+
+      <div className="metrics">
+        <div className="metric">
+          <div className="label">Runs</div>
+          <div className="value">{formatNumber(results?.summary.runs)}</div>
+        </div>
+        <div className="metric">
+          <div className="label">Sellers</div>
+          <div className="value">{formatNumber(results?.summary.sellers)}</div>
+        </div>
+        <div className="metric">
+          <div className="label">Sellable products</div>
+          <div className="value">{formatNumber(results?.summary.products)}</div>
+        </div>
+      </div>
+
+      <div className="card">
+        <div className="toolbar">
+          <input value={search} onChange={(e) => { setPage(1); setSearch(e.target.value); }} placeholder="Search ASIN or seller" />
+          <input value={sellerId} onChange={(e) => { setPage(1); setSellerId(e.target.value.toUpperCase()); }} placeholder="Seller ID" />
+          <input value={runId} onChange={(e) => { setPage(1); setRunId(e.target.value); }} placeholder="Run ID" />
+          <select value={String(pageSize)} onChange={(e) => { setPage(1); setPageSize(Number(e.target.value)); }}>
+            <option value="25">25 / page</option>
+            <option value="50">50 / page</option>
+            <option value="100">100 / page</option>
+          </select>
+        </div>
+      </div>
+
+      <div className="card">
+        <div className="table-wrap">
+          <table className="stalker-table">
+            <thead>
+              <tr>
+                <th><button onClick={() => setSort(nextSort(sort, "asin"))}>ASIN</button></th>
+                <th><button onClick={() => setSort(nextSort(sort, "seller_id"))}>Seller ID</button></th>
+                <th><button onClick={() => setSort(nextSort(sort, "seller_name"))}>Seller</button></th>
+                <th><button onClick={() => setSort(nextSort(sort, "rating_count"))}>Rating Count</button></th>
+                <th>Status</th>
+                <th><button onClick={() => setSort(nextSort(sort, "runId"))}>Run</button></th>
+                <th><button onClick={() => setSort(nextSort(sort, "last_seen_at"))}>Last Seen</button></th>
+              </tr>
+            </thead>
+            <tbody>
+              {loading ? (
+                <tr><td colSpan={7}>Loading...</td></tr>
+              ) : results?.items.length ? (
+                results.items.map((item) => (
+                  <tr key={`${item.runId}-${item.seller_id}-${item.asin}`}>
+                    <td><a href={`http://amazon.com/dp/${item.asin}`} target="_blank" rel="noreferrer">{item.asin}</a></td>
+                    <td>{item.seller_id}</td>
+                    <td>{item.seller_name || "-"}</td>
+                    <td>{formatNumber(item.rating_count)}</td>
+                    <td><span className="badge badge-ok">{item.sellability_status}</span></td>
+                    <td>{item.runId}</td>
+                    <td>{formatDate(item.last_seen_at)}</td>
+                  </tr>
+                ))
+              ) : (
+                <tr><td colSpan={7}>No sellable Stalker products found</td></tr>
+              )}
+            </tbody>
+          </table>
+        </div>
+        <div className="pager">
+          <div>Showing {results?.items.length ?? 0} of {results?.total ?? 0}</div>
+          <div>
+            <button disabled={page <= 1} onClick={() => setPage((p) => Math.max(1, p - 1))}>Previous</button>
+            <span style={{ padding: "0 8px" }}>Page {results?.page ?? page} / {results?.totalPages ?? 1}</span>
+            <button disabled={Boolean(results && page >= results.totalPages)} onClick={() => setPage((p) => p + 1)}>Next</button>
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+}
+
 type AppRoute =
  | { kind: "dashboard" }
  | { kind: "run"; processType: ProcessType; runId: number }
  | { kind: "products"; verdict: VerdictFilter }
-  | { kind: "stalker" };
+  | { kind: "stalker" }
+  | { kind: "stalker-products" };

 function parseRoute(pathname: string, search: string): AppRoute {
  const runMatch = pathname.match(/^\/runs\/(lead_analysis|category_analysis)\/(\d+)$/);
@@ -1055,6 +1253,10 @@ function parseRoute(pathname: string, search: string): AppRoute {
    return { kind: "stalker" };
  }

+  if (pathname === "/stalker/products") {
+    return { kind: "stalker-products" };
+  }
+
  return { kind: "dashboard" };
 }

@@ -1084,6 +1286,11 @@ function App() {
    setRoute({ kind: "stalker" });
  }

+  function openStalkerProducts() {
+    history.pushState({}, "", "/stalker/products");
+    setRoute({ kind: "stalker-products" });
+  }
+
  function backToDashboard() {
    history.pushState({}, "", "/");
    setRoute({ kind: "dashboard" });
@@ -1098,10 +1305,21 @@ function App() {
  }

  if (route.kind === "stalker") {
-    return <StalkerExplorer onBack={backToDashboard} />;
+    return <StalkerExplorer onBack={backToDashboard} onOpenProducts={openStalkerProducts} />;
  }

-  return <Dashboard onOpenRun={openRun} onOpenProducts={openProducts} onOpenStalker={openStalker} />;
+  if (route.kind === "stalker-products") {
+    return <StalkerProductsExplorer onBack={backToDashboard} onOpenSellers={openStalker} />;
+  }
+
+  return (
+    <Dashboard
+      onOpenRun={openRun}
+      onOpenProducts={openProducts}
+      onOpenStalker={openStalker}
+      onOpenStalkerProducts={openStalkerProducts}
+    />
+  );
 }

 const root = document.getElementById("root");
--- a/src/web/styles.css
+++ b/src/web/styles.css
@@ -48,6 +48,13 @@ p {
  gap: 12px;
 }

+.button-row {
+  display: flex;
+  align-items: center;
+  gap: 8px;
+  flex-wrap: wrap;
+}
+
 .toolbar input,
 .toolbar select,
 button {
@@ -63,6 +70,17 @@ button {
  cursor: pointer;
 }

+button.danger {
+  border-color: #efb8b8;
+  color: #9f1c1c;
+  background: #fff6f6;
+}
+
+button:disabled {
+  cursor: not-allowed;
+  opacity: 0.58;
+}
+
 .table-wrap {
  overflow: auto;
  border: 1px solid #eceef0;