feat: add supplier scoring and UPC file analysis functionality
- Implemented supplier scoring logic in `supplier-scoring.ts` with functions to compute demand score, competition penalty, and overall supplier product score. - Created unit tests for supplier scoring in `supplier-scoring.test.ts` to validate scoring logic against various scenarios. - Developed UPC file analysis tool in `upc-file-analysis.ts` to process UPCs in batches, fetch product data from Keepa and SP-API, and generate supplier results. - Added UPC input reading functionality in `upc-file-reader.ts` to handle XLSX and XLS files, including validation for UPC formats. - Introduced a command-line tool in `upc-lookup.ts` for looking up UPCs and displaying detailed results or mappings to ASINs. - Enhanced error handling and logging throughout the new modules for better traceability and user feedback.
This commit is contained in:
Victor Noguera
350
src/integrations/searxng.test.ts
Normal file
350
src/integrations/searxng.test.ts
Normal file
@@ -0,0 +1,350 @@
|
||||
import { afterAll, beforeEach, expect, mock, test } from "bun:test";
|
||||
import { normalizeAsin, searchProductOffers } from "./searxng.ts";
|
||||
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
beforeEach(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
});
|
||||
|
||||
test("normalizeAsin uppercases and validates ASINs", () => {
|
||||
expect(normalizeAsin(" b07sn9bhvv ")).toBe("B07SN9BHVV");
|
||||
expect(() => normalizeAsin("not-an-asin")).toThrow("Invalid ASIN");
|
||||
});
|
||||
|
||||
test("searchProductOffers derives ASIN search behavior for ASIN-only queries", async () => {
|
||||
const fetchMock = mock(async (input: string | URL | Request) => {
|
||||
const url = input instanceof URL ? input : new URL(String(input));
|
||||
expect(url.pathname).toBe("/search");
|
||||
expect(url.searchParams.get("format")).toBe("json");
|
||||
expect(url.searchParams.get("q")).toBe("B07SN9BHVV price sale offer buy online");
|
||||
|
||||
return Response.json({
|
||||
results: [
|
||||
{
|
||||
title: "Amazon listing B07SN9BHVV",
|
||||
url: "https://www.amazon.com/dp/B07SN9BHVV",
|
||||
content: "Official marketplace listing.",
|
||||
engines: ["duckduckgo"],
|
||||
},
|
||||
{
|
||||
title: "Romand palette offer",
|
||||
url: "https://example-shop.com/item",
|
||||
content: "Buy product ASIN B07SN9BHVV. Offer price: $12.99 today.",
|
||||
engines: ["brave"],
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
const results = await searchProductOffers("B07SN9BHVV", {
|
||||
provider: "searxng",
|
||||
baseUrl: "https://searxng.test/",
|
||||
fetchImpl: fetchMock as unknown as typeof fetch,
|
||||
maxResults: 10,
|
||||
});
|
||||
|
||||
expect(results).toHaveLength(2);
|
||||
expect(results[0]?.domain).toBe("example-shop.com");
|
||||
expect(results[0]?.matchedAsin).toBe("B07SN9BHVV");
|
||||
expect(results[0]?.detectedPrice).toBe(12.99);
|
||||
expect(results[0]?.detectedPriceCurrency).toBe("USD");
|
||||
expect(results[0]?.detectedPriceLabel).toBe("offer price");
|
||||
expect(results[0]?.detectedPriceText).toBe("$12.99");
|
||||
expect(results[0]?.engines).toEqual(["brave"]);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test("searchProductOffers falls back to HTML when JSON is unavailable", async () => {
|
||||
const html = `
|
||||
<article class="result result-default category-general">
|
||||
<a class="url_header" href="https://supplier.example/products/romand"></a>
|
||||
<h3><a href="https://supplier.example/products/romand">Supplier offer B07SN9BHVV</a></h3>
|
||||
<p class="content">Wholesale product sale price: USD 9.50 with ASIN B07SN9BHVV.</p>
|
||||
<div class="engines"><span>duckduckgo</span></div>
|
||||
</article>
|
||||
`;
|
||||
const fetchMock = mock(async (input: string | URL | Request) => {
|
||||
const url = input instanceof URL ? input : new URL(String(input));
|
||||
if (url.searchParams.get("format") === "json") {
|
||||
return new Response("forbidden", { status: 403 });
|
||||
}
|
||||
return new Response(html, {
|
||||
status: 200,
|
||||
headers: { "content-type": "text/html" },
|
||||
});
|
||||
});
|
||||
|
||||
const results = await searchProductOffers("B07SN9BHVV", {
|
||||
provider: "searxng",
|
||||
baseUrl: "https://searxng.test/",
|
||||
fetchImpl: fetchMock as unknown as typeof fetch,
|
||||
});
|
||||
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0]?.title).toBe("Supplier offer B07SN9BHVV");
|
||||
expect(results[0]?.domain).toBe("supplier.example");
|
||||
expect(results[0]?.detectedPrice).toBe(9.5);
|
||||
expect(results[0]?.detectedPriceLabel).toBe("sale price");
|
||||
expect(results[0]?.detectedPriceText).toBe("USD 9.50");
|
||||
expect(results[0]?.matchedAsin).toBe("B07SN9BHVV");
|
||||
expect(results[0]?.engines).toEqual(["duckduckgo"]);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
test("searchProductOffers detects common selling and sale price formats", async () => {
|
||||
const fetchMock = mock(async () =>
|
||||
Response.json({
|
||||
results: [
|
||||
{
|
||||
title: "Supplier page",
|
||||
url: "https://supplier.example/item",
|
||||
content: "Selling price is €18.75 and list price is $24.00.",
|
||||
},
|
||||
{
|
||||
title: "Backup page",
|
||||
url: "https://backup.example/item",
|
||||
content: "Available now for 22.10 USD.",
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
const results = await searchProductOffers("romand palette price", {
|
||||
provider: "searxng",
|
||||
baseUrl: "https://searxng.test/",
|
||||
fetchImpl: fetchMock as unknown as typeof fetch,
|
||||
maxResults: 2,
|
||||
});
|
||||
|
||||
expect(results[0]?.detectedPrice).toBe(18.75);
|
||||
expect(results[0]?.detectedPriceCurrency).toBe("EUR");
|
||||
expect(results[0]?.detectedPriceLabel).toBe("selling price");
|
||||
expect(results[1]?.detectedPrice).toBe(22.1);
|
||||
expect(results[1]?.detectedPriceCurrency).toBe("USD");
|
||||
});
|
||||
|
||||
test("searchProductOffers filters unrelated priced results for ASIN-only queries", async () => {
|
||||
const fetchMock = mock(async () =>
|
||||
Response.json({
|
||||
results: [
|
||||
{
|
||||
title: "Unrelated deal",
|
||||
url: "https://deals.example/phones",
|
||||
content: "This price is $449 but it is for another product.",
|
||||
},
|
||||
{
|
||||
title: "Amazon listing B07SN9BHVV",
|
||||
url: "https://www.amazon.in/dp/B07SN9BHVV",
|
||||
content: "1 offer from ₹550.00 · Buying options.",
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
const results = await searchProductOffers("B07SN9BHVV", {
|
||||
provider: "searxng",
|
||||
baseUrl: "https://searxng.test/",
|
||||
fetchImpl: fetchMock as unknown as typeof fetch,
|
||||
});
|
||||
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0]?.matchedAsin).toBe("B07SN9BHVV");
|
||||
expect(results[0]?.detectedPrice).toBe(550);
|
||||
expect(results[0]?.detectedPriceCurrency).toBe("INR");
|
||||
expect(results[0]?.detectedPriceText).toBe("₹550.00");
|
||||
});
|
||||
|
||||
test("searchProductOffers keeps arbitrary query strings generic", async () => {
|
||||
const fetchMock = mock(async (input: string | URL | Request) => {
|
||||
const url = input instanceof URL ? input : new URL(String(input));
|
||||
expect(url.searchParams.get("q")).toBe("romand dry mango tulip price");
|
||||
|
||||
return Response.json({
|
||||
results: [
|
||||
{
|
||||
title: "Generic result",
|
||||
url: "https://shop.example/romand",
|
||||
content: "Sale price: $14.25",
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
const results = await searchProductOffers("romand dry mango tulip price", {
|
||||
provider: "searxng",
|
||||
baseUrl: "https://searxng.test/",
|
||||
fetchImpl: fetchMock as unknown as typeof fetch,
|
||||
});
|
||||
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0]?.asin).toBeUndefined();
|
||||
expect(results[0]?.detectedPrice).toBe(14.25);
|
||||
});
|
||||
|
||||
test("searchProductOffers sends configured categories", async () => {
|
||||
const fetchMock = mock(async (input: string | URL | Request) => {
|
||||
const url = input instanceof URL ? input : new URL(String(input));
|
||||
expect(url.searchParams.get("categories")).toBe("shopping");
|
||||
|
||||
return Response.json({
|
||||
results: [
|
||||
{
|
||||
title: "Shopping result",
|
||||
url: "https://shop.example/item",
|
||||
content: "Offer price: $10.00",
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
const results = await searchProductOffers("romand price", {
|
||||
provider: "searxng",
|
||||
baseUrl: "https://searxng.test/",
|
||||
categories: "shopping",
|
||||
fetchImpl: fetchMock as unknown as typeof fetch,
|
||||
});
|
||||
|
||||
expect(results[0]?.detectedPrice).toBe(10);
|
||||
});
|
||||
|
||||
test("searchProductOffers sends configured SearXNG engines", async () => {
|
||||
const fetchMock = mock(async (input: string | URL | Request) => {
|
||||
const url = input instanceof URL ? input : new URL(String(input));
|
||||
expect(url.searchParams.get("engines")).toBe("google");
|
||||
expect(url.searchParams.get("q")).toBe("!go romand price");
|
||||
|
||||
return Response.json({
|
||||
results: [
|
||||
{
|
||||
title: "Google-backed result",
|
||||
url: "https://shop.example/item",
|
||||
content: "Offer price: $11.00",
|
||||
engine: "google",
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
const results = await searchProductOffers("romand price", {
|
||||
provider: "searxng",
|
||||
baseUrl: "https://searxng.test/",
|
||||
engines: "google",
|
||||
fetchImpl: fetchMock as unknown as typeof fetch,
|
||||
});
|
||||
|
||||
expect(results[0]?.detectedPrice).toBe(11);
|
||||
expect(results[0]?.engines).toEqual(["google"]);
|
||||
});
|
||||
|
||||
test("searchProductOffers uses Google Custom Search API and pagemap offer prices", async () => {
|
||||
const fetchMock = mock(async (input: string | URL | Request) => {
|
||||
const url = input instanceof URL ? input : new URL(String(input));
|
||||
expect(url.hostname).toBe("googleapis.test");
|
||||
expect(url.searchParams.get("key")).toBe("test-key");
|
||||
expect(url.searchParams.get("cx")).toBe("test-cx");
|
||||
expect(url.searchParams.get("num")).toBe("5");
|
||||
expect(url.searchParams.get("q")).toBe("romand dry mango tulip");
|
||||
|
||||
return Response.json({
|
||||
items: [
|
||||
{
|
||||
title: "Romand Dry Mango Tulip",
|
||||
link: "https://store.example/romand",
|
||||
snippet: "Buy from Store Example.",
|
||||
pagemap: {
|
||||
offer: [{ price: "12.50", pricecurrency: "USD" }],
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
const results = await searchProductOffers("romand dry mango tulip", {
|
||||
provider: "google-custom-search",
|
||||
baseUrl: "https://googleapis.test/customsearch/v1",
|
||||
googleApiKey: "test-key",
|
||||
googleCx: "test-cx",
|
||||
maxResults: 5,
|
||||
fetchImpl: fetchMock as unknown as typeof fetch,
|
||||
});
|
||||
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0]?.title).toContain("Romand Dry Mango Tulip");
|
||||
expect(results[0]?.domain).toBe("store.example");
|
||||
expect(results[0]?.detectedPrice).toBe(12.5);
|
||||
expect(results[0]?.detectedPriceLabel).toBe("offer price");
|
||||
expect(results[0]?.engines).toEqual(["google custom search"]);
|
||||
});
|
||||
|
||||
test("searchProductOffers defaults to SerpApi Google Shopping results", async () => {
|
||||
const fetchMock = mock(async (input: string | URL | Request) => {
|
||||
const url = input instanceof URL ? input : new URL(String(input));
|
||||
expect(url.hostname).toBe("serpapi.test");
|
||||
expect(url.searchParams.get("engine")).toBe("google_shopping");
|
||||
expect(url.searchParams.get("q")).toBe("dry mango tulip price");
|
||||
expect(url.searchParams.get("api_key")).toBe("serpapi-key");
|
||||
expect(url.searchParams.get("gl")).toBe("us");
|
||||
expect(url.searchParams.get("hl")).toBe("en");
|
||||
|
||||
return Response.json({
|
||||
shopping_results: [
|
||||
{
|
||||
position: 1,
|
||||
title: "Romand Better Than Eyes Dry Mango Tulip",
|
||||
source: "K-Beauty Store",
|
||||
link: "https://store.example/products/romand",
|
||||
price: "$13.40",
|
||||
extracted_price: 13.4,
|
||||
delivery: "$4.99 delivery",
|
||||
rating: 4.7,
|
||||
reviews: 128,
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
const results = await searchProductOffers("dry mango tulip price", {
|
||||
baseUrl: "https://serpapi.test/search.json",
|
||||
serpapiApiKey: "serpapi-key",
|
||||
fetchImpl: fetchMock as unknown as typeof fetch,
|
||||
});
|
||||
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0]?.domain).toBe("store.example");
|
||||
expect(results[0]?.detectedPrice).toBe(13.4);
|
||||
expect(results[0]?.detectedPriceText).toBe("$13.40");
|
||||
expect(results[0]?.engines).toEqual(["serpapi google shopping"]);
|
||||
});
|
||||
|
||||
test("searchProductOffers applies result limits and handles empty results", async () => {
|
||||
const fetchMock = mock(async () =>
|
||||
Response.json({
|
||||
results: [
|
||||
{ title: "One", url: "https://one.example", content: "No price" },
|
||||
{ title: "Two", url: "https://two.example", content: "$20.00" },
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
const limited = await searchProductOffers("romand palette", {
|
||||
provider: "searxng",
|
||||
baseUrl: "https://searxng.test/",
|
||||
fetchImpl: fetchMock as unknown as typeof fetch,
|
||||
maxResults: 1,
|
||||
});
|
||||
expect(limited).toHaveLength(1);
|
||||
expect(limited[0]?.domain).toBe("two.example");
|
||||
|
||||
const emptyFetch = mock(async () => Response.json({ results: [] }));
|
||||
const empty = await searchProductOffers("missing product", {
|
||||
provider: "searxng",
|
||||
baseUrl: "https://searxng.test/",
|
||||
fetchImpl: emptyFetch as unknown as typeof fetch,
|
||||
});
|
||||
expect(empty).toEqual([]);
|
||||
});
|
||||
Reference in New Issue
Block a user