Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrapeURL/index): locations array querying bug (#2164)
Gergő Móricz8f4b73b6f1f4
diff --git a/apps/api/src/__tests__/snips/v1/scrape.test.ts b/apps/api/src/__tests__/snips/v1/scrape.test.tsindex e4a6b6291..d6f95edf7 100644--- a/apps/api/src/__tests__/snips/v1/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v1/scrape.test.ts@@ -666,7 +666,7 @@ describe("Scrape tests", () => {const response1 = await scrape({url,- location: { country: "DE" },+ location: { country: "DE", languages: ["hu-HU", "de-DE"] },maxAge: scrapeTimeout * 2,timeout: scrapeTimeout,},@@ -680,7 +680,7 @@ describe("Scrape tests", () => {const response2 = await scrape({url,- location: { country: "DE" },+ location: { country: "DE", languages: ["de-DE", "hu-HU"] },timeout: scrapeTimeout,maxAge: scrapeTimeout * 3,},diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.tsindex e413fbbc4..2f6f57cc6 100644--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts@@ -688,7 +688,7 @@ describe("Scrape tests", () => {const response1 = await scrape({url,- location: { country: "DE" },+ location: { country: "DE", languages: ["hu-HU", "de-DE"] },maxAge: scrapeTimeout * 2,},identity,@@ -701,7 +701,7 @@ describe("Scrape tests", () => {const response2 = await scrape({url,- location: { country: "DE" },+ location: { country: "DE", languages: ["de-DE", "hu-HU"] },maxAge: scrapeTimeout * 3,},identity,diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex cfe622261..3a51ae435 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -241,38 +241,21 @@ export async function scrapeURLWithIndex(}}- let selector = index_supabase_service- .from("index")- .select("id, created_at, status")- .eq("url_hash", urlHash)- .is("invalidated_at", null)- .gte("created_at", new Date(Date.now() - maxAge).toISOString())- .eq("is_mobile", meta.options.mobile)- .eq("block_ads", meta.options.blockAds);-- if (meta.featureFlags.has("screenshot")) {- selector = selector.eq("has_screenshot", true);- }- if (meta.featureFlags.has("screenshot@fullScreen")) {- selector = selector.eq("has_screenshot_fullscreen", true);- }- if (meta.options.location?.country) {- selector = selector.eq("location_country", meta.options.location.country);- } else {- selector = selector.is("location_country", null);- }- if (meta.options.location?.languages) {- selector = selector.eq(- "location_languages",- meta.options.location.languages,- );- } else {- selector = selector.is("location_languages", null);- }-- const { data, error } = await selector- .order("created_at", { ascending: false })- .limit(5);+ const { data, error } = await index_supabase_service.rpc("index_get_recent", {+ p_url_hash: urlHash,+ p_max_age_ms: maxAge,+ p_is_mobile: meta.options.mobile,+ p_block_ads: meta.options.blockAds,+ p_feature_screenshot: meta.featureFlags.has("screenshot"),+ p_feature_screenshot_fullscreen: meta.featureFlags.has(+ "screenshot@fullScreen",+ ),+ p_location_country: meta.options.location?.country ?? null,+ p_location_languages:+ (meta.options.location?.languages?.length ?? 0) > 0+ ? meta.options.location?.languages+ : null,+ });if (error || !data) {throw new EngineError("Failed to retrieve URL from DB index", {