Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(scrapeURL/index): locations array querying bug (#2164)

Gergő Móricz8f4b73b6f1f4
diff --git a/apps/api/src/__tests__/snips/v1/scrape.test.ts b/apps/api/src/__tests__/snips/v1/scrape.test.ts
index e4a6b6291..d6f95edf7 100644
--- a/apps/api/src/__tests__/snips/v1/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/v1/scrape.test.ts
@@ -666,7 +666,7 @@ describe("Scrape tests", () => {
const response1 = await scrape(
{
url,
- location: { country: "DE" },
+ location: { country: "DE", languages: ["hu-HU", "de-DE"] },
maxAge: scrapeTimeout * 2,
timeout: scrapeTimeout,
},
@@ -680,7 +680,7 @@ describe("Scrape tests", () => {
const response2 = await scrape(
{
url,
- location: { country: "DE" },
+ location: { country: "DE", languages: ["de-DE", "hu-HU"] },
timeout: scrapeTimeout,
maxAge: scrapeTimeout * 3,
},
diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.ts
index e413fbbc4..2f6f57cc6 100644
--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts
@@ -688,7 +688,7 @@ describe("Scrape tests", () => {
const response1 = await scrape(
{
url,
- location: { country: "DE" },
+ location: { country: "DE", languages: ["hu-HU", "de-DE"] },
maxAge: scrapeTimeout * 2,
},
identity,
@@ -701,7 +701,7 @@ describe("Scrape tests", () => {
const response2 = await scrape(
{
url,
- location: { country: "DE" },
+ location: { country: "DE", languages: ["de-DE", "hu-HU"] },
maxAge: scrapeTimeout * 3,
},
identity,
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index cfe622261..3a51ae435 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -241,38 +241,21 @@ export async function scrapeURLWithIndex(
}
}
- let selector = index_supabase_service
- .from("index")
- .select("id, created_at, status")
- .eq("url_hash", urlHash)
- .is("invalidated_at", null)
- .gte("created_at", new Date(Date.now() - maxAge).toISOString())
- .eq("is_mobile", meta.options.mobile)
- .eq("block_ads", meta.options.blockAds);
-
- if (meta.featureFlags.has("screenshot")) {
- selector = selector.eq("has_screenshot", true);
- }
- if (meta.featureFlags.has("screenshot@fullScreen")) {
- selector = selector.eq("has_screenshot_fullscreen", true);
- }
- if (meta.options.location?.country) {
- selector = selector.eq("location_country", meta.options.location.country);
- } else {
- selector = selector.is("location_country", null);
- }
- if (meta.options.location?.languages) {
- selector = selector.eq(
- "location_languages",
- meta.options.location.languages,
- );
- } else {
- selector = selector.is("location_languages", null);
- }
-
- const { data, error } = await selector
- .order("created_at", { ascending: false })
- .limit(5);
+ const { data, error } = await index_supabase_service.rpc("index_get_recent", {
+ p_url_hash: urlHash,
+ p_max_age_ms: maxAge,
+ p_is_mobile: meta.options.mobile,
+ p_block_ads: meta.options.blockAds,
+ p_feature_screenshot: meta.featureFlags.has("screenshot"),
+ p_feature_screenshot_fullscreen: meta.featureFlags.has(
+ "screenshot@fullScreen",
+ ),
+ p_location_country: meta.options.location?.country ?? null,
+ p_location_languages:
+ (meta.options.location?.languages?.length ?? 0) > 0
+ ? meta.options.location?.languages
+ : null,
+ });
if (error || !data) {
throw new EngineError("Failed to retrieve URL from DB index", {