Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: use index in search and extract (#1660)

Gergő Móricz07224b8cd496
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index 136df1f2f..b716d5a9a 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -99,7 +99,10 @@ async function scrapeSearchResult(
url: searchResult.url,
mode: "single_urls" as Mode,
team_id: options.teamId,
- scrapeOptions: options.scrapeOptions,
+ scrapeOptions: {
+ ...options.scrapeOptions,
+ maxAge: 4 * 60 * 60 * 1000, // 4 hours, same as useCache
+ },
internalOptions: { teamId: options.teamId, useCache: true, bypassBilling: true },
origin: options.origin,
is_scrape: true,
diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.ts
index 799062b69..f06a8b257 100644
--- a/apps/api/src/lib/extract/document-scraper.ts
+++ b/apps/api/src/lib/extract/document-scraper.ts
@@ -40,7 +40,10 @@ export async function scrapeDocument(
url: options.url,
mode: "single_urls",
team_id: options.teamId,
- scrapeOptions: scrapeOptions.parse({ ...internalScrapeOptions }),
+ scrapeOptions: scrapeOptions.parse({
+ ...internalScrapeOptions,
+ maxAge: 4 * 60 * 60 * 1000, // 4 hours, same as useCache
+ }),
internalOptions: {
useCache: true,
teamId: options.teamId,
diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
index fa2f289e3..f81bb8dd9 100644
--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
@@ -39,7 +39,10 @@ export async function scrapeDocument_F0(
url: options.url,
mode: "single_urls",
team_id: options.teamId,
- scrapeOptions: scrapeOptions.parse({ ...internalScrapeOptions }),
+ scrapeOptions: scrapeOptions.parse({
+ ...internalScrapeOptions,
+ maxAge: 4 * 60 * 60 * 1000, // 4 hours, same as useCache
+ }),
internalOptions: {
useCache: true,
teamId: options.teamId,