Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: use index in search and extract (#1660)
Gergő Móricz07224b8cd496
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 136df1f2f..b716d5a9a 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -99,7 +99,10 @@ async function scrapeSearchResult(url: searchResult.url,mode: "single_urls" as Mode,team_id: options.teamId,- scrapeOptions: options.scrapeOptions,+ scrapeOptions: {+ ...options.scrapeOptions,+ maxAge: 4 * 60 * 60 * 1000, // 4 hours, same as useCache+ },internalOptions: { teamId: options.teamId, useCache: true, bypassBilling: true },origin: options.origin,is_scrape: true,diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.tsindex 799062b69..f06a8b257 100644--- a/apps/api/src/lib/extract/document-scraper.ts+++ b/apps/api/src/lib/extract/document-scraper.ts@@ -40,7 +40,10 @@ export async function scrapeDocument(url: options.url,mode: "single_urls",team_id: options.teamId,- scrapeOptions: scrapeOptions.parse({ ...internalScrapeOptions }),+ scrapeOptions: scrapeOptions.parse({+ ...internalScrapeOptions,+ maxAge: 4 * 60 * 60 * 1000, // 4 hours, same as useCache+ }),internalOptions: {useCache: true,teamId: options.teamId,diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.tsindex fa2f289e3..f81bb8dd9 100644--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts@@ -39,7 +39,10 @@ export async function scrapeDocument_F0(url: options.url,mode: "single_urls",team_id: options.teamId,- scrapeOptions: scrapeOptions.parse({ ...internalScrapeOptions }),+ scrapeOptions: scrapeOptions.parse({+ ...internalScrapeOptions,+ maxAge: 4 * 60 * 60 * 1000, // 4 hours, same as useCache+ }),internalOptions: {useCache: true,teamId: options.teamId,