Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(scrapeURL): AI maxAge
Gergő Móriczf0665fa8eafa
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex b13c5953a..9f806b825 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -320,7 +320,7 @@ const baseScrapeOptions = zuseMock: z.string().optional(),blockAds: z.boolean().default(true),proxy: z.enum(["basic", "stealth", "auto"]).default("auto"),- maxAge: z.number().int().gte(0).safe().default(2 * 24 * 60 * 60 * 1000), // 2 days+ maxAge: z.number().int().gte(0).safe().optional(),storeInCache: z.boolean().default(true),// @deprecated__searchPreviewToken: z.string().optional(),diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex d02e21f31..740dda874 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -140,11 +140,51 @@ export async function scrapeURLWithIndex(meta: Meta): Promise<EngineScrapeResultconst normalizedURL = normalizeURLForIndex(meta.url);const urlHash = hashURL(normalizedURL);+ let maxAge: number;+ if (meta.options.maxAge !== undefined) {+ maxAge = meta.options.maxAge;+ } else {+ const domainSplitsHash = generateDomainSplits(new URL(meta.url).hostname).map(x => hashURL(x));+ const level = domainSplitsHash.length - 1;++ if (domainSplitsHash.length === 0 || process.env.FIRECRAWL_INDEX_WRITE_ONLY === "true" || process.env.USE_DB_AUTHENTICATION !== "true") {+ maxAge = 2 * 24 * 60 * 60 * 1000; // 2 days+ } else {+ try {+ maxAge = await Promise.race([+ (async () => {+ const { data, error } = await index_supabase_service+ .rpc("query_max_age", {+ i_domain_hash: domainSplitsHash[level],+ });++ if (error || !data || data.length === 0) {+ meta.logger.warn("Failed to get max age from DB", {+ error,+ });+ return 2 * 24 * 60 * 60 * 1000; // 2 days+ }++ return data[0].max_age ?? 2 * 24 * 60 * 60 * 1000; // 2 days+ })(),+ new Promise(resolve => setTimeout(() => {+ resolve(2 * 24 * 60 * 60 * 1000); // 2 days+ }, 200)),+ ]);+ } catch (e) {+ meta.logger.warn("Failed to get max age from DB", {+ error: e,+ });+ maxAge = 2 * 24 * 60 * 60 * 1000; // 2 days+ }+ }+ }+let selector = index_supabase_service.from("index").select("id, created_at, status").eq("url_hash", urlHash)- .gte("created_at", new Date(Date.now() - meta.options.maxAge).toISOString())+ .gte("created_at", new Date(Date.now() - maxAge).toISOString()).eq("is_mobile", meta.options.mobile).eq("block_ads", meta.options.blockAds);