Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: cache /extract scrapes
Nicolas6b2e1cbb2813
diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.tsindex 04194b0b8..91d515dfc 100644--- a/apps/api/src/lib/extract/document-scraper.ts+++ b/apps/api/src/lib/extract/document-scraper.ts@@ -14,10 +14,13 @@ interface ScrapeDocumentOptions {timeout: number;}-export async function scrapeDocument(options: ScrapeDocumentOptions, urlTraces: URLTrace[]): Promise<Document | null> {+export async function scrapeDocument(+ options: ScrapeDocumentOptions,+ urlTraces: URLTrace[],+): Promise<Document | null> {const trace = urlTraces.find((t) => t.url === options.url);if (trace) {- trace.status = 'scraped';+ trace.status = "scraped";trace.timing.scrapedAt = new Date().toISOString();}@@ -35,7 +38,9 @@ export async function scrapeDocument(options: ScrapeDocumentOptions, urlTraces:mode: "single_urls",team_id: options.teamId,scrapeOptions: scrapeOptions.parse({}),- internalOptions: {},+ internalOptions: {+ useCache: true,+ },plan: options.plan,origin: options.origin,is_scrape: true,@@ -61,9 +66,9 @@ export async function scrapeDocument(options: ScrapeDocumentOptions, urlTraces:} catch (error) {logger.error(`Error in scrapeDocument: ${error}`);if (trace) {- trace.status = 'error';+ trace.status = "error";trace.error = error.message;}return null;}-}\ No newline at end of file+}diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex bf51ac946..956fc3ab9 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -298,6 +298,12 @@ export function buildFallbackList(meta: Meta): {engine: Engine;unsupportedFeatures: Set<FeatureFlag>;}[] {++ if (meta.internalOptions.useCache !== true) {+ engines.splice(engines.indexOf("cache"), 1);+ }else{+ meta.logger.debug("Cache engine enabled by useCache option");+ }const prioritySum = [...meta.featureFlags].reduce((a, x) => a + featureFlagOptions[x].priority,0,diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 549ce9d14..b13f7d9a8 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -151,7 +151,7 @@ export type InternalOptions = {v0CrawlOnlyUrls?: boolean;v0DisableJsDom?: boolean;-+ useCache?: boolean;disableSmartWaitCache?: boolean; // Passed along to fire-engineisBackgroundIndex?: boolean;};