Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: add scrapeOptions.fastMode
Gergő Móricz842b522b445d
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex f7226338d..2c0545608 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -182,6 +182,7 @@ export const scrapeOptions = z.optional(),skipTlsVerification: z.boolean().default(false),removeBase64Images: z.boolean().default(true),+ fastMode: z.boolean().default(false),}).strict(strictMessage);@@ -685,11 +686,11 @@ export function fromLegacyScrapeOptions(}: undefined,mobile: pageOptions.mobile,+ fastMode: pageOptions.useFastMode,}),internalOptions: {atsv: pageOptions.atsv,v0DisableJsDom: pageOptions.disableJsDom,- v0UseFastMode: pageOptions.useFastMode,},// TODO: fallback, fetchPageContent, replaceAllPathsWithAbsolutePaths, includeLinks};diff --git a/apps/api/src/lib/cache.ts b/apps/api/src/lib/cache.tsindex 7dcbf88b6..cbab4e053 100644--- a/apps/api/src/lib/cache.ts+++ b/apps/api/src/lib/cache.ts@@ -21,7 +21,7 @@ export function cacheKey(if (internalOptions.v0CrawlOnlyUrls ||internalOptions.forceEngine ||- internalOptions.v0UseFastMode ||+ scrapeOptions.fastMode ||internalOptions.atsv ||(scrapeOptions.actions && scrapeOptions.actions.length > 0)) {diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex a3eb6f1eb..d3b33418a 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -86,7 +86,7 @@ function buildFeatureFlags(flags.add("skipTlsVerification");}- if (internalOptions.v0UseFastMode) {+ if (options.fastMode) {flags.add("useFastMode");}@@ -148,7 +148,6 @@ export type InternalOptions = {atsv?: boolean; // anti-bot solver, betav0CrawlOnlyUrls?: boolean;- v0UseFastMode?: boolean;v0DisableJsDom?: boolean;disableSmartWaitCache?: boolean; // Passed along to fire-engine