Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: add scrapeOptions.fastMode

Gergő Móricz842b522b445d
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index f7226338d..2c0545608 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -182,6 +182,7 @@ export const scrapeOptions = z
.optional(),
skipTlsVerification: z.boolean().default(false),
removeBase64Images: z.boolean().default(true),
+ fastMode: z.boolean().default(false),
})
.strict(strictMessage);
@@ -685,11 +686,11 @@ export function fromLegacyScrapeOptions(
}
: undefined,
mobile: pageOptions.mobile,
+ fastMode: pageOptions.useFastMode,
}),
internalOptions: {
atsv: pageOptions.atsv,
v0DisableJsDom: pageOptions.disableJsDom,
- v0UseFastMode: pageOptions.useFastMode,
},
// TODO: fallback, fetchPageContent, replaceAllPathsWithAbsolutePaths, includeLinks
};
diff --git a/apps/api/src/lib/cache.ts b/apps/api/src/lib/cache.ts
index 7dcbf88b6..cbab4e053 100644
--- a/apps/api/src/lib/cache.ts
+++ b/apps/api/src/lib/cache.ts
@@ -21,7 +21,7 @@ export function cacheKey(
if (
internalOptions.v0CrawlOnlyUrls ||
internalOptions.forceEngine ||
- internalOptions.v0UseFastMode ||
+ scrapeOptions.fastMode ||
internalOptions.atsv ||
(scrapeOptions.actions && scrapeOptions.actions.length > 0)
) {
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index a3eb6f1eb..d3b33418a 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -86,7 +86,7 @@ function buildFeatureFlags(
flags.add("skipTlsVerification");
}
- if (internalOptions.v0UseFastMode) {
+ if (options.fastMode) {
flags.add("useFastMode");
}
@@ -148,7 +148,6 @@ export type InternalOptions = {
atsv?: boolean; // anti-bot solver, beta
v0CrawlOnlyUrls?: boolean;
- v0UseFastMode?: boolean;
v0DisableJsDom?: boolean;
disableSmartWaitCache?: boolean; // Passed along to fire-engine