Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Merge branch 'main' into fix/remove-base64-images

Rafael Miller8297e5beef3c
diff --cc apps/api/src/controllers/v1/types.ts
index 14d420256,54eb3741e..9585175e8
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@@ -470,7 -470,7 +471,8 @@@ export function legacyScrapeOptions(x:
actions: x.actions as Action[], // no strict null checking grrrr - mogery
geolocation: x.location ?? x.geolocation,
skipTlsVerification: x.skipTlsVerification,
+ removeBase64Images: x.removeBase64Images,
+ mobile: x.mobile,
};
}
diff --cc apps/api/src/lib/entities.ts
index 87aa44ac2,a1ebb4a62..2c8f376de
--- a/apps/api/src/lib/entities.ts
+++ b/apps/api/src/lib/entities.ts
@@@ -58,7 -58,7 +58,8 @@@ export type PageOptions =
country?: string;
};
skipTlsVerification?: boolean;
+ removeBase64Images?: boolean;
+ mobile?: boolean;
};
export type ExtractorOptions = {
diff --cc apps/api/src/scraper/WebScraper/index.ts
index 615e49b40,4e39e4266..79b312d59
--- a/apps/api/src/scraper/WebScraper/index.ts
+++ b/apps/api/src/scraper/WebScraper/index.ts
@@@ -595,7 -595,7 +595,8 @@@ export class WebScraperDataProvider
actions: options.pageOptions?.actions ?? undefined,
geolocation: options.pageOptions?.geolocation ?? undefined,
skipTlsVerification: options.pageOptions?.skipTlsVerification ?? false,
+ removeBase64Images: options.pageOptions?.removeBase64Images ?? true,
+ mobile: options.pageOptions?.mobile ?? false,
};
this.extractorOptions = options.extractorOptions ?? { mode: "markdown" };
this.replaceAllPathsWithAbsolutePaths =
diff --cc apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts
index 8a37457ca,058e82074..7616774a7
--- a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts
+++ b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts
@@@ -28,7 -28,7 +28,7 @@@ export async function scrapWithFireEngi
waitFor = 0,
screenshot = false,
fullPageScreenshot = false,
- pageOptions = { parsePDF: true, atsv: false, useFastMode: false, disableJsDom: false, geolocation: { country: "US" }, skipTlsVerification: false, removeBase64Images: true },
- pageOptions = { parsePDF: true, atsv: false, useFastMode: false, disableJsDom: false, geolocation: { country: "US" }, skipTlsVerification: false, mobile: false },
++ pageOptions = { parsePDF: true, atsv: false, useFastMode: false, disableJsDom: false, geolocation: { country: "US" }, skipTlsVerification: false, removeBase64Images: true, mobile: false },
fireEngineOptions = {},
headers,
options,
@@@ -40,7 -40,7 +40,7 @@@
waitFor?: number;
screenshot?: boolean;
fullPageScreenshot?: boolean;
- pageOptions?: { scrollXPaths?: string[]; parsePDF?: boolean, atsv?: boolean, useFastMode?: boolean, disableJsDom?: boolean, geolocation?: { country?: string }, skipTlsVerification?: boolean, removeBase64Images?: boolean };
- pageOptions?: { scrollXPaths?: string[]; parsePDF?: boolean, atsv?: boolean, useFastMode?: boolean, disableJsDom?: boolean, geolocation?: { country?: string }, skipTlsVerification?: boolean, mobile?: boolean };
++ pageOptions?: { scrollXPaths?: string[]; parsePDF?: boolean, atsv?: boolean, useFastMode?: boolean, disableJsDom?: boolean, geolocation?: { country?: string }, skipTlsVerification?: boolean, removeBase64Images?: boolean, mobile?: boolean };
fireEngineOptions?: FireEngineOptions;
headers?: Record<string, string>;
options?: any;
diff --cc apps/api/src/scraper/WebScraper/single_url.ts
index 6f5ec541f,085cdb04b..38ec74f03
--- a/apps/api/src/scraper/WebScraper/single_url.ts
+++ b/apps/api/src/scraper/WebScraper/single_url.ts
@@@ -160,7 -159,7 +160,8 @@@ export async function scrapSingleUrl
actions: pageOptions.actions ?? undefined,
geolocation: pageOptions.geolocation ?? undefined,
skipTlsVerification: pageOptions.skipTlsVerification ?? false,
+ removeBase64Images: pageOptions.removeBase64Images ?? true,
+ mobile: pageOptions.mobile ?? false,
}
if (extractorOptions) {