Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Merge branch 'main' into fix/remove-base64-images
Rafael Miller8297e5beef3c
diff --cc apps/api/src/controllers/v1/types.tsindex 14d420256,54eb3741e..9585175e8--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@@ -470,7 -470,7 +471,8 @@@ export function legacyScrapeOptions(x:actions: x.actions as Action[], // no strict null checking grrrr - mogerygeolocation: x.location ?? x.geolocation,skipTlsVerification: x.skipTlsVerification,+ removeBase64Images: x.removeBase64Images,+ mobile: x.mobile,};}diff --cc apps/api/src/lib/entities.tsindex 87aa44ac2,a1ebb4a62..2c8f376de--- a/apps/api/src/lib/entities.ts+++ b/apps/api/src/lib/entities.ts@@@ -58,7 -58,7 +58,8 @@@ export type PageOptions =country?: string;};skipTlsVerification?: boolean;+ removeBase64Images?: boolean;+ mobile?: boolean;};export type ExtractorOptions = {diff --cc apps/api/src/scraper/WebScraper/index.tsindex 615e49b40,4e39e4266..79b312d59--- a/apps/api/src/scraper/WebScraper/index.ts+++ b/apps/api/src/scraper/WebScraper/index.ts@@@ -595,7 -595,7 +595,8 @@@ export class WebScraperDataProvideractions: options.pageOptions?.actions ?? undefined,geolocation: options.pageOptions?.geolocation ?? undefined,skipTlsVerification: options.pageOptions?.skipTlsVerification ?? false,+ removeBase64Images: options.pageOptions?.removeBase64Images ?? true,+ mobile: options.pageOptions?.mobile ?? false,};this.extractorOptions = options.extractorOptions ?? { mode: "markdown" };this.replaceAllPathsWithAbsolutePaths =diff --cc apps/api/src/scraper/WebScraper/scrapers/fireEngine.tsindex 8a37457ca,058e82074..7616774a7--- a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts+++ b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts@@@ -28,7 -28,7 +28,7 @@@ export async function scrapWithFireEngiwaitFor = 0,screenshot = false,fullPageScreenshot = false,- pageOptions = { parsePDF: true, atsv: false, useFastMode: false, disableJsDom: false, geolocation: { country: "US" }, skipTlsVerification: false, removeBase64Images: true },- pageOptions = { parsePDF: true, atsv: false, useFastMode: false, disableJsDom: false, geolocation: { country: "US" }, skipTlsVerification: false, mobile: false },++ pageOptions = { parsePDF: true, atsv: false, useFastMode: false, disableJsDom: false, geolocation: { country: "US" }, skipTlsVerification: false, removeBase64Images: true, mobile: false },fireEngineOptions = {},headers,options,@@@ -40,7 -40,7 +40,7 @@@waitFor?: number;screenshot?: boolean;fullPageScreenshot?: boolean;- pageOptions?: { scrollXPaths?: string[]; parsePDF?: boolean, atsv?: boolean, useFastMode?: boolean, disableJsDom?: boolean, geolocation?: { country?: string }, skipTlsVerification?: boolean, removeBase64Images?: boolean };- pageOptions?: { scrollXPaths?: string[]; parsePDF?: boolean, atsv?: boolean, useFastMode?: boolean, disableJsDom?: boolean, geolocation?: { country?: string }, skipTlsVerification?: boolean, mobile?: boolean };++ pageOptions?: { scrollXPaths?: string[]; parsePDF?: boolean, atsv?: boolean, useFastMode?: boolean, disableJsDom?: boolean, geolocation?: { country?: string }, skipTlsVerification?: boolean, removeBase64Images?: boolean, mobile?: boolean };fireEngineOptions?: FireEngineOptions;headers?: Record<string, string>;options?: any;diff --cc apps/api/src/scraper/WebScraper/single_url.tsindex 6f5ec541f,085cdb04b..38ec74f03--- a/apps/api/src/scraper/WebScraper/single_url.ts+++ b/apps/api/src/scraper/WebScraper/single_url.ts@@@ -160,7 -159,7 +160,8 @@@ export async function scrapSingleUrlactions: pageOptions.actions ?? undefined,geolocation: pageOptions.geolocation ?? undefined,skipTlsVerification: pageOptions.skipTlsVerification ?? false,+ removeBase64Images: pageOptions.removeBase64Images ?? true,+ mobile: pageOptions.mobile ?? false,}if (extractorOptions) {