Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
added remove base64 images options (true by default)
rafaelmmiller4c5bb21a6fa8
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 45aa3c9bd..14d420256 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -140,6 +140,7 @@ export const scrapeOptions = z.object({languages: z.string().array().optional(),}).optional(),skipTlsVerification: z.boolean().default(false),+ removeBase64Images: z.boolean().default(true),}).strict(strictMessage)@@ -468,7 +469,8 @@ export function legacyScrapeOptions(x: ScrapeOptions): PageOptions {parsePDF: x.parsePDF,actions: x.actions as Action[], // no strict null checking grrrr - mogerygeolocation: x.location ?? x.geolocation,- skipTlsVerification: x.skipTlsVerification+ skipTlsVerification: x.skipTlsVerification,+ removeBase64Images: x.removeBase64Images,};}diff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.tsindex 81bca5719..87aa44ac2 100644--- a/apps/api/src/lib/entities.ts+++ b/apps/api/src/lib/entities.ts@@ -58,6 +58,7 @@ export type PageOptions = {country?: string;};skipTlsVerification?: boolean;+ removeBase64Images?: boolean;};export type ExtractorOptions = {diff --git a/apps/api/src/scraper/WebScraper/index.ts b/apps/api/src/scraper/WebScraper/index.tsindex 1817a07b8..615e49b40 100644--- a/apps/api/src/scraper/WebScraper/index.ts+++ b/apps/api/src/scraper/WebScraper/index.ts@@ -595,6 +595,7 @@ export class WebScraperDataProvider {actions: options.pageOptions?.actions ?? undefined,geolocation: options.pageOptions?.geolocation ?? undefined,skipTlsVerification: options.pageOptions?.skipTlsVerification ?? false,+ removeBase64Images: options.pageOptions?.removeBase64Images ?? true,};this.extractorOptions = options.extractorOptions ?? { mode: "markdown" };this.replaceAllPathsWithAbsolutePaths =diff --git a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.tsindex 7332874f2..8a37457ca 100644--- a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts+++ b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts@@ -28,7 +28,7 @@ export async function scrapWithFireEngine({waitFor = 0,screenshot = false,fullPageScreenshot = false,- pageOptions = { parsePDF: true, atsv: false, useFastMode: false, disableJsDom: false, geolocation: { country: "US" }, skipTlsVerification: false },+ pageOptions = { parsePDF: true, atsv: false, useFastMode: false, disableJsDom: false, geolocation: { country: "US" }, skipTlsVerification: false, removeBase64Images: true },fireEngineOptions = {},headers,options,@@ -40,7 +40,7 @@ export async function scrapWithFireEngine({waitFor?: number;screenshot?: boolean;fullPageScreenshot?: boolean;- pageOptions?: { scrollXPaths?: string[]; parsePDF?: boolean, atsv?: boolean, useFastMode?: boolean, disableJsDom?: boolean, geolocation?: { country?: string }, skipTlsVerification?: boolean };+ pageOptions?: { scrollXPaths?: string[]; parsePDF?: boolean, atsv?: boolean, useFastMode?: boolean, disableJsDom?: boolean, geolocation?: { country?: string }, skipTlsVerification?: boolean, removeBase64Images?: boolean };fireEngineOptions?: FireEngineOptions;headers?: Record<string, string>;options?: any;@@ -120,6 +120,7 @@ export async function scrapWithFireEngine({scrollXPaths: pageOptions?.scrollXPaths ?? [],geolocation: pageOptions?.geolocation,skipTlsVerification: pageOptions?.skipTlsVerification ?? false,+ removeBase64Images: pageOptions?.removeBase64Images ?? true,actions: actions,},{diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.tsindex 611a7b5c3..6f5ec541f 100644--- a/apps/api/src/scraper/WebScraper/single_url.ts+++ b/apps/api/src/scraper/WebScraper/single_url.ts@@ -22,6 +22,7 @@ import { Logger } from "../../lib/logger";import { ScrapeEvents } from "../../lib/scrape-events";import { clientSideError } from "../../strings";import { ScrapeActionContent } from "../../lib/entities";+import { removeBase64Images } from "./utils/removeBase64Images";dotenv.config();@@ -159,6 +160,7 @@ export async function scrapSingleUrl(actions: pageOptions.actions ?? undefined,geolocation: pageOptions.geolocation ?? undefined,skipTlsVerification: pageOptions.skipTlsVerification ?? false,+ removeBase64Images: pageOptions.removeBase64Images ?? true,}if (extractorOptions) {@@ -350,7 +352,10 @@ export async function scrapSingleUrl(}//* TODO: add an optional to return markdown or structured/extracted contentlet cleanedHtml = removeUnwantedElements(scraperResponse.text, pageOptions);- const text = await parseMarkdown(cleanedHtml);+ let text = await parseMarkdown(cleanedHtml);+ if (pageOptions.removeBase64Images) {+ text = await removeBase64Images(text);+ }const insertedLogId = await logInsertPromise;ScrapeEvents.updateScrapeResult(insertedLogId, {diff --git a/apps/api/src/scraper/WebScraper/utils/removeBase64Images.ts b/apps/api/src/scraper/WebScraper/utils/removeBase64Images.tsnew file mode 100644index 000000000..2845589c6--- /dev/null+++ b/apps/api/src/scraper/WebScraper/utils/removeBase64Images.ts@@ -0,0 +1,7 @@+export const removeBase64Images = async (+ markdown: string,+) => {+ const regex = /(!\[.*?\])\(data:image\/.*?;base64,.*?\)/g;+ markdown = markdown.replace(regex, '$1(<Base64-Image-Removed>)');+ return markdown;+};