Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
raw base64 format (#4387)
rowinsgca0be9b7d91e
diff --git a/apps/api/openapi.json b/apps/api/openapi.jsonindex 98cacd817..76809d30b 100644--- a/apps/api/openapi.json+++ b/apps/api/openapi.json@@ -2277,6 +2277,7 @@"markdown","html","rawHtml",+ "rawBase64","links","screenshot","screenshot@fullPage",@@ -2285,7 +2286,7 @@"branding"]},- "description": "Formats to include in the output.",+ "description": "Formats to include in the output. `rawBase64` must be requested by itself.","default": ["markdown"]},"onlyMainContent": {@@ -2606,6 +2607,11 @@"nullable": true,"description": "Raw HTML content of the page if `rawHtml` is in `formats`"},+ "rawBase64": {+ "type": "string",+ "nullable": true,+ "description": "Base64-encoded original response body if `rawBase64` is in `formats`"+ },"screenshot": {"type": "string","nullable": true,@@ -2866,6 +2872,11 @@"nullable": true,"description": "Raw HTML content of the page if `includeRawHtml` is true"},+ "rawBase64": {+ "type": "string",+ "nullable": true,+ "description": "Base64-encoded original response body if `rawBase64` is in `formats`"+ },"links": {"type": "array","items": {@@ -3003,6 +3014,11 @@"nullable": true,"description": "Raw HTML content of the page if `includeRawHtml` is true"},+ "rawBase64": {+ "type": "string",+ "nullable": true,+ "description": "Base64-encoded original response body if `rawBase64` is in `formats`"+ },"links": {"type": "array","items": {diff --git a/apps/api/src/__tests__/snips/mocks/raw-base64.json b/apps/api/src/__tests__/snips/mocks/raw-base64.jsonnew file mode 100644index 000000000..c80ae3b13--- /dev/null+++ b/apps/api/src/__tests__/snips/mocks/raw-base64.json@@ -0,0 +1,70 @@+[+ {+ "time": 1,+ "options": {+ "url": "<fire-engine>/scrape",+ "method": "POST",+ "body": {+ "url": "https://example.com/raw",+ "engine": "chrome-cdp",+ "format": "rawBase64"+ }+ },+ "result": {+ "status": 200,+ "headers": {},+ "body": "{\"timeTaken\":0.1,\"content\":\"\",\"url\":\"https://example.com/raw\",\"pageStatusCode\":200,\"responseHeaders\":{\"content-type\":\"text/html; charset=utf-8\"},\"file\":{\"name\":\"raw.html\",\"content\":\"PGh0bWw+cmF3PC9odG1sPg==\"}}"+ }+ },+ {+ "time": 2,+ "options": {+ "url": "<fire-engine>/scrape",+ "method": "POST",+ "body": {+ "url": "https://example.com/raw.pdf",+ "engine": "chrome-cdp",+ "format": "rawBase64"+ }+ },+ "result": {+ "status": 200,+ "headers": {},+ "body": "{\"timeTaken\":0.1,\"content\":\"\",\"url\":\"https://example.com/raw.pdf\",\"pageStatusCode\":200,\"responseHeaders\":{\"content-type\":\"application/pdf\"},\"file\":{\"name\":\"raw.pdf\",\"content\":\"PGh0bWw+cmF3PC9odG1sPg==\"}}"+ }+ },+ {+ "time": 3,+ "options": {+ "url": "<fire-engine>/scrape",+ "method": "POST",+ "body": {+ "url": "https://example.com/raw.docx",+ "engine": "chrome-cdp",+ "format": "rawBase64"+ }+ },+ "result": {+ "status": 200,+ "headers": {},+ "body": "{\"timeTaken\":0.1,\"content\":\"\",\"url\":\"https://example.com/raw.docx\",\"pageStatusCode\":200,\"responseHeaders\":{\"content-type\":\"application/vnd.openxmlformats-officedocument.wordprocessingml.document\"},\"file\":{\"name\":\"raw.docx\",\"content\":\"PGh0bWw+cmF3PC9odG1sPg==\"}}"+ }+ },+ {+ "time": 4,+ "options": {+ "url": "<fire-engine>/scrape",+ "method": "POST",+ "body": {+ "url": "https://example.com/raw-error",+ "engine": "chrome-cdp",+ "format": "rawBase64"+ }+ },+ "result": {+ "status": 200,+ "headers": {},+ "body": "{\"timeTaken\":0.1,\"content\":\"Not found\",\"url\":\"https://example.com/raw-error\",\"pageStatusCode\":404,\"responseHeaders\":{\"content-type\":\"text/html; charset=utf-8\"}}"+ }+ }+]diff --git a/apps/api/src/__tests__/snips/v1/types-validation.test.ts b/apps/api/src/__tests__/snips/v1/types-validation.test.tsindex 79d763931..8831f958b 100644--- a/apps/api/src/__tests__/snips/v1/types-validation.test.ts+++ b/apps/api/src/__tests__/snips/v1/types-validation.test.ts@@ -59,6 +59,22 @@ describe("V1 Types Validation", () => {expect(result.timeout).toBe(60000);});+ it("should only allow rawBase64 as the sole format", () => {+ expect(+ scrapeRequestSchema.parse({+ url: "https://example.com/file",+ formats: ["rawBase64"],+ }).formats,+ ).toEqual(["rawBase64"]);++ expect(() =>+ scrapeRequestSchema.parse({+ url: "https://example.com/file",+ formats: ["markdown", "rawBase64"],+ }),+ ).toThrow("The rawBase64 format cannot be combined with other formats");+ });+it("should reject invalid URL", () => {const input = {url: "not-a-url",diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.tsindex da202c891..113747c94 100644--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts@@ -1,6 +1,7 @@import { z } from "zod";import {scrapeRequestSchema,+ parseRequestSchema,scrapeOptions,extractRequestSchema,crawlRequestSchema,@@ -83,6 +84,22 @@ describe("V2 Types Validation", () => {expect(result.formats).toEqual([{ type: "markdown" }, { type: "html" }]);});+ it("should only allow rawBase64 as the sole format", () => {+ expect(+ scrapeRequestSchema.parse({+ url: "https://example.com/file",+ formats: ["rawBase64"],+ }).formats,+ ).toEqual([{ type: "rawBase64" }]);++ expect(() =>+ scrapeRequestSchema.parse({+ url: "https://example.com/file",+ formats: ["markdown", "rawBase64"],+ }),+ ).toThrow("The rawBase64 format cannot be combined with other formats");+ });+it("should accept video format as string and object", () => {const stringInput: ScrapeRequestInput = {url: "https://example.com",@@ -685,6 +702,22 @@ describe("V2 Types Validation", () => {});});+ describe("parseRequestSchema", () => {+ it("should reject rawBase64 for file uploads", () => {+ expect(() =>+ parseRequestSchema.parse({+ formats: ["rawBase64"],+ file: {+ buffer: Buffer.from("raw upload"),+ filename: "upload.html",+ contentType: "text/html",+ kind: "html",+ },+ }),+ ).toThrow("The rawBase64 format is not supported for parse uploads");+ });+ });+describe("extractRequestSchema", () => {it("should accept valid extract request with urls", () => {const input: ExtractRequestInput = {diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex df135e902..026ef952d 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -29,6 +29,7 @@ type Format =| "markdown"| "html"| "rawHtml"+ | "rawBase64"| "links"| "screenshot"| "screenshot@fullPage"@@ -436,6 +437,7 @@ const baseScrapeOptions = z.strictObject({"markdown","html","rawHtml",+ "rawBase64","links","screenshot","screenshot@fullPage",@@ -457,6 +459,10 @@ const baseScrapeOptions = z.strictObject({.refine(x => !x.includes("changeTracking") || x.includes("markdown"),"The changeTracking format requires the markdown format to be specified as well",+ )+ .refine(+ x => !x.includes("rawBase64") || x.length === 1,+ "The rawBase64 format cannot be combined with other formats",),headers: z.record(z.string(), z.string()).optional(),includeTags: z@@ -1014,6 +1020,7 @@ export type Document = {blocks?: PdfPageBlocks[];html?: string;rawHtml?: string;+ rawBase64?: string;links?: string[];images?: string[];screenshot?: string;diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 496d00384..ce7c0279a 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -458,6 +458,7 @@ export type FormatObject =| { type: "markdown" }| { type: "html" }| { type: "rawHtml" }+ | { type: "rawBase64" }| { type: "links" }| { type: "images" }| { type: "summary" }@@ -702,6 +703,7 @@ const baseScrapeOptions = z.strictObject({z.strictObject({ type: z.literal("markdown") }),z.strictObject({ type: z.literal("html") }),z.strictObject({ type: z.literal("rawHtml") }),+ z.strictObject({ type: z.literal("rawBase64") }),z.strictObject({ type: z.literal("links") }),z.strictObject({ type: z.literal("images") }),z.strictObject({ type: z.literal("summary") }),@@ -737,7 +739,11 @@ const baseScrapeOptions = z.strictObject({const hasJson = x.some(f => f.type === "json");const hasDeterministicJson = x.some(f => f.type === "deterministicJson");return !(hasJson && hasDeterministicJson);- }, "Cannot specify both json and deterministicJson formats"),+ }, "Cannot specify both json and deterministicJson formats")+ .refine(+ x => !x.some(f => f.type === "rawBase64") || x.length === 1,+ "The rawBase64 format cannot be combined with other formats",+ ),headers: z.record(z.string(), z.string()).optional(),includeTags: z.string()@@ -1105,6 +1111,10 @@ const parseRequestSchemaBase = baseScrapeOptions.extend({});export const parseRequestSchema = strictWithMessage(parseRequestSchemaBase)+ .refine(+ x => !x.formats.some(format => format.type === "rawBase64"),+ "The rawBase64 format is not supported for parse uploads",+ ).refine(waitForRefine, waitForRefineOpts).transform(x => {const { file, ...scrapeLike } = x;@@ -1304,6 +1314,7 @@ export type Document = {blocks?: PdfPageBlocks[];html?: string;rawHtml?: string;+ rawBase64?: string;links?: string[];images?: string[];screenshot?: string;diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 942b51314..06a4d3151 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -195,24 +195,29 @@ async function performFireEngineScrape<status = scrape as FireEngineCheckStatusSuccess;}- await specialtyScrapeCheck(- logger.child({- method: "performFireEngineScrape/specialtyScrapeCheck",- }),- status.responseHeaders,- status,- );+ const wantsRawBase64 =+ hasFormatOfType(meta.options.formats, "rawBase64") !== undefined;++ if (!wantsRawBase64) {+ await specialtyScrapeCheck(+ logger.child({+ method: "performFireEngineScrape/specialtyScrapeCheck",+ }),+ status.responseHeaders,+ status,+ );+ }const contentType =(Object.entries(status.responseHeaders ?? {}).find(x => x[0].toLowerCase() === "content-type",) ?? [])[1] ?? "";- if (contentType.includes("application/json")) {+ if (!wantsRawBase64 && contentType.includes("application/json")) {status.content = await getInnerJson(status.content);}- if (status.file) {+ if (status.file && !wantsRawBase64) {const content = status.file.content;delete status.file;let buffer = Buffer.from(content, "base64");@@ -318,16 +323,34 @@ export async function scrapeURLWithFireEngineChromeCDP("engine.url": meta.url,"engine.team_id": meta.internalOptions.teamId,});+ const wantsRawBase64 =+ hasFormatOfType(meta.options.formats, "rawBase64") !== undefined;+ if (+ wantsRawBase64 &&+ ((meta.options.waitFor ?? 0) > 0 ||+ (meta.options.actions?.length ?? 0) > 0)+ ) {+ meta.logger.warn(+ "rawBase64 returns the original response body; waitFor and actions are ignored.",+ {+ waitFor: meta.options.waitFor,+ actionTypes: meta.options.actions?.map(action => action.type),+ },+ );+ }const hasBranding = hasFormatOfType(meta.options.formats, "branding");const hasAudio = hasFormatOfType(meta.options.formats, "audio");const hasVideo = hasFormatOfType(meta.options.formats, "video");- const shouldRunYoutubePostprocessor = youtubePostprocessor.shouldRun(- meta,- new URL(meta.rewrittenUrl ?? meta.url),- );+ const shouldRunYoutubePostprocessor =+ !wantsRawBase64 &&+ youtubePostprocessor.shouldRun(+ meta,+ new URL(meta.rewrittenUrl ?? meta.url),+ );const defaultWait = hasBranding ? BRANDING_DEFAULT_WAIT_MS : 0;- const effectiveWait =- meta.options.waitFor != null && meta.options.waitFor !== 0+ const effectiveWait = wantsRawBase64+ ? 0+ : meta.options.waitFor != null && meta.options.waitFor !== 0? meta.options.waitFor: defaultWait;@@ -344,7 +367,7 @@ export async function scrapeURLWithFireEngineChromeCDP(: []),// Include specified actions- ...(meta.options.actions ?? []).map(action => {+ ...(!wantsRawBase64 ? (meta.options.actions ?? []) : []).map(action => {const { metadata: _, ...rest } = action as InternalAction;return rest;}),@@ -407,6 +430,7 @@ export async function scrapeURLWithFireEngineChromeCDP(url: meta.rewrittenUrl ?? meta.url,scrapeId: meta.id,engine: "chrome-cdp",+ ...(wantsRawBase64 ? { format: "rawBase64" as const } : {}),instantReturn: false,skipTlsVerification: meta.options.skipTlsVerification,headers: meta.options.headers,@@ -508,6 +532,7 @@ export async function scrapeURLWithFireEngineChromeCDP(url: response.url ?? meta.url,html: response.content,+ rawBase64: response.file?.content,markdown: contentType?.includes("text/markdown")? response.content: undefined,diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex 6e028c566..03f481702 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -30,6 +30,7 @@ const browserCookieSchema = zexport type FireEngineScrapeRequestCommon = {url: string;scrapeId?: string;+ format?: "html" | "rawBase64";headers?: { [K: string]: string };diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 040180633..890712962 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -34,7 +34,11 @@ import {} from "../../../controllers/v2/types";import type { PdfMetadata, PdfPageBlocks } from "./pdf/types";import { BrandingProfile } from "../../../types/branding";-import { AgentIndexOnlyError, BrandingNotSupportedError } from "../error";+import {+ AgentIndexOnlyError,+ BrandingNotSupportedError,+ NoCachedDataError,+} from "../error";import { isUrlBlocked } from "../../WebScraper/utils/blocklist";import {canUseExchangeForRequest,@@ -141,6 +145,7 @@ export type EngineScrapeResult = {url: string;html: string;+ rawBase64?: string;markdown?: string;pages?: Array<{ pageNumber: number; markdown: string }>;blocks?: PdfPageBlocks[];@@ -587,6 +592,27 @@ export async function buildFallbackList(meta: Meta): Promise<unsupportedFeatures: Set<FeatureFlag>;}[]> {+ if (hasFormatOfType(meta.options.formats, "rawBase64")) {+ if (meta.internalOptions.agentIndexOnly) {+ throw new AgentIndexOnlyError();+ }++ if (meta.options.minAge !== undefined) {+ throw new NoCachedDataError();+ }++ if (meta.options.lockdown || (!useFireEngine && meta.mock === null)) {+ return [];+ }++ return [+ {+ engine: "fire-engine;chrome-cdp",+ unsupportedFeatures: new Set(),+ },+ ];+ }+if (!meta.internalOptions.agentIndexOnly &&meta.internalOptions.forceEngine === undefineddiff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 26e8b3cc3..c2792ab2c 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -535,6 +535,7 @@ async function scrapeURLLoopIter(const hasQuestion = hasFormatOfType(meta.options.formats, "question");const hasHighlights = hasFormatOfType(meta.options.formats, "highlights");const hasQuery = hasFormatOfType(meta.options.formats, "query");+ const hasRawBase64 = hasFormatOfType(meta.options.formats, "rawBase64");const needsMarkdown =hasMarkdown ||hasChangeTracking ||@@ -548,7 +549,9 @@ async function scrapeURLLoopIter(const htmlSize = engineResult.html?.length ?? 0;const shouldSkipMarkdownCheck = htmlSize > MAX_HTML_SIZE_FOR_MARKDOWN_CHECK;- if (+ if (hasRawBase64) {+ checkMarkdown = engineResult.rawBase64 !== undefined ? "rawBase64" : "";+ } else if (meta.internalOptions.teamId === "sitemap" ||meta.internalOptions.teamId === "robots-txt") {@@ -597,6 +600,9 @@ async function scrapeURLLoopIter((engineResult.statusCode >= 200 && engineResult.statusCode < 300) ||engineResult.statusCode === 304;const hasNoPageError = engineResult.error === undefined;+ const hasRequiredOutput = hasRawBase64+ ? engineResult.rawBase64 !== undefined+ : isLongEnough || !isGoodStatusCode;const isLikelyProxyError = [401, 403, 429].includes(engineResult.statusCode,);@@ -622,7 +628,7 @@ async function scrapeURLLoopIter(// NOTE: TODO: what to do when status code is bad is tough...// we cannot just rely on text because error messages can be brief and not hit the limit// should we just use all the fallbacks and pick the one with the longest text? - mogery- if (isLongEnough || !isGoodStatusCode) {+ if (hasRequiredOutput) {meta.logger.info("Scrape via " + engine + " deemed successful.", {factors: { isLongEnough, isGoodStatusCode, hasNoPageError },});@@ -954,6 +960,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {for (const postprocessor of postprocessors) {if (+ !hasFormatOfType(meta.options.formats, "rawBase64") &&postprocessor.shouldRun(meta,new URL(engineResult.url),@@ -987,6 +994,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {pages: engineResult.pages,blocks: engineResult.blocks,rawHtml: engineResult.html,+ rawBase64: engineResult.rawBase64,json: engineResult.json,screenshot: engineResult.screenshot,actions: engineResult.actions,diff --git a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts b/apps/api/src/scraper/scrapeURL/scrapeURL.test.tsindex c5d4a6d9a..4abfae155 100644--- a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts+++ b/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts@@ -8,6 +8,7 @@ import { scrapeURL } from ".";import { scrapeOptions } from "../../controllers/v2/types";import { Engine } from "./engines";import { CostTracking } from "../../lib/cost-tracking";+import { AgentIndexOnlyError, NoCachedDataError } from "./error";// Mock parseMarkdown but delegate to real implementation for other testsvi.mock("../../lib/html-to-markdown", async importOriginal => {@@ -34,6 +35,92 @@ const testEnginesScreenshot: (Engine | undefined)[] = [];describe("Standalone scrapeURL tests", () => {+ it.each([+ ["https://example.com/raw", "text/html; charset=utf-8"],+ ["https://example.com/raw.pdf", "application/pdf"],+ [+ "https://example.com/raw.docx",+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",+ ],+ ])(+ "returns rawBase64 through fire-engine Chrome for %s",+ async (url, contentType) => {+ const out = await scrapeURL(+ "test:raw-base64",+ url,+ scrapeOptions.parse({+ formats: ["rawBase64"],+ useMock: "raw-base64",+ fastMode: true,+ blockAds: false,+ proxy: "stealth",+ }),+ { forceEngine: "fetch", teamId: "test", orgId: null },+ new CostTracking(),+ );++ expect(out.success).toBe(true);+ if (out.success) {+ expect(out.document.rawBase64).toBe("PGh0bWw+cmF3PC9odG1sPg==");+ expect(out.document).not.toHaveProperty("markdown");+ expect(out.document).not.toHaveProperty("html");+ expect(out.document).not.toHaveProperty("rawHtml");+ expect(out.document.metadata.contentType).toBe(contentType);+ }+ },+ );++ it("rejects a rawBase64 engine response without file content", async () => {+ const out = await scrapeURL(+ "test:raw-base64-missing",+ "https://example.com/raw-error",+ scrapeOptions.parse({+ formats: ["rawBase64"],+ useMock: "raw-base64",+ }),+ { teamId: "test", orgId: null },+ new CostTracking(),+ );++ expect(out.success).toBe(false);+ });++ it("returns the canonical agent-index-only error for rawBase64", async () => {+ const out = await scrapeURL(+ "test:raw-base64-agent-index-only",+ "https://example.com/raw",+ scrapeOptions.parse({+ formats: ["rawBase64"],+ useMock: "raw-base64",+ }),+ { teamId: "test", orgId: null, agentIndexOnly: true },+ new CostTracking(),+ );++ expect(out.success).toBe(false);+ if (!out.success) {+ expect(out.error).toBeInstanceOf(AgentIndexOnlyError);+ }+ });++ it("rejects rawBase64 when minAge is set", async () => {+ const out = await scrapeURL(+ "test:raw-base64-min-age",+ "https://example.com/raw",+ scrapeOptions.parse({+ formats: ["rawBase64"],+ minAge: 0,+ }),+ { teamId: "test", orgId: null },+ new CostTracking(),+ );++ expect(out.success).toBe(false);+ if (!out.success) {+ expect(out.error).toBeInstanceOf(NoCachedDataError);+ }+ });+describe.each(testEngines)("Engine %s", (forceEngine: Engine | undefined) => {it("Basic scrape", async () => {const out = await scrapeURL(diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex 2df9b5da8..3695ae78f 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -366,6 +366,7 @@ async function performLLMExtractUnlessNativeJson(function coerceFieldsToFormats(meta: Meta, document: Document): Document {const hasMarkdown = hasFormatOfType(meta.options.formats, "markdown");const hasRawHtml = hasFormatOfType(meta.options.formats, "rawHtml");+ const hasRawBase64 = hasFormatOfType(meta.options.formats, "rawBase64");const hasHtml = hasFormatOfType(meta.options.formats, "html");const hasLinks = hasFormatOfType(meta.options.formats, "links");const hasImages = hasFormatOfType(meta.options.formats, "images");@@ -407,6 +408,14 @@ function coerceFieldsToFormats(meta: Meta, document: Document): Document {);}+ if (!hasRawBase64 && document.rawBase64 !== undefined) {+ delete document.rawBase64;+ } else if (hasRawBase64 && document.rawBase64 === undefined) {+ meta.logger.warn(+ "Request had format: rawBase64, but there was no rawBase64 field in the result.",+ );+ }+if (!hasHtml && document.html !== undefined) {delete document.html;} else if (hasHtml && document.html === undefined) {@@ -661,6 +670,10 @@ export async function executeTransformers(meta: Meta,document: Document,): Promise<Document> {+ if (hasFormatOfType(meta.options.formats, "rawBase64")) {+ return coerceFieldsToFormats(meta, document);+ }+const executions: [string, number][] = [];for (const transformer of transformerStack) {diff --git a/apps/api/v1-openapi.json b/apps/api/v1-openapi.jsonindex 9fec50fcf..d18d70826 100644--- a/apps/api/v1-openapi.json+++ b/apps/api/v1-openapi.json@@ -2097,6 +2097,7 @@"markdown","html","rawHtml",+ "rawBase64","links","screenshot","screenshot@fullPage",@@ -2105,7 +2106,7 @@"branding"]},- "description": "Formats to include in the output.",+ "description": "Formats to include in the output. `rawBase64` must be requested by itself.","default": ["markdown"]},"onlyMainContent": {@@ -2426,6 +2427,11 @@"nullable": true,"description": "Raw HTML content of the page if `rawHtml` is in `formats`"},+ "rawBase64": {+ "type": "string",+ "nullable": true,+ "description": "Base64-encoded original response body if `rawBase64` is in `formats`"+ },"screenshot": {"type": "string","nullable": true,@@ -2672,6 +2678,11 @@"nullable": true,"description": "Raw HTML content of the page if `includeRawHtml` is true"},+ "rawBase64": {+ "type": "string",+ "nullable": true,+ "description": "Base64-encoded original response body if `rawBase64` is in `formats`"+ },"links": {"type": "array","items": {@@ -2809,6 +2820,11 @@"nullable": true,"description": "Raw HTML content of the page if `includeRawHtml` is true"},+ "rawBase64": {+ "type": "string",+ "nullable": true,+ "description": "Base64-encoded original response body if `rawBase64` is in `formats`"+ },"links": {"type": "array","items": {