Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

raw base64 format (#4387)

rowinsgca0be9b7d91e
diff --git a/apps/api/openapi.json b/apps/api/openapi.json
index 98cacd817..76809d30b 100644
--- a/apps/api/openapi.json
+++ b/apps/api/openapi.json
@@ -2277,6 +2277,7 @@
"markdown",
"html",
"rawHtml",
+ "rawBase64",
"links",
"screenshot",
"screenshot@fullPage",
@@ -2285,7 +2286,7 @@
"branding"
]
},
- "description": "Formats to include in the output.",
+ "description": "Formats to include in the output. `rawBase64` must be requested by itself.",
"default": ["markdown"]
},
"onlyMainContent": {
@@ -2606,6 +2607,11 @@
"nullable": true,
"description": "Raw HTML content of the page if `rawHtml` is in `formats`"
},
+ "rawBase64": {
+ "type": "string",
+ "nullable": true,
+ "description": "Base64-encoded original response body if `rawBase64` is in `formats`"
+ },
"screenshot": {
"type": "string",
"nullable": true,
@@ -2866,6 +2872,11 @@
"nullable": true,
"description": "Raw HTML content of the page if `includeRawHtml` is true"
},
+ "rawBase64": {
+ "type": "string",
+ "nullable": true,
+ "description": "Base64-encoded original response body if `rawBase64` is in `formats`"
+ },
"links": {
"type": "array",
"items": {
@@ -3003,6 +3014,11 @@
"nullable": true,
"description": "Raw HTML content of the page if `includeRawHtml` is true"
},
+ "rawBase64": {
+ "type": "string",
+ "nullable": true,
+ "description": "Base64-encoded original response body if `rawBase64` is in `formats`"
+ },
"links": {
"type": "array",
"items": {
diff --git a/apps/api/src/__tests__/snips/mocks/raw-base64.json b/apps/api/src/__tests__/snips/mocks/raw-base64.json
new file mode 100644
index 000000000..c80ae3b13
--- /dev/null
+++ b/apps/api/src/__tests__/snips/mocks/raw-base64.json
@@ -0,0 +1,70 @@
+[
+ {
+ "time": 1,
+ "options": {
+ "url": "<fire-engine>/scrape",
+ "method": "POST",
+ "body": {
+ "url": "https://example.com/raw",
+ "engine": "chrome-cdp",
+ "format": "rawBase64"
+ }
+ },
+ "result": {
+ "status": 200,
+ "headers": {},
+ "body": "{\"timeTaken\":0.1,\"content\":\"\",\"url\":\"https://example.com/raw\",\"pageStatusCode\":200,\"responseHeaders\":{\"content-type\":\"text/html; charset=utf-8\"},\"file\":{\"name\":\"raw.html\",\"content\":\"PGh0bWw+cmF3PC9odG1sPg==\"}}"
+ }
+ },
+ {
+ "time": 2,
+ "options": {
+ "url": "<fire-engine>/scrape",
+ "method": "POST",
+ "body": {
+ "url": "https://example.com/raw.pdf",
+ "engine": "chrome-cdp",
+ "format": "rawBase64"
+ }
+ },
+ "result": {
+ "status": 200,
+ "headers": {},
+ "body": "{\"timeTaken\":0.1,\"content\":\"\",\"url\":\"https://example.com/raw.pdf\",\"pageStatusCode\":200,\"responseHeaders\":{\"content-type\":\"application/pdf\"},\"file\":{\"name\":\"raw.pdf\",\"content\":\"PGh0bWw+cmF3PC9odG1sPg==\"}}"
+ }
+ },
+ {
+ "time": 3,
+ "options": {
+ "url": "<fire-engine>/scrape",
+ "method": "POST",
+ "body": {
+ "url": "https://example.com/raw.docx",
+ "engine": "chrome-cdp",
+ "format": "rawBase64"
+ }
+ },
+ "result": {
+ "status": 200,
+ "headers": {},
+ "body": "{\"timeTaken\":0.1,\"content\":\"\",\"url\":\"https://example.com/raw.docx\",\"pageStatusCode\":200,\"responseHeaders\":{\"content-type\":\"application/vnd.openxmlformats-officedocument.wordprocessingml.document\"},\"file\":{\"name\":\"raw.docx\",\"content\":\"PGh0bWw+cmF3PC9odG1sPg==\"}}"
+ }
+ },
+ {
+ "time": 4,
+ "options": {
+ "url": "<fire-engine>/scrape",
+ "method": "POST",
+ "body": {
+ "url": "https://example.com/raw-error",
+ "engine": "chrome-cdp",
+ "format": "rawBase64"
+ }
+ },
+ "result": {
+ "status": 200,
+ "headers": {},
+ "body": "{\"timeTaken\":0.1,\"content\":\"Not found\",\"url\":\"https://example.com/raw-error\",\"pageStatusCode\":404,\"responseHeaders\":{\"content-type\":\"text/html; charset=utf-8\"}}"
+ }
+ }
+]
diff --git a/apps/api/src/__tests__/snips/v1/types-validation.test.ts b/apps/api/src/__tests__/snips/v1/types-validation.test.ts
index 79d763931..8831f958b 100644
--- a/apps/api/src/__tests__/snips/v1/types-validation.test.ts
+++ b/apps/api/src/__tests__/snips/v1/types-validation.test.ts
@@ -59,6 +59,22 @@ describe("V1 Types Validation", () => {
expect(result.timeout).toBe(60000);
});
+ it("should only allow rawBase64 as the sole format", () => {
+ expect(
+ scrapeRequestSchema.parse({
+ url: "https://example.com/file",
+ formats: ["rawBase64"],
+ }).formats,
+ ).toEqual(["rawBase64"]);
+
+ expect(() =>
+ scrapeRequestSchema.parse({
+ url: "https://example.com/file",
+ formats: ["markdown", "rawBase64"],
+ }),
+ ).toThrow("The rawBase64 format cannot be combined with other formats");
+ });
+
it("should reject invalid URL", () => {
const input = {
url: "not-a-url",
diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.ts
index da202c891..113747c94 100644
--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts
+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts
@@ -1,6 +1,7 @@
import { z } from "zod";
import {
scrapeRequestSchema,
+ parseRequestSchema,
scrapeOptions,
extractRequestSchema,
crawlRequestSchema,
@@ -83,6 +84,22 @@ describe("V2 Types Validation", () => {
expect(result.formats).toEqual([{ type: "markdown" }, { type: "html" }]);
});
+ it("should only allow rawBase64 as the sole format", () => {
+ expect(
+ scrapeRequestSchema.parse({
+ url: "https://example.com/file",
+ formats: ["rawBase64"],
+ }).formats,
+ ).toEqual([{ type: "rawBase64" }]);
+
+ expect(() =>
+ scrapeRequestSchema.parse({
+ url: "https://example.com/file",
+ formats: ["markdown", "rawBase64"],
+ }),
+ ).toThrow("The rawBase64 format cannot be combined with other formats");
+ });
+
it("should accept video format as string and object", () => {
const stringInput: ScrapeRequestInput = {
url: "https://example.com",
@@ -685,6 +702,22 @@ describe("V2 Types Validation", () => {
});
});
+ describe("parseRequestSchema", () => {
+ it("should reject rawBase64 for file uploads", () => {
+ expect(() =>
+ parseRequestSchema.parse({
+ formats: ["rawBase64"],
+ file: {
+ buffer: Buffer.from("raw upload"),
+ filename: "upload.html",
+ contentType: "text/html",
+ kind: "html",
+ },
+ }),
+ ).toThrow("The rawBase64 format is not supported for parse uploads");
+ });
+ });
+
describe("extractRequestSchema", () => {
it("should accept valid extract request with urls", () => {
const input: ExtractRequestInput = {
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index df135e902..026ef952d 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -29,6 +29,7 @@ type Format =
| "markdown"
| "html"
| "rawHtml"
+ | "rawBase64"
| "links"
| "screenshot"
| "screenshot@fullPage"
@@ -436,6 +437,7 @@ const baseScrapeOptions = z.strictObject({
"markdown",
"html",
"rawHtml",
+ "rawBase64",
"links",
"screenshot",
"screenshot@fullPage",
@@ -457,6 +459,10 @@ const baseScrapeOptions = z.strictObject({
.refine(
x => !x.includes("changeTracking") || x.includes("markdown"),
"The changeTracking format requires the markdown format to be specified as well",
+ )
+ .refine(
+ x => !x.includes("rawBase64") || x.length === 1,
+ "The rawBase64 format cannot be combined with other formats",
),
headers: z.record(z.string(), z.string()).optional(),
includeTags: z
@@ -1014,6 +1020,7 @@ export type Document = {
blocks?: PdfPageBlocks[];
html?: string;
rawHtml?: string;
+ rawBase64?: string;
links?: string[];
images?: string[];
screenshot?: string;
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 496d00384..ce7c0279a 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -458,6 +458,7 @@ export type FormatObject =
| { type: "markdown" }
| { type: "html" }
| { type: "rawHtml" }
+ | { type: "rawBase64" }
| { type: "links" }
| { type: "images" }
| { type: "summary" }
@@ -702,6 +703,7 @@ const baseScrapeOptions = z.strictObject({
z.strictObject({ type: z.literal("markdown") }),
z.strictObject({ type: z.literal("html") }),
z.strictObject({ type: z.literal("rawHtml") }),
+ z.strictObject({ type: z.literal("rawBase64") }),
z.strictObject({ type: z.literal("links") }),
z.strictObject({ type: z.literal("images") }),
z.strictObject({ type: z.literal("summary") }),
@@ -737,7 +739,11 @@ const baseScrapeOptions = z.strictObject({
const hasJson = x.some(f => f.type === "json");
const hasDeterministicJson = x.some(f => f.type === "deterministicJson");
return !(hasJson && hasDeterministicJson);
- }, "Cannot specify both json and deterministicJson formats"),
+ }, "Cannot specify both json and deterministicJson formats")
+ .refine(
+ x => !x.some(f => f.type === "rawBase64") || x.length === 1,
+ "The rawBase64 format cannot be combined with other formats",
+ ),
headers: z.record(z.string(), z.string()).optional(),
includeTags: z
.string()
@@ -1105,6 +1111,10 @@ const parseRequestSchemaBase = baseScrapeOptions.extend({
});
export const parseRequestSchema = strictWithMessage(parseRequestSchemaBase)
+ .refine(
+ x => !x.formats.some(format => format.type === "rawBase64"),
+ "The rawBase64 format is not supported for parse uploads",
+ )
.refine(waitForRefine, waitForRefineOpts)
.transform(x => {
const { file, ...scrapeLike } = x;
@@ -1304,6 +1314,7 @@ export type Document = {
blocks?: PdfPageBlocks[];
html?: string;
rawHtml?: string;
+ rawBase64?: string;
links?: string[];
images?: string[];
screenshot?: string;
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index 942b51314..06a4d3151 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -195,24 +195,29 @@ async function performFireEngineScrape<
status = scrape as FireEngineCheckStatusSuccess;
}
- await specialtyScrapeCheck(
- logger.child({
- method: "performFireEngineScrape/specialtyScrapeCheck",
- }),
- status.responseHeaders,
- status,
- );
+ const wantsRawBase64 =
+ hasFormatOfType(meta.options.formats, "rawBase64") !== undefined;
+
+ if (!wantsRawBase64) {
+ await specialtyScrapeCheck(
+ logger.child({
+ method: "performFireEngineScrape/specialtyScrapeCheck",
+ }),
+ status.responseHeaders,
+ status,
+ );
+ }
const contentType =
(Object.entries(status.responseHeaders ?? {}).find(
x => x[0].toLowerCase() === "content-type",
) ?? [])[1] ?? "";
- if (contentType.includes("application/json")) {
+ if (!wantsRawBase64 && contentType.includes("application/json")) {
status.content = await getInnerJson(status.content);
}
- if (status.file) {
+ if (status.file && !wantsRawBase64) {
const content = status.file.content;
delete status.file;
let buffer = Buffer.from(content, "base64");
@@ -318,16 +323,34 @@ export async function scrapeURLWithFireEngineChromeCDP(
"engine.url": meta.url,
"engine.team_id": meta.internalOptions.teamId,
});
+ const wantsRawBase64 =
+ hasFormatOfType(meta.options.formats, "rawBase64") !== undefined;
+ if (
+ wantsRawBase64 &&
+ ((meta.options.waitFor ?? 0) > 0 ||
+ (meta.options.actions?.length ?? 0) > 0)
+ ) {
+ meta.logger.warn(
+ "rawBase64 returns the original response body; waitFor and actions are ignored.",
+ {
+ waitFor: meta.options.waitFor,
+ actionTypes: meta.options.actions?.map(action => action.type),
+ },
+ );
+ }
const hasBranding = hasFormatOfType(meta.options.formats, "branding");
const hasAudio = hasFormatOfType(meta.options.formats, "audio");
const hasVideo = hasFormatOfType(meta.options.formats, "video");
- const shouldRunYoutubePostprocessor = youtubePostprocessor.shouldRun(
- meta,
- new URL(meta.rewrittenUrl ?? meta.url),
- );
+ const shouldRunYoutubePostprocessor =
+ !wantsRawBase64 &&
+ youtubePostprocessor.shouldRun(
+ meta,
+ new URL(meta.rewrittenUrl ?? meta.url),
+ );
const defaultWait = hasBranding ? BRANDING_DEFAULT_WAIT_MS : 0;
- const effectiveWait =
- meta.options.waitFor != null && meta.options.waitFor !== 0
+ const effectiveWait = wantsRawBase64
+ ? 0
+ : meta.options.waitFor != null && meta.options.waitFor !== 0
? meta.options.waitFor
: defaultWait;
@@ -344,7 +367,7 @@ export async function scrapeURLWithFireEngineChromeCDP(
: []),
// Include specified actions
- ...(meta.options.actions ?? []).map(action => {
+ ...(!wantsRawBase64 ? (meta.options.actions ?? []) : []).map(action => {
const { metadata: _, ...rest } = action as InternalAction;
return rest;
}),
@@ -407,6 +430,7 @@ export async function scrapeURLWithFireEngineChromeCDP(
url: meta.rewrittenUrl ?? meta.url,
scrapeId: meta.id,
engine: "chrome-cdp",
+ ...(wantsRawBase64 ? { format: "rawBase64" as const } : {}),
instantReturn: false,
skipTlsVerification: meta.options.skipTlsVerification,
headers: meta.options.headers,
@@ -508,6 +532,7 @@ export async function scrapeURLWithFireEngineChromeCDP(
url: response.url ?? meta.url,
html: response.content,
+ rawBase64: response.file?.content,
markdown: contentType?.includes("text/markdown")
? response.content
: undefined,
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
index 6e028c566..03f481702 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -30,6 +30,7 @@ const browserCookieSchema = z
export type FireEngineScrapeRequestCommon = {
url: string;
scrapeId?: string;
+ format?: "html" | "rawBase64";
headers?: { [K: string]: string };
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 040180633..890712962 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -34,7 +34,11 @@ import {
} from "../../../controllers/v2/types";
import type { PdfMetadata, PdfPageBlocks } from "./pdf/types";
import { BrandingProfile } from "../../../types/branding";
-import { AgentIndexOnlyError, BrandingNotSupportedError } from "../error";
+import {
+ AgentIndexOnlyError,
+ BrandingNotSupportedError,
+ NoCachedDataError,
+} from "../error";
import { isUrlBlocked } from "../../WebScraper/utils/blocklist";
import {
canUseExchangeForRequest,
@@ -141,6 +145,7 @@ export type EngineScrapeResult = {
url: string;
html: string;
+ rawBase64?: string;
markdown?: string;
pages?: Array<{ pageNumber: number; markdown: string }>;
blocks?: PdfPageBlocks[];
@@ -587,6 +592,27 @@ export async function buildFallbackList(meta: Meta): Promise<
unsupportedFeatures: Set<FeatureFlag>;
}[]
> {
+ if (hasFormatOfType(meta.options.formats, "rawBase64")) {
+ if (meta.internalOptions.agentIndexOnly) {
+ throw new AgentIndexOnlyError();
+ }
+
+ if (meta.options.minAge !== undefined) {
+ throw new NoCachedDataError();
+ }
+
+ if (meta.options.lockdown || (!useFireEngine && meta.mock === null)) {
+ return [];
+ }
+
+ return [
+ {
+ engine: "fire-engine;chrome-cdp",
+ unsupportedFeatures: new Set(),
+ },
+ ];
+ }
+
if (
!meta.internalOptions.agentIndexOnly &&
meta.internalOptions.forceEngine === undefined
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 26e8b3cc3..c2792ab2c 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -535,6 +535,7 @@ async function scrapeURLLoopIter(
const hasQuestion = hasFormatOfType(meta.options.formats, "question");
const hasHighlights = hasFormatOfType(meta.options.formats, "highlights");
const hasQuery = hasFormatOfType(meta.options.formats, "query");
+ const hasRawBase64 = hasFormatOfType(meta.options.formats, "rawBase64");
const needsMarkdown =
hasMarkdown ||
hasChangeTracking ||
@@ -548,7 +549,9 @@ async function scrapeURLLoopIter(
const htmlSize = engineResult.html?.length ?? 0;
const shouldSkipMarkdownCheck = htmlSize > MAX_HTML_SIZE_FOR_MARKDOWN_CHECK;
- if (
+ if (hasRawBase64) {
+ checkMarkdown = engineResult.rawBase64 !== undefined ? "rawBase64" : "";
+ } else if (
meta.internalOptions.teamId === "sitemap" ||
meta.internalOptions.teamId === "robots-txt"
) {
@@ -597,6 +600,9 @@ async function scrapeURLLoopIter(
(engineResult.statusCode >= 200 && engineResult.statusCode < 300) ||
engineResult.statusCode === 304;
const hasNoPageError = engineResult.error === undefined;
+ const hasRequiredOutput = hasRawBase64
+ ? engineResult.rawBase64 !== undefined
+ : isLongEnough || !isGoodStatusCode;
const isLikelyProxyError = [401, 403, 429].includes(
engineResult.statusCode,
);
@@ -622,7 +628,7 @@ async function scrapeURLLoopIter(
// NOTE: TODO: what to do when status code is bad is tough...
// we cannot just rely on text because error messages can be brief and not hit the limit
// should we just use all the fallbacks and pick the one with the longest text? - mogery
- if (isLongEnough || !isGoodStatusCode) {
+ if (hasRequiredOutput) {
meta.logger.info("Scrape via " + engine + " deemed successful.", {
factors: { isLongEnough, isGoodStatusCode, hasNoPageError },
});
@@ -954,6 +960,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
for (const postprocessor of postprocessors) {
if (
+ !hasFormatOfType(meta.options.formats, "rawBase64") &&
postprocessor.shouldRun(
meta,
new URL(engineResult.url),
@@ -987,6 +994,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
pages: engineResult.pages,
blocks: engineResult.blocks,
rawHtml: engineResult.html,
+ rawBase64: engineResult.rawBase64,
json: engineResult.json,
screenshot: engineResult.screenshot,
actions: engineResult.actions,
diff --git a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts b/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts
index c5d4a6d9a..4abfae155 100644
--- a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts
+++ b/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts
@@ -8,6 +8,7 @@ import { scrapeURL } from ".";
import { scrapeOptions } from "../../controllers/v2/types";
import { Engine } from "./engines";
import { CostTracking } from "../../lib/cost-tracking";
+import { AgentIndexOnlyError, NoCachedDataError } from "./error";
// Mock parseMarkdown but delegate to real implementation for other tests
vi.mock("../../lib/html-to-markdown", async importOriginal => {
@@ -34,6 +35,92 @@ const testEnginesScreenshot: (Engine | undefined)[] = [
];
describe("Standalone scrapeURL tests", () => {
+ it.each([
+ ["https://example.com/raw", "text/html; charset=utf-8"],
+ ["https://example.com/raw.pdf", "application/pdf"],
+ [
+ "https://example.com/raw.docx",
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ ],
+ ])(
+ "returns rawBase64 through fire-engine Chrome for %s",
+ async (url, contentType) => {
+ const out = await scrapeURL(
+ "test:raw-base64",
+ url,
+ scrapeOptions.parse({
+ formats: ["rawBase64"],
+ useMock: "raw-base64",
+ fastMode: true,
+ blockAds: false,
+ proxy: "stealth",
+ }),
+ { forceEngine: "fetch", teamId: "test", orgId: null },
+ new CostTracking(),
+ );
+
+ expect(out.success).toBe(true);
+ if (out.success) {
+ expect(out.document.rawBase64).toBe("PGh0bWw+cmF3PC9odG1sPg==");
+ expect(out.document).not.toHaveProperty("markdown");
+ expect(out.document).not.toHaveProperty("html");
+ expect(out.document).not.toHaveProperty("rawHtml");
+ expect(out.document.metadata.contentType).toBe(contentType);
+ }
+ },
+ );
+
+ it("rejects a rawBase64 engine response without file content", async () => {
+ const out = await scrapeURL(
+ "test:raw-base64-missing",
+ "https://example.com/raw-error",
+ scrapeOptions.parse({
+ formats: ["rawBase64"],
+ useMock: "raw-base64",
+ }),
+ { teamId: "test", orgId: null },
+ new CostTracking(),
+ );
+
+ expect(out.success).toBe(false);
+ });
+
+ it("returns the canonical agent-index-only error for rawBase64", async () => {
+ const out = await scrapeURL(
+ "test:raw-base64-agent-index-only",
+ "https://example.com/raw",
+ scrapeOptions.parse({
+ formats: ["rawBase64"],
+ useMock: "raw-base64",
+ }),
+ { teamId: "test", orgId: null, agentIndexOnly: true },
+ new CostTracking(),
+ );
+
+ expect(out.success).toBe(false);
+ if (!out.success) {
+ expect(out.error).toBeInstanceOf(AgentIndexOnlyError);
+ }
+ });
+
+ it("rejects rawBase64 when minAge is set", async () => {
+ const out = await scrapeURL(
+ "test:raw-base64-min-age",
+ "https://example.com/raw",
+ scrapeOptions.parse({
+ formats: ["rawBase64"],
+ minAge: 0,
+ }),
+ { teamId: "test", orgId: null },
+ new CostTracking(),
+ );
+
+ expect(out.success).toBe(false);
+ if (!out.success) {
+ expect(out.error).toBeInstanceOf(NoCachedDataError);
+ }
+ });
+
describe.each(testEngines)("Engine %s", (forceEngine: Engine | undefined) => {
it("Basic scrape", async () => {
const out = await scrapeURL(
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts
index 2df9b5da8..3695ae78f 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts
@@ -366,6 +366,7 @@ async function performLLMExtractUnlessNativeJson(
function coerceFieldsToFormats(meta: Meta, document: Document): Document {
const hasMarkdown = hasFormatOfType(meta.options.formats, "markdown");
const hasRawHtml = hasFormatOfType(meta.options.formats, "rawHtml");
+ const hasRawBase64 = hasFormatOfType(meta.options.formats, "rawBase64");
const hasHtml = hasFormatOfType(meta.options.formats, "html");
const hasLinks = hasFormatOfType(meta.options.formats, "links");
const hasImages = hasFormatOfType(meta.options.formats, "images");
@@ -407,6 +408,14 @@ function coerceFieldsToFormats(meta: Meta, document: Document): Document {
);
}
+ if (!hasRawBase64 && document.rawBase64 !== undefined) {
+ delete document.rawBase64;
+ } else if (hasRawBase64 && document.rawBase64 === undefined) {
+ meta.logger.warn(
+ "Request had format: rawBase64, but there was no rawBase64 field in the result.",
+ );
+ }
+
if (!hasHtml && document.html !== undefined) {
delete document.html;
} else if (hasHtml && document.html === undefined) {
@@ -661,6 +670,10 @@ export async function executeTransformers(
meta: Meta,
document: Document,
): Promise<Document> {
+ if (hasFormatOfType(meta.options.formats, "rawBase64")) {
+ return coerceFieldsToFormats(meta, document);
+ }
+
const executions: [string, number][] = [];
for (const transformer of transformerStack) {
diff --git a/apps/api/v1-openapi.json b/apps/api/v1-openapi.json
index 9fec50fcf..d18d70826 100644
--- a/apps/api/v1-openapi.json
+++ b/apps/api/v1-openapi.json
@@ -2097,6 +2097,7 @@
"markdown",
"html",
"rawHtml",
+ "rawBase64",
"links",
"screenshot",
"screenshot@fullPage",
@@ -2105,7 +2106,7 @@
"branding"
]
},
- "description": "Formats to include in the output.",
+ "description": "Formats to include in the output. `rawBase64` must be requested by itself.",
"default": ["markdown"]
},
"onlyMainContent": {
@@ -2426,6 +2427,11 @@
"nullable": true,
"description": "Raw HTML content of the page if `rawHtml` is in `formats`"
},
+ "rawBase64": {
+ "type": "string",
+ "nullable": true,
+ "description": "Base64-encoded original response body if `rawBase64` is in `formats`"
+ },
"screenshot": {
"type": "string",
"nullable": true,
@@ -2672,6 +2678,11 @@
"nullable": true,
"description": "Raw HTML content of the page if `includeRawHtml` is true"
},
+ "rawBase64": {
+ "type": "string",
+ "nullable": true,
+ "description": "Base64-encoded original response body if `rawBase64` is in `formats`"
+ },
"links": {
"type": "array",
"items": {
@@ -2809,6 +2820,11 @@
"nullable": true,
"description": "Raw HTML content of the page if `includeRawHtml` is true"
},
+ "rawBase64": {
+ "type": "string",
+ "nullable": true,
+ "description": "Base64-encoded original response body if `rawBase64` is in `formats`"
+ },
"links": {
"type": "array",
"items": {