Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
ENG-3089: Support both string and object format inputs in v2 scrape API (#1932)
Gergő Móricz64e3e794ee7a
diff --git a/apps/api/src/__tests__/queue-concurrency-integration.test.ts b/apps/api/src/__tests__/queue-concurrency-integration.test.tsindex 6eb1d9586..6d2b6f6a3 100644--- a/apps/api/src/__tests__/queue-concurrency-integration.test.ts+++ b/apps/api/src/__tests__/queue-concurrency-integration.test.ts@@ -35,16 +35,7 @@ describe("Queue Concurrency Integration", () => {const mockNow = Date.now();const defaultScrapeOptions = {- formats: ["markdown"] as (- | "markdown"- | "html"- | "rawHtml"- | "links"- | "screenshot"- | "screenshot@fullPage"- | "extract"- | "json"- )[],+ formats: [{ type: "markdown" }],onlyMainContent: true,waitFor: 0,mobile: false,diff --git a/apps/api/src/__tests__/snips/v2/scrape-formats.test.ts b/apps/api/src/__tests__/snips/v2/scrape-formats.test.tsnew file mode 100644index 000000000..85e4fba9e--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/scrape-formats.test.ts@@ -0,0 +1,278 @@+import { scrape, scrapeWithFailure, scrapeTimeout, idmux, Identity, scrapeRaw } from "./lib";++let identity: Identity;++beforeAll(async () => {+ identity = await idmux({+ name: "scrape-formats",+ concurrency: 100,+ credits: 1000000,+ });+}, 10000 + scrapeTimeout);++describe("Scrape format variations", () => {+ describe("String format inputs", () => {+ it.concurrent("accepts string format for markdown", async () => {+ const response = await scrape({+ url: "https://firecrawl.dev",+ formats: [{ type: "markdown" }],+ timeout: scrapeTimeout,+ }, identity);++ expect(response.markdown).toBeDefined();+ expect(typeof response.markdown).toBe("string");+ expect(response.markdown?.length).toBeGreaterThan(0);+ }, scrapeTimeout);++ it.concurrent("accepts multiple string formats", async () => {+ const response = await scrape({+ url: "https://firecrawl.dev",+ formats: ["markdown", "html", "links"],+ timeout: scrapeTimeout,+ }, identity);++ expect(response.markdown).toBeDefined();+ expect(response.html).toBeDefined();+ expect(response.links).toBeDefined();+ expect(Array.isArray(response.links)).toBe(true);+ }, scrapeTimeout);++ it.concurrent("accepts string format for screenshot", async () => {+ if (process.env.TEST_SUITE_SELF_HOSTED) {+ return;+ }++ const response = await scrape({+ url: "https://firecrawl.dev",+ formats: ["screenshot"],+ timeout: scrapeTimeout,+ }, identity);++ expect(response.screenshot).toBeDefined();+ expect(typeof response.screenshot).toBe("string");+ }, scrapeTimeout);+ });++ describe("Object format inputs", () => {+ it.concurrent("accepts object format for markdown", async () => {+ const response = await scrape({+ url: "https://firecrawl.dev",+ formats: [{ type: "markdown" }],+ timeout: scrapeTimeout,+ }, identity);++ expect(response.markdown).toBeDefined();+ expect(typeof response.markdown).toBe("string");+ expect(response.markdown?.length).toBeGreaterThan(0);+ }, scrapeTimeout);++ it.concurrent("accepts multiple object formats", async () => {+ const response = await scrape({+ url: "https://firecrawl.dev",+ formats: [+ { type: "markdown" },+ { type: "html" },+ { type: "links" }+ ],+ timeout: scrapeTimeout,+ }, identity);++ expect(response.markdown).toBeDefined();+ expect(response.html).toBeDefined();+ expect(response.links).toBeDefined();+ expect(Array.isArray(response.links)).toBe(true);+ }, scrapeTimeout);++ it.concurrent("accepts object format for screenshot with options", async () => {+ if (process.env.TEST_SUITE_SELF_HOSTED) {+ return;+ }++ const response = await scrape({+ url: "https://firecrawl.dev",+ formats: [{+ type: "screenshot",+ fullPage: true,+ quality: 80,+ viewport: { width: 1920, height: 1080 }+ }],+ timeout: scrapeTimeout,+ }, identity);++ expect(response.screenshot).toBeDefined();+ expect(typeof response.screenshot).toBe("string");+ }, scrapeTimeout);+ });++ describe("Mixed format inputs", () => {+ it.concurrent("accepts mixed string and object formats", async () => {+ const response = await scrape({+ url: "https://firecrawl.dev",+ formats: [+ "markdown",+ { type: "html" },+ "links"+ ],+ timeout: scrapeTimeout,+ }, identity);++ expect(response.markdown).toBeDefined();+ expect(response.html).toBeDefined();+ expect(response.links).toBeDefined();+ }, scrapeTimeout);++ it.concurrent("handles complex formats alongside simple ones", async () => {+ if (!process.env.TEST_SUITE_SELF_HOSTED) {+ const response = await scrape({+ url: "https://firecrawl.dev",+ formats: [+ "markdown",+ {+ type: "screenshot",+ fullPage: false,+ quality: 90+ },+ { type: "links" }+ ],+ timeout: scrapeTimeout,+ }, identity);++ expect(response.markdown).toBeDefined();+ expect(response.screenshot).toBeDefined();+ expect(response.links).toBeDefined();+ }+ }, scrapeTimeout);+ });++ describe("Format with options that already exist", () => {+ it.concurrent("handles json format with schema", async () => {+ if (!process.env.TEST_SUITE_SELF_HOSTED || process.env.OPENAI_API_KEY || process.env.OLLAMA_BASE_URL) {+ const response = await scrape({+ url: "https://firecrawl.dev",+ formats: [{+ type: "json",+ prompt: "Extract the main heading and description",+ schema: {+ type: "object",+ properties: {+ heading: { type: "string" },+ description: { type: "string" }+ }+ }+ }],+ timeout: scrapeTimeout,+ }, identity);++ expect(response.json).toBeDefined();+ expect(typeof response.json).toBe("object");+ }+ }, scrapeTimeout);++ it.concurrent("handles changeTracking format with options", async () => {+ if (!process.env.TEST_SUITE_SELF_HOSTED) {+ const response = await scrape({+ url: "https://firecrawl.dev",+ formats: [+ "markdown",+ {+ type: "changeTracking",+ modes: ["json"],+ tag: "test-tag"+ }+ ],+ timeout: scrapeTimeout,+ }, identity);++ expect(response.markdown).toBeDefined();+ expect(response.changeTracking).toBeDefined();+ }+ }, scrapeTimeout);+ });++ describe("Edge cases and validation", () => {+ it.concurrent("default format is markdown when formats not specified", async () => {+ const response = await scrape({+ url: "https://firecrawl.dev",+ timeout: scrapeTimeout,+ }, identity);++ expect(response.markdown).toBeDefined();+ expect(typeof response.markdown).toBe("string");+ }, scrapeTimeout);++ it.concurrent("rejects invalid format type in object", async () => {+ const raw = await scrapeRaw({+ url: "https://firecrawl.dev",+ formats: [{ type: "invalid-format" } as any],+ timeout: scrapeTimeout,+ }, identity);++ expect(raw.statusCode).toBe(400);+ expect(raw.body.success).toBe(false);+ }, scrapeTimeout);++ it.concurrent("maintains backward compatibility with string-only formats", async () => {+ const response = await scrape({+ url: "https://firecrawl.dev",+ formats: ["markdown", "html", "rawHtml", "links"],+ timeout: scrapeTimeout,+ }, identity);++ expect(response.markdown).toBeDefined();+ expect(response.html).toBeDefined();+ expect(response.rawHtml).toBeDefined();+ expect(response.links).toBeDefined();+ }, scrapeTimeout);+ });++ describe("Format type consistency in output", () => {+ it.concurrent("string input produces consistent output structure", async () => {+ const response = await scrape({+ url: "https://firecrawl.dev",+ formats: ["markdown", "html"],+ timeout: scrapeTimeout,+ }, identity);++ const keys = Object.keys(response);+ expect(keys).toContain("markdown");+ expect(keys).toContain("html");+ expect(keys).toContain("metadata");+ }, scrapeTimeout);++ it.concurrent("object input produces identical output structure", async () => {+ const response = await scrape({+ url: "https://firecrawl.dev",+ formats: [{ type: "markdown" }, { type: "html" }],+ timeout: scrapeTimeout,+ }, identity);++ const keys = Object.keys(response);+ expect(keys).toContain("markdown");+ expect(keys).toContain("html");+ expect(keys).toContain("metadata");+ }, scrapeTimeout);++ it.concurrent("mixed input produces consistent output", async () => {+ const response1 = await scrape({+ url: "https://firecrawl.dev",+ formats: ["markdown", "html"],+ timeout: scrapeTimeout,+ }, identity);++ const response2 = await scrape({+ url: "https://firecrawl.dev",+ formats: [{ type: "markdown" }, { type: "html" }],+ timeout: scrapeTimeout,+ }, identity);++ const response3 = await scrape({+ url: "https://firecrawl.dev",+ formats: ["markdown", { type: "html" }],+ timeout: scrapeTimeout,+ }, identity);++ expect(Object.keys(response1).sort()).toEqual(Object.keys(response2).sort());+ expect(Object.keys(response2).sort()).toEqual(Object.keys(response3).sort());+ }, scrapeTimeout);+ });+});\ No newline at end of filediff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.tsindex 9b684d6ea..98a50f93f 100644--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts@@ -3,6 +3,8 @@ import crypto from "crypto";let identity: Identity;+let changeTrackingTestUrl = "https://firecrawl.dev?testId=" + crypto.randomUUID();+beforeAll(async () => {identity = await idmux({name: "scrape",@@ -13,7 +15,7 @@ beforeAll(async () => {if (!process.env.TEST_SUITE_SELF_HOSTED) {// Needed for change tracking tests to workawait scrape({- url: "https://example.com",+ url: changeTrackingTestUrl,formats: ["markdown", "changeTracking"],timeout: scrapeTimeout,}, identity);@@ -296,7 +298,7 @@ describe("Scrape tests", () => {const response3 = await scrape({url,- formats: ["screenshot@fullPage"],+ formats: [{ type: "screenshot", fullPage: true }],timeout: scrapeTimeout,maxAge: scrapeTimeout * 3,}, identity);@@ -311,7 +313,7 @@ describe("Scrape tests", () => {const response1 = await scrape({url,- formats: ["screenshot@fullPage"],+ formats: [{ type: "screenshot", fullPage: true }],timeout: scrapeTimeout,maxAge: 0,}, identity);@@ -322,7 +324,7 @@ describe("Scrape tests", () => {const response2 = await scrape({url,- formats: ["screenshot@fullPage"],+ formats: [{ type: "screenshot", fullPage: true }],timeout: scrapeTimeout,maxAge: scrapeTimeout * 2,}, identity);@@ -591,7 +593,7 @@ describe("Scrape tests", () => {describe("Change Tracking format", () => {it.concurrent("works", async () => {const response = await scrape({- url: "https://example.com",+ url: changeTrackingTestUrl,formats: ["markdown", "changeTracking"],timeout: scrapeTimeout,}, identity);@@ -602,7 +604,7 @@ describe("Scrape tests", () => {it.concurrent("includes git diff when requested", async () => {const response = await scrape({- url: "https://example.com",+ url: changeTrackingTestUrl,formats: ["markdown", { type: "changeTracking", modes: ["git-diff"] }],timeout: scrapeTimeout,}, identity);@@ -620,7 +622,7 @@ describe("Scrape tests", () => {it.concurrent("includes structured output when requested", async () => {const response = await scrape({- url: "https://example.com",+ url: changeTrackingTestUrl,formats: ["markdown", { type: "changeTracking", modes: ["json"], prompt: "Summarize the changes between the previous and current content" }],timeout: scrapeTimeout,}, identity);@@ -635,7 +637,7 @@ describe("Scrape tests", () => {it.concurrent("supports schema-based extraction for change tracking", async () => {const response = await scrape({- url: "https://example.com",+ url: changeTrackingTestUrl,formats: ["markdown",{@@ -680,7 +682,7 @@ describe("Scrape tests", () => {it.concurrent("supports both git-diff and structured modes together", async () => {const response = await scrape({- url: "https://example.com",+ url: changeTrackingTestUrl,formats: ["markdown", { type: "changeTracking", modes: ["git-diff", "json"], schema: {type: "object",properties: {@@ -711,13 +713,13 @@ describe("Scrape tests", () => {const uuid2 = crypto.randomUUID();const response1 = await scrape({- url: "https://firecrawl.dev/",+ url: changeTrackingTestUrl,formats: ["markdown", { type: "changeTracking", tag: uuid1 }],timeout: scrapeTimeout,}, identity);const response2 = await scrape({- url: "https://firecrawl.dev/",+ url: changeTrackingTestUrl,formats: ["markdown", { type: "changeTracking", tag: uuid2 }],timeout: scrapeTimeout,}, identity);@@ -728,7 +730,7 @@ describe("Scrape tests", () => {expect(response2.changeTracking?.changeStatus).toBe("new");const response3 = await scrape({- url: "https://firecrawl.dev/",+ url: changeTrackingTestUrl,formats: ["markdown", { type: "changeTracking", tag: uuid1 }],timeout: scrapeTimeout,}, identity);@@ -771,7 +773,7 @@ describe("Scrape tests", () => {it.concurrent("screenshot@fullPage format works", async () => {const response = await scrape({url: "http://firecrawl.dev",- formats: ["screenshot@fullPage"],+ formats: [{ type: "screenshot", fullPage: true }],timeout: scrapeTimeout,}, identity);diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex 254bc53a9..a9542e55e 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -11,6 +11,7 @@ import { v4 as uuidv4 } from "uuid";import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import { getJobPriority } from "../../lib/job-priority";import { getScrapeQueue } from "../../services/queue-service";+import { hasFormatOfType } from "../../lib/format-utils";export async function scrapeController(req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,@@ -124,7 +125,7 @@ export async function scrapeController(await getScrapeQueue().remove(jobId);- if (!req.body.formats.includes("rawHtml")) {+ if (!hasFormatOfType(req.body.formats, "rawHtml")) {if (doc && doc.rawHtml) {delete doc.rawHtml;}diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex a080fae13..2136b8c20 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -209,6 +209,16 @@ export const screenshotFormatWithOptions = z.object({export type ScreenshotFormatWithOptions = z.output<typeof screenshotFormatWithOptions>;+export type FormatObject =+ | { type: "markdown" }+ | { type: "html" }+ | { type: "rawHtml" }+ | { type: "links" }+ | { type: "summary" }+ | JsonFormatWithOptions+ | ChangeTrackingFormatWithOptions+ | ScreenshotFormatWithOptions;+export const parsersSchema = z.array(z.enum(["pdf"])).default(["pdf"]);export type Parsers = z.infer<typeof parsersSchema>;@@ -222,40 +232,43 @@ function transformIframeSelector(selector: string): string {const baseScrapeOptions = z.object({- formats: z- .union([- z.enum([- "markdown",- "html",- "rawHtml",- "links",- "screenshot",- "screenshot@fullPage",- "extract",- "summary",- "changeTracking",- ]),- jsonFormatWithOptions,- changeTrackingFormatWithOptions,- screenshotFormatWithOptions,- ])- .array()- .optional()- .default(["markdown"])+ formats: z.preprocess(+ (val) => {+ if (!Array.isArray(val)) return val;+ return val.map(format => {+ if (typeof format === 'string') {+ return { type: format };+ }+ return format;+ });+ },+ z+ .union([+ z.object({ type: z.literal("markdown") }),+ z.object({ type: z.literal("html") }),+ z.object({ type: z.literal("rawHtml") }),+ z.object({ type: z.literal("links") }),+ z.object({ type: z.literal("summary") }),+ jsonFormatWithOptions,+ changeTrackingFormatWithOptions,+ screenshotFormatWithOptions,+ ])+ .array()+ .optional()+ .default([{ type: "markdown" }])+ ).refine((x) => {- const hasStringScreenshot = x.includes("screenshot") || x.includes("screenshot@fullPage");- const hasObjectScreenshot = x.find(f => typeof f === "object" && f.type === "screenshot");- return !(hasStringScreenshot && hasObjectScreenshot);+ return x.filter(f => f.type === "screenshot").length <= 1;},- "You may only specify either string screenshot formats or object screenshot format, not both",+ "You may only specify one screenshot format",).refine(- (x) => !(x.includes("screenshot") && x.includes("screenshot@fullPage")),- "You may only specify either screenshot or screenshot@fullPage",- )- .refine(- (x) => !x.includes("changeTracking") || x.includes("markdown"),+ (x) => {+ const hasChangeTracking = x.find(f => f.type === "changeTracking");+ const hasMarkdown = x.find(f => f.type === "markdown");+ return !hasChangeTracking || hasMarkdown;+ },"The changeTracking format requires the markdown format to be specified as well",),headers: z.record(z.string(), z.string()).optional(),@@ -1068,7 +1081,7 @@ export function fromV0ScrapeOptions((pageOptions.includeRawHtml ?? false) ? ("rawHtml" as const) : null,(pageOptions.screenshot ?? false) ? ("screenshot" as const) : null,(pageOptions.fullPageScreenshot ?? false)- ? ("screenshot@fullPage" as const)+ ? ({ type: "screenshot" as const, fullPage: true }): null,extractorOptions !== undefined &&extractorOptions.mode.includes("llm-extraction")@@ -1184,6 +1197,8 @@ export function fromV1ScrapeOptions(prompt: opts?.prompt,};return fmt;+ } else if (x === "screenshot@fullPage") {+ return { type: "screenshot" as const, fullPage: true };} else {return x;}@@ -1309,20 +1324,36 @@ export const searchRequestSchema = z__searchPreviewToken: z.string().optional(),scrapeOptions: baseScrapeOptions.extend({- formats: z- .array(- z.enum([- "markdown",- "html",- "rawHtml",- "links",- "screenshot",- "screenshot@fullPage",- "extract",- "json",- ]),- )- .default([]),+ formats: z.preprocess(+ (val) => {+ if (!Array.isArray(val)) return val;+ return val.map(format => {+ if (typeof format === 'string') {+ return { type: format };+ }+ return format;+ });+ },+ z+ .union([+ z.object({ type: z.literal("markdown") }),+ z.object({ type: z.literal("html") }),+ z.object({ type: z.literal("rawHtml") }),+ z.object({ type: z.literal("links") }),+ z.object({ type: z.literal("summary") }),+ jsonFormatWithOptions,+ screenshotFormatWithOptions,+ ])+ .array()+ .optional()+ .default([])+ )+ .refine(+ (x) => {+ return x.filter(f => f.type === "screenshot").length <= 1;+ },+ "You may only specify one screenshot format",+ ),}).default({}),})diff --git a/apps/api/src/lib/format-utils.ts b/apps/api/src/lib/format-utils.tsnew file mode 100644index 000000000..a668d6f17--- /dev/null+++ b/apps/api/src/lib/format-utils.ts@@ -0,0 +1,79 @@+import { FormatObject } from "../controllers/v2/types";++/**+ * Checks if a format of a specific type exists in the formats array.+ * Returns the format object if found, undefined otherwise.+ *+ * This function handles both simple formats (e.g., { type: "markdown" })+ * and complex formats with additional properties (e.g., { type: "screenshot", fullPage: true }).+ *+ * @param formats - Array of format objects+ * @param type - The format type to search for+ * @returns The format object if found, undefined otherwise+ */+export function hasFormatOfType<T extends FormatObject["type"]>(+ formats: FormatObject[] | undefined,+ type: T+): Extract<FormatObject, { type: T }> | undefined {+ if (!formats) {+ return undefined;+ }++ const found = formats.find(f => f.type === type);+ return found as Extract<FormatObject, { type: T }> | undefined;+}++/**+ * Checks if any of the specified format types exist in the formats array.+ * Returns true if at least one of the types is found.+ *+ * @param formats - Array of format objects+ * @param types - Array of format types to search for+ * @returns true if any of the types are found, false otherwise+ */+export function hasAnyFormatOfTypes(+ formats: FormatObject[] | undefined,+ types: FormatObject["type"][]+): boolean {+ if (!formats) {+ return false;+ }++ return formats.some(f => types.includes(f.type));+}++/**+ * Gets all format objects of a specific type from the formats array.+ * This is useful when multiple formats of the same type might exist.+ *+ * @param formats - Array of format objects+ * @param type - The format type to filter by+ * @returns Array of format objects matching the type+ */+export function getFormatsOfType<T extends FormatObject["type"]>(+ formats: FormatObject[] | undefined,+ type: T+): Extract<FormatObject, { type: T }>[] {+ if (!formats) {+ return [];+ }++ return formats.filter(f => f.type === type) as Extract<FormatObject, { type: T }>[];+}++/**+ * Extracts just the format types from an array of format objects.+ * Useful for checking which formats are present without their options.+ *+ * @param formats - Array of format objects+ * @returns Array of format type strings+ */+export function getFormatTypes(+ formats: FormatObject[] | undefined+): FormatObject["type"][] {+ if (!formats) {+ return [];+ }++ return formats.map(f => f.type);+}\ No newline at end of filediff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.tsindex 925d967e0..ec47a82c5 100644--- a/apps/api/src/lib/scrape-billing.ts+++ b/apps/api/src/lib/scrape-billing.ts@@ -1,6 +1,7 @@import { InternalOptions } from "src/scraper/scrapeURL";import { Document, ScrapeOptions, TeamFlags } from "../controllers/v2/types";import { CostTracking } from "./extract/extraction-service";+import { hasFormatOfType } from "./format-utils";const creditsPerPDFPage = 1;const stealthProxyCostBonus = 4;@@ -20,7 +21,8 @@ export async function calculateCreditsToBeBilled(options: ScrapeOptions, interna}let creditsToBeBilled = 1; // Assuming 1 credit per document- if (options.formats.find(x => typeof x === "object" && x.type === "json") || (options.formats.find(x => typeof x === "object" && x.type === "changeTracking")?.modes?.includes("json"))) {+ const changeTrackingFormat = hasFormatOfType(options.formats, "changeTracking");+ if (hasFormatOfType(options.formats, "json") || changeTrackingFormat?.modes?.includes("json")) {creditsToBeBilled = 5;}diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.tsindex d3a967de4..d89a51783 100644--- a/apps/api/src/main/runWebScraper.ts+++ b/apps/api/src/main/runWebScraper.ts@@ -32,7 +32,7 @@ export async function startWebScraperPipeline({...job.data.scrapeOptions,...(job.data.crawl_id? {- formats: job.data.scrapeOptions.formats.concat(["rawHtml"]),+ formats: job.data.scrapeOptions.formats.concat([{ type: "rawHtml" }]),}: {}),},diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex f8059d82e..b681bddea 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -28,6 +28,7 @@ import { specialtyScrapeCheck } from "../utils/specialtyHandler";import { fireEngineDelete } from "./delete";import { MockState } from "../../lib/mock";import { getInnerJSON } from "../../../../lib/html-transformer";+import { hasFormatOfType, hasAnyFormatOfTypes } from "../../../../lib/format-utils";import { Action, TimeoutSignal } from "../../../../controllers/v1/types";// This function does not take `Meta` on purpose. It may not access any@@ -205,16 +206,13 @@ export async function scrapeURLWithFireEngineChromeCDP(...(meta.options.actions ?? []),// Transform screenshot format into an action (unsupported by chrome-cdp)- ...(meta.options.formats.includes("screenshot") ||- meta.options.formats.includes("screenshot@fullPage") ||- meta.options.formats.find(x => typeof x === "object" && x.type === "screenshot")+ ...(hasFormatOfType(meta.options.formats, "screenshot")? [{type: "screenshot" as const,- fullPage: meta.options.formats.includes("screenshot@fullPage") ||- meta.options.formats.find(x => typeof x === "object" && x.type === "screenshot")?.fullPage || false,- ...(meta.options.formats.find(x => typeof x === "object" && x.type === "screenshot")?.viewport ?- { viewport: meta.options.formats.find(x => typeof x === "object" && x.type === "screenshot")?.viewport } : {}),+ fullPage: hasFormatOfType(meta.options.formats, "screenshot")?.fullPage || false,+ ...(hasFormatOfType(meta.options.formats, "screenshot")?.viewport ?+ { viewport: hasFormatOfType(meta.options.formats, "screenshot")!.viewport } : {}),},]: []),@@ -290,11 +288,7 @@ export async function scrapeURLWithFireEngineChromeCDP(true,);- if (- meta.options.formats.includes("screenshot") ||- meta.options.formats.includes("screenshot@fullPage") ||- meta.options.formats.find(x => typeof x === "object" && x.type === "screenshot")- ) {+ if (hasFormatOfType(meta.options.formats, "screenshot")) {// meta.logger.debug(// "Transforming screenshots from actions into screenshot field",// { screenshots: response.screenshots },@@ -358,10 +352,8 @@ export async function scrapeURLWithFireEnginePlaywright(headers: meta.options.headers,priority: meta.internalOptions.priority,- screenshot: meta.options.formats.includes("screenshot") ||- !!meta.options.formats.find(x => typeof x === "object" && x.type === "screenshot"),- fullPageScreenshot: meta.options.formats.includes("screenshot@fullPage") ||- meta.options.formats.find(x => typeof x === "object" && x.type === "screenshot")?.fullPage,+ screenshot: hasFormatOfType(meta.options.formats, "screenshot") !== undefined,+ fullPageScreenshot: hasFormatOfType(meta.options.formats, "screenshot")?.fullPage,wait: meta.options.waitFor,geolocation: meta.options.location,blockAds: meta.options.blockAds,diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 797b7494f..bc734e010 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -11,6 +11,7 @@ import { scrapeURLWithFetch } from "./fetch";import { scrapeURLWithPlaywright } from "./playwright";import { scrapeURLWithIndex } from "./index/index";import { useIndex } from "../../../services";+import { hasFormatOfType } from "../../../lib/format-utils";export type Engine =| "fire-engine;chrome-cdp"@@ -424,7 +425,7 @@ export function buildFallbackList(meta: Meta): {useIndex&& process.env.FIRECRAWL_INDEX_WRITE_ONLY !== "true"&& meta.options.waitFor === 0- && !meta.options.formats.includes("changeTracking")+ && !hasFormatOfType(meta.options.formats, "changeTracking")&& meta.options.maxAge !== 0&& (meta.options.headers === undefineddiff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 90e35adde..bd3afe5d2 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -12,6 +12,7 @@ import {scrapeURLWithEngine,} from "./engines";import { parseMarkdown } from "../../lib/html-to-markdown";+import { hasFormatOfType } from "../../lib/format-utils";import {ActionError,AddFeatureError,@@ -84,17 +85,12 @@ function buildFeatureFlags(flags.add("actions");}- if (options.formats.includes("screenshot")) {- flags.add("screenshot");- }-- if (options.formats.includes("screenshot@fullPage")) {- flags.add("screenshot@fullScreen");- }-- const screenshotFormat = options.formats.find(x => typeof x === "object" && x.type === "screenshot");- if (screenshotFormat) {- flags.add(screenshotFormat.fullPage ? "screenshot@fullScreen" : "screenshot");+ if (hasFormatOfType(options.formats, "screenshot")) {+ if (hasFormatOfType(options.formats, "screenshot")?.fullPage) {+ flags.add("screenshot@fullScreen");+ } else {+ flags.add("screenshot");+ }}if (options.waitFor !== 0) {diff --git a/apps/api/src/scraper/scrapeURL/transformers/agent.ts b/apps/api/src/scraper/scrapeURL/transformers/agent.tsindex 207d1c480..b00a6063a 100644--- a/apps/api/src/scraper/scrapeURL/transformers/agent.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/agent.ts@@ -5,6 +5,7 @@ import { Meta } from "..";import { logger } from "../../../lib/logger";import { parseMarkdown } from "../../../lib/html-to-markdown";import { smartScrape, SmartScrapeResult } from "../lib/smartScrape";+import { hasFormatOfType } from "../../../lib/format-utils";export async function performAgent(@@ -49,11 +50,11 @@ export async function performAgent(const html = smartscrapeResults.scrapedPages[smartscrapeResults.scrapedPages.length - 1].html- if (meta.options.formats.includes("markdown")) {+ if (hasFormatOfType(meta.options.formats, "markdown")) {const markdown = await parseMarkdown(html)document.markdown = markdown}- if (meta.options.formats.includes("html")) {+ if (hasFormatOfType(meta.options.formats, "html")) {document.html = html}}diff --git a/apps/api/src/scraper/scrapeURL/transformers/diff.ts b/apps/api/src/scraper/scrapeURL/transformers/diff.tsindex 890d24ba4..1c3b2a6f0 100644--- a/apps/api/src/scraper/scrapeURL/transformers/diff.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/diff.ts@@ -5,9 +5,10 @@ import { getJob } from "../../../controllers/v1/crawl-status";import gitDiff from 'git-diff';import parseDiff from 'parse-diff';import { generateCompletions } from "./llmExtract";+import { hasFormatOfType } from "../../../lib/format-utils";async function extractDataWithSchema(content: string, meta: Meta): Promise<{ extract: any } | null> {- const changeTrackingFormat = meta.options.formats.find(x => typeof x === "object" && x.type === "changeTracking")!;+ const changeTrackingFormat = hasFormatOfType(meta.options.formats, "changeTracking")!;try {const { extract } = await generateCompletions({@@ -15,7 +16,7 @@ async function extractDataWithSchema(content: string, meta: Meta): Promise<{ extmethod: "extractDataWithSchema/generateCompletions",}),options: {- schema: changeTrackingFormat.schema,+ schema: changeTrackingFormat?.schema as any,systemPrompt: "Extract the requested information from the content based on the provided schema.",temperature: 0},@@ -59,10 +60,9 @@ function compareExtractedData(previousData: any, currentData: any): any {}export async function deriveDiff(meta: Meta, document: Document): Promise<Document> {- const changeTrackingFormat = meta.options.formats.find(x => typeof x === "object" && x.type === "changeTracking");- const changeTrackingSimple = meta.options.formats.includes("changeTracking");+ const changeTrackingFormat = hasFormatOfType(meta.options.formats, "changeTracking");- if (changeTrackingFormat || changeTrackingSimple) {+ if (changeTrackingFormat) {if (meta.internalOptions.zeroDataRetention) {document.warning = "Change tracking is not supported with zero data retention." + (document.warning ? " " + document.warning : "")return document;@@ -70,7 +70,7 @@ export async function deriveDiff(meta: Meta, document: Document): Promise<Documeconst start = Date.now();const res = await supabase_service- .rpc("diff_get_last_scrape_4", {+ .rpc("diff_get_last_scrape_5", {i_team_id: meta.internalOptions.teamId,i_url: document.metadata.sourceURL ?? meta.rewrittenUrl ?? meta.url,i_tag: changeTrackingFormat?.tag ?? null,@@ -157,10 +157,10 @@ export async function deriveDiff(meta: Meta, document: Document): Promise<Documeif (changeTrackingFormat?.modes?.includes("json") && changeStatus === "changed") {try {- const previousData = changeTrackingFormat.schema ?+ const previousData = changeTrackingFormat?.schema ?await extractDataWithSchema(previousMarkdown, meta) : null;- const currentData = changeTrackingFormat.schema ?+ const currentData = changeTrackingFormat?.schema ?await extractDataWithSchema(currentMarkdown, meta) : null;if (previousData && currentData) {@@ -172,8 +172,8 @@ export async function deriveDiff(meta: Meta, document: Document): Promise<Docume}),options: {systemPrompt: "Analyze the differences between the previous and current content and provide a structured summary of the changes.",- schema: changeTrackingFormat.schema,- prompt: changeTrackingFormat.prompt,+ schema: changeTrackingFormat?.schema,+ prompt: changeTrackingFormat?.prompt,temperature: 0},markdown: `Previous Content:\n${previousMarkdown}\n\nCurrent Content:\n${currentMarkdown}`,diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex 2515d4062..f0cde5b22 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -12,6 +12,7 @@ import { performAgent } from "./agent";import { deriveDiff } from "./diff";import { useIndex } from "../../../services/index";import { sendDocumentToIndex } from "../engines/index/index";+import { hasFormatOfType, hasAnyFormatOfTypes } from "../../../lib/format-utils";export type Transformer = (meta: Meta,@@ -102,7 +103,7 @@ export async function deriveMarkdownFromHTML(export async function deriveLinksFromHTML(meta: Meta, document: Document): Promise<Document> {// Only derive if the formats has links- if (meta.options.formats.includes("links")) {+ if (hasFormatOfType(meta.options.formats, "links")) {if (document.html === undefined) {throw new Error("html is undefined -- this transformer is being called out of order",@@ -119,38 +120,40 @@ export function coerceFieldsToFormats(meta: Meta,document: Document,): Document {- const formats = new Set(meta.options.formats);- const hasJson = meta.options.formats.find(x => typeof x === "object" && x.type === "json");- const hasChangeTrackingObject = meta.options.formats.find(x => typeof x === "object" && x.type === "changeTracking");- const hasScreenshot = meta.options.formats.find(x => typeof x === "object" && x.type === "screenshot");-- if (!formats.has("markdown") && document.markdown !== undefined) {+ const hasMarkdown = hasFormatOfType(meta.options.formats, "markdown");+ const hasRawHtml = hasFormatOfType(meta.options.formats, "rawHtml");+ const hasHtml = hasFormatOfType(meta.options.formats, "html");+ const hasLinks = hasFormatOfType(meta.options.formats, "links");+ const hasChangeTracking = hasFormatOfType(meta.options.formats, "changeTracking");+ const hasJson = hasFormatOfType(meta.options.formats, "json");+ const hasScreenshot = hasFormatOfType(meta.options.formats, "screenshot");+ const hasSummary = hasFormatOfType(meta.options.formats, "summary");++ if (!hasMarkdown && document.markdown !== undefined) {delete document.markdown;- } else if (formats.has("markdown") && document.markdown === undefined) {+ } else if (hasMarkdown && document.markdown === undefined) {meta.logger.warn("Request had format: markdown, but there was no markdown field in the result.",);}- if (!formats.has("rawHtml") && document.rawHtml !== undefined) {+ if (!hasRawHtml && document.rawHtml !== undefined) {delete document.rawHtml;- } else if (formats.has("rawHtml") && document.rawHtml === undefined) {+ } else if (hasRawHtml && document.rawHtml === undefined) {meta.logger.warn("Request had format: rawHtml, but there was no rawHtml field in the result.",);}- if (!formats.has("html") && document.html !== undefined) {+ if (!hasHtml && document.html !== undefined) {delete document.html;- } else if (formats.has("html") && document.html === undefined) {+ } else if (hasHtml && document.html === undefined) {meta.logger.warn("Request had format: html, but there was no html field in the result.",);}if (- !formats.has("screenshot") &&- !formats.has("screenshot@fullPage") &&!hasScreenshot &&document.screenshot !== undefined) {@@ -159,7 +162,7 @@ export function coerceFieldsToFormats();delete document.screenshot;} else if (- (formats.has("screenshot") || formats.has("screenshot@fullPage") || hasScreenshot) &&+ hasScreenshot &&document.screenshot === undefined) {meta.logger.warn(@@ -167,12 +170,12 @@ export function coerceFieldsToFormats();}- if (!formats.has("links") && document.links !== undefined) {+ if (!hasLinks && document.links !== undefined) {meta.logger.warn("Removed links from Document because it wasn't in formats -- this is wasteful and indicates a bug.",);delete document.links;- } else if (formats.has("links") && document.links === undefined) {+ } else if (hasLinks && document.links === undefined) {meta.logger.warn("Request had format: links, but there was no links field in the result.",);@@ -214,30 +217,30 @@ export function coerceFieldsToFormats();}- if (!formats.has("summary") && document.summary !== undefined) {+ if (!hasSummary && document.summary !== undefined) {meta.logger.warn("Removed summary from Document because it wasn't in formats -- this is wasteful and indicates a bug.",);delete document.summary;- } else if (formats.has("summary") && document.summary === undefined) {+ } else if (hasSummary && document.summary === undefined) {meta.logger.warn("Request had format summary, but there was no summary field in the result.",);}- if (!hasChangeTrackingObject && !formats.has("changeTracking") && document.changeTracking !== undefined) {+ if (!hasChangeTracking && document.changeTracking !== undefined) {meta.logger.warn("Removed changeTracking from Document because it wasn't in formats -- this is extremely wasteful and indicates a bug.",);delete document.changeTracking;- } else if ((formats.has("changeTracking") || hasChangeTrackingObject) && document.changeTracking === undefined) {+ } else if (hasChangeTracking && document.changeTracking === undefined) {meta.logger.warn("Request had format changeTracking, but there was no changeTracking field in the result.",);}if (document.changeTracking &&- (!hasChangeTrackingObject?.modes?.includes("git-diff")) &&+ (!hasChangeTracking?.modes?.includes("git-diff")) &&document.changeTracking.diff !== undefined) {meta.logger.warn("Removed diff from changeTracking because git-diff mode wasn't specified in changeTrackingOptions.modes.",@@ -246,7 +249,7 @@ export function coerceFieldsToFormats(}if (document.changeTracking &&- (!hasChangeTrackingObject?.modes?.includes("json")) &&+ (!hasChangeTracking?.modes?.includes("json")) &&document.changeTracking.json !== undefined) {meta.logger.warn("Removed structured from changeTracking because structured mode wasn't specified in changeTrackingOptions.modes.",diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex e5f1860f8..ff7071bbc 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -24,6 +24,7 @@ import Ajv from "ajv";import { extractData } from "../lib/extractSmartScrape";import { CostTracking } from "../../../lib/extract/extraction-service";import { isAgentExtractModelValid } from "../../../controllers/v1/types";+import { hasFormatOfType } from "../../../lib/format-utils";// TODO: fix this, it's horribletype LanguageModelV1ProviderMetadata = {anthropic?: {@@ -657,7 +658,7 @@ export async function performLLMExtract(meta: Meta,document: Document,): Promise<Document> {- const jsonFormat = meta.options.formats.find(x => typeof x === "object" && x.type === "json") as JsonFormatWithOptions | undefined;+ const jsonFormat = hasFormatOfType(meta.options.formats, "json");// Debug logging for v1 format investigationif (meta.internalOptions.v1OriginalFormat) {@@ -823,7 +824,7 @@ export async function performSummary(meta: Meta,document: Document,): Promise<Document> {- if (meta.options.formats.includes("summary")) {+ if (hasFormatOfType(meta.options.formats, "summary")) {if (meta.internalOptions.zeroDataRetention) {document.warning = "Summary mode is not supported with zero data retention." + (document.warning ? " " + document.warning : "")return document;diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.tsindex 9a3d568b3..aee617f70 100644--- a/apps/api/src/services/worker/crawl-logic.ts+++ b/apps/api/src/services/worker/crawl-logic.ts@@ -12,6 +12,7 @@ import { addScrapeJobs } from "../queue-jobs";import { getJobs } from "../../controllers/v1/crawl-status";import { logJob } from "../logging/log_job";import { callWebhook } from "../webhook";+import { hasFormatOfType } from "../../lib/format-utils";export async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {const logger = _logger.child({@@ -52,7 +53,7 @@ export async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredClet lastUrls: string[] = [];const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";- if (useDbAuthentication && sc.scrapeOptions.formats.includes("changeTracking")) {+ if (useDbAuthentication && hasFormatOfType(sc.scrapeOptions.formats, "changeTracking")) {lastUrls = ((await supabase_service.rpc("diff_get_last_crawl_urls", {diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex a80bc4a58..518099d28 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -29,6 +29,7 @@ import {import { addScrapeJob, addScrapeJobs } from "../queue-jobs";import { getJobPriority } from "../../lib/job-priority";import { Document, scrapeOptions, TeamFlags } from "../../controllers/v2/types";+import { hasFormatOfType } from "../../lib/format-utils";import { getACUCTeam } from "../../controllers/auth";import { callWebhook } from "../webhook";import { CustomError } from "../../lib/custom-error";@@ -185,7 +186,7 @@ async function processJob(job: Job & { id: string }) {const rawHtml = doc.rawHtml ?? "";- if (!job.data.scrapeOptions.formats.includes("rawHtml")) {+ if (!hasFormatOfType(job.data.scrapeOptions.formats, "rawHtml")) {delete doc.rawHtml;}