Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Fix v1 API JSON/extract format backward compatibility on v2 (#1917)
Gergő Móricz40267b6e6c8a
diff --git a/apps/api/src/__tests__/snips/v1-json-extract-format.test.ts b/apps/api/src/__tests__/snips/v1-json-extract-format.test.tsnew file mode 100644index 000000000..b85f9e924--- /dev/null+++ b/apps/api/src/__tests__/snips/v1-json-extract-format.test.ts@@ -0,0 +1,177 @@+import { scrape, scrapeTimeout, idmux, Identity } from "./lib";++let identity: Identity;++beforeAll(async () => {+ identity = await idmux({+ name: "v1-json-extract-format",+ concurrency: 100,+ credits: 1000000,+ });+}, 10000 + scrapeTimeout);++describe("V1 JSON/Extract Format Backward Compatibility", () => {+ if (!process.env.TEST_SUITE_SELF_HOSTED || process.env.OPENAI_API_KEY || process.env.OLLAMA_BASE_URL) {+ describe("extract format", () => {+ it.concurrent("should return extracted data in 'extract' field when using format='extract'", async () => {+ const response = await scrape({+ url: "https://jsonplaceholder.typicode.com/posts/1",+ formats: ["extract"],+ extract: {+ schema: {+ type: "object",+ properties: {+ userId: { type: "number" },+ id: { type: "number" },+ title: { type: "string" },+ body: { type: "string" }+ },+ required: ["userId", "id", "title", "body"]+ }+ },+ timeout: scrapeTimeout,+ }, identity);++ // Data should be in extract field, not json field+ expect(response.extract).toBeDefined();+ expect(response.json).toBeUndefined();+ expect(response.extract.userId).toBe(1);+ expect(response.extract.id).toBe(1);+ expect(response.extract.title).toBeDefined();+ expect(response.extract.body).toBeDefined();+ }, scrapeTimeout);++ it.concurrent("should work with extract format and custom prompt", async () => {+ const response = await scrape({+ url: "https://firecrawl.dev",+ formats: ["extract"],+ extract: {+ prompt: "Extract the main heading from the page",+ schema: {+ type: "object",+ properties: {+ mainHeading: { type: "string" }+ },+ required: ["mainHeading"]+ }+ },+ timeout: scrapeTimeout,+ }, identity);++ // Data should be in extract field, not json field+ expect(response.extract).toBeDefined();+ expect(response.json).toBeUndefined();+ expect(response.extract.mainHeading).toBeDefined();+ }, scrapeTimeout);+ });++ describe("json format", () => {+ it.concurrent("should return extracted data in 'json' field when using format='json'", async () => {+ const response = await scrape({+ url: "https://jsonplaceholder.typicode.com/posts/1",+ formats: ["json"],+ jsonOptions: {+ schema: {+ type: "object",+ properties: {+ userId: { type: "number" },+ id: { type: "number" },+ title: { type: "string" },+ body: { type: "string" }+ },+ required: ["userId", "id", "title", "body"]+ }+ },+ timeout: scrapeTimeout,+ }, identity);++ // Data should be in json field, not extract field+ expect(response.json).toBeDefined();+ expect(response.extract).toBeUndefined();+ expect(response.json.userId).toBe(1);+ expect(response.json.id).toBe(1);+ expect(response.json.title).toBeDefined();+ expect(response.json.body).toBeDefined();+ }, scrapeTimeout);++ it.concurrent("should work with json format and custom prompt", async () => {+ const response = await scrape({+ url: "https://firecrawl.dev",+ formats: ["json"],+ jsonOptions: {+ prompt: "Extract the main heading from the page",+ schema: {+ type: "object",+ properties: {+ mainHeading: { type: "string" }+ },+ required: ["mainHeading"]+ }+ },+ timeout: scrapeTimeout,+ }, identity);++ // Data should be in json field, not extract field+ expect(response.json).toBeDefined();+ expect(response.extract).toBeUndefined();+ expect(response.json.mainHeading).toBeDefined();+ }, scrapeTimeout);+ });++ describe("multiple formats", () => {+ it.concurrent("should return markdown and extracted data in correct fields", async () => {+ const response = await scrape({+ url: "https://firecrawl.dev",+ formats: ["markdown", "extract"],+ extract: {+ schema: {+ type: "object",+ properties: {+ mainHeading: { type: "string" },+ hasLinks: { type: "boolean" }+ },+ required: ["mainHeading", "hasLinks"]+ }+ },+ timeout: scrapeTimeout,+ }, identity);++ // Both markdown and extract should be present+ expect(response.markdown).toBeDefined();+ expect(response.extract).toBeDefined();+ expect(response.json).toBeUndefined();+ expect(response.extract.mainHeading).toBeDefined();+ expect(typeof response.extract.hasLinks).toBe("boolean");+ }, scrapeTimeout);++ it.concurrent("should return markdown and json data in correct fields", async () => {+ const response = await scrape({+ url: "https://firecrawl.dev",+ formats: ["markdown", "json"],+ jsonOptions: {+ schema: {+ type: "object",+ properties: {+ mainHeading: { type: "string" },+ hasLinks: { type: "boolean" }+ },+ required: ["mainHeading", "hasLinks"]+ }+ },+ timeout: scrapeTimeout,+ }, identity);++ // Both markdown and json should be present+ expect(response.markdown).toBeDefined();+ expect(response.json).toBeDefined();+ expect(response.extract).toBeUndefined();+ expect(response.json.mainHeading).toBeDefined();+ expect(typeof response.json.hasLinks).toBe("boolean");+ }, scrapeTimeout);+ });+ } else {+ it("should skip LLM tests in self-hosted mode without LLM keys", () => {+ expect(true).toBe(true);+ });+ }+});\ No newline at end of filediff --git a/apps/api/src/__tests__/snips/v2-scrape-skip-tls.test.ts b/apps/api/src/__tests__/snips/v2-scrape-skip-tls.test.tsindex cce6c8e6d..709dde822 100644--- a/apps/api/src/__tests__/snips/v2-scrape-skip-tls.test.ts+++ b/apps/api/src/__tests__/snips/v2-scrape-skip-tls.test.ts@@ -59,7 +59,7 @@ describe("V2 Scrape skipTlsVerification Default", () => {.set("Authorization", `Bearer ${identity.apiKey}`).set("Content-Type", "application/json").send({- url: "https://example.com",+ url: "https://firecrawl.dev",timeout: scrapeTimeout,});@@ -70,7 +70,7 @@ describe("V2 Scrape skipTlsVerification Default", () => {expect(response.status).toBe(200);expect(response.body.success).toBe(true);expect(response.body.data).toBeDefined();- expect(response.body.data.markdown).toContain("Example Domain");+ expect(response.body.data.markdown).toContain("Firecrawl");}, scrapeTimeout);if (!process.env.TEST_SUITE_SELF_HOSTED) {@@ -80,7 +80,7 @@ describe("V2 Scrape skipTlsVerification Default", () => {.set("Authorization", `Bearer ${identity.apiKey}`).set("Content-Type", "application/json").send({- url: "https://example.com",+ url: "https://firecrawl.dev",formats: [{ type: "screenshot", fullPage: false }],timeout: scrapeTimeout,});@@ -102,7 +102,7 @@ describe("V2 Scrape skipTlsVerification Default", () => {.set("Authorization", `Bearer ${identity.apiKey}`).set("Content-Type", "application/json").send({- url: "https://example.com",+ url: "https://firecrawl.dev",formats: [{ type: "screenshot", fullPage: true }],timeout: scrapeTimeout,});diff --git a/apps/api/src/__tests__/snips/v2-system-prompt-rejection.test.ts b/apps/api/src/__tests__/snips/v2-system-prompt-rejection.test.tsindex 7d6fed06d..0e2947014 100644--- a/apps/api/src/__tests__/snips/v2-system-prompt-rejection.test.ts+++ b/apps/api/src/__tests__/snips/v2-system-prompt-rejection.test.ts@@ -27,7 +27,7 @@ describe("V2 System Prompt Rejection", () => {it("should reject systemPrompt in json format options for v2 scrape", async () => {const response = await scrapeV2Raw({- url: "https://example.com",+ url: "https://firecrawl.dev",formats: [{type: "json",@@ -51,7 +51,7 @@ describe("V2 System Prompt Rejection", () => {it("should accept valid json format options without systemPrompt", async () => {const response = await scrapeV2Raw({- url: "https://example.com",+ url: "https://firecrawl.dev",formats: [{type: "json",diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 90b4b5eb1..8942ea93a 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1113,6 +1113,8 @@ export function fromV1ScrapeOptions(delete (spreadScrapeOptions as any).origin;delete (spreadScrapeOptions as any).integration;delete (spreadScrapeOptions as any).webhook;+ delete (spreadScrapeOptions as any).zeroDataRetention;+ delete (spreadScrapeOptions as any).maxConcurrency;delete spreadScrapeOptions.__experimental_cache;delete spreadScrapeOptions.jsonOptions;@@ -1121,6 +1123,16 @@ export function fromV1ScrapeOptions(delete spreadScrapeOptions.geolocation;delete (spreadScrapeOptions as any).parsePDF;+ // Track the original format for v1 backward compatibility+ // Check json first since when user specifies "json", both "json" and "extract" are present+ // When user specifies "extract", only "extract" is present+ let v1OriginalFormat: "extract" | "json" | undefined;+ if (v1ScrapeOptions.formats.includes("json")) {+ v1OriginalFormat = "json";+ } else if (v1ScrapeOptions.formats.includes("extract")) {+ v1OriginalFormat = "extract";+ }+return {scrapeOptions: scrapeOptions.parse({...spreadScrapeOptions,@@ -1140,6 +1152,16 @@ export function fromV1ScrapeOptions(};return fmt;} else if (x === "json") {+ // If jsonOptions are provided with json format, create JsonFormatWithOptions+ const opts = v1ScrapeOptions.jsonOptions;+ if (opts) {+ const fmt: JsonFormatWithOptions = {+ type: "json",+ schema: opts.schema,+ prompt: opts.prompt,+ };+ return v1ScrapeOptions.formats.includes("extract") ? null : fmt;+ }return null;} else if (x === "changeTracking") {const opts = v1ScrapeOptions.changeTrackingOptions;@@ -1162,6 +1184,7 @@ export function fromV1ScrapeOptions(v1Agent: v1ScrapeOptions.agent,v1JSONSystemPrompt: (v1ScrapeOptions.jsonOptions || v1ScrapeOptions.extract)?.systemPrompt,v1JSONAgent: (v1ScrapeOptions.jsonOptions || v1ScrapeOptions.extract)?.agent,+ v1OriginalFormat,},};}diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 1a477502c..90e35adde 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -236,6 +236,7 @@ export type InternalOptions = {v1Agent?: ScrapeOptionsV1["agent"];v1JSONAgent?: Exclude<ScrapeOptionsV1["jsonOptions"], undefined>["agent"];v1JSONSystemPrompt?: string;+ v1OriginalFormat?: "extract" | "json"; // Track original v1 format for backward compatibility};export type EngineResultsTracker = {diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex f8fabef45..0d6380be4 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -178,11 +178,36 @@ export function coerceFieldsToFormats();}+ // Handle v1 backward compatibility - don't delete fields based on v1OriginalFormat+ const shouldKeepExtract = meta.internalOptions.v1OriginalFormat === "extract";+ const shouldKeepJson = meta.internalOptions.v1OriginalFormat === "json";++ // Debug logging for v1 format investigation+ if (meta.internalOptions.v1OriginalFormat) {+ meta.logger.debug("coerceFieldsToFormats v1 format debug", {+ v1OriginalFormat: meta.internalOptions.v1OriginalFormat,+ hasJson: !!hasJson,+ shouldKeepExtract,+ shouldKeepJson,+ hasExtractField: document.extract !== undefined,+ hasJsonField: document.json !== undefined+ });+ }+if (!hasJson && (document.extract !== undefined || document.json !== undefined)) {- meta.logger.warn(- "Removed json from Document because it wasn't in formats -- this is extremely wasteful and indicates a bug.",- );- delete document.extract;+ // For v1 API, keep the field specified by v1OriginalFormat+ if (!shouldKeepExtract && document.extract !== undefined) {+ meta.logger.warn(+ "Removed extract from Document because it wasn't in formats -- this is extremely wasteful and indicates a bug.",+ );+ delete document.extract;+ }+ if (!shouldKeepJson && document.json !== undefined) {+ meta.logger.warn(+ "Removed json from Document because it wasn't in formats -- this is extremely wasteful and indicates a bug.",+ );+ delete document.json;+ }} else if (hasJson && document.extract === undefined && document.json === undefined) {meta.logger.warn("Request had format json, but there was no json field in the result.",diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex 99fc01545..e5f1860f8 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -658,6 +658,16 @@ export async function performLLMExtract(document: Document,): Promise<Document> {const jsonFormat = meta.options.formats.find(x => typeof x === "object" && x.type === "json") as JsonFormatWithOptions | undefined;++ // Debug logging for v1 format investigation+ if (meta.internalOptions.v1OriginalFormat) {+ meta.logger.debug("performLLMExtract v1 format debug", {+ v1OriginalFormat: meta.internalOptions.v1OriginalFormat,+ hasJsonFormat: !!jsonFormat,+ formats: meta.options.formats.map(f => typeof f === "object" ? f.type : f)+ });+ }+if (jsonFormat) {if (meta.internalOptions.zeroDataRetention) {document.warning = "JSON mode is not supported with zero data retention." + (document.warning ? " " + document.warning : "")@@ -787,7 +797,22 @@ export async function performLLMExtract(// }// Assign the final extracted data- document.json = extractedData;+ // For v1 API backward compatibility, check the original format+ meta.logger.debug("Assigning extracted data", {+ v1OriginalFormat: meta.internalOptions.v1OriginalFormat,+ hasExtractedData: !!extractedData,+ assigningTo: meta.internalOptions.v1OriginalFormat === "extract" ? "extract" :+ meta.internalOptions.v1OriginalFormat === "json" ? "json" : "json (default)"+ });++ if (meta.internalOptions.v1OriginalFormat === "extract") {+ document.extract = extractedData;+ } else if (meta.internalOptions.v1OriginalFormat === "json") {+ document.json = extractedData;+ } else {+ // v2 API or no v1OriginalFormat - use json field+ document.json = extractedData;+ }// document.warning = warning;}