Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Fix v1 API JSON/extract format backward compatibility on v2 (#1917)

Gergő Móricz40267b6e6c8a
diff --git a/apps/api/src/__tests__/snips/v1-json-extract-format.test.ts b/apps/api/src/__tests__/snips/v1-json-extract-format.test.ts
new file mode 100644
index 000000000..b85f9e924
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v1-json-extract-format.test.ts
@@ -0,0 +1,177 @@
+import { scrape, scrapeTimeout, idmux, Identity } from "./lib";
+
+let identity: Identity;
+
+beforeAll(async () => {
+ identity = await idmux({
+ name: "v1-json-extract-format",
+ concurrency: 100,
+ credits: 1000000,
+ });
+}, 10000 + scrapeTimeout);
+
+describe("V1 JSON/Extract Format Backward Compatibility", () => {
+ if (!process.env.TEST_SUITE_SELF_HOSTED || process.env.OPENAI_API_KEY || process.env.OLLAMA_BASE_URL) {
+ describe("extract format", () => {
+ it.concurrent("should return extracted data in 'extract' field when using format='extract'", async () => {
+ const response = await scrape({
+ url: "https://jsonplaceholder.typicode.com/posts/1",
+ formats: ["extract"],
+ extract: {
+ schema: {
+ type: "object",
+ properties: {
+ userId: { type: "number" },
+ id: { type: "number" },
+ title: { type: "string" },
+ body: { type: "string" }
+ },
+ required: ["userId", "id", "title", "body"]
+ }
+ },
+ timeout: scrapeTimeout,
+ }, identity);
+
+ // Data should be in extract field, not json field
+ expect(response.extract).toBeDefined();
+ expect(response.json).toBeUndefined();
+ expect(response.extract.userId).toBe(1);
+ expect(response.extract.id).toBe(1);
+ expect(response.extract.title).toBeDefined();
+ expect(response.extract.body).toBeDefined();
+ }, scrapeTimeout);
+
+ it.concurrent("should work with extract format and custom prompt", async () => {
+ const response = await scrape({
+ url: "https://firecrawl.dev",
+ formats: ["extract"],
+ extract: {
+ prompt: "Extract the main heading from the page",
+ schema: {
+ type: "object",
+ properties: {
+ mainHeading: { type: "string" }
+ },
+ required: ["mainHeading"]
+ }
+ },
+ timeout: scrapeTimeout,
+ }, identity);
+
+ // Data should be in extract field, not json field
+ expect(response.extract).toBeDefined();
+ expect(response.json).toBeUndefined();
+ expect(response.extract.mainHeading).toBeDefined();
+ }, scrapeTimeout);
+ });
+
+ describe("json format", () => {
+ it.concurrent("should return extracted data in 'json' field when using format='json'", async () => {
+ const response = await scrape({
+ url: "https://jsonplaceholder.typicode.com/posts/1",
+ formats: ["json"],
+ jsonOptions: {
+ schema: {
+ type: "object",
+ properties: {
+ userId: { type: "number" },
+ id: { type: "number" },
+ title: { type: "string" },
+ body: { type: "string" }
+ },
+ required: ["userId", "id", "title", "body"]
+ }
+ },
+ timeout: scrapeTimeout,
+ }, identity);
+
+ // Data should be in json field, not extract field
+ expect(response.json).toBeDefined();
+ expect(response.extract).toBeUndefined();
+ expect(response.json.userId).toBe(1);
+ expect(response.json.id).toBe(1);
+ expect(response.json.title).toBeDefined();
+ expect(response.json.body).toBeDefined();
+ }, scrapeTimeout);
+
+ it.concurrent("should work with json format and custom prompt", async () => {
+ const response = await scrape({
+ url: "https://firecrawl.dev",
+ formats: ["json"],
+ jsonOptions: {
+ prompt: "Extract the main heading from the page",
+ schema: {
+ type: "object",
+ properties: {
+ mainHeading: { type: "string" }
+ },
+ required: ["mainHeading"]
+ }
+ },
+ timeout: scrapeTimeout,
+ }, identity);
+
+ // Data should be in json field, not extract field
+ expect(response.json).toBeDefined();
+ expect(response.extract).toBeUndefined();
+ expect(response.json.mainHeading).toBeDefined();
+ }, scrapeTimeout);
+ });
+
+ describe("multiple formats", () => {
+ it.concurrent("should return markdown and extracted data in correct fields", async () => {
+ const response = await scrape({
+ url: "https://firecrawl.dev",
+ formats: ["markdown", "extract"],
+ extract: {
+ schema: {
+ type: "object",
+ properties: {
+ mainHeading: { type: "string" },
+ hasLinks: { type: "boolean" }
+ },
+ required: ["mainHeading", "hasLinks"]
+ }
+ },
+ timeout: scrapeTimeout,
+ }, identity);
+
+ // Both markdown and extract should be present
+ expect(response.markdown).toBeDefined();
+ expect(response.extract).toBeDefined();
+ expect(response.json).toBeUndefined();
+ expect(response.extract.mainHeading).toBeDefined();
+ expect(typeof response.extract.hasLinks).toBe("boolean");
+ }, scrapeTimeout);
+
+ it.concurrent("should return markdown and json data in correct fields", async () => {
+ const response = await scrape({
+ url: "https://firecrawl.dev",
+ formats: ["markdown", "json"],
+ jsonOptions: {
+ schema: {
+ type: "object",
+ properties: {
+ mainHeading: { type: "string" },
+ hasLinks: { type: "boolean" }
+ },
+ required: ["mainHeading", "hasLinks"]
+ }
+ },
+ timeout: scrapeTimeout,
+ }, identity);
+
+ // Both markdown and json should be present
+ expect(response.markdown).toBeDefined();
+ expect(response.json).toBeDefined();
+ expect(response.extract).toBeUndefined();
+ expect(response.json.mainHeading).toBeDefined();
+ expect(typeof response.json.hasLinks).toBe("boolean");
+ }, scrapeTimeout);
+ });
+ } else {
+ it("should skip LLM tests in self-hosted mode without LLM keys", () => {
+ expect(true).toBe(true);
+ });
+ }
+});
\ No newline at end of file
diff --git a/apps/api/src/__tests__/snips/v2-scrape-skip-tls.test.ts b/apps/api/src/__tests__/snips/v2-scrape-skip-tls.test.ts
index cce6c8e6d..709dde822 100644
--- a/apps/api/src/__tests__/snips/v2-scrape-skip-tls.test.ts
+++ b/apps/api/src/__tests__/snips/v2-scrape-skip-tls.test.ts
@@ -59,7 +59,7 @@ describe("V2 Scrape skipTlsVerification Default", () => {
.set("Authorization", `Bearer ${identity.apiKey}`)
.set("Content-Type", "application/json")
.send({
- url: "https://example.com",
+ url: "https://firecrawl.dev",
timeout: scrapeTimeout,
});
@@ -70,7 +70,7 @@ describe("V2 Scrape skipTlsVerification Default", () => {
expect(response.status).toBe(200);
expect(response.body.success).toBe(true);
expect(response.body.data).toBeDefined();
- expect(response.body.data.markdown).toContain("Example Domain");
+ expect(response.body.data.markdown).toContain("Firecrawl");
}, scrapeTimeout);
if (!process.env.TEST_SUITE_SELF_HOSTED) {
@@ -80,7 +80,7 @@ describe("V2 Scrape skipTlsVerification Default", () => {
.set("Authorization", `Bearer ${identity.apiKey}`)
.set("Content-Type", "application/json")
.send({
- url: "https://example.com",
+ url: "https://firecrawl.dev",
formats: [{ type: "screenshot", fullPage: false }],
timeout: scrapeTimeout,
});
@@ -102,7 +102,7 @@ describe("V2 Scrape skipTlsVerification Default", () => {
.set("Authorization", `Bearer ${identity.apiKey}`)
.set("Content-Type", "application/json")
.send({
- url: "https://example.com",
+ url: "https://firecrawl.dev",
formats: [{ type: "screenshot", fullPage: true }],
timeout: scrapeTimeout,
});
diff --git a/apps/api/src/__tests__/snips/v2-system-prompt-rejection.test.ts b/apps/api/src/__tests__/snips/v2-system-prompt-rejection.test.ts
index 7d6fed06d..0e2947014 100644
--- a/apps/api/src/__tests__/snips/v2-system-prompt-rejection.test.ts
+++ b/apps/api/src/__tests__/snips/v2-system-prompt-rejection.test.ts
@@ -27,7 +27,7 @@ describe("V2 System Prompt Rejection", () => {
it("should reject systemPrompt in json format options for v2 scrape", async () => {
const response = await scrapeV2Raw(
{
- url: "https://example.com",
+ url: "https://firecrawl.dev",
formats: [
{
type: "json",
@@ -51,7 +51,7 @@ describe("V2 System Prompt Rejection", () => {
it("should accept valid json format options without systemPrompt", async () => {
const response = await scrapeV2Raw(
{
- url: "https://example.com",
+ url: "https://firecrawl.dev",
formats: [
{
type: "json",
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 90b4b5eb1..8942ea93a 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -1113,6 +1113,8 @@ export function fromV1ScrapeOptions(
delete (spreadScrapeOptions as any).origin;
delete (spreadScrapeOptions as any).integration;
delete (spreadScrapeOptions as any).webhook;
+ delete (spreadScrapeOptions as any).zeroDataRetention;
+ delete (spreadScrapeOptions as any).maxConcurrency;
delete spreadScrapeOptions.__experimental_cache;
delete spreadScrapeOptions.jsonOptions;
@@ -1121,6 +1123,16 @@ export function fromV1ScrapeOptions(
delete spreadScrapeOptions.geolocation;
delete (spreadScrapeOptions as any).parsePDF;
+ // Track the original format for v1 backward compatibility
+ // Check json first since when user specifies "json", both "json" and "extract" are present
+ // When user specifies "extract", only "extract" is present
+ let v1OriginalFormat: "extract" | "json" | undefined;
+ if (v1ScrapeOptions.formats.includes("json")) {
+ v1OriginalFormat = "json";
+ } else if (v1ScrapeOptions.formats.includes("extract")) {
+ v1OriginalFormat = "extract";
+ }
+
return {
scrapeOptions: scrapeOptions.parse({
...spreadScrapeOptions,
@@ -1140,6 +1152,16 @@ export function fromV1ScrapeOptions(
};
return fmt;
} else if (x === "json") {
+ // If jsonOptions are provided with json format, create JsonFormatWithOptions
+ const opts = v1ScrapeOptions.jsonOptions;
+ if (opts) {
+ const fmt: JsonFormatWithOptions = {
+ type: "json",
+ schema: opts.schema,
+ prompt: opts.prompt,
+ };
+ return v1ScrapeOptions.formats.includes("extract") ? null : fmt;
+ }
return null;
} else if (x === "changeTracking") {
const opts = v1ScrapeOptions.changeTrackingOptions;
@@ -1162,6 +1184,7 @@ export function fromV1ScrapeOptions(
v1Agent: v1ScrapeOptions.agent,
v1JSONSystemPrompt: (v1ScrapeOptions.jsonOptions || v1ScrapeOptions.extract)?.systemPrompt,
v1JSONAgent: (v1ScrapeOptions.jsonOptions || v1ScrapeOptions.extract)?.agent,
+ v1OriginalFormat,
},
};
}
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 1a477502c..90e35adde 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -236,6 +236,7 @@ export type InternalOptions = {
v1Agent?: ScrapeOptionsV1["agent"];
v1JSONAgent?: Exclude<ScrapeOptionsV1["jsonOptions"], undefined>["agent"];
v1JSONSystemPrompt?: string;
+ v1OriginalFormat?: "extract" | "json"; // Track original v1 format for backward compatibility
};
export type EngineResultsTracker = {
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts
index f8fabef45..0d6380be4 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts
@@ -178,11 +178,36 @@ export function coerceFieldsToFormats(
);
}
+ // Handle v1 backward compatibility - don't delete fields based on v1OriginalFormat
+ const shouldKeepExtract = meta.internalOptions.v1OriginalFormat === "extract";
+ const shouldKeepJson = meta.internalOptions.v1OriginalFormat === "json";
+
+ // Debug logging for v1 format investigation
+ if (meta.internalOptions.v1OriginalFormat) {
+ meta.logger.debug("coerceFieldsToFormats v1 format debug", {
+ v1OriginalFormat: meta.internalOptions.v1OriginalFormat,
+ hasJson: !!hasJson,
+ shouldKeepExtract,
+ shouldKeepJson,
+ hasExtractField: document.extract !== undefined,
+ hasJsonField: document.json !== undefined
+ });
+ }
+
if (!hasJson && (document.extract !== undefined || document.json !== undefined)) {
- meta.logger.warn(
- "Removed json from Document because it wasn't in formats -- this is extremely wasteful and indicates a bug.",
- );
- delete document.extract;
+ // For v1 API, keep the field specified by v1OriginalFormat
+ if (!shouldKeepExtract && document.extract !== undefined) {
+ meta.logger.warn(
+ "Removed extract from Document because it wasn't in formats -- this is extremely wasteful and indicates a bug.",
+ );
+ delete document.extract;
+ }
+ if (!shouldKeepJson && document.json !== undefined) {
+ meta.logger.warn(
+ "Removed json from Document because it wasn't in formats -- this is extremely wasteful and indicates a bug.",
+ );
+ delete document.json;
+ }
} else if (hasJson && document.extract === undefined && document.json === undefined) {
meta.logger.warn(
"Request had format json, but there was no json field in the result.",
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
index 99fc01545..e5f1860f8 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@ -658,6 +658,16 @@ export async function performLLMExtract(
document: Document,
): Promise<Document> {
const jsonFormat = meta.options.formats.find(x => typeof x === "object" && x.type === "json") as JsonFormatWithOptions | undefined;
+
+ // Debug logging for v1 format investigation
+ if (meta.internalOptions.v1OriginalFormat) {
+ meta.logger.debug("performLLMExtract v1 format debug", {
+ v1OriginalFormat: meta.internalOptions.v1OriginalFormat,
+ hasJsonFormat: !!jsonFormat,
+ formats: meta.options.formats.map(f => typeof f === "object" ? f.type : f)
+ });
+ }
+
if (jsonFormat) {
if (meta.internalOptions.zeroDataRetention) {
document.warning = "JSON mode is not supported with zero data retention." + (document.warning ? " " + document.warning : "")
@@ -787,7 +797,22 @@ export async function performLLMExtract(
// }
// Assign the final extracted data
- document.json = extractedData;
+ // For v1 API backward compatibility, check the original format
+ meta.logger.debug("Assigning extracted data", {
+ v1OriginalFormat: meta.internalOptions.v1OriginalFormat,
+ hasExtractedData: !!extractedData,
+ assigningTo: meta.internalOptions.v1OriginalFormat === "extract" ? "extract" :
+ meta.internalOptions.v1OriginalFormat === "json" ? "json" : "json (default)"
+ });
+
+ if (meta.internalOptions.v1OriginalFormat === "extract") {
+ document.extract = extractedData;
+ } else if (meta.internalOptions.v1OriginalFormat === "json") {
+ document.json = extractedData;
+ } else {
+ // v2 API or no v1OriginalFormat - use json field
+ document.json = extractedData;
+ }
// document.warning = warning;
}