Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(scrapeURL): handle contentType JSON better in markdown conversion (#1604)

Gergő Móricz38c96b524f12
diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.ts
index 9c2a9e2d5..a27057f78 100644
--- a/apps/api/src/__tests__/snips/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/scrape.test.ts
@@ -373,5 +373,14 @@ describe("Scrape tests", () => {
});
expect(response.metadata.sourceURL).toBe("https://firecrawl.dev/?pagewanted=all&et_blog");
- }, 35000);
+ }, 30000);
+
+ it.concurrent("application/json content type is markdownified properly", async () => {
+ const response = await scrape({
+ url: "https://jsonplaceholder.typicode.com/todos/1",
+ formats: ["markdown"],
+ });
+
+ expect(response.markdown).toContain("```json");
+ }, 30000);
});
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 908a51d3d..5ee1328d0 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -750,6 +750,7 @@ export type Document = {
scrapeId?: string;
error?: string;
numPages?: number;
+ contentType?: string;
proxyUsed: "basic" | "stealth";
// [key: string]: string | string[] | number | { smartScrape: number; other: number; total: number } | undefined;
};
diff --git a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts b/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts
index 40c343997..299b0d355 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts
@@ -30,7 +30,7 @@ export async function scrapeURLWithFetch(
url: string;
body: string,
status: number;
- headers: any;
+ headers: [string, string][];
};
if (meta.mock !== null) {
@@ -117,5 +117,8 @@ export async function scrapeURLWithFetch(
url: response.url,
html: response.body,
statusCode: response.status,
+ contentType: (response.headers.find(
+ (x) => x[0].toLowerCase() === "content-type",
+ ) ?? [])[1] ?? undefined,
};
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index 6c1648bfd..8bbacb4c0 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -273,6 +273,10 @@ export async function scrapeURLWithFireEngineChromeCDP(
error: response.pageError,
statusCode: response.pageStatusCode,
+ contentType: (Object.entries(response.responseHeaders ?? {}).find(
+ (x) => x[0].toLowerCase() === "content-type",
+ ) ?? [])[1] ?? undefined,
+
screenshot: response.screenshot,
...(actions.length > 0
? {
@@ -336,6 +340,10 @@ export async function scrapeURLWithFireEnginePlaywright(
error: response.pageError,
statusCode: response.pageStatusCode,
+ contentType: (Object.entries(response.responseHeaders ?? {}).find(
+ (x) => x[0].toLowerCase() === "content-type",
+ ) ?? [])[1] ?? undefined,
+
...(response.screenshots !== undefined && response.screenshots.length > 0
? {
screenshot: response.screenshots[0],
@@ -391,5 +399,9 @@ export async function scrapeURLWithFireEngineTLSClient(
html: response.content,
error: response.pageError,
statusCode: response.pageStatusCode,
+
+ contentType: (Object.entries(response.responseHeaders ?? {}).find(
+ (x) => x[0].toLowerCase() === "content-type",
+ ) ?? [])[1] ?? undefined,
};
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 06c5e0725..fe0ae8c70 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -111,6 +111,8 @@ export type EngineScrapeResult = {
};
numPages?: number;
+
+ contentType?: string;
};
const engineHandlers: {
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index fbc0c53b3..90873ca6e 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -379,6 +379,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
statusCode: result.result.statusCode,
error: result.result.error,
numPages: result.result.numPages,
+ contentType: result.result.contentType,
proxyUsed: meta.featureFlags.has("stealthProxy") ? "stealth" : "basic",
},
};
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts
index d9fc4b8c5..5186d6b4f 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts
@@ -61,6 +61,17 @@ export async function deriveMarkdownFromHTML(
);
}
+ if (document.metadata.contentType?.includes("application/json")) {
+ if (document.rawHtml === undefined) {
+ throw new Error(
+ "rawHtml is undefined -- this transformer is being called out of order",
+ );
+ }
+
+ document.markdown = "```json\n" + document.rawHtml + "\n```";
+ return document;
+ }
+
document.markdown = await parseMarkdown(document.html);
return document;
}