Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(scrapeURL): handle contentType JSON better in markdown conversion (#1604)
Gergő Móricz38c96b524f12
diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.tsindex 9c2a9e2d5..a27057f78 100644--- a/apps/api/src/__tests__/snips/scrape.test.ts+++ b/apps/api/src/__tests__/snips/scrape.test.ts@@ -373,5 +373,14 @@ describe("Scrape tests", () => {});expect(response.metadata.sourceURL).toBe("https://firecrawl.dev/?pagewanted=all&et_blog");- }, 35000);+ }, 30000);++ it.concurrent("application/json content type is markdownified properly", async () => {+ const response = await scrape({+ url: "https://jsonplaceholder.typicode.com/todos/1",+ formats: ["markdown"],+ });++ expect(response.markdown).toContain("```json");+ }, 30000);});diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 908a51d3d..5ee1328d0 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -750,6 +750,7 @@ export type Document = {scrapeId?: string;error?: string;numPages?: number;+ contentType?: string;proxyUsed: "basic" | "stealth";// [key: string]: string | string[] | number | { smartScrape: number; other: number; total: number } | undefined;};diff --git a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts b/apps/api/src/scraper/scrapeURL/engines/fetch/index.tsindex 40c343997..299b0d355 100644--- a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts@@ -30,7 +30,7 @@ export async function scrapeURLWithFetch(url: string;body: string,status: number;- headers: any;+ headers: [string, string][];};if (meta.mock !== null) {@@ -117,5 +117,8 @@ export async function scrapeURLWithFetch(url: response.url,html: response.body,statusCode: response.status,+ contentType: (response.headers.find(+ (x) => x[0].toLowerCase() === "content-type",+ ) ?? [])[1] ?? undefined,};}diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 6c1648bfd..8bbacb4c0 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -273,6 +273,10 @@ export async function scrapeURLWithFireEngineChromeCDP(error: response.pageError,statusCode: response.pageStatusCode,+ contentType: (Object.entries(response.responseHeaders ?? {}).find(+ (x) => x[0].toLowerCase() === "content-type",+ ) ?? [])[1] ?? undefined,+screenshot: response.screenshot,...(actions.length > 0? {@@ -336,6 +340,10 @@ export async function scrapeURLWithFireEnginePlaywright(error: response.pageError,statusCode: response.pageStatusCode,+ contentType: (Object.entries(response.responseHeaders ?? {}).find(+ (x) => x[0].toLowerCase() === "content-type",+ ) ?? [])[1] ?? undefined,+...(response.screenshots !== undefined && response.screenshots.length > 0? {screenshot: response.screenshots[0],@@ -391,5 +399,9 @@ export async function scrapeURLWithFireEngineTLSClient(html: response.content,error: response.pageError,statusCode: response.pageStatusCode,++ contentType: (Object.entries(response.responseHeaders ?? {}).find(+ (x) => x[0].toLowerCase() === "content-type",+ ) ?? [])[1] ?? undefined,};}diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 06c5e0725..fe0ae8c70 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -111,6 +111,8 @@ export type EngineScrapeResult = {};numPages?: number;++ contentType?: string;};const engineHandlers: {diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex fbc0c53b3..90873ca6e 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -379,6 +379,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {statusCode: result.result.statusCode,error: result.result.error,numPages: result.result.numPages,+ contentType: result.result.contentType,proxyUsed: meta.featureFlags.has("stealthProxy") ? "stealth" : "basic",},};diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex d9fc4b8c5..5186d6b4f 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -61,6 +61,17 @@ export async function deriveMarkdownFromHTML();}+ if (document.metadata.contentType?.includes("application/json")) {+ if (document.rawHtml === undefined) {+ throw new Error(+ "rawHtml is undefined -- this transformer is being called out of order",+ );+ }++ document.markdown = "```json\n" + document.rawHtml + "\n```";+ return document;+ }+document.markdown = await parseMarkdown(document.html);return document;}