Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(js-sdk): add getParseFormats for GET /v2/parse/formats (#4909)
Gergő Móricz5e46be7cb505
diff --git a/.github/workflows/test-js-sdk.yml b/.github/workflows/test-js-sdk.ymlindex 0818b473d..11a09e25c 100644--- a/.github/workflows/test-js-sdk.yml+++ b/.github/workflows/test-js-sdk.yml@@ -32,8 +32,8 @@ jobs:- name: Install dependenciesrun: pnpm install --frozen-lockfileworking-directory: ./apps/js-sdk/firecrawl- - name: Run next URL pinning unit tests- run: pnpm exec jest --verbose src/__tests__/unit/v1/next-url-pinning.test.ts src/__tests__/unit/v2/next-url-pinning.test.ts+ - name: Run selected unit tests+ run: pnpm exec jest --verbose src/__tests__/unit/v1/next-url-pinning.test.ts src/__tests__/unit/v2/next-url-pinning.test.ts src/__tests__/unit/v2/parse-formats.unit.test.tsworking-directory: ./apps/js-sdk/firecrawlenv:NODE_OPTIONS: --experimental-vm-modulesdiff --git a/apps/js-sdk/firecrawl/README.md b/apps/js-sdk/firecrawl/README.mdindex 49a7d03d9..5d2e3ef34 100644--- a/apps/js-sdk/firecrawl/README.md+++ b/apps/js-sdk/firecrawl/README.md@@ -102,6 +102,13 @@ const parsed = await app.parse(console.log(parsed.markdown);```+To see which file formats `parse` accepts, call `getParseFormats`. Each entry has `format`, `kind` (for example, `document` or `image`), `extensions`, `mimeTypes`, and `available` (false when the format is not enabled on this deployment).++```js+const formats = await app.getParseFormats();+console.log(formats.filter((f) => f.available).map((f) => f.format));+```+### Crawling a WebsiteTo crawl a website with error handling, use the `crawl` method. It takes the starting URL and optional parameters, including limits and per‑page `scrapeOptions`.diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex 487a6eb77..e577eb2d4 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.42.1",+ "version": "4.42.2","description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/parse.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/parse.test.tsindex 21d54e199..7ad241458 100644--- a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/parse.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/parse.test.ts@@ -3,6 +3,7 @@ import { config } from "dotenv";import { getIdentity, getApiUrl } from "./utils/idmux";import { testTimeoutMs, withRateLimitRetry } from "./utils/rateLimit";import { describe, test, expect, beforeAll } from "@jest/globals";+import { SdkError } from "../../../v2/types";config();@@ -65,4 +66,29 @@ describe("v2.parse e2e", () => {},testTimeoutMs(60_000),);++ test(+ "lists supported parse formats",+ async () => {+ if (!client) throw new Error();++ let formats;+ try {+ formats = await client.getParseFormats();+ } catch (err) {+ if (err instanceof SdkError && err.status === 404) {+ console.warn("Skipping: /v2/parse/formats is not deployed on this API");+ return;+ }+ throw err;+ }++ expect(formats.length).toBeGreaterThan(0);+ const pdf = formats.find(f => f.format === "pdf");+ expect(pdf?.kind).toBe("document");+ expect(pdf?.extensions).toContain(".pdf");+ expect(pdf?.mimeTypes).toContain("application/pdf");+ },+ testTimeoutMs(60_000),+ );});diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/parse-formats.unit.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/parse-formats.unit.test.tsnew file mode 100644index 000000000..38a5563f7--- /dev/null+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/parse-formats.unit.test.ts@@ -0,0 +1,84 @@+import { describe, test, expect } from "@jest/globals";+import axios, { type AxiosAdapter } from "axios";+import { Firecrawl } from "../../../index";+import { SdkError } from "../../../v2/types";++const API_URL = "https://api.firecrawl.dev";+const API_KEY = "fc-test";++function makeClient(status: number, data: unknown) {+ const client = new Firecrawl({ apiKey: API_KEY, apiUrl: API_URL });+ const sent: Array<{ url: string; method?: string; authorization: unknown }> = [];+ const adapter: AxiosAdapter = async config => {+ sent.push({ url: axios.getUri(config), method: config.method, authorization: config.headers.Authorization });+ const response = { data, status, statusText: "", headers: {}, config };+ if (status >= 400) {+ throw new axios.AxiosError(`Request failed with status code ${status}`, "ERR_BAD_RESPONSE", config, null, response);+ }+ return response;+ };+ (client as any).http.instance.defaults.adapter = adapter;+ return { client, sent };+}++describe("v2.getParseFormats unit", () => {+ test("GETs /v2/parse/formats with auth and unwraps data.formats", async () => {+ const { client, sent } = makeClient(200, {+ success: true,+ data: {+ formats: [+ { format: "pdf", kind: "document", extensions: [".pdf"], mimeTypes: ["application/pdf"], available: true },+ { format: "png", kind: "image", extensions: [".png"], mimeTypes: ["image/png"], available: false },+ ],+ },+ });++ const formats = await client.getParseFormats();++ expect(sent).toEqual([{ url: `${API_URL}/v2/parse/formats`, method: "get", authorization: `Bearer ${API_KEY}` }]);+ expect(formats).toHaveLength(2);+ expect(formats[0]).toEqual({ format: "pdf", kind: "document", extensions: [".pdf"], mimeTypes: ["application/pdf"], available: true });+ expect(formats[1]!.kind).toBe("image");+ expect(formats[1]!.mimeTypes).toEqual(["image/png"]);+ expect(formats[1]!.available).toBe(false);+ });++ test("tolerates unknown kinds and extra fields", async () => {+ const { client } = makeClient(200, {+ success: true,+ data: {+ formats: [+ { format: "mp3", kind: "audio", extensions: [".mp3"], mimeTypes: ["audio/mpeg"], available: true, maxSizeMb: 50 },+ ],+ },+ });++ const [format] = await client.getParseFormats();++ expect(format!.format).toBe("mp3");+ expect(format!.kind).toBe("audio");+ expect(format!.mimeTypes).toEqual(["audio/mpeg"]);+ });++ test("returns an empty list when formats is missing", async () => {+ const { client } = makeClient(200, { success: true, data: {} });++ await expect(client.getParseFormats()).resolves.toEqual([]);+ });++ test.each([401, 500])("throws SdkError on %i", async status => {+ const { client } = makeClient(status, { success: false, error: "Unauthorized" });++ const err = await client.getParseFormats().catch(e => e);++ expect(err).toBeInstanceOf(SdkError);+ expect(err.status).toBe(status);+ expect(err.message).toBe("Unauthorized");+ });++ test("throws SdkError when success is false", async () => {+ const { client } = makeClient(200, { success: false, error: "nope" });++ await expect(client.getParseFormats()).rejects.toBeInstanceOf(SdkError);+ });+});diff --git a/apps/js-sdk/firecrawl/src/v2/client.ts b/apps/js-sdk/firecrawl/src/v2/client.tsindex aa6a6ca9b..8554c7d7d 100644--- a/apps/js-sdk/firecrawl/src/v2/client.ts+++ b/apps/js-sdk/firecrawl/src/v2/client.ts@@ -4,7 +4,7 @@ import {interact as interactMethod,stopInteraction as stopInteractionMethod,} from "./methods/scrape";-import { parse as parseMethod } from "./methods/parse";+import { parse as parseMethod, getParseFormats } from "./methods/parse";import { search } from "./methods/search";import { scrapeAlexandria, findTools } from "./methods/tools";import { developerSearch as developerSearchMethod } from "./methods/developer";@@ -53,6 +53,7 @@ import type {FindToolsData,AlexandriaScrapeData,ParseFile,+ ParseFormatInfo,ParseOptions,ScrapeOptions,SearchData,@@ -273,6 +274,14 @@ export class FirecrawlClient {return parseMethod(this.http, file, options);}+ /**+ * List the file formats the parse endpoint accepts on this deployment.+ * @returns Formats with their kind, extensions, MIME types, and availability.+ */+ async getParseFormats(): Promise<ParseFormatInfo[]> {+ return getParseFormats(this.http);+ }+// Search/*** Search the web and optionally scrape each result.diff --git a/apps/js-sdk/firecrawl/src/v2/methods/parse.ts b/apps/js-sdk/firecrawl/src/v2/methods/parse.tsindex d85f90cc8..183ca6730 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/parse.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/parse.ts@@ -1,4 +1,4 @@-import { type Document, type ParseFile, type ParseOptions } from "../types";+import { type Document, type ParseFile, type ParseFormatInfo, type ParseOptions } from "../types";import { HttpClient } from "../utils/httpClient";import { agentHintMetadata } from "../utils/agentHints";import { ensureValidParseOptions } from "../utils/validation";@@ -95,3 +95,14 @@ export async function parse(throw err;}}++export async function getParseFormats(http: HttpClient): Promise<ParseFormatInfo[]> {+ try {+ const res = await http.get<{ success: boolean; data?: { formats: ParseFormatInfo[] } }>("/v2/parse/formats");+ if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get parse formats");+ return res.data.data?.formats ?? [];+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "get parse formats");+ throw err;+ }+}diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex b485d9e13..f82e0c6e1 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -366,6 +366,18 @@ export type ParseOptions = Omit<proxy?: "basic" | "auto";};+/** A file format accepted by the parse endpoint. */+export interface ParseFormatInfo {+ /** Format identifier, e.g. "pdf". */+ format: string;+ kind: "document" | "image" | (string & {});+ /** File extensions including the leading dot, e.g. ".pdf". */+ extensions: string[];+ mimeTypes: string[];+ /** False when the format is recognized but not enabled on this deployment. */+ available: boolean;+}+export interface WebhookConfig {url: string;headers?: Record<string, string>;