Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(js-sdk): stop status pagination spinning, report the SDK's own version, name rejected fields in errors (#5200)

Erik Engervallbd7a3c073706
diff --git a/apps/js-sdk/firecrawl/README.md b/apps/js-sdk/firecrawl/README.md
index d52246c6d..c48084744 100644
--- a/apps/js-sdk/firecrawl/README.md
+++ b/apps/js-sdk/firecrawl/README.md
@@ -13,7 +13,7 @@ npm install firecrawl
## Usage
1. Get an API key from [firecrawl.dev](https://firecrawl.dev)
-2. Set the API key as an environment variable named `FIRECRAWL_API_KEY` or pass it as a parameter to the `FirecrawlApp` class.
+2. Set the API key as an environment variable named `FIRECRAWL_API_KEY` or pass it as the `apiKey` option to the `Firecrawl` class.
Here's an example of how to use the SDK with error handling:
@@ -137,7 +137,7 @@ const start = await app.startCrawl('https://mendable.ai', {
To check the status of a crawl job with error handling, use the `getCrawlStatus` method. It takes the job ID as a parameter and returns the current status.
```js
-const status = await app.getCrawlStatus(id);
+const status = await app.getCrawlStatus(start.id);
```
### Extracting structured data from URLs
diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/errorHandler.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/errorHandler.test.ts
index 89e6bc3f0..c10d69cee 100644
--- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/errorHandler.test.ts
+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/errorHandler.test.ts
@@ -15,5 +15,28 @@ describe("v2 utils: errorHandler", () => {
};
expect(() => normalizeAxiosError(err, "action")).toThrow(/payment required/);
});
+
+ test("normalizeAxiosError: names the fields a 400 validation error rejected", () => {
+ const err: any = {
+ isAxiosError: true,
+ response: {
+ status: 400,
+ data: {
+ error: "Unrecognized key in body -- please review the v2 API documentation for request body changes",
+ details: [{ code: "unrecognized_keys", keys: ["parsePDF"], path: [], message: "Unrecognized key(s) in object: 'parsePDF'" }],
+ },
+ },
+ message: "Request failed with status code 400",
+ };
+ expect(() => normalizeAxiosError(err, "scrape")).toThrow(/unrecognized key\(s\) parsePDF/);
+ });
+
+ test("throwForBadResponse: appends the path and message of each issue", () => {
+ const resp: any = {
+ status: 400,
+ data: { error: "Bad Request", details: [{ code: "invalid_union", path: ["formats", 0], message: "Invalid input" }] },
+ };
+ expect(() => throwForBadResponse(resp, "scrape")).toThrow("Bad Request (formats.0: Invalid input)");
+ });
});
diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/pagination.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/pagination.test.ts
index 8a1cc01ab..622eb477a 100644
--- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/pagination.test.ts
+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/pagination.test.ts
@@ -107,6 +107,26 @@ describe("JS SDK v2 pagination", () => {
}
});
+ test("crawl: stops when a running job's cursor does not advance", async () => {
+ const next = "https://api/v2/crawl/jobR?skip=2";
+ const first = { status: 200, data: { success: true, status: "scraping", completed: 2, total: 2, next, data: [{ markdown: "a" }, { markdown: "b" }] } };
+ const stalled = { status: 200, data: { success: true, status: "scraping", next, data: [] } };
+ const http = makeHttp((url) => (url === next ? stalled : first));
+ const res = await getCrawlStatus(http, "jobR");
+ expect(res.data.length).toBe(2);
+ expect((http.get as Mock).mock.calls.length).toBe(2);
+ });
+
+ test("batch: stops when a running job's cursor does not advance", async () => {
+ const next = "https://api/v2/batch/scrape/jobR?skip=2";
+ const first = { status: 200, data: { success: true, status: "scraping", completed: 2, total: 2, next, data: [{ markdown: "a" }, { markdown: "b" }] } };
+ const stalled = { status: 200, data: { success: true, status: "scraping", next, data: [] } };
+ const http = makeHttp((url) => (url === next ? stalled : first));
+ const res = await getBatchScrapeStatus(http, "jobR");
+ expect(res.data.length).toBe(2);
+ expect((http.get as Mock).mock.calls.length).toBe(2);
+ });
+
test("batch: maxWaitTime stops pagination after first page", async () => {
const first = { status: 200, data: { success: true, status: "completed", completed: 1, total: 5, next: "https://api/b1", data: [{ markdown: "a" }] } };
const p1 = { status: 200, data: { success: true, next: "https://api/b2", data: [{ markdown: "b" }] } };
diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.ts
index 2d4d6445d..6a9d2872c 100644
--- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.ts
+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.ts
@@ -1,9 +1,47 @@
import { describe, test, expect } from "vitest";
-import { ensureValidFormats, ensureValidScrapeOptions } from "../../../v2/utils/validation";
+import { ensureNoV1Options, ensureValidFormats, ensureValidScrapeOptions } from "../../../v2/utils/validation";
import type { FormatOption } from "../../../v2/types";
import { z } from "zod";
describe("v2 utils: validation", () => {
+ const scrapeLevel = {
+ pageOptions: "use scrapeOptions",
+ includeLinks: "add 'links' to scrapeOptions.formats",
+ parsePDF: "use scrapeOptions.parsers",
+ };
+ const crawlOnly = {
+ crawlerOptions: "move its fields to the top level",
+ maxDepth: "use maxDiscoveryDepth",
+ allowBackwardLinks: "use crawlEntireDomain",
+ ignoreSitemap: "use sitemap: 'skip'",
+ };
+
+ test.each(Object.entries({ ...crawlOnly, ...scrapeLevel }))("ensureNoV1Options: crawl names %s and its replacement", (option, replacement) => {
+ expect(() => ensureNoV1Options({ url: "https://a.dev", [option]: true }, "crawl")).toThrow(
+ `crawl: ${option} is a v1 option, ${replacement}`,
+ );
+ });
+
+ test.each(Object.entries(scrapeLevel))("ensureNoV1Options: search names %s and its replacement", (option, replacement) => {
+ expect(() => ensureNoV1Options({ query: "q", [option]: true }, "search")).toThrow(
+ `search: ${option} is a v1 option, ${replacement}`,
+ );
+ });
+
+ test("ensureNoV1Options: search does not apply crawl-only hints", () => {
+ expect(() => ensureNoV1Options({ query: "q", maxDepth: 2, crawlerOptions: {} }, "search")).not.toThrow();
+ });
+
+ test("ensureNoV1Options: lists every v1 option found", () => {
+ expect(() => ensureNoV1Options({ maxDepth: 2, allowBackwardLinks: true }, "crawl")).toThrow(/maxDepth.*maxDiscoveryDepth.*allowBackwardLinks.*crawlEntireDomain/);
+ });
+
+ test("ensureNoV1Options: accepts v2 options and unset v1 keys", () => {
+ expect(() => ensureNoV1Options({ url: "https://a.dev", limit: 2, maxDiscoveryDepth: 1 }, "crawl")).not.toThrow();
+ expect(() => ensureNoV1Options({ url: "https://a.dev", maxDepth: undefined }, "crawl")).not.toThrow();
+ expect(() => ensureNoV1Options(undefined, "search")).not.toThrow();
+ });
+
test("ensureValidFormats: plain 'json' string is invalid", () => {
const formats: FormatOption[] = ["markdown", "json"] as unknown as FormatOption[];
expect(() => ensureValidFormats(formats)).toThrow(/json format must be an object/i);
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts b/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts
index 01405d702..1212ed912 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts
@@ -10,7 +10,7 @@ import {
SdkError,
} from "../types";
import { HttpClient } from "../utils/httpClient";
-import { ensureValidScrapeOptions } from "../utils/validation";
+import { ensureNoV1Options, ensureValidScrapeOptions } from "../utils/validation";
import { normalizeAxiosError, throwForBadResponse, isRetryableError } from "../utils/errorHandler";
import type { HttpClient as _Http } from "../utils/httpClient";
import { fetchAllPages } from "../utils/pagination";
@@ -21,6 +21,7 @@ export type CrawlRequest = CrawlOptions & {
function prepareCrawlPayload(request: CrawlRequest): Record<string, unknown> {
if (!request.url || !request.url.trim()) throw new Error("URL cannot be empty");
+ ensureNoV1Options(request, "crawl");
const data: Record<string, unknown> = { url: request.url.trim() };
if (request.prompt) data.prompt = request.prompt;
if (request.excludePaths) data.excludePaths = request.excludePaths;
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/search.ts b/apps/js-sdk/firecrawl/src/v2/methods/search.ts
index 08509ca2e..b29f8cd83 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/search.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/search.ts
@@ -9,7 +9,7 @@ import {
} from "../types";
import { HttpClient } from "../utils/httpClient";
import { agentHintMetadata } from "../utils/agentHints";
-import { ensureValidScrapeOptions } from "../utils/validation";
+import { ensureNoV1Options, ensureValidScrapeOptions } from "../utils/validation";
import {
throwForBadResponse,
normalizeAxiosError,
@@ -17,6 +17,7 @@ import {
function prepareSearchPayload(req: SearchRequest): Record<string, unknown> {
if (!req.query || !req.query.trim()) throw new Error("Query cannot be empty");
+ ensureNoV1Options(req, "search");
if (req.limit != null && req.limit <= 0)
throw new Error("limit must be positive");
if (req.timeout != null && req.timeout <= 0)
diff --git a/apps/js-sdk/firecrawl/src/v2/utils/errorHandler.ts b/apps/js-sdk/firecrawl/src/v2/utils/errorHandler.ts
index e14dd5ace..f7f8e59e1 100644
--- a/apps/js-sdk/firecrawl/src/v2/utils/errorHandler.ts
+++ b/apps/js-sdk/firecrawl/src/v2/utils/errorHandler.ts
@@ -2,12 +2,33 @@ import { type AxiosError, type AxiosResponse } from "axios";
import { SdkError, JobTimeoutError, parseRequiresAction } from "../types";
import { agentHintMetadata } from "./agentHints";
+/** Names the fields a validation error rejected, so a bare "Bad Request" says what to change. */
+function describeValidationIssues(details: unknown): string | undefined {
+ if (!Array.isArray(details)) return undefined;
+ const parts = details.slice(0, 3).flatMap((issue) => {
+ if (!issue || typeof issue !== "object") return [];
+ const { path, message, keys } = issue as { path?: unknown; message?: unknown; keys?: unknown };
+ const what = Array.isArray(keys) && keys.length > 0
+ ? `unrecognized key(s) ${keys.join(", ")}`
+ : typeof message === "string" ? message : undefined;
+ if (!what) return [];
+ return Array.isArray(path) && path.length > 0 ? [`${path.join(".")}: ${what}`] : [what];
+ });
+ return parts.length > 0 ? parts.join("; ") : undefined;
+}
+
+function withIssues(message: unknown, details: unknown): string {
+ const text = typeof message === "string" ? message : JSON.stringify(message);
+ const issues = describeValidationIssues(details);
+ return issues && !text.includes(issues) ? `${text} (${issues})` : text;
+}
+
export function throwForBadResponse(resp: AxiosResponse, action: string): never {
const status = resp.status;
const body = resp.data || {};
const msg = body?.error || body?.message || `Request failed (${status}) while trying to ${action}`;
throw Object.assign(new SdkError(
- msg,
+ withIssues(msg, body?.details),
status,
body?.code,
body?.details,
@@ -23,7 +44,7 @@ export function normalizeAxiosError(err: AxiosError, action: string): never {
const message = body?.error || err.message || `Request failed${status ? ` (${status})` : ""} while trying to ${action}`;
const code = (body?.code as string) || err.code;
throw Object.assign(new SdkError(
- message,
+ withIssues(message, body?.details),
status,
code,
body?.details ?? body,
diff --git a/apps/js-sdk/firecrawl/src/v2/utils/getVersion.ts b/apps/js-sdk/firecrawl/src/v2/utils/getVersion.ts
index d1df63538..836eb24c7 100644
--- a/apps/js-sdk/firecrawl/src/v2/utils/getVersion.ts
+++ b/apps/js-sdk/firecrawl/src/v2/utils/getVersion.ts
@@ -1,14 +1,6 @@
-export function getVersion(): string {
- try {
- if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
- return process.env.npm_package_version as string;
- }
+import { version } from "../../../package.json";
- // eslint-disable-next-line @typescript-eslint/no-var-requires
- const pkg = require("../../../package.json");
- return (pkg?.version as string) || "3.x.x";
- } catch {
- return "3.x.x";
- }
+/** The SDK's own version, inlined at build time so a host app's npm_package_version never replaces it. */
+export function getVersion(): string {
+ return version;
}
-
diff --git a/apps/js-sdk/firecrawl/src/v2/utils/pagination.ts b/apps/js-sdk/firecrawl/src/v2/utils/pagination.ts
index cb8273cfc..14bc93a45 100644
--- a/apps/js-sdk/firecrawl/src/v2/utils/pagination.ts
+++ b/apps/js-sdk/firecrawl/src/v2/utils/pagination.ts
@@ -39,7 +39,10 @@ export async function fetchAllPages<T = Document>(
docs.push(d as T);
}
if (maxResults != null && docs.length >= maxResults) break;
- current = (payload.next ?? (Array.isArray(payload.data) ? null : payload.data?.next) ?? null) as string | null;
+ const next = (payload.next ?? (Array.isArray(payload.data) ? null : payload.data?.next) ?? null) as string | null;
+ // A running job's cursor can point back at itself until new documents land; re-requesting it here only spins.
+ if (next === current) break;
+ current = next;
pageCount += 1;
}
return docs;
diff --git a/apps/js-sdk/firecrawl/src/v2/utils/validation.ts b/apps/js-sdk/firecrawl/src/v2/utils/validation.ts
index e42e2242c..37c047446 100644
--- a/apps/js-sdk/firecrawl/src/v2/utils/validation.ts
+++ b/apps/js-sdk/firecrawl/src/v2/utils/validation.ts
@@ -86,6 +86,34 @@ export function ensureValidFormats(formats?: FormatOption[]): void {
}
}
+const V1_SCRAPE_OPTION_REPLACEMENTS: Record<string, string> = {
+ pageOptions: "use scrapeOptions",
+ includeLinks: "add 'links' to scrapeOptions.formats",
+ parsePDF: "use scrapeOptions.parsers",
+};
+
+const V1_OPTION_REPLACEMENTS: Record<"crawl" | "search", Record<string, string>> = {
+ crawl: {
+ crawlerOptions: "move its fields to the top level",
+ maxDepth: "use maxDiscoveryDepth",
+ allowBackwardLinks: "use crawlEntireDomain",
+ ignoreSitemap: "use sitemap: 'skip'",
+ ...V1_SCRAPE_OPTION_REPLACEMENTS,
+ },
+ search: V1_SCRAPE_OPTION_REPLACEMENTS,
+};
+
+/** Throws on v1 option names, which the v2 payload builders would otherwise drop without a word. */
+export function ensureNoV1Options(options: object | undefined, method: "crawl" | "search"): void {
+ if (!options) return;
+ const replacements = V1_OPTION_REPLACEMENTS[method];
+ const found = Object.keys(replacements).filter((key) => (options as Record<string, unknown>)[key] != null);
+ if (found.length === 0) return;
+ throw new Error(
+ `${method}: ${found.map((key) => `${key} is a v1 option, ${replacements[key]}`).join("; ")}`,
+ );
+}
+
export function ensureValidScrapeOptions(options?: ScrapeOptions): void {
if (!options) return;
if (options.timeout != null && options.timeout <= 0) {