Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(js-sdk): stop status pagination spinning, report the SDK's own version, name rejected fields in errors (#5200)
Erik Engervallbd7a3c073706
diff --git a/apps/js-sdk/firecrawl/README.md b/apps/js-sdk/firecrawl/README.mdindex d52246c6d..c48084744 100644--- a/apps/js-sdk/firecrawl/README.md+++ b/apps/js-sdk/firecrawl/README.md@@ -13,7 +13,7 @@ npm install firecrawl## Usage1. Get an API key from [firecrawl.dev](https://firecrawl.dev)-2. Set the API key as an environment variable named `FIRECRAWL_API_KEY` or pass it as a parameter to the `FirecrawlApp` class.+2. Set the API key as an environment variable named `FIRECRAWL_API_KEY` or pass it as the `apiKey` option to the `Firecrawl` class.Here's an example of how to use the SDK with error handling:@@ -137,7 +137,7 @@ const start = await app.startCrawl('https://mendable.ai', {To check the status of a crawl job with error handling, use the `getCrawlStatus` method. It takes the job ID as a parameter and returns the current status.```js-const status = await app.getCrawlStatus(id);+const status = await app.getCrawlStatus(start.id);```### Extracting structured data from URLsdiff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/errorHandler.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/errorHandler.test.tsindex 89e6bc3f0..c10d69cee 100644--- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/errorHandler.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/errorHandler.test.ts@@ -15,5 +15,28 @@ describe("v2 utils: errorHandler", () => {};expect(() => normalizeAxiosError(err, "action")).toThrow(/payment required/);});++ test("normalizeAxiosError: names the fields a 400 validation error rejected", () => {+ const err: any = {+ isAxiosError: true,+ response: {+ status: 400,+ data: {+ error: "Unrecognized key in body -- please review the v2 API documentation for request body changes",+ details: [{ code: "unrecognized_keys", keys: ["parsePDF"], path: [], message: "Unrecognized key(s) in object: 'parsePDF'" }],+ },+ },+ message: "Request failed with status code 400",+ };+ expect(() => normalizeAxiosError(err, "scrape")).toThrow(/unrecognized key\(s\) parsePDF/);+ });++ test("throwForBadResponse: appends the path and message of each issue", () => {+ const resp: any = {+ status: 400,+ data: { error: "Bad Request", details: [{ code: "invalid_union", path: ["formats", 0], message: "Invalid input" }] },+ };+ expect(() => throwForBadResponse(resp, "scrape")).toThrow("Bad Request (formats.0: Invalid input)");+ });});diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/pagination.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/pagination.test.tsindex 8a1cc01ab..622eb477a 100644--- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/pagination.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/pagination.test.ts@@ -107,6 +107,26 @@ describe("JS SDK v2 pagination", () => {}});+ test("crawl: stops when a running job's cursor does not advance", async () => {+ const next = "https://api/v2/crawl/jobR?skip=2";+ const first = { status: 200, data: { success: true, status: "scraping", completed: 2, total: 2, next, data: [{ markdown: "a" }, { markdown: "b" }] } };+ const stalled = { status: 200, data: { success: true, status: "scraping", next, data: [] } };+ const http = makeHttp((url) => (url === next ? stalled : first));+ const res = await getCrawlStatus(http, "jobR");+ expect(res.data.length).toBe(2);+ expect((http.get as Mock).mock.calls.length).toBe(2);+ });++ test("batch: stops when a running job's cursor does not advance", async () => {+ const next = "https://api/v2/batch/scrape/jobR?skip=2";+ const first = { status: 200, data: { success: true, status: "scraping", completed: 2, total: 2, next, data: [{ markdown: "a" }, { markdown: "b" }] } };+ const stalled = { status: 200, data: { success: true, status: "scraping", next, data: [] } };+ const http = makeHttp((url) => (url === next ? stalled : first));+ const res = await getBatchScrapeStatus(http, "jobR");+ expect(res.data.length).toBe(2);+ expect((http.get as Mock).mock.calls.length).toBe(2);+ });+test("batch: maxWaitTime stops pagination after first page", async () => {const first = { status: 200, data: { success: true, status: "completed", completed: 1, total: 5, next: "https://api/b1", data: [{ markdown: "a" }] } };const p1 = { status: 200, data: { success: true, next: "https://api/b2", data: [{ markdown: "b" }] } };diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.tsindex 2d4d6445d..6a9d2872c 100644--- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.ts@@ -1,9 +1,47 @@import { describe, test, expect } from "vitest";-import { ensureValidFormats, ensureValidScrapeOptions } from "../../../v2/utils/validation";+import { ensureNoV1Options, ensureValidFormats, ensureValidScrapeOptions } from "../../../v2/utils/validation";import type { FormatOption } from "../../../v2/types";import { z } from "zod";describe("v2 utils: validation", () => {+ const scrapeLevel = {+ pageOptions: "use scrapeOptions",+ includeLinks: "add 'links' to scrapeOptions.formats",+ parsePDF: "use scrapeOptions.parsers",+ };+ const crawlOnly = {+ crawlerOptions: "move its fields to the top level",+ maxDepth: "use maxDiscoveryDepth",+ allowBackwardLinks: "use crawlEntireDomain",+ ignoreSitemap: "use sitemap: 'skip'",+ };++ test.each(Object.entries({ ...crawlOnly, ...scrapeLevel }))("ensureNoV1Options: crawl names %s and its replacement", (option, replacement) => {+ expect(() => ensureNoV1Options({ url: "https://a.dev", [option]: true }, "crawl")).toThrow(+ `crawl: ${option} is a v1 option, ${replacement}`,+ );+ });++ test.each(Object.entries(scrapeLevel))("ensureNoV1Options: search names %s and its replacement", (option, replacement) => {+ expect(() => ensureNoV1Options({ query: "q", [option]: true }, "search")).toThrow(+ `search: ${option} is a v1 option, ${replacement}`,+ );+ });++ test("ensureNoV1Options: search does not apply crawl-only hints", () => {+ expect(() => ensureNoV1Options({ query: "q", maxDepth: 2, crawlerOptions: {} }, "search")).not.toThrow();+ });++ test("ensureNoV1Options: lists every v1 option found", () => {+ expect(() => ensureNoV1Options({ maxDepth: 2, allowBackwardLinks: true }, "crawl")).toThrow(/maxDepth.*maxDiscoveryDepth.*allowBackwardLinks.*crawlEntireDomain/);+ });++ test("ensureNoV1Options: accepts v2 options and unset v1 keys", () => {+ expect(() => ensureNoV1Options({ url: "https://a.dev", limit: 2, maxDiscoveryDepth: 1 }, "crawl")).not.toThrow();+ expect(() => ensureNoV1Options({ url: "https://a.dev", maxDepth: undefined }, "crawl")).not.toThrow();+ expect(() => ensureNoV1Options(undefined, "search")).not.toThrow();+ });+test("ensureValidFormats: plain 'json' string is invalid", () => {const formats: FormatOption[] = ["markdown", "json"] as unknown as FormatOption[];expect(() => ensureValidFormats(formats)).toThrow(/json format must be an object/i);diff --git a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts b/apps/js-sdk/firecrawl/src/v2/methods/crawl.tsindex 01405d702..1212ed912 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts@@ -10,7 +10,7 @@ import {SdkError,} from "../types";import { HttpClient } from "../utils/httpClient";-import { ensureValidScrapeOptions } from "../utils/validation";+import { ensureNoV1Options, ensureValidScrapeOptions } from "../utils/validation";import { normalizeAxiosError, throwForBadResponse, isRetryableError } from "../utils/errorHandler";import type { HttpClient as _Http } from "../utils/httpClient";import { fetchAllPages } from "../utils/pagination";@@ -21,6 +21,7 @@ export type CrawlRequest = CrawlOptions & {function prepareCrawlPayload(request: CrawlRequest): Record<string, unknown> {if (!request.url || !request.url.trim()) throw new Error("URL cannot be empty");+ ensureNoV1Options(request, "crawl");const data: Record<string, unknown> = { url: request.url.trim() };if (request.prompt) data.prompt = request.prompt;if (request.excludePaths) data.excludePaths = request.excludePaths;diff --git a/apps/js-sdk/firecrawl/src/v2/methods/search.ts b/apps/js-sdk/firecrawl/src/v2/methods/search.tsindex 08509ca2e..b29f8cd83 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/search.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/search.ts@@ -9,7 +9,7 @@ import {} from "../types";import { HttpClient } from "../utils/httpClient";import { agentHintMetadata } from "../utils/agentHints";-import { ensureValidScrapeOptions } from "../utils/validation";+import { ensureNoV1Options, ensureValidScrapeOptions } from "../utils/validation";import {throwForBadResponse,normalizeAxiosError,@@ -17,6 +17,7 @@ import {function prepareSearchPayload(req: SearchRequest): Record<string, unknown> {if (!req.query || !req.query.trim()) throw new Error("Query cannot be empty");+ ensureNoV1Options(req, "search");if (req.limit != null && req.limit <= 0)throw new Error("limit must be positive");if (req.timeout != null && req.timeout <= 0)diff --git a/apps/js-sdk/firecrawl/src/v2/utils/errorHandler.ts b/apps/js-sdk/firecrawl/src/v2/utils/errorHandler.tsindex e14dd5ace..f7f8e59e1 100644--- a/apps/js-sdk/firecrawl/src/v2/utils/errorHandler.ts+++ b/apps/js-sdk/firecrawl/src/v2/utils/errorHandler.ts@@ -2,12 +2,33 @@ import { type AxiosError, type AxiosResponse } from "axios";import { SdkError, JobTimeoutError, parseRequiresAction } from "../types";import { agentHintMetadata } from "./agentHints";+/** Names the fields a validation error rejected, so a bare "Bad Request" says what to change. */+function describeValidationIssues(details: unknown): string | undefined {+ if (!Array.isArray(details)) return undefined;+ const parts = details.slice(0, 3).flatMap((issue) => {+ if (!issue || typeof issue !== "object") return [];+ const { path, message, keys } = issue as { path?: unknown; message?: unknown; keys?: unknown };+ const what = Array.isArray(keys) && keys.length > 0+ ? `unrecognized key(s) ${keys.join(", ")}`+ : typeof message === "string" ? message : undefined;+ if (!what) return [];+ return Array.isArray(path) && path.length > 0 ? [`${path.join(".")}: ${what}`] : [what];+ });+ return parts.length > 0 ? parts.join("; ") : undefined;+}++function withIssues(message: unknown, details: unknown): string {+ const text = typeof message === "string" ? message : JSON.stringify(message);+ const issues = describeValidationIssues(details);+ return issues && !text.includes(issues) ? `${text} (${issues})` : text;+}+export function throwForBadResponse(resp: AxiosResponse, action: string): never {const status = resp.status;const body = resp.data || {};const msg = body?.error || body?.message || `Request failed (${status}) while trying to ${action}`;throw Object.assign(new SdkError(- msg,+ withIssues(msg, body?.details),status,body?.code,body?.details,@@ -23,7 +44,7 @@ export function normalizeAxiosError(err: AxiosError, action: string): never {const message = body?.error || err.message || `Request failed${status ? ` (${status})` : ""} while trying to ${action}`;const code = (body?.code as string) || err.code;throw Object.assign(new SdkError(- message,+ withIssues(message, body?.details),status,code,body?.details ?? body,diff --git a/apps/js-sdk/firecrawl/src/v2/utils/getVersion.ts b/apps/js-sdk/firecrawl/src/v2/utils/getVersion.tsindex d1df63538..836eb24c7 100644--- a/apps/js-sdk/firecrawl/src/v2/utils/getVersion.ts+++ b/apps/js-sdk/firecrawl/src/v2/utils/getVersion.ts@@ -1,14 +1,6 @@-export function getVersion(): string {- try {- if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {- return process.env.npm_package_version as string;- }+import { version } from "../../../package.json";- // eslint-disable-next-line @typescript-eslint/no-var-requires- const pkg = require("../../../package.json");- return (pkg?.version as string) || "3.x.x";- } catch {- return "3.x.x";- }+/** The SDK's own version, inlined at build time so a host app's npm_package_version never replaces it. */+export function getVersion(): string {+ return version;}-diff --git a/apps/js-sdk/firecrawl/src/v2/utils/pagination.ts b/apps/js-sdk/firecrawl/src/v2/utils/pagination.tsindex cb8273cfc..14bc93a45 100644--- a/apps/js-sdk/firecrawl/src/v2/utils/pagination.ts+++ b/apps/js-sdk/firecrawl/src/v2/utils/pagination.ts@@ -39,7 +39,10 @@ export async function fetchAllPages<T = Document>(docs.push(d as T);}if (maxResults != null && docs.length >= maxResults) break;- current = (payload.next ?? (Array.isArray(payload.data) ? null : payload.data?.next) ?? null) as string | null;+ const next = (payload.next ?? (Array.isArray(payload.data) ? null : payload.data?.next) ?? null) as string | null;+ // A running job's cursor can point back at itself until new documents land; re-requesting it here only spins.+ if (next === current) break;+ current = next;pageCount += 1;}return docs;diff --git a/apps/js-sdk/firecrawl/src/v2/utils/validation.ts b/apps/js-sdk/firecrawl/src/v2/utils/validation.tsindex e42e2242c..37c047446 100644--- a/apps/js-sdk/firecrawl/src/v2/utils/validation.ts+++ b/apps/js-sdk/firecrawl/src/v2/utils/validation.ts@@ -86,6 +86,34 @@ export function ensureValidFormats(formats?: FormatOption[]): void {}}+const V1_SCRAPE_OPTION_REPLACEMENTS: Record<string, string> = {+ pageOptions: "use scrapeOptions",+ includeLinks: "add 'links' to scrapeOptions.formats",+ parsePDF: "use scrapeOptions.parsers",+};++const V1_OPTION_REPLACEMENTS: Record<"crawl" | "search", Record<string, string>> = {+ crawl: {+ crawlerOptions: "move its fields to the top level",+ maxDepth: "use maxDiscoveryDepth",+ allowBackwardLinks: "use crawlEntireDomain",+ ignoreSitemap: "use sitemap: 'skip'",+ ...V1_SCRAPE_OPTION_REPLACEMENTS,+ },+ search: V1_SCRAPE_OPTION_REPLACEMENTS,+};++/** Throws on v1 option names, which the v2 payload builders would otherwise drop without a word. */+export function ensureNoV1Options(options: object | undefined, method: "crawl" | "search"): void {+ if (!options) return;+ const replacements = V1_OPTION_REPLACEMENTS[method];+ const found = Object.keys(replacements).filter((key) => (options as Record<string, unknown>)[key] != null);+ if (found.length === 0) return;+ throw new Error(+ `${method}: ${found.map((key) => `${key} is a v1 option, ${replacements[key]}`).join("; ")}`,+ );+}+export function ensureValidScrapeOptions(options?: ScrapeOptions): void {if (!options) return;if (options.timeout != null && options.timeout <= 0) {