Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(js-sdk): abortable crawl() that cancels its job; cancelCrawl returns false on 409 (#4898)
claude[bot]23b68aefd291
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex ba1780a84..1265ebb80 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.44.1",+ "version": "4.45.0","description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/crawl-cancel-abort.unit.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/crawl-cancel-abort.unit.test.tsnew file mode 100644index 000000000..e33f15d7c--- /dev/null+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/crawl-cancel-abort.unit.test.ts@@ -0,0 +1,189 @@+import { describe, test, expect } from "@jest/globals";+import axios, { type AxiosAdapter } from "axios";+import { FirecrawlClient } from "../../../v2/client";+import { HttpClient } from "../../../v2/utils/httpClient";+import { waitForCrawlCompletion } from "../../../v2/methods/crawl";+import { SdkError } from "../../../v2/types";++const API_URL = "https://api.firecrawl.dev";+const JOB_ID = "job-123";++type Sent = { method: string; path: string };+type Reply = { status: number; data: unknown };++function withAdapter<T extends { defaults: any }>(instance: T, route: (req: Sent) => Reply): Sent[] {+ const sent: Sent[] = [];+ const adapter: AxiosAdapter = async (config) => {+ const req = {+ method: (config.method || "get").toLowerCase(),+ path: new URL(axios.getUri(config)).pathname,+ };+ sent.push(req);+ const { status, data } = route(req);+ const response = { data, status, statusText: String(status), headers: {}, config };+ const validate = config.validateStatus;+ if (validate && !validate(status)) {+ throw new axios.AxiosError(`Request failed with status code ${status}`, undefined, config, null, response);+ }+ return response;+ };+ instance.defaults.adapter = adapter;+ return sent;+}++function makeClient(route: (req: Sent) => Reply) {+ const client = new FirecrawlClient({ apiKey: "fc-test", apiUrl: API_URL });+ const sent = withAdapter((client as any).http.instance, route);+ return { client, sent };+}++// The cancel runs in the background, so let its request reach the adapter.+const flush = () => new Promise((r) => setTimeout(r, 0));++const running: Reply = { status: 200, data: { success: true, status: "scraping", completed: 0, total: 1, data: [] } };+const started: Reply = { status: 200, data: { success: true, id: JOB_ID, url: "https://example.com" } };++describe("v2 cancelCrawl", () => {+ test("returns false when the crawl is already completed (409)", async () => {+ const { client, sent } = makeClient(() => ({ status: 409, data: { error: "Crawl is already completed" } }));+ await expect(client.cancelCrawl(JOB_ID)).resolves.toBe(false);+ expect(sent).toEqual([{ method: "delete", path: `/v2/crawl/${JOB_ID}` }]);+ });++ test("returns false on 409 when the transport does not throw for 4xx", async () => {+ const { client } = makeClient(() => ({ status: 409, data: { error: "Crawl is already completed" } }));+ (client as any).http.instance.defaults.validateStatus = () => true;+ await expect(client.cancelCrawl(JOB_ID)).resolves.toBe(false);+ });++ test("returns true when the crawl is cancelled", async () => {+ const { client } = makeClient(() => ({ status: 200, data: { status: "cancelled" } }));+ await expect(client.cancelCrawl(JOB_ID)).resolves.toBe(true);+ });++ test("still throws for other errors", async () => {+ const { client } = makeClient(() => ({ status: 404, data: { error: "Job not found" } }));+ const err = await client.cancelCrawl(JOB_ID).catch((e) => e);+ expect(err).toBeInstanceOf(SdkError);+ expect(err.status).toBe(404);+ });+});++describe("v2 crawl with an AbortSignal", () => {+ test("abort during polling stops polling, cancels the job, and rejects with the abort reason", async () => {+ const controller = new AbortController();+ let polls = 0;+ const { client, sent } = makeClient((req) => {+ if (req.method === "post") return started;+ if (req.method === "delete") return { status: 200, data: { status: "cancelled" } };+ polls += 1;+ if (polls === 1) setTimeout(() => controller.abort(), 10);+ return running;+ });++ const t0 = Date.now();+ const err = await client.crawl("https://example.com", { pollInterval: 30, signal: controller.signal }).catch((e) => e);++ expect(Date.now() - t0).toBeLessThan(5000);+ expect(err).toBe(controller.signal.reason);+ expect((err as Error).name).toBe("AbortError");+ await flush();+ expect(sent.map((r) => r.method)).toEqual(["post", "get", "delete"]);+ expect(sent[2]!.path).toBe(`/v2/crawl/${JOB_ID}`);+ });++ test("rejects with a custom abort reason and ignores a failing cancel", async () => {+ const controller = new AbortController();+ const reason = new Error("caller gave up");+ const { client, sent } = makeClient((req) => {+ if (req.method === "post") return started;+ if (req.method === "delete") return { status: 500, data: { error: "boom" } };+ setTimeout(() => controller.abort(reason), 10);+ return running;+ });++ await expect(client.crawl("https://example.com", { pollInterval: 30, signal: controller.signal })).rejects.toBe(reason);+ await flush();+ expect(sent.some((r) => r.method === "delete")).toBe(true);+ });++ test("a hanging cancel does not delay the rejection", async () => {+ const controller = new AbortController();+ const client = new FirecrawlClient({ apiKey: "fc-test", apiUrl: API_URL });+ const methods: string[] = [];+ (client as any).http.instance.defaults.adapter = (async (config) => {+ methods.push(config.method!);+ if (config.method === "delete") return new Promise(() => {});+ if (config.method === "get") setTimeout(() => controller.abort(), 10);+ const data = config.method === "post" ? started.data : running.data;+ return { data, status: 200, statusText: "OK", headers: {}, config };+ }) as AxiosAdapter;++ // Read the reason after the abort: before it, signal.reason is undefined.+ const err = await client.crawl("https://example.com", { pollInterval: 30, signal: controller.signal }).catch((e) => e);+ expect(err).toBe(controller.signal.reason);+ expect((err as Error).name).toBe("AbortError");+ await flush();+ expect(methods).toEqual(["post", "get", "delete"]);+ });++ test("abort while the start request is in flight cancels the job once the id arrives", async () => {+ const controller = new AbortController();+ const { client, sent } = makeClient((req) => {+ if (req.method === "post") {+ controller.abort();+ return started;+ }+ if (req.method === "delete") return { status: 200, data: { status: "cancelled" } };+ return running;+ });++ const err = await client.crawl("https://example.com", { signal: controller.signal }).catch((e) => e);+ expect(err).toBe(controller.signal.reason);+ expect((err as Error).name).toBe("AbortError");+ await flush();+ expect(sent.map((r) => r.method)).toEqual(["post", "delete"]);+ });++ test("an already aborted signal sends no request", async () => {+ const controller = new AbortController();+ controller.abort();+ const { client, sent } = makeClient(() => started);++ await expect(client.crawl("https://example.com", { signal: controller.signal })).rejects.toBe(controller.signal.reason);+ expect(sent).toHaveLength(0);+ });++ test("the signal is not sent in the crawl request body", async () => {+ const controller = new AbortController();+ const client = new FirecrawlClient({ apiKey: "fc-test", apiUrl: API_URL });+ const bodies: unknown[] = [];+ (client as any).http.instance.defaults.adapter = (async (config) => {+ if (config.method === "post") bodies.push(JSON.parse(config.data as string));+ const data = config.method === "post" ? started.data : { success: true, status: "completed", completed: 1, total: 1, data: [] };+ return { data, status: 200, statusText: "OK", headers: {}, config };+ }) as AxiosAdapter;++ const job = await client.crawl("https://example.com", { limit: 1, signal: controller.signal });+ expect(job.status).toBe("completed");+ expect(bodies[0]).toMatchObject({ url: "https://example.com", limit: 1 });+ expect(Object.keys(bodies[0] as object)).not.toContain("signal");+ });+});++describe("v2 waitForCrawlCompletion with an AbortSignal", () => {+ test("stops polling and rejects without cancelling the job", async () => {+ const http = new HttpClient({ apiKey: "fc-test", apiUrl: API_URL });+ const controller = new AbortController();+ const sent = withAdapter((http as any).instance, () => {+ setTimeout(() => controller.abort(), 10);+ return running;+ });++ const err = await waitForCrawlCompletion(http, JOB_ID, 30, undefined, controller.signal).catch((e) => e);+ expect(err).toBe(controller.signal.reason);+ expect((err as Error).name).toBe("AbortError");+ await new Promise((r) => setTimeout(r, 50));+ expect(sent.map((r) => r.method)).toEqual(["get"]);+ });+});diff --git a/apps/js-sdk/firecrawl/src/v2/client.ts b/apps/js-sdk/firecrawl/src/v2/client.tsindex 40c7c6bc2..694fe04e0 100644--- a/apps/js-sdk/firecrawl/src/v2/client.ts+++ b/apps/js-sdk/firecrawl/src/v2/client.ts@@ -394,7 +394,7 @@ export class FirecrawlClient {/*** Cancel a crawl job.* @param jobId Crawl job id.- * @returns True if cancelled.+ * @returns True if cancelled. False if the crawl was already completed (the API answers 409).*/async cancelCrawl(jobId: string): Promise<boolean> {return cancelCrawl(this.http, jobId);@@ -402,11 +402,14 @@ export class FirecrawlClient {/*** Convenience waiter: start a crawl and poll until it finishes.* @param url Root URL to crawl.- * @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).+ * @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds, signal).+ * When `signal` aborts, polling stops, the job gets a best-effort cancel request,+ * and the promise rejects with `signal.reason`.* @returns Final job snapshot.*/- async crawl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number } = {}): Promise<CrawlJob> {- return crawlWaiter(this.http, { url, ...req }, req.pollInterval, req.timeout);+ async crawl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number; signal?: AbortSignal } = {}): Promise<CrawlJob> {+ const { pollInterval, timeout, signal, ...options } = req;+ return crawlWaiter(this.http, { url, ...options }, pollInterval, timeout, signal);}/*** Retrieve crawl errors and robots.txt blocks.@@ -724,7 +727,7 @@ export class FirecrawlClient {}/** @deprecated V1 compatibility alias for agent recovery. Prefer crawl(). */- async crawlUrl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number } = {}): Promise<CrawlJob> {+ async crawlUrl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number; signal?: AbortSignal } = {}): Promise<CrawlJob> {return this.crawl(url, req);}diff --git a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts b/apps/js-sdk/firecrawl/src/v2/methods/crawl.tsindex 834444c13..01405d702 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts@@ -107,28 +107,94 @@ export async function getCrawlStatus(}}+/**+ * Cancel a crawl job. Resolves `false` when the API answers 409 because the+ * crawl already finished, so there is nothing left to cancel.+ */export async function cancelCrawl(http: HttpClient, jobId: string): Promise<boolean> {try {const res = await http.delete<{ status: string }>(`/v2/crawl/${jobId}`);+ if (res.status === 409) return false;if (res.status !== 200) throwForBadResponse(res, "cancel crawl");return res.data?.status === "cancelled";} catch (err: any) {- if (err?.isAxiosError) return normalizeAxiosError(err, "cancel crawl");+ if (err?.isAxiosError) {+ if (err.response?.status === 409) return false;+ return normalizeAxiosError(err, "cancel crawl");+ }throw err;}}-export async function waitForCrawlCompletion(http: HttpClient, jobId: string, pollInterval = 2, timeout?: number): Promise<CrawlJob> {+function abortReason(signal: AbortSignal): unknown {+ if (signal.reason !== undefined) return signal.reason;+ return new DOMException("This operation was aborted", "AbortError");+}++/** Reject with the abort reason as soon as `signal` aborts. */+function raceAbort<T>(promise: Promise<T>, signal?: AbortSignal): Promise<T> {+ if (!signal) return promise;+ if (signal.aborted) {+ promise.catch(() => {});+ return Promise.reject(abortReason(signal));+ }+ return new Promise<T>((resolve, reject) => {+ const onAbort = () => {+ promise.catch(() => {});+ reject(abortReason(signal));+ };+ signal.addEventListener("abort", onAbort, { once: true });+ promise.then(+ (value) => {+ signal.removeEventListener("abort", onAbort);+ resolve(value);+ },+ (err) => {+ signal.removeEventListener("abort", onAbort);+ reject(err);+ },+ );+ });+}++function sleep(ms: number, signal?: AbortSignal): Promise<void> {+ return new Promise<void>((resolve, reject) => {+ if (signal?.aborted) return reject(abortReason(signal));+ const onAbort = () => {+ clearTimeout(timer);+ reject(abortReason(signal!));+ };+ const timer = setTimeout(() => {+ signal?.removeEventListener("abort", onAbort);+ resolve();+ }, ms);+ signal?.addEventListener("abort", onAbort, { once: true });+ });+}++/**+ * Poll a crawl until it reaches a terminal status. When `signal` aborts, polling+ * stops and the promise rejects with the abort reason. The job is not cancelled.+ */+export async function waitForCrawlCompletion(+ http: HttpClient,+ jobId: string,+ pollInterval = 2,+ timeout?: number,+ signal?: AbortSignal,+): Promise<CrawlJob> {const start = Date.now();while (true) {+ if (signal?.aborted) throw abortReason(signal);try {- const status = await getCrawlStatus(http, jobId);+ const status = await raceAbort(getCrawlStatus(http, jobId), signal);if (["completed", "failed", "cancelled"].includes(status.status)) {return status;}} catch (err: any) {+ if (signal?.aborted) throw abortReason(signal);// Don't retry on permanent errors (4xx) - re-throw immediately with jobId contextif (!isRetryableError(err)) {// Create new error with jobId for better debugging (non-retryable errors like 404)@@ -151,13 +217,31 @@ export async function waitForCrawlCompletion(http: HttpClient, jobId: string, pothrow new JobTimeoutError(jobId, timeout, 'crawl');}- await new Promise((r) => setTimeout(r, Math.max(1000, pollInterval * 1000)));+ await sleep(Math.max(1000, pollInterval * 1000), signal);}}-export async function crawl(http: HttpClient, request: CrawlRequest, pollInterval = 2, timeout?: number): Promise<CrawlJob> {+/**+ * Start a crawl and wait for it to finish. When `signal` aborts, polling stops,+ * the job gets a best-effort cancel request, and the promise rejects with the+ * abort reason at once, without waiting for the cancel. A signal that is already+ * aborted rejects before any request.+ */+export async function crawl(+ http: HttpClient,+ request: CrawlRequest,+ pollInterval = 2,+ timeout?: number,+ signal?: AbortSignal,+): Promise<CrawlJob> {+ if (signal?.aborted) throw abortReason(signal);const started = await startCrawl(http, request);- return waitForCrawlCompletion(http, started.id, pollInterval, timeout);+ try {+ return await waitForCrawlCompletion(http, started.id, pollInterval, timeout, signal);+ } catch (err) {+ if (signal?.aborted) cancelCrawl(http, started.id).catch(() => {});+ throw err;+ }}export async function getCrawlErrors(http: HttpClient, crawlId: string): Promise<CrawlErrorsResponse> {