Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(js-sdk): abortable crawl() that cancels its job; cancelCrawl returns false on 409 (#4898)

claude[bot]23b68aefd291
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index ba1780a84..1265ebb80 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.44.1",
+ "version": "4.45.0",
"description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
"main": "dist/index.js",
"types": "dist/index.d.ts",
diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/crawl-cancel-abort.unit.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/crawl-cancel-abort.unit.test.ts
new file mode 100644
index 000000000..e33f15d7c
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/crawl-cancel-abort.unit.test.ts
@@ -0,0 +1,189 @@
+import { describe, test, expect } from "@jest/globals";
+import axios, { type AxiosAdapter } from "axios";
+import { FirecrawlClient } from "../../../v2/client";
+import { HttpClient } from "../../../v2/utils/httpClient";
+import { waitForCrawlCompletion } from "../../../v2/methods/crawl";
+import { SdkError } from "../../../v2/types";
+
+const API_URL = "https://api.firecrawl.dev";
+const JOB_ID = "job-123";
+
+type Sent = { method: string; path: string };
+type Reply = { status: number; data: unknown };
+
+function withAdapter<T extends { defaults: any }>(instance: T, route: (req: Sent) => Reply): Sent[] {
+ const sent: Sent[] = [];
+ const adapter: AxiosAdapter = async (config) => {
+ const req = {
+ method: (config.method || "get").toLowerCase(),
+ path: new URL(axios.getUri(config)).pathname,
+ };
+ sent.push(req);
+ const { status, data } = route(req);
+ const response = { data, status, statusText: String(status), headers: {}, config };
+ const validate = config.validateStatus;
+ if (validate && !validate(status)) {
+ throw new axios.AxiosError(`Request failed with status code ${status}`, undefined, config, null, response);
+ }
+ return response;
+ };
+ instance.defaults.adapter = adapter;
+ return sent;
+}
+
+function makeClient(route: (req: Sent) => Reply) {
+ const client = new FirecrawlClient({ apiKey: "fc-test", apiUrl: API_URL });
+ const sent = withAdapter((client as any).http.instance, route);
+ return { client, sent };
+}
+
+// The cancel runs in the background, so let its request reach the adapter.
+const flush = () => new Promise((r) => setTimeout(r, 0));
+
+const running: Reply = { status: 200, data: { success: true, status: "scraping", completed: 0, total: 1, data: [] } };
+const started: Reply = { status: 200, data: { success: true, id: JOB_ID, url: "https://example.com" } };
+
+describe("v2 cancelCrawl", () => {
+ test("returns false when the crawl is already completed (409)", async () => {
+ const { client, sent } = makeClient(() => ({ status: 409, data: { error: "Crawl is already completed" } }));
+ await expect(client.cancelCrawl(JOB_ID)).resolves.toBe(false);
+ expect(sent).toEqual([{ method: "delete", path: `/v2/crawl/${JOB_ID}` }]);
+ });
+
+ test("returns false on 409 when the transport does not throw for 4xx", async () => {
+ const { client } = makeClient(() => ({ status: 409, data: { error: "Crawl is already completed" } }));
+ (client as any).http.instance.defaults.validateStatus = () => true;
+ await expect(client.cancelCrawl(JOB_ID)).resolves.toBe(false);
+ });
+
+ test("returns true when the crawl is cancelled", async () => {
+ const { client } = makeClient(() => ({ status: 200, data: { status: "cancelled" } }));
+ await expect(client.cancelCrawl(JOB_ID)).resolves.toBe(true);
+ });
+
+ test("still throws for other errors", async () => {
+ const { client } = makeClient(() => ({ status: 404, data: { error: "Job not found" } }));
+ const err = await client.cancelCrawl(JOB_ID).catch((e) => e);
+ expect(err).toBeInstanceOf(SdkError);
+ expect(err.status).toBe(404);
+ });
+});
+
+describe("v2 crawl with an AbortSignal", () => {
+ test("abort during polling stops polling, cancels the job, and rejects with the abort reason", async () => {
+ const controller = new AbortController();
+ let polls = 0;
+ const { client, sent } = makeClient((req) => {
+ if (req.method === "post") return started;
+ if (req.method === "delete") return { status: 200, data: { status: "cancelled" } };
+ polls += 1;
+ if (polls === 1) setTimeout(() => controller.abort(), 10);
+ return running;
+ });
+
+ const t0 = Date.now();
+ const err = await client.crawl("https://example.com", { pollInterval: 30, signal: controller.signal }).catch((e) => e);
+
+ expect(Date.now() - t0).toBeLessThan(5000);
+ expect(err).toBe(controller.signal.reason);
+ expect((err as Error).name).toBe("AbortError");
+ await flush();
+ expect(sent.map((r) => r.method)).toEqual(["post", "get", "delete"]);
+ expect(sent[2]!.path).toBe(`/v2/crawl/${JOB_ID}`);
+ });
+
+ test("rejects with a custom abort reason and ignores a failing cancel", async () => {
+ const controller = new AbortController();
+ const reason = new Error("caller gave up");
+ const { client, sent } = makeClient((req) => {
+ if (req.method === "post") return started;
+ if (req.method === "delete") return { status: 500, data: { error: "boom" } };
+ setTimeout(() => controller.abort(reason), 10);
+ return running;
+ });
+
+ await expect(client.crawl("https://example.com", { pollInterval: 30, signal: controller.signal })).rejects.toBe(reason);
+ await flush();
+ expect(sent.some((r) => r.method === "delete")).toBe(true);
+ });
+
+ test("a hanging cancel does not delay the rejection", async () => {
+ const controller = new AbortController();
+ const client = new FirecrawlClient({ apiKey: "fc-test", apiUrl: API_URL });
+ const methods: string[] = [];
+ (client as any).http.instance.defaults.adapter = (async (config) => {
+ methods.push(config.method!);
+ if (config.method === "delete") return new Promise(() => {});
+ if (config.method === "get") setTimeout(() => controller.abort(), 10);
+ const data = config.method === "post" ? started.data : running.data;
+ return { data, status: 200, statusText: "OK", headers: {}, config };
+ }) as AxiosAdapter;
+
+ // Read the reason after the abort: before it, signal.reason is undefined.
+ const err = await client.crawl("https://example.com", { pollInterval: 30, signal: controller.signal }).catch((e) => e);
+ expect(err).toBe(controller.signal.reason);
+ expect((err as Error).name).toBe("AbortError");
+ await flush();
+ expect(methods).toEqual(["post", "get", "delete"]);
+ });
+
+ test("abort while the start request is in flight cancels the job once the id arrives", async () => {
+ const controller = new AbortController();
+ const { client, sent } = makeClient((req) => {
+ if (req.method === "post") {
+ controller.abort();
+ return started;
+ }
+ if (req.method === "delete") return { status: 200, data: { status: "cancelled" } };
+ return running;
+ });
+
+ const err = await client.crawl("https://example.com", { signal: controller.signal }).catch((e) => e);
+ expect(err).toBe(controller.signal.reason);
+ expect((err as Error).name).toBe("AbortError");
+ await flush();
+ expect(sent.map((r) => r.method)).toEqual(["post", "delete"]);
+ });
+
+ test("an already aborted signal sends no request", async () => {
+ const controller = new AbortController();
+ controller.abort();
+ const { client, sent } = makeClient(() => started);
+
+ await expect(client.crawl("https://example.com", { signal: controller.signal })).rejects.toBe(controller.signal.reason);
+ expect(sent).toHaveLength(0);
+ });
+
+ test("the signal is not sent in the crawl request body", async () => {
+ const controller = new AbortController();
+ const client = new FirecrawlClient({ apiKey: "fc-test", apiUrl: API_URL });
+ const bodies: unknown[] = [];
+ (client as any).http.instance.defaults.adapter = (async (config) => {
+ if (config.method === "post") bodies.push(JSON.parse(config.data as string));
+ const data = config.method === "post" ? started.data : { success: true, status: "completed", completed: 1, total: 1, data: [] };
+ return { data, status: 200, statusText: "OK", headers: {}, config };
+ }) as AxiosAdapter;
+
+ const job = await client.crawl("https://example.com", { limit: 1, signal: controller.signal });
+ expect(job.status).toBe("completed");
+ expect(bodies[0]).toMatchObject({ url: "https://example.com", limit: 1 });
+ expect(Object.keys(bodies[0] as object)).not.toContain("signal");
+ });
+});
+
+describe("v2 waitForCrawlCompletion with an AbortSignal", () => {
+ test("stops polling and rejects without cancelling the job", async () => {
+ const http = new HttpClient({ apiKey: "fc-test", apiUrl: API_URL });
+ const controller = new AbortController();
+ const sent = withAdapter((http as any).instance, () => {
+ setTimeout(() => controller.abort(), 10);
+ return running;
+ });
+
+ const err = await waitForCrawlCompletion(http, JOB_ID, 30, undefined, controller.signal).catch((e) => e);
+ expect(err).toBe(controller.signal.reason);
+ expect((err as Error).name).toBe("AbortError");
+ await new Promise((r) => setTimeout(r, 50));
+ expect(sent.map((r) => r.method)).toEqual(["get"]);
+ });
+});
diff --git a/apps/js-sdk/firecrawl/src/v2/client.ts b/apps/js-sdk/firecrawl/src/v2/client.ts
index 40c7c6bc2..694fe04e0 100644
--- a/apps/js-sdk/firecrawl/src/v2/client.ts
+++ b/apps/js-sdk/firecrawl/src/v2/client.ts
@@ -394,7 +394,7 @@ export class FirecrawlClient {
/**
* Cancel a crawl job.
* @param jobId Crawl job id.
- * @returns True if cancelled.
+ * @returns True if cancelled. False if the crawl was already completed (the API answers 409).
*/
async cancelCrawl(jobId: string): Promise<boolean> {
return cancelCrawl(this.http, jobId);
@@ -402,11 +402,14 @@ export class FirecrawlClient {
/**
* Convenience waiter: start a crawl and poll until it finishes.
* @param url Root URL to crawl.
- * @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).
+ * @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds, signal).
+ * When `signal` aborts, polling stops, the job gets a best-effort cancel request,
+ * and the promise rejects with `signal.reason`.
* @returns Final job snapshot.
*/
- async crawl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number } = {}): Promise<CrawlJob> {
- return crawlWaiter(this.http, { url, ...req }, req.pollInterval, req.timeout);
+ async crawl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number; signal?: AbortSignal } = {}): Promise<CrawlJob> {
+ const { pollInterval, timeout, signal, ...options } = req;
+ return crawlWaiter(this.http, { url, ...options }, pollInterval, timeout, signal);
}
/**
* Retrieve crawl errors and robots.txt blocks.
@@ -724,7 +727,7 @@ export class FirecrawlClient {
}
/** @deprecated V1 compatibility alias for agent recovery. Prefer crawl(). */
- async crawlUrl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number } = {}): Promise<CrawlJob> {
+ async crawlUrl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number; signal?: AbortSignal } = {}): Promise<CrawlJob> {
return this.crawl(url, req);
}
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts b/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts
index 834444c13..01405d702 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts
@@ -107,28 +107,94 @@ export async function getCrawlStatus(
}
}
+/**
+ * Cancel a crawl job. Resolves `false` when the API answers 409 because the
+ * crawl already finished, so there is nothing left to cancel.
+ */
export async function cancelCrawl(http: HttpClient, jobId: string): Promise<boolean> {
try {
const res = await http.delete<{ status: string }>(`/v2/crawl/${jobId}`);
+ if (res.status === 409) return false;
if (res.status !== 200) throwForBadResponse(res, "cancel crawl");
return res.data?.status === "cancelled";
} catch (err: any) {
- if (err?.isAxiosError) return normalizeAxiosError(err, "cancel crawl");
+ if (err?.isAxiosError) {
+ if (err.response?.status === 409) return false;
+ return normalizeAxiosError(err, "cancel crawl");
+ }
throw err;
}
}
-export async function waitForCrawlCompletion(http: HttpClient, jobId: string, pollInterval = 2, timeout?: number): Promise<CrawlJob> {
+function abortReason(signal: AbortSignal): unknown {
+ if (signal.reason !== undefined) return signal.reason;
+ return new DOMException("This operation was aborted", "AbortError");
+}
+
+/** Reject with the abort reason as soon as `signal` aborts. */
+function raceAbort<T>(promise: Promise<T>, signal?: AbortSignal): Promise<T> {
+ if (!signal) return promise;
+ if (signal.aborted) {
+ promise.catch(() => {});
+ return Promise.reject(abortReason(signal));
+ }
+ return new Promise<T>((resolve, reject) => {
+ const onAbort = () => {
+ promise.catch(() => {});
+ reject(abortReason(signal));
+ };
+ signal.addEventListener("abort", onAbort, { once: true });
+ promise.then(
+ (value) => {
+ signal.removeEventListener("abort", onAbort);
+ resolve(value);
+ },
+ (err) => {
+ signal.removeEventListener("abort", onAbort);
+ reject(err);
+ },
+ );
+ });
+}
+
+function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+ return new Promise<void>((resolve, reject) => {
+ if (signal?.aborted) return reject(abortReason(signal));
+ const onAbort = () => {
+ clearTimeout(timer);
+ reject(abortReason(signal!));
+ };
+ const timer = setTimeout(() => {
+ signal?.removeEventListener("abort", onAbort);
+ resolve();
+ }, ms);
+ signal?.addEventListener("abort", onAbort, { once: true });
+ });
+}
+
+/**
+ * Poll a crawl until it reaches a terminal status. When `signal` aborts, polling
+ * stops and the promise rejects with the abort reason. The job is not cancelled.
+ */
+export async function waitForCrawlCompletion(
+ http: HttpClient,
+ jobId: string,
+ pollInterval = 2,
+ timeout?: number,
+ signal?: AbortSignal,
+): Promise<CrawlJob> {
const start = Date.now();
while (true) {
+ if (signal?.aborted) throw abortReason(signal);
try {
- const status = await getCrawlStatus(http, jobId);
+ const status = await raceAbort(getCrawlStatus(http, jobId), signal);
if (["completed", "failed", "cancelled"].includes(status.status)) {
return status;
}
} catch (err: any) {
+ if (signal?.aborted) throw abortReason(signal);
// Don't retry on permanent errors (4xx) - re-throw immediately with jobId context
if (!isRetryableError(err)) {
// Create new error with jobId for better debugging (non-retryable errors like 404)
@@ -151,13 +217,31 @@ export async function waitForCrawlCompletion(http: HttpClient, jobId: string, po
throw new JobTimeoutError(jobId, timeout, 'crawl');
}
- await new Promise((r) => setTimeout(r, Math.max(1000, pollInterval * 1000)));
+ await sleep(Math.max(1000, pollInterval * 1000), signal);
}
}
-export async function crawl(http: HttpClient, request: CrawlRequest, pollInterval = 2, timeout?: number): Promise<CrawlJob> {
+/**
+ * Start a crawl and wait for it to finish. When `signal` aborts, polling stops,
+ * the job gets a best-effort cancel request, and the promise rejects with the
+ * abort reason at once, without waiting for the cancel. A signal that is already
+ * aborted rejects before any request.
+ */
+export async function crawl(
+ http: HttpClient,
+ request: CrawlRequest,
+ pollInterval = 2,
+ timeout?: number,
+ signal?: AbortSignal,
+): Promise<CrawlJob> {
+ if (signal?.aborted) throw abortReason(signal);
const started = await startCrawl(http, request);
- return waitForCrawlCompletion(http, started.id, pollInterval, timeout);
+ try {
+ return await waitForCrawlCompletion(http, started.id, pollInterval, timeout, signal);
+ } catch (err) {
+ if (signal?.aborted) cancelCrawl(http, started.id).catch(() => {});
+ throw err;
+ }
}
export async function getCrawlErrors(http: HttpClient, crawlId: string): Promise<CrawlErrorsResponse> {