Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

js-sdk: Retry polling after errors, add jobId to error exception (#2608)

Abimael Martellc61a3f03e805
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index a3750a106..33d81a7a6 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.9.3",
+ "version": "4.10.0",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/batch.ts b/apps/js-sdk/firecrawl/src/v2/methods/batch.ts
index 684736f8f..58ebf6b1a 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/batch.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/batch.ts
@@ -5,11 +5,13 @@ import {
type Document,
type BatchScrapeOptions,
type PaginationConfig,
+ JobTimeoutError,
+ SdkError,
} from "../types";
import { HttpClient } from "../utils/httpClient";
import { ensureValidScrapeOptions } from "../utils/validation";
import { fetchAllPages } from "../utils/pagination";
-import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";
+import { normalizeAxiosError, throwForBadResponse, isRetryableError } from "../utils/errorHandler";
export async function startBatchScrape(
http: HttpClient,
@@ -115,12 +117,37 @@ export async function getBatchScrapeErrors(http: HttpClient, jobId: string): Pro
export async function waitForBatchCompletion(http: HttpClient, jobId: string, pollInterval = 2, timeout?: number): Promise<BatchScrapeJob> {
const start = Date.now();
+
while (true) {
- const status = await getBatchScrapeStatus(http, jobId);
- if (["completed", "failed", "cancelled"].includes(status.status)) return status;
+ try {
+ const status = await getBatchScrapeStatus(http, jobId);
+
+ if (["completed", "failed", "cancelled"].includes(status.status)) {
+ return status;
+ }
+ } catch (err: any) {
+ // Don't retry on permanent errors (4xx) - re-throw immediately with jobId context
+ if (!isRetryableError(err)) {
+ // Create new error with jobId for better debugging (non-retryable errors like 404)
+ if (err instanceof SdkError) {
+ const errorWithJobId = new SdkError(
+ err.message,
+ err.status,
+ err.code,
+ err.details,
+ jobId
+ );
+ throw errorWithJobId;
+ }
+ throw err;
+ }
+ // Otherwise, retry after delay - error might be transient (network issue, timeout, 5xx, etc.)
+ }
+
if (timeout != null && Date.now() - start > timeout * 1000) {
- throw new Error(`Batch scrape job ${jobId} did not complete within ${timeout} seconds`);
+ throw new JobTimeoutError(jobId, timeout, 'batch');
}
+
await new Promise((r) => setTimeout(r, Math.max(1000, pollInterval * 1000)));
}
}
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts b/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts
index b9201fb20..ba110ca37 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts
@@ -6,10 +6,12 @@ import {
type Document,
type CrawlOptions,
type PaginationConfig,
+ JobTimeoutError,
+ SdkError,
} from "../types";
import { HttpClient } from "../utils/httpClient";
import { ensureValidScrapeOptions } from "../utils/validation";
-import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";
+import { normalizeAxiosError, throwForBadResponse, isRetryableError } from "../utils/errorHandler";
import type { HttpClient as _Http } from "../utils/httpClient";
import { fetchAllPages } from "../utils/pagination";
@@ -114,12 +116,37 @@ export async function cancelCrawl(http: HttpClient, jobId: string): Promise<bool
export async function waitForCrawlCompletion(http: HttpClient, jobId: string, pollInterval = 2, timeout?: number): Promise<CrawlJob> {
const start = Date.now();
+
while (true) {
- const status = await getCrawlStatus(http, jobId);
- if (["completed", "failed", "cancelled"].includes(status.status)) return status;
+ try {
+ const status = await getCrawlStatus(http, jobId);
+
+ if (["completed", "failed", "cancelled"].includes(status.status)) {
+ return status;
+ }
+ } catch (err: any) {
+ // Don't retry on permanent errors (4xx) - re-throw immediately with jobId context
+ if (!isRetryableError(err)) {
+ // Create new error with jobId for better debugging (non-retryable errors like 404)
+ if (err instanceof SdkError) {
+ const errorWithJobId = new SdkError(
+ err.message,
+ err.status,
+ err.code,
+ err.details,
+ jobId
+ );
+ throw errorWithJobId;
+ }
+ throw err;
+ }
+ // Otherwise, retry after delay - error might be transient (network issue, timeout, 5xx, etc.)
+ }
+
if (timeout != null && Date.now() - start > timeout * 1000) {
- throw new Error(`Crawl job ${jobId} did not complete within ${timeout} seconds`);
+ throw new JobTimeoutError(jobId, timeout, 'crawl');
}
+
await new Promise((r) => setTimeout(r, Math.max(1000, pollInterval * 1000)));
}
}
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index 87523e64d..5869be0d6 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -631,17 +631,36 @@ export class SdkError extends Error {
status?: number;
code?: string;
details?: unknown;
+ jobId?: string;
constructor(
message: string,
status?: number,
code?: string,
- details?: unknown
+ details?: unknown,
+ jobId?: string
) {
super(message);
this.name = 'FirecrawlSdkError';
this.status = status;
this.code = code;
this.details = details;
+ this.jobId = jobId;
+ }
+}
+
+export class JobTimeoutError extends SdkError {
+ timeoutSeconds: number;
+ constructor(jobId: string, timeoutSeconds: number, jobType: 'batch' | 'crawl' = 'batch') {
+ const jobTypeLabel = jobType === 'batch' ? 'batch scrape' : 'crawl';
+ super(
+ `${jobTypeLabel.charAt(0).toUpperCase() + jobTypeLabel.slice(1)} job ${jobId} did not complete within ${timeoutSeconds} seconds`,
+ undefined,
+ 'JOB_TIMEOUT',
+ undefined,
+ jobId
+ );
+ this.name = 'JobTimeoutError';
+ this.timeoutSeconds = timeoutSeconds;
}
}
diff --git a/apps/js-sdk/firecrawl/src/v2/utils/errorHandler.ts b/apps/js-sdk/firecrawl/src/v2/utils/errorHandler.ts
index 10adfc974..cc0907013 100644
--- a/apps/js-sdk/firecrawl/src/v2/utils/errorHandler.ts
+++ b/apps/js-sdk/firecrawl/src/v2/utils/errorHandler.ts
@@ -1,5 +1,5 @@
import { type AxiosError, type AxiosResponse } from "axios";
-import { SdkError } from "../types";
+import { SdkError, JobTimeoutError } from "../types";
export function throwForBadResponse(resp: AxiosResponse, action: string): never {
const status = resp.status;
@@ -16,3 +16,36 @@ export function normalizeAxiosError(err: AxiosError, action: string): never {
throw new SdkError(message, status, code, body?.details ?? body);
}
+export function isRetryableError(err: any): boolean {
+ // JobTimeoutError should never be retried - it's the overall timeout
+ if (err instanceof JobTimeoutError) {
+ return false;
+ }
+
+ // If it's an SdkError with a status code, check if it's retryable
+ if (err instanceof SdkError || (err && typeof err === 'object' && 'status' in err)) {
+ const status = err.status;
+ // 4xx errors are client errors and shouldn't be retried
+ if (status && status >= 400 && status < 500) {
+ return false; // Don't retry client errors (401, 404, etc.)
+ }
+ // 5xx errors are server errors and can be retried
+ if (status && status >= 500) {
+ return true;
+ }
+ }
+
+ // Network errors (no response) are retryable
+ if (err?.isAxiosError && !err.response) {
+ return true;
+ }
+
+ // HTTP timeout errors are retryable (different from JobTimeoutError)
+ if (err?.code === 'ECONNABORTED' || err?.message?.includes('timeout')) {
+ return true;
+ }
+
+ // Default: retry on unknown errors (safer than not retrying)
+ return true;
+}
+