Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
js-sdk: Retry polling after errors, add jobId to error exception (#2608)
Abimael Martellc61a3f03e805
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex a3750a106..33d81a7a6 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.9.3",+ "version": "4.10.0","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/v2/methods/batch.ts b/apps/js-sdk/firecrawl/src/v2/methods/batch.tsindex 684736f8f..58ebf6b1a 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/batch.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/batch.ts@@ -5,11 +5,13 @@ import {type Document,type BatchScrapeOptions,type PaginationConfig,+ JobTimeoutError,+ SdkError,} from "../types";import { HttpClient } from "../utils/httpClient";import { ensureValidScrapeOptions } from "../utils/validation";import { fetchAllPages } from "../utils/pagination";-import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";+import { normalizeAxiosError, throwForBadResponse, isRetryableError } from "../utils/errorHandler";export async function startBatchScrape(http: HttpClient,@@ -115,12 +117,37 @@ export async function getBatchScrapeErrors(http: HttpClient, jobId: string): Proexport async function waitForBatchCompletion(http: HttpClient, jobId: string, pollInterval = 2, timeout?: number): Promise<BatchScrapeJob> {const start = Date.now();+while (true) {- const status = await getBatchScrapeStatus(http, jobId);- if (["completed", "failed", "cancelled"].includes(status.status)) return status;+ try {+ const status = await getBatchScrapeStatus(http, jobId);++ if (["completed", "failed", "cancelled"].includes(status.status)) {+ return status;+ }+ } catch (err: any) {+ // Don't retry on permanent errors (4xx) - re-throw immediately with jobId context+ if (!isRetryableError(err)) {+ // Create new error with jobId for better debugging (non-retryable errors like 404)+ if (err instanceof SdkError) {+ const errorWithJobId = new SdkError(+ err.message,+ err.status,+ err.code,+ err.details,+ jobId+ );+ throw errorWithJobId;+ }+ throw err;+ }+ // Otherwise, retry after delay - error might be transient (network issue, timeout, 5xx, etc.)+ }+if (timeout != null && Date.now() - start > timeout * 1000) {- throw new Error(`Batch scrape job ${jobId} did not complete within ${timeout} seconds`);+ throw new JobTimeoutError(jobId, timeout, 'batch');}+await new Promise((r) => setTimeout(r, Math.max(1000, pollInterval * 1000)));}}diff --git a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts b/apps/js-sdk/firecrawl/src/v2/methods/crawl.tsindex b9201fb20..ba110ca37 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts@@ -6,10 +6,12 @@ import {type Document,type CrawlOptions,type PaginationConfig,+ JobTimeoutError,+ SdkError,} from "../types";import { HttpClient } from "../utils/httpClient";import { ensureValidScrapeOptions } from "../utils/validation";-import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";+import { normalizeAxiosError, throwForBadResponse, isRetryableError } from "../utils/errorHandler";import type { HttpClient as _Http } from "../utils/httpClient";import { fetchAllPages } from "../utils/pagination";@@ -114,12 +116,37 @@ export async function cancelCrawl(http: HttpClient, jobId: string): Promise<boolexport async function waitForCrawlCompletion(http: HttpClient, jobId: string, pollInterval = 2, timeout?: number): Promise<CrawlJob> {const start = Date.now();+while (true) {- const status = await getCrawlStatus(http, jobId);- if (["completed", "failed", "cancelled"].includes(status.status)) return status;+ try {+ const status = await getCrawlStatus(http, jobId);++ if (["completed", "failed", "cancelled"].includes(status.status)) {+ return status;+ }+ } catch (err: any) {+ // Don't retry on permanent errors (4xx) - re-throw immediately with jobId context+ if (!isRetryableError(err)) {+ // Create new error with jobId for better debugging (non-retryable errors like 404)+ if (err instanceof SdkError) {+ const errorWithJobId = new SdkError(+ err.message,+ err.status,+ err.code,+ err.details,+ jobId+ );+ throw errorWithJobId;+ }+ throw err;+ }+ // Otherwise, retry after delay - error might be transient (network issue, timeout, 5xx, etc.)+ }+if (timeout != null && Date.now() - start > timeout * 1000) {- throw new Error(`Crawl job ${jobId} did not complete within ${timeout} seconds`);+ throw new JobTimeoutError(jobId, timeout, 'crawl');}+await new Promise((r) => setTimeout(r, Math.max(1000, pollInterval * 1000)));}}diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex 87523e64d..5869be0d6 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -631,17 +631,36 @@ export class SdkError extends Error {status?: number;code?: string;details?: unknown;+ jobId?: string;constructor(message: string,status?: number,code?: string,- details?: unknown+ details?: unknown,+ jobId?: string) {super(message);this.name = 'FirecrawlSdkError';this.status = status;this.code = code;this.details = details;+ this.jobId = jobId;+ }+}++export class JobTimeoutError extends SdkError {+ timeoutSeconds: number;+ constructor(jobId: string, timeoutSeconds: number, jobType: 'batch' | 'crawl' = 'batch') {+ const jobTypeLabel = jobType === 'batch' ? 'batch scrape' : 'crawl';+ super(+ `${jobTypeLabel.charAt(0).toUpperCase() + jobTypeLabel.slice(1)} job ${jobId} did not complete within ${timeoutSeconds} seconds`,+ undefined,+ 'JOB_TIMEOUT',+ undefined,+ jobId+ );+ this.name = 'JobTimeoutError';+ this.timeoutSeconds = timeoutSeconds;}}diff --git a/apps/js-sdk/firecrawl/src/v2/utils/errorHandler.ts b/apps/js-sdk/firecrawl/src/v2/utils/errorHandler.tsindex 10adfc974..cc0907013 100644--- a/apps/js-sdk/firecrawl/src/v2/utils/errorHandler.ts+++ b/apps/js-sdk/firecrawl/src/v2/utils/errorHandler.ts@@ -1,5 +1,5 @@import { type AxiosError, type AxiosResponse } from "axios";-import { SdkError } from "../types";+import { SdkError, JobTimeoutError } from "../types";export function throwForBadResponse(resp: AxiosResponse, action: string): never {const status = resp.status;@@ -16,3 +16,36 @@ export function normalizeAxiosError(err: AxiosError, action: string): never {throw new SdkError(message, status, code, body?.details ?? body);}+export function isRetryableError(err: any): boolean {+ // JobTimeoutError should never be retried - it's the overall timeout+ if (err instanceof JobTimeoutError) {+ return false;+ }++ // If it's an SdkError with a status code, check if it's retryable+ if (err instanceof SdkError || (err && typeof err === 'object' && 'status' in err)) {+ const status = err.status;+ // 4xx errors are client errors and shouldn't be retried+ if (status && status >= 400 && status < 500) {+ return false; // Don't retry client errors (401, 404, etc.)+ }+ // 5xx errors are server errors and can be retried+ if (status && status >= 500) {+ return true;+ }+ }++ // Network errors (no response) are retryable+ if (err?.isAxiosError && !err.response) {+ return true;+ }++ // HTTP timeout errors are retryable (different from JobTimeoutError)+ if (err?.code === 'ECONNABORTED' || err?.message?.includes('timeout')) {+ return true;+ }++ // Default: retry on unknown errors (safer than not retrying)+ return true;+}+