Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(js-sdk): add retry logic for socket hang up errors in monitorJobStatus (ENG-3029) (#1893)

devin-ai-integration[bot]049d2e6bcc5e
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index 96ed80ae1..1aa0a0420 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "1.29.2",
+ "version": "1.29.3",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",
@@ -16,7 +16,8 @@
"build": "tsup",
"build-and-publish": "npm run build && npm publish --access public",
"publish-beta": "npm run build && npm publish --access public --tag beta",
- "test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts"
+ "test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts",
+ "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/unit/*.test.ts"
},
"repository": {
"type": "git",
diff --git a/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts b/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts
index ffe3ac747..a0a2d345a 100644
--- a/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts
+++ b/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts
@@ -297,4 +297,5 @@ describe('FirecrawlApp E2E Tests', () => {
const app = new FirecrawlApp({ apiUrl: API_URL, apiKey: "invalid_api_key" });
await expect(app.search("test query")).rejects.toThrow("Request failed with status code 401");
});
+
});
diff --git a/apps/js-sdk/firecrawl/src/__tests__/v1/unit/monitor-job-status-retry.test.ts b/apps/js-sdk/firecrawl/src/__tests__/v1/unit/monitor-job-status-retry.test.ts
new file mode 100644
index 000000000..75563aa3b
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/__tests__/v1/unit/monitor-job-status-retry.test.ts
@@ -0,0 +1,154 @@
+import FirecrawlApp from '../../../index';
+import { describe, test, expect, jest, beforeEach, afterEach } from '@jest/globals';
+
+describe('monitorJobStatus retry logic', () => {
+ let app: FirecrawlApp;
+ let originalConsoleWarn: typeof console.warn;
+
+ beforeEach(() => {
+ app = new FirecrawlApp({ apiKey: 'test-key', apiUrl: 'https://test.com' });
+ originalConsoleWarn = console.warn;
+ console.warn = jest.fn();
+ });
+
+ afterEach(() => {
+ console.warn = originalConsoleWarn;
+ jest.clearAllMocks();
+ });
+
+ test('should retry on socket hang up error', async () => {
+ const socketHangUpError = new Error('socket hang up') as any;
+ socketHangUpError.code = 'ECONNRESET';
+
+ const successResponse = {
+ status: 200,
+ data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
+ };
+
+ const originalGetRequest = app.getRequest;
+ let callCount = 0;
+
+ app.getRequest = async function(url: string, headers: any) {
+ callCount++;
+ if (callCount === 1) {
+ throw socketHangUpError;
+ }
+ return successResponse;
+ };
+
+ const result = await app.monitorJobStatus('test-id', {}, 1);
+
+ expect(callCount).toBe(2);
+ expect(result).toEqual(successResponse.data);
+ expect(console.warn).toHaveBeenCalledWith(
+ expect.stringContaining('Network error during job status check (attempt 1/3): socket hang up')
+ );
+ });
+
+ test('should retry on ETIMEDOUT error', async () => {
+ const timeoutError = new Error('timeout') as any;
+ timeoutError.code = 'ETIMEDOUT';
+
+ const successResponse = {
+ status: 200,
+ data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
+ };
+
+ const originalGetRequest = app.getRequest;
+ let callCount = 0;
+
+ app.getRequest = async function(url: string, headers: any) {
+ callCount++;
+ if (callCount === 1) {
+ throw timeoutError;
+ }
+ return successResponse;
+ };
+
+ const result = await app.monitorJobStatus('test-id', {}, 1);
+
+ expect(callCount).toBe(2);
+ expect(result).toEqual(successResponse.data);
+ });
+
+ test('should fail after max retries exceeded', async () => {
+ const socketHangUpError = new Error('socket hang up') as any;
+ socketHangUpError.code = 'ECONNRESET';
+
+ app.getRequest = async function(url: string, headers: any) {
+ throw socketHangUpError;
+ };
+
+ await expect(app.monitorJobStatus('test-id', {}, 1)).rejects.toThrow('socket hang up');
+
+ expect(console.warn).toHaveBeenCalledTimes(3);
+ }, 15000);
+
+ test('should not retry on non-retryable errors', async () => {
+ const authError = new Error('Unauthorized') as any;
+ authError.response = { status: 401, data: { error: 'Unauthorized' } };
+
+ app.getRequest = async function(url: string, headers: any) {
+ throw authError;
+ };
+
+ await expect(app.monitorJobStatus('test-id', {}, 1)).rejects.toThrow('Unauthorized');
+
+ expect(console.warn).not.toHaveBeenCalled();
+ });
+
+ test('should retry on HTTP timeout status codes', async () => {
+ const timeoutError = new Error('Request timeout') as any;
+ timeoutError.response = { status: 408, data: { error: 'Request timeout' } };
+
+ const successResponse = {
+ status: 200,
+ data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
+ };
+
+ const originalGetRequest = app.getRequest;
+ let callCount = 0;
+
+ app.getRequest = async function(url: string, headers: any) {
+ callCount++;
+ if (callCount === 1) {
+ throw timeoutError;
+ }
+ return successResponse;
+ };
+
+ const result = await app.monitorJobStatus('test-id', {}, 1);
+
+ expect(callCount).toBe(2);
+ expect(result).toEqual(successResponse.data);
+ });
+
+ test('should use exponential backoff for retries', async () => {
+ const socketHangUpError = new Error('socket hang up') as any;
+ socketHangUpError.code = 'ECONNRESET';
+
+ const successResponse = {
+ status: 200,
+ data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
+ };
+
+ const originalGetRequest = app.getRequest;
+ let callCount = 0;
+
+ app.getRequest = async function(url: string, headers: any) {
+ callCount++;
+ if (callCount <= 2) {
+ throw socketHangUpError;
+ }
+ return successResponse;
+ };
+
+ const startTime = Date.now();
+ const result = await app.monitorJobStatus('test-id', {}, 1);
+ const endTime = Date.now();
+
+ expect(callCount).toBe(3);
+ expect(result).toEqual(successResponse.data);
+ expect(endTime - startTime).toBeGreaterThan(3000);
+ });
+});
diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts
index 139f7a919..f93bb48e3 100644
--- a/apps/js-sdk/firecrawl/src/index.ts
+++ b/apps/js-sdk/firecrawl/src/index.ts
@@ -1455,33 +1455,39 @@ export default class FirecrawlApp {
headers: AxiosRequestHeaders,
checkInterval: number
): Promise<CrawlStatusResponse | ErrorResponse> {
- try {
- let failedTries = 0;
- while (true) {
+ let failedTries = 0;
+ let networkRetries = 0;
+ const maxNetworkRetries = 3;
+
+ while (true) {
+ try {
let statusResponse: AxiosResponse = await this.getRequest(
`${this.apiUrl}/v1/crawl/${id}`,
headers
);
+
if (statusResponse.status === 200) {
failedTries = 0;
+ networkRetries = 0;
let statusData = statusResponse.data;
- if (statusData.status === "completed") {
- if ("data" in statusData) {
- let data = statusData.data;
- while (typeof statusData === 'object' && 'next' in statusData) {
- if (data.length === 0) {
- break
- }
- statusResponse = await this.getRequest(statusData.next, headers);
- statusData = statusResponse.data;
- data = data.concat(statusData.data);
+
+ if (statusData.status === "completed") {
+ if ("data" in statusData) {
+ let data = statusData.data;
+ while (typeof statusData === 'object' && 'next' in statusData) {
+ if (data.length === 0) {
+ break
}
- statusData.data = data;
- return statusData;
- } else {
- throw new FirecrawlError("Crawl job completed but no data was returned", 500);
+ statusResponse = await this.getRequest(statusData.next, headers);
+ statusData = statusResponse.data;
+ data = data.concat(statusData.data);
}
- } else if (
+ statusData.data = data;
+ return statusData;
+ } else {
+ throw new FirecrawlError("Crawl job completed but no data was returned", 500);
+ }
+ } else if (
["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)
) {
checkInterval = Math.max(checkInterval, 2);
@@ -1500,12 +1506,69 @@ export default class FirecrawlApp {
this.handleError(statusResponse, "check crawl status");
}
}
+ } catch (error: any) {
+ if (this.isRetryableError(error) && networkRetries < maxNetworkRetries) {
+ networkRetries++;
+ const backoffDelay = Math.min(1000 * Math.pow(2, networkRetries - 1), 10000);
+
+ await new Promise((resolve) => setTimeout(resolve, backoffDelay));
+ continue;
+ }
+
+ throw new FirecrawlError(error, 500);
}
- } catch (error: any) {
- throw new FirecrawlError(error, 500);
}
}
+ /**
+ * Determines if an error is retryable (transient network error)
+ * @param error - The error to check
+ * @returns True if the error should be retried
+ */
+ private isRetryableError(error: any): boolean {
+ if (error instanceof AxiosError) {
+ if (!error.response) {
+ const code = error.code;
+ const message = error.message?.toLowerCase() || '';
+
+ return (
+ code === 'ECONNRESET' ||
+ code === 'ETIMEDOUT' ||
+ code === 'ENOTFOUND' ||
+ code === 'ECONNREFUSED' ||
+ message.includes('socket hang up') ||
+ message.includes('network error') ||
+ message.includes('timeout')
+ );
+ }
+
+ if (error.response?.status === 408 || error.response?.status === 504) {
+ return true;
+ }
+ }
+
+ if (error && typeof error === 'object') {
+ const code = error.code;
+ const message = error.message?.toLowerCase() || '';
+
+ if (code === 'ECONNRESET' ||
+ code === 'ETIMEDOUT' ||
+ code === 'ENOTFOUND' ||
+ code === 'ECONNREFUSED' ||
+ message.includes('socket hang up') ||
+ message.includes('network error') ||
+ message.includes('timeout')) {
+ return true;
+ }
+
+ if (error.response?.status === 408 || error.response?.status === 504) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
/**
* Handles errors from API responses.
* @param {AxiosResponse} response - The response from the API.