Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(js-sdk): add retry logic for socket hang up errors in monitorJobStatus (ENG-3029) (#1893)
devin-ai-integration[bot]049d2e6bcc5e
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex 96ed80ae1..1aa0a0420 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "1.29.2",+ "version": "1.29.3","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",@@ -16,7 +16,8 @@"build": "tsup","build-and-publish": "npm run build && npm publish --access public","publish-beta": "npm run build && npm publish --access public --tag beta",- "test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts"+ "test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts",+ "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/unit/*.test.ts"},"repository": {"type": "git",diff --git a/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts b/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.tsindex ffe3ac747..a0a2d345a 100644--- a/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts@@ -297,4 +297,5 @@ describe('FirecrawlApp E2E Tests', () => {const app = new FirecrawlApp({ apiUrl: API_URL, apiKey: "invalid_api_key" });await expect(app.search("test query")).rejects.toThrow("Request failed with status code 401");});+});diff --git a/apps/js-sdk/firecrawl/src/__tests__/v1/unit/monitor-job-status-retry.test.ts b/apps/js-sdk/firecrawl/src/__tests__/v1/unit/monitor-job-status-retry.test.tsnew file mode 100644index 000000000..75563aa3b--- /dev/null+++ b/apps/js-sdk/firecrawl/src/__tests__/v1/unit/monitor-job-status-retry.test.ts@@ -0,0 +1,154 @@+import FirecrawlApp from '../../../index';+import { describe, test, expect, jest, beforeEach, afterEach } from '@jest/globals';++describe('monitorJobStatus retry logic', () => {+ let app: FirecrawlApp;+ let originalConsoleWarn: typeof console.warn;++ beforeEach(() => {+ app = new FirecrawlApp({ apiKey: 'test-key', apiUrl: 'https://test.com' });+ originalConsoleWarn = console.warn;+ console.warn = jest.fn();+ });++ afterEach(() => {+ console.warn = originalConsoleWarn;+ jest.clearAllMocks();+ });++ test('should retry on socket hang up error', async () => {+ const socketHangUpError = new Error('socket hang up') as any;+ socketHangUpError.code = 'ECONNRESET';++ const successResponse = {+ status: 200,+ data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }+ };++ const originalGetRequest = app.getRequest;+ let callCount = 0;++ app.getRequest = async function(url: string, headers: any) {+ callCount++;+ if (callCount === 1) {+ throw socketHangUpError;+ }+ return successResponse;+ };++ const result = await app.monitorJobStatus('test-id', {}, 1);++ expect(callCount).toBe(2);+ expect(result).toEqual(successResponse.data);+ expect(console.warn).toHaveBeenCalledWith(+ expect.stringContaining('Network error during job status check (attempt 1/3): socket hang up')+ );+ });++ test('should retry on ETIMEDOUT error', async () => {+ const timeoutError = new Error('timeout') as any;+ timeoutError.code = 'ETIMEDOUT';++ const successResponse = {+ status: 200,+ data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }+ };++ const originalGetRequest = app.getRequest;+ let callCount = 0;++ app.getRequest = async function(url: string, headers: any) {+ callCount++;+ if (callCount === 1) {+ throw timeoutError;+ }+ return successResponse;+ };++ const result = await app.monitorJobStatus('test-id', {}, 1);++ expect(callCount).toBe(2);+ expect(result).toEqual(successResponse.data);+ });++ test('should fail after max retries exceeded', async () => {+ const socketHangUpError = new Error('socket hang up') as any;+ socketHangUpError.code = 'ECONNRESET';++ app.getRequest = async function(url: string, headers: any) {+ throw socketHangUpError;+ };++ await expect(app.monitorJobStatus('test-id', {}, 1)).rejects.toThrow('socket hang up');++ expect(console.warn).toHaveBeenCalledTimes(3);+ }, 15000);++ test('should not retry on non-retryable errors', async () => {+ const authError = new Error('Unauthorized') as any;+ authError.response = { status: 401, data: { error: 'Unauthorized' } };++ app.getRequest = async function(url: string, headers: any) {+ throw authError;+ };++ await expect(app.monitorJobStatus('test-id', {}, 1)).rejects.toThrow('Unauthorized');++ expect(console.warn).not.toHaveBeenCalled();+ });++ test('should retry on HTTP timeout status codes', async () => {+ const timeoutError = new Error('Request timeout') as any;+ timeoutError.response = { status: 408, data: { error: 'Request timeout' } };++ const successResponse = {+ status: 200,+ data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }+ };++ const originalGetRequest = app.getRequest;+ let callCount = 0;++ app.getRequest = async function(url: string, headers: any) {+ callCount++;+ if (callCount === 1) {+ throw timeoutError;+ }+ return successResponse;+ };++ const result = await app.monitorJobStatus('test-id', {}, 1);++ expect(callCount).toBe(2);+ expect(result).toEqual(successResponse.data);+ });++ test('should use exponential backoff for retries', async () => {+ const socketHangUpError = new Error('socket hang up') as any;+ socketHangUpError.code = 'ECONNRESET';++ const successResponse = {+ status: 200,+ data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }+ };++ const originalGetRequest = app.getRequest;+ let callCount = 0;++ app.getRequest = async function(url: string, headers: any) {+ callCount++;+ if (callCount <= 2) {+ throw socketHangUpError;+ }+ return successResponse;+ };++ const startTime = Date.now();+ const result = await app.monitorJobStatus('test-id', {}, 1);+ const endTime = Date.now();++ expect(callCount).toBe(3);+ expect(result).toEqual(successResponse.data);+ expect(endTime - startTime).toBeGreaterThan(3000);+ });+});diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.tsindex 139f7a919..f93bb48e3 100644--- a/apps/js-sdk/firecrawl/src/index.ts+++ b/apps/js-sdk/firecrawl/src/index.ts@@ -1455,33 +1455,39 @@ export default class FirecrawlApp {headers: AxiosRequestHeaders,checkInterval: number): Promise<CrawlStatusResponse | ErrorResponse> {- try {- let failedTries = 0;- while (true) {+ let failedTries = 0;+ let networkRetries = 0;+ const maxNetworkRetries = 3;++ while (true) {+ try {let statusResponse: AxiosResponse = await this.getRequest(`${this.apiUrl}/v1/crawl/${id}`,headers);+if (statusResponse.status === 200) {failedTries = 0;+ networkRetries = 0;let statusData = statusResponse.data;- if (statusData.status === "completed") {- if ("data" in statusData) {- let data = statusData.data;- while (typeof statusData === 'object' && 'next' in statusData) {- if (data.length === 0) {- break- }- statusResponse = await this.getRequest(statusData.next, headers);- statusData = statusResponse.data;- data = data.concat(statusData.data);++ if (statusData.status === "completed") {+ if ("data" in statusData) {+ let data = statusData.data;+ while (typeof statusData === 'object' && 'next' in statusData) {+ if (data.length === 0) {+ break}- statusData.data = data;- return statusData;- } else {- throw new FirecrawlError("Crawl job completed but no data was returned", 500);+ statusResponse = await this.getRequest(statusData.next, headers);+ statusData = statusResponse.data;+ data = data.concat(statusData.data);}- } else if (+ statusData.data = data;+ return statusData;+ } else {+ throw new FirecrawlError("Crawl job completed but no data was returned", 500);+ }+ } else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {checkInterval = Math.max(checkInterval, 2);@@ -1500,12 +1506,69 @@ export default class FirecrawlApp {this.handleError(statusResponse, "check crawl status");}}+ } catch (error: any) {+ if (this.isRetryableError(error) && networkRetries < maxNetworkRetries) {+ networkRetries++;+ const backoffDelay = Math.min(1000 * Math.pow(2, networkRetries - 1), 10000);++ await new Promise((resolve) => setTimeout(resolve, backoffDelay));+ continue;+ }++ throw new FirecrawlError(error, 500);}- } catch (error: any) {- throw new FirecrawlError(error, 500);}}+ /**+ * Determines if an error is retryable (transient network error)+ * @param error - The error to check+ * @returns True if the error should be retried+ */+ private isRetryableError(error: any): boolean {+ if (error instanceof AxiosError) {+ if (!error.response) {+ const code = error.code;+ const message = error.message?.toLowerCase() || '';++ return (+ code === 'ECONNRESET' ||+ code === 'ETIMEDOUT' ||+ code === 'ENOTFOUND' ||+ code === 'ECONNREFUSED' ||+ message.includes('socket hang up') ||+ message.includes('network error') ||+ message.includes('timeout')+ );+ }++ if (error.response?.status === 408 || error.response?.status === 504) {+ return true;+ }+ }++ if (error && typeof error === 'object') {+ const code = error.code;+ const message = error.message?.toLowerCase() || '';++ if (code === 'ECONNRESET' ||+ code === 'ETIMEDOUT' ||+ code === 'ENOTFOUND' ||+ code === 'ECONNREFUSED' ||+ message.includes('socket hang up') ||+ message.includes('network error') ||+ message.includes('timeout')) {+ return true;+ }++ if (error.response?.status === 408 || error.response?.status === 504) {+ return true;+ }+ }++ return false;+ }+/*** Handles errors from API responses.* @param {AxiosResponse} response - The response from the API.