Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
js-sdk: fix stale unit tests and run the full unit suite in CI (#4917)
Gergő Móriczcdc1e47f24dd
diff --git a/.github/workflows/test-js-sdk.yml b/.github/workflows/test-js-sdk.ymlindex 11a09e25c..9227a7ed5 100644--- a/.github/workflows/test-js-sdk.yml+++ b/.github/workflows/test-js-sdk.yml@@ -32,11 +32,9 @@ jobs:- name: Install dependenciesrun: pnpm install --frozen-lockfileworking-directory: ./apps/js-sdk/firecrawl- - name: Run selected unit tests- run: pnpm exec jest --verbose src/__tests__/unit/v1/next-url-pinning.test.ts src/__tests__/unit/v2/next-url-pinning.test.ts src/__tests__/unit/v2/parse-formats.unit.test.ts+ - name: Run unit tests+ run: pnpm run test:unitworking-directory: ./apps/js-sdk/firecrawl- env:- NODE_OPTIONS: --experimental-vm-modulestest:name: Run testsdiff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex e577eb2d4..abfea8474 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -17,7 +17,7 @@"build-and-publish": "pnpm run build && pnpm publish --access public","publish-beta": "pnpm run build && pnpm publish --access public --tag beta","test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/e2e/v2/*.test.ts --detectOpenHandles",- "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit/v2/*.test.ts"+ "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit"},"repository": {"type": "git",diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v1/monitor-job-status-retry.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v1/monitor-job-status-retry.test.tsindex 75563aa3b..0132e87dd 100644--- a/apps/js-sdk/firecrawl/src/__tests__/unit/v1/monitor-job-status-retry.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v1/monitor-job-status-retry.test.ts@@ -1,154 +1,107 @@-import FirecrawlApp from '../../../index';+import FirecrawlApp from '../../../v1';import { describe, test, expect, jest, beforeEach, afterEach } from '@jest/globals';+const successResponse = {+ status: 200,+ data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] },+};++function networkError(message: string, code: string) {+ return Object.assign(new Error(message), { code });+}++function httpError(message: string, status: number) {+ return Object.assign(new Error(message), { response: { status, data: { error: message } } });+}+describe('monitorJobStatus retry logic', () => {let app: FirecrawlApp;- let originalConsoleWarn: typeof console.warn;-+ let delays: number[];+beforeEach(() => {app = new FirecrawlApp({ apiKey: 'test-key', apiUrl: 'https://test.com' });- originalConsoleWarn = console.warn;- console.warn = jest.fn();+ delays = [];+ jest.useFakeTimers();+ const fakeSetTimeout = globalThis.setTimeout;+ jest.spyOn(globalThis, 'setTimeout').mockImplementation(((fn: () => void, ms?: number) => {+ delays.push(ms ?? 0);+ return fakeSetTimeout(fn, ms);+ }) as any);});afterEach(() => {- console.warn = originalConsoleWarn;- jest.clearAllMocks();+ jest.restoreAllMocks();+ jest.useRealTimers();});- test('should retry on socket hang up error', async () => {- const socketHangUpError = new Error('socket hang up') as any;- socketHangUpError.code = 'ECONNRESET';-- const successResponse = {- status: 200,- data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }- };-- const originalGetRequest = app.getRequest;- let callCount = 0;-- app.getRequest = async function(url: string, headers: any) {- callCount++;- if (callCount === 1) {- throw socketHangUpError;- }- return successResponse;- };-- const result = await app.monitorJobStatus('test-id', {}, 1);-- expect(callCount).toBe(2);+ function failFirst(n: number, error: Error, statuses: string[] = []) {+ let calls = 0;+ app.getRequest = (async () => {+ calls++;+ if (calls <= n) throw error;+ const status = statuses.shift();+ return status ? { status: 200, data: { status } } : successResponse;+ }) as any;+ return () => calls;+ }++ async function monitor() {+ const result = app.monitorJobStatus('test-id', {} as any, 1);+ result.catch(() => {});+ await jest.runAllTimersAsync();+ return result;+ }++ test.each([+ ['socket hang up', networkError('socket hang up', 'ECONNRESET')],+ ['ETIMEDOUT', networkError('timeout', 'ETIMEDOUT')],+ ['HTTP 408', httpError('Request timeout', 408)],+ ['HTTP 504', httpError('Gateway timeout', 504)],+ ])('retries once after a %s error and returns the job data', async (_label, error) => {+ const calls = failFirst(1, error);++ const result = await monitor();++ expect(calls()).toBe(2);expect(result).toEqual(successResponse.data);- expect(console.warn).toHaveBeenCalledWith(- expect.stringContaining('Network error during job status check (attempt 1/3): socket hang up')- );+ expect(delays).toEqual([1000]);});- test('should retry on ETIMEDOUT error', async () => {- const timeoutError = new Error('timeout') as any;- timeoutError.code = 'ETIMEDOUT';-- const successResponse = {- status: 200,- data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }- };-- const originalGetRequest = app.getRequest;- let callCount = 0;-- app.getRequest = async function(url: string, headers: any) {- callCount++;- if (callCount === 1) {- throw timeoutError;- }- return successResponse;- };-- const result = await app.monitorJobStatus('test-id', {}, 1);-- expect(callCount).toBe(2);+ test('uses exponential backoff between retries', async () => {+ const calls = failFirst(2, networkError('socket hang up', 'ECONNRESET'));++ const result = await monitor();++ expect(calls()).toBe(3);expect(result).toEqual(successResponse.data);+ expect(delays).toEqual([1000, 2000]);});- test('should fail after max retries exceeded', async () => {- const socketHangUpError = new Error('socket hang up') as any;- socketHangUpError.code = 'ECONNRESET';-- app.getRequest = async function(url: string, headers: any) {- throw socketHangUpError;- };-- await expect(app.monitorJobStatus('test-id', {}, 1)).rejects.toThrow('socket hang up');-- expect(console.warn).toHaveBeenCalledTimes(3);- }, 15000);-- test('should not retry on non-retryable errors', async () => {- const authError = new Error('Unauthorized') as any;- authError.response = { status: 401, data: { error: 'Unauthorized' } };-- app.getRequest = async function(url: string, headers: any) {- throw authError;- };-- await expect(app.monitorJobStatus('test-id', {}, 1)).rejects.toThrow('Unauthorized');-- expect(console.warn).not.toHaveBeenCalled();- });+ test('records both the retry backoff and the 2s poll wait for an active job', async () => {+ const calls = failFirst(1, networkError('socket hang up', 'ECONNRESET'), ['active']);++ const result = await monitor();- test('should retry on HTTP timeout status codes', async () => {- const timeoutError = new Error('Request timeout') as any;- timeoutError.response = { status: 408, data: { error: 'Request timeout' } };-- const successResponse = {- status: 200,- data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }- };-- const originalGetRequest = app.getRequest;- let callCount = 0;-- app.getRequest = async function(url: string, headers: any) {- callCount++;- if (callCount === 1) {- throw timeoutError;- }- return successResponse;- };-- const result = await app.monitorJobStatus('test-id', {}, 1);-- expect(callCount).toBe(2);+ expect(calls()).toBe(3);expect(result).toEqual(successResponse.data);+ expect(delays).toEqual([1000, 2000]);});- test('should use exponential backoff for retries', async () => {- const socketHangUpError = new Error('socket hang up') as any;- socketHangUpError.code = 'ECONNRESET';-- const successResponse = {- status: 200,- data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }- };-- const originalGetRequest = app.getRequest;- let callCount = 0;-- app.getRequest = async function(url: string, headers: any) {- callCount++;- if (callCount <= 2) {- throw socketHangUpError;- }- return successResponse;- };-- const startTime = Date.now();- const result = await app.monitorJobStatus('test-id', {}, 1);- const endTime = Date.now();-- expect(callCount).toBe(3);- expect(result).toEqual(successResponse.data);- expect(endTime - startTime).toBeGreaterThan(3000);+ test('fails after max retries are exceeded', async () => {+ const calls = failFirst(Infinity, networkError('socket hang up', 'ECONNRESET'));++ await expect(monitor()).rejects.toThrow('socket hang up');++ expect(calls()).toBe(4);+ expect(delays).toEqual([1000, 2000, 4000]);+ });++ test('does not retry non-retryable errors', async () => {+ const calls = failFirst(Infinity, httpError('Unauthorized', 401));++ await expect(monitor()).rejects.toThrow('Unauthorized');++ expect(calls()).toBe(1);+ expect(delays).toEqual([]);});});diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/scrape.unit.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/scrape.unit.test.tsindex 1784cdc85..859660f0c 100644--- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/scrape.unit.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/scrape.unit.test.ts@@ -1,11 +1,60 @@-/**- * Minimal unit test for v2 scrape (no mocking; sanity check payload path)- */+import { describe, test, expect } from "@jest/globals";+import axios, { type AxiosAdapter } from "axios";import { FirecrawlClient } from "../../../v2/client";+const API_URL = "https://api.firecrawl.dev";++function makeClient(apiKey: string) {+ const client = new FirecrawlClient({ apiKey, apiUrl: API_URL });+ const sent: Array<{ method?: string; url: string; authorization: unknown; body: unknown }> = [];+ const adapter: AxiosAdapter = async config => {+ sent.push({+ method: config.method,+ url: axios.getUri(config),+ authorization: config.headers.Authorization,+ body: JSON.parse(config.data),+ });+ return {+ data: { success: true, data: { markdown: "# hello" } },+ status: 200,+ statusText: "OK",+ headers: {},+ config,+ };+ };+ (client as any).http.instance.defaults.adapter = adapter;+ return { client, sent };+}+describe("v2.scrape unit", () => {- test("constructor requires apiKey", () => {- expect(() => new FirecrawlClient({ apiKey: "", apiUrl: "https://api.firecrawl.dev" })).toThrow();+ test("constructs without an API key and scrapes keyless, without an Authorization header", async () => {+ const { client, sent } = makeClient("");+ const doc = await client.scrape(" https://example.com ", { formats: ["markdown"] });++ expect(doc.markdown).toBe("# hello");+ expect(sent).toHaveLength(1);+ expect(sent[0]!.method).toBe("post");+ expect(sent[0]!.url).toBe(`${API_URL}/v2/scrape`);+ expect(sent[0]!.authorization).toBeUndefined();+ expect(sent[0]!.body).toEqual({+ url: "https://example.com",+ formats: ["markdown"],+ origin: expect.stringMatching(/^js-sdk@/),+ });});-});+ test("sends the API key as a bearer token when one is set", async () => {+ const { client, sent } = makeClient("fc-test");+ await client.scrape("https://example.com");++ expect(sent).toHaveLength(1);+ expect(sent[0]!.authorization).toBe("Bearer fc-test");+ });++ test("rejects an empty URL before sending a request", async () => {+ const { client, sent } = makeClient("fc-test");++ await expect(client.scrape(" ")).rejects.toThrow("URL cannot be empty");+ expect(sent).toHaveLength(0);+ });+});