Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(js-sdk): pin pagination next URLs to the api_url origin (#4869)
Gergő Móricz444f312880f0
diff --git a/.github/workflows/test-js-sdk.yml b/.github/workflows/test-js-sdk.ymlindex 34409e94f..0818b473d 100644--- a/.github/workflows/test-js-sdk.yml+++ b/.github/workflows/test-js-sdk.yml@@ -6,22 +6,51 @@ on:- mainpaths:- apps/js-sdk/firecrawl/**+ - .github/workflows/test-js-sdk.yml-env:- IDMUX_URL: ${{ secrets.IDMUX_URL }}+permissions:+ contents: readjobs:+ unit:+ name: Unit tests+ runs-on: blacksmith-2vcpu-ubuntu-2404+ steps:+ - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0+ with:+ persist-credentials: false+ - name: Install pnpm+ uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4+ with:+ version: 10+ - name: Set up Node.js+ uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0+ with:+ node-version: "22"+ cache: "pnpm"+ cache-dependency-path: './apps/js-sdk/firecrawl/pnpm-lock.yaml'+ - name: Install dependencies+ run: pnpm install --frozen-lockfile+ working-directory: ./apps/js-sdk/firecrawl+ - name: Run next URL pinning unit tests+ run: pnpm exec jest --verbose src/__tests__/unit/v1/next-url-pinning.test.ts src/__tests__/unit/v2/next-url-pinning.test.ts+ working-directory: ./apps/js-sdk/firecrawl+ env:+ NODE_OPTIONS: --experimental-vm-modules+test:name: Run testsruns-on: blacksmith-4vcpu-ubuntu-2404+ env:+ IDMUX_URL: ${{ secrets.IDMUX_URL }}steps:- - uses: actions/checkout@v5+ - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0- name: Install pnpmuses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4with:version: 10- name: Set up Node.js- uses: actions/setup-node@v4+ uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0with:node-version: "20"cache: "pnpm"diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex 2c0d9bcce..08b378786 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.42.0",+ "version": "4.42.1","description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v1/next-url-pinning.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v1/next-url-pinning.test.tsnew file mode 100644index 000000000..f5b5cc425--- /dev/null+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v1/next-url-pinning.test.ts@@ -0,0 +1,81 @@+import { describe, test, expect, jest, afterEach } from "@jest/globals";+import axios from "axios";+import FirecrawlApp from "../../../v1";++const API_URL = "https://api.firecrawl.dev";+const API_KEY = "fc-test";++const origins: Array<[string, string]> = [+ ["same origin", API_URL],+ ["cross host", "https://evil.example"],+ ["protocol-relative", "//evil.example"],+ ["different port", "https://api.firecrawl.dev:8443"],+ ["different scheme", "http://api.firecrawl.dev"],+ ["whitespace-prefixed", "\n https://evil.example"],+];++function mockGet(respond: (url: URL) => unknown) {+ const sent: Array<{ url: string; authorization: unknown }> = [];+ jest.spyOn(axios, "get").mockImplementation(async (url: string, config?: any) => {+ sent.push({ url, authorization: config?.headers?.Authorization });+ return { status: 200, data: respond(new URL(url)) };+ });+ return sent;+}++function expectPinned(sent: Array<{ url: string; authorization: unknown }>, expectedUrls: string[]) {+ expect(sent.map(req => req.url)).toEqual(expectedUrls);+ for (const req of sent) {+ expect(new URL(req.url).origin).toBe(API_URL);+ expect(req.url).not.toContain("evil.example");+ expect(req.authorization).toBe(`Bearer ${API_KEY}`);+ }+}++const completedPage = (next: string) => ({+ success: true,+ status: "completed",+ total: 2,+ completed: 2,+ expiresAt: "2030-01-01T00:00:00Z",+ data: [{ markdown: "a" }],+ next,+});+const lastPage = () => ({ success: true, status: "completed", data: [{ markdown: "b" }] });++describe("v1 pins next URLs to the api_url origin", () => {+ afterEach(() => {+ jest.restoreAllMocks();+ });++ describe.each(origins)("%s next", (_label, origin) => {+ const app = new FirecrawlApp({ apiKey: API_KEY, apiUrl: API_URL });++ test.each([+ ["checkCrawlStatus", "/v1/crawl/abc"],+ ["checkBatchScrapeStatus", "/v1/batch/scrape/abc"],+ ] as const)("%s getAllData follows server next", async (method, path) => {+ const sent = mockGet(url => (url.searchParams.has("skip") ? lastPage() : completedPage(`${origin}${path}?skip=10`)));+ const res: any = await app[method]("abc", true);+ expect(res.data).toHaveLength(2);+ expectPinned(sent, [`${API_URL}${path}`, `${API_URL}${path}?skip=10`]);+ });++ test.each([+ ["checkCrawlStatus", "/v1/crawl/abc"],+ ["checkBatchScrapeStatus", "/v1/batch/scrape/abc"],+ ] as const)("%s caller-supplied nextURL", async (method, path) => {+ const sent = mockGet(lastPage);+ await app[method]("abc", false, `${origin}${path}?skip=10`);+ expectPinned(sent, [`${API_URL}${path}?skip=10`]);+ });++ test("monitorJobStatus follows server next", async () => {+ const path = "/v1/crawl/abc";+ const sent = mockGet(url => (url.searchParams.has("skip") ? lastPage() : completedPage(`${origin}${path}?skip=10`)));+ const res: any = await app.monitorJobStatus("abc", app.prepareHeaders(), 0);+ expect(res.data).toHaveLength(2);+ expectPinned(sent, [`${API_URL}${path}`, `${API_URL}${path}?skip=10`]);+ });+ });+});diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/next-url-pinning.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/next-url-pinning.test.tsnew file mode 100644index 000000000..21366c05d--- /dev/null+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/next-url-pinning.test.ts@@ -0,0 +1,104 @@+import { describe, test, expect } from "@jest/globals";+import axios, { type AxiosAdapter } from "axios";+import { FirecrawlClient } from "../../../v2/client";+import { pinToApiOrigin } from "../../../utils/apiOrigin";++const API_URL = "https://api.firecrawl.dev";+const API_KEY = "fc-test";++const origins: Array<[string, string]> = [+ ["same origin", API_URL],+ ["cross host", "https://evil.example"],+ ["protocol-relative", "//evil.example"],+ ["different port", "https://api.firecrawl.dev:8443"],+ ["different scheme", "http://api.firecrawl.dev"],+ ["whitespace-prefixed", "\n https://evil.example"],+];++function makeClient(firstPage: (next: string) => unknown, lastPage: unknown, next: string) {+ const client = new FirecrawlClient({ apiKey: API_KEY, apiUrl: API_URL });+ const sent: Array<{ url: string; authorization: unknown }> = [];+ const adapter: AxiosAdapter = async config => {+ const url = axios.getUri(config);+ sent.push({ url, authorization: config.headers.Authorization });+ const data = new URL(url).searchParams.has("skip") ? lastPage : firstPage(next);+ return { data, status: 200, statusText: "OK", headers: {}, config };+ };+ (client as any).http.instance.defaults.adapter = adapter;+ return { client, sent };+}++function expectPinned(sent: Array<{ url: string; authorization: unknown }>, path: string) {+ expect(sent).toHaveLength(2);+ expect(sent[1]!.url).toBe(`${API_URL}${path}?skip=10`);+ for (const req of sent) {+ expect(new URL(req.url).origin).toBe(API_URL);+ expect(req.url).not.toContain("evil.example");+ expect(req.authorization).toBe(`Bearer ${API_KEY}`);+ }+}++describe("v2 pagination pins next URLs to the api_url origin", () => {+ describe.each(origins)("%s next", (_label, origin) => {+ test("crawl status", async () => {+ const path = "/v2/crawl/abc";+ const { client, sent } = makeClient(+ next => ({ success: true, status: "completed", completed: 2, total: 2, next, data: [{ markdown: "a" }] }),+ { success: true, next: null, data: [{ markdown: "b" }] },+ `${origin}${path}?skip=10`,+ );+ const job = await client.getCrawlStatus("abc");+ expect(job.data).toHaveLength(2);+ expectPinned(sent, path);+ });++ test("batch scrape status", async () => {+ const path = "/v2/batch/scrape/abc";+ const { client, sent } = makeClient(+ next => ({ success: true, status: "completed", completed: 2, total: 2, next, data: [{ markdown: "a" }] }),+ { success: true, next: null, data: [{ markdown: "b" }] },+ `${origin}${path}?skip=10`,+ );+ const job = await client.getBatchScrapeStatus("abc");+ expect(job.data).toHaveLength(2);+ expectPinned(sent, path);+ });++ test("monitor check", async () => {+ const path = "/v2/monitor/m1/checks/c1";+ const { client, sent } = makeClient(+ next => ({ success: true, data: { id: "c1", pages: [{ url: "https://a.example" }], next } }),+ { success: true, data: { pages: [{ url: "https://b.example" }], next: null } },+ `${origin}${path}?skip=10`,+ );+ const check = await client.getMonitorCheck("m1", "c1");+ expect(check.pages).toHaveLength(2);+ expectPinned(sent, path);+ });+ });+});++describe("pinToApiOrigin", () => {+ test("rewrites absolute URLs onto the api origin, keeping path and query and dropping the fragment", () => {+ expect(pinToApiOrigin(API_URL, "https://evil.example/v2/crawl/abc?skip=10#frag")).toBe(+ `${API_URL}/v2/crawl/abc?skip=10`,+ );+ expect(pinToApiOrigin("http://localhost:3002", "https://evil.example:444/v2/crawl/abc?skip=10")).toBe(+ "http://localhost:3002/v2/crawl/abc?skip=10",+ );+ expect(pinToApiOrigin(API_URL, "\\\\evil.example/v2/crawl/abc")).toBe(`${API_URL}/v2/crawl/abc`);+ for (const url of ["\nhttps://evil.example/v2/crawl/abc", "h\tttps://evil.example/v2/crawl/abc", "\u0000 //evil.example/v2/crawl/abc", "/\\evil.example/v2/crawl/abc"]) {+ expect(pinToApiOrigin(API_URL, url)).toBe(`${API_URL}/v2/crawl/abc`);+ }+ });++ test("leaves relative URLs unchanged", () => {+ expect(pinToApiOrigin(API_URL, "/v2/crawl/abc?skip=10")).toBe("/v2/crawl/abc?skip=10");+ });++ test("throws when apiUrl is not absolute and the URL needs pinning", () => {+ expect(() => pinToApiOrigin("api.firecrawl.dev", "https://evil.example/v2/crawl/abc")).toThrow(+ "apiUrl must be an absolute URL",+ );+ });+});diff --git a/apps/js-sdk/firecrawl/src/utils/apiOrigin.ts b/apps/js-sdk/firecrawl/src/utils/apiOrigin.tsnew file mode 100644index 000000000..995d58fa0--- /dev/null+++ b/apps/js-sdk/firecrawl/src/utils/apiOrigin.ts@@ -0,0 +1,19 @@+/**+ * Rewrite an absolute or protocol-relative URL onto apiUrl's scheme, host and+ * port (keeping path and query, dropping any fragment) so credentials never+ * leave the configured API origin. Relative URLs are returned unchanged.+ * Throws if apiUrl is not an absolute URL.+ */+export function pinToApiOrigin(apiUrl: string, url: string): string {+ const normalized = url.replace(/[\t\n\r]/g, "").replace(/^[\u0000-\u0020]+/, "");+ if (!/^([a-z][a-z\d+\-.]*:|[\\/]{2})/i.test(normalized)) return url;+ const base = URL.canParse(apiUrl) ? new URL(apiUrl) : null;+ if (!base?.host) {+ throw new Error(`apiUrl must be an absolute URL, got ${JSON.stringify(apiUrl)}`);+ }+ const target = new URL(normalized, base);+ const pinned = new URL(base.origin);+ pinned.pathname = target.pathname;+ pinned.search = target.search;+ return pinned.href;+}diff --git a/apps/js-sdk/firecrawl/src/v1/index.ts b/apps/js-sdk/firecrawl/src/v1/index.tsindex 8b0564bf1..7a0f7bdd4 100644--- a/apps/js-sdk/firecrawl/src/v1/index.ts+++ b/apps/js-sdk/firecrawl/src/v1/index.ts@@ -3,6 +3,7 @@ import * as zt from "zod";import { zodSchemaToJsonSchema } from "../utils/zodSchemaToJson";import { TypedEventTarget } from "typescript-event-target";import type { RequiresAction } from "../v2/types";+import { pinToApiOrigin } from "../utils/apiOrigin";/*** Configuration interface for FirecrawlApp.@@ -922,7 +923,7 @@ export default class FirecrawlApp {}const headers: AxiosRequestHeaders = this.prepareHeaders();- const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/crawl/${id}`);+ const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/crawl/${id}`, this.apiUrl);if (skip !== undefined) {targetURL.searchParams.set("skip", skip.toString());}@@ -1210,7 +1211,7 @@ export default class FirecrawlApp {}const headers: AxiosRequestHeaders = this.prepareHeaders();- const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/batch/scrape/${id}`);+ const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/batch/scrape/${id}`, this.apiUrl);if (skip !== undefined) {targetURL.searchParams.set("skip", skip.toString());}@@ -1451,7 +1452,7 @@ export default class FirecrawlApp {headers: AxiosRequestHeaders): Promise<AxiosResponse> {try {- return await axios.get(url, { headers });+ return await axios.get(pinToApiOrigin(this.apiUrl, url), { headers });} catch (error) {if (error instanceof AxiosError && error.response) {return error.response as AxiosResponse;diff --git a/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts b/apps/js-sdk/firecrawl/src/v2/utils/httpClient.tsindex 64bb2e601..b59c047a8 100644--- a/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts+++ b/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts@@ -4,6 +4,7 @@ import axios, {type AxiosResponse,} from "axios";import { getVersion } from "./getVersion";+import { pinToApiOrigin } from "../../utils/apiOrigin";export interface HttpClientOptions {apiKey: string;@@ -64,6 +65,7 @@ export class HttpClient {config.headers = {...(config.headers || {}),};+ if (config.url) config.url = pinToApiOrigin(this.apiUrl, config.url);let lastError: any;for (let attempt = 0; attempt < this.maxRetries; attempt++) {