Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(js-sdk): pin pagination next URLs to the api_url origin (#4869)

Gergő Móricz444f312880f0
diff --git a/.github/workflows/test-js-sdk.yml b/.github/workflows/test-js-sdk.yml
index 34409e94f..0818b473d 100644
--- a/.github/workflows/test-js-sdk.yml
+++ b/.github/workflows/test-js-sdk.yml
@@ -6,22 +6,51 @@ on:
- main
paths:
- apps/js-sdk/firecrawl/**
+ - .github/workflows/test-js-sdk.yml
-env:
- IDMUX_URL: ${{ secrets.IDMUX_URL }}
+permissions:
+ contents: read
jobs:
+ unit:
+ name: Unit tests
+ runs-on: blacksmith-2vcpu-ubuntu-2404
+ steps:
+ - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0
+ with:
+ persist-credentials: false
+ - name: Install pnpm
+ uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4
+ with:
+ version: 10
+ - name: Set up Node.js
+ uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+ with:
+ node-version: "22"
+ cache: "pnpm"
+ cache-dependency-path: './apps/js-sdk/firecrawl/pnpm-lock.yaml'
+ - name: Install dependencies
+ run: pnpm install --frozen-lockfile
+ working-directory: ./apps/js-sdk/firecrawl
+ - name: Run next URL pinning unit tests
+ run: pnpm exec jest --verbose src/__tests__/unit/v1/next-url-pinning.test.ts src/__tests__/unit/v2/next-url-pinning.test.ts
+ working-directory: ./apps/js-sdk/firecrawl
+ env:
+ NODE_OPTIONS: --experimental-vm-modules
+
test:
name: Run tests
runs-on: blacksmith-4vcpu-ubuntu-2404
+ env:
+ IDMUX_URL: ${{ secrets.IDMUX_URL }}
steps:
- - uses: actions/checkout@v5
+ - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0
- name: Install pnpm
uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4
with:
version: 10
- name: Set up Node.js
- uses: actions/setup-node@v4
+ uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version: "20"
cache: "pnpm"
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index 2c0d9bcce..08b378786 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.42.0",
+ "version": "4.42.1",
"description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
"main": "dist/index.js",
"types": "dist/index.d.ts",
diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v1/next-url-pinning.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v1/next-url-pinning.test.ts
new file mode 100644
index 000000000..f5b5cc425
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v1/next-url-pinning.test.ts
@@ -0,0 +1,81 @@
+import { describe, test, expect, jest, afterEach } from "@jest/globals";
+import axios from "axios";
+import FirecrawlApp from "../../../v1";
+
+const API_URL = "https://api.firecrawl.dev";
+const API_KEY = "fc-test";
+
+const origins: Array<[string, string]> = [
+ ["same origin", API_URL],
+ ["cross host", "https://evil.example"],
+ ["protocol-relative", "//evil.example"],
+ ["different port", "https://api.firecrawl.dev:8443"],
+ ["different scheme", "http://api.firecrawl.dev"],
+ ["whitespace-prefixed", "\n https://evil.example"],
+];
+
+function mockGet(respond: (url: URL) => unknown) {
+ const sent: Array<{ url: string; authorization: unknown }> = [];
+ jest.spyOn(axios, "get").mockImplementation(async (url: string, config?: any) => {
+ sent.push({ url, authorization: config?.headers?.Authorization });
+ return { status: 200, data: respond(new URL(url)) };
+ });
+ return sent;
+}
+
+function expectPinned(sent: Array<{ url: string; authorization: unknown }>, expectedUrls: string[]) {
+ expect(sent.map(req => req.url)).toEqual(expectedUrls);
+ for (const req of sent) {
+ expect(new URL(req.url).origin).toBe(API_URL);
+ expect(req.url).not.toContain("evil.example");
+ expect(req.authorization).toBe(`Bearer ${API_KEY}`);
+ }
+}
+
+const completedPage = (next: string) => ({
+ success: true,
+ status: "completed",
+ total: 2,
+ completed: 2,
+ expiresAt: "2030-01-01T00:00:00Z",
+ data: [{ markdown: "a" }],
+ next,
+});
+const lastPage = () => ({ success: true, status: "completed", data: [{ markdown: "b" }] });
+
+describe("v1 pins next URLs to the api_url origin", () => {
+ afterEach(() => {
+ jest.restoreAllMocks();
+ });
+
+ describe.each(origins)("%s next", (_label, origin) => {
+ const app = new FirecrawlApp({ apiKey: API_KEY, apiUrl: API_URL });
+
+ test.each([
+ ["checkCrawlStatus", "/v1/crawl/abc"],
+ ["checkBatchScrapeStatus", "/v1/batch/scrape/abc"],
+ ] as const)("%s getAllData follows server next", async (method, path) => {
+ const sent = mockGet(url => (url.searchParams.has("skip") ? lastPage() : completedPage(`${origin}${path}?skip=10`)));
+ const res: any = await app[method]("abc", true);
+ expect(res.data).toHaveLength(2);
+ expectPinned(sent, [`${API_URL}${path}`, `${API_URL}${path}?skip=10`]);
+ });
+
+ test.each([
+ ["checkCrawlStatus", "/v1/crawl/abc"],
+ ["checkBatchScrapeStatus", "/v1/batch/scrape/abc"],
+ ] as const)("%s caller-supplied nextURL", async (method, path) => {
+ const sent = mockGet(lastPage);
+ await app[method]("abc", false, `${origin}${path}?skip=10`);
+ expectPinned(sent, [`${API_URL}${path}?skip=10`]);
+ });
+
+ test("monitorJobStatus follows server next", async () => {
+ const path = "/v1/crawl/abc";
+ const sent = mockGet(url => (url.searchParams.has("skip") ? lastPage() : completedPage(`${origin}${path}?skip=10`)));
+ const res: any = await app.monitorJobStatus("abc", app.prepareHeaders(), 0);
+ expect(res.data).toHaveLength(2);
+ expectPinned(sent, [`${API_URL}${path}`, `${API_URL}${path}?skip=10`]);
+ });
+ });
+});
diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/next-url-pinning.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/next-url-pinning.test.ts
new file mode 100644
index 000000000..21366c05d
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/next-url-pinning.test.ts
@@ -0,0 +1,104 @@
+import { describe, test, expect } from "@jest/globals";
+import axios, { type AxiosAdapter } from "axios";
+import { FirecrawlClient } from "../../../v2/client";
+import { pinToApiOrigin } from "../../../utils/apiOrigin";
+
+const API_URL = "https://api.firecrawl.dev";
+const API_KEY = "fc-test";
+
+const origins: Array<[string, string]> = [
+ ["same origin", API_URL],
+ ["cross host", "https://evil.example"],
+ ["protocol-relative", "//evil.example"],
+ ["different port", "https://api.firecrawl.dev:8443"],
+ ["different scheme", "http://api.firecrawl.dev"],
+ ["whitespace-prefixed", "\n https://evil.example"],
+];
+
+function makeClient(firstPage: (next: string) => unknown, lastPage: unknown, next: string) {
+ const client = new FirecrawlClient({ apiKey: API_KEY, apiUrl: API_URL });
+ const sent: Array<{ url: string; authorization: unknown }> = [];
+ const adapter: AxiosAdapter = async config => {
+ const url = axios.getUri(config);
+ sent.push({ url, authorization: config.headers.Authorization });
+ const data = new URL(url).searchParams.has("skip") ? lastPage : firstPage(next);
+ return { data, status: 200, statusText: "OK", headers: {}, config };
+ };
+ (client as any).http.instance.defaults.adapter = adapter;
+ return { client, sent };
+}
+
+function expectPinned(sent: Array<{ url: string; authorization: unknown }>, path: string) {
+ expect(sent).toHaveLength(2);
+ expect(sent[1]!.url).toBe(`${API_URL}${path}?skip=10`);
+ for (const req of sent) {
+ expect(new URL(req.url).origin).toBe(API_URL);
+ expect(req.url).not.toContain("evil.example");
+ expect(req.authorization).toBe(`Bearer ${API_KEY}`);
+ }
+}
+
+describe("v2 pagination pins next URLs to the api_url origin", () => {
+ describe.each(origins)("%s next", (_label, origin) => {
+ test("crawl status", async () => {
+ const path = "/v2/crawl/abc";
+ const { client, sent } = makeClient(
+ next => ({ success: true, status: "completed", completed: 2, total: 2, next, data: [{ markdown: "a" }] }),
+ { success: true, next: null, data: [{ markdown: "b" }] },
+ `${origin}${path}?skip=10`,
+ );
+ const job = await client.getCrawlStatus("abc");
+ expect(job.data).toHaveLength(2);
+ expectPinned(sent, path);
+ });
+
+ test("batch scrape status", async () => {
+ const path = "/v2/batch/scrape/abc";
+ const { client, sent } = makeClient(
+ next => ({ success: true, status: "completed", completed: 2, total: 2, next, data: [{ markdown: "a" }] }),
+ { success: true, next: null, data: [{ markdown: "b" }] },
+ `${origin}${path}?skip=10`,
+ );
+ const job = await client.getBatchScrapeStatus("abc");
+ expect(job.data).toHaveLength(2);
+ expectPinned(sent, path);
+ });
+
+ test("monitor check", async () => {
+ const path = "/v2/monitor/m1/checks/c1";
+ const { client, sent } = makeClient(
+ next => ({ success: true, data: { id: "c1", pages: [{ url: "https://a.example" }], next } }),
+ { success: true, data: { pages: [{ url: "https://b.example" }], next: null } },
+ `${origin}${path}?skip=10`,
+ );
+ const check = await client.getMonitorCheck("m1", "c1");
+ expect(check.pages).toHaveLength(2);
+ expectPinned(sent, path);
+ });
+ });
+});
+
+describe("pinToApiOrigin", () => {
+ test("rewrites absolute URLs onto the api origin, keeping path and query and dropping the fragment", () => {
+ expect(pinToApiOrigin(API_URL, "https://evil.example/v2/crawl/abc?skip=10#frag")).toBe(
+ `${API_URL}/v2/crawl/abc?skip=10`,
+ );
+ expect(pinToApiOrigin("http://localhost:3002", "https://evil.example:444/v2/crawl/abc?skip=10")).toBe(
+ "http://localhost:3002/v2/crawl/abc?skip=10",
+ );
+ expect(pinToApiOrigin(API_URL, "\\\\evil.example/v2/crawl/abc")).toBe(`${API_URL}/v2/crawl/abc`);
+ for (const url of ["\nhttps://evil.example/v2/crawl/abc", "h\tttps://evil.example/v2/crawl/abc", "\u0000 //evil.example/v2/crawl/abc", "/\\evil.example/v2/crawl/abc"]) {
+ expect(pinToApiOrigin(API_URL, url)).toBe(`${API_URL}/v2/crawl/abc`);
+ }
+ });
+
+ test("leaves relative URLs unchanged", () => {
+ expect(pinToApiOrigin(API_URL, "/v2/crawl/abc?skip=10")).toBe("/v2/crawl/abc?skip=10");
+ });
+
+ test("throws when apiUrl is not absolute and the URL needs pinning", () => {
+ expect(() => pinToApiOrigin("api.firecrawl.dev", "https://evil.example/v2/crawl/abc")).toThrow(
+ "apiUrl must be an absolute URL",
+ );
+ });
+});
diff --git a/apps/js-sdk/firecrawl/src/utils/apiOrigin.ts b/apps/js-sdk/firecrawl/src/utils/apiOrigin.ts
new file mode 100644
index 000000000..995d58fa0
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/utils/apiOrigin.ts
@@ -0,0 +1,19 @@
+/**
+ * Rewrite an absolute or protocol-relative URL onto apiUrl's scheme, host and
+ * port (keeping path and query, dropping any fragment) so credentials never
+ * leave the configured API origin. Relative URLs are returned unchanged.
+ * Throws if apiUrl is not an absolute URL.
+ */
+export function pinToApiOrigin(apiUrl: string, url: string): string {
+ const normalized = url.replace(/[\t\n\r]/g, "").replace(/^[\u0000-\u0020]+/, "");
+ if (!/^([a-z][a-z\d+\-.]*:|[\\/]{2})/i.test(normalized)) return url;
+ const base = URL.canParse(apiUrl) ? new URL(apiUrl) : null;
+ if (!base?.host) {
+ throw new Error(`apiUrl must be an absolute URL, got ${JSON.stringify(apiUrl)}`);
+ }
+ const target = new URL(normalized, base);
+ const pinned = new URL(base.origin);
+ pinned.pathname = target.pathname;
+ pinned.search = target.search;
+ return pinned.href;
+}
diff --git a/apps/js-sdk/firecrawl/src/v1/index.ts b/apps/js-sdk/firecrawl/src/v1/index.ts
index 8b0564bf1..7a0f7bdd4 100644
--- a/apps/js-sdk/firecrawl/src/v1/index.ts
+++ b/apps/js-sdk/firecrawl/src/v1/index.ts
@@ -3,6 +3,7 @@ import * as zt from "zod";
import { zodSchemaToJsonSchema } from "../utils/zodSchemaToJson";
import { TypedEventTarget } from "typescript-event-target";
import type { RequiresAction } from "../v2/types";
+import { pinToApiOrigin } from "../utils/apiOrigin";
/**
* Configuration interface for FirecrawlApp.
@@ -922,7 +923,7 @@ export default class FirecrawlApp {
}
const headers: AxiosRequestHeaders = this.prepareHeaders();
- const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/crawl/${id}`);
+ const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/crawl/${id}`, this.apiUrl);
if (skip !== undefined) {
targetURL.searchParams.set("skip", skip.toString());
}
@@ -1210,7 +1211,7 @@ export default class FirecrawlApp {
}
const headers: AxiosRequestHeaders = this.prepareHeaders();
- const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/batch/scrape/${id}`);
+ const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/batch/scrape/${id}`, this.apiUrl);
if (skip !== undefined) {
targetURL.searchParams.set("skip", skip.toString());
}
@@ -1451,7 +1452,7 @@ export default class FirecrawlApp {
headers: AxiosRequestHeaders
): Promise<AxiosResponse> {
try {
- return await axios.get(url, { headers });
+ return await axios.get(pinToApiOrigin(this.apiUrl, url), { headers });
} catch (error) {
if (error instanceof AxiosError && error.response) {
return error.response as AxiosResponse;
diff --git a/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts b/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts
index 64bb2e601..b59c047a8 100644
--- a/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts
+++ b/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts
@@ -4,6 +4,7 @@ import axios, {
type AxiosResponse,
} from "axios";
import { getVersion } from "./getVersion";
+import { pinToApiOrigin } from "../../utils/apiOrigin";
export interface HttpClientOptions {
apiKey: string;
@@ -64,6 +65,7 @@ export class HttpClient {
config.headers = {
...(config.headers || {}),
};
+ if (config.url) config.url = pinToApiOrigin(this.apiUrl, config.url);
let lastError: any;
for (let attempt = 0; attempt < this.maxRetries; attempt++) {