Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(java-sdk): pin pagination next URLs to the api_url origin (#4870)

Gergő Móricz997ae07419ea
diff --git a/apps/java-sdk/build.gradle.kts b/apps/java-sdk/build.gradle.kts
index 7619661f6..8e05d95d7 100644
--- a/apps/java-sdk/build.gradle.kts
+++ b/apps/java-sdk/build.gradle.kts
@@ -4,7 +4,7 @@ plugins {
}
group = "com.firecrawl"
-version = "1.18.0"
+version = "1.18.1"
java {
sourceCompatibility = JavaVersion.VERSION_11
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java
index 789480ebe..3cad63856 100644
--- a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java
+++ b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java
@@ -37,7 +37,7 @@ import java.util.concurrent.ForkJoinPool;
public class FirecrawlClient {
private static final String DEFAULT_API_URL = "https://api.firecrawl.dev";
- private static final String SDK_ORIGIN = "[email protected]";
+ private static final String SDK_ORIGIN = "[email protected]";
private static final long DEFAULT_TIMEOUT_MS = 300_000; // 5 minutes
private static final int DEFAULT_MAX_RETRIES = 3;
private static final double DEFAULT_BACKOFF_FACTOR = 0.5;
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlHttpClient.java b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlHttpClient.java
index b6fe62d7a..4fcccf48b 100644
--- a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlHttpClient.java
+++ b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlHttpClient.java
@@ -174,17 +174,41 @@ class FirecrawlHttpClient {
}
/**
- * Sends a GET request with full URL (for following next-page cursors).
+ * Sends a GET request to a next-page cursor URL, pinned to the API origin.
*/
- <T> T getAbsolute(String absoluteUrl, Class<T> responseType) {
+ <T> T getAbsolute(String nextUrl, Class<T> responseType) {
Request.Builder builder = new Request.Builder()
- .url(absoluteUrl)
+ .url(pinToApiOrigin(baseUrl + "/", nextUrl))
.get();
applyAuth(builder);
Request request = builder.build();
return executeWithRetry(request, responseType);
}
+ /**
+ * Resolves url against apiUrl and rewrites it onto apiUrl's scheme, host and port so
+ * credentials never leave the configured API origin. Path and query are kept; the
+ * fragment is dropped.
+ *
+ * @throws IllegalArgumentException if apiUrl is not an absolute http(s) URL
+ * @throws FirecrawlException if url cannot be resolved against apiUrl
+ */
+ static HttpUrl pinToApiOrigin(String apiUrl, String url) {
+ HttpUrl base = HttpUrl.get(apiUrl);
+ HttpUrl resolved = base.resolve(url);
+ if (resolved == null) {
+ throw new FirecrawlException("Invalid next page URL: " + url);
+ }
+ return resolved.newBuilder()
+ .scheme(base.scheme())
+ .encodedUsername(base.encodedUsername())
+ .encodedPassword(base.encodedPassword())
+ .host(base.host())
+ .port(base.port())
+ .fragment(null)
+ .build();
+ }
+
/**
* Sends a DELETE request.
*/
diff --git a/apps/java-sdk/src/test/java/com/firecrawl/PaginationOriginTest.java b/apps/java-sdk/src/test/java/com/firecrawl/PaginationOriginTest.java
new file mode 100644
index 000000000..62458af03
--- /dev/null
+++ b/apps/java-sdk/src/test/java/com/firecrawl/PaginationOriginTest.java
@@ -0,0 +1,149 @@
+package com.firecrawl;
+
+import com.firecrawl.client.FirecrawlClient;
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpServer;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+/**
+ * Verifies that auto-pagination only ever sends requests (and the API key) to the
+ * configured API origin, whatever host the server-supplied "next" URL points at.
+ */
+class PaginationOriginTest {
+
+ private static final String API_KEY = "fc-test-key";
+
+ private HttpServer api;
+ private HttpServer foreign;
+ private final List<String> apiRequests = new CopyOnWriteArrayList<>();
+ private final List<String> foreignRequests = new CopyOnWriteArrayList<>();
+ private volatile String nextUrl;
+
+ @BeforeEach
+ void setup() throws IOException {
+ foreign = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
+ foreign.createContext("/", exchange -> {
+ foreignRequests.add(exchange.getRequestURI().toString());
+ respond(exchange, "{\"success\":true,\"status\":\"completed\",\"data\":[]}");
+ });
+ foreign.start();
+
+ api = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
+ api.createContext("/", this::handleApi);
+ api.start();
+ }
+
+ @AfterEach
+ void teardown() {
+ api.stop(0);
+ foreign.stop(0);
+ }
+
+ private void handleApi(HttpExchange exchange) throws IOException {
+ String uri = exchange.getRequestURI().toString();
+ apiRequests.add(exchange.getRequestMethod() + " " + uri + " "
+ + exchange.getRequestHeaders().getFirst("Authorization"));
+ switch (uri) {
+ case "/v2/crawl":
+ case "/v2/batch/scrape":
+ respond(exchange, "{\"success\":true,\"id\":\"abc\"}");
+ return;
+ case "/v2/crawl/abc":
+ case "/v2/batch/scrape/abc":
+ respond(exchange, "{\"status\":\"completed\",\"data\":[{\"markdown\":\"page-1\"}],"
+ + "\"next\":\"" + nextUrl + "\"}");
+ return;
+ case "/v2/crawl/abc?skip=10":
+ case "/v2/batch/scrape/abc?skip=10":
+ respond(exchange, "{\"status\":\"completed\",\"data\":[{\"markdown\":\"page-2\"}]}");
+ return;
+ case "/v2/monitor/m1/checks/c1":
+ respond(exchange, "{\"success\":true,\"data\":{\"id\":\"c1\","
+ + "\"pages\":[{\"url\":\"https://a.example\"}],\"next\":\"" + nextUrl + "\"}}");
+ return;
+ case "/v2/monitor/m1/checks/c1?skip=10":
+ respond(exchange, "{\"success\":true,\"data\":{\"id\":\"c1\","
+ + "\"pages\":[{\"url\":\"https://b.example\"}]}}");
+ return;
+ default:
+ exchange.sendResponseHeaders(404, -1);
+ exchange.close();
+ }
+ }
+
+ private static void respond(HttpExchange exchange, String body) throws IOException {
+ byte[] bytes = body.getBytes(StandardCharsets.UTF_8);
+ exchange.getResponseHeaders().set("Content-Type", "application/json");
+ exchange.sendResponseHeaders(200, bytes.length);
+ try (OutputStream os = exchange.getResponseBody()) {
+ os.write(bytes);
+ }
+ }
+
+ static Stream<Arguments> cases() {
+ String[] followers = {"crawl", "batch/scrape", "monitor"};
+ String[] origins = {
+ "same-origin|http://127.0.0.1:{api}",
+ "cross-host|https://evil.example",
+ "cross-host-reachable|http://localhost:{foreign}",
+ "protocol-relative|//evil.example",
+ "different-port|http://127.0.0.1:{foreign}",
+ "different-scheme|https://127.0.0.1:{api}",
+ "relative|",
+ };
+ return Stream.of(followers).flatMap(follower -> Stream.of(origins).map(origin -> {
+ String[] parts = origin.split("\\|", -1);
+ return Arguments.of(follower, parts[0], parts[1]);
+ }));
+ }
+
+ @ParameterizedTest(name = "{0} follows {1} next URL on the API origin")
+ @MethodSource("cases")
+ void nextUrlIsPinnedToApiOrigin(String follower, String label, String originTemplate) {
+ String path = follower.equals("monitor") ? "/v2/monitor/m1/checks/c1" : "/v2/" + follower + "/abc";
+ nextUrl = originTemplate
+ .replace("{api}", String.valueOf(api.getAddress().getPort()))
+ .replace("{foreign}", String.valueOf(foreign.getAddress().getPort()))
+ + path + "?skip=10";
+
+ FirecrawlClient client = FirecrawlClient.builder()
+ .apiKey(API_KEY)
+ .apiUrl("http://127.0.0.1:" + api.getAddress().getPort())
+ .maxRetries(0)
+ .timeoutMs(5_000)
+ .build();
+
+ int results;
+ switch (follower) {
+ case "crawl":
+ results = client.crawl("https://example.com", null, 1, 30).getData().size();
+ break;
+ case "batch/scrape":
+ results = client.batchScrape(Collections.singletonList("https://example.com"), null, 1, 30)
+ .getData().size();
+ break;
+ default:
+ results = client.getMonitorCheck("m1", "c1").getPages().size();
+ }
+
+ assertEquals(2, results);
+ assertTrue(apiRequests.contains("GET " + path + "?skip=10 Bearer " + API_KEY),
+ "next page was not requested on the API origin with the API key: " + apiRequests);
+ assertTrue(foreignRequests.isEmpty(), "request leaked to foreign origin: " + foreignRequests);
+ }
+}
diff --git a/apps/java-sdk/src/test/java/com/firecrawl/client/ApiOriginTest.java b/apps/java-sdk/src/test/java/com/firecrawl/client/ApiOriginTest.java
new file mode 100644
index 000000000..0a4a9d796
--- /dev/null
+++ b/apps/java-sdk/src/test/java/com/firecrawl/client/ApiOriginTest.java
@@ -0,0 +1,77 @@
+package com.firecrawl.client;
+
+import com.firecrawl.errors.FirecrawlException;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+class ApiOriginTest {
+
+ private static final String API = "https://api.firecrawl.dev";
+
+ private static String pin(String apiUrl, String url) {
+ return FirecrawlHttpClient.pinToApiOrigin(apiUrl, url).toString();
+ }
+
+ @Test
+ void sameOriginUrlIsUnchanged() {
+ assertEquals("https://api.firecrawl.dev/v2/crawl/abc?skip=10",
+ pin(API, "https://api.firecrawl.dev/v2/crawl/abc?skip=10"));
+ }
+
+ @Test
+ void crossHostUrlIsRewrittenOntoApiOrigin() {
+ assertEquals("https://api.firecrawl.dev/v2/crawl/abc?skip=10",
+ pin(API, "https://evil.example/v2/crawl/abc?skip=10"));
+ }
+
+ @Test
+ void protocolRelativeUrlIsRewrittenOntoApiOrigin() {
+ assertEquals("https://api.firecrawl.dev/v2/crawl/abc?skip=10",
+ pin(API, "//evil.example/v2/crawl/abc?skip=10"));
+ }
+
+ @Test
+ void differentPortIsRewrittenOntoApiPort() {
+ assertEquals("https://api.firecrawl.dev/v2/crawl/abc?skip=10",
+ pin(API, "https://api.firecrawl.dev:8443/v2/crawl/abc?skip=10"));
+ assertEquals("http://localhost:3002/v2/crawl/abc?skip=10",
+ pin("http://localhost:3002", "http://localhost:9999/v2/crawl/abc?skip=10"));
+ }
+
+ @Test
+ void differentSchemeIsRewrittenOntoApiScheme() {
+ assertEquals("https://api.firecrawl.dev/v2/crawl/abc?skip=10",
+ pin(API, "http://api.firecrawl.dev/v2/crawl/abc?skip=10"));
+ }
+
+ @Test
+ void userInfoAndFragmentAreDropped() {
+ assertEquals("https://api.firecrawl.dev/v2/crawl/abc?skip=10",
+ pin(API, "https://user:[email protected]/v2/crawl/abc?skip=10#frag"));
+ }
+
+ @Test
+ void relativeUrlIsResolvedAgainstApiUrl() {
+ assertEquals("https://api.firecrawl.dev/v2/crawl/abc?skip=10",
+ pin(API, "/v2/crawl/abc?skip=10"));
+ }
+
+ @Test
+ void pathRelativeUrlKeepsApiUrlPathPrefix() {
+ assertEquals("https://api.example/prefix/next?skip=10",
+ pin("https://api.example/prefix/", "next?skip=10"));
+ }
+
+ @Test
+ void unresolvableUrlIsRejected() {
+ assertThrows(FirecrawlException.class,
+ () -> pin(API, "ftp://evil.example/v2/crawl/abc"));
+ }
+
+ @Test
+ void nonAbsoluteApiUrlIsRejected() {
+ assertThrows(IllegalArgumentException.class,
+ () -> pin("api.firecrawl.dev", "https://evil.example/v2/crawl/abc"));
+ }
+}