Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(java-sdk): pin pagination next URLs to the api_url origin (#4870)
Gergő Móricz997ae07419ea
diff --git a/apps/java-sdk/build.gradle.kts b/apps/java-sdk/build.gradle.ktsindex 7619661f6..8e05d95d7 100644--- a/apps/java-sdk/build.gradle.kts+++ b/apps/java-sdk/build.gradle.kts@@ -4,7 +4,7 @@ plugins {}group = "com.firecrawl"-version = "1.18.0"+version = "1.18.1"java {sourceCompatibility = JavaVersion.VERSION_11diff --git a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.javaindex 789480ebe..3cad63856 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java@@ -37,7 +37,7 @@ import java.util.concurrent.ForkJoinPool;public class FirecrawlClient {private static final String DEFAULT_API_URL = "https://api.firecrawl.dev";- private static final String SDK_ORIGIN = "[email protected]";+ private static final String SDK_ORIGIN = "[email protected]";private static final long DEFAULT_TIMEOUT_MS = 300_000; // 5 minutesprivate static final int DEFAULT_MAX_RETRIES = 3;private static final double DEFAULT_BACKOFF_FACTOR = 0.5;diff --git a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlHttpClient.java b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlHttpClient.javaindex b6fe62d7a..4fcccf48b 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlHttpClient.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlHttpClient.java@@ -174,17 +174,41 @@ class FirecrawlHttpClient {}/**- * Sends a GET request with full URL (for following next-page cursors).+ * Sends a GET request to a next-page cursor URL, pinned to the API origin.*/- <T> T getAbsolute(String absoluteUrl, Class<T> responseType) {+ <T> T getAbsolute(String nextUrl, Class<T> responseType) {Request.Builder builder = new Request.Builder()- .url(absoluteUrl)+ .url(pinToApiOrigin(baseUrl + "/", nextUrl)).get();applyAuth(builder);Request request = builder.build();return executeWithRetry(request, responseType);}+ /**+ * Resolves url against apiUrl and rewrites it onto apiUrl's scheme, host and port so+ * credentials never leave the configured API origin. Path and query are kept; the+ * fragment is dropped.+ *+ * @throws IllegalArgumentException if apiUrl is not an absolute http(s) URL+ * @throws FirecrawlException if url cannot be resolved against apiUrl+ */+ static HttpUrl pinToApiOrigin(String apiUrl, String url) {+ HttpUrl base = HttpUrl.get(apiUrl);+ HttpUrl resolved = base.resolve(url);+ if (resolved == null) {+ throw new FirecrawlException("Invalid next page URL: " + url);+ }+ return resolved.newBuilder()+ .scheme(base.scheme())+ .encodedUsername(base.encodedUsername())+ .encodedPassword(base.encodedPassword())+ .host(base.host())+ .port(base.port())+ .fragment(null)+ .build();+ }+/*** Sends a DELETE request.*/diff --git a/apps/java-sdk/src/test/java/com/firecrawl/PaginationOriginTest.java b/apps/java-sdk/src/test/java/com/firecrawl/PaginationOriginTest.javanew file mode 100644index 000000000..62458af03--- /dev/null+++ b/apps/java-sdk/src/test/java/com/firecrawl/PaginationOriginTest.java@@ -0,0 +1,149 @@+package com.firecrawl;++import com.firecrawl.client.FirecrawlClient;+import com.sun.net.httpserver.HttpExchange;+import com.sun.net.httpserver.HttpServer;+import org.junit.jupiter.api.AfterEach;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.Arguments;+import org.junit.jupiter.params.provider.MethodSource;++import java.io.IOException;+import java.io.OutputStream;+import java.net.InetSocketAddress;+import java.nio.charset.StandardCharsets;+import java.util.Collections;+import java.util.List;+import java.util.concurrent.CopyOnWriteArrayList;+import java.util.stream.Stream;++import static org.junit.jupiter.api.Assertions.*;++/**+ * Verifies that auto-pagination only ever sends requests (and the API key) to the+ * configured API origin, whatever host the server-supplied "next" URL points at.+ */+class PaginationOriginTest {++ private static final String API_KEY = "fc-test-key";++ private HttpServer api;+ private HttpServer foreign;+ private final List<String> apiRequests = new CopyOnWriteArrayList<>();+ private final List<String> foreignRequests = new CopyOnWriteArrayList<>();+ private volatile String nextUrl;++ @BeforeEach+ void setup() throws IOException {+ foreign = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);+ foreign.createContext("/", exchange -> {+ foreignRequests.add(exchange.getRequestURI().toString());+ respond(exchange, "{\"success\":true,\"status\":\"completed\",\"data\":[]}");+ });+ foreign.start();++ api = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);+ api.createContext("/", this::handleApi);+ api.start();+ }++ @AfterEach+ void teardown() {+ api.stop(0);+ foreign.stop(0);+ }++ private void handleApi(HttpExchange exchange) throws IOException {+ String uri = exchange.getRequestURI().toString();+ apiRequests.add(exchange.getRequestMethod() + " " + uri + " "+ + exchange.getRequestHeaders().getFirst("Authorization"));+ switch (uri) {+ case "/v2/crawl":+ case "/v2/batch/scrape":+ respond(exchange, "{\"success\":true,\"id\":\"abc\"}");+ return;+ case "/v2/crawl/abc":+ case "/v2/batch/scrape/abc":+ respond(exchange, "{\"status\":\"completed\",\"data\":[{\"markdown\":\"page-1\"}],"+ + "\"next\":\"" + nextUrl + "\"}");+ return;+ case "/v2/crawl/abc?skip=10":+ case "/v2/batch/scrape/abc?skip=10":+ respond(exchange, "{\"status\":\"completed\",\"data\":[{\"markdown\":\"page-2\"}]}");+ return;+ case "/v2/monitor/m1/checks/c1":+ respond(exchange, "{\"success\":true,\"data\":{\"id\":\"c1\","+ + "\"pages\":[{\"url\":\"https://a.example\"}],\"next\":\"" + nextUrl + "\"}}");+ return;+ case "/v2/monitor/m1/checks/c1?skip=10":+ respond(exchange, "{\"success\":true,\"data\":{\"id\":\"c1\","+ + "\"pages\":[{\"url\":\"https://b.example\"}]}}");+ return;+ default:+ exchange.sendResponseHeaders(404, -1);+ exchange.close();+ }+ }++ private static void respond(HttpExchange exchange, String body) throws IOException {+ byte[] bytes = body.getBytes(StandardCharsets.UTF_8);+ exchange.getResponseHeaders().set("Content-Type", "application/json");+ exchange.sendResponseHeaders(200, bytes.length);+ try (OutputStream os = exchange.getResponseBody()) {+ os.write(bytes);+ }+ }++ static Stream<Arguments> cases() {+ String[] followers = {"crawl", "batch/scrape", "monitor"};+ String[] origins = {+ "same-origin|http://127.0.0.1:{api}",+ "cross-host|https://evil.example",+ "cross-host-reachable|http://localhost:{foreign}",+ "protocol-relative|//evil.example",+ "different-port|http://127.0.0.1:{foreign}",+ "different-scheme|https://127.0.0.1:{api}",+ "relative|",+ };+ return Stream.of(followers).flatMap(follower -> Stream.of(origins).map(origin -> {+ String[] parts = origin.split("\\|", -1);+ return Arguments.of(follower, parts[0], parts[1]);+ }));+ }++ @ParameterizedTest(name = "{0} follows {1} next URL on the API origin")+ @MethodSource("cases")+ void nextUrlIsPinnedToApiOrigin(String follower, String label, String originTemplate) {+ String path = follower.equals("monitor") ? "/v2/monitor/m1/checks/c1" : "/v2/" + follower + "/abc";+ nextUrl = originTemplate+ .replace("{api}", String.valueOf(api.getAddress().getPort()))+ .replace("{foreign}", String.valueOf(foreign.getAddress().getPort()))+ + path + "?skip=10";++ FirecrawlClient client = FirecrawlClient.builder()+ .apiKey(API_KEY)+ .apiUrl("http://127.0.0.1:" + api.getAddress().getPort())+ .maxRetries(0)+ .timeoutMs(5_000)+ .build();++ int results;+ switch (follower) {+ case "crawl":+ results = client.crawl("https://example.com", null, 1, 30).getData().size();+ break;+ case "batch/scrape":+ results = client.batchScrape(Collections.singletonList("https://example.com"), null, 1, 30)+ .getData().size();+ break;+ default:+ results = client.getMonitorCheck("m1", "c1").getPages().size();+ }++ assertEquals(2, results);+ assertTrue(apiRequests.contains("GET " + path + "?skip=10 Bearer " + API_KEY),+ "next page was not requested on the API origin with the API key: " + apiRequests);+ assertTrue(foreignRequests.isEmpty(), "request leaked to foreign origin: " + foreignRequests);+ }+}diff --git a/apps/java-sdk/src/test/java/com/firecrawl/client/ApiOriginTest.java b/apps/java-sdk/src/test/java/com/firecrawl/client/ApiOriginTest.javanew file mode 100644index 000000000..0a4a9d796--- /dev/null+++ b/apps/java-sdk/src/test/java/com/firecrawl/client/ApiOriginTest.java@@ -0,0 +1,77 @@+package com.firecrawl.client;++import com.firecrawl.errors.FirecrawlException;+import org.junit.jupiter.api.Test;++import static org.junit.jupiter.api.Assertions.*;++class ApiOriginTest {++ private static final String API = "https://api.firecrawl.dev";++ private static String pin(String apiUrl, String url) {+ return FirecrawlHttpClient.pinToApiOrigin(apiUrl, url).toString();+ }++ @Test+ void sameOriginUrlIsUnchanged() {+ assertEquals("https://api.firecrawl.dev/v2/crawl/abc?skip=10",+ pin(API, "https://api.firecrawl.dev/v2/crawl/abc?skip=10"));+ }++ @Test+ void crossHostUrlIsRewrittenOntoApiOrigin() {+ assertEquals("https://api.firecrawl.dev/v2/crawl/abc?skip=10",+ pin(API, "https://evil.example/v2/crawl/abc?skip=10"));+ }++ @Test+ void protocolRelativeUrlIsRewrittenOntoApiOrigin() {+ assertEquals("https://api.firecrawl.dev/v2/crawl/abc?skip=10",+ pin(API, "//evil.example/v2/crawl/abc?skip=10"));+ }++ @Test+ void differentPortIsRewrittenOntoApiPort() {+ assertEquals("https://api.firecrawl.dev/v2/crawl/abc?skip=10",+ pin(API, "https://api.firecrawl.dev:8443/v2/crawl/abc?skip=10"));+ assertEquals("http://localhost:3002/v2/crawl/abc?skip=10",+ pin("http://localhost:3002", "http://localhost:9999/v2/crawl/abc?skip=10"));+ }++ @Test+ void differentSchemeIsRewrittenOntoApiScheme() {+ assertEquals("https://api.firecrawl.dev/v2/crawl/abc?skip=10",+ pin(API, "http://api.firecrawl.dev/v2/crawl/abc?skip=10"));+ }++ @Test+ void userInfoAndFragmentAreDropped() {+ assertEquals("https://api.firecrawl.dev/v2/crawl/abc?skip=10",+ pin(API, "https://user:[email protected]/v2/crawl/abc?skip=10#frag"));+ }++ @Test+ void relativeUrlIsResolvedAgainstApiUrl() {+ assertEquals("https://api.firecrawl.dev/v2/crawl/abc?skip=10",+ pin(API, "/v2/crawl/abc?skip=10"));+ }++ @Test+ void pathRelativeUrlKeepsApiUrlPathPrefix() {+ assertEquals("https://api.example/prefix/next?skip=10",+ pin("https://api.example/prefix/", "next?skip=10"));+ }++ @Test+ void unresolvableUrlIsRejected() {+ assertThrows(FirecrawlException.class,+ () -> pin(API, "ftp://evil.example/v2/crawl/abc"));+ }++ @Test+ void nonAbsoluteApiUrlIsRejected() {+ assertThrows(IllegalArgumentException.class,+ () -> pin("api.firecrawl.dev", "https://evil.example/v2/crawl/abc"));+ }+}