Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(java-sdk): send the SDK origin on crawl, batch scrape and map (#5233)

Erik Engervalldcbc9473aa53
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java
index 768056a9b..4b93c0da8 100644
--- a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java
+++ b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java
@@ -297,6 +297,7 @@ public class FirecrawlClient {
if (options != null) {
mergeOptions(body, options);
}
+ body.putIfAbsent("origin", SDK_ORIGIN);
return http.post("/v2/crawl", body, CrawlResponse.class);
}
@@ -396,6 +397,7 @@ public class FirecrawlClient {
body.putAll(batchFields);
}
}
+ body.putIfAbsent("origin", SDK_ORIGIN);
return http.post("/v2/batch/scrape", body, BatchScrapeResponse.class, extraHeaders);
}
@@ -476,6 +478,7 @@ public class FirecrawlClient {
if (options != null) {
mergeOptions(body, options);
}
+ body.putIfAbsent("origin", SDK_ORIGIN);
return extractData(http.post("/v2/map", body, Map.class), MapData.class);
}
diff --git a/apps/java-sdk/src/test/java/com/firecrawl/JobOriginMockTest.java b/apps/java-sdk/src/test/java/com/firecrawl/JobOriginMockTest.java
new file mode 100644
index 000000000..f4bfbefc9
--- /dev/null
+++ b/apps/java-sdk/src/test/java/com/firecrawl/JobOriginMockTest.java
@@ -0,0 +1,83 @@
+package com.firecrawl;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.firecrawl.client.FirecrawlClient;
+import com.firecrawl.models.BatchScrapeOptions;
+import com.firecrawl.models.ScrapeOptions;
+import com.sun.net.httpserver.HttpServer;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+/**
+ * Checks that job-starting calls tag their request bodies with the SDK origin.
+ */
+class JobOriginMockTest {
+
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ private HttpServer server;
+ private FirecrawlClient client;
+ private final Map<String, String> bodies = new ConcurrentHashMap<>();
+
+ @BeforeEach
+ void setup() throws IOException {
+ server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
+ server.createContext("/", exchange -> {
+ String path = exchange.getRequestURI().getPath();
+ bodies.put(path, new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8));
+ String response = path.equals("/v2/map")
+ ? "{\"success\":true,\"links\":[]}"
+ : "{\"success\":true,\"id\":\"job-1\",\"url\":\"https://api.firecrawl.dev/job-1\"}";
+ byte[] bytes = response.getBytes(StandardCharsets.UTF_8);
+ exchange.getResponseHeaders().set("Content-Type", "application/json");
+ exchange.sendResponseHeaders(200, bytes.length);
+ try (OutputStream os = exchange.getResponseBody()) {
+ os.write(bytes);
+ }
+ });
+ server.start();
+ client = FirecrawlClient.builder()
+ .apiKey("fc-test-key")
+ .apiUrl("http://127.0.0.1:" + server.getAddress().getPort())
+ .maxRetries(0)
+ .build();
+ }
+
+ @AfterEach
+ void teardown() {
+ server.stop(0);
+ }
+
+ private JsonNode body(String path) throws IOException {
+ String raw = bodies.get(path);
+ assertNotNull(raw, "no request to " + path);
+ return MAPPER.readTree(raw);
+ }
+
+ @Test
+ void startCrawlBatchScrapeAndMapSendSdkOrigin() throws IOException {
+ client.startCrawl("https://example.com", null);
+ client.startBatchScrape(List.of("https://example.com"), BatchScrapeOptions.builder()
+ .options(ScrapeOptions.builder().formats(List.<Object>of("markdown")).build())
+ .build());
+ client.map("https://example.com");
+
+ for (String path : List.of("/v2/crawl", "/v2/batch/scrape", "/v2/map")) {
+ String origin = body(path).path("origin").asText();
+ assertTrue(origin.matches("java-sdk@\\d+\\.\\d+\\.\\d+"), path + " origin was '" + origin + "'");
+ }
+ assertEquals("markdown", body("/v2/batch/scrape").path("formats").path(0).asText());
+ }
+}