Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(java-sdk): add getBatchScrapeErrors (#5258)
Erik Engervall0300ff529e14
diff --git a/apps/api/src/controllers/v2/agent-credit-gate.ts b/apps/api/src/controllers/v2/agent-credit-gate.tsindex 22c0e2cfe..b2d4421eb 100644--- a/apps/api/src/controllers/v2/agent-credit-gate.ts+++ b/apps/api/src/controllers/v2/agent-credit-gate.ts@@ -2,7 +2,7 @@ import { autumnService } from "../../services/autumn/autumn.service";import { logger } from "../../lib/logger";export const AGENT_DEFAULT_MAX_CREDITS = 2500;-export const AGENT_MIN_CREDITS = 20;+const AGENT_MIN_CREDITS = 20;export const CREDIT_CHECK_EXEMPT_TEAM_ID ="8c528896-7882-4587-a4b6-768b721b0b53";diff --git a/apps/java-sdk/build.gradle.kts b/apps/java-sdk/build.gradle.ktsindex d94e6aa63..a765cb938 100644--- a/apps/java-sdk/build.gradle.kts+++ b/apps/java-sdk/build.gradle.kts@@ -4,7 +4,7 @@ plugins {}group = "com.firecrawl"-version = "1.18.5"+version = "1.18.6"java {sourceCompatibility = JavaVersion.VERSION_11diff --git a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.javaindex 4b93c0da8..a0a27df63 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java@@ -37,7 +37,7 @@ import java.util.concurrent.ForkJoinPool;public class FirecrawlClient {private static final String DEFAULT_API_URL = "https://api.firecrawl.dev";- private static final String SDK_ORIGIN = "[email protected]";+ private static final String SDK_ORIGIN = "[email protected]";private static final long DEFAULT_TIMEOUT_MS = 300_000; // 5 minutesprivate static final int DEFAULT_MAX_RETRIES = 3;private static final double DEFAULT_BACKOFF_FACTOR = 0.5;@@ -450,6 +450,18 @@ public class FirecrawlClient {return http.delete("/v2/batch/scrape/" + jobId, Map.class);}+ /**+ * Gets errors and robots.txt blocks from a batch scrape job.+ *+ * @param jobId the batch scrape job ID+ * @return error details+ */+ @SuppressWarnings("unchecked")+ public Map<String, Object> getBatchScrapeErrors(String jobId) {+ Objects.requireNonNull(jobId, "Job ID is required");+ return http.get("/v2/batch/scrape/" + jobId + "/errors", Map.class);+ }+// ================================================================// MAP// ================================================================diff --git a/apps/java-sdk/src/test/java/com/firecrawl/BatchScrapeErrorsMockTest.java b/apps/java-sdk/src/test/java/com/firecrawl/BatchScrapeErrorsMockTest.javanew file mode 100644index 000000000..5f840895a--- /dev/null+++ b/apps/java-sdk/src/test/java/com/firecrawl/BatchScrapeErrorsMockTest.java@@ -0,0 +1,70 @@+package com.firecrawl;++import com.firecrawl.client.FirecrawlClient;+import com.sun.net.httpserver.HttpServer;+import org.junit.jupiter.api.AfterEach;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.io.IOException;+import java.io.OutputStream;+import java.net.InetSocketAddress;+import java.nio.charset.StandardCharsets;+import java.util.List;+import java.util.Map;+import java.util.concurrent.atomic.AtomicReference;++import static org.junit.jupiter.api.Assertions.*;++class BatchScrapeErrorsMockTest {++ private HttpServer server;+ private FirecrawlClient client;+ private final AtomicReference<String> requestLine = new AtomicReference<>();++ @BeforeEach+ void setup() throws IOException {+ server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);+ server.createContext("/", exchange -> {+ requestLine.set(exchange.getRequestMethod() + " " + exchange.getRequestURI().getPath());+ byte[] bytes = ("{\"errors\":[{\"id\":\"e1\",\"timestamp\":\"2026-01-01T00:00:00Z\","+ + "\"url\":\"https://example.com/a\",\"error\":\"timeout\"}],"+ + "\"robotsBlocked\":[\"https://example.com/private\"]}")+ .getBytes(StandardCharsets.UTF_8);+ exchange.getResponseHeaders().set("Content-Type", "application/json");+ exchange.sendResponseHeaders(200, bytes.length);+ try (OutputStream os = exchange.getResponseBody()) {+ os.write(bytes);+ }+ });+ server.start();+ client = FirecrawlClient.builder()+ .apiKey("fc-test-key")+ .apiUrl("http://127.0.0.1:" + server.getAddress().getPort())+ .maxRetries(0)+ .build();+ }++ @AfterEach+ void teardown() {+ server.stop(0);+ }++ @Test+ @SuppressWarnings("unchecked")+ void getBatchScrapeErrorsReturnsErrorsAndRobotsBlocks() {+ Map<String, Object> result = client.getBatchScrapeErrors("job-1");++ assertEquals("GET /v2/batch/scrape/job-1/errors", requestLine.get());+ List<Map<String, Object>> errors = (List<Map<String, Object>>) result.get("errors");+ assertEquals(1, errors.size());+ assertEquals("https://example.com/a", errors.get(0).get("url"));+ assertEquals(List.of("https://example.com/private"), result.get("robotsBlocked"));+ }++ @Test+ void getBatchScrapeErrorsRequiresJobId() {+ assertThrows(NullPointerException.class, () -> client.getBatchScrapeErrors(null));+ assertNull(requestLine.get());+ }+}