Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(java): add getParseFormats for GET /v2/parse/formats (#4910)
Gergő Móricz40cb54f4feee
diff --git a/apps/java-sdk/README.md b/apps/java-sdk/README.mdindex 27cdccc45..cc37aa8af 100644--- a/apps/java-sdk/README.md+++ b/apps/java-sdk/README.md@@ -180,6 +180,17 @@ Document parsed = client.parse(file,System.out.println(parsed.getMarkdown());```+To see which upload types the API accepts, call `getParseFormats()`. Each entry has a `format`, a `kind` (`document` or `image`), its `extensions` and `mimeTypes`, and whether it is `available` on the current deployment (image formats need image OCR enabled).++```java+import com.firecrawl.models.ParseFormat;++for (ParseFormat format : client.getParseFormats()) {+ System.out.println(format.getFormat() + " " + format.getKind()+ + " " + format.getExtensions() + " available=" + format.isAvailable());+}+```+#### JSON Extraction```javadiff --git a/apps/java-sdk/build.gradle.kts b/apps/java-sdk/build.gradle.ktsindex 8e05d95d7..36ed0dc14 100644--- a/apps/java-sdk/build.gradle.kts+++ b/apps/java-sdk/build.gradle.kts@@ -4,7 +4,7 @@ plugins {}group = "com.firecrawl"-version = "1.18.1"+version = "1.18.2"java {sourceCompatibility = JavaVersion.VERSION_11diff --git a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.javaindex 3cad63856..eeb552b1c 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java@@ -37,7 +37,7 @@ import java.util.concurrent.ForkJoinPool;public class FirecrawlClient {private static final String DEFAULT_API_URL = "https://api.firecrawl.dev";- private static final String SDK_ORIGIN = "[email protected]";+ private static final String SDK_ORIGIN = "[email protected]";private static final long DEFAULT_TIMEOUT_MS = 300_000; // 5 minutesprivate static final int DEFAULT_MAX_RETRIES = 3;private static final double DEFAULT_BACKOFF_FACTOR = 0.5;@@ -249,6 +249,25 @@ public class FirecrawlClient {);}+ /**+ * Lists the upload types accepted by {@link #parse(ParseFile, ParseOptions)}.+ *+ * @return the supported parse formats+ */+ public List<ParseFormat> getParseFormats() {+ Map<?, ?> raw = http.get("/v2/parse/formats", Map.class);+ Object data = raw.get("data");+ Object formats = data instanceof Map<?, ?> ? ((Map<?, ?>) data).get("formats") : null;+ if (!(formats instanceof List<?>)) {+ return Collections.emptyList();+ }+ List<ParseFormat> result = new ArrayList<>();+ for (Object item : (List<?>) formats) {+ result.add(http.objectMapper.convertValue(item, ParseFormat.class));+ }+ return result;+ }+// ================================================================// CRAWL// ================================================================@@ -1007,6 +1026,15 @@ public class FirecrawlClient {return CompletableFuture.supplyAsync(() -> parse(file, options), asyncExecutor);}+ /**+ * Asynchronously lists the upload types accepted by parse.+ *+ * @return a CompletableFuture that resolves to the supported parse formats+ */+ public CompletableFuture<List<ParseFormat>> getParseFormatsAsync() {+ return CompletableFuture.supplyAsync(this::getParseFormats, asyncExecutor);+ }+/*** Asynchronously crawls a website and waits for completion.*diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/ParseFormat.java b/apps/java-sdk/src/main/java/com/firecrawl/models/ParseFormat.javanew file mode 100644index 000000000..47d6025b0--- /dev/null+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/ParseFormat.java@@ -0,0 +1,74 @@+package com.firecrawl.models;++import com.fasterxml.jackson.annotation.JsonIgnore;+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;++import java.util.Collections;+import java.util.List;++/**+ * An upload type accepted by {@code POST /v2/parse}, as returned by+ * {@code GET /v2/parse/formats}.+ */+@JsonIgnoreProperties(ignoreUnknown = true)+public class ParseFormat {++ /**+ * Category of a parse format. Values the SDK does not know yet map to {@link #UNKNOWN};+ * the raw value stays available via {@link ParseFormat#getKind()}.+ */+ public enum Kind {+ DOCUMENT("document"),+ IMAGE("image"),+ UNKNOWN(null);++ private final String value;++ Kind(String value) {+ this.value = value;+ }++ public String getValue() {+ return value;+ }++ public static Kind fromValue(String value) {+ for (Kind kind : values()) {+ if (kind.value != null && kind.value.equals(value)) {+ return kind;+ }+ }+ return UNKNOWN;+ }+ }++ private String format;+ private String kind;+ private List<String> extensions;+ private List<String> mimeTypes;+ private boolean available;++ /** Format identifier, e.g. {@code "pdf"} or {@code "png"}. */+ public String getFormat() { return format; }++ /** Raw kind value, e.g. {@code "document"} or {@code "image"}. */+ public String getKind() { return kind; }++ /** Kind as an enum; {@link Kind#UNKNOWN} for values this SDK version does not know. */+ @JsonIgnore+ public Kind getKindType() { return Kind.fromValue(kind); }++ /** File extensions including the leading dot, e.g. {@code ".pdf"}. */+ public List<String> getExtensions() { return extensions != null ? extensions : Collections.emptyList(); }++ /** Accepted MIME types, e.g. {@code "application/pdf"}. */+ public List<String> getMimeTypes() { return mimeTypes != null ? mimeTypes : Collections.emptyList(); }++ /** Whether this format can be parsed on the current deployment. */+ public boolean isAvailable() { return available; }++ @Override+ public String toString() {+ return "ParseFormat{format=" + format + ", kind=" + kind + ", available=" + available + "}";+ }+}diff --git a/apps/java-sdk/src/test/java/com/firecrawl/FirecrawlClientTest.java b/apps/java-sdk/src/test/java/com/firecrawl/FirecrawlClientTest.javaindex 1cefffcdd..f1c1da01d 100644--- a/apps/java-sdk/src/test/java/com/firecrawl/FirecrawlClientTest.java+++ b/apps/java-sdk/src/test/java/com/firecrawl/FirecrawlClientTest.java@@ -4,6 +4,7 @@ import com.firecrawl.client.FirecrawlClient;import com.firecrawl.errors.FirecrawlException;import com.firecrawl.models.*;import okhttp3.OkHttpClient;+import org.junit.jupiter.api.Assumptions;import org.junit.jupiter.api.Test;import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;@@ -490,4 +491,21 @@ class FirecrawlClientTest {assertFalse(doc.getMarkdown().isEmpty());assertTrue(doc.getMarkdown().contains("Java SDK Parse E2E"));}++ @Test+ @EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")+ void testGetParseFormatsE2E() {+ FirecrawlClient client = FirecrawlClient.fromEnv();+ List<ParseFormat> formats;+ try {+ formats = client.getParseFormats();+ } catch (FirecrawlException e) {+ Assumptions.assumeTrue(e.getStatusCode() != 404, "GET /v2/parse/formats is not deployed yet");+ throw e;+ }++ assertFalse(formats.isEmpty());+ assertTrue(formats.stream().anyMatch(f ->+ "pdf".equals(f.getFormat()) && f.getKindType() == ParseFormat.Kind.DOCUMENT));+ }}diff --git a/apps/java-sdk/src/test/java/com/firecrawl/ParseFormatsMockTest.java b/apps/java-sdk/src/test/java/com/firecrawl/ParseFormatsMockTest.javanew file mode 100644index 000000000..0cccf84fd--- /dev/null+++ b/apps/java-sdk/src/test/java/com/firecrawl/ParseFormatsMockTest.java@@ -0,0 +1,167 @@+package com.firecrawl;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.firecrawl.client.FirecrawlClient;+import com.firecrawl.errors.AuthenticationException;+import com.firecrawl.errors.FirecrawlException;+import com.firecrawl.models.ParseFormat;+import com.sun.net.httpserver.HttpServer;+import org.junit.jupiter.api.AfterEach;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.io.IOException;+import java.io.OutputStream;+import java.net.InetSocketAddress;+import java.nio.charset.StandardCharsets;+import java.util.List;+import java.util.concurrent.atomic.AtomicInteger;+import java.util.concurrent.atomic.AtomicReference;++import static org.junit.jupiter.api.Assertions.*;++/**+ * GET /v2/parse/formats tests against a local mock HTTP server.+ */+class ParseFormatsMockTest {++ private HttpServer server;+ private FirecrawlClient client;+ private final AtomicReference<String> lastMethod = new AtomicReference<>();+ private final AtomicReference<String> lastPath = new AtomicReference<>();+ private final AtomicReference<String> lastAuth = new AtomicReference<>();+ private final AtomicInteger responseStatus = new AtomicInteger(200);+ private final AtomicReference<String> responseBody = new AtomicReference<>();++ @BeforeEach+ void setup() throws IOException {+ server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);+ server.createContext("/", exchange -> {+ lastMethod.set(exchange.getRequestMethod());+ lastPath.set(exchange.getRequestURI().toString());+ lastAuth.set(exchange.getRequestHeaders().getFirst("Authorization"));+ byte[] bytes = responseBody.get().getBytes(StandardCharsets.UTF_8);+ exchange.getResponseHeaders().set("Content-Type", "application/json");+ exchange.sendResponseHeaders(responseStatus.get(), bytes.length);+ try (OutputStream os = exchange.getResponseBody()) {+ os.write(bytes);+ }+ });+ server.start();+ client = FirecrawlClient.builder()+ .apiKey("fc-test-key")+ .apiUrl("http://127.0.0.1:" + server.getAddress().getPort())+ .maxRetries(0)+ .build();+ }++ @AfterEach+ void teardown() {+ server.stop(0);+ }++ private void respondWith(int status, String body) {+ responseStatus.set(status);+ responseBody.set(body);+ }++ @Test+ void testGetParseFormats() {+ respondWith(200, "{\"success\":true,\"data\":{\"formats\":["+ + "{\"format\":\"pdf\",\"kind\":\"document\",\"extensions\":[\".pdf\"],"+ + "\"mimeTypes\":[\"application/pdf\"],\"available\":true},"+ + "{\"format\":\"png\",\"kind\":\"image\",\"extensions\":[\".png\"],"+ + "\"mimeTypes\":[\"image/png\"],\"available\":false}"+ + "]}}");++ List<ParseFormat> formats = client.getParseFormats();++ assertEquals("GET", lastMethod.get());+ assertEquals("/v2/parse/formats", lastPath.get());+ assertEquals("Bearer fc-test-key", lastAuth.get());++ assertEquals(2, formats.size());+ ParseFormat pdf = formats.get(0);+ assertEquals("pdf", pdf.getFormat());+ assertEquals("document", pdf.getKind());+ assertEquals(ParseFormat.Kind.DOCUMENT, pdf.getKindType());+ assertEquals(List.of(".pdf"), pdf.getExtensions());+ assertEquals(List.of("application/pdf"), pdf.getMimeTypes());+ assertTrue(pdf.isAvailable());++ ParseFormat png = formats.get(1);+ assertEquals("png", png.getFormat());+ assertEquals(ParseFormat.Kind.IMAGE, png.getKindType());+ assertEquals(List.of("image/png"), png.getMimeTypes());+ assertFalse(png.isAvailable());+ }++ @Test+ void testGetParseFormatsToleratesUnknownKindAndFields() {+ respondWith(200, "{\"success\":true,\"data\":{\"formats\":["+ + "{\"format\":\"mp3\",\"kind\":\"audio\",\"extensions\":[\".mp3\"],"+ + "\"mimeTypes\":[\"audio/mpeg\"],\"available\":true,\"maxSizeMb\":50}"+ + "]},\"extra\":true}");++ List<ParseFormat> formats = client.getParseFormats();++ assertEquals(1, formats.size());+ ParseFormat mp3 = formats.get(0);+ assertEquals("mp3", mp3.getFormat());+ assertEquals("audio", mp3.getKind());+ assertEquals(ParseFormat.Kind.UNKNOWN, mp3.getKindType());+ assertEquals(List.of("audio/mpeg"), mp3.getMimeTypes());+ assertTrue(mp3.isAvailable());+ }++ @Test+ void testGetParseFormatsReturnsEmptyListWhenFormatsMissing() {+ respondWith(200, "{\"success\":true,\"data\":{}}");+ assertTrue(client.getParseFormats().isEmpty());++ respondWith(200, "{\"success\":true}");+ assertTrue(client.getParseFormats().isEmpty());+ }++ @Test+ void testParseFormatSerializesOnlyWireFields() throws Exception {+ ObjectMapper mapper = new ObjectMapper();+ ParseFormat pdf = mapper.readValue("{\"format\":\"pdf\",\"kind\":\"document\","+ + "\"extensions\":[\".pdf\"],\"mimeTypes\":[\"application/pdf\"],\"available\":true}", ParseFormat.class);++ JsonNode json = mapper.valueToTree(pdf);+ assertEquals("document", json.get("kind").asText());+ assertFalse(json.has("kindType"));+ }++ @Test+ void testGetParseFormatsUnauthorized() {+ respondWith(401, "{\"success\":false,\"error\":\"Unauthorized: Invalid token\"}");++ AuthenticationException e = assertThrows(AuthenticationException.class, () -> client.getParseFormats());+ assertEquals(401, e.getStatusCode());+ assertEquals("Unauthorized: Invalid token", e.getMessage());+ }++ @Test+ void testGetParseFormatsServerError() {+ respondWith(500, "{\"success\":false,\"error\":\"Internal server error\"}");++ FirecrawlException e = assertThrows(FirecrawlException.class, () -> client.getParseFormats());+ assertEquals(500, e.getStatusCode());+ }++ @Test+ void testGetParseFormatsAsync() throws Exception {+ respondWith(200, "{\"success\":true,\"data\":{\"formats\":["+ + "{\"format\":\"pdf\",\"kind\":\"document\",\"extensions\":[\".pdf\"],"+ + "\"mimeTypes\":[\"application/pdf\"],\"available\":true}"+ + "]}}");++ List<ParseFormat> formats = client.getParseFormatsAsync().get();++ assertEquals(1, formats.size());+ assertEquals("pdf", formats.get(0).getFormat());+ }+}