Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(elixir-sdk): add get_parse_formats for GET /v2/parse/formats (#4914)

Gergő Móricz2ffb1ab59b83
diff --git a/apps/elixir-sdk/README.md b/apps/elixir-sdk/README.md
index 2bf45feca..04814691c 100644
--- a/apps/elixir-sdk/README.md
+++ b/apps/elixir-sdk/README.md
@@ -88,6 +88,13 @@ All params are passed as keyword lists with snake_case keys. Invalid keys, missi
formats: ["markdown"]
)
+# List the file formats parse_file accepts on this deployment
+{:ok, formats} = Firecrawl.get_parse_formats()
+
+for %Firecrawl.ParseFormat{format: format, kind: kind, mime_types: mime_types, available: true} <- formats do
+ IO.puts("#{format} (#{kind}): #{Enum.join(mime_types, ", ")}")
+end
+
# Self-hosted instance
{:ok, response} = Firecrawl.scrape_and_extract_from_url(
[url: "https://example.com"],
diff --git a/apps/elixir-sdk/generate.exs b/apps/elixir-sdk/generate.exs
index 959b825b9..ea421c438 100644
--- a/apps/elixir-sdk/generate.exs
+++ b/apps/elixir-sdk/generate.exs
@@ -23,6 +23,10 @@ defmodule Firecrawl.Generator do
"getHistoricalTokenUsage"
])
+ # Routes emitted by hand-written template code instead of the spec, so a
+ # spec entry for them never generates a second, untyped definition.
+ @hand_written_routes MapSet.new([{"get", "/parse/formats"}])
+
# The method key becomes the Req function name in generated code, a position
# no escaping can protect, so anything else stops generation outright.
@http_methods ~w(get post put patch delete head options)
@@ -82,7 +86,7 @@ defmodule Firecrawl.Generator do
# Module template
# ---------------------------------------------------------------------------
- defp generate_module(spec) do
+ def generate_module(spec) do
base_url =
case get_in(spec, ["servers"]) do
[%{"url" => url} | _] -> url
@@ -107,8 +111,9 @@ defmodule Firecrawl.Generator do
{method, path, operation, path_level_params}
end)
end)
- |> Enum.reject(fn {_method, _path, op, _} ->
- MapSet.member?(@skip_operations, Map.get(op, "operationId", ""))
+ |> Enum.reject(fn {method, path, op, _} ->
+ MapSet.member?(@skip_operations, Map.get(op, "operationId", "")) or
+ MapSet.member?(@hand_written_routes, {method, path})
end)
|> Enum.sort_by(fn {_method, _path, op, _} -> Map.get(op, "operationId", "") end)
|> Enum.map(&generate_function(&1, spec))
@@ -242,10 +247,61 @@ defmodule Firecrawl.Generator do
Enum.join([first | Enum.map(rest, &String.capitalize/1)])
end
+ #{parse_formats_code()}
#{functions}end
"""
end
+ # get_parse_formats returns typed structs rather than a raw response, so it
+ # is written by hand and emitted verbatim on every regeneration.
+ def parse_formats_code do
+ ~S'''
+ @doc """
+ List the file formats `parse_file/3` accepts
+
+ `GET /parse/formats`
+
+ Tag: Parsing
+
+ ## Returns
+
+ * `{:ok, [%Firecrawl.ParseFormat{}]}` on success
+ * `{:error, exception}` on HTTP failure or an unexpected response shape
+ """
+ @spec get_parse_formats(keyword()) ::
+ {:ok, [Firecrawl.ParseFormat.t()]} | {:error, Exception.t() | Firecrawl.Error.t()}
+ def get_parse_formats(opts \\ []) do
+ with {:ok, response} <- Req.get(client(opts), url: "/parse/formats") do
+ decode_parse_formats(response)
+ end
+ end
+
+
+ @doc """
+ Bang variant of `get_parse_formats`. Raises on error.
+ """
+ @spec get_parse_formats!(keyword()) :: [Firecrawl.ParseFormat.t()]
+ def get_parse_formats!(opts \\ []) do
+ case get_parse_formats(opts) do
+ {:ok, formats} -> formats
+ {:error, exception} -> raise exception
+ end
+ end
+
+ defp decode_parse_formats(%Req.Response{body: %{"data" => %{"formats" => formats}}})
+ when is_list(formats) do
+ {:ok, Enum.map(formats, &Firecrawl.ParseFormat.from_map/1)}
+ end
+
+ defp decode_parse_formats(%Req.Response{status: status, body: body}) do
+ {:error,
+ RuntimeError.exception(
+ "unexpected GET /parse/formats response (HTTP #{status}): #{inspect(body)}"
+ )}
+ end
+ '''
+ end
+
# ---------------------------------------------------------------------------
# Per-function generation
# ---------------------------------------------------------------------------
diff --git a/apps/elixir-sdk/lib/firecrawl.ex b/apps/elixir-sdk/lib/firecrawl.ex
index 11e5485cb..39eacacce 100644
--- a/apps/elixir-sdk/lib/firecrawl.ex
+++ b/apps/elixir-sdk/lib/firecrawl.ex
@@ -124,6 +124,50 @@ defmodule Firecrawl do
Enum.join([first | Enum.map(rest, &String.capitalize/1)])
end
+ @doc """
+ List the file formats `parse_file/3` accepts
+
+ `GET /parse/formats`
+
+ Tag: Parsing
+
+ ## Returns
+
+ * `{:ok, [%Firecrawl.ParseFormat{}]}` on success
+ * `{:error, exception}` on HTTP failure or an unexpected response shape
+ """
+ @spec get_parse_formats(keyword()) ::
+ {:ok, [Firecrawl.ParseFormat.t()]} | {:error, Exception.t() | Firecrawl.Error.t()}
+ def get_parse_formats(opts \\ []) do
+ with {:ok, response} <- Req.get(client(opts), url: "/parse/formats") do
+ decode_parse_formats(response)
+ end
+ end
+
+
+ @doc """
+ Bang variant of `get_parse_formats`. Raises on error.
+ """
+ @spec get_parse_formats!(keyword()) :: [Firecrawl.ParseFormat.t()]
+ def get_parse_formats!(opts \\ []) do
+ case get_parse_formats(opts) do
+ {:ok, formats} -> formats
+ {:error, exception} -> raise exception
+ end
+ end
+
+ defp decode_parse_formats(%Req.Response{body: %{"data" => %{"formats" => formats}}})
+ when is_list(formats) do
+ {:ok, Enum.map(formats, &Firecrawl.ParseFormat.from_map/1)}
+ end
+
+ defp decode_parse_formats(%Req.Response{status: status, body: body}) do
+ {:error,
+ RuntimeError.exception(
+ "unexpected GET /parse/formats response (HTTP #{status}): #{inspect(body)}"
+ )}
+ end
+
defp fetch_file_field(file, key) do
case Keyword.fetch(file, key) do
{:ok, _value} = ok -> ok
diff --git a/apps/elixir-sdk/lib/firecrawl/parse_format.ex b/apps/elixir-sdk/lib/firecrawl/parse_format.ex
new file mode 100644
index 000000000..f06872e40
--- /dev/null
+++ b/apps/elixir-sdk/lib/firecrawl/parse_format.ex
@@ -0,0 +1,42 @@
+defmodule Firecrawl.ParseFormat do
+ @moduledoc """
+ A file format accepted by `Firecrawl.parse_file/3`, as returned by
+ `Firecrawl.get_parse_formats/1`.
+
+ ## Fields
+
+ * `:format` - Format identifier, e.g. `"pdf"`
+ * `:kind` - `:document` or `:image`; any other value the API sends is kept as a string
+ * `:extensions` - File extensions, e.g. `[".pdf"]`
+ * `:mime_types` - MIME types, e.g. `["application/pdf"]`
+ * `:available` - Whether this deployment can parse the format right now
+ """
+
+ defstruct [:format, :kind, extensions: [], mime_types: [], available: false]
+
+ @type kind :: :document | :image | String.t()
+
+ @type t :: %__MODULE__{
+ format: String.t(),
+ kind: kind(),
+ extensions: [String.t()],
+ mime_types: [String.t()],
+ available: boolean()
+ }
+
+ @doc false
+ @spec from_map(map()) :: t()
+ def from_map(map) when is_map(map) do
+ %__MODULE__{
+ format: map["format"] || "",
+ kind: to_kind(map["kind"] || ""),
+ extensions: map["extensions"] || [],
+ mime_types: map["mimeTypes"] || [],
+ available: map["available"] == true
+ }
+ end
+
+ defp to_kind("document"), do: :document
+ defp to_kind("image"), do: :image
+ defp to_kind(other), do: other
+end
diff --git a/apps/elixir-sdk/mix.exs b/apps/elixir-sdk/mix.exs
index 75b4bbf30..9cdab7394 100644
--- a/apps/elixir-sdk/mix.exs
+++ b/apps/elixir-sdk/mix.exs
@@ -1,7 +1,7 @@
defmodule Firecrawl.MixProject do
use Mix.Project
- @version "1.11.1"
+ @version "1.11.2"
@source_url "https://github.com/firecrawl/firecrawl/tree/main/apps/elixir-sdk"
def project do
diff --git a/apps/elixir-sdk/test/firecrawl_test.exs b/apps/elixir-sdk/test/firecrawl_test.exs
index 73b7add3a..da6fb09f2 100644
--- a/apps/elixir-sdk/test/firecrawl_test.exs
+++ b/apps/elixir-sdk/test/firecrawl_test.exs
@@ -676,6 +676,139 @@ defmodule FirecrawlTest do
assert body["highlights"] == false
end
+ defp parse_formats_adapter(parent, status, body) do
+ fn request ->
+ send(parent, {:request, request})
+
+ resp =
+ Req.Response.new(
+ status: status,
+ headers: %{"content-type" => ["application/json"]},
+ body: Jason.encode!(body)
+ )
+
+ {request, resp}
+ end
+ end
+
+ test "get_parse_formats sends GET /v2/parse/formats and returns typed formats" do
+ body = %{
+ "success" => true,
+ "data" => %{
+ "formats" => [
+ %{
+ "format" => "pdf",
+ "kind" => "document",
+ "extensions" => [".pdf"],
+ "mimeTypes" => ["application/pdf"],
+ "available" => true
+ },
+ %{
+ "format" => "png",
+ "kind" => "image",
+ "extensions" => [".png"],
+ "mimeTypes" => ["image/png"],
+ "available" => false
+ }
+ ]
+ }
+ }
+
+ assert {:ok, [pdf, png]} =
+ Firecrawl.get_parse_formats(
+ api_key: "test-key",
+ adapter: parse_formats_adapter(self(), 200, body)
+ )
+
+ assert_receive {:request, request}
+ assert request.method == :get
+ assert URI.to_string(request.url) == "https://api.firecrawl.dev/v2/parse/formats"
+ assert request.headers["authorization"] == ["Bearer test-key"]
+
+ assert pdf == %Firecrawl.ParseFormat{
+ format: "pdf",
+ kind: :document,
+ extensions: [".pdf"],
+ mime_types: ["application/pdf"],
+ available: true
+ }
+
+ assert png == %Firecrawl.ParseFormat{
+ format: "png",
+ kind: :image,
+ extensions: [".png"],
+ mime_types: ["image/png"],
+ available: false
+ }
+ end
+
+ test "get_parse_formats keeps unknown kinds as strings and ignores unknown fields" do
+ body = %{
+ "success" => true,
+ "data" => %{
+ "formats" => [
+ %{
+ "format" => "glb",
+ "kind" => "model",
+ "extensions" => [".glb"],
+ "mimeTypes" => ["model/gltf-binary"],
+ "available" => true,
+ "maxSizeBytes" => 1024
+ }
+ ]
+ }
+ }
+
+ formats =
+ Firecrawl.get_parse_formats!(
+ api_key: "test-key",
+ adapter: parse_formats_adapter(self(), 200, body)
+ )
+
+ assert [%Firecrawl.ParseFormat{format: "glb", kind: "model", mime_types: ["model/gltf-binary"]}] =
+ formats
+ end
+
+ test "get_parse_formats returns {:error, %Firecrawl.Error{}} on API errors" do
+ adapter =
+ parse_formats_adapter(self(), 401, %{"success" => false, "error" => "Unauthorized"})
+
+ assert {:error, %Firecrawl.Error{status: 401}} =
+ Firecrawl.get_parse_formats(api_key: "bad-key", adapter: adapter, retry: false)
+
+ assert_raise Firecrawl.Error, ~r/Unauthorized/, fn ->
+ Firecrawl.get_parse_formats!(api_key: "bad-key", adapter: adapter, retry: false)
+ end
+ end
+
+ test "get_parse_formats returns a non-API error for an unexpected success body" do
+ adapter = parse_formats_adapter(self(), 200, %{"success" => true, "data" => %{}})
+
+ assert {:error, %RuntimeError{message: msg}} =
+ Firecrawl.get_parse_formats(api_key: "test-key", adapter: adapter)
+
+ assert msg =~ "unexpected GET /parse/formats response (HTTP 200)"
+ end
+
+ test "get_parse_formats fills missing format and kind with empty strings" do
+ body = %{"success" => true, "data" => %{"formats" => [%{"extensions" => [".x"]}]}}
+
+ assert {:ok, [%Firecrawl.ParseFormat{format: "", kind: "", extensions: [".x"], available: false}]} =
+ Firecrawl.get_parse_formats(
+ api_key: "test-key",
+ adapter: parse_formats_adapter(self(), 200, body)
+ )
+ end
+
+ test "get_parse_formats! raises Firecrawl.Error on server errors" do
+ adapter =
+ parse_formats_adapter(self(), 500, %{"success" => false, "error" => "Internal error"})
+
+ assert_raise Firecrawl.Error, ~r/HTTP 500/, fn ->
+ Firecrawl.get_parse_formats!(api_key: "test-key", adapter: adapter, retry: false)
+ end
+ end
+
test "all expected API functions are defined with bang variants" do
functions = Firecrawl.__info__(:functions)
@@ -709,7 +842,11 @@ defmodule FirecrawlTest do
{:list_agents, 2},
{:list_agents!, 0},
{:list_agents!, 1},
- {:list_agents!, 2}
+ {:list_agents!, 2},
+ {:get_parse_formats, 0},
+ {:get_parse_formats, 1},
+ {:get_parse_formats!, 0},
+ {:get_parse_formats!, 1}
]
for {name, arity} <- expected do
diff --git a/apps/elixir-sdk/test/generator_test.exs b/apps/elixir-sdk/test/generator_test.exs
index e6018d8bd..5eefdca99 100644
--- a/apps/elixir-sdk/test/generator_test.exs
+++ b/apps/elixir-sdk/test/generator_test.exs
@@ -159,4 +159,28 @@ defmodule Firecrawl.GeneratorTest do
assert function_exported?(mod, :f, 0)
end
end
+
+ describe "get_parse_formats" do
+ test "is emitted once even when the spec declares GET /parse/formats" do
+ spec = %{
+ "paths" => %{
+ "/parse/formats" => %{
+ "get" => %{"operationId" => "getParseFormats", "summary" => "List formats"}
+ }
+ }
+ }
+
+ code = Firecrawl.Generator.generate_module(spec)
+
+ assert length(Regex.scan(~r/^ def get_parse_formats\(/m, code)) == 1
+ assert length(Regex.scan(~r/^ def get_parse_formats!\(/m, code)) == 1
+ end
+
+ test "lib/firecrawl.ex carries the generator's hand-written block verbatim" do
+ lib = File.read!(Path.expand("../lib/firecrawl.ex", __DIR__))
+ block = Firecrawl.Generator.parse_formats_code()
+
+ assert String.contains?(lib, block)
+ end
+ end
end