Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(elixir-sdk): add get_parse_formats for GET /v2/parse/formats (#4914)
Gergő Móricz2ffb1ab59b83
diff --git a/apps/elixir-sdk/README.md b/apps/elixir-sdk/README.mdindex 2bf45feca..04814691c 100644--- a/apps/elixir-sdk/README.md+++ b/apps/elixir-sdk/README.md@@ -88,6 +88,13 @@ All params are passed as keyword lists with snake_case keys. Invalid keys, missiformats: ["markdown"])+# List the file formats parse_file accepts on this deployment+{:ok, formats} = Firecrawl.get_parse_formats()++for %Firecrawl.ParseFormat{format: format, kind: kind, mime_types: mime_types, available: true} <- formats do+ IO.puts("#{format} (#{kind}): #{Enum.join(mime_types, ", ")}")+end+# Self-hosted instance{:ok, response} = Firecrawl.scrape_and_extract_from_url([url: "https://example.com"],diff --git a/apps/elixir-sdk/generate.exs b/apps/elixir-sdk/generate.exsindex 959b825b9..ea421c438 100644--- a/apps/elixir-sdk/generate.exs+++ b/apps/elixir-sdk/generate.exs@@ -23,6 +23,10 @@ defmodule Firecrawl.Generator do"getHistoricalTokenUsage"])+ # Routes emitted by hand-written template code instead of the spec, so a+ # spec entry for them never generates a second, untyped definition.+ @hand_written_routes MapSet.new([{"get", "/parse/formats"}])+# The method key becomes the Req function name in generated code, a position# no escaping can protect, so anything else stops generation outright.@http_methods ~w(get post put patch delete head options)@@ -82,7 +86,7 @@ defmodule Firecrawl.Generator do# Module template# ---------------------------------------------------------------------------- defp generate_module(spec) do+ def generate_module(spec) dobase_url =case get_in(spec, ["servers"]) do[%{"url" => url} | _] -> url@@ -107,8 +111,9 @@ defmodule Firecrawl.Generator do{method, path, operation, path_level_params}end)end)- |> Enum.reject(fn {_method, _path, op, _} ->- MapSet.member?(@skip_operations, Map.get(op, "operationId", ""))+ |> Enum.reject(fn {method, path, op, _} ->+ MapSet.member?(@skip_operations, Map.get(op, "operationId", "")) or+ MapSet.member?(@hand_written_routes, {method, path})end)|> Enum.sort_by(fn {_method, _path, op, _} -> Map.get(op, "operationId", "") end)|> Enum.map(&generate_function(&1, spec))@@ -242,10 +247,61 @@ defmodule Firecrawl.Generator doEnum.join([first | Enum.map(rest, &String.capitalize/1)])end+ #{parse_formats_code()}#{functions}end"""end+ # get_parse_formats returns typed structs rather than a raw response, so it+ # is written by hand and emitted verbatim on every regeneration.+ def parse_formats_code do+ ~S'''+ @doc """+ List the file formats `parse_file/3` accepts++ `GET /parse/formats`++ Tag: Parsing++ ## Returns++ * `{:ok, [%Firecrawl.ParseFormat{}]}` on success+ * `{:error, exception}` on HTTP failure or an unexpected response shape+ """+ @spec get_parse_formats(keyword()) ::+ {:ok, [Firecrawl.ParseFormat.t()]} | {:error, Exception.t() | Firecrawl.Error.t()}+ def get_parse_formats(opts \\ []) do+ with {:ok, response} <- Req.get(client(opts), url: "/parse/formats") do+ decode_parse_formats(response)+ end+ end+++ @doc """+ Bang variant of `get_parse_formats`. Raises on error.+ """+ @spec get_parse_formats!(keyword()) :: [Firecrawl.ParseFormat.t()]+ def get_parse_formats!(opts \\ []) do+ case get_parse_formats(opts) do+ {:ok, formats} -> formats+ {:error, exception} -> raise exception+ end+ end++ defp decode_parse_formats(%Req.Response{body: %{"data" => %{"formats" => formats}}})+ when is_list(formats) do+ {:ok, Enum.map(formats, &Firecrawl.ParseFormat.from_map/1)}+ end++ defp decode_parse_formats(%Req.Response{status: status, body: body}) do+ {:error,+ RuntimeError.exception(+ "unexpected GET /parse/formats response (HTTP #{status}): #{inspect(body)}"+ )}+ end+ '''+ end+# ---------------------------------------------------------------------------# Per-function generation# ---------------------------------------------------------------------------diff --git a/apps/elixir-sdk/lib/firecrawl.ex b/apps/elixir-sdk/lib/firecrawl.exindex 11e5485cb..39eacacce 100644--- a/apps/elixir-sdk/lib/firecrawl.ex+++ b/apps/elixir-sdk/lib/firecrawl.ex@@ -124,6 +124,50 @@ defmodule Firecrawl doEnum.join([first | Enum.map(rest, &String.capitalize/1)])end+ @doc """+ List the file formats `parse_file/3` accepts++ `GET /parse/formats`++ Tag: Parsing++ ## Returns++ * `{:ok, [%Firecrawl.ParseFormat{}]}` on success+ * `{:error, exception}` on HTTP failure or an unexpected response shape+ """+ @spec get_parse_formats(keyword()) ::+ {:ok, [Firecrawl.ParseFormat.t()]} | {:error, Exception.t() | Firecrawl.Error.t()}+ def get_parse_formats(opts \\ []) do+ with {:ok, response} <- Req.get(client(opts), url: "/parse/formats") do+ decode_parse_formats(response)+ end+ end+++ @doc """+ Bang variant of `get_parse_formats`. Raises on error.+ """+ @spec get_parse_formats!(keyword()) :: [Firecrawl.ParseFormat.t()]+ def get_parse_formats!(opts \\ []) do+ case get_parse_formats(opts) do+ {:ok, formats} -> formats+ {:error, exception} -> raise exception+ end+ end++ defp decode_parse_formats(%Req.Response{body: %{"data" => %{"formats" => formats}}})+ when is_list(formats) do+ {:ok, Enum.map(formats, &Firecrawl.ParseFormat.from_map/1)}+ end++ defp decode_parse_formats(%Req.Response{status: status, body: body}) do+ {:error,+ RuntimeError.exception(+ "unexpected GET /parse/formats response (HTTP #{status}): #{inspect(body)}"+ )}+ end+defp fetch_file_field(file, key) docase Keyword.fetch(file, key) do{:ok, _value} = ok -> okdiff --git a/apps/elixir-sdk/lib/firecrawl/parse_format.ex b/apps/elixir-sdk/lib/firecrawl/parse_format.exnew file mode 100644index 000000000..f06872e40--- /dev/null+++ b/apps/elixir-sdk/lib/firecrawl/parse_format.ex@@ -0,0 +1,42 @@+defmodule Firecrawl.ParseFormat do+ @moduledoc """+ A file format accepted by `Firecrawl.parse_file/3`, as returned by+ `Firecrawl.get_parse_formats/1`.++ ## Fields++ * `:format` - Format identifier, e.g. `"pdf"`+ * `:kind` - `:document` or `:image`; any other value the API sends is kept as a string+ * `:extensions` - File extensions, e.g. `[".pdf"]`+ * `:mime_types` - MIME types, e.g. `["application/pdf"]`+ * `:available` - Whether this deployment can parse the format right now+ """++ defstruct [:format, :kind, extensions: [], mime_types: [], available: false]++ @type kind :: :document | :image | String.t()++ @type t :: %__MODULE__{+ format: String.t(),+ kind: kind(),+ extensions: [String.t()],+ mime_types: [String.t()],+ available: boolean()+ }++ @doc false+ @spec from_map(map()) :: t()+ def from_map(map) when is_map(map) do+ %__MODULE__{+ format: map["format"] || "",+ kind: to_kind(map["kind"] || ""),+ extensions: map["extensions"] || [],+ mime_types: map["mimeTypes"] || [],+ available: map["available"] == true+ }+ end++ defp to_kind("document"), do: :document+ defp to_kind("image"), do: :image+ defp to_kind(other), do: other+enddiff --git a/apps/elixir-sdk/mix.exs b/apps/elixir-sdk/mix.exsindex 75b4bbf30..9cdab7394 100644--- a/apps/elixir-sdk/mix.exs+++ b/apps/elixir-sdk/mix.exs@@ -1,7 +1,7 @@defmodule Firecrawl.MixProject douse Mix.Project- @version "1.11.1"+ @version "1.11.2"@source_url "https://github.com/firecrawl/firecrawl/tree/main/apps/elixir-sdk"def project dodiff --git a/apps/elixir-sdk/test/firecrawl_test.exs b/apps/elixir-sdk/test/firecrawl_test.exsindex 73b7add3a..da6fb09f2 100644--- a/apps/elixir-sdk/test/firecrawl_test.exs+++ b/apps/elixir-sdk/test/firecrawl_test.exs@@ -676,6 +676,139 @@ defmodule FirecrawlTest doassert body["highlights"] == falseend+ defp parse_formats_adapter(parent, status, body) do+ fn request ->+ send(parent, {:request, request})++ resp =+ Req.Response.new(+ status: status,+ headers: %{"content-type" => ["application/json"]},+ body: Jason.encode!(body)+ )++ {request, resp}+ end+ end++ test "get_parse_formats sends GET /v2/parse/formats and returns typed formats" do+ body = %{+ "success" => true,+ "data" => %{+ "formats" => [+ %{+ "format" => "pdf",+ "kind" => "document",+ "extensions" => [".pdf"],+ "mimeTypes" => ["application/pdf"],+ "available" => true+ },+ %{+ "format" => "png",+ "kind" => "image",+ "extensions" => [".png"],+ "mimeTypes" => ["image/png"],+ "available" => false+ }+ ]+ }+ }++ assert {:ok, [pdf, png]} =+ Firecrawl.get_parse_formats(+ api_key: "test-key",+ adapter: parse_formats_adapter(self(), 200, body)+ )++ assert_receive {:request, request}+ assert request.method == :get+ assert URI.to_string(request.url) == "https://api.firecrawl.dev/v2/parse/formats"+ assert request.headers["authorization"] == ["Bearer test-key"]++ assert pdf == %Firecrawl.ParseFormat{+ format: "pdf",+ kind: :document,+ extensions: [".pdf"],+ mime_types: ["application/pdf"],+ available: true+ }++ assert png == %Firecrawl.ParseFormat{+ format: "png",+ kind: :image,+ extensions: [".png"],+ mime_types: ["image/png"],+ available: false+ }+ end++ test "get_parse_formats keeps unknown kinds as strings and ignores unknown fields" do+ body = %{+ "success" => true,+ "data" => %{+ "formats" => [+ %{+ "format" => "glb",+ "kind" => "model",+ "extensions" => [".glb"],+ "mimeTypes" => ["model/gltf-binary"],+ "available" => true,+ "maxSizeBytes" => 1024+ }+ ]+ }+ }++ formats =+ Firecrawl.get_parse_formats!(+ api_key: "test-key",+ adapter: parse_formats_adapter(self(), 200, body)+ )++ assert [%Firecrawl.ParseFormat{format: "glb", kind: "model", mime_types: ["model/gltf-binary"]}] =+ formats+ end++ test "get_parse_formats returns {:error, %Firecrawl.Error{}} on API errors" do+ adapter =+ parse_formats_adapter(self(), 401, %{"success" => false, "error" => "Unauthorized"})++ assert {:error, %Firecrawl.Error{status: 401}} =+ Firecrawl.get_parse_formats(api_key: "bad-key", adapter: adapter, retry: false)++ assert_raise Firecrawl.Error, ~r/Unauthorized/, fn ->+ Firecrawl.get_parse_formats!(api_key: "bad-key", adapter: adapter, retry: false)+ end+ end++ test "get_parse_formats returns a non-API error for an unexpected success body" do+ adapter = parse_formats_adapter(self(), 200, %{"success" => true, "data" => %{}})++ assert {:error, %RuntimeError{message: msg}} =+ Firecrawl.get_parse_formats(api_key: "test-key", adapter: adapter)++ assert msg =~ "unexpected GET /parse/formats response (HTTP 200)"+ end++ test "get_parse_formats fills missing format and kind with empty strings" do+ body = %{"success" => true, "data" => %{"formats" => [%{"extensions" => [".x"]}]}}++ assert {:ok, [%Firecrawl.ParseFormat{format: "", kind: "", extensions: [".x"], available: false}]} =+ Firecrawl.get_parse_formats(+ api_key: "test-key",+ adapter: parse_formats_adapter(self(), 200, body)+ )+ end++ test "get_parse_formats! raises Firecrawl.Error on server errors" do+ adapter =+ parse_formats_adapter(self(), 500, %{"success" => false, "error" => "Internal error"})++ assert_raise Firecrawl.Error, ~r/HTTP 500/, fn ->+ Firecrawl.get_parse_formats!(api_key: "test-key", adapter: adapter, retry: false)+ end+ end+test "all expected API functions are defined with bang variants" dofunctions = Firecrawl.__info__(:functions)@@ -709,7 +842,11 @@ defmodule FirecrawlTest do{:list_agents, 2},{:list_agents!, 0},{:list_agents!, 1},- {:list_agents!, 2}+ {:list_agents!, 2},+ {:get_parse_formats, 0},+ {:get_parse_formats, 1},+ {:get_parse_formats!, 0},+ {:get_parse_formats!, 1}]for {name, arity} <- expected dodiff --git a/apps/elixir-sdk/test/generator_test.exs b/apps/elixir-sdk/test/generator_test.exsindex e6018d8bd..5eefdca99 100644--- a/apps/elixir-sdk/test/generator_test.exs+++ b/apps/elixir-sdk/test/generator_test.exs@@ -159,4 +159,28 @@ defmodule Firecrawl.GeneratorTest doassert function_exported?(mod, :f, 0)endend++ describe "get_parse_formats" do+ test "is emitted once even when the spec declares GET /parse/formats" do+ spec = %{+ "paths" => %{+ "/parse/formats" => %{+ "get" => %{"operationId" => "getParseFormats", "summary" => "List formats"}+ }+ }+ }++ code = Firecrawl.Generator.generate_module(spec)++ assert length(Regex.scan(~r/^ def get_parse_formats\(/m, code)) == 1+ assert length(Regex.scan(~r/^ def get_parse_formats!\(/m, code)) == 1+ end++ test "lib/firecrawl.ex carries the generator's hand-written block verbatim" do+ lib = File.read!(Path.expand("../lib/firecrawl.ex", __DIR__))+ block = Firecrawl.Generator.parse_formats_code()++ assert String.contains?(lib, block)+ end+ endend