Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(elixir-sdk): add parse_file for /parse endpoint
firecrawl-spring[bot]92cbe08c89da
diff --git a/apps/elixir-sdk/README.md b/apps/elixir-sdk/README.mdindex db1b8127e..ec3efb68c 100644--- a/apps/elixir-sdk/README.md+++ b/apps/elixir-sdk/README.md@@ -59,6 +59,12 @@ All params are passed as keyword lists with snake_case keys. Invalid keys, missi# Check crawl status{:ok, response} = Firecrawl.get_crawl_status("job-uuid")+# Parse a file (PDF, DOCX, HTML, etc.)+{:ok, response} = Firecrawl.parse_file(+ [filename: "report.pdf", data: File.read!("report.pdf"), content_type: "application/pdf"],+ formats: ["markdown"]+)+# Self-hosted instance{:ok, response} = Firecrawl.scrape_and_extract_from_url([url: "https://example.com"],diff --git a/apps/elixir-sdk/generate.exs b/apps/elixir-sdk/generate.exsindex 7c97f522c..e0327912a 100644--- a/apps/elixir-sdk/generate.exs+++ b/apps/elixir-sdk/generate.exs@@ -263,6 +263,38 @@ defmodule Firecrawl.Generator dohas_body = Map.has_key?(operation, "requestBody")http_method = String.upcase(method)+ if has_body and multipart?(operation) do+ generate_multipart_function(method, path, operation, spec, func_name, summary, tag)+ else+ generate_json_function(+ method,+ path,+ operation,+ spec,+ func_name,+ summary,+ tag,+ path_params,+ query_params,+ has_body,+ http_method+ )+ end+ end++ defp generate_json_function(+ method,+ path,+ operation,+ spec,+ func_name,+ summary,+ tag,+ path_params,+ query_params,+ has_body,+ http_method+ ) do# Extract request body schemabody_properties = if has_body, do: extract_body_properties(operation, spec), else: []required_keys = if has_body, do: extract_required_keys(operation, spec), else: []@@ -348,6 +380,227 @@ defmodule Firecrawl.Generator doparts |> Enum.reject(&is_nil/1) |> Enum.join("\n")end+ # ---------------------------------------------------------------------------+ # Multipart Support+ # ---------------------------------------------------------------------------++ defp multipart?(operation) do+ content = get_in(operation, ["requestBody", "content"]) || %{}+ Map.has_key?(content, "multipart/form-data")+ end++ defp extract_multipart_meta(operation, spec) do+ schema = get_in(operation, ["requestBody", "content", "multipart/form-data", "schema"]) || %{}+ props = resolve_properties(schema, spec)++ {file_props, other_props} =+ Enum.split_with(props, fn {_name, ps} ->+ Map.get(ps, "format") == "binary"+ end)++ file_field =+ case file_props do+ [{name, _} | _] -> name+ _ -> nil+ end++ {options_field, options_props, options_required} =+ case Enum.find(other_props, fn {_n, ps} ->+ Map.has_key?(ps, "properties") or Map.has_key?(ps, "$ref") or+ Map.has_key?(ps, "allOf")+ end) do+ {name, ps} ->+ inner_props = resolve_properties(ps, spec)+ inner_required = resolve_required(ps, spec)+ {name, inner_props, inner_required}++ nil ->+ {nil, [], []}+ end++ %{+ file_field: file_field,+ options_field: options_field,+ options_props: options_props,+ options_required: options_required+ }+ end++ defp generate_multipart_function(method, path, operation, spec, func_name, summary, tag) do+ meta = extract_multipart_meta(operation, spec)+ http_method = String.upcase(method)+ has_options? = meta.options_props != []++ body_schema_code =+ if has_options?, do: generate_schema(func_name, meta.options_props, meta.options_required), else: nil++ body_key_mapping_code =+ if has_options?, do: generate_key_mapping(func_name, meta.options_props), else: nil++ doc = build_multipart_doc(summary, http_method, path, tag, func_name, has_options?, meta)++ {sig, body} = build_multipart_function_body(func_name, method, path, meta, has_options?, false)+ {bang_sig, bang_body} = build_multipart_function_body(func_name, method, path, meta, has_options?, true)++ spec_code = build_multipart_typespec(func_name, has_options?, false)+ bang_spec_code = build_multipart_typespec(func_name, has_options?, true)++ parts = [+ body_schema_code,+ body_key_mapping_code,+ doc,+ spec_code,+ " #{sig}",+ body,+ "",+ doc_bang(func_name),+ bang_spec_code,+ " #{bang_sig}",+ bang_body,+ ""+ ]++ parts |> Enum.reject(&is_nil/1) |> Enum.join("\n")+ end++ defp build_multipart_function_body(func_name, method, path, meta, has_options?, bang?) do+ req_method = String.to_atom(method)+ fn_name = if bang?, do: "#{func_name}!", else: func_name+ req_fn = if bang?, do: "#{req_method}!", else: "#{req_method}"+ options_field = meta.options_field+ file_field = meta.file_field++ sig =+ if has_options? do+ "def #{fn_name}(file, params \\\\ [], opts \\\\ []) do"+ else+ "def #{fn_name}(file, opts \\\\ []) do"+ end++ options_part_text =+ cond do+ has_options? and not is_nil(options_field) ->+ "{\"#{options_field}\", Jason.encode!(to_body(params, @#{func_name}_key_mapping))}, "++ true ->+ ""+ end++ file_part_text = "{\"#{file_field}\", file_part}"++ indent = if not bang? and has_options?, do: " ", else: " "++ core_lines = [+ "#{indent}filename = Keyword.fetch!(file, :filename)",+ "#{indent}data = Keyword.fetch!(file, :data)",+ "#{indent}content_type = Keyword.get(file, :content_type)",+ "",+ "#{indent}if not is_binary(filename) or filename == \"\" do",+ "#{indent} raise ArgumentError, \"filename cannot be empty\"",+ "#{indent}end",+ "",+ "#{indent}if is_nil(data) do",+ "#{indent} raise ArgumentError, \"file data cannot be empty\"",+ "#{indent}end",+ "",+ "#{indent}file_part =",+ "#{indent} case content_type do",+ "#{indent} nil -> {data, filename: filename}",+ "#{indent} ct -> {data, filename: filename, content_type: ct}",+ "#{indent} end",+ "",+ "#{indent}multipart = [#{options_part_text}#{file_part_text}]",+ "",+ "#{indent}Req.#{req_fn}(client(opts), url: \"#{path}\", form_multipart: multipart)"+ ]++ core = Enum.join(core_lines, "\n")++ body =+ cond do+ bang? and has_options? ->+ " params = NimbleOptions.validate!(params, @#{func_name}_schema)\n#{core}\n end\n"++ not bang? and has_options? ->+ " with {:ok, params} <- NimbleOptions.validate(params, @#{func_name}_schema) do\n#{core}\n end\n end\n"++ true ->+ "#{core}\n end\n"+ end++ {sig, body}+ end++ defp build_multipart_typespec(func_name, has_options?, bang?) do+ name = if bang?, do: "#{func_name}!", else: func_name+ return_type = if bang?, do: "Req.Response.t()", else: "response()"++ args =+ if has_options? do+ "keyword(), keyword(), keyword()"+ else+ "keyword(), keyword()"+ end++ " @spec #{name}(#{args}) :: #{return_type}"+ end++ defp build_multipart_doc(summary, http_method, path, tag, func_name, has_options?, meta) do+ bt = <<96>>++ parts = [+ " @doc \"\"\"",+ " #{summary}",+ "",+ " #{bt}#{http_method} #{path}#{bt}",+ "",+ " Sends a #{bt}multipart/form-data#{bt} request."+ ]++ parts = if tag != "", do: parts ++ ["", " Tag: #{tag}"], else: parts++ parts =+ parts +++ [+ "",+ " ## File",+ "",+ " Pass #{bt}file#{bt} as a keyword list:",+ "",+ " * #{bt}:filename#{bt} (required) - The filename to send.",+ " * #{bt}:data#{bt} (required) - The file contents as a binary.",+ " * #{bt}:content_type#{bt} (optional) - The MIME type of the file."+ ]++ parts =+ if has_options? do+ parts +++ [+ "",+ " ## Parameters",+ "",+ " Validated by #{bt}NimbleOptions#{bt}. Pass options as a keyword list with snake_case keys.",+ " These are JSON-encoded and sent as the #{bt}#{meta.options_field}#{bt} multipart field.",+ " See #{bt}@#{func_name}_schema#{bt} for the full schema."+ ]+ else+ parts+ end++ parts =+ parts +++ [+ "",+ " ## Returns",+ "",+ " * #{bt}{:ok, %Req.Response{}}#{bt} on success",+ " * #{bt}{:error, exception}#{bt} on HTTP or validation failure",+ " \"\"\""+ ]++ Enum.join(parts, "\n")+ end+# ---------------------------------------------------------------------------# Schema Extraction# ---------------------------------------------------------------------------diff --git a/apps/elixir-sdk/lib/firecrawl.ex b/apps/elixir-sdk/lib/firecrawl.exindex d974d853b..b38318ce7 100644--- a/apps/elixir-sdk/lib/firecrawl.ex+++ b/apps/elixir-sdk/lib/firecrawl.ex@@ -901,6 +901,111 @@ defmodule Firecrawl doend+ @parse_file_schema NimbleOptions.new!([+ block_ads: [type: :boolean, doc: "Enable ad and cookie popup blocking."],+ exclude_tags: [type: {:list, :string}, doc: "Tags to exclude from the output."],+ formats: [type: {:list, :any}, doc: "Output formats supported for `/parse` uploads. Browser-rendering formats and change tracking are not supported."],+ headers: [type: :any, doc: "Headers to send when additional network requests are required."],+ include_tags: [type: {:list, :string}, doc: "Tags to include in the output."],+ integration: [type: :string, doc: "Optional integration identifier."],+ only_main_content: [type: :boolean, doc: "Only return the main content of the page excluding headers, navs, footers, etc."],+ origin: [type: :string, doc: "Origin identifier for analytics and logging."],+ parsers: [type: {:list, :any}, doc: "Controls file parser behavior when relevant (for example PDF parser mode)."],+ proxy: [type: {:or, [{:in, [:basic, :auto]}, :string]}, doc: "Proxy mode for parse uploads. `/parse` supports only `basic` and `auto`."],+ remove_base64_images: [type: :boolean, doc: "Remove base64-encoded images from output and keep alt text placeholders."],+ skip_tls_verification: [type: :boolean, doc: "Skip TLS certificate verification when making requests."],+ timeout: [type: :integer, doc: "Timeout in milliseconds for the request. Default is 30000 (30 seconds). Maximum is 300000 (300 seconds)."],+ zero_data_retention: [type: :boolean, doc: "If true, this will enable zero data retention for this parse. To enable this feature, please contact [email protected]"]+ ])++ @parse_file_key_mapping %{block_ads: "blockAds", exclude_tags: "excludeTags", formats: "formats", headers: "headers", include_tags: "includeTags", integration: "integration", only_main_content: "onlyMainContent", origin: "origin", parsers: "parsers", proxy: "proxy", remove_base64_images: "removeBase64Images", skip_tls_verification: "skipTlsVerification", timeout: "timeout", zero_data_retention: "zeroDataRetention"}++ @doc """+ Upload and parse a file++ `POST /parse`++ Sends a `multipart/form-data` request.++ Tag: Scraping++ ## File++ Pass `file` as a keyword list:++ * `:filename` (required) - The filename to send.+ * `:data` (required) - The file contents as a binary.+ * `:content_type` (optional) - The MIME type of the file.++ ## Parameters++ Validated by `NimbleOptions`. Pass options as a keyword list with snake_case keys.+ These are JSON-encoded and sent as the `options` multipart field.+ See `@parse_file_schema` for the full schema.++ ## Returns++ * `{:ok, %Req.Response{}}` on success+ * `{:error, exception}` on HTTP or validation failure+ """+ @spec parse_file(keyword(), keyword(), keyword()) :: response()+ def parse_file(file, params \\ [], opts \\ []) do+ with {:ok, params} <- NimbleOptions.validate(params, @parse_file_schema) do+ filename = Keyword.fetch!(file, :filename)+ data = Keyword.fetch!(file, :data)+ content_type = Keyword.get(file, :content_type)++ if not is_binary(filename) or filename == "" do+ raise ArgumentError, "filename cannot be empty"+ end++ if is_nil(data) do+ raise ArgumentError, "file data cannot be empty"+ end++ file_part =+ case content_type do+ nil -> {data, filename: filename}+ ct -> {data, filename: filename, content_type: ct}+ end++ multipart = [{"options", Jason.encode!(to_body(params, @parse_file_key_mapping))}, {"file", file_part}]++ Req.post(client(opts), url: "/parse", form_multipart: multipart)+ end+ end+++ @doc """+ Bang variant of `parse_file`. Raises on error.+ """+ @spec parse_file!(keyword(), keyword(), keyword()) :: Req.Response.t()+ def parse_file!(file, params \\ [], opts \\ []) do+ params = NimbleOptions.validate!(params, @parse_file_schema)+ filename = Keyword.fetch!(file, :filename)+ data = Keyword.fetch!(file, :data)+ content_type = Keyword.get(file, :content_type)++ if not is_binary(filename) or filename == "" do+ raise ArgumentError, "filename cannot be empty"+ end++ if is_nil(data) do+ raise ArgumentError, "file data cannot be empty"+ end++ file_part =+ case content_type do+ nil -> {data, filename: filename}+ ct -> {data, filename: filename, content_type: ct}+ end++ multipart = [{"options", Jason.encode!(to_body(params, @parse_file_key_mapping))}, {"file", file_part}]++ Req.post!(client(opts), url: "/parse", form_multipart: multipart)+ end++@scrape_and_extract_from_url_schema NimbleOptions.new!([url: [type: :string, required: true, doc: "The URL to scrape"],actions: [type: {:list, :any}, doc: "Actions to perform on the page before grabbing the content"],diff --git a/apps/elixir-sdk/mix.exs b/apps/elixir-sdk/mix.exsindex 479cf2217..e15468af2 100644--- a/apps/elixir-sdk/mix.exs+++ b/apps/elixir-sdk/mix.exs@@ -1,7 +1,7 @@defmodule Firecrawl.MixProject douse Mix.Project- @version "1.2.1"+ @version "1.3.0"@source_url "https://github.com/firecrawl/firecrawl/tree/main/apps/elixir-sdk"def project dodiff --git a/apps/elixir-sdk/test/firecrawl_test.exs b/apps/elixir-sdk/test/firecrawl_test.exsindex 72f6a9d14..f6c6b6daa 100644--- a/apps/elixir-sdk/test/firecrawl_test.exs+++ b/apps/elixir-sdk/test/firecrawl_test.exs@@ -124,6 +124,37 @@ defmodule FirecrawlTest do"Expected connection error, got validation error: #{inspect(err)}"end+ test "parse_file raises ArgumentError when filename is empty" do+ Application.put_env(:firecrawl, :api_key, "test-key")+ on_exit(fn -> Application.delete_env(:firecrawl, :api_key) end)++ assert_raise ArgumentError, ~r/filename cannot be empty/, fn ->+ Firecrawl.parse_file([filename: "", data: "x"])+ end+ end++ test "parse_file raises ArgumentError when data is nil" do+ Application.put_env(:firecrawl, :api_key, "test-key")+ on_exit(fn -> Application.delete_env(:firecrawl, :api_key) end)++ assert_raise ArgumentError, ~r/file data cannot be empty/, fn ->+ Firecrawl.parse_file([filename: "doc.pdf", data: nil])+ end+ end++ test "parse_file rejects unknown options" do+ Application.put_env(:firecrawl, :api_key, "test-key")+ on_exit(fn -> Application.delete_env(:firecrawl, :api_key) end)++ assert {:error, %NimbleOptions.ValidationError{message: msg}} =+ Firecrawl.parse_file(+ [filename: "doc.pdf", data: "x"],+ typo_option: true+ )++ assert msg =~ "unknown options"+ end+test "non-bang returns {:error, %Firecrawl.Error{}} for API errors" doadapter = fn request ->resp = Req.Response.new(@@ -199,7 +230,13 @@ defmodule FirecrawlTest do{:get_queue_status, 0},{:get_queue_status!, 0},{:cancel_crawl, 1},- {:cancel_crawl!, 1}+ {:cancel_crawl!, 1},+ {:parse_file, 1},+ {:parse_file, 2},+ {:parse_file, 3},+ {:parse_file!, 1},+ {:parse_file!, 2},+ {:parse_file!, 3}]for {name, arity} <- expected do