Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
# This file is generated by generate.exs from openapi.json. Edit by hand only
# inside the HAND-WRITTEN region, which regeneration keeps verbatim.
# Re-generate with: mix run generate.exs
defmodule Firecrawl do
@moduledoc """
Auto-generated Firecrawl API v2 client.
Generated from the OpenAPI spec at:
https://raw.githubusercontent.com/firecrawl/firecrawl-docs/main/api-reference/v2-openapi.json
## Configuration
Set your API key in application config:
config :firecrawl, api_key: "fc-your-api-key"
Or pass it as an option to any function:
Firecrawl.scrape_and_extract_from_url([url: "https://example.com"], api_key: "fc-your-api-key")
## Options
All functions accept an optional keyword list as the last argument with:
* `:api_key` - Override the API key for this request
* `:base_url` - Override the default API base URL (useful for self-hosted instances)
* Any other options are passed through to the `Req` request
## Usage
{:ok, response} = Firecrawl.scrape_and_extract_from_url(
url: "https://example.com"
)
{:ok, response} = Firecrawl.crawl_urls(
url: "https://example.com",
limit: 100
)
"""
@type response :: {:ok, Req.Response.t()} | {:error, Exception.t() | Firecrawl.Error.t()}
@base_url "https://api.firecrawl.dev/v2"
# Sourced from mix.exs at compile time so the origin header cannot drift
# from the published package version.
@version Mix.Project.config()[:version]
@sdk_origin "elixir-sdk@" <> @version
defp client(opts) do
api_key =
Keyword.get_lazy(opts, :api_key, fn ->
Application.get_env(:firecrawl, :api_key)
end)
# A nil/empty key is allowed: scrape, search, and interact fall back to the
# keyless free tier (rate-limited per IP). Other endpoints return 401 from the
# API until a key is provided.
api_key =
case api_key do
key when is_binary(key) ->
case String.trim(key) do
"" -> nil
trimmed -> trimmed
end
_ ->
nil
end
{base_url, opts} = Keyword.pop(opts, :base_url, @base_url)
opts = Keyword.delete(opts, :api_key)
headers = if api_key, do: [{"authorization", "Bearer #{api_key}"}], else: []
Req.new(
base_url: base_url,
headers: headers
)
|> Req.merge(opts)
|> Req.Request.append_response_steps(firecrawl_error_handler: &handle_api_error/1)
end
defp handle_api_error({request, %Req.Response{status: status} = response}) when status >= 400 do
{request, Firecrawl.Error.exception(status: status, body: response.body)}
end
defp handle_api_error({request, response}), do: {request, response}
defp to_body(validated_params, key_mapping) do
validated_params
|> to_json_object(key_mapping)
# Identify the SDK so the API can grant the keyless free tier; harmless
# telemetry on keyed requests.
|> Map.put_new("origin", @sdk_origin)
end
defp to_json_object(params, key_mapping) do
Map.new(params, fn {k, v} ->
case Map.fetch!(key_mapping, k) do
{json_key, nested_mapping} -> {json_key, to_json_object(v, nested_mapping)}
json_key -> {json_key, to_json_value(v)}
end
end)
end
defp to_query(validated_params, key_mapping) do
Enum.map(validated_params, fn {k, v} ->
{Map.fetch!(key_mapping, k), to_json_value(v)}
end)
end
defp to_json_value(atom) when is_atom(atom) and atom not in [true, false, nil] do
Atom.to_string(atom)
end
defp to_json_value(map) when is_map(map), do: map
defp to_json_value(list) when is_list(list) do
if Keyword.keyword?(list) and list != [] do
Map.new(list, fn {k, v} -> {camelize(Atom.to_string(k)), to_json_value(v)} end)
else
Enum.map(list, &to_json_value/1)
end
end
defp to_json_value(value), do: value
defp camelize(string) do
[first | rest] = String.split(string, "_")
Enum.join([first | Enum.map(rest, &String.capitalize/1)])
end
defp fetch_file_field(file, key) do
case Keyword.fetch(file, key) do
{:ok, _value} = ok -> ok
:error -> {:error, %ArgumentError{message: "missing required file field: #{key}"}}
end
end
defp validate_filename(filename) do
if is_binary(filename) and filename != "" do
:ok
else
{:error, %ArgumentError{message: "filename cannot be empty"}}
end
end
defp validate_data(data) do
if is_nil(data) do
{:error, %ArgumentError{message: "file data cannot be empty"}}
else
:ok
end
end
# --- BEGIN HAND-WRITTEN ---
# generate.exs keeps everything between these markers verbatim. Any route
# defined here must be listed in @hand_written_routes in generate.exs.
@doc """
List the file formats `parse_file/3` accepts
`GET /parse/formats`
Tag: Parsing
## Returns
* `{:ok, [%Firecrawl.ParseFormat{}]}` on success
* `{:error, exception}` on HTTP failure or an unexpected response shape
"""
@spec get_parse_formats(keyword()) ::
{:ok, [Firecrawl.ParseFormat.t()]} | {:error, Exception.t() | Firecrawl.Error.t()}
def get_parse_formats(opts \\ []) do
with {:ok, response} <- Req.get(client(opts), url: "/parse/formats") do
decode_parse_formats(response)
end
end
@doc """
Bang variant of `get_parse_formats`. Raises on error.
"""
@spec get_parse_formats!(keyword()) :: [Firecrawl.ParseFormat.t()]
def get_parse_formats!(opts \\ []) do
case get_parse_formats(opts) do
{:ok, formats} -> formats
{:error, exception} -> raise exception
end
end
defp decode_parse_formats(%Req.Response{body: %{"data" => %{"formats" => formats}}})
when is_list(formats) do
{:ok, Enum.map(formats, &Firecrawl.ParseFormat.from_map/1)}
end
defp decode_parse_formats(%Req.Response{status: status, body: body}) do
{:error,
RuntimeError.exception(
"unexpected GET /parse/formats response (HTTP #{status}): #{inspect(body)}"
)}
end
@doc """
Get the execution trace of an agent job
`GET /agent/{jobId}/trace`
Tag: Agent
The trace is only available for agent runs on spark-2.
## Path Parameters
* `job_id` - Path parameter `jobId`
## Options
* `:live_view` - when `true`, the query parameter `liveView=true` is sent and
the response may additionally carry `activeBrowserSessions`, a list of
`%{"id" => "...", "liveViewUrl" => "...", "viewport" => %{"width" => 1280, "height" => 720}}`
entries for the run's live browser sessions.
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
Success JSON: `%{"success" => true, "id" => "...", "events" => [...], "creditsUsed" => 5}`.
Each event has a `type` field (e.g. `"run.started"`, `"tool_call.started"`,
`"artifact.updated"`) plus the base fields `schemaVersion`, `eventId`, `runId`,
`occurredAt`, `producerSequence`, and `agent`.
"""
@spec get_agent_trace(String.t(), keyword()) :: response()
def get_agent_trace(job_id, opts \\ []) do
{live_view, opts} = Keyword.pop(opts, :live_view, false)
params = if live_view, do: [{"liveView", "true"}], else: []
Req.get(client(opts), url: "/agent/#{job_id}/trace", params: params)
end
@doc """
Bang variant of `get_agent_trace`. Raises on error.
"""
@spec get_agent_trace!(String.t(), keyword()) :: Req.Response.t()
def get_agent_trace!(job_id, opts \\ []) do
{live_view, opts} = Keyword.pop(opts, :live_view, false)
params = if live_view, do: [{"liveView", "true"}], else: []
Req.get!(client(opts), url: "/agent/#{job_id}/trace", params: params)
end
@research_search_papers_schema NimbleOptions.new!([
authors: [type: {:list, :string}],
categories: [type: {:list, :string}],
from: [type: :string],
k: [type: :integer],
query: [type: :string, required: true],
sort: [type: {:in, [:relevance, :date_desc, "relevance", "date_desc"]}],
to: [type: :string]
])
@research_search_papers_key_mapping %{authors: "authors", categories: "categories", from: "from", k: "k", query: "query", sort: "sort", to: "to"}
@doc """
Search research papers.
`sort` accepts `relevance` (default) or `date_desc`, which returns the newest
of the most relevant matches, newest first by created date. It does not list
every paper in a date range; add `from` (and `to`) for that.
`GET /search/research/papers`
"""
@spec search_papers(keyword(), keyword()) :: response()
def search_papers(params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @research_search_papers_schema) do
Req.get(client(opts), url: "/search/research/papers", params: [{"origin", @sdk_origin} | to_query(params, @research_search_papers_key_mapping)])
end
end
@doc """
Bang variant of `search_papers`. Raises on error.
"""
@spec search_papers!(keyword(), keyword()) :: Req.Response.t()
def search_papers!(params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @research_search_papers_schema)
Req.get!(client(opts), url: "/search/research/papers", params: [{"origin", @sdk_origin} | to_query(params, @research_search_papers_key_mapping)])
end
@doc """
Inspect paper metadata.
`GET /search/research/papers/{id}`
"""
@spec inspect_paper(String.t(), keyword()) :: response()
def inspect_paper(id, opts \\ []) do
Req.get(client(opts),
url: "/search/research/papers/#{URI.encode_www_form(id)}",
params: [{"origin", @sdk_origin}]
)
end
@doc """
Bang variant of `inspect_paper`. Raises on error.
"""
@spec inspect_paper!(String.t(), keyword()) :: Req.Response.t()
def inspect_paper!(id, opts \\ []) do
Req.get!(client(opts),
url: "/search/research/papers/#{URI.encode_www_form(id)}",
params: [{"origin", @sdk_origin}]
)
end
@research_read_paper_schema NimbleOptions.new!([
k: [type: :integer],
query: [type: :string, required: true]
])
@research_read_paper_key_mapping %{k: "k", query: "query"}
@doc """
Read a paper with query-guided passages.
`GET /search/research/papers/{id}`
"""
@spec read_paper(String.t(), keyword(), keyword()) :: response()
def read_paper(id, params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @research_read_paper_schema) do
Req.get(client(opts), url: "/search/research/papers/#{URI.encode_www_form(id)}", params: [{"origin", @sdk_origin} | to_query(params, @research_read_paper_key_mapping)])
end
end
@doc """
Bang variant of `read_paper`. Raises on error.
"""
@spec read_paper!(String.t(), keyword(), keyword()) :: Req.Response.t()
def read_paper!(id, params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @research_read_paper_schema)
Req.get!(client(opts), url: "/search/research/papers/#{URI.encode_www_form(id)}", params: [{"origin", @sdk_origin} | to_query(params, @research_read_paper_key_mapping)])
end
@research_related_papers_schema NimbleOptions.new!([
anchor: [type: {:list, :string}],
intent: [type: :string, required: true],
k: [type: :integer],
mode: [type: {:in, [:similar, :citers, :references, "similar", "citers", "references"]}],
rerank: [type: :boolean]
])
@research_related_papers_key_mapping %{anchor: "anchor", intent: "intent", k: "k", mode: "mode", rerank: "rerank"}
@doc """
Find papers related to a paper.
`GET /search/research/papers/{id}/similar`
"""
@spec related_papers(String.t(), keyword(), keyword()) :: response()
def related_papers(id, params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @research_related_papers_schema) do
Req.get(client(opts), url: "/search/research/papers/#{URI.encode_www_form(id)}/similar", params: [{"origin", @sdk_origin} | to_query(params, @research_related_papers_key_mapping)])
end
end
@doc """
Bang variant of `related_papers`. Raises on error.
"""
@spec related_papers!(String.t(), keyword(), keyword()) :: Req.Response.t()
def related_papers!(id, params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @research_related_papers_schema)
Req.get!(client(opts), url: "/search/research/papers/#{URI.encode_www_form(id)}/similar", params: [{"origin", @sdk_origin} | to_query(params, @research_related_papers_key_mapping)])
end
@research_search_github_schema NimbleOptions.new!([
k: [type: :integer],
query: [type: :string, required: true]
])
@research_search_github_key_mapping %{k: "k", query: "query"}
@doc """
Search GitHub research content.
`GET /search/research/github`
Deprecated: the research index GitHub search endpoint was removed after
2026-11-03. Use `search_developer/2` instead.
"""
@deprecated "The research index GitHub search endpoint was removed after 2026-11-03. Use search_developer/2 instead."
@spec search_github(keyword(), keyword()) :: response()
def search_github(params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @research_search_github_schema) do
Req.get(client(opts), url: "/search/research/github", params: [{"origin", @sdk_origin} | to_query(params, @research_search_github_key_mapping)])
end
end
@doc """
Bang variant of `search_github`. Raises on error.
Deprecated: the research index GitHub search endpoint was removed after
2026-11-03. Use `search_developer!/2` instead.
"""
@deprecated "The research index GitHub search endpoint was removed after 2026-11-03. Use search_developer!/2 instead."
@spec search_github!(keyword(), keyword()) :: Req.Response.t()
def search_github!(params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @research_search_github_schema)
Req.get!(client(opts), url: "/search/research/github", params: [{"origin", @sdk_origin} | to_query(params, @research_search_github_key_mapping)])
end
@search_developer_schema NimbleOptions.new!([
query: [type: :string, required: true, doc: "Natural-language question or search phrase. Must not be blank."],
k: [type: :integer, doc: "Total ranked results, 1 to 100 (default 10)."],
passages: [type: :integer, doc: "Passages per result, 1 to 5 (default 1)."],
types: [type: {:list, :string}, doc: "Result kinds: `doc`, `issue`, `pull_request`, `readme` (default all)."],
repos: [type: {:list, :string}, doc: "GitHub `owner/name` filters, at most 20."],
sources: [type: {:list, :string}, doc: "Developer documentation source ids, at most 20."],
language: [type: :string, doc: "One case-insensitive GitHub Linguist primary language."],
topic: [type: {:or, [:string, {:list, :string}]}, doc: "Repository topic, or topics that must all match."],
license: [type: :string, doc: "One case-insensitive SPDX repository license identifier."],
min_stars: [type: :integer, doc: "Lower bound on repository stars."],
max_stars: [type: :integer, doc: "Upper bound on repository stars."],
archived: [type: :boolean, doc: "Include or exclude archived repositories."],
fork: [type: :boolean, doc: "Include or exclude forks."],
skills: [type: {:or, [{:in, [:only]}, :string]}, doc: "Set to `only` to return packaged agent-skill evidence only."]
])
@search_developer_key_mapping %{query: "query", k: "k", passages: "passages", types: "types", repos: "repos", sources: "sources", language: "language", topic: "topic", license: "license", min_stars: "min_stars", max_stars: "max_stars", archived: "archived", fork: "fork", skills: "skills"}
@doc """
Search the developer index: public repositories, GitHub issues, merged pull
requests, READMEs and code documentation. Replaces `search_github/2`, whose
endpoint was removed after 2026-11-03.
`POST /search/developer`
Tag: Developer
## Parameters
Validated by `NimbleOptions`. A blank `query` is rejected before any request is sent.
#{NimbleOptions.docs(@search_developer_schema)}
## Returns
* `{:ok, %Firecrawl.DeveloperSearch{}}` on success
* `{:error, exception}` on a blank query, validation failure, HTTP failure or an unexpected response shape
## Example
{:ok, %Firecrawl.DeveloperSearch{results: results}} =
Firecrawl.search_developer(query: "retry an http request", repos: ["square/okhttp"], k: 5)
"""
@spec search_developer(keyword(), keyword()) ::
{:ok, Firecrawl.DeveloperSearch.t()} | {:error, Exception.t() | Firecrawl.Error.t()}
def search_developer(params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @search_developer_schema),
:ok <- validate_query(params[:query]),
{:ok, response} <-
Req.post(client(opts), url: "/search/developer", json: to_body(params, @search_developer_key_mapping)) do
decode_developer_search(response)
end
end
@doc """
Bang variant of `search_developer`. Raises on error.
"""
@spec search_developer!(keyword(), keyword()) :: Firecrawl.DeveloperSearch.t()
def search_developer!(params \\ [], opts \\ []) do
case search_developer(params, opts) do
{:ok, result} -> result
{:error, exception} -> raise exception
end
end
defp validate_query(query) do
if String.trim(query) == "" do
{:error, %ArgumentError{message: "query cannot be empty"}}
else
:ok
end
end
defp decode_developer_search(%Req.Response{body: %{"results" => results} = body})
when is_list(results) do
{:ok, Firecrawl.DeveloperSearch.from_map(body)}
rescue
exception -> {:error, exception}
end
defp decode_developer_search(%Req.Response{status: status, body: body}) do
{:error,
RuntimeError.exception(
"unexpected POST /search/developer response (HTTP #{status}): #{inspect(body)}"
)}
end
@monitor_key_mapping %{
name: "name",
schedule: "schedule",
targets: "targets",
webhook: "webhook",
notification: "notification",
retention_days: "retentionDays",
status: "status",
goal: "goal",
judge_enabled: "judgeEnabled"
}
@monitor_list_key_mapping %{limit: "limit", offset: "offset"}
@monitor_check_list_key_mapping %{limit: "limit", offset: "offset", status: "status"}
@monitor_check_key_mapping %{limit: "limit", skip: "skip", status: "status"}
@monitor_schema NimbleOptions.new!([
name: [type: :string],
schedule: [type: :any],
targets: [type: {:list, :any}],
webhook: [type: :any],
notification: [type: :any],
retention_days: [type: :integer],
status: [type: :any],
goal: [type: :any],
judge_enabled: [type: :boolean]
])
@monitor_list_schema NimbleOptions.new!([
limit: [type: :integer],
offset: [type: :integer]
])
@monitor_check_list_schema NimbleOptions.new!([
limit: [type: :integer],
offset: [type: :integer],
status: [type: :any]
])
@monitor_check_schema NimbleOptions.new!([
limit: [type: :integer],
skip: [type: :integer],
status: [type: :any]
])
@doc """
Create a scheduled monitor.
`POST /monitor`
## Targets
`targets` is a list of maps/keyword lists. Each target is sent as-is (keys are
camelCased), so the supported target shapes are:
* Scrape: `[type: "scrape", url: "https://example.com", ...]`
* Crawl: `[type: "crawl", url: "https://example.com", ...]`
* Search:
`[type: "search", queries: ["term"], search_window: "24h",
include_domains: ["example.com"], exclude_domains: [],
max_results: 10]`
(`search_window` is one of `"5m"`, `"15m"`, `"1h"`, `"6h"`, `"24h"`,
`"7d"`; only `queries` is required.)
Optionally pass `goal` and `judge_enabled: true` to enable AI judging.
"""
@spec create_monitor(keyword(), keyword()) :: response()
def create_monitor(params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @monitor_schema) do
Req.post(client(opts), url: "/monitor", json: to_body(params, @monitor_key_mapping))
end
end
@doc """
Bang variant of `create_monitor`. Raises on error.
"""
@spec create_monitor!(keyword(), keyword()) :: Req.Response.t()
def create_monitor!(params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @monitor_schema)
Req.post!(client(opts), url: "/monitor", json: to_body(params, @monitor_key_mapping))
end
@doc """
List monitors for the authenticated team.
`GET /monitor`
"""
@spec list_monitors(keyword(), keyword()) :: response()
def list_monitors(params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @monitor_list_schema) do
Req.get(client(opts), url: "/monitor", params: to_query(params, @monitor_list_key_mapping))
end
end
@doc """
Bang variant of `list_monitors`. Raises on error.
"""
@spec list_monitors!(keyword(), keyword()) :: Req.Response.t()
def list_monitors!(params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @monitor_list_schema)
Req.get!(client(opts), url: "/monitor", params: to_query(params, @monitor_list_key_mapping))
end
@doc """
Get a monitor by ID.
`GET /monitor/{monitorId}`
"""
@spec get_monitor(String.t(), keyword()) :: response()
def get_monitor(monitor_id, opts \\ []) do
Req.get(client(opts), url: "/monitor/#{monitor_id}")
end
@doc """
Bang variant of `get_monitor`. Raises on error.
"""
@spec get_monitor!(String.t(), keyword()) :: Req.Response.t()
def get_monitor!(monitor_id, opts \\ []) do
Req.get!(client(opts), url: "/monitor/#{monitor_id}")
end
@doc """
Update a monitor.
`PATCH /monitor/{monitorId}`
"""
@spec update_monitor(String.t(), keyword(), keyword()) :: response()
def update_monitor(monitor_id, params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @monitor_schema) do
Req.patch(client(opts), url: "/monitor/#{monitor_id}", json: to_body(params, @monitor_key_mapping))
end
end
@doc """
Bang variant of `update_monitor`. Raises on error.
"""
@spec update_monitor!(String.t(), keyword(), keyword()) :: Req.Response.t()
def update_monitor!(monitor_id, params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @monitor_schema)
Req.patch!(client(opts), url: "/monitor/#{monitor_id}", json: to_body(params, @monitor_key_mapping))
end
@doc """
Delete a monitor.
`DELETE /monitor/{monitorId}`
"""
@spec delete_monitor(String.t(), keyword()) :: response()
def delete_monitor(monitor_id, opts \\ []) do
Req.delete(client(opts), url: "/monitor/#{monitor_id}")
end
@doc """
Bang variant of `delete_monitor`. Raises on error.
"""
@spec delete_monitor!(String.t(), keyword()) :: Req.Response.t()
def delete_monitor!(monitor_id, opts \\ []) do
Req.delete!(client(opts), url: "/monitor/#{monitor_id}")
end
@doc """
Trigger a manual monitor check.
`POST /monitor/{monitorId}/run`
"""
@spec run_monitor(String.t(), keyword()) :: response()
def run_monitor(monitor_id, opts \\ []) do
Req.post(client(opts), url: "/monitor/#{monitor_id}/run", json: %{})
end
@doc """
Bang variant of `run_monitor`. Raises on error.
"""
@spec run_monitor!(String.t(), keyword()) :: Req.Response.t()
def run_monitor!(monitor_id, opts \\ []) do
Req.post!(client(opts), url: "/monitor/#{monitor_id}/run", json: %{})
end
@doc """
List checks for a monitor.
`GET /monitor/{monitorId}/checks`
"""
@spec list_monitor_checks(String.t(), keyword(), keyword()) :: response()
def list_monitor_checks(monitor_id, params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @monitor_check_list_schema) do
Req.get(client(opts),
url: "/monitor/#{monitor_id}/checks",
params: to_query(params, @monitor_check_list_key_mapping)
)
end
end
@doc """
Bang variant of `list_monitor_checks`. Raises on error.
"""
@spec list_monitor_checks!(String.t(), keyword(), keyword()) :: Req.Response.t()
def list_monitor_checks!(monitor_id, params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @monitor_check_list_schema)
Req.get!(client(opts),
url: "/monitor/#{monitor_id}/checks",
params: to_query(params, @monitor_check_list_key_mapping)
)
end
@doc """
Get a monitor check with paginated page results and inline diffs.
`GET /monitor/{monitorId}/checks/{checkId}`
"""
@spec get_monitor_check(String.t(), String.t(), keyword(), keyword()) :: response()
def get_monitor_check(monitor_id, check_id, params \\ [], opts \\ []) do
{auto_paginate, opts} = Keyword.pop(opts, :auto_paginate, true)
with {:ok, params} <- NimbleOptions.validate(params, @monitor_check_schema) do
case Req.get(client(opts),
url: "/monitor/#{monitor_id}/checks/#{check_id}",
params: to_query(params, @monitor_check_key_mapping)
) do
{:ok, response} when auto_paginate -> {:ok, paginate_monitor_check_response(response, opts)}
result -> result
end
end
end
@doc """
Bang variant of `get_monitor_check`. Raises on error.
"""
@spec get_monitor_check!(String.t(), String.t(), keyword(), keyword()) :: Req.Response.t()
def get_monitor_check!(monitor_id, check_id, params \\ [], opts \\ []) do
{auto_paginate, opts} = Keyword.pop(opts, :auto_paginate, true)
params = NimbleOptions.validate!(params, @monitor_check_schema)
response = Req.get!(client(opts),
url: "/monitor/#{monitor_id}/checks/#{check_id}",
params: to_query(params, @monitor_check_key_mapping)
)
if auto_paginate, do: paginate_monitor_check_response!(response, opts), else: response
end
defp paginate_monitor_check_response(%Req.Response{body: body} = response, opts) when is_map(body) do
data = Map.get(body, "data") || %{}
pages = Map.get(data, "pages") || []
next = Map.get(body, "next") || Map.get(data, "next")
pages = fetch_monitor_check_pages(next, pages, opts)
data = data |> Map.put("pages", pages) |> Map.put("next", nil)
%{response | body: body |> Map.put("data", data) |> Map.put("next", nil)}
end
defp paginate_monitor_check_response(response, _opts), do: response
defp paginate_monitor_check_response!(%Req.Response{body: body} = response, opts) when is_map(body) do
data = Map.get(body, "data") || %{}
pages = Map.get(data, "pages") || []
next = Map.get(body, "next") || Map.get(data, "next")
pages = fetch_monitor_check_pages!(next, pages, opts)
data = data |> Map.put("pages", pages) |> Map.put("next", nil)
%{response | body: body |> Map.put("data", data) |> Map.put("next", nil)}
end
defp paginate_monitor_check_response!(response, _opts), do: response
defp fetch_monitor_check_pages(nil, pages, _opts), do: pages
defp fetch_monitor_check_pages("", pages, _opts), do: pages
defp fetch_monitor_check_pages(next, pages, opts) do
req = client(opts)
case Req.get(req, url: Firecrawl.ApiOrigin.pin(next, req.options.base_url)) do
{:ok, %Req.Response{body: body}} when is_map(body) ->
data = Map.get(body, "data") || %{}
next_pages = Map.get(data, "pages") || []
next_url = Map.get(body, "next") || Map.get(data, "next")
fetch_monitor_check_pages(next_url, pages ++ next_pages, opts)
_ ->
pages
end
end
defp fetch_monitor_check_pages!(nil, pages, _opts), do: pages
defp fetch_monitor_check_pages!("", pages, _opts), do: pages
defp fetch_monitor_check_pages!(next, pages, opts) do
req = client(opts)
response = Req.get!(req, url: Firecrawl.ApiOrigin.pin(next, req.options.base_url))
body = response.body
data = if is_map(body), do: Map.get(body, "data") || %{}, else: %{}
next_pages = Map.get(data, "pages") || []
next_url = if is_map(body), do: Map.get(body, "next") || Map.get(data, "next"), else: nil
fetch_monitor_check_pages!(next_url, pages ++ next_pages, opts)
end
# --- END HAND-WRITTEN ---
@ask_support_agent_schema NimbleOptions.new!([
question: [type: :string, required: true, doc: "Question or issue for the support agent to diagnose."],
rationale: [type: :string, doc: "Optional context about what the end user is trying to accomplish."]
])
@ask_support_agent_key_mapping %{question: "question", rationale: "rationale"}
@doc """
Ask the Firecrawl support agent
`POST /support/ask`
Tag: Support
## Parameters
Validated by `NimbleOptions`. Pass params as a keyword list with snake_case keys.
See `@ask_support_agent_schema` for the full schema.
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec ask_support_agent(keyword(), keyword()) :: response()
def ask_support_agent(params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @ask_support_agent_schema) do
Req.post(client(opts), url: "/support/ask", json: to_body(params, @ask_support_agent_key_mapping))
end
end
@doc """
Bang variant of `ask_support_agent`. Raises on error.
"""
@spec ask_support_agent!(keyword(), keyword()) :: Req.Response.t()
def ask_support_agent!(params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @ask_support_agent_schema)
Req.post!(client(opts), url: "/support/ask", json: to_body(params, @ask_support_agent_key_mapping))
end
@doc """
Cancel an agent job
`DELETE /agent/{jobId}`
Tag: Agent
## Path Parameters
* `job_id` - Path parameter `jobId`
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec cancel_agent(String.t(), keyword()) :: response()
def cancel_agent(job_id, opts \\ []) do
Req.delete(client(opts), url: "/agent/#{job_id}")
end
@doc """
Bang variant of `cancel_agent`. Raises on error.
"""
@spec cancel_agent!(String.t(), keyword()) :: Req.Response.t()
def cancel_agent!(job_id, opts \\ []) do
Req.delete!(client(opts), url: "/agent/#{job_id}")
end
@doc """
Cancel a batch scrape job
`DELETE /batch/scrape/{id}`
Tag: Scraping
## Path Parameters
* `id` - Path parameter `id`
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec cancel_batch_scrape(String.t(), keyword()) :: response()
def cancel_batch_scrape(id, opts \\ []) do
Req.delete(client(opts), url: "/batch/scrape/#{id}")
end
@doc """
Bang variant of `cancel_batch_scrape`. Raises on error.
"""
@spec cancel_batch_scrape!(String.t(), keyword()) :: Req.Response.t()
def cancel_batch_scrape!(id, opts \\ []) do
Req.delete!(client(opts), url: "/batch/scrape/#{id}")
end
@doc """
Cancel a crawl job
`DELETE /crawl/{id}`
Tag: Crawling
## Path Parameters
* `id` - Path parameter `id`
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec cancel_crawl(String.t(), keyword()) :: response()
def cancel_crawl(id, opts \\ []) do
Req.delete(client(opts), url: "/crawl/#{id}")
end
@doc """
Bang variant of `cancel_crawl`. Raises on error.
"""
@spec cancel_crawl!(String.t(), keyword()) :: Req.Response.t()
def cancel_crawl!(id, opts \\ []) do
Req.delete!(client(opts), url: "/crawl/#{id}")
end
@crawl_params_preview_schema NimbleOptions.new!([
prompt: [type: :string, required: true, doc: "Natural language prompt describing what you want to crawl"],
url: [type: :string, required: true, doc: "The URL to crawl"]
])
@crawl_params_preview_key_mapping %{prompt: "prompt", url: "url"}
@doc """
Preview crawl parameters generated from natural language prompt
`POST /crawl/params-preview`
Tag: Crawling
## Parameters
Validated by `NimbleOptions`. Pass params as a keyword list with snake_case keys.
See `@crawl_params_preview_schema` for the full schema.
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec crawl_params_preview(keyword(), keyword()) :: response()
def crawl_params_preview(params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @crawl_params_preview_schema) do
Req.post(client(opts), url: "/crawl/params-preview", json: to_body(params, @crawl_params_preview_key_mapping))
end
end
@doc """
Bang variant of `crawl_params_preview`. Raises on error.
"""
@spec crawl_params_preview!(keyword(), keyword()) :: Req.Response.t()
def crawl_params_preview!(params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @crawl_params_preview_schema)
Req.post!(client(opts), url: "/crawl/params-preview", json: to_body(params, @crawl_params_preview_key_mapping))
end
@crawl_urls_schema NimbleOptions.new!([
allow_external_links: [type: :boolean, doc: "Allows the crawler to follow links to external websites. External links are followed one hop (the links found on those external pages are not crawled). Links pointing to an external site's homepage (a root URL with no path) are skipped and reported in Get Crawl Errors with the code EXTERNAL_LINK; redirects to an external homepage are skipped for the same reason."],
allow_subdomains: [type: :boolean, doc: "Allows the crawler to follow links to subdomains of the main domain."],
crawl_entire_domain: [type: :boolean, doc: "Allows the crawler to follow internal links to sibling or parent URLs, not just child paths.\n\nfalse: Only crawls deeper (child) URLs.\nβ e.g. /features/feature-1 β /features/feature-1/tips β
\nβ Won't follow /pricing or / β\n\ntrue: Crawls any internal links, including siblings and parents.\nβ e.g. /features/feature-1 β /pricing, /, etc. β
\n\nUse true for broader internal coverage beyond nested paths."],
delay: [type: {:or, [:integer, :float]}, doc: "Delay in seconds between scrapes. This helps respect website rate limits. Setting this forces concurrency to 1."],
exclude_paths: [type: {:list, :string}, doc: "URL pathname regex patterns that exclude matching URLs from the crawl. For example, if you set \"excludePaths\": [\"blog/.*\"] for the base URL firecrawl.dev, any results matching that pattern will be excluded, such as https://www.firecrawl.dev/blog/firecrawl-launch-week-1-recap. Patterns use Rust regex (RE2-style) syntax: look-around and backreferences are not supported, and a pattern that does not compile is rejected with a 400. Each field accepts at most 1000 patterns of at most 2000 characters each, and includePaths and excludePaths together may contain at most 1000 patterns and 100,000 characters in total. For keyword-style filtering, send one short pattern per term rather than combining terms into a single long alternation."],
ignore_query_parameters: [type: :boolean, doc: "Do not re-scrape the same path with different (or none) query parameters"],
ignore_robots_txt: [type: :boolean, doc: "Ignore the website's robots.txt rules. Enterprise only β contact [email protected] to enable."],
include_paths: [type: {:list, :string}, doc: "URL pathname regex patterns that include matching URLs in the crawl. Only the paths that match the specified patterns will be included in the response. Note: the starting URL is also checked against these patterns β if it does not match, the crawl may return 0 pages. For example, if you set \"includePaths\": [\"blog/.*\"] for the base URL firecrawl.dev/blog, only pages under /blog/ will be included in the results, such as https://www.firecrawl.dev/blog/firecrawl-launch-week-1-recap. Patterns use Rust regex (RE2-style) syntax: look-around and backreferences are not supported, and a pattern that does not compile is rejected with a 400. Each field accepts at most 1000 patterns of at most 2000 characters each, and includePaths and excludePaths together may contain at most 1000 patterns and 100,000 characters in total. For keyword-style filtering, send one short pattern per term rather than combining terms into a single long alternation."],
limit: [type: :integer, doc: "Maximum number of pages to crawl. Default limit is 10000."],
max_concurrency: [type: :integer, doc: "Maximum number of concurrent scrapes. This parameter allows you to set a concurrency limit for this crawl. If not specified, the crawl adheres to your team's concurrency limit."],
max_discovery_depth: [type: :integer, doc: "Maximum depth to crawl based on discovery order. The root site and sitemapped pages has a discovery depth of 0. For example, if you set it to 1, and you set `sitemap: 'skip'`, you will only crawl the entered URL and all URLs that are linked on that page."],
prompt: [type: :string, doc: "A prompt to use to generate the crawler options (all the parameters below) from natural language. Explicitly set parameters will override the generated equivalents."],
regex_on_full_url: [type: :boolean, doc: "When true, includePaths and excludePaths regex patterns are matched against the full URL (including query parameters) instead of just the URL pathname. Useful when you need to filter URLs based on query strings."],
robots_user_agent: [type: :string, doc: "Custom User-Agent string for robots.txt evaluation. When set, robots.txt is fetched with this User-Agent and allow/disallow rules are matched against it instead of the default. Enterprise only β contact [email protected] to enable."],
scrape_options: [type: :keyword_list],
sitemap: [type: {:or, [{:in, [:skip, :include, :only]}, :string]}, doc: "Sitemap mode when crawling. If you set it to 'skip', the crawler will ignore the website sitemap and only crawl the entered URL and discover pages from there onwards. If you set it to 'only', the crawler will only crawl URLs from the sitemap (plus the start URL) and will not discover links from HTML."],
url: [type: :string, required: true, doc: "The base URL to start crawling from"],
webhook: [type: :keyword_list, doc: "A webhook specification object."],
zero_data_retention: [type: :boolean, doc: "If true, this will enable zero data retention for this crawl. To enable this feature, please contact [email protected]"]
])
@crawl_urls_key_mapping %{allow_external_links: "allowExternalLinks", allow_subdomains: "allowSubdomains", crawl_entire_domain: "crawlEntireDomain", delay: "delay", exclude_paths: "excludePaths", ignore_query_parameters: "ignoreQueryParameters", ignore_robots_txt: "ignoreRobotsTxt", include_paths: "includePaths", limit: "limit", max_concurrency: "maxConcurrency", max_discovery_depth: "maxDiscoveryDepth", prompt: "prompt", regex_on_full_url: "regexOnFullURL", robots_user_agent: "robotsUserAgent", scrape_options: "scrapeOptions", sitemap: "sitemap", url: "url", webhook: "webhook", zero_data_retention: "zeroDataRetention"}
@doc """
Crawl multiple URLs based on options
`POST /crawl`
Tag: Crawling
## Parameters
Validated by `NimbleOptions`. Pass params as a keyword list with snake_case keys.
See `@crawl_urls_schema` for the full schema.
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec crawl_urls(keyword(), keyword()) :: response()
def crawl_urls(params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @crawl_urls_schema) do
Req.post(client(opts), url: "/crawl", json: to_body(params, @crawl_urls_key_mapping))
end
end
@doc """
Bang variant of `crawl_urls`. Raises on error.
"""
@spec crawl_urls!(keyword(), keyword()) :: Req.Response.t()
def crawl_urls!(params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @crawl_urls_schema)
Req.post!(client(opts), url: "/crawl", json: to_body(params, @crawl_urls_key_mapping))
end
@create_browser_session_schema NimbleOptions.new!([
activity_ttl: [type: :integer, doc: "Time in seconds before the session is destroyed due to inactivity"],
block_ads: [type: :boolean, doc: "Enable ad and cookie popup blocking, as in scrape. Sessions started from a scrape with interact use the scrape's blockAds."],
location: [type: :keyword_list, keys: [country: [type: :string, required: true]], doc: "Country the session browses from. Defaults to the US. Sessions started from a scrape with interact use the scrape's location."],
profile: [type: :keyword_list, doc: "Enable persistent storage across interact sessions. Data saved in one session can be loaded in a later session using the same name."],
stream_web_view: [type: :boolean, doc: "Whether to stream a live view of the browser"],
ttl: [type: :integer, doc: "Total time-to-live in seconds for the interact session"]
])
@create_browser_session_key_mapping %{activity_ttl: "activityTtl", block_ads: "blockAds", location: {"location", %{country: "country"}}, profile: "profile", stream_web_view: "streamWebView", ttl: "ttl"}
@doc """
Create an interact session
`POST /interact`
Tag: Interact
## Parameters
Validated by `NimbleOptions`. Pass params as a keyword list with snake_case keys.
See `@create_browser_session_schema` for the full schema.
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec create_browser_session(keyword(), keyword()) :: response()
def create_browser_session(params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @create_browser_session_schema) do
Req.post(client(opts), url: "/interact", json: to_body(params, @create_browser_session_key_mapping))
end
end
@doc """
Bang variant of `create_browser_session`. Raises on error.
"""
@spec create_browser_session!(keyword(), keyword()) :: Req.Response.t()
def create_browser_session!(params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @create_browser_session_schema)
Req.post!(client(opts), url: "/interact", json: to_body(params, @create_browser_session_key_mapping))
end
@doc """
Delete an interact session
`DELETE /interact/{sessionId}`
Tag: Interact
## Path Parameters
* `session_id` - Path parameter `sessionId`
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec delete_browser_session(String.t(), keyword()) :: response()
def delete_browser_session(session_id, opts \\ []) do
Req.delete(client(opts), url: "/interact/#{session_id}")
end
@doc """
Bang variant of `delete_browser_session`. Raises on error.
"""
@spec delete_browser_session!(String.t(), keyword()) :: Req.Response.t()
def delete_browser_session!(session_id, opts \\ []) do
Req.delete!(client(opts), url: "/interact/#{session_id}")
end
@developer_search_query_schema NimbleOptions.new!([
query: [type: :string, required: true, doc: "Natural-language question or search phrase."],
k: [type: :integer, doc: "Number of ranked results to return."],
types: [type: {:list, :string}, doc: "Result kinds to search. Defaults to all four. Accepts a repeated parameter (`types=issue&types=pull_request`) or one comma-separated value (`types=issue,pull_request`)."],
repos: [type: {:list, :string}, doc: "Repository slugs to scope the repository half of the index to, such as `firecrawl/firecrawl`. Applies to the `issue`, `pull_request`, and `readme` types only. Sent together with `sources`, the two halves are combined rather than intersected, so matching results come back from either. Returns 400 when no repository type is in `types`, reporting that `repos` cannot match any requested type and that you should add repository types or drop `repos`."],
sources: [type: {:list, :string}, doc: "Documentation source ids to scope the documentation half to, at most 20. Applies to the `doc` type only. Not a fixed enum: ids reflect the documentation sites in the index and the set grows over time, so confirm an id resolves by sending it and reading the `sources` array on the response. Returns 400 with `sources cannot match any requested type; add doc or drop sources` when `doc` is not in `types`."],
skills: [type: {:or, [{:in, [:only]}, :string]}, doc: "Set to `only` to limit the search to indexed agent-skill files."],
passages: [type: :integer, doc: "Matched passages to return per result."],
language: [type: :string, doc: "Repository primary language, such as `Rust`. Applies to repository results only; sending it with no `sources` scope returns no `doc` results. See [how the repository filters scope a search](/api-reference/endpoint/developer-search#how-the-repository-filters-scope-a-search)."],
topic: [type: :string, doc: "Repository topic, such as `async`. Applies to repository results only; sending it with no `sources` scope returns no `doc` results."],
license: [type: :string, doc: "Repository license, such as `MIT`. Applies to repository results only; sending it with no `sources` scope returns no `doc` results."],
min_stars: [type: :integer, doc: "Lower bound on repository stars. Applies to repository results only; sending it with no `sources` scope returns no `doc` results."],
max_stars: [type: :integer, doc: "Upper bound on repository stars. Applies to repository results only; sending it with no `sources` scope returns no `doc` results."],
archived: [type: :boolean, doc: "Include or exclude archived repositories. Applies to repository results only; sending it with no `sources` scope returns no `doc` results."],
fork: [type: :boolean, doc: "Include or exclude forks. Applies to repository results only; sending it with no `sources` scope returns no `doc` results."]
])
@developer_search_query_key_mapping %{query: "query", k: "k", types: "types", repos: "repos", sources: "sources", skills: "skills", passages: "passages", language: "language", topic: "topic", license: "license", min_stars: "min_stars", max_stars: "max_stars", archived: "archived", fork: "fork"}
@doc """
Search the developer index
`GET /search/developer`
Tag: Developer
## Query Parameters
* `query` β query parameter `query`
* `k` β query parameter `k`
* `types` β query parameter `types`
* `repos` β query parameter `repos`
* `sources` β query parameter `sources`
* `skills` β query parameter `skills`
* `passages` β query parameter `passages`
* `language` β query parameter `language`
* `topic` β query parameter `topic`
* `license` β query parameter `license`
* `min_stars` β query parameter `min_stars`
* `max_stars` β query parameter `max_stars`
* `archived` β query parameter `archived`
* `fork` β query parameter `fork`
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec developer_search(keyword(), keyword()) :: response()
def developer_search(params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @developer_search_query_schema) do
Req.get(client(opts), url: "/search/developer", params: to_query(params, @developer_search_query_key_mapping))
end
end
@doc """
Bang variant of `developer_search`. Raises on error.
"""
@spec developer_search!(keyword(), keyword()) :: Req.Response.t()
def developer_search!(params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @developer_search_query_schema)
Req.get!(client(opts), url: "/search/developer", params: to_query(params, @developer_search_query_key_mapping))
end
@developer_search_post_schema NimbleOptions.new!([
archived: [type: :boolean, doc: "Include or exclude archived repositories. Applies to repository results only; sending it with no `sources` scope returns no `doc` results."],
fork: [type: :boolean, doc: "Include or exclude forks. Applies to repository results only; sending it with no `sources` scope returns no `doc` results."],
k: [type: :integer],
language: [type: :string, doc: "Repository primary language, such as `Rust`. Applies to repository results only; sending it with no `sources` scope returns no `doc` results. See [how the repository filters scope a search](/api-reference/endpoint/developer-search#how-the-repository-filters-scope-a-search)."],
license: [type: :string, doc: "Repository license, such as `MIT`. Applies to repository results only; sending it with no `sources` scope returns no `doc` results."],
max_stars: [type: :integer, doc: "Upper bound on repository stars. Applies to repository results only; sending it with no `sources` scope returns no `doc` results."],
min_stars: [type: :integer, doc: "Lower bound on repository stars. Applies to repository results only; sending it with no `sources` scope returns no `doc` results."],
passages: [type: :integer],
query: [type: :string, required: true],
repos: [type: {:list, :string}, doc: "Repository slugs to scope the repository half of the index to. Applies to the `issue`, `pull_request`, and `readme` types only. Sent together with `sources`, the two halves are combined rather than intersected. Returns 400 when no repository type is in `types`, reporting that `repos` cannot match any requested type and that you should add repository types or drop `repos`."],
skills: [type: {:or, [{:in, [:only]}, :string]}, doc: "Set to `only` to limit the search to indexed agent-skill files."],
sources: [type: {:list, :string}, doc: "Documentation source ids to scope the documentation half to, at most 20. Applies to the `doc` type only. Not a fixed enum: ids reflect the documentation sites in the index and the set grows over time. Returns 400 with `sources cannot match any requested type; add doc or drop sources` when `doc` is not in `types`."],
topic: [type: :string, doc: "Repository topic, such as `async`. Applies to repository results only; sending it with no `sources` scope returns no `doc` results."],
types: [type: {:list, :string}]
])
@developer_search_post_key_mapping %{archived: "archived", fork: "fork", k: "k", language: "language", license: "license", max_stars: "max_stars", min_stars: "min_stars", passages: "passages", query: "query", repos: "repos", skills: "skills", sources: "sources", topic: "topic", types: "types"}
@doc """
Search the developer index
`POST /search/developer`
Tag: Developer
## Parameters
Validated by `NimbleOptions`. Pass params as a keyword list with snake_case keys.
See `@developer_search_post_schema` for the full schema.
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec developer_search_post(keyword(), keyword()) :: response()
def developer_search_post(params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @developer_search_post_schema) do
Req.post(client(opts), url: "/search/developer", json: to_body(params, @developer_search_post_key_mapping))
end
end
@doc """
Bang variant of `developer_search_post`. Raises on error.
"""
@spec developer_search_post!(keyword(), keyword()) :: Req.Response.t()
def developer_search_post!(params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @developer_search_post_schema)
Req.post!(client(opts), url: "/search/developer", json: to_body(params, @developer_search_post_key_mapping))
end
@execute_browser_code_schema NimbleOptions.new!([
code: [type: :string, required: true, doc: "Code to execute in the browser sandbox"],
language: [type: {:or, [{:in, [:python, :node, :bash]}, :string]}, doc: "Language of the code to execute. Use `node` for JavaScript or `bash` for agent-browser CLI commands."],
timeout: [type: :integer, doc: "Execution timeout in seconds"]
])
@execute_browser_code_key_mapping %{code: "code", language: "language", timeout: "timeout"}
@doc """
Execute code in an interact session
`POST /interact/{sessionId}/execute`
Tag: Interact
## Path Parameters
* `session_id` - Path parameter `sessionId`
## Parameters
Validated by `NimbleOptions`. Pass params as a keyword list with snake_case keys.
See `@execute_browser_code_schema` for the full schema.
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec execute_browser_code(String.t(), keyword(), keyword()) :: response()
def execute_browser_code(session_id, params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @execute_browser_code_schema) do
Req.post(client(opts), url: "/interact/#{session_id}/execute", json: to_body(params, @execute_browser_code_key_mapping))
end
end
@doc """
Bang variant of `execute_browser_code`. Raises on error.
"""
@spec execute_browser_code!(String.t(), keyword(), keyword()) :: Req.Response.t()
def execute_browser_code!(session_id, params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @execute_browser_code_schema)
Req.post!(client(opts), url: "/interact/#{session_id}/execute", json: to_body(params, @execute_browser_code_key_mapping))
end
@doc """
Get all active crawls for the authenticated team
`GET /crawl/active`
Tag: Crawling
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec get_active_crawls(keyword()) :: response()
def get_active_crawls(opts \\ []) do
Req.get(client(opts), url: "/crawl/active")
end
@doc """
Bang variant of `get_active_crawls`. Raises on error.
"""
@spec get_active_crawls!(keyword()) :: Req.Response.t()
def get_active_crawls!(opts \\ []) do
Req.get!(client(opts), url: "/crawl/active")
end
@get_activity_query_schema NimbleOptions.new!([
endpoint: [type: {:or, [{:in, [:scrape, :crawl, :batch_scrape, :search, :extract, :llmstxt, :deep_research, :map, :agent, :browser, :interact]}, :string]}, doc: "Filter by endpoint"],
limit: [type: :integer, doc: "Maximum number of results per page"],
cursor: [type: :string, doc: "Cursor for pagination. Use the cursor value from the previous response."]
])
@get_activity_query_key_mapping %{endpoint: "endpoint", limit: "limit", cursor: "cursor"}
@doc """
List recent API activity
`GET /team/activity`
Tag: Account
## Query Parameters
* `endpoint` β query parameter `endpoint`
* `limit` β query parameter `limit`
* `cursor` β query parameter `cursor`
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec get_activity(keyword(), keyword()) :: response()
def get_activity(params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @get_activity_query_schema) do
Req.get(client(opts), url: "/team/activity", params: to_query(params, @get_activity_query_key_mapping))
end
end
@doc """
Bang variant of `get_activity`. Raises on error.
"""
@spec get_activity!(keyword(), keyword()) :: Req.Response.t()
def get_activity!(params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @get_activity_query_schema)
Req.get!(client(opts), url: "/team/activity", params: to_query(params, @get_activity_query_key_mapping))
end
@doc """
Get an output snapshot of an agent job
`GET /agent/{jobId}/snapshots/{snapshotId}`
Tag: Agent
## Path Parameters
* `job_id` - Path parameter `jobId`
* `snapshot_id` - Path parameter `snapshotId`
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec get_agent_snapshot(String.t(), String.t(), keyword()) :: response()
def get_agent_snapshot(job_id, snapshot_id, opts \\ []) do
Req.get(client(opts), url: "/agent/#{job_id}/snapshots/#{snapshot_id}")
end
@doc """
Bang variant of `get_agent_snapshot`. Raises on error.
"""
@spec get_agent_snapshot!(String.t(), String.t(), keyword()) :: Req.Response.t()
def get_agent_snapshot!(job_id, snapshot_id, opts \\ []) do
Req.get!(client(opts), url: "/agent/#{job_id}/snapshots/#{snapshot_id}")
end
@doc """
Get the status of an agent job
`GET /agent/{jobId}`
Tag: Agent
## Path Parameters
* `job_id` - Path parameter `jobId`
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec get_agent_status(String.t(), keyword()) :: response()
def get_agent_status(job_id, opts \\ []) do
Req.get(client(opts), url: "/agent/#{job_id}")
end
@doc """
Bang variant of `get_agent_status`. Raises on error.
"""
@spec get_agent_status!(String.t(), keyword()) :: Req.Response.t()
def get_agent_status!(job_id, opts \\ []) do
Req.get!(client(opts), url: "/agent/#{job_id}")
end
@doc """
Get the errors of a batch scrape job
`GET /batch/scrape/{id}/errors`
Tag: Scraping
## Path Parameters
* `id` - Path parameter `id`
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec get_batch_scrape_errors(String.t(), keyword()) :: response()
def get_batch_scrape_errors(id, opts \\ []) do
Req.get(client(opts), url: "/batch/scrape/#{id}/errors")
end
@doc """
Bang variant of `get_batch_scrape_errors`. Raises on error.
"""
@spec get_batch_scrape_errors!(String.t(), keyword()) :: Req.Response.t()
def get_batch_scrape_errors!(id, opts \\ []) do
Req.get!(client(opts), url: "/batch/scrape/#{id}/errors")
end
@doc """
Get the status of a batch scrape job
`GET /batch/scrape/{id}`
Tag: Scraping
## Path Parameters
* `id` - Path parameter `id`
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec get_batch_scrape_status(String.t(), keyword()) :: response()
def get_batch_scrape_status(id, opts \\ []) do
Req.get(client(opts), url: "/batch/scrape/#{id}")
end
@doc """
Bang variant of `get_batch_scrape_status`. Raises on error.
"""
@spec get_batch_scrape_status!(String.t(), keyword()) :: Req.Response.t()
def get_batch_scrape_status!(id, opts \\ []) do
Req.get!(client(opts), url: "/batch/scrape/#{id}")
end
@doc """
Get the errors of a crawl job
`GET /crawl/{id}/errors`
Tag: Crawling
## Path Parameters
* `id` - Path parameter `id`
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec get_crawl_errors(String.t(), keyword()) :: response()
def get_crawl_errors(id, opts \\ []) do
Req.get(client(opts), url: "/crawl/#{id}/errors")
end
@doc """
Bang variant of `get_crawl_errors`. Raises on error.
"""
@spec get_crawl_errors!(String.t(), keyword()) :: Req.Response.t()
def get_crawl_errors!(id, opts \\ []) do
Req.get!(client(opts), url: "/crawl/#{id}/errors")
end
@doc """
Get the status of a crawl job
`GET /crawl/{id}`
Tag: Crawling
## Path Parameters
* `id` - Path parameter `id`
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec get_crawl_status(String.t(), keyword()) :: response()
def get_crawl_status(id, opts \\ []) do
Req.get(client(opts), url: "/crawl/#{id}")
end
@doc """
Bang variant of `get_crawl_status`. Raises on error.
"""
@spec get_crawl_status!(String.t(), keyword()) :: Req.Response.t()
def get_crawl_status!(id, opts \\ []) do
Req.get!(client(opts), url: "/crawl/#{id}")
end
@doc """
Get remaining credits for the authenticated team
`GET /team/credit-usage`
Tag: Billing
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec get_credit_usage(keyword()) :: response()
def get_credit_usage(opts \\ []) do
Req.get(client(opts), url: "/team/credit-usage")
end
@doc """
Bang variant of `get_credit_usage`. Raises on error.
"""
@spec get_credit_usage!(keyword()) :: Req.Response.t()
def get_credit_usage!(opts \\ []) do
Req.get!(client(opts), url: "/team/credit-usage")
end
@get_historical_credit_usage_query_schema NimbleOptions.new!([
by_api_key: [type: :boolean, doc: "Get historical credit usage by API key"]
])
@get_historical_credit_usage_query_key_mapping %{by_api_key: "byApiKey"}
@doc """
Get historical credit usage for the authenticated team
`GET /team/credit-usage/historical`
Tag: Billing
## Query Parameters
* `by_api_key` β query parameter `byApiKey`
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec get_historical_credit_usage(keyword(), keyword()) :: response()
def get_historical_credit_usage(params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @get_historical_credit_usage_query_schema) do
Req.get(client(opts), url: "/team/credit-usage/historical", params: to_query(params, @get_historical_credit_usage_query_key_mapping))
end
end
@doc """
Bang variant of `get_historical_credit_usage`. Raises on error.
"""
@spec get_historical_credit_usage!(keyword(), keyword()) :: Req.Response.t()
def get_historical_credit_usage!(params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @get_historical_credit_usage_query_schema)
Req.get!(client(opts), url: "/team/credit-usage/historical", params: to_query(params, @get_historical_credit_usage_query_key_mapping))
end
@doc """
Metrics about your team's scrape queue
`GET /team/queue-status`
Tag: Miscellaneous
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec get_queue_status(keyword()) :: response()
def get_queue_status(opts \\ []) do
Req.get(client(opts), url: "/team/queue-status")
end
@doc """
Bang variant of `get_queue_status`. Raises on error.
"""
@spec get_queue_status!(keyword()) :: Req.Response.t()
def get_queue_status!(opts \\ []) do
Req.get!(client(opts), url: "/team/queue-status")
end
@doc """
Get the status of a scrape job
`GET /scrape/{jobId}`
Tag: Scraping
## Path Parameters
* `job_id` - Path parameter `jobId`
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec get_scrape_status(String.t(), keyword()) :: response()
def get_scrape_status(job_id, opts \\ []) do
Req.get(client(opts), url: "/scrape/#{job_id}")
end
@doc """
Bang variant of `get_scrape_status`. Raises on error.
"""
@spec get_scrape_status!(String.t(), keyword()) :: Req.Response.t()
def get_scrape_status!(job_id, opts \\ []) do
Req.get!(client(opts), url: "/scrape/#{job_id}")
end
@doc """
Get the team's threat protection policy
`GET /team/threat-protection`
Tag: Threat Protection
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec get_threat_protection(keyword()) :: response()
def get_threat_protection(opts \\ []) do
Req.get(client(opts), url: "/team/threat-protection")
end
@doc """
Bang variant of `get_threat_protection`. Raises on error.
"""
@spec get_threat_protection!(keyword()) :: Req.Response.t()
def get_threat_protection!(opts \\ []) do
Req.get!(client(opts), url: "/team/threat-protection")
end
@gov_search_query_schema NimbleOptions.new!([
query: [type: :string, required: true, doc: "Natural-language question or search phrase."],
k: [type: :integer, doc: "Number of results to return."],
jurisdiction: [type: :string, doc: "Keep only pages of one jurisdiction: `us` for federal, or `us-` plus a two-letter state or territory code, such as `us-ca` or `us-dc`."],
court: [type: :string, doc: "Keep only pages labeled for one court, by CourtListener court id, such as `scotus`, `ca9` or `nysd`."],
court_type: [type: {:or, [{:in, [:"federal-supreme", :"federal-appellate", :"federal-district", :"federal-bankruptcy", :"federal-special", :military, :"state-supreme", :"state-appellate", :"state-trial", :"state-special", :"state-attorney-general"]}, :string]}, doc: "Keep only pages from one type of court: `federal-supreme`, `federal-appellate`, `federal-district`, `federal-bankruptcy`, `federal-special`, `military`, `state-supreme`, `state-appellate`, `state-trial`, `state-special`, `state-attorney-general`."],
cites: [type: :string, doc: "Keep only pages that cite one law or case, as a printed citation such as `42 U.S.C. Β§ 1983` or `5 U.S. 137`, or a citation id such as `us:usc:42:1983`."],
domain: [type: :string, doc: "Keep only pages from one site, such as `irs.gov`."]
])
@gov_search_query_key_mapping %{query: "query", k: "k", jurisdiction: "jurisdiction", court: "court", court_type: "courtType", cites: "cites", domain: "domain"}
@doc """
Search the Government Index
`GET /search/gov`
Tag: Government
## Query Parameters
* `query` β query parameter `query`
* `k` β query parameter `k`
* `jurisdiction` β query parameter `jurisdiction`
* `court` β query parameter `court`
* `court_type` β query parameter `courtType`
* `cites` β query parameter `cites`
* `domain` β query parameter `domain`
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec gov_search(keyword(), keyword()) :: response()
def gov_search(params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @gov_search_query_schema) do
Req.get(client(opts), url: "/search/gov", params: to_query(params, @gov_search_query_key_mapping))
end
end
@doc """
Bang variant of `gov_search`. Raises on error.
"""
@spec gov_search!(keyword(), keyword()) :: Req.Response.t()
def gov_search!(params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @gov_search_query_schema)
Req.get!(client(opts), url: "/search/gov", params: to_query(params, @gov_search_query_key_mapping))
end
@gov_search_post_schema NimbleOptions.new!([
cites: [type: :string, doc: "Keep only pages that cite one law or case, as a printed citation such as `42 U.S.C. Β§ 1983` or `5 U.S. 137`, or a citation id such as `us:usc:42:1983`."],
court: [type: :string, doc: "Keep only pages labeled for one court, by CourtListener court id, such as `scotus`, `ca9` or `nysd`."],
court_type: [type: {:or, [{:in, [:"federal-supreme", :"federal-appellate", :"federal-district", :"federal-bankruptcy", :"federal-special", :military, :"state-supreme", :"state-appellate", :"state-trial", :"state-special", :"state-attorney-general"]}, :string]}, doc: "Keep only pages from one type of court: `federal-supreme`, `federal-appellate`, `federal-district`, `federal-bankruptcy`, `federal-special`, `military`, `state-supreme`, `state-appellate`, `state-trial`, `state-special`, `state-attorney-general`."],
domain: [type: :string, doc: "Keep only pages from one site, such as `irs.gov`."],
jurisdiction: [type: :string, doc: "Keep only pages of one jurisdiction: `us` for federal, or `us-` plus a two-letter state or territory code, such as `us-ca` or `us-dc`."],
k: [type: :integer, doc: "Number of results to return."],
query: [type: :string, required: true, doc: "Natural-language question or search phrase."]
])
@gov_search_post_key_mapping %{cites: "cites", court: "court", court_type: "courtType", domain: "domain", jurisdiction: "jurisdiction", k: "k", query: "query"}
@doc """
Search the Government Index
`POST /search/gov`
Tag: Government
## Parameters
Validated by `NimbleOptions`. Pass params as a keyword list with snake_case keys.
See `@gov_search_post_schema` for the full schema.
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec gov_search_post(keyword(), keyword()) :: response()
def gov_search_post(params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @gov_search_post_schema) do
Req.post(client(opts), url: "/search/gov", json: to_body(params, @gov_search_post_key_mapping))
end
end
@doc """
Bang variant of `gov_search_post`. Raises on error.
"""
@spec gov_search_post!(keyword(), keyword()) :: Req.Response.t()
def gov_search_post!(params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @gov_search_post_schema)
Req.post!(client(opts), url: "/search/gov", json: to_body(params, @gov_search_post_key_mapping))
end
@interact_with_scrape_browser_session_schema NimbleOptions.new!([
code: [type: :string, required: true, doc: "Code to execute in the scrape-bound browser sandbox"],
language: [type: {:or, [{:in, [:python, :node, :bash]}, :string]}, doc: "Language of the code to execute. Use `node` for JavaScript or `bash` for agent-browser CLI commands."],
origin: [type: :string, doc: "Optional origin label used for execution telemetry"],
timeout: [type: :integer, doc: "Execution timeout in seconds"]
])
@interact_with_scrape_browser_session_key_mapping %{code: "code", language: "language", origin: "origin", timeout: "timeout"}
@doc """
Interact with the browser session associated with a scrape job
`POST /scrape/{jobId}/interact`
Tag: Scraping
## Path Parameters
* `job_id` - Path parameter `jobId`
## Parameters
Validated by `NimbleOptions`. Pass params as a keyword list with snake_case keys.
See `@interact_with_scrape_browser_session_schema` for the full schema.
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec interact_with_scrape_browser_session(String.t(), keyword(), keyword()) :: response()
def interact_with_scrape_browser_session(job_id, params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @interact_with_scrape_browser_session_schema) do
Req.post(client(opts), url: "/scrape/#{job_id}/interact", json: to_body(params, @interact_with_scrape_browser_session_key_mapping))
end
end
@doc """
Bang variant of `interact_with_scrape_browser_session`. Raises on error.
"""
@spec interact_with_scrape_browser_session!(String.t(), keyword(), keyword()) :: Req.Response.t()
def interact_with_scrape_browser_session!(job_id, params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @interact_with_scrape_browser_session_schema)
Req.post!(client(opts), url: "/scrape/#{job_id}/interact", json: to_body(params, @interact_with_scrape_browser_session_key_mapping))
end
@list_agents_query_schema NimbleOptions.new!([
before: [type: :integer, doc: "Only return agent runs created before this unix millisecond timestamp. Pages are fixed at 20 runs; use the `before` value from the previous page's `next` URL to fetch the next page."]
])
@list_agents_query_key_mapping %{before: "before"}
@doc """
List agent runs
`GET /agent`
Tag: Agent
## Query Parameters
* `before` β query parameter `before`
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec list_agents(keyword(), keyword()) :: response()
def list_agents(params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @list_agents_query_schema) do
Req.get(client(opts), url: "/agent", params: to_query(params, @list_agents_query_key_mapping))
end
end
@doc """
Bang variant of `list_agents`. Raises on error.
"""
@spec list_agents!(keyword(), keyword()) :: Req.Response.t()
def list_agents!(params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @list_agents_query_schema)
Req.get!(client(opts), url: "/agent", params: to_query(params, @list_agents_query_key_mapping))
end
@list_browser_sessions_query_schema NimbleOptions.new!([
status: [type: {:or, [{:in, [:active, :destroyed]}, :string]}, doc: "Filter sessions by status"]
])
@list_browser_sessions_query_key_mapping %{status: "status"}
@doc """
List interact sessions
`GET /interact`
Tag: Interact
## Query Parameters
* `status` β query parameter `status`
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec list_browser_sessions(keyword(), keyword()) :: response()
def list_browser_sessions(params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @list_browser_sessions_query_schema) do
Req.get(client(opts), url: "/interact", params: to_query(params, @list_browser_sessions_query_key_mapping))
end
end
@doc """
Bang variant of `list_browser_sessions`. Raises on error.
"""
@spec list_browser_sessions!(keyword(), keyword()) :: Req.Response.t()
def list_browser_sessions!(params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @list_browser_sessions_query_schema)
Req.get!(client(opts), url: "/interact", params: to_query(params, @list_browser_sessions_query_key_mapping))
end
@map_urls_schema NimbleOptions.new!([
audit_metadata: [type: :keyword_list, keys: [username: [type: :string, required: true]], doc: "User attribution included with SIEM logging events when SIEM Logging is enabled for the organization."],
ignore_cache: [type: :boolean, doc: "Bypass the sitemap cache to retrieve fresh URLs. Sitemap data is cached for up to 7 days; use this parameter when your sitemap has been recently updated."],
ignore_query_parameters: [type: :boolean, doc: "Do not return URLs with query parameters"],
include_subdomains: [type: :boolean, doc: "Include subdomains of the website"],
limit: [type: :integer, doc: "Maximum number of links to return"],
location: [type: :keyword_list, doc: "Location settings for the request. When specified, this will use an appropriate proxy if available and emulate the corresponding language and timezone settings. Defaults to 'US' if not specified."],
search: [type: :string, doc: "Specify a search query to order the results by relevance. Example: 'blog' will return URLs that contain the word 'blog' in the URL ordered by relevance."],
sitemap: [type: {:or, [{:in, [:skip, :include, :only]}, :string]}, doc: "Sitemap mode when mapping. If you set it to `skip`, the sitemap won't be used to find URLs. If you set it to `only`, only URLs that are in the sitemap will be returned. By default (`include`), the sitemap and other methods will be used together to find URLs."],
threat_protection: [type: :keyword_list, doc: "Per-request [Threat Protection](https://docs.firecrawl.dev/features/threat-protection) override. Fields you provide replace the corresponding fields of your organization's policy for this request only; omitted fields keep their organization-level values. Requires Threat Protection to be enabled for your team (enterprise feature) β otherwise the request is rejected with a 403. If your organization has disabled request overrides, any request that includes this object is rejected with a 403. If Threat Protection is enforced for your team, `mode` may not be set to `off`."],
timeout: [type: :integer, doc: "Timeout in milliseconds. There is no timeout by default."],
url: [type: :string, required: true, doc: "The base URL to start crawling from"]
])
@map_urls_key_mapping %{audit_metadata: {"auditMetadata", %{username: "username"}}, ignore_cache: "ignoreCache", ignore_query_parameters: "ignoreQueryParameters", include_subdomains: "includeSubdomains", limit: "limit", location: "location", search: "search", sitemap: "sitemap", threat_protection: "threatProtection", timeout: "timeout", url: "url"}
@doc """
Map multiple URLs based on options
`POST /map`
Tag: Mapping
## Parameters
Validated by `NimbleOptions`. Pass params as a keyword list with snake_case keys.
See `@map_urls_schema` for the full schema.
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec map_urls(keyword(), keyword()) :: response()
def map_urls(params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @map_urls_schema) do
Req.post(client(opts), url: "/map", json: to_body(params, @map_urls_key_mapping))
end
end
@doc """
Bang variant of `map_urls`. Raises on error.
"""
@spec map_urls!(keyword(), keyword()) :: Req.Response.t()
def map_urls!(params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @map_urls_schema)
Req.post!(client(opts), url: "/map", json: to_body(params, @map_urls_key_mapping))
end
@parse_file_schema NimbleOptions.new!([
audit_metadata: [type: :keyword_list, keys: [username: [type: :string, required: true]], doc: "User attribution included with SIEM logging events when SIEM Logging is enabled for the organization."],
block_ads: [type: :boolean, doc: "Enable ad and cookie popup blocking."],
check_prompt_injection: [type: :boolean, doc: "When enabled, scans the parsed content for prompt injection attempts before any LLM-backed format (such as json, summary or question) runs. If an injection is detected, the request fails with a 403 and error code SCRAPE_PROMPT_INJECTION_DETECTED. Adds 4 credits when the check runs. If the check cannot scan all of the content, the parse continues with a warning and the check is not billed."],
exclude_tags: [type: {:list, :string}, doc: "Tags to exclude from the output."],
formats: [type: {:list, :any}, doc: "Output formats supported for `/parse` uploads. Browser-rendering formats and change tracking are not supported."],
headers: [type: :any, doc: "Headers to send when additional network requests are required."],
include_tags: [type: {:list, :string}, doc: "Tags to include in the output."],
integration: [type: :string, doc: "Optional integration identifier."],
only_main_content: [type: :boolean, doc: "Only return the main content of the page excluding headers, navs, footers, etc."],
origin: [type: :string, doc: "Origin identifier for analytics and logging."],
parsers: [type: {:list, :any}, doc: "Controls file parser behavior when relevant (for example PDF parser mode)."],
proxy: [type: {:or, [{:in, [:basic, :auto]}, :string]}, doc: "Proxy mode for parse uploads. `/parse` supports only `basic` and `auto`."],
redact_pii: [type: :any, doc: "Redact personally identifiable information from returned markdown. Pass `true` to use defaults, or an object to tune mode, entities, and replacement style."],
remove_base64_images: [type: :boolean, doc: "Remove base64-encoded images from output and keep alt text placeholders."],
skip_tls_verification: [type: :boolean, doc: "Skip TLS certificate verification when making requests."],
timeout: [type: :integer, doc: "Timeout in milliseconds for the request. Default is 30000 (30 seconds). Maximum is 300000 (300 seconds)."],
zero_data_retention: [type: :boolean, doc: "If true, this will enable zero data retention for this parse. To enable this feature, please contact [email protected]"]
])
@parse_file_key_mapping %{audit_metadata: {"auditMetadata", %{username: "username"}}, block_ads: "blockAds", check_prompt_injection: "checkPromptInjection", exclude_tags: "excludeTags", formats: "formats", headers: "headers", include_tags: "includeTags", integration: "integration", only_main_content: "onlyMainContent", origin: "origin", parsers: "parsers", proxy: "proxy", redact_pii: "redactPII", remove_base64_images: "removeBase64Images", skip_tls_verification: "skipTlsVerification", timeout: "timeout", zero_data_retention: "zeroDataRetention"}
@doc """
Upload and parse a file
`POST /parse`
Sends a `multipart/form-data` request.
Tag: Scraping
## File
Pass `file` as a keyword list:
* `:filename` (required) - The filename to send.
* `:data` (required) - The file contents as a binary.
* `:content_type` (optional) - The MIME type of the file.
## Parameters
Validated by `NimbleOptions`. Pass options as a keyword list with snake_case keys.
These are JSON-encoded and sent as the `options` multipart field.
See `@parse_file_schema` for the full schema.
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec parse_file(keyword(), keyword(), keyword()) :: response()
def parse_file(file, params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @parse_file_schema),
{:ok, filename} <- fetch_file_field(file, :filename),
:ok <- validate_filename(filename),
{:ok, data} <- fetch_file_field(file, :data),
:ok <- validate_data(data) do
content_type = Keyword.get(file, :content_type)
file_part =
case content_type do
nil -> {data, filename: filename}
ct -> {data, filename: filename, content_type: ct}
end
multipart = [{"options", Jason.encode!(to_body(params, @parse_file_key_mapping))}, {"file", file_part}]
Req.post(client(opts), url: "/parse", form_multipart: multipart)
end
end
@doc """
Bang variant of `parse_file`. Raises on error.
"""
@spec parse_file!(keyword(), keyword(), keyword()) :: Req.Response.t()
def parse_file!(file, params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @parse_file_schema)
filename = Keyword.fetch!(file, :filename)
data = Keyword.fetch!(file, :data)
content_type = Keyword.get(file, :content_type)
if not is_binary(filename) or filename == "" do
raise ArgumentError, "filename cannot be empty"
end
if is_nil(data) do
raise ArgumentError, "file data cannot be empty"
end
file_part =
case content_type do
nil -> {data, filename: filename}
ct -> {data, filename: filename, content_type: ct}
end
multipart = [{"options", Jason.encode!(to_body(params, @parse_file_key_mapping))}, {"file", file_part}]
Req.post!(client(opts), url: "/parse", form_multipart: multipart)
end
@scrape_and_extract_from_url_schema NimbleOptions.new!([
url: [type: :string, required: true, doc: "The URL to scrape"],
actions: [type: {:list, :any}, doc: "Actions to perform on the page before grabbing the content"],
audit_metadata: [type: :keyword_list, keys: [username: [type: :string, required: true]], doc: "User attribution included with SIEM logging events when SIEM Logging is enabled for the organization."],
block_ads: [type: :boolean, doc: "Enables ad-blocking and cookie popup blocking."],
check_prompt_injection: [type: :boolean, doc: "When enabled, scans the page content for prompt injection attempts before any LLM-backed format (such as json, summary or question) runs. Works with any format except rawBase64. If an injection is detected, the request fails with a 403 and error code SCRAPE_PROMPT_INJECTION_DETECTED. Adds 4 credits when the check runs. If the check cannot scan all of the content, the scrape continues with a warning and the check is not billed."],
exclude_tags: [type: {:list, :string}, doc: "Tags to exclude from the output."],
formats: [type: {:list, :any}, doc: "Output formats to include in the response. You can specify one or more formats, either as strings (e.g., `'markdown'`) or as objects with additional options (e.g., `{ type: 'json', schema: {...} }`). Some formats require specific options to be set. Example: `['markdown', { type: 'json', schema: {...} }]`."],
headers: [type: :any, doc: "Headers to send with the request. Can be used to send cookies, user-agent, etc."],
include_tags: [type: {:list, :string}, doc: "Tags to include in the output."],
location: [type: :keyword_list, doc: "Location settings for the request. When specified, this will use an appropriate proxy if available and emulate the corresponding language and timezone settings. Defaults to 'US' if not specified."],
lockdown: [type: :boolean, doc: "If true, serves the request from Firecrawl's cache only and never makes an outbound request to the target URL. Designed for compliance-constrained or air-gapped environments where the scrape request itself could leak sensitive information. On cache miss, returns a 404 with error code SCRAPE_LOCKDOWN_CACHE_MISS (the URL is never logged on miss). Lockdown requests are treated as zero data retention. Default maxAge is extended to 2 years so existing cached pages remain eligible. Billed at 5 credits on hit, 1 credit on cache miss."],
max_age: [type: :integer, doc: "Returns a cached version of the page if it is younger than this age in milliseconds. If a cached version of the page is older than this value, the page will be scraped. If you do not need extremely fresh data, enabling this can speed up your scrapes by 500%. Defaults to 2 days."],
min_age: [type: :integer, doc: "When set, the request only checks the cache and never triggers a fresh scrape. The value is in milliseconds and specifies the minimum age the cached data must be. If matching cached data exists, it is returned instantly. If no cached data is found, a 404 with error code SCRAPE_NO_CACHED_DATA is returned. Set to 1 to accept any cached data regardless of age."],
mobile: [type: :boolean, doc: "Set to true if you want to emulate scraping from a mobile device. Useful for testing responsive pages and taking mobile screenshots."],
only_clean_content: [type: :boolean, doc: "Beta. Run an additional LLM-based pass over the generated markdown to remove residual boilerplate that `onlyMainContent` can miss (cookie banners, ad blocks, social share widgets, breadcrumbs, newsletter signups, comment sections, related-article lists). Headings, lists, tables, code blocks, image references, and inline links are preserved. Can be combined with `onlyMainContent` (the most common setup) or used on its own. Skipped with a warning when the markdown exceeds the cleaning model's output token limit (the original markdown is preserved). Not supported on zero-data-retention requests."],
only_main_content: [type: :boolean, doc: "Only return the main content of the page excluding headers, navs, footers, etc. This is a deterministic HTML-level filter applied before markdown is generated; no LLM is involved."],
parsers: [type: {:list, :any}, doc: "Controls how files are processed during scraping. Each item is a parser name (`\"pdf\"` or `\"image\"`) or a parser object. Both parsers are on by default.\n\n - **pdf**: PDF content is extracted and converted to markdown format, with billing based on the number of pages (1 credit per page).\n - **image**: Raster images (PNG, JPEG, JPEG 2000, TIFF, GIF, BMP, WebP, AVIF) are OCR'd as one-page documents and converted to markdown. SVG and HEIC are not supported.\n\nIf you pass a list, parsers that are not in the list are off. For example, `[\"pdf\"]` turns off image OCR, so image URLs fail as unsupported files. When an empty array is passed, the PDF file is returned in base64 encoding with a flat rate of 1 credit for the entire PDF."],
profile: [type: :keyword_list, doc: "Enable persistent browser storage across scrape and interact sessions. Pass a profile when scraping to preserve cookies, localStorage, and session data. Sessions with the same profile name share browser state."],
proxy: [type: {:or, [{:in, [:basic, :enhanced, :auto]}, :string]}, doc: "Specifies the type of proxy to use.\n\n - **basic**: Proxies for scraping sites with none to basic anti-bot solutions. Fast and usually works.\n - **enhanced**: Enhanced proxies for scraping sites with advanced anti-bot solutions. Slower, but more reliable on certain sites. Billed at the same credit cost as basic.\n - **auto**: Firecrawl will automatically retry scraping with enhanced proxies if the basic proxy fails. Enhanced proxies carry no credit surcharge, so either way only the regular cost is billed."],
redact_pii: [type: :any, doc: "Redact personally identifiable information from returned markdown. Pass `true` to use defaults, or an object to tune mode, entities, and replacement style."],
remove_base64_images: [type: :boolean, doc: "Removes all base 64 images from the markdown output, which may be overwhelmingly long. This does not affect html or rawHtml formats. The image's alt text remains in the output, but the URL is replaced with a placeholder."],
skip_tls_verification: [type: :boolean, doc: "Skip TLS certificate verification when making requests."],
store_in_cache: [type: :boolean, doc: "If true, the page will be stored in the Firecrawl index and cache. Setting this to false is useful if your scraping activity may have data protection concerns. Using some parameters associated with sensitive scraping (e.g. actions, headers) will force this parameter to be false."],
threat_protection: [type: :keyword_list, doc: "Per-request [Threat Protection](https://docs.firecrawl.dev/features/threat-protection) override. Fields you provide replace the corresponding fields of your organization's policy for this request only; omitted fields keep their organization-level values. Requires Threat Protection to be enabled for your team (enterprise feature) β otherwise the request is rejected with a 403. If your organization has disabled request overrides, any request that includes this object is rejected with a 403. If Threat Protection is enforced for your team, `mode` may not be set to `off`."],
timeout: [type: :integer, doc: "Timeout in milliseconds for the request. Minimum is 1000 (1 second). Default is 60000 (60 seconds). Maximum is 300000 (300 seconds)."],
wait_for: [type: :integer, doc: "Specify a delay in milliseconds before fetching the content, allowing the page sufficient time to load. This waiting time is in addition to Firecrawl's smart wait feature."],
zero_data_retention: [type: :boolean, doc: "If true, this will enable zero data retention for this scrape. To enable this feature, please contact [email protected]"],
alexandria: [type: :any, doc: "Execute one or more catalogued provider tools instead of scraping a URL. Cannot be combined with `url`, `formats`, or other scrape options (400); the only allowed sibling keys are `timeout`, `origin`, and `integration`. Use the `x-request-id` request header as a client-chosen idempotency key (1 to 128 characters of letters, digits, `.`, `_`, `:`, `-`) β it is echoed back, and a completed request with the same key replays its original response and `scrape_id` instead of re-executing. To browse the catalogue and read tool contracts for free, call Find Tools: set `provider` to `firecrawl`, `capability` to `find-tools`, and `options` to a `FindToolsOptions` object. The result `data` is a `FindToolsData` page."],
domain_tools: [type: :boolean, doc: "When true on an ordinary URL scrape, `data.tools` lists tool contracts matched to the scraped page's domain (same `DiscoveredTool` shape as search). Requires the team's Alexandria access to be enabled and no zero data retention (403 otherwise). Free."],
tool_detail: [type: {:or, [{:in, [:compact, :summary, :full]}, :string]}, doc: "How much of each tool contract to return in `data.tools` when `domainTools` is true. `compact` returns only `provider`, `capability`, and `description`. `summary` returns the full tool entry without `options` and `response`. `full` also returns `options`, `response`, and `examples`. If you omit it, tools are returned as with `summary`."]
])
@scrape_and_extract_from_url_key_mapping %{url: "url", actions: "actions", audit_metadata: {"auditMetadata", %{username: "username"}}, block_ads: "blockAds", check_prompt_injection: "checkPromptInjection", exclude_tags: "excludeTags", formats: "formats", headers: "headers", include_tags: "includeTags", location: "location", lockdown: "lockdown", max_age: "maxAge", min_age: "minAge", mobile: "mobile", only_clean_content: "onlyCleanContent", only_main_content: "onlyMainContent", parsers: "parsers", profile: "profile", proxy: "proxy", redact_pii: "redactPII", remove_base64_images: "removeBase64Images", skip_tls_verification: "skipTlsVerification", store_in_cache: "storeInCache", threat_protection: "threatProtection", timeout: "timeout", wait_for: "waitFor", zero_data_retention: "zeroDataRetention", alexandria: "alexandria", domain_tools: "domainTools", tool_detail: "toolDetail"}
@doc """
Scrape a single URL and optionally extract information using an LLM
`POST /scrape`
Tag: Scraping
## Parameters
Validated by `NimbleOptions`. Pass params as a keyword list with snake_case keys.
See `@scrape_and_extract_from_url_schema` for the full schema.
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec scrape_and_extract_from_url(keyword(), keyword()) :: response()
def scrape_and_extract_from_url(params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @scrape_and_extract_from_url_schema) do
Req.post(client(opts), url: "/scrape", json: to_body(params, @scrape_and_extract_from_url_key_mapping))
end
end
@doc """
Bang variant of `scrape_and_extract_from_url`. Raises on error.
"""
@spec scrape_and_extract_from_url!(keyword(), keyword()) :: Req.Response.t()
def scrape_and_extract_from_url!(params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @scrape_and_extract_from_url_schema)
Req.post!(client(opts), url: "/scrape", json: to_body(params, @scrape_and_extract_from_url_key_mapping))
end
@scrape_and_extract_from_urls_schema NimbleOptions.new!([
ignore_invalid_urls: [type: :boolean, doc: "If invalid URLs are specified in the urls array, they will be ignored. Instead of them failing the entire request, a batch scrape using the remaining valid URLs will be created, and the invalid URLs will be returned in the invalidURLs field of the response."],
max_concurrency: [type: :integer, doc: "Maximum number of concurrent scrapes. This parameter allows you to set a concurrency limit for this batch scrape. If not specified, the batch scrape adheres to your team's concurrency limit."],
urls: [type: {:list, :string}, required: true],
webhook: [type: :keyword_list, doc: "A webhook specification object."],
actions: [type: {:list, :any}, doc: "Actions to perform on the page before grabbing the content"],
audit_metadata: [type: :keyword_list, keys: [username: [type: :string, required: true]], doc: "User attribution included with SIEM logging events when SIEM Logging is enabled for the organization."],
block_ads: [type: :boolean, doc: "Enables ad-blocking and cookie popup blocking."],
check_prompt_injection: [type: :boolean, doc: "When enabled, scans the page content for prompt injection attempts before any LLM-backed format (such as json, summary or question) runs. Works with any format except rawBase64. If an injection is detected, the request fails with a 403 and error code SCRAPE_PROMPT_INJECTION_DETECTED. Adds 4 credits when the check runs. If the check cannot scan all of the content, the scrape continues with a warning and the check is not billed."],
exclude_tags: [type: {:list, :string}, doc: "Tags to exclude from the output."],
formats: [type: {:list, :any}, doc: "Output formats to include in the response. You can specify one or more formats, either as strings (e.g., `'markdown'`) or as objects with additional options (e.g., `{ type: 'json', schema: {...} }`). Some formats require specific options to be set. Example: `['markdown', { type: 'json', schema: {...} }]`."],
headers: [type: :any, doc: "Headers to send with the request. Can be used to send cookies, user-agent, etc."],
include_tags: [type: {:list, :string}, doc: "Tags to include in the output."],
location: [type: :keyword_list, doc: "Location settings for the request. When specified, this will use an appropriate proxy if available and emulate the corresponding language and timezone settings. Defaults to 'US' if not specified."],
lockdown: [type: :boolean, doc: "If true, serves the request from Firecrawl's cache only and never makes an outbound request to the target URL. Designed for compliance-constrained or air-gapped environments where the scrape request itself could leak sensitive information. On cache miss, returns a 404 with error code SCRAPE_LOCKDOWN_CACHE_MISS (the URL is never logged on miss). Lockdown requests are treated as zero data retention. Default maxAge is extended to 2 years so existing cached pages remain eligible. Billed at 5 credits on hit, 1 credit on cache miss."],
max_age: [type: :integer, doc: "Returns a cached version of the page if it is younger than this age in milliseconds. If a cached version of the page is older than this value, the page will be scraped. If you do not need extremely fresh data, enabling this can speed up your scrapes by 500%. Defaults to 2 days."],
min_age: [type: :integer, doc: "When set, the request only checks the cache and never triggers a fresh scrape. The value is in milliseconds and specifies the minimum age the cached data must be. If matching cached data exists, it is returned instantly. If no cached data is found, a 404 with error code SCRAPE_NO_CACHED_DATA is returned. Set to 1 to accept any cached data regardless of age."],
mobile: [type: :boolean, doc: "Set to true if you want to emulate scraping from a mobile device. Useful for testing responsive pages and taking mobile screenshots."],
only_clean_content: [type: :boolean, doc: "Beta. Run an additional LLM-based pass over the generated markdown to remove residual boilerplate that `onlyMainContent` can miss (cookie banners, ad blocks, social share widgets, breadcrumbs, newsletter signups, comment sections, related-article lists). Headings, lists, tables, code blocks, image references, and inline links are preserved. Can be combined with `onlyMainContent` (the most common setup) or used on its own. Skipped with a warning when the markdown exceeds the cleaning model's output token limit (the original markdown is preserved). Not supported on zero-data-retention requests."],
only_main_content: [type: :boolean, doc: "Only return the main content of the page excluding headers, navs, footers, etc. This is a deterministic HTML-level filter applied before markdown is generated; no LLM is involved."],
parsers: [type: {:list, :any}, doc: "Controls how files are processed during scraping. Each item is a parser name (`\"pdf\"` or `\"image\"`) or a parser object. Both parsers are on by default.\n\n - **pdf**: PDF content is extracted and converted to markdown format, with billing based on the number of pages (1 credit per page).\n - **image**: Raster images (PNG, JPEG, JPEG 2000, TIFF, GIF, BMP, WebP, AVIF) are OCR'd as one-page documents and converted to markdown. SVG and HEIC are not supported.\n\nIf you pass a list, parsers that are not in the list are off. For example, `[\"pdf\"]` turns off image OCR, so image URLs fail as unsupported files. When an empty array is passed, the PDF file is returned in base64 encoding with a flat rate of 1 credit for the entire PDF."],
profile: [type: :keyword_list, doc: "Enable persistent browser storage across scrape and interact sessions. Pass a profile when scraping to preserve cookies, localStorage, and session data. Sessions with the same profile name share browser state."],
proxy: [type: {:or, [{:in, [:basic, :enhanced, :auto]}, :string]}, doc: "Specifies the type of proxy to use.\n\n - **basic**: Proxies for scraping sites with none to basic anti-bot solutions. Fast and usually works.\n - **enhanced**: Enhanced proxies for scraping sites with advanced anti-bot solutions. Slower, but more reliable on certain sites. Billed at the same credit cost as basic.\n - **auto**: Firecrawl will automatically retry scraping with enhanced proxies if the basic proxy fails. Enhanced proxies carry no credit surcharge, so either way only the regular cost is billed."],
redact_pii: [type: :any, doc: "Redact personally identifiable information from returned markdown. Pass `true` to use defaults, or an object to tune mode, entities, and replacement style."],
remove_base64_images: [type: :boolean, doc: "Removes all base 64 images from the markdown output, which may be overwhelmingly long. This does not affect html or rawHtml formats. The image's alt text remains in the output, but the URL is replaced with a placeholder."],
skip_tls_verification: [type: :boolean, doc: "Skip TLS certificate verification when making requests."],
store_in_cache: [type: :boolean, doc: "If true, the page will be stored in the Firecrawl index and cache. Setting this to false is useful if your scraping activity may have data protection concerns. Using some parameters associated with sensitive scraping (e.g. actions, headers) will force this parameter to be false."],
threat_protection: [type: :keyword_list, doc: "Per-request [Threat Protection](https://docs.firecrawl.dev/features/threat-protection) override. Fields you provide replace the corresponding fields of your organization's policy for this request only; omitted fields keep their organization-level values. Requires Threat Protection to be enabled for your team (enterprise feature) β otherwise the request is rejected with a 403. If your organization has disabled request overrides, any request that includes this object is rejected with a 403. If Threat Protection is enforced for your team, `mode` may not be set to `off`."],
timeout: [type: :integer, doc: "Timeout in milliseconds for the request. Minimum is 1000 (1 second). Default is 60000 (60 seconds). Maximum is 300000 (300 seconds)."],
wait_for: [type: :integer, doc: "Specify a delay in milliseconds before fetching the content, allowing the page sufficient time to load. This waiting time is in addition to Firecrawl's smart wait feature."],
zero_data_retention: [type: :boolean, doc: "If true, this will enable zero data retention for this batch scrape. To enable this feature, please contact [email protected]"]
])
@scrape_and_extract_from_urls_key_mapping %{ignore_invalid_urls: "ignoreInvalidURLs", max_concurrency: "maxConcurrency", urls: "urls", webhook: "webhook", actions: "actions", audit_metadata: {"auditMetadata", %{username: "username"}}, block_ads: "blockAds", check_prompt_injection: "checkPromptInjection", exclude_tags: "excludeTags", formats: "formats", headers: "headers", include_tags: "includeTags", location: "location", lockdown: "lockdown", max_age: "maxAge", min_age: "minAge", mobile: "mobile", only_clean_content: "onlyCleanContent", only_main_content: "onlyMainContent", parsers: "parsers", profile: "profile", proxy: "proxy", redact_pii: "redactPII", remove_base64_images: "removeBase64Images", skip_tls_verification: "skipTlsVerification", store_in_cache: "storeInCache", threat_protection: "threatProtection", timeout: "timeout", wait_for: "waitFor", zero_data_retention: "zeroDataRetention"}
@doc """
Scrape multiple URLs and optionally extract information using an LLM
`POST /batch/scrape`
Tag: Scraping
## Parameters
Validated by `NimbleOptions`. Pass params as a keyword list with snake_case keys.
See `@scrape_and_extract_from_urls_schema` for the full schema.
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec scrape_and_extract_from_urls(keyword(), keyword()) :: response()
def scrape_and_extract_from_urls(params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @scrape_and_extract_from_urls_schema) do
Req.post(client(opts), url: "/batch/scrape", json: to_body(params, @scrape_and_extract_from_urls_key_mapping))
end
end
@doc """
Bang variant of `scrape_and_extract_from_urls`. Raises on error.
"""
@spec scrape_and_extract_from_urls!(keyword(), keyword()) :: Req.Response.t()
def scrape_and_extract_from_urls!(params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @scrape_and_extract_from_urls_schema)
Req.post!(client(opts), url: "/batch/scrape", json: to_body(params, @scrape_and_extract_from_urls_key_mapping))
end
@search_and_scrape_schema NimbleOptions.new!([
categories: [type: {:list, :any}, doc: "Categories to filter results by. Defaults to [], which means results will not be filtered by any categories."],
country: [type: :string, doc: "ISO country code for geo-targeting search results (e.g. `US`). For best results, set both this and the `location` parameter."],
domain_tools: [type: :boolean, doc: "Include tool contracts whose provider matches the domains of the returned web results. Defaults to on when `alexandria` is among the sources; set `false` to disable, or `true` alone to add domain-matched tools beside ordinary web results. Free."],
enterprise: [type: {:list, :string}, doc: "Enterprise search options for Zero Data Retention (ZDR). Use `[\"zdr\"]` for end-to-end ZDR (10 credits / 10 results) or `[\"anon\"]` for anonymized ZDR (2 credits / 10 results). Must be enabled for your team."],
exclude_domains: [type: {:list, :string}, doc: "Excludes search results from the specified domains. Domains should be hostnames only, without protocol or path. Cannot be used with includeDomains."],
highlights: [type: :boolean, doc: "Generate query-relevant highlights for search results. Set to false to return provider descriptions or snippets without highlighting."],
ignore_invalid_urls: [type: :boolean, doc: "Excludes URLs from the search results that are invalid for other Firecrawl endpoints. This helps reduce errors if you are piping data from search into other Firecrawl API endpoints."],
include_domains: [type: {:list, :string}, doc: "Restricts search results to the specified domains. Domains should be hostnames only, without protocol or path. Cannot be used with excludeDomains."],
limit: [type: :integer, doc: "Maximum number of results to return (per source type when using multiple sources)"],
location: [type: :string, doc: "Location parameter for search results (e.g. `San Francisco,California,United States`). For best results, set both this and the `country` parameter."],
query: [type: :string, required: true, doc: "The search query"],
safe: [type: :boolean, doc: "When `true`, filters explicit content from search results (SafeSearch). Omit to keep the default behavior, which does not apply the filter."],
scrape_options: [type: :keyword_list, doc: "Options for scraping search results"],
sources: [type: {:list, :any}, doc: "Sources to search. Will determine the arrays available in the response. Defaults to ['web']. The plain string form (e.g. `[\"web\", \"alexandria\"]`) is also accepted. The `alexandria` source returns tool contracts discovered from the Alexandria catalogue in `data.tools` instead of search results; it is free and is never counted in `creditsUsed`. Requires the team's Alexandria access to be enabled (403 otherwise)."],
tbs: [type: :string, doc: "Time-based search parameter. Supports predefined time ranges (`qdr:h`, `qdr:d`, `qdr:w`, `qdr:m`, `qdr:y`), custom date ranges (`cdr:1,cd_min:MM/DD/YYYY,cd_max:MM/DD/YYYY`), and sort by date (`sbd:1`). Values can be combined, e.g. `sbd:1,qdr:w`."],
threat_protection: [type: :keyword_list, doc: "Per-request [Threat Protection](https://docs.firecrawl.dev/features/threat-protection) override. Fields you provide replace the corresponding fields of your organization's policy for this request only; omitted fields keep their organization-level values. Requires Threat Protection to be enabled for your team (enterprise feature) β otherwise the request is rejected with a 403. If your organization has disabled request overrides, any request that includes this object is rejected with a 403. If Threat Protection is enforced for your team, `mode` may not be set to `off`."],
timeout: [type: :integer, doc: "Timeout in milliseconds"],
tool_detail: [type: {:or, [{:in, [:compact, :summary, :full]}, :string]}, doc: "How much of each tool contract to return in `data.tools`. `compact` returns only `provider`, `capability`, and `description`. `summary` returns the full tool entry without `options` and `response`. `full` also returns `options`, `response`, and `examples`."]
])
@search_and_scrape_key_mapping %{categories: "categories", country: "country", domain_tools: "domainTools", enterprise: "enterprise", exclude_domains: "excludeDomains", highlights: "highlights", ignore_invalid_urls: "ignoreInvalidURLs", include_domains: "includeDomains", limit: "limit", location: "location", query: "query", safe: "safe", scrape_options: "scrapeOptions", sources: "sources", tbs: "tbs", threat_protection: "threatProtection", timeout: "timeout", tool_detail: "toolDetail"}
@doc """
Search and optionally scrape search results
`POST /search`
Tag: Search
## Parameters
Validated by `NimbleOptions`. Pass params as a keyword list with snake_case keys.
See `@search_and_scrape_schema` for the full schema.
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec search_and_scrape(keyword(), keyword()) :: response()
def search_and_scrape(params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @search_and_scrape_schema) do
Req.post(client(opts), url: "/search", json: to_body(params, @search_and_scrape_key_mapping))
end
end
@doc """
Bang variant of `search_and_scrape`. Raises on error.
"""
@spec search_and_scrape!(keyword(), keyword()) :: Req.Response.t()
def search_and_scrape!(params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @search_and_scrape_schema)
Req.post!(client(opts), url: "/search", json: to_body(params, @search_and_scrape_key_mapping))
end
@search_support_docs_schema NimbleOptions.new!([
question: [type: :string, required: true, doc: "Documentation question to answer."]
])
@search_support_docs_key_mapping %{question: "question"}
@doc """
Search Firecrawl docs with citations
`POST /support/docs-search`
Tag: Support
## Parameters
Validated by `NimbleOptions`. Pass params as a keyword list with snake_case keys.
See `@search_support_docs_schema` for the full schema.
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec search_support_docs(keyword(), keyword()) :: response()
def search_support_docs(params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @search_support_docs_schema) do
Req.post(client(opts), url: "/support/docs-search", json: to_body(params, @search_support_docs_key_mapping))
end
end
@doc """
Bang variant of `search_support_docs`. Raises on error.
"""
@spec search_support_docs!(keyword(), keyword()) :: Req.Response.t()
def search_support_docs!(params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @search_support_docs_schema)
Req.post!(client(opts), url: "/support/docs-search", json: to_body(params, @search_support_docs_key_mapping))
end
@start_agent_schema NimbleOptions.new!([
audit_metadata: [type: :keyword_list, keys: [username: [type: :string, required: true]], doc: "User attribution included with SIEM logging events when SIEM Logging is enabled for the organization."],
effort: [type: {:in, ["low", "medium", "high"]}, doc: "Reasoning budget for the agent task. Every run executes on spark-2, so effort can be sent with or without model."],
exchange: [type: :keyword_list, keys: [approve: [type: :keyword_list, keys: [always: [type: :boolean], approval_id: [type: :string, required: true], call_ids: [type: {:list, :string}]]], decline: [type: :keyword_list, keys: [approval_id: [type: :string, required: true]]], enabled: [type: :boolean], max_calls: [type: :integer], on_terms_required: [type: {:or, [{:in, [:skip, :ask]}, :string]}], require_approval: [type: :boolean], toolkits: [type: {:list, :string}]], doc: "Let the agent call your team's [Alexandria](https://docs.firecrawl.dev/features/alexandria) data providers during the run. Without this object the run uses the web only (a follow-up turn inherits the previous turn's settings). See [Use your connected Alexandria tools](https://docs.firecrawl.dev/features/agent#use-your-connected-alexandria-tools)."],
max_credits: [type: {:or, [:integer, :float]}, doc: "Maximum credits to spend on this agent task. Defaults to 2500 if not set. Values above 2,500 are always billed as paid requests."],
mode: [type: {:or, [{:in, [:extract, :chat]}, :string]}, doc: "`extract` returns the complete structured result in `data` every turn. `chat` lets a follow-up that asks for no new data get a short reply in `message` instead of a re-run. `exchange.requireApproval` needs `chat` on the same request. Omitted on a follow-up turn keeps the thread's mode."],
model: [type: {:or, [{:in, [:"spark-2", :"spark-1-mini", :"spark-1-pro"]}, :string]}, doc: "The model to use for the agent task. spark-2 is the default and the model every run executes on. The Spark 1 model names remain accepted for backwards compatibility but are deprecated and route to spark-2."],
prompt: [type: :string, required: true, doc: "The prompt describing what data to extract"],
schema: [type: :any, doc: "Optional JSON schema to structure the extracted data"],
strict_constrain_to_urls: [type: :boolean, doc: "If true, agent will only visit URLs provided in the urls array"],
thread_id: [type: :string, doc: "Continue an existing thread: pass the `threadId` an earlier run returned, and this request runs as that thread's next turn. Omitted `urls`, `schema`, `effort`, `mode` and `exchange` settings carry over from the previous turn. Omit `threadId` to start a new thread."],
threat_protection: [type: :keyword_list, doc: "Per-request [Threat Protection](https://docs.firecrawl.dev/features/threat-protection) override. Fields you provide replace the corresponding fields of your organization's policy for this request only; omitted fields keep their organization-level values. Requires Threat Protection to be enabled for your team (enterprise feature) β otherwise the request is rejected with a 403. If your organization has disabled request overrides, any request that includes this object is rejected with a 403. If Threat Protection is enforced for your team, `mode` may not be set to `off`."],
urls: [type: {:list, :string}, doc: "Optional list of URLs to constrain the agent to"],
webhook: [type: :keyword_list, doc: "A webhook specification object. Subscribes to agent lifecycle events (agent.started, agent.action, agent.completed, agent.failed, agent.cancelled)."]
])
@start_agent_key_mapping %{audit_metadata: {"auditMetadata", %{username: "username"}}, effort: "effort", exchange: {"exchange", %{approve: {"approve", %{always: "always", approval_id: "approvalId", call_ids: "callIds"}}, decline: {"decline", %{approval_id: "approvalId"}}, enabled: "enabled", max_calls: "maxCalls", on_terms_required: "onTermsRequired", require_approval: "requireApproval", toolkits: "toolkits"}}, max_credits: "maxCredits", mode: "mode", model: "model", prompt: "prompt", schema: "schema", strict_constrain_to_urls: "strictConstrainToURLs", thread_id: "threadId", threat_protection: "threatProtection", urls: "urls", webhook: "webhook"}
@doc """
Start an agent task for agentic data extraction
`POST /agent`
Tag: Agent
## Parameters
Validated by `NimbleOptions`. Pass params as a keyword list with snake_case keys.
See `@start_agent_schema` for the full schema.
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec start_agent(keyword(), keyword()) :: response()
def start_agent(params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @start_agent_schema) do
Req.post(client(opts), url: "/agent", json: to_body(params, @start_agent_key_mapping))
end
end
@doc """
Bang variant of `start_agent`. Raises on error.
"""
@spec start_agent!(keyword(), keyword()) :: Req.Response.t()
def start_agent!(params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @start_agent_schema)
Req.post!(client(opts), url: "/agent", json: to_body(params, @start_agent_key_mapping))
end
@doc """
Stop the interactive browser session associated with a scrape job
`DELETE /scrape/{jobId}/interact`
Tag: Scraping
## Path Parameters
* `job_id` - Path parameter `jobId`
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec stop_interactive_scrape_browser_session(String.t(), keyword()) :: response()
def stop_interactive_scrape_browser_session(job_id, opts \\ []) do
Req.delete(client(opts), url: "/scrape/#{job_id}/interact")
end
@doc """
Bang variant of `stop_interactive_scrape_browser_session`. Raises on error.
"""
@spec stop_interactive_scrape_browser_session!(String.t(), keyword()) :: Req.Response.t()
def stop_interactive_scrape_browser_session!(job_id, opts \\ []) do
Req.delete!(client(opts), url: "/scrape/#{job_id}/interact")
end
@submit_endpoint_feedback_schema NimbleOptions.new!([
integration: [type: :string],
missing_content: [type: {:list, :any}],
origin: [type: :string],
query_suggestions: [type: :string],
rating: [type: {:or, [{:in, [:good, :partial, :bad]}, :string]}, required: true],
valuable_sources: [type: {:list, :any}],
endpoint: [type: {:or, [{:in, [:search, :scrape, :parse, :map]}, :string]}, required: true],
issues: [type: {:list, :string}],
job_id: [type: :string, required: true],
metadata: [type: :any, doc: "Small endpoint-specific metadata object. Must be 8KB or smaller; do not include full endpoint results."],
note: [type: :string],
page_numbers: [type: {:list, :any}],
tags: [type: {:list, :string}],
url: [type: :string]
])
@submit_endpoint_feedback_key_mapping %{integration: "integration", missing_content: "missingContent", origin: "origin", query_suggestions: "querySuggestions", rating: "rating", valuable_sources: "valuableSources", endpoint: "endpoint", issues: "issues", job_id: "jobId", metadata: "metadata", note: "note", page_numbers: "pageNumbers", tags: "tags", url: "url"}
@doc """
Submit feedback for a v2 job
`POST /feedback`
Tag: Feedback
## Parameters
Validated by `NimbleOptions`. Pass params as a keyword list with snake_case keys.
See `@submit_endpoint_feedback_schema` for the full schema.
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec submit_endpoint_feedback(keyword(), keyword()) :: response()
def submit_endpoint_feedback(params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @submit_endpoint_feedback_schema) do
Req.post(client(opts), url: "/feedback", json: to_body(params, @submit_endpoint_feedback_key_mapping))
end
end
@doc """
Bang variant of `submit_endpoint_feedback`. Raises on error.
"""
@spec submit_endpoint_feedback!(keyword(), keyword()) :: Req.Response.t()
def submit_endpoint_feedback!(params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @submit_endpoint_feedback_schema)
Req.post!(client(opts), url: "/feedback", json: to_body(params, @submit_endpoint_feedback_key_mapping))
end
@submit_search_feedback_schema NimbleOptions.new!([
integration: [type: :string],
missing_content: [type: {:list, :any}],
origin: [type: :string],
query_suggestions: [type: :string],
rating: [type: {:or, [{:in, [:good, :partial, :bad]}, :string]}, required: true],
valuable_sources: [type: {:list, :any}]
])
@submit_search_feedback_key_mapping %{integration: "integration", missing_content: "missingContent", origin: "origin", query_suggestions: "querySuggestions", rating: "rating", valuable_sources: "valuableSources"}
@doc """
Submit feedback for a search job
`POST /search/{jobId}/feedback`
Tag: Search
## Path Parameters
* `job_id` - Path parameter `jobId`
## Parameters
Validated by `NimbleOptions`. Pass params as a keyword list with snake_case keys.
See `@submit_search_feedback_schema` for the full schema.
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec submit_search_feedback(String.t(), keyword(), keyword()) :: response()
def submit_search_feedback(job_id, params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @submit_search_feedback_schema) do
Req.post(client(opts), url: "/search/#{job_id}/feedback", json: to_body(params, @submit_search_feedback_key_mapping))
end
end
@doc """
Bang variant of `submit_search_feedback`. Raises on error.
"""
@spec submit_search_feedback!(String.t(), keyword(), keyword()) :: Req.Response.t()
def submit_search_feedback!(job_id, params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @submit_search_feedback_schema)
Req.post!(client(opts), url: "/search/#{job_id}/feedback", json: to_body(params, @submit_search_feedback_key_mapping))
end
@update_threat_protection_schema NimbleOptions.new!([
allow_request_overrides: [type: :boolean, doc: "Whether individual requests may pass a `threatProtection` object. When false, such requests are rejected with 403."],
blacklist: [type: {:list, :string}, doc: "Exact domains or globs (e.g. `*.example.com`) always blocked, without a classifier call."],
blocked_tlds: [type: {:list, :string}, doc: "Top-level domains to block outright, lowercase without a leading dot."],
failure_policy: [type: {:or, [{:in, [:open, :closed]}, :string]}, doc: "Behavior when the classifier is unreachable: `closed` blocks (default), `open` allows."],
mode: [type: {:or, [{:in, [:off, :normal]}, :string]}, required: true, doc: "Threat protection mode. `off` disables checks; `normal` checks URLs against Google Web Risk (+2 credits per URL scanned)."],
risk_score_threshold: [type: :integer, doc: "Normalized score (0-100) at or above which a classifier verdict is blocked. Lower is stricter."],
whitelist: [type: {:list, :string}, doc: "Exact domains or globs always allowed. Wins over every other rule."]
])
@update_threat_protection_key_mapping %{allow_request_overrides: "allowRequestOverrides", blacklist: "blacklist", blocked_tlds: "blockedTlds", failure_policy: "failurePolicy", mode: "mode", risk_score_threshold: "riskScoreThreshold", whitelist: "whitelist"}
@doc """
Update the team's threat protection policy
`PUT /team/threat-protection`
Tag: Threat Protection
## Parameters
Validated by `NimbleOptions`. Pass params as a keyword list with snake_case keys.
See `@update_threat_protection_schema` for the full schema.
## Returns
* `{:ok, %Req.Response{}}` on success
* `{:error, exception}` on HTTP or validation failure
"""
@spec update_threat_protection(keyword(), keyword()) :: response()
def update_threat_protection(params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @update_threat_protection_schema) do
Req.put(client(opts), url: "/team/threat-protection", json: to_body(params, @update_threat_protection_key_mapping))
end
end
@doc """
Bang variant of `update_threat_protection`. Raises on error.
"""
@spec update_threat_protection!(keyword(), keyword()) :: Req.Response.t()
def update_threat_protection!(params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @update_threat_protection_schema)
Req.put!(client(opts), url: "/team/threat-protection", json: to_body(params, @update_threat_protection_key_mapping))
end
end