Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(elixir-sdk): add developer search (#5278)

Erik Engervallb8d9b08f48c1
diff --git a/apps/elixir-sdk/README.md b/apps/elixir-sdk/README.md
index fa5d9ad3a..88fe380e5 100644
--- a/apps/elixir-sdk/README.md
+++ b/apps/elixir-sdk/README.md
@@ -79,6 +79,19 @@ All params are passed as keyword lists with snake_case keys. Invalid keys, missi
# Search
{:ok, response} = Firecrawl.search_and_scrape(query: "firecrawl web scraping")
+# Search the developer index: public repositories, GitHub issues, merged pull
+# requests, READMEs and code documentation. Replaces the deprecated search_github.
+{:ok, %Firecrawl.DeveloperSearch{results: results}} = Firecrawl.search_developer(
+ query: "retry an http request with exponential backoff",
+ repos: ["square/okhttp"],
+ types: ["issue", "pull_request"],
+ k: 5
+)
+
+for %Firecrawl.DeveloperSearch.Result{title: title, url: url} <- results do
+ IO.puts("#{title} #{url}")
+end
+
# Check crawl status
{:ok, response} = Firecrawl.get_crawl_status("job-uuid")
diff --git a/apps/elixir-sdk/lib/firecrawl.ex b/apps/elixir-sdk/lib/firecrawl.ex
index 4b3e70bd1..2fb03e760 100644
--- a/apps/elixir-sdk/lib/firecrawl.ex
+++ b/apps/elixir-sdk/lib/firecrawl.ex
@@ -377,7 +377,11 @@ defmodule Firecrawl do
Search GitHub research content.
`GET /search/research/github`
+
+ Deprecated: stops responding after 2026-11-03. Use `search_developer/2`, which
+ does not carry over the score breakdown or the web fallback results.
"""
+ @deprecated "Use search_developer/2; the endpoint stops responding after 2026-11-03"
@spec search_github(keyword(), keyword()) :: response()
def search_github(params \\ [], opts \\ []) do
with {:ok, params} <- NimbleOptions.validate(params, @research_search_github_schema) do
@@ -387,13 +391,103 @@ defmodule Firecrawl do
@doc """
Bang variant of `search_github`. Raises on error.
+
+ Deprecated: use `search_developer!/2`.
"""
+ @deprecated "Use search_developer!/2; the endpoint stops responding after 2026-11-03"
@spec search_github!(keyword(), keyword()) :: Req.Response.t()
def search_github!(params \\ [], opts \\ []) do
params = NimbleOptions.validate!(params, @research_search_github_schema)
Req.get!(client(opts), url: "/search/research/github", params: [{"origin", @sdk_origin} | to_query(params, @research_search_github_key_mapping)])
end
+ @search_developer_schema NimbleOptions.new!([
+ query: [type: :string, required: true, doc: "Natural-language question or search phrase. Must not be blank."],
+ k: [type: :integer, doc: "Total ranked results, 1 to 100 (default 10)."],
+ passages: [type: :integer, doc: "Passages per result, 1 to 5 (default 1)."],
+ types: [type: {:list, :string}, doc: "Result kinds: `doc`, `issue`, `pull_request`, `readme` (default all)."],
+ repos: [type: {:list, :string}, doc: "GitHub `owner/name` filters, at most 20."],
+ sources: [type: {:list, :string}, doc: "Developer documentation source ids, at most 20."],
+ language: [type: :string, doc: "One case-insensitive GitHub Linguist primary language."],
+ topic: [type: {:or, [:string, {:list, :string}]}, doc: "Repository topic, or topics that must all match."],
+ license: [type: :string, doc: "One case-insensitive SPDX repository license identifier."],
+ min_stars: [type: :integer, doc: "Lower bound on repository stars."],
+ max_stars: [type: :integer, doc: "Upper bound on repository stars."],
+ archived: [type: :boolean, doc: "Include or exclude archived repositories."],
+ fork: [type: :boolean, doc: "Include or exclude forks."],
+ skills: [type: {:or, [{:in, [:only]}, :string]}, doc: "Set to `only` to return packaged agent-skill evidence only."]
+ ])
+
+ @search_developer_key_mapping %{query: "query", k: "k", passages: "passages", types: "types", repos: "repos", sources: "sources", language: "language", topic: "topic", license: "license", min_stars: "min_stars", max_stars: "max_stars", archived: "archived", fork: "fork", skills: "skills"}
+
+ @doc """
+ Search the developer index: public repositories, GitHub issues, merged pull
+ requests, READMEs and code documentation. Replaces the deprecated `search_github/2`.
+
+ `POST /search/developer`
+
+ Tag: Developer
+
+ ## Parameters
+
+ Validated by `NimbleOptions`. A blank `query` is rejected before any request is sent.
+
+ #{NimbleOptions.docs(@search_developer_schema)}
+
+ ## Returns
+
+ * `{:ok, %Firecrawl.DeveloperSearch{}}` on success
+ * `{:error, exception}` on a blank query, validation failure, HTTP failure or an unexpected response shape
+
+ ## Example
+
+ {:ok, %Firecrawl.DeveloperSearch{results: results}} =
+ Firecrawl.search_developer(query: "retry an http request", repos: ["square/okhttp"], k: 5)
+ """
+ @spec search_developer(keyword(), keyword()) ::
+ {:ok, Firecrawl.DeveloperSearch.t()} | {:error, Exception.t() | Firecrawl.Error.t()}
+ def search_developer(params \\ [], opts \\ []) do
+ with {:ok, params} <- NimbleOptions.validate(params, @search_developer_schema),
+ :ok <- validate_query(params[:query]),
+ {:ok, response} <-
+ Req.post(client(opts), url: "/search/developer", json: to_body(params, @search_developer_key_mapping)) do
+ decode_developer_search(response)
+ end
+ end
+
+ @doc """
+ Bang variant of `search_developer`. Raises on error.
+ """
+ @spec search_developer!(keyword(), keyword()) :: Firecrawl.DeveloperSearch.t()
+ def search_developer!(params \\ [], opts \\ []) do
+ case search_developer(params, opts) do
+ {:ok, result} -> result
+ {:error, exception} -> raise exception
+ end
+ end
+
+ defp validate_query(query) do
+ if String.trim(query) == "" do
+ {:error, %ArgumentError{message: "query cannot be empty"}}
+ else
+ :ok
+ end
+ end
+
+ defp decode_developer_search(%Req.Response{body: %{"results" => results} = body})
+ when is_list(results) do
+ {:ok, Firecrawl.DeveloperSearch.from_map(body)}
+ rescue
+ exception -> {:error, exception}
+ end
+
+ defp decode_developer_search(%Req.Response{status: status, body: body}) do
+ {:error,
+ RuntimeError.exception(
+ "unexpected POST /search/developer response (HTTP #{status}): #{inspect(body)}"
+ )}
+ end
+
@monitor_key_mapping %{
name: "name",
schedule: "schedule",
diff --git a/apps/elixir-sdk/lib/firecrawl/developer_search.ex b/apps/elixir-sdk/lib/firecrawl/developer_search.ex
new file mode 100644
index 000000000..f8f627c26
--- /dev/null
+++ b/apps/elixir-sdk/lib/firecrawl/developer_search.ex
@@ -0,0 +1,115 @@
+defmodule Firecrawl.DeveloperSearch do
+ @moduledoc """
+ Response of `Firecrawl.search_developer/2`.
+
+ ## Fields
+
+ * `:success` - Whether the request succeeded
+ * `:partial` - Whether some of the index could not be searched, so results may be incomplete
+ * `:results` - Ranked `Firecrawl.DeveloperSearch.Result` structs
+ * `:repos` - Index status per requested repository, or `nil` when `repos` was not sent
+ * `:sources` - Index status per requested documentation source, or `nil` when `sources` was not sent
+ """
+
+ alias Firecrawl.DeveloperSearch.{Repo, Result, Source}
+
+ defstruct [:success, :partial, :repos, :sources, results: []]
+
+ @type t :: %__MODULE__{
+ success: boolean(),
+ partial: boolean(),
+ results: [Result.t()],
+ repos: [Repo.t()] | nil,
+ sources: [Source.t()] | nil
+ }
+
+ @doc false
+ @spec from_map(map()) :: t()
+ def from_map(map) when is_map(map) do
+ %__MODULE__{
+ success: map["success"] == true,
+ partial: map["partial"] == true,
+ results: Enum.map(map["results"] || [], &Result.from_map/1),
+ repos: map["repos"] && Enum.map(map["repos"], &Repo.from_map/1),
+ sources: map["sources"] && Enum.map(map["sources"], &Source.from_map/1)
+ }
+ end
+end
+
+defmodule Firecrawl.DeveloperSearch.Result do
+ @moduledoc """
+ One ranked developer search result.
+
+ `:license` is either an SPDX string or a disclosure map, so it is kept as
+ returned. `:passages` are the matched passage texts, in markdown.
+ """
+
+ defstruct [:id, :url, :title, :license, passages: []]
+
+ @type t :: %__MODULE__{
+ id: String.t(),
+ url: String.t(),
+ title: String.t() | nil,
+ license: String.t() | map() | nil,
+ passages: [String.t()]
+ }
+
+ @doc false
+ @spec from_map(map()) :: t()
+ def from_map(map) when is_map(map) do
+ %__MODULE__{
+ id: map["id"],
+ url: map["url"],
+ title: map["title"],
+ license: map["license"],
+ passages: for(%{"text" => text} <- map["passages"] || [], do: text)
+ }
+ end
+end
+
+defmodule Firecrawl.DeveloperSearch.Repo do
+ @moduledoc """
+ Whether a requested repository is indexed.
+
+ `:types` reports which result types are indexed for it, with the keys
+ `:issue`, `:pull_request` and `:readme`.
+ """
+
+ defstruct [:repo, indexed: false, types: %{issue: false, pull_request: false, readme: false}]
+
+ @type t :: %__MODULE__{
+ repo: String.t(),
+ indexed: boolean(),
+ types: %{issue: boolean(), pull_request: boolean(), readme: boolean()}
+ }
+
+ @doc false
+ @spec from_map(map()) :: t()
+ def from_map(map) when is_map(map) do
+ types = map["types"] || %{}
+
+ %__MODULE__{
+ repo: map["repo"],
+ indexed: map["indexed"] == true,
+ types: %{
+ issue: types["issue"] == true,
+ pull_request: types["pullRequest"] == true,
+ readme: types["readme"] == true
+ }
+ }
+ end
+end
+
+defmodule Firecrawl.DeveloperSearch.Source do
+ @moduledoc "Whether a requested documentation source is indexed."
+
+ defstruct [:source, indexed: false]
+
+ @type t :: %__MODULE__{source: String.t(), indexed: boolean()}
+
+ @doc false
+ @spec from_map(map()) :: t()
+ def from_map(map) when is_map(map) do
+ %__MODULE__{source: map["source"], indexed: map["indexed"] == true}
+ end
+end
diff --git a/apps/elixir-sdk/mix.exs b/apps/elixir-sdk/mix.exs
index 6c705e40b..a0b636dc8 100644
--- a/apps/elixir-sdk/mix.exs
+++ b/apps/elixir-sdk/mix.exs
@@ -1,7 +1,7 @@
defmodule Firecrawl.MixProject do
use Mix.Project
- @version "1.12.2"
+ @version "1.12.4"
@source_url "https://github.com/firecrawl/firecrawl/tree/main/apps/elixir-sdk"
def project do
diff --git a/apps/elixir-sdk/test/firecrawl_test.exs b/apps/elixir-sdk/test/firecrawl_test.exs
index 59440f7a8..0f274cf7a 100644
--- a/apps/elixir-sdk/test/firecrawl_test.exs
+++ b/apps/elixir-sdk/test/firecrawl_test.exs
@@ -1051,6 +1051,154 @@ defmodule FirecrawlTest do
end
end
+ defp developer_search_body do
+ %{
+ "success" => true,
+ "partial" => false,
+ "results" => [
+ %{
+ "id" => "issue:square/okhttp#4963",
+ "url" => "https://github.com/square/okhttp/issues/4963",
+ "title" => "Issues with retry on connection failure",
+ "passages" => [%{"text" => "OkHttp retries when a client reuses a closed connection."}],
+ "license" => "Apache-2.0"
+ },
+ %{
+ "id" => "doc:okhttp/recipes",
+ "url" => "https://square.github.io/okhttp/recipes/",
+ "passages" => [],
+ "license" => %{"state" => "unknown"}
+ }
+ ],
+ "repos" => [
+ %{
+ "repo" => "square/okhttp",
+ "indexed" => true,
+ "types" => %{"issue" => true, "pullRequest" => true, "readme" => false}
+ }
+ ]
+ }
+ end
+
+ test "search_developer posts the query, filters and origin and returns a typed response" do
+ assert {:ok, %Firecrawl.DeveloperSearch{} = response} =
+ Firecrawl.search_developer(
+ [
+ query: "http client retry",
+ k: 3,
+ passages: 2,
+ types: ["issue"],
+ repos: ["square/okhttp"],
+ sources: ["okhttp-docs"],
+ language: "Java",
+ topic: ["http"],
+ license: "apache-2.0",
+ min_stars: 100,
+ max_stars: 100_000,
+ archived: false,
+ fork: false,
+ skills: :only
+ ],
+ api_key: "test-key",
+ adapter: parse_formats_adapter(self(), 200, developer_search_body())
+ )
+
+ assert_receive {:request, request}
+ assert request.method == :post
+ assert URI.to_string(request.url) == "https://api.firecrawl.dev/v2/search/developer"
+ assert request.headers["authorization"] == ["Bearer test-key"]
+
+ assert Jason.decode!(request.body) == %{
+ "query" => "http client retry",
+ "k" => 3,
+ "passages" => 2,
+ "types" => ["issue"],
+ "repos" => ["square/okhttp"],
+ "sources" => ["okhttp-docs"],
+ "language" => "Java",
+ "topic" => ["http"],
+ "license" => "apache-2.0",
+ "min_stars" => 100,
+ "max_stars" => 100_000,
+ "archived" => false,
+ "fork" => false,
+ "skills" => "only",
+ "origin" => "elixir-sdk@" <> Mix.Project.config()[:version]
+ }
+
+ assert %Firecrawl.DeveloperSearch{success: true, partial: false, sources: nil} = response
+
+ assert [
+ %Firecrawl.DeveloperSearch.Result{
+ id: "issue:square/okhttp#4963",
+ title: "Issues with retry on connection failure",
+ passages: ["OkHttp retries when a client reuses a closed connection."],
+ license: "Apache-2.0"
+ },
+ %Firecrawl.DeveloperSearch.Result{title: nil, passages: [], license: %{"state" => "unknown"}}
+ ] = response.results
+
+ assert [
+ %Firecrawl.DeveloperSearch.Repo{
+ repo: "square/okhttp",
+ indexed: true,
+ types: %{issue: true, pull_request: true, readme: false}
+ }
+ ] = response.repos
+ end
+
+ test "search_developer! returns the typed response" do
+ assert %Firecrawl.DeveloperSearch{results: [_, _]} =
+ Firecrawl.search_developer!(
+ [query: "http client retry"],
+ api_key: "test-key",
+ adapter: parse_formats_adapter(self(), 200, developer_search_body())
+ )
+ end
+
+ test "search_developer rejects a blank query before sending a request" do
+ adapter = parse_formats_adapter(self(), 200, developer_search_body())
+
+ for query <- ["", " "] do
+ assert {:error, %ArgumentError{message: "query cannot be empty"}} =
+ Firecrawl.search_developer([query: query], api_key: "test-key", adapter: adapter)
+
+ assert_raise ArgumentError, "query cannot be empty", fn ->
+ Firecrawl.search_developer!([query: query], api_key: "test-key", adapter: adapter)
+ end
+ end
+
+ assert {:error, %NimbleOptions.ValidationError{}} =
+ Firecrawl.search_developer([], api_key: "test-key", adapter: adapter)
+
+ refute_received {:request, _}
+ end
+
+ test "search_developer returns {:error, %Firecrawl.Error{}} on API errors" do
+ adapter = parse_formats_adapter(self(), 401, %{"success" => false, "error" => "Unauthorized"})
+
+ assert {:error, %Firecrawl.Error{status: 401}} =
+ Firecrawl.search_developer([query: "retry"], api_key: "bad-key", adapter: adapter, retry: false)
+ end
+
+ test "search_developer returns a non-API error for an unexpected success body" do
+ adapter = parse_formats_adapter(self(), 200, %{"success" => true})
+
+ assert {:error, %RuntimeError{message: msg}} =
+ Firecrawl.search_developer([query: "retry"], api_key: "test-key", adapter: adapter)
+
+ assert msg =~ "unexpected POST /search/developer response (HTTP 200)"
+ end
+
+ test "search_developer returns an error tuple for malformed result entries" do
+ adapter = parse_formats_adapter(self(), 200, %{"success" => true, "results" => ["not a map"]})
+
+ assert {:error, exception} =
+ Firecrawl.search_developer([query: "retry"], api_key: "test-key", adapter: adapter)
+
+ assert is_exception(exception)
+ end
+
test "all expected API functions are defined with bang variants" do
functions = Firecrawl.__info__(:functions)
@@ -1088,7 +1236,13 @@ defmodule FirecrawlTest do
{:get_parse_formats, 0},
{:get_parse_formats, 1},
{:get_parse_formats!, 0},
- {:get_parse_formats!, 1}
+ {:get_parse_formats!, 1},
+ {:search_developer, 0},
+ {:search_developer, 1},
+ {:search_developer, 2},
+ {:search_developer!, 0},
+ {:search_developer!, 1},
+ {:search_developer!, 2}
]
for {name, arity} <- expected do