Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(elixir-sdk): add developer search (#5278)
Erik Engervallb8d9b08f48c1
diff --git a/apps/elixir-sdk/README.md b/apps/elixir-sdk/README.mdindex fa5d9ad3a..88fe380e5 100644--- a/apps/elixir-sdk/README.md+++ b/apps/elixir-sdk/README.md@@ -79,6 +79,19 @@ All params are passed as keyword lists with snake_case keys. Invalid keys, missi# Search{:ok, response} = Firecrawl.search_and_scrape(query: "firecrawl web scraping")+# Search the developer index: public repositories, GitHub issues, merged pull+# requests, READMEs and code documentation. Replaces the deprecated search_github.+{:ok, %Firecrawl.DeveloperSearch{results: results}} = Firecrawl.search_developer(+ query: "retry an http request with exponential backoff",+ repos: ["square/okhttp"],+ types: ["issue", "pull_request"],+ k: 5+)++for %Firecrawl.DeveloperSearch.Result{title: title, url: url} <- results do+ IO.puts("#{title} #{url}")+end+# Check crawl status{:ok, response} = Firecrawl.get_crawl_status("job-uuid")diff --git a/apps/elixir-sdk/lib/firecrawl.ex b/apps/elixir-sdk/lib/firecrawl.exindex 4b3e70bd1..2fb03e760 100644--- a/apps/elixir-sdk/lib/firecrawl.ex+++ b/apps/elixir-sdk/lib/firecrawl.ex@@ -377,7 +377,11 @@ defmodule Firecrawl doSearch GitHub research content.`GET /search/research/github`++ Deprecated: stops responding after 2026-11-03. Use `search_developer/2`, which+ does not carry over the score breakdown or the web fallback results."""+ @deprecated "Use search_developer/2; the endpoint stops responding after 2026-11-03"@spec search_github(keyword(), keyword()) :: response()def search_github(params \\ [], opts \\ []) dowith {:ok, params} <- NimbleOptions.validate(params, @research_search_github_schema) do@@ -387,13 +391,103 @@ defmodule Firecrawl do@doc """Bang variant of `search_github`. Raises on error.++ Deprecated: use `search_developer!/2`."""+ @deprecated "Use search_developer!/2; the endpoint stops responding after 2026-11-03"@spec search_github!(keyword(), keyword()) :: Req.Response.t()def search_github!(params \\ [], opts \\ []) doparams = NimbleOptions.validate!(params, @research_search_github_schema)Req.get!(client(opts), url: "/search/research/github", params: [{"origin", @sdk_origin} | to_query(params, @research_search_github_key_mapping)])end+ @search_developer_schema NimbleOptions.new!([+ query: [type: :string, required: true, doc: "Natural-language question or search phrase. Must not be blank."],+ k: [type: :integer, doc: "Total ranked results, 1 to 100 (default 10)."],+ passages: [type: :integer, doc: "Passages per result, 1 to 5 (default 1)."],+ types: [type: {:list, :string}, doc: "Result kinds: `doc`, `issue`, `pull_request`, `readme` (default all)."],+ repos: [type: {:list, :string}, doc: "GitHub `owner/name` filters, at most 20."],+ sources: [type: {:list, :string}, doc: "Developer documentation source ids, at most 20."],+ language: [type: :string, doc: "One case-insensitive GitHub Linguist primary language."],+ topic: [type: {:or, [:string, {:list, :string}]}, doc: "Repository topic, or topics that must all match."],+ license: [type: :string, doc: "One case-insensitive SPDX repository license identifier."],+ min_stars: [type: :integer, doc: "Lower bound on repository stars."],+ max_stars: [type: :integer, doc: "Upper bound on repository stars."],+ archived: [type: :boolean, doc: "Include or exclude archived repositories."],+ fork: [type: :boolean, doc: "Include or exclude forks."],+ skills: [type: {:or, [{:in, [:only]}, :string]}, doc: "Set to `only` to return packaged agent-skill evidence only."]+ ])++ @search_developer_key_mapping %{query: "query", k: "k", passages: "passages", types: "types", repos: "repos", sources: "sources", language: "language", topic: "topic", license: "license", min_stars: "min_stars", max_stars: "max_stars", archived: "archived", fork: "fork", skills: "skills"}++ @doc """+ Search the developer index: public repositories, GitHub issues, merged pull+ requests, READMEs and code documentation. Replaces the deprecated `search_github/2`.++ `POST /search/developer`++ Tag: Developer++ ## Parameters++ Validated by `NimbleOptions`. A blank `query` is rejected before any request is sent.++ #{NimbleOptions.docs(@search_developer_schema)}++ ## Returns++ * `{:ok, %Firecrawl.DeveloperSearch{}}` on success+ * `{:error, exception}` on a blank query, validation failure, HTTP failure or an unexpected response shape++ ## Example++ {:ok, %Firecrawl.DeveloperSearch{results: results}} =+ Firecrawl.search_developer(query: "retry an http request", repos: ["square/okhttp"], k: 5)+ """+ @spec search_developer(keyword(), keyword()) ::+ {:ok, Firecrawl.DeveloperSearch.t()} | {:error, Exception.t() | Firecrawl.Error.t()}+ def search_developer(params \\ [], opts \\ []) do+ with {:ok, params} <- NimbleOptions.validate(params, @search_developer_schema),+ :ok <- validate_query(params[:query]),+ {:ok, response} <-+ Req.post(client(opts), url: "/search/developer", json: to_body(params, @search_developer_key_mapping)) do+ decode_developer_search(response)+ end+ end++ @doc """+ Bang variant of `search_developer`. Raises on error.+ """+ @spec search_developer!(keyword(), keyword()) :: Firecrawl.DeveloperSearch.t()+ def search_developer!(params \\ [], opts \\ []) do+ case search_developer(params, opts) do+ {:ok, result} -> result+ {:error, exception} -> raise exception+ end+ end++ defp validate_query(query) do+ if String.trim(query) == "" do+ {:error, %ArgumentError{message: "query cannot be empty"}}+ else+ :ok+ end+ end++ defp decode_developer_search(%Req.Response{body: %{"results" => results} = body})+ when is_list(results) do+ {:ok, Firecrawl.DeveloperSearch.from_map(body)}+ rescue+ exception -> {:error, exception}+ end++ defp decode_developer_search(%Req.Response{status: status, body: body}) do+ {:error,+ RuntimeError.exception(+ "unexpected POST /search/developer response (HTTP #{status}): #{inspect(body)}"+ )}+ end+@monitor_key_mapping %{name: "name",schedule: "schedule",diff --git a/apps/elixir-sdk/lib/firecrawl/developer_search.ex b/apps/elixir-sdk/lib/firecrawl/developer_search.exnew file mode 100644index 000000000..f8f627c26--- /dev/null+++ b/apps/elixir-sdk/lib/firecrawl/developer_search.ex@@ -0,0 +1,115 @@+defmodule Firecrawl.DeveloperSearch do+ @moduledoc """+ Response of `Firecrawl.search_developer/2`.++ ## Fields++ * `:success` - Whether the request succeeded+ * `:partial` - Whether some of the index could not be searched, so results may be incomplete+ * `:results` - Ranked `Firecrawl.DeveloperSearch.Result` structs+ * `:repos` - Index status per requested repository, or `nil` when `repos` was not sent+ * `:sources` - Index status per requested documentation source, or `nil` when `sources` was not sent+ """++ alias Firecrawl.DeveloperSearch.{Repo, Result, Source}++ defstruct [:success, :partial, :repos, :sources, results: []]++ @type t :: %__MODULE__{+ success: boolean(),+ partial: boolean(),+ results: [Result.t()],+ repos: [Repo.t()] | nil,+ sources: [Source.t()] | nil+ }++ @doc false+ @spec from_map(map()) :: t()+ def from_map(map) when is_map(map) do+ %__MODULE__{+ success: map["success"] == true,+ partial: map["partial"] == true,+ results: Enum.map(map["results"] || [], &Result.from_map/1),+ repos: map["repos"] && Enum.map(map["repos"], &Repo.from_map/1),+ sources: map["sources"] && Enum.map(map["sources"], &Source.from_map/1)+ }+ end+end++defmodule Firecrawl.DeveloperSearch.Result do+ @moduledoc """+ One ranked developer search result.++ `:license` is either an SPDX string or a disclosure map, so it is kept as+ returned. `:passages` are the matched passage texts, in markdown.+ """++ defstruct [:id, :url, :title, :license, passages: []]++ @type t :: %__MODULE__{+ id: String.t(),+ url: String.t(),+ title: String.t() | nil,+ license: String.t() | map() | nil,+ passages: [String.t()]+ }++ @doc false+ @spec from_map(map()) :: t()+ def from_map(map) when is_map(map) do+ %__MODULE__{+ id: map["id"],+ url: map["url"],+ title: map["title"],+ license: map["license"],+ passages: for(%{"text" => text} <- map["passages"] || [], do: text)+ }+ end+end++defmodule Firecrawl.DeveloperSearch.Repo do+ @moduledoc """+ Whether a requested repository is indexed.++ `:types` reports which result types are indexed for it, with the keys+ `:issue`, `:pull_request` and `:readme`.+ """++ defstruct [:repo, indexed: false, types: %{issue: false, pull_request: false, readme: false}]++ @type t :: %__MODULE__{+ repo: String.t(),+ indexed: boolean(),+ types: %{issue: boolean(), pull_request: boolean(), readme: boolean()}+ }++ @doc false+ @spec from_map(map()) :: t()+ def from_map(map) when is_map(map) do+ types = map["types"] || %{}++ %__MODULE__{+ repo: map["repo"],+ indexed: map["indexed"] == true,+ types: %{+ issue: types["issue"] == true,+ pull_request: types["pullRequest"] == true,+ readme: types["readme"] == true+ }+ }+ end+end++defmodule Firecrawl.DeveloperSearch.Source do+ @moduledoc "Whether a requested documentation source is indexed."++ defstruct [:source, indexed: false]++ @type t :: %__MODULE__{source: String.t(), indexed: boolean()}++ @doc false+ @spec from_map(map()) :: t()+ def from_map(map) when is_map(map) do+ %__MODULE__{source: map["source"], indexed: map["indexed"] == true}+ end+enddiff --git a/apps/elixir-sdk/mix.exs b/apps/elixir-sdk/mix.exsindex 6c705e40b..a0b636dc8 100644--- a/apps/elixir-sdk/mix.exs+++ b/apps/elixir-sdk/mix.exs@@ -1,7 +1,7 @@defmodule Firecrawl.MixProject douse Mix.Project- @version "1.12.2"+ @version "1.12.4"@source_url "https://github.com/firecrawl/firecrawl/tree/main/apps/elixir-sdk"def project dodiff --git a/apps/elixir-sdk/test/firecrawl_test.exs b/apps/elixir-sdk/test/firecrawl_test.exsindex 59440f7a8..0f274cf7a 100644--- a/apps/elixir-sdk/test/firecrawl_test.exs+++ b/apps/elixir-sdk/test/firecrawl_test.exs@@ -1051,6 +1051,154 @@ defmodule FirecrawlTest doendend+ defp developer_search_body do+ %{+ "success" => true,+ "partial" => false,+ "results" => [+ %{+ "id" => "issue:square/okhttp#4963",+ "url" => "https://github.com/square/okhttp/issues/4963",+ "title" => "Issues with retry on connection failure",+ "passages" => [%{"text" => "OkHttp retries when a client reuses a closed connection."}],+ "license" => "Apache-2.0"+ },+ %{+ "id" => "doc:okhttp/recipes",+ "url" => "https://square.github.io/okhttp/recipes/",+ "passages" => [],+ "license" => %{"state" => "unknown"}+ }+ ],+ "repos" => [+ %{+ "repo" => "square/okhttp",+ "indexed" => true,+ "types" => %{"issue" => true, "pullRequest" => true, "readme" => false}+ }+ ]+ }+ end++ test "search_developer posts the query, filters and origin and returns a typed response" do+ assert {:ok, %Firecrawl.DeveloperSearch{} = response} =+ Firecrawl.search_developer(+ [+ query: "http client retry",+ k: 3,+ passages: 2,+ types: ["issue"],+ repos: ["square/okhttp"],+ sources: ["okhttp-docs"],+ language: "Java",+ topic: ["http"],+ license: "apache-2.0",+ min_stars: 100,+ max_stars: 100_000,+ archived: false,+ fork: false,+ skills: :only+ ],+ api_key: "test-key",+ adapter: parse_formats_adapter(self(), 200, developer_search_body())+ )++ assert_receive {:request, request}+ assert request.method == :post+ assert URI.to_string(request.url) == "https://api.firecrawl.dev/v2/search/developer"+ assert request.headers["authorization"] == ["Bearer test-key"]++ assert Jason.decode!(request.body) == %{+ "query" => "http client retry",+ "k" => 3,+ "passages" => 2,+ "types" => ["issue"],+ "repos" => ["square/okhttp"],+ "sources" => ["okhttp-docs"],+ "language" => "Java",+ "topic" => ["http"],+ "license" => "apache-2.0",+ "min_stars" => 100,+ "max_stars" => 100_000,+ "archived" => false,+ "fork" => false,+ "skills" => "only",+ "origin" => "elixir-sdk@" <> Mix.Project.config()[:version]+ }++ assert %Firecrawl.DeveloperSearch{success: true, partial: false, sources: nil} = response++ assert [+ %Firecrawl.DeveloperSearch.Result{+ id: "issue:square/okhttp#4963",+ title: "Issues with retry on connection failure",+ passages: ["OkHttp retries when a client reuses a closed connection."],+ license: "Apache-2.0"+ },+ %Firecrawl.DeveloperSearch.Result{title: nil, passages: [], license: %{"state" => "unknown"}}+ ] = response.results++ assert [+ %Firecrawl.DeveloperSearch.Repo{+ repo: "square/okhttp",+ indexed: true,+ types: %{issue: true, pull_request: true, readme: false}+ }+ ] = response.repos+ end++ test "search_developer! returns the typed response" do+ assert %Firecrawl.DeveloperSearch{results: [_, _]} =+ Firecrawl.search_developer!(+ [query: "http client retry"],+ api_key: "test-key",+ adapter: parse_formats_adapter(self(), 200, developer_search_body())+ )+ end++ test "search_developer rejects a blank query before sending a request" do+ adapter = parse_formats_adapter(self(), 200, developer_search_body())++ for query <- ["", " "] do+ assert {:error, %ArgumentError{message: "query cannot be empty"}} =+ Firecrawl.search_developer([query: query], api_key: "test-key", adapter: adapter)++ assert_raise ArgumentError, "query cannot be empty", fn ->+ Firecrawl.search_developer!([query: query], api_key: "test-key", adapter: adapter)+ end+ end++ assert {:error, %NimbleOptions.ValidationError{}} =+ Firecrawl.search_developer([], api_key: "test-key", adapter: adapter)++ refute_received {:request, _}+ end++ test "search_developer returns {:error, %Firecrawl.Error{}} on API errors" do+ adapter = parse_formats_adapter(self(), 401, %{"success" => false, "error" => "Unauthorized"})++ assert {:error, %Firecrawl.Error{status: 401}} =+ Firecrawl.search_developer([query: "retry"], api_key: "bad-key", adapter: adapter, retry: false)+ end++ test "search_developer returns a non-API error for an unexpected success body" do+ adapter = parse_formats_adapter(self(), 200, %{"success" => true})++ assert {:error, %RuntimeError{message: msg}} =+ Firecrawl.search_developer([query: "retry"], api_key: "test-key", adapter: adapter)++ assert msg =~ "unexpected POST /search/developer response (HTTP 200)"+ end++ test "search_developer returns an error tuple for malformed result entries" do+ adapter = parse_formats_adapter(self(), 200, %{"success" => true, "results" => ["not a map"]})++ assert {:error, exception} =+ Firecrawl.search_developer([query: "retry"], api_key: "test-key", adapter: adapter)++ assert is_exception(exception)+ end+test "all expected API functions are defined with bang variants" dofunctions = Firecrawl.__info__(:functions)@@ -1088,7 +1236,13 @@ defmodule FirecrawlTest do{:get_parse_formats, 0},{:get_parse_formats, 1},{:get_parse_formats!, 0},- {:get_parse_formats!, 1}+ {:get_parse_formats!, 1},+ {:search_developer, 0},+ {:search_developer, 1},+ {:search_developer, 2},+ {:search_developer!, 0},+ {:search_developer!, 1},+ {:search_developer!, 2}]for {name, arity} <- expected do