Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(elixir-sdk): pin pagination next URLs to the api_url origin (#4868)
Gergő Móricz1f52d600ffcd
diff --git a/.github/workflows/test-elixir-sdk.yml b/.github/workflows/test-elixir-sdk.ymlnew file mode 100644index 000000000..050b635b6--- /dev/null+++ b/.github/workflows/test-elixir-sdk.yml@@ -0,0 +1,46 @@+name: Elixir SDK Test Suite++on:+ pull_request:+ branches:+ - main+ paths:+ - apps/elixir-sdk/**+ - .github/workflows/test-elixir-sdk.yml+ push:+ branches:+ - main+ paths:+ - apps/elixir-sdk/**+ - .github/workflows/test-elixir-sdk.yml+ workflow_dispatch:++permissions:+ contents: read++jobs:+ test:+ name: Build and Test+ runs-on: blacksmith-2vcpu-ubuntu-2404+ defaults:+ run:+ working-directory: ./apps/elixir-sdk+ steps:+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2+ with:+ persist-credentials: false++ - name: Set up Elixir+ uses: erlef/setup-beam@ee09b1e59bb240681c382eb1f0abc6a04af72764 # v1.23.0+ with:+ elixir-version: '1.17'+ otp-version: '27'++ - name: Install dependencies+ run: mix deps.get++ - name: Compile+ run: mix compile --warnings-as-errors++ - name: Run unit tests+ run: mix testdiff --git a/apps/elixir-sdk/lib/firecrawl.ex b/apps/elixir-sdk/lib/firecrawl.exindex 7f479fcdd..11e5485cb 100644--- a/apps/elixir-sdk/lib/firecrawl.ex+++ b/apps/elixir-sdk/lib/firecrawl.ex@@ -1819,7 +1819,9 @@ defmodule Firecrawl dodefp fetch_monitor_check_pages("", pages, _opts), do: pagesdefp fetch_monitor_check_pages(next, pages, opts) do- case Req.get(client(opts), url: next) do+ req = client(opts)++ case Req.get(req, url: Firecrawl.ApiOrigin.pin(next, req.options.base_url)) do{:ok, %Req.Response{body: body}} when is_map(body) ->data = Map.get(body, "data") || %{}next_pages = Map.get(data, "pages") || []@@ -1835,7 +1837,8 @@ defmodule Firecrawl dodefp fetch_monitor_check_pages!("", pages, _opts), do: pagesdefp fetch_monitor_check_pages!(next, pages, opts) do- response = Req.get!(client(opts), url: next)+ req = client(opts)+ response = Req.get!(req, url: Firecrawl.ApiOrigin.pin(next, req.options.base_url))body = response.bodydata = if is_map(body), do: Map.get(body, "data") || %{}, else: %{}next_pages = Map.get(data, "pages") || []diff --git a/apps/elixir-sdk/lib/firecrawl/api_origin.ex b/apps/elixir-sdk/lib/firecrawl/api_origin.exnew file mode 100644index 000000000..1c563bd41--- /dev/null+++ b/apps/elixir-sdk/lib/firecrawl/api_origin.ex@@ -0,0 +1,36 @@+defmodule Firecrawl.ApiOrigin do+ @moduledoc false++ @doc """+ Rewrites an absolute or protocol-relative `url` onto `api_url`'s scheme, host,+ and port so credentials never leave it, keeping the path and query and+ dropping the fragment.++ Relative URLs are returned unchanged. Raises `ArgumentError` if `api_url` is+ not an absolute URL.+ """+ @spec pin(String.t(), String.t() | URI.t()) :: String.t()+ def pin(url, api_url) do+ case URI.parse(url) do+ %URI{scheme: nil, host: nil} ->+ url++ parsed ->+ base = if is_binary(api_url) or is_struct(api_url, URI), do: URI.parse(api_url)++ case base do+ %URI{scheme: scheme, host: host} = base+ when is_binary(scheme) and host not in [nil, ""] ->+ URI.to_string(%URI{+ base+ | path: parsed.path || "/",+ query: parsed.query,+ fragment: nil+ })++ _ ->+ raise ArgumentError, "api_url must be an absolute URL, got: #{inspect(api_url)}"+ end+ end+ end+enddiff --git a/apps/elixir-sdk/mix.exs b/apps/elixir-sdk/mix.exsindex 138192e42..75b4bbf30 100644--- a/apps/elixir-sdk/mix.exs+++ b/apps/elixir-sdk/mix.exs@@ -1,7 +1,7 @@defmodule Firecrawl.MixProject douse Mix.Project- @version "1.11.0"+ @version "1.11.1"@source_url "https://github.com/firecrawl/firecrawl/tree/main/apps/elixir-sdk"def project dodiff --git a/apps/elixir-sdk/test/api_origin_test.exs b/apps/elixir-sdk/test/api_origin_test.exsnew file mode 100644index 000000000..49a561efa--- /dev/null+++ b/apps/elixir-sdk/test/api_origin_test.exs@@ -0,0 +1,69 @@+defmodule Firecrawl.ApiOriginTest do+ use ExUnit.Case, async: true++ alias Firecrawl.ApiOrigin++ @api_url "https://api.firecrawl.dev/v2"++ test "keeps same-origin URLs" do+ assert ApiOrigin.pin("https://api.firecrawl.dev/v2/crawl/abc?skip=10", @api_url) ==+ "https://api.firecrawl.dev/v2/crawl/abc?skip=10"+ end++ test "rewrites a foreign host onto the api_url origin" do+ assert ApiOrigin.pin("https://evil.example/v2/crawl/abc?skip=10", @api_url) ==+ "https://api.firecrawl.dev/v2/crawl/abc?skip=10"+ end++ test "drops userinfo from a foreign URL" do+ assert ApiOrigin.pin("https://user:[email protected]/v2/crawl/abc?skip=10", @api_url) ==+ "https://api.firecrawl.dev/v2/crawl/abc?skip=10"+ end++ test "accepts a URI struct as api_url" do+ assert ApiOrigin.pin("https://evil.example/v2/crawl/abc", URI.parse(@api_url)) ==+ "https://api.firecrawl.dev/v2/crawl/abc"+ end++ test "rewrites protocol-relative URLs" do+ assert ApiOrigin.pin("//evil.example/v2/crawl/abc?skip=10", @api_url) ==+ "https://api.firecrawl.dev/v2/crawl/abc?skip=10"+ end++ test "rewrites a different port" do+ assert ApiOrigin.pin("https://api.firecrawl.dev:8443/v2/crawl/abc", @api_url) ==+ "https://api.firecrawl.dev/v2/crawl/abc"++ assert ApiOrigin.pin("https://evil.example/v2/crawl/abc", "http://localhost:3002/v2") ==+ "http://localhost:3002/v2/crawl/abc"+ end++ test "rewrites a different scheme" do+ assert ApiOrigin.pin("http://api.firecrawl.dev/v2/crawl/abc", @api_url) ==+ "https://api.firecrawl.dev/v2/crawl/abc"+ end++ test "preserves path and query and drops the fragment" do+ assert ApiOrigin.pin(+ "https://evil.example/v2/batch/scrape/abc?skip=10&limit=5#frag",+ @api_url+ ) ==+ "https://api.firecrawl.dev/v2/batch/scrape/abc?skip=10&limit=5"++ assert ApiOrigin.pin("https://evil.example", @api_url) == "https://api.firecrawl.dev/"+ end++ test "returns relative URLs unchanged" do+ assert ApiOrigin.pin("/v2/crawl/abc?skip=10", @api_url) == "/v2/crawl/abc?skip=10"+ end++ test "raises when api_url is not absolute" do+ assert_raise ArgumentError, ~r/api_url must be an absolute URL/, fn ->+ ApiOrigin.pin("https://evil.example/v2/crawl/abc", "/v2")+ end++ assert_raise ArgumentError, ~r/api_url must be an absolute URL/, fn ->+ ApiOrigin.pin("https://evil.example/v2/crawl/abc", fn -> @api_url end)+ end+ end+enddiff --git a/apps/elixir-sdk/test/firecrawl_test.exs b/apps/elixir-sdk/test/firecrawl_test.exsindex 68db83368..73b7add3a 100644--- a/apps/elixir-sdk/test/firecrawl_test.exs+++ b/apps/elixir-sdk/test/firecrawl_test.exs@@ -1,14 +1,21 @@defmodule FirecrawlTest douse ExUnit.Case- test "raises when no API key is configured" do+ test "sends no authorization header when no API key is configured" doold = Application.get_env(:firecrawl, :api_key)Application.delete_env(:firecrawl, :api_key)on_exit(fn -> if old, do: Application.put_env(:firecrawl, :api_key, old) end)- assert_raise RuntimeError, ~r/Firecrawl API key not found/, fn ->- Firecrawl.get_credit_usage()+ parent = self()++ adapter = fn request ->+ send(parent, {:request, request})+ {request, Req.Response.new(status: 200, body: "")}end++ assert {:ok, %Req.Response{status: 200}} = Firecrawl.get_credit_usage(adapter: adapter)+ assert_receive {:request, request}+ refute Map.has_key?(request.headers, "authorization")endtest "does not raise when API key is in application config" do@@ -315,8 +322,7 @@ defmodule FirecrawlTest doassert request.url.path == "/v2/agent"refute Map.has_key?(URI.decode_query(request.url.query || ""), "before")- body = Jason.decode!(response.body)- assert [%{"id" => "job-123", "status" => "completed"}] = body["agents"]+ assert [%{"id" => "job-123", "status" => "completed"}] = response.body["agents"]endtest "list_agents sends before query param" do@@ -626,7 +632,7 @@ defmodule FirecrawlTest dotest "audit_metadata rejects unsupported fields" doassert_raise NimbleOptions.ValidationError, fn ->- Firecrawl.scrape_and_extract_from_url(+ Firecrawl.scrape_and_extract_from_url!([url: "https://example.com",audit_metadata: [username: "[email protected]", session: "session-123"]@@ -711,4 +717,117 @@ defmodule FirecrawlTest do"Expected #{name}/#{arity} to be defined in Firecrawl"endend++ defp monitor_check_adapter(parent, next) do+ fn request ->+ send(parent, {:request, request})++ body =+ case request.url.path do+ "/v2/monitor/mon-1/checks/chk-1" ->+ %{"success" => true, "data" => %{"id" => "chk-1", "pages" => [%{"url" => "https://a.example"}], "next" => next}}++ "/v2/monitor/mon-1/checks/chk-1/pages" ->+ %{+ "success" => true,+ "data" => %{+ "pages" => [%{"url" => "https://b.example"}],+ "next" => "https://evil2.example/v2/monitor/mon-1/checks/chk-1/last?skip=20"+ }+ }++ _ ->+ %{"success" => true, "data" => %{"pages" => [%{"url" => "https://c.example"}]}}+ end++ resp =+ Req.Response.new(+ status: 200,+ headers: %{"content-type" => ["application/json"]},+ body: Jason.encode!(body)+ )++ {request, resp}+ end+ end++ defp assert_monitor_next_pinned(get_check, next, expected_url) do+ adapter = monitor_check_adapter(self(), next)+ response = get_check.(adapter)++ assert response.body["data"]["pages"] == [+ %{"url" => "https://a.example"},+ %{"url" => "https://b.example"},+ %{"url" => "https://c.example"}+ ]++ assert_receive {:request, first}+ assert URI.to_string(first.url) == "https://api.firecrawl.dev/v2/monitor/mon-1/checks/chk-1"+ assert_receive {:request, followed}+ assert URI.to_string(followed.url) == expected_url+ assert followed.headers["authorization"] == ["Bearer test-key"]+ assert_receive {:request, last}+ assert URI.to_string(last.url) == "https://api.firecrawl.dev/v2/monitor/mon-1/checks/chk-1/last?skip=20"+ assert last.headers["authorization"] == ["Bearer test-key"]+ refute_receive {:request, _}+ end++ @monitor_next_cases [+ {"same origin", "https://api.firecrawl.dev/v2/monitor/mon-1/checks/chk-1/pages?skip=1",+ "https://api.firecrawl.dev/v2/monitor/mon-1/checks/chk-1/pages?skip=1"},+ {"cross host", "https://evil.example/v2/monitor/mon-1/checks/chk-1/pages?skip=10",+ "https://api.firecrawl.dev/v2/monitor/mon-1/checks/chk-1/pages?skip=10"},+ {"protocol-relative", "//evil.example/v2/monitor/mon-1/checks/chk-1/pages?skip=10",+ "https://api.firecrawl.dev/v2/monitor/mon-1/checks/chk-1/pages?skip=10"},+ {"different port", "https://api.firecrawl.dev:8443/v2/monitor/mon-1/checks/chk-1/pages?skip=10",+ "https://api.firecrawl.dev/v2/monitor/mon-1/checks/chk-1/pages?skip=10"},+ {"relative", "/monitor/mon-1/checks/chk-1/pages?skip=10",+ "https://api.firecrawl.dev/v2/monitor/mon-1/checks/chk-1/pages?skip=10"},+ {"userinfo", "https://user:[email protected]/v2/monitor/mon-1/checks/chk-1/pages?skip=10",+ "https://api.firecrawl.dev/v2/monitor/mon-1/checks/chk-1/pages?skip=10"},+ {"different scheme", "http://api.firecrawl.dev/v2/monitor/mon-1/checks/chk-1/pages?skip=10#frag",+ "https://api.firecrawl.dev/v2/monitor/mon-1/checks/chk-1/pages?skip=10"}+ ]++ for {name, next, expected} <- @monitor_next_cases do+ test "get_monitor_check pins #{name} next URL to the api_url origin" do+ assert_monitor_next_pinned(+ fn adapter ->+ assert {:ok, response} =+ Firecrawl.get_monitor_check("mon-1", "chk-1", [], api_key: "test-key", adapter: adapter)++ response+ end,+ unquote(next),+ unquote(expected)+ )+ end++ test "get_monitor_check! pins #{name} next URL to the api_url origin" do+ assert_monitor_next_pinned(+ fn adapter ->+ Firecrawl.get_monitor_check!("mon-1", "chk-1", [], api_key: "test-key", adapter: adapter)+ end,+ unquote(next),+ unquote(expected)+ )+ end+ end++ test "get_monitor_check pins next URLs to a self-hosted base_url origin" do+ adapter = monitor_check_adapter(self(), "https://evil.example/v2/monitor/mon-1/checks/chk-1/pages?skip=10")++ assert {:ok, _} =+ Firecrawl.get_monitor_check("mon-1", "chk-1", [],+ api_key: "test-key",+ base_url: "http://localhost:3002/v2",+ adapter: adapter+ )++ assert_receive {:request, first}+ assert URI.to_string(first.url) == "http://localhost:3002/v2/monitor/mon-1/checks/chk-1"+ assert_receive {:request, followed}+ assert URI.to_string(followed.url) == "http://localhost:3002/v2/monitor/mon-1/checks/chk-1/pages?skip=10"+ assert followed.headers["authorization"] == ["Bearer test-key"]+ endend