Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(elixir-sdk): pin pagination next URLs to the api_url origin (#4868)

Gergő Móricz1f52d600ffcd
diff --git a/.github/workflows/test-elixir-sdk.yml b/.github/workflows/test-elixir-sdk.yml
new file mode 100644
index 000000000..050b635b6
--- /dev/null
+++ b/.github/workflows/test-elixir-sdk.yml
@@ -0,0 +1,46 @@
+name: Elixir SDK Test Suite
+
+on:
+ pull_request:
+ branches:
+ - main
+ paths:
+ - apps/elixir-sdk/**
+ - .github/workflows/test-elixir-sdk.yml
+ push:
+ branches:
+ - main
+ paths:
+ - apps/elixir-sdk/**
+ - .github/workflows/test-elixir-sdk.yml
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+jobs:
+ test:
+ name: Build and Test
+ runs-on: blacksmith-2vcpu-ubuntu-2404
+ defaults:
+ run:
+ working-directory: ./apps/elixir-sdk
+ steps:
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+ with:
+ persist-credentials: false
+
+ - name: Set up Elixir
+ uses: erlef/setup-beam@ee09b1e59bb240681c382eb1f0abc6a04af72764 # v1.23.0
+ with:
+ elixir-version: '1.17'
+ otp-version: '27'
+
+ - name: Install dependencies
+ run: mix deps.get
+
+ - name: Compile
+ run: mix compile --warnings-as-errors
+
+ - name: Run unit tests
+ run: mix test
diff --git a/apps/elixir-sdk/lib/firecrawl.ex b/apps/elixir-sdk/lib/firecrawl.ex
index 7f479fcdd..11e5485cb 100644
--- a/apps/elixir-sdk/lib/firecrawl.ex
+++ b/apps/elixir-sdk/lib/firecrawl.ex
@@ -1819,7 +1819,9 @@ defmodule Firecrawl do
defp fetch_monitor_check_pages("", pages, _opts), do: pages
defp fetch_monitor_check_pages(next, pages, opts) do
- case Req.get(client(opts), url: next) do
+ req = client(opts)
+
+ case Req.get(req, url: Firecrawl.ApiOrigin.pin(next, req.options.base_url)) do
{:ok, %Req.Response{body: body}} when is_map(body) ->
data = Map.get(body, "data") || %{}
next_pages = Map.get(data, "pages") || []
@@ -1835,7 +1837,8 @@ defmodule Firecrawl do
defp fetch_monitor_check_pages!("", pages, _opts), do: pages
defp fetch_monitor_check_pages!(next, pages, opts) do
- response = Req.get!(client(opts), url: next)
+ req = client(opts)
+ response = Req.get!(req, url: Firecrawl.ApiOrigin.pin(next, req.options.base_url))
body = response.body
data = if is_map(body), do: Map.get(body, "data") || %{}, else: %{}
next_pages = Map.get(data, "pages") || []
diff --git a/apps/elixir-sdk/lib/firecrawl/api_origin.ex b/apps/elixir-sdk/lib/firecrawl/api_origin.ex
new file mode 100644
index 000000000..1c563bd41
--- /dev/null
+++ b/apps/elixir-sdk/lib/firecrawl/api_origin.ex
@@ -0,0 +1,36 @@
+defmodule Firecrawl.ApiOrigin do
+ @moduledoc false
+
+ @doc """
+ Rewrites an absolute or protocol-relative `url` onto `api_url`'s scheme, host,
+ and port so credentials never leave it, keeping the path and query and
+ dropping the fragment.
+
+ Relative URLs are returned unchanged. Raises `ArgumentError` if `api_url` is
+ not an absolute URL.
+ """
+ @spec pin(String.t(), String.t() | URI.t()) :: String.t()
+ def pin(url, api_url) do
+ case URI.parse(url) do
+ %URI{scheme: nil, host: nil} ->
+ url
+
+ parsed ->
+ base = if is_binary(api_url) or is_struct(api_url, URI), do: URI.parse(api_url)
+
+ case base do
+ %URI{scheme: scheme, host: host} = base
+ when is_binary(scheme) and host not in [nil, ""] ->
+ URI.to_string(%URI{
+ base
+ | path: parsed.path || "/",
+ query: parsed.query,
+ fragment: nil
+ })
+
+ _ ->
+ raise ArgumentError, "api_url must be an absolute URL, got: #{inspect(api_url)}"
+ end
+ end
+ end
+end
diff --git a/apps/elixir-sdk/mix.exs b/apps/elixir-sdk/mix.exs
index 138192e42..75b4bbf30 100644
--- a/apps/elixir-sdk/mix.exs
+++ b/apps/elixir-sdk/mix.exs
@@ -1,7 +1,7 @@
defmodule Firecrawl.MixProject do
use Mix.Project
- @version "1.11.0"
+ @version "1.11.1"
@source_url "https://github.com/firecrawl/firecrawl/tree/main/apps/elixir-sdk"
def project do
diff --git a/apps/elixir-sdk/test/api_origin_test.exs b/apps/elixir-sdk/test/api_origin_test.exs
new file mode 100644
index 000000000..49a561efa
--- /dev/null
+++ b/apps/elixir-sdk/test/api_origin_test.exs
@@ -0,0 +1,69 @@
+defmodule Firecrawl.ApiOriginTest do
+ use ExUnit.Case, async: true
+
+ alias Firecrawl.ApiOrigin
+
+ @api_url "https://api.firecrawl.dev/v2"
+
+ test "keeps same-origin URLs" do
+ assert ApiOrigin.pin("https://api.firecrawl.dev/v2/crawl/abc?skip=10", @api_url) ==
+ "https://api.firecrawl.dev/v2/crawl/abc?skip=10"
+ end
+
+ test "rewrites a foreign host onto the api_url origin" do
+ assert ApiOrigin.pin("https://evil.example/v2/crawl/abc?skip=10", @api_url) ==
+ "https://api.firecrawl.dev/v2/crawl/abc?skip=10"
+ end
+
+ test "drops userinfo from a foreign URL" do
+ assert ApiOrigin.pin("https://user:[email protected]/v2/crawl/abc?skip=10", @api_url) ==
+ "https://api.firecrawl.dev/v2/crawl/abc?skip=10"
+ end
+
+ test "accepts a URI struct as api_url" do
+ assert ApiOrigin.pin("https://evil.example/v2/crawl/abc", URI.parse(@api_url)) ==
+ "https://api.firecrawl.dev/v2/crawl/abc"
+ end
+
+ test "rewrites protocol-relative URLs" do
+ assert ApiOrigin.pin("//evil.example/v2/crawl/abc?skip=10", @api_url) ==
+ "https://api.firecrawl.dev/v2/crawl/abc?skip=10"
+ end
+
+ test "rewrites a different port" do
+ assert ApiOrigin.pin("https://api.firecrawl.dev:8443/v2/crawl/abc", @api_url) ==
+ "https://api.firecrawl.dev/v2/crawl/abc"
+
+ assert ApiOrigin.pin("https://evil.example/v2/crawl/abc", "http://localhost:3002/v2") ==
+ "http://localhost:3002/v2/crawl/abc"
+ end
+
+ test "rewrites a different scheme" do
+ assert ApiOrigin.pin("http://api.firecrawl.dev/v2/crawl/abc", @api_url) ==
+ "https://api.firecrawl.dev/v2/crawl/abc"
+ end
+
+ test "preserves path and query and drops the fragment" do
+ assert ApiOrigin.pin(
+ "https://evil.example/v2/batch/scrape/abc?skip=10&limit=5#frag",
+ @api_url
+ ) ==
+ "https://api.firecrawl.dev/v2/batch/scrape/abc?skip=10&limit=5"
+
+ assert ApiOrigin.pin("https://evil.example", @api_url) == "https://api.firecrawl.dev/"
+ end
+
+ test "returns relative URLs unchanged" do
+ assert ApiOrigin.pin("/v2/crawl/abc?skip=10", @api_url) == "/v2/crawl/abc?skip=10"
+ end
+
+ test "raises when api_url is not absolute" do
+ assert_raise ArgumentError, ~r/api_url must be an absolute URL/, fn ->
+ ApiOrigin.pin("https://evil.example/v2/crawl/abc", "/v2")
+ end
+
+ assert_raise ArgumentError, ~r/api_url must be an absolute URL/, fn ->
+ ApiOrigin.pin("https://evil.example/v2/crawl/abc", fn -> @api_url end)
+ end
+ end
+end
diff --git a/apps/elixir-sdk/test/firecrawl_test.exs b/apps/elixir-sdk/test/firecrawl_test.exs
index 68db83368..73b7add3a 100644
--- a/apps/elixir-sdk/test/firecrawl_test.exs
+++ b/apps/elixir-sdk/test/firecrawl_test.exs
@@ -1,14 +1,21 @@
defmodule FirecrawlTest do
use ExUnit.Case
- test "raises when no API key is configured" do
+ test "sends no authorization header when no API key is configured" do
old = Application.get_env(:firecrawl, :api_key)
Application.delete_env(:firecrawl, :api_key)
on_exit(fn -> if old, do: Application.put_env(:firecrawl, :api_key, old) end)
- assert_raise RuntimeError, ~r/Firecrawl API key not found/, fn ->
- Firecrawl.get_credit_usage()
+ parent = self()
+
+ adapter = fn request ->
+ send(parent, {:request, request})
+ {request, Req.Response.new(status: 200, body: "")}
end
+
+ assert {:ok, %Req.Response{status: 200}} = Firecrawl.get_credit_usage(adapter: adapter)
+ assert_receive {:request, request}
+ refute Map.has_key?(request.headers, "authorization")
end
test "does not raise when API key is in application config" do
@@ -315,8 +322,7 @@ defmodule FirecrawlTest do
assert request.url.path == "/v2/agent"
refute Map.has_key?(URI.decode_query(request.url.query || ""), "before")
- body = Jason.decode!(response.body)
- assert [%{"id" => "job-123", "status" => "completed"}] = body["agents"]
+ assert [%{"id" => "job-123", "status" => "completed"}] = response.body["agents"]
end
test "list_agents sends before query param" do
@@ -626,7 +632,7 @@ defmodule FirecrawlTest do
test "audit_metadata rejects unsupported fields" do
assert_raise NimbleOptions.ValidationError, fn ->
- Firecrawl.scrape_and_extract_from_url(
+ Firecrawl.scrape_and_extract_from_url!(
[
url: "https://example.com",
audit_metadata: [username: "[email protected]", session: "session-123"]
@@ -711,4 +717,117 @@ defmodule FirecrawlTest do
"Expected #{name}/#{arity} to be defined in Firecrawl"
end
end
+
+ defp monitor_check_adapter(parent, next) do
+ fn request ->
+ send(parent, {:request, request})
+
+ body =
+ case request.url.path do
+ "/v2/monitor/mon-1/checks/chk-1" ->
+ %{"success" => true, "data" => %{"id" => "chk-1", "pages" => [%{"url" => "https://a.example"}], "next" => next}}
+
+ "/v2/monitor/mon-1/checks/chk-1/pages" ->
+ %{
+ "success" => true,
+ "data" => %{
+ "pages" => [%{"url" => "https://b.example"}],
+ "next" => "https://evil2.example/v2/monitor/mon-1/checks/chk-1/last?skip=20"
+ }
+ }
+
+ _ ->
+ %{"success" => true, "data" => %{"pages" => [%{"url" => "https://c.example"}]}}
+ end
+
+ resp =
+ Req.Response.new(
+ status: 200,
+ headers: %{"content-type" => ["application/json"]},
+ body: Jason.encode!(body)
+ )
+
+ {request, resp}
+ end
+ end
+
+ defp assert_monitor_next_pinned(get_check, next, expected_url) do
+ adapter = monitor_check_adapter(self(), next)
+ response = get_check.(adapter)
+
+ assert response.body["data"]["pages"] == [
+ %{"url" => "https://a.example"},
+ %{"url" => "https://b.example"},
+ %{"url" => "https://c.example"}
+ ]
+
+ assert_receive {:request, first}
+ assert URI.to_string(first.url) == "https://api.firecrawl.dev/v2/monitor/mon-1/checks/chk-1"
+ assert_receive {:request, followed}
+ assert URI.to_string(followed.url) == expected_url
+ assert followed.headers["authorization"] == ["Bearer test-key"]
+ assert_receive {:request, last}
+ assert URI.to_string(last.url) == "https://api.firecrawl.dev/v2/monitor/mon-1/checks/chk-1/last?skip=20"
+ assert last.headers["authorization"] == ["Bearer test-key"]
+ refute_receive {:request, _}
+ end
+
+ @monitor_next_cases [
+ {"same origin", "https://api.firecrawl.dev/v2/monitor/mon-1/checks/chk-1/pages?skip=1",
+ "https://api.firecrawl.dev/v2/monitor/mon-1/checks/chk-1/pages?skip=1"},
+ {"cross host", "https://evil.example/v2/monitor/mon-1/checks/chk-1/pages?skip=10",
+ "https://api.firecrawl.dev/v2/monitor/mon-1/checks/chk-1/pages?skip=10"},
+ {"protocol-relative", "//evil.example/v2/monitor/mon-1/checks/chk-1/pages?skip=10",
+ "https://api.firecrawl.dev/v2/monitor/mon-1/checks/chk-1/pages?skip=10"},
+ {"different port", "https://api.firecrawl.dev:8443/v2/monitor/mon-1/checks/chk-1/pages?skip=10",
+ "https://api.firecrawl.dev/v2/monitor/mon-1/checks/chk-1/pages?skip=10"},
+ {"relative", "/monitor/mon-1/checks/chk-1/pages?skip=10",
+ "https://api.firecrawl.dev/v2/monitor/mon-1/checks/chk-1/pages?skip=10"},
+ {"userinfo", "https://user:[email protected]/v2/monitor/mon-1/checks/chk-1/pages?skip=10",
+ "https://api.firecrawl.dev/v2/monitor/mon-1/checks/chk-1/pages?skip=10"},
+ {"different scheme", "http://api.firecrawl.dev/v2/monitor/mon-1/checks/chk-1/pages?skip=10#frag",
+ "https://api.firecrawl.dev/v2/monitor/mon-1/checks/chk-1/pages?skip=10"}
+ ]
+
+ for {name, next, expected} <- @monitor_next_cases do
+ test "get_monitor_check pins #{name} next URL to the api_url origin" do
+ assert_monitor_next_pinned(
+ fn adapter ->
+ assert {:ok, response} =
+ Firecrawl.get_monitor_check("mon-1", "chk-1", [], api_key: "test-key", adapter: adapter)
+
+ response
+ end,
+ unquote(next),
+ unquote(expected)
+ )
+ end
+
+ test "get_monitor_check! pins #{name} next URL to the api_url origin" do
+ assert_monitor_next_pinned(
+ fn adapter ->
+ Firecrawl.get_monitor_check!("mon-1", "chk-1", [], api_key: "test-key", adapter: adapter)
+ end,
+ unquote(next),
+ unquote(expected)
+ )
+ end
+ end
+
+ test "get_monitor_check pins next URLs to a self-hosted base_url origin" do
+ adapter = monitor_check_adapter(self(), "https://evil.example/v2/monitor/mon-1/checks/chk-1/pages?skip=10")
+
+ assert {:ok, _} =
+ Firecrawl.get_monitor_check("mon-1", "chk-1", [],
+ api_key: "test-key",
+ base_url: "http://localhost:3002/v2",
+ adapter: adapter
+ )
+
+ assert_receive {:request, first}
+ assert URI.to_string(first.url) == "http://localhost:3002/v2/monitor/mon-1/checks/chk-1"
+ assert_receive {:request, followed}
+ assert URI.to_string(followed.url) == "http://localhost:3002/v2/monitor/mon-1/checks/chk-1/pages?skip=10"
+ assert followed.headers["authorization"] == ["Bearer test-key"]
+ end
end