Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Fix favicons (#2257)

Uku Tahte849e030586d
diff --git a/config/test.exs b/config/test.exs
index c311c15a92..d148d4ca17 100644
--- a/config/test.exs
+++ b/config/test.exs
@@ -62,4 +62,5 @@ config :geolix,
]
config :plausible,
- session_timeout: 0
+ session_timeout: 0,
+ http_impl: Plausible.HTTPClient.Mock
diff --git a/lib/plausible/http_client.ex b/lib/plausible/http_client.ex
index c65f5331db..5de47e0954 100644
--- a/lib/plausible/http_client.ex
+++ b/lib/plausible/http_client.ex
@@ -1,49 +1,64 @@
-defmodule Plausible.HTTPClient do
- @moduledoc """
- HTTP Client built on top of Finch.
-
- By default, request parameters are json-encoded.
-
- If a raw binary value is supplied, no encoding is performed.
- If x-www-form-urlencoded content-type is set in headers,
- URL encoding is invoked.
- """
+defmodule Plausible.HTTPClient.Non200Error do
+ defstruct reason: nil
- defmodule Non200Error do
- defstruct reason: nil
+ @type t :: %__MODULE__{reason: Finch.Response.t()}
- @type t :: %__MODULE__{reason: Finch.Response.t()}
-
- @spec new(Finch.Response.t()) :: t()
- def new(%Finch.Response{status: status} = response)
- when is_integer(status) and (status < 200 or status >= 300) do
- %__MODULE__{reason: response}
- end
+ @spec new(Finch.Response.t()) :: t()
+ def new(%Finch.Response{status: status} = response)
+ when is_integer(status) and (status < 200 or status >= 300) do
+ %__MODULE__{reason: response}
end
+end
+defmodule Plausible.HTTPClient.Interface do
@type url() :: Finch.Request.url()
@type headers() :: Finch.Request.headers()
@type params() :: Finch.Request.body() | map()
@type response() ::
{:ok, Finch.Response.t()}
- | {:error, Mint.Types.error() | Finch.Error.t() | Non200Error.t()}
+ | {:error, Mint.Types.error() | Finch.Error.t() | Plausible.HTTPClient.Non200Error.t()}
+
+ @callback get(url(), headers()) :: response()
+ @callback get(url()) :: response()
+ @callback post(url(), headers(), params()) :: response()
+end
+
+defmodule Plausible.HTTPClient do
+ @moduledoc """
+ HTTP Client built on top of Finch.
+
+ By default, request parameters are json-encoded.
+
+ If a raw binary value is supplied, no encoding is performed.
+ If x-www-form-urlencoded content-type is set in headers,
+ URL encoding is invoked.
+ """
@doc """
Make a POST request
"""
- @spec post(url(), headers(), params()) :: response()
- def(post(url, headers \\ [], params \\ nil)) do
+ @behaviour Plausible.HTTPClient.Interface
+
+ @impl Plausible.HTTPClient.Interface
+ def post(url, headers \\ [], params \\ nil) do
call(:post, url, headers, params)
end
@doc """
Make a GET request
"""
- @spec get(url(), headers()) :: response()
+ @impl Plausible.HTTPClient.Interface
def get(url, headers \\ []) do
call(:get, url, headers, nil)
end
+ # TODO: Is it possible to tell the type checker that we're returning a module that conforms to the
+ # Plausible.HTTPClient.Interface behaviour?
+ @spec impl() :: Plausible.HTTPClient
+ def impl() do
+ Application.get_env(:plausible, :http_impl, __MODULE__)
+ end
+
defp call(method, url, headers, params) do
{params, headers} = maybe_encode_params(params, headers)
@@ -91,7 +106,7 @@ defmodule Plausible.HTTPClient do
end
defp tag_error({:ok, %Finch.Response{status: _} = response}) do
- {:error, Non200Error.new(response)}
+ {:error, Plausible.HTTPClient.Non200Error.new(response)}
end
defp tag_error({:error, _} = error) do
diff --git a/lib/plausible_web/plugs/favicon.ex b/lib/plausible_web/plugs/favicon.ex
index d60e4d3266..02bce94b03 100644
--- a/lib/plausible_web/plugs/favicon.ex
+++ b/lib/plausible_web/plugs/favicon.ex
@@ -1,29 +1,58 @@
defmodule PlausibleWeb.Favicon do
+ @referer_domains_file "priv/referer_favicon_domains.json"
+ @moduledoc """
+ A Plug that fetches favicon images from DuckDuckGo and returns them
+ to the Plausible frontend.
+
+ The proxying is there so we can reduce the number of third-party domains that
+ the browser clients need to connect to. Our goal is to have 0 third-party domain
+ connections on the website for privacy reasons.
+
+ This module also maps between categorized sources and their respective URLs for favicons.
+ What does that mean exactly? During ingestion we use `PlausibleWeb.RefInspector.parse/1` to
+ categorize our referrer sources like so:
+
+ google.com -> Google
+ google.co.uk -> Google
+ google.com.au -> Google
+
+ So when we show Google as a source in the dashboard, the request to this plug will come as:
+ https://plausible/io/favicon/sources/Google
+
+ Now, when we want to show a favicon for Google, we need to convert Google -> google.com or
+ some other hostname owned by Google:
+ https://icons.duckduckgo.com/ip3/google.com.ico
+
+ The mapping from source category -> source hostname is stored in "#{@referer_domains_file}" and
+ managed by `Mix.Tasks.GenerateReferrerFavicons.run/1`
+ """
import Plug.Conn
alias Plausible.HTTPClient
def init(_) do
domains =
- case File.read(Application.app_dir(:plausible, "priv/referer_favicon_domains.json")) do
- {:ok, contents} ->
- Jason.decode!(contents)
-
- _ ->
- %{}
- end
+ File.read!(Application.app_dir(:plausible, @referer_domains_file))
+ |> Jason.decode!()
[favicon_domains: domains]
end
+ @doc """
+ Proxies HTTP request to DuckDuckGo favicon service. Swallows hop-by-hop HTTP headers that
+ should not be forwarded as defined in RFC 2616 (https://www.rfc-editor.org/rfc/rfc2616#section-13.5.1)
+ """
def call(conn, favicon_domains: favicon_domains) do
case conn.path_info do
["favicon", "sources", source] ->
clean_source = URI.decode_www_form(source)
domain = Map.get(favicon_domains, clean_source, clean_source)
- case HTTPClient.get("https://icons.duckduckgo.com/ip3/#{domain}.ico") do
+ case HTTPClient.impl().get("https://icons.duckduckgo.com/ip3/#{domain}.ico") do
{:ok, res} ->
- send_response(conn, res)
+ conn
+ |> forward_headers(res)
+ |> send_resp(200, res.body)
+ |> halt
_ ->
send_resp(conn, 503, "") |> halt
@@ -34,13 +63,9 @@ defmodule PlausibleWeb.Favicon do
end
end
- defp send_response(conn, res) do
- conn =
- Enum.filter(res.headers, fn {key, _val} -> key != "Transfer-Encoding" end)
- |> Enum.reduce(conn, fn {key, val}, conn ->
- put_resp_header(conn, key, val)
- end)
-
- send_resp(conn, 200, res.body) |> halt
+ @forwarded_headers ["content-type", "cache-control", "expires"]
+ defp forward_headers(conn, response) do
+ headers = Enum.filter(response.headers, fn {k, _} -> k in @forwarded_headers end)
+ %Plug.Conn{conn | resp_headers: headers}
end
end
diff --git a/mix.exs b/mix.exs
index e3b8cd8bc1..b56e726482 100644
--- a/mix.exs
+++ b/mix.exs
@@ -88,6 +88,7 @@ defmodule Plausible.MixProject do
{:kaffy, "~> 0.9.0"},
{:location, git: "https://github.com/plausible/location.git"},
{:mimic, "~> 1.7", only: :test},
+ {:mox, "~> 1.0", only: :test},
{:nanoid, "~> 2.0.2"},
{:oauther, "~> 1.3"},
{:oban, "~> 2.11"},
diff --git a/mix.lock b/mix.lock
index c24149f13e..c8fd7411e9 100644
--- a/mix.lock
+++ b/mix.lock
@@ -76,6 +76,7 @@
"mimic": {:hex, :mimic, "1.7.2", "27007e4e0c746ddb6d56a386c40585088b35621ae2d7167160e8c3283e8cd585", [:mix], [], "hexpm", "e4d40550523841055aa469f5125d124ab89ce8b2d3686cab908b98dff5e6111b"},
"mint": {:hex, :mint, "1.4.2", "50330223429a6e1260b2ca5415f69b0ab086141bc76dc2fbf34d7c389a6675b2", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "ce75a5bbcc59b4d7d8d70f8b2fc284b1751ffb35c7b6a6302b5192f8ab4ddd80"},
"mmdb2_decoder": {:hex, :mmdb2_decoder, "3.0.1", "78e3aedde88035c6873ada5ceaf41b7f15a6259ed034e0eaca72ccfa937798f0", [:mix], [], "hexpm", "316af0f388fac824782d944f54efe78e7c9691bbbdb0afd5cccdd0510adf559d"},
+ "mox": {:hex, :mox, "1.0.2", "dc2057289ac478b35760ba74165b4b3f402f68803dd5aecd3bfd19c183815d64", [:mix], [], "hexpm", "f9864921b3aaf763c8741b5b8e6f908f44566f1e427b2630e89e9a73b981fef2"},
"nanoid": {:hex, :nanoid, "2.0.5", "1d2948d8967ef2d948a58c3fef02385040bd9823fc6394bd604b8d98e5516b22", [:mix], [], "hexpm", "956e8876321104da72aa48770539ff26b36b744cd26753ec8e7a8a37e53d5f58"},
"nimble_options": {:hex, :nimble_options, "0.4.0", "c89babbab52221a24b8d1ff9e7d838be70f0d871be823165c94dd3418eea728f", [:mix], [], "hexpm", "e6701c1af326a11eea9634a3b1c62b475339ace9456c1a23ec3bc9a847bca02d"},
"nimble_parsec": {:hex, :nimble_parsec, "1.2.3", "244836e6e3f1200c7f30cb56733fd808744eca61fd182f731eac4af635cc6d0b", [:mix], [], "hexpm", "c8d789e39b9131acf7b99291e93dae60ab48ef14a7ee9d58c6964f59efb570b0"},
diff --git a/test/plausible/http_client_test.exs b/test/plausible/http_client_test.exs
index b38c3dabca..669270d4f2 100644
--- a/test/plausible/http_client_test.exs
+++ b/test/plausible/http_client_test.exs
@@ -122,6 +122,16 @@ defmodule Plausible.HTTPClientTest do
}} = HTTPClient.get(bypass_url(bypass, path: "/get"))
end
+ test "header keys are downcased but values are not", %{bypass: bypass} do
+ Bypass.expect_once(bypass, "GET", "/get", fn conn ->
+ Conn.resp(conn, 200, "ok")
+ |> Conn.put_resp_header("Some-Header", "Header-Value")
+ end)
+
+ assert {:ok, res} = HTTPClient.get(bypass_url(bypass, path: "/get"))
+ assert {"some-header", "Header-Value"} in res.headers
+ end
+
defp bypass_url(bypass, opts) do
port = bypass.port
path = Keyword.get(opts, :path, "/")
diff --git a/test/plausible_web/plugs/favicon_test.exs b/test/plausible_web/plugs/favicon_test.exs
new file mode 100644
index 0000000000..b15beca005
--- /dev/null
+++ b/test/plausible_web/plugs/favicon_test.exs
@@ -0,0 +1,87 @@
+defmodule PlausibleWeb.FaviconTest do
+ use Plausible.DataCase
+ use Plug.Test
+ alias PlausibleWeb.Favicon
+
+ import Mox
+ setup :verify_on_exit!
+
+ setup_all do
+ opts = PlausibleWeb.Favicon.init(nil)
+
+ %{plug_opts: opts}
+ end
+
+ test "ignores request on a URL it does not need to handle", %{plug_opts: plug_opts} do
+ old_conn = conn(:get, "/irrelevant")
+ new_conn = Favicon.call(old_conn, plug_opts)
+
+ refute new_conn.halted
+ assert old_conn == new_conn
+ end
+
+ test "proxies request on favicon URL to duckduckgo", %{plug_opts: plug_opts} do
+ expect(
+ Plausible.HTTPClient.Mock,
+ :get,
+ fn "https://icons.duckduckgo.com/ip3/plausible.io.ico" ->
+ {:ok, %Finch.Response{status: 200, body: "favicon response body"}}
+ end
+ )
+
+ conn =
+ conn(:get, "/favicon/sources/plausible.io")
+ |> Favicon.call(plug_opts)
+
+ assert conn.halted
+ assert conn.status == 200
+ assert conn.resp_body == "favicon response body"
+ end
+
+ test "maps a categorized source to URL for favicon", %{plug_opts: plug_opts} do
+ expect(
+ Plausible.HTTPClient.Mock,
+ :get,
+ fn "https://icons.duckduckgo.com/ip3/facebook.com.ico" ->
+ {:ok, %Finch.Response{status: 200, body: "favicon response body"}}
+ end
+ )
+
+ conn =
+ conn(:get, "/favicon/sources/Facebook")
+ |> Favicon.call(plug_opts)
+
+ assert conn.halted
+ assert conn.status == 200
+ assert conn.resp_body == "favicon response body"
+ end
+
+ test "copies content-type header from the proxied response", %{plug_opts: plug_opts} do
+ expect(
+ Plausible.HTTPClient.Mock,
+ :get,
+ fn "https://icons.duckduckgo.com/ip3/plausible.io.ico" ->
+ {:ok,
+ %Finch.Response{
+ status: 200,
+ body: "favicon response body",
+ headers: [
+ {"transfer-encoding", "chunked"},
+ {"content-type", "should-pass-through"}
+ ]
+ }}
+ end
+ )
+
+ conn =
+ conn(:get, "/favicon/sources/plausible.io")
+ |> Favicon.call(plug_opts)
+
+ assert conn.halted
+ assert conn.status == 200
+
+ assert conn.resp_headers == [
+ {"content-type", "should-pass-through"}
+ ]
+ end
+end
diff --git a/test/test_helper.exs b/test/test_helper.exs
index 78603315c3..3fb7b96fe5 100644
--- a/test/test_helper.exs
+++ b/test/test_helper.exs
@@ -1,6 +1,7 @@
{:ok, _} = Application.ensure_all_started(:ex_machina)
Plausible.Test.ClickhouseSetup.run()
Mimic.copy(FunWithFlags)
+Mox.defmock(Plausible.HTTPClient.Mock, for: Plausible.HTTPClient.Interface)
ExUnit.start()
Application.ensure_all_started(:double)
Ecto.Adapters.SQL.Sandbox.mode(Plausible.Repo, :manual)