Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Fix favicons (#2257)
Uku Tahte849e030586d
diff --git a/config/test.exs b/config/test.exsindex c311c15a92..d148d4ca17 100644--- a/config/test.exs+++ b/config/test.exs@@ -62,4 +62,5 @@ config :geolix,]config :plausible,- session_timeout: 0+ session_timeout: 0,+ http_impl: Plausible.HTTPClient.Mockdiff --git a/lib/plausible/http_client.ex b/lib/plausible/http_client.exindex c65f5331db..5de47e0954 100644--- a/lib/plausible/http_client.ex+++ b/lib/plausible/http_client.ex@@ -1,49 +1,64 @@-defmodule Plausible.HTTPClient do- @moduledoc """- HTTP Client built on top of Finch.-- By default, request parameters are json-encoded.-- If a raw binary value is supplied, no encoding is performed.- If x-www-form-urlencoded content-type is set in headers,- URL encoding is invoked.- """+defmodule Plausible.HTTPClient.Non200Error do+ defstruct reason: nil- defmodule Non200Error do- defstruct reason: nil+ @type t :: %__MODULE__{reason: Finch.Response.t()}- @type t :: %__MODULE__{reason: Finch.Response.t()}-- @spec new(Finch.Response.t()) :: t()- def new(%Finch.Response{status: status} = response)- when is_integer(status) and (status < 200 or status >= 300) do- %__MODULE__{reason: response}- end+ @spec new(Finch.Response.t()) :: t()+ def new(%Finch.Response{status: status} = response)+ when is_integer(status) and (status < 200 or status >= 300) do+ %__MODULE__{reason: response}end+end+defmodule Plausible.HTTPClient.Interface do@type url() :: Finch.Request.url()@type headers() :: Finch.Request.headers()@type params() :: Finch.Request.body() | map()@type response() ::{:ok, Finch.Response.t()}- | {:error, Mint.Types.error() | Finch.Error.t() | Non200Error.t()}+ | {:error, Mint.Types.error() | Finch.Error.t() | Plausible.HTTPClient.Non200Error.t()}++ @callback get(url(), headers()) :: response()+ @callback get(url()) :: response()+ @callback post(url(), headers(), params()) :: response()+end++defmodule Plausible.HTTPClient do+ @moduledoc """+ HTTP Client built on top of Finch.++ By default, request parameters are json-encoded.++ If a raw binary value is supplied, no encoding is performed.+ If x-www-form-urlencoded content-type is set in headers,+ URL encoding is invoked.+ """@doc """Make a POST request"""- @spec post(url(), headers(), params()) :: response()- def(post(url, headers \\ [], params \\ nil)) do+ @behaviour Plausible.HTTPClient.Interface++ @impl Plausible.HTTPClient.Interface+ def post(url, headers \\ [], params \\ nil) docall(:post, url, headers, params)end@doc """Make a GET request"""- @spec get(url(), headers()) :: response()+ @impl Plausible.HTTPClient.Interfacedef get(url, headers \\ []) docall(:get, url, headers, nil)end+ # TODO: Is it possible to tell the type checker that we're returning a module that conforms to the+ # Plausible.HTTPClient.Interface behaviour?+ @spec impl() :: Plausible.HTTPClient+ def impl() do+ Application.get_env(:plausible, :http_impl, __MODULE__)+ end+defp call(method, url, headers, params) do{params, headers} = maybe_encode_params(params, headers)@@ -91,7 +106,7 @@ defmodule Plausible.HTTPClient doenddefp tag_error({:ok, %Finch.Response{status: _} = response}) do- {:error, Non200Error.new(response)}+ {:error, Plausible.HTTPClient.Non200Error.new(response)}enddefp tag_error({:error, _} = error) dodiff --git a/lib/plausible_web/plugs/favicon.ex b/lib/plausible_web/plugs/favicon.exindex d60e4d3266..02bce94b03 100644--- a/lib/plausible_web/plugs/favicon.ex+++ b/lib/plausible_web/plugs/favicon.ex@@ -1,29 +1,58 @@defmodule PlausibleWeb.Favicon do+ @referer_domains_file "priv/referer_favicon_domains.json"+ @moduledoc """+ A Plug that fetches favicon images from DuckDuckGo and returns them+ to the Plausible frontend.++ The proxying is there so we can reduce the number of third-party domains that+ the browser clients need to connect to. Our goal is to have 0 third-party domain+ connections on the website for privacy reasons.++ This module also maps between categorized sources and their respective URLs for favicons.+ What does that mean exactly? During ingestion we use `PlausibleWeb.RefInspector.parse/1` to+ categorize our referrer sources like so:++ google.com -> Google+ google.co.uk -> Google+ google.com.au -> Google++ So when we show Google as a source in the dashboard, the request to this plug will come as:+ https://plausible/io/favicon/sources/Google++ Now, when we want to show a favicon for Google, we need to convert Google -> google.com or+ some other hostname owned by Google:+ https://icons.duckduckgo.com/ip3/google.com.ico++ The mapping from source category -> source hostname is stored in "#{@referer_domains_file}" and+ managed by `Mix.Tasks.GenerateReferrerFavicons.run/1`+ """import Plug.Connalias Plausible.HTTPClientdef init(_) dodomains =- case File.read(Application.app_dir(:plausible, "priv/referer_favicon_domains.json")) do- {:ok, contents} ->- Jason.decode!(contents)-- _ ->- %{}- end+ File.read!(Application.app_dir(:plausible, @referer_domains_file))+ |> Jason.decode!()[favicon_domains: domains]end+ @doc """+ Proxies HTTP request to DuckDuckGo favicon service. Swallows hop-by-hop HTTP headers that+ should not be forwarded as defined in RFC 2616 (https://www.rfc-editor.org/rfc/rfc2616#section-13.5.1)+ """def call(conn, favicon_domains: favicon_domains) docase conn.path_info do["favicon", "sources", source] ->clean_source = URI.decode_www_form(source)domain = Map.get(favicon_domains, clean_source, clean_source)- case HTTPClient.get("https://icons.duckduckgo.com/ip3/#{domain}.ico") do+ case HTTPClient.impl().get("https://icons.duckduckgo.com/ip3/#{domain}.ico") do{:ok, res} ->- send_response(conn, res)+ conn+ |> forward_headers(res)+ |> send_resp(200, res.body)+ |> halt_ ->send_resp(conn, 503, "") |> halt@@ -34,13 +63,9 @@ defmodule PlausibleWeb.Favicon doendend- defp send_response(conn, res) do- conn =- Enum.filter(res.headers, fn {key, _val} -> key != "Transfer-Encoding" end)- |> Enum.reduce(conn, fn {key, val}, conn ->- put_resp_header(conn, key, val)- end)-- send_resp(conn, 200, res.body) |> halt+ @forwarded_headers ["content-type", "cache-control", "expires"]+ defp forward_headers(conn, response) do+ headers = Enum.filter(response.headers, fn {k, _} -> k in @forwarded_headers end)+ %Plug.Conn{conn | resp_headers: headers}endenddiff --git a/mix.exs b/mix.exsindex e3b8cd8bc1..b56e726482 100644--- a/mix.exs+++ b/mix.exs@@ -88,6 +88,7 @@ defmodule Plausible.MixProject do{:kaffy, "~> 0.9.0"},{:location, git: "https://github.com/plausible/location.git"},{:mimic, "~> 1.7", only: :test},+ {:mox, "~> 1.0", only: :test},{:nanoid, "~> 2.0.2"},{:oauther, "~> 1.3"},{:oban, "~> 2.11"},diff --git a/mix.lock b/mix.lockindex c24149f13e..c8fd7411e9 100644--- a/mix.lock+++ b/mix.lock@@ -76,6 +76,7 @@"mimic": {:hex, :mimic, "1.7.2", "27007e4e0c746ddb6d56a386c40585088b35621ae2d7167160e8c3283e8cd585", [:mix], [], "hexpm", "e4d40550523841055aa469f5125d124ab89ce8b2d3686cab908b98dff5e6111b"},"mint": {:hex, :mint, "1.4.2", "50330223429a6e1260b2ca5415f69b0ab086141bc76dc2fbf34d7c389a6675b2", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "ce75a5bbcc59b4d7d8d70f8b2fc284b1751ffb35c7b6a6302b5192f8ab4ddd80"},"mmdb2_decoder": {:hex, :mmdb2_decoder, "3.0.1", "78e3aedde88035c6873ada5ceaf41b7f15a6259ed034e0eaca72ccfa937798f0", [:mix], [], "hexpm", "316af0f388fac824782d944f54efe78e7c9691bbbdb0afd5cccdd0510adf559d"},+ "mox": {:hex, :mox, "1.0.2", "dc2057289ac478b35760ba74165b4b3f402f68803dd5aecd3bfd19c183815d64", [:mix], [], "hexpm", "f9864921b3aaf763c8741b5b8e6f908f44566f1e427b2630e89e9a73b981fef2"},"nanoid": {:hex, :nanoid, "2.0.5", "1d2948d8967ef2d948a58c3fef02385040bd9823fc6394bd604b8d98e5516b22", [:mix], [], "hexpm", "956e8876321104da72aa48770539ff26b36b744cd26753ec8e7a8a37e53d5f58"},"nimble_options": {:hex, :nimble_options, "0.4.0", "c89babbab52221a24b8d1ff9e7d838be70f0d871be823165c94dd3418eea728f", [:mix], [], "hexpm", "e6701c1af326a11eea9634a3b1c62b475339ace9456c1a23ec3bc9a847bca02d"},"nimble_parsec": {:hex, :nimble_parsec, "1.2.3", "244836e6e3f1200c7f30cb56733fd808744eca61fd182f731eac4af635cc6d0b", [:mix], [], "hexpm", "c8d789e39b9131acf7b99291e93dae60ab48ef14a7ee9d58c6964f59efb570b0"},diff --git a/test/plausible/http_client_test.exs b/test/plausible/http_client_test.exsindex b38c3dabca..669270d4f2 100644--- a/test/plausible/http_client_test.exs+++ b/test/plausible/http_client_test.exs@@ -122,6 +122,16 @@ defmodule Plausible.HTTPClientTest do}} = HTTPClient.get(bypass_url(bypass, path: "/get"))end+ test "header keys are downcased but values are not", %{bypass: bypass} do+ Bypass.expect_once(bypass, "GET", "/get", fn conn ->+ Conn.resp(conn, 200, "ok")+ |> Conn.put_resp_header("Some-Header", "Header-Value")+ end)++ assert {:ok, res} = HTTPClient.get(bypass_url(bypass, path: "/get"))+ assert {"some-header", "Header-Value"} in res.headers+ end+defp bypass_url(bypass, opts) doport = bypass.portpath = Keyword.get(opts, :path, "/")diff --git a/test/plausible_web/plugs/favicon_test.exs b/test/plausible_web/plugs/favicon_test.exsnew file mode 100644index 0000000000..b15beca005--- /dev/null+++ b/test/plausible_web/plugs/favicon_test.exs@@ -0,0 +1,87 @@+defmodule PlausibleWeb.FaviconTest do+ use Plausible.DataCase+ use Plug.Test+ alias PlausibleWeb.Favicon++ import Mox+ setup :verify_on_exit!++ setup_all do+ opts = PlausibleWeb.Favicon.init(nil)++ %{plug_opts: opts}+ end++ test "ignores request on a URL it does not need to handle", %{plug_opts: plug_opts} do+ old_conn = conn(:get, "/irrelevant")+ new_conn = Favicon.call(old_conn, plug_opts)++ refute new_conn.halted+ assert old_conn == new_conn+ end++ test "proxies request on favicon URL to duckduckgo", %{plug_opts: plug_opts} do+ expect(+ Plausible.HTTPClient.Mock,+ :get,+ fn "https://icons.duckduckgo.com/ip3/plausible.io.ico" ->+ {:ok, %Finch.Response{status: 200, body: "favicon response body"}}+ end+ )++ conn =+ conn(:get, "/favicon/sources/plausible.io")+ |> Favicon.call(plug_opts)++ assert conn.halted+ assert conn.status == 200+ assert conn.resp_body == "favicon response body"+ end++ test "maps a categorized source to URL for favicon", %{plug_opts: plug_opts} do+ expect(+ Plausible.HTTPClient.Mock,+ :get,+ fn "https://icons.duckduckgo.com/ip3/facebook.com.ico" ->+ {:ok, %Finch.Response{status: 200, body: "favicon response body"}}+ end+ )++ conn =+ conn(:get, "/favicon/sources/Facebook")+ |> Favicon.call(plug_opts)++ assert conn.halted+ assert conn.status == 200+ assert conn.resp_body == "favicon response body"+ end++ test "copies content-type header from the proxied response", %{plug_opts: plug_opts} do+ expect(+ Plausible.HTTPClient.Mock,+ :get,+ fn "https://icons.duckduckgo.com/ip3/plausible.io.ico" ->+ {:ok,+ %Finch.Response{+ status: 200,+ body: "favicon response body",+ headers: [+ {"transfer-encoding", "chunked"},+ {"content-type", "should-pass-through"}+ ]+ }}+ end+ )++ conn =+ conn(:get, "/favicon/sources/plausible.io")+ |> Favicon.call(plug_opts)++ assert conn.halted+ assert conn.status == 200++ assert conn.resp_headers == [+ {"content-type", "should-pass-through"}+ ]+ end+enddiff --git a/test/test_helper.exs b/test/test_helper.exsindex 78603315c3..3fb7b96fe5 100644--- a/test/test_helper.exs+++ b/test/test_helper.exs@@ -1,6 +1,7 @@{:ok, _} = Application.ensure_all_started(:ex_machina)Plausible.Test.ClickhouseSetup.run()Mimic.copy(FunWithFlags)+Mox.defmock(Plausible.HTTPClient.Mock, for: Plausible.HTTPClient.Interface)ExUnit.start()Application.ensure_all_started(:double)Ecto.Adapters.SQL.Sandbox.mode(Plausible.Repo, :manual)