Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Kick off sites by domain cache implementation (#2434)

Adam Rutkowski457a5584712f
diff --git a/config/config.exs b/config/config.exs
index 369b11b00b..32456c2a75 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -41,4 +41,9 @@ config :plausible, Plausible.Repo,
connect_timeout: 300_000,
handshake_timeout: 300_000
+config :plausible,
+ sites_by_domain_cache_enabled: true,
+ sites_by_domain_cache_refresh_interval_max_jitter: :timer.seconds(5),
+ sites_by_domain_cache_refresh_interval: :timer.minutes(15)
+
import_config "#{config_env()}.exs"
diff --git a/config/test.exs b/config/test.exs
index d148d4ca17..475cd355a3 100644
--- a/config/test.exs
+++ b/config/test.exs
@@ -63,4 +63,5 @@ config :geolix,
config :plausible,
session_timeout: 0,
- http_impl: Plausible.HTTPClient.Mock
+ http_impl: Plausible.HTTPClient.Mock,
+ sites_by_domain_cache_enabled: false
diff --git a/lib/plausible/application.ex b/lib/plausible/application.ex
index a267567f90..b060c3449e 100644
--- a/lib/plausible/application.ex
+++ b/lib/plausible/application.ex
@@ -21,6 +21,8 @@ defmodule Plausible.Application do
Supervisor.child_spec({Cachex, name: :sessions, limit: nil, stats: true},
id: :cachex_sessions
),
+ {Plausible.Site.Cache, []},
+ {Plausible.Site.Cache.Warmer, []},
PlausibleWeb.Endpoint,
{Oban, Application.get_env(:plausible, Oban)},
Plausible.PromEx
diff --git a/lib/plausible/site.ex b/lib/plausible/site.ex
index adfb124a53..bbe8c90b82 100644
--- a/lib/plausible/site.ex
+++ b/lib/plausible/site.ex
@@ -31,6 +31,11 @@ defmodule Plausible.Site do
has_one :custom_domain, Plausible.Site.CustomDomain
has_one :spike_notification, Plausible.Site.SpikeNotification
+ # If `from_cache?` is set, the struct might be incomplete - see `Plausible.Site.Cache`.
+ # Use `Plausible.Repo.reload!(cached_site)` to pre-fill missing fields if
+ # strictly necessary.
+ field :from_cache?, :boolean, virtual: true, default: false
+
timestamps()
end
diff --git a/lib/plausible/site/cache.ex b/lib/plausible/site/cache.ex
new file mode 100644
index 0000000000..c24a7c0aa2
--- /dev/null
+++ b/lib/plausible/site/cache.ex
@@ -0,0 +1,112 @@
+defmodule Plausible.Site.Cache do
+ @moduledoc """
+ A sites by domain caching interface.
+
+ Serves as a thin wrapper around Cachex, but the underlying
+ implementation can be transparently swapped.
+
+ Even though the Cachex process is started, cache access is disabled
+ during tests via the `:sites_by_domain_cache_enabled` application env key.
+ This can be overriden on case by case basis, using the child specs options.
+
+ When Cache is disabled via application env, the `get/1` function
+ falls back to pure database lookups. This should help with introducing
+ cached lookups in existing code, so that no existing tests should break.
+
+ To differentiate cached Site structs from those retrieved directly from the
+ database, a virtual schema field `from_cache?: true` is set.
+ This indicates the `Plausible.Site` struct is incomplete in comparison to its
+ database counterpart -- to spare bandwidth and query execution time,
+ only selected database columns are retrieved and cached.
+
+ The `@cached_schema_fields` attribute defines the list of DB columns
+ queried on cache pre-fill.
+
+ Also see tests for more comprehensive examples.
+ """
+ require Logger
+
+ import Ecto.Query
+
+ alias Plausible.Site
+
+ @cache_name :sites_by_domain
+
+ @cached_schema_fields ~w(
+ id
+ domain
+ ingest_rate_limit_scale_seconds
+ ingest_rate_limit_threshold
+ )a
+
+ def name(), do: @cache_name
+
+ @spec child_spec(Keyword.t()) :: Supervisor.child_spec()
+ def child_spec(opts) do
+ cache_name = Keyword.get(opts, :cache_name, @cache_name)
+ child_id = Keyword.get(opts, :child_id, :cachex_sites)
+
+ Supervisor.child_spec(
+ {Cachex, name: cache_name, limit: nil, stats: true},
+ id: child_id
+ )
+ end
+
+ @spec prefill(Keyword.t()) :: :ok
+ def prefill(opts) do
+ cache_name = Keyword.fetch!(opts, :cache_name)
+
+ sites_by_domain_query =
+ from s in Site,
+ select: {
+ s.domain,
+ %{struct(s, ^@cached_schema_fields) | from_cache?: true}
+ }
+
+ sites_by_domain = Plausible.Repo.all(sites_by_domain_query)
+
+ true = Cachex.put_many!(cache_name, sites_by_domain)
+ :ok
+ end
+
+ @spec size() :: non_neg_integer()
+ def size(cache_name \\ @cache_name) do
+ {:ok, size} = Cachex.size(cache_name)
+ size
+ end
+
+ @spec hit_rate() :: number()
+ def hit_rate(cache_name \\ @cache_name) do
+ {:ok, stats} = Cachex.stats(cache_name)
+ Map.get(stats, :hit_rate, 0)
+ end
+
+ @spec get(String.t(), Keyword.t()) :: nil | Site.t()
+ def get(domain, opts \\ []) do
+ cache_name = Keyword.get(opts, :cache_name, @cache_name)
+ force? = Keyword.get(opts, :force?, false)
+
+ if enabled?() or force? do
+ case Cachex.get(cache_name, domain) do
+ {:ok, nil} ->
+ nil
+
+ {:ok, site} ->
+ site
+
+ {:error, e} ->
+ Logger.error(
+ "Error retrieving '#{domain}' from '#{inspect(cache_name)}': #{inspect(e)}"
+ )
+
+ nil
+ end
+ else
+ Plausible.Sites.get_by_domain(domain)
+ end
+ end
+
+ def enabled?() do
+ Application.fetch_env!(:plausible, :sites_by_domain_cache_enabled) == true
+ end
+end
diff --git a/lib/plausible/site/cache/warmer.ex b/lib/plausible/site/cache/warmer.ex
new file mode 100644
index 0000000000..cbb705ac72
--- /dev/null
+++ b/lib/plausible/site/cache/warmer.ex
@@ -0,0 +1,111 @@
+defmodule Plausible.Site.Cache.Warmer do
+ @moduledoc """
+ A periodic cache warmer.
+ Queries all Sites from Postgres, every `interval` and pre-populates the cache.
+ After each run the process is hibernated, triggering garbage collection.
+
+ Currently Cachex is used, but the underlying implementation can be transparently swapped.
+
+ Child specification options available:
+
+ * `interval` - the number of milliseconds for each warm-up cycle, defaults
+ to `:sites_by_domain_cache_refresh_interval` application env value
+ with random jitter added, for which the maximum is stored under
+ `:sites_by_domain_cache_refresh_interval_max_jitter` key.
+ * `cache_name` - defaults to Cache.name() but can be overriden for testing
+ * `force_start?` - enforcess process startup for testing, even if it's barred
+ by `Cache.enabled?`. This is useful for avoiding issues with DB ownership
+ and async tests.
+ * `warmer_fn` - used for testing, a custom function to retrieve the items meant
+ to be cached during the warm-up cycle.
+
+ On each warm-up cycle, a telemetry event is emitted with warm-up `duration` stored.
+ in the measurments map. The event name defaults to:
+
+ `[:prom_ex, :plugin, :cachex, Cache.name(), :refresh]`
+
+ See tests for more comprehensive examples.
+ """
+
+ @behaviour :gen_cycle
+
+ require Logger
+
+ alias Plausible.Site.Cache
+
+ @spec child_spec(Keyword.t()) :: Supervisor.child_spec() | :ignore
+ def child_spec(opts) do
+ %{
+ id: __MODULE__,
+ start: {:gen_cycle, :start_link, [{:local, __MODULE__}, __MODULE__, opts]}
+ }
+ end
+
+ @impl true
+ def init_cycle(opts) do
+ interval = Keyword.get(opts, :interval, interval())
+ force_start? = Keyword.get(opts, :force_start?, false)
+ cache_name = Keyword.get(opts, :cache_name, Cache.name())
+
+ if Cache.enabled?() or force_start? do
+ Logger.info("Initializing #{__MODULE__} with interval #{interval}")
+
+ {:ok, {interval, Keyword.put(opts, :cache_name, cache_name)}}
+ else
+ :ignore
+ end
+ end
+
+ @impl true
+ def handle_cycle(opts) do
+ cache_name = Keyword.fetch!(opts, :cache_name)
+
+ measure_duration(telemetry_event_refresh(cache_name), fn ->
+ Logger.info("Refreshing #{cache_name} cache...")
+
+ warmer_fn = Keyword.get(opts, :warmer_fn, &Cache.prefill/1)
+
+ warmer_fn.(opts)
+ end)
+
+ {:continue_hibernated, opts}
+ end
+
+ @impl true
+ def handle_info(_msg, state) do
+ {:continue, state}
+ end
+
+ @spec telemetry_event_refresh(atom()) :: list(atom())
+ def telemetry_event_refresh(cache_name) do
+ [:prom_ex, :plugin, :cachex, cache_name, :refresh]
+ end
+
+ @spec interval() :: pos_integer()
+ def interval() do
+ interval = Application.fetch_env!(:plausible, :sites_by_domain_cache_refresh_interval)
+
+ interval + jitter()
+ end
+
+ defp jitter() do
+ max_jitter =
+ Application.fetch_env!(:plausible, :sites_by_domain_cache_refresh_interval_max_jitter)
+
+ Enum.random(1..max_jitter)
+ end
+
+ defp measure_duration(event, fun) when is_function(fun, 0) do
+ start = System.monotonic_time()
+ result = fun.()
+ stop = System.monotonic_time()
+
+ :telemetry.execute(
+ event,
+ %{duration: stop - start},
+ %{}
+ )
+
+ result
+ end
+end
diff --git a/mix.exs b/mix.exs
index 1eeb1d0dd5..a78b939875 100644
--- a/mix.exs
+++ b/mix.exs
@@ -80,6 +80,7 @@ defmodule Plausible.MixProject do
{:floki, "~> 0.32.0", only: :test},
{:fun_with_flags, "~> 1.9.0"},
{:fun_with_flags_ui, "~> 0.8"},
+ {:gen_cycle, "~> 1.0"},
{:geolix, "~> 2.0"},
{:geolix_adapter_mmdb2, "~> 0.6.0"},
{:hackney, "~> 1.8"},
diff --git a/mix.lock b/mix.lock
index 635786835b..473e80adc6 100644
--- a/mix.lock
+++ b/mix.lock
@@ -47,6 +47,7 @@
"floki": {:hex, :floki, "0.32.1", "dfe3b8db3b793939c264e6f785bca01753d17318d144bd44b407fb3493acaa87", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "d4b91c713e4a784a3f7b1e3cc016eefc619f6b1c3898464222867cafd3c681a3"},
"fun_with_flags": {:hex, :fun_with_flags, "1.9.0", "0be8692727623af0c00e353f7bdcc9934dd6e1f87798ca6bfec9e739d42b63e6", [:mix], [{:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: true]}, {:redix, "~> 1.0", [hex: :redix, repo: "hexpm", optional: true]}], "hexpm", "8145dc009d549389f1b1915a715fb7e357fcd8fd7b91c74f04aae74912eef27a"},
"fun_with_flags_ui": {:hex, :fun_with_flags_ui, "0.8.0", "70587e344ba2035516a639e7bd8cb2ce8d54ee6c5f5dfd3e4e6f6a776e14ac1d", [:mix], [{:cowboy, ">= 2.0.0", [hex: :cowboy, repo: "hexpm", optional: true]}, {:fun_with_flags, "~> 1.8", [hex: :fun_with_flags, repo: "hexpm", optional: false]}, {:plug, "~> 1.12", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 2.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "95e1e5afae77e259a4a43f930f3da97ff3c388a72d4622f7848cb7ee34de6338"},
+ "gen_cycle": {:hex, :gen_cycle, "1.0.3", "ee0117b9c65814f58827abfe84967d4cb5380db0fbac992358e5b669b3ebcbcb", [:rebar3], [], "hexpm", "fffdadf7fb73de75e3429eb8a6982768e7240803561942602350b689991749e4"},
"gen_smtp": {:hex, :gen_smtp, "1.2.0", "9cfc75c72a8821588b9b9fe947ae5ab2aed95a052b81237e0928633a13276fd3", [:rebar3], [{:ranch, ">= 1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "5ee0375680bca8f20c4d85f58c2894441443a743355430ff33a783fe03296779"},
"geolix": {:hex, :geolix, "2.0.0", "7e65764bedfc98d08a3ddb24c417657c9d438eff163280b45fbb7de289626acd", [:mix], [], "hexpm", "8742bf588ed0bb7def2c443204d09d355990846c6efdff96ded66aac24c301df"},
"geolix_adapter_mmdb2": {:hex, :geolix_adapter_mmdb2, "0.6.0", "6ab9dbf6ea395817aa1fd2597be25d0dda1853c7f664e62716e47728d18bc4f9", [:mix], [{:geolix, "~> 2.0", [hex: :geolix, repo: "hexpm", optional: false]}, {:mmdb2_decoder, "~> 3.0", [hex: :mmdb2_decoder, repo: "hexpm", optional: false]}], "hexpm", "06ff962feae8a310cffdf86b74bfcda6e2d0dccb439bb1f62df2b657b1c0269b"},
diff --git a/test/plausible/site/cache_test.exs b/test/plausible/site/cache_test.exs
new file mode 100644
index 0000000000..7232f90423
--- /dev/null
+++ b/test/plausible/site/cache_test.exs
@@ -0,0 +1,126 @@
+defmodule Plausible.Site.CacheTest do
+ use Plausible.DataCase, async: true
+
+ alias Plausible.Site
+ alias Plausible.Site.Cache
+
+ import ExUnit.CaptureLog
+
+ test "cache process is started, but falls back to the database if cache is disabled" do
+ insert(:site, domain: "example.test")
+ refute Cache.enabled?()
+ assert Process.alive?(Process.whereis(Cache.name()))
+ refute Process.whereis(Cache.Warmer)
+ assert %Site{domain: "example.test", from_cache?: false} = Cache.get("example.test")
+ assert Cache.size() == 0
+ refute Cache.get("other.test")
+ end
+
+ test "cache warmer process warms up the cache", %{test: test} do
+ test_pid = self()
+ opts = [force_start?: true, warmer_fn: report_back(test_pid), cache_name: test]
+
+ {:ok, _} = Supervisor.start_link([{Cache.Warmer, opts}], strategy: :one_for_one, name: test)
+
+ assert Process.whereis(Cache.Warmer)
+ assert_receive {:cache_warmed, %{opts: got_opts}}
+ assert got_opts[:cache_name] == test
+ end
+
+ test "cache warmer sends telemetry event", %{test: test} do
+ test_pid = self()
+
+ :telemetry.attach(
+ "test-cache-warmer",
+ Cache.Warmer.telemetry_event_refresh(test),
+ fn _event, %{duration: d}, %{}, %{} when is_integer(d) ->
+ send(test_pid, :telemetry_handled)
+ end,
+ %{}
+ )
+
+ opts = [force_start?: true, warmer_fn: report_back(test_pid), cache_name: test]
+ {:ok, _} = Supervisor.start_link([{Cache.Warmer, opts}], strategy: :one_for_one, name: test)
+
+ assert_receive {:cache_warmed, _}
+ assert_receive :telemetry_handled
+ end
+
+ test "critical cache errors are logged and nil is returned" do
+ log =
+ capture_log(fn ->
+ assert Cache.get("key", force?: true, cache_name: NonExistingCache) == nil
+ end)
+
+ assert log =~ "Error retrieving 'key' from 'NonExistingCache': :no_cache"
+ end
+
+ test "cache warmer warms periodically with an interval", %{test: test} do
+ test_pid = self()
+ opts = [force_start?: true, warmer_fn: report_back(test_pid), cache_name: test, interval: 30]
+
+ {:ok, _} = Supervisor.start_link([{Cache.Warmer, opts}], strategy: :one_for_one, name: test)
+
+ assert_receive {:cache_warmed, %{at: at1}}, 100
+ assert_receive {:cache_warmed, %{at: at2}}, 100
+ assert_receive {:cache_warmed, %{at: at3}}, 100
+
+ assert is_integer(at1)
+ assert is_integer(at2)
+ assert is_integer(at3)
+
+ assert at1 < at2
+ assert at3 > at2
+ end
+
+ test "cache caches", %{test: test} do
+ {:ok, _} =
+ Supervisor.start_link([{Cache, [cache_name: test, child_id: :test_cache_caches_id]}],
+ strategy: :one_for_one,
+ name: Test.Supervisor.Cache
+ )
+
+ %{id: first_id} = site1 = insert(:site, domain: "site1.example.com")
+ _ = insert(:site, domain: "site2.example.com")
+
+ :ok = Cache.prefill(cache_name: test)
+
+ {:ok, _} = Plausible.Repo.delete(site1)
+
+ assert Cache.size(test) == 2
+
+ assert %Site{from_cache?: true, id: ^first_id} =
+ Cache.get("site1.example.com", force?: true, cache_name: test)
+
+ assert %Site{from_cache?: true} =
+ Cache.get("site2.example.com", force?: true, cache_name: test)
+
+ assert %Site{from_cache?: false} = Cache.get("site2.example.com", cache_name: test)
+
+ refute Cache.get("site3.example.com", cache_name: test, force?: true)
+ end
+
+ test "cache exposes hit rate", %{test: test} do
+ {:ok, _} =
+ Supervisor.start_link([{Cache, [cache_name: test, child_id: :test_cache_caches_id]}],
+ strategy: :one_for_one,
+ name: Test.Supervisor.HitRateCache
+ )
+
+ insert(:site, domain: "site1.example.com")
+ :ok = Cache.prefill(cache_name: test)
+
+ assert Cache.hit_rate(test) == 0
+ assert Cache.get("site1.example.com", force?: true, cache_name: test)
+ assert Cache.hit_rate(test) == 100
+ refute Cache.get("nonexisting.example.com", force?: true, cache_name: test)
+ assert Cache.hit_rate(test) == 50
+ end
+
+ defp report_back(test_pid) do
+ fn opts ->
+ send(test_pid, {:cache_warmed, %{at: System.monotonic_time(), opts: opts}})
+ :ok
+ end
+ end
+end