Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Update sites cache implementation (#2444)

Adam Rutkowski3fe6a956960d
diff --git a/lib/plausible/site/cache.ex b/lib/plausible/site/cache.ex
index c24a7c0aa2..f0df6e33b4 100644
--- a/lib/plausible/site/cache.ex
+++ b/lib/plausible/site/cache.ex
@@ -1,13 +1,13 @@
defmodule Plausible.Site.Cache do
@moduledoc """
- A sites by domain caching interface.
+ A "sites by domain" caching interface.
Serves as a thin wrapper around Cachex, but the underlying
implementation can be transparently swapped.
Even though the Cachex process is started, cache access is disabled
during tests via the `:sites_by_domain_cache_enabled` application env key.
- This can be overriden on case by case basis, using the child specs options.
+ This can be overridden on case by case basis, using the child specs options.
When Cache is disabled via application env, the `get/1` function
falls back to pure database lookups. This should help with introducing
@@ -19,8 +19,26 @@ defmodule Plausible.Site.Cache do
database counterpart -- to spare bandwidth and query execution time,
only selected database columns are retrieved and cached.
+ There are two modes of refreshing the cache: `:all` and `:single`.
+
+ * `:all` means querying the database for all Site entries and should be done
+ periodically (via `Cache.Warmer`). All existing Cache entries all cleared
+ prior to writing the new batch.
+
+ * `:single` attempts to re-query a specific site by domain and should be done
+ only when the initial Cache.get attempt resulted with `nil`. Single refresh will
+ write `%Ecto.NoResultsError{}` to the cache so that subsequent Cache.get calls
+ indicate that we already failed to retrieve a Site.
+
+ This helps in recognising missing/deleted Sites with minimal number of DB lookups
+ across a disconnected cluster within the periodic refresh window.
+
+ Refreshing a single Site emits a telemetry event including `duration` measurement
+ and meta-data indicating whether the site was found in the DB or is missing still.
+ The telemetry event is defined with `telemetry_event_refresh/2`.
+
The `@cached_schema_fields` attribute defines the list of DB columns
- queried on cache pre-fill.
+ queried on each cache refresh.
Also see tests for more comprehensive examples.
"""
@@ -39,6 +57,9 @@ defmodule Plausible.Site.Cache do
ingest_rate_limit_threshold
)a
+ @type not_found_in_db() :: %Ecto.NoResultsError{}
+ @type t() :: Site.t() | not_found_in_db()
+
def name(), do: @cache_name
@spec child_spec(Keyword.t()) :: Supervisor.child_spec()
@@ -52,20 +73,27 @@ defmodule Plausible.Site.Cache do
)
end
- @spec prefill(Keyword.t()) :: :ok
- def prefill(opts) do
+ @spec refresh_all(Keyword.t()) :: :ok
+ def refresh_all(opts) do
cache_name = Keyword.fetch!(opts, :cache_name)
- sites_by_domain_query =
- from s in Site,
- select: {
- s.domain,
- %{struct(s, ^@cached_schema_fields) | from_cache?: true}
- }
+ measure_duration(telemetry_event_refresh(cache_name, :all), fn ->
+ sites_by_domain_query =
+ from s in Site,
+ select: {
+ s.domain,
+ %{struct(s, ^@cached_schema_fields) | from_cache?: true}
+ }
- sites_by_domain = Plausible.Repo.all(sites_by_domain_query)
+ sites_by_domain = Plausible.Repo.all(sites_by_domain_query)
+
+ Cachex.clear!(cache_name)
+
+ if not Enum.empty?(sites_by_domain) do
+ true = Cachex.put_many!(cache_name, sites_by_domain)
+ end
+ end)
- true = Cachex.put_many!(cache_name, sites_by_domain)
:ok
end
@@ -81,7 +109,7 @@ defmodule Plausible.Site.Cache do
Map.get(stats, :hit_rate, 0)
end
- @spec get(String.t(), Keyword.t()) :: nil | Site.t()
+ @spec get(String.t(), Keyword.t()) :: t() | nil
def get(domain, opts \\ []) do
cache_name = Keyword.get(opts, :cache_name, @cache_name)
force? = Keyword.get(opts, :force?, false)
@@ -106,7 +134,68 @@ defmodule Plausible.Site.Cache do
end
end
+ @spec refresh_one(String.t(), Keyword.t()) :: {:ok, t()} | {:error, any()}
+ def refresh_one(domain, opts) do
+ cache_name = Keyword.get(opts, :cache_name, @cache_name)
+ force? = Keyword.get(opts, :force?, false)
+
+ if not enabled?() and not force?, do: raise("Cache: '#{cache_name}' is disabled")
+
+ measure_duration_with_metadata(telemetry_event_refresh(cache_name, :one), fn ->
+ {found_in_db?, item_to_cache} = select_one(domain)
+
+ case Cachex.put(cache_name, domain, item_to_cache) do
+ {:ok, _} ->
+ result = {:ok, item_to_cache}
+ {result, with_telemetry_metadata(found_in_db?: found_in_db?)}
+
+ {:error, _} = error ->
+ {error, with_telemetry_metadata(error: true)}
+ end
+ end)
+ end
+
+ @spec telemetry_event_refresh(atom(), :all | :one) :: list(atom())
+ def telemetry_event_refresh(cache_name, mode) when mode in [:all, :one] do
+ [:plausible, :cache, cache_name, :refresh, mode]
+ end
+
def enabled?() do
Application.fetch_env!(:plausible, :sites_by_domain_cache_enabled) == true
end
+
+ defp select_one(domain) do
+ site_by_domain_query =
+ from s in Site,
+ where: s.domain == ^domain,
+ select: %{struct(s, ^@cached_schema_fields) | from_cache?: true}
+
+ case Plausible.Repo.one(site_by_domain_query) do
+ nil -> {false, %Ecto.NoResultsError{}}
+ site -> {true, site}
+ end
+ end
+
+ defp with_telemetry_metadata(props) do
+ Enum.into(props, %{})
+ end
+
+ defp measure_duration_with_metadata(event, fun) when is_function(fun, 0) do
+ {duration, {result, telemetry_metadata}} = time_it(fun)
+ :telemetry.execute(event, %{duration: duration}, telemetry_metadata)
+ result
+ end
+
+ defp measure_duration(event, fun) when is_function(fun, 0) do
+ {duration, result} = time_it(fun)
+ :telemetry.execute(event, %{duration: duration}, %{})
+ result
+ end
+
+ defp time_it(fun) do
+ start = System.monotonic_time()
+ result = fun.()
+ stop = System.monotonic_time()
+ {stop - start, result}
+ end
end
diff --git a/lib/plausible/site/cache/warmer.ex b/lib/plausible/site/cache/warmer.ex
index cbb705ac72..cd31863eb0 100644
--- a/lib/plausible/site/cache/warmer.ex
+++ b/lib/plausible/site/cache/warmer.ex
@@ -19,11 +19,6 @@ defmodule Plausible.Site.Cache.Warmer do
* `warmer_fn` - used for testing, a custom function to retrieve the items meant
to be cached during the warm-up cycle.
- On each warm-up cycle, a telemetry event is emitted with warm-up `duration` stored.
- in the measurments map. The event name defaults to:
-
- `[:prom_ex, :plugin, :cachex, Cache.name(), :refresh]`
-
See tests for more comprehensive examples.
"""
@@ -35,9 +30,11 @@ defmodule Plausible.Site.Cache.Warmer do
@spec child_spec(Keyword.t()) :: Supervisor.child_spec() | :ignore
def child_spec(opts) do
+ child_name = Keyword.get(opts, :child_name, {:local, __MODULE__})
+
%{
id: __MODULE__,
- start: {:gen_cycle, :start_link, [{:local, __MODULE__}, __MODULE__, opts]}
+ start: {:gen_cycle, :start_link, [child_name, __MODULE__, opts]}
}
end
@@ -59,14 +56,10 @@ defmodule Plausible.Site.Cache.Warmer do
@impl true
def handle_cycle(opts) do
cache_name = Keyword.fetch!(opts, :cache_name)
+ Logger.info("Refreshing #{cache_name} cache...")
- measure_duration(telemetry_event_refresh(cache_name), fn ->
- Logger.info("Refreshing #{cache_name} cache...")
-
- warmer_fn = Keyword.get(opts, :warmer_fn, &Cache.prefill/1)
-
- warmer_fn.(opts)
- end)
+ warmer_fn = Keyword.get(opts, :warmer_fn, &Cache.refresh_all/1)
+ warmer_fn.(opts)
{:continue_hibernated, opts}
end
@@ -76,11 +69,6 @@ defmodule Plausible.Site.Cache.Warmer do
{:continue, state}
end
- @spec telemetry_event_refresh(atom()) :: list(atom())
- def telemetry_event_refresh(cache_name) do
- [:prom_ex, :plugin, :cachex, cache_name, :refresh]
- end
-
@spec interval() :: pos_integer()
def interval() do
interval = Application.fetch_env!(:plausible, :sites_by_domain_cache_refresh_interval)
@@ -94,18 +82,4 @@ defmodule Plausible.Site.Cache.Warmer do
Enum.random(1..max_jitter)
end
-
- defp measure_duration(event, fun) when is_function(fun, 0) do
- start = System.monotonic_time()
- result = fun.()
- stop = System.monotonic_time()
-
- :telemetry.execute(
- event,
- %{duration: stop - start},
- %{}
- )
-
- result
- end
end
diff --git a/test/plausible/site/cache_test.exs b/test/plausible/site/cache_test.exs
index 7232f90423..4b6edb7a55 100644
--- a/test/plausible/site/cache_test.exs
+++ b/test/plausible/site/cache_test.exs
@@ -6,115 +6,193 @@ defmodule Plausible.Site.CacheTest do
import ExUnit.CaptureLog
- test "cache process is started, but falls back to the database if cache is disabled" do
- insert(:site, domain: "example.test")
- refute Cache.enabled?()
- assert Process.alive?(Process.whereis(Cache.name()))
- refute Process.whereis(Cache.Warmer)
- assert %Site{domain: "example.test", from_cache?: false} = Cache.get("example.test")
- assert Cache.size() == 0
- refute Cache.get("other.test")
- end
+ describe "public cache interface" do
+ test "cache process is started, but falls back to the database if cache is disabled" do
+ insert(:site, domain: "example.test")
+ refute Cache.enabled?()
+ assert Process.alive?(Process.whereis(Cache.name()))
+ refute Process.whereis(Cache.Warmer)
+ assert %Site{domain: "example.test", from_cache?: false} = Cache.get("example.test")
+ assert Cache.size() == 0
+ refute Cache.get("other.test")
+ end
- test "cache warmer process warms up the cache", %{test: test} do
- test_pid = self()
- opts = [force_start?: true, warmer_fn: report_back(test_pid), cache_name: test]
+ test "critical cache errors are logged and nil is returned" do
+ log =
+ capture_log(fn ->
+ assert Cache.get("key", force?: true, cache_name: NonExistingCache) == nil
+ end)
- {:ok, _} = Supervisor.start_link([{Cache.Warmer, opts}], strategy: :one_for_one, name: test)
+ assert log =~ "Error retrieving 'key' from 'NonExistingCache': :no_cache"
+ end
- assert Process.whereis(Cache.Warmer)
- assert_receive {:cache_warmed, %{opts: got_opts}}
- assert got_opts[:cache_name] == test
- end
+ test "cache caches", %{test: test} do
+ {:ok, _} =
+ Supervisor.start_link([{Cache, [cache_name: test, child_id: :test_cache_caches_id]}],
+ strategy: :one_for_one,
+ name: Test.Supervisor.Cache
+ )
- test "cache warmer sends telemetry event", %{test: test} do
- test_pid = self()
+ %{id: first_id} = site1 = insert(:site, domain: "site1.example.com")
+ _ = insert(:site, domain: "site2.example.com")
- :telemetry.attach(
- "test-cache-warmer",
- Cache.Warmer.telemetry_event_refresh(test),
- fn _event, %{duration: d}, %{}, %{} when is_integer(d) ->
- send(test_pid, :telemetry_handled)
- end,
- %{}
- )
+ :ok = Cache.refresh_all(cache_name: test)
- opts = [force_start?: true, warmer_fn: report_back(test_pid), cache_name: test]
- {:ok, _} = Supervisor.start_link([{Cache.Warmer, opts}], strategy: :one_for_one, name: test)
+ {:ok, _} = Plausible.Repo.delete(site1)
- assert_receive {:cache_warmed, _}
- assert_receive :telemetry_handled
- end
+ assert Cache.size(test) == 2
- test "critical cache errors are logged and nil is returned" do
- log =
- capture_log(fn ->
- assert Cache.get("key", force?: true, cache_name: NonExistingCache) == nil
- end)
+ assert %Site{from_cache?: true, id: ^first_id} =
+ Cache.get("site1.example.com", force?: true, cache_name: test)
- assert log =~ "Error retrieving 'key' from 'NonExistingCache': :no_cache"
- end
+ assert %Site{from_cache?: true} =
+ Cache.get("site2.example.com", force?: true, cache_name: test)
- test "cache warmer warms periodically with an interval", %{test: test} do
- test_pid = self()
- opts = [force_start?: true, warmer_fn: report_back(test_pid), cache_name: test, interval: 30]
+ assert %Site{from_cache?: false} = Cache.get("site2.example.com", cache_name: test)
- {:ok, _} = Supervisor.start_link([{Cache.Warmer, opts}], strategy: :one_for_one, name: test)
+ refute Cache.get("site3.example.com", cache_name: test, force?: true)
+ end
- assert_receive {:cache_warmed, %{at: at1}}, 100
- assert_receive {:cache_warmed, %{at: at2}}, 100
- assert_receive {:cache_warmed, %{at: at3}}, 100
+ test "cache exposes hit rate", %{test: test} do
+ {:ok, _} = start_test_cache(test)
- assert is_integer(at1)
- assert is_integer(at2)
- assert is_integer(at3)
+ insert(:site, domain: "site1.example.com")
+ :ok = Cache.refresh_all(cache_name: test)
- assert at1 < at2
- assert at3 > at2
- end
+ assert Cache.hit_rate(test) == 0
+ assert Cache.get("site1.example.com", force?: true, cache_name: test)
+ assert Cache.hit_rate(test) == 100
+ refute Cache.get("nonexisting.example.com", force?: true, cache_name: test)
+ assert Cache.hit_rate(test) == 50
+ end
+
+ test "a single cached site can be refreshed", %{test: test} do
+ {:ok, _} = start_test_cache(test)
+
+ domain1 = "site1.example.com"
+ domain2 = "nonexisting.example.com"
+
+ cache_opts = [cache_name: test, force?: true]
- test "cache caches", %{test: test} do
- {:ok, _} =
- Supervisor.start_link([{Cache, [cache_name: test, child_id: :test_cache_caches_id]}],
- strategy: :one_for_one,
- name: Test.Supervisor.Cache
- )
+ assert Cache.get(domain1) == nil
- %{id: first_id} = site1 = insert(:site, domain: "site1.example.com")
- _ = insert(:site, domain: "site2.example.com")
+ insert(:site, domain: domain1)
+
+ assert {:ok, %{domain: ^domain1}} = Cache.refresh_one(domain1, cache_opts)
+ assert %Site{domain: ^domain1} = Cache.get(domain1, cache_opts)
+
+ assert {:ok, %Ecto.NoResultsError{}} = Cache.refresh_one(domain2, cache_opts)
+ assert %Ecto.NoResultsError{} = Cache.get(domain2, cache_opts)
+ end
- :ok = Cache.prefill(cache_name: test)
+ test "refreshing a single site sends a telemetry event indicating record not found in the database",
+ %{
+ test: test
+ } do
+ :ok =
+ start_test_cache_with_telemetry_handler(test,
+ event: Cache.telemetry_event_refresh(test, :one)
+ )
+
+ Cache.refresh_one("missing.example.com", force?: true, cache_name: test)
+ assert_receive {:telemetry_handled, %{found_in_db?: false}}
+ end
- {:ok, _} = Plausible.Repo.delete(site1)
+ test "refreshing a single site sends a telemetry event indicating record found in the database",
+ %{
+ test: test
+ } do
+ domain = "site1.example.com"
+ insert(:site, domain: domain)
- assert Cache.size(test) == 2
+ :ok =
+ start_test_cache_with_telemetry_handler(test,
+ event: Cache.telemetry_event_refresh(test, :one)
+ )
- assert %Site{from_cache?: true, id: ^first_id} =
- Cache.get("site1.example.com", force?: true, cache_name: test)
+ Cache.refresh_one(domain, force?: true, cache_name: test)
+ assert_receive {:telemetry_handled, %{found_in_db?: true}}
+ end
- assert %Site{from_cache?: true} =
- Cache.get("site2.example.com", force?: true, cache_name: test)
+ test "refreshing all sites sends a telemetry event",
+ %{
+ test: test
+ } do
+ domain = "site1.example.com"
+ insert(:site, domain: domain)
- assert %Site{from_cache?: false} = Cache.get("site2.example.com", cache_name: test)
+ :ok =
+ start_test_cache_with_telemetry_handler(test,
+ event: Cache.telemetry_event_refresh(test, :all)
+ )
- refute Cache.get("site3.example.com", cache_name: test, force?: true)
+ Cache.refresh_all(force?: true, cache_name: test)
+ assert_receive {:telemetry_handled, %{}}
+ end
end
- test "cache exposes hit rate", %{test: test} do
- {:ok, _} =
- Supervisor.start_link([{Cache, [cache_name: test, child_id: :test_cache_caches_id]}],
- strategy: :one_for_one,
- name: Test.Supervisor.HitRateCache
- )
-
- insert(:site, domain: "site1.example.com")
- :ok = Cache.prefill(cache_name: test)
-
- assert Cache.hit_rate(test) == 0
- assert Cache.get("site1.example.com", force?: true, cache_name: test)
- assert Cache.hit_rate(test) == 100
- refute Cache.get("nonexisting.example.com", force?: true, cache_name: test)
- assert Cache.hit_rate(test) == 50
+ describe "warming the cache" do
+ test "cache warmer process warms up the cache", %{test: test} do
+ test_pid = self()
+ opts = [force_start?: true, warmer_fn: report_back(test_pid), cache_name: test]
+
+ {:ok, _} = Supervisor.start_link([{Cache.Warmer, opts}], strategy: :one_for_one, name: test)
+ assert Process.whereis(Cache.Warmer)
+
+ assert_receive {:cache_warmed, %{opts: got_opts}}
+ assert got_opts[:cache_name] == test
+ end
+
+ test "cache warmer warms periodically with an interval", %{test: test} do
+ test_pid = self()
+
+ opts = [
+ force_start?: true,
+ warmer_fn: report_back(test_pid),
+ cache_name: test,
+ interval: 30
+ ]
+
+ {:ok, _} = start_test_warmer(opts)
+
+ assert_receive {:cache_warmed, %{at: at1}}, 100
+ assert_receive {:cache_warmed, %{at: at2}}, 100
+ assert_receive {:cache_warmed, %{at: at3}}, 100
+
+ assert is_integer(at1)
+ assert is_integer(at2)
+ assert is_integer(at3)
+
+ assert at1 < at2
+ assert at3 > at2
+ end
+
+ test "deleted sites don't stay in cache on another refresh", %{test: test} do
+ {:ok, _} = start_test_cache(test)
+
+ domain1 = "site1.example.com"
+ domain2 = "site2.example.com"
+
+ site1 = insert(:site, domain: domain1)
+ _site2 = insert(:site, domain: domain2)
+
+ cache_opts = [cache_name: test, force?: true]
+
+ :ok = Cache.refresh_all(cache_opts)
+
+ assert Cache.get(domain1, cache_opts)
+ assert Cache.get(domain2, cache_opts)
+
+ Repo.delete!(site1)
+
+ :ok = Cache.refresh_all(cache_opts)
+
+ assert Cache.get(domain2, cache_opts)
+
+ refute Cache.get(domain1, cache_opts)
+ Cache.refresh_one(domain1, cache_opts)
+ assert Cache.get(domain1, cache_opts) == %Ecto.NoResultsError{}
+ end
end
defp report_back(test_pid) do
@@ -123,4 +201,29 @@ defmodule Plausible.Site.CacheTest do
:ok
end
end
+
+ defp start_test_cache(cache_name) do
+ %{start: {m, f, a}} = Cache.child_spec(cache_name: cache_name)
+ apply(m, f, a)
+ end
+
+ defp start_test_warmer(opts) do
+ child_name_opt = {:child_name, {:local, Keyword.fetch!(opts, :cache_name)}}
+ %{start: {m, f, a}} = Cache.Warmer.child_spec([child_name_opt | opts])
+ apply(m, f, a)
+ end
+
+ defp start_test_cache_with_telemetry_handler(test, event: event) do
+ {:ok, _} = start_test_cache(test)
+ test_pid = self()
+
+ :telemetry.attach(
+ "#{test}-telemetry-handler",
+ event,
+ fn ^event, %{duration: d}, metadata, _ when is_integer(d) ->
+ send(test_pid, {:telemetry_handled, metadata})
+ end,
+ %{}
+ )
+ end
end