Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Update sites cache implementation (#2444)
Adam Rutkowski3fe6a956960d
diff --git a/lib/plausible/site/cache.ex b/lib/plausible/site/cache.exindex c24a7c0aa2..f0df6e33b4 100644--- a/lib/plausible/site/cache.ex+++ b/lib/plausible/site/cache.ex@@ -1,13 +1,13 @@defmodule Plausible.Site.Cache do@moduledoc """- A sites by domain caching interface.+ A "sites by domain" caching interface.Serves as a thin wrapper around Cachex, but the underlyingimplementation can be transparently swapped.Even though the Cachex process is started, cache access is disabledduring tests via the `:sites_by_domain_cache_enabled` application env key.- This can be overriden on case by case basis, using the child specs options.+ This can be overridden on case by case basis, using the child specs options.When Cache is disabled via application env, the `get/1` functionfalls back to pure database lookups. This should help with introducing@@ -19,8 +19,26 @@ defmodule Plausible.Site.Cache dodatabase counterpart -- to spare bandwidth and query execution time,only selected database columns are retrieved and cached.+ There are two modes of refreshing the cache: `:all` and `:single`.++ * `:all` means querying the database for all Site entries and should be done+ periodically (via `Cache.Warmer`). All existing Cache entries all cleared+ prior to writing the new batch.++ * `:single` attempts to re-query a specific site by domain and should be done+ only when the initial Cache.get attempt resulted with `nil`. Single refresh will+ write `%Ecto.NoResultsError{}` to the cache so that subsequent Cache.get calls+ indicate that we already failed to retrieve a Site.++ This helps in recognising missing/deleted Sites with minimal number of DB lookups+ across a disconnected cluster within the periodic refresh window.++ Refreshing a single Site emits a telemetry event including `duration` measurement+ and meta-data indicating whether the site was found in the DB or is missing still.+ The telemetry event is defined with `telemetry_event_refresh/2`.+The `@cached_schema_fields` attribute defines the list of DB columns- queried on cache pre-fill.+ queried on each cache refresh.Also see tests for more comprehensive examples."""@@ -39,6 +57,9 @@ defmodule Plausible.Site.Cache doingest_rate_limit_threshold)a+ @type not_found_in_db() :: %Ecto.NoResultsError{}+ @type t() :: Site.t() | not_found_in_db()+def name(), do: @cache_name@spec child_spec(Keyword.t()) :: Supervisor.child_spec()@@ -52,20 +73,27 @@ defmodule Plausible.Site.Cache do)end- @spec prefill(Keyword.t()) :: :ok- def prefill(opts) do+ @spec refresh_all(Keyword.t()) :: :ok+ def refresh_all(opts) docache_name = Keyword.fetch!(opts, :cache_name)- sites_by_domain_query =- from s in Site,- select: {- s.domain,- %{struct(s, ^@cached_schema_fields) | from_cache?: true}- }+ measure_duration(telemetry_event_refresh(cache_name, :all), fn ->+ sites_by_domain_query =+ from s in Site,+ select: {+ s.domain,+ %{struct(s, ^@cached_schema_fields) | from_cache?: true}+ }- sites_by_domain = Plausible.Repo.all(sites_by_domain_query)+ sites_by_domain = Plausible.Repo.all(sites_by_domain_query)++ Cachex.clear!(cache_name)++ if not Enum.empty?(sites_by_domain) do+ true = Cachex.put_many!(cache_name, sites_by_domain)+ end+ end)- true = Cachex.put_many!(cache_name, sites_by_domain):okend@@ -81,7 +109,7 @@ defmodule Plausible.Site.Cache doMap.get(stats, :hit_rate, 0)end- @spec get(String.t(), Keyword.t()) :: nil | Site.t()+ @spec get(String.t(), Keyword.t()) :: t() | nildef get(domain, opts \\ []) docache_name = Keyword.get(opts, :cache_name, @cache_name)force? = Keyword.get(opts, :force?, false)@@ -106,7 +134,68 @@ defmodule Plausible.Site.Cache doendend+ @spec refresh_one(String.t(), Keyword.t()) :: {:ok, t()} | {:error, any()}+ def refresh_one(domain, opts) do+ cache_name = Keyword.get(opts, :cache_name, @cache_name)+ force? = Keyword.get(opts, :force?, false)++ if not enabled?() and not force?, do: raise("Cache: '#{cache_name}' is disabled")++ measure_duration_with_metadata(telemetry_event_refresh(cache_name, :one), fn ->+ {found_in_db?, item_to_cache} = select_one(domain)++ case Cachex.put(cache_name, domain, item_to_cache) do+ {:ok, _} ->+ result = {:ok, item_to_cache}+ {result, with_telemetry_metadata(found_in_db?: found_in_db?)}++ {:error, _} = error ->+ {error, with_telemetry_metadata(error: true)}+ end+ end)+ end++ @spec telemetry_event_refresh(atom(), :all | :one) :: list(atom())+ def telemetry_event_refresh(cache_name, mode) when mode in [:all, :one] do+ [:plausible, :cache, cache_name, :refresh, mode]+ end+def enabled?() doApplication.fetch_env!(:plausible, :sites_by_domain_cache_enabled) == trueend++ defp select_one(domain) do+ site_by_domain_query =+ from s in Site,+ where: s.domain == ^domain,+ select: %{struct(s, ^@cached_schema_fields) | from_cache?: true}++ case Plausible.Repo.one(site_by_domain_query) do+ nil -> {false, %Ecto.NoResultsError{}}+ site -> {true, site}+ end+ end++ defp with_telemetry_metadata(props) do+ Enum.into(props, %{})+ end++ defp measure_duration_with_metadata(event, fun) when is_function(fun, 0) do+ {duration, {result, telemetry_metadata}} = time_it(fun)+ :telemetry.execute(event, %{duration: duration}, telemetry_metadata)+ result+ end++ defp measure_duration(event, fun) when is_function(fun, 0) do+ {duration, result} = time_it(fun)+ :telemetry.execute(event, %{duration: duration}, %{})+ result+ end++ defp time_it(fun) do+ start = System.monotonic_time()+ result = fun.()+ stop = System.monotonic_time()+ {stop - start, result}+ endenddiff --git a/lib/plausible/site/cache/warmer.ex b/lib/plausible/site/cache/warmer.exindex cbb705ac72..cd31863eb0 100644--- a/lib/plausible/site/cache/warmer.ex+++ b/lib/plausible/site/cache/warmer.ex@@ -19,11 +19,6 @@ defmodule Plausible.Site.Cache.Warmer do* `warmer_fn` - used for testing, a custom function to retrieve the items meantto be cached during the warm-up cycle.- On each warm-up cycle, a telemetry event is emitted with warm-up `duration` stored.- in the measurments map. The event name defaults to:-- `[:prom_ex, :plugin, :cachex, Cache.name(), :refresh]`-See tests for more comprehensive examples."""@@ -35,9 +30,11 @@ defmodule Plausible.Site.Cache.Warmer do@spec child_spec(Keyword.t()) :: Supervisor.child_spec() | :ignoredef child_spec(opts) do+ child_name = Keyword.get(opts, :child_name, {:local, __MODULE__})+%{id: __MODULE__,- start: {:gen_cycle, :start_link, [{:local, __MODULE__}, __MODULE__, opts]}+ start: {:gen_cycle, :start_link, [child_name, __MODULE__, opts]}}end@@ -59,14 +56,10 @@ defmodule Plausible.Site.Cache.Warmer do@impl truedef handle_cycle(opts) docache_name = Keyword.fetch!(opts, :cache_name)+ Logger.info("Refreshing #{cache_name} cache...")- measure_duration(telemetry_event_refresh(cache_name), fn ->- Logger.info("Refreshing #{cache_name} cache...")-- warmer_fn = Keyword.get(opts, :warmer_fn, &Cache.prefill/1)-- warmer_fn.(opts)- end)+ warmer_fn = Keyword.get(opts, :warmer_fn, &Cache.refresh_all/1)+ warmer_fn.(opts){:continue_hibernated, opts}end@@ -76,11 +69,6 @@ defmodule Plausible.Site.Cache.Warmer do{:continue, state}end- @spec telemetry_event_refresh(atom()) :: list(atom())- def telemetry_event_refresh(cache_name) do- [:prom_ex, :plugin, :cachex, cache_name, :refresh]- end-@spec interval() :: pos_integer()def interval() dointerval = Application.fetch_env!(:plausible, :sites_by_domain_cache_refresh_interval)@@ -94,18 +82,4 @@ defmodule Plausible.Site.Cache.Warmer doEnum.random(1..max_jitter)end-- defp measure_duration(event, fun) when is_function(fun, 0) do- start = System.monotonic_time()- result = fun.()- stop = System.monotonic_time()-- :telemetry.execute(- event,- %{duration: stop - start},- %{}- )-- result- endenddiff --git a/test/plausible/site/cache_test.exs b/test/plausible/site/cache_test.exsindex 7232f90423..4b6edb7a55 100644--- a/test/plausible/site/cache_test.exs+++ b/test/plausible/site/cache_test.exs@@ -6,115 +6,193 @@ defmodule Plausible.Site.CacheTest doimport ExUnit.CaptureLog- test "cache process is started, but falls back to the database if cache is disabled" do- insert(:site, domain: "example.test")- refute Cache.enabled?()- assert Process.alive?(Process.whereis(Cache.name()))- refute Process.whereis(Cache.Warmer)- assert %Site{domain: "example.test", from_cache?: false} = Cache.get("example.test")- assert Cache.size() == 0- refute Cache.get("other.test")- end+ describe "public cache interface" do+ test "cache process is started, but falls back to the database if cache is disabled" do+ insert(:site, domain: "example.test")+ refute Cache.enabled?()+ assert Process.alive?(Process.whereis(Cache.name()))+ refute Process.whereis(Cache.Warmer)+ assert %Site{domain: "example.test", from_cache?: false} = Cache.get("example.test")+ assert Cache.size() == 0+ refute Cache.get("other.test")+ end- test "cache warmer process warms up the cache", %{test: test} do- test_pid = self()- opts = [force_start?: true, warmer_fn: report_back(test_pid), cache_name: test]+ test "critical cache errors are logged and nil is returned" do+ log =+ capture_log(fn ->+ assert Cache.get("key", force?: true, cache_name: NonExistingCache) == nil+ end)- {:ok, _} = Supervisor.start_link([{Cache.Warmer, opts}], strategy: :one_for_one, name: test)+ assert log =~ "Error retrieving 'key' from 'NonExistingCache': :no_cache"+ end- assert Process.whereis(Cache.Warmer)- assert_receive {:cache_warmed, %{opts: got_opts}}- assert got_opts[:cache_name] == test- end+ test "cache caches", %{test: test} do+ {:ok, _} =+ Supervisor.start_link([{Cache, [cache_name: test, child_id: :test_cache_caches_id]}],+ strategy: :one_for_one,+ name: Test.Supervisor.Cache+ )- test "cache warmer sends telemetry event", %{test: test} do- test_pid = self()+ %{id: first_id} = site1 = insert(:site, domain: "site1.example.com")+ _ = insert(:site, domain: "site2.example.com")- :telemetry.attach(- "test-cache-warmer",- Cache.Warmer.telemetry_event_refresh(test),- fn _event, %{duration: d}, %{}, %{} when is_integer(d) ->- send(test_pid, :telemetry_handled)- end,- %{}- )+ :ok = Cache.refresh_all(cache_name: test)- opts = [force_start?: true, warmer_fn: report_back(test_pid), cache_name: test]- {:ok, _} = Supervisor.start_link([{Cache.Warmer, opts}], strategy: :one_for_one, name: test)+ {:ok, _} = Plausible.Repo.delete(site1)- assert_receive {:cache_warmed, _}- assert_receive :telemetry_handled- end+ assert Cache.size(test) == 2- test "critical cache errors are logged and nil is returned" do- log =- capture_log(fn ->- assert Cache.get("key", force?: true, cache_name: NonExistingCache) == nil- end)+ assert %Site{from_cache?: true, id: ^first_id} =+ Cache.get("site1.example.com", force?: true, cache_name: test)- assert log =~ "Error retrieving 'key' from 'NonExistingCache': :no_cache"- end+ assert %Site{from_cache?: true} =+ Cache.get("site2.example.com", force?: true, cache_name: test)- test "cache warmer warms periodically with an interval", %{test: test} do- test_pid = self()- opts = [force_start?: true, warmer_fn: report_back(test_pid), cache_name: test, interval: 30]+ assert %Site{from_cache?: false} = Cache.get("site2.example.com", cache_name: test)- {:ok, _} = Supervisor.start_link([{Cache.Warmer, opts}], strategy: :one_for_one, name: test)+ refute Cache.get("site3.example.com", cache_name: test, force?: true)+ end- assert_receive {:cache_warmed, %{at: at1}}, 100- assert_receive {:cache_warmed, %{at: at2}}, 100- assert_receive {:cache_warmed, %{at: at3}}, 100+ test "cache exposes hit rate", %{test: test} do+ {:ok, _} = start_test_cache(test)- assert is_integer(at1)- assert is_integer(at2)- assert is_integer(at3)+ insert(:site, domain: "site1.example.com")+ :ok = Cache.refresh_all(cache_name: test)- assert at1 < at2- assert at3 > at2- end+ assert Cache.hit_rate(test) == 0+ assert Cache.get("site1.example.com", force?: true, cache_name: test)+ assert Cache.hit_rate(test) == 100+ refute Cache.get("nonexisting.example.com", force?: true, cache_name: test)+ assert Cache.hit_rate(test) == 50+ end++ test "a single cached site can be refreshed", %{test: test} do+ {:ok, _} = start_test_cache(test)++ domain1 = "site1.example.com"+ domain2 = "nonexisting.example.com"++ cache_opts = [cache_name: test, force?: true]- test "cache caches", %{test: test} do- {:ok, _} =- Supervisor.start_link([{Cache, [cache_name: test, child_id: :test_cache_caches_id]}],- strategy: :one_for_one,- name: Test.Supervisor.Cache- )+ assert Cache.get(domain1) == nil- %{id: first_id} = site1 = insert(:site, domain: "site1.example.com")- _ = insert(:site, domain: "site2.example.com")+ insert(:site, domain: domain1)++ assert {:ok, %{domain: ^domain1}} = Cache.refresh_one(domain1, cache_opts)+ assert %Site{domain: ^domain1} = Cache.get(domain1, cache_opts)++ assert {:ok, %Ecto.NoResultsError{}} = Cache.refresh_one(domain2, cache_opts)+ assert %Ecto.NoResultsError{} = Cache.get(domain2, cache_opts)+ end- :ok = Cache.prefill(cache_name: test)+ test "refreshing a single site sends a telemetry event indicating record not found in the database",+ %{+ test: test+ } do+ :ok =+ start_test_cache_with_telemetry_handler(test,+ event: Cache.telemetry_event_refresh(test, :one)+ )++ Cache.refresh_one("missing.example.com", force?: true, cache_name: test)+ assert_receive {:telemetry_handled, %{found_in_db?: false}}+ end- {:ok, _} = Plausible.Repo.delete(site1)+ test "refreshing a single site sends a telemetry event indicating record found in the database",+ %{+ test: test+ } do+ domain = "site1.example.com"+ insert(:site, domain: domain)- assert Cache.size(test) == 2+ :ok =+ start_test_cache_with_telemetry_handler(test,+ event: Cache.telemetry_event_refresh(test, :one)+ )- assert %Site{from_cache?: true, id: ^first_id} =- Cache.get("site1.example.com", force?: true, cache_name: test)+ Cache.refresh_one(domain, force?: true, cache_name: test)+ assert_receive {:telemetry_handled, %{found_in_db?: true}}+ end- assert %Site{from_cache?: true} =- Cache.get("site2.example.com", force?: true, cache_name: test)+ test "refreshing all sites sends a telemetry event",+ %{+ test: test+ } do+ domain = "site1.example.com"+ insert(:site, domain: domain)- assert %Site{from_cache?: false} = Cache.get("site2.example.com", cache_name: test)+ :ok =+ start_test_cache_with_telemetry_handler(test,+ event: Cache.telemetry_event_refresh(test, :all)+ )- refute Cache.get("site3.example.com", cache_name: test, force?: true)+ Cache.refresh_all(force?: true, cache_name: test)+ assert_receive {:telemetry_handled, %{}}+ endend- test "cache exposes hit rate", %{test: test} do- {:ok, _} =- Supervisor.start_link([{Cache, [cache_name: test, child_id: :test_cache_caches_id]}],- strategy: :one_for_one,- name: Test.Supervisor.HitRateCache- )-- insert(:site, domain: "site1.example.com")- :ok = Cache.prefill(cache_name: test)-- assert Cache.hit_rate(test) == 0- assert Cache.get("site1.example.com", force?: true, cache_name: test)- assert Cache.hit_rate(test) == 100- refute Cache.get("nonexisting.example.com", force?: true, cache_name: test)- assert Cache.hit_rate(test) == 50+ describe "warming the cache" do+ test "cache warmer process warms up the cache", %{test: test} do+ test_pid = self()+ opts = [force_start?: true, warmer_fn: report_back(test_pid), cache_name: test]++ {:ok, _} = Supervisor.start_link([{Cache.Warmer, opts}], strategy: :one_for_one, name: test)+ assert Process.whereis(Cache.Warmer)++ assert_receive {:cache_warmed, %{opts: got_opts}}+ assert got_opts[:cache_name] == test+ end++ test "cache warmer warms periodically with an interval", %{test: test} do+ test_pid = self()++ opts = [+ force_start?: true,+ warmer_fn: report_back(test_pid),+ cache_name: test,+ interval: 30+ ]++ {:ok, _} = start_test_warmer(opts)++ assert_receive {:cache_warmed, %{at: at1}}, 100+ assert_receive {:cache_warmed, %{at: at2}}, 100+ assert_receive {:cache_warmed, %{at: at3}}, 100++ assert is_integer(at1)+ assert is_integer(at2)+ assert is_integer(at3)++ assert at1 < at2+ assert at3 > at2+ end++ test "deleted sites don't stay in cache on another refresh", %{test: test} do+ {:ok, _} = start_test_cache(test)++ domain1 = "site1.example.com"+ domain2 = "site2.example.com"++ site1 = insert(:site, domain: domain1)+ _site2 = insert(:site, domain: domain2)++ cache_opts = [cache_name: test, force?: true]++ :ok = Cache.refresh_all(cache_opts)++ assert Cache.get(domain1, cache_opts)+ assert Cache.get(domain2, cache_opts)++ Repo.delete!(site1)++ :ok = Cache.refresh_all(cache_opts)++ assert Cache.get(domain2, cache_opts)++ refute Cache.get(domain1, cache_opts)+ Cache.refresh_one(domain1, cache_opts)+ assert Cache.get(domain1, cache_opts) == %Ecto.NoResultsError{}+ endenddefp report_back(test_pid) do@@ -123,4 +201,29 @@ defmodule Plausible.Site.CacheTest do:okendend++ defp start_test_cache(cache_name) do+ %{start: {m, f, a}} = Cache.child_spec(cache_name: cache_name)+ apply(m, f, a)+ end++ defp start_test_warmer(opts) do+ child_name_opt = {:child_name, {:local, Keyword.fetch!(opts, :cache_name)}}+ %{start: {m, f, a}} = Cache.Warmer.child_spec([child_name_opt | opts])+ apply(m, f, a)+ end++ defp start_test_cache_with_telemetry_handler(test, event: event) do+ {:ok, _} = start_test_cache(test)+ test_pid = self()++ :telemetry.attach(+ "#{test}-telemetry-handler",+ event,+ fn ^event, %{duration: d}, metadata, _ when is_integer(d) ->+ send(test_pid, {:telemetry_handled, metadata})+ end,+ %{}+ )+ endend