Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Kick off sites by domain cache implementation (#2434)
Adam Rutkowski457a5584712f
diff --git a/config/config.exs b/config/config.exsindex 369b11b00b..32456c2a75 100644--- a/config/config.exs+++ b/config/config.exs@@ -41,4 +41,9 @@ config :plausible, Plausible.Repo,connect_timeout: 300_000,handshake_timeout: 300_000+config :plausible,+ sites_by_domain_cache_enabled: true,+ sites_by_domain_cache_refresh_interval_max_jitter: :timer.seconds(5),+ sites_by_domain_cache_refresh_interval: :timer.minutes(15)+import_config "#{config_env()}.exs"diff --git a/config/test.exs b/config/test.exsindex d148d4ca17..475cd355a3 100644--- a/config/test.exs+++ b/config/test.exs@@ -63,4 +63,5 @@ config :geolix,config :plausible,session_timeout: 0,- http_impl: Plausible.HTTPClient.Mock+ http_impl: Plausible.HTTPClient.Mock,+ sites_by_domain_cache_enabled: falsediff --git a/lib/plausible/application.ex b/lib/plausible/application.exindex a267567f90..b060c3449e 100644--- a/lib/plausible/application.ex+++ b/lib/plausible/application.ex@@ -21,6 +21,8 @@ defmodule Plausible.Application doSupervisor.child_spec({Cachex, name: :sessions, limit: nil, stats: true},id: :cachex_sessions),+ {Plausible.Site.Cache, []},+ {Plausible.Site.Cache.Warmer, []},PlausibleWeb.Endpoint,{Oban, Application.get_env(:plausible, Oban)},Plausible.PromExdiff --git a/lib/plausible/site.ex b/lib/plausible/site.exindex adfb124a53..bbe8c90b82 100644--- a/lib/plausible/site.ex+++ b/lib/plausible/site.ex@@ -31,6 +31,11 @@ defmodule Plausible.Site dohas_one :custom_domain, Plausible.Site.CustomDomainhas_one :spike_notification, Plausible.Site.SpikeNotification+ # If `from_cache?` is set, the struct might be incomplete - see `Plausible.Site.Cache`.+ # Use `Plausible.Repo.reload!(cached_site)` to pre-fill missing fields if+ # strictly necessary.+ field :from_cache?, :boolean, virtual: true, default: false+timestamps()enddiff --git a/lib/plausible/site/cache.ex b/lib/plausible/site/cache.exnew file mode 100644index 0000000000..c24a7c0aa2--- /dev/null+++ b/lib/plausible/site/cache.ex@@ -0,0 +1,112 @@+defmodule Plausible.Site.Cache do+ @moduledoc """+ A sites by domain caching interface.++ Serves as a thin wrapper around Cachex, but the underlying+ implementation can be transparently swapped.++ Even though the Cachex process is started, cache access is disabled+ during tests via the `:sites_by_domain_cache_enabled` application env key.+ This can be overriden on case by case basis, using the child specs options.++ When Cache is disabled via application env, the `get/1` function+ falls back to pure database lookups. This should help with introducing+ cached lookups in existing code, so that no existing tests should break.++ To differentiate cached Site structs from those retrieved directly from the+ database, a virtual schema field `from_cache?: true` is set.+ This indicates the `Plausible.Site` struct is incomplete in comparison to its+ database counterpart -- to spare bandwidth and query execution time,+ only selected database columns are retrieved and cached.++ The `@cached_schema_fields` attribute defines the list of DB columns+ queried on cache pre-fill.++ Also see tests for more comprehensive examples.+ """+ require Logger++ import Ecto.Query++ alias Plausible.Site++ @cache_name :sites_by_domain++ @cached_schema_fields ~w(+ id+ domain+ ingest_rate_limit_scale_seconds+ ingest_rate_limit_threshold+ )a++ def name(), do: @cache_name++ @spec child_spec(Keyword.t()) :: Supervisor.child_spec()+ def child_spec(opts) do+ cache_name = Keyword.get(opts, :cache_name, @cache_name)+ child_id = Keyword.get(opts, :child_id, :cachex_sites)++ Supervisor.child_spec(+ {Cachex, name: cache_name, limit: nil, stats: true},+ id: child_id+ )+ end++ @spec prefill(Keyword.t()) :: :ok+ def prefill(opts) do+ cache_name = Keyword.fetch!(opts, :cache_name)++ sites_by_domain_query =+ from s in Site,+ select: {+ s.domain,+ %{struct(s, ^@cached_schema_fields) | from_cache?: true}+ }++ sites_by_domain = Plausible.Repo.all(sites_by_domain_query)++ true = Cachex.put_many!(cache_name, sites_by_domain)+ :ok+ end++ @spec size() :: non_neg_integer()+ def size(cache_name \\ @cache_name) do+ {:ok, size} = Cachex.size(cache_name)+ size+ end++ @spec hit_rate() :: number()+ def hit_rate(cache_name \\ @cache_name) do+ {:ok, stats} = Cachex.stats(cache_name)+ Map.get(stats, :hit_rate, 0)+ end++ @spec get(String.t(), Keyword.t()) :: nil | Site.t()+ def get(domain, opts \\ []) do+ cache_name = Keyword.get(opts, :cache_name, @cache_name)+ force? = Keyword.get(opts, :force?, false)++ if enabled?() or force? do+ case Cachex.get(cache_name, domain) do+ {:ok, nil} ->+ nil++ {:ok, site} ->+ site++ {:error, e} ->+ Logger.error(+ "Error retrieving '#{domain}' from '#{inspect(cache_name)}': #{inspect(e)}"+ )++ nil+ end+ else+ Plausible.Sites.get_by_domain(domain)+ end+ end++ def enabled?() do+ Application.fetch_env!(:plausible, :sites_by_domain_cache_enabled) == true+ end+enddiff --git a/lib/plausible/site/cache/warmer.ex b/lib/plausible/site/cache/warmer.exnew file mode 100644index 0000000000..cbb705ac72--- /dev/null+++ b/lib/plausible/site/cache/warmer.ex@@ -0,0 +1,111 @@+defmodule Plausible.Site.Cache.Warmer do+ @moduledoc """+ A periodic cache warmer.+ Queries all Sites from Postgres, every `interval` and pre-populates the cache.+ After each run the process is hibernated, triggering garbage collection.++ Currently Cachex is used, but the underlying implementation can be transparently swapped.++ Child specification options available:++ * `interval` - the number of milliseconds for each warm-up cycle, defaults+ to `:sites_by_domain_cache_refresh_interval` application env value+ with random jitter added, for which the maximum is stored under+ `:sites_by_domain_cache_refresh_interval_max_jitter` key.+ * `cache_name` - defaults to Cache.name() but can be overriden for testing+ * `force_start?` - enforcess process startup for testing, even if it's barred+ by `Cache.enabled?`. This is useful for avoiding issues with DB ownership+ and async tests.+ * `warmer_fn` - used for testing, a custom function to retrieve the items meant+ to be cached during the warm-up cycle.++ On each warm-up cycle, a telemetry event is emitted with warm-up `duration` stored.+ in the measurments map. The event name defaults to:++ `[:prom_ex, :plugin, :cachex, Cache.name(), :refresh]`++ See tests for more comprehensive examples.+ """++ @behaviour :gen_cycle++ require Logger++ alias Plausible.Site.Cache++ @spec child_spec(Keyword.t()) :: Supervisor.child_spec() | :ignore+ def child_spec(opts) do+ %{+ id: __MODULE__,+ start: {:gen_cycle, :start_link, [{:local, __MODULE__}, __MODULE__, opts]}+ }+ end++ @impl true+ def init_cycle(opts) do+ interval = Keyword.get(opts, :interval, interval())+ force_start? = Keyword.get(opts, :force_start?, false)+ cache_name = Keyword.get(opts, :cache_name, Cache.name())++ if Cache.enabled?() or force_start? do+ Logger.info("Initializing #{__MODULE__} with interval #{interval}")++ {:ok, {interval, Keyword.put(opts, :cache_name, cache_name)}}+ else+ :ignore+ end+ end++ @impl true+ def handle_cycle(opts) do+ cache_name = Keyword.fetch!(opts, :cache_name)++ measure_duration(telemetry_event_refresh(cache_name), fn ->+ Logger.info("Refreshing #{cache_name} cache...")++ warmer_fn = Keyword.get(opts, :warmer_fn, &Cache.prefill/1)++ warmer_fn.(opts)+ end)++ {:continue_hibernated, opts}+ end++ @impl true+ def handle_info(_msg, state) do+ {:continue, state}+ end++ @spec telemetry_event_refresh(atom()) :: list(atom())+ def telemetry_event_refresh(cache_name) do+ [:prom_ex, :plugin, :cachex, cache_name, :refresh]+ end++ @spec interval() :: pos_integer()+ def interval() do+ interval = Application.fetch_env!(:plausible, :sites_by_domain_cache_refresh_interval)++ interval + jitter()+ end++ defp jitter() do+ max_jitter =+ Application.fetch_env!(:plausible, :sites_by_domain_cache_refresh_interval_max_jitter)++ Enum.random(1..max_jitter)+ end++ defp measure_duration(event, fun) when is_function(fun, 0) do+ start = System.monotonic_time()+ result = fun.()+ stop = System.monotonic_time()++ :telemetry.execute(+ event,+ %{duration: stop - start},+ %{}+ )++ result+ end+enddiff --git a/mix.exs b/mix.exsindex 1eeb1d0dd5..a78b939875 100644--- a/mix.exs+++ b/mix.exs@@ -80,6 +80,7 @@ defmodule Plausible.MixProject do{:floki, "~> 0.32.0", only: :test},{:fun_with_flags, "~> 1.9.0"},{:fun_with_flags_ui, "~> 0.8"},+ {:gen_cycle, "~> 1.0"},{:geolix, "~> 2.0"},{:geolix_adapter_mmdb2, "~> 0.6.0"},{:hackney, "~> 1.8"},diff --git a/mix.lock b/mix.lockindex 635786835b..473e80adc6 100644--- a/mix.lock+++ b/mix.lock@@ -47,6 +47,7 @@"floki": {:hex, :floki, "0.32.1", "dfe3b8db3b793939c264e6f785bca01753d17318d144bd44b407fb3493acaa87", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "d4b91c713e4a784a3f7b1e3cc016eefc619f6b1c3898464222867cafd3c681a3"},"fun_with_flags": {:hex, :fun_with_flags, "1.9.0", "0be8692727623af0c00e353f7bdcc9934dd6e1f87798ca6bfec9e739d42b63e6", [:mix], [{:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: true]}, {:redix, "~> 1.0", [hex: :redix, repo: "hexpm", optional: true]}], "hexpm", "8145dc009d549389f1b1915a715fb7e357fcd8fd7b91c74f04aae74912eef27a"},"fun_with_flags_ui": {:hex, :fun_with_flags_ui, "0.8.0", "70587e344ba2035516a639e7bd8cb2ce8d54ee6c5f5dfd3e4e6f6a776e14ac1d", [:mix], [{:cowboy, ">= 2.0.0", [hex: :cowboy, repo: "hexpm", optional: true]}, {:fun_with_flags, "~> 1.8", [hex: :fun_with_flags, repo: "hexpm", optional: false]}, {:plug, "~> 1.12", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 2.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "95e1e5afae77e259a4a43f930f3da97ff3c388a72d4622f7848cb7ee34de6338"},+ "gen_cycle": {:hex, :gen_cycle, "1.0.3", "ee0117b9c65814f58827abfe84967d4cb5380db0fbac992358e5b669b3ebcbcb", [:rebar3], [], "hexpm", "fffdadf7fb73de75e3429eb8a6982768e7240803561942602350b689991749e4"},"gen_smtp": {:hex, :gen_smtp, "1.2.0", "9cfc75c72a8821588b9b9fe947ae5ab2aed95a052b81237e0928633a13276fd3", [:rebar3], [{:ranch, ">= 1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "5ee0375680bca8f20c4d85f58c2894441443a743355430ff33a783fe03296779"},"geolix": {:hex, :geolix, "2.0.0", "7e65764bedfc98d08a3ddb24c417657c9d438eff163280b45fbb7de289626acd", [:mix], [], "hexpm", "8742bf588ed0bb7def2c443204d09d355990846c6efdff96ded66aac24c301df"},"geolix_adapter_mmdb2": {:hex, :geolix_adapter_mmdb2, "0.6.0", "6ab9dbf6ea395817aa1fd2597be25d0dda1853c7f664e62716e47728d18bc4f9", [:mix], [{:geolix, "~> 2.0", [hex: :geolix, repo: "hexpm", optional: false]}, {:mmdb2_decoder, "~> 3.0", [hex: :mmdb2_decoder, repo: "hexpm", optional: false]}], "hexpm", "06ff962feae8a310cffdf86b74bfcda6e2d0dccb439bb1f62df2b657b1c0269b"},diff --git a/test/plausible/site/cache_test.exs b/test/plausible/site/cache_test.exsnew file mode 100644index 0000000000..7232f90423--- /dev/null+++ b/test/plausible/site/cache_test.exs@@ -0,0 +1,126 @@+defmodule Plausible.Site.CacheTest do+ use Plausible.DataCase, async: true++ alias Plausible.Site+ alias Plausible.Site.Cache++ import ExUnit.CaptureLog++ test "cache process is started, but falls back to the database if cache is disabled" do+ insert(:site, domain: "example.test")+ refute Cache.enabled?()+ assert Process.alive?(Process.whereis(Cache.name()))+ refute Process.whereis(Cache.Warmer)+ assert %Site{domain: "example.test", from_cache?: false} = Cache.get("example.test")+ assert Cache.size() == 0+ refute Cache.get("other.test")+ end++ test "cache warmer process warms up the cache", %{test: test} do+ test_pid = self()+ opts = [force_start?: true, warmer_fn: report_back(test_pid), cache_name: test]++ {:ok, _} = Supervisor.start_link([{Cache.Warmer, opts}], strategy: :one_for_one, name: test)++ assert Process.whereis(Cache.Warmer)+ assert_receive {:cache_warmed, %{opts: got_opts}}+ assert got_opts[:cache_name] == test+ end++ test "cache warmer sends telemetry event", %{test: test} do+ test_pid = self()++ :telemetry.attach(+ "test-cache-warmer",+ Cache.Warmer.telemetry_event_refresh(test),+ fn _event, %{duration: d}, %{}, %{} when is_integer(d) ->+ send(test_pid, :telemetry_handled)+ end,+ %{}+ )++ opts = [force_start?: true, warmer_fn: report_back(test_pid), cache_name: test]+ {:ok, _} = Supervisor.start_link([{Cache.Warmer, opts}], strategy: :one_for_one, name: test)++ assert_receive {:cache_warmed, _}+ assert_receive :telemetry_handled+ end++ test "critical cache errors are logged and nil is returned" do+ log =+ capture_log(fn ->+ assert Cache.get("key", force?: true, cache_name: NonExistingCache) == nil+ end)++ assert log =~ "Error retrieving 'key' from 'NonExistingCache': :no_cache"+ end++ test "cache warmer warms periodically with an interval", %{test: test} do+ test_pid = self()+ opts = [force_start?: true, warmer_fn: report_back(test_pid), cache_name: test, interval: 30]++ {:ok, _} = Supervisor.start_link([{Cache.Warmer, opts}], strategy: :one_for_one, name: test)++ assert_receive {:cache_warmed, %{at: at1}}, 100+ assert_receive {:cache_warmed, %{at: at2}}, 100+ assert_receive {:cache_warmed, %{at: at3}}, 100++ assert is_integer(at1)+ assert is_integer(at2)+ assert is_integer(at3)++ assert at1 < at2+ assert at3 > at2+ end++ test "cache caches", %{test: test} do+ {:ok, _} =+ Supervisor.start_link([{Cache, [cache_name: test, child_id: :test_cache_caches_id]}],+ strategy: :one_for_one,+ name: Test.Supervisor.Cache+ )++ %{id: first_id} = site1 = insert(:site, domain: "site1.example.com")+ _ = insert(:site, domain: "site2.example.com")++ :ok = Cache.prefill(cache_name: test)++ {:ok, _} = Plausible.Repo.delete(site1)++ assert Cache.size(test) == 2++ assert %Site{from_cache?: true, id: ^first_id} =+ Cache.get("site1.example.com", force?: true, cache_name: test)++ assert %Site{from_cache?: true} =+ Cache.get("site2.example.com", force?: true, cache_name: test)++ assert %Site{from_cache?: false} = Cache.get("site2.example.com", cache_name: test)++ refute Cache.get("site3.example.com", cache_name: test, force?: true)+ end++ test "cache exposes hit rate", %{test: test} do+ {:ok, _} =+ Supervisor.start_link([{Cache, [cache_name: test, child_id: :test_cache_caches_id]}],+ strategy: :one_for_one,+ name: Test.Supervisor.HitRateCache+ )++ insert(:site, domain: "site1.example.com")+ :ok = Cache.prefill(cache_name: test)++ assert Cache.hit_rate(test) == 0+ assert Cache.get("site1.example.com", force?: true, cache_name: test)+ assert Cache.hit_rate(test) == 100+ refute Cache.get("nonexisting.example.com", force?: true, cache_name: test)+ assert Cache.hit_rate(test) == 50+ end++ defp report_back(test_pid) do+ fn opts ->+ send(test_pid, {:cache_warmed, %{at: System.monotonic_time(), opts: opts}})+ :ok+ end+ end+end