Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Implement support for cache partitioning for sessions (#5073)
Adrian Gruntkowskia0c13383e762
diff --git a/config/runtime.exs b/config/runtime.exsindex 17d0f8f0af..23134105b7 100644--- a/config/runtime.exs+++ b/config/runtime.exs@@ -970,4 +970,6 @@ unless s3_disabled? doimports_bucket: s3_env_value.("S3_IMPORTS_BUCKET")end+config :plausible, Plausible.Cache.Adapter, sessions: [partitions: 4]+config :phoenix_storybook, enabled: env !== "prod"diff --git a/lib/plausible/application.ex b/lib/plausible/application.exindex e8a504b1a3..8b7b9401ee 100644--- a/lib/plausible/application.ex+++ b/lib/plausible/application.ex@@ -31,7 +31,7 @@ defmodule Plausible.Application dottl_check_interval: :timer.minutes(5),global_ttl: :timer.minutes(60)),- Plausible.Cache.Adapter.child_spec(:sessions, :cache_sessions,+ Plausible.Cache.Adapter.child_specs(:sessions, :cache_sessions,ttl_check_interval: :timer.seconds(10),global_ttl: :timer.minutes(30)),@@ -109,6 +109,7 @@ defmodule Plausible.Application dohelp_scout_vault()end]+ |> List.flatten()|> Enum.reject(&is_nil/1)opts = [strategy: :one_for_one, name: Plausible.Supervisor]diff --git a/lib/plausible/cache/adapter.ex b/lib/plausible/cache/adapter.exindex 0c77a28c97..ce8f0658d1 100644--- a/lib/plausible/cache/adapter.ex+++ b/lib/plausible/cache/adapter.ex@@ -9,6 +9,23 @@ defmodule Plausible.Cache.Adapter dorequire Logger+ @spec child_specs(atom(), atom(), Keyword.t()) :: [Supervisor.child_spec()]+ def child_specs(name, child_id, opts \\ [])+ when is_atom(name) and is_atom(child_id) and is_list(opts) do+ partitions = partitions(name)++ if partitions == 1 do+ [child_spec(name, child_id, opts)]+ else+ Enum.map(1..partitions, fn partition ->+ partition_name = String.to_atom("#{name}_#{partition}")+ partition_child_id = String.to_atom("#{child_id}_#{partition}")++ child_spec(partition_name, partition_child_id, opts)+ end)+ end+ end+@spec child_spec(atom(), atom(), Keyword.t()) :: Supervisor.child_spec()def child_spec(name, child_id, opts \\ [])when is_atom(name) and is_atom(child_id) and is_list(opts) do@@ -29,14 +46,18 @@ defmodule Plausible.Cache.Adapter do@spec size(atom()) :: non_neg_integer() | nildef size(cache_name) do- ConCache.size(cache_name)+ cache_name+ |> get_names()+ |> Enum.map(&ConCache.size/1)+ |> Enum.sum()catch:exit, _ -> nilend@spec get(atom(), any()) :: any()def get(cache_name, key) do- ConCache.get(cache_name, key)+ full_cache_name = get_name(cache_name, key)+ ConCache.get(full_cache_name, key)catch:exit, _ ->Logger.error("Error retrieving key from '#{inspect(cache_name)}'")@@ -45,7 +66,8 @@ defmodule Plausible.Cache.Adapter do@spec get(atom(), any(), (-> any())) :: any()def get(cache_name, key, fallback_fn) do- ConCache.get_or_store(cache_name, key, fallback_fn)+ full_cache_name = get_name(cache_name, key)+ ConCache.get_or_store(full_cache_name, key, fallback_fn)catch:exit, _ ->Logger.error("Error retrieving key from '#{inspect(cache_name)}'")@@ -54,7 +76,8 @@ defmodule Plausible.Cache.Adapter do@spec fetch(atom(), any(), (-> any())) :: any()def fetch(cache_name, key, fallback_fn) do- ConCache.fetch_or_store(cache_name, key, fallback_fn)+ full_cache_name = get_name(cache_name, key)+ ConCache.fetch_or_store(full_cache_name, key, fallback_fn)catch:exit, _ ->Logger.error("Error fetching key from '#{inspect(cache_name)}'")@@ -63,10 +86,12 @@ defmodule Plausible.Cache.Adapter do@spec put(atom(), any(), any()) :: any()def put(cache_name, key, value, opts \\ []) do+ full_cache_name = get_name(cache_name, key)+if opts[:dirty?] do- :ok = ConCache.dirty_put(cache_name, key, value)+ :ok = ConCache.dirty_put(full_cache_name, key, value)else- :ok = ConCache.put(cache_name, key, value)+ :ok = ConCache.put(full_cache_name, key, value)endvalue@@ -78,7 +103,12 @@ defmodule Plausible.Cache.Adapter do@spec put_many(atom(), [any()]) :: :okdef put_many(cache_name, items) when is_list(items) do- true = :ets.insert(ConCache.ets(cache_name), items)+ items+ |> Enum.group_by(fn {key, _} -> get_name(cache_name, key) end)+ |> Enum.each(fn {full_cache_name, items} ->+ true = :ets.insert(ConCache.ets(full_cache_name), items)+ end)+:okcatch:exit, _ ->@@ -88,7 +118,8 @@ defmodule Plausible.Cache.Adapter do@spec delete(atom(), any()) :: :okdef delete(cache_name, key) do- ConCache.dirty_delete(cache_name, key)+ full_cache_name = get_name(cache_name, key)+ ConCache.dirty_delete(full_cache_name, key)catch:exit, _ ->Logger.error("Error deleting a key in '#{cache_name}'")@@ -97,16 +128,11 @@ defmodule Plausible.Cache.Adapter do@spec keys(atom()) :: Enumerable.t()def keys(cache_name) do- ets = ConCache.ets(cache_name)-- Stream.resource(- fn -> :ets.first(ets) end,- fn- :"$end_of_table" -> {:halt, nil}- prev_key -> {[prev_key], :ets.next(ets, prev_key)}- end,- fn _ -> :ok end- )+ cache_name+ |> get_names()+ |> Enum.reduce([], fn full_cache_name, stream ->+ Stream.concat(stream, get_keys(full_cache_name))+ end)catch:exit, _ ->Logger.error("Error retrieving key from '#{inspect(cache_name)}'")@@ -116,7 +142,8 @@ defmodule Plausible.Cache.Adapter do@spec with_lock(atom(), any(), pos_integer(), (-> result)) :: {:ok, result} | {:error, :timeout}when result: any()def with_lock(cache_name, key, timeout, fun) do- result = ConCache.isolated(cache_name, key, timeout, fun)+ full_cache_name = get_name(cache_name, key)+ result = ConCache.isolated(full_cache_name, key, timeout, fun){:ok, result}catch:exit, {:timeout, _} ->@@ -127,4 +154,43 @@ defmodule Plausible.Cache.Adapter do{:error, :timeout}end++ @spec get_names(atom()) :: [atom()]+ def get_names(cache_name) do+ partitions = partitions(cache_name)++ if partitions == 1 do+ [cache_name]+ else+ Enum.map(1..partitions, &String.to_existing_atom("#{cache_name}_#{&1}"))+ end+ end++ defp get_keys(full_cache_name) do+ ets = ConCache.ets(full_cache_name)++ Stream.resource(+ fn -> :ets.first(ets) end,+ fn+ :"$end_of_table" -> {:halt, nil}+ prev_key -> {[prev_key], :ets.next(ets, prev_key)}+ end,+ fn _ -> :ok end+ )+ end++ defp get_name(cache_name, key) do+ partitions = partitions(cache_name)++ if partitions == 1 do+ cache_name+ else+ chosen_partition = :erlang.phash2(key, partitions) + 1+ String.to_existing_atom("#{cache_name}_#{chosen_partition}")+ end+ end++ defp partitions(cache_name) do+ Application.get_env(:plausible, __MODULE__)[cache_name][:partitions] || 1+ endenddiff --git a/lib/plausible/cache/stats.ex b/lib/plausible/cache/stats.exindex f88688bced..9ee4f0947f 100644--- a/lib/plausible/cache/stats.ex+++ b/lib/plausible/cache/stats.ex@@ -63,14 +63,30 @@ defmodule Plausible.Cache.Stats doenddef hit_rate(cache_name) do- hit = :ets.lookup_element(__MODULE__, {cache_name, @hit}, 2, 0)- miss = :ets.lookup_element(__MODULE__, {cache_name, @miss}, 2, 0)- hit_miss = hit + miss-- if hit_miss == 0 do- 0.0- else- hit / hit_miss * 100- end+ cache_name+ |> Plausible.Cache.Adapter.get_names()+ |> Enum.reduce(+ %{hit: 0, miss: 0, hit_miss: 0.0},+ fn name, acc ->+ hit =+ acc.hit + :ets.lookup_element(__MODULE__, {name, @hit}, 2, 0)++ miss =+ acc.miss + :ets.lookup_element(__MODULE__, {name, @miss}, 2, 0)++ hit_miss = hit + miss++ hit_miss = if(hit_miss == 0, do: 0.0, else: hit / hit_miss * 100)++ acc+ |> Map.put(:hit, hit)+ |> Map.put(:miss, miss)+ |> Map.put(+ :hit_miss,+ hit_miss+ )+ end+ )+ |> Map.fetch!(:hit_miss)endenddiff --git a/test/plausible/cache/adapter_test.exs b/test/plausible/cache/adapter_test.exsnew file mode 100644index 0000000000..474ca08f0e--- /dev/null+++ b/test/plausible/cache/adapter_test.exs@@ -0,0 +1,40 @@+defmodule Plausible.Cache.AdapterTest do+ use Plausible.DataCase, async: false++ alias Plausible.Cache.Adapter++ describe "adapter - partitioning" do+ test "multiple partitions are routed to", %{test: test} do+ name = :cache_partitions_test++ iterations = 100+ partitions = 4++ patch_env(Adapter, [{name, partitions: partitions}])++ {:ok, _} =+ Supervisor.start_link(+ Adapter.child_specs(name, name, []),+ strategy: :one_for_one,+ name: :"cache_supervisor_#{test}"+ )++ half = div(iterations, 2)+ for i <- 1..half, do: Adapter.put(name, i, i)+ Adapter.put_many(name, for(i <- half..iterations, do: {i, i}))++ assert Adapter.size(name) == iterations++ for i <- 1..iterations, do: assert(Adapter.get(name, i) == i)++ assert name |> Adapter.keys() |> Enum.sort() == Enum.to_list(1..iterations)++ for i <- 1..partitions do+ assert ConCache.size(:"#{name}_#{i}") > 0+ assert ConCache.size(:"#{name}_#{i}") < iterations+ end++ assert {:ok, %{hit_rate: 100.0, count: ^iterations}} = Plausible.Cache.Stats.gather(name)+ end+ end+end