Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Balance sessions across worker queues instead of explicit locks (#5113)
hq1ccdf50b7349b
diff --git a/config/runtime.exs b/config/runtime.exsindex 47b702e5ad..16b2befd1a 100644--- a/config/runtime.exs+++ b/config/runtime.exs@@ -977,6 +977,4 @@ unless s3_disabled? doimports_bucket: s3_env_value.("S3_IMPORTS_BUCKET")end-config :plausible, Plausible.Cache.Adapter, sessions: [partitions: 4]-config :phoenix_storybook, enabled: env !== "prod"diff --git a/lib/plausible/application.ex b/lib/plausible/application.exindex 2ff5961707..391047e74d 100644--- a/lib/plausible/application.ex+++ b/lib/plausible/application.ex@@ -15,6 +15,7 @@ defmodule Plausible.Application dochildren =[+ Plausible.Session.BalancerSupervisor,Plausible.Cache.Stats,Plausible.PromEx,{Plausible.Auth.TOTP.Vault, key: totp_vault_key()},diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.exindex 9c7c356150..d08e8d4c26 100644--- a/lib/plausible/ingestion/event.ex+++ b/lib/plausible/ingestion/event.ex@@ -393,7 +393,7 @@ defmodule Plausible.Ingestion.Event doevent.clickhouse_event,event.clickhouse_session_attrs,previous_user_id,- write_buffer_insert+ buffer_insert: write_buffer_insert)case session_result dodiff --git a/lib/plausible/session/balancer.ex b/lib/plausible/session/balancer.exnew file mode 100644index 0000000000..11235adcd0--- /dev/null+++ b/lib/plausible/session/balancer.ex@@ -0,0 +1,39 @@+defmodule Plausible.Session.Balancer do+ @moduledoc "Serialize session processing to avoid explicit locks"+ use GenServer++ def start_link(id) do+ GenServer.start_link(__MODULE__, id, name: via(id))+ end++ @impl true+ def init(id) do+ {:ok, %{id: id}}+ end++ def dispatch(user_id, fun, opts) do+ timeout = Keyword.fetch!(opts, :timeout)+ local? = Keyword.get(opts, :local?, false)++ if local? do+ fun.()+ else+ worker = :erlang.phash2(user_id, Plausible.Session.BalancerSupervisor.size()) + 1+ [{pid, _}] = Registry.lookup(Plausible.Session.Balancer.Registry, worker)+ GenServer.call(pid, {:process, fun}, timeout)+ end+ end++ @impl true+ def handle_call({:process, fun}, _from, state) do+ try do+ response = fun.()+ {:reply, response, state}+ rescue+ e ->+ {:reply, {:error, e}, state}+ end+ end++ defp via(id), do: {:via, Registry, {Plausible.Session.Balancer.Registry, id}}+enddiff --git a/lib/plausible/session/balancer_supervisor.ex b/lib/plausible/session/balancer_supervisor.exnew file mode 100644index 0000000000..e35969841d--- /dev/null+++ b/lib/plausible/session/balancer_supervisor.ex@@ -0,0 +1,33 @@+defmodule Plausible.Session.BalancerSupervisor do+ @moduledoc "Serialize session processing to avoid explicit locks"+ use Supervisor++ if Mix.env() in [:test, :ce_test] do+ def size(),+ do: 10+ else+ def size(), do: 100+ end++ def start_link(_) do+ Supervisor.start_link(__MODULE__, size(), name: __MODULE__)+ end++ def init(size) do+ children =+ for id <- 1..size do+ %{+ id: id,+ start: {Plausible.Session.Balancer, :start_link, [id]},+ restart: :permanent+ }+ end++ Supervisor.init(+ [+ {Registry, [keys: :unique, name: Plausible.Session.Balancer.Registry]} | children+ ],+ strategy: :one_for_one+ )+ end+enddiff --git a/lib/plausible/session/cache_store.ex b/lib/plausible/session/cache_store.exindex 54a5bb488f..f115698f46 100644--- a/lib/plausible/session/cache_store.ex+++ b/lib/plausible/session/cache_store.ex@@ -2,27 +2,43 @@ defmodule Plausible.Session.CacheStore dorequire Loggeralias Plausible.Session.WriteBuffer- @lock_timeout 500+ @lock_timeout 1000@lock_telemetry_event [:plausible, :sessions, :cache, :lock]def lock_telemetry_event, do: @lock_telemetry_event- def on_event(event, session_attributes, prev_user_id, buffer_insert \\ &WriteBuffer.insert/1) do+ def on_event(event, session_attributes, prev_user_id, opts \\ []) do+ buffer_insert = Keyword.get(opts, :buffer_insert, &WriteBuffer.insert/1)+ skip_balancer? = Keyword.get(opts, :skip_balancer?, false)lock_requested_at = System.monotonic_time()- Plausible.Cache.Adapter.with_lock(- :sessions,- {event.site_id, event.user_id},- @lock_timeout,- fn ->- lock_duration = System.monotonic_time() - lock_requested_at- :telemetry.execute(@lock_telemetry_event, %{duration: lock_duration}, %{})- found_session = find_session(event, event.user_id) || find_session(event, prev_user_id)-- handle_event(event, found_session, session_attributes, buffer_insert)+ try do+ response =+ Plausible.Session.Balancer.dispatch(+ event.user_id,+ fn ->+ lock_duration = System.monotonic_time() - lock_requested_at+ :telemetry.execute(@lock_telemetry_event, %{duration: lock_duration}, %{})++ found_session =+ find_session(event, event.user_id) || find_session(event, prev_user_id)++ handle_event(event, found_session, session_attributes, buffer_insert)+ end,+ timeout: @lock_timeout,+ local?: skip_balancer?+ )++ case response do+ {:error, e} -> raise e+ _ -> {:ok, response}end- )+ catch+ :exit, {:timeout, _} ->+ Sentry.capture_message("Timeout while handling session event")+ {:error, :timeout}+ endenddefp handle_event(%{name: "engagement"} = event, found_session, _, _) dodiff --git a/test/plausible/ingestion/event_test.exs b/test/plausible/ingestion/event_test.exsindex 0b64ff341f..38e185ebc6 100644--- a/test/plausible/ingestion/event_test.exs+++ b/test/plausible/ingestion/event_test.exs@@ -1,5 +1,5 @@defmodule Plausible.Ingestion.EventTest do- use Plausible.DataCase, async: true+ use Plausible.DataCase, async: falseuse Plausible.Teams.Testimport Phoenix.ConnTest@@ -282,10 +282,9 @@ defmodule Plausible.Ingestion.EventTest dotest = self()- very_slow_buffer = fn sessions ->+ very_slow_buffer = fn _sessions ->send(test, :slow_buffer_insert_started)- Process.sleep(1000)- Plausible.Session.WriteBuffer.insert(sessions)+ Process.sleep(800)endfirst_conn =@@ -315,7 +314,11 @@ defmodule Plausible.Ingestion.EventTest doreceive do:slow_buffer_insert_started ->- assert {:ok, %{buffered: [], dropped: [dropped]}} = Event.build_and_buffer(second_request)+ assert {:ok, %{buffered: [], dropped: [dropped]}} =+ Event.build_and_buffer(second_request,+ session_write_buffer_insert: very_slow_buffer+ )+assert dropped.drop_reason == :lock_timeoutendenddiff --git a/test/plausible/session/cache_store_test.exs b/test/plausible/session/cache_store_test.exsindex 1bf2a45fc0..dd215b4442 100644--- a/test/plausible/session/cache_store_test.exs+++ b/test/plausible/session/cache_store_test.exs@@ -43,11 +43,13 @@ defmodule Plausible.Session.CacheStoreTest do} dotelemetry_event = CacheStore.lock_telemetry_event()+ test_pid = self()+:telemetry.attach("#{test}-telemetry-handler",telemetry_event,fn ^telemetry_event, %{duration: d}, _, _ when is_integer(d) ->- send(self(), {:telemetry_handled, d})+ send(test_pid, {:telemetry_handled, d})end,%{})@@ -56,7 +58,7 @@ defmodule Plausible.Session.CacheStoreTest doevent2 = build(:event, name: "pageview", user_id: event1.user_id, site_id: event1.site_id)event3 = build(:event, name: "pageview", user_id: event1.user_id, site_id: event1.site_id)- CacheStore.on_event(event1, @session_params, nil, buffer)+ CacheStore.on_event(event1, @session_params, nil, buffer_insert: buffer)assert_receive({:buffer, :insert, [[session1]]})assert_receive({:telemetry_handled, duration})@@ -65,7 +67,7 @@ defmodule Plausible.Session.CacheStoreTest do[event2, event3]|> Enum.map(fn e ->Task.async(fn ->- CacheStore.on_event(e, @session_params, nil, slow_buffer)+ CacheStore.on_event(e, @session_params, nil, buffer_insert: slow_buffer)end)end)|> Task.await_many()@@ -122,7 +124,7 @@ defmodule Plausible.Session.CacheStoreTest doasync1 =Task.async(fn ->- CacheStore.on_event(event1, @session_params, nil, very_slow_buffer)+ CacheStore.on_event(event1, @session_params, nil, buffer_insert: very_slow_buffer)end)# Ensure next events are executed after processing event1 starts@@ -130,12 +132,12 @@ defmodule Plausible.Session.CacheStoreTest doasync2 =Task.async(fn ->- CacheStore.on_event(event2, @session_params, nil, buffer)+ CacheStore.on_event(event2, @session_params, nil, buffer_insert: buffer)end)async3 =Task.async(fn ->- CacheStore.on_event(event3, @session_params, nil, buffer)+ CacheStore.on_event(event3, @session_params, nil, buffer_insert: buffer)end)Task.await_many([async1, async2, async3])@@ -160,7 +162,7 @@ defmodule Plausible.Session.CacheStoreTest doasync1 =Task.async(fn ->- CacheStore.on_event(event1, @session_params, nil, very_slow_buffer)+ CacheStore.on_event(event1, @session_params, nil, buffer_insert: very_slow_buffer)end)# Ensure next events are executed after processing event1 starts@@ -168,14 +170,14 @@ defmodule Plausible.Session.CacheStoreTest doasync2 =Task.async(fn ->- CacheStore.on_event(event2, @session_params, nil, buffer)+ CacheStore.on_event(event2, @session_params, nil, buffer_insert: buffer)end)Process.sleep(100)async3 =Task.async(fn ->- CacheStore.on_event(event3, @session_params, nil, buffer)+ CacheStore.on_event(event3, @session_params, nil, buffer_insert: buffer)end)Task.await_many([async1, async2, async3])@@ -198,7 +200,7 @@ defmodule Plausible.Session.CacheStoreTest doevent = build(:event, name: "pageview")assert_raise RuntimeError, "boom", fn ->- CacheStore.on_event(event, @session_params, nil, crashing_buffer)+ CacheStore.on_event(event, @session_params, nil, buffer_insert: crashing_buffer)endend@@ -210,7 +212,7 @@ defmodule Plausible.Session.CacheStoreTest do"meta.value": ["true", "false"])- CacheStore.on_event(event, @session_params, nil, buffer)+ CacheStore.on_event(event, @session_params, nil, buffer_insert: buffer)assert_receive({:buffer, :insert, [sessions]})assert [session] = sessions@@ -253,8 +255,8 @@ defmodule Plausible.Session.CacheStoreTest do| timestamp: timestamp}- CacheStore.on_event(event1, %{}, nil, buffer)- CacheStore.on_event(event2, %{}, nil, buffer)+ CacheStore.on_event(event1, %{}, nil, buffer_insert: buffer)+ CacheStore.on_event(event2, %{}, nil, buffer_insert: buffer)assert_receive({:buffer, :insert, [[_negative_record, session]]})assert session.is_bounce == falseassert session.duration == 10@@ -267,8 +269,8 @@ defmodule Plausible.Session.CacheStoreTest dopageview = build(:pageview, timestamp: NaiveDateTime.shift(now, second: -10))engagement = %{pageview | name: "engagement", timestamp: now}- CacheStore.on_event(pageview, %{}, nil, buffer)- CacheStore.on_event(engagement, %{}, nil, buffer)+ CacheStore.on_event(pageview, %{}, nil, buffer_insert: buffer)+ CacheStore.on_event(engagement, %{}, nil, buffer_insert: buffer)assert_receive({:buffer, :insert, [[session]]})assert session.is_bounce == true@@ -282,7 +284,7 @@ defmodule Plausible.Session.CacheStoreTest dopageview1 = build(:event, name: "pageview", timestamp: start)- CacheStore.on_event(pageview1, %{}, nil, buffer)+ CacheStore.on_event(pageview1, %{}, nil, buffer_insert: buffer)assert_receive({:buffer, :insert, [[start_session]]})for delta <- [20, 40, 60] do@@ -292,11 +294,11 @@ defmodule Plausible.Session.CacheStoreTest dotimestamp: start |> NaiveDateTime.shift(minute: delta)})- CacheStore.on_event(engagement, %{}, nil, buffer)+ CacheStore.on_event(engagement, %{}, nil, buffer_insert: buffer)endpageview2 = Map.put(pageview1, :timestamp, start |> NaiveDateTime.shift(minute: 80))- CacheStore.on_event(pageview2, %{}, nil, buffer)+ CacheStore.on_event(pageview2, %{}, nil, buffer_insert: buffer)assert_receive({:buffer, :insert, [[_negative_record, updated_session]]})assert updated_session.session_id == start_session.session_id@@ -500,8 +502,8 @@ defmodule Plausible.Session.CacheStoreTest doevent2 = %{event1 | timestamp: timestamp}- CacheStore.on_event(event1, %{}, nil, buffer)- CacheStore.on_event(event2, %{}, nil, buffer)+ CacheStore.on_event(event1, %{}, nil, buffer_insert: buffer)+ CacheStore.on_event(event2, %{}, nil, buffer_insert: buffer)assert_receive({:buffer, :insert, [[_negative_record, session]]})assert session.duration == 10diff --git a/test/support/test_utils.ex b/test/support/test_utils.exindex 0707b72f15..f404457aa7 100644--- a/test/support/test_utils.ex+++ b/test/support/test_utils.ex@@ -191,7 +191,10 @@ defmodule Plausible.TestUtils dodefp populate_native_stats(events) dofor event_params <- events do- {:ok, session} = Plausible.Session.CacheStore.on_event(event_params, event_params, nil)+ {:ok, session} =+ Plausible.Session.CacheStore.on_event(event_params, event_params, nil,+ skip_balancer?: true+ )event_params|> Plausible.ClickhouseEventV2.merge_session(session)