Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Enforce sequential processing of session events (#4493)
Adrian Gruntkowski8a1c6e091369
diff --git a/lib/plausible/cache/adapter.ex b/lib/plausible/cache/adapter.exindex 65f8c254c4..ae79375807 100644--- a/lib/plausible/cache/adapter.ex+++ b/lib/plausible/cache/adapter.ex@@ -62,8 +62,13 @@ defmodule Plausible.Cache.Adapter doend@spec put(atom(), any(), any()) :: any()- def put(cache_name, key, value) do- :ok = ConCache.put(cache_name, key, value)+ def put(cache_name, key, value, opts \\ []) do+ if opts[:dirty?] do+ :ok = ConCache.dirty_put(cache_name, key, value)+ else+ :ok = ConCache.put(cache_name, key, value)+ end+valuecatch:exit, _ ->@@ -107,4 +112,12 @@ defmodule Plausible.Cache.Adapter doLogger.error("Error retrieving key from '#{inspect(cache_name)}'")[]end++ @spec with_lock!(atom(), any(), pos_integer(), (-> result)) :: result when result: any()+ def with_lock!(cache_name, key, timeout, fun) do+ ConCache.isolated(cache_name, key, timeout, fun)+ catch+ :exit, {:timeout, _} ->+ Logger.error("Timeout while executing with lock on key in '#{inspect(cache_name)}'")+ endenddiff --git a/lib/plausible/session/cache_store.ex b/lib/plausible/session/cache_store.exindex d9fa2f9761..51bbe858a3 100644--- a/lib/plausible/session/cache_store.ex+++ b/lib/plausible/session/cache_store.ex@@ -2,18 +2,35 @@ defmodule Plausible.Session.CacheStore dorequire Loggeralias Plausible.Session.WriteBuffer- def on_event(event, session_attributes, prev_user_id, buffer \\ WriteBuffer) do- found_session = find_session(event, event.user_id) || find_session(event, prev_user_id)+ @lock_timeout 500- if found_session do- updated_session = update_session(found_session, event)- buffer.insert([%{found_session | sign: -1}, %{updated_session | sign: 1}])- persist_session(updated_session)- else- new_session = new_session_from_event(event, session_attributes)- buffer.insert([new_session])- persist_session(new_session)- end+ @lock_telemetry_event [:plausible, :sessions, :cache, :lock]++ def lock_telemetry_event, do: @lock_telemetry_event++ def on_event(event, session_attributes, prev_user_id, buffer_insert \\ &WriteBuffer.insert/1) do+ lock_requested_at = System.monotonic_time()++ Plausible.Cache.Adapter.with_lock!(+ :sessions,+ {event.site_id, event.user_id},+ @lock_timeout,+ fn ->+ lock_duration = System.monotonic_time() - lock_requested_at+ :telemetry.execute(@lock_telemetry_event, %{duration: lock_duration}, %{})+ found_session = find_session(event, event.user_id) || find_session(event, prev_user_id)++ if found_session do+ updated_session = update_session(found_session, event)+ buffer_insert.([%{found_session | sign: -1}, %{updated_session | sign: 1}])+ persist_session(updated_session)+ else+ new_session = new_session_from_event(event, session_attributes)+ buffer_insert.([new_session])+ persist_session(new_session)+ end+ end+ )enddefp find_session(_domain, nil), do: nil@@ -34,7 +51,7 @@ defmodule Plausible.Session.CacheStore dodefp persist_session(session) dokey = {session.site_id, session.user_id}- Plausible.Cache.Adapter.put(:sessions, key, session)+ Plausible.Cache.Adapter.put(:sessions, key, session, dirty?: true)enddefp update_session(session, event) dodiff --git a/lib/plausible/telemetry/plausible_metrics.ex b/lib/plausible/telemetry/plausible_metrics.exindex 17e24e3371..b6f836b2f7 100644--- a/lib/plausible/telemetry/plausible_metrics.ex+++ b/lib/plausible/telemetry/plausible_metrics.ex@@ -57,6 +57,15 @@ defmodule Plausible.PromEx.Plugins.PlausibleMetrics domeasurement: :duration,tags: [:step]),+ distribution(+ metric_prefix ++ [:sessions, :cache, :register, :lock],+ event_name: Plausible.Session.CacheStore.lock_telemetry_event(),+ reporter_options: [+ buckets: [10, 50, 100, 250, 350, 500, 1000, 5000, 10_000, 100_000, 500_000]+ ],+ unit: {:native, :microsecond},+ measurement: :duration+ ),counter(metric_prefix ++ [:ingest, :events, :buffered, :total],event_name: Ingestion.Event.telemetry_event_buffered()diff --git a/test/plausible/session/cache_store_test.exs b/test/plausible/session/cache_store_test.exsindex 58fd5f32c5..b3423e84c1 100644--- a/test/plausible/session/cache_store_test.exs+++ b/test/plausible/session/cache_store_test.exs@@ -1,16 +1,257 @@defmodule Plausible.Session.CacheStoreTest douse Plausible.DataCase++ import ExUnit.CaptureLog+alias Plausible.Session.CacheStore- defmodule FakeBuffer do- def insert(sessions) do- send(self(), {WriteBuffer, :insert, [sessions]})+ setup do+ current_pid = self()++ buffer = fn sessions ->+ send(current_pid, {:buffer, :insert, [sessions]})+ {:ok, sessions}+ end++ slow_buffer = fn sessions ->+ Process.sleep(200)+ send(current_pid, {:slow_buffer, :insert, [sessions]}){:ok, sessions}end++ [buffer: buffer, slow_buffer: slow_buffer]end- setup do- [buffer: FakeBuffer]+ test "event processing is sequential within session", %{+ buffer: buffer,+ slow_buffer: slow_buffer,+ test: test+ } do+ telemetry_event = CacheStore.lock_telemetry_event()++ :telemetry.attach(+ "#{test}-telemetry-handler",+ telemetry_event,+ fn ^telemetry_event, %{duration: d}, _, _ when is_integer(d) ->+ send(self(), {:telemetry_handled, d})+ end,+ %{}+ )++ event1 = build(:event, name: "pageview")+ event2 = build(:event, name: "pageview", user_id: event1.user_id, site_id: event1.site_id)+ event3 = build(:event, name: "pageview", user_id: event1.user_id, site_id: event1.site_id)++ session_params = %{+ referrer: "ref",+ referrer_source: "refsource",+ utm_medium: "medium",+ utm_source: "source",+ utm_campaign: "campaign",+ utm_content: "content",+ utm_term: "term",+ browser: "browser",+ browser_version: "55",+ country_code: "EE",+ screen_size: "Desktop",+ operating_system: "Mac",+ operating_system_version: "11"+ }++ CacheStore.on_event(event1, session_params, nil, buffer)++ assert_receive({:buffer, :insert, [[session1]]})+ assert_receive({:telemetry_handled, duration})+ assert is_integer(duration)++ [event2, event3]+ |> Enum.map(fn e ->+ Task.async(fn ->+ CacheStore.on_event(e, session_params, nil, slow_buffer)+ end)+ end)+ |> Task.await_many()++ assert_receive({:slow_buffer, :insert, [[removed_session11, updated_session12]]})+ assert_receive({:slow_buffer, :insert, [[removed_session12, updated_session13]]})++ # Without isolation enforced in `CacheStore.on_event/4`,+ # event2 and event3 would both get executed in parallel+ # and treat event1 as event to be updated. This would result+ # in a following set of entries in Clickhouse sessions+ # table for _the same_ session:+ #+ # session_id | is_bounce | sign+ # (event1) 123 1 1+ # (event2) 123 1 -1+ # 123 0 1+ # (event3) 123 1 -1+ # 123 0 1+ #+ # Once collapsing merge tree table does collapsing, we'd end up with:+ #+ # session_id | is_bounce | sign+ # 123 0 1+ # 123 1 -1+ # 123 0 1+ #+ # This in turn led to sum(sign * is_bounce) < 0 which, after underflowed casting,+ # ended up with 2^32-(small n) for bounce_rate value.++ assert removed_session11 == %{session1 | sign: -1}+ assert updated_session12.sign == 1+ assert updated_session12.events == 2+ assert updated_session12.pageviews == 2+ assert removed_session12 == %{updated_session12 | sign: -1}+ assert updated_session13.sign == 1+ assert updated_session13.events == 3+ assert updated_session13.pageviews == 3+ end++ @tag :slow+ test "in case of lock kicking in, the slow event finishes processing", %{buffer: buffer} do+ current_pid = self()++ very_slow_buffer = fn sessions ->+ Process.sleep(1000)+ send(current_pid, {:very_slow_buffer, :insert, [sessions]})+ {:ok, sessions}+ end++ event1 = build(:event, name: "pageview")+ event2 = build(:event, name: "pageview", user_id: event1.user_id, site_id: event1.site_id)+ event3 = build(:event, name: "pageview", user_id: event1.user_id, site_id: event1.site_id)++ session_params = %{+ referrer: "ref",+ referrer_source: "refsource",+ utm_medium: "medium",+ utm_source: "source",+ utm_campaign: "campaign",+ utm_content: "content",+ utm_term: "term",+ browser: "browser",+ browser_version: "55",+ country_code: "EE",+ screen_size: "Desktop",+ operating_system: "Mac",+ operating_system_version: "11"+ }++ async1 =+ Task.async(fn ->+ CacheStore.on_event(event1, session_params, nil, very_slow_buffer)+ end)++ # Ensure next events are executed after processing event1 starts+ Process.sleep(100)++ async2 =+ Task.async(fn ->+ CacheStore.on_event(event2, session_params, nil, buffer)+ end)++ async3 =+ Task.async(fn ->+ CacheStore.on_event(event3, session_params, nil, buffer)+ end)++ capture_log(fn ->+ Task.await_many([async1, async2, async3])+ end) =~ "Timeout while executing with lock on key in ':sessions'"++ assert_receive({:very_slow_buffer, :insert, [[_session]]})+ refute_receive({:buffer, :insert, [[_updated_session]]})+ end++ @tag :slow+ test "lock on slow processing of one event does not affect unrelated events", %{buffer: buffer} do+ current_pid = self()++ very_slow_buffer = fn sessions ->+ Process.sleep(1000)+ send(current_pid, {:very_slow_buffer, :insert, [sessions]})+ {:ok, sessions}+ end++ event1 = build(:event, name: "pageview")+ event2 = build(:event, name: "pageview")+ event3 = build(:event, name: "pageview", user_id: event2.user_id, site_id: event2.site_id)++ session_params = %{+ referrer: "ref",+ referrer_source: "refsource",+ utm_medium: "medium",+ utm_source: "source",+ utm_campaign: "campaign",+ utm_content: "content",+ utm_term: "term",+ browser: "browser",+ browser_version: "55",+ country_code: "EE",+ screen_size: "Desktop",+ operating_system: "Mac",+ operating_system_version: "11"+ }++ async1 =+ Task.async(fn ->+ CacheStore.on_event(event1, session_params, nil, very_slow_buffer)+ end)++ # Ensure next events are executed after processing event1 starts+ Process.sleep(100)++ async2 =+ Task.async(fn ->+ CacheStore.on_event(event2, session_params, nil, buffer)+ end)++ Process.sleep(100)++ async3 =+ Task.async(fn ->+ CacheStore.on_event(event3, session_params, nil, buffer)+ end)++ Task.await_many([async1, async2, async3])++ assert_receive({:very_slow_buffer, :insert, [[_slow_session]]})+ assert_receive({:buffer, :insert, [[new_session1]]})+ assert_receive({:buffer, :insert, [[removed_session1, updated_session1]]})+ assert new_session1.sign == 1+ assert removed_session1.session_id == new_session1.session_id+ assert removed_session1.sign == -1+ assert updated_session1.session_id == removed_session1.session_id+ assert updated_session1.sign == 1+ end++ test "exploding event processing is passed through by locking mechanism" do+ crashing_buffer = fn _sessions ->+ raise "boom"+ end++ event = build(:event, name: "pageview")++ session_params = %{+ referrer: "ref",+ referrer_source: "refsource",+ utm_medium: "medium",+ utm_source: "source",+ utm_campaign: "campaign",+ utm_term: "term",+ utm_content: "content",+ browser: "browser",+ browser_version: "55",+ country_code: "EE",+ screen_size: "Desktop",+ operating_system: "Mac",+ operating_system_version: "11"+ }++ assert_raise RuntimeError, "boom", fn ->+ CacheStore.on_event(event, session_params, nil, crashing_buffer)+ endendtest "creates a session from an event", %{buffer: buffer} do@@ -39,7 +280,7 @@ defmodule Plausible.Session.CacheStoreTest doCacheStore.on_event(event, session_params, nil, buffer)- assert_receive({WriteBuffer, :insert, [sessions]})+ assert_receive({:buffer, :insert, [sessions]})assert [session] = sessionsassert session.hostname == event.hostname@@ -82,7 +323,7 @@ defmodule Plausible.Session.CacheStoreTest doCacheStore.on_event(event1, %{}, nil, buffer)CacheStore.on_event(event2, %{}, nil, buffer)- assert_receive({WriteBuffer, :insert, [[_negative_record, session]]})+ assert_receive({:buffer, :insert, [[_negative_record, session]]})assert session.is_bounce == falseassert session.duration == 10assert session.pageviews == 2@@ -286,7 +527,7 @@ defmodule Plausible.Session.CacheStoreTest doCacheStore.on_event(event1, %{}, nil, buffer)CacheStore.on_event(event2, %{}, nil, buffer)- assert_receive({WriteBuffer, :insert, [[_negative_record, session]]})+ assert_receive({:buffer, :insert, [[_negative_record, session]]})assert session.duration == 10end