Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Use new Session.CacheStore in favour of Session.Store (#1934)
Uku Taht3e5695408af7
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 5d3e0e8029..55d2e2ed8a 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -40,6 +40,7 @@ All notable changes to this project will be documented in this file.- Paginate /api/sites results and add a `View all` link to the site-switcher dropdown in the dashboard.- Remove the `+ Add Site` link to the site-switcher dropdown in the dashboard.- `DISABLE_REGISTRATIONS` configuration parameter can now accept `invite_only` to allow invited users to register an account while keeping regular registrations disabled plausible/analytics#1841+- New and improved Session tracking module for higher throughput and lower latency. [PR#1934](https://github.com/plausible/analytics#1934)## v1.4.1diff --git a/lib/plausible/application.ex b/lib/plausible/application.exindex f150f11a63..ecdf04e8c6 100644--- a/lib/plausible/application.ex+++ b/lib/plausible/application.ex@@ -13,7 +13,6 @@ defmodule Plausible.Application doPlausible.Session.Salts,Plausible.Event.WriteBuffer,Plausible.Session.WriteBuffer,- Plausible.Session.Store,ReferrerBlocklist,Supervisor.child_spec({Cachex, name: :user_agents, limit: 1000}, id: :cachex_user_agents),Supervisor.child_spec({Cachex, name: :sessions, limit: nil}, id: :cachex_sessions),diff --git a/lib/plausible/session/cache_store.ex b/lib/plausible/session/cache_store.exindex 940d119ce0..fe342a573b 100644--- a/lib/plausible/session/cache_store.ex+++ b/lib/plausible/session/cache_store.ex@@ -2,18 +2,17 @@ defmodule Plausible.Session.CacheStore dorequire Loggeralias Plausible.Session.WriteBuffer- def on_event(event, prev_user_id) do- found_session =- find_session(event.domain, event.user_id) || find_session(event.domain, prev_user_id)+ def on_event(event, prev_user_id, buffer \\ WriteBuffer) do+ found_session = find_session(event, event.user_id) || find_session(event, prev_user_id)session =if found_session doupdated_session = update_session(found_session, event)- WriteBuffer.insert([%{updated_session | sign: 1}, %{found_session | sign: -1}])+ buffer.insert([%{updated_session | sign: 1}, %{found_session | sign: -1}])persist_session(updated_session)elsenew_session = new_session_from_event(event)- WriteBuffer.insert([new_session])+ buffer.insert([new_session])persist_session(new_session)end@@ -22,10 +21,15 @@ defmodule Plausible.Session.CacheStore dodefp find_session(_domain, nil), do: nil- defp find_session(domain, user_id) do- case Cachex.get(:sessions, {domain, user_id}) do- {:ok, val} ->- val+ defp find_session(event, user_id) do+ case Cachex.get(:sessions, {event.domain, user_id}) do+ {:ok, nil} ->+ nil++ {:ok, session} ->+ if Timex.diff(event.timestamp, session.timestamp, :minutes) <= 30 do+ session+ end{:error, e} ->Sentry.capture_message("Cachex error", extra: %{error: e})@@ -118,7 +122,9 @@ defmodule Plausible.Session.CacheStore dobrowser: event.browser,browser_version: event.browser_version,timestamp: event.timestamp,- start: event.timestamp+ start: event.timestamp,+ "entry_meta.key": Map.get(event, :"meta.key"),+ "entry_meta.value": Map.get(event, :"meta.value")}endenddiff --git a/lib/plausible/session/store.ex b/lib/plausible/session/store.exdeleted file mode 100644index 25ab936bd9..0000000000--- a/lib/plausible/session/store.ex+++ /dev/null@@ -1,200 +0,0 @@-defmodule Plausible.Session.Store do- use GenServer- use Plausible.Repo- require Logger-- @garbage_collect_interval_milliseconds 60 * 1000-- def start_link(opts) do- GenServer.start_link(__MODULE__, opts, name: __MODULE__)- end-- def init(opts) do- buffer = Keyword.get(opts, :buffer, Plausible.Session.WriteBuffer)- timer = Process.send_after(self(), :garbage_collect, @garbage_collect_interval_milliseconds)-- {:ok, %{timer: timer, sessions: %{}, buffer: buffer}}- end-- def on_event(event, prev_user_id, pid \\ __MODULE__) do- GenServer.call(pid, {:on_event, event, prev_user_id})- end-- def handle_call(- {:on_event, event, prev_user_id},- _from,- %{sessions: sessions, buffer: buffer} = state- ) do- session_key = {event.domain, event.user_id}-- found_session =- sessions[session_key] || (prev_user_id && sessions[{event.domain, prev_user_id}])-- active = is_active?(found_session, event)-- updated_sessions =- cond do- found_session && active ->- new_session = update_session(found_session, event)- buffer.insert([%{new_session | sign: 1}, %{found_session | sign: -1}])- Map.put(sessions, session_key, new_session)-- found_session && !active ->- new_session = new_session_from_event(event)- buffer.insert([new_session])- Map.put(sessions, session_key, new_session)-- true ->- new_session = new_session_from_event(event)- buffer.insert([new_session])- Map.put(sessions, session_key, new_session)- end-- session_id = updated_sessions[session_key].session_id- {:reply, session_id, %{state | sessions: updated_sessions}}- end-- def reconcile_event(sessions, event) do- session_key = {event.domain, event.user_id}- found_session = sessions[session_key]- active = is_active?(found_session, event)-- updated_sessions =- cond do- found_session && active ->- new_session = update_session(found_session, event)- Map.put(sessions, session_key, new_session)-- found_session && !active ->- new_session = new_session_from_event(event)- Map.put(sessions, session_key, new_session)-- true ->- new_session = new_session_from_event(event)- Map.put(sessions, session_key, new_session)- end-- updated_sessions- end-- defp is_active?(session, event) do- session && Timex.diff(event.timestamp, session.timestamp, :second) < session_length_seconds()- end-- defp update_session(session, event) do- %{- session- | user_id: event.user_id,- timestamp: event.timestamp,- exit_page: event.pathname,- is_bounce: false,- duration: Timex.diff(event.timestamp, session.start, :second) |> abs,- pageviews:- if(event.name == "pageview", do: session.pageviews + 1, else: session.pageviews),- country_code:- if(session.country_code == "", do: event.country_code, else: session.country_code),- subdivision1_code:- if(session.subdivision1_code == "",- do: event.subdivision1_code,- else: session.subdivision1_code- ),- subdivision2_code:- if(session.subdivision2_code == "",- do: event.subdivision2_code,- else: session.subdivision2_code- ),- city_geoname_id:- if(session.city_geoname_id == 0,- do: event.city_geoname_id,- else: session.city_geoname_id- ),- operating_system:- if(session.operating_system == "",- do: event.operating_system,- else: session.operating_system- ),- operating_system_version:- if(session.operating_system_version == "",- do: event.operating_system_version,- else: session.operating_system_version- ),- browser: if(session.browser == "", do: event.browser, else: session.browser),- browser_version:- if(session.browser_version == "",- do: event.browser_version,- else: session.browser_version- ),- screen_size:- if(session.screen_size == "", do: event.screen_size, else: session.screen_size),- events: session.events + 1- }- end-- defp new_session_from_event(event) do- %Plausible.ClickhouseSession{- sign: 1,- session_id: Plausible.ClickhouseSession.random_uint64(),- hostname: event.hostname,- domain: event.domain,- user_id: event.user_id,- entry_page: event.pathname,- exit_page: event.pathname,- is_bounce: true,- duration: 0,- pageviews: if(event.name == "pageview", do: 1, else: 0),- events: 1,- referrer: event.referrer,- referrer_source: event.referrer_source,- utm_medium: event.utm_medium,- utm_source: event.utm_source,- utm_campaign: event.utm_campaign,- utm_content: event.utm_content,- utm_term: event.utm_term,- country_code: event.country_code,- subdivision1_code: event.subdivision1_code,- subdivision2_code: event.subdivision2_code,- city_geoname_id: event.city_geoname_id,- screen_size: event.screen_size,- operating_system: event.operating_system,- operating_system_version: event.operating_system_version,- browser: event.browser,- browser_version: event.browser_version,- timestamp: event.timestamp,- start: event.timestamp,- "entry_meta.key": Map.get(event, :"meta.key"),- "entry_meta.value": Map.get(event, :"meta.value")- }- end-- def handle_info(:garbage_collect, state) do- Logger.debug("Session store collecting garbage")-- now = Timex.now()-- new_sessions =- Enum.reduce(state[:sessions], %{}, fn {key, session}, acc ->- if Timex.diff(now, session.timestamp, :second) <= forget_session_after() do- Map.put(acc, key, session)- else- # forget the session- acc- end- end)-- Process.cancel_timer(state[:timer])-- new_timer =- Process.send_after(self(), :garbage_collect, @garbage_collect_interval_milliseconds)-- Logger.debug(fn ->- n_old = Enum.count(state[:sessions])- n_new = Enum.count(new_sessions)- "Removed #{n_old - n_new} sessions from store"- end)-- {:noreply, %{state | sessions: new_sessions, timer: new_timer}}- end-- defp session_length_seconds(), do: Application.get_env(:plausible, :session_length_minutes) * 60- defp forget_session_after(), do: session_length_seconds() * 2-enddiff --git a/lib/plausible/session/write_buffer.ex b/lib/plausible/session/write_buffer.exindex 12f2664c5a..563b6d7a82 100644--- a/lib/plausible/session/write_buffer.ex+++ b/lib/plausible/session/write_buffer.ex@@ -17,13 +17,18 @@ defmodule Plausible.Session.WriteBuffer do{:ok, sessions}end+ def flush() do+ GenServer.call(__MODULE__, :flush, :infinity)+ :ok+ end+def handle_cast({:insert, sessions}, %{buffer: buffer} = state) donew_buffer = sessions ++ bufferif length(new_buffer) >= max_buffer_size() doLogger.info("Buffer full, flushing to disk")Process.cancel_timer(state[:timer])- flush(new_buffer)+ do_flush(new_buffer)new_timer = Process.send_after(self(), :tick, flush_interval_ms()){:noreply, %{buffer: [], timer: new_timer}}else@@ -32,17 +37,24 @@ defmodule Plausible.Session.WriteBuffer doenddef handle_info(:tick, %{buffer: buffer}) do- flush(buffer)+ do_flush(buffer)timer = Process.send_after(self(), :tick, flush_interval_ms()){:noreply, %{buffer: [], timer: timer}}end+ def handle_call(:flush, _from, %{buffer: buffer} = state) do+ Process.cancel_timer(state[:timer])+ do_flush(buffer)+ new_timer = Process.send_after(self(), :tick, flush_interval_ms())+ {:reply, nil, %{buffer: [], timer: new_timer}}+ end+def terminate(_reason, %{buffer: buffer}) doLogger.info("Flushing session buffer before shutdown...")- flush(buffer)+ do_flush(buffer)end- defp flush(buffer) do+ defp do_flush(buffer) docase buffer do[] ->nildiff --git a/lib/plausible_web/controllers/api/external_controller.ex b/lib/plausible_web/controllers/api/external_controller.exindex c01be7ab9c..b1d1970e71 100644--- a/lib/plausible_web/controllers/api/external_controller.ex+++ b/lib/plausible_web/controllers/api/external_controller.ex@@ -178,14 +178,8 @@ defmodule PlausibleWeb.Api.ExternalController doevent = Ecto.Changeset.apply_changes(changeset)session_id =- if FunWithFlags.enabled?(:cache_store, for: "domain:" <> domain) do- Tracer.with_span "cache_store_event" do- Plausible.Session.CacheStore.on_event(event, previous_user_id)- end- else- Tracer.with_span "store_event" do- Plausible.Session.Store.on_event(event, previous_user_id)- end+ Tracer.with_span "cache_store_event" do+ Plausible.Session.CacheStore.on_event(event, previous_user_id)endeventdiff --git a/test/plausible/session/store_test.exs b/test/plausible/session/cache_store_test.exssimilarity index 88%rename from test/plausible/session/store_test.exsrename to test/plausible/session/cache_store_test.exsindex 6738e1454e..6f14c52064 100644--- a/test/plausible/session/store_test.exs+++ b/test/plausible/session/cache_store_test.exs@@ -1,18 +1,17 @@-defmodule Plausible.Session.StoreTest do+defmodule Plausible.Session.CacheStoreTest douse Plausible.DataCaseimport Double- alias Plausible.Session.{Store, WriteBuffer}+ alias Plausible.Session.{CacheStore, WriteBuffer}setup dobuffer =WriteBuffer|> stub(:insert, fn _sessions -> nil end)- {:ok, store} = GenServer.start_link(Store, buffer: buffer)- [store: store, buffer: buffer]+ [buffer: buffer]end- test "creates a session from an event", %{store: store} do+ test "creates a session from an event", %{buffer: buffer} doevent =build(:event,name: "pageview",@@ -33,7 +32,7 @@ defmodule Plausible.Session.StoreTest do"meta.value": ["true", "false"])- Store.on_event(event, nil, store)+ CacheStore.on_event(event, nil, buffer)assert_receive({WriteBuffer, :insert, [sessions]})assert [session] = sessions@@ -65,7 +64,7 @@ defmodule Plausible.Session.StoreTest do# assert Map.get(session, :"entry.meta.value") == ["true", "false"]end- test "updates a session", %{store: store} do+ test "updates a session", %{buffer: buffer} dotimestamp = Timex.now()event1 = build(:event, name: "pageview", timestamp: timestamp |> Timex.shift(seconds: -10))@@ -86,8 +85,8 @@ defmodule Plausible.Session.StoreTest dobrowser_version: "10")- Store.on_event(event1, nil, store)- Store.on_event(event2, nil, store)+ CacheStore.on_event(event1, nil, buffer)+ CacheStore.on_event(event2, nil, buffer)assert_receive({WriteBuffer, :insert, [[session, _negative_record]]})assert session.is_bounce == falseassert session.duration == 10@@ -104,7 +103,7 @@ defmodule Plausible.Session.StoreTest doassert session.screen_size == "Desktop"end- test "calculates duration correctly for out-of-order events", %{store: store} do+ test "calculates duration correctly for out-of-order events", %{buffer: buffer} dotimestamp = Timex.now()event1 = build(:event, name: "pageview", timestamp: timestamp |> Timex.shift(seconds: 10))@@ -116,8 +115,8 @@ defmodule Plausible.Session.StoreTest dotimestamp: timestamp)- Store.on_event(event1, nil, store)- Store.on_event(event2, nil, store)+ CacheStore.on_event(event1, nil, buffer)+ CacheStore.on_event(event2, nil, buffer)assert_receive({WriteBuffer, :insert, [[session, _negative_record]]})assert session.duration == 10diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exsindex 5e0d021597..8d4eef0d06 100644--- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs@@ -645,10 +645,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest dopathname: "/page2",user_id: @user_id,timestamp: ~N[2021-01-01 00:15:00]- )- ])-- populate_stats(site, [+ ),build(:pageview,pathname: "/page2",user_id: @user_id,diff --git a/test/support/test_utils.ex b/test/support/test_utils.exindex bb9076597f..350e1398b9 100644--- a/test/support/test_utils.ex+++ b/test/support/test_utils.ex@@ -120,23 +120,17 @@ defmodule Plausible.TestUtils dodefp populate_native_stats(events) dosessions =Enum.reduce(events, %{}, fn event, sessions ->- Plausible.Session.Store.reconcile_event(sessions, event)+ session_id = Plausible.Session.CacheStore.on_event(event, nil)+ Map.put(sessions, {event.domain, event.user_id}, session_id)end)- events =- Enum.map(events, fn event ->- Map.put(event, :session_id, sessions[{event.domain, event.user_id}].session_id)- end)-- Plausible.ClickhouseRepo.insert_all(- Plausible.ClickhouseEvent,- Enum.map(events, &schema_to_map/1)- )+ Enum.each(events, fn event ->+ event = Map.put(event, :session_id, sessions[{event.domain, event.user_id}])+ Plausible.Event.WriteBuffer.insert(event)+ end)- Plausible.ClickhouseRepo.insert_all(- Plausible.ClickhouseSession,- Enum.map(Map.values(sessions), &schema_to_map/1)- )+ Plausible.Session.WriteBuffer.flush()+ Plausible.Event.WriteBuffer.flush()enddefp populate_imported_stats(events) do@@ -149,9 +143,4 @@ defmodule Plausible.TestUtils do|> Timex.shift(shifts)|> NaiveDateTime.truncate(:second)end-- defp schema_to_map(schema) do- Map.from_struct(schema)- |> Map.delete(:__meta__)- endend