Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Sessions transfer (#5229)
ruslandoga9bfb1992d96b
diff --git a/config/runtime.exs b/config/runtime.exsindex e6428b43b1..3052d4f3e2 100644--- a/config/runtime.exs+++ b/config/runtime.exs@@ -238,6 +238,13 @@ persistent_cache_dir = get_var_from_path_or_env(config_dir, "PERSISTENT_CACHE_DIdata_dir = data_dir || persistent_cache_dir || System.get_env("DEFAULT_DATA_DIR")persistent_cache_dir = persistent_cache_dir || data_dir+session_transfer_dir =+ if get_bool_from_path_or_env(config_dir, "ENABLE_SESSION_TRANSFER", config_env() == :prod) do+ if persistent_cache_dir do+ Path.join(persistent_cache_dir, "sessions")+ end+ end+enable_email_verification =get_bool_from_path_or_env(config_dir, "ENABLE_EMAIL_VERIFICATION", false)@@ -305,7 +312,8 @@ config :plausible,custom_script_name: custom_script_name,log_failed_login_attempts: log_failed_login_attempts,license_key: license_key,- data_dir: data_dir+ data_dir: data_dir,+ session_transfer_dir: session_transfer_dirconfig :plausible, :selfhost,enable_email_verification: enable_email_verification,diff --git a/lib/plausible/application.ex b/lib/plausible/application.exindex 751521a0cb..1afd6a826e 100644--- a/lib/plausible/application.ex+++ b/lib/plausible/application.ex@@ -43,6 +43,8 @@ defmodule Plausible.Application don_lock_partitions: 1,ets_options: [read_concurrency: true, write_concurrency: true]),+ {Plausible.Session.Transfer,+ base_path: Application.get_env(:plausible, :session_transfer_dir)},warmed_cache(Plausible.Site.Cache,adapter_opts: [n_lock_partitions: 1,diff --git a/lib/plausible/cache/adapter.ex b/lib/plausible/cache/adapter.exindex 644276004a..fb0f62f9fd 100644--- a/lib/plausible/cache/adapter.ex+++ b/lib/plausible/cache/adapter.ex@@ -172,4 +172,8 @@ defmodule Plausible.Cache.Adapter dodefp partitions(cache_name) doApplication.get_env(:plausible, __MODULE__)[cache_name][:partitions] || 1end++ def cache2list(full_cache_name) do+ :ets.tab2list(ConCache.ets(full_cache_name))+ endenddiff --git a/lib/plausible/session/transfer.ex b/lib/plausible/session/transfer.exnew file mode 100644index 0000000000..4de7ddff7b--- /dev/null+++ b/lib/plausible/session/transfer.ex@@ -0,0 +1,185 @@+defmodule Plausible.Session.Transfer do+ @moduledoc """+ Cross-deployment transfer for `:sessions` cache.++ It works by establishing a client-server architecture where:+ - The "replica" one-time task retrieves `:sessions` data from other OS processes via Unix domain sockets+ - The "primary" server process responds to requests for `:sessions` data via Unix domain sockets+ - The "alive" process waits on shutdown for at least one replica, for 15 seconds+ """++ @behaviour Supervisor++ require Logger+ alias Plausible.Session.Transfer.{TinySock, Alive}+ alias Plausible.{Cache, ClickhouseSessionV2, Session}++ @cmd_list_cache_names :list+ @cmd_dump_cache :get+ @cmd_takeover_done :done++ def telemetry_event, do: [:plausible, :sessions, :transfer]++ @doc """+ Starts the `:sessions` transfer supervisor.++ Options:+ - `:name` - the name of the supervisor (default: `Plausible.Session.Transfer`)+ - `:base_path` - the base path for the Unix domain sockets (required)+ """+ def start_link(opts) do+ name = Keyword.get(opts, :name, __MODULE__)+ base_path = Keyword.fetch!(opts, :base_path)+ Supervisor.start_link(__MODULE__, base_path, name: name)+ end++ @impl true+ def init(nil) do+ Logger.info(+ "Session transfer: ignoring, no socket base path configured (make sure ENABLE_SESSION_TRANSFER/PERSISTENT_CACHE_DIR are set)"+ )++ :ignore+ end++ def init(base_path) do+ File.mkdir_p!(base_path)++ replica =+ Supervisor.child_spec(+ {Task, fn -> init_takeover(base_path) end},+ id: :transfer_replica+ )++ given_counter = :counters.new(1, [])+ parent = self()++ primary =+ {TinySock,+ base_path: base_path,+ handler: fn message -> handle_replica(message, parent, given_counter) end}++ alive =+ Supervisor.child_spec(+ {Alive,+ _until = fn ->+ result = :counters.get(given_counter, 1) > 0++ Logger.info(+ "Session transfer delayed shut down. Checking if session takeover happened?: #{result}"+ )++ result+ end},+ shutdown: :timer.seconds(15)+ )++ Logger.info("Session transfer init: #{base_path}")+ Supervisor.init([replica, primary, alive], strategy: :one_for_one)+ end++ @doc """+ Returns `true` if the transfer has been attempted (successfully or not).+ Returns `false` if the transfer is still in progress.+ """+ def attempted?(transfer_sup \\ __MODULE__) do+ result = not replica_alive?(transfer_sup)+ Logger.info("Session transfer attempted?: #{result}")+ result+ end++ @doc """+ Returns the child specification for the `:sessions` transfer supervisor.+ See `start_link/1` for options.+ """+ def child_spec(opts) do+ %{+ id: __MODULE__,+ start: {__MODULE__, :start_link, [opts]},+ type: :supervisor,+ restart: :temporary+ }+ end++ defp handle_replica(request, parent, given_counter) do+ Logger.info("Session transfer message received at #{node()}: #{inspect(request, limit: 10)}")++ case request do+ {@cmd_list_cache_names, session_version} ->+ if session_version == session_version() and attempted?(parent) do+ Cache.Adapter.get_names(:sessions)+ else+ []+ end++ {@cmd_dump_cache, cache} ->+ Cache.Adapter.cache2list(cache)++ @cmd_takeover_done ->+ :counters.add(given_counter, 1, 1)+ end+ end++ defp init_takeover(base_path) do+ started = System.monotonic_time()++ base_path+ |> TinySock.list!()+ |> Enum.sort_by(&file_stat_ctime/1, :asc)+ |> Enum.each(&request_takeover/1)++ :telemetry.execute(telemetry_event(), %{duration: System.monotonic_time() - started})+ end++ defp request_takeover(sock) do+ Logger.info("Session transfer: requesting takeover at #{node()}")++ with {:ok, names} <- TinySock.call(sock, {@cmd_list_cache_names, session_version()}) do+ tasks = Enum.map(names, fn name -> Task.async(fn -> takeover_cache(sock, name) end) end)+ Task.await_many(tasks, :timer.seconds(10))+ end+ after+ Logger.info("Session transfer: marking takeover as done at #{node()}")+ TinySock.call(sock, @cmd_takeover_done)+ end++ defp takeover_cache(sock, cache) do+ Logger.info("Session transfer: requesting cache #{cache} dump at #{node()}")++ with {:ok, records} <- TinySock.call(sock, {@cmd_dump_cache, cache}) do+ Enum.each(records, fn record ->+ {key, %ClickhouseSessionV2{} = session} = record+ Cache.Adapter.put(:sessions, key, session)+ end)++ Logger.info("Session transfer: restored cache #{cache} at #{node()}")+ end+ end++ defp file_stat_ctime(path) do+ case File.stat(path) do+ {:ok, stat} -> stat.ctime+ {:error, _} -> nil+ end+ end++ defp session_version do+ [+ ClickhouseSessionV2.module_info(:md5),+ Cache.Adapter.module_info(:md5),+ Session.CacheStore.module_info(:md5),+ Session.Transfer.module_info(:md5)+ ]+ end++ defp replica_alive?(transfer_sup) do+ children = Supervisor.which_children(transfer_sup)++ replica =+ Enum.find_value(children, fn {id, pid, _, _} -> id == :transfer_replica && pid end)++ is_pid(replica) and Process.alive?(replica)+ catch+ :exit, {:noproc, _} -> false+ end+enddiff --git a/lib/plausible/session/transfer/alive.ex b/lib/plausible/session/transfer/alive.exnew file mode 100644index 0000000000..bfaab0ccdb--- /dev/null+++ b/lib/plausible/session/transfer/alive.ex@@ -0,0 +1,31 @@+defmodule Plausible.Session.Transfer.Alive do+ @moduledoc false+ use GenServer++ @spec start_link((-> boolean)) :: GenServer.on_start()+ def start_link(until) do+ GenServer.start_link(__MODULE__, until)+ end++ @impl true+ def init(until) when is_function(until, 0) do+ Process.flag(:trap_exit, true)+ {:ok, until}+ end++ @impl true+ def terminate(_reason, until) do+ loop(until)+ end++ defp loop(until) do+ case until.() do+ true ->+ :ok++ false ->+ :timer.sleep(500)+ loop(until)+ end+ end+enddiff --git a/lib/plausible/session/transfer/tinysock.ex b/lib/plausible/session/transfer/tinysock.exnew file mode 100644index 0000000000..f409895e05--- /dev/null+++ b/lib/plausible/session/transfer/tinysock.ex@@ -0,0 +1,170 @@+defmodule Plausible.Session.Transfer.TinySock do+ @moduledoc false+ use GenServer+ require Logger++ @tag_data "tinysock"+ @tag_size byte_size(@tag_data)++ def start_link(opts) do+ base_path = Keyword.fetch!(opts, :base_path)+ handler = Keyword.fetch!(opts, :handler)+ GenServer.start_link(__MODULE__, {base_path, handler})+ end++ @impl true+ def init({base_path, handler}) do+ File.mkdir_p!(base_path)+ socket = sock_listen_or_retry!(base_path)++ Process.flag(:trap_exit, true)+ state = %{socket: socket, handler: handler}+ for _ <- 1..10, do: spawn_acceptor(state)++ {:ok, state}+ end++ @spec list!(Path.t()) :: [Path.t()]+ def list!(base_path) do+ for @tag_data <> _rand = name <- File.ls!(base_path) do+ Path.join(base_path, name)+ end+ end++ @spec call(Path.t(), term, timeout) :: {:ok, reply :: term} | {:error, :timeout | :inet.posix()}+ def call(sock_path, message, timeout \\ :timer.seconds(5)) do+ with {:ok, socket} <- sock_connect_or_rm(sock_path, timeout) do+ try do+ with :ok <- sock_send(socket, :erlang.term_to_iovec(message)) do+ sock_recv(socket, timeout)+ end+ after+ sock_shut_and_close(socket)+ end+ end+ end++ @impl true+ def handle_cast(:accepted, state) do+ spawn_acceptor(state)+ {:noreply, state}+ end++ @impl true+ def handle_info({:EXIT, pid, reason}, state) do+ case reason do+ :normal ->+ {:noreply, state}++ :emfile ->+ Logger.error("tinysock ran out of file descriptors, stopping")+ {:stop, reason, state}++ reason ->+ log = "tinysock request handler #{inspect(pid)} terminating\n" <> format_error(reason)+ Logger.error(log, crash_reason: reason)+ {:noreply, state}+ end+ end++ defp format_error(reason) do+ case reason do+ {e, stacktrace} when is_list(stacktrace) -> Exception.format(:error, e, stacktrace)+ _other -> Exception.format(:exit, reason)+ end+ end++ @impl true+ def terminate(reason, %{socket: socket}) do+ with {:ok, {:local, path}} <- :inet.sockname(socket), do: File.rm(path)+ reason+ end++ defp spawn_acceptor(%{socket: socket, handler: handler}) do+ :proc_lib.spawn_link(__MODULE__, :accept, [_parent = self(), socket, handler])+ end++ @doc false+ def accept(parent, listen_socket, handler) do+ case :gen_tcp.accept(listen_socket, :infinity) do+ {:ok, socket} ->+ GenServer.cast(parent, :accepted)+ handle_message(socket, handler)++ {:error, reason} ->+ exit(reason)+ end+ end++ defp handle_message(socket, handler) do+ {:ok, message} = sock_recv(socket, _timeout = :timer.seconds(5))+ sock_send(socket, :erlang.term_to_iovec(handler.(message)))+ after+ sock_shut_and_close(socket)+ end++ @listen_opts mode: :binary, active: false, backlog: 1024+ @connect_opts mode: :binary, active: false++ defp sock_listen_or_retry!(base_path) do+ sock_name = @tag_data <> Base.url_encode64(:crypto.strong_rand_bytes(4), padding: false)+ sock_path = Path.join(base_path, sock_name)++ case :gen_tcp.listen(0, [{:ifaddr, {:local, sock_path}} | @listen_opts]) do+ {:ok, socket} -> socket+ {:error, :eaddrinuse} -> sock_listen_or_retry!(base_path)+ {:error, reason} -> raise File.Error, path: sock_path, reason: reason, action: "bind"+ end+ end++ defp sock_connect_or_rm(sock_path, timeout) do+ case :gen_tcp.connect({:local, sock_path}, 0, @connect_opts, timeout) do+ {:ok, socket} ->+ {:ok, socket}++ {:error, reason} = error ->+ if reason != :timeout do+ # removes stale socket file+ # possible - but unlikely - race condition+ File.rm(sock_path)+ end++ error+ end+ end++ @dialyzer :no_improper_lists+ defp sock_send(socket, data) do+ :gen_tcp.send(socket, [<<@tag_data, IO.iodata_length(data)::64-little>> | data])+ end++ defp sock_recv(socket, timeout) do+ with {:ok, <<@tag_data, size::64-little>>} <- :gen_tcp.recv(socket, @tag_size + 8, timeout),+ {:ok, binary} <- sock_recv_continue(socket, size, timeout, []) do+ try do+ {:ok, :erlang.binary_to_term(binary, [:safe])}+ rescue+ e -> {:error, e}+ end+ end+ end++ @five_mb 5 * 1024 * 1024++ # for larger messages (>64MB) we need to read in chunks or we get {:error, :enomem}+ defp sock_recv_continue(socket, size, timeout, acc) do+ with {:ok, data} <- :gen_tcp.recv(socket, min(size, @five_mb), timeout) do+ acc = [acc | data]++ case size - byte_size(data) do+ 0 -> {:ok, IO.iodata_to_binary(acc)}+ left -> sock_recv_continue(socket, left, timeout, acc)+ end+ end+ end++ defp sock_shut_and_close(socket) do+ :gen_tcp.shutdown(socket, :read_write)+ :gen_tcp.close(socket)+ end+enddiff --git a/lib/plausible/telemetry/plausible_metrics.ex b/lib/plausible/telemetry/plausible_metrics.exindex 1cad45895c..d7c7d5a3dd 100644--- a/lib/plausible/telemetry/plausible_metrics.ex+++ b/lib/plausible/telemetry/plausible_metrics.ex@@ -78,6 +78,15 @@ defmodule Plausible.PromEx.Plugins.PlausibleMetrics docounter(metric_prefix ++ [:ingest, :user_agent_parse, :timeout, :total],event_name: Ingestion.Event.telemetry_ua_parse_timeout()+ ),+ distribution(+ metric_prefix ++ [:sessions, :transfer, :duration],+ event_name: Plausible.Session.Transfer.telemetry_event(),+ reporter_options: [+ buckets: [100, 250, 500, 750, 1000, 2500, 5000, 7500, 10_000]+ ],+ unit: {:native, :millisecond},+ measurement: :duration)])diff --git a/lib/plausible_web/controllers/api/system_controller.ex b/lib/plausible_web/controllers/api/system_controller.exindex 4073d42ba7..95482e0d52 100644--- a/lib/plausible_web/controllers/api/system_controller.ex+++ b/lib/plausible_web/controllers/api/system_controller.ex@@ -59,9 +59,16 @@ defmodule PlausibleWeb.Api.SystemController do"ok"end+ sessions_health =+ if Plausible.Session.Transfer.attempted?() do+ "ok"+ else+ "waiting"+ end+status =- case {postgres_health, clickhouse_health, cache_health} do- {"ok", "ok", "ok"} -> 200+ case {postgres_health, clickhouse_health, cache_health, sessions_health} do+ {"ok", "ok", "ok", "ok"} -> 200_ -> 500end@@ -69,7 +76,8 @@ defmodule PlausibleWeb.Api.SystemController do|> json(%{postgres: postgres_health,clickhouse: clickhouse_health,- sites_cache: cache_health+ sites_cache: cache_health,+ sessions: sessions_health})endenddiff --git a/test/plausible/config_test.exs b/test/plausible/config_test.exsindex 46eaa2c8d1..de11bb66ec 100644--- a/test/plausible/config_test.exs+++ b/test/plausible/config_test.exs@@ -407,6 +407,8 @@ defmodule Plausible.ConfigTest doassert get_in(config, [:plausible, Plausible.Geo, :cache_dir]) == "/var/lib/plausible"# tzdata (timezones cache)assert get_in(config, [:tzdata, :data_dir]) == "/var/lib/plausible/tzdata_data"+ # session transfer+ assert get_in(config, [:plausible, :session_transfer_dir]) == "/var/lib/plausible/sessions"endtest "with only DATA_DIR set", %{env: env} do@@ -418,6 +420,8 @@ defmodule Plausible.ConfigTest doassert get_in(config, [:plausible, Plausible.Geo, :cache_dir]) == "/data"# tzdata (timezones cache)assert get_in(config, [:tzdata, :data_dir]) == "/data/tzdata_data"+ # session transfer+ assert get_in(config, [:plausible, :session_transfer_dir]) == "/data/sessions"endtest "with only PERSISTENT_CACHE_DIR set", %{env: env} do@@ -429,6 +433,8 @@ defmodule Plausible.ConfigTest doassert get_in(config, [:plausible, Plausible.Geo, :cache_dir]) == "/cache"# tzdata (timezones cache)assert get_in(config, [:tzdata, :data_dir]) == "/cache/tzdata_data"+ # session transfer+ assert get_in(config, [:plausible, :session_transfer_dir]) == "/cache/sessions"endtest "with both DATA_DIR and PERSISTENT_CACHE_DIR set", %{env: env} do@@ -440,6 +446,8 @@ defmodule Plausible.ConfigTest doassert get_in(config, [:plausible, Plausible.Geo, :cache_dir]) == "/cache"# tzdata (timezones cache)assert get_in(config, [:tzdata, :data_dir]) == "/cache/tzdata_data"+ # session transfer+ assert get_in(config, [:plausible, :session_transfer_dir]) == "/cache/sessions"endenddiff --git a/test/plausible/session/transfer_test.exs b/test/plausible/session/transfer_test.exsnew file mode 100644index 0000000000..b10084ade6--- /dev/null+++ b/test/plausible/session/transfer_test.exs@@ -0,0 +1,107 @@+defmodule Plausible.Session.TransferTest do+ use ExUnit.Case+ import Plausible.Factory+ import Plausible.TestUtils, only: [tmp_dir: 0]++ @tag :slow+ test "it works" do+ tmp_dir = tmp_dir()++ old = start_another_plausible(tmp_dir)+ await_transfer(old)++ Enum.each(1..250, fn _ -> process_event(old, build(:event, name: "pageview")) end)++ new = start_another_plausible(tmp_dir)+ await_transfer(new)++ assert all_sessions_sorted(new) == all_sessions_sorted(old)+ end++ defp start_another_plausible(tmp_dir) do+ {:ok, pid, _node} = :peer.start_link(%{connection: {{127, 0, 0, 1}, 0}})+ add_code_paths(pid)+ transfer_configuration(pid)+ :ok = :peer.call(pid, Application, :put_env, [:plausible, :session_transfer_dir, tmp_dir])+ ensure_applications_started(pid)+ pid+ end++ defp add_code_paths(pid) do+ :ok = :peer.call(pid, :code, :add_paths, [:code.get_path()])+ end++ defp transfer_configuration(pid) do+ for {app_name, _, _} <- Application.loaded_applications() do+ for {key, val} <- Application.get_all_env(app_name) do+ :ok = :peer.call(pid, Application, :put_env, [app_name, key, val])+ end+ end+ end++ defp ensure_applications_started(pid) do+ {:ok, _apps} = :peer.call(pid, Application, :ensure_all_started, [:mix])+ :ok = :peer.call(pid, Mix, :env, [Mix.env()])++ for {app_name, _, _} <- Application.loaded_applications(), app_name != :dialyxir do+ {:ok, _apps} = :peer.call(pid, Application, :ensure_all_started, [app_name])+ end+ end++ @session_params %{+ referrer: "ref",+ referrer_source: "refsource",+ utm_medium: "medium",+ utm_source: "source",+ utm_campaign: "campaign",+ utm_content: "content",+ utm_term: "term",+ browser: "browser",+ browser_version: "55",+ country_code: "EE",+ screen_size: "Desktop",+ operating_system: "Mac",+ operating_system_version: "11"+ }++ defp process_event(pid, event) do+ :peer.call(pid, Plausible.Session.CacheStore, :on_event, [+ event,+ @session_params,+ _prev_user_id = nil,+ [buffer_insert: &Function.identity/1]+ ])+ end++ defp all_sessions_sorted(pid) do+ cache_names = :peer.call(pid, Plausible.Cache.Adapter, :get_names, [:sessions])++ records =+ Enum.flat_map(cache_names, fn cache_name ->+ tab = :peer.call(pid, ConCache, :ets, [cache_name])+ :peer.call(pid, :ets, :tab2list, [tab])+ end)++ Enum.sort_by(records, fn {key, _} -> key end)+ end++ defp await_transfer(pid, timeout \\ :timer.seconds(1)) do+ test = self()++ spawn_link(fn ->+ await_loop(fn -> :peer.call(pid, Plausible.Session.Transfer, :attempted?, []) end)+ send(test, :took)+ end)++ assert_receive :took, timeout+ end++ defp await_loop(f) do+ :timer.sleep(100)++ case f.() do+ true -> :done+ false -> await_loop(f)+ end+ end+enddiff --git a/test/support/test_utils.ex b/test/support/test_utils.exindex cc004235d4..681e49636a 100644--- a/test/support/test_utils.ex+++ b/test/support/test_utils.ex@@ -333,4 +333,16 @@ defmodule Plausible.TestUtils do|> Application.fetch_env!(PlausibleWeb.Endpoint)|> Keyword.fetch!(:secret_key_base)end++ # normal `@tag :tmp_dir` might not work in Plausible.Session.Transfer tests+ # if the path is too long for unix domain sockets (>104)+ # this one makes paths a bit shorter+ def tmp_dir do+ name = "plausible-#{System.unique_integer([:positive])}"+ tmp_dir = Path.join(System.tmp_dir!(), name)+ File.rm_rf!(tmp_dir)+ File.mkdir_p!(tmp_dir)+ ExUnit.Callbacks.on_exit(fn -> File.rm_rf!(tmp_dir) end)+ tmp_dir+ endend