Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Sessions transfer (#5229)

ruslandoga9bfb1992d96b
diff --git a/config/runtime.exs b/config/runtime.exs
index e6428b43b1..3052d4f3e2 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -238,6 +238,13 @@ persistent_cache_dir = get_var_from_path_or_env(config_dir, "PERSISTENT_CACHE_DI
data_dir = data_dir || persistent_cache_dir || System.get_env("DEFAULT_DATA_DIR")
persistent_cache_dir = persistent_cache_dir || data_dir
+session_transfer_dir =
+ if get_bool_from_path_or_env(config_dir, "ENABLE_SESSION_TRANSFER", config_env() == :prod) do
+ if persistent_cache_dir do
+ Path.join(persistent_cache_dir, "sessions")
+ end
+ end
+
enable_email_verification =
get_bool_from_path_or_env(config_dir, "ENABLE_EMAIL_VERIFICATION", false)
@@ -305,7 +312,8 @@ config :plausible,
custom_script_name: custom_script_name,
log_failed_login_attempts: log_failed_login_attempts,
license_key: license_key,
- data_dir: data_dir
+ data_dir: data_dir,
+ session_transfer_dir: session_transfer_dir
config :plausible, :selfhost,
enable_email_verification: enable_email_verification,
diff --git a/lib/plausible/application.ex b/lib/plausible/application.ex
index 751521a0cb..1afd6a826e 100644
--- a/lib/plausible/application.ex
+++ b/lib/plausible/application.ex
@@ -43,6 +43,8 @@ defmodule Plausible.Application do
n_lock_partitions: 1,
ets_options: [read_concurrency: true, write_concurrency: true]
),
+ {Plausible.Session.Transfer,
+ base_path: Application.get_env(:plausible, :session_transfer_dir)},
warmed_cache(Plausible.Site.Cache,
adapter_opts: [
n_lock_partitions: 1,
diff --git a/lib/plausible/cache/adapter.ex b/lib/plausible/cache/adapter.ex
index 644276004a..fb0f62f9fd 100644
--- a/lib/plausible/cache/adapter.ex
+++ b/lib/plausible/cache/adapter.ex
@@ -172,4 +172,8 @@ defmodule Plausible.Cache.Adapter do
defp partitions(cache_name) do
Application.get_env(:plausible, __MODULE__)[cache_name][:partitions] || 1
end
+
+ def cache2list(full_cache_name) do
+ :ets.tab2list(ConCache.ets(full_cache_name))
+ end
end
diff --git a/lib/plausible/session/transfer.ex b/lib/plausible/session/transfer.ex
new file mode 100644
index 0000000000..4de7ddff7b
--- /dev/null
+++ b/lib/plausible/session/transfer.ex
@@ -0,0 +1,185 @@
+defmodule Plausible.Session.Transfer do
+ @moduledoc """
+ Cross-deployment transfer for `:sessions` cache.
+
+ It works by establishing a client-server architecture where:
+ - The "replica" one-time task retrieves `:sessions` data from other OS processes via Unix domain sockets
+ - The "primary" server process responds to requests for `:sessions` data via Unix domain sockets
+ - The "alive" process waits on shutdown for at least one replica, for 15 seconds
+ """
+
+ @behaviour Supervisor
+
+ require Logger
+ alias Plausible.Session.Transfer.{TinySock, Alive}
+ alias Plausible.{Cache, ClickhouseSessionV2, Session}
+
+ @cmd_list_cache_names :list
+ @cmd_dump_cache :get
+ @cmd_takeover_done :done
+
+ def telemetry_event, do: [:plausible, :sessions, :transfer]
+
+ @doc """
+ Starts the `:sessions` transfer supervisor.
+
+ Options:
+ - `:name` - the name of the supervisor (default: `Plausible.Session.Transfer`)
+ - `:base_path` - the base path for the Unix domain sockets (required)
+ """
+ def start_link(opts) do
+ name = Keyword.get(opts, :name, __MODULE__)
+ base_path = Keyword.fetch!(opts, :base_path)
+ Supervisor.start_link(__MODULE__, base_path, name: name)
+ end
+
+ @impl true
+ def init(nil) do
+ Logger.info(
+ "Session transfer: ignoring, no socket base path configured (make sure ENABLE_SESSION_TRANSFER/PERSISTENT_CACHE_DIR are set)"
+ )
+
+ :ignore
+ end
+
+ def init(base_path) do
+ File.mkdir_p!(base_path)
+
+ replica =
+ Supervisor.child_spec(
+ {Task, fn -> init_takeover(base_path) end},
+ id: :transfer_replica
+ )
+
+ given_counter = :counters.new(1, [])
+ parent = self()
+
+ primary =
+ {TinySock,
+ base_path: base_path,
+ handler: fn message -> handle_replica(message, parent, given_counter) end}
+
+ alive =
+ Supervisor.child_spec(
+ {Alive,
+ _until = fn ->
+ result = :counters.get(given_counter, 1) > 0
+
+ Logger.info(
+ "Session transfer delayed shut down. Checking if session takeover happened?: #{result}"
+ )
+
+ result
+ end},
+ shutdown: :timer.seconds(15)
+ )
+
+ Logger.info("Session transfer init: #{base_path}")
+ Supervisor.init([replica, primary, alive], strategy: :one_for_one)
+ end
+
+ @doc """
+ Returns `true` if the transfer has been attempted (successfully or not).
+ Returns `false` if the transfer is still in progress.
+ """
+ def attempted?(transfer_sup \\ __MODULE__) do
+ result = not replica_alive?(transfer_sup)
+ Logger.info("Session transfer attempted?: #{result}")
+ result
+ end
+
+ @doc """
+ Returns the child specification for the `:sessions` transfer supervisor.
+ See `start_link/1` for options.
+ """
+ def child_spec(opts) do
+ %{
+ id: __MODULE__,
+ start: {__MODULE__, :start_link, [opts]},
+ type: :supervisor,
+ restart: :temporary
+ }
+ end
+
+ defp handle_replica(request, parent, given_counter) do
+ Logger.info("Session transfer message received at #{node()}: #{inspect(request, limit: 10)}")
+
+ case request do
+ {@cmd_list_cache_names, session_version} ->
+ if session_version == session_version() and attempted?(parent) do
+ Cache.Adapter.get_names(:sessions)
+ else
+ []
+ end
+
+ {@cmd_dump_cache, cache} ->
+ Cache.Adapter.cache2list(cache)
+
+ @cmd_takeover_done ->
+ :counters.add(given_counter, 1, 1)
+ end
+ end
+
+ defp init_takeover(base_path) do
+ started = System.monotonic_time()
+
+ base_path
+ |> TinySock.list!()
+ |> Enum.sort_by(&file_stat_ctime/1, :asc)
+ |> Enum.each(&request_takeover/1)
+
+ :telemetry.execute(telemetry_event(), %{duration: System.monotonic_time() - started})
+ end
+
+ defp request_takeover(sock) do
+ Logger.info("Session transfer: requesting takeover at #{node()}")
+
+ with {:ok, names} <- TinySock.call(sock, {@cmd_list_cache_names, session_version()}) do
+ tasks = Enum.map(names, fn name -> Task.async(fn -> takeover_cache(sock, name) end) end)
+ Task.await_many(tasks, :timer.seconds(10))
+ end
+ after
+ Logger.info("Session transfer: marking takeover as done at #{node()}")
+ TinySock.call(sock, @cmd_takeover_done)
+ end
+
+ defp takeover_cache(sock, cache) do
+ Logger.info("Session transfer: requesting cache #{cache} dump at #{node()}")
+
+ with {:ok, records} <- TinySock.call(sock, {@cmd_dump_cache, cache}) do
+ Enum.each(records, fn record ->
+ {key, %ClickhouseSessionV2{} = session} = record
+ Cache.Adapter.put(:sessions, key, session)
+ end)
+
+ Logger.info("Session transfer: restored cache #{cache} at #{node()}")
+ end
+ end
+
+ defp file_stat_ctime(path) do
+ case File.stat(path) do
+ {:ok, stat} -> stat.ctime
+ {:error, _} -> nil
+ end
+ end
+
+ defp session_version do
+ [
+ ClickhouseSessionV2.module_info(:md5),
+ Cache.Adapter.module_info(:md5),
+ Session.CacheStore.module_info(:md5),
+ Session.Transfer.module_info(:md5)
+ ]
+ end
+
+ defp replica_alive?(transfer_sup) do
+ children = Supervisor.which_children(transfer_sup)
+
+ replica =
+ Enum.find_value(children, fn {id, pid, _, _} -> id == :transfer_replica && pid end)
+
+ is_pid(replica) and Process.alive?(replica)
+ catch
+ :exit, {:noproc, _} -> false
+ end
+end
diff --git a/lib/plausible/session/transfer/alive.ex b/lib/plausible/session/transfer/alive.ex
new file mode 100644
index 0000000000..bfaab0ccdb
--- /dev/null
+++ b/lib/plausible/session/transfer/alive.ex
@@ -0,0 +1,31 @@
+defmodule Plausible.Session.Transfer.Alive do
+ @moduledoc false
+ use GenServer
+
+ @spec start_link((-> boolean)) :: GenServer.on_start()
+ def start_link(until) do
+ GenServer.start_link(__MODULE__, until)
+ end
+
+ @impl true
+ def init(until) when is_function(until, 0) do
+ Process.flag(:trap_exit, true)
+ {:ok, until}
+ end
+
+ @impl true
+ def terminate(_reason, until) do
+ loop(until)
+ end
+
+ defp loop(until) do
+ case until.() do
+ true ->
+ :ok
+
+ false ->
+ :timer.sleep(500)
+ loop(until)
+ end
+ end
+end
diff --git a/lib/plausible/session/transfer/tinysock.ex b/lib/plausible/session/transfer/tinysock.ex
new file mode 100644
index 0000000000..f409895e05
--- /dev/null
+++ b/lib/plausible/session/transfer/tinysock.ex
@@ -0,0 +1,170 @@
+defmodule Plausible.Session.Transfer.TinySock do
+ @moduledoc false
+ use GenServer
+ require Logger
+
+ @tag_data "tinysock"
+ @tag_size byte_size(@tag_data)
+
+ def start_link(opts) do
+ base_path = Keyword.fetch!(opts, :base_path)
+ handler = Keyword.fetch!(opts, :handler)
+ GenServer.start_link(__MODULE__, {base_path, handler})
+ end
+
+ @impl true
+ def init({base_path, handler}) do
+ File.mkdir_p!(base_path)
+ socket = sock_listen_or_retry!(base_path)
+
+ Process.flag(:trap_exit, true)
+ state = %{socket: socket, handler: handler}
+ for _ <- 1..10, do: spawn_acceptor(state)
+
+ {:ok, state}
+ end
+
+ @spec list!(Path.t()) :: [Path.t()]
+ def list!(base_path) do
+ for @tag_data <> _rand = name <- File.ls!(base_path) do
+ Path.join(base_path, name)
+ end
+ end
+
+ @spec call(Path.t(), term, timeout) :: {:ok, reply :: term} | {:error, :timeout | :inet.posix()}
+ def call(sock_path, message, timeout \\ :timer.seconds(5)) do
+ with {:ok, socket} <- sock_connect_or_rm(sock_path, timeout) do
+ try do
+ with :ok <- sock_send(socket, :erlang.term_to_iovec(message)) do
+ sock_recv(socket, timeout)
+ end
+ after
+ sock_shut_and_close(socket)
+ end
+ end
+ end
+
+ @impl true
+ def handle_cast(:accepted, state) do
+ spawn_acceptor(state)
+ {:noreply, state}
+ end
+
+ @impl true
+ def handle_info({:EXIT, pid, reason}, state) do
+ case reason do
+ :normal ->
+ {:noreply, state}
+
+ :emfile ->
+ Logger.error("tinysock ran out of file descriptors, stopping")
+ {:stop, reason, state}
+
+ reason ->
+ log = "tinysock request handler #{inspect(pid)} terminating\n" <> format_error(reason)
+ Logger.error(log, crash_reason: reason)
+ {:noreply, state}
+ end
+ end
+
+ defp format_error(reason) do
+ case reason do
+ {e, stacktrace} when is_list(stacktrace) -> Exception.format(:error, e, stacktrace)
+ _other -> Exception.format(:exit, reason)
+ end
+ end
+
+ @impl true
+ def terminate(reason, %{socket: socket}) do
+ with {:ok, {:local, path}} <- :inet.sockname(socket), do: File.rm(path)
+ reason
+ end
+
+ defp spawn_acceptor(%{socket: socket, handler: handler}) do
+ :proc_lib.spawn_link(__MODULE__, :accept, [_parent = self(), socket, handler])
+ end
+
+ @doc false
+ def accept(parent, listen_socket, handler) do
+ case :gen_tcp.accept(listen_socket, :infinity) do
+ {:ok, socket} ->
+ GenServer.cast(parent, :accepted)
+ handle_message(socket, handler)
+
+ {:error, reason} ->
+ exit(reason)
+ end
+ end
+
+ defp handle_message(socket, handler) do
+ {:ok, message} = sock_recv(socket, _timeout = :timer.seconds(5))
+ sock_send(socket, :erlang.term_to_iovec(handler.(message)))
+ after
+ sock_shut_and_close(socket)
+ end
+
+ @listen_opts mode: :binary, active: false, backlog: 1024
+ @connect_opts mode: :binary, active: false
+
+ defp sock_listen_or_retry!(base_path) do
+ sock_name = @tag_data <> Base.url_encode64(:crypto.strong_rand_bytes(4), padding: false)
+ sock_path = Path.join(base_path, sock_name)
+
+ case :gen_tcp.listen(0, [{:ifaddr, {:local, sock_path}} | @listen_opts]) do
+ {:ok, socket} -> socket
+ {:error, :eaddrinuse} -> sock_listen_or_retry!(base_path)
+ {:error, reason} -> raise File.Error, path: sock_path, reason: reason, action: "bind"
+ end
+ end
+
+ defp sock_connect_or_rm(sock_path, timeout) do
+ case :gen_tcp.connect({:local, sock_path}, 0, @connect_opts, timeout) do
+ {:ok, socket} ->
+ {:ok, socket}
+
+ {:error, reason} = error ->
+ if reason != :timeout do
+ # removes stale socket file
+ # possible - but unlikely - race condition
+ File.rm(sock_path)
+ end
+
+ error
+ end
+ end
+
+ @dialyzer :no_improper_lists
+ defp sock_send(socket, data) do
+ :gen_tcp.send(socket, [<<@tag_data, IO.iodata_length(data)::64-little>> | data])
+ end
+
+ defp sock_recv(socket, timeout) do
+ with {:ok, <<@tag_data, size::64-little>>} <- :gen_tcp.recv(socket, @tag_size + 8, timeout),
+ {:ok, binary} <- sock_recv_continue(socket, size, timeout, []) do
+ try do
+ {:ok, :erlang.binary_to_term(binary, [:safe])}
+ rescue
+ e -> {:error, e}
+ end
+ end
+ end
+
+ @five_mb 5 * 1024 * 1024
+
+ # for larger messages (>64MB) we need to read in chunks or we get {:error, :enomem}
+ defp sock_recv_continue(socket, size, timeout, acc) do
+ with {:ok, data} <- :gen_tcp.recv(socket, min(size, @five_mb), timeout) do
+ acc = [acc | data]
+
+ case size - byte_size(data) do
+ 0 -> {:ok, IO.iodata_to_binary(acc)}
+ left -> sock_recv_continue(socket, left, timeout, acc)
+ end
+ end
+ end
+
+ defp sock_shut_and_close(socket) do
+ :gen_tcp.shutdown(socket, :read_write)
+ :gen_tcp.close(socket)
+ end
+end
diff --git a/lib/plausible/telemetry/plausible_metrics.ex b/lib/plausible/telemetry/plausible_metrics.ex
index 1cad45895c..d7c7d5a3dd 100644
--- a/lib/plausible/telemetry/plausible_metrics.ex
+++ b/lib/plausible/telemetry/plausible_metrics.ex
@@ -78,6 +78,15 @@ defmodule Plausible.PromEx.Plugins.PlausibleMetrics do
counter(
metric_prefix ++ [:ingest, :user_agent_parse, :timeout, :total],
event_name: Ingestion.Event.telemetry_ua_parse_timeout()
+ ),
+ distribution(
+ metric_prefix ++ [:sessions, :transfer, :duration],
+ event_name: Plausible.Session.Transfer.telemetry_event(),
+ reporter_options: [
+ buckets: [100, 250, 500, 750, 1000, 2500, 5000, 7500, 10_000]
+ ],
+ unit: {:native, :millisecond},
+ measurement: :duration
)
]
)
diff --git a/lib/plausible_web/controllers/api/system_controller.ex b/lib/plausible_web/controllers/api/system_controller.ex
index 4073d42ba7..95482e0d52 100644
--- a/lib/plausible_web/controllers/api/system_controller.ex
+++ b/lib/plausible_web/controllers/api/system_controller.ex
@@ -59,9 +59,16 @@ defmodule PlausibleWeb.Api.SystemController do
"ok"
end
+ sessions_health =
+ if Plausible.Session.Transfer.attempted?() do
+ "ok"
+ else
+ "waiting"
+ end
+
status =
- case {postgres_health, clickhouse_health, cache_health} do
- {"ok", "ok", "ok"} -> 200
+ case {postgres_health, clickhouse_health, cache_health, sessions_health} do
+ {"ok", "ok", "ok", "ok"} -> 200
_ -> 500
end
@@ -69,7 +76,8 @@ defmodule PlausibleWeb.Api.SystemController do
|> json(%{
postgres: postgres_health,
clickhouse: clickhouse_health,
- sites_cache: cache_health
+ sites_cache: cache_health,
+ sessions: sessions_health
})
end
end
diff --git a/test/plausible/config_test.exs b/test/plausible/config_test.exs
index 46eaa2c8d1..de11bb66ec 100644
--- a/test/plausible/config_test.exs
+++ b/test/plausible/config_test.exs
@@ -407,6 +407,8 @@ defmodule Plausible.ConfigTest do
assert get_in(config, [:plausible, Plausible.Geo, :cache_dir]) == "/var/lib/plausible"
# tzdata (timezones cache)
assert get_in(config, [:tzdata, :data_dir]) == "/var/lib/plausible/tzdata_data"
+ # session transfer
+ assert get_in(config, [:plausible, :session_transfer_dir]) == "/var/lib/plausible/sessions"
end
test "with only DATA_DIR set", %{env: env} do
@@ -418,6 +420,8 @@ defmodule Plausible.ConfigTest do
assert get_in(config, [:plausible, Plausible.Geo, :cache_dir]) == "/data"
# tzdata (timezones cache)
assert get_in(config, [:tzdata, :data_dir]) == "/data/tzdata_data"
+ # session transfer
+ assert get_in(config, [:plausible, :session_transfer_dir]) == "/data/sessions"
end
test "with only PERSISTENT_CACHE_DIR set", %{env: env} do
@@ -429,6 +433,8 @@ defmodule Plausible.ConfigTest do
assert get_in(config, [:plausible, Plausible.Geo, :cache_dir]) == "/cache"
# tzdata (timezones cache)
assert get_in(config, [:tzdata, :data_dir]) == "/cache/tzdata_data"
+ # session transfer
+ assert get_in(config, [:plausible, :session_transfer_dir]) == "/cache/sessions"
end
test "with both DATA_DIR and PERSISTENT_CACHE_DIR set", %{env: env} do
@@ -440,6 +446,8 @@ defmodule Plausible.ConfigTest do
assert get_in(config, [:plausible, Plausible.Geo, :cache_dir]) == "/cache"
# tzdata (timezones cache)
assert get_in(config, [:tzdata, :data_dir]) == "/cache/tzdata_data"
+ # session transfer
+ assert get_in(config, [:plausible, :session_transfer_dir]) == "/cache/sessions"
end
end
diff --git a/test/plausible/session/transfer_test.exs b/test/plausible/session/transfer_test.exs
new file mode 100644
index 0000000000..b10084ade6
--- /dev/null
+++ b/test/plausible/session/transfer_test.exs
@@ -0,0 +1,107 @@
+defmodule Plausible.Session.TransferTest do
+ use ExUnit.Case
+ import Plausible.Factory
+ import Plausible.TestUtils, only: [tmp_dir: 0]
+
+ @tag :slow
+ test "it works" do
+ tmp_dir = tmp_dir()
+
+ old = start_another_plausible(tmp_dir)
+ await_transfer(old)
+
+ Enum.each(1..250, fn _ -> process_event(old, build(:event, name: "pageview")) end)
+
+ new = start_another_plausible(tmp_dir)
+ await_transfer(new)
+
+ assert all_sessions_sorted(new) == all_sessions_sorted(old)
+ end
+
+ defp start_another_plausible(tmp_dir) do
+ {:ok, pid, _node} = :peer.start_link(%{connection: {{127, 0, 0, 1}, 0}})
+ add_code_paths(pid)
+ transfer_configuration(pid)
+ :ok = :peer.call(pid, Application, :put_env, [:plausible, :session_transfer_dir, tmp_dir])
+ ensure_applications_started(pid)
+ pid
+ end
+
+ defp add_code_paths(pid) do
+ :ok = :peer.call(pid, :code, :add_paths, [:code.get_path()])
+ end
+
+ defp transfer_configuration(pid) do
+ for {app_name, _, _} <- Application.loaded_applications() do
+ for {key, val} <- Application.get_all_env(app_name) do
+ :ok = :peer.call(pid, Application, :put_env, [app_name, key, val])
+ end
+ end
+ end
+
+ defp ensure_applications_started(pid) do
+ {:ok, _apps} = :peer.call(pid, Application, :ensure_all_started, [:mix])
+ :ok = :peer.call(pid, Mix, :env, [Mix.env()])
+
+ for {app_name, _, _} <- Application.loaded_applications(), app_name != :dialyxir do
+ {:ok, _apps} = :peer.call(pid, Application, :ensure_all_started, [app_name])
+ end
+ end
+
+ @session_params %{
+ referrer: "ref",
+ referrer_source: "refsource",
+ utm_medium: "medium",
+ utm_source: "source",
+ utm_campaign: "campaign",
+ utm_content: "content",
+ utm_term: "term",
+ browser: "browser",
+ browser_version: "55",
+ country_code: "EE",
+ screen_size: "Desktop",
+ operating_system: "Mac",
+ operating_system_version: "11"
+ }
+
+ defp process_event(pid, event) do
+ :peer.call(pid, Plausible.Session.CacheStore, :on_event, [
+ event,
+ @session_params,
+ _prev_user_id = nil,
+ [buffer_insert: &Function.identity/1]
+ ])
+ end
+
+ defp all_sessions_sorted(pid) do
+ cache_names = :peer.call(pid, Plausible.Cache.Adapter, :get_names, [:sessions])
+
+ records =
+ Enum.flat_map(cache_names, fn cache_name ->
+ tab = :peer.call(pid, ConCache, :ets, [cache_name])
+ :peer.call(pid, :ets, :tab2list, [tab])
+ end)
+
+ Enum.sort_by(records, fn {key, _} -> key end)
+ end
+
+ defp await_transfer(pid, timeout \\ :timer.seconds(1)) do
+ test = self()
+
+ spawn_link(fn ->
+ await_loop(fn -> :peer.call(pid, Plausible.Session.Transfer, :attempted?, []) end)
+ send(test, :took)
+ end)
+
+ assert_receive :took, timeout
+ end
+
+ defp await_loop(f) do
+ :timer.sleep(100)
+
+ case f.() do
+ true -> :done
+ false -> await_loop(f)
+ end
+ end
+end
diff --git a/test/support/test_utils.ex b/test/support/test_utils.ex
index cc004235d4..681e49636a 100644
--- a/test/support/test_utils.ex
+++ b/test/support/test_utils.ex
@@ -333,4 +333,16 @@ defmodule Plausible.TestUtils do
|> Application.fetch_env!(PlausibleWeb.Endpoint)
|> Keyword.fetch!(:secret_key_base)
end
+
+ # normal `@tag :tmp_dir` might not work in Plausible.Session.Transfer tests
+ # if the path is too long for unix domain sockets (>104)
+ # this one makes paths a bit shorter
+ def tmp_dir do
+ name = "plausible-#{System.unique_integer([:positive])}"
+ tmp_dir = Path.join(System.tmp_dir!(), name)
+ File.rm_rf!(tmp_dir)
+ File.mkdir_p!(tmp_dir)
+ ExUnit.Callbacks.on_exit(fn -> File.rm_rf!(tmp_dir) end)
+ tmp_dir
+ end
end