Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Implement directing percentage of traffic to a given persistor backend (#5700)

Adrian Gruntkowskid170d9cc9c68
diff --git a/config/runtime.exs b/config/runtime.exs
index 2e0ec97bbf..0acb5dfaa2 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -148,6 +148,11 @@ persistor_backend =
"remote" -> Plausible.Ingestion.Persistor.Remote
end
+persistor_backend_percent_enabled =
+ config_dir
+ |> get_var_from_path_or_env("PERSISTOR_BACKEND_PERCENT_ENABLED", "0")
+ |> Integer.parse()
+
persistor_url =
get_var_from_path_or_env(config_dir, "PERSISTOR_URL", "http://localhost:8001/event")
@@ -660,7 +665,9 @@ config :plausible, Plausible.ImportDeletionRepo,
transport_opts: ch_transport_opts,
pool_size: 1
-config :plausible, Plausible.Ingestion.Persistor, backend: persistor_backend
+config :plausible, Plausible.Ingestion.Persistor,
+ backend: persistor_backend,
+ backend_percent_enabled: persistor_backend_percent_enabled
config :plausible, Plausible.Ingestion.Persistor.Remote,
url: persistor_url,
diff --git a/lib/plausible/ingestion/persistor.ex b/lib/plausible/ingestion/persistor.ex
index b54a2421f7..6e8b8f3a72 100644
--- a/lib/plausible/ingestion/persistor.ex
+++ b/lib/plausible/ingestion/persistor.ex
@@ -3,17 +3,37 @@ defmodule Plausible.Ingestion.Persistor do
Registers and persists sessions and events.
"""
+ @fallback_backend Plausible.Ingestion.Persistor.Embedded
+
def persist_event(event, previous_user_id, opts) do
{backend_override, opts} = Keyword.pop(opts, :backend)
+ user_id = event.clickhouse_event.user_id
- backend(backend_override).persist_event(event, previous_user_id, opts)
+ backend(backend_override, user_id).persist_event(event, previous_user_id, opts)
end
- defp backend(nil) do
- :plausible
- |> Application.fetch_env!(__MODULE__)
- |> Keyword.fetch!(:backend)
+ defp backend(nil, user_id) do
+ percent_enabled =
+ :plausible
+ |> Application.fetch_env!(__MODULE__)
+ |> Keyword.fetch!(:backend_percent_enabled)
+
+ backend =
+ :plausible
+ |> Application.fetch_env!(__MODULE__)
+ |> Keyword.fetch!(:backend)
+
+ cond do
+ backend == @fallback_backend or percent_enabled >= 100 ->
+ backend
+
+ :erlang.phash2(user_id, 100) + 1 <= percent_enabled ->
+ backend
+
+ true ->
+ @fallback_backend
+ end
end
- defp backend(override), do: override
+ defp backend(override, _user_id), do: override
end
diff --git a/test/plausible/ingestion/persistor_sync_test.exs b/test/plausible/ingestion/persistor_sync_test.exs
new file mode 100644
index 0000000000..31aed23f81
--- /dev/null
+++ b/test/plausible/ingestion/persistor_sync_test.exs
@@ -0,0 +1,136 @@
+defmodule Plausible.Ingestion.PersistorSyncTest do
+ use Plausible.DataCase, async: false
+
+ alias Plausible.Ingestion.Event
+ alias Plausible.Ingestion.Persistor
+
+ @session_params %{
+ referrer: "ref",
+ referrer_source: "refsource",
+ utm_medium: "medium",
+ utm_source: "source",
+ utm_campaign: "campaign",
+ utm_content: "content",
+ utm_term: "term",
+ browser: "browser",
+ browser_version: "55",
+ country_code: "EE",
+ screen_size: "Desktop",
+ operating_system: "Mac",
+ operating_system_version: "11"
+ }
+
+ describe "backend_percent_enabled" do
+ test "ingests with embedded persistor when backend set to 100% and left on default" do
+ Application.put_env(:plausible, Persistor,
+ backend_percent_enabled: 100,
+ backend: Persistor.Embedded
+ )
+
+ event = build(:event, name: "pageview")
+ ingest_event = %Event{clickhouse_event: event, clickhouse_session_attrs: @session_params}
+
+ assert {:ok, ingested_event} = Persistor.persist_event(ingest_event, nil, [])
+ refute ingested_event.dropped?
+ assert ingested_event.clickhouse_event.operating_system == "Mac"
+ assert is_integer(ingested_event.clickhouse_event.session_id)
+ end
+
+ test "ingests with embedded persistor when backend set to < 100%" do
+ Application.put_env(:plausible, Persistor,
+ backend_percent_enabled: 10,
+ backend: Persistor.Embedded
+ )
+
+ event = build(:event, name: "pageview")
+ ingest_event = %Event{clickhouse_event: event, clickhouse_session_attrs: @session_params}
+
+ assert {:ok, ingested_event} = Persistor.persist_event(ingest_event, nil, [])
+ refute ingested_event.dropped?
+ assert ingested_event.clickhouse_event.operating_system == "Mac"
+ assert is_integer(ingested_event.clickhouse_event.session_id)
+ end
+
+ test "ingests with remote persistor for a fraction of users with fallback to embedded when set to < 100%" do
+ Application.put_env(:plausible, Persistor,
+ backend_percent_enabled: 50,
+ backend: Persistor.Remote
+ )
+
+ bypass = Bypass.open()
+
+ Application.put_env(:plausible, Persistor.Remote, url: bypass_url(bypass))
+
+ current_pid = self()
+
+ expect_persistor(bypass, fn input_event, session_attrs ->
+ send(current_pid, :remote_ingest)
+
+ input_event
+ |> Map.merge(session_attrs)
+ |> Map.put(:session_id, System.unique_integer())
+ end)
+
+ assert ingest_via_both_backends(false, false, 10) == :ok
+ end
+
+ defp ingest_via_both_backends(true, true, _), do: :ok
+
+ defp ingest_via_both_backends(_, _, 0), do: :out_of_attempts
+
+ defp ingest_via_both_backends(embedded_ingested?, remote_ingested?, attempts_left) do
+ event = build(:event, name: "pageview")
+ ingest_event = %Event{clickhouse_event: event, clickhouse_session_attrs: @session_params}
+
+ current_pid = self()
+
+ assert {:ok, _ingested_event} =
+ Persistor.persist_event(ingest_event, nil,
+ session_write_buffer_insert: fn [session] = sessions ->
+ assert session.user_id == event.user_id
+ assert session.site_id == event.site_id
+ send(current_pid, :embedded_ingest)
+ {:ok, sessions}
+ end
+ )
+
+ receive do
+ :embedded_ingest ->
+ ingest_via_both_backends(true, remote_ingested?, attempts_left - 1)
+
+ :remote_ingest ->
+ ingest_via_both_backends(embedded_ingested?, true, attempts_left - 1)
+ after
+ 100 ->
+ :ingest_not_called
+ end
+ end
+
+ defp bypass_url(bypass) do
+ "http://localhost:#{bypass.port}/event"
+ end
+
+ defp expect_persistor(bypass, callback_fn) do
+ Bypass.expect(bypass, "POST", "/event", fn conn ->
+ {:ok, body, conn} = Plug.Conn.read_body(conn)
+
+ {input_event, session_attrs} =
+ body
+ |> Base.decode64!(padding: false)
+ |> :erlang.binary_to_term()
+
+ output_event = callback_fn.(input_event, session_attrs)
+
+ event_payload =
+ output_event
+ |> Map.merge(session_attrs)
+ |> Map.put(:session_id, 123)
+ |> :erlang.term_to_binary()
+ |> Base.encode64(padding: false)
+
+ conn
+ |> Plug.Conn.resp(200, event_payload)
+ end)
+ end
+ end
+end