Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Implement directing percentage of traffic to a given persistor backend (#5700)
Adrian Gruntkowskid170d9cc9c68
diff --git a/config/runtime.exs b/config/runtime.exsindex 2e0ec97bbf..0acb5dfaa2 100644--- a/config/runtime.exs+++ b/config/runtime.exs@@ -148,6 +148,11 @@ persistor_backend ="remote" -> Plausible.Ingestion.Persistor.Remoteend+persistor_backend_percent_enabled =+ config_dir+ |> get_var_from_path_or_env("PERSISTOR_BACKEND_PERCENT_ENABLED", "0")+ |> Integer.parse()+persistor_url =get_var_from_path_or_env(config_dir, "PERSISTOR_URL", "http://localhost:8001/event")@@ -660,7 +665,9 @@ config :plausible, Plausible.ImportDeletionRepo,transport_opts: ch_transport_opts,pool_size: 1-config :plausible, Plausible.Ingestion.Persistor, backend: persistor_backend+config :plausible, Plausible.Ingestion.Persistor,+ backend: persistor_backend,+ backend_percent_enabled: persistor_backend_percent_enabledconfig :plausible, Plausible.Ingestion.Persistor.Remote,url: persistor_url,diff --git a/lib/plausible/ingestion/persistor.ex b/lib/plausible/ingestion/persistor.exindex b54a2421f7..6e8b8f3a72 100644--- a/lib/plausible/ingestion/persistor.ex+++ b/lib/plausible/ingestion/persistor.ex@@ -3,17 +3,37 @@ defmodule Plausible.Ingestion.Persistor doRegisters and persists sessions and events."""+ @fallback_backend Plausible.Ingestion.Persistor.Embedded+def persist_event(event, previous_user_id, opts) do{backend_override, opts} = Keyword.pop(opts, :backend)+ user_id = event.clickhouse_event.user_id- backend(backend_override).persist_event(event, previous_user_id, opts)+ backend(backend_override, user_id).persist_event(event, previous_user_id, opts)end- defp backend(nil) do- :plausible- |> Application.fetch_env!(__MODULE__)- |> Keyword.fetch!(:backend)+ defp backend(nil, user_id) do+ percent_enabled =+ :plausible+ |> Application.fetch_env!(__MODULE__)+ |> Keyword.fetch!(:backend_percent_enabled)++ backend =+ :plausible+ |> Application.fetch_env!(__MODULE__)+ |> Keyword.fetch!(:backend)++ cond do+ backend == @fallback_backend or percent_enabled >= 100 ->+ backend++ :erlang.phash2(user_id, 100) + 1 <= percent_enabled ->+ backend++ true ->+ @fallback_backend+ endend- defp backend(override), do: override+ defp backend(override, _user_id), do: overrideenddiff --git a/test/plausible/ingestion/persistor_sync_test.exs b/test/plausible/ingestion/persistor_sync_test.exsnew file mode 100644index 0000000000..31aed23f81--- /dev/null+++ b/test/plausible/ingestion/persistor_sync_test.exs@@ -0,0 +1,136 @@+defmodule Plausible.Ingestion.PersistorSyncTest do+ use Plausible.DataCase, async: false++ alias Plausible.Ingestion.Event+ alias Plausible.Ingestion.Persistor++ @session_params %{+ referrer: "ref",+ referrer_source: "refsource",+ utm_medium: "medium",+ utm_source: "source",+ utm_campaign: "campaign",+ utm_content: "content",+ utm_term: "term",+ browser: "browser",+ browser_version: "55",+ country_code: "EE",+ screen_size: "Desktop",+ operating_system: "Mac",+ operating_system_version: "11"+ }++ describe "backend_percent_enabled" do+ test "ingests with embedded persistor when backend set to 100% and left on default" do+ Application.put_env(:plausible, Persistor,+ backend_percent_enabled: 100,+ backend: Persistor.Embedded+ )++ event = build(:event, name: "pageview")+ ingest_event = %Event{clickhouse_event: event, clickhouse_session_attrs: @session_params}++ assert {:ok, ingested_event} = Persistor.persist_event(ingest_event, nil, [])+ refute ingested_event.dropped?+ assert ingested_event.clickhouse_event.operating_system == "Mac"+ assert is_integer(ingested_event.clickhouse_event.session_id)+ end++ test "ingests with embedded persistor when backend set to < 100%" do+ Application.put_env(:plausible, Persistor,+ backend_percent_enabled: 10,+ backend: Persistor.Embedded+ )++ event = build(:event, name: "pageview")+ ingest_event = %Event{clickhouse_event: event, clickhouse_session_attrs: @session_params}++ assert {:ok, ingested_event} = Persistor.persist_event(ingest_event, nil, [])+ refute ingested_event.dropped?+ assert ingested_event.clickhouse_event.operating_system == "Mac"+ assert is_integer(ingested_event.clickhouse_event.session_id)+ end++ test "ingests with remote persistor for a fraction of users with fallback to embedded when set to < 100%" do+ Application.put_env(:plausible, Persistor,+ backend_percent_enabled: 50,+ backend: Persistor.Remote+ )++ bypass = Bypass.open()++ Application.put_env(:plausible, Persistor.Remote, url: bypass_url(bypass))++ current_pid = self()++ expect_persistor(bypass, fn input_event, session_attrs ->+ send(current_pid, :remote_ingest)++ input_event+ |> Map.merge(session_attrs)+ |> Map.put(:session_id, System.unique_integer())+ end)++ assert ingest_via_both_backends(false, false, 10) == :ok+ end++ defp ingest_via_both_backends(true, true, _), do: :ok++ defp ingest_via_both_backends(_, _, 0), do: :out_of_attempts++ defp ingest_via_both_backends(embedded_ingested?, remote_ingested?, attempts_left) do+ event = build(:event, name: "pageview")+ ingest_event = %Event{clickhouse_event: event, clickhouse_session_attrs: @session_params}++ current_pid = self()++ assert {:ok, _ingested_event} =+ Persistor.persist_event(ingest_event, nil,+ session_write_buffer_insert: fn [session] = sessions ->+ assert session.user_id == event.user_id+ assert session.site_id == event.site_id+ send(current_pid, :embedded_ingest)+ {:ok, sessions}+ end+ )++ receive do+ :embedded_ingest ->+ ingest_via_both_backends(true, remote_ingested?, attempts_left - 1)++ :remote_ingest ->+ ingest_via_both_backends(embedded_ingested?, true, attempts_left - 1)+ after+ 100 ->+ :ingest_not_called+ end+ end++ defp bypass_url(bypass) do+ "http://localhost:#{bypass.port}/event"+ end++ defp expect_persistor(bypass, callback_fn) do+ Bypass.expect(bypass, "POST", "/event", fn conn ->+ {:ok, body, conn} = Plug.Conn.read_body(conn)++ {input_event, session_attrs} =+ body+ |> Base.decode64!(padding: false)+ |> :erlang.binary_to_term()++ output_event = callback_fn.(input_event, session_attrs)++ event_payload =+ output_event+ |> Map.merge(session_attrs)+ |> Map.put(:session_id, 123)+ |> :erlang.term_to_binary()+ |> Base.encode64(padding: false)++ conn+ |> Plug.Conn.resp(200, event_payload)+ end)+ end+ end+end