Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

fractional_sample_rate: Vary sample rate according to traffic estimation (#4839)

Karl-Aksel Puulmann969f094e6135
diff --git a/extra/lib/plausible/stats/sampling.ex b/extra/lib/plausible/stats/sampling.ex
index 19b132e40e..16ee807c86 100644
--- a/extra/lib/plausible/stats/sampling.ex
+++ b/extra/lib/plausible/stats/sampling.ex
@@ -3,13 +3,22 @@ defmodule Plausible.Stats.Sampling do
Sampling related functions
"""
@default_sample_threshold 20_000_000
+ # 1 percent
+ @min_sample_rate 0.01
import Ecto.Query
+ alias Plausible.Stats.{Query, SamplingCache}
+
+ def default_sample_threshold(), do: @default_sample_threshold
+
@spec add_query_hint(Ecto.Query.t(), Plausible.Stats.Query.t()) :: Ecto.Query.t()
def add_query_hint(%Ecto.Query{} = db_query, %Plausible.Stats.Query{} = query) do
case query.sample_threshold do
- :infinite ->
+ :no_sampling ->
+ db_query
+
+ nil ->
db_query
threshold ->
@@ -33,10 +42,10 @@ defmodule Plausible.Stats.Sampling do
sample_threshold =
case params["sample_threshold"] do
nil ->
- site_default_threshold(site)
+ decide_sample_rate(site, query)
"infinite" ->
- :infinite
+ :no_sampling
value_string ->
{value, _} = Float.parse(value_string)
@@ -46,13 +55,41 @@ defmodule Plausible.Stats.Sampling do
Map.put(query, :sample_threshold, sample_threshold)
end
- defp site_default_threshold(site) do
- if FunWithFlags.enabled?(:fractional_hardcoded_sample_rate, for: site) do
- # Hard-coded sample rate to temporarily fix an issue for a client.
- # To be solved as part of https://3.basecamp.com/5308029/buckets/39750953/messages/7978775089
- 0.1
- else
- @default_sample_threshold
+ defp decide_sample_rate(site, query) do
+ cond do
+ FunWithFlags.enabled?(:fractional_hardcoded_sample_rate, for: site) ->
+ # Hard-coded sample rate to temporarily fix an issue for a client.
+ # To be solved as part of https://3.basecamp.com/5308029/buckets/39750953/messages/7978775089
+ 0.1
+
+ FunWithFlags.enabled?(:fractional_sample_rate, for: site) ->
+ traffic_30_day = SamplingCache.get(site.id)
+ fractional_sample_rate(traffic_30_day, query)
+
+ true ->
+ @default_sample_threshold
+ end
+ end
+
+ def fractional_sample_rate(nil = _traffic_30_day, _query), do: :no_sampling
+
+ def fractional_sample_rate(traffic_30_day, query) do
+ date_range = Query.date_range(query)
+ duration = Date.diff(date_range.last, date_range.first)
+ estimated_traffic = traffic_30_day / 30.0 * duration
+
+ fraction =
+ if(estimated_traffic > 0,
+ do: Float.round(@default_sample_threshold / estimated_traffic, 2),
+ else: 1.0
+ )
+
+ cond do
+ # Don't sample small time ranges
+ duration < 1 -> :no_sampling
+ # If sampling doesn't have a significant effect, don't sample
+ fraction > 0.4 -> :no_sampling
+ true -> max(fraction, @min_sample_rate)
end
end
end
diff --git a/extra/lib/plausible/stats/sampling_cache.ex b/extra/lib/plausible/stats/sampling_cache.ex
new file mode 100644
index 0000000000..7f91e39470
--- /dev/null
+++ b/extra/lib/plausible/stats/sampling_cache.ex
@@ -0,0 +1,55 @@
+defmodule Plausible.Stats.SamplingCache do
+ @moduledoc """
+ Cache storing estimation for events ingested by a team in the past month.
+
+ Used for sampling rate calculations in Plausible.Stats.Sampling.
+ """
+ alias Plausible.Ingestion
+ alias Plausible.Stats.Sampling
+
+ import Ecto.Query
+ use Plausible.Cache
+
+ @cache_name :stats_sampling_cache
+
+ @impl true
+ def name(), do: @cache_name
+
+ @impl true
+ def child_id(), do: :cache_stats_sampling
+
+ @impl true
+ def repo(), do: Plausible.ClickhouseRepo
+
+ @impl true
+ def count_all() do
+ base_db_query()
+ |> repo().all()
+ |> length()
+ end
+
+ @impl true
+ def base_db_query() do
+ from(r in Ingestion.Counters.Record,
+ select: {
+ r.site_id,
+ selected_as(fragment("sumIf(value, metric = 'buffered')"), :events_ingested)
+ },
+ where: fragment("toDate(event_timebucket) >= ?", ^thirty_days_ago()),
+ group_by: r.site_id,
+ having: selected_as(:events_ingested) > ^Sampling.default_sample_threshold()
+ )
+ end
+
+ @impl true
+ def get_from_source(site_id) do
+ base_db_query()
+ |> repo().all()
+ |> Map.new()
+ |> Map.get(site_id)
+ end
+
+ def thirty_days_ago() do
+ Date.shift(Date.utc_today(), day: -30)
+ end
+end
diff --git a/lib/plausible/application.ex b/lib/plausible/application.ex
index 32bc19658a..1e0e6a23c5 100644
--- a/lib/plausible/application.ex
+++ b/lib/plausible/application.ex
@@ -13,92 +13,103 @@ defmodule Plausible.Application do
# in CE we start the endpoint under site_encrypt for automatic https
endpoint = on_ee(do: PlausibleWeb.Endpoint, else: maybe_https_endpoint())
- children = [
- Plausible.Cache.Stats,
- Plausible.Repo,
- Plausible.ClickhouseRepo,
- Plausible.IngestRepo,
- Plausible.AsyncInsertRepo,
- Plausible.ImportDeletionRepo,
- {Plausible.Auth.TOTP.Vault, key: totp_vault_key()},
- {Plausible.RateLimit, clean_period: :timer.minutes(10)},
- Plausible.Ingestion.Counters,
- {Finch, name: Plausible.Finch, pools: finch_pool_config()},
- {Phoenix.PubSub, name: Plausible.PubSub},
- Plausible.Session.Salts,
- Supervisor.child_spec(Plausible.Event.WriteBuffer, id: Plausible.Event.WriteBuffer),
- Supervisor.child_spec(Plausible.Session.WriteBuffer, id: Plausible.Session.WriteBuffer),
- ReferrerBlocklist,
- Plausible.Cache.Adapter.child_spec(:customer_currency, :cache_customer_currency,
- ttl_check_interval: :timer.minutes(5),
- global_ttl: :timer.minutes(60)
- ),
- Plausible.Cache.Adapter.child_spec(:user_agents, :cache_user_agents,
- ttl_check_interval: :timer.seconds(5),
- global_ttl: :timer.minutes(60)
- ),
- Plausible.Cache.Adapter.child_spec(:sessions, :cache_sessions,
- ttl_check_interval: :timer.seconds(1),
- global_ttl: :timer.minutes(30)
- ),
- warmed_cache(Plausible.Site.Cache,
- adapter_opts: [ttl_check_interval: false],
- warmers: [
- refresh_all:
- {Plausible.Site.Cache.All,
- interval: :timer.minutes(15) + Enum.random(1..:timer.seconds(10))},
- refresh_updated_recently:
- {Plausible.Site.Cache.RecentlyUpdated, interval: :timer.seconds(30)}
- ]
- ),
- warmed_cache(Plausible.Shield.IPRuleCache,
- adapter_opts: [ttl_check_interval: false],
- warmers: [
- refresh_all:
- {Plausible.Shield.IPRuleCache.All,
- interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10))},
- refresh_updated_recently:
- {Plausible.Shield.IPRuleCache.RecentlyUpdated, interval: :timer.seconds(35)}
- ]
- ),
- warmed_cache(Plausible.Shield.CountryRuleCache,
- adapter_opts: [ttl_check_interval: false],
- warmers: [
- refresh_all:
- {Plausible.Shield.CountryRuleCache.All,
- interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10))},
- refresh_updated_recently:
- {Plausible.Shield.CountryRuleCache.RecentlyUpdated, interval: :timer.seconds(35)}
- ]
- ),
- warmed_cache(Plausible.Shield.PageRuleCache,
- adapter_opts: [ttl_check_interval: false, ets_options: [:bag]],
- warmers: [
- refresh_all:
- {Plausible.Shield.PageRuleCache.All,
- interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10))},
- refresh_updated_recently:
- {Plausible.Shield.PageRuleCache.RecentlyUpdated, interval: :timer.seconds(35)}
- ]
- ),
- warmed_cache(Plausible.Shield.HostnameRuleCache,
- adapter_opts: [ttl_check_interval: false, ets_options: [:bag]],
- warmers: [
- refresh_all:
- {Plausible.Shield.HostnameRuleCache.All,
- interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10))},
- refresh_updated_recently:
- {Plausible.Shield.HostnameRuleCache.RecentlyUpdated, interval: :timer.seconds(25)}
- ]
- ),
- endpoint,
- {Oban, Application.get_env(:plausible, Oban)},
- Plausible.PromEx
- ]
-
- on_ee do
- children = children ++ help_scout_vault()
- end
+ children =
+ [
+ Plausible.Cache.Stats,
+ Plausible.Repo,
+ Plausible.ClickhouseRepo,
+ Plausible.IngestRepo,
+ Plausible.AsyncInsertRepo,
+ Plausible.ImportDeletionRepo,
+ {Plausible.Auth.TOTP.Vault, key: totp_vault_key()},
+ {Plausible.RateLimit, clean_period: :timer.minutes(10)},
+ Plausible.Ingestion.Counters,
+ {Finch, name: Plausible.Finch, pools: finch_pool_config()},
+ {Phoenix.PubSub, name: Plausible.PubSub},
+ Plausible.Session.Salts,
+ Supervisor.child_spec(Plausible.Event.WriteBuffer, id: Plausible.Event.WriteBuffer),
+ Supervisor.child_spec(Plausible.Session.WriteBuffer, id: Plausible.Session.WriteBuffer),
+ ReferrerBlocklist,
+ Plausible.Cache.Adapter.child_spec(:customer_currency, :cache_customer_currency,
+ ttl_check_interval: :timer.minutes(5),
+ global_ttl: :timer.minutes(60)
+ ),
+ Plausible.Cache.Adapter.child_spec(:user_agents, :cache_user_agents,
+ ttl_check_interval: :timer.seconds(5),
+ global_ttl: :timer.minutes(60)
+ ),
+ Plausible.Cache.Adapter.child_spec(:sessions, :cache_sessions,
+ ttl_check_interval: :timer.seconds(1),
+ global_ttl: :timer.minutes(30)
+ ),
+ warmed_cache(Plausible.Site.Cache,
+ adapter_opts: [ttl_check_interval: false],
+ warmers: [
+ refresh_all:
+ {Plausible.Site.Cache.All,
+ interval: :timer.minutes(15) + Enum.random(1..:timer.seconds(10))},
+ refresh_updated_recently:
+ {Plausible.Site.Cache.RecentlyUpdated, interval: :timer.seconds(30)}
+ ]
+ ),
+ warmed_cache(Plausible.Shield.IPRuleCache,
+ adapter_opts: [ttl_check_interval: false],
+ warmers: [
+ refresh_all:
+ {Plausible.Shield.IPRuleCache.All,
+ interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10))},
+ refresh_updated_recently:
+ {Plausible.Shield.IPRuleCache.RecentlyUpdated, interval: :timer.seconds(35)}
+ ]
+ ),
+ warmed_cache(Plausible.Shield.CountryRuleCache,
+ adapter_opts: [ttl_check_interval: false],
+ warmers: [
+ refresh_all:
+ {Plausible.Shield.CountryRuleCache.All,
+ interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10))},
+ refresh_updated_recently:
+ {Plausible.Shield.CountryRuleCache.RecentlyUpdated, interval: :timer.seconds(35)}
+ ]
+ ),
+ warmed_cache(Plausible.Shield.PageRuleCache,
+ adapter_opts: [ttl_check_interval: false, ets_options: [:bag]],
+ warmers: [
+ refresh_all:
+ {Plausible.Shield.PageRuleCache.All,
+ interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10))},
+ refresh_updated_recently:
+ {Plausible.Shield.PageRuleCache.RecentlyUpdated, interval: :timer.seconds(35)}
+ ]
+ ),
+ warmed_cache(Plausible.Shield.HostnameRuleCache,
+ adapter_opts: [ttl_check_interval: false, ets_options: [:bag]],
+ warmers: [
+ refresh_all:
+ {Plausible.Shield.HostnameRuleCache.All,
+ interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10))},
+ refresh_updated_recently:
+ {Plausible.Shield.HostnameRuleCache.RecentlyUpdated, interval: :timer.seconds(25)}
+ ]
+ ),
+ on_ee do
+ warmed_cache(Plausible.Stats.SamplingCache,
+ adapter_opts: [ttl_check_interval: false],
+ warmers: [
+ refresh_all:
+ {Plausible.Stats.SamplingCache.All,
+ interval: :timer.hours(24) + Enum.random(1..:timer.minutes(60))}
+ ]
+ )
+ end,
+ endpoint,
+ {Oban, Application.get_env(:plausible, Oban)},
+ Plausible.PromEx,
+ on_ee do
+ help_scout_vault()
+ end
+ ]
+ |> Enum.reject(&is_nil/1)
opts = [strategy: :one_for_one, name: Plausible.Supervisor]
diff --git a/lib/plausible/cache.ex b/lib/plausible/cache.ex
index 781883c93e..2ffe2f2886 100644
--- a/lib/plausible/cache.ex
+++ b/lib/plausible/cache.ex
@@ -15,22 +15,22 @@ defmodule Plausible.Cache do
during tests via the `:plausible, #{__MODULE__}, enabled: bool()` application env key.
This can be overridden on case by case basis, using the child specs options.
- The `base_db_query/0` callback is used to generate the base query that is
- executed on every cache refresh.
+ The `base_db_query/0` callback is used to generate the base query that is
+ executed on every cache refresh.
There are two modes of refresh operation: `:all` and `:updated_recently`;
the former will invoke the query as is and clear all the existing entries,
- while the latter will attempt to limit the query to only the records that
+ while the latter will attempt to limit the query to only the records that
have been updated in the last 15 minutes and try to merge the new results with
existing cache entries.
Both refresh modes are normally executed periodically from within a warmer process;
- see: `Plausible.Cache.Warmer`. The reason for two modes is that the latter is lighter
+ see: `Plausible.Cache.Warmer`. The reason for two modes is that the latter is lighter
on the database and can be executed more frequently.
When Cache is disabled via application env, the `get/1` function
- falls back to pure database lookups (implemented via `get_from_source/1` callback.
- This should help with introducing cached lookups in existing code,
+ falls back to pure database lookups (implemented via `get_from_source/1` callback.
+ This should help with introducing cached lookups in existing code,
so that no existing tests should break.
Refreshing the cache emits telemetry event defined as per `telemetry_event_refresh/2`.
@@ -39,6 +39,8 @@ defmodule Plausible.Cache do
@callback name() :: atom()
@doc "Supervisor child id, must be unique within the supervision tree"
@callback child_id() :: atom()
+ @doc "Optional repo to use. Defaults to Plausible.Repo"
+ @callback repo() :: Ecto.Repo.t()
@doc "Counts all items at the source, an aggregate query most likely"
@callback count_all() :: integer()
@doc "Optionally unwraps the keys of the cache items, in case one item is stored under multiple keys"
@@ -89,6 +91,9 @@ defmodule Plausible.Cache do
def unwrap_cache_keys(items), do: items
defoverridable unwrap_cache_keys: 1
+ def repo(), do: Plausible.Repo
+ defoverridable repo: 0
+
@spec refresh_all(Keyword.t()) :: :ok
def refresh_all(opts \\ []) do
refresh(
@@ -177,7 +182,7 @@ defmodule Plausible.Cache do
Plausible.PromEx.Plugins.PlausibleMetrics.measure_duration(
telemetry_event_refresh(cache_name, mode),
fn ->
- items = Plausible.Repo.all(query)
+ items = repo().all(query)
:ok = merge_items(items, opts)
end
)
diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exs
index ce5af98cd8..8716128b7b 100644
--- a/test/plausible/stats/query_test.exs
+++ b/test/plausible/stats/query_test.exs
@@ -183,9 +183,9 @@ defmodule Plausible.Stats.QueryTest do
end
@tag :ee_only
- test "adds sample_threshold :infinite to query struct", %{site: site} do
+ test "adds sample_threshold :no_sampling to query struct", %{site: site} do
q = Query.from(site, %{"period" => "30d", "sample_threshold" => "infinite"})
- assert q.sample_threshold == :infinite
+ assert q.sample_threshold == :no_sampling
end
@tag :ee_only
diff --git a/test/plausible/stats/sampling_cache_test.exs b/test/plausible/stats/sampling_cache_test.exs
new file mode 100644
index 0000000000..095d8b6145
--- /dev/null
+++ b/test/plausible/stats/sampling_cache_test.exs
@@ -0,0 +1,94 @@
+defmodule Plausible.Stats.SamplingCacheTest do
+ use Plausible.DataCase, async: true
+
+ use Plausible
+
+ on_ee do
+ alias Plausible.Stats.SamplingCache
+
+ describe "getter" do
+ test "returns cached values for traffic in past 30 days", %{test: test} do
+ now = DateTime.utc_now()
+
+ Plausible.IngestRepo.insert_all(Plausible.Ingestion.Counters.Record, [
+ %{
+ site_id: 1,
+ domain: "1.com",
+ value: 11_000_000,
+ event_timebucket: add(now, -1, :day),
+ metric: "buffered"
+ },
+ %{
+ site_id: 2,
+ domain: "2.com",
+ value: 11_000_000,
+ event_timebucket: add(now, -1, :day),
+ metric: "buffered"
+ },
+ %{
+ site_id: 2,
+ domain: "2.com",
+ value: 11_000_000,
+ event_timebucket: add(now, -5, :day),
+ metric: "buffered"
+ },
+ %{
+ site_id: 2,
+ domain: "2.com",
+ value: 11_000_000,
+ event_timebucket: add(now, -35, :day),
+ metric: "buffered"
+ },
+ %{
+ site_id: 3,
+ domain: "3.com",
+ value: 44_000_000,
+ event_timebucket: add(now, -35, :day),
+ metric: "buffered"
+ },
+ %{
+ site_id: 4,
+ domain: "4.com",
+ value: 11_000_000,
+ event_timebucket: add(now, -35, :day),
+ metric: "buffered"
+ }
+ ])
+
+ start_test_cache(test)
+
+ assert SamplingCache.count_all() == 1
+ assert SamplingCache.get(1, force?: true, cache_name: test) == nil
+ assert SamplingCache.get(2, force?: true, cache_name: test) == 22_000_000
+ assert SamplingCache.get(3, force?: true, cache_name: test) == nil
+ assert SamplingCache.get(4, force?: true, cache_name: test) == nil
+
+ Plausible.IngestRepo.insert_all(Plausible.Ingestion.Counters.Record, [
+ %{
+ site_id: 1,
+ value: 11_000_000,
+ event_timebucket: add(now, -1, :day),
+ metric: "buffered"
+ }
+ ])
+
+ :ok = SamplingCache.refresh_all(cache_name: test)
+
+ assert SamplingCache.count_all() == 2
+ assert SamplingCache.get(1, force?: true, cache_name: test) == 22_000_000
+ assert SamplingCache.get(2, force?: true, cache_name: test) == 22_000_000
+ end
+ end
+
+ def add(datetime, n, unit) do
+ DateTime.add(datetime, n, unit) |> DateTime.truncate(:second)
+ end
+
+ defp start_test_cache(cache_name) do
+ %{start: {m, f, a}} = SamplingCache.child_spec(cache_name: cache_name)
+ apply(m, f, a)
+
+ :ok = SamplingCache.refresh_all(cache_name: cache_name)
+ end
+ end
+end
diff --git a/test/plausible/stats/sampling_test.exs b/test/plausible/stats/sampling_test.exs
new file mode 100644
index 0000000000..b2ef635172
--- /dev/null
+++ b/test/plausible/stats/sampling_test.exs
@@ -0,0 +1,52 @@
+defmodule Plausible.Stats.SamplingTest do
+ use Plausible.DataCase, async: true
+
+ use Plausible
+
+ on_ee do
+ import Plausible.Stats.Sampling, only: [fractional_sample_rate: 2]
+ alias Plausible.Stats.{Query, DateTimeRange}
+
+ describe "&fractional_sample_rate/2" do
+ test "no traffic estimate" do
+ assert fractional_sample_rate(nil, query(30)) == :no_sampling
+ end
+
+ test "scales sampling rate according to query duration" do
+ assert fractional_sample_rate(40_000_000, query(30)) == :no_sampling
+ assert fractional_sample_rate(40_000_000, query(60)) == 0.25
+ assert fractional_sample_rate(40_000_000, query(100)) == 0.15
+
+ assert fractional_sample_rate(100_000_000, query(1)) == :no_sampling
+ assert fractional_sample_rate(100_000_000, query(5)) == :no_sampling
+ assert fractional_sample_rate(100_000_000, query(10)) == :no_sampling
+ assert fractional_sample_rate(100_000_000, query(15)) == 0.40
+ assert fractional_sample_rate(100_000_000, query(30)) == 0.20
+ assert fractional_sample_rate(100_000_000, query(60)) == 0.10
+ assert fractional_sample_rate(100_000_000, query(100)) == 0.06
+
+ assert fractional_sample_rate(300_000_000, query(2)) == :no_sampling
+ assert fractional_sample_rate(300_000_000, query(5)) == 0.40
+ assert fractional_sample_rate(300_000_000, query(10)) == 0.20
+ end
+
+ test "short durations" do
+ assert fractional_sample_rate(300_000_000_000, query(1, :hour)) == :no_sampling
+ end
+
+ test "very low sampling rate" do
+ assert fractional_sample_rate(300_000_000_000, query(30)) == 0.01
+ end
+ end
+
+ def query(duration, unit \\ :day) do
+ first = DateTime.utc_now()
+ last = DateTime.add(first, duration, unit)
+
+ %Query{
+ utc_time_range: DateTimeRange.new!(first, last),
+ timezone: "UTC"
+ }
+ end
+ end
+end