Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Fix sample rate calculator estimated traffic logic (#6506)
Artur Pata641d1e2aca67
diff --git a/extra/lib/plausible/stats/sampling.ex b/extra/lib/plausible/stats/sampling.exindex a1172281d7..bc854cb81b 100644--- a/extra/lib/plausible/stats/sampling.ex+++ b/extra/lib/plausible/stats/sampling.ex@@ -1,7 +1,23 @@defmodule Plausible.Stats.Sampling do@moduledoc """- Sampling related functions+ This module contains the logic to calculate the sample fraction to use for a particular query.++ To do so, it+ 1) estimates how much traffic the site has per day+ 2) how many days of traffic does the query cover+ 3) calculates the fraction so as not to scan more than 10M events++ ### Example calculation+ traffic = 3M / day+ query covers = 14 days+ events in query = 3M / day * 14 days = 42M++ 10M = 42M * fraction+ ->+ fraction = 10M / 42M+ fraction = 0.24"""+@default_sample_threshold 10_000_000import Ecto.Query@@ -69,8 +85,7 @@ defmodule Plausible.Stats.Sampling dodo: :no_samplingdef fractional_sample_rate(traffic_30_day, query) do- date_range = Query.date_range(query)- duration = Date.diff(date_range.last, date_range.first)+ duration = queried_duration_days(query)estimated_traffic = estimate_traffic(traffic_30_day, duration, query)@@ -89,12 +104,41 @@ defmodule Plausible.Stats.Sampling doendend+ @doc """+ Number of days of stats the query's range covers, clamped to the site's native stats+ begin date (traffic before that isn't queried, so it shouldn't figure in the+ traffic estimation).+ """+ def queried_duration_days(%Query{} = query) do+ {first, last} = Plausible.Stats.Time.utc_boundaries(query)+ days_in_range(first, last)+ end+defp min_sample_rate(), do: 0.013- defp estimate_traffic(traffic_30_day, duration, query) do- duration_adjusted_traffic = traffic_30_day / 30.0 * duration+ defp estimate_traffic(traffic_30_day, duration, %Query{} = query) do+ daily_traffic =+ traffic_30_day / days_of_traffic_in_30d_window(query.now, query.site_native_stats_start_at)++ estimate_by_filters(daily_traffic * duration, query.filters)+ end++ defp days_of_traffic_in_30d_window(now, native_stats_start_at) do+ days_from_stats_start = days_in_range(native_stats_start_at, DateTime.to_naive(now))++ cond do+ days_from_stats_start <= 1 -> 1+ days_from_stats_start <= 30 -> days_from_stats_start+ true -> 30+ end+ end- estimate_by_filters(duration_adjusted_traffic, query.filters)+ @seconds_per_day 86_400+ # Whole days spanned by the inclusive range [first, last]. Rounds the real+ # elapsed time so a range ending at 23:59:59 counts its final day in full,+ # rather than diffing the date parts (which undercounts by one).+ defp days_in_range(first, last) do+ round(NaiveDateTime.diff(last, first, :second) / @seconds_per_day)end@filter_traffic_multiplier 1 / 4.0diff --git a/extra/lib/plausible/stats/sampling_cache.ex b/extra/lib/plausible/stats/sampling_cache.exindex 98fea262d2..aeb463b651 100644--- a/extra/lib/plausible/stats/sampling_cache.ex+++ b/extra/lib/plausible/stats/sampling_cache.ex@@ -1,6 +1,6 @@defmodule Plausible.Stats.SamplingCache do@moduledoc """- Cache storing estimation for events ingested by a site in the past month.+ Cache storing estimation for events ingested by a site in the past 30 days.Used for sampling rate calculations in Plausible.Stats.Sampling."""diff --git a/lib/plausible/sites.ex b/lib/plausible/sites.exindex aa9ca3ccdd..3b9e16dd8b 100644--- a/lib/plausible/sites.ex+++ b/lib/plausible/sites.ex@@ -378,7 +378,9 @@ defmodule Plausible.Sites dodef stats_start_date(site)on_ee do- # for now, we're going to always update consolidated views+ # for now, we're going to always update consolidated views,+ # though Repo.update! runs the actual update query only when+ # the value has changeddef stats_start_date(%Site{consolidated: true} = site) doteam = Repo.preload(site, :team).teamdiff --git a/lib/plausible/stats/time.ex b/lib/plausible/stats/time.exindex 220f9b134f..7be814526a 100644--- a/lib/plausible/stats/time.ex+++ b/lib/plausible/stats/time.ex@@ -5,6 +5,7 @@ defmodule Plausible.Stats.Time doalias Plausible.Stats.{Query, DateTimeRange}+ @spec utc_boundaries(Query.t()) :: {NaiveDateTime.t(), NaiveDateTime.t()}def utc_boundaries(%Query{utc_time_range: time_range,site_native_stats_start_at: native_stats_start_atdiff --git a/test/plausible/stats/sampling_test.exs b/test/plausible/stats/sampling_test.exsindex b323705bfa..92fb2622e7 100644--- a/test/plausible/stats/sampling_test.exs+++ b/test/plausible/stats/sampling_test.exs@@ -1,13 +1,40 @@defmodule Plausible.Stats.SamplingTest do- use Plausible.DataCase, async: true+ use Plausible.DataCase, async: falseon_ee do- import Plausible.Stats.Sampling, only: [fractional_sample_rate: 2]+ import Plausible.Stats.Sampling, only: [fractional_sample_rate: 2, queried_duration_days: 1]alias Plausible.Stats.{Query, DateTimeRange}- describe "&fractional_sample_rate/2" do- @threshold Plausible.Stats.Sampling.default_sample_threshold()+ @threshold Plausible.Stats.Sampling.default_sample_threshold()+ describe "&queried_duration_days/1" do+ test "handles 1 day range" do+ query = range_query(~D[2026-07-20], ~D[2026-07-20], ~N[2000-01-01 00:00:00])++ assert queried_duration_days(query) == 1+ end++ test "handles range in non-leap year" do+ query = range_query(~D[2025-01-01], ~D[2025-12-31], ~N[2000-01-01 00:00:00])++ assert queried_duration_days(query) == 365+ end++ test "handles range in leap year" do+ query = range_query(~D[2024-01-01], ~D[2024-12-31], ~N[2000-01-01 00:00:00])++ assert queried_duration_days(query) == 366+ end++ test "is clamped to the site's native stats start for a range extending beyond native stats start" do+ query = range_query(~D[2026-07-01], ~D[2026-07-10], ~N[2026-07-08 00:00:00])++ # Stats begin Jul 8, range ends Jul 10 -> Jul 8, 9, 10 = 3 days.+ assert queried_duration_days(query) == 3+ end+ end++ describe "&fractional_sample_rate/2 for date ranges of different lengths" dotest "no traffic estimate" doassert fractional_sample_rate(nil, query(30)) == :no_samplingend@@ -47,8 +74,7 @@ defmodule Plausible.Stats.SamplingTest doendtest "short durations" do- assert fractional_sample_rate(@threshold * 15, query(1, unit: :hour)) ==- :no_sampling+ assert fractional_sample_rate(@threshold * 15, query(1, unit: :hour)) == :no_samplingendtest "very low sampling rate" do@@ -58,24 +84,43 @@ defmodule Plausible.Stats.SamplingTest do@filter ["is", "event:name", ["pageview"]]test "scales sampling rate according to query filters (when sampling adjustments are enabled)" doassert fractional_sample_rate(@threshold * 50, query(30, filters: [])) == 0.02+ assert fractional_sample_rate(@threshold * 50, query(30, filters: [@filter])) == 0.08- assert fractional_sample_rate(@threshold * 50, query(30, filters: [@filter])) ==- 0.08-- assert fractional_sample_rate(- @threshold * 50,- query(30, filters: [@filter, @filter])- ) == 0.32+ assert fractional_sample_rate(@threshold * 50, query(30, filters: [@filter, @filter])) ==+ 0.32assert fractional_sample_rate(@threshold * 50,query(30, filters: [@filter, @filter, @filter])) == 0.32- assert fractional_sample_rate(- @threshold * 10,- query(30, filters: [@filter, @filter])- ) == :no_sampling+ assert fractional_sample_rate(@threshold * 10, query(30, filters: [@filter, @filter])) ==+ :no_sampling+ end+ end++ describe "&fractional_sample_rate/2 clamping" do+ test "a range starting before stats begin is treated as starting when stats begin" do+ stats_start = ~N[2026-01-01 00:00:00]+ range_end = ~D[2026-07-15]++ from_stats_start =+ fractional_sample_rate(@threshold, range_query(~D[2026-01-01], range_end, stats_start))++ from_1970 =+ fractional_sample_rate(@threshold, range_query(~D[1970-01-01], range_end, stats_start))++ from_2025 =+ fractional_sample_rate(@threshold, range_query(~D[2025-01-01], range_end, stats_start))++ assert from_stats_start == 0.15+ assert from_1970 == from_stats_start+ assert from_2025 == from_stats_start+ end++ test "sample rate is correct if a site has ingested 30M events in the 3 days it has existed" do+ query = range_query(~D[2026-06-15], ~D[2026-07-15], ~N[2026-07-12 00:00:00])+ assert fractional_sample_rate(30_000_000, query) == 0.33endend@@ -88,8 +133,23 @@ defmodule Plausible.Stats.SamplingTest do%Query{utc_time_range: DateTimeRange.new!(first, last),- timezone: "UTC",- filters: filters+ timezone: "Etc/UTC",+ filters: filters,+ now: last,+ # A stats start well before the range, so nothing gets clamped.+ site_native_stats_start_at: ~N[2000-01-01 00:00:00]+ }+ end++ defp range_query(first_date, last_date, native_stats_start_at) do+ utc_time_range = DateTimeRange.new!(first_date, last_date, "Etc/UTC")++ %Query{+ utc_time_range: utc_time_range,+ timezone: "Etc/UTC",+ filters: [],+ now: utc_time_range.last,+ site_native_stats_start_at: native_stats_start_at}endend