Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Fix sample rate calculator estimated traffic logic (#6506)

Artur Pata641d1e2aca67
diff --git a/extra/lib/plausible/stats/sampling.ex b/extra/lib/plausible/stats/sampling.ex
index a1172281d7..bc854cb81b 100644
--- a/extra/lib/plausible/stats/sampling.ex
+++ b/extra/lib/plausible/stats/sampling.ex
@@ -1,7 +1,23 @@
defmodule Plausible.Stats.Sampling do
@moduledoc """
- Sampling related functions
+ This module contains the logic to calculate the sample fraction to use for a particular query.
+
+ To do so, it
+ 1) estimates how much traffic the site has per day
+ 2) how many days of traffic does the query cover
+ 3) calculates the fraction so as not to scan more than 10M events
+
+ ### Example calculation
+ traffic = 3M / day
+ query covers = 14 days
+ events in query = 3M / day * 14 days = 42M
+
+ 10M = 42M * fraction
+ ->
+ fraction = 10M / 42M
+ fraction = 0.24
"""
+
@default_sample_threshold 10_000_000
import Ecto.Query
@@ -69,8 +85,7 @@ defmodule Plausible.Stats.Sampling do
do: :no_sampling
def fractional_sample_rate(traffic_30_day, query) do
- date_range = Query.date_range(query)
- duration = Date.diff(date_range.last, date_range.first)
+ duration = queried_duration_days(query)
estimated_traffic = estimate_traffic(traffic_30_day, duration, query)
@@ -89,12 +104,41 @@ defmodule Plausible.Stats.Sampling do
end
end
+ @doc """
+ Number of days of stats the query's range covers, clamped to the site's native stats
+ begin date (traffic before that isn't queried, so it shouldn't figure in the
+ traffic estimation).
+ """
+ def queried_duration_days(%Query{} = query) do
+ {first, last} = Plausible.Stats.Time.utc_boundaries(query)
+ days_in_range(first, last)
+ end
+
defp min_sample_rate(), do: 0.013
- defp estimate_traffic(traffic_30_day, duration, query) do
- duration_adjusted_traffic = traffic_30_day / 30.0 * duration
+ defp estimate_traffic(traffic_30_day, duration, %Query{} = query) do
+ daily_traffic =
+ traffic_30_day / days_of_traffic_in_30d_window(query.now, query.site_native_stats_start_at)
+
+ estimate_by_filters(daily_traffic * duration, query.filters)
+ end
+
+ defp days_of_traffic_in_30d_window(now, native_stats_start_at) do
+ days_from_stats_start = days_in_range(native_stats_start_at, DateTime.to_naive(now))
+
+ cond do
+ days_from_stats_start <= 1 -> 1
+ days_from_stats_start <= 30 -> days_from_stats_start
+ true -> 30
+ end
+ end
- estimate_by_filters(duration_adjusted_traffic, query.filters)
+ @seconds_per_day 86_400
+ # Whole days spanned by the inclusive range [first, last]. Rounds the real
+ # elapsed time so a range ending at 23:59:59 counts its final day in full,
+ # rather than diffing the date parts (which undercounts by one).
+ defp days_in_range(first, last) do
+ round(NaiveDateTime.diff(last, first, :second) / @seconds_per_day)
end
@filter_traffic_multiplier 1 / 4.0
diff --git a/extra/lib/plausible/stats/sampling_cache.ex b/extra/lib/plausible/stats/sampling_cache.ex
index 98fea262d2..aeb463b651 100644
--- a/extra/lib/plausible/stats/sampling_cache.ex
+++ b/extra/lib/plausible/stats/sampling_cache.ex
@@ -1,6 +1,6 @@
defmodule Plausible.Stats.SamplingCache do
@moduledoc """
- Cache storing estimation for events ingested by a site in the past month.
+ Cache storing estimation for events ingested by a site in the past 30 days.
Used for sampling rate calculations in Plausible.Stats.Sampling.
"""
diff --git a/lib/plausible/sites.ex b/lib/plausible/sites.ex
index aa9ca3ccdd..3b9e16dd8b 100644
--- a/lib/plausible/sites.ex
+++ b/lib/plausible/sites.ex
@@ -378,7 +378,9 @@ defmodule Plausible.Sites do
def stats_start_date(site)
on_ee do
- # for now, we're going to always update consolidated views
+ # for now, we're going to always update consolidated views,
+ # though Repo.update! runs the actual update query only when
+ # the value has changed
def stats_start_date(%Site{consolidated: true} = site) do
team = Repo.preload(site, :team).team
diff --git a/lib/plausible/stats/time.ex b/lib/plausible/stats/time.ex
index 220f9b134f..7be814526a 100644
--- a/lib/plausible/stats/time.ex
+++ b/lib/plausible/stats/time.ex
@@ -5,6 +5,7 @@ defmodule Plausible.Stats.Time do
alias Plausible.Stats.{Query, DateTimeRange}
+ @spec utc_boundaries(Query.t()) :: {NaiveDateTime.t(), NaiveDateTime.t()}
def utc_boundaries(%Query{
utc_time_range: time_range,
site_native_stats_start_at: native_stats_start_at
diff --git a/test/plausible/stats/sampling_test.exs b/test/plausible/stats/sampling_test.exs
index b323705bfa..92fb2622e7 100644
--- a/test/plausible/stats/sampling_test.exs
+++ b/test/plausible/stats/sampling_test.exs
@@ -1,13 +1,40 @@
defmodule Plausible.Stats.SamplingTest do
- use Plausible.DataCase, async: true
+ use Plausible.DataCase, async: false
on_ee do
- import Plausible.Stats.Sampling, only: [fractional_sample_rate: 2]
+ import Plausible.Stats.Sampling, only: [fractional_sample_rate: 2, queried_duration_days: 1]
alias Plausible.Stats.{Query, DateTimeRange}
- describe "&fractional_sample_rate/2" do
- @threshold Plausible.Stats.Sampling.default_sample_threshold()
+ @threshold Plausible.Stats.Sampling.default_sample_threshold()
+ describe "&queried_duration_days/1" do
+ test "handles 1 day range" do
+ query = range_query(~D[2026-07-20], ~D[2026-07-20], ~N[2000-01-01 00:00:00])
+
+ assert queried_duration_days(query) == 1
+ end
+
+ test "handles range in non-leap year" do
+ query = range_query(~D[2025-01-01], ~D[2025-12-31], ~N[2000-01-01 00:00:00])
+
+ assert queried_duration_days(query) == 365
+ end
+
+ test "handles range in leap year" do
+ query = range_query(~D[2024-01-01], ~D[2024-12-31], ~N[2000-01-01 00:00:00])
+
+ assert queried_duration_days(query) == 366
+ end
+
+ test "is clamped to the site's native stats start for a range extending beyond native stats start" do
+ query = range_query(~D[2026-07-01], ~D[2026-07-10], ~N[2026-07-08 00:00:00])
+
+ # Stats begin Jul 8, range ends Jul 10 -> Jul 8, 9, 10 = 3 days.
+ assert queried_duration_days(query) == 3
+ end
+ end
+
+ describe "&fractional_sample_rate/2 for date ranges of different lengths" do
test "no traffic estimate" do
assert fractional_sample_rate(nil, query(30)) == :no_sampling
end
@@ -47,8 +74,7 @@ defmodule Plausible.Stats.SamplingTest do
end
test "short durations" do
- assert fractional_sample_rate(@threshold * 15, query(1, unit: :hour)) ==
- :no_sampling
+ assert fractional_sample_rate(@threshold * 15, query(1, unit: :hour)) == :no_sampling
end
test "very low sampling rate" do
@@ -58,24 +84,43 @@ defmodule Plausible.Stats.SamplingTest do
@filter ["is", "event:name", ["pageview"]]
test "scales sampling rate according to query filters (when sampling adjustments are enabled)" do
assert fractional_sample_rate(@threshold * 50, query(30, filters: [])) == 0.02
+ assert fractional_sample_rate(@threshold * 50, query(30, filters: [@filter])) == 0.08
- assert fractional_sample_rate(@threshold * 50, query(30, filters: [@filter])) ==
- 0.08
-
- assert fractional_sample_rate(
- @threshold * 50,
- query(30, filters: [@filter, @filter])
- ) == 0.32
+ assert fractional_sample_rate(@threshold * 50, query(30, filters: [@filter, @filter])) ==
+ 0.32
assert fractional_sample_rate(
@threshold * 50,
query(30, filters: [@filter, @filter, @filter])
) == 0.32
- assert fractional_sample_rate(
- @threshold * 10,
- query(30, filters: [@filter, @filter])
- ) == :no_sampling
+ assert fractional_sample_rate(@threshold * 10, query(30, filters: [@filter, @filter])) ==
+ :no_sampling
+ end
+ end
+
+ describe "&fractional_sample_rate/2 clamping" do
+ test "a range starting before stats begin is treated as starting when stats begin" do
+ stats_start = ~N[2026-01-01 00:00:00]
+ range_end = ~D[2026-07-15]
+
+ from_stats_start =
+ fractional_sample_rate(@threshold, range_query(~D[2026-01-01], range_end, stats_start))
+
+ from_1970 =
+ fractional_sample_rate(@threshold, range_query(~D[1970-01-01], range_end, stats_start))
+
+ from_2025 =
+ fractional_sample_rate(@threshold, range_query(~D[2025-01-01], range_end, stats_start))
+
+ assert from_stats_start == 0.15
+ assert from_1970 == from_stats_start
+ assert from_2025 == from_stats_start
+ end
+
+ test "sample rate is correct if a site has ingested 30M events in the 3 days it has existed" do
+ query = range_query(~D[2026-06-15], ~D[2026-07-15], ~N[2026-07-12 00:00:00])
+ assert fractional_sample_rate(30_000_000, query) == 0.33
end
end
@@ -88,8 +133,23 @@ defmodule Plausible.Stats.SamplingTest do
%Query{
utc_time_range: DateTimeRange.new!(first, last),
- timezone: "UTC",
- filters: filters
+ timezone: "Etc/UTC",
+ filters: filters,
+ now: last,
+ # A stats start well before the range, so nothing gets clamped.
+ site_native_stats_start_at: ~N[2000-01-01 00:00:00]
+ }
+ end
+
+ defp range_query(first_date, last_date, native_stats_start_at) do
+ utc_time_range = DateTimeRange.new!(first_date, last_date, "Etc/UTC")
+
+ %Query{
+ utc_time_range: utc_time_range,
+ timezone: "Etc/UTC",
+ filters: [],
+ now: utc_time_range.last,
+ site_native_stats_start_at: native_stats_start_at
}
end
end