Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Experiment: Adjust sampling minimum and by filters (#5515)
Karl-Aksel Puulmann7da74b3031ec
diff --git a/extra/lib/plausible/stats/sampling.ex b/extra/lib/plausible/stats/sampling.exindex fc78534743..33abb160ab 100644--- a/extra/lib/plausible/stats/sampling.ex+++ b/extra/lib/plausible/stats/sampling.ex@@ -3,8 +3,6 @@ defmodule Plausible.Stats.Sampling doSampling related functions"""@default_sample_threshold 10_000_000- # 1 percent- @min_sample_rate 0.01import Ecto.Query@@ -56,17 +54,21 @@ defmodule Plausible.Stats.Sampling doenddefp decide_sample_rate(site, query) do+ sampling_adjustments? = FunWithFlags.enabled?(:sampling_adjustments, for: site)+site.id|> SamplingCache.get()- |> fractional_sample_rate(query)+ |> fractional_sample_rate(query, sampling_adjustments?)end- def fractional_sample_rate(nil = _traffic_30_day, _query), do: :no_sampling+ def fractional_sample_rate(nil = _traffic_30_day, _query, _sampling_adjustments?),+ do: :no_sampling- def fractional_sample_rate(traffic_30_day, query) do+ def fractional_sample_rate(traffic_30_day, query, sampling_adjustments?) dodate_range = Query.date_range(query)duration = Date.diff(date_range.last, date_range.first)- estimated_traffic = traffic_30_day / 30.0 * duration++ estimated_traffic = estimate_traffic(traffic_30_day, duration, query, sampling_adjustments?)fraction =if(estimated_traffic > 0,@@ -79,7 +81,25 @@ defmodule Plausible.Stats.Sampling doduration < 1 -> :no_sampling# If sampling doesn't have a significant effect, don't samplefraction > 0.4 -> :no_sampling- true -> max(fraction, @min_sample_rate)+ true -> max(fraction, min_sample_rate(sampling_adjustments?))+ end+ end++ defp min_sample_rate(false = _sampling_adjustments?), do: 0.01+ defp min_sample_rate(true = _sampling_adjustments?), do: 0.013++ defp estimate_traffic(traffic_30_day, duration, query, sampling_adjustments?) do+ duration_adjusted_traffic = traffic_30_day / 30.0 * duration++ if sampling_adjustments? do+ duration_adjusted_traffic+ |> estimate_by_filters(query.filters)+ else+ duration_adjusted_trafficendend++ @filter_traffic_multiplier 1 / 20.0+ defp estimate_by_filters(estimation, filters),+ do: estimation * @filter_traffic_multiplier ** length(filters)enddiff --git a/test/plausible/stats/sampling_test.exs b/test/plausible/stats/sampling_test.exsindex 3714bec36e..4d9e8e6f25 100644--- a/test/plausible/stats/sampling_test.exs+++ b/test/plausible/stats/sampling_test.exs@@ -4,50 +4,70 @@ defmodule Plausible.Stats.SamplingTest douse Plausibleon_ee do- import Plausible.Stats.Sampling, only: [fractional_sample_rate: 2]+ import Plausible.Stats.Sampling, only: [fractional_sample_rate: 3]alias Plausible.Stats.{Query, DateTimeRange, Sampling}describe "&fractional_sample_rate/2" do@threshold Sampling.default_sample_threshold()test "no traffic estimate" do- assert fractional_sample_rate(nil, query(30)) == :no_sampling+ assert fractional_sample_rate(nil, query(30), false) == :no_samplingendtest "scales sampling rate according to query duration" do- assert fractional_sample_rate(@threshold * 2, query(30)) == :no_sampling- assert fractional_sample_rate(@threshold * 2, query(60)) == 0.25- assert fractional_sample_rate(@threshold * 2, query(100)) == 0.15-- assert fractional_sample_rate(@threshold * 5, query(1)) == :no_sampling- assert fractional_sample_rate(@threshold * 5, query(5)) == :no_sampling- assert fractional_sample_rate(@threshold * 5, query(10)) == :no_sampling- assert fractional_sample_rate(@threshold * 5, query(15)) == 0.40- assert fractional_sample_rate(@threshold * 5, query(30)) == 0.20- assert fractional_sample_rate(@threshold * 5, query(60)) == 0.10- assert fractional_sample_rate(@threshold * 5, query(100)) == 0.06-- assert fractional_sample_rate(@threshold * 15, query(2)) == :no_sampling- assert fractional_sample_rate(@threshold * 15, query(5)) == 0.40- assert fractional_sample_rate(@threshold * 15, query(10)) == 0.20+ assert fractional_sample_rate(@threshold * 2, query(30), false) == :no_sampling+ assert fractional_sample_rate(@threshold * 2, query(60), false) == 0.25+ assert fractional_sample_rate(@threshold * 2, query(100), false) == 0.15++ assert fractional_sample_rate(@threshold * 5, query(1), false) == :no_sampling+ assert fractional_sample_rate(@threshold * 5, query(5), false) == :no_sampling+ assert fractional_sample_rate(@threshold * 5, query(10), false) == :no_sampling+ assert fractional_sample_rate(@threshold * 5, query(15), false) == 0.40+ assert fractional_sample_rate(@threshold * 5, query(30), false) == 0.20+ assert fractional_sample_rate(@threshold * 5, query(60), false) == 0.10+ assert fractional_sample_rate(@threshold * 5, query(100), false) == 0.06++ assert fractional_sample_rate(@threshold * 15, query(2), false) == :no_sampling+ assert fractional_sample_rate(@threshold * 15, query(5), false) == 0.40+ assert fractional_sample_rate(@threshold * 15, query(10), false) == 0.20endtest "short durations" do- assert fractional_sample_rate(@threshold * 15, query(1, :hour)) == :no_sampling+ assert fractional_sample_rate(@threshold * 15, query(1, unit: :hour), false) ==+ :no_samplingendtest "very low sampling rate" do- assert fractional_sample_rate(@threshold * 500, query(30)) == 0.01+ assert fractional_sample_rate(@threshold * 500, query(30), false) == 0.01+ assert fractional_sample_rate(@threshold * 500, query(30), true) == 0.013+ end++ @filter ["is", "event:name", ["pageview"]]+ test "scales sampling rate according to query filters (when sampling adjustments are enabled)" do+ assert fractional_sample_rate(@threshold * 50, query(30, filters: []), true) == 0.02++ assert fractional_sample_rate(@threshold * 50, query(30, filters: [@filter]), true) ==+ 0.40++ assert fractional_sample_rate(+ @threshold * 50,+ query(30, filters: [@filter, @filter]),+ true+ ) == :no_samplingendend- def query(duration, unit \\ :day) do+ def query(duration, opts \\ []) do+ unit = Keyword.get(opts, :unit, :day)+ filters = Keyword.get(opts, :filters, [])+first = DateTime.utc_now()last = DateTime.add(first, duration, unit)%Query{utc_time_range: DateTimeRange.new!(first, last),- timezone: "UTC"+ timezone: "UTC",+ filters: filters}endend