Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Remove sampling flag code and reduce sampling threshold (#5299)
Karl-Aksel Puulmannc18961b2d2ad
diff --git a/extra/lib/plausible/stats/sampling.ex b/extra/lib/plausible/stats/sampling.exindex 16ee807c86..fc78534743 100644--- a/extra/lib/plausible/stats/sampling.ex+++ b/extra/lib/plausible/stats/sampling.ex@@ -2,7 +2,7 @@ defmodule Plausible.Stats.Sampling do@moduledoc """Sampling related functions"""- @default_sample_threshold 20_000_000+ @default_sample_threshold 10_000_000# 1 percent@min_sample_rate 0.01@@ -56,19 +56,9 @@ defmodule Plausible.Stats.Sampling doenddefp decide_sample_rate(site, query) do- cond do- FunWithFlags.enabled?(:fractional_hardcoded_sample_rate, for: site) ->- # Hard-coded sample rate to temporarily fix an issue for a client.- # To be solved as part of https://3.basecamp.com/5308029/buckets/39750953/messages/7978775089- 0.1-- FunWithFlags.enabled?(:fractional_sample_rate, for: site) ->- traffic_30_day = SamplingCache.get(site.id)- fractional_sample_rate(traffic_30_day, query)-- true ->- @default_sample_threshold- end+ site.id+ |> SamplingCache.get()+ |> fractional_sample_rate(query)enddef fractional_sample_rate(nil = _traffic_30_day, _query), do: :no_samplingdiff --git a/test/plausible/stats/sampling_cache_test.exs b/test/plausible/stats/sampling_cache_test.exsindex d9953d3117..59cba435e9 100644--- a/test/plausible/stats/sampling_cache_test.exs+++ b/test/plausible/stats/sampling_cache_test.exs@@ -12,6 +12,8 @@ defmodule Plausible.Stats.SamplingCacheTest do@site_id4 400_000describe "getter" do+ @threshold Plausible.Stats.Sampling.default_sample_threshold()+test "returns cached values for traffic in past 30 days", %{test: test} donow = DateTime.utc_now()@@ -19,42 +21,42 @@ defmodule Plausible.Stats.SamplingCacheTest do%{site_id: @site_id1,domain: "1.com",- value: 11_000_000,+ value: (@threshold * 0.55) |> trunc(),event_timebucket: add(now, -1, :day),metric: "buffered"},%{site_id: @site_id2,domain: "2.com",- value: 11_000_000,+ value: (@threshold * 0.55) |> trunc(),event_timebucket: add(now, -1, :day),metric: "buffered"},%{site_id: @site_id2,domain: "2.com",- value: 11_000_000,+ value: (@threshold * 0.55) |> trunc(),event_timebucket: add(now, -5, :day),metric: "buffered"},%{site_id: @site_id2,domain: "2.com",- value: 11_000_000,+ value: (@threshold * 0.55) |> trunc(),event_timebucket: add(now, -35, :day),metric: "buffered"},%{site_id: @site_id3,domain: "3.com",- value: 44_000_000,+ value: (@threshold * 2.05) |> trunc(),event_timebucket: add(now, -35, :day),metric: "buffered"},%{site_id: @site_id4,domain: "4.com",- value: 11_000_000,+ value: (@threshold * 0.55) |> trunc(),event_timebucket: add(now, -35, :day),metric: "buffered"}@@ -64,14 +66,14 @@ defmodule Plausible.Stats.SamplingCacheTest doassert SamplingCache.count_all() == 1assert SamplingCache.get(@site_id1, force?: true, cache_name: test) == nil- assert SamplingCache.get(@site_id2, force?: true, cache_name: test) == 22_000_000+ assert SamplingCache.get(@site_id2, force?: true, cache_name: test) == 1.1 * @thresholdassert SamplingCache.get(@site_id3, force?: true, cache_name: test) == nilassert SamplingCache.get(@site_id4, force?: true, cache_name: test) == nilPlausible.IngestRepo.insert_all(Plausible.Ingestion.Counters.Record, [%{site_id: @site_id1,- value: 11_000_000,+ value: (@threshold * 0.55) |> trunc(),event_timebucket: add(now, -1, :day),metric: "buffered"}@@ -80,8 +82,8 @@ defmodule Plausible.Stats.SamplingCacheTest do:ok = SamplingCache.refresh_all(cache_name: test)assert SamplingCache.count_all() == 2- assert SamplingCache.get(@site_id1, force?: true, cache_name: test) == 22_000_000- assert SamplingCache.get(@site_id2, force?: true, cache_name: test) == 22_000_000+ assert SamplingCache.get(@site_id1, force?: true, cache_name: test) == 1.1 * @threshold+ assert SamplingCache.get(@site_id2, force?: true, cache_name: test) == 1.1 * @thresholdendenddiff --git a/test/plausible/stats/sampling_test.exs b/test/plausible/stats/sampling_test.exsindex b2ef635172..3714bec36e 100644--- a/test/plausible/stats/sampling_test.exs+++ b/test/plausible/stats/sampling_test.exs@@ -5,37 +5,39 @@ defmodule Plausible.Stats.SamplingTest doon_ee doimport Plausible.Stats.Sampling, only: [fractional_sample_rate: 2]- alias Plausible.Stats.{Query, DateTimeRange}+ alias Plausible.Stats.{Query, DateTimeRange, Sampling}describe "&fractional_sample_rate/2" do+ @threshold Sampling.default_sample_threshold()+test "no traffic estimate" doassert fractional_sample_rate(nil, query(30)) == :no_samplingendtest "scales sampling rate according to query duration" do- assert fractional_sample_rate(40_000_000, query(30)) == :no_sampling- assert fractional_sample_rate(40_000_000, query(60)) == 0.25- assert fractional_sample_rate(40_000_000, query(100)) == 0.15-- assert fractional_sample_rate(100_000_000, query(1)) == :no_sampling- assert fractional_sample_rate(100_000_000, query(5)) == :no_sampling- assert fractional_sample_rate(100_000_000, query(10)) == :no_sampling- assert fractional_sample_rate(100_000_000, query(15)) == 0.40- assert fractional_sample_rate(100_000_000, query(30)) == 0.20- assert fractional_sample_rate(100_000_000, query(60)) == 0.10- assert fractional_sample_rate(100_000_000, query(100)) == 0.06-- assert fractional_sample_rate(300_000_000, query(2)) == :no_sampling- assert fractional_sample_rate(300_000_000, query(5)) == 0.40- assert fractional_sample_rate(300_000_000, query(10)) == 0.20+ assert fractional_sample_rate(@threshold * 2, query(30)) == :no_sampling+ assert fractional_sample_rate(@threshold * 2, query(60)) == 0.25+ assert fractional_sample_rate(@threshold * 2, query(100)) == 0.15++ assert fractional_sample_rate(@threshold * 5, query(1)) == :no_sampling+ assert fractional_sample_rate(@threshold * 5, query(5)) == :no_sampling+ assert fractional_sample_rate(@threshold * 5, query(10)) == :no_sampling+ assert fractional_sample_rate(@threshold * 5, query(15)) == 0.40+ assert fractional_sample_rate(@threshold * 5, query(30)) == 0.20+ assert fractional_sample_rate(@threshold * 5, query(60)) == 0.10+ assert fractional_sample_rate(@threshold * 5, query(100)) == 0.06++ assert fractional_sample_rate(@threshold * 15, query(2)) == :no_sampling+ assert fractional_sample_rate(@threshold * 15, query(5)) == 0.40+ assert fractional_sample_rate(@threshold * 15, query(10)) == 0.20endtest "short durations" do- assert fractional_sample_rate(300_000_000_000, query(1, :hour)) == :no_sampling+ assert fractional_sample_rate(@threshold * 15, query(1, :hour)) == :no_samplingendtest "very low sampling rate" do- assert fractional_sample_rate(300_000_000_000, query(30)) == 0.01+ assert fractional_sample_rate(@threshold * 500, query(30)) == 0.01endend