Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Sampling Cache: ensure consolidated values are reported; cache all sites, not only >threshold (#5783)
Adam Rutkowski5976a6ae6a30
diff --git a/extra/lib/plausible/stats/sampling_cache.ex b/extra/lib/plausible/stats/sampling_cache.exindex d4d3daabb4..98fea262d2 100644--- a/extra/lib/plausible/stats/sampling_cache.ex+++ b/extra/lib/plausible/stats/sampling_cache.ex@@ -1,11 +1,10 @@defmodule Plausible.Stats.SamplingCache do@moduledoc """- Cache storing estimation for events ingested by a team in the past month.+ Cache storing estimation for events ingested by a site in the past month.Used for sampling rate calculations in Plausible.Stats.Sampling."""alias Plausible.Ingestion- alias Plausible.Stats.Samplingimport Ecto.Queryuse Plausible.Cache@@ -36,8 +35,7 @@ defmodule Plausible.Stats.SamplingCache doselected_as(fragment("sumIf(value, metric = 'buffered')"), :events_ingested)},where: fragment("toDate(event_timebucket) >= ?", ^thirty_days_ago()),- group_by: r.site_id,- having: selected_as(:events_ingested) > ^Sampling.default_sample_threshold()+ group_by: r.site_id)enddiff --git a/lib/plausible/cache.ex b/lib/plausible/cache.exindex 969909d0e3..8790d67399 100644--- a/lib/plausible/cache.ex+++ b/lib/plausible/cache.ex@@ -78,7 +78,9 @@ defmodule Plausible.Cache doend@spec get(any(), Keyword.t()) :: any() | nil- def get(key, opts \\ []) when is_list(opts) do+ def get(key, opts \\ [])++ def get(key, opts) when is_list(opts) docache_name = Keyword.get(opts, :cache_name, name())force? = Keyword.get(opts, :force?, false)@@ -89,6 +91,8 @@ defmodule Plausible.Cache doendend+ defoverridable get: 2+@spec get_or_store(any(), (-> any()), Keyword.t()) :: any() | nildef get_or_store(key, fallback_fn, opts \\ [])when is_function(fallback_fn, 0) and is_list(opts) dodiff --git a/test/plausible/stats/sampling_cache_test.exs b/test/plausible/stats/sampling_cache_test.exsindex c32636579e..06364e8716 100644--- a/test/plausible/stats/sampling_cache_test.exs+++ b/test/plausible/stats/sampling_cache_test.exs@@ -12,6 +12,9 @@ defmodule Plausible.Stats.SamplingCacheTest do@site_id4 400_000@site_id5 500_000@site_id6 600_000+ @site_id7 700_000+ @site_id8 800_000+ @site_id9 900_000setup doPlausible.IngestRepo.query!("truncate ingest_counters")@@ -77,13 +80,11 @@ defmodule Plausible.Stats.SamplingCacheTest dostart_test_cache(test)- assert SamplingCache.get(@site_id1, force?: true, cache_name: test) == nil+ assert SamplingCache.get(@site_id1, force?: true, cache_name: test) == 5_500_000assert SamplingCache.get(@site_id2, force?: true, cache_name: test) == 1.1 * @thresholdassert SamplingCache.get(@site_id3, force?: true, cache_name: test) == nilassert SamplingCache.get(@site_id4, force?: true, cache_name: test) == nil- assert SamplingCache.count_all() == 1-Plausible.IngestRepo.insert_all(Plausible.Ingestion.Counters.Record, [%{site_id: @site_id1,@@ -95,8 +96,6 @@ defmodule Plausible.Stats.SamplingCacheTest do:ok = SamplingCache.refresh_all(cache_name: test)- assert SamplingCache.count_all() == 2- assert SamplingCache.size(test) == 2assert SamplingCache.get(@site_id1, force?: true, cache_name: test) == 1.1 * @thresholdassert SamplingCache.get(@site_id2, force?: true, cache_name: test) == 1.1 * @thresholdend@@ -152,6 +151,48 @@ defmodule Plausible.Stats.SamplingCacheTest do) == 1.1 * @threshold * 2end+ test "consolidated_get sum over threshold", %{+ test: test+ } do+ now = DateTime.utc_now()++ Plausible.IngestRepo.insert_all(Plausible.Ingestion.Counters.Record, [+ %{+ site_id: @site_id7,+ domain: "7.com",+ value: div(@threshold, 2),+ event_timebucket: add(now, -1, :day),+ metric: "buffered"+ },+ %{+ site_id: @site_id8,+ domain: "8.com",+ value: div(@threshold, 2),+ event_timebucket: add(now, -1, :day),+ metric: "buffered"+ },+ %{+ site_id: @site_id9,+ domain: "9.com",+ value: div(@threshold, 2),+ event_timebucket: add(now, -1, :day),+ metric: "buffered"+ }+ ])++ start_test_cache(test)++ assert SamplingCache.consolidated_get([@site_id7, @site_id8],+ cache_name: test,+ force?: true+ ) == @threshold++ assert SamplingCache.consolidated_get([@site_id9],+ cache_name: test,+ force?: true+ ) == div(@threshold, 2)+ end+test "conslidated_get returns nil", %{test: test} dostart_test_cache(test)diff --git a/test/plausible/stats/sampling_test.exs b/test/plausible/stats/sampling_test.exsindex 02faf5cf56..fc011f72e0 100644--- a/test/plausible/stats/sampling_test.exs+++ b/test/plausible/stats/sampling_test.exs@@ -5,16 +5,32 @@ defmodule Plausible.Stats.SamplingTest doon_ee doimport Plausible.Stats.Sampling, only: [fractional_sample_rate: 2]- alias Plausible.Stats.{Query, DateTimeRange, Sampling}+ alias Plausible.Stats.{Query, DateTimeRange}describe "&fractional_sample_rate/2" do- @threshold Sampling.default_sample_threshold()+ @threshold Plausible.Stats.Sampling.default_sample_threshold()test "no traffic estimate" doassert fractional_sample_rate(nil, query(30)) == :no_samplingendtest "scales sampling rate according to query duration" do+ for n <- [1, 100, 1000, 10_000, 100_000, 1_000_000, 7_000_000] do+ assert {n, fractional_sample_rate(n, query(1))} == {n, :no_sampling}+ assert {n, fractional_sample_rate(n, query(5))} == {n, :no_sampling}+ assert {n, fractional_sample_rate(n, query(10))} == {n, :no_sampling}+ assert {n, fractional_sample_rate(n, query(15))} == {n, :no_sampling}+ assert {n, fractional_sample_rate(n, query(30))} == {n, :no_sampling}+ assert {n, fractional_sample_rate(n, query(60))} == {n, :no_sampling}+ assert {n, fractional_sample_rate(n, query(100))} == {n, :no_sampling}+ end++ assert fractional_sample_rate(6_000_000, query(300)) == 0.17+ assert fractional_sample_rate(7_500_000, query(100)) == 0.4+ assert fractional_sample_rate(7_500_000, query(364)) == 0.11+ assert fractional_sample_rate(7_500_000, query(900)) == 0.04+ assert fractional_sample_rate(7_500_000, query(2900)) == 0.013+assert fractional_sample_rate(@threshold * 2, query(30)) == :no_samplingassert fractional_sample_rate(@threshold * 2, query(60)) == 0.25assert fractional_sample_rate(@threshold * 2, query(100)) == 0.15