Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Sampling Cache: ensure consolidated values are reported; cache all sites, not only >threshold (#5783)

Adam Rutkowski5976a6ae6a30
diff --git a/extra/lib/plausible/stats/sampling_cache.ex b/extra/lib/plausible/stats/sampling_cache.ex
index d4d3daabb4..98fea262d2 100644
--- a/extra/lib/plausible/stats/sampling_cache.ex
+++ b/extra/lib/plausible/stats/sampling_cache.ex
@@ -1,11 +1,10 @@
defmodule Plausible.Stats.SamplingCache do
@moduledoc """
- Cache storing estimation for events ingested by a team in the past month.
+ Cache storing estimation for events ingested by a site in the past month.
Used for sampling rate calculations in Plausible.Stats.Sampling.
"""
alias Plausible.Ingestion
- alias Plausible.Stats.Sampling
import Ecto.Query
use Plausible.Cache
@@ -36,8 +35,7 @@ defmodule Plausible.Stats.SamplingCache do
selected_as(fragment("sumIf(value, metric = 'buffered')"), :events_ingested)
},
where: fragment("toDate(event_timebucket) >= ?", ^thirty_days_ago()),
- group_by: r.site_id,
- having: selected_as(:events_ingested) > ^Sampling.default_sample_threshold()
+ group_by: r.site_id
)
end
diff --git a/lib/plausible/cache.ex b/lib/plausible/cache.ex
index 969909d0e3..8790d67399 100644
--- a/lib/plausible/cache.ex
+++ b/lib/plausible/cache.ex
@@ -78,7 +78,9 @@ defmodule Plausible.Cache do
end
@spec get(any(), Keyword.t()) :: any() | nil
- def get(key, opts \\ []) when is_list(opts) do
+ def get(key, opts \\ [])
+
+ def get(key, opts) when is_list(opts) do
cache_name = Keyword.get(opts, :cache_name, name())
force? = Keyword.get(opts, :force?, false)
@@ -89,6 +91,8 @@ defmodule Plausible.Cache do
end
end
+ defoverridable get: 2
+
@spec get_or_store(any(), (-> any()), Keyword.t()) :: any() | nil
def get_or_store(key, fallback_fn, opts \\ [])
when is_function(fallback_fn, 0) and is_list(opts) do
diff --git a/test/plausible/stats/sampling_cache_test.exs b/test/plausible/stats/sampling_cache_test.exs
index c32636579e..06364e8716 100644
--- a/test/plausible/stats/sampling_cache_test.exs
+++ b/test/plausible/stats/sampling_cache_test.exs
@@ -12,6 +12,9 @@ defmodule Plausible.Stats.SamplingCacheTest do
@site_id4 400_000
@site_id5 500_000
@site_id6 600_000
+ @site_id7 700_000
+ @site_id8 800_000
+ @site_id9 900_000
setup do
Plausible.IngestRepo.query!("truncate ingest_counters")
@@ -77,13 +80,11 @@ defmodule Plausible.Stats.SamplingCacheTest do
start_test_cache(test)
- assert SamplingCache.get(@site_id1, force?: true, cache_name: test) == nil
+ assert SamplingCache.get(@site_id1, force?: true, cache_name: test) == 5_500_000
assert SamplingCache.get(@site_id2, force?: true, cache_name: test) == 1.1 * @threshold
assert SamplingCache.get(@site_id3, force?: true, cache_name: test) == nil
assert SamplingCache.get(@site_id4, force?: true, cache_name: test) == nil
- assert SamplingCache.count_all() == 1
-
Plausible.IngestRepo.insert_all(Plausible.Ingestion.Counters.Record, [
%{
site_id: @site_id1,
@@ -95,8 +96,6 @@ defmodule Plausible.Stats.SamplingCacheTest do
:ok = SamplingCache.refresh_all(cache_name: test)
- assert SamplingCache.count_all() == 2
- assert SamplingCache.size(test) == 2
assert SamplingCache.get(@site_id1, force?: true, cache_name: test) == 1.1 * @threshold
assert SamplingCache.get(@site_id2, force?: true, cache_name: test) == 1.1 * @threshold
end
@@ -152,6 +151,48 @@ defmodule Plausible.Stats.SamplingCacheTest do
) == 1.1 * @threshold * 2
end
+ test "consolidated_get sum over threshold", %{
+ test: test
+ } do
+ now = DateTime.utc_now()
+
+ Plausible.IngestRepo.insert_all(Plausible.Ingestion.Counters.Record, [
+ %{
+ site_id: @site_id7,
+ domain: "7.com",
+ value: div(@threshold, 2),
+ event_timebucket: add(now, -1, :day),
+ metric: "buffered"
+ },
+ %{
+ site_id: @site_id8,
+ domain: "8.com",
+ value: div(@threshold, 2),
+ event_timebucket: add(now, -1, :day),
+ metric: "buffered"
+ },
+ %{
+ site_id: @site_id9,
+ domain: "9.com",
+ value: div(@threshold, 2),
+ event_timebucket: add(now, -1, :day),
+ metric: "buffered"
+ }
+ ])
+
+ start_test_cache(test)
+
+ assert SamplingCache.consolidated_get([@site_id7, @site_id8],
+ cache_name: test,
+ force?: true
+ ) == @threshold
+
+ assert SamplingCache.consolidated_get([@site_id9],
+ cache_name: test,
+ force?: true
+ ) == div(@threshold, 2)
+ end
+
test "conslidated_get returns nil", %{test: test} do
start_test_cache(test)
diff --git a/test/plausible/stats/sampling_test.exs b/test/plausible/stats/sampling_test.exs
index 02faf5cf56..fc011f72e0 100644
--- a/test/plausible/stats/sampling_test.exs
+++ b/test/plausible/stats/sampling_test.exs
@@ -5,16 +5,32 @@ defmodule Plausible.Stats.SamplingTest do
on_ee do
import Plausible.Stats.Sampling, only: [fractional_sample_rate: 2]
- alias Plausible.Stats.{Query, DateTimeRange, Sampling}
+ alias Plausible.Stats.{Query, DateTimeRange}
describe "&fractional_sample_rate/2" do
- @threshold Sampling.default_sample_threshold()
+ @threshold Plausible.Stats.Sampling.default_sample_threshold()
test "no traffic estimate" do
assert fractional_sample_rate(nil, query(30)) == :no_sampling
end
test "scales sampling rate according to query duration" do
+ for n <- [1, 100, 1000, 10_000, 100_000, 1_000_000, 7_000_000] do
+ assert {n, fractional_sample_rate(n, query(1))} == {n, :no_sampling}
+ assert {n, fractional_sample_rate(n, query(5))} == {n, :no_sampling}
+ assert {n, fractional_sample_rate(n, query(10))} == {n, :no_sampling}
+ assert {n, fractional_sample_rate(n, query(15))} == {n, :no_sampling}
+ assert {n, fractional_sample_rate(n, query(30))} == {n, :no_sampling}
+ assert {n, fractional_sample_rate(n, query(60))} == {n, :no_sampling}
+ assert {n, fractional_sample_rate(n, query(100))} == {n, :no_sampling}
+ end
+
+ assert fractional_sample_rate(6_000_000, query(300)) == 0.17
+ assert fractional_sample_rate(7_500_000, query(100)) == 0.4
+ assert fractional_sample_rate(7_500_000, query(364)) == 0.11
+ assert fractional_sample_rate(7_500_000, query(900)) == 0.04
+ assert fractional_sample_rate(7_500_000, query(2900)) == 0.013
+
assert fractional_sample_rate(@threshold * 2, query(30)) == :no_sampling
assert fractional_sample_rate(@threshold * 2, query(60)) == 0.25
assert fractional_sample_rate(@threshold * 2, query(100)) == 0.15