Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Consolidated sampling cache (#5768)

Adam Rutkowskie5ea7a59fc05
diff --git a/extra/lib/plausible/stats/sampling.ex b/extra/lib/plausible/stats/sampling.ex
index 33abb160ab..84b9f16915 100644
--- a/extra/lib/plausible/stats/sampling.ex
+++ b/extra/lib/plausible/stats/sampling.ex
@@ -56,9 +56,15 @@ defmodule Plausible.Stats.Sampling do
defp decide_sample_rate(site, query) do
sampling_adjustments? = FunWithFlags.enabled?(:sampling_adjustments, for: site)
- site.id
- |> SamplingCache.get()
- |> fractional_sample_rate(query, sampling_adjustments?)
+ if Plausible.Sites.consolidated?(site) and not Enum.empty?(query.consolidated_site_ids) do
+ query.consolidated_site_ids
+ |> SamplingCache.consolidated_get()
+ |> fractional_sample_rate(query, sampling_adjustments?)
+ else
+ site.id
+ |> SamplingCache.get()
+ |> fractional_sample_rate(query, sampling_adjustments?)
+ end
end
def fractional_sample_rate(nil = _traffic_30_day, _query, _sampling_adjustments?),
diff --git a/extra/lib/plausible/stats/sampling_cache.ex b/extra/lib/plausible/stats/sampling_cache.ex
index 7f91e39470..d4d3daabb4 100644
--- a/extra/lib/plausible/stats/sampling_cache.ex
+++ b/extra/lib/plausible/stats/sampling_cache.ex
@@ -49,7 +49,13 @@ defmodule Plausible.Stats.SamplingCache do
|> Map.get(site_id)
end
- def thirty_days_ago() do
+ @spec consolidated_get(list(pos_integer()), Keyword.t()) :: pos_integer() | nil
+ def consolidated_get(site_ids, opts \\ []) when is_list(site_ids) do
+ events_ingested = Enum.sum_by(site_ids, &(get(&1, opts) || 0))
+ if events_ingested > 0, do: events_ingested
+ end
+
+ defp thirty_days_ago() do
Date.shift(Date.utc_today(), day: -30)
end
end
diff --git a/test/plausible/stats/sampling_cache_test.exs b/test/plausible/stats/sampling_cache_test.exs
index 59cba435e9..c32636579e 100644
--- a/test/plausible/stats/sampling_cache_test.exs
+++ b/test/plausible/stats/sampling_cache_test.exs
@@ -10,6 +10,18 @@ defmodule Plausible.Stats.SamplingCacheTest do
@site_id2 200_000
@site_id3 300_000
@site_id4 400_000
+ @site_id5 500_000
+ @site_id6 600_000
+
+ setup do
+ Plausible.IngestRepo.query!("truncate ingest_counters")
+
+ on_exit(fn ->
+ Plausible.IngestRepo.query!("truncate ingest_counters")
+ end)
+
+ :ok
+ end
describe "getter" do
@threshold Plausible.Stats.Sampling.default_sample_threshold()
@@ -17,45 +29,115 @@ defmodule Plausible.Stats.SamplingCacheTest do
test "returns cached values for traffic in past 30 days", %{test: test} do
now = DateTime.utc_now()
+ {6, _} =
+ Plausible.IngestRepo.insert_all(Plausible.Ingestion.Counters.Record, [
+ %{
+ site_id: @site_id1,
+ domain: "1.com",
+ value: (@threshold * 0.55) |> trunc(),
+ event_timebucket: add(now, -1, :day),
+ metric: "buffered"
+ },
+ %{
+ site_id: @site_id2,
+ domain: "2.com",
+ value: (@threshold * 0.55) |> trunc(),
+ event_timebucket: add(now, -1, :day),
+ metric: "buffered"
+ },
+ %{
+ site_id: @site_id2,
+ domain: "2.com",
+ value: (@threshold * 0.55) |> trunc(),
+ event_timebucket: add(now, -5, :day),
+ metric: "buffered"
+ },
+ %{
+ site_id: @site_id2,
+ domain: "2.com",
+ value: (@threshold * 0.55) |> trunc(),
+ event_timebucket: add(now, -35, :day),
+ metric: "buffered"
+ },
+ %{
+ site_id: @site_id3,
+ domain: "3.com",
+ value: (@threshold * 2.05) |> trunc(),
+ event_timebucket: add(now, -35, :day),
+ metric: "buffered"
+ },
+ %{
+ site_id: @site_id4,
+ domain: "4.com",
+ value: (@threshold * 0.55) |> trunc(),
+ event_timebucket: add(now, -35, :day),
+ metric: "buffered"
+ }
+ ])
+
+ start_test_cache(test)
+
+ assert SamplingCache.get(@site_id1, force?: true, cache_name: test) == nil
+ assert SamplingCache.get(@site_id2, force?: true, cache_name: test) == 1.1 * @threshold
+ assert SamplingCache.get(@site_id3, force?: true, cache_name: test) == nil
+ assert SamplingCache.get(@site_id4, force?: true, cache_name: test) == nil
+
+ assert SamplingCache.count_all() == 1
+
Plausible.IngestRepo.insert_all(Plausible.Ingestion.Counters.Record, [
%{
site_id: @site_id1,
- domain: "1.com",
value: (@threshold * 0.55) |> trunc(),
event_timebucket: add(now, -1, :day),
metric: "buffered"
- },
+ }
+ ])
+
+ :ok = SamplingCache.refresh_all(cache_name: test)
+
+ assert SamplingCache.count_all() == 2
+ assert SamplingCache.size(test) == 2
+ assert SamplingCache.get(@site_id1, force?: true, cache_name: test) == 1.1 * @threshold
+ assert SamplingCache.get(@site_id2, force?: true, cache_name: test) == 1.1 * @threshold
+ end
+
+ test "returns cached values for traffic in past 30 days for consolidated site", %{
+ test: test
+ } do
+ now = DateTime.utc_now()
+
+ Plausible.IngestRepo.insert_all(Plausible.Ingestion.Counters.Record, [
%{
- site_id: @site_id2,
- domain: "2.com",
+ site_id: @site_id5,
+ domain: "5.com",
value: (@threshold * 0.55) |> trunc(),
event_timebucket: add(now, -1, :day),
metric: "buffered"
},
%{
- site_id: @site_id2,
- domain: "2.com",
+ site_id: @site_id5,
+ domain: "5.com",
value: (@threshold * 0.55) |> trunc(),
- event_timebucket: add(now, -5, :day),
+ event_timebucket: add(now, -1, :day),
metric: "buffered"
},
%{
- site_id: @site_id2,
- domain: "2.com",
+ site_id: @site_id6,
+ domain: "6.com",
value: (@threshold * 0.55) |> trunc(),
- event_timebucket: add(now, -35, :day),
+ event_timebucket: add(now, -1, :day),
metric: "buffered"
},
%{
- site_id: @site_id3,
- domain: "3.com",
- value: (@threshold * 2.05) |> trunc(),
- event_timebucket: add(now, -35, :day),
+ site_id: @site_id6,
+ domain: "6.com",
+ value: (@threshold * 0.55) |> trunc(),
+ event_timebucket: add(now, -5, :day),
metric: "buffered"
},
%{
- site_id: @site_id4,
- domain: "4.com",
+ site_id: @site_id6,
+ domain: "6.com",
value: (@threshold * 0.55) |> trunc(),
event_timebucket: add(now, -35, :day),
metric: "buffered"
@@ -64,37 +146,32 @@ defmodule Plausible.Stats.SamplingCacheTest do
start_test_cache(test)
- assert SamplingCache.count_all() == 1
- assert SamplingCache.get(@site_id1, force?: true, cache_name: test) == nil
- assert SamplingCache.get(@site_id2, force?: true, cache_name: test) == 1.1 * @threshold
- assert SamplingCache.get(@site_id3, force?: true, cache_name: test) == nil
- assert SamplingCache.get(@site_id4, force?: true, cache_name: test) == nil
-
- Plausible.IngestRepo.insert_all(Plausible.Ingestion.Counters.Record, [
- %{
- site_id: @site_id1,
- value: (@threshold * 0.55) |> trunc(),
- event_timebucket: add(now, -1, :day),
- metric: "buffered"
- }
- ])
+ assert SamplingCache.consolidated_get([@site_id5, @site_id6],
+ cache_name: test,
+ force?: true
+ ) == 1.1 * @threshold * 2
+ end
- :ok = SamplingCache.refresh_all(cache_name: test)
+ test "conslidated_get returns nil", %{test: test} do
+ start_test_cache(test)
- assert SamplingCache.count_all() == 2
- assert SamplingCache.get(@site_id1, force?: true, cache_name: test) == 1.1 * @threshold
- assert SamplingCache.get(@site_id2, force?: true, cache_name: test) == 1.1 * @threshold
+ assert is_nil(
+ SamplingCache.consolidated_get([@site_id5, @site_id6],
+ cache_name: test,
+ force?: true
+ )
+ )
end
end
- def add(datetime, n, unit) do
+ defp add(datetime, n, unit) do
DateTime.add(datetime, n, unit) |> DateTime.truncate(:second)
end
defp start_test_cache(cache_name) do
%{start: {m, f, a}} = SamplingCache.child_spec(cache_name: cache_name)
- apply(m, f, a)
+ {:ok, _pid} = apply(m, f, a)
:ok = SamplingCache.refresh_all(cache_name: cache_name)
end
end