Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Consolidated sampling cache (#5768)
Adam Rutkowskie5ea7a59fc05
diff --git a/extra/lib/plausible/stats/sampling.ex b/extra/lib/plausible/stats/sampling.exindex 33abb160ab..84b9f16915 100644--- a/extra/lib/plausible/stats/sampling.ex+++ b/extra/lib/plausible/stats/sampling.ex@@ -56,9 +56,15 @@ defmodule Plausible.Stats.Sampling dodefp decide_sample_rate(site, query) dosampling_adjustments? = FunWithFlags.enabled?(:sampling_adjustments, for: site)- site.id- |> SamplingCache.get()- |> fractional_sample_rate(query, sampling_adjustments?)+ if Plausible.Sites.consolidated?(site) and not Enum.empty?(query.consolidated_site_ids) do+ query.consolidated_site_ids+ |> SamplingCache.consolidated_get()+ |> fractional_sample_rate(query, sampling_adjustments?)+ else+ site.id+ |> SamplingCache.get()+ |> fractional_sample_rate(query, sampling_adjustments?)+ endenddef fractional_sample_rate(nil = _traffic_30_day, _query, _sampling_adjustments?),diff --git a/extra/lib/plausible/stats/sampling_cache.ex b/extra/lib/plausible/stats/sampling_cache.exindex 7f91e39470..d4d3daabb4 100644--- a/extra/lib/plausible/stats/sampling_cache.ex+++ b/extra/lib/plausible/stats/sampling_cache.ex@@ -49,7 +49,13 @@ defmodule Plausible.Stats.SamplingCache do|> Map.get(site_id)end- def thirty_days_ago() do+ @spec consolidated_get(list(pos_integer()), Keyword.t()) :: pos_integer() | nil+ def consolidated_get(site_ids, opts \\ []) when is_list(site_ids) do+ events_ingested = Enum.sum_by(site_ids, &(get(&1, opts) || 0))+ if events_ingested > 0, do: events_ingested+ end++ defp thirty_days_ago() doDate.shift(Date.utc_today(), day: -30)endenddiff --git a/test/plausible/stats/sampling_cache_test.exs b/test/plausible/stats/sampling_cache_test.exsindex 59cba435e9..c32636579e 100644--- a/test/plausible/stats/sampling_cache_test.exs+++ b/test/plausible/stats/sampling_cache_test.exs@@ -10,6 +10,18 @@ defmodule Plausible.Stats.SamplingCacheTest do@site_id2 200_000@site_id3 300_000@site_id4 400_000+ @site_id5 500_000+ @site_id6 600_000++ setup do+ Plausible.IngestRepo.query!("truncate ingest_counters")++ on_exit(fn ->+ Plausible.IngestRepo.query!("truncate ingest_counters")+ end)++ :ok+ enddescribe "getter" do@threshold Plausible.Stats.Sampling.default_sample_threshold()@@ -17,45 +29,115 @@ defmodule Plausible.Stats.SamplingCacheTest dotest "returns cached values for traffic in past 30 days", %{test: test} donow = DateTime.utc_now()+ {6, _} =+ Plausible.IngestRepo.insert_all(Plausible.Ingestion.Counters.Record, [+ %{+ site_id: @site_id1,+ domain: "1.com",+ value: (@threshold * 0.55) |> trunc(),+ event_timebucket: add(now, -1, :day),+ metric: "buffered"+ },+ %{+ site_id: @site_id2,+ domain: "2.com",+ value: (@threshold * 0.55) |> trunc(),+ event_timebucket: add(now, -1, :day),+ metric: "buffered"+ },+ %{+ site_id: @site_id2,+ domain: "2.com",+ value: (@threshold * 0.55) |> trunc(),+ event_timebucket: add(now, -5, :day),+ metric: "buffered"+ },+ %{+ site_id: @site_id2,+ domain: "2.com",+ value: (@threshold * 0.55) |> trunc(),+ event_timebucket: add(now, -35, :day),+ metric: "buffered"+ },+ %{+ site_id: @site_id3,+ domain: "3.com",+ value: (@threshold * 2.05) |> trunc(),+ event_timebucket: add(now, -35, :day),+ metric: "buffered"+ },+ %{+ site_id: @site_id4,+ domain: "4.com",+ value: (@threshold * 0.55) |> trunc(),+ event_timebucket: add(now, -35, :day),+ metric: "buffered"+ }+ ])++ start_test_cache(test)++ assert SamplingCache.get(@site_id1, force?: true, cache_name: test) == nil+ assert SamplingCache.get(@site_id2, force?: true, cache_name: test) == 1.1 * @threshold+ assert SamplingCache.get(@site_id3, force?: true, cache_name: test) == nil+ assert SamplingCache.get(@site_id4, force?: true, cache_name: test) == nil++ assert SamplingCache.count_all() == 1+Plausible.IngestRepo.insert_all(Plausible.Ingestion.Counters.Record, [%{site_id: @site_id1,- domain: "1.com",value: (@threshold * 0.55) |> trunc(),event_timebucket: add(now, -1, :day),metric: "buffered"- },+ }+ ])++ :ok = SamplingCache.refresh_all(cache_name: test)++ assert SamplingCache.count_all() == 2+ assert SamplingCache.size(test) == 2+ assert SamplingCache.get(@site_id1, force?: true, cache_name: test) == 1.1 * @threshold+ assert SamplingCache.get(@site_id2, force?: true, cache_name: test) == 1.1 * @threshold+ end++ test "returns cached values for traffic in past 30 days for consolidated site", %{+ test: test+ } do+ now = DateTime.utc_now()++ Plausible.IngestRepo.insert_all(Plausible.Ingestion.Counters.Record, [%{- site_id: @site_id2,- domain: "2.com",+ site_id: @site_id5,+ domain: "5.com",value: (@threshold * 0.55) |> trunc(),event_timebucket: add(now, -1, :day),metric: "buffered"},%{- site_id: @site_id2,- domain: "2.com",+ site_id: @site_id5,+ domain: "5.com",value: (@threshold * 0.55) |> trunc(),- event_timebucket: add(now, -5, :day),+ event_timebucket: add(now, -1, :day),metric: "buffered"},%{- site_id: @site_id2,- domain: "2.com",+ site_id: @site_id6,+ domain: "6.com",value: (@threshold * 0.55) |> trunc(),- event_timebucket: add(now, -35, :day),+ event_timebucket: add(now, -1, :day),metric: "buffered"},%{- site_id: @site_id3,- domain: "3.com",- value: (@threshold * 2.05) |> trunc(),- event_timebucket: add(now, -35, :day),+ site_id: @site_id6,+ domain: "6.com",+ value: (@threshold * 0.55) |> trunc(),+ event_timebucket: add(now, -5, :day),metric: "buffered"},%{- site_id: @site_id4,- domain: "4.com",+ site_id: @site_id6,+ domain: "6.com",value: (@threshold * 0.55) |> trunc(),event_timebucket: add(now, -35, :day),metric: "buffered"@@ -64,37 +146,32 @@ defmodule Plausible.Stats.SamplingCacheTest dostart_test_cache(test)- assert SamplingCache.count_all() == 1- assert SamplingCache.get(@site_id1, force?: true, cache_name: test) == nil- assert SamplingCache.get(@site_id2, force?: true, cache_name: test) == 1.1 * @threshold- assert SamplingCache.get(@site_id3, force?: true, cache_name: test) == nil- assert SamplingCache.get(@site_id4, force?: true, cache_name: test) == nil-- Plausible.IngestRepo.insert_all(Plausible.Ingestion.Counters.Record, [- %{- site_id: @site_id1,- value: (@threshold * 0.55) |> trunc(),- event_timebucket: add(now, -1, :day),- metric: "buffered"- }- ])+ assert SamplingCache.consolidated_get([@site_id5, @site_id6],+ cache_name: test,+ force?: true+ ) == 1.1 * @threshold * 2+ end- :ok = SamplingCache.refresh_all(cache_name: test)+ test "conslidated_get returns nil", %{test: test} do+ start_test_cache(test)- assert SamplingCache.count_all() == 2- assert SamplingCache.get(@site_id1, force?: true, cache_name: test) == 1.1 * @threshold- assert SamplingCache.get(@site_id2, force?: true, cache_name: test) == 1.1 * @threshold+ assert is_nil(+ SamplingCache.consolidated_get([@site_id5, @site_id6],+ cache_name: test,+ force?: true+ )+ )endend- def add(datetime, n, unit) do+ defp add(datetime, n, unit) doDateTime.add(datetime, n, unit) |> DateTime.truncate(:second)enddefp start_test_cache(cache_name) do%{start: {m, f, a}} = SamplingCache.child_spec(cache_name: cache_name)- apply(m, f, a)+ {:ok, _pid} = apply(m, f, a):ok = SamplingCache.refresh_all(cache_name: cache_name)endend