Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Cut off 24h sites stats at respective native stats start dates (#4439)
Adrian Gruntkowski77d841221b63
diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.exindex bf25559150..69437de694 100644--- a/lib/plausible/stats/clickhouse.ex+++ b/lib/plausible/stats/clickhouse.ex@@ -4,7 +4,7 @@ defmodule Plausible.Stats.Clickhouse douse Plausible.ClickhouseRepouse Plausible.Stats.SQL.Fragments- import Ecto.Query, only: [from: 2]+ import Ecto.Query, only: [from: 2, dynamic: 1, dynamic: 2]alias Plausible.Stats.Queryalias Plausible.Timezones@@ -147,7 +147,9 @@ defmodule Plausible.Stats.Clickhouse doplaceholder = empty_24h_visitors_hourly_intervals(sites, now)- previous_query = visitors_24h_total(now, -48, -24, site_id_to_domain_mapping)+ base_query = base_24h_events_query(sites)++ previous_query = visitors_24h_total(base_query, now, -48, -24)previous_result =previous_query@@ -156,14 +158,13 @@ defmodule Plausible.Stats.Clickhouse do%{total_visitors: total, site_id: site_id}, acc -> Map.put_new(acc, site_id, total)end)- total_q = visitors_24h_total(now, -24, 0, site_id_to_domain_mapping)+ total_q = visitors_24h_total(base_query, now, -24, 0)current_q =from(- e in "events_v2",+ e in base_query,join: total_q in subquery(total_q),on: e.site_id == total_q.site_id,- where: e.site_id in ^Map.keys(site_id_to_domain_mapping),where: e.timestamp >= ^NaiveDateTime.add(now, -24, :hour),where: e.timestamp <= ^now,select: %{@@ -202,10 +203,9 @@ defmodule Plausible.Stats.Clickhouse doMap.merge(placeholder, result)end- defp visitors_24h_total(now, offset1, offset2, site_id_to_domain_mapping) do+ defp visitors_24h_total(base_query, now, offset1, offset2) doquery =- from e in "events_v2",- where: e.site_id in ^Map.keys(site_id_to_domain_mapping),+ from e in base_query,where: e.timestamp >= ^NaiveDateTime.add(now, offset1, :hour),where: e.timestamp <= ^NaiveDateTime.add(now, offset2, :hour),select: %{@@ -221,6 +221,24 @@ defmodule Plausible.Stats.Clickhouse doqueryend+ defp base_24h_events_query(sites) do+ cutoff_times_condition =+ Enum.reduce(sites, dynamic(false), fn+ %{native_stats_start_at: nil}, dynamic ->+ dynamic++ site, dynamic ->+ dynamic(+ [e],+ ^dynamic or (e.site_id == ^site.id and e.timestamp >= ^site.native_stats_start_at)+ )+ end)++ from e in "events_v2",+ where: e.site_id in ^Enum.map(sites, & &1.id),+ where: ^cutoff_times_condition+ end+defp empty_24h_intervals(now) dofirst = NaiveDateTime.add(now, -23, :hour){:ok, time} = Time.new(first.hour, 0, 0)diff --git a/test/plausible/stats/clickhouse_test.exs b/test/plausible/stats/clickhouse_test.exsindex 7175d719f4..6dfd9f1990 100644--- a/test/plausible/stats/clickhouse_test.exs+++ b/test/plausible/stats/clickhouse_test.exs@@ -98,6 +98,93 @@ defmodule Plausible.Stats.ClickhouseTest do} = Clickhouse.last_24h_visitors_hourly_intervals([site], fixed_now)[site.domain]end+ test "ignores visits before native stats start time (after reset)" do+ fixed_now = ~N[2023-10-26 10:00:15]+ site1 = insert(:site, native_stats_start_at: ~N[2023-10-25 14:15:00])+ site2 = insert(:site, native_stats_start_at: ~N[2023-10-23 12:00:00])++ user_id1 = 111+ user_id2 = 222++ populate_stats(site1, [+ build(:pageview, timestamp: ~N[2023-10-25 13:59:00]),+ build(:pageview, timestamp: ~N[2023-10-25 13:58:00]),+ build(:pageview, user_id: user_id1, timestamp: ~N[2023-10-25 15:00:00]),+ build(:pageview, user_id: user_id1, timestamp: ~N[2023-10-25 15:01:00])+ ])++ populate_stats(site2, [+ build(:pageview, timestamp: ~N[2023-10-25 13:59:00]),+ build(:pageview, timestamp: ~N[2023-10-25 13:58:00]),+ build(:pageview, user_id: user_id2, timestamp: ~N[2023-10-25 15:00:00]),+ build(:pageview, user_id: user_id2, timestamp: ~N[2023-10-25 15:01:00])+ ])++ assert %{+ change: 100,+ visitors: 1,+ intervals: [+ %{interval: ~N[2023-10-25 11:00:00], visitors: 0},+ %{interval: ~N[2023-10-25 12:00:00], visitors: 0},+ %{interval: ~N[2023-10-25 13:00:00], visitors: 0},+ %{interval: ~N[2023-10-25 14:00:00], visitors: 0},+ %{interval: ~N[2023-10-25 15:00:00], visitors: 1},+ %{interval: ~N[2023-10-25 16:00:00], visitors: 0},+ %{interval: ~N[2023-10-25 17:00:00], visitors: 0},+ %{interval: ~N[2023-10-25 18:00:00], visitors: 0},+ %{interval: ~N[2023-10-25 19:00:00], visitors: 0},+ %{interval: ~N[2023-10-25 20:00:00], visitors: 0},+ %{interval: ~N[2023-10-25 21:00:00], visitors: 0},+ %{interval: ~N[2023-10-25 22:00:00], visitors: 0},+ %{interval: ~N[2023-10-25 23:00:00], visitors: 0},+ %{interval: ~N[2023-10-26 00:00:00], visitors: 0},+ %{interval: ~N[2023-10-26 01:00:00], visitors: 0},+ %{interval: ~N[2023-10-26 02:00:00], visitors: 0},+ %{interval: ~N[2023-10-26 03:00:00], visitors: 0},+ %{interval: ~N[2023-10-26 04:00:00], visitors: 0},+ %{interval: ~N[2023-10-26 05:00:00], visitors: 0},+ %{interval: ~N[2023-10-26 06:00:00], visitors: 0},+ %{interval: ~N[2023-10-26 07:00:00], visitors: 0},+ %{interval: ~N[2023-10-26 08:00:00], visitors: 0},+ %{interval: ~N[2023-10-26 09:00:00], visitors: 0},+ %{interval: ~N[2023-10-26 10:00:00], visitors: 0},+ %{interval: ~N[2023-10-26 11:00:00], visitors: 0}+ ]+ } = Clickhouse.last_24h_visitors_hourly_intervals([site1], fixed_now)[site1.domain]++ assert %{+ change: 100,+ visitors: 3,+ intervals: [+ %{interval: ~N[2023-10-25 11:00:00], visitors: 0},+ %{interval: ~N[2023-10-25 12:00:00], visitors: 0},+ %{interval: ~N[2023-10-25 13:00:00], visitors: 2},+ %{interval: ~N[2023-10-25 14:00:00], visitors: 0},+ %{interval: ~N[2023-10-25 15:00:00], visitors: 1},+ %{interval: ~N[2023-10-25 16:00:00], visitors: 0},+ %{interval: ~N[2023-10-25 17:00:00], visitors: 0},+ %{interval: ~N[2023-10-25 18:00:00], visitors: 0},+ %{interval: ~N[2023-10-25 19:00:00], visitors: 0},+ %{interval: ~N[2023-10-25 20:00:00], visitors: 0},+ %{interval: ~N[2023-10-25 21:00:00], visitors: 0},+ %{interval: ~N[2023-10-25 22:00:00], visitors: 0},+ %{interval: ~N[2023-10-25 23:00:00], visitors: 0},+ %{interval: ~N[2023-10-26 00:00:00], visitors: 0},+ %{interval: ~N[2023-10-26 01:00:00], visitors: 0},+ %{interval: ~N[2023-10-26 02:00:00], visitors: 0},+ %{interval: ~N[2023-10-26 03:00:00], visitors: 0},+ %{interval: ~N[2023-10-26 04:00:00], visitors: 0},+ %{interval: ~N[2023-10-26 05:00:00], visitors: 0},+ %{interval: ~N[2023-10-26 06:00:00], visitors: 0},+ %{interval: ~N[2023-10-26 07:00:00], visitors: 0},+ %{interval: ~N[2023-10-26 08:00:00], visitors: 0},+ %{interval: ~N[2023-10-26 09:00:00], visitors: 0},+ %{interval: ~N[2023-10-26 10:00:00], visitors: 0},+ %{interval: ~N[2023-10-26 11:00:00], visitors: 0}+ ]+ } = Clickhouse.last_24h_visitors_hourly_intervals([site2], fixed_now)[site2.domain]+ end+test "returns clickhouse data merged with placeholder for multiple sites" dofixed_now = ~N[2023-10-26 10:00:15]site1 = insert(:site)