Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Cut off 24h sites stats at respective native stats start dates (#4439)

Adrian Gruntkowski77d841221b63
diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex
index bf25559150..69437de694 100644
--- a/lib/plausible/stats/clickhouse.ex
+++ b/lib/plausible/stats/clickhouse.ex
@@ -4,7 +4,7 @@ defmodule Plausible.Stats.Clickhouse do
use Plausible.ClickhouseRepo
use Plausible.Stats.SQL.Fragments
- import Ecto.Query, only: [from: 2]
+ import Ecto.Query, only: [from: 2, dynamic: 1, dynamic: 2]
alias Plausible.Stats.Query
alias Plausible.Timezones
@@ -147,7 +147,9 @@ defmodule Plausible.Stats.Clickhouse do
placeholder = empty_24h_visitors_hourly_intervals(sites, now)
- previous_query = visitors_24h_total(now, -48, -24, site_id_to_domain_mapping)
+ base_query = base_24h_events_query(sites)
+
+ previous_query = visitors_24h_total(base_query, now, -48, -24)
previous_result =
previous_query
@@ -156,14 +158,13 @@ defmodule Plausible.Stats.Clickhouse do
%{total_visitors: total, site_id: site_id}, acc -> Map.put_new(acc, site_id, total)
end)
- total_q = visitors_24h_total(now, -24, 0, site_id_to_domain_mapping)
+ total_q = visitors_24h_total(base_query, now, -24, 0)
current_q =
from(
- e in "events_v2",
+ e in base_query,
join: total_q in subquery(total_q),
on: e.site_id == total_q.site_id,
- where: e.site_id in ^Map.keys(site_id_to_domain_mapping),
where: e.timestamp >= ^NaiveDateTime.add(now, -24, :hour),
where: e.timestamp <= ^now,
select: %{
@@ -202,10 +203,9 @@ defmodule Plausible.Stats.Clickhouse do
Map.merge(placeholder, result)
end
- defp visitors_24h_total(now, offset1, offset2, site_id_to_domain_mapping) do
+ defp visitors_24h_total(base_query, now, offset1, offset2) do
query =
- from e in "events_v2",
- where: e.site_id in ^Map.keys(site_id_to_domain_mapping),
+ from e in base_query,
where: e.timestamp >= ^NaiveDateTime.add(now, offset1, :hour),
where: e.timestamp <= ^NaiveDateTime.add(now, offset2, :hour),
select: %{
@@ -221,6 +221,24 @@ defmodule Plausible.Stats.Clickhouse do
query
end
+ defp base_24h_events_query(sites) do
+ cutoff_times_condition =
+ Enum.reduce(sites, dynamic(false), fn
+ %{native_stats_start_at: nil}, dynamic ->
+ dynamic
+
+ site, dynamic ->
+ dynamic(
+ [e],
+ ^dynamic or (e.site_id == ^site.id and e.timestamp >= ^site.native_stats_start_at)
+ )
+ end)
+
+ from e in "events_v2",
+ where: e.site_id in ^Enum.map(sites, & &1.id),
+ where: ^cutoff_times_condition
+ end
+
defp empty_24h_intervals(now) do
first = NaiveDateTime.add(now, -23, :hour)
{:ok, time} = Time.new(first.hour, 0, 0)
diff --git a/test/plausible/stats/clickhouse_test.exs b/test/plausible/stats/clickhouse_test.exs
index 7175d719f4..6dfd9f1990 100644
--- a/test/plausible/stats/clickhouse_test.exs
+++ b/test/plausible/stats/clickhouse_test.exs
@@ -98,6 +98,93 @@ defmodule Plausible.Stats.ClickhouseTest do
} = Clickhouse.last_24h_visitors_hourly_intervals([site], fixed_now)[site.domain]
end
+ test "ignores visits before native stats start time (after reset)" do
+ fixed_now = ~N[2023-10-26 10:00:15]
+ site1 = insert(:site, native_stats_start_at: ~N[2023-10-25 14:15:00])
+ site2 = insert(:site, native_stats_start_at: ~N[2023-10-23 12:00:00])
+
+ user_id1 = 111
+ user_id2 = 222
+
+ populate_stats(site1, [
+ build(:pageview, timestamp: ~N[2023-10-25 13:59:00]),
+ build(:pageview, timestamp: ~N[2023-10-25 13:58:00]),
+ build(:pageview, user_id: user_id1, timestamp: ~N[2023-10-25 15:00:00]),
+ build(:pageview, user_id: user_id1, timestamp: ~N[2023-10-25 15:01:00])
+ ])
+
+ populate_stats(site2, [
+ build(:pageview, timestamp: ~N[2023-10-25 13:59:00]),
+ build(:pageview, timestamp: ~N[2023-10-25 13:58:00]),
+ build(:pageview, user_id: user_id2, timestamp: ~N[2023-10-25 15:00:00]),
+ build(:pageview, user_id: user_id2, timestamp: ~N[2023-10-25 15:01:00])
+ ])
+
+ assert %{
+ change: 100,
+ visitors: 1,
+ intervals: [
+ %{interval: ~N[2023-10-25 11:00:00], visitors: 0},
+ %{interval: ~N[2023-10-25 12:00:00], visitors: 0},
+ %{interval: ~N[2023-10-25 13:00:00], visitors: 0},
+ %{interval: ~N[2023-10-25 14:00:00], visitors: 0},
+ %{interval: ~N[2023-10-25 15:00:00], visitors: 1},
+ %{interval: ~N[2023-10-25 16:00:00], visitors: 0},
+ %{interval: ~N[2023-10-25 17:00:00], visitors: 0},
+ %{interval: ~N[2023-10-25 18:00:00], visitors: 0},
+ %{interval: ~N[2023-10-25 19:00:00], visitors: 0},
+ %{interval: ~N[2023-10-25 20:00:00], visitors: 0},
+ %{interval: ~N[2023-10-25 21:00:00], visitors: 0},
+ %{interval: ~N[2023-10-25 22:00:00], visitors: 0},
+ %{interval: ~N[2023-10-25 23:00:00], visitors: 0},
+ %{interval: ~N[2023-10-26 00:00:00], visitors: 0},
+ %{interval: ~N[2023-10-26 01:00:00], visitors: 0},
+ %{interval: ~N[2023-10-26 02:00:00], visitors: 0},
+ %{interval: ~N[2023-10-26 03:00:00], visitors: 0},
+ %{interval: ~N[2023-10-26 04:00:00], visitors: 0},
+ %{interval: ~N[2023-10-26 05:00:00], visitors: 0},
+ %{interval: ~N[2023-10-26 06:00:00], visitors: 0},
+ %{interval: ~N[2023-10-26 07:00:00], visitors: 0},
+ %{interval: ~N[2023-10-26 08:00:00], visitors: 0},
+ %{interval: ~N[2023-10-26 09:00:00], visitors: 0},
+ %{interval: ~N[2023-10-26 10:00:00], visitors: 0},
+ %{interval: ~N[2023-10-26 11:00:00], visitors: 0}
+ ]
+ } = Clickhouse.last_24h_visitors_hourly_intervals([site1], fixed_now)[site1.domain]
+
+ assert %{
+ change: 100,
+ visitors: 3,
+ intervals: [
+ %{interval: ~N[2023-10-25 11:00:00], visitors: 0},
+ %{interval: ~N[2023-10-25 12:00:00], visitors: 0},
+ %{interval: ~N[2023-10-25 13:00:00], visitors: 2},
+ %{interval: ~N[2023-10-25 14:00:00], visitors: 0},
+ %{interval: ~N[2023-10-25 15:00:00], visitors: 1},
+ %{interval: ~N[2023-10-25 16:00:00], visitors: 0},
+ %{interval: ~N[2023-10-25 17:00:00], visitors: 0},
+ %{interval: ~N[2023-10-25 18:00:00], visitors: 0},
+ %{interval: ~N[2023-10-25 19:00:00], visitors: 0},
+ %{interval: ~N[2023-10-25 20:00:00], visitors: 0},
+ %{interval: ~N[2023-10-25 21:00:00], visitors: 0},
+ %{interval: ~N[2023-10-25 22:00:00], visitors: 0},
+ %{interval: ~N[2023-10-25 23:00:00], visitors: 0},
+ %{interval: ~N[2023-10-26 00:00:00], visitors: 0},
+ %{interval: ~N[2023-10-26 01:00:00], visitors: 0},
+ %{interval: ~N[2023-10-26 02:00:00], visitors: 0},
+ %{interval: ~N[2023-10-26 03:00:00], visitors: 0},
+ %{interval: ~N[2023-10-26 04:00:00], visitors: 0},
+ %{interval: ~N[2023-10-26 05:00:00], visitors: 0},
+ %{interval: ~N[2023-10-26 06:00:00], visitors: 0},
+ %{interval: ~N[2023-10-26 07:00:00], visitors: 0},
+ %{interval: ~N[2023-10-26 08:00:00], visitors: 0},
+ %{interval: ~N[2023-10-26 09:00:00], visitors: 0},
+ %{interval: ~N[2023-10-26 10:00:00], visitors: 0},
+ %{interval: ~N[2023-10-26 11:00:00], visitors: 0}
+ ]
+ } = Clickhouse.last_24h_visitors_hourly_intervals([site2], fixed_now)[site2.domain]
+ end
+
test "returns clickhouse data merged with placeholder for multiple sites" do
fixed_now = ~N[2023-10-26 10:00:15]
site1 = insert(:site)