Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
fix(TimeSeries): Work around weird time zones (#4337)
Karl-Aksel Puulmann011468977914
diff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.exindex b3b8a8dab1..7d5d21aae6 100644--- a/lib/plausible/stats/sql/expression.ex+++ b/lib/plausible/stats/sql/expression.ex@@ -17,19 +17,19 @@ defmodule Plausible.Stats.SQL.Expression do@no_ref "Direct / None"@not_set "(not set)"- defmacrop field_or_blank_value(key, expr, empty_value) do+ defmacrop field_or_blank_value(q, key, expr, empty_value) doquote do- wrap_alias([t], %{+ select_merge_as(unquote(q), [t], %{unquote(key) =>fragment("if(empty(?), ?, ?)", unquote(expr), unquote(empty_value), unquote(expr))})endend- defmacrop regular_time_slots(query, period_in_seconds) do+ defmacrop time_slots(query, period_in_seconds) doquote dofragment(- "arrayJoin(timeSlots(toTimeZone(?, ?), toUInt32(timeDiff(?, ?)), toUInt32(?)))",+ "timeSlots(toTimeZone(?, ?), toUInt32(timeDiff(?, ?)), toUInt32(?))",s.start,^unquote(query).timezone,s.start,@@ -39,14 +39,14 @@ defmodule Plausible.Stats.SQL.Expression doendend- def dimension(key, "time:month", _table, query) do- wrap_alias([t], %{+ def select_dimension(q, key, "time:month", _table, query) do+ select_merge_as(q, [t], %{key => fragment("toStartOfMonth(toTimeZone(?, ?))", t.timestamp, ^query.timezone)})end- def dimension(key, "time:week", _table, query) do- wrap_alias([t], %{+ def select_dimension(q, key, "time:week", _table, query) do+ select_merge_as(q, [t], %{key =>weekstart_not_before(to_timezone(t.timestamp, ^query.timezone),@@ -55,29 +55,35 @@ defmodule Plausible.Stats.SQL.Expression do})end- def dimension(key, "time:day", _table, query) do- wrap_alias([t], %{+ def select_dimension(q, key, "time:day", _table, query) do+ select_merge_as(q, [t], %{key => fragment("toDate(toTimeZone(?, ?))", t.timestamp, ^query.timezone)})end- def dimension(key, "time:hour", :sessions, query) do- wrap_alias([s], %{- key => regular_time_slots(query, 3600)+ def select_dimension(q, key, "time:hour", :sessions, query) do+ # :TRICKY: ClickHouse timeSlots works off of unix epoch and is not+ # timezone-aware. This means that for e.g. Asia/Katmandu (GMT+5:45)+ # to work, we divide time into 15-minute buckets and later combine these+ # via toStartOfHour+ q+ |> join(:array, [s], time_slot in time_slots(query, 15 * 60), as: :time_slot)+ |> select_merge_as([s, time_slot: time_slot], %{+ key => fragment("toStartOfHour(?)", time_slot)})end- def dimension(key, "time:hour", _table, query) do- wrap_alias([t], %{+ def select_dimension(q, key, "time:hour", _table, query) do+ select_merge_as(q, [t], %{key => fragment("toStartOfHour(toTimeZone(?, ?))", t.timestamp, ^query.timezone)})end# :NOTE: This is not exposed in Query APIv2- def dimension(key, "time:minute", :sessions, %Query{+ def select_dimension(q, key, "time:minute", :sessions, %Query{period: "30m"}) do- wrap_alias([s], %{+ select_merge_as(q, [s], %{key =>fragment("arrayJoin(range(dateDiff('minute', now(), ?), dateDiff('minute', now(), ?) + 1))",@@ -88,37 +94,39 @@ defmodule Plausible.Stats.SQL.Expression doend# :NOTE: This is not exposed in Query APIv2- def dimension(key, "time:minute", _table, %Query{period: "30m"}) do- wrap_alias([t], %{+ def select_dimension(q, key, "time:minute", _table, %Query{period: "30m"}) do+ select_merge_as(q, [t], %{key => fragment("dateDiff('minute', now(), ?)", t.timestamp)})end# :NOTE: This is not exposed in Query APIv2- def dimension(key, "time:minute", :sessions, query) do- wrap_alias([s], %{- key => regular_time_slots(query, 60)+ def select_dimension(q, key, "time:minute", :sessions, query) do+ q+ |> join(:array, [s], time_slot in time_slots(query, 60), as: :time_slot)+ |> select_merge_as([s, time_slot: time_slot], %{+ key => fragment("?", time_slot)})end# :NOTE: This is not exposed in Query APIv2- def dimension(key, "time:minute", _table, query) do- wrap_alias([t], %{+ def select_dimension(q, key, "time:minute", _table, query) do+ select_merge_as(q, [t], %{key => fragment("toStartOfMinute(toTimeZone(?, ?))", t.timestamp, ^query.timezone)})end- def dimension(key, "event:name", _table, _query),- do: wrap_alias([t], %{key => t.name})+ def select_dimension(q, key, "event:name", _table, _query),+ do: select_merge_as(q, [t], %{key => t.name})- def dimension(key, "event:page", _table, _query),- do: wrap_alias([t], %{key => t.pathname})+ def select_dimension(q, key, "event:page", _table, _query),+ do: select_merge_as(q, [t], %{key => t.pathname})- def dimension(key, "event:hostname", _table, _query),- do: wrap_alias([t], %{key => t.hostname})+ def select_dimension(q, key, "event:hostname", _table, _query),+ do: select_merge_as(q, [t], %{key => t.hostname})- def dimension(key, "event:props:" <> property_name, _table, _query) do- wrap_alias([t], %{+ def select_dimension(q, key, "event:props:" <> property_name, _table, _query) do+ select_merge_as(q, [t], %{key =>fragment("if(not empty(?), ?, '(none)')",@@ -128,56 +136,56 @@ defmodule Plausible.Stats.SQL.Expression do})end- def dimension(key, "visit:entry_page", _table, _query),- do: wrap_alias([t], %{key => t.entry_page})+ def select_dimension(q, key, "visit:entry_page", _table, _query),+ do: select_merge_as(q, [t], %{key => t.entry_page})- def dimension(key, "visit:exit_page", _table, _query),- do: wrap_alias([t], %{key => t.exit_page})+ def select_dimension(q, key, "visit:exit_page", _table, _query),+ do: select_merge_as(q, [t], %{key => t.exit_page})- def dimension(key, "visit:utm_medium", _table, _query),- do: field_or_blank_value(key, t.utm_medium, @not_set)+ def select_dimension(q, key, "visit:utm_medium", _table, _query),+ do: field_or_blank_value(q, key, t.utm_medium, @not_set)- def dimension(key, "visit:utm_source", _table, _query),- do: field_or_blank_value(key, t.utm_source, @not_set)+ def select_dimension(q, key, "visit:utm_source", _table, _query),+ do: field_or_blank_value(q, key, t.utm_source, @not_set)- def dimension(key, "visit:utm_campaign", _table, _query),- do: field_or_blank_value(key, t.utm_campaign, @not_set)+ def select_dimension(q, key, "visit:utm_campaign", _table, _query),+ do: field_or_blank_value(q, key, t.utm_campaign, @not_set)- def dimension(key, "visit:utm_content", _table, _query),- do: field_or_blank_value(key, t.utm_content, @not_set)+ def select_dimension(q, key, "visit:utm_content", _table, _query),+ do: field_or_blank_value(q, key, t.utm_content, @not_set)- def dimension(key, "visit:utm_term", _table, _query),- do: field_or_blank_value(key, t.utm_term, @not_set)+ def select_dimension(q, key, "visit:utm_term", _table, _query),+ do: field_or_blank_value(q, key, t.utm_term, @not_set)- def dimension(key, "visit:source", _table, _query),- do: field_or_blank_value(key, t.source, @no_ref)+ def select_dimension(q, key, "visit:source", _table, _query),+ do: field_or_blank_value(q, key, t.source, @no_ref)- def dimension(key, "visit:referrer", _table, _query),- do: field_or_blank_value(key, t.referrer, @no_ref)+ def select_dimension(q, key, "visit:referrer", _table, _query),+ do: field_or_blank_value(q, key, t.referrer, @no_ref)- def dimension(key, "visit:device", _table, _query),- do: field_or_blank_value(key, t.device, @not_set)+ def select_dimension(q, key, "visit:device", _table, _query),+ do: field_or_blank_value(q, key, t.device, @not_set)- def dimension(key, "visit:os", _table, _query),- do: field_or_blank_value(key, t.os, @not_set)+ def select_dimension(q, key, "visit:os", _table, _query),+ do: field_or_blank_value(q, key, t.os, @not_set)- def dimension(key, "visit:os_version", _table, _query),- do: field_or_blank_value(key, t.os_version, @not_set)+ def select_dimension(q, key, "visit:os_version", _table, _query),+ do: field_or_blank_value(q, key, t.os_version, @not_set)- def dimension(key, "visit:browser", _table, _query),- do: field_or_blank_value(key, t.browser, @not_set)+ def select_dimension(q, key, "visit:browser", _table, _query),+ do: field_or_blank_value(q, key, t.browser, @not_set)- def dimension(key, "visit:browser_version", _table, _query),- do: field_or_blank_value(key, t.browser_version, @not_set)+ def select_dimension(q, key, "visit:browser_version", _table, _query),+ do: field_or_blank_value(q, key, t.browser_version, @not_set)- def dimension(key, "visit:country", _table, _query),- do: wrap_alias([t], %{key => t.country})+ def select_dimension(q, key, "visit:country", _table, _query),+ do: select_merge_as(q, [t], %{key => t.country})- def dimension(key, "visit:region", _table, _query),- do: wrap_alias([t], %{key => t.region})+ def select_dimension(q, key, "visit:region", _table, _query),+ do: select_merge_as(q, [t], %{key => t.region})- def dimension(key, "visit:city", _table, _query),- do: wrap_alias([t], %{key => t.city})+ def select_dimension(q, key, "visit:city", _table, _query),+ do: select_merge_as(q, [t], %{key => t.city})def event_metric(:pageviews) dowrap_alias([e], %{diff --git a/lib/plausible/stats/sql/query_builder.ex b/lib/plausible/stats/sql/query_builder.exindex aaf3eb8c9d..4a7d801fb2 100644--- a/lib/plausible/stats/sql/query_builder.ex+++ b/lib/plausible/stats/sql/query_builder.ex@@ -144,7 +144,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder dokey = shortname(query, dimension)q- |> select_merge_as([], Expression.dimension(key, dimension, table, query))+ |> Expression.select_dimension(key, dimension, table, query)|> group_by([], selected_as(^key))enddiff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exsindex f9a61acbbc..3c1a15d207 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs@@ -1133,6 +1133,35 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do%{"dimensions" => ["2021-01-01"], "metrics" => [1]}]end++ test "timeseries with quarter-hour timezone", %{conn: conn, user: user} do+ # GMT+05:45+ site = insert(:site, timezone: "Asia/Katmandu", members: [user])++ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-02 05:00:00]),+ build(:pageview, timestamp: ~N[2021-01-02 05:15:00]),+ build(:pageview, timestamp: ~N[2021-01-02 05:30:00]),+ build(:pageview, timestamp: ~N[2021-01-02 05:45:00]),+ build(:pageview, timestamp: ~N[2021-01-02 06:00:00]),+ build(:pageview, timestamp: ~N[2021-01-02 06:15:00]),+ build(:pageview, timestamp: ~N[2021-01-02 06:30:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visits"],+ "date_range" => ["2021-01-02", "2021-01-02"],+ "dimensions" => ["time:hour"]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["2021-01-02 10:00:00"], "metrics" => [1]},+ %{"dimensions" => ["2021-01-02 11:00:00"], "metrics" => [4]},+ %{"dimensions" => ["2021-01-02 12:00:00"], "metrics" => [2]}+ ]+ endendtest "breakdown by visit:source", %{conn: conn, site: site} do