Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
APIv2: Standard iso8601 timestamps, operate on UTC (#4563)
Karl-Aksel Puulmannbd11b4cf67a3
diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.exindex 1442c9013f..9cdb024571 100644--- a/lib/plausible/google/api.ex+++ b/lib/plausible/google/api.ex@@ -7,7 +7,7 @@ defmodule Plausible.Google.API doalias Plausible.Google.HTTPalias Plausible.Google.SearchConsole- alias Plausible.Stats.DateTimeRange+ alias Plausible.Stats.Queryrequire Logger@@ -65,12 +65,11 @@ defmodule Plausible.Google.API do{:ok, access_token} <- maybe_refresh_token(site.google_auth),{:ok, gsc_filters} <-SearchConsole.Filters.transform(site.google_auth.property, query.filters, search),- date_range = DateTimeRange.to_date_range(query.date_range),{:ok, stats} <-HTTP.list_stats(access_token,site.google_auth.property,- date_range,+ Query.date_range(query),pagination,gsc_filters) dodiff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.exindex 76f08c37eb..be636e1231 100644--- a/lib/plausible/stats/breakdown.ex+++ b/lib/plausible/stats/breakdown.ex@@ -116,6 +116,8 @@ defmodule Plausible.Stats.Breakdown dofrom e in subquery(timed_page_transitions_q),group_by: e.pathname+ date_range = Query.date_range(query)+timed_pages_q =if query.include_imported do# Imported page views have pre-calculated values@@ -123,9 +125,7 @@ defmodule Plausible.Stats.Breakdown dofrom i in "imported_pages",group_by: i.page,where: i.site_id == ^site.id,- where:- i.date >= ^DateTime.to_naive(query.date_range.first) and- i.date <= ^DateTime.to_naive(query.date_range.last),+ where: i.date >= ^date_range.first and i.date <= ^date_range.last,where: i.page in ^pages,select: %{page: i.page,diff --git a/lib/plausible/stats/comparisons.ex b/lib/plausible/stats/comparisons.exindex 3f5c1ab7c3..22caaa5667 100644--- a/lib/plausible/stats/comparisons.ex+++ b/lib/plausible/stats/comparisons.ex@@ -64,15 +64,17 @@ defmodule Plausible.Stats.Comparisons do|> Keyword.put_new(:now, DateTime.now!(site.timezone))|> Keyword.put_new(:match_day_of_week?, false)- source_date_range = DateTimeRange.to_date_range(source_query.date_range)+ source_date_range = Query.date_range(source_query)with :ok <- validate_mode(source_query, mode),{:ok, comparison_date_range} <- get_comparison_date_range(source_date_range, mode, opts) do- %Date.Range{first: first, last: last} = comparison_date_range+ new_range =+ DateTimeRange.new!(comparison_date_range.first, comparison_date_range.last, site.timezone)+ |> DateTimeRange.to_timezone("Etc/UTC")comparison_query =source_query- |> Query.set(date_range: DateTimeRange.new!(first, last, site.timezone))+ |> Query.set(utc_time_range: new_range)|> maybe_include_imported(source_query){:ok, comparison_query}diff --git a/lib/plausible/stats/datetime_range.ex b/lib/plausible/stats/datetime_range.exindex e8e787a71c..e6df6cb8d1 100644--- a/lib/plausible/stats/datetime_range.ex+++ b/lib/plausible/stats/datetime_range.ex@@ -45,7 +45,16 @@ defmodule Plausible.Stats.DateTimeRange do%__MODULE__{first: first, last: last}end- def to_date_range(%__MODULE__{first: first, last: last}) do+ def to_timezone(%__MODULE__{first: first, last: last}, timezone) do+ first = DateTime.shift_zone!(first, timezone)+ last = DateTime.shift_zone!(last, timezone)++ %__MODULE__{first: first, last: last}+ end++ def to_date_range(datetime_range, timezone) do+ %__MODULE__{first: first, last: last} = to_timezone(datetime_range, timezone)+first = DateTime.to_date(first)last = DateTime.to_date(last)diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.exindex a0878122bc..d800a9b1ad 100644--- a/lib/plausible/stats/filters/query_parser.ex+++ b/lib/plausible/stats/filters/query_parser.ex@@ -18,7 +18,9 @@ defmodule Plausible.Stats.Filters.QueryParser dowith :ok <- JSONSchema.validate(schema_type, params),{:ok, date} <- parse_date(site, Map.get(params, "date"), date),- {:ok, date_range} <- parse_date_range(site, Map.get(params, "date_range"), date, now),+ {:ok, raw_time_range} <-+ parse_time_range(site, Map.get(params, "date_range"), date, now),+ utc_time_range = raw_time_range |> DateTimeRange.to_timezone("Etc/UTC"),{:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])),{:ok, filters} <- parse_filters(Map.get(params, "filters", [])),{:ok, dimensions} <- parse_dimensions(Map.get(params, "dimensions", [])),@@ -28,10 +30,10 @@ defmodule Plausible.Stats.Filters.QueryParser doquery = %{metrics: metrics,filters: filters,- date_range: date_range,+ utc_time_range: utc_time_range,dimensions: dimensions,order_by: order_by,- timezone: date_range.first.time_zone,+ timezone: site.timezone,preloaded_goals: preloaded_goals,include: include},@@ -151,7 +153,7 @@ defmodule Plausible.Stats.Filters.QueryParser do{:ok, date}end- defp parse_date_range(_site, date_range, _date, now) when date_range in ["realtime", "30m"] do+ defp parse_time_range(_site, date_range, _date, now) when date_range in ["realtime", "30m"] doduration_minutes =case date_range do"realtime" -> 5@@ -164,27 +166,27 @@ defmodule Plausible.Stats.Filters.QueryParser do{:ok, DateTimeRange.new!(first_datetime, last_datetime)}end- defp parse_date_range(site, "day", date, _now) do+ defp parse_time_range(site, "day", date, _now) do{:ok, DateTimeRange.new!(date, date, site.timezone)}end- defp parse_date_range(site, "7d", date, _now) do+ defp parse_time_range(site, "7d", date, _now) dofirst = date |> Date.add(-6){:ok, DateTimeRange.new!(first, date, site.timezone)}end- defp parse_date_range(site, "30d", date, _now) do+ defp parse_time_range(site, "30d", date, _now) dofirst = date |> Date.add(-30){:ok, DateTimeRange.new!(first, date, site.timezone)}end- defp parse_date_range(site, "month", date, _now) do+ defp parse_time_range(site, "month", date, _now) dolast = date |> Date.end_of_month()first = last |> Date.beginning_of_month(){:ok, DateTimeRange.new!(first, last, site.timezone)}end- defp parse_date_range(site, "6mo", date, _now) do+ defp parse_time_range(site, "6mo", date, _now) dolast = date |> Date.end_of_month()first =@@ -195,7 +197,7 @@ defmodule Plausible.Stats.Filters.QueryParser do{:ok, DateTimeRange.new!(first, last, site.timezone)}end- defp parse_date_range(site, "12mo", date, _now) do+ defp parse_time_range(site, "12mo", date, _now) dolast = date |> Date.end_of_month()first =@@ -206,19 +208,19 @@ defmodule Plausible.Stats.Filters.QueryParser do{:ok, DateTimeRange.new!(first, last, site.timezone)}end- defp parse_date_range(site, "year", date, _now) do+ defp parse_time_range(site, "year", date, _now) dolast = date |> Timex.end_of_year()first = last |> Timex.beginning_of_year(){:ok, DateTimeRange.new!(first, last, site.timezone)}end- defp parse_date_range(site, "all", date, _now) do+ defp parse_time_range(site, "all", date, _now) dostart_date = Plausible.Sites.stats_start_date(site) || date{:ok, DateTimeRange.new!(start_date, date, site.timezone)}end- defp parse_date_range(site, [from, to], _date, _now)+ defp parse_time_range(site, [from, to], _date, _now)when is_binary(from) and is_binary(to) docase date_range_from_date_strings(site, from, to) do{:ok, date_range} -> {:ok, date_range}@@ -226,7 +228,7 @@ defmodule Plausible.Stats.Filters.QueryParser doendend- defp parse_date_range(_site, unknown, _date, _now),+ defp parse_time_range(_site, unknown, _date, _now),do: {:error, "Invalid date_range '#{i(unknown)}'."}defp date_range_from_date_strings(site, from, to) do@@ -237,22 +239,14 @@ defmodule Plausible.Stats.Filters.QueryParser doenddefp date_range_from_timestamps(from, to) do- with {:ok, from_datetime} <- datetime_from_timestamp(from),- {:ok, to_datetime} <- datetime_from_timestamp(to),- true <- from_datetime.time_zone == to_datetime.time_zone do+ with {:ok, from_datetime, _offset} <- DateTime.from_iso8601(from),+ {:ok, to_datetime, _offset} <- DateTime.from_iso8601(to) do{:ok, DateTimeRange.new!(from_datetime, to_datetime)}else_ -> {:error, "Invalid date_range '#{i([from, to])}'."}endend- defp datetime_from_timestamp(timestamp_string) do- with [timestamp, timezone] <- String.split(timestamp_string),- {:ok, naive_datetime} <- NaiveDateTime.from_iso8601(timestamp) do- DateTime.from_naive(naive_datetime, timezone)- end- end-defp today(site), do: DateTime.now!(site.timezone) |> DateTime.to_date()defp parse_dimensions(dimensions) when is_list(dimensions) dodiff --git a/lib/plausible/stats/goal_suggestions.ex b/lib/plausible/stats/goal_suggestions.exindex d3192bfb81..93ee2cd57e 100644--- a/lib/plausible/stats/goal_suggestions.ex+++ b/lib/plausible/stats/goal_suggestions.ex@@ -57,13 +57,13 @@ defmodule Plausible.Stats.GoalSuggestions do)|> maybe_set_limit(limit)+ date_range = Query.date_range(query)+imported_q =from(i in "imported_custom_events",where: i.site_id == ^site.id,where: i.import_id in ^site.complete_import_ids,- where:- i.date >= ^DateTime.to_naive(query.date_range.first) and- i.date <= ^DateTime.to_naive(query.date_range.last),+ where: i.date >= ^date_range.first and i.date <= ^date_range.last,where: i.visitors > 0,where: fragment("? ilike ?", i.name, ^matches),where: fragment("trim(?)", i.name) != "",diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.exindex 4a8744b5d5..49fea32c9f 100644--- a/lib/plausible/stats/imported/base.ex+++ b/lib/plausible/stats/imported/base.ex@@ -6,7 +6,7 @@ defmodule Plausible.Stats.Imported.Base doimport Ecto.Queryalias Plausible.Imported- alias Plausible.Stats.{Query, DateTimeRange}+ alias Plausible.Stats.Queryimport Plausible.Stats.Filters, only: [dimensions_used_in_filters: 1]@@ -59,7 +59,8 @@ defmodule Plausible.Stats.Imported.Base dodef query_imported(table, site, query) doimport_ids = site.complete_import_ids- %{first: date_from, last: date_to} = DateTimeRange.to_date_range(query.date_range)+ # Assumption: dates in imported table are in user-local timezone.+ %{first: date_from, last: date_to} = Query.date_range(query)from(i in table,where: i.site_id == ^site.id,diff --git a/lib/plausible/stats/imported/sql/expression.ex b/lib/plausible/stats/imported/sql/expression.exindex 776796950e..4ef755f3da 100644--- a/lib/plausible/stats/imported/sql/expression.ex+++ b/lib/plausible/stats/imported/sql/expression.ex@@ -9,6 +9,8 @@ defmodule Plausible.Stats.Imported.SQL.Expression doimport Plausible.Stats.Util, only: [shortname: 2]import Ecto.Query+ alias Plausible.Stats.Query+@no_ref "Direct / None"@not_set "(not set)"@none "(none)"@@ -292,8 +294,10 @@ defmodule Plausible.Stats.Imported.SQL.Expression doenddefp select_group_fields(q, "time:week", key, query) do+ date_range = Query.date_range(query)+select_merge_as(q, [i], %{- key => weekstart_not_before(i.date, ^DateTime.to_naive(query.date_range.first))+ key => weekstart_not_before(i.date, ^date_range.first)})enddiff --git a/lib/plausible/stats/interval.ex b/lib/plausible/stats/interval.exindex fc3324d4c5..8f053f1e1c 100644--- a/lib/plausible/stats/interval.ex+++ b/lib/plausible/stats/interval.ex@@ -40,9 +40,7 @@ defmodule Plausible.Stats.Interval do@doc """Returns the suggested interval for the given `DateTimeRange` struct."""- def default_for_date_range(%DateTimeRange{} = date_range) do- %Date.Range{first: first, last: last} = DateTimeRange.to_date_range(date_range)-+ def default_for_date_range(%DateTimeRange{first: first, last: last}) docond doTimex.diff(last, first, :months) > 0 ->"month"diff --git a/lib/plausible/stats/legacy/legacy_query_builder.ex b/lib/plausible/stats/legacy/legacy_query_builder.exindex 7165e3d18f..065da1a3c8 100644--- a/lib/plausible/stats/legacy/legacy_query_builder.ex+++ b/lib/plausible/stats/legacy/legacy_query_builder.ex@@ -8,14 +8,14 @@ defmodule Plausible.Stats.Legacy.QueryBuilder doalias Plausible.Stats.{Filters, Interval, Query, DateTimeRange, Metrics}- def from(site, params, debug_metadata) do- now = DateTime.utc_now(:second)+ def from(site, params, debug_metadata, now \\ nil) do+ now = now || DateTime.utc_now(:second)query =Query|> struct!(now: now, debug_metadata: debug_metadata)|> put_period(site, params)- |> put_timezone()+ |> put_timezone(site)|> put_dimensions(params)|> put_interval(params)|> put_parsed_filters(params)@@ -53,86 +53,109 @@ defmodule Plausible.Stats.Legacy.QueryBuilder dofirst_datetime = DateTime.shift(now, minute: -duration_minutes)last_datetime = DateTime.shift(now, second: 5)- struct!(query, period: period, date_range: DateTimeRange.new!(first_datetime, last_datetime))+ datetime_range =+ DateTimeRange.new!(first_datetime, last_datetime) |> DateTimeRange.to_timezone("Etc/UTC")++ struct!(query, period: period, utc_time_range: datetime_range)enddefp put_period(query, site, %{"period" => "day"} = params) do- date = parse_single_date(site.timezone, params)- datetime_range = DateTimeRange.new!(date, date, site.timezone)+ date = parse_single_date(query, params)++ datetime_range =+ DateTimeRange.new!(date, date, site.timezone) |> DateTimeRange.to_timezone("Etc/UTC")- struct!(query, period: "day", date_range: datetime_range)+ struct!(query, period: "day", utc_time_range: datetime_range)enddefp put_period(query, site, %{"period" => "7d"} = params) do- end_date = parse_single_date(site.timezone, params)+ end_date = parse_single_date(query, params)start_date = end_date |> Date.shift(day: -6)- datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone)- struct!(query, period: "7d", date_range: datetime_range)+ datetime_range =+ DateTimeRange.new!(start_date, end_date, site.timezone)+ |> DateTimeRange.to_timezone("Etc/UTC")++ struct!(query, period: "7d", utc_time_range: datetime_range)enddefp put_period(query, site, %{"period" => "30d"} = params) do- end_date = parse_single_date(site.timezone, params)+ end_date = parse_single_date(query, params)start_date = end_date |> Date.shift(day: -30)- datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone)- struct!(query, period: "30d", date_range: datetime_range)+ datetime_range =+ DateTimeRange.new!(start_date, end_date, site.timezone)+ |> DateTimeRange.to_timezone("Etc/UTC")++ struct!(query, period: "30d", utc_time_range: datetime_range)enddefp put_period(query, site, %{"period" => "month"} = params) do- date = parse_single_date(site.timezone, params)+ date = parse_single_date(query, params)start_date = Timex.beginning_of_month(date)end_date = Timex.end_of_month(date)- datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone)- struct!(query, period: "month", date_range: datetime_range)+ datetime_range =+ DateTimeRange.new!(start_date, end_date, site.timezone)+ |> DateTimeRange.to_timezone("Etc/UTC")++ struct!(query, period: "month", utc_time_range: datetime_range)enddefp put_period(query, site, %{"period" => "6mo"} = params) doend_date =- parse_single_date(site.timezone, params)+ parse_single_date(query, params)|> Timex.end_of_month()start_date =Date.shift(end_date, month: -5)|> Timex.beginning_of_month()- datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone)+ datetime_range =+ DateTimeRange.new!(start_date, end_date, site.timezone)+ |> DateTimeRange.to_timezone("Etc/UTC")- struct!(query, period: "6mo", date_range: datetime_range)+ struct!(query, period: "6mo", utc_time_range: datetime_range)enddefp put_period(query, site, %{"period" => "12mo"} = params) doend_date =- parse_single_date(site.timezone, params)+ parse_single_date(query, params)|> Timex.end_of_month()start_date =Date.shift(end_date, month: -11)|> Timex.beginning_of_month()- datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone)+ datetime_range =+ DateTimeRange.new!(start_date, end_date, site.timezone)+ |> DateTimeRange.to_timezone("Etc/UTC")- struct!(query, period: "12mo", date_range: datetime_range)+ struct!(query, period: "12mo", utc_time_range: datetime_range)enddefp put_period(query, site, %{"period" => "year"} = params) doend_date =- parse_single_date(site.timezone, params)+ parse_single_date(query, params)|> Timex.end_of_year()start_date = Timex.beginning_of_year(end_date)- datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone)- struct!(query, period: "year", date_range: datetime_range)+ datetime_range =+ DateTimeRange.new!(start_date, end_date, site.timezone)+ |> DateTimeRange.to_timezone("Etc/UTC")++ struct!(query, period: "year", utc_time_range: datetime_range)enddefp put_period(query, site, %{"period" => "all"}) do- today = today(site.timezone)+ today = today(query)start_date = Plausible.Sites.stats_start_date(site) || today- datetime_range = DateTimeRange.new!(start_date, today, site.timezone)- struct!(query, period: "all", date_range: datetime_range)+ datetime_range =+ DateTimeRange.new!(start_date, today, site.timezone) |> DateTimeRange.to_timezone("Etc/UTC")++ struct!(query, period: "all", utc_time_range: datetime_range)enddefp put_period(query, site, %{"period" => "custom", "from" => from, "to" => to} = params) do@@ -148,17 +171,20 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do[from, to] = String.split(date, ",")from_date = Date.from_iso8601!(String.trim(from))to_date = Date.from_iso8601!(String.trim(to))- datetime_range = DateTimeRange.new!(from_date, to_date, site.timezone)- struct!(query, period: "custom", date_range: datetime_range)+ datetime_range =+ DateTimeRange.new!(from_date, to_date, site.timezone)+ |> DateTimeRange.to_timezone("Etc/UTC")++ struct!(query, period: "custom", utc_time_range: datetime_range)enddefp put_period(query, site, params) doput_period(query, site, Map.merge(params, %{"period" => "30d"}))end- defp put_timezone(query) do- struct!(query, timezone: query.date_range.first.time_zone)+ defp put_timezone(query, site) do+ struct!(query, timezone: site.timezone)enddefp put_dimensions(query, params) do@@ -225,7 +251,7 @@ defmodule Plausible.Stats.Legacy.QueryBuilder doenddefp put_interval(%{:period => "all"} = query, params) do- interval = Map.get(params, "interval", Interval.default_for_date_range(query.date_range))+ interval = Map.get(params, "interval", Interval.default_for_date_range(query.utc_time_range))struct!(query, interval: interval)end@@ -238,15 +264,15 @@ defmodule Plausible.Stats.Legacy.QueryBuilder dostruct!(query, filters: Filters.parse(params["filters"]))end- defp today(tz) do- DateTime.now!(tz) |> Timex.to_date()+ defp today(query) do+ query.now |> Timex.to_date()end- defp parse_single_date(tz, params) do+ defp parse_single_date(query, params) docase params["date"] do- "today" -> DateTime.now!(tz) |> Timex.to_date()+ "today" -> query.now |> Timex.to_date()date when is_binary(date) -> Date.from_iso8601!(date)- _ -> today(tz)+ _ -> today(query)endendenddiff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.exindex ee2d6556f4..682e854c10 100644--- a/lib/plausible/stats/query.ex+++ b/lib/plausible/stats/query.ex@@ -1,7 +1,7 @@defmodule Plausible.Stats.Query douse Plausible- defstruct date_range: nil,+ defstruct utc_time_range: nil,interval: nil,period: nil,dimensions: [],@@ -44,8 +44,8 @@ defmodule Plausible.Stats.Query do@doc """Builds query from old-style params. New code should prefer Query.build"""- def from(site, params, debug_metadata \\ %{}) do- Legacy.QueryBuilder.from(site, params, debug_metadata)+ def from(site, params, debug_metadata \\ %{}, now \\ nil) do+ Legacy.QueryBuilder.from(site, params, debug_metadata, now)enddef put_experimental_reduced_joins(query, site, params) do@@ -60,6 +60,10 @@ defmodule Plausible.Stats.Query doendend+ def date_range(query) do+ Plausible.Stats.DateTimeRange.to_date_range(query.utc_time_range, query.timezone)+ end+def set(query, keywords) donew_query = struct!(query, keywords)@@ -141,7 +145,7 @@ defmodule Plausible.Stats.Query do"time:minute" in query.dimensions or "time:hour" in query.dimensions ->{:error, :unsupported_interval}- Date.after?(query.date_range.first, query.latest_import_end_date) ->+ Date.after?(date_range(query).first, query.latest_import_end_date) ->{:error, :out_of_range}not Imported.schema_supports_query?(query) ->diff --git a/lib/plausible/stats/query_optimizer.ex b/lib/plausible/stats/query_optimizer.exindex c93958503e..7d43f7b2af 100644--- a/lib/plausible/stats/query_optimizer.ex+++ b/lib/plausible/stats/query_optimizer.ex@@ -61,7 +61,7 @@ defmodule Plausible.Stats.QueryOptimizer dodefp update_group_by_time(%Query{- date_range: %DateTimeRange{first: first, last: last}+ utc_time_range: %DateTimeRange{first: first, last: last}} = query) dodimensions =diff --git a/lib/plausible/stats/query_result.ex b/lib/plausible/stats/query_result.exindex 66835d646c..c332c88a0a 100644--- a/lib/plausible/stats/query_result.ex+++ b/lib/plausible/stats/query_result.ex@@ -32,8 +32,8 @@ defmodule Plausible.Stats.QueryResult dosite_id: site.domain,metrics: query.metrics,date_range: [- to_iso_8601_with_timezone(query.date_range.first),- to_iso_8601_with_timezone(query.date_range.last)+ to_iso8601(query.utc_time_range.first, query.timezone),+ to_iso8601(query.utc_time_range.last, query.timezone)],filters: query.filters,dimensions: query.dimensions,@@ -88,13 +88,10 @@ defmodule Plausible.Stats.QueryResult do|> Enum.into(%{})end- defp to_iso_8601_with_timezone(%DateTime{time_zone: timezone} = datetime) do- naive_iso8601 =- datetime- |> DateTime.to_naive()- |> NaiveDateTime.to_iso8601()-- naive_iso8601 <> " " <> timezone+ defp to_iso8601(datetime, timezone) do+ datetime+ |> DateTime.shift_zone!(timezone)+ |> DateTime.to_iso8601(:extended)endenddiff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.exindex e5e8fb4472..4a051e0c6d 100644--- a/lib/plausible/stats/sql/expression.ex+++ b/lib/plausible/stats/sql/expression.ex@@ -12,7 +12,7 @@ defmodule Plausible.Stats.SQL.Expression doimport Ecto.Query- alias Plausible.Stats.{Filters, SQL}+ alias Plausible.Stats.{Query, Filters, SQL}@no_ref "Direct / None"@not_set "(not set)"@@ -46,11 +46,13 @@ defmodule Plausible.Stats.SQL.Expression doenddef select_dimension(q, key, "time:week", _table, query) do+ date_range = Query.date_range(query)+select_merge_as(q, [t], %{key =>weekstart_not_before(to_timezone(t.timestamp, ^query.timezone),- ^DateTime.to_naive(query.date_range.first)+ ^date_range.first)})enddiff --git a/lib/plausible/stats/time.ex b/lib/plausible/stats/time.exindex 4b9b4b6a53..7be4866e37 100644--- a/lib/plausible/stats/time.ex+++ b/lib/plausible/stats/time.ex@@ -5,16 +5,13 @@ defmodule Plausible.Stats.Time doalias Plausible.Stats.{Query, DateTimeRange}- def utc_boundaries(%Query{date_range: date_range}, site) do- %DateTimeRange{first: first, last: last} = date_range-+ def utc_boundaries(%Query{utc_time_range: time_range}, site) dofirst =- first- |> DateTime.shift_zone!("Etc/UTC")+ time_range.first|> DateTime.to_naive()|> beginning_of_time(site.native_stats_start_at)- last = DateTime.shift_zone!(last, "Etc/UTC") |> DateTime.to_naive()+ last = DateTime.to_naive(time_range.last){first, last}end@@ -47,7 +44,7 @@ defmodule Plausible.Stats.Time doenddefp time_labels_for_dimension("time:month", query) do- date_range = DateTimeRange.to_date_range(query.date_range)+ date_range = Query.date_range(query)n_buckets =Timex.diff(@@ -65,7 +62,7 @@ defmodule Plausible.Stats.Time doenddefp time_labels_for_dimension("time:week", query) do- date_range = DateTimeRange.to_date_range(query.date_range)+ date_range = Query.date_range(query)n_buckets =Timex.diff(@@ -77,23 +74,25 @@ defmodule Plausible.Stats.Time doEnum.map(0..n_buckets, fn shift ->date_range.first|> Date.shift(week: shift)- |> date_or_weekstart(query)+ |> date_or_weekstart(date_range)|> format_datetime()end)enddefp time_labels_for_dimension("time:day", query) do- query.date_range- |> DateTimeRange.to_date_range()+ Query.date_range(query)|> Enum.into([])|> Enum.map(&format_datetime/1)enddefp time_labels_for_dimension("time:hour", query) do- n_buckets = DateTime.diff(query.date_range.last, query.date_range.first, :hour)+ time_range = query.utc_time_range |> DateTimeRange.to_timezone(query.timezone)++ from_timestamp = time_range.first |> Map.merge(%{minute: 0, second: 0})+ n_buckets = DateTime.diff(time_range.last, from_timestamp, :hour)Enum.map(0..n_buckets, fn step ->- query.date_range.first+ from_timestamp|> DateTime.to_naive()|> NaiveDateTime.shift(hour: step)|> format_datetime()@@ -101,24 +100,23 @@ defmodule Plausible.Stats.Time doenddefp time_labels_for_dimension("time:minute", query) do- first_datetime = Map.put(query.date_range.first, :second, 0)+ time_range = query.utc_time_range |> DateTimeRange.to_timezone(query.timezone)+ first_datetime = Map.put(time_range.first, :second, 0)first_datetime|> Stream.iterate(fn datetime -> DateTime.shift(datetime, minute: 1) end)|> Enum.take_while(fn datetime ->current_minute = Map.put(query.now, :second, 0)- DateTime.before?(datetime, query.date_range.last) &&+ DateTime.before?(datetime, time_range.last) &&DateTime.before?(datetime, current_minute)end)|> Enum.map(&format_datetime/1)end- def date_or_weekstart(date, query) do+ def date_or_weekstart(date, date_range) doweekstart = Date.beginning_of_week(date)- date_range = DateTimeRange.to_date_range(query.date_range)-if Enum.member?(date_range, weekstart) doweekstartelsediff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex 689c6aa157..8fa9a895eb 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -5,7 +5,7 @@ defmodule PlausibleWeb.Api.StatsController douse PlausibleWeb.Plugs.ErrorHandleralias Plausible.Stats- alias Plausible.Stats.{Query, Comparisons, Filters, Time, TableDecider, DateTimeRange}+ alias Plausible.Stats.{Query, Comparisons, Filters, Time, TableDecider}alias Plausible.Stats.Filters.LegacyDashboardFilterParseralias PlausibleWeb.Api.Helpers, as: H@@ -164,13 +164,19 @@ defmodule PlausibleWeb.Api.StatsController doendend- defp build_full_intervals(%{interval: "week", date_range: date_range}, labels) do- date_range = DateTimeRange.to_date_range(date_range)+ defp build_full_intervals(+ %Query{interval: "week"} = query,+ labels+ ) do+ date_range = Query.date_range(query)build_intervals(labels, date_range, &Timex.beginning_of_week/1, &Timex.end_of_week/1)end- defp build_full_intervals(%{interval: "month", date_range: date_range}, labels) do- date_range = DateTimeRange.to_date_range(date_range)+ defp build_full_intervals(+ %Query{interval: "month"} = query,+ labels+ ) do+ date_range = Query.date_range(query)build_intervals(labels, date_range, &Timex.beginning_of_month/1, &Timex.end_of_month/1)end@@ -220,10 +226,10 @@ defmodule PlausibleWeb.Api.StatsController dowith_imported_switch: with_imported_switch_info(query, comparison_query),includes_imported: includes_imported?(query, comparison_query),imports_exist: site.complete_import_ids != [],- comparing_from: comparison_query && DateTime.to_date(comparison_query.date_range.first),- comparing_to: comparison_query && DateTime.to_date(comparison_query.date_range.last),- from: DateTime.to_date(query.date_range.first),- to: DateTime.to_date(query.date_range.last)+ comparing_from: comparison_query && Query.date_range(comparison_query).first,+ comparing_to: comparison_query && Query.date_range(comparison_query).last,+ from: Query.date_range(query).first,+ to: Query.date_range(query).last})end@@ -277,10 +283,12 @@ defmodule PlausibleWeb.Api.StatsController doEnum.find_index(dates, &(&1 == current_date))"week" ->+ date_range = Query.date_range(query)+current_date =DateTime.now!(site.timezone)|> Timex.to_date()- |> Time.date_or_weekstart(query)+ |> Time.date_or_weekstart(date_range)|> Date.to_string()Enum.find_index(dates, &(&1 == current_date))diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.exindex e5e568cc27..dd1d52e450 100644--- a/lib/plausible_web/controllers/stats_controller.ex+++ b/lib/plausible_web/controllers/stats_controller.ex@@ -110,8 +110,10 @@ defmodule PlausibleWeb.StatsController dosite = Plausible.Repo.preload(conn.assigns.site, :owner)query = Query.from(site, params, debug_metadata(conn))+ date_range = Query.date_range(query)+filename =- ~c"Plausible export #{params["domain"]} #{Date.to_iso8601(query.date_range.first)} to #{Date.to_iso8601(query.date_range.last)} .zip"+ ~c"Plausible export #{params["domain"]} #{Date.to_iso8601(date_range.first)} to #{Date.to_iso8601(date_range.last)} .zip"params = Map.merge(params, %{"limit" => "300", "csv" => "True", "detailed" => "True"})limited_params = Map.merge(params, %{"limit" => "100"})diff --git a/lib/plausible_web/mjml/templates/stats_report.mjml.eex b/lib/plausible_web/mjml/templates/stats_report.mjml.eexindex 2818c93307..51782cba66 100644--- a/lib/plausible_web/mjml/templates/stats_report.mjml.eex+++ b/lib/plausible_web/mjml/templates/stats_report.mjml.eex@@ -22,7 +22,7 @@<%= @site.domain %></mj-text><mj-text mj-class="text-sm" padding="0 25px 0 25px">- <%= @name %> Report (<%= Calendar.strftime(@query.date_range.last, "%-d %b %Y") %>)+ <%= @name %> Report (<%= @date %>)</mj-text></mj-column></mj-section>diff --git a/lib/workers/send_email_report.ex b/lib/workers/send_email_report.exindex 85b31986ad..4494ea3950 100644--- a/lib/workers/send_email_report.ex+++ b/lib/workers/send_email_report.ex@@ -9,9 +9,11 @@ defmodule Plausible.Workers.SendEmailReport doif site && site.weekly_report do%{site: site}+ |> put_last_week_query()+ |> put_date_range()|> Map.put(:type, :weekly)|> Map.put(:name, "Weekly")- |> put_last_week_query()+ |> put(:date, &Calendar.strftime(&1.date_range.last, "%-d %b %Y"))|> put_stats()|> send_report_for_all(site.weekly_report.recipients)else@@ -25,9 +27,11 @@ defmodule Plausible.Workers.SendEmailReport doif site && site.monthly_report do%{site: site}- |> Map.put(:type, :monthly)|> put_last_month_query()- |> put_monthly_report_name()+ |> put_date_range()+ |> Map.put(:type, :monthly)+ |> put(:name, &Calendar.strftime(&1.date_range.first, "%B"))+ |> put(:date, &Calendar.strftime(&1.date_range.last, "%-d %b %Y"))|> put_stats()|> send_report_for_all(site.monthly_report.recipients)else@@ -76,11 +80,15 @@ defmodule Plausible.Workers.SendEmailReport doMap.put(assigns, :query, query)end- defp put_monthly_report_name(%{query: query} = assigns) do- Map.put(assigns, :name, Calendar.strftime(query.date_range.first, "%B"))+ defp put_date_range(%{query: query} = assigns) do+ Map.put(assigns, :date_range, Query.date_range(query))enddefp put_stats(%{site: site, query: query} = assigns) doMap.put(assigns, :stats, Plausible.Stats.EmailReport.get(site, query))end++ defp put(assigns, key, value_fn) do+ Map.put(assigns, key, value_fn.(assigns))+ endenddiff --git a/priv/json-schemas/query-api-schema.json b/priv/json-schemas/query-api-schema.jsonindex 9b80aac218..b6c062f334 100644--- a/priv/json-schemas/query-api-schema.json+++ b/priv/json-schemas/query-api-schema.json@@ -95,15 +95,23 @@"minItems": 2,"maxItems": 2,"items": {- "type": "string",- "pattern": "^\\d{4}-\\d{2}-\\d{2}(?:T\\d{2}:\\d{2}:\\d{2}\\s[A-Za-z/_]+)?$"+ "oneOf": [+ {+ "type": "string",+ "format": "date"+ },+ {+ "type": "string",+ "format": "date-time"+ }+ ]},- "markdownDescription": "A list of two elements to determine the query date range. Both elements should have the same format - either `YYYY-MM-DD` or `YYYY-MM-DDThh:mm:ss <timezone>`",+ "markdownDescription": "A list of two ISO8601 dates or timestamps to determine the query date range.","examples": [["2024-01-01", "2024-01-31"],[- "2024-01-01T00:00:00 Europe/Tallinn",- "2024-01-01T12:00:00 Europe/Tallinn"+ "2024-01-01T00:00:00+03:00",+ "2024-01-02T12:00:00+03:00"]]}diff --git a/test/plausible/google/api_test.exs b/test/plausible/google/api_test.exsindex ef6a96d93a..4724d360a7 100644--- a/test/plausible/google/api_test.exs+++ b/test/plausible/google/api_test.exs@@ -3,7 +3,7 @@ defmodule Plausible.Google.APITest douse Plausible.Test.Support.HTTPMockeralias Plausible.Google- alias Plausible.Stats.Query+ alias Plausible.Stats.{DateTimeRange, Query}import ExUnit.CaptureLogimport Mox@@ -111,7 +111,8 @@ defmodule Plausible.Google.APITest doendtest "returns error when google auth not configured", %{site: site} do- query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}+ time_range = DateTimeRange.new!(~U[2022-01-01 00:00:00Z], ~U[2022-01-05 23:59:59Z])+ query = %Plausible.Stats.Query{utc_time_range: time_range}assert {:error, :google_property_not_configured} = Google.API.fetch_stats(site, query, 5, "")enddiff --git a/test/plausible/stats/comparisons_test.exs b/test/plausible/stats/comparisons_test.exsindex 7d2504c6a7..f2408dc173 100644--- a/test/plausible/stats/comparisons_test.exs+++ b/test/plausible/stats/comparisons_test.exs@@ -11,11 +11,8 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} = Comparisons.compare(site, query, "previous_period", now: now)- assert comparison.date_range.first ==- DateTime.new!(~D[2023-02-27], ~T[00:00:00], site.timezone)-- assert comparison.date_range.last ==- DateTime.new!(~D[2023-02-28], ~T[23:59:59], site.timezone)+ assert comparison.utc_time_range.first == ~U[2023-02-27 00:00:00Z]+ assert comparison.utc_time_range.last == ~U[2023-02-28 23:59:59Z]endtest "shifts back this month period when it's the first day of the month and mode is previous_period" do@@ -25,11 +22,8 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} = Comparisons.compare(site, query, "previous_period", now: now)- assert comparison.date_range.first ==- DateTime.new!(~D[2023-02-28], ~T[00:00:00], site.timezone)-- assert comparison.date_range.last ==- DateTime.new!(~D[2023-02-28], ~T[23:59:59], site.timezone)+ assert comparison.utc_time_range.first == ~U[2023-02-28 00:00:00Z]+ assert comparison.utc_time_range.last == ~U[2023-02-28 23:59:59Z]endtest "matches the day of the week when nearest day is original query start date and mode is previous_period" do@@ -40,11 +34,19 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} =Comparisons.compare(site, query, "previous_period", now: now, match_day_of_week?: true)- assert comparison.date_range.first ==- DateTime.new!(~D[2023-02-22], ~T[00:00:00], site.timezone)+ assert comparison.utc_time_range.first == ~U[2023-02-22 00:00:00Z]+ assert comparison.utc_time_range.last == ~U[2023-02-23 23:59:59Z]+ end++ test "custom time zone sets timezone to UTC" do+ site = insert(:site, timezone: "US/Eastern")+ query = Query.from(site, %{"period" => "month", "date" => "2023-03-02"})+ now = ~N[2023-03-02 14:00:00]- assert comparison.date_range.last ==- DateTime.new!(~D[2023-02-23], ~T[23:59:59], site.timezone)+ {:ok, comparison} = Comparisons.compare(site, query, "previous_period", now: now)++ assert comparison.utc_time_range.first == ~U[2023-02-27 05:00:00Z]+ assert comparison.utc_time_range.last == ~U[2023-03-01 04:59:59Z]endend@@ -56,11 +58,8 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} = Comparisons.compare(site, query, "previous_period", now: now)- assert comparison.date_range.first ==- DateTime.new!(~D[2023-01-04], ~T[00:00:00], site.timezone)-- assert comparison.date_range.last ==- DateTime.new!(~D[2023-01-31], ~T[23:59:59], site.timezone)+ assert comparison.utc_time_range.first == ~U[2023-01-04 00:00:00Z]+ assert comparison.utc_time_range.last == ~U[2023-01-31 23:59:59Z]endtest "shifts back the full month when mode is year_over_year" do@@ -70,11 +69,8 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} = Comparisons.compare(site, query, "year_over_year", now: now)- assert comparison.date_range.first ==- DateTime.new!(~D[2022-02-01], ~T[00:00:00], site.timezone)-- assert comparison.date_range.last ==- DateTime.new!(~D[2022-02-28], ~T[23:59:59], site.timezone)+ assert comparison.utc_time_range.first == ~U[2022-02-01 00:00:00Z]+ assert comparison.utc_time_range.last == ~U[2022-02-28 23:59:59Z]endtest "shifts back whole month plus one day when mode is year_over_year and a leap year" do@@ -84,11 +80,8 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} = Comparisons.compare(site, query, "year_over_year", now: now)- assert comparison.date_range.first ==- DateTime.new!(~D[2019-02-01], ~T[00:00:00], site.timezone)-- assert comparison.date_range.last ==- DateTime.new!(~D[2019-03-01], ~T[23:59:59], site.timezone)+ assert comparison.utc_time_range.first == ~U[2019-02-01 00:00:00Z]+ assert comparison.utc_time_range.last == ~U[2019-03-01 23:59:59Z]endtest "matches the day of the week when mode is previous_period keeping the same day" do@@ -99,11 +92,8 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} =Comparisons.compare(site, query, "previous_period", now: now, match_day_of_week?: true)- assert comparison.date_range.first ==- DateTime.new!(~D[2023-01-04], ~T[00:00:00], site.timezone)-- assert comparison.date_range.last ==- DateTime.new!(~D[2023-01-31], ~T[23:59:59], site.timezone)+ assert comparison.utc_time_range.first == ~U[2023-01-04 00:00:00Z]+ assert comparison.utc_time_range.last == ~U[2023-01-31 23:59:59Z]endtest "matches the day of the week when mode is previous_period" do@@ -114,11 +104,8 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} =Comparisons.compare(site, query, "previous_period", now: now, match_day_of_week?: true)- assert comparison.date_range.first ==- DateTime.new!(~D[2022-12-04], ~T[00:00:00], site.timezone)-- assert comparison.date_range.last ==- DateTime.new!(~D[2023-01-03], ~T[23:59:59], site.timezone)+ assert comparison.utc_time_range.first == ~U[2022-12-04 00:00:00Z]+ assert comparison.utc_time_range.last == ~U[2023-01-03 23:59:59Z]endend@@ -130,11 +117,8 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} = Comparisons.compare(site, query, "previous_period", now: now)- assert comparison.date_range.first ==- DateTime.new!(~D[2022-11-02], ~T[00:00:00], site.timezone)-- assert comparison.date_range.last ==- DateTime.new!(~D[2022-12-31], ~T[23:59:59], site.timezone)+ assert comparison.utc_time_range.first == ~U[2022-11-02 00:00:00Z]+ assert comparison.utc_time_range.last == ~U[2022-12-31 23:59:59Z]endtest "shifts back by the same number of days when mode is year_over_year" do@@ -144,11 +128,8 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} = Comparisons.compare(site, query, "year_over_year", now: now)- assert comparison.date_range.first ==- DateTime.new!(~D[2022-01-01], ~T[00:00:00], site.timezone)-- assert comparison.date_range.last ==- DateTime.new!(~D[2022-03-01], ~T[23:59:59], site.timezone)+ assert comparison.utc_time_range.first == ~U[2022-01-01 00:00:00Z]+ assert comparison.utc_time_range.last == ~U[2022-03-01 23:59:59Z]endtest "matches the day of the week when mode is year_over_year" do@@ -159,11 +140,8 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} =Comparisons.compare(site, query, "year_over_year", now: now, match_day_of_week?: true)- assert comparison.date_range.first ==- DateTime.new!(~D[2022-01-02], ~T[00:00:00], site.timezone)-- assert comparison.date_range.last ==- DateTime.new!(~D[2022-03-02], ~T[23:59:59], site.timezone)+ assert comparison.utc_time_range.first == ~U[2022-01-02 00:00:00Z]+ assert comparison.utc_time_range.last == ~U[2022-03-02 23:59:59Z]endend@@ -174,11 +152,8 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} = Comparisons.compare(site, query, "year_over_year")- assert comparison.date_range.first ==- DateTime.new!(~D[2021-01-01], ~T[00:00:00], site.timezone)-- assert comparison.date_range.last ==- DateTime.new!(~D[2021-12-31], ~T[23:59:59], site.timezone)+ assert comparison.utc_time_range.first == ~U[2021-01-01 00:00:00Z]+ assert comparison.utc_time_range.last == ~U[2021-12-31 23:59:59Z]endtest "shifts back a whole year when mode is previous_period" do@@ -187,11 +162,8 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} = Comparisons.compare(site, query, "previous_period")- assert comparison.date_range.first ==- DateTime.new!(~D[2021-01-01], ~T[00:00:00], site.timezone)-- assert comparison.date_range.last ==- DateTime.new!(~D[2021-12-31], ~T[23:59:59], site.timezone)+ assert comparison.utc_time_range.first == ~U[2021-01-01 00:00:00Z]+ assert comparison.utc_time_range.last == ~U[2021-12-31 23:59:59Z]endend@@ -202,11 +174,8 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} = Comparisons.compare(site, query, "previous_period")- assert comparison.date_range.first ==- DateTime.new!(~D[2022-12-25], ~T[00:00:00], site.timezone)-- assert comparison.date_range.last ==- DateTime.new!(~D[2022-12-31], ~T[23:59:59], site.timezone)+ assert comparison.utc_time_range.first == ~U[2022-12-25 00:00:00Z]+ assert comparison.utc_time_range.last == ~U[2022-12-31 23:59:59Z]endtest "shifts back to last year when mode is year_over_year" do@@ -215,11 +184,8 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} = Comparisons.compare(site, query, "year_over_year")- assert comparison.date_range.first ==- DateTime.new!(~D[2022-01-01], ~T[00:00:00], site.timezone)-- assert comparison.date_range.last ==- DateTime.new!(~D[2022-01-07], ~T[23:59:59], site.timezone)+ assert comparison.utc_time_range.first == ~U[2022-01-01 00:00:00Z]+ assert comparison.utc_time_range.last == ~U[2022-01-07 23:59:59Z]endend@@ -231,11 +197,8 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} =Comparisons.compare(site, query, "custom", from: "2022-05-25", to: "2022-05-30")- assert comparison.date_range.first ==- DateTime.new!(~D[2022-05-25], ~T[00:00:00], site.timezone)-- assert comparison.date_range.last ==- DateTime.new!(~D[2022-05-30], ~T[23:59:59], site.timezone)+ assert comparison.utc_time_range.first == ~U[2022-05-25 00:00:00Z]+ assert comparison.utc_time_range.last == ~U[2022-05-30 23:59:59Z]endtest "validates from and to dates" dodiff --git a/test/plausible/stats/query_optimizer_test.exs b/test/plausible/stats/query_optimizer_test.exsindex 259d540287..dc9e9a7d1b 100644--- a/test/plausible/stats/query_optimizer_test.exs+++ b/test/plausible/stats/query_optimizer_test.exs@@ -24,7 +24,7 @@ defmodule Plausible.Stats.QueryOptimizerTest dotest "adds time and first metric to order_by if order_by not specified" doassert perform(%{- date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-31], "UTC"),+ utc_time_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-31], "UTC"),metrics: [:pageviews, :visitors],dimensions: ["time", "event:page"]}).order_by ==@@ -35,14 +35,14 @@ defmodule Plausible.Stats.QueryOptimizerTest dodescribe "update_group_by_time" dotest "does nothing if `time` dimension not passed" doassert perform(%{- date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-04], "UTC"),+ utc_time_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-04], "UTC"),dimensions: ["time:month"]}).dimensions == ["time:month"]endtest "updating time dimension" doassert perform(%{- date_range:+ utc_time_range:DateTimeRange.new!(DateTime.new!(~D[2022-01-01], ~T[00:00:00], "UTC"),DateTime.new!(~D[2022-01-01], ~T[05:00:00], "UTC")@@ -51,57 +51,57 @@ defmodule Plausible.Stats.QueryOptimizerTest do}).dimensions == ["time:hour"]assert perform(%{- date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-01], "UTC"),+ utc_time_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-01], "UTC"),dimensions: ["time"]}).dimensions == ["time:hour"]assert perform(%{- date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-02], "UTC"),+ utc_time_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-02], "UTC"),dimensions: ["time"]}).dimensions == ["time:hour"]assert perform(%{- date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-03], "UTC"),+ utc_time_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-03], "UTC"),dimensions: ["time"]}).dimensions == ["time:day"]assert perform(%{- date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-10], "UTC"),+ utc_time_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-10], "UTC"),dimensions: ["time"]}).dimensions == ["time:day"]assert perform(%{- date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-16], "UTC"),+ utc_time_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-16], "UTC"),dimensions: ["time"]}).dimensions == ["time:day"]assert perform(%{- date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-02-16], "UTC"),+ utc_time_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-02-16], "UTC"),dimensions: ["time"]}).dimensions == ["time:week"]assert perform(%{- date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-03-16], "UTC"),+ utc_time_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-03-16], "UTC"),dimensions: ["time"]}).dimensions == ["time:week"]assert perform(%{- date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-03-16], "UTC"),+ utc_time_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-03-16], "UTC"),dimensions: ["time"]}).dimensions == ["time:week"]assert perform(%{- date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2023-11-16], "UTC"),+ utc_time_range: DateTimeRange.new!(~D[2022-01-01], ~D[2023-11-16], "UTC"),dimensions: ["time"]}).dimensions == ["time:month"]assert perform(%{- date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2024-01-16], "UTC"),+ utc_time_range: DateTimeRange.new!(~D[2022-01-01], ~D[2024-01-16], "UTC"),dimensions: ["time"]}).dimensions == ["time:month"]assert perform(%{- date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2026-01-01], "UTC"),+ utc_time_range: DateTimeRange.new!(~D[2022-01-01], ~D[2026-01-01], "UTC"),dimensions: ["time"]}).dimensions == ["time:month"]end@@ -110,7 +110,7 @@ defmodule Plausible.Stats.QueryOptimizerTest dodescribe "update_time_in_order_by" dotest "updates explicit time dimension in order_by" doassert perform(%{- date_range:+ utc_time_range:DateTimeRange.new!(DateTime.new!(~D[2022-01-01], ~T[00:00:00], "UTC"),DateTime.new!(~D[2022-01-01], ~T[05:00:00], "UTC")@@ -124,7 +124,7 @@ defmodule Plausible.Stats.QueryOptimizerTest dodescribe "extend_hostname_filters_to_visit" dotest "updates filters it filtering by event:hostname and visit:referrer and visit:exit_page dimensions" doassert perform(%{- date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]),+ utc_time_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]),filters: [[:is, "event:hostname", ["example.com"]],[:matches_wildcard, "event:hostname", ["*.com"]]@@ -142,7 +142,7 @@ defmodule Plausible.Stats.QueryOptimizerTest dotest "does not update filters if not needed" doassert perform(%{- date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]),+ utc_time_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]),filters: [[:is, "event:hostname", ["example.com"]]],diff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exsindex 14f4e5ec8d..8163ff8107 100644--- a/test/plausible/stats/query_parser_test.exs+++ b/test/plausible/stats/query_parser_test.exs@@ -7,42 +7,42 @@ defmodule Plausible.Stats.Filters.QueryParserTest dosetup [:create_user, :create_new_site]- @now DateTime.new!(~D[2021-05-05], ~T[12:30:00], "UTC")+ @now DateTime.new!(~D[2021-05-05], ~T[12:30:00], "Etc/UTC")@date_range_realtime %DateTimeRange{- first: DateTime.new!(~D[2021-05-05], ~T[12:25:00], "UTC"),- last: DateTime.new!(~D[2021-05-05], ~T[12:30:05], "UTC")+ first: DateTime.new!(~D[2021-05-05], ~T[12:25:00], "Etc/UTC"),+ last: DateTime.new!(~D[2021-05-05], ~T[12:30:05], "Etc/UTC")}@date_range_30m %DateTimeRange{- first: DateTime.new!(~D[2021-05-05], ~T[12:00:00], "UTC"),- last: DateTime.new!(~D[2021-05-05], ~T[12:30:05], "UTC")+ first: DateTime.new!(~D[2021-05-05], ~T[12:00:00], "Etc/UTC"),+ last: DateTime.new!(~D[2021-05-05], ~T[12:30:05], "Etc/UTC")}@date_range_day %DateTimeRange{- first: DateTime.new!(~D[2021-05-05], ~T[00:00:00], "UTC"),- last: DateTime.new!(~D[2021-05-05], ~T[23:59:59], "UTC")+ first: DateTime.new!(~D[2021-05-05], ~T[00:00:00], "Etc/UTC"),+ last: DateTime.new!(~D[2021-05-05], ~T[23:59:59], "Etc/UTC")}@date_range_7d %DateTimeRange{- first: DateTime.new!(~D[2021-04-29], ~T[00:00:00], "UTC"),- last: DateTime.new!(~D[2021-05-05], ~T[23:59:59], "UTC")+ first: DateTime.new!(~D[2021-04-29], ~T[00:00:00], "Etc/UTC"),+ last: DateTime.new!(~D[2021-05-05], ~T[23:59:59], "Etc/UTC")}@date_range_30d %DateTimeRange{- first: DateTime.new!(~D[2021-04-05], ~T[00:00:00], "UTC"),- last: DateTime.new!(~D[2021-05-05], ~T[23:59:59], "UTC")+ first: DateTime.new!(~D[2021-04-05], ~T[00:00:00], "Etc/UTC"),+ last: DateTime.new!(~D[2021-05-05], ~T[23:59:59], "Etc/UTC")}@date_range_month %DateTimeRange{- first: DateTime.new!(~D[2021-05-01], ~T[00:00:00], "UTC"),- last: DateTime.new!(~D[2021-05-31], ~T[23:59:59], "UTC")+ first: DateTime.new!(~D[2021-05-01], ~T[00:00:00], "Etc/UTC"),+ last: DateTime.new!(~D[2021-05-31], ~T[23:59:59], "Etc/UTC")}@date_range_6mo %DateTimeRange{- first: DateTime.new!(~D[2020-12-01], ~T[00:00:00], "UTC"),- last: DateTime.new!(~D[2021-05-31], ~T[23:59:59], "UTC")+ first: DateTime.new!(~D[2020-12-01], ~T[00:00:00], "Etc/UTC"),+ last: DateTime.new!(~D[2021-05-31], ~T[23:59:59], "Etc/UTC")}@date_range_year %DateTimeRange{- first: DateTime.new!(~D[2021-01-01], ~T[00:00:00], "UTC"),- last: DateTime.new!(~D[2021-12-31], ~T[23:59:59], "UTC")+ first: DateTime.new!(~D[2021-01-01], ~T[00:00:00], "Etc/UTC"),+ last: DateTime.new!(~D[2021-12-31], ~T[23:59:59], "Etc/UTC")}@date_range_12mo %DateTimeRange{- first: DateTime.new!(~D[2020-06-01], ~T[00:00:00], "UTC"),- last: DateTime.new!(~D[2021-05-31], ~T[23:59:59], "UTC")+ first: DateTime.new!(~D[2020-06-01], ~T[00:00:00], "Etc/UTC"),+ last: DateTime.new!(~D[2021-05-31], ~T[23:59:59], "Etc/UTC")}def check_success(params, site, expected_result, schema_type \\ :public) do@@ -55,7 +55,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest doassert message == expected_error_messageend- def check_date_range(date_params, site, expected_fields, schema_type \\ :public) do+ def check_date_range(date_params, site, expected_date_range, schema_type \\ :public) doparams =%{"site_id" => site.domain, "metrics" => ["visitors", "events"]}|> Map.merge(date_params)@@ -63,11 +63,11 @@ defmodule Plausible.Stats.Filters.QueryParserTest doexpected_parsed =%{metrics: [:visitors, :events],- date_range: expected_fields.date_range,+ utc_time_range: expected_date_range,filters: [],dimensions: [],order_by: nil,- timezone: Map.get(expected_fields, :timezone, site.timezone),+ timezone: site.timezone,include: %{imports: false, time_labels: false},preloaded_goals: []}@@ -85,7 +85,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do%{"site_id" => site.domain, "metrics" => ["visitors", "events"], "date_range" => "all"}|> check_success(site, %{metrics: [:visitors, :events],- date_range: @date_range_day,+ utc_time_range: @date_range_day,filters: [],dimensions: [],order_by: nil,@@ -126,7 +126,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do:bounce_rate,:visit_duration],- date_range: @date_range_day,+ utc_time_range: @date_range_day,filters: [],dimensions: [],order_by: nil,@@ -190,7 +190,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dosite,%{metrics: [:visitors],- date_range: @date_range_day,+ utc_time_range: @date_range_day,filters: [[unquote(operation), "event:name", ["foo"]]],@@ -315,7 +315,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:visitors],- date_range: @date_range_day,+ utc_time_range: @date_range_day,filters: [[:is, "event:props:foobar", ["value"]]],@@ -340,7 +340,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:visitors],- date_range: @date_range_day,+ utc_time_range: @date_range_day,filters: [[:is, "event:#{unquote(dimension)}", ["foo"]]],@@ -366,7 +366,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:visitors],- date_range: @date_range_day,+ utc_time_range: @date_range_day,filters: [[:is, "visit:#{unquote(dimension)}", ["ab"]]],@@ -432,7 +432,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:visitors],- date_range: @date_range_day,+ utc_time_range: @date_range_day,filters: [[:is, "visit:city", [123, 456]]],@@ -451,7 +451,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:visitors],- date_range: @date_range_day,+ utc_time_range: @date_range_day,filters: [[:is, "visit:city", ["123", "456"]]],@@ -499,7 +499,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:visitors],- date_range: @date_range_day,+ utc_time_range: @date_range_day,filters: [[:or,@@ -558,7 +558,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:visitors],- date_range: @date_range_day,+ utc_time_range: @date_range_day,filters: [[:is, "event:hostname", ["a.plausible.io"]]],@@ -595,7 +595,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:visitors],- date_range: @date_range_day,+ utc_time_range: @date_range_day,filters: [],dimensions: ["time"],order_by: nil,@@ -645,7 +645,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest doassert %{metrics: [:visitors],- date_range: @date_range_day,+ utc_time_range: @date_range_day,filters: [[:is, "event:goal", ["Signup", "Visit /thank-you"]]],@@ -729,42 +729,22 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodescribe "date range validation" dotest "parsing shortcut options", %{site: site} do- check_date_range(%{"date_range" => "day"}, site, %{date_range: @date_range_day})- check_date_range(%{"date_range" => "7d"}, site, %{date_range: @date_range_7d})- check_date_range(%{"date_range" => "30d"}, site, %{date_range: @date_range_30d})- check_date_range(%{"date_range" => "month"}, site, %{date_range: @date_range_month})- check_date_range(%{"date_range" => "6mo"}, site, %{date_range: @date_range_6mo})- check_date_range(%{"date_range" => "12mo"}, site, %{date_range: @date_range_12mo})- check_date_range(%{"date_range" => "year"}, site, %{date_range: @date_range_year})+ check_date_range(%{"date_range" => "day"}, site, @date_range_day)+ check_date_range(%{"date_range" => "7d"}, site, @date_range_7d)+ check_date_range(%{"date_range" => "30d"}, site, @date_range_30d)+ check_date_range(%{"date_range" => "month"}, site, @date_range_month)+ check_date_range(%{"date_range" => "6mo"}, site, @date_range_6mo)+ check_date_range(%{"date_range" => "12mo"}, site, @date_range_12mo)+ check_date_range(%{"date_range" => "year"}, site, @date_range_year)endtest "30m and realtime are available in internal API", %{site: site} do- check_date_range(%{"date_range" => "30m"}, site, %{date_range: @date_range_30m}, :internal)-- check_date_range(- %{"date_range" => "realtime"},- site,- %{date_range: @date_range_realtime},- :internal- )- end-- test "timezone is UTC instead of site.timezone for realtime and 30m periods", %{- site: site- } do- site = struct!(site, timezone: "Europe/Tallinn")-- check_date_range(- %{"date_range" => "30m"},- site,- %{date_range: @date_range_30m, timezone: "UTC"},- :internal- )+ check_date_range(%{"date_range" => "30m"}, site, @date_range_30m, :internal)check_date_range(%{"date_range" => "realtime"},site,- %{date_range: @date_range_realtime, timezone: "UTC"},+ @date_range_realtime,:internal)end@@ -780,50 +760,41 @@ defmodule Plausible.Stats.Filters.QueryParserTest dotest "parsing `all` with previous data", %{site: site} dosite = Map.put(site, :stats_start_date, ~D[2020-01-01])- expected_date_range = DateTimeRange.new!(~D[2020-01-01], ~D[2021-05-05], "UTC")- check_date_range(%{"date_range" => "all"}, site, %{date_range: expected_date_range})+ expected_date_range = DateTimeRange.new!(~D[2020-01-01], ~D[2021-05-05], "Etc/UTC")+ check_date_range(%{"date_range" => "all"}, site, expected_date_range)endtest "parsing `all` with no previous data", %{site: site} dosite = Map.put(site, :stats_start_date, nil)- check_date_range(%{"date_range" => "all"}, site, %{date_range: @date_range_day})+ check_date_range(%{"date_range" => "all"}, site, @date_range_day)endtest "parsing custom date range from simple date strings", %{site: site} do- check_date_range(%{"date_range" => ["2021-05-05", "2021-05-05"]}, site, %{- date_range: @date_range_day- })+ check_date_range(%{"date_range" => ["2021-05-05", "2021-05-05"]}, site, @date_range_day)endtest "parsing custom date range from iso8601 timestamps", %{site: site} docheck_date_range(- %{"date_range" => ["2024-01-01T00:00:00 UTC", "2024-01-02T23:59:59 UTC"]},+ %{"date_range" => ["2024-01-01T00:00:00Z", "2024-01-02T23:59:59Z"]},site,- %{- date_range:- DateTimeRange.new!(- DateTime.new!(~D[2024-01-01], ~T[00:00:00], "UTC"),- DateTime.new!(~D[2024-01-02], ~T[23:59:59], "UTC")- )- }+ DateTimeRange.new!(+ DateTime.new!(~D[2024-01-01], ~T[00:00:00], "Etc/UTC"),+ DateTime.new!(~D[2024-01-02], ~T[23:59:59], "Etc/UTC")+ ))check_date_range(%{"date_range" => [- "2024-08-29T07:12:34 America/Los_Angeles",- "2024-08-29T10:12:34 America/Los_Angeles"+ "2024-08-29T07:12:34-07:00",+ "2024-08-29T10:12:34-07:00"]},site,- %{- date_range:- DateTimeRange.new!(- DateTime.new!(~D[2024-08-29], ~T[07:12:34], "America/Los_Angeles"),- DateTime.new!(~D[2024-08-29], ~T[10:12:34], "America/Los_Angeles")- ),- timezone: "America/Los_Angeles"- }+ DateTimeRange.new!(+ ~U[2024-08-29 14:12:34Z],+ ~U[2024-08-29 17:12:34Z]+ ))end@@ -845,59 +816,31 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_error(site,- "#/date_range: Invalid date range [\"2021-02-03T00:00:00\", \"2021-02-03T23:59:59\"]"- )- end-- test "custom date range is invalid when timestamp timezones are different", %{site: site} do- %{- "site_id" => site.domain,- "date_range" => ["2021-02-03T00:00:00 Europe/Tallinn", "2021-02-03T23:59:59 UTC"],- "metrics" => ["visitors"]- }- |> check_error(- site,- "Invalid date_range '[\"2021-02-03T00:00:00 Europe/Tallinn\", \"2021-02-03T23:59:59 UTC\"]'."+ "Invalid date_range '[\"2021-02-03T00:00:00\", \"2021-02-03T23:59:59\"]'.")endtest "custom date range is invalid when timestamp timezone is invalid", %{site: site} do%{"site_id" => site.domain,- "date_range" => ["2021-02-03T00:00:00 Fake/Timezone", "2021-02-03T23:59:59 Fake/Timezone"],+ "date_range" => ["2021-02-03T00:00:00-25:00", "2021-02-03T23:59:59-25:00"],"metrics" => ["visitors"]}|> check_error(site,- "Invalid date_range '[\"2021-02-03T00:00:00 Fake/Timezone\", \"2021-02-03T23:59:59 Fake/Timezone\"]'."+ "#/date_range: Invalid date range [\"2021-02-03T00:00:00-25:00\", \"2021-02-03T23:59:59-25:00\"]")endtest "custom date range is invalid when date and timestamp are combined", %{site: site} do%{"site_id" => site.domain,- "date_range" => ["2021-02-03T00:00:00 UTC", "2021-02-04"],+ "date_range" => ["2021-02-03T00:00:00Z", "2021-02-04"],"metrics" => ["visitors"]}|> check_error(site,- "Invalid date_range '[\"2021-02-03T00:00:00 UTC\", \"2021-02-04\"]'."- )- end-- test "custom date range is invalid when timestamp cannot be converted to datetime due to a gap in timezone",- %{site: site} do- %{- "site_id" => site.domain,- "date_range" => [- "2024-03-31T03:30:00 Europe/Tallinn",- "2024-04-15T10:00:00 Europe/Tallinn"- ],- "metrics" => ["visitors"]- }- |> check_error(- site,- "Invalid date_range '[\"2024-03-31T03:30:00 Europe/Tallinn\", \"2024-04-15T10:00:00 Europe/Tallinn\"]'."+ "Invalid date_range '[\"2021-02-03T00:00:00Z\", \"2021-02-04\"]'.")end@@ -914,7 +857,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do{"year", @date_range_year}] do%{"date_range" => date_range_shortcut, "date" => date}- |> check_date_range(site, %{date_range: expected_date_range}, :internal)+ |> check_date_range(site, expected_date_range, :internal)endend@@ -933,14 +876,11 @@ defmodule Plausible.Stats.Filters.QueryParserTest do} dosite = %{site | timezone: "America/Santiago"}- expected_date_range =- DateTimeRange.new!(- DateTime.new!(~D[2022-09-11], ~T[01:00:00], site.timezone),- DateTime.new!(~D[2022-09-11], ~T[23:59:59], site.timezone)- )-%{"date_range" => ["2022-09-11", "2022-09-11"]}- |> check_date_range(site, %{date_range: expected_date_range})+ |> check_date_range(+ site,+ DateTimeRange.new!(~U[2022-09-11 04:00:00Z], ~U[2022-09-12 02:59:59Z])+ )endtest "parses date_range.first into the latest of ambiguous datetimes in site.timezone", %{@@ -948,17 +888,11 @@ defmodule Plausible.Stats.Filters.QueryParserTest do} dosite = %{site | timezone: "America/Havana"}- {:ambiguous, _, expected_first_datetime} =- DateTime.new(~D[2023-11-05], ~T[00:00:00], site.timezone)-- expected_date_range =- DateTimeRange.new!(- expected_first_datetime,- DateTime.new!(~D[2023-11-05], ~T[23:59:59], site.timezone)- )-%{"date_range" => ["2023-11-05", "2023-11-05"]}- |> check_date_range(site, %{date_range: expected_date_range})+ |> check_date_range(+ site,+ DateTimeRange.new!(~U[2023-11-05 05:00:00Z], ~U[2023-11-06 04:59:59Z])+ )endtest "parses date_range.last into the earliest of ambiguous datetimes in site.timezone", %{@@ -966,17 +900,11 @@ defmodule Plausible.Stats.Filters.QueryParserTest do} dosite = %{site | timezone: "America/Asuncion"}- {:ambiguous, first_dt, _second_dt} =- DateTime.new(~D[2024-03-23], ~T[23:59:59], site.timezone)-- expected_date_range =- DateTimeRange.new!(- DateTime.new!(~D[2024-03-23], ~T[00:00:00], site.timezone),- first_dt- )-%{"date_range" => ["2024-03-23", "2024-03-23"]}- |> check_date_range(site, %{date_range: expected_date_range})+ |> check_date_range(+ site,+ DateTimeRange.new!(~U[2024-03-23 03:00:00Z], ~U[2024-03-24 02:59:59Z])+ )endend@@ -991,7 +919,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:visitors],- date_range: @date_range_day,+ utc_time_range: @date_range_day,filters: [],dimensions: ["event:#{unquote(dimension)}"],order_by: nil,@@ -1012,7 +940,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:visitors],- date_range: @date_range_day,+ utc_time_range: @date_range_day,filters: [],dimensions: ["visit:#{unquote(dimension)}"],order_by: nil,@@ -1032,7 +960,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:visitors],- date_range: @date_range_day,+ utc_time_range: @date_range_day,filters: [],dimensions: ["event:props:foobar"],order_by: nil,@@ -1093,7 +1021,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:visitors, :events],- date_range: @date_range_day,+ utc_time_range: @date_range_day,filters: [],dimensions: [],order_by: [{:events, :desc}, {:visitors, :asc}],@@ -1113,7 +1041,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:visitors],- date_range: @date_range_day,+ utc_time_range: @date_range_day,filters: [],dimensions: ["event:name"],order_by: [{"event:name", :desc}],@@ -1221,7 +1149,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do# }# |> check_success(site, %{# metrics: [:conversion_rate],- # date_range: @date_range_day,+ # utc_time_range: @date_range_day,# filters: [[:is, "event:goal", [event: "Signup"]]],# dimensions: [],# order_by: nil,@@ -1241,7 +1169,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do# }# |> check_success(site, %{# metrics: [:conversion_rate],- # date_range: @date_range_day,+ # utc_time_range: @date_range_day,# filters: [],# dimensions: ["event:goal"],# order_by: nil,@@ -1261,7 +1189,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:conversion_rate, :group_conversion_rate],- date_range: @date_range_day,+ utc_time_range: @date_range_day,filters: [[:is, "event:props:foo", ["bar"]]],@@ -1299,7 +1227,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do# }# |> check_success(site, %{# metrics: [:views_per_visit],- # date_range: @date_range_day,+ # utc_time_range: @date_range_day,# filters: [[:is, "event:goal", [event: "Signup"]]],# dimensions: [],# order_by: nil,@@ -1346,7 +1274,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:bounce_rate],- date_range: @date_range_day,+ utc_time_range: @date_range_day,filters: [],dimensions: ["visit:device"],order_by: nil,@@ -1378,7 +1306,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:bounce_rate],- date_range: @date_range_day,+ utc_time_range: @date_range_day,filters: [],dimensions: ["event:page"],order_by: nil,@@ -1397,7 +1325,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:bounce_rate],- date_range: @date_range_day,+ utc_time_range: @date_range_day,filters: [[:is, "event:props:foo", ["(none)"]]],dimensions: [],order_by: nil,diff --git a/test/plausible/stats/query_result_test.exs b/test/plausible/stats/query_result_test.exsindex b06f296fdb..3782a0e9c7 100644--- a/test/plausible/stats/query_result_test.exs+++ b/test/plausible/stats/query_result_test.exs@@ -51,8 +51,8 @@ defmodule Plausible.Stats.QueryResultTest do"pageviews"],"date_range": [- "2024-01-01T00:00:00 UTC",- "2024-02-01T23:59:59 UTC"+ "2024-01-01T00:00:00+00:00",+ "2024-02-01T23:59:59+00:00"],"filters": [],"dimensions": [],diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exsindex 9634e1aece..89425a62e8 100644--- a/test/plausible/stats/query_test.exs+++ b/test/plausible/stats/query_test.exs@@ -1,6 +1,6 @@defmodule Plausible.Stats.QueryTest douse Plausible.DataCase, async: true- alias Plausible.Stats.{Query, DateTimeRange}+ alias Plausible.Stats.Queryalias Plausible.Stats.Legacy.QueryBuilderdoctest Plausible.Stats.Legacy.QueryBuilder@@ -12,12 +12,15 @@ defmodule Plausible.Stats.QueryTest doinsert(:site,members: [user],inserted_at: ~N[2020-01-01T00:00:00],- stats_start_date: ~D[2020-01-01]+ stats_start_date: ~D[2020-01-01],+ timezone: "US/Eastern"){:ok, site: site, user: user}end+ @now ~U[2024-05-03 16:30:00Z]+@tag :slowtest "keeps current timestamp so that utc_boundaries don't depend on time passing by", %{site: site@@ -32,193 +35,130 @@ defmodule Plausible.Stats.QueryTest doendtest "parses day format", %{site: site} do- q = Query.from(site, %{"period" => "day", "date" => "2019-01-01"})+ q = Query.from(site, %{"period" => "day", "date" => "2019-01-01"}, %{}, @now)- assert q.date_range.first == DateTime.new!(~D[2019-01-01], ~T[00:00:00], site.timezone)- assert q.date_range.last == DateTime.new!(~D[2019-01-01], ~T[23:59:59], site.timezone)+ assert q.utc_time_range.first == ~U[2019-01-01 05:00:00Z]+ assert q.utc_time_range.last == ~U[2019-01-02 04:59:59Z]assert q.interval == "hour"endtest "day format defaults to today", %{site: site} do- q = Query.from(site, %{"period" => "day"})-- expected_first_datetime = Date.utc_today() |> DateTime.new!(~T[00:00:00], site.timezone)- expected_last_datetime = Date.utc_today() |> DateTime.new!(~T[23:59:59], site.timezone)+ q = Query.from(site, %{"period" => "day"}, %{}, @now)- assert q.date_range.first == expected_first_datetime- assert q.date_range.last == expected_last_datetime+ assert q.utc_time_range.first == ~U[2024-05-03 04:00:00Z]+ assert q.utc_time_range.last == ~U[2024-05-04 03:59:59Z]assert q.interval == "hour"endtest "parses realtime format", %{site: site} do- q = Query.from(site, %{"period" => "realtime"})-- utc_now = DateTime.shift_zone!(q.now, "Etc/UTC")+ q = Query.from(site, %{"period" => "realtime"}, %{}, @now)- expected_first_datetime = utc_now |> DateTime.shift(minute: -5)- expected_last_datetime = utc_now |> DateTime.shift(second: 5)-- assert q.date_range.first == expected_first_datetime- assert q.date_range.last == expected_last_datetime+ assert q.utc_time_range.first == ~U[2024-05-03 16:25:00Z]+ assert q.utc_time_range.last == ~U[2024-05-03 16:30:05Z]assert q.period == "realtime"endtest "parses month format", %{site: site} do- q = Query.from(site, %{"period" => "month", "date" => "2019-01-01"})+ q = Query.from(site, %{"period" => "month", "date" => "2019-01-01"}, %{}, @now)- assert q.date_range.first == DateTime.new!(~D[2019-01-01], ~T[00:00:00], site.timezone)- assert q.date_range.last == DateTime.new!(~D[2019-01-31], ~T[23:59:59], site.timezone)+ assert q.utc_time_range.first == ~U[2019-01-01 05:00:00Z]+ assert q.utc_time_range.last == ~U[2019-02-01 04:59:59Z]assert q.interval == "day"endtest "parses 6 month format", %{site: site} do- q = Query.from(site, %{"period" => "6mo"})-- expected_first_datetime =- q.now- |> DateTime.to_date()- |> Date.shift(month: -5)- |> Date.beginning_of_month()- |> DateTime.new!(~T[00:00:00], site.timezone)-- expected_last_datetime =- q.now- |> DateTime.to_date()- |> Date.end_of_month()- |> DateTime.new!(~T[23:59:59], site.timezone)-- assert q.date_range.first == expected_first_datetime- assert q.date_range.last == expected_last_datetime+ q = Query.from(site, %{"period" => "6mo"}, %{}, @now)++ assert q.utc_time_range.first == ~U[2023-12-01 05:00:00Z]+ assert q.utc_time_range.last == ~U[2024-06-01 03:59:59Z]assert q.interval == "month"endtest "parses 12 month format", %{site: site} do- q = Query.from(site, %{"period" => "12mo"})-- expected_first_datetime =- q.now- |> DateTime.to_date()- |> Date.shift(month: -11)- |> Date.beginning_of_month()- |> DateTime.new!(~T[00:00:00], site.timezone)-- expected_last_datetime =- q.now- |> DateTime.to_date()- |> Date.end_of_month()- |> DateTime.new!(~T[23:59:59], site.timezone)-- assert q.date_range.first == expected_first_datetime- assert q.date_range.last == expected_last_datetime+ q = Query.from(site, %{"period" => "12mo"}, %{}, @now)++ assert q.utc_time_range.first == ~U[2023-06-01 04:00:00Z]+ assert q.utc_time_range.last == ~U[2024-06-01 03:59:59Z]assert q.interval == "month"endtest "parses year to date format", %{site: site} do- q = Query.from(site, %{"period" => "year"})-- %Date{year: current_year} = DateTime.to_date(q.now)-- expected_first_datetime =- Date.new!(current_year, 1, 1)- |> DateTime.new!(~T[00:00:00], site.timezone)+ q = Query.from(site, %{"period" => "year"}, %{}, @now)- expected_last_datetime =- Date.new!(current_year, 12, 31)- |> DateTime.new!(~T[23:59:59], site.timezone)-- assert q.date_range.first == expected_first_datetime- assert q.date_range.last == expected_last_datetime+ assert q.utc_time_range.first == ~U[2024-01-01 05:00:00Z]+ assert q.utc_time_range.last == ~U[2025-01-01 04:59:59Z]assert q.interval == "month"endtest "parses all time", %{site: site} do- q = Query.from(site, %{"period" => "all"})-- expected_last_datetime =- q.now- |> DateTime.to_date()- |> DateTime.new!(~T[23:59:59], site.timezone)+ q = Query.from(site, %{"period" => "all"}, %{}, @now)- assert DateTime.to_naive(q.date_range.first) == site.inserted_at- assert q.date_range.last == expected_last_datetime+ assert q.utc_time_range.first == ~U[2020-01-01 05:00:00Z]+ assert q.utc_time_range.last == ~U[2024-05-04 03:59:59Z]assert q.period == "all"assert q.interval == "month"end- test "parses all time in site timezone", %{site: site} do- for timezone <- ["Etc/GMT+12", "Etc/GMT-12"] do- site = Map.put(site, :timezone, timezone)- query = Query.from(site, %{"period" => "all"})+ test "parses all time in GMT+12 timezone", %{site: site} do+ site = Map.put(site, :timezone, "Etc/GMT+12")+ q = Query.from(site, %{"period" => "all"}, %{}, @now)- expected_first_datetime = DateTime.new!(~D[2020-01-01], ~T[00:00:00], site.timezone)-- expected_last_datetime =- DateTime.now!(site.timezone)- |> DateTime.to_date()- |> DateTime.new!(~T[23:59:59], site.timezone)-- assert query.date_range.first == expected_first_datetime- assert query.date_range.last == expected_last_datetime- end+ assert q.utc_time_range.first == ~U[2020-01-01 12:00:00Z]+ assert q.utc_time_range.last == ~U[2024-05-04 11:59:59Z]endtest "all time shows today if site has no start date", %{site: site} dosite = Map.put(site, :stats_start_date, nil)- q = Query.from(site, %{"period" => "all"})+ q = Query.from(site, %{"period" => "all"}, %{}, @now)- today = Date.utc_today()-- assert q.date_range == DateTimeRange.new!(today, today, site.timezone)+ assert q.utc_time_range.first == ~U[2024-05-03 04:00:00Z]+ assert q.utc_time_range.last == ~U[2024-05-04 03:59:59Z]assert q.period == "all"assert q.interval == "hour"endtest "all time shows hourly if site is completely new", %{site: site} do- site = Map.put(site, :stats_start_date, Date.utc_today())- q = Query.from(site, %{"period" => "all"})-- today = Date.utc_today()+ site = Map.put(site, :stats_start_date, @now |> DateTime.to_date())+ q = Query.from(site, %{"period" => "all"}, %{}, @now)- assert q.date_range == DateTimeRange.new!(today, today, site.timezone)+ assert q.utc_time_range.first == ~U[2024-05-03 04:00:00Z]+ assert q.utc_time_range.last == ~U[2024-05-04 03:59:59Z]assert q.period == "all"assert q.interval == "hour"endtest "all time shows daily if site is more than a day old", %{site: site} do- today = Date.utc_today()- yesterday = today |> Date.shift(day: -1)-+ yesterday = @now |> DateTime.to_date() |> Date.shift(day: -1)site = Map.put(site, :stats_start_date, yesterday)- q = Query.from(site, %{"period" => "all"})+ q = Query.from(site, %{"period" => "all"}, %{}, @now)- assert q.date_range == DateTimeRange.new!(yesterday, today, site.timezone)+ assert q.utc_time_range.first == ~U[2024-05-02 04:00:00Z]+ assert q.utc_time_range.last == ~U[2024-05-04 03:59:59Z]assert q.period == "all"assert q.interval == "day"endtest "all time shows monthly if site is more than a month old", %{site: site} do- today = Date.utc_today()- last_month = today |> Date.shift(month: -1)-+ last_month = @now |> DateTime.to_date() |> Date.shift(month: -1)site = Map.put(site, :stats_start_date, last_month)- q = Query.from(site, %{"period" => "all"})+ q = Query.from(site, %{"period" => "all"}, %{}, @now)- assert q.date_range == DateTimeRange.new!(last_month, today, site.timezone)+ assert q.utc_time_range.first == ~U[2024-04-03 04:00:00Z]+ assert q.utc_time_range.last == ~U[2024-05-04 03:59:59Z]assert q.period == "all"assert q.interval == "month"endtest "all time uses passed interval different from the default interval", %{site: site} do- today = Date.utc_today()- last_month = today |> Date.shift(month: -1)-+ last_month = @now |> DateTime.to_date() |> Date.shift(month: -1)site = Map.put(site, :stats_start_date, last_month)- q = Query.from(site, %{"period" => "all", "interval" => "week"})+ q = Query.from(site, %{"period" => "all", "interval" => "week"}, %{}, @now)- assert q.date_range == DateTimeRange.new!(last_month, today, site.timezone)+ assert q.utc_time_range.first == ~U[2024-04-03 04:00:00Z]+ assert q.utc_time_range.last == ~U[2024-05-04 03:59:59Z]assert q.period == "all"assert q.interval == "week"end@@ -228,10 +168,16 @@ defmodule Plausible.Stats.QueryTest doendtest "parses custom format", %{site: site} do- q = Query.from(site, %{"period" => "custom", "from" => "2019-01-01", "to" => "2019-01-15"})+ q =+ Query.from(+ site,+ %{"period" => "custom", "from" => "2019-01-01", "to" => "2019-01-15"},+ %{},+ @now+ )- assert q.date_range.first == DateTime.new!(~D[2019-01-01], ~T[00:00:00], site.timezone)- assert q.date_range.last == DateTime.new!(~D[2019-01-15], ~T[23:59:59], site.timezone)+ assert q.utc_time_range.first == ~U[2019-01-01 05:00:00Z]+ assert q.utc_time_range.last == ~U[2019-01-16 04:59:59Z]assert q.interval == "day"enddiff --git a/test/plausible/stats/time_test.exs b/test/plausible/stats/time_test.exsindex 37cb983fb9..ba3dfb7a46 100644--- a/test/plausible/stats/time_test.exs+++ b/test/plausible/stats/time_test.exs@@ -8,7 +8,8 @@ defmodule Plausible.Stats.TimeTest dotest "with time:month dimension" doassert time_labels(%{dimensions: ["visit:device", "time:month"],- date_range: DateTimeRange.new!(~D[2022-01-17], ~D[2022-02-01], "UTC")+ utc_time_range: DateTimeRange.new!(~D[2022-01-17], ~D[2022-02-01], "UTC"),+ timezone: "UTC"}) == ["2022-01-01","2022-02-01"@@ -16,7 +17,8 @@ defmodule Plausible.Stats.TimeTest doassert time_labels(%{dimensions: ["visit:device", "time:month"],- date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-03-07], "UTC")+ utc_time_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-03-07], "UTC"),+ timezone: "UTC"}) == ["2022-01-01","2022-02-01",@@ -27,7 +29,8 @@ defmodule Plausible.Stats.TimeTest dotest "with time:week dimension" doassert time_labels(%{dimensions: ["time:week"],- date_range: DateTimeRange.new!(~D[2020-12-20], ~D[2021-01-08], "UTC")+ utc_time_range: DateTimeRange.new!(~D[2020-12-20], ~D[2021-01-08], "UTC"),+ timezone: "UTC"}) == ["2020-12-20","2020-12-21",@@ -37,7 +40,8 @@ defmodule Plausible.Stats.TimeTest doassert time_labels(%{dimensions: ["time:week"],- date_range: DateTimeRange.new!(~D[2020-12-21], ~D[2021-01-03], "UTC")+ utc_time_range: DateTimeRange.new!(~D[2020-12-21], ~D[2021-01-03], "UTC"),+ timezone: "UTC"}) == ["2020-12-21","2020-12-28"@@ -47,7 +51,8 @@ defmodule Plausible.Stats.TimeTest dotest "with time:day dimension" doassert time_labels(%{dimensions: ["time:day"],- date_range: DateTimeRange.new!(~D[2022-01-17], ~D[2022-02-02], "UTC")+ utc_time_range: DateTimeRange.new!(~D[2022-01-17], ~D[2022-02-02], "UTC"),+ timezone: "UTC"}) == ["2022-01-17","2022-01-18",@@ -72,7 +77,8 @@ defmodule Plausible.Stats.TimeTest dotest "with time:hour dimension" doassert time_labels(%{dimensions: ["time:hour"],- date_range: DateTimeRange.new!(~D[2022-01-17], ~D[2022-01-17], "UTC")+ utc_time_range: DateTimeRange.new!(~D[2022-01-17], ~D[2022-01-17], "UTC"),+ timezone: "UTC"}) == ["2022-01-17 00:00:00","2022-01-17 01:00:00",@@ -102,7 +108,8 @@ defmodule Plausible.Stats.TimeTest doassert time_labels(%{dimensions: ["time:hour"],- date_range: DateTimeRange.new!(~D[2022-01-17], ~D[2022-01-18], "UTC")+ utc_time_range: DateTimeRange.new!(~D[2022-01-17], ~D[2022-01-18], "UTC"),+ timezone: "UTC"}) == ["2022-01-17 00:00:00","2022-01-17 01:00:00",@@ -154,51 +161,75 @@ defmodule Plausible.Stats.TimeTest do"2022-01-18 23:00:00"]end- end- test "with time:minute dimension" do- now = DateTime.new!(~D[2024-01-01], ~T[12:30:57], "UTC")+ test "with a different time range" do+ {:ok, from_timestamp, _} = DateTime.from_iso8601("2024-09-04T21:53:17+03:00")+ {:ok, to_timestamp, _} = DateTime.from_iso8601("2024-09-05T05:59:59+03:00")++ assert time_labels(%{+ dimensions: ["time:hour"],+ utc_time_range:+ DateTimeRange.new!(from_timestamp, to_timestamp)+ |> DateTimeRange.to_timezone("Etc/UTC"),+ timezone: "Europe/Tallinn"+ }) == [+ "2024-09-04 21:00:00",+ "2024-09-04 22:00:00",+ "2024-09-04 23:00:00",+ "2024-09-05 00:00:00",+ "2024-09-05 01:00:00",+ "2024-09-05 02:00:00",+ "2024-09-05 03:00:00",+ "2024-09-05 04:00:00",+ "2024-09-05 05:00:00"+ ]+ end++ test "with time:minute dimension" do+ now = DateTime.new!(~D[2024-01-01], ~T[12:30:57], "UTC")- # ~U[2024-01-01 12:00:57Z]- first_dt = DateTime.shift(now, minute: -30)- # ~U[2024-01-01 12:31:02Z]- last_dt = DateTime.shift(now, second: 5)+ # ~U[2024-01-01 12:00:57Z]+ first_dt = DateTime.shift(now, minute: -30)+ # ~U[2024-01-01 12:31:02Z]+ last_dt = DateTime.shift(now, second: 5)- assert time_labels(%{- dimensions: ["time:minute"],- now: now,- date_range: DateTimeRange.new!(first_dt, last_dt)- }) == [- "2024-01-01 12:00:00",- "2024-01-01 12:01:00",- "2024-01-01 12:02:00",- "2024-01-01 12:03:00",- "2024-01-01 12:04:00",- "2024-01-01 12:05:00",- "2024-01-01 12:06:00",- "2024-01-01 12:07:00",- "2024-01-01 12:08:00",- "2024-01-01 12:09:00",- "2024-01-01 12:10:00",- "2024-01-01 12:11:00",- "2024-01-01 12:12:00",- "2024-01-01 12:13:00",- "2024-01-01 12:14:00",- "2024-01-01 12:15:00",- "2024-01-01 12:16:00",- "2024-01-01 12:17:00",- "2024-01-01 12:18:00",- "2024-01-01 12:19:00",- "2024-01-01 12:20:00",- "2024-01-01 12:21:00",- "2024-01-01 12:22:00",- "2024-01-01 12:23:00",- "2024-01-01 12:24:00",- "2024-01-01 12:25:00",- "2024-01-01 12:26:00",- "2024-01-01 12:27:00",- "2024-01-01 12:28:00",- "2024-01-01 12:29:00"- ]+ assert time_labels(%{+ dimensions: ["time:minute"],+ now: now,+ utc_time_range: DateTimeRange.new!(first_dt, last_dt),+ timezone: "UTC"+ }) == [+ "2024-01-01 12:00:00",+ "2024-01-01 12:01:00",+ "2024-01-01 12:02:00",+ "2024-01-01 12:03:00",+ "2024-01-01 12:04:00",+ "2024-01-01 12:05:00",+ "2024-01-01 12:06:00",+ "2024-01-01 12:07:00",+ "2024-01-01 12:08:00",+ "2024-01-01 12:09:00",+ "2024-01-01 12:10:00",+ "2024-01-01 12:11:00",+ "2024-01-01 12:12:00",+ "2024-01-01 12:13:00",+ "2024-01-01 12:14:00",+ "2024-01-01 12:15:00",+ "2024-01-01 12:16:00",+ "2024-01-01 12:17:00",+ "2024-01-01 12:18:00",+ "2024-01-01 12:19:00",+ "2024-01-01 12:20:00",+ "2024-01-01 12:21:00",+ "2024-01-01 12:22:00",+ "2024-01-01 12:23:00",+ "2024-01-01 12:24:00",+ "2024-01-01 12:25:00",+ "2024-01-01 12:26:00",+ "2024-01-01 12:27:00",+ "2024-01-01 12:28:00",+ "2024-01-01 12:29:00"+ ]+ endendend