Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Support realtime periods in API v2 (#4469)
RobertJoonasf04c47f8818d
diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.exindex 7f186e4779..22e04da220 100644--- a/lib/plausible/google/api.ex+++ b/lib/plausible/google/api.ex@@ -7,6 +7,7 @@ defmodule Plausible.Google.API doalias Plausible.Google.HTTPalias Plausible.Google.SearchConsole+ alias Plausible.Stats.DateTimeRangerequire Logger@@ -64,11 +65,12 @@ defmodule Plausible.Google.API do{:ok, access_token} <- maybe_refresh_token(site.google_auth),{:ok, gsc_filters} <-SearchConsole.Filters.transform(site.google_auth.property, query.filters, search),+ date_range = DateTimeRange.to_date_range(query.date_range),{:ok, stats} <-HTTP.list_stats(access_token,site.google_auth.property,- query.date_range,+ date_range,pagination,gsc_filters) dodiff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.exindex 37481f0e47..15b8964e4a 100644--- a/lib/plausible/stats/breakdown.ex+++ b/lib/plausible/stats/breakdown.ex@@ -117,7 +117,9 @@ defmodule Plausible.Stats.Breakdown dofrom i in "imported_pages",group_by: i.page,where: i.site_id == ^site.id,- where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last,+ where:+ i.date >= ^DateTime.to_naive(query.date_range.first) and+ i.date <= ^DateTime.to_naive(query.date_range.last),where: i.page in ^pages,select: %{page: i.page,diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.exindex 69437de694..219051e686 100644--- a/lib/plausible/stats/clickhouse.ex+++ b/lib/plausible/stats/clickhouse.ex@@ -6,7 +6,6 @@ defmodule Plausible.Stats.Clickhouse doimport Ecto.Query, only: [from: 2, dynamic: 1, dynamic: 2]- alias Plausible.Stats.Queryalias Plausible.Timezones@spec pageview_start_date_local(Plausible.Site.t()) :: Date.t() | nil@@ -78,7 +77,7 @@ defmodule Plausible.Stats.Clickhouse dodef top_sources_for_spike(site, query, limit, page) dooffset = (page - 1) * limit- {first_datetime, last_datetime} = utc_boundaries(query, site)+ {first_datetime, last_datetime} = Plausible.Stats.Time.utc_boundaries(query, site)referrers =from(s in "sessions_v2",@@ -251,52 +250,4 @@ defmodule Plausible.Stats.Clickhouse do}endend-- defp utc_boundaries(%Query{now: now, period: "30m"}, site) do- last_datetime = now |> NaiveDateTime.truncate(:second)-- first_datetime =- last_datetime- |> NaiveDateTime.shift(minute: -30)- |> beginning_of_time(site.native_stats_start_at)- |> NaiveDateTime.truncate(:second)-- {first_datetime, last_datetime}- end-- defp utc_boundaries(%Query{now: now, period: "realtime"}, site) do- last_datetime = now |> NaiveDateTime.truncate(:second)-- first_datetime =- last_datetime- |> NaiveDateTime.shift(minute: -5)- |> beginning_of_time(site.native_stats_start_at)- |> NaiveDateTime.truncate(:second)-- {first_datetime, last_datetime}- end-- defp utc_boundaries(%Query{date_range: date_range}, site) do- {:ok, first} = NaiveDateTime.new(date_range.first, ~T[00:00:00])-- first_datetime =- first- |> Timezones.to_utc_datetime(site.timezone)- |> beginning_of_time(site.native_stats_start_at)-- {:ok, last} = NaiveDateTime.new(date_range.last |> Date.shift(day: 1), ~T[00:00:00])-- last_datetime =- Timezones.to_utc_datetime(last, site.timezone)-- {first_datetime, last_datetime}- end-- defp beginning_of_time(candidate, site_creation_date) do- if Timex.after?(site_creation_date, candidate) do- site_creation_date- else- candidate- end- endenddiff --git a/lib/plausible/stats/comparisons.ex b/lib/plausible/stats/comparisons.exindex 157be97c1f..23f2572861 100644--- a/lib/plausible/stats/comparisons.ex+++ b/lib/plausible/stats/comparisons.ex@@ -8,7 +8,7 @@ defmodule Plausible.Stats.Comparisons do"""alias Plausible.Stats- alias Plausible.Stats.Query+ alias Plausible.Stats.{Query, DateTimeRange}@modes ~w(previous_period year_over_year custom)@disallowed_periods ~w(realtime all)@@ -21,6 +21,9 @@ defmodule Plausible.Stats.Comparisons do@doc """Generates a comparison query based on the source query and comparison mode.+ Currently only historical periods are supported for comparisons (not `realtime`+ and `30m` periods).+The mode parameter specifies the type of comparison and can be one of thefollowing:@@ -61,52 +64,55 @@ defmodule Plausible.Stats.Comparisons do|> Keyword.put_new(:now, Timex.now(site.timezone))|> Keyword.put_new(:match_day_of_week?, false)+ source_date_range = DateTimeRange.to_date_range(source_query.date_range)+with :ok <- validate_mode(source_query, mode),- {:ok, comparison_query} <- do_compare(source_query, mode, opts),- comparison_query <- maybe_include_imported(comparison_query, source_query),- do: {:ok, comparison_query}+ {:ok, comparison_date_range} <- get_comparison_date_range(source_date_range, mode, opts) do+ %Date.Range{first: first, last: last} = comparison_date_range++ comparison_query =+ source_query+ |> Query.set(date_range: DateTimeRange.new!(first, last, site.timezone))+ |> maybe_include_imported(source_query)++ {:ok, comparison_query}+ endend- defp do_compare(source_query, "year_over_year", opts) do+ defp get_comparison_date_range(source_date_range, "year_over_year", opts) donow = Keyword.fetch!(opts, :now)- start_date = Date.add(source_query.date_range.first, -365)- end_date = earliest(source_query.date_range.last, now) |> Date.add(-365)-- range = Date.range(start_date, end_date)+ start_date = Date.add(source_date_range.first, -365)+ end_date = earliest(source_date_range.last, now) |> Date.add(-365)- comparison_query =- source_query- |> Map.put(:date_range, range)- |> maybe_match_day_of_week(source_query, opts)+ comparison_date_range =+ Date.range(start_date, end_date)+ |> maybe_match_day_of_week(source_date_range, opts)- {:ok, comparison_query}+ {:ok, comparison_date_range}end- defp do_compare(source_query, "previous_period", opts) do+ defp get_comparison_date_range(source_date_range, "previous_period", opts) donow = Keyword.fetch!(opts, :now)- last = earliest(source_query.date_range.last, now)- diff_in_days = Date.diff(source_query.date_range.first, last) - 1+ last = earliest(source_date_range.last, now)+ diff_in_days = Date.diff(source_date_range.first, last) - 1- new_first = Date.add(source_query.date_range.first, diff_in_days)+ new_first = Date.add(source_date_range.first, diff_in_days)new_last = Date.add(last, diff_in_days)- range = Date.range(new_first, new_last)-- comparison_query =- source_query- |> Map.put(:date_range, range)- |> maybe_match_day_of_week(source_query, opts)+ comparison_date_range =+ Date.range(new_first, new_last)+ |> maybe_match_day_of_week(source_date_range, opts)- {:ok, comparison_query}+ {:ok, comparison_date_range}end- defp do_compare(source_query, "custom", opts) do+ defp get_comparison_date_range(_source_date_range, "custom", opts) dowith {:ok, from} <- opts |> Keyword.fetch!(:from) |> Date.from_iso8601(),{:ok, to} <- opts |> Keyword.fetch!(:to) |> Date.from_iso8601(),result when result in [:eq, :lt] <- Date.compare(from, to) do- {:ok, %Stats.Query{source_query | date_range: Date.range(from, to)}}+ {:ok, Date.range(from, to)}else_error -> {:error, :invalid_dates}end@@ -116,24 +122,23 @@ defmodule Plausible.Stats.Comparisons doif Date.compare(a, b) in [:eq, :lt], do: a, else: bend- defp maybe_match_day_of_week(comparison_query, source_query, opts) do+ defp maybe_match_day_of_week(comparison_date_range, source_date_range, opts) doif Keyword.fetch!(opts, :match_day_of_week?) do- day_to_match = Date.day_of_week(source_query.date_range.first)+ day_to_match = Date.day_of_week(source_date_range.first)new_first =shift_to_nearest(day_to_match,- comparison_query.date_range.first,- source_query.date_range.first+ comparison_date_range.first,+ source_date_range.first)- days_shifted = Date.diff(new_first, comparison_query.date_range.first)- new_last = Date.add(comparison_query.date_range.last, days_shifted)+ days_shifted = Date.diff(new_first, comparison_date_range.first)+ new_last = Date.add(comparison_date_range.last, days_shifted)- new_range = Date.range(new_first, new_last)- %Stats.Query{comparison_query | date_range: new_range}+ Date.range(new_first, new_last)else- comparison_query+ comparison_date_rangeendenddiff --git a/lib/plausible/stats/datetime_range.ex b/lib/plausible/stats/datetime_range.exnew file mode 100644index 0000000000..e8e787a71c--- /dev/null+++ b/lib/plausible/stats/datetime_range.ex@@ -0,0 +1,54 @@+defmodule Plausible.Stats.DateTimeRange do+ @moduledoc """+ Defines a struct similar `Date.Range`, but with `DateTime` instead of `Date`.++ The structs should be created with the `new!/2` function.+ """++ @enforce_keys [:first, :last]+ defstruct [:first, :last]++ @type t() :: %__MODULE__{+ first: %DateTime{},+ last: %DateTime{}+ }++ @doc """+ Creates a `DateTimeRange` struct from the given `%Date{}` structs.++ The first datetime will become the first date at 00:00:00, and the last datetime+ will become the last date at 23:59:59. Both dates will be turned into `%DateTime{}`+ structs in the given timezone.+ """+ def new!(%Date{} = first, %Date{} = last, timezone) do+ first =+ case DateTime.new(first, ~T[00:00:00], timezone) do+ {:ok, datetime} -> datetime+ {:gap, _just_before, just_after} -> just_after+ {:ambiguous, _first_datetime, second_datetime} -> second_datetime+ end++ last =+ case DateTime.new(last, ~T[23:59:59], timezone) do+ {:ok, datetime} -> datetime+ {:gap, just_before, _just_after} -> just_before+ {:ambiguous, first_datetime, _second_datetime} -> first_datetime+ end++ new!(first, last)+ end++ def new!(%DateTime{} = first, %DateTime{} = last) do+ first = DateTime.truncate(first, :second)+ last = DateTime.truncate(last, :second)++ %__MODULE__{first: first, last: last}+ end++ def to_date_range(%__MODULE__{first: first, last: last}) do+ first = DateTime.to_date(first)+ last = DateTime.to_date(last)++ Date.range(first, last)+ end+enddiff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.exindex f5f93e72f2..6ee02bd883 100644--- a/lib/plausible/stats/filters/query_parser.ex+++ b/lib/plausible/stats/filters/query_parser.ex@@ -1,11 +1,7 @@defmodule Plausible.Stats.Filters.QueryParser do@moduledoc false- alias Plausible.Stats.TableDecider- alias Plausible.Stats.Filters- alias Plausible.Stats.Query- alias Plausible.Stats.Metrics- alias Plausible.Stats.JSONSchema+ alias Plausible.Stats.{TableDecider, Filters, Query, Metrics, DateTimeRange, JSONSchema}@default_include %{imports: false,@@ -13,11 +9,18 @@ defmodule Plausible.Stats.Filters.QueryParser do}def parse(site, schema_type, params, now \\ nil) when is_map(params) do+ {now, date} =+ if now do+ {now, DateTime.shift_zone!(now, site.timezone) |> DateTime.to_date()}+ else+ {DateTime.utc_now(:second), today(site)}+ end+with :ok <- JSONSchema.validate(schema_type, params),- {:ok, date} <- parse_date(site, Map.get(params, "date"), now),+ {:ok, date} <- parse_date(site, Map.get(params, "date"), date),+ {:ok, date_range} <- parse_date_range(site, Map.get(params, "date_range"), date, now),{:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])),{:ok, filters} <- parse_filters(Map.get(params, "filters", [])),- {:ok, date_range} <- parse_date_range(site, Map.get(params, "date_range"), date),{:ok, dimensions} <- parse_dimensions(Map.get(params, "dimensions", [])),{:ok, order_by} <- parse_order_by(Map.get(params, "order_by")),{:ok, include} <- parse_include(Map.get(params, "include", %{})),@@ -110,84 +113,118 @@ defmodule Plausible.Stats.Filters.QueryParser dodefp parse_clauses_list(filter), do: {:error, "Invalid filter '#{i(filter)}'"}- defp parse_date(site, nil, nil), do: {:ok, today(site)}-- defp parse_date(_site, date_string, nil) when is_binary(date_string) do+ defp parse_date(_site, date_string, _date) when is_binary(date_string) docase Date.from_iso8601(date_string) do{:ok, date} -> {:ok, date}_ -> {:error, "Invalid date '#{date_string}'."}endend- defp parse_date(_site, _date_string, param_date), do: {:ok, param_date}+ defp parse_date(_site, _date_string, date) do+ {:ok, date}+ end++ defp parse_date_range(_site, date_range, _date, now) when date_range in ["realtime", "30m"] do+ duration_minutes =+ case date_range do+ "realtime" -> 5+ "30m" -> 30+ end++ first_datetime = DateTime.shift(now, minute: -duration_minutes)+ last_datetime = DateTime.shift(now, second: 5)- defp parse_date_range(_site, "day", date) do- {:ok, Date.range(date, date)}+ {:ok, DateTimeRange.new!(first_datetime, last_datetime)}end- defp parse_date_range(_site, "7d", last) do- first = last |> Date.add(-6)- {:ok, Date.range(first, last)}+ defp parse_date_range(site, "day", date, _now) do+ {:ok, DateTimeRange.new!(date, date, site.timezone)}end- defp parse_date_range(_site, "30d", last) do- first = last |> Date.add(-30)- {:ok, Date.range(first, last)}+ defp parse_date_range(site, "7d", date, _now) do+ first = date |> Date.add(-6)+ {:ok, DateTimeRange.new!(first, date, site.timezone)}end- defp parse_date_range(_site, "month", today) do- last = today |> Date.end_of_month()+ defp parse_date_range(site, "30d", date, _now) do+ first = date |> Date.add(-30)+ {:ok, DateTimeRange.new!(first, date, site.timezone)}+ end++ defp parse_date_range(site, "month", date, _now) do+ last = date |> Date.end_of_month()first = last |> Date.beginning_of_month()- {:ok, Date.range(first, last)}+ {:ok, DateTimeRange.new!(first, last, site.timezone)}end- defp parse_date_range(_site, "6mo", today) do- last = today |> Date.end_of_month()+ defp parse_date_range(site, "6mo", date, _now) do+ last = date |> Date.end_of_month()first =last|> Date.shift(month: -5)|> Date.beginning_of_month()- {:ok, Date.range(first, last)}+ {:ok, DateTimeRange.new!(first, last, site.timezone)}end- defp parse_date_range(_site, "12mo", today) do- last = today |> Date.end_of_month()+ defp parse_date_range(site, "12mo", date, _now) do+ last = date |> Date.end_of_month()first =last|> Date.shift(month: -11)|> Date.beginning_of_month()- {:ok, Date.range(first, last)}+ {:ok, DateTimeRange.new!(first, last, site.timezone)}end- defp parse_date_range(_site, "year", today) do- last = today |> Timex.end_of_year()+ defp parse_date_range(site, "year", date, _now) do+ last = date |> Timex.end_of_year()first = last |> Timex.beginning_of_year()- {:ok, Date.range(first, last)}+ {:ok, DateTimeRange.new!(first, last, site.timezone)}end- defp parse_date_range(site, "all", today) do- start_date = Plausible.Sites.stats_start_date(site) || today+ defp parse_date_range(site, "all", date, _now) do+ start_date = Plausible.Sites.stats_start_date(site) || date- {:ok, Date.range(start_date, today)}+ {:ok, DateTimeRange.new!(start_date, date, site.timezone)}end- defp parse_date_range(_site, [from_date_string, to_date_string], _date)- when is_binary(from_date_string) and is_binary(to_date_string) do- with {:ok, from_date} <- Date.from_iso8601(from_date_string),- {:ok, to_date} <- Date.from_iso8601(to_date_string) do- {:ok, Date.range(from_date, to_date)}- else- _ -> {:error, "Invalid date_range '#{i([from_date_string, to_date_string])}'."}+ defp parse_date_range(site, [from, to], _date, _now)+ when is_binary(from) and is_binary(to) do+ case date_range_from_date_strings(site, from, to) do+ {:ok, date_range} -> {:ok, date_range}+ {:error, _} -> date_range_from_timestamps(from, to)endend- defp parse_date_range(_site, unknown, _),+ defp parse_date_range(_site, unknown, _date, _now),do: {:error, "Invalid date_range '#{i(unknown)}'."}+ defp date_range_from_date_strings(site, from, to) do+ with {:ok, from_date} <- Date.from_iso8601(from),+ {:ok, to_date} <- Date.from_iso8601(to) do+ {:ok, DateTimeRange.new!(from_date, to_date, site.timezone)}+ end+ end++ defp date_range_from_timestamps(from, to) do+ with {:ok, from_datetime} <- datetime_from_timestamp(from),+ {:ok, to_datetime} <- datetime_from_timestamp(to) do+ {:ok, DateTimeRange.new!(from_datetime, to_datetime)}+ else+ _ -> {:error, "Invalid date_range '#{i([from, to])}'."}+ end+ end++ defp datetime_from_timestamp(timestamp_string) do+ with [timestamp, timezone] <- String.split(timestamp_string),+ {:ok, naive_datetime} <- NaiveDateTime.from_iso8601(timestamp) do+ DateTime.from_naive(naive_datetime, timezone)+ end+ end+defp today(site), do: DateTime.now!(site.timezone) |> DateTime.to_date()defp parse_dimensions(dimensions) when is_list(dimensions) dodiff --git a/lib/plausible/stats/goal_suggestions.ex b/lib/plausible/stats/goal_suggestions.exindex 4ff23bc8e6..d3192bfb81 100644--- a/lib/plausible/stats/goal_suggestions.ex+++ b/lib/plausible/stats/goal_suggestions.ex@@ -61,7 +61,9 @@ defmodule Plausible.Stats.GoalSuggestions dofrom(i in "imported_custom_events",where: i.site_id == ^site.id,where: i.import_id in ^site.complete_import_ids,- where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last,+ where:+ i.date >= ^DateTime.to_naive(query.date_range.first) and+ i.date <= ^DateTime.to_naive(query.date_range.last),where: i.visitors > 0,where: fragment("? ilike ?", i.name, ^matches),where: fragment("trim(?)", i.name) != "",diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.exindex 5bc46d07ee..23bfebda41 100644--- a/lib/plausible/stats/imported/base.ex+++ b/lib/plausible/stats/imported/base.ex@@ -6,7 +6,7 @@ defmodule Plausible.Stats.Imported.Base doimport Ecto.Queryalias Plausible.Imported- alias Plausible.Stats.{Filters, Query, SQL}+ alias Plausible.Stats.{Filters, Query, SQL, DateTimeRange}@property_to_table_mappings %{"visit:source" => "imported_sources",@@ -71,7 +71,7 @@ defmodule Plausible.Stats.Imported.Base dodef query_imported(table, site, query) doimport_ids = site.complete_import_ids- %{first: date_from, last: date_to} = query.date_range+ %{first: date_from, last: date_to} = DateTimeRange.to_date_range(query.date_range)from(i in table,where: i.site_id == ^site.id,diff --git a/lib/plausible/stats/imported/sql/builder.ex b/lib/plausible/stats/imported/sql/builder.exindex 96a846121c..76a8b011d0 100644--- a/lib/plausible/stats/imported/sql/builder.ex+++ b/lib/plausible/stats/imported/sql/builder.ex@@ -293,7 +293,7 @@ defmodule Plausible.Stats.Imported.SQL.Builder dodefp select_group_fields(q, "time:week", key, query) doselect_merge_as(q, [i], %{- key => weekstart_not_before(i.date, ^query.date_range.first)+ key => weekstart_not_before(i.date, ^DateTime.to_naive(query.date_range.first))})enddiff --git a/lib/plausible/stats/interval.ex b/lib/plausible/stats/interval.exindex eb501adf75..fc3324d4c5 100644--- a/lib/plausible/stats/interval.ex+++ b/lib/plausible/stats/interval.ex@@ -7,6 +7,8 @@ defmodule Plausible.Stats.Interval do`week`, and `month`."""+ alias Plausible.Stats.DateTimeRange+@type t() :: String.t()@type(opt() :: {:site, Plausible.Site.t()} | {:from, Date.t()}, {:to, Date.t()})@type opts :: list(opt())@@ -27,18 +29,20 @@ defmodule Plausible.Stats.Interval do"""def default_for_period(period) docase period do- "realtime" -> "minute"+ period when period in ["realtime", "30m"] -> "minute""day" -> "hour"period when period in ["custom", "7d", "30d", "month"] -> "day"period when period in ["6mo", "12mo", "year"] -> "month"endend- @spec default_for_date_range(Date.Range.t()) :: t()+ @spec default_for_date_range(DateTimeRange.t()) :: t()@doc """- Returns the suggested interval for the given `Date.Range` struct.+ Returns the suggested interval for the given `DateTimeRange` struct."""- def default_for_date_range(%Date.Range{first: first, last: last}) do+ def default_for_date_range(%DateTimeRange{} = date_range) do+ %Date.Range{first: first, last: last} = DateTimeRange.to_date_range(date_range)+cond doTimex.diff(last, first, :months) > 0 ->"month"diff --git a/lib/plausible/stats/json_schema.ex b/lib/plausible/stats/json_schema.exindex 08c22467a6..c1f9beedef 100644--- a/lib/plausible/stats/json_schema.ex+++ b/lib/plausible/stats/json_schema.ex@@ -27,6 +27,13 @@ defmodule Plausible.Stats.JSONSchema do|> JSONPointer.add!("#/definitions/metric/oneOf/0", %{"const" => "time_on_page"})+ |> JSONPointer.add!("#/definitions/date_range/oneOf/0", %{+ "const" => "30m"+ })+ |> JSONPointer.add!("#/definitions/date_range/oneOf/0", %{+ "const" => "realtime"+ })+ |> JSONPointer.add!("#/properties/date", %{"type" => "string"})|> ExJsonSchema.Schema.resolve()def validate(schema_type, params) dodiff --git a/lib/plausible/stats/legacy/legacy_query_builder.ex b/lib/plausible/stats/legacy/legacy_query_builder.exindex 3abd8194c5..cd87b86703 100644--- a/lib/plausible/stats/legacy/legacy_query_builder.ex+++ b/lib/plausible/stats/legacy/legacy_query_builder.ex@@ -3,10 +3,10 @@ defmodule Plausible.Stats.Legacy.QueryBuilder douse Plausible- alias Plausible.Stats.{Filters, Interval, Query}+ alias Plausible.Stats.{Filters, Interval, Query, DateTimeRange}def from(site, params, debug_metadata) do- now = NaiveDateTime.utc_now(:second)+ now = DateTime.utc_now(:second)query =Query@@ -37,46 +37,50 @@ defmodule Plausible.Stats.Legacy.QueryBuilder dostruct!(query, preloaded_goals: goals)end- defp put_period(query, site, %{"period" => "realtime"}) do- date = today(site.timezone)+ defp put_period(%Query{now: now} = query, _site, %{"period" => period})+ when period in ["realtime", "30m"] do+ duration_minutes =+ case period do+ "realtime" -> 5+ "30m" -> 30+ end- struct!(query, period: "realtime", date_range: Date.range(date, date))+ first_datetime = DateTime.shift(now, minute: -duration_minutes)+ last_datetime = DateTime.shift(now, second: 5)++ struct!(query, period: period, date_range: DateTimeRange.new!(first_datetime, last_datetime))enddefp put_period(query, site, %{"period" => "day"} = params) dodate = parse_single_date(site.timezone, params)+ datetime_range = DateTimeRange.new!(date, date, site.timezone)- struct!(query, period: "day", date_range: Date.range(date, date))+ struct!(query, period: "day", date_range: datetime_range)enddefp put_period(query, site, %{"period" => "7d"} = params) doend_date = parse_single_date(site.timezone, params)start_date = end_date |> Date.shift(day: -6)+ datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone)- struct!(- query,- period: "7d",- date_range: Date.range(start_date, end_date)- )+ struct!(query, period: "7d", date_range: datetime_range)enddefp put_period(query, site, %{"period" => "30d"} = params) doend_date = parse_single_date(site.timezone, params)start_date = end_date |> Date.shift(day: -30)+ datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone)- struct!(query, period: "30d", date_range: Date.range(start_date, end_date))+ struct!(query, period: "30d", date_range: datetime_range)enddefp put_period(query, site, %{"period" => "month"} = params) dodate = parse_single_date(site.timezone, params)-start_date = Timex.beginning_of_month(date)end_date = Timex.end_of_month(date)+ datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone)- struct!(query,- period: "month",- date_range: Date.range(start_date, end_date)- )+ struct!(query, period: "month", date_range: datetime_range)enddefp put_period(query, site, %{"period" => "6mo"} = params) do@@ -88,10 +92,9 @@ defmodule Plausible.Stats.Legacy.QueryBuilder doDate.shift(end_date, month: -5)|> Timex.beginning_of_month()- struct!(query,- period: "6mo",- date_range: Date.range(start_date, end_date)- )+ datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone)++ struct!(query, period: "6mo", date_range: datetime_range)enddefp put_period(query, site, %{"period" => "12mo"} = params) do@@ -103,10 +106,9 @@ defmodule Plausible.Stats.Legacy.QueryBuilder doDate.shift(end_date, month: -11)|> Timex.beginning_of_month()- struct!(query,- period: "12mo",- date_range: Date.range(start_date, end_date)- )+ datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone)++ struct!(query, period: "12mo", date_range: datetime_range)enddefp put_period(query, site, %{"period" => "year"} = params) do@@ -115,21 +117,17 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do|> Timex.end_of_year()start_date = Timex.beginning_of_year(end_date)+ datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone)- struct!(query,- period: "year",- date_range: Date.range(start_date, end_date)- )+ struct!(query, period: "year", date_range: datetime_range)enddefp put_period(query, site, %{"period" => "all"}) do- now = today(site.timezone)- start_date = Plausible.Sites.stats_start_date(site) || now+ today = today(site.timezone)+ start_date = Plausible.Sites.stats_start_date(site) || today+ datetime_range = DateTimeRange.new!(start_date, today, site.timezone)- struct!(query,- period: "all",- date_range: Date.range(start_date, now)- )+ struct!(query, period: "all", date_range: datetime_range)enddefp put_period(query, site, %{"period" => "custom", "from" => from, "to" => to} = params) do@@ -141,15 +139,13 @@ defmodule Plausible.Stats.Legacy.QueryBuilder doput_period(query, site, new_params)end- defp put_period(query, _site, %{"period" => "custom", "date" => date}) do+ defp put_period(query, site, %{"period" => "custom", "date" => date}) do[from, to] = String.split(date, ",")from_date = Date.from_iso8601!(String.trim(from))to_date = Date.from_iso8601!(String.trim(to))+ datetime_range = DateTimeRange.new!(from_date, to_date, site.timezone)- struct!(query,- period: "custom",- date_range: Date.range(from_date, to_date)- )+ struct!(query, period: "custom", date_range: datetime_range)enddefp put_period(query, site, params) dodiff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.exindex cbbf6ae31f..71f5f200e1 100644--- a/lib/plausible/stats/query.ex+++ b/lib/plausible/stats/query.ex@@ -35,7 +35,7 @@ defmodule Plausible.Stats.Query dostruct!(__MODULE__, Map.to_list(query_data))|> put_imported_opts(site, %{})|> put_experimental_reduced_joins(site, params)- |> struct!(v2: true, now: NaiveDateTime.utc_now(:second), debug_metadata: debug_metadata)+ |> struct!(v2: true, now: DateTime.utc_now(:second), debug_metadata: debug_metadata){:ok, query}end@@ -142,9 +142,9 @@ defmodule Plausible.Stats.Query dodef ensure_include_imported(query, requested?) docond dois_nil(query.latest_import_end_date) -> {:error, :no_imported_data}+ query.period in ["realtime", "30m"] -> {:error, :unsupported_query}Date.after?(query.date_range.first, query.latest_import_end_date) -> {:error, :out_of_range}not Imported.schema_supports_query?(query) -> {:error, :unsupported_query}- query.period == "realtime" -> {:error, :unsupported_query}not requested? -> {:error, :not_requested}true -> :okenddiff --git a/lib/plausible/stats/query_optimizer.ex b/lib/plausible/stats/query_optimizer.exindex 5f3bf706f8..f5e9b272b2 100644--- a/lib/plausible/stats/query_optimizer.ex+++ b/lib/plausible/stats/query_optimizer.ex@@ -4,7 +4,7 @@ defmodule Plausible.Stats.QueryOptimizer do"""use Plausible- alias Plausible.Stats.{Query, TableDecider, Util}+ alias Plausible.Stats.{Query, TableDecider, Util, DateTimeRange}@doc """This module manipulates an existing query, updating it according to business logic.@@ -61,7 +61,7 @@ defmodule Plausible.Stats.QueryOptimizer dodefp update_group_by_time(%Query{- date_range: %Date.Range{first: first, last: last}+ date_range: %DateTimeRange{first: first, last: last}} = query) dodimensions =diff --git a/lib/plausible/stats/query_result.ex b/lib/plausible/stats/query_result.exindex 3a9ce3c562..22ecc654a7 100644--- a/lib/plausible/stats/query_result.ex+++ b/lib/plausible/stats/query_result.ex@@ -7,8 +7,7 @@ defmodule Plausible.Stats.QueryResult doproduced by Jason.encode(query_result) is ordered."""- alias Plausible.Stats.Util- alias Plausible.Stats.Filters+ alias Plausible.Stats.{Util, Filters}defstruct results: [],meta: %{},@@ -32,7 +31,10 @@ defmodule Plausible.Stats.QueryResult doJason.OrderedObject.new(site_id: site.domain,metrics: query.metrics,- date_range: [query.date_range.first, query.date_range.last],+ date_range: [+ to_iso_8601_with_timezone(query.date_range.first),+ to_iso_8601_with_timezone(query.date_range.last)+ ],filters: query.filters,dimensions: query.dimensions,order_by: query.order_by |> Enum.map(&Tuple.to_list/1),@@ -79,6 +81,15 @@ defmodule Plausible.Stats.QueryResult do|> Enum.reject(fn {_, value} -> is_nil(value) end)|> Enum.into(%{})end++ defp to_iso_8601_with_timezone(%DateTime{time_zone: timezone} = datetime) do+ naive_iso8601 =+ datetime+ |> DateTime.to_naive()+ |> NaiveDateTime.to_iso8601()++ naive_iso8601 <> " " <> timezone+ endenddefimpl Jason.Encoder, for: Plausible.Stats.QueryResult dodiff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.exindex 4638ea7fc4..cdec499d15 100644--- a/lib/plausible/stats/sql/expression.ex+++ b/lib/plausible/stats/sql/expression.ex@@ -50,7 +50,7 @@ defmodule Plausible.Stats.SQL.Expression dokey =>weekstart_not_before(to_timezone(t.timestamp, ^query.timezone),- ^query.date_range.first+ ^DateTime.to_naive(query.date_range.first))})end@@ -83,27 +83,6 @@ defmodule Plausible.Stats.SQL.Expression do})end- # :NOTE: This is not exposed in Query APIv2- def select_dimension(q, key, "time:minute", :sessions, %Query{- period: "30m"- }) do- select_merge_as(q, [s], %{- key =>- fragment(- "arrayJoin(range(dateDiff('minute', now(), ?), dateDiff('minute', now(), ?) + 1))",- s.start,- s.timestamp- )- })- end-- # :NOTE: This is not exposed in Query APIv2- def select_dimension(q, key, "time:minute", _table, %Query{period: "30m"}) do- select_merge_as(q, [t], %{- key => fragment("dateDiff('minute', now(), ?)", t.timestamp)- })- end-# :NOTE: This is not exposed in Query APIv2def select_dimension(q, key, "time:minute", :sessions, query) doqdiff --git a/lib/plausible/stats/sql/where_builder.ex b/lib/plausible/stats/sql/where_builder.exindex 6cda55cf12..51a7b70904 100644--- a/lib/plausible/stats/sql/where_builder.ex+++ b/lib/plausible/stats/sql/where_builder.ex@@ -48,7 +48,7 @@ defmodule Plausible.Stats.SQL.WhereBuilder dodynamic([e],- e.site_id == ^site.id and e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime+ e.site_id == ^site.id and e.timestamp >= ^first_datetime and e.timestamp <= ^last_datetime)end@@ -71,7 +71,7 @@ defmodule Plausible.Stats.SQL.WhereBuilder dos.site_id == ^site.id ands.start >= ^NaiveDateTime.add(first_datetime, -7, :day) ands.timestamp >= ^first_datetime and- s.start < ^last_datetime+ s.start <= ^last_datetime)enddiff --git a/lib/plausible/stats/time.ex b/lib/plausible/stats/time.exindex f87c40e4ed..4b9b4b6a53 100644--- a/lib/plausible/stats/time.ex+++ b/lib/plausible/stats/time.ex@@ -3,52 +3,20 @@ defmodule Plausible.Stats.Time doCollection of functions to work with time in queries."""- alias Plausible.Stats.Query- alias Plausible.Timezones-- def utc_boundaries(%Query{period: "realtime", now: now}, site) do- last_datetime =- now- |> NaiveDateTime.shift(second: 5)- |> NaiveDateTime.truncate(:second)-- first_datetime =- now- |> NaiveDateTime.shift(minute: -5)- |> NaiveDateTime.truncate(:second)- |> beginning_of_time(site.native_stats_start_at)-- {first_datetime, last_datetime}- end-- def utc_boundaries(%Query{period: "30m", now: now}, site) do- last_datetime =- now- |> NaiveDateTime.shift(second: 5)- |> NaiveDateTime.truncate(:second)-- first_datetime =- now- |> NaiveDateTime.shift(minute: -30)- |> NaiveDateTime.truncate(:second)- |> beginning_of_time(site.native_stats_start_at)-- {first_datetime, last_datetime}- end+ alias Plausible.Stats.{Query, DateTimeRange}def utc_boundaries(%Query{date_range: date_range}, site) do- {:ok, first} = NaiveDateTime.new(date_range.first, ~T[00:00:00])+ %DateTimeRange{first: first, last: last} = date_range- first_datetime =+ first =first- |> Timezones.to_utc_datetime(site.timezone)+ |> DateTime.shift_zone!("Etc/UTC")+ |> DateTime.to_naive()|> beginning_of_time(site.native_stats_start_at)- {:ok, last} = NaiveDateTime.new(date_range.last |> Date.shift(day: 1), ~T[00:00:00])-- last_datetime = Timezones.to_utc_datetime(last, site.timezone)+ last = DateTime.shift_zone!(last, "Etc/UTC") |> DateTime.to_naive()- {first_datetime, last_datetime}+ {first, last}enddefp beginning_of_time(candidate, native_stats_start_at) do@@ -61,7 +29,7 @@ defmodule Plausible.Stats.Time dodef format_datetime(%Date{} = date), do: Date.to_string(date)- def format_datetime(%DateTime{} = datetime),+ def format_datetime(%mod{} = datetime) when mod in [NaiveDateTime, DateTime],do: Calendar.strftime(datetime, "%Y-%m-%d %H:%M:%S")# Realtime graphs return numbers@@ -79,15 +47,17 @@ defmodule Plausible.Stats.Time doenddefp time_labels_for_dimension("time:month", query) do+ date_range = DateTimeRange.to_date_range(query.date_range)+n_buckets =Timex.diff(- query.date_range.last,- Date.beginning_of_month(query.date_range.first),+ date_range.last,+ Date.beginning_of_month(date_range.first),:months)Enum.map(n_buckets..0, fn shift ->- query.date_range.last+ date_range.last|> Date.beginning_of_month()|> Date.shift(month: -shift)|> format_datetime()@@ -95,15 +65,17 @@ defmodule Plausible.Stats.Time doenddefp time_labels_for_dimension("time:week", query) do+ date_range = DateTimeRange.to_date_range(query.date_range)+n_buckets =Timex.diff(- query.date_range.last,- Date.beginning_of_week(query.date_range.first),+ date_range.last,+ Date.beginning_of_week(date_range.first),:weeks)Enum.map(0..n_buckets, fn shift ->- query.date_range.first+ date_range.first|> Date.shift(week: shift)|> date_or_weekstart(query)|> format_datetime()@@ -112,59 +84,42 @@ defmodule Plausible.Stats.Time dodefp time_labels_for_dimension("time:day", query) doquery.date_range+ |> DateTimeRange.to_date_range()|> Enum.into([])|> Enum.map(&format_datetime/1)end- @full_day_in_hours 23defp time_labels_for_dimension("time:hour", query) do- n_buckets =- if query.date_range.first == query.date_range.last do- @full_day_in_hours- else- end_time =- query.date_range.last- |> Timex.to_datetime()- |> Timex.end_of_day()-- Timex.diff(end_time, query.date_range.first, :hours)- end+ n_buckets = DateTime.diff(query.date_range.last, query.date_range.first, :hour)Enum.map(0..n_buckets, fn step ->query.date_range.first- |> Timex.to_datetime()- |> DateTime.shift(hour: step)- |> DateTime.truncate(:second)+ |> DateTime.to_naive()+ |> NaiveDateTime.shift(hour: step)|> format_datetime()end)end- # Only supported in dashboards not via API- defp time_labels_for_dimension("time:minute", %Query{period: "30m"}) do- Enum.into(-30..-1, [])- end-- @full_day_in_minutes 24 * 60 - 1defp time_labels_for_dimension("time:minute", query) do- n_buckets =- if query.date_range.first == query.date_range.last do- @full_day_in_minutes- else- Timex.diff(query.date_range.last, query.date_range.first, :minutes)- end+ first_datetime = Map.put(query.date_range.first, :second, 0)- Enum.map(0..n_buckets, fn step ->- query.date_range.first- |> Timex.to_datetime()- |> DateTime.shift(minute: step)- |> format_datetime()+ first_datetime+ |> Stream.iterate(fn datetime -> DateTime.shift(datetime, minute: 1) end)+ |> Enum.take_while(fn datetime ->+ current_minute = Map.put(query.now, :second, 0)++ DateTime.before?(datetime, query.date_range.last) &&+ DateTime.before?(datetime, current_minute)end)+ |> Enum.map(&format_datetime/1)end- defp date_or_weekstart(date, query) do- weekstart = Timex.beginning_of_week(date)+ def date_or_weekstart(date, query) do+ weekstart = Date.beginning_of_week(date)++ date_range = DateTimeRange.to_date_range(query.date_range)- if Enum.member?(query.date_range, weekstart) do+ if Enum.member?(date_range, weekstart) doweekstartelsedatediff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.exindex 8b3ddbcc38..8c6bf85125 100644--- a/lib/plausible/stats/timeseries.ex+++ b/lib/plausible/stats/timeseries.ex@@ -69,6 +69,7 @@ defmodule Plausible.Stats.Timeseries do)|> cast_revenue_metrics_to_money(currency)end)+ |> transform_realtime_labels(query)enddefp empty_row(date, metrics) do@@ -102,6 +103,13 @@ defmodule Plausible.Stats.Timeseries doend)end+ defp transform_realtime_labels(results, %Query{period: "30m"}) do+ Enum.with_index(results)+ |> Enum.map(fn {entry, index} -> %{entry | date: -30 + index} end)+ end++ defp transform_realtime_labels(results, _query), do: results+on_ee dodefp cast_revenue_metrics_to_money(results, revenue_goals) doPlausible.Stats.Goal.Revenue.cast_revenue_metrics_to_money(results, revenue_goals)diff --git a/lib/plausible/timezones.ex b/lib/plausible/timezones.exindex 9aa030e3dc..bcd161ad49 100644--- a/lib/plausible/timezones.ex+++ b/lib/plausible/timezones.ex@@ -6,20 +6,6 @@ defmodule Plausible.Timezones do|> Enum.sort_by(& &1[:offset], :desc)end- @spec to_utc_datetime(NaiveDateTime.t(), String.t()) :: DateTime.t()- def to_utc_datetime(naive_date_time, timezone) do- case Timex.to_datetime(naive_date_time, timezone) do- %DateTime{} = tz_dt ->- Timex.Timezone.convert(tz_dt, "UTC")-- %Timex.AmbiguousDateTime{after: after_dt} ->- Timex.Timezone.convert(after_dt, "UTC")-- {:error, {:could_not_resolve_timezone, _, _, _}} ->- Timex.Timezone.convert(naive_date_time, "UTC")- end- end-@spec to_date_in_timezone(Date.t() | NaiveDateTime.t() | DateTime.t(), String.t()) :: Date.t()def to_date_in_timezone(dt, timezone) doto_datetime_in_timezone(dt, timezone) |> Timex.to_date()diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex 4db14ec22c..687e288255 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -5,7 +5,7 @@ defmodule PlausibleWeb.Api.StatsController douse PlausibleWeb.Plugs.ErrorHandleralias Plausible.Stats- alias Plausible.Stats.{Query, Comparisons}+ alias Plausible.Stats.{Query, Comparisons, Time, DateTimeRange}alias Plausible.Stats.Filters.LegacyDashboardFilterParseralias PlausibleWeb.Api.Helpers, as: H@@ -104,16 +104,10 @@ defmodule PlausibleWeb.Api.StatsController dowith {:ok, dates} <- parse_date_params(params),:ok <- validate_interval(params),:ok <- validate_interval_granularity(site, params, dates),+ params <- realtime_period_to_30m(params),query = Query.from(site, params, debug_metadata(conn)),{:ok, metric} <- parse_and_validate_graph_metric(params, query) do- timeseries_query =- if query.period == "realtime" do- %Query{query | period: "30m"}- else- query- end-- timeseries_result = Stats.timeseries(site, timeseries_query, [metric])+ timeseries_result = Stats.timeseries(site, query, [metric])comparison_opts = parse_comparison_opts(params)@@ -173,10 +167,12 @@ defmodule PlausibleWeb.Api.StatsController doenddefp build_full_intervals(%{interval: "week", date_range: date_range}, labels) do+ date_range = DateTimeRange.to_date_range(date_range)build_intervals(labels, date_range, &Timex.beginning_of_week/1, &Timex.end_of_week/1)enddefp build_full_intervals(%{interval: "month", date_range: date_range}, labels) do+ date_range = DateTimeRange.to_date_range(date_range)build_intervals(labels, date_range, &Timex.beginning_of_month/1, &Timex.end_of_month/1)end@@ -205,6 +201,8 @@ defmodule PlausibleWeb.Api.StatsController dodef top_stats(conn, params) dosite = conn.assigns[:site]+ params = realtime_period_to_30m(params)+query = Query.from(site, params, debug_metadata(conn))comparison_opts = parse_comparison_opts(params)@@ -224,14 +222,14 @@ defmodule PlausibleWeb.Api.StatsController dowith_imported_switch: with_imported_switch_info(query, comparison_query),includes_imported: includes_imported?(query, comparison_query),imports_exist: site.complete_import_ids != [],- comparing_from: comparison_query && comparison_query.date_range.first,- comparing_to: comparison_query && comparison_query.date_range.last,- from: query.date_range.first,- to: query.date_range.last+ comparing_from: comparison_query && DateTime.to_date(comparison_query.date_range.first),+ comparing_to: comparison_query && DateTime.to_date(comparison_query.date_range.last),+ from: DateTime.to_date(query.date_range.first),+ to: DateTime.to_date(query.date_range.last)})end- defp with_imported_switch_info(%Query{period: "realtime"}, _) do+ defp with_imported_switch_info(%Query{period: "30m"}, _) do%{visible: false, togglable: false, tooltip_msg: nil}end@@ -284,7 +282,7 @@ defmodule PlausibleWeb.Api.StatsController docurrent_date =Timex.now(site.timezone)|> Timex.to_date()- |> date_or_weekstart(query)+ |> Time.date_or_weekstart(query)|> Date.to_string()Enum.find_index(dates, &(&1 == current_date))@@ -307,24 +305,14 @@ defmodule PlausibleWeb.Api.StatsController doendend- defp date_or_weekstart(date, query) do- weekstart = Timex.beginning_of_week(date)-- if Enum.member?(query.date_range, weekstart) do- weekstart- else- date- end- end-defp fetch_top_stats(site, query, comparison_query) dogoal_filter = Query.get_filter(query, "event:goal")cond do- query.period == "realtime" && goal_filter ->+ query.period == "30m" && goal_filter ->fetch_goal_realtime_top_stats(site, query, comparison_query)- query.period == "realtime" ->+ query.period == "30m" ->fetch_realtime_top_stats(site, query, comparison_query)goal_filter ->@@ -336,12 +324,10 @@ defmodule PlausibleWeb.Api.StatsController doenddefp fetch_goal_realtime_top_stats(site, query, _comparison_query) do- query_30m = %Query{query | period: "30m"}-%{visitors: %{value: unique_conversions},events: %{value: total_conversions}- } = Stats.aggregate(site, query_30m, [:visitors, :events])+ } = Stats.aggregate(site, query, [:visitors, :events])stats = [%{@@ -364,12 +350,10 @@ defmodule PlausibleWeb.Api.StatsController doenddefp fetch_realtime_top_stats(site, query, _comparison_query) do- query_30m = %Query{query | period: "30m"}-%{visitors: %{value: visitors},pageviews: %{value: pageviews}- } = Stats.aggregate(site, query_30m, [:visitors, :pageviews])+ } = Stats.aggregate(site, query, [:visitors, :pageviews])stats = [%{@@ -1270,15 +1254,13 @@ defmodule PlausibleWeb.Api.StatsController dodef conversions(conn, params) dopagination = parse_pagination(params)site = Plausible.Repo.preload(conn.assigns.site, :goals)- params = Map.put(params, "property", "event:goal")- query = Query.from(site, params, debug_metadata(conn))- query =- if query.period == "realtime" do- %Query{query | period: "30m"}- else- query- end+ params =+ params+ |> realtime_period_to_30m()+ |> Map.put("property", "event:goal")++ query = Query.from(site, params, debug_metadata(conn))metrics = [:visitors, :events, :conversion_rate] ++ @revenue_metrics@@ -1583,4 +1565,12 @@ defmodule PlausibleWeb.Api.StatsController doMap.put(row, :name, name)end++ defp realtime_period_to_30m(%{"period" => _} = params) do+ Map.update!(params, "period", fn period ->+ if period == "realtime", do: "30m", else: period+ end)+ end++ defp realtime_period_to_30m(params), do: paramsenddiff --git a/priv/json-schemas/query-api-schema.json b/priv/json-schemas/query-api-schema.jsonindex 007575ae85..8e62b288e3 100644--- a/priv/json-schemas/query-api-schema.json+++ b/priv/json-schemas/query-api-schema.json@@ -99,13 +99,17 @@"type": "array","items": {"type": "string",- "pattern": "^\\d{4}-\\d{2}-\\d{2}$"+ "pattern": "^\\d{4}-\\d{2}-\\d{2}(?:T\\d{2}:\\d{2}:\\d{2}\\s[A-Za-z/_]+)?$"},- "markdownDescription": "Start and end dates of custom date range. Both dates are in format `YYYY-MM-DD`",+ "markdownDescription": "A list of two elements to determine the query date range. Both elements should have the same format - either `YYYY-MM-DD` or `YYYY-MM-DDThh:mm:ss <timezone>`","examples": [["2024-01-01","2024-01-31"+ ],+ [+ "2024-01-01T00:00:00 Europe/Tallinn",+ "2024-01-01T12:00:00 Europe/Tallinn"]],"minItems": 2,diff --git a/test/plausible/google/api_test.exs b/test/plausible/google/api_test.exsindex cd020ca5ec..ef6a96d93a 100644--- a/test/plausible/google/api_test.exs+++ b/test/plausible/google/api_test.exs@@ -3,6 +3,7 @@ defmodule Plausible.Google.APITest douse Plausible.Test.Support.HTTPMockeralias Plausible.Google+ alias Plausible.Stats.Queryimport ExUnit.CaptureLogimport Mox@@ -41,7 +42,8 @@ defmodule Plausible.Google.APITest doend)- query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}+ query =+ Query.from(site, %{"period" => "custom", "from" => "2022-01-01", "to" => "2022-01-05"})assert {:error, "google_auth_error"} = Google.API.fetch_stats(site, query, {5, 0}, "")end@@ -58,7 +60,8 @@ defmodule Plausible.Google.APITest doend)- query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}+ query =+ Query.from(site, %{"period" => "custom", "from" => "2022-01-01", "to" => "2022-01-05"})assert {:error, "some_error"} = Google.API.fetch_stats(site, query, {5, 0}, "")end@@ -75,7 +78,8 @@ defmodule Plausible.Google.APITest doend)- query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}+ query =+ Query.from(site, %{"period" => "custom", "from" => "2022-01-01", "to" => "2022-01-05"})log =capture_log(fn ->@@ -100,7 +104,8 @@ defmodule Plausible.Google.APITest doexpires: NaiveDateTime.add(NaiveDateTime.utc_now(), -3600))- query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}+ query =+ Query.from(site, %{"period" => "custom", "from" => "2022-01-01", "to" => "2022-01-05"})assert {:error, "invalid_grant"} = Google.API.fetch_stats(site, query, 5, "")end@@ -126,7 +131,8 @@ defmodule Plausible.Google.APITest dotest "returns name and visitor count", %{site: site} domock_http_with("google_search_console.json")- query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}+ query =+ Query.from(site, %{"period" => "custom", "from" => "2022-01-01", "to" => "2022-01-05"})assert {:ok,[diff --git a/test/plausible/stats/comparisons_test.exs b/test/plausible/stats/comparisons_test.exsindex 9f72a35f96..7d2504c6a7 100644--- a/test/plausible/stats/comparisons_test.exs+++ b/test/plausible/stats/comparisons_test.exs@@ -11,8 +11,11 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} = Comparisons.compare(site, query, "previous_period", now: now)- assert comparison.date_range.first == ~D[2023-02-27]- assert comparison.date_range.last == ~D[2023-02-28]+ assert comparison.date_range.first ==+ DateTime.new!(~D[2023-02-27], ~T[00:00:00], site.timezone)++ assert comparison.date_range.last ==+ DateTime.new!(~D[2023-02-28], ~T[23:59:59], site.timezone)endtest "shifts back this month period when it's the first day of the month and mode is previous_period" do@@ -22,8 +25,11 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} = Comparisons.compare(site, query, "previous_period", now: now)- assert comparison.date_range.first == ~D[2023-02-28]- assert comparison.date_range.last == ~D[2023-02-28]+ assert comparison.date_range.first ==+ DateTime.new!(~D[2023-02-28], ~T[00:00:00], site.timezone)++ assert comparison.date_range.last ==+ DateTime.new!(~D[2023-02-28], ~T[23:59:59], site.timezone)endtest "matches the day of the week when nearest day is original query start date and mode is previous_period" do@@ -34,8 +40,11 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} =Comparisons.compare(site, query, "previous_period", now: now, match_day_of_week?: true)- assert comparison.date_range.first == ~D[2023-02-22]- assert comparison.date_range.last == ~D[2023-02-23]+ assert comparison.date_range.first ==+ DateTime.new!(~D[2023-02-22], ~T[00:00:00], site.timezone)++ assert comparison.date_range.last ==+ DateTime.new!(~D[2023-02-23], ~T[23:59:59], site.timezone)endend@@ -47,8 +56,11 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} = Comparisons.compare(site, query, "previous_period", now: now)- assert comparison.date_range.first == ~D[2023-01-04]- assert comparison.date_range.last == ~D[2023-01-31]+ assert comparison.date_range.first ==+ DateTime.new!(~D[2023-01-04], ~T[00:00:00], site.timezone)++ assert comparison.date_range.last ==+ DateTime.new!(~D[2023-01-31], ~T[23:59:59], site.timezone)endtest "shifts back the full month when mode is year_over_year" do@@ -58,8 +70,11 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} = Comparisons.compare(site, query, "year_over_year", now: now)- assert comparison.date_range.first == ~D[2022-02-01]- assert comparison.date_range.last == ~D[2022-02-28]+ assert comparison.date_range.first ==+ DateTime.new!(~D[2022-02-01], ~T[00:00:00], site.timezone)++ assert comparison.date_range.last ==+ DateTime.new!(~D[2022-02-28], ~T[23:59:59], site.timezone)endtest "shifts back whole month plus one day when mode is year_over_year and a leap year" do@@ -69,8 +84,11 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} = Comparisons.compare(site, query, "year_over_year", now: now)- assert comparison.date_range.first == ~D[2019-02-01]- assert comparison.date_range.last == ~D[2019-03-01]+ assert comparison.date_range.first ==+ DateTime.new!(~D[2019-02-01], ~T[00:00:00], site.timezone)++ assert comparison.date_range.last ==+ DateTime.new!(~D[2019-03-01], ~T[23:59:59], site.timezone)endtest "matches the day of the week when mode is previous_period keeping the same day" do@@ -81,8 +99,11 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} =Comparisons.compare(site, query, "previous_period", now: now, match_day_of_week?: true)- assert comparison.date_range.first == ~D[2023-01-04]- assert comparison.date_range.last == ~D[2023-01-31]+ assert comparison.date_range.first ==+ DateTime.new!(~D[2023-01-04], ~T[00:00:00], site.timezone)++ assert comparison.date_range.last ==+ DateTime.new!(~D[2023-01-31], ~T[23:59:59], site.timezone)endtest "matches the day of the week when mode is previous_period" do@@ -93,8 +114,11 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} =Comparisons.compare(site, query, "previous_period", now: now, match_day_of_week?: true)- assert comparison.date_range.first == ~D[2022-12-04]- assert comparison.date_range.last == ~D[2023-01-03]+ assert comparison.date_range.first ==+ DateTime.new!(~D[2022-12-04], ~T[00:00:00], site.timezone)++ assert comparison.date_range.last ==+ DateTime.new!(~D[2023-01-03], ~T[23:59:59], site.timezone)endend@@ -106,8 +130,11 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} = Comparisons.compare(site, query, "previous_period", now: now)- assert comparison.date_range.first == ~D[2022-11-02]- assert comparison.date_range.last == ~D[2022-12-31]+ assert comparison.date_range.first ==+ DateTime.new!(~D[2022-11-02], ~T[00:00:00], site.timezone)++ assert comparison.date_range.last ==+ DateTime.new!(~D[2022-12-31], ~T[23:59:59], site.timezone)endtest "shifts back by the same number of days when mode is year_over_year" do@@ -117,8 +144,11 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} = Comparisons.compare(site, query, "year_over_year", now: now)- assert comparison.date_range.first == ~D[2022-01-01]- assert comparison.date_range.last == ~D[2022-03-01]+ assert comparison.date_range.first ==+ DateTime.new!(~D[2022-01-01], ~T[00:00:00], site.timezone)++ assert comparison.date_range.last ==+ DateTime.new!(~D[2022-03-01], ~T[23:59:59], site.timezone)endtest "matches the day of the week when mode is year_over_year" do@@ -129,8 +159,11 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} =Comparisons.compare(site, query, "year_over_year", now: now, match_day_of_week?: true)- assert comparison.date_range.first == ~D[2022-01-02]- assert comparison.date_range.last == ~D[2022-03-02]+ assert comparison.date_range.first ==+ DateTime.new!(~D[2022-01-02], ~T[00:00:00], site.timezone)++ assert comparison.date_range.last ==+ DateTime.new!(~D[2022-03-02], ~T[23:59:59], site.timezone)endend@@ -141,8 +174,11 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} = Comparisons.compare(site, query, "year_over_year")- assert comparison.date_range.first == ~D[2021-01-01]- assert comparison.date_range.last == ~D[2021-12-31]+ assert comparison.date_range.first ==+ DateTime.new!(~D[2021-01-01], ~T[00:00:00], site.timezone)++ assert comparison.date_range.last ==+ DateTime.new!(~D[2021-12-31], ~T[23:59:59], site.timezone)endtest "shifts back a whole year when mode is previous_period" do@@ -151,8 +187,11 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} = Comparisons.compare(site, query, "previous_period")- assert comparison.date_range.first == ~D[2021-01-01]- assert comparison.date_range.last == ~D[2021-12-31]+ assert comparison.date_range.first ==+ DateTime.new!(~D[2021-01-01], ~T[00:00:00], site.timezone)++ assert comparison.date_range.last ==+ DateTime.new!(~D[2021-12-31], ~T[23:59:59], site.timezone)endend@@ -163,8 +202,11 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} = Comparisons.compare(site, query, "previous_period")- assert comparison.date_range.first == ~D[2022-12-25]- assert comparison.date_range.last == ~D[2022-12-31]+ assert comparison.date_range.first ==+ DateTime.new!(~D[2022-12-25], ~T[00:00:00], site.timezone)++ assert comparison.date_range.last ==+ DateTime.new!(~D[2022-12-31], ~T[23:59:59], site.timezone)endtest "shifts back to last year when mode is year_over_year" do@@ -173,8 +215,11 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} = Comparisons.compare(site, query, "year_over_year")- assert comparison.date_range.first == ~D[2022-01-01]- assert comparison.date_range.last == ~D[2022-01-07]+ assert comparison.date_range.first ==+ DateTime.new!(~D[2022-01-01], ~T[00:00:00], site.timezone)++ assert comparison.date_range.last ==+ DateTime.new!(~D[2022-01-07], ~T[23:59:59], site.timezone)endend@@ -186,8 +231,11 @@ defmodule Plausible.Stats.ComparisonsTest do{:ok, comparison} =Comparisons.compare(site, query, "custom", from: "2022-05-25", to: "2022-05-30")- assert comparison.date_range.first == ~D[2022-05-25]- assert comparison.date_range.last == ~D[2022-05-30]+ assert comparison.date_range.first ==+ DateTime.new!(~D[2022-05-25], ~T[00:00:00], site.timezone)++ assert comparison.date_range.last ==+ DateTime.new!(~D[2022-05-30], ~T[23:59:59], site.timezone)endtest "validates from and to dates" dodiff --git a/test/plausible/stats/interval_test.exs b/test/plausible/stats/interval_test.exsindex c18771edf3..1ec223ed3d 100644--- a/test/plausible/stats/interval_test.exs+++ b/test/plausible/stats/interval_test.exs@@ -2,6 +2,7 @@ defmodule Plausible.Stats.IntervalTest douse Plausible.DataCase, async: trueimport Plausible.Stats.Interval+ alias Plausible.Stats.DateTimeRangetest "default_for_period/1" doassert default_for_period("realtime") == "minute"@@ -11,9 +12,13 @@ defmodule Plausible.Stats.IntervalTest doendtest "default_for_date_range/1" do- assert default_for_date_range(Date.range(~D[2022-01-01], ~D[2023-01-01])) == "month"- assert default_for_date_range(Date.range(~D[2022-01-01], ~D[2022-01-15])) == "day"- assert default_for_date_range(Date.range(~D[2022-01-01], ~D[2022-01-01])) == "hour"+ year = DateTimeRange.new!(~D[2022-01-01], ~D[2023-01-01], "UTC")+ fifteen_days = DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-15], "UTC")+ day = DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-01], "UTC")++ assert default_for_date_range(year) == "month"+ assert default_for_date_range(fifteen_days) == "day"+ assert default_for_date_range(day) == "hour"enddescribe "valid_by_period/1" dodiff --git a/test/plausible/stats/query_optimizer_test.exs b/test/plausible/stats/query_optimizer_test.exsindex d35b958bcd..64d526edda 100644--- a/test/plausible/stats/query_optimizer_test.exs+++ b/test/plausible/stats/query_optimizer_test.exs@@ -1,7 +1,7 @@defmodule Plausible.Stats.QueryOptimizerTest douse Plausible.DataCase, async: true- alias Plausible.Stats.{Query, QueryOptimizer}+ alias Plausible.Stats.{Query, QueryOptimizer, DateTimeRange}@default_params %{metrics: [:visitors]}@@ -24,7 +24,7 @@ defmodule Plausible.Stats.QueryOptimizerTest dotest "adds time and first metric to order_by if order_by not specified" doassert perform(%{- date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-02-01 00:00:00]),+ date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-31], "UTC"),metrics: [:pageviews, :visitors],dimensions: ["time", "event:page"]}).order_by ==@@ -35,69 +35,73 @@ defmodule Plausible.Stats.QueryOptimizerTest dodescribe "update_group_by_time" dotest "does nothing if `time` dimension not passed" doassert perform(%{- date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-05 00:00:00]),+ date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-04], "UTC"),dimensions: ["time:month"]}).dimensions == ["time:month"]endtest "updating time dimension" doassert perform(%{- date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]),+ date_range:+ DateTimeRange.new!(+ DateTime.new!(~D[2022-01-01], ~T[00:00:00], "UTC"),+ DateTime.new!(~D[2022-01-01], ~T[05:00:00], "UTC")+ ),dimensions: ["time"]}).dimensions == ["time:hour"]assert perform(%{- date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-02 00:00:00]),+ date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-01], "UTC"),dimensions: ["time"]}).dimensions == ["time:hour"]assert perform(%{- date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-02 16:00:00]),+ date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-02], "UTC"),dimensions: ["time"]}).dimensions == ["time:hour"]assert perform(%{- date_range: Date.range(~D[2022-01-01], ~D[2022-01-04]),+ date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-03], "UTC"),dimensions: ["time"]}).dimensions == ["time:day"]assert perform(%{- date_range: Date.range(~D[2022-01-01], ~D[2022-01-10]),+ date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-10], "UTC"),dimensions: ["time"]}).dimensions == ["time:day"]assert perform(%{- date_range: Date.range(~D[2022-01-01], ~D[2022-01-16]),+ date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-16], "UTC"),dimensions: ["time"]}).dimensions == ["time:day"]assert perform(%{- date_range: Date.range(~D[2022-01-01], ~D[2022-02-16]),+ date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-02-16], "UTC"),dimensions: ["time"]}).dimensions == ["time:week"]assert perform(%{- date_range: Date.range(~D[2022-01-01], ~D[2022-03-16]),+ date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-03-16], "UTC"),dimensions: ["time"]}).dimensions == ["time:week"]assert perform(%{- date_range: Date.range(~D[2022-01-01], ~D[2022-03-16]),+ date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-03-16], "UTC"),dimensions: ["time"]}).dimensions == ["time:week"]assert perform(%{- date_range: Date.range(~D[2022-01-01], ~D[2023-11-16]),+ date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2023-11-16], "UTC"),dimensions: ["time"]}).dimensions == ["time:month"]assert perform(%{- date_range: Date.range(~D[2022-01-01], ~D[2024-01-16]),+ date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2024-01-16], "UTC"),dimensions: ["time"]}).dimensions == ["time:month"]assert perform(%{- date_range: Date.range(~D[2022-01-01], ~D[2026-01-01]),+ date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2026-01-01], "UTC"),dimensions: ["time"]}).dimensions == ["time:month"]end@@ -106,7 +110,11 @@ defmodule Plausible.Stats.QueryOptimizerTest dodescribe "update_time_in_order_by" dotest "updates explicit time dimension in order_by" doassert perform(%{- date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]),+ date_range:+ DateTimeRange.new!(+ DateTime.new!(~D[2022-01-01], ~T[00:00:00], "UTC"),+ DateTime.new!(~D[2022-01-01], ~T[05:00:00], "UTC")+ ),dimensions: ["time:hour"],order_by: [{"time", :asc}]}).order_by == [{"time:hour", :asc}]diff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exsindex 062abe1cd2..64775fa190 100644--- a/test/plausible/stats/query_parser_test.exs+++ b/test/plausible/stats/query_parser_test.exs@@ -1,41 +1,77 @@defmodule Plausible.Stats.Filters.QueryParserTest douse Plausible.DataCase+ alias Plausible.Stats.DateTimeRangealias Plausible.Stats.Filtersimport Plausible.Stats.Filters.QueryParsersetup [:create_user, :create_new_site]- @today ~D[2021-05-05]- @date_range Date.range(@today, @today)-- def check_success(params, site, expected_result),- do: check_success(params, site, :public, expected_result)-- def check_success(params, site, schema_type, expected_result) do- assert {:ok, ^expected_result} = parse(site, schema_type, params, @today)+ @now DateTime.new!(~D[2021-05-05], ~T[12:30:00], "UTC")+ @date_range_realtime %DateTimeRange{+ first: DateTime.new!(~D[2021-05-05], ~T[12:25:00], "UTC"),+ last: DateTime.new!(~D[2021-05-05], ~T[12:30:05], "UTC")+ }+ @date_range_30m %DateTimeRange{+ first: DateTime.new!(~D[2021-05-05], ~T[12:00:00], "UTC"),+ last: DateTime.new!(~D[2021-05-05], ~T[12:30:05], "UTC")+ }+ @date_range_day %DateTimeRange{+ first: DateTime.new!(~D[2021-05-05], ~T[00:00:00], "UTC"),+ last: DateTime.new!(~D[2021-05-05], ~T[23:59:59], "UTC")+ }+ @date_range_7d %DateTimeRange{+ first: DateTime.new!(~D[2021-04-29], ~T[00:00:00], "UTC"),+ last: DateTime.new!(~D[2021-05-05], ~T[23:59:59], "UTC")+ }+ @date_range_30d %DateTimeRange{+ first: DateTime.new!(~D[2021-04-05], ~T[00:00:00], "UTC"),+ last: DateTime.new!(~D[2021-05-05], ~T[23:59:59], "UTC")+ }+ @date_range_month %DateTimeRange{+ first: DateTime.new!(~D[2021-05-01], ~T[00:00:00], "UTC"),+ last: DateTime.new!(~D[2021-05-31], ~T[23:59:59], "UTC")+ }+ @date_range_6mo %DateTimeRange{+ first: DateTime.new!(~D[2020-12-01], ~T[00:00:00], "UTC"),+ last: DateTime.new!(~D[2021-05-31], ~T[23:59:59], "UTC")+ }+ @date_range_year %DateTimeRange{+ first: DateTime.new!(~D[2021-01-01], ~T[00:00:00], "UTC"),+ last: DateTime.new!(~D[2021-12-31], ~T[23:59:59], "UTC")+ }+ @date_range_12mo %DateTimeRange{+ first: DateTime.new!(~D[2020-06-01], ~T[00:00:00], "UTC"),+ last: DateTime.new!(~D[2021-05-31], ~T[23:59:59], "UTC")+ }++ def check_success(params, site, expected_result, schema_type \\ :public) do+ assert {:ok, result} = parse(site, schema_type, params, @now)+ assert result == expected_resultend- def check_error(params, site, expected_error_message),- do: check_error(params, site, :public, expected_error_message)-- def check_error(params, site, schema_type, expected_error_message) do- {:error, message} = parse(site, schema_type, params, @today)+ def check_error(params, site, expected_error_message, schema_type \\ :public) do+ {:error, message} = parse(site, schema_type, params, @now)assert message == expected_error_messageend- def check_date_range(date_range, site, expected_date_range) do- %{"site_id" => site.domain, "metrics" => ["visitors", "events"], "date_range" => date_range}- |> check_success(site, %{- metrics: [:visitors, :events],- date_range: expected_date_range,- filters: [],- dimensions: [],- order_by: nil,- timezone: site.timezone,- include: %{imports: false, time_labels: false},- preloaded_goals: []- })+ def check_date_range(date_params, site, expected_date_range, schema_type \\ :public) do+ %{"site_id" => site.domain, "metrics" => ["visitors", "events"]}+ |> Map.merge(date_params)+ |> check_success(+ site,+ %{+ metrics: [:visitors, :events],+ date_range: expected_date_range,+ filters: [],+ dimensions: [],+ order_by: nil,+ timezone: site.timezone,+ include: %{imports: false, time_labels: false},+ preloaded_goals: []+ },+ schema_type+ )endtest "parsing empty map fails", %{site: site} do@@ -48,7 +84,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do%{"site_id" => site.domain, "metrics" => ["visitors", "events"], "date_range" => "all"}|> check_success(site, %{metrics: [:visitors, :events],- date_range: @date_range,+ date_range: @date_range_day,filters: [],dimensions: [],order_by: nil,@@ -77,24 +113,28 @@ defmodule Plausible.Stats.Filters.QueryParserTest do],"date_range" => "all"}- |> check_success(site, :internal, %{- metrics: [- :time_on_page,- :visitors,- :pageviews,- :visits,- :events,- :bounce_rate,- :visit_duration- ],- date_range: @date_range,- filters: [],- dimensions: [],- order_by: nil,- timezone: site.timezone,- include: %{imports: false, time_labels: false},- preloaded_goals: []- })+ |> check_success(+ site,+ %{+ metrics: [+ :time_on_page,+ :visitors,+ :pageviews,+ :visits,+ :events,+ :bounce_rate,+ :visit_duration+ ],+ date_range: @date_range_day,+ filters: [],+ dimensions: [],+ order_by: nil,+ timezone: site.timezone,+ include: %{imports: false, time_labels: false},+ preloaded_goals: []+ },+ :internal+ )endtest "time_on_page is not a valid metric in public API", %{site: site} do@@ -103,7 +143,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do"metrics" => ["time_on_page"],"date_range" => "all"}- |> check_error(site, :public, "#/metrics/0: Invalid metric \"time_on_page\"")+ |> check_error(site, "#/metrics/0: Invalid metric \"time_on_page\"")endtest "same metric queried multiple times", %{site: site} do@@ -136,18 +176,22 @@ defmodule Plausible.Stats.Filters.QueryParserTest do[Atom.to_string(unquote(operation)), "event:name", ["foo"]]]}- |> check_success(site, :internal, %{- metrics: [:visitors],- date_range: @date_range,- filters: [- [unquote(operation), "event:name", ["foo"]]- ],- dimensions: [],- order_by: nil,- timezone: site.timezone,- include: %{imports: false, time_labels: false},- preloaded_goals: []- })+ |> check_success(+ site,+ %{+ metrics: [:visitors],+ date_range: @date_range_day,+ filters: [+ [unquote(operation), "event:name", ["foo"]]+ ],+ dimensions: [],+ order_by: nil,+ timezone: site.timezone,+ include: %{imports: false, time_labels: false},+ preloaded_goals: []+ },+ :internal+ )endtest "#{operation} filter with invalid clause", %{site: site} do@@ -161,8 +205,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_error(site,- :internal,- "#/filters/0: Invalid filter [\"#{unquote(operation)}\", \"event:name\", \"foo\"]"+ "#/filters/0: Invalid filter [\"#{unquote(operation)}\", \"event:name\", \"foo\"]",+ :internal)endend@@ -179,7 +223,6 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_error(site,- :public,"#/filters/0: Invalid filter [\"#{unquote(operation)}\", \"event:name\", [\"foo\"]]")end@@ -208,7 +251,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:visitors],- date_range: @date_range,+ date_range: @date_range_day,filters: [[:is, "event:props:foobar", ["value"]]],@@ -233,7 +276,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:visitors],- date_range: @date_range,+ date_range: @date_range_day,filters: [[:is, "event:#{unquote(dimension)}", ["foo"]]],@@ -259,7 +302,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:visitors],- date_range: @date_range,+ date_range: @date_range_day,filters: [[:is, "visit:#{unquote(dimension)}", ["ab"]]],@@ -325,7 +368,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:visitors],- date_range: @date_range,+ date_range: @date_range_day,filters: [[:is, "visit:city", [123, 456]]],@@ -344,7 +387,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:visitors],- date_range: @date_range,+ date_range: @date_range_day,filters: [[:is, "visit:city", ["123", "456"]]],@@ -381,7 +424,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:visitors],- date_range: @date_range,+ date_range: @date_range_day,filters: [],dimensions: ["time"],order_by: nil,@@ -426,12 +469,12 @@ defmodule Plausible.Stats.Filters.QueryParserTest do]}- assert {:ok, res} = parse(site, :public, params, @today)+ assert {:ok, res} = parse(site, :public, params, @now)expected_timezone = site.timezoneassert %{metrics: [:visitors],- date_range: @date_range,+ date_range: @date_range_day,filters: [[:is, "event:goal", ["Signup", "Visit /thank-you"]]],@@ -479,41 +522,208 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodescribe "date range validation" dotest "parsing shortcut options", %{site: site} do- check_date_range("day", site, Date.range(~D[2021-05-05], ~D[2021-05-05]))- check_date_range("7d", site, Date.range(~D[2021-04-29], ~D[2021-05-05]))- check_date_range("30d", site, Date.range(~D[2021-04-05], ~D[2021-05-05]))- check_date_range("month", site, Date.range(~D[2021-05-01], ~D[2021-05-31]))- check_date_range("6mo", site, Date.range(~D[2020-12-01], ~D[2021-05-31]))- check_date_range("12mo", site, Date.range(~D[2020-06-01], ~D[2021-05-31]))- check_date_range("year", site, Date.range(~D[2021-01-01], ~D[2021-12-31]))+ check_date_range(%{"date_range" => "day"}, site, @date_range_day)+ check_date_range(%{"date_range" => "7d"}, site, @date_range_7d)+ check_date_range(%{"date_range" => "30d"}, site, @date_range_30d)+ check_date_range(%{"date_range" => "month"}, site, @date_range_month)+ check_date_range(%{"date_range" => "6mo"}, site, @date_range_6mo)+ check_date_range(%{"date_range" => "12mo"}, site, @date_range_12mo)+ check_date_range(%{"date_range" => "year"}, site, @date_range_year)+ end++ test "30m and realtime are available in internal API", %{site: site} do+ check_date_range(%{"date_range" => "30m"}, site, @date_range_30m, :internal)+ check_date_range(%{"date_range" => "realtime"}, site, @date_range_realtime, :internal)+ end++ test "30m and realtime date_ranges are unavailable in public API", %{+ site: site+ } do+ for date_range <- ["realtime", "30m"] do+ %{"site_id" => site.domain, "metrics" => ["visitors"], "date_range" => date_range}+ |> check_error(site, "#/date_range: Invalid date range \"#{date_range}\"")+ endendtest "parsing `all` with previous data", %{site: site} dosite = Map.put(site, :stats_start_date, ~D[2020-01-01])- check_date_range("all", site, Date.range(~D[2020-01-01], ~D[2021-05-05]))+ expected_date_range = DateTimeRange.new!(~D[2020-01-01], ~D[2021-05-05], "UTC")+ check_date_range(%{"date_range" => "all"}, site, expected_date_range)endtest "parsing `all` with no previous data", %{site: site} dosite = Map.put(site, :stats_start_date, nil)+ check_date_range(%{"date_range" => "all"}, site, @date_range_day)+ end- check_date_range("all", site, Date.range(~D[2021-05-05], ~D[2021-05-05]))+ test "parsing custom date range from simple date strings", %{site: site} do+ check_date_range(%{"date_range" => ["2021-05-05", "2021-05-05"]}, site, @date_range_day)end- test "parsing custom date range", %{site: site} do+ test "parsing custom date range from iso8601 timestamps", %{site: site} docheck_date_range(- ["2021-05-05", "2021-05-05"],+ %{"date_range" => ["2024-01-01T00:00:00 UTC", "2024-01-02T23:59:59 UTC"]},+ site,+ DateTimeRange.new!(+ DateTime.new!(~D[2024-01-01], ~T[00:00:00], "UTC"),+ DateTime.new!(~D[2024-01-02], ~T[23:59:59], "UTC")+ )+ )++ check_date_range(+ %{+ "date_range" => [+ "2024-08-29T07:12:34 America/Los_Angeles",+ "2024-08-29T10:12:34 America/Los_Angeles"+ ]+ },site,- Date.range(~D[2021-05-05], ~D[2021-05-05])+ DateTimeRange.new!(+ DateTime.new!(~D[2024-08-29], ~T[07:12:34], "America/Los_Angeles"),+ DateTime.new!(~D[2024-08-29], ~T[10:12:34], "America/Los_Angeles")+ ))end- test "parsing invalid custom date range", %{site: site} do+ test "parsing invalid custom date range with invalid dates", %{site: site} do%{"site_id" => site.domain, "date_range" => "foo", "metrics" => ["visitors"]}|> check_error(site, "#/date_range: Invalid date range \"foo\"")%{"site_id" => site.domain, "date_range" => ["21415-00", "eee"], "metrics" => ["visitors"]}|> check_error(site, "#/date_range: Invalid date range [\"21415-00\", \"eee\"]")end++ test "custom date range is invalid when timestamps do not include timezone info", %{+ site: site+ } do+ %{+ "site_id" => site.domain,+ "date_range" => ["2021-02-03T00:00:00", "2021-02-03T23:59:59"],+ "metrics" => ["visitors"]+ }+ |> check_error(+ site,+ "#/date_range: Invalid date range [\"2021-02-03T00:00:00\", \"2021-02-03T23:59:59\"]"+ )+ end++ test "custom date range is invalid when timestamp timezone is invalid", %{site: site} do+ %{+ "site_id" => site.domain,+ "date_range" => ["2021-02-03T00:00:00 Fake/Timezone", "2021-02-03T23:59:59 UTC"],+ "metrics" => ["visitors"]+ }+ |> check_error(+ site,+ "Invalid date_range '[\"2021-02-03T00:00:00 Fake/Timezone\", \"2021-02-03T23:59:59 UTC\"]'."+ )+ end++ test "custom date range is invalid when date and timestamp are combined", %{site: site} do+ %{+ "site_id" => site.domain,+ "date_range" => ["2021-02-03T00:00:00 UTC", "2021-02-04"],+ "metrics" => ["visitors"]+ }+ |> check_error(+ site,+ "Invalid date_range '[\"2021-02-03T00:00:00 UTC\", \"2021-02-04\"]'."+ )+ end++ test "custom date range is invalid when timestamp cannot be converted to datetime due to a gap in timezone",+ %{site: site} do+ %{+ "site_id" => site.domain,+ "date_range" => [+ "2024-03-31T03:30:00 Europe/Tallinn",+ "2024-04-15T10:00:00 Europe/Tallinn"+ ],+ "metrics" => ["visitors"]+ }+ |> check_error(+ site,+ "Invalid date_range '[\"2024-03-31T03:30:00 Europe/Tallinn\", \"2024-04-15T10:00:00 Europe/Tallinn\"]'."+ )+ end++ test "parses date_range relative to date param", %{site: site} do+ date = @now |> DateTime.to_date() |> Date.to_string()++ for {date_range_shortcut, expected_date_range} <- [+ {"day", @date_range_day},+ {"7d", @date_range_7d},+ {"30d", @date_range_30d},+ {"month", @date_range_month},+ {"6mo", @date_range_6mo},+ {"12mo", @date_range_12mo},+ {"year", @date_range_year}+ ] do+ %{"date_range" => date_range_shortcut, "date" => date}+ |> check_date_range(site, expected_date_range, :internal)+ end+ end++ test "date parameter is not available in the public API", %{site: site} do+ %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "events"],+ "date_range" => "month",+ "date" => "2021-05-05"+ }+ |> check_error(site, "#/date: Schema does not allow additional properties.")+ end++ test "parses date_range.first into a datetime right after the gap in site.timezone", %{+ site: site+ } do+ site = %{site | timezone: "America/Santiago"}++ expected_date_range =+ DateTimeRange.new!(+ DateTime.new!(~D[2022-09-11], ~T[01:00:00], site.timezone),+ DateTime.new!(~D[2022-09-11], ~T[23:59:59], site.timezone)+ )++ %{"date_range" => ["2022-09-11", "2022-09-11"]}+ |> check_date_range(site, expected_date_range)+ end++ test "parses date_range.first into the latest of ambiguous datetimes in site.timezone", %{+ site: site+ } do+ site = %{site | timezone: "America/Havana"}++ {:ambiguous, _, expected_first_datetime} =+ DateTime.new(~D[2023-11-05], ~T[00:00:00], site.timezone)++ expected_date_range =+ DateTimeRange.new!(+ expected_first_datetime,+ DateTime.new!(~D[2023-11-05], ~T[23:59:59], site.timezone)+ )++ %{"date_range" => ["2023-11-05", "2023-11-05"]}+ |> check_date_range(site, expected_date_range)+ end++ test "parses date_range.last into the earliest of ambiguous datetimes in site.timezone", %{+ site: site+ } do+ site = %{site | timezone: "America/Asuncion"}++ {:ambiguous, first_dt, _second_dt} =+ DateTime.new(~D[2024-03-23], ~T[23:59:59], site.timezone)++ expected_date_range =+ DateTimeRange.new!(+ DateTime.new!(~D[2024-03-23], ~T[00:00:00], site.timezone),+ first_dt+ )++ %{"date_range" => ["2024-03-23", "2024-03-23"]}+ |> check_date_range(site, expected_date_range)+ endenddescribe "dimensions validation" do@@ -527,7 +737,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:visitors],- date_range: @date_range,+ date_range: @date_range_day,filters: [],dimensions: ["event:#{unquote(dimension)}"],order_by: nil,@@ -548,7 +758,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:visitors],- date_range: @date_range,+ date_range: @date_range_day,filters: [],dimensions: ["visit:#{unquote(dimension)}"],order_by: nil,@@ -568,7 +778,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:visitors],- date_range: @date_range,+ date_range: @date_range_day,filters: [],dimensions: ["event:props:foobar"],order_by: nil,@@ -629,7 +839,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:visitors, :events],- date_range: @date_range,+ date_range: @date_range_day,filters: [],dimensions: [],order_by: [{:events, :desc}, {:visitors, :asc}],@@ -649,7 +859,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:visitors],- date_range: @date_range,+ date_range: @date_range_day,filters: [],dimensions: ["event:name"],order_by: [{"event:name", :desc}],@@ -757,7 +967,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do# }# |> check_success(site, %{# metrics: [:conversion_rate],- # date_range: @date_range,+ # date_range: @date_range_day,# filters: [[:is, "event:goal", [event: "Signup"]]],# dimensions: [],# order_by: nil,@@ -777,7 +987,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do# }# |> check_success(site, %{# metrics: [:conversion_rate],- # date_range: @date_range,+ # date_range: @date_range_day,# filters: [],# dimensions: ["event:goal"],# order_by: nil,@@ -799,7 +1009,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do# }# |> check_success(site, %{# metrics: [:views_per_visit],- # date_range: @date_range,+ # date_range: @date_range_day,# filters: [[:is, "event:goal", [event: "Signup"]]],# dimensions: [],# order_by: nil,@@ -846,7 +1056,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:bounce_rate],- date_range: @date_range,+ date_range: @date_range_day,filters: [],dimensions: ["visit:device"],order_by: nil,@@ -878,7 +1088,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:bounce_rate],- date_range: @date_range,+ date_range: @date_range_day,filters: [],dimensions: ["event:page"],order_by: nil,@@ -897,7 +1107,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_success(site, %{metrics: [:bounce_rate],- date_range: @date_range,+ date_range: @date_range_day,filters: [[:is, "event:props:foo", ["(none)"]]],dimensions: [],order_by: nil,diff --git a/test/plausible/stats/query_result_test.exs b/test/plausible/stats/query_result_test.exsindex 5e2ce5b4dd..f7f3db75c6 100644--- a/test/plausible/stats/query_result_test.exs+++ b/test/plausible/stats/query_result_test.exs@@ -48,8 +48,8 @@ defmodule Plausible.Stats.QueryResultTest do"pageviews"],"date_range": [- "2024-01-01",- "2024-02-01"+ "2024-01-01T00:00:00 UTC",+ "2024-02-01T23:59:59 UTC"],"filters": [],"dimensions": [],diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exsindex ee0517cbdf..688c8a14cc 100644--- a/test/plausible/stats/query_test.exs+++ b/test/plausible/stats/query_test.exs@@ -1,6 +1,6 @@defmodule Plausible.Stats.QueryTest douse Plausible.DataCase, async: true- alias Plausible.Stats.Query+ alias Plausible.Stats.{Query, DateTimeRange}setup douser = insert(:user)@@ -19,8 +19,8 @@ defmodule Plausible.Stats.QueryTest dotest "keeps current timestamp so that utc_boundaries don't depend on time passing by", %{site: site} do- q1 = %{now: %NaiveDateTime{}} = Query.from(site, %{"period" => "realtime"})- q2 = %{now: %NaiveDateTime{}} = Query.from(site, %{"period" => "30m"})+ q1 = %{now: %DateTime{}} = Query.from(site, %{"period" => "realtime"})+ q2 = %{now: %DateTime{}} = Query.from(site, %{"period" => "30m"})boundaries1 = Plausible.Stats.Time.utc_boundaries(q1, site)boundaries2 = Plausible.Stats.Time.utc_boundaries(q2, site):timer.sleep(1500)@@ -31,136 +31,190 @@ defmodule Plausible.Stats.QueryTest dotest "parses day format", %{site: site} doq = Query.from(site, %{"period" => "day", "date" => "2019-01-01"})- assert q.date_range.first == ~D[2019-01-01]- assert q.date_range.last == ~D[2019-01-01]+ assert q.date_range.first == DateTime.new!(~D[2019-01-01], ~T[00:00:00], site.timezone)+ assert q.date_range.last == DateTime.new!(~D[2019-01-01], ~T[23:59:59], site.timezone)assert q.interval == "hour"endtest "day format defaults to today", %{site: site} doq = Query.from(site, %{"period" => "day"})- assert q.date_range.first == Timex.today()- assert q.date_range.last == Timex.today()+ expected_first_datetime = Date.utc_today() |> DateTime.new!(~T[00:00:00], site.timezone)+ expected_last_datetime = Date.utc_today() |> DateTime.new!(~T[23:59:59], site.timezone)++ assert q.date_range.first == expected_first_datetime+ assert q.date_range.last == expected_last_datetimeassert q.interval == "hour"endtest "parses realtime format", %{site: site} doq = Query.from(site, %{"period" => "realtime"})- assert q.date_range.first == Timex.today()- assert q.date_range.last == Timex.today()+ utc_now = DateTime.shift_zone!(q.now, "Etc/UTC")++ expected_first_datetime = utc_now |> DateTime.shift(minute: -5)+ expected_last_datetime = utc_now |> DateTime.shift(second: 5)++ assert q.date_range.first == expected_first_datetime+ assert q.date_range.last == expected_last_datetimeassert q.period == "realtime"endtest "parses month format", %{site: site} doq = Query.from(site, %{"period" => "month", "date" => "2019-01-01"})- assert q.date_range.first == ~D[2019-01-01]- assert q.date_range.last == ~D[2019-01-31]+ assert q.date_range.first == DateTime.new!(~D[2019-01-01], ~T[00:00:00], site.timezone)+ assert q.date_range.last == DateTime.new!(~D[2019-01-31], ~T[23:59:59], site.timezone)assert q.interval == "day"endtest "parses 6 month format", %{site: site} doq = Query.from(site, %{"period" => "6mo"})- assert q.date_range.first ==- Timex.shift(Timex.today(), months: -5) |> Timex.beginning_of_month()-- assert q.date_range.last == Timex.today() |> Timex.end_of_month()+ expected_first_datetime =+ q.now+ |> DateTime.to_date()+ |> Date.shift(month: -5)+ |> Date.beginning_of_month()+ |> DateTime.new!(~T[00:00:00], site.timezone)++ expected_last_datetime =+ q.now+ |> DateTime.to_date()+ |> Date.end_of_month()+ |> DateTime.new!(~T[23:59:59], site.timezone)++ assert q.date_range.first == expected_first_datetime+ assert q.date_range.last == expected_last_datetimeassert q.interval == "month"endtest "parses 12 month format", %{site: site} doq = Query.from(site, %{"period" => "12mo"})- assert q.date_range.first ==- Timex.shift(Timex.today(), months: -11) |> Timex.beginning_of_month()-- assert q.date_range.last == Timex.today() |> Timex.end_of_month()+ expected_first_datetime =+ q.now+ |> DateTime.to_date()+ |> Date.shift(month: -11)+ |> Date.beginning_of_month()+ |> DateTime.new!(~T[00:00:00], site.timezone)++ expected_last_datetime =+ q.now+ |> DateTime.to_date()+ |> Date.end_of_month()+ |> DateTime.new!(~T[23:59:59], site.timezone)++ assert q.date_range.first == expected_first_datetime+ assert q.date_range.last == expected_last_datetimeassert q.interval == "month"endtest "parses year to date format", %{site: site} doq = Query.from(site, %{"period" => "year"})- assert q.date_range.first ==- Timex.now(site.timezone) |> Timex.to_date() |> Timex.beginning_of_year()+ %Date{year: current_year} = DateTime.to_date(q.now)- assert q.date_range.last ==- Timex.now(site.timezone) |> Timex.to_date() |> Timex.end_of_year()+ expected_first_datetime =+ Date.new!(current_year, 1, 1)+ |> DateTime.new!(~T[00:00:00], site.timezone)+ expected_last_datetime =+ Date.new!(current_year, 12, 31)+ |> DateTime.new!(~T[23:59:59], site.timezone)++ assert q.date_range.first == expected_first_datetime+ assert q.date_range.last == expected_last_datetimeassert q.interval == "month"endtest "parses all time", %{site: site} doq = Query.from(site, %{"period" => "all"})- assert q.date_range.first == NaiveDateTime.to_date(site.inserted_at)- assert q.date_range.last == Timex.today()+ expected_last_datetime =+ q.now+ |> DateTime.to_date()+ |> DateTime.new!(~T[23:59:59], site.timezone)++ assert DateTime.to_naive(q.date_range.first) == site.inserted_at+ assert q.date_range.last == expected_last_datetimeassert q.period == "all"assert q.interval == "month"end- test "parses all time in correct timezone", %{site: site} do- site = Map.put(site, :timezone, "America/Cancun")- q = Query.from(site, %{"period" => "all"})+ test "parses all time in site timezone", %{site: site} do+ for timezone <- ["Etc/GMT+12", "Etc/GMT-12"] do+ site = Map.put(site, :timezone, timezone)+ query = Query.from(site, %{"period" => "all"})++ expected_first_datetime = DateTime.new!(~D[2020-01-01], ~T[00:00:00], site.timezone)++ expected_last_datetime =+ Timex.today(site.timezone)+ |> DateTime.new!(~T[23:59:59], site.timezone)- assert q.date_range.first == ~D[2020-01-01]- assert q.date_range.last == Timex.today("America/Cancun")+ assert query.date_range.first == expected_first_datetime+ assert query.date_range.last == expected_last_datetime+ endendtest "all time shows today if site has no start date", %{site: site} dosite = Map.put(site, :stats_start_date, nil)q = Query.from(site, %{"period" => "all"})- assert q.date_range.first == Timex.today()- assert q.date_range.last == Timex.today()+ today = Date.utc_today()++ assert q.date_range == DateTimeRange.new!(today, today, site.timezone)assert q.period == "all"assert q.interval == "hour"endtest "all time shows hourly if site is completely new", %{site: site} do- site = Map.put(site, :stats_start_date, Timex.now() |> Timex.to_date())+ site = Map.put(site, :stats_start_date, Date.utc_today())q = Query.from(site, %{"period" => "all"})- assert q.date_range.first == Timex.today()- assert q.date_range.last == Timex.today()+ today = Date.utc_today()++ assert q.date_range == DateTimeRange.new!(today, today, site.timezone)assert q.period == "all"assert q.interval == "hour"endtest "all time shows daily if site is more than a day old", %{site: site} do- site =- Map.put(site, :stats_start_date, Timex.now() |> Timex.shift(days: -1) |> Timex.to_date())+ today = Date.utc_today()+ yesterday = today |> Date.shift(day: -1)++ site = Map.put(site, :stats_start_date, yesterday)q = Query.from(site, %{"period" => "all"})- assert q.date_range.first == Timex.today() |> Timex.shift(days: -1)- assert q.date_range.last == Timex.today()+ assert q.date_range == DateTimeRange.new!(yesterday, today, site.timezone)assert q.period == "all"assert q.interval == "day"endtest "all time shows monthly if site is more than a month old", %{site: site} do- site =- Map.put(site, :stats_start_date, Timex.now() |> Timex.shift(months: -1) |> Timex.to_date())+ today = Date.utc_today()+ last_month = today |> Date.shift(month: -1)++ site = Map.put(site, :stats_start_date, last_month)q = Query.from(site, %{"period" => "all"})- assert q.date_range.first == Timex.today() |> Timex.shift(months: -1)- assert q.date_range.last == Timex.today()+ assert q.date_range == DateTimeRange.new!(last_month, today, site.timezone)assert q.period == "all"assert q.interval == "month"endtest "all time uses passed interval different from the default interval", %{site: site} do- site =- Map.put(site, :stats_start_date, Timex.now() |> Timex.shift(months: -1) |> Timex.to_date())+ today = Date.utc_today()+ last_month = today |> Date.shift(month: -1)++ site = Map.put(site, :stats_start_date, last_month)q = Query.from(site, %{"period" => "all", "interval" => "week"})- assert q.date_range.first == Timex.today() |> Timex.shift(months: -1)- assert q.date_range.last == Timex.today()+ assert q.date_range == DateTimeRange.new!(last_month, today, site.timezone)assert q.period == "all"assert q.interval == "week"end@@ -172,8 +226,8 @@ defmodule Plausible.Stats.QueryTest dotest "parses custom format", %{site: site} doq = Query.from(site, %{"period" => "custom", "from" => "2019-01-01", "to" => "2019-01-15"})- assert q.date_range.first == ~D[2019-01-01]- assert q.date_range.last == ~D[2019-01-15]+ assert q.date_range.first == DateTime.new!(~D[2019-01-01], ~T[00:00:00], site.timezone)+ assert q.date_range.last == DateTime.new!(~D[2019-01-15], ~T[23:59:59], site.timezone)assert q.interval == "day"enddiff --git a/test/plausible/stats/time_test.exs b/test/plausible/stats/time_test.exsindex e368a0cbc5..37cb983fb9 100644--- a/test/plausible/stats/time_test.exs+++ b/test/plausible/stats/time_test.exs@@ -2,12 +2,13 @@ defmodule Plausible.Stats.TimeTest douse Plausible.DataCase, async: trueimport Plausible.Stats.Time+ alias Plausible.Stats.DateTimeRangedescribe "time_labels/1" dotest "with time:month dimension" doassert time_labels(%{dimensions: ["visit:device", "time:month"],- date_range: Date.range(~D[2022-01-17], ~D[2022-02-01])+ date_range: DateTimeRange.new!(~D[2022-01-17], ~D[2022-02-01], "UTC")}) == ["2022-01-01","2022-02-01"@@ -15,7 +16,7 @@ defmodule Plausible.Stats.TimeTest doassert time_labels(%{dimensions: ["visit:device", "time:month"],- date_range: Date.range(~D[2022-01-01], ~D[2022-03-07])+ date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-03-07], "UTC")}) == ["2022-01-01","2022-02-01",@@ -26,7 +27,7 @@ defmodule Plausible.Stats.TimeTest dotest "with time:week dimension" doassert time_labels(%{dimensions: ["time:week"],- date_range: Date.range(~D[2020-12-20], ~D[2021-01-08])+ date_range: DateTimeRange.new!(~D[2020-12-20], ~D[2021-01-08], "UTC")}) == ["2020-12-20","2020-12-21",@@ -36,7 +37,7 @@ defmodule Plausible.Stats.TimeTest doassert time_labels(%{dimensions: ["time:week"],- date_range: Date.range(~D[2020-12-21], ~D[2021-01-03])+ date_range: DateTimeRange.new!(~D[2020-12-21], ~D[2021-01-03], "UTC")}) == ["2020-12-21","2020-12-28"@@ -46,7 +47,7 @@ defmodule Plausible.Stats.TimeTest dotest "with time:day dimension" doassert time_labels(%{dimensions: ["time:day"],- date_range: Date.range(~D[2022-01-17], ~D[2022-02-02])+ date_range: DateTimeRange.new!(~D[2022-01-17], ~D[2022-02-02], "UTC")}) == ["2022-01-17","2022-01-18",@@ -71,7 +72,7 @@ defmodule Plausible.Stats.TimeTest dotest "with time:hour dimension" doassert time_labels(%{dimensions: ["time:hour"],- date_range: Date.range(~D[2022-01-17], ~D[2022-01-17])+ date_range: DateTimeRange.new!(~D[2022-01-17], ~D[2022-01-17], "UTC")}) == ["2022-01-17 00:00:00","2022-01-17 01:00:00",@@ -101,7 +102,7 @@ defmodule Plausible.Stats.TimeTest doassert time_labels(%{dimensions: ["time:hour"],- date_range: Date.range(~D[2022-01-17], ~D[2022-01-18])+ date_range: DateTimeRange.new!(~D[2022-01-17], ~D[2022-01-18], "UTC")}) == ["2022-01-17 00:00:00","2022-01-17 01:00:00",@@ -154,4 +155,50 @@ defmodule Plausible.Stats.TimeTest do]endend++ test "with time:minute dimension" do+ now = DateTime.new!(~D[2024-01-01], ~T[12:30:57], "UTC")++ # ~U[2024-01-01 12:00:57Z]+ first_dt = DateTime.shift(now, minute: -30)+ # ~U[2024-01-01 12:31:02Z]+ last_dt = DateTime.shift(now, second: 5)++ assert time_labels(%{+ dimensions: ["time:minute"],+ now: now,+ date_range: DateTimeRange.new!(first_dt, last_dt)+ }) == [+ "2024-01-01 12:00:00",+ "2024-01-01 12:01:00",+ "2024-01-01 12:02:00",+ "2024-01-01 12:03:00",+ "2024-01-01 12:04:00",+ "2024-01-01 12:05:00",+ "2024-01-01 12:06:00",+ "2024-01-01 12:07:00",+ "2024-01-01 12:08:00",+ "2024-01-01 12:09:00",+ "2024-01-01 12:10:00",+ "2024-01-01 12:11:00",+ "2024-01-01 12:12:00",+ "2024-01-01 12:13:00",+ "2024-01-01 12:14:00",+ "2024-01-01 12:15:00",+ "2024-01-01 12:16:00",+ "2024-01-01 12:17:00",+ "2024-01-01 12:18:00",+ "2024-01-01 12:19:00",+ "2024-01-01 12:20:00",+ "2024-01-01 12:21:00",+ "2024-01-01 12:22:00",+ "2024-01-01 12:23:00",+ "2024-01-01 12:24:00",+ "2024-01-01 12:25:00",+ "2024-01-01 12:26:00",+ "2024-01-01 12:27:00",+ "2024-01-01 12:28:00",+ "2024-01-01 12:29:00"+ ]+ endenddiff --git a/test/plausible/timezones_test.exs b/test/plausible/timezones_test.exsindex 890d2e2f22..06cae19b83 100644--- a/test/plausible/timezones_test.exs+++ b/test/plausible/timezones_test.exs@@ -19,15 +19,6 @@ defmodule Plausible.TimezonesTest dorefute Enum.empty?(options)end- test "to_utc_datetime/2" do- assert to_utc_datetime(~N[2022-09-11 00:00:00], "Etc/UTC") == ~U[2022-09-11 00:00:00Z]-- assert to_utc_datetime(~N[2022-09-11 00:00:00], "America/Santiago") ==- ~U[2022-09-11 00:00:00Z]-- assert to_utc_datetime(~N[2023-10-29 00:00:00], "Atlantic/Azores") == ~U[2023-10-29 01:00:00Z]- end-test "to_date_in_timezone/1" doassert to_date_in_timezone(~D[2021-01-03], "Etc/UTC") == ~D[2021-01-03]assert to_date_in_timezone(~U[2015-01-13 13:00:07Z], "Etc/UTC") == ~D[2015-01-13]diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exsindex cbfda8e256..6a74ca02af 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs@@ -2896,6 +2896,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doreferrer_source: "Google",timestamp: ~N[2021-01-02 00:00:00]),+ build(:pageview,+ referrer_source: "Google",+ timestamp: ~N[2021-01-02 00:00:00]+ ),+ build(:pageview, timestamp: ~N[2021-01-03 00:00:00]),build(:pageview, timestamp: ~N[2021-01-03 00:00:00]),build(:pageview,referrer_source: "Twitter",@@ -2912,11 +2917,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do})assert json_response(conn, 200)["results"] == [- %{"dimensions" => ["2021-01-01 00:00:00", "Google"], "metrics" => [1]},- %{"dimensions" => ["2021-01-02 00:00:00", "Google"], "metrics" => [1]},- %{"dimensions" => ["2021-01-02 00:00:00", "Direct / None"], "metrics" => [1]},- %{"dimensions" => ["2021-01-03 00:00:00", "Direct / None"], "metrics" => [1]},- %{"dimensions" => ["2021-01-03 00:00:00", "Twitter"], "metrics" => [1]}+ %{"dimensions" => ["2021-01-01", "Google"], "metrics" => [1]},+ %{"dimensions" => ["2021-01-02", "Google"], "metrics" => [2]},+ %{"dimensions" => ["2021-01-02", "Direct / None"], "metrics" => [1]},+ %{"dimensions" => ["2021-01-03", "Direct / None"], "metrics" => [2]},+ %{"dimensions" => ["2021-01-03", "Twitter"], "metrics" => [1]}]end@@ -2975,4 +2980,32 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do%{"dimensions" => ["United Kingdom", "London", "London"], "metrics" => [1]}]end++ describe "using the returned query object in a new POST request" do+ test "yields the same results for a simple aggregate query", %{conn: conn, site: site} do+ Plausible.Site.changeset(site, %{timezone: "Europe/Tallinn"})+ |> Plausible.Repo.update()++ populate_stats(site, [+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:25:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn1 =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["pageviews"],+ "date_range" => "all"+ })++ assert %{"results" => results1, "query" => query} = json_response(conn1, 200)+ assert results1 == [%{"metrics" => [3], "dimensions" => []}]++ conn2 = post(conn, "/api/v2/query", query)++ assert %{"results" => results2} = json_response(conn2, 200)+ assert results2 == [%{"metrics" => [3], "dimensions" => []}]+ end+ endenddiff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exsindex bfecd730b6..c5bfe35559 100644--- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs@@ -13,8 +13,9 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest doconn = get(conn, "/api/stats/#{site.domain}/main-graph?period=realtime&metric=pageviews")- assert %{"plot" => plot} = json_response(conn, 200)+ assert %{"plot" => plot, "labels" => labels} = json_response(conn, 200)+ assert labels == Enum.to_list(-30..-1)assert Enum.count(plot) == 30assert Enum.any?(plot, fn pageviews -> pageviews > 0 end)end@@ -1072,6 +1073,28 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do"2021-12-01" => false}end++ test "returns stats for a day with a minute interval", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2023-03-01 12:00:00])+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=day&metric=visitors&date=2023-03-01&interval=minute"+ )++ %{"labels" => labels, "plot" => plot} = json_response(conn, 200)++ assert length(labels) == 24 * 60++ assert List.first(labels) == "2023-03-01 00:00:00"+ assert Enum.at(labels, 1) == "2023-03-01 00:01:00"+ assert List.last(labels) == "2023-03-01 23:59:00"++ assert Enum.at(plot, Enum.find_index(labels, &(&1 == "2023-03-01 12:00:00"))) == 1+ endenddescribe "GET /api/stats/main-graph - comparisons" do