Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Support realtime periods in API v2 (#4469)

RobertJoonasf04c47f8818d
diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex
index 7f186e4779..22e04da220 100644
--- a/lib/plausible/google/api.ex
+++ b/lib/plausible/google/api.ex
@@ -7,6 +7,7 @@ defmodule Plausible.Google.API do
alias Plausible.Google.HTTP
alias Plausible.Google.SearchConsole
+ alias Plausible.Stats.DateTimeRange
require Logger
@@ -64,11 +65,12 @@ defmodule Plausible.Google.API do
{:ok, access_token} <- maybe_refresh_token(site.google_auth),
{:ok, gsc_filters} <-
SearchConsole.Filters.transform(site.google_auth.property, query.filters, search),
+ date_range = DateTimeRange.to_date_range(query.date_range),
{:ok, stats} <-
HTTP.list_stats(
access_token,
site.google_auth.property,
- query.date_range,
+ date_range,
pagination,
gsc_filters
) do
diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex
index 37481f0e47..15b8964e4a 100644
--- a/lib/plausible/stats/breakdown.ex
+++ b/lib/plausible/stats/breakdown.ex
@@ -117,7 +117,9 @@ defmodule Plausible.Stats.Breakdown do
from i in "imported_pages",
group_by: i.page,
where: i.site_id == ^site.id,
- where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last,
+ where:
+ i.date >= ^DateTime.to_naive(query.date_range.first) and
+ i.date <= ^DateTime.to_naive(query.date_range.last),
where: i.page in ^pages,
select: %{
page: i.page,
diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex
index 69437de694..219051e686 100644
--- a/lib/plausible/stats/clickhouse.ex
+++ b/lib/plausible/stats/clickhouse.ex
@@ -6,7 +6,6 @@ defmodule Plausible.Stats.Clickhouse do
import Ecto.Query, only: [from: 2, dynamic: 1, dynamic: 2]
- alias Plausible.Stats.Query
alias Plausible.Timezones
@spec pageview_start_date_local(Plausible.Site.t()) :: Date.t() | nil
@@ -78,7 +77,7 @@ defmodule Plausible.Stats.Clickhouse do
def top_sources_for_spike(site, query, limit, page) do
offset = (page - 1) * limit
- {first_datetime, last_datetime} = utc_boundaries(query, site)
+ {first_datetime, last_datetime} = Plausible.Stats.Time.utc_boundaries(query, site)
referrers =
from(s in "sessions_v2",
@@ -251,52 +250,4 @@ defmodule Plausible.Stats.Clickhouse do
}
end
end
-
- defp utc_boundaries(%Query{now: now, period: "30m"}, site) do
- last_datetime = now |> NaiveDateTime.truncate(:second)
-
- first_datetime =
- last_datetime
- |> NaiveDateTime.shift(minute: -30)
- |> beginning_of_time(site.native_stats_start_at)
- |> NaiveDateTime.truncate(:second)
-
- {first_datetime, last_datetime}
- end
-
- defp utc_boundaries(%Query{now: now, period: "realtime"}, site) do
- last_datetime = now |> NaiveDateTime.truncate(:second)
-
- first_datetime =
- last_datetime
- |> NaiveDateTime.shift(minute: -5)
- |> beginning_of_time(site.native_stats_start_at)
- |> NaiveDateTime.truncate(:second)
-
- {first_datetime, last_datetime}
- end
-
- defp utc_boundaries(%Query{date_range: date_range}, site) do
- {:ok, first} = NaiveDateTime.new(date_range.first, ~T[00:00:00])
-
- first_datetime =
- first
- |> Timezones.to_utc_datetime(site.timezone)
- |> beginning_of_time(site.native_stats_start_at)
-
- {:ok, last} = NaiveDateTime.new(date_range.last |> Date.shift(day: 1), ~T[00:00:00])
-
- last_datetime =
- Timezones.to_utc_datetime(last, site.timezone)
-
- {first_datetime, last_datetime}
- end
-
- defp beginning_of_time(candidate, site_creation_date) do
- if Timex.after?(site_creation_date, candidate) do
- site_creation_date
- else
- candidate
- end
- end
end
diff --git a/lib/plausible/stats/comparisons.ex b/lib/plausible/stats/comparisons.ex
index 157be97c1f..23f2572861 100644
--- a/lib/plausible/stats/comparisons.ex
+++ b/lib/plausible/stats/comparisons.ex
@@ -8,7 +8,7 @@ defmodule Plausible.Stats.Comparisons do
"""
alias Plausible.Stats
- alias Plausible.Stats.Query
+ alias Plausible.Stats.{Query, DateTimeRange}
@modes ~w(previous_period year_over_year custom)
@disallowed_periods ~w(realtime all)
@@ -21,6 +21,9 @@ defmodule Plausible.Stats.Comparisons do
@doc """
Generates a comparison query based on the source query and comparison mode.
+ Currently only historical periods are supported for comparisons (not `realtime`
+ and `30m` periods).
+
The mode parameter specifies the type of comparison and can be one of the
following:
@@ -61,52 +64,55 @@ defmodule Plausible.Stats.Comparisons do
|> Keyword.put_new(:now, Timex.now(site.timezone))
|> Keyword.put_new(:match_day_of_week?, false)
+ source_date_range = DateTimeRange.to_date_range(source_query.date_range)
+
with :ok <- validate_mode(source_query, mode),
- {:ok, comparison_query} <- do_compare(source_query, mode, opts),
- comparison_query <- maybe_include_imported(comparison_query, source_query),
- do: {:ok, comparison_query}
+ {:ok, comparison_date_range} <- get_comparison_date_range(source_date_range, mode, opts) do
+ %Date.Range{first: first, last: last} = comparison_date_range
+
+ comparison_query =
+ source_query
+ |> Query.set(date_range: DateTimeRange.new!(first, last, site.timezone))
+ |> maybe_include_imported(source_query)
+
+ {:ok, comparison_query}
+ end
end
- defp do_compare(source_query, "year_over_year", opts) do
+ defp get_comparison_date_range(source_date_range, "year_over_year", opts) do
now = Keyword.fetch!(opts, :now)
- start_date = Date.add(source_query.date_range.first, -365)
- end_date = earliest(source_query.date_range.last, now) |> Date.add(-365)
-
- range = Date.range(start_date, end_date)
+ start_date = Date.add(source_date_range.first, -365)
+ end_date = earliest(source_date_range.last, now) |> Date.add(-365)
- comparison_query =
- source_query
- |> Map.put(:date_range, range)
- |> maybe_match_day_of_week(source_query, opts)
+ comparison_date_range =
+ Date.range(start_date, end_date)
+ |> maybe_match_day_of_week(source_date_range, opts)
- {:ok, comparison_query}
+ {:ok, comparison_date_range}
end
- defp do_compare(source_query, "previous_period", opts) do
+ defp get_comparison_date_range(source_date_range, "previous_period", opts) do
now = Keyword.fetch!(opts, :now)
- last = earliest(source_query.date_range.last, now)
- diff_in_days = Date.diff(source_query.date_range.first, last) - 1
+ last = earliest(source_date_range.last, now)
+ diff_in_days = Date.diff(source_date_range.first, last) - 1
- new_first = Date.add(source_query.date_range.first, diff_in_days)
+ new_first = Date.add(source_date_range.first, diff_in_days)
new_last = Date.add(last, diff_in_days)
- range = Date.range(new_first, new_last)
-
- comparison_query =
- source_query
- |> Map.put(:date_range, range)
- |> maybe_match_day_of_week(source_query, opts)
+ comparison_date_range =
+ Date.range(new_first, new_last)
+ |> maybe_match_day_of_week(source_date_range, opts)
- {:ok, comparison_query}
+ {:ok, comparison_date_range}
end
- defp do_compare(source_query, "custom", opts) do
+ defp get_comparison_date_range(_source_date_range, "custom", opts) do
with {:ok, from} <- opts |> Keyword.fetch!(:from) |> Date.from_iso8601(),
{:ok, to} <- opts |> Keyword.fetch!(:to) |> Date.from_iso8601(),
result when result in [:eq, :lt] <- Date.compare(from, to) do
- {:ok, %Stats.Query{source_query | date_range: Date.range(from, to)}}
+ {:ok, Date.range(from, to)}
else
_error -> {:error, :invalid_dates}
end
@@ -116,24 +122,23 @@ defmodule Plausible.Stats.Comparisons do
if Date.compare(a, b) in [:eq, :lt], do: a, else: b
end
- defp maybe_match_day_of_week(comparison_query, source_query, opts) do
+ defp maybe_match_day_of_week(comparison_date_range, source_date_range, opts) do
if Keyword.fetch!(opts, :match_day_of_week?) do
- day_to_match = Date.day_of_week(source_query.date_range.first)
+ day_to_match = Date.day_of_week(source_date_range.first)
new_first =
shift_to_nearest(
day_to_match,
- comparison_query.date_range.first,
- source_query.date_range.first
+ comparison_date_range.first,
+ source_date_range.first
)
- days_shifted = Date.diff(new_first, comparison_query.date_range.first)
- new_last = Date.add(comparison_query.date_range.last, days_shifted)
+ days_shifted = Date.diff(new_first, comparison_date_range.first)
+ new_last = Date.add(comparison_date_range.last, days_shifted)
- new_range = Date.range(new_first, new_last)
- %Stats.Query{comparison_query | date_range: new_range}
+ Date.range(new_first, new_last)
else
- comparison_query
+ comparison_date_range
end
end
diff --git a/lib/plausible/stats/datetime_range.ex b/lib/plausible/stats/datetime_range.ex
new file mode 100644
index 0000000000..e8e787a71c
--- /dev/null
+++ b/lib/plausible/stats/datetime_range.ex
@@ -0,0 +1,54 @@
+defmodule Plausible.Stats.DateTimeRange do
+ @moduledoc """
+ Defines a struct similar `Date.Range`, but with `DateTime` instead of `Date`.
+
+ The structs should be created with the `new!/2` function.
+ """
+
+ @enforce_keys [:first, :last]
+ defstruct [:first, :last]
+
+ @type t() :: %__MODULE__{
+ first: %DateTime{},
+ last: %DateTime{}
+ }
+
+ @doc """
+ Creates a `DateTimeRange` struct from the given `%Date{}` structs.
+
+ The first datetime will become the first date at 00:00:00, and the last datetime
+ will become the last date at 23:59:59. Both dates will be turned into `%DateTime{}`
+ structs in the given timezone.
+ """
+ def new!(%Date{} = first, %Date{} = last, timezone) do
+ first =
+ case DateTime.new(first, ~T[00:00:00], timezone) do
+ {:ok, datetime} -> datetime
+ {:gap, _just_before, just_after} -> just_after
+ {:ambiguous, _first_datetime, second_datetime} -> second_datetime
+ end
+
+ last =
+ case DateTime.new(last, ~T[23:59:59], timezone) do
+ {:ok, datetime} -> datetime
+ {:gap, just_before, _just_after} -> just_before
+ {:ambiguous, first_datetime, _second_datetime} -> first_datetime
+ end
+
+ new!(first, last)
+ end
+
+ def new!(%DateTime{} = first, %DateTime{} = last) do
+ first = DateTime.truncate(first, :second)
+ last = DateTime.truncate(last, :second)
+
+ %__MODULE__{first: first, last: last}
+ end
+
+ def to_date_range(%__MODULE__{first: first, last: last}) do
+ first = DateTime.to_date(first)
+ last = DateTime.to_date(last)
+
+ Date.range(first, last)
+ end
+end
diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.ex
index f5f93e72f2..6ee02bd883 100644
--- a/lib/plausible/stats/filters/query_parser.ex
+++ b/lib/plausible/stats/filters/query_parser.ex
@@ -1,11 +1,7 @@
defmodule Plausible.Stats.Filters.QueryParser do
@moduledoc false
- alias Plausible.Stats.TableDecider
- alias Plausible.Stats.Filters
- alias Plausible.Stats.Query
- alias Plausible.Stats.Metrics
- alias Plausible.Stats.JSONSchema
+ alias Plausible.Stats.{TableDecider, Filters, Query, Metrics, DateTimeRange, JSONSchema}
@default_include %{
imports: false,
@@ -13,11 +9,18 @@ defmodule Plausible.Stats.Filters.QueryParser do
}
def parse(site, schema_type, params, now \\ nil) when is_map(params) do
+ {now, date} =
+ if now do
+ {now, DateTime.shift_zone!(now, site.timezone) |> DateTime.to_date()}
+ else
+ {DateTime.utc_now(:second), today(site)}
+ end
+
with :ok <- JSONSchema.validate(schema_type, params),
- {:ok, date} <- parse_date(site, Map.get(params, "date"), now),
+ {:ok, date} <- parse_date(site, Map.get(params, "date"), date),
+ {:ok, date_range} <- parse_date_range(site, Map.get(params, "date_range"), date, now),
{:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])),
{:ok, filters} <- parse_filters(Map.get(params, "filters", [])),
- {:ok, date_range} <- parse_date_range(site, Map.get(params, "date_range"), date),
{:ok, dimensions} <- parse_dimensions(Map.get(params, "dimensions", [])),
{:ok, order_by} <- parse_order_by(Map.get(params, "order_by")),
{:ok, include} <- parse_include(Map.get(params, "include", %{})),
@@ -110,84 +113,118 @@ defmodule Plausible.Stats.Filters.QueryParser do
defp parse_clauses_list(filter), do: {:error, "Invalid filter '#{i(filter)}'"}
- defp parse_date(site, nil, nil), do: {:ok, today(site)}
-
- defp parse_date(_site, date_string, nil) when is_binary(date_string) do
+ defp parse_date(_site, date_string, _date) when is_binary(date_string) do
case Date.from_iso8601(date_string) do
{:ok, date} -> {:ok, date}
_ -> {:error, "Invalid date '#{date_string}'."}
end
end
- defp parse_date(_site, _date_string, param_date), do: {:ok, param_date}
+ defp parse_date(_site, _date_string, date) do
+ {:ok, date}
+ end
+
+ defp parse_date_range(_site, date_range, _date, now) when date_range in ["realtime", "30m"] do
+ duration_minutes =
+ case date_range do
+ "realtime" -> 5
+ "30m" -> 30
+ end
+
+ first_datetime = DateTime.shift(now, minute: -duration_minutes)
+ last_datetime = DateTime.shift(now, second: 5)
- defp parse_date_range(_site, "day", date) do
- {:ok, Date.range(date, date)}
+ {:ok, DateTimeRange.new!(first_datetime, last_datetime)}
end
- defp parse_date_range(_site, "7d", last) do
- first = last |> Date.add(-6)
- {:ok, Date.range(first, last)}
+ defp parse_date_range(site, "day", date, _now) do
+ {:ok, DateTimeRange.new!(date, date, site.timezone)}
end
- defp parse_date_range(_site, "30d", last) do
- first = last |> Date.add(-30)
- {:ok, Date.range(first, last)}
+ defp parse_date_range(site, "7d", date, _now) do
+ first = date |> Date.add(-6)
+ {:ok, DateTimeRange.new!(first, date, site.timezone)}
end
- defp parse_date_range(_site, "month", today) do
- last = today |> Date.end_of_month()
+ defp parse_date_range(site, "30d", date, _now) do
+ first = date |> Date.add(-30)
+ {:ok, DateTimeRange.new!(first, date, site.timezone)}
+ end
+
+ defp parse_date_range(site, "month", date, _now) do
+ last = date |> Date.end_of_month()
first = last |> Date.beginning_of_month()
- {:ok, Date.range(first, last)}
+ {:ok, DateTimeRange.new!(first, last, site.timezone)}
end
- defp parse_date_range(_site, "6mo", today) do
- last = today |> Date.end_of_month()
+ defp parse_date_range(site, "6mo", date, _now) do
+ last = date |> Date.end_of_month()
first =
last
|> Date.shift(month: -5)
|> Date.beginning_of_month()
- {:ok, Date.range(first, last)}
+ {:ok, DateTimeRange.new!(first, last, site.timezone)}
end
- defp parse_date_range(_site, "12mo", today) do
- last = today |> Date.end_of_month()
+ defp parse_date_range(site, "12mo", date, _now) do
+ last = date |> Date.end_of_month()
first =
last
|> Date.shift(month: -11)
|> Date.beginning_of_month()
- {:ok, Date.range(first, last)}
+ {:ok, DateTimeRange.new!(first, last, site.timezone)}
end
- defp parse_date_range(_site, "year", today) do
- last = today |> Timex.end_of_year()
+ defp parse_date_range(site, "year", date, _now) do
+ last = date |> Timex.end_of_year()
first = last |> Timex.beginning_of_year()
- {:ok, Date.range(first, last)}
+ {:ok, DateTimeRange.new!(first, last, site.timezone)}
end
- defp parse_date_range(site, "all", today) do
- start_date = Plausible.Sites.stats_start_date(site) || today
+ defp parse_date_range(site, "all", date, _now) do
+ start_date = Plausible.Sites.stats_start_date(site) || date
- {:ok, Date.range(start_date, today)}
+ {:ok, DateTimeRange.new!(start_date, date, site.timezone)}
end
- defp parse_date_range(_site, [from_date_string, to_date_string], _date)
- when is_binary(from_date_string) and is_binary(to_date_string) do
- with {:ok, from_date} <- Date.from_iso8601(from_date_string),
- {:ok, to_date} <- Date.from_iso8601(to_date_string) do
- {:ok, Date.range(from_date, to_date)}
- else
- _ -> {:error, "Invalid date_range '#{i([from_date_string, to_date_string])}'."}
+ defp parse_date_range(site, [from, to], _date, _now)
+ when is_binary(from) and is_binary(to) do
+ case date_range_from_date_strings(site, from, to) do
+ {:ok, date_range} -> {:ok, date_range}
+ {:error, _} -> date_range_from_timestamps(from, to)
end
end
- defp parse_date_range(_site, unknown, _),
+ defp parse_date_range(_site, unknown, _date, _now),
do: {:error, "Invalid date_range '#{i(unknown)}'."}
+ defp date_range_from_date_strings(site, from, to) do
+ with {:ok, from_date} <- Date.from_iso8601(from),
+ {:ok, to_date} <- Date.from_iso8601(to) do
+ {:ok, DateTimeRange.new!(from_date, to_date, site.timezone)}
+ end
+ end
+
+ defp date_range_from_timestamps(from, to) do
+ with {:ok, from_datetime} <- datetime_from_timestamp(from),
+ {:ok, to_datetime} <- datetime_from_timestamp(to) do
+ {:ok, DateTimeRange.new!(from_datetime, to_datetime)}
+ else
+ _ -> {:error, "Invalid date_range '#{i([from, to])}'."}
+ end
+ end
+
+ defp datetime_from_timestamp(timestamp_string) do
+ with [timestamp, timezone] <- String.split(timestamp_string),
+ {:ok, naive_datetime} <- NaiveDateTime.from_iso8601(timestamp) do
+ DateTime.from_naive(naive_datetime, timezone)
+ end
+ end
+
defp today(site), do: DateTime.now!(site.timezone) |> DateTime.to_date()
defp parse_dimensions(dimensions) when is_list(dimensions) do
diff --git a/lib/plausible/stats/goal_suggestions.ex b/lib/plausible/stats/goal_suggestions.ex
index 4ff23bc8e6..d3192bfb81 100644
--- a/lib/plausible/stats/goal_suggestions.ex
+++ b/lib/plausible/stats/goal_suggestions.ex
@@ -61,7 +61,9 @@ defmodule Plausible.Stats.GoalSuggestions do
from(i in "imported_custom_events",
where: i.site_id == ^site.id,
where: i.import_id in ^site.complete_import_ids,
- where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last,
+ where:
+ i.date >= ^DateTime.to_naive(query.date_range.first) and
+ i.date <= ^DateTime.to_naive(query.date_range.last),
where: i.visitors > 0,
where: fragment("? ilike ?", i.name, ^matches),
where: fragment("trim(?)", i.name) != "",
diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.ex
index 5bc46d07ee..23bfebda41 100644
--- a/lib/plausible/stats/imported/base.ex
+++ b/lib/plausible/stats/imported/base.ex
@@ -6,7 +6,7 @@ defmodule Plausible.Stats.Imported.Base do
import Ecto.Query
alias Plausible.Imported
- alias Plausible.Stats.{Filters, Query, SQL}
+ alias Plausible.Stats.{Filters, Query, SQL, DateTimeRange}
@property_to_table_mappings %{
"visit:source" => "imported_sources",
@@ -71,7 +71,7 @@ defmodule Plausible.Stats.Imported.Base do
def query_imported(table, site, query) do
import_ids = site.complete_import_ids
- %{first: date_from, last: date_to} = query.date_range
+ %{first: date_from, last: date_to} = DateTimeRange.to_date_range(query.date_range)
from(i in table,
where: i.site_id == ^site.id,
diff --git a/lib/plausible/stats/imported/sql/builder.ex b/lib/plausible/stats/imported/sql/builder.ex
index 96a846121c..76a8b011d0 100644
--- a/lib/plausible/stats/imported/sql/builder.ex
+++ b/lib/plausible/stats/imported/sql/builder.ex
@@ -293,7 +293,7 @@ defmodule Plausible.Stats.Imported.SQL.Builder do
defp select_group_fields(q, "time:week", key, query) do
select_merge_as(q, [i], %{
- key => weekstart_not_before(i.date, ^query.date_range.first)
+ key => weekstart_not_before(i.date, ^DateTime.to_naive(query.date_range.first))
})
end
diff --git a/lib/plausible/stats/interval.ex b/lib/plausible/stats/interval.ex
index eb501adf75..fc3324d4c5 100644
--- a/lib/plausible/stats/interval.ex
+++ b/lib/plausible/stats/interval.ex
@@ -7,6 +7,8 @@ defmodule Plausible.Stats.Interval do
`week`, and `month`.
"""
+ alias Plausible.Stats.DateTimeRange
+
@type t() :: String.t()
@type(opt() :: {:site, Plausible.Site.t()} | {:from, Date.t()}, {:to, Date.t()})
@type opts :: list(opt())
@@ -27,18 +29,20 @@ defmodule Plausible.Stats.Interval do
"""
def default_for_period(period) do
case period do
- "realtime" -> "minute"
+ period when period in ["realtime", "30m"] -> "minute"
"day" -> "hour"
period when period in ["custom", "7d", "30d", "month"] -> "day"
period when period in ["6mo", "12mo", "year"] -> "month"
end
end
- @spec default_for_date_range(Date.Range.t()) :: t()
+ @spec default_for_date_range(DateTimeRange.t()) :: t()
@doc """
- Returns the suggested interval for the given `Date.Range` struct.
+ Returns the suggested interval for the given `DateTimeRange` struct.
"""
- def default_for_date_range(%Date.Range{first: first, last: last}) do
+ def default_for_date_range(%DateTimeRange{} = date_range) do
+ %Date.Range{first: first, last: last} = DateTimeRange.to_date_range(date_range)
+
cond do
Timex.diff(last, first, :months) > 0 ->
"month"
diff --git a/lib/plausible/stats/json_schema.ex b/lib/plausible/stats/json_schema.ex
index 08c22467a6..c1f9beedef 100644
--- a/lib/plausible/stats/json_schema.ex
+++ b/lib/plausible/stats/json_schema.ex
@@ -27,6 +27,13 @@ defmodule Plausible.Stats.JSONSchema do
|> JSONPointer.add!("#/definitions/metric/oneOf/0", %{
"const" => "time_on_page"
})
+ |> JSONPointer.add!("#/definitions/date_range/oneOf/0", %{
+ "const" => "30m"
+ })
+ |> JSONPointer.add!("#/definitions/date_range/oneOf/0", %{
+ "const" => "realtime"
+ })
+ |> JSONPointer.add!("#/properties/date", %{"type" => "string"})
|> ExJsonSchema.Schema.resolve()
def validate(schema_type, params) do
diff --git a/lib/plausible/stats/legacy/legacy_query_builder.ex b/lib/plausible/stats/legacy/legacy_query_builder.ex
index 3abd8194c5..cd87b86703 100644
--- a/lib/plausible/stats/legacy/legacy_query_builder.ex
+++ b/lib/plausible/stats/legacy/legacy_query_builder.ex
@@ -3,10 +3,10 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do
use Plausible
- alias Plausible.Stats.{Filters, Interval, Query}
+ alias Plausible.Stats.{Filters, Interval, Query, DateTimeRange}
def from(site, params, debug_metadata) do
- now = NaiveDateTime.utc_now(:second)
+ now = DateTime.utc_now(:second)
query =
Query
@@ -37,46 +37,50 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do
struct!(query, preloaded_goals: goals)
end
- defp put_period(query, site, %{"period" => "realtime"}) do
- date = today(site.timezone)
+ defp put_period(%Query{now: now} = query, _site, %{"period" => period})
+ when period in ["realtime", "30m"] do
+ duration_minutes =
+ case period do
+ "realtime" -> 5
+ "30m" -> 30
+ end
- struct!(query, period: "realtime", date_range: Date.range(date, date))
+ first_datetime = DateTime.shift(now, minute: -duration_minutes)
+ last_datetime = DateTime.shift(now, second: 5)
+
+ struct!(query, period: period, date_range: DateTimeRange.new!(first_datetime, last_datetime))
end
defp put_period(query, site, %{"period" => "day"} = params) do
date = parse_single_date(site.timezone, params)
+ datetime_range = DateTimeRange.new!(date, date, site.timezone)
- struct!(query, period: "day", date_range: Date.range(date, date))
+ struct!(query, period: "day", date_range: datetime_range)
end
defp put_period(query, site, %{"period" => "7d"} = params) do
end_date = parse_single_date(site.timezone, params)
start_date = end_date |> Date.shift(day: -6)
+ datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone)
- struct!(
- query,
- period: "7d",
- date_range: Date.range(start_date, end_date)
- )
+ struct!(query, period: "7d", date_range: datetime_range)
end
defp put_period(query, site, %{"period" => "30d"} = params) do
end_date = parse_single_date(site.timezone, params)
start_date = end_date |> Date.shift(day: -30)
+ datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone)
- struct!(query, period: "30d", date_range: Date.range(start_date, end_date))
+ struct!(query, period: "30d", date_range: datetime_range)
end
defp put_period(query, site, %{"period" => "month"} = params) do
date = parse_single_date(site.timezone, params)
-
start_date = Timex.beginning_of_month(date)
end_date = Timex.end_of_month(date)
+ datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone)
- struct!(query,
- period: "month",
- date_range: Date.range(start_date, end_date)
- )
+ struct!(query, period: "month", date_range: datetime_range)
end
defp put_period(query, site, %{"period" => "6mo"} = params) do
@@ -88,10 +92,9 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do
Date.shift(end_date, month: -5)
|> Timex.beginning_of_month()
- struct!(query,
- period: "6mo",
- date_range: Date.range(start_date, end_date)
- )
+ datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone)
+
+ struct!(query, period: "6mo", date_range: datetime_range)
end
defp put_period(query, site, %{"period" => "12mo"} = params) do
@@ -103,10 +106,9 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do
Date.shift(end_date, month: -11)
|> Timex.beginning_of_month()
- struct!(query,
- period: "12mo",
- date_range: Date.range(start_date, end_date)
- )
+ datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone)
+
+ struct!(query, period: "12mo", date_range: datetime_range)
end
defp put_period(query, site, %{"period" => "year"} = params) do
@@ -115,21 +117,17 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do
|> Timex.end_of_year()
start_date = Timex.beginning_of_year(end_date)
+ datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone)
- struct!(query,
- period: "year",
- date_range: Date.range(start_date, end_date)
- )
+ struct!(query, period: "year", date_range: datetime_range)
end
defp put_period(query, site, %{"period" => "all"}) do
- now = today(site.timezone)
- start_date = Plausible.Sites.stats_start_date(site) || now
+ today = today(site.timezone)
+ start_date = Plausible.Sites.stats_start_date(site) || today
+ datetime_range = DateTimeRange.new!(start_date, today, site.timezone)
- struct!(query,
- period: "all",
- date_range: Date.range(start_date, now)
- )
+ struct!(query, period: "all", date_range: datetime_range)
end
defp put_period(query, site, %{"period" => "custom", "from" => from, "to" => to} = params) do
@@ -141,15 +139,13 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do
put_period(query, site, new_params)
end
- defp put_period(query, _site, %{"period" => "custom", "date" => date}) do
+ defp put_period(query, site, %{"period" => "custom", "date" => date}) do
[from, to] = String.split(date, ",")
from_date = Date.from_iso8601!(String.trim(from))
to_date = Date.from_iso8601!(String.trim(to))
+ datetime_range = DateTimeRange.new!(from_date, to_date, site.timezone)
- struct!(query,
- period: "custom",
- date_range: Date.range(from_date, to_date)
- )
+ struct!(query, period: "custom", date_range: datetime_range)
end
defp put_period(query, site, params) do
diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex
index cbbf6ae31f..71f5f200e1 100644
--- a/lib/plausible/stats/query.ex
+++ b/lib/plausible/stats/query.ex
@@ -35,7 +35,7 @@ defmodule Plausible.Stats.Query do
struct!(__MODULE__, Map.to_list(query_data))
|> put_imported_opts(site, %{})
|> put_experimental_reduced_joins(site, params)
- |> struct!(v2: true, now: NaiveDateTime.utc_now(:second), debug_metadata: debug_metadata)
+ |> struct!(v2: true, now: DateTime.utc_now(:second), debug_metadata: debug_metadata)
{:ok, query}
end
@@ -142,9 +142,9 @@ defmodule Plausible.Stats.Query do
def ensure_include_imported(query, requested?) do
cond do
is_nil(query.latest_import_end_date) -> {:error, :no_imported_data}
+ query.period in ["realtime", "30m"] -> {:error, :unsupported_query}
Date.after?(query.date_range.first, query.latest_import_end_date) -> {:error, :out_of_range}
not Imported.schema_supports_query?(query) -> {:error, :unsupported_query}
- query.period == "realtime" -> {:error, :unsupported_query}
not requested? -> {:error, :not_requested}
true -> :ok
end
diff --git a/lib/plausible/stats/query_optimizer.ex b/lib/plausible/stats/query_optimizer.ex
index 5f3bf706f8..f5e9b272b2 100644
--- a/lib/plausible/stats/query_optimizer.ex
+++ b/lib/plausible/stats/query_optimizer.ex
@@ -4,7 +4,7 @@ defmodule Plausible.Stats.QueryOptimizer do
"""
use Plausible
- alias Plausible.Stats.{Query, TableDecider, Util}
+ alias Plausible.Stats.{Query, TableDecider, Util, DateTimeRange}
@doc """
This module manipulates an existing query, updating it according to business logic.
@@ -61,7 +61,7 @@ defmodule Plausible.Stats.QueryOptimizer do
defp update_group_by_time(
%Query{
- date_range: %Date.Range{first: first, last: last}
+ date_range: %DateTimeRange{first: first, last: last}
} = query
) do
dimensions =
diff --git a/lib/plausible/stats/query_result.ex b/lib/plausible/stats/query_result.ex
index 3a9ce3c562..22ecc654a7 100644
--- a/lib/plausible/stats/query_result.ex
+++ b/lib/plausible/stats/query_result.ex
@@ -7,8 +7,7 @@ defmodule Plausible.Stats.QueryResult do
produced by Jason.encode(query_result) is ordered.
"""
- alias Plausible.Stats.Util
- alias Plausible.Stats.Filters
+ alias Plausible.Stats.{Util, Filters}
defstruct results: [],
meta: %{},
@@ -32,7 +31,10 @@ defmodule Plausible.Stats.QueryResult do
Jason.OrderedObject.new(
site_id: site.domain,
metrics: query.metrics,
- date_range: [query.date_range.first, query.date_range.last],
+ date_range: [
+ to_iso_8601_with_timezone(query.date_range.first),
+ to_iso_8601_with_timezone(query.date_range.last)
+ ],
filters: query.filters,
dimensions: query.dimensions,
order_by: query.order_by |> Enum.map(&Tuple.to_list/1),
@@ -79,6 +81,15 @@ defmodule Plausible.Stats.QueryResult do
|> Enum.reject(fn {_, value} -> is_nil(value) end)
|> Enum.into(%{})
end
+
+ defp to_iso_8601_with_timezone(%DateTime{time_zone: timezone} = datetime) do
+ naive_iso8601 =
+ datetime
+ |> DateTime.to_naive()
+ |> NaiveDateTime.to_iso8601()
+
+ naive_iso8601 <> " " <> timezone
+ end
end
defimpl Jason.Encoder, for: Plausible.Stats.QueryResult do
diff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.ex
index 4638ea7fc4..cdec499d15 100644
--- a/lib/plausible/stats/sql/expression.ex
+++ b/lib/plausible/stats/sql/expression.ex
@@ -50,7 +50,7 @@ defmodule Plausible.Stats.SQL.Expression do
key =>
weekstart_not_before(
to_timezone(t.timestamp, ^query.timezone),
- ^query.date_range.first
+ ^DateTime.to_naive(query.date_range.first)
)
})
end
@@ -83,27 +83,6 @@ defmodule Plausible.Stats.SQL.Expression do
})
end
- # :NOTE: This is not exposed in Query APIv2
- def select_dimension(q, key, "time:minute", :sessions, %Query{
- period: "30m"
- }) do
- select_merge_as(q, [s], %{
- key =>
- fragment(
- "arrayJoin(range(dateDiff('minute', now(), ?), dateDiff('minute', now(), ?) + 1))",
- s.start,
- s.timestamp
- )
- })
- end
-
- # :NOTE: This is not exposed in Query APIv2
- def select_dimension(q, key, "time:minute", _table, %Query{period: "30m"}) do
- select_merge_as(q, [t], %{
- key => fragment("dateDiff('minute', now(), ?)", t.timestamp)
- })
- end
-
# :NOTE: This is not exposed in Query APIv2
def select_dimension(q, key, "time:minute", :sessions, query) do
q
diff --git a/lib/plausible/stats/sql/where_builder.ex b/lib/plausible/stats/sql/where_builder.ex
index 6cda55cf12..51a7b70904 100644
--- a/lib/plausible/stats/sql/where_builder.ex
+++ b/lib/plausible/stats/sql/where_builder.ex
@@ -48,7 +48,7 @@ defmodule Plausible.Stats.SQL.WhereBuilder do
dynamic(
[e],
- e.site_id == ^site.id and e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime
+ e.site_id == ^site.id and e.timestamp >= ^first_datetime and e.timestamp <= ^last_datetime
)
end
@@ -71,7 +71,7 @@ defmodule Plausible.Stats.SQL.WhereBuilder do
s.site_id == ^site.id and
s.start >= ^NaiveDateTime.add(first_datetime, -7, :day) and
s.timestamp >= ^first_datetime and
- s.start < ^last_datetime
+ s.start <= ^last_datetime
)
end
diff --git a/lib/plausible/stats/time.ex b/lib/plausible/stats/time.ex
index f87c40e4ed..4b9b4b6a53 100644
--- a/lib/plausible/stats/time.ex
+++ b/lib/plausible/stats/time.ex
@@ -3,52 +3,20 @@ defmodule Plausible.Stats.Time do
Collection of functions to work with time in queries.
"""
- alias Plausible.Stats.Query
- alias Plausible.Timezones
-
- def utc_boundaries(%Query{period: "realtime", now: now}, site) do
- last_datetime =
- now
- |> NaiveDateTime.shift(second: 5)
- |> NaiveDateTime.truncate(:second)
-
- first_datetime =
- now
- |> NaiveDateTime.shift(minute: -5)
- |> NaiveDateTime.truncate(:second)
- |> beginning_of_time(site.native_stats_start_at)
-
- {first_datetime, last_datetime}
- end
-
- def utc_boundaries(%Query{period: "30m", now: now}, site) do
- last_datetime =
- now
- |> NaiveDateTime.shift(second: 5)
- |> NaiveDateTime.truncate(:second)
-
- first_datetime =
- now
- |> NaiveDateTime.shift(minute: -30)
- |> NaiveDateTime.truncate(:second)
- |> beginning_of_time(site.native_stats_start_at)
-
- {first_datetime, last_datetime}
- end
+ alias Plausible.Stats.{Query, DateTimeRange}
def utc_boundaries(%Query{date_range: date_range}, site) do
- {:ok, first} = NaiveDateTime.new(date_range.first, ~T[00:00:00])
+ %DateTimeRange{first: first, last: last} = date_range
- first_datetime =
+ first =
first
- |> Timezones.to_utc_datetime(site.timezone)
+ |> DateTime.shift_zone!("Etc/UTC")
+ |> DateTime.to_naive()
|> beginning_of_time(site.native_stats_start_at)
- {:ok, last} = NaiveDateTime.new(date_range.last |> Date.shift(day: 1), ~T[00:00:00])
-
- last_datetime = Timezones.to_utc_datetime(last, site.timezone)
+ last = DateTime.shift_zone!(last, "Etc/UTC") |> DateTime.to_naive()
- {first_datetime, last_datetime}
+ {first, last}
end
defp beginning_of_time(candidate, native_stats_start_at) do
@@ -61,7 +29,7 @@ defmodule Plausible.Stats.Time do
def format_datetime(%Date{} = date), do: Date.to_string(date)
- def format_datetime(%DateTime{} = datetime),
+ def format_datetime(%mod{} = datetime) when mod in [NaiveDateTime, DateTime],
do: Calendar.strftime(datetime, "%Y-%m-%d %H:%M:%S")
# Realtime graphs return numbers
@@ -79,15 +47,17 @@ defmodule Plausible.Stats.Time do
end
defp time_labels_for_dimension("time:month", query) do
+ date_range = DateTimeRange.to_date_range(query.date_range)
+
n_buckets =
Timex.diff(
- query.date_range.last,
- Date.beginning_of_month(query.date_range.first),
+ date_range.last,
+ Date.beginning_of_month(date_range.first),
:months
)
Enum.map(n_buckets..0, fn shift ->
- query.date_range.last
+ date_range.last
|> Date.beginning_of_month()
|> Date.shift(month: -shift)
|> format_datetime()
@@ -95,15 +65,17 @@ defmodule Plausible.Stats.Time do
end
defp time_labels_for_dimension("time:week", query) do
+ date_range = DateTimeRange.to_date_range(query.date_range)
+
n_buckets =
Timex.diff(
- query.date_range.last,
- Date.beginning_of_week(query.date_range.first),
+ date_range.last,
+ Date.beginning_of_week(date_range.first),
:weeks
)
Enum.map(0..n_buckets, fn shift ->
- query.date_range.first
+ date_range.first
|> Date.shift(week: shift)
|> date_or_weekstart(query)
|> format_datetime()
@@ -112,59 +84,42 @@ defmodule Plausible.Stats.Time do
defp time_labels_for_dimension("time:day", query) do
query.date_range
+ |> DateTimeRange.to_date_range()
|> Enum.into([])
|> Enum.map(&format_datetime/1)
end
- @full_day_in_hours 23
defp time_labels_for_dimension("time:hour", query) do
- n_buckets =
- if query.date_range.first == query.date_range.last do
- @full_day_in_hours
- else
- end_time =
- query.date_range.last
- |> Timex.to_datetime()
- |> Timex.end_of_day()
-
- Timex.diff(end_time, query.date_range.first, :hours)
- end
+ n_buckets = DateTime.diff(query.date_range.last, query.date_range.first, :hour)
Enum.map(0..n_buckets, fn step ->
query.date_range.first
- |> Timex.to_datetime()
- |> DateTime.shift(hour: step)
- |> DateTime.truncate(:second)
+ |> DateTime.to_naive()
+ |> NaiveDateTime.shift(hour: step)
|> format_datetime()
end)
end
- # Only supported in dashboards not via API
- defp time_labels_for_dimension("time:minute", %Query{period: "30m"}) do
- Enum.into(-30..-1, [])
- end
-
- @full_day_in_minutes 24 * 60 - 1
defp time_labels_for_dimension("time:minute", query) do
- n_buckets =
- if query.date_range.first == query.date_range.last do
- @full_day_in_minutes
- else
- Timex.diff(query.date_range.last, query.date_range.first, :minutes)
- end
+ first_datetime = Map.put(query.date_range.first, :second, 0)
- Enum.map(0..n_buckets, fn step ->
- query.date_range.first
- |> Timex.to_datetime()
- |> DateTime.shift(minute: step)
- |> format_datetime()
+ first_datetime
+ |> Stream.iterate(fn datetime -> DateTime.shift(datetime, minute: 1) end)
+ |> Enum.take_while(fn datetime ->
+ current_minute = Map.put(query.now, :second, 0)
+
+ DateTime.before?(datetime, query.date_range.last) &&
+ DateTime.before?(datetime, current_minute)
end)
+ |> Enum.map(&format_datetime/1)
end
- defp date_or_weekstart(date, query) do
- weekstart = Timex.beginning_of_week(date)
+ def date_or_weekstart(date, query) do
+ weekstart = Date.beginning_of_week(date)
+
+ date_range = DateTimeRange.to_date_range(query.date_range)
- if Enum.member?(query.date_range, weekstart) do
+ if Enum.member?(date_range, weekstart) do
weekstart
else
date
diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex
index 8b3ddbcc38..8c6bf85125 100644
--- a/lib/plausible/stats/timeseries.ex
+++ b/lib/plausible/stats/timeseries.ex
@@ -69,6 +69,7 @@ defmodule Plausible.Stats.Timeseries do
)
|> cast_revenue_metrics_to_money(currency)
end)
+ |> transform_realtime_labels(query)
end
defp empty_row(date, metrics) do
@@ -102,6 +103,13 @@ defmodule Plausible.Stats.Timeseries do
end)
end
+ defp transform_realtime_labels(results, %Query{period: "30m"}) do
+ Enum.with_index(results)
+ |> Enum.map(fn {entry, index} -> %{entry | date: -30 + index} end)
+ end
+
+ defp transform_realtime_labels(results, _query), do: results
+
on_ee do
defp cast_revenue_metrics_to_money(results, revenue_goals) do
Plausible.Stats.Goal.Revenue.cast_revenue_metrics_to_money(results, revenue_goals)
diff --git a/lib/plausible/timezones.ex b/lib/plausible/timezones.ex
index 9aa030e3dc..bcd161ad49 100644
--- a/lib/plausible/timezones.ex
+++ b/lib/plausible/timezones.ex
@@ -6,20 +6,6 @@ defmodule Plausible.Timezones do
|> Enum.sort_by(& &1[:offset], :desc)
end
- @spec to_utc_datetime(NaiveDateTime.t(), String.t()) :: DateTime.t()
- def to_utc_datetime(naive_date_time, timezone) do
- case Timex.to_datetime(naive_date_time, timezone) do
- %DateTime{} = tz_dt ->
- Timex.Timezone.convert(tz_dt, "UTC")
-
- %Timex.AmbiguousDateTime{after: after_dt} ->
- Timex.Timezone.convert(after_dt, "UTC")
-
- {:error, {:could_not_resolve_timezone, _, _, _}} ->
- Timex.Timezone.convert(naive_date_time, "UTC")
- end
- end
-
@spec to_date_in_timezone(Date.t() | NaiveDateTime.t() | DateTime.t(), String.t()) :: Date.t()
def to_date_in_timezone(dt, timezone) do
to_datetime_in_timezone(dt, timezone) |> Timex.to_date()
diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
index 4db14ec22c..687e288255 100644
--- a/lib/plausible_web/controllers/api/stats_controller.ex
+++ b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -5,7 +5,7 @@ defmodule PlausibleWeb.Api.StatsController do
use PlausibleWeb.Plugs.ErrorHandler
alias Plausible.Stats
- alias Plausible.Stats.{Query, Comparisons}
+ alias Plausible.Stats.{Query, Comparisons, Time, DateTimeRange}
alias Plausible.Stats.Filters.LegacyDashboardFilterParser
alias PlausibleWeb.Api.Helpers, as: H
@@ -104,16 +104,10 @@ defmodule PlausibleWeb.Api.StatsController do
with {:ok, dates} <- parse_date_params(params),
:ok <- validate_interval(params),
:ok <- validate_interval_granularity(site, params, dates),
+ params <- realtime_period_to_30m(params),
query = Query.from(site, params, debug_metadata(conn)),
{:ok, metric} <- parse_and_validate_graph_metric(params, query) do
- timeseries_query =
- if query.period == "realtime" do
- %Query{query | period: "30m"}
- else
- query
- end
-
- timeseries_result = Stats.timeseries(site, timeseries_query, [metric])
+ timeseries_result = Stats.timeseries(site, query, [metric])
comparison_opts = parse_comparison_opts(params)
@@ -173,10 +167,12 @@ defmodule PlausibleWeb.Api.StatsController do
end
defp build_full_intervals(%{interval: "week", date_range: date_range}, labels) do
+ date_range = DateTimeRange.to_date_range(date_range)
build_intervals(labels, date_range, &Timex.beginning_of_week/1, &Timex.end_of_week/1)
end
defp build_full_intervals(%{interval: "month", date_range: date_range}, labels) do
+ date_range = DateTimeRange.to_date_range(date_range)
build_intervals(labels, date_range, &Timex.beginning_of_month/1, &Timex.end_of_month/1)
end
@@ -205,6 +201,8 @@ defmodule PlausibleWeb.Api.StatsController do
def top_stats(conn, params) do
site = conn.assigns[:site]
+ params = realtime_period_to_30m(params)
+
query = Query.from(site, params, debug_metadata(conn))
comparison_opts = parse_comparison_opts(params)
@@ -224,14 +222,14 @@ defmodule PlausibleWeb.Api.StatsController do
with_imported_switch: with_imported_switch_info(query, comparison_query),
includes_imported: includes_imported?(query, comparison_query),
imports_exist: site.complete_import_ids != [],
- comparing_from: comparison_query && comparison_query.date_range.first,
- comparing_to: comparison_query && comparison_query.date_range.last,
- from: query.date_range.first,
- to: query.date_range.last
+ comparing_from: comparison_query && DateTime.to_date(comparison_query.date_range.first),
+ comparing_to: comparison_query && DateTime.to_date(comparison_query.date_range.last),
+ from: DateTime.to_date(query.date_range.first),
+ to: DateTime.to_date(query.date_range.last)
})
end
- defp with_imported_switch_info(%Query{period: "realtime"}, _) do
+ defp with_imported_switch_info(%Query{period: "30m"}, _) do
%{visible: false, togglable: false, tooltip_msg: nil}
end
@@ -284,7 +282,7 @@ defmodule PlausibleWeb.Api.StatsController do
current_date =
Timex.now(site.timezone)
|> Timex.to_date()
- |> date_or_weekstart(query)
+ |> Time.date_or_weekstart(query)
|> Date.to_string()
Enum.find_index(dates, &(&1 == current_date))
@@ -307,24 +305,14 @@ defmodule PlausibleWeb.Api.StatsController do
end
end
- defp date_or_weekstart(date, query) do
- weekstart = Timex.beginning_of_week(date)
-
- if Enum.member?(query.date_range, weekstart) do
- weekstart
- else
- date
- end
- end
-
defp fetch_top_stats(site, query, comparison_query) do
goal_filter = Query.get_filter(query, "event:goal")
cond do
- query.period == "realtime" && goal_filter ->
+ query.period == "30m" && goal_filter ->
fetch_goal_realtime_top_stats(site, query, comparison_query)
- query.period == "realtime" ->
+ query.period == "30m" ->
fetch_realtime_top_stats(site, query, comparison_query)
goal_filter ->
@@ -336,12 +324,10 @@ defmodule PlausibleWeb.Api.StatsController do
end
defp fetch_goal_realtime_top_stats(site, query, _comparison_query) do
- query_30m = %Query{query | period: "30m"}
-
%{
visitors: %{value: unique_conversions},
events: %{value: total_conversions}
- } = Stats.aggregate(site, query_30m, [:visitors, :events])
+ } = Stats.aggregate(site, query, [:visitors, :events])
stats = [
%{
@@ -364,12 +350,10 @@ defmodule PlausibleWeb.Api.StatsController do
end
defp fetch_realtime_top_stats(site, query, _comparison_query) do
- query_30m = %Query{query | period: "30m"}
-
%{
visitors: %{value: visitors},
pageviews: %{value: pageviews}
- } = Stats.aggregate(site, query_30m, [:visitors, :pageviews])
+ } = Stats.aggregate(site, query, [:visitors, :pageviews])
stats = [
%{
@@ -1270,15 +1254,13 @@ defmodule PlausibleWeb.Api.StatsController do
def conversions(conn, params) do
pagination = parse_pagination(params)
site = Plausible.Repo.preload(conn.assigns.site, :goals)
- params = Map.put(params, "property", "event:goal")
- query = Query.from(site, params, debug_metadata(conn))
- query =
- if query.period == "realtime" do
- %Query{query | period: "30m"}
- else
- query
- end
+ params =
+ params
+ |> realtime_period_to_30m()
+ |> Map.put("property", "event:goal")
+
+ query = Query.from(site, params, debug_metadata(conn))
metrics = [:visitors, :events, :conversion_rate] ++ @revenue_metrics
@@ -1583,4 +1565,12 @@ defmodule PlausibleWeb.Api.StatsController do
Map.put(row, :name, name)
end
+
+ defp realtime_period_to_30m(%{"period" => _} = params) do
+ Map.update!(params, "period", fn period ->
+ if period == "realtime", do: "30m", else: period
+ end)
+ end
+
+ defp realtime_period_to_30m(params), do: params
end
diff --git a/priv/json-schemas/query-api-schema.json b/priv/json-schemas/query-api-schema.json
index 007575ae85..8e62b288e3 100644
--- a/priv/json-schemas/query-api-schema.json
+++ b/priv/json-schemas/query-api-schema.json
@@ -99,13 +99,17 @@
"type": "array",
"items": {
"type": "string",
- "pattern": "^\\d{4}-\\d{2}-\\d{2}$"
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}(?:T\\d{2}:\\d{2}:\\d{2}\\s[A-Za-z/_]+)?$"
},
- "markdownDescription": "Start and end dates of custom date range. Both dates are in format `YYYY-MM-DD`",
+ "markdownDescription": "A list of two elements to determine the query date range. Both elements should have the same format - either `YYYY-MM-DD` or `YYYY-MM-DDThh:mm:ss <timezone>`",
"examples": [
[
"2024-01-01",
"2024-01-31"
+ ],
+ [
+ "2024-01-01T00:00:00 Europe/Tallinn",
+ "2024-01-01T12:00:00 Europe/Tallinn"
]
],
"minItems": 2,
diff --git a/test/plausible/google/api_test.exs b/test/plausible/google/api_test.exs
index cd020ca5ec..ef6a96d93a 100644
--- a/test/plausible/google/api_test.exs
+++ b/test/plausible/google/api_test.exs
@@ -3,6 +3,7 @@ defmodule Plausible.Google.APITest do
use Plausible.Test.Support.HTTPMocker
alias Plausible.Google
+ alias Plausible.Stats.Query
import ExUnit.CaptureLog
import Mox
@@ -41,7 +42,8 @@ defmodule Plausible.Google.APITest do
end
)
- query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}
+ query =
+ Query.from(site, %{"period" => "custom", "from" => "2022-01-01", "to" => "2022-01-05"})
assert {:error, "google_auth_error"} = Google.API.fetch_stats(site, query, {5, 0}, "")
end
@@ -58,7 +60,8 @@ defmodule Plausible.Google.APITest do
end
)
- query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}
+ query =
+ Query.from(site, %{"period" => "custom", "from" => "2022-01-01", "to" => "2022-01-05"})
assert {:error, "some_error"} = Google.API.fetch_stats(site, query, {5, 0}, "")
end
@@ -75,7 +78,8 @@ defmodule Plausible.Google.APITest do
end
)
- query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}
+ query =
+ Query.from(site, %{"period" => "custom", "from" => "2022-01-01", "to" => "2022-01-05"})
log =
capture_log(fn ->
@@ -100,7 +104,8 @@ defmodule Plausible.Google.APITest do
expires: NaiveDateTime.add(NaiveDateTime.utc_now(), -3600)
)
- query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}
+ query =
+ Query.from(site, %{"period" => "custom", "from" => "2022-01-01", "to" => "2022-01-05"})
assert {:error, "invalid_grant"} = Google.API.fetch_stats(site, query, 5, "")
end
@@ -126,7 +131,8 @@ defmodule Plausible.Google.APITest do
test "returns name and visitor count", %{site: site} do
mock_http_with("google_search_console.json")
- query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}
+ query =
+ Query.from(site, %{"period" => "custom", "from" => "2022-01-01", "to" => "2022-01-05"})
assert {:ok,
[
diff --git a/test/plausible/stats/comparisons_test.exs b/test/plausible/stats/comparisons_test.exs
index 9f72a35f96..7d2504c6a7 100644
--- a/test/plausible/stats/comparisons_test.exs
+++ b/test/plausible/stats/comparisons_test.exs
@@ -11,8 +11,11 @@ defmodule Plausible.Stats.ComparisonsTest do
{:ok, comparison} = Comparisons.compare(site, query, "previous_period", now: now)
- assert comparison.date_range.first == ~D[2023-02-27]
- assert comparison.date_range.last == ~D[2023-02-28]
+ assert comparison.date_range.first ==
+ DateTime.new!(~D[2023-02-27], ~T[00:00:00], site.timezone)
+
+ assert comparison.date_range.last ==
+ DateTime.new!(~D[2023-02-28], ~T[23:59:59], site.timezone)
end
test "shifts back this month period when it's the first day of the month and mode is previous_period" do
@@ -22,8 +25,11 @@ defmodule Plausible.Stats.ComparisonsTest do
{:ok, comparison} = Comparisons.compare(site, query, "previous_period", now: now)
- assert comparison.date_range.first == ~D[2023-02-28]
- assert comparison.date_range.last == ~D[2023-02-28]
+ assert comparison.date_range.first ==
+ DateTime.new!(~D[2023-02-28], ~T[00:00:00], site.timezone)
+
+ assert comparison.date_range.last ==
+ DateTime.new!(~D[2023-02-28], ~T[23:59:59], site.timezone)
end
test "matches the day of the week when nearest day is original query start date and mode is previous_period" do
@@ -34,8 +40,11 @@ defmodule Plausible.Stats.ComparisonsTest do
{:ok, comparison} =
Comparisons.compare(site, query, "previous_period", now: now, match_day_of_week?: true)
- assert comparison.date_range.first == ~D[2023-02-22]
- assert comparison.date_range.last == ~D[2023-02-23]
+ assert comparison.date_range.first ==
+ DateTime.new!(~D[2023-02-22], ~T[00:00:00], site.timezone)
+
+ assert comparison.date_range.last ==
+ DateTime.new!(~D[2023-02-23], ~T[23:59:59], site.timezone)
end
end
@@ -47,8 +56,11 @@ defmodule Plausible.Stats.ComparisonsTest do
{:ok, comparison} = Comparisons.compare(site, query, "previous_period", now: now)
- assert comparison.date_range.first == ~D[2023-01-04]
- assert comparison.date_range.last == ~D[2023-01-31]
+ assert comparison.date_range.first ==
+ DateTime.new!(~D[2023-01-04], ~T[00:00:00], site.timezone)
+
+ assert comparison.date_range.last ==
+ DateTime.new!(~D[2023-01-31], ~T[23:59:59], site.timezone)
end
test "shifts back the full month when mode is year_over_year" do
@@ -58,8 +70,11 @@ defmodule Plausible.Stats.ComparisonsTest do
{:ok, comparison} = Comparisons.compare(site, query, "year_over_year", now: now)
- assert comparison.date_range.first == ~D[2022-02-01]
- assert comparison.date_range.last == ~D[2022-02-28]
+ assert comparison.date_range.first ==
+ DateTime.new!(~D[2022-02-01], ~T[00:00:00], site.timezone)
+
+ assert comparison.date_range.last ==
+ DateTime.new!(~D[2022-02-28], ~T[23:59:59], site.timezone)
end
test "shifts back whole month plus one day when mode is year_over_year and a leap year" do
@@ -69,8 +84,11 @@ defmodule Plausible.Stats.ComparisonsTest do
{:ok, comparison} = Comparisons.compare(site, query, "year_over_year", now: now)
- assert comparison.date_range.first == ~D[2019-02-01]
- assert comparison.date_range.last == ~D[2019-03-01]
+ assert comparison.date_range.first ==
+ DateTime.new!(~D[2019-02-01], ~T[00:00:00], site.timezone)
+
+ assert comparison.date_range.last ==
+ DateTime.new!(~D[2019-03-01], ~T[23:59:59], site.timezone)
end
test "matches the day of the week when mode is previous_period keeping the same day" do
@@ -81,8 +99,11 @@ defmodule Plausible.Stats.ComparisonsTest do
{:ok, comparison} =
Comparisons.compare(site, query, "previous_period", now: now, match_day_of_week?: true)
- assert comparison.date_range.first == ~D[2023-01-04]
- assert comparison.date_range.last == ~D[2023-01-31]
+ assert comparison.date_range.first ==
+ DateTime.new!(~D[2023-01-04], ~T[00:00:00], site.timezone)
+
+ assert comparison.date_range.last ==
+ DateTime.new!(~D[2023-01-31], ~T[23:59:59], site.timezone)
end
test "matches the day of the week when mode is previous_period" do
@@ -93,8 +114,11 @@ defmodule Plausible.Stats.ComparisonsTest do
{:ok, comparison} =
Comparisons.compare(site, query, "previous_period", now: now, match_day_of_week?: true)
- assert comparison.date_range.first == ~D[2022-12-04]
- assert comparison.date_range.last == ~D[2023-01-03]
+ assert comparison.date_range.first ==
+ DateTime.new!(~D[2022-12-04], ~T[00:00:00], site.timezone)
+
+ assert comparison.date_range.last ==
+ DateTime.new!(~D[2023-01-03], ~T[23:59:59], site.timezone)
end
end
@@ -106,8 +130,11 @@ defmodule Plausible.Stats.ComparisonsTest do
{:ok, comparison} = Comparisons.compare(site, query, "previous_period", now: now)
- assert comparison.date_range.first == ~D[2022-11-02]
- assert comparison.date_range.last == ~D[2022-12-31]
+ assert comparison.date_range.first ==
+ DateTime.new!(~D[2022-11-02], ~T[00:00:00], site.timezone)
+
+ assert comparison.date_range.last ==
+ DateTime.new!(~D[2022-12-31], ~T[23:59:59], site.timezone)
end
test "shifts back by the same number of days when mode is year_over_year" do
@@ -117,8 +144,11 @@ defmodule Plausible.Stats.ComparisonsTest do
{:ok, comparison} = Comparisons.compare(site, query, "year_over_year", now: now)
- assert comparison.date_range.first == ~D[2022-01-01]
- assert comparison.date_range.last == ~D[2022-03-01]
+ assert comparison.date_range.first ==
+ DateTime.new!(~D[2022-01-01], ~T[00:00:00], site.timezone)
+
+ assert comparison.date_range.last ==
+ DateTime.new!(~D[2022-03-01], ~T[23:59:59], site.timezone)
end
test "matches the day of the week when mode is year_over_year" do
@@ -129,8 +159,11 @@ defmodule Plausible.Stats.ComparisonsTest do
{:ok, comparison} =
Comparisons.compare(site, query, "year_over_year", now: now, match_day_of_week?: true)
- assert comparison.date_range.first == ~D[2022-01-02]
- assert comparison.date_range.last == ~D[2022-03-02]
+ assert comparison.date_range.first ==
+ DateTime.new!(~D[2022-01-02], ~T[00:00:00], site.timezone)
+
+ assert comparison.date_range.last ==
+ DateTime.new!(~D[2022-03-02], ~T[23:59:59], site.timezone)
end
end
@@ -141,8 +174,11 @@ defmodule Plausible.Stats.ComparisonsTest do
{:ok, comparison} = Comparisons.compare(site, query, "year_over_year")
- assert comparison.date_range.first == ~D[2021-01-01]
- assert comparison.date_range.last == ~D[2021-12-31]
+ assert comparison.date_range.first ==
+ DateTime.new!(~D[2021-01-01], ~T[00:00:00], site.timezone)
+
+ assert comparison.date_range.last ==
+ DateTime.new!(~D[2021-12-31], ~T[23:59:59], site.timezone)
end
test "shifts back a whole year when mode is previous_period" do
@@ -151,8 +187,11 @@ defmodule Plausible.Stats.ComparisonsTest do
{:ok, comparison} = Comparisons.compare(site, query, "previous_period")
- assert comparison.date_range.first == ~D[2021-01-01]
- assert comparison.date_range.last == ~D[2021-12-31]
+ assert comparison.date_range.first ==
+ DateTime.new!(~D[2021-01-01], ~T[00:00:00], site.timezone)
+
+ assert comparison.date_range.last ==
+ DateTime.new!(~D[2021-12-31], ~T[23:59:59], site.timezone)
end
end
@@ -163,8 +202,11 @@ defmodule Plausible.Stats.ComparisonsTest do
{:ok, comparison} = Comparisons.compare(site, query, "previous_period")
- assert comparison.date_range.first == ~D[2022-12-25]
- assert comparison.date_range.last == ~D[2022-12-31]
+ assert comparison.date_range.first ==
+ DateTime.new!(~D[2022-12-25], ~T[00:00:00], site.timezone)
+
+ assert comparison.date_range.last ==
+ DateTime.new!(~D[2022-12-31], ~T[23:59:59], site.timezone)
end
test "shifts back to last year when mode is year_over_year" do
@@ -173,8 +215,11 @@ defmodule Plausible.Stats.ComparisonsTest do
{:ok, comparison} = Comparisons.compare(site, query, "year_over_year")
- assert comparison.date_range.first == ~D[2022-01-01]
- assert comparison.date_range.last == ~D[2022-01-07]
+ assert comparison.date_range.first ==
+ DateTime.new!(~D[2022-01-01], ~T[00:00:00], site.timezone)
+
+ assert comparison.date_range.last ==
+ DateTime.new!(~D[2022-01-07], ~T[23:59:59], site.timezone)
end
end
@@ -186,8 +231,11 @@ defmodule Plausible.Stats.ComparisonsTest do
{:ok, comparison} =
Comparisons.compare(site, query, "custom", from: "2022-05-25", to: "2022-05-30")
- assert comparison.date_range.first == ~D[2022-05-25]
- assert comparison.date_range.last == ~D[2022-05-30]
+ assert comparison.date_range.first ==
+ DateTime.new!(~D[2022-05-25], ~T[00:00:00], site.timezone)
+
+ assert comparison.date_range.last ==
+ DateTime.new!(~D[2022-05-30], ~T[23:59:59], site.timezone)
end
test "validates from and to dates" do
diff --git a/test/plausible/stats/interval_test.exs b/test/plausible/stats/interval_test.exs
index c18771edf3..1ec223ed3d 100644
--- a/test/plausible/stats/interval_test.exs
+++ b/test/plausible/stats/interval_test.exs
@@ -2,6 +2,7 @@ defmodule Plausible.Stats.IntervalTest do
use Plausible.DataCase, async: true
import Plausible.Stats.Interval
+ alias Plausible.Stats.DateTimeRange
test "default_for_period/1" do
assert default_for_period("realtime") == "minute"
@@ -11,9 +12,13 @@ defmodule Plausible.Stats.IntervalTest do
end
test "default_for_date_range/1" do
- assert default_for_date_range(Date.range(~D[2022-01-01], ~D[2023-01-01])) == "month"
- assert default_for_date_range(Date.range(~D[2022-01-01], ~D[2022-01-15])) == "day"
- assert default_for_date_range(Date.range(~D[2022-01-01], ~D[2022-01-01])) == "hour"
+ year = DateTimeRange.new!(~D[2022-01-01], ~D[2023-01-01], "UTC")
+ fifteen_days = DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-15], "UTC")
+ day = DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-01], "UTC")
+
+ assert default_for_date_range(year) == "month"
+ assert default_for_date_range(fifteen_days) == "day"
+ assert default_for_date_range(day) == "hour"
end
describe "valid_by_period/1" do
diff --git a/test/plausible/stats/query_optimizer_test.exs b/test/plausible/stats/query_optimizer_test.exs
index d35b958bcd..64d526edda 100644
--- a/test/plausible/stats/query_optimizer_test.exs
+++ b/test/plausible/stats/query_optimizer_test.exs
@@ -1,7 +1,7 @@
defmodule Plausible.Stats.QueryOptimizerTest do
use Plausible.DataCase, async: true
- alias Plausible.Stats.{Query, QueryOptimizer}
+ alias Plausible.Stats.{Query, QueryOptimizer, DateTimeRange}
@default_params %{metrics: [:visitors]}
@@ -24,7 +24,7 @@ defmodule Plausible.Stats.QueryOptimizerTest do
test "adds time and first metric to order_by if order_by not specified" do
assert perform(%{
- date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-02-01 00:00:00]),
+ date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-31], "UTC"),
metrics: [:pageviews, :visitors],
dimensions: ["time", "event:page"]
}).order_by ==
@@ -35,69 +35,73 @@ defmodule Plausible.Stats.QueryOptimizerTest do
describe "update_group_by_time" do
test "does nothing if `time` dimension not passed" do
assert perform(%{
- date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-05 00:00:00]),
+ date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-04], "UTC"),
dimensions: ["time:month"]
}).dimensions == ["time:month"]
end
test "updating time dimension" do
assert perform(%{
- date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]),
+ date_range:
+ DateTimeRange.new!(
+ DateTime.new!(~D[2022-01-01], ~T[00:00:00], "UTC"),
+ DateTime.new!(~D[2022-01-01], ~T[05:00:00], "UTC")
+ ),
dimensions: ["time"]
}).dimensions == ["time:hour"]
assert perform(%{
- date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-02 00:00:00]),
+ date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-01], "UTC"),
dimensions: ["time"]
}).dimensions == ["time:hour"]
assert perform(%{
- date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-02 16:00:00]),
+ date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-02], "UTC"),
dimensions: ["time"]
}).dimensions == ["time:hour"]
assert perform(%{
- date_range: Date.range(~D[2022-01-01], ~D[2022-01-04]),
+ date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-03], "UTC"),
dimensions: ["time"]
}).dimensions == ["time:day"]
assert perform(%{
- date_range: Date.range(~D[2022-01-01], ~D[2022-01-10]),
+ date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-10], "UTC"),
dimensions: ["time"]
}).dimensions == ["time:day"]
assert perform(%{
- date_range: Date.range(~D[2022-01-01], ~D[2022-01-16]),
+ date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-16], "UTC"),
dimensions: ["time"]
}).dimensions == ["time:day"]
assert perform(%{
- date_range: Date.range(~D[2022-01-01], ~D[2022-02-16]),
+ date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-02-16], "UTC"),
dimensions: ["time"]
}).dimensions == ["time:week"]
assert perform(%{
- date_range: Date.range(~D[2022-01-01], ~D[2022-03-16]),
+ date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-03-16], "UTC"),
dimensions: ["time"]
}).dimensions == ["time:week"]
assert perform(%{
- date_range: Date.range(~D[2022-01-01], ~D[2022-03-16]),
+ date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-03-16], "UTC"),
dimensions: ["time"]
}).dimensions == ["time:week"]
assert perform(%{
- date_range: Date.range(~D[2022-01-01], ~D[2023-11-16]),
+ date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2023-11-16], "UTC"),
dimensions: ["time"]
}).dimensions == ["time:month"]
assert perform(%{
- date_range: Date.range(~D[2022-01-01], ~D[2024-01-16]),
+ date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2024-01-16], "UTC"),
dimensions: ["time"]
}).dimensions == ["time:month"]
assert perform(%{
- date_range: Date.range(~D[2022-01-01], ~D[2026-01-01]),
+ date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2026-01-01], "UTC"),
dimensions: ["time"]
}).dimensions == ["time:month"]
end
@@ -106,7 +110,11 @@ defmodule Plausible.Stats.QueryOptimizerTest do
describe "update_time_in_order_by" do
test "updates explicit time dimension in order_by" do
assert perform(%{
- date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]),
+ date_range:
+ DateTimeRange.new!(
+ DateTime.new!(~D[2022-01-01], ~T[00:00:00], "UTC"),
+ DateTime.new!(~D[2022-01-01], ~T[05:00:00], "UTC")
+ ),
dimensions: ["time:hour"],
order_by: [{"time", :asc}]
}).order_by == [{"time:hour", :asc}]
diff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exs
index 062abe1cd2..64775fa190 100644
--- a/test/plausible/stats/query_parser_test.exs
+++ b/test/plausible/stats/query_parser_test.exs
@@ -1,41 +1,77 @@
defmodule Plausible.Stats.Filters.QueryParserTest do
use Plausible.DataCase
+ alias Plausible.Stats.DateTimeRange
alias Plausible.Stats.Filters
import Plausible.Stats.Filters.QueryParser
setup [:create_user, :create_new_site]
- @today ~D[2021-05-05]
- @date_range Date.range(@today, @today)
-
- def check_success(params, site, expected_result),
- do: check_success(params, site, :public, expected_result)
-
- def check_success(params, site, schema_type, expected_result) do
- assert {:ok, ^expected_result} = parse(site, schema_type, params, @today)
+ @now DateTime.new!(~D[2021-05-05], ~T[12:30:00], "UTC")
+ @date_range_realtime %DateTimeRange{
+ first: DateTime.new!(~D[2021-05-05], ~T[12:25:00], "UTC"),
+ last: DateTime.new!(~D[2021-05-05], ~T[12:30:05], "UTC")
+ }
+ @date_range_30m %DateTimeRange{
+ first: DateTime.new!(~D[2021-05-05], ~T[12:00:00], "UTC"),
+ last: DateTime.new!(~D[2021-05-05], ~T[12:30:05], "UTC")
+ }
+ @date_range_day %DateTimeRange{
+ first: DateTime.new!(~D[2021-05-05], ~T[00:00:00], "UTC"),
+ last: DateTime.new!(~D[2021-05-05], ~T[23:59:59], "UTC")
+ }
+ @date_range_7d %DateTimeRange{
+ first: DateTime.new!(~D[2021-04-29], ~T[00:00:00], "UTC"),
+ last: DateTime.new!(~D[2021-05-05], ~T[23:59:59], "UTC")
+ }
+ @date_range_30d %DateTimeRange{
+ first: DateTime.new!(~D[2021-04-05], ~T[00:00:00], "UTC"),
+ last: DateTime.new!(~D[2021-05-05], ~T[23:59:59], "UTC")
+ }
+ @date_range_month %DateTimeRange{
+ first: DateTime.new!(~D[2021-05-01], ~T[00:00:00], "UTC"),
+ last: DateTime.new!(~D[2021-05-31], ~T[23:59:59], "UTC")
+ }
+ @date_range_6mo %DateTimeRange{
+ first: DateTime.new!(~D[2020-12-01], ~T[00:00:00], "UTC"),
+ last: DateTime.new!(~D[2021-05-31], ~T[23:59:59], "UTC")
+ }
+ @date_range_year %DateTimeRange{
+ first: DateTime.new!(~D[2021-01-01], ~T[00:00:00], "UTC"),
+ last: DateTime.new!(~D[2021-12-31], ~T[23:59:59], "UTC")
+ }
+ @date_range_12mo %DateTimeRange{
+ first: DateTime.new!(~D[2020-06-01], ~T[00:00:00], "UTC"),
+ last: DateTime.new!(~D[2021-05-31], ~T[23:59:59], "UTC")
+ }
+
+ def check_success(params, site, expected_result, schema_type \\ :public) do
+ assert {:ok, result} = parse(site, schema_type, params, @now)
+ assert result == expected_result
end
- def check_error(params, site, expected_error_message),
- do: check_error(params, site, :public, expected_error_message)
-
- def check_error(params, site, schema_type, expected_error_message) do
- {:error, message} = parse(site, schema_type, params, @today)
+ def check_error(params, site, expected_error_message, schema_type \\ :public) do
+ {:error, message} = parse(site, schema_type, params, @now)
assert message == expected_error_message
end
- def check_date_range(date_range, site, expected_date_range) do
- %{"site_id" => site.domain, "metrics" => ["visitors", "events"], "date_range" => date_range}
- |> check_success(site, %{
- metrics: [:visitors, :events],
- date_range: expected_date_range,
- filters: [],
- dimensions: [],
- order_by: nil,
- timezone: site.timezone,
- include: %{imports: false, time_labels: false},
- preloaded_goals: []
- })
+ def check_date_range(date_params, site, expected_date_range, schema_type \\ :public) do
+ %{"site_id" => site.domain, "metrics" => ["visitors", "events"]}
+ |> Map.merge(date_params)
+ |> check_success(
+ site,
+ %{
+ metrics: [:visitors, :events],
+ date_range: expected_date_range,
+ filters: [],
+ dimensions: [],
+ order_by: nil,
+ timezone: site.timezone,
+ include: %{imports: false, time_labels: false},
+ preloaded_goals: []
+ },
+ schema_type
+ )
end
test "parsing empty map fails", %{site: site} do
@@ -48,7 +84,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
%{"site_id" => site.domain, "metrics" => ["visitors", "events"], "date_range" => "all"}
|> check_success(site, %{
metrics: [:visitors, :events],
- date_range: @date_range,
+ date_range: @date_range_day,
filters: [],
dimensions: [],
order_by: nil,
@@ -77,24 +113,28 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
],
"date_range" => "all"
}
- |> check_success(site, :internal, %{
- metrics: [
- :time_on_page,
- :visitors,
- :pageviews,
- :visits,
- :events,
- :bounce_rate,
- :visit_duration
- ],
- date_range: @date_range,
- filters: [],
- dimensions: [],
- order_by: nil,
- timezone: site.timezone,
- include: %{imports: false, time_labels: false},
- preloaded_goals: []
- })
+ |> check_success(
+ site,
+ %{
+ metrics: [
+ :time_on_page,
+ :visitors,
+ :pageviews,
+ :visits,
+ :events,
+ :bounce_rate,
+ :visit_duration
+ ],
+ date_range: @date_range_day,
+ filters: [],
+ dimensions: [],
+ order_by: nil,
+ timezone: site.timezone,
+ include: %{imports: false, time_labels: false},
+ preloaded_goals: []
+ },
+ :internal
+ )
end
test "time_on_page is not a valid metric in public API", %{site: site} do
@@ -103,7 +143,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
"metrics" => ["time_on_page"],
"date_range" => "all"
}
- |> check_error(site, :public, "#/metrics/0: Invalid metric \"time_on_page\"")
+ |> check_error(site, "#/metrics/0: Invalid metric \"time_on_page\"")
end
test "same metric queried multiple times", %{site: site} do
@@ -136,18 +176,22 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
[Atom.to_string(unquote(operation)), "event:name", ["foo"]]
]
}
- |> check_success(site, :internal, %{
- metrics: [:visitors],
- date_range: @date_range,
- filters: [
- [unquote(operation), "event:name", ["foo"]]
- ],
- dimensions: [],
- order_by: nil,
- timezone: site.timezone,
- include: %{imports: false, time_labels: false},
- preloaded_goals: []
- })
+ |> check_success(
+ site,
+ %{
+ metrics: [:visitors],
+ date_range: @date_range_day,
+ filters: [
+ [unquote(operation), "event:name", ["foo"]]
+ ],
+ dimensions: [],
+ order_by: nil,
+ timezone: site.timezone,
+ include: %{imports: false, time_labels: false},
+ preloaded_goals: []
+ },
+ :internal
+ )
end
test "#{operation} filter with invalid clause", %{site: site} do
@@ -161,8 +205,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
}
|> check_error(
site,
- :internal,
- "#/filters/0: Invalid filter [\"#{unquote(operation)}\", \"event:name\", \"foo\"]"
+ "#/filters/0: Invalid filter [\"#{unquote(operation)}\", \"event:name\", \"foo\"]",
+ :internal
)
end
end
@@ -179,7 +223,6 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
}
|> check_error(
site,
- :public,
"#/filters/0: Invalid filter [\"#{unquote(operation)}\", \"event:name\", [\"foo\"]]"
)
end
@@ -208,7 +251,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
}
|> check_success(site, %{
metrics: [:visitors],
- date_range: @date_range,
+ date_range: @date_range_day,
filters: [
[:is, "event:props:foobar", ["value"]]
],
@@ -233,7 +276,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
}
|> check_success(site, %{
metrics: [:visitors],
- date_range: @date_range,
+ date_range: @date_range_day,
filters: [
[:is, "event:#{unquote(dimension)}", ["foo"]]
],
@@ -259,7 +302,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
}
|> check_success(site, %{
metrics: [:visitors],
- date_range: @date_range,
+ date_range: @date_range_day,
filters: [
[:is, "visit:#{unquote(dimension)}", ["ab"]]
],
@@ -325,7 +368,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
}
|> check_success(site, %{
metrics: [:visitors],
- date_range: @date_range,
+ date_range: @date_range_day,
filters: [
[:is, "visit:city", [123, 456]]
],
@@ -344,7 +387,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
}
|> check_success(site, %{
metrics: [:visitors],
- date_range: @date_range,
+ date_range: @date_range_day,
filters: [
[:is, "visit:city", ["123", "456"]]
],
@@ -381,7 +424,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
}
|> check_success(site, %{
metrics: [:visitors],
- date_range: @date_range,
+ date_range: @date_range_day,
filters: [],
dimensions: ["time"],
order_by: nil,
@@ -426,12 +469,12 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
]
}
- assert {:ok, res} = parse(site, :public, params, @today)
+ assert {:ok, res} = parse(site, :public, params, @now)
expected_timezone = site.timezone
assert %{
metrics: [:visitors],
- date_range: @date_range,
+ date_range: @date_range_day,
filters: [
[:is, "event:goal", ["Signup", "Visit /thank-you"]]
],
@@ -479,41 +522,208 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
describe "date range validation" do
test "parsing shortcut options", %{site: site} do
- check_date_range("day", site, Date.range(~D[2021-05-05], ~D[2021-05-05]))
- check_date_range("7d", site, Date.range(~D[2021-04-29], ~D[2021-05-05]))
- check_date_range("30d", site, Date.range(~D[2021-04-05], ~D[2021-05-05]))
- check_date_range("month", site, Date.range(~D[2021-05-01], ~D[2021-05-31]))
- check_date_range("6mo", site, Date.range(~D[2020-12-01], ~D[2021-05-31]))
- check_date_range("12mo", site, Date.range(~D[2020-06-01], ~D[2021-05-31]))
- check_date_range("year", site, Date.range(~D[2021-01-01], ~D[2021-12-31]))
+ check_date_range(%{"date_range" => "day"}, site, @date_range_day)
+ check_date_range(%{"date_range" => "7d"}, site, @date_range_7d)
+ check_date_range(%{"date_range" => "30d"}, site, @date_range_30d)
+ check_date_range(%{"date_range" => "month"}, site, @date_range_month)
+ check_date_range(%{"date_range" => "6mo"}, site, @date_range_6mo)
+ check_date_range(%{"date_range" => "12mo"}, site, @date_range_12mo)
+ check_date_range(%{"date_range" => "year"}, site, @date_range_year)
+ end
+
+ test "30m and realtime are available in internal API", %{site: site} do
+ check_date_range(%{"date_range" => "30m"}, site, @date_range_30m, :internal)
+ check_date_range(%{"date_range" => "realtime"}, site, @date_range_realtime, :internal)
+ end
+
+ test "30m and realtime date_ranges are unavailable in public API", %{
+ site: site
+ } do
+ for date_range <- ["realtime", "30m"] do
+ %{"site_id" => site.domain, "metrics" => ["visitors"], "date_range" => date_range}
+ |> check_error(site, "#/date_range: Invalid date range \"#{date_range}\"")
+ end
end
test "parsing `all` with previous data", %{site: site} do
site = Map.put(site, :stats_start_date, ~D[2020-01-01])
- check_date_range("all", site, Date.range(~D[2020-01-01], ~D[2021-05-05]))
+ expected_date_range = DateTimeRange.new!(~D[2020-01-01], ~D[2021-05-05], "UTC")
+ check_date_range(%{"date_range" => "all"}, site, expected_date_range)
end
test "parsing `all` with no previous data", %{site: site} do
site = Map.put(site, :stats_start_date, nil)
+ check_date_range(%{"date_range" => "all"}, site, @date_range_day)
+ end
- check_date_range("all", site, Date.range(~D[2021-05-05], ~D[2021-05-05]))
+ test "parsing custom date range from simple date strings", %{site: site} do
+ check_date_range(%{"date_range" => ["2021-05-05", "2021-05-05"]}, site, @date_range_day)
end
- test "parsing custom date range", %{site: site} do
+ test "parsing custom date range from iso8601 timestamps", %{site: site} do
check_date_range(
- ["2021-05-05", "2021-05-05"],
+ %{"date_range" => ["2024-01-01T00:00:00 UTC", "2024-01-02T23:59:59 UTC"]},
+ site,
+ DateTimeRange.new!(
+ DateTime.new!(~D[2024-01-01], ~T[00:00:00], "UTC"),
+ DateTime.new!(~D[2024-01-02], ~T[23:59:59], "UTC")
+ )
+ )
+
+ check_date_range(
+ %{
+ "date_range" => [
+ "2024-08-29T07:12:34 America/Los_Angeles",
+ "2024-08-29T10:12:34 America/Los_Angeles"
+ ]
+ },
site,
- Date.range(~D[2021-05-05], ~D[2021-05-05])
+ DateTimeRange.new!(
+ DateTime.new!(~D[2024-08-29], ~T[07:12:34], "America/Los_Angeles"),
+ DateTime.new!(~D[2024-08-29], ~T[10:12:34], "America/Los_Angeles")
+ )
)
end
- test "parsing invalid custom date range", %{site: site} do
+ test "parsing invalid custom date range with invalid dates", %{site: site} do
%{"site_id" => site.domain, "date_range" => "foo", "metrics" => ["visitors"]}
|> check_error(site, "#/date_range: Invalid date range \"foo\"")
%{"site_id" => site.domain, "date_range" => ["21415-00", "eee"], "metrics" => ["visitors"]}
|> check_error(site, "#/date_range: Invalid date range [\"21415-00\", \"eee\"]")
end
+
+ test "custom date range is invalid when timestamps do not include timezone info", %{
+ site: site
+ } do
+ %{
+ "site_id" => site.domain,
+ "date_range" => ["2021-02-03T00:00:00", "2021-02-03T23:59:59"],
+ "metrics" => ["visitors"]
+ }
+ |> check_error(
+ site,
+ "#/date_range: Invalid date range [\"2021-02-03T00:00:00\", \"2021-02-03T23:59:59\"]"
+ )
+ end
+
+ test "custom date range is invalid when timestamp timezone is invalid", %{site: site} do
+ %{
+ "site_id" => site.domain,
+ "date_range" => ["2021-02-03T00:00:00 Fake/Timezone", "2021-02-03T23:59:59 UTC"],
+ "metrics" => ["visitors"]
+ }
+ |> check_error(
+ site,
+ "Invalid date_range '[\"2021-02-03T00:00:00 Fake/Timezone\", \"2021-02-03T23:59:59 UTC\"]'."
+ )
+ end
+
+ test "custom date range is invalid when date and timestamp are combined", %{site: site} do
+ %{
+ "site_id" => site.domain,
+ "date_range" => ["2021-02-03T00:00:00 UTC", "2021-02-04"],
+ "metrics" => ["visitors"]
+ }
+ |> check_error(
+ site,
+ "Invalid date_range '[\"2021-02-03T00:00:00 UTC\", \"2021-02-04\"]'."
+ )
+ end
+
+ test "custom date range is invalid when timestamp cannot be converted to datetime due to a gap in timezone",
+ %{site: site} do
+ %{
+ "site_id" => site.domain,
+ "date_range" => [
+ "2024-03-31T03:30:00 Europe/Tallinn",
+ "2024-04-15T10:00:00 Europe/Tallinn"
+ ],
+ "metrics" => ["visitors"]
+ }
+ |> check_error(
+ site,
+ "Invalid date_range '[\"2024-03-31T03:30:00 Europe/Tallinn\", \"2024-04-15T10:00:00 Europe/Tallinn\"]'."
+ )
+ end
+
+ test "parses date_range relative to date param", %{site: site} do
+ date = @now |> DateTime.to_date() |> Date.to_string()
+
+ for {date_range_shortcut, expected_date_range} <- [
+ {"day", @date_range_day},
+ {"7d", @date_range_7d},
+ {"30d", @date_range_30d},
+ {"month", @date_range_month},
+ {"6mo", @date_range_6mo},
+ {"12mo", @date_range_12mo},
+ {"year", @date_range_year}
+ ] do
+ %{"date_range" => date_range_shortcut, "date" => date}
+ |> check_date_range(site, expected_date_range, :internal)
+ end
+ end
+
+ test "date parameter is not available in the public API", %{site: site} do
+ %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors", "events"],
+ "date_range" => "month",
+ "date" => "2021-05-05"
+ }
+ |> check_error(site, "#/date: Schema does not allow additional properties.")
+ end
+
+ test "parses date_range.first into a datetime right after the gap in site.timezone", %{
+ site: site
+ } do
+ site = %{site | timezone: "America/Santiago"}
+
+ expected_date_range =
+ DateTimeRange.new!(
+ DateTime.new!(~D[2022-09-11], ~T[01:00:00], site.timezone),
+ DateTime.new!(~D[2022-09-11], ~T[23:59:59], site.timezone)
+ )
+
+ %{"date_range" => ["2022-09-11", "2022-09-11"]}
+ |> check_date_range(site, expected_date_range)
+ end
+
+ test "parses date_range.first into the latest of ambiguous datetimes in site.timezone", %{
+ site: site
+ } do
+ site = %{site | timezone: "America/Havana"}
+
+ {:ambiguous, _, expected_first_datetime} =
+ DateTime.new(~D[2023-11-05], ~T[00:00:00], site.timezone)
+
+ expected_date_range =
+ DateTimeRange.new!(
+ expected_first_datetime,
+ DateTime.new!(~D[2023-11-05], ~T[23:59:59], site.timezone)
+ )
+
+ %{"date_range" => ["2023-11-05", "2023-11-05"]}
+ |> check_date_range(site, expected_date_range)
+ end
+
+ test "parses date_range.last into the earliest of ambiguous datetimes in site.timezone", %{
+ site: site
+ } do
+ site = %{site | timezone: "America/Asuncion"}
+
+ {:ambiguous, first_dt, _second_dt} =
+ DateTime.new(~D[2024-03-23], ~T[23:59:59], site.timezone)
+
+ expected_date_range =
+ DateTimeRange.new!(
+ DateTime.new!(~D[2024-03-23], ~T[00:00:00], site.timezone),
+ first_dt
+ )
+
+ %{"date_range" => ["2024-03-23", "2024-03-23"]}
+ |> check_date_range(site, expected_date_range)
+ end
end
describe "dimensions validation" do
@@ -527,7 +737,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
}
|> check_success(site, %{
metrics: [:visitors],
- date_range: @date_range,
+ date_range: @date_range_day,
filters: [],
dimensions: ["event:#{unquote(dimension)}"],
order_by: nil,
@@ -548,7 +758,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
}
|> check_success(site, %{
metrics: [:visitors],
- date_range: @date_range,
+ date_range: @date_range_day,
filters: [],
dimensions: ["visit:#{unquote(dimension)}"],
order_by: nil,
@@ -568,7 +778,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
}
|> check_success(site, %{
metrics: [:visitors],
- date_range: @date_range,
+ date_range: @date_range_day,
filters: [],
dimensions: ["event:props:foobar"],
order_by: nil,
@@ -629,7 +839,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
}
|> check_success(site, %{
metrics: [:visitors, :events],
- date_range: @date_range,
+ date_range: @date_range_day,
filters: [],
dimensions: [],
order_by: [{:events, :desc}, {:visitors, :asc}],
@@ -649,7 +859,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
}
|> check_success(site, %{
metrics: [:visitors],
- date_range: @date_range,
+ date_range: @date_range_day,
filters: [],
dimensions: ["event:name"],
order_by: [{"event:name", :desc}],
@@ -757,7 +967,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
# }
# |> check_success(site, %{
# metrics: [:conversion_rate],
- # date_range: @date_range,
+ # date_range: @date_range_day,
# filters: [[:is, "event:goal", [event: "Signup"]]],
# dimensions: [],
# order_by: nil,
@@ -777,7 +987,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
# }
# |> check_success(site, %{
# metrics: [:conversion_rate],
- # date_range: @date_range,
+ # date_range: @date_range_day,
# filters: [],
# dimensions: ["event:goal"],
# order_by: nil,
@@ -799,7 +1009,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
# }
# |> check_success(site, %{
# metrics: [:views_per_visit],
- # date_range: @date_range,
+ # date_range: @date_range_day,
# filters: [[:is, "event:goal", [event: "Signup"]]],
# dimensions: [],
# order_by: nil,
@@ -846,7 +1056,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
}
|> check_success(site, %{
metrics: [:bounce_rate],
- date_range: @date_range,
+ date_range: @date_range_day,
filters: [],
dimensions: ["visit:device"],
order_by: nil,
@@ -878,7 +1088,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
}
|> check_success(site, %{
metrics: [:bounce_rate],
- date_range: @date_range,
+ date_range: @date_range_day,
filters: [],
dimensions: ["event:page"],
order_by: nil,
@@ -897,7 +1107,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
}
|> check_success(site, %{
metrics: [:bounce_rate],
- date_range: @date_range,
+ date_range: @date_range_day,
filters: [[:is, "event:props:foo", ["(none)"]]],
dimensions: [],
order_by: nil,
diff --git a/test/plausible/stats/query_result_test.exs b/test/plausible/stats/query_result_test.exs
index 5e2ce5b4dd..f7f3db75c6 100644
--- a/test/plausible/stats/query_result_test.exs
+++ b/test/plausible/stats/query_result_test.exs
@@ -48,8 +48,8 @@ defmodule Plausible.Stats.QueryResultTest do
"pageviews"
],
"date_range": [
- "2024-01-01",
- "2024-02-01"
+ "2024-01-01T00:00:00 UTC",
+ "2024-02-01T23:59:59 UTC"
],
"filters": [],
"dimensions": [],
diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exs
index ee0517cbdf..688c8a14cc 100644
--- a/test/plausible/stats/query_test.exs
+++ b/test/plausible/stats/query_test.exs
@@ -1,6 +1,6 @@
defmodule Plausible.Stats.QueryTest do
use Plausible.DataCase, async: true
- alias Plausible.Stats.Query
+ alias Plausible.Stats.{Query, DateTimeRange}
setup do
user = insert(:user)
@@ -19,8 +19,8 @@ defmodule Plausible.Stats.QueryTest do
test "keeps current timestamp so that utc_boundaries don't depend on time passing by", %{
site: site
} do
- q1 = %{now: %NaiveDateTime{}} = Query.from(site, %{"period" => "realtime"})
- q2 = %{now: %NaiveDateTime{}} = Query.from(site, %{"period" => "30m"})
+ q1 = %{now: %DateTime{}} = Query.from(site, %{"period" => "realtime"})
+ q2 = %{now: %DateTime{}} = Query.from(site, %{"period" => "30m"})
boundaries1 = Plausible.Stats.Time.utc_boundaries(q1, site)
boundaries2 = Plausible.Stats.Time.utc_boundaries(q2, site)
:timer.sleep(1500)
@@ -31,136 +31,190 @@ defmodule Plausible.Stats.QueryTest do
test "parses day format", %{site: site} do
q = Query.from(site, %{"period" => "day", "date" => "2019-01-01"})
- assert q.date_range.first == ~D[2019-01-01]
- assert q.date_range.last == ~D[2019-01-01]
+ assert q.date_range.first == DateTime.new!(~D[2019-01-01], ~T[00:00:00], site.timezone)
+ assert q.date_range.last == DateTime.new!(~D[2019-01-01], ~T[23:59:59], site.timezone)
assert q.interval == "hour"
end
test "day format defaults to today", %{site: site} do
q = Query.from(site, %{"period" => "day"})
- assert q.date_range.first == Timex.today()
- assert q.date_range.last == Timex.today()
+ expected_first_datetime = Date.utc_today() |> DateTime.new!(~T[00:00:00], site.timezone)
+ expected_last_datetime = Date.utc_today() |> DateTime.new!(~T[23:59:59], site.timezone)
+
+ assert q.date_range.first == expected_first_datetime
+ assert q.date_range.last == expected_last_datetime
assert q.interval == "hour"
end
test "parses realtime format", %{site: site} do
q = Query.from(site, %{"period" => "realtime"})
- assert q.date_range.first == Timex.today()
- assert q.date_range.last == Timex.today()
+ utc_now = DateTime.shift_zone!(q.now, "Etc/UTC")
+
+ expected_first_datetime = utc_now |> DateTime.shift(minute: -5)
+ expected_last_datetime = utc_now |> DateTime.shift(second: 5)
+
+ assert q.date_range.first == expected_first_datetime
+ assert q.date_range.last == expected_last_datetime
assert q.period == "realtime"
end
test "parses month format", %{site: site} do
q = Query.from(site, %{"period" => "month", "date" => "2019-01-01"})
- assert q.date_range.first == ~D[2019-01-01]
- assert q.date_range.last == ~D[2019-01-31]
+ assert q.date_range.first == DateTime.new!(~D[2019-01-01], ~T[00:00:00], site.timezone)
+ assert q.date_range.last == DateTime.new!(~D[2019-01-31], ~T[23:59:59], site.timezone)
assert q.interval == "day"
end
test "parses 6 month format", %{site: site} do
q = Query.from(site, %{"period" => "6mo"})
- assert q.date_range.first ==
- Timex.shift(Timex.today(), months: -5) |> Timex.beginning_of_month()
-
- assert q.date_range.last == Timex.today() |> Timex.end_of_month()
+ expected_first_datetime =
+ q.now
+ |> DateTime.to_date()
+ |> Date.shift(month: -5)
+ |> Date.beginning_of_month()
+ |> DateTime.new!(~T[00:00:00], site.timezone)
+
+ expected_last_datetime =
+ q.now
+ |> DateTime.to_date()
+ |> Date.end_of_month()
+ |> DateTime.new!(~T[23:59:59], site.timezone)
+
+ assert q.date_range.first == expected_first_datetime
+ assert q.date_range.last == expected_last_datetime
assert q.interval == "month"
end
test "parses 12 month format", %{site: site} do
q = Query.from(site, %{"period" => "12mo"})
- assert q.date_range.first ==
- Timex.shift(Timex.today(), months: -11) |> Timex.beginning_of_month()
-
- assert q.date_range.last == Timex.today() |> Timex.end_of_month()
+ expected_first_datetime =
+ q.now
+ |> DateTime.to_date()
+ |> Date.shift(month: -11)
+ |> Date.beginning_of_month()
+ |> DateTime.new!(~T[00:00:00], site.timezone)
+
+ expected_last_datetime =
+ q.now
+ |> DateTime.to_date()
+ |> Date.end_of_month()
+ |> DateTime.new!(~T[23:59:59], site.timezone)
+
+ assert q.date_range.first == expected_first_datetime
+ assert q.date_range.last == expected_last_datetime
assert q.interval == "month"
end
test "parses year to date format", %{site: site} do
q = Query.from(site, %{"period" => "year"})
- assert q.date_range.first ==
- Timex.now(site.timezone) |> Timex.to_date() |> Timex.beginning_of_year()
+ %Date{year: current_year} = DateTime.to_date(q.now)
- assert q.date_range.last ==
- Timex.now(site.timezone) |> Timex.to_date() |> Timex.end_of_year()
+ expected_first_datetime =
+ Date.new!(current_year, 1, 1)
+ |> DateTime.new!(~T[00:00:00], site.timezone)
+ expected_last_datetime =
+ Date.new!(current_year, 12, 31)
+ |> DateTime.new!(~T[23:59:59], site.timezone)
+
+ assert q.date_range.first == expected_first_datetime
+ assert q.date_range.last == expected_last_datetime
assert q.interval == "month"
end
test "parses all time", %{site: site} do
q = Query.from(site, %{"period" => "all"})
- assert q.date_range.first == NaiveDateTime.to_date(site.inserted_at)
- assert q.date_range.last == Timex.today()
+ expected_last_datetime =
+ q.now
+ |> DateTime.to_date()
+ |> DateTime.new!(~T[23:59:59], site.timezone)
+
+ assert DateTime.to_naive(q.date_range.first) == site.inserted_at
+ assert q.date_range.last == expected_last_datetime
assert q.period == "all"
assert q.interval == "month"
end
- test "parses all time in correct timezone", %{site: site} do
- site = Map.put(site, :timezone, "America/Cancun")
- q = Query.from(site, %{"period" => "all"})
+ test "parses all time in site timezone", %{site: site} do
+ for timezone <- ["Etc/GMT+12", "Etc/GMT-12"] do
+ site = Map.put(site, :timezone, timezone)
+ query = Query.from(site, %{"period" => "all"})
+
+ expected_first_datetime = DateTime.new!(~D[2020-01-01], ~T[00:00:00], site.timezone)
+
+ expected_last_datetime =
+ Timex.today(site.timezone)
+ |> DateTime.new!(~T[23:59:59], site.timezone)
- assert q.date_range.first == ~D[2020-01-01]
- assert q.date_range.last == Timex.today("America/Cancun")
+ assert query.date_range.first == expected_first_datetime
+ assert query.date_range.last == expected_last_datetime
+ end
end
test "all time shows today if site has no start date", %{site: site} do
site = Map.put(site, :stats_start_date, nil)
q = Query.from(site, %{"period" => "all"})
- assert q.date_range.first == Timex.today()
- assert q.date_range.last == Timex.today()
+ today = Date.utc_today()
+
+ assert q.date_range == DateTimeRange.new!(today, today, site.timezone)
assert q.period == "all"
assert q.interval == "hour"
end
test "all time shows hourly if site is completely new", %{site: site} do
- site = Map.put(site, :stats_start_date, Timex.now() |> Timex.to_date())
+ site = Map.put(site, :stats_start_date, Date.utc_today())
q = Query.from(site, %{"period" => "all"})
- assert q.date_range.first == Timex.today()
- assert q.date_range.last == Timex.today()
+ today = Date.utc_today()
+
+ assert q.date_range == DateTimeRange.new!(today, today, site.timezone)
assert q.period == "all"
assert q.interval == "hour"
end
test "all time shows daily if site is more than a day old", %{site: site} do
- site =
- Map.put(site, :stats_start_date, Timex.now() |> Timex.shift(days: -1) |> Timex.to_date())
+ today = Date.utc_today()
+ yesterday = today |> Date.shift(day: -1)
+
+ site = Map.put(site, :stats_start_date, yesterday)
q = Query.from(site, %{"period" => "all"})
- assert q.date_range.first == Timex.today() |> Timex.shift(days: -1)
- assert q.date_range.last == Timex.today()
+ assert q.date_range == DateTimeRange.new!(yesterday, today, site.timezone)
assert q.period == "all"
assert q.interval == "day"
end
test "all time shows monthly if site is more than a month old", %{site: site} do
- site =
- Map.put(site, :stats_start_date, Timex.now() |> Timex.shift(months: -1) |> Timex.to_date())
+ today = Date.utc_today()
+ last_month = today |> Date.shift(month: -1)
+
+ site = Map.put(site, :stats_start_date, last_month)
q = Query.from(site, %{"period" => "all"})
- assert q.date_range.first == Timex.today() |> Timex.shift(months: -1)
- assert q.date_range.last == Timex.today()
+ assert q.date_range == DateTimeRange.new!(last_month, today, site.timezone)
assert q.period == "all"
assert q.interval == "month"
end
test "all time uses passed interval different from the default interval", %{site: site} do
- site =
- Map.put(site, :stats_start_date, Timex.now() |> Timex.shift(months: -1) |> Timex.to_date())
+ today = Date.utc_today()
+ last_month = today |> Date.shift(month: -1)
+
+ site = Map.put(site, :stats_start_date, last_month)
q = Query.from(site, %{"period" => "all", "interval" => "week"})
- assert q.date_range.first == Timex.today() |> Timex.shift(months: -1)
- assert q.date_range.last == Timex.today()
+ assert q.date_range == DateTimeRange.new!(last_month, today, site.timezone)
assert q.period == "all"
assert q.interval == "week"
end
@@ -172,8 +226,8 @@ defmodule Plausible.Stats.QueryTest do
test "parses custom format", %{site: site} do
q = Query.from(site, %{"period" => "custom", "from" => "2019-01-01", "to" => "2019-01-15"})
- assert q.date_range.first == ~D[2019-01-01]
- assert q.date_range.last == ~D[2019-01-15]
+ assert q.date_range.first == DateTime.new!(~D[2019-01-01], ~T[00:00:00], site.timezone)
+ assert q.date_range.last == DateTime.new!(~D[2019-01-15], ~T[23:59:59], site.timezone)
assert q.interval == "day"
end
diff --git a/test/plausible/stats/time_test.exs b/test/plausible/stats/time_test.exs
index e368a0cbc5..37cb983fb9 100644
--- a/test/plausible/stats/time_test.exs
+++ b/test/plausible/stats/time_test.exs
@@ -2,12 +2,13 @@ defmodule Plausible.Stats.TimeTest do
use Plausible.DataCase, async: true
import Plausible.Stats.Time
+ alias Plausible.Stats.DateTimeRange
describe "time_labels/1" do
test "with time:month dimension" do
assert time_labels(%{
dimensions: ["visit:device", "time:month"],
- date_range: Date.range(~D[2022-01-17], ~D[2022-02-01])
+ date_range: DateTimeRange.new!(~D[2022-01-17], ~D[2022-02-01], "UTC")
}) == [
"2022-01-01",
"2022-02-01"
@@ -15,7 +16,7 @@ defmodule Plausible.Stats.TimeTest do
assert time_labels(%{
dimensions: ["visit:device", "time:month"],
- date_range: Date.range(~D[2022-01-01], ~D[2022-03-07])
+ date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-03-07], "UTC")
}) == [
"2022-01-01",
"2022-02-01",
@@ -26,7 +27,7 @@ defmodule Plausible.Stats.TimeTest do
test "with time:week dimension" do
assert time_labels(%{
dimensions: ["time:week"],
- date_range: Date.range(~D[2020-12-20], ~D[2021-01-08])
+ date_range: DateTimeRange.new!(~D[2020-12-20], ~D[2021-01-08], "UTC")
}) == [
"2020-12-20",
"2020-12-21",
@@ -36,7 +37,7 @@ defmodule Plausible.Stats.TimeTest do
assert time_labels(%{
dimensions: ["time:week"],
- date_range: Date.range(~D[2020-12-21], ~D[2021-01-03])
+ date_range: DateTimeRange.new!(~D[2020-12-21], ~D[2021-01-03], "UTC")
}) == [
"2020-12-21",
"2020-12-28"
@@ -46,7 +47,7 @@ defmodule Plausible.Stats.TimeTest do
test "with time:day dimension" do
assert time_labels(%{
dimensions: ["time:day"],
- date_range: Date.range(~D[2022-01-17], ~D[2022-02-02])
+ date_range: DateTimeRange.new!(~D[2022-01-17], ~D[2022-02-02], "UTC")
}) == [
"2022-01-17",
"2022-01-18",
@@ -71,7 +72,7 @@ defmodule Plausible.Stats.TimeTest do
test "with time:hour dimension" do
assert time_labels(%{
dimensions: ["time:hour"],
- date_range: Date.range(~D[2022-01-17], ~D[2022-01-17])
+ date_range: DateTimeRange.new!(~D[2022-01-17], ~D[2022-01-17], "UTC")
}) == [
"2022-01-17 00:00:00",
"2022-01-17 01:00:00",
@@ -101,7 +102,7 @@ defmodule Plausible.Stats.TimeTest do
assert time_labels(%{
dimensions: ["time:hour"],
- date_range: Date.range(~D[2022-01-17], ~D[2022-01-18])
+ date_range: DateTimeRange.new!(~D[2022-01-17], ~D[2022-01-18], "UTC")
}) == [
"2022-01-17 00:00:00",
"2022-01-17 01:00:00",
@@ -154,4 +155,50 @@ defmodule Plausible.Stats.TimeTest do
]
end
end
+
+ test "with time:minute dimension" do
+ now = DateTime.new!(~D[2024-01-01], ~T[12:30:57], "UTC")
+
+ # ~U[2024-01-01 12:00:57Z]
+ first_dt = DateTime.shift(now, minute: -30)
+ # ~U[2024-01-01 12:31:02Z]
+ last_dt = DateTime.shift(now, second: 5)
+
+ assert time_labels(%{
+ dimensions: ["time:minute"],
+ now: now,
+ date_range: DateTimeRange.new!(first_dt, last_dt)
+ }) == [
+ "2024-01-01 12:00:00",
+ "2024-01-01 12:01:00",
+ "2024-01-01 12:02:00",
+ "2024-01-01 12:03:00",
+ "2024-01-01 12:04:00",
+ "2024-01-01 12:05:00",
+ "2024-01-01 12:06:00",
+ "2024-01-01 12:07:00",
+ "2024-01-01 12:08:00",
+ "2024-01-01 12:09:00",
+ "2024-01-01 12:10:00",
+ "2024-01-01 12:11:00",
+ "2024-01-01 12:12:00",
+ "2024-01-01 12:13:00",
+ "2024-01-01 12:14:00",
+ "2024-01-01 12:15:00",
+ "2024-01-01 12:16:00",
+ "2024-01-01 12:17:00",
+ "2024-01-01 12:18:00",
+ "2024-01-01 12:19:00",
+ "2024-01-01 12:20:00",
+ "2024-01-01 12:21:00",
+ "2024-01-01 12:22:00",
+ "2024-01-01 12:23:00",
+ "2024-01-01 12:24:00",
+ "2024-01-01 12:25:00",
+ "2024-01-01 12:26:00",
+ "2024-01-01 12:27:00",
+ "2024-01-01 12:28:00",
+ "2024-01-01 12:29:00"
+ ]
+ end
end
diff --git a/test/plausible/timezones_test.exs b/test/plausible/timezones_test.exs
index 890d2e2f22..06cae19b83 100644
--- a/test/plausible/timezones_test.exs
+++ b/test/plausible/timezones_test.exs
@@ -19,15 +19,6 @@ defmodule Plausible.TimezonesTest do
refute Enum.empty?(options)
end
- test "to_utc_datetime/2" do
- assert to_utc_datetime(~N[2022-09-11 00:00:00], "Etc/UTC") == ~U[2022-09-11 00:00:00Z]
-
- assert to_utc_datetime(~N[2022-09-11 00:00:00], "America/Santiago") ==
- ~U[2022-09-11 00:00:00Z]
-
- assert to_utc_datetime(~N[2023-10-29 00:00:00], "Atlantic/Azores") == ~U[2023-10-29 01:00:00Z]
- end
-
test "to_date_in_timezone/1" do
assert to_date_in_timezone(~D[2021-01-03], "Etc/UTC") == ~D[2021-01-03]
assert to_date_in_timezone(~U[2015-01-13 13:00:07Z], "Etc/UTC") == ~D[2015-01-13]
diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs
index cbfda8e256..6a74ca02af 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs
@@ -2896,6 +2896,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
referrer_source: "Google",
timestamp: ~N[2021-01-02 00:00:00]
),
+ build(:pageview,
+ referrer_source: "Google",
+ timestamp: ~N[2021-01-02 00:00:00]
+ ),
+ build(:pageview, timestamp: ~N[2021-01-03 00:00:00]),
build(:pageview, timestamp: ~N[2021-01-03 00:00:00]),
build(:pageview,
referrer_source: "Twitter",
@@ -2912,11 +2917,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
})
assert json_response(conn, 200)["results"] == [
- %{"dimensions" => ["2021-01-01 00:00:00", "Google"], "metrics" => [1]},
- %{"dimensions" => ["2021-01-02 00:00:00", "Google"], "metrics" => [1]},
- %{"dimensions" => ["2021-01-02 00:00:00", "Direct / None"], "metrics" => [1]},
- %{"dimensions" => ["2021-01-03 00:00:00", "Direct / None"], "metrics" => [1]},
- %{"dimensions" => ["2021-01-03 00:00:00", "Twitter"], "metrics" => [1]}
+ %{"dimensions" => ["2021-01-01", "Google"], "metrics" => [1]},
+ %{"dimensions" => ["2021-01-02", "Google"], "metrics" => [2]},
+ %{"dimensions" => ["2021-01-02", "Direct / None"], "metrics" => [1]},
+ %{"dimensions" => ["2021-01-03", "Direct / None"], "metrics" => [2]},
+ %{"dimensions" => ["2021-01-03", "Twitter"], "metrics" => [1]}
]
end
@@ -2975,4 +2980,32 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
%{"dimensions" => ["United Kingdom", "London", "London"], "metrics" => [1]}
]
end
+
+ describe "using the returned query object in a new POST request" do
+ test "yields the same results for a simple aggregate query", %{conn: conn, site: site} do
+ Plausible.Site.changeset(site, %{timezone: "Europe/Tallinn"})
+ |> Plausible.Repo.update()
+
+ populate_stats(site, [
+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:25:00]),
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
+ ])
+
+ conn1 =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["pageviews"],
+ "date_range" => "all"
+ })
+
+ assert %{"results" => results1, "query" => query} = json_response(conn1, 200)
+ assert results1 == [%{"metrics" => [3], "dimensions" => []}]
+
+ conn2 = post(conn, "/api/v2/query", query)
+
+ assert %{"results" => results2} = json_response(conn2, 200)
+ assert results2 == [%{"metrics" => [3], "dimensions" => []}]
+ end
+ end
end
diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
index bfecd730b6..c5bfe35559 100644
--- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
@@ -13,8 +13,9 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=realtime&metric=pageviews")
- assert %{"plot" => plot} = json_response(conn, 200)
+ assert %{"plot" => plot, "labels" => labels} = json_response(conn, 200)
+ assert labels == Enum.to_list(-30..-1)
assert Enum.count(plot) == 30
assert Enum.any?(plot, fn pageviews -> pageviews > 0 end)
end
@@ -1072,6 +1073,28 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
"2021-12-01" => false
}
end
+
+ test "returns stats for a day with a minute interval", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2023-03-01 12:00:00])
+ ])
+
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/main-graph?period=day&metric=visitors&date=2023-03-01&interval=minute"
+ )
+
+ %{"labels" => labels, "plot" => plot} = json_response(conn, 200)
+
+ assert length(labels) == 24 * 60
+
+ assert List.first(labels) == "2023-03-01 00:00:00"
+ assert Enum.at(labels, 1) == "2023-03-01 00:01:00"
+ assert List.last(labels) == "2023-03-01 23:59:00"
+
+ assert Enum.at(plot, Enum.find_index(labels, &(&1 == "2023-03-01 12:00:00"))) == 1
+ end
end
describe "GET /api/stats/main-graph - comparisons" do