Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Add API support for intervals (#2417)

Vini Brasil9c98a3f2e82d
diff --git a/lib/plausible/purge.ex b/lib/plausible/purge.ex
index 2a778a2b09..3bb91c9a03 100644
--- a/lib/plausible/purge.ex
+++ b/lib/plausible/purge.ex
@@ -5,7 +5,7 @@ defmodule Plausible.Purge do
Stats are stored on Clickhouse, and unlike other databases data deletion is
done asynchronously.
- - [Clickhouse `ALTER TABLE ... DELETE` Statement`](https://clickhouse.com/docs/en/sql-reference/statements/alter/delete)
+ - [Clickhouse `ALTER TABLE ... DELETE` Statement](https://clickhouse.com/docs/en/sql-reference/statements/alter/delete)
- [Synchronicity of `ALTER` Queries](https://clickhouse.com/docs/en/sql-reference/statements/alter/#synchronicity-of-alter-queries)
"""
diff --git a/lib/plausible/stats/fragments.ex b/lib/plausible/stats/fragments.ex
index 692233f107..19808288d2 100644
--- a/lib/plausible/stats/fragments.ex
+++ b/lib/plausible/stats/fragments.ex
@@ -35,6 +35,68 @@ defmodule Plausible.Stats.Fragments do
end
end
+ @doc """
+ Converts time or date and time to the specified timezone.
+
+ Reference: https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#totimezone
+ """
+ defmacro to_timezone(date, timezone) do
+ quote do
+ fragment("toTimeZone(?, ?)", unquote(date), unquote(timezone))
+ end
+ end
+
+ @doc """
+ Returns the weekstart for `date`. If the weekstart is before the `not_before`
+ boundary, `not_before` is returned.
+
+ ## Examples
+
+ In this pseudo-code example, the fragment returns the weekstart. The
+ `not_before` boundary is set to the past Saturday, which is before the
+ weekstart, therefore the cap does not apply.
+
+ iex> this_wednesday = ~D[2022-11-09]
+ ...> past_saturday = ~D[2022-11-05]
+ ...> weekstart_not_before(this_wednesday, past_saturday)
+ ~D[2022-11-07]
+
+
+ In this other example, the fragment returns Tuesday and not the weekstart.
+ The `not_before` boundary is set to Tuesday, which is past the weekstart,
+ therefore the cap applies.
+
+ iex> this_wednesday = ~D[2022-11-09]
+ ...> this_tuesday = ~D[2022-11-08]
+ ...> weekstart_not_before(this_wednesday, this_tuesday)
+ ~D[2022-11-08]
+
+ """
+ defmacro weekstart_not_before(date, not_before) do
+ quote do
+ fragment(
+ "if(toMonday(?) < toDate(?), toDate(?), toMonday(?))",
+ unquote(date),
+ unquote(not_before),
+ unquote(not_before),
+ unquote(date)
+ )
+ end
+ end
+
+ @doc """
+ Same as Plausible.Stats.Fragments.weekstart_not_before/2 but converts dates to
+ the specified timezone.
+ """
+ defmacro weekstart_not_before(date, not_before, timezone) do
+ quote do
+ weekstart_not_before(
+ to_timezone(unquote(date), unquote(timezone)),
+ unquote(not_before)
+ )
+ end
+ end
+
defmacro __using__(_) do
quote do
import Plausible.Stats.Fragments
diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.ex
index 9888bba5f6..37d1de6753 100644
--- a/lib/plausible/stats/imported.ex
+++ b/lib/plausible/stats/imported.ex
@@ -2,6 +2,8 @@ defmodule Plausible.Stats.Imported do
use Plausible.ClickhouseRepo
alias Plausible.Stats.Query
import Ecto.Query
+ import Plausible.Stats.Base
+ import Plausible.Stats.Fragments
@no_ref "Direct / None"
@@ -21,7 +23,7 @@ defmodule Plausible.Stats.Imported do
select: %{}
)
|> select_imported_metrics(metrics)
- |> apply_interval(query)
+ |> apply_interval(query, site.timezone)
from(s in Ecto.Query.subquery(native_q),
full_join: i in subquery(imported_q),
@@ -31,13 +33,21 @@ defmodule Plausible.Stats.Imported do
|> select_joined_metrics(metrics)
end
- defp apply_interval(imported_q, %Plausible.Stats.Query{interval: "month"}) do
+ defp apply_interval(imported_q, %Plausible.Stats.Query{interval: "month"}, _timezone) do
imported_q
|> group_by([i], fragment("toStartOfMonth(?)", i.date))
|> select_merge([i], %{date: fragment("toStartOfMonth(?)", i.date)})
end
- defp apply_interval(imported_q, _query) do
+ defp apply_interval(imported_q, %Plausible.Stats.Query{interval: "week"} = query, timezone) do
+ {first_datetime, _} = utc_boundaries(query, timezone)
+
+ imported_q
+ |> group_by([i], weekstart_not_before(i.date, ^first_datetime))
+ |> select_merge([i], %{date: weekstart_not_before(i.date, ^first_datetime)})
+ end
+
+ defp apply_interval(imported_q, _query, _timezone) do
imported_q
|> group_by([i], i.date)
|> select_merge([i], %{date: i.date})
diff --git a/lib/plausible/stats/interval.ex b/lib/plausible/stats/interval.ex
new file mode 100644
index 0000000000..75f55f78bc
--- /dev/null
+++ b/lib/plausible/stats/interval.ex
@@ -0,0 +1,78 @@
+defmodule Plausible.Stats.Interval do
+ @moduledoc """
+ Collection of functions to work with intervals.
+
+ The interval of a query defines the granularity of the data. You can think of
+ it as a `GROUP BY` clause. Possible values are `minute`, `hour`, `date`,
+ `week`, and `month`.
+ """
+
+ @type t() :: String.t()
+ @typep period() :: String.t()
+
+ @intervals ~w(minute hour date week month)
+ @spec list() :: [t()]
+ def list, do: @intervals
+
+ @spec valid?(term()) :: boolean()
+ def valid?(interval) do
+ interval in @intervals
+ end
+
+ @spec default_for_period(period()) :: t()
+ @doc """
+ Returns the suggested interval for the given time period.
+
+ ## Examples
+
+ iex> Plausible.Stats.Interval.default_for_period("7d")
+ "date"
+
+ """
+ def default_for_period(period) do
+ case period do
+ "realtime" -> "minute"
+ "day" -> "hour"
+ period when period in ["custom", "7d", "30d", "month"] -> "date"
+ period when period in ["6mo", "12mo", "year"] -> "month"
+ end
+ end
+
+ @allowed_intervals_for_period %{
+ "realtime" => ["minute"],
+ "day" => ["minute", "hour"],
+ "7d" => ["minute", "hour", "date"],
+ "month" => ["minute", "hour", "date", "week"],
+ "30d" => ["minute", "hour", "date", "week"],
+ "6mo" => ["minute", "hour", "date", "week", "month"],
+ "12mo" => ["minute", "hour", "date", "week", "month"],
+ "year" => ["minute", "hour", "date", "week", "month"],
+ "custom" => ["minute", "hour", "date", "week", "month"],
+ "all" => ["minute", "hour", "date", "week", "month"]
+ }
+
+ @spec allowed_for_period?(period(), t()) :: boolean()
+ @doc """
+ Returns whether the given interval is valid for a time period.
+
+ Intervals longer than periods are not supported, e.g. current month stats with
+ a month interval, or today stats with a week interval.
+
+ ## Examples
+
+
+ iex> Plausible.Stats.Interval.allowed_for_period?("month", "date")
+ true
+
+ iex> Plausible.Stats.Interval.allowed_for_period?("30d", "month")
+ false
+
+ iex> Plausible.Stats.Interval.allowed_for_period?("realtime", "week")
+ false
+
+ """
+ def allowed_for_period?(period, interval) do
+ allowed = Map.get(@allowed_intervals_for_period, period, [])
+ interval in allowed
+ end
+end
diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex
index c645e68af0..a53c689e54 100644
--- a/lib/plausible/stats/query.ex
+++ b/lib/plausible/stats/query.ex
@@ -7,7 +7,7 @@ defmodule Plausible.Stats.Query do
include_imported: false
@default_sample_threshold 20_000_000
- alias Plausible.Stats.FilterParser
+ alias Plausible.Stats.{FilterParser, Interval}
def shift_back(%__MODULE__{period: "year"} = query, site) do
# Querying current year to date
@@ -67,7 +67,7 @@ defmodule Plausible.Stats.Query do
%__MODULE__{
period: "realtime",
- interval: "minute",
+ interval: params["interval"] || Interval.default_for_period(params["period"]),
date_range: Date.range(date, date),
filters: FilterParser.parse_filters(params["filters"]),
sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold),
@@ -81,7 +81,7 @@ defmodule Plausible.Stats.Query do
%__MODULE__{
period: "day",
date_range: Date.range(date, date),
- interval: "hour",
+ interval: params["interval"] || Interval.default_for_period(params["period"]),
filters: FilterParser.parse_filters(params["filters"]),
sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)
}
@@ -95,7 +95,7 @@ defmodule Plausible.Stats.Query do
%__MODULE__{
period: "7d",
date_range: Date.range(start_date, end_date),
- interval: "date",
+ interval: params["interval"] || Interval.default_for_period(params["period"]),
filters: FilterParser.parse_filters(params["filters"]),
sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)
}
@@ -109,7 +109,7 @@ defmodule Plausible.Stats.Query do
%__MODULE__{
period: "30d",
date_range: Date.range(start_date, end_date),
- interval: "date",
+ interval: params["interval"] || Interval.default_for_period(params["period"]),
filters: FilterParser.parse_filters(params["filters"]),
sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)
}
@@ -125,7 +125,7 @@ defmodule Plausible.Stats.Query do
%__MODULE__{
period: "month",
date_range: Date.range(start_date, end_date),
- interval: "date",
+ interval: params["interval"] || Interval.default_for_period(params["period"]),
filters: FilterParser.parse_filters(params["filters"]),
sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)
}
@@ -144,7 +144,7 @@ defmodule Plausible.Stats.Query do
%__MODULE__{
period: "6mo",
date_range: Date.range(start_date, end_date),
- interval: Map.get(params, "interval", "month"),
+ interval: params["interval"] || Interval.default_for_period(params["period"]),
filters: FilterParser.parse_filters(params["filters"]),
sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)
}
@@ -163,7 +163,7 @@ defmodule Plausible.Stats.Query do
%__MODULE__{
period: "12mo",
date_range: Date.range(start_date, end_date),
- interval: Map.get(params, "interval", "month"),
+ interval: params["interval"] || Interval.default_for_period(params["period"]),
filters: FilterParser.parse_filters(params["filters"]),
sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)
}
@@ -180,7 +180,7 @@ defmodule Plausible.Stats.Query do
%__MODULE__{
period: "year",
date_range: Date.range(start_date, end_date),
- interval: Map.get(params, "interval", "month"),
+ interval: params["interval"] || Interval.default_for_period(params["period"]),
filters: FilterParser.parse_filters(params["filters"]),
sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)
}
@@ -199,7 +199,7 @@ defmodule Plausible.Stats.Query do
"period" => "custom",
"from" => Date.to_iso8601(start_date),
"to" => Date.to_iso8601(now),
- "interval" => "month"
+ "interval" => params["interval"] || "month"
})
)
|> Map.put(:period, "all")
@@ -211,7 +211,7 @@ defmodule Plausible.Stats.Query do
"period" => "custom",
"from" => Date.to_iso8601(start_date),
"to" => Date.to_iso8601(now),
- "interval" => "date"
+ "interval" => params["interval"] || "date"
})
)
|> Map.put(:period, "all")
@@ -240,7 +240,7 @@ defmodule Plausible.Stats.Query do
%__MODULE__{
period: "custom",
date_range: Date.range(from_date, to_date),
- interval: Map.get(params, "interval", "date"),
+ interval: params["interval"] || Interval.default_for_period(params["period"]),
filters: FilterParser.parse_filters(params["filters"]),
sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)
}
diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex
index 694a7dd21a..dfd94d7058 100644
--- a/lib/plausible/stats/timeseries.ex
+++ b/lib/plausible/stats/timeseries.ex
@@ -50,7 +50,7 @@ defmodule Plausible.Stats.Timeseries do
|> ClickhouseRepo.all()
end
- def buckets(%Query{interval: "month"} = query) do
+ defp buckets(%Query{interval: "month"} = query) do
n_buckets = Timex.diff(query.date_range.last, query.date_range.first, :months)
Enum.map(n_buckets..0, fn shift ->
@@ -60,23 +60,59 @@ defmodule Plausible.Stats.Timeseries do
end)
end
- def buckets(%Query{interval: "date"} = query) do
+ defp buckets(%Query{interval: "week"} = query) do
+ n_buckets = Timex.diff(query.date_range.last, query.date_range.first, :weeks)
+
+ Enum.map(0..n_buckets, fn shift ->
+ query.date_range.first
+ |> Timex.shift(weeks: shift)
+ |> date_or_weekstart(query)
+ end)
+ end
+
+ defp buckets(%Query{interval: "date"} = query) do
Enum.into(query.date_range, [])
end
- def buckets(%Query{interval: "hour"} = query) do
- Enum.map(0..23, fn step ->
- Timex.to_datetime(query.date_range.first)
+ @full_day_in_hours 23
+ defp buckets(%Query{interval: "hour"} = query) do
+ n_buckets =
+ if query.date_range.first == query.date_range.last do
+ @full_day_in_hours
+ else
+ Timex.diff(query.date_range.last, query.date_range.first, :hours)
+ end
+
+ Enum.map(0..n_buckets, fn step ->
+ query.date_range.first
+ |> Timex.to_datetime()
|> Timex.shift(hours: step)
|> Timex.format!("{YYYY}-{0M}-{0D} {h24}:{m}:{s}")
end)
end
- def buckets(%Query{period: "30m", interval: "minute"}) do
+ defp buckets(%Query{period: "30m", interval: "minute"}) do
Enum.into(-30..-1, [])
end
- def select_bucket(q, site, %Query{interval: "month"}) do
+ @full_day_in_minutes 1439
+ defp buckets(%Query{interval: "minute"} = query) do
+ n_buckets =
+ if query.date_range.first == query.date_range.last do
+ @full_day_in_minutes
+ else
+ Timex.diff(query.date_range.last, query.date_range.first, :minutes)
+ end
+
+ Enum.map(0..n_buckets, fn step ->
+ query.date_range.first
+ |> Timex.to_datetime()
+ |> Timex.shift(minutes: step)
+ |> Timex.format!("{YYYY}-{0M}-{0D} {h24}:{m}:{s}")
+ end)
+ end
+
+ defp select_bucket(q, site, %Query{interval: "month"}) do
from(
e in q,
group_by: fragment("toStartOfMonth(toTimeZone(?, ?))", e.timestamp, ^site.timezone),
@@ -87,7 +123,18 @@ defmodule Plausible.Stats.Timeseries do
)
end
- def select_bucket(q, site, %Query{interval: "date"}) do
+ defp select_bucket(q, site, %Query{interval: "week"} = query) do
+ {first_datetime, _} = utc_boundaries(query, site.timezone)
+
+ from(
+ e in q,
+ select_merge: %{date: weekstart_not_before(e.timestamp, ^first_datetime, ^site.timezone)},
+ group_by: weekstart_not_before(e.timestamp, ^first_datetime, ^site.timezone),
+ order_by: weekstart_not_before(e.timestamp, ^first_datetime, ^site.timezone)
+ )
+ end
+
+ defp select_bucket(q, site, %Query{interval: "date"}) do
from(
e in q,
group_by: fragment("toDate(toTimeZone(?, ?))", e.timestamp, ^site.timezone),
@@ -98,7 +145,7 @@ defmodule Plausible.Stats.Timeseries do
)
end
- def select_bucket(q, site, %Query{interval: "hour"}) do
+ defp select_bucket(q, site, %Query{interval: "hour"}) do
from(
e in q,
group_by: fragment("toStartOfHour(toTimeZone(?, ?))", e.timestamp, ^site.timezone),
@@ -109,7 +156,7 @@ defmodule Plausible.Stats.Timeseries do
)
end
- def select_bucket(q, _site, %Query{interval: "minute"}) do
+ defp select_bucket(q, _site, %Query{interval: "minute", period: "30m"}) do
from(
e in q,
group_by: fragment("dateDiff('minute', now(), ?)", e.timestamp),
@@ -120,6 +167,27 @@ defmodule Plausible.Stats.Timeseries do
)
end
+ defp select_bucket(q, site, %Query{interval: "minute"}) do
+ from(
+ e in q,
+ group_by: fragment("toStartOfMinute(toTimeZone(?, ?))", e.timestamp, ^site.timezone),
+ order_by: fragment("toStartOfMinute(toTimeZone(?, ?))", e.timestamp, ^site.timezone),
+ select_merge: %{
+ date: fragment("toStartOfMinute(toTimeZone(?, ?))", e.timestamp, ^site.timezone)
+ }
+ )
+ end
+
+ defp date_or_weekstart(date, query) do
+ weekstart = Timex.beginning_of_week(date)
+
+ if Enum.member?(query.date_range, weekstart) do
+ weekstart
+ else
+ date
+ end
+ end
+
defp empty_row(date, metrics) do
Enum.reduce(metrics, %{date: date}, fn metric, row ->
case metric do
diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
index 04cf3237db..cfac6aad84 100644
--- a/lib/plausible_web/controllers/api/stats_controller.ex
+++ b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -5,6 +5,88 @@ defmodule PlausibleWeb.Api.StatsController do
alias Plausible.Stats
alias Plausible.Stats.{Query, Filters}
+ @doc """
+ Returns a time-series based on given parameters.
+
+ ## Parameters
+
+ This API accepts the following parameters:
+
+ * `period` - x-axis of the graph, e.g. `12mo`, `day`, `custom`.
+
+ * `metric` - y-axis of the graph, e.g. `visits`, `visitors`, `pageviews`.
+ See the Stats API ["Metrics"](https://plausible.io/docs/stats-api#metrics)
+ section for more details. Defaults to `visitors`.
+
+ * `interval` - granularity of the time-series data. You can think of it as
+ a `GROUP BY` clause. Possible values are `minute`, `hour`, `date`, `week`,
+ and `month`. The default depends on the `period` parameter. Check
+ `Plausible.Query.from/2` for each default.
+
+ * `filters` - optional filters to drill down data. See the Stats API
+ ["Filtering"](https://plausible.io/docs/stats-api#filtering) section for
+ more details.
+
+ * `with_imported` - boolean indicating whether to include Google Analytics
+ imported data or not. Defaults to `false`.
+
+ Full example:
+ ```elixir
+ %{
+ "from" => "2021-09-06",
+ "interval" => "month",
+ "metric" => "visitors",
+ "period" => "custom",
+ "to" => "2021-12-13"
+ }
+ ```
+
+ ## Response
+
+ Returns a map with the following keys:
+
+ * `plot` - list of values for the requested metric representing the y-axis
+ of the graph.
+
+ * `labels` - list of date times representing the x-axis of the graph.
+
+ * `present_index` - index of the element representing the current date in
+ `labels` and `plot` lists.
+
+ * `interval` - the interval used for querying.
+
+ * `with_imported` - boolean indicating whether the Google Analytics data
+ was queried or not.
+
+ * `imported_source` - the source of the imported data, when applicable.
+ Currently only Google Analytics is supported.
+
+ * `full_intervals` - map of dates indicating whether the interval has been
+ cut off by the requested date range or not. For example, if looking at a
+ month week-by-week, some weeks may be cut off by the month boundaries.
+ It's useful to adjust the graph display slightly in case the interval is
+ not 'full' so that the user understands why the numbers might be lower for
+ those partial periods.
+
+ Full example:
+ ```elixir
+ %{
+ "full_intervals" => %{
+ "2021-09-01" => false,
+ "2021-10-01" => true,
+ "2021-11-01" => true,
+ "2021-12-01" => false
+ },
+ "imported_source" => nil,
+ "interval" => "month",
+ "labels" => ["2021-09-01", "2021-10-01", "2021-11-01", "2021-12-01"],
+ "plot" => [0, 0, 0, 0],
+ "present_index" => nil,
+ "with_imported" => false
+ }
+ ```
+
+ """
def main_graph(conn, params) do
site = conn.assigns[:site]
@@ -35,6 +117,7 @@ defmodule PlausibleWeb.Api.StatsController do
labels = Enum.map(timeseries_result, fn row -> row[:date] end)
present_index = present_index_for(site, query, labels)
+ full_intervals = build_full_intervals(query, labels)
json(conn, %{
plot: plot,
@@ -42,14 +125,40 @@ defmodule PlausibleWeb.Api.StatsController do
present_index: present_index,
interval: query.interval,
with_imported: query.include_imported,
- imported_source: site.imported_data && site.imported_data.source
+ imported_source: site.imported_data && site.imported_data.source,
+ full_intervals: full_intervals
})
else
- _ ->
- bad_request(conn)
+ {:error, message} when is_binary(message) -> bad_request(conn, message)
+ end
+ end
+
+ defp build_full_intervals(%{interval: "week", date_range: range}, labels) do
+ for label <- labels, into: %{} do
+ interval_start = Timex.beginning_of_week(label)
+ interval_end = Timex.end_of_week(label)
+
+ within_interval? = Enum.member?(range, interval_start) && Enum.member?(range, interval_end)
+
+ {label, within_interval?}
+ end
+ end
+
+ defp build_full_intervals(%{interval: "month", date_range: range}, labels) do
+ for label <- labels, into: %{} do
+ interval_start = Timex.beginning_of_month(label)
+ interval_end = Timex.end_of_month(label)
+
+ within_interval? = Enum.member?(range, interval_start) && Enum.member?(range, interval_end)
+
+ {label, within_interval?}
end
end
+ defp build_full_intervals(_query, _labels) do
+ nil
+ end
+
def top_stats(conn, params) do
site = conn.assigns[:site]
@@ -66,8 +175,7 @@ defmodule PlausibleWeb.Api.StatsController do
imported_source: site.imported_data && site.imported_data.source
})
else
- _ ->
- bad_request(conn)
+ {:error, message} when is_binary(message) -> bad_request(conn, message)
end
end
@@ -87,6 +195,14 @@ defmodule PlausibleWeb.Api.StatsController do
Enum.find_index(dates, &(&1 == current_date))
+ "week" ->
+ current_date =
+ Timex.now(site.timezone)
+ |> Timex.to_date()
+ |> date_or_weekstart(query)
+
+ Enum.find_index(dates, &(&1 == current_date))
+
"month" ->
current_date =
Timex.now(site.timezone)
@@ -96,7 +212,21 @@ defmodule PlausibleWeb.Api.StatsController do
Enum.find_index(dates, &(&1 == current_date))
"minute" ->
- nil
+ current_date =
+ Timex.now(site.timezone)
+ |> Timex.format!("{YYYY}-{0M}-{0D} {h24}:{0m}:00")
+
+ Enum.find_index(dates, &(&1 == current_date))
+ end
+ end
+
+ defp date_or_weekstart(date, query) do
+ weekstart = Timex.beginning_of_week(date)
+
+ if Enum.member?(query.date_range, weekstart) do
+ weekstart
+ else
+ date
end
end
@@ -935,8 +1065,7 @@ defmodule PlausibleWeb.Api.StatsController do
json(conn, Stats.filter_suggestions(site, query, params["filter_name"], params["q"]))
else
- _ ->
- bad_request(conn)
+ {:error, message} when is_binary(message) -> bad_request(conn, message)
end
end
@@ -1046,19 +1175,57 @@ defmodule PlausibleWeb.Api.StatsController do
end
end
- defp validate_params(%{"date" => date}) do
- with {:ok, _} <- Date.from_iso8601(date) do
+ defp validate_params(params) do
+ with :ok <- validate_date(params),
+ :ok <- validate_interval(params),
+ do: validate_interval_granularity(params)
+ end
+
+ defp validate_date(params) do
+ with %{"date" => date} <- params,
+ {:ok, _} <- Date.from_iso8601(date) do
+ :ok
+ else
+ %{} ->
+ :ok
+
+ {:error, _reason} ->
+ {:error,
+ "Failed to parse date argument. Only ISO 8601 dates are allowed, e.g. `2019-09-07`, `2020-01-01`"}
+ end
+ end
+
+ defp validate_interval(params) do
+ with %{"interval" => interval} <- params,
+ true <- Plausible.Stats.Interval.valid?(interval) do
:ok
+ else
+ %{} ->
+ :ok
+
+ false ->
+ values = Enum.join(Plausible.Stats.Interval.list(), ", ")
+ {:error, "Invalid value for interval. Accepted values are: #{values}"}
end
end
- defp validate_params(_) do
- :ok
+ defp validate_interval_granularity(params) do
+ with %{"interval" => interval, "period" => period} <- params,
+ true <- Plausible.Stats.Interval.allowed_for_period?(period, interval) do
+ :ok
+ else
+ %{} ->
+ :ok
+
+ false ->
+ {:error,
+ "Invalid combination of interval and period. Interval must be smaller than the selected period, e.g. `period=day,interval=minute`"}
+ end
end
- defp bad_request(conn) do
+ defp bad_request(conn, message) do
conn
|> put_status(400)
- |> json(%{error: "input validation error"})
+ |> json(%{error: message})
end
end
diff --git a/test/plausible/imported/imported_test.exs b/test/plausible/imported/imported_test.exs
index 68700af95f..3e58aeb1dc 100644
--- a/test/plausible/imported/imported_test.exs
+++ b/test/plausible/imported/imported_test.exs
@@ -60,6 +60,52 @@ defmodule Plausible.ImportedTest do
assert Enum.sum(plot) == 4
end
+ test "returns data grouped by week", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-31 00:00:00])
+ ])
+
+ import_data(
+ [
+ %{
+ dimensions: %{"ga:date" => "20210101"},
+ metrics: %{
+ "ga:users" => "1",
+ "ga:pageviews" => "1",
+ "ga:bounces" => "0",
+ "ga:sessions" => "1",
+ "ga:sessionDuration" => "60"
+ }
+ },
+ %{
+ dimensions: %{"ga:date" => "20210131"},
+ metrics: %{
+ "ga:users" => "1",
+ "ga:pageviews" => "1",
+ "ga:bounces" => "0",
+ "ga:sessions" => "1",
+ "ga:sessionDuration" => "60"
+ }
+ }
+ ],
+ site.id,
+ "imported_visitors"
+ )
+
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true&interval=week"
+ )
+
+ assert %{"plot" => plot, "imported_source" => "Google Analytics"} = json_response(conn, 200)
+ assert Enum.count(plot) == 5
+ assert List.first(plot) == 2
+ assert List.last(plot) == 2
+ assert Enum.sum(plot) == 4
+ end
+
test "Sources are imported", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
diff --git a/test/plausible/stats/interval_test.exs b/test/plausible/stats/interval_test.exs
new file mode 100644
index 0000000000..635c4efa28
--- /dev/null
+++ b/test/plausible/stats/interval_test.exs
@@ -0,0 +1,4 @@
+defmodule Plausible.Stats.IntervalTest do
+ use ExUnit.Case, async: true
+ doctest Plausible.Stats.Interval
+end
diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
index 81def8a272..8d80e95afc 100644
--- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
@@ -383,4 +383,273 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert List.first(plot) == 200
end
end
+
+ describe "GET /api/stats/main-graph - varying intervals" do
+ setup [:create_user, :log_in, :create_new_site]
+
+ test "displays visitors for a month on an hourly scale", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-01 01:01:00])
+ ])
+
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=visitors&interval=hour"
+ )
+
+ assert %{"plot" => plot} = json_response(conn, 200)
+
+ assert Enum.count(plot) == 721
+ assert List.first(plot) == 1
+ assert Enum.at(plot, 1) == 1
+ end
+
+ test "displays visitors for a day on a minute scale", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-01 00:15:01]),
+ build(:pageview, timestamp: ~N[2021-01-01 00:15:02])
+ ])
+
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01&metric=visitors&interval=minute"
+ )
+
+ assert %{"plot" => plot} = json_response(conn, 200)
+
+ assert Enum.count(plot) == 1440
+ assert List.first(plot) == 1
+ assert Enum.at(plot, 15) == 2
+ end
+
+ test "displays visitors for date range on a minute scale", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-01 00:15:01]),
+ build(:pageview, timestamp: ~N[2021-01-01 00:15:02]),
+ build(:pageview, timestamp: ~N[2021-01-02 00:10:00]),
+ build(:pageview, timestamp: ~N[2021-01-02 00:11:01]),
+ build(:pageview, timestamp: ~N[2021-01-02 01:00:02]),
+ build(:pageview, timestamp: ~N[2021-01-04 03:10:00]),
+ build(:pageview, timestamp: ~N[2021-01-04 04:11:01]),
+ build(:pageview, timestamp: ~N[2021-01-04 05:00:02])
+ ])
+
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/main-graph?period=custom&from=2021-01-01&to=2021-01-04&metric=visitors&interval=minute"
+ )
+
+ assert %{"plot" => plot} = json_response(conn, 200)
+
+ assert Enum.count(plot) == 4321
+ assert List.first(plot) == 1
+ assert Enum.at(plot, 15) == 2
+ assert Enum.at(plot, 1450) == 1
+ end
+
+ test "displays visitors for 6mo on a day scale", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-15 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-15 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-02-15 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-06-30 01:00:00])
+ ])
+
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/main-graph?period=6mo&date=2021-06-01&metric=visitors&interval=date"
+ )
+
+ assert %{"plot" => plot} = json_response(conn, 200)
+
+ assert Enum.count(plot) == 181
+ assert List.first(plot) == 1
+ assert Enum.at(plot, 14) == 2
+ assert Enum.at(plot, 45) == 1
+ assert List.last(plot) == 1
+ end
+
+ test "displays visitors for a custom period on a monthly scale", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-15 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-02-15 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-06-01 00:00:00])
+ ])
+
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/main-graph?period=custom&from=2021-01-01&to=2021-06-30&metric=visitors&interval=month"
+ )
+
+ assert %{"plot" => plot} = json_response(conn, 200)
+
+ assert Enum.count(plot) == 6
+ assert List.first(plot) == 2
+ assert Enum.at(plot, 1) == 1
+ assert List.last(plot) == 1
+ end
+
+ test "returns error when requesting an interval longer than the time period", %{
+ conn: conn,
+ site: site
+ } do
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01&metric=visitors&interval=month"
+ )
+
+ assert %{
+ "error" =>
+ "Invalid combination of interval and period. Interval must be smaller than the selected period, e.g. `period=day,interval=minute`"
+ } == json_response(conn, 400)
+ end
+
+ test "returns error when the interval is not valid", %{
+ conn: conn,
+ site: site
+ } do
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01&metric=visitors&interval=biweekly"
+ )
+
+ assert %{
+ "error" =>
+ "Invalid value for interval. Accepted values are: minute, hour, date, week, month"
+ } == json_response(conn, 400)
+ end
+
+ test "displays visitors for a month on a weekly scale", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-01 00:15:01]),
+ build(:pageview, timestamp: ~N[2021-01-05 00:15:02])
+ ])
+
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=visitors&interval=week"
+ )
+
+ assert %{"plot" => plot} = json_response(conn, 200)
+
+ assert Enum.count(plot) == 5
+ assert List.first(plot) == 2
+ assert Enum.at(plot, 1) == 1
+ end
+
+ test "shows imperfect week-split month on week scale with full week indicators", %{
+ conn: conn,
+ site: site
+ } do
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/main-graph?period=month&metric=visitors&interval=week&date=2021-09-01"
+ )
+
+ assert %{"labels" => labels, "full_intervals" => full_intervals} = json_response(conn, 200)
+
+ assert labels == ["2021-09-01", "2021-09-06", "2021-09-13", "2021-09-20", "2021-09-27"]
+
+ assert full_intervals == %{
+ "2021-09-01" => false,
+ "2021-09-06" => true,
+ "2021-09-13" => true,
+ "2021-09-20" => true,
+ "2021-09-27" => false
+ }
+ end
+
+ test "shows half-perfect week-split month on week scale with full week indicators", %{
+ conn: conn,
+ site: site
+ } do
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/main-graph?period=month&metric=visitors&interval=week&date=2021-10-01"
+ )
+
+ assert %{"labels" => labels, "full_intervals" => full_intervals} = json_response(conn, 200)
+
+ assert labels == ["2021-10-01", "2021-10-04", "2021-10-11", "2021-10-18", "2021-10-25"]
+
+ assert full_intervals == %{
+ "2021-10-01" => false,
+ "2021-10-04" => true,
+ "2021-10-11" => true,
+ "2021-10-18" => true,
+ "2021-10-25" => true
+ }
+ end
+
+ test "shows perfect week-split range on week scale with full week indicators", %{
+ conn: conn,
+ site: site
+ } do
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/main-graph?period=custom&metric=visitors&interval=week&from=2020-12-21&to=2021-02-07"
+ )
+
+ assert %{"labels" => labels, "full_intervals" => full_intervals} = json_response(conn, 200)
+
+ assert labels == [
+ "2020-12-21",
+ "2020-12-28",
+ "2021-01-04",
+ "2021-01-11",
+ "2021-01-18",
+ "2021-01-25",
+ "2021-02-01"
+ ]
+
+ assert full_intervals == %{
+ "2020-12-21" => true,
+ "2020-12-28" => true,
+ "2021-01-04" => true,
+ "2021-01-11" => true,
+ "2021-01-18" => true,
+ "2021-01-25" => true,
+ "2021-02-01" => true
+ }
+ end
+
+ test "shows imperfect month-split period on month scale with full month indicators", %{
+ conn: conn,
+ site: site
+ } do
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/main-graph?period=custom&metric=visitors&interval=month&from=2021-09-06&to=2021-12-13"
+ )
+
+ assert %{"labels" => labels, "full_intervals" => full_intervals} = json_response(conn, 200)
+
+ assert labels == ["2021-09-01", "2021-10-01", "2021-11-01", "2021-12-01"]
+
+ assert full_intervals == %{
+ "2021-09-01" => false,
+ "2021-10-01" => true,
+ "2021-11-01" => true,
+ "2021-12-01" => false
+ }
+ end
+ end
end
diff --git a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs
index b5513f84ba..ff0dbaab00 100644
--- a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs
@@ -299,7 +299,10 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
"/api/stats/#{site.domain}/suggestions/prop_value?period=all&date=CLEVER_SECURITY_RESEARCH&filters=#{filters}"
)
- assert json_response(conn, 400) == %{"error" => "input validation error"}
+ assert json_response(conn, 400) == %{
+ "error" =>
+ "Failed to parse date argument. Only ISO 8601 dates are allowed, e.g. `2019-09-07`, `2020-01-01`"
+ }
end
end
end