Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Add API support for intervals (#2417)
Vini Brasil9c98a3f2e82d
diff --git a/lib/plausible/purge.ex b/lib/plausible/purge.exindex 2a778a2b09..3bb91c9a03 100644--- a/lib/plausible/purge.ex+++ b/lib/plausible/purge.ex@@ -5,7 +5,7 @@ defmodule Plausible.Purge doStats are stored on Clickhouse, and unlike other databases data deletion isdone asynchronously.- - [Clickhouse `ALTER TABLE ... DELETE` Statement`](https://clickhouse.com/docs/en/sql-reference/statements/alter/delete)+ - [Clickhouse `ALTER TABLE ... DELETE` Statement](https://clickhouse.com/docs/en/sql-reference/statements/alter/delete)- [Synchronicity of `ALTER` Queries](https://clickhouse.com/docs/en/sql-reference/statements/alter/#synchronicity-of-alter-queries)"""diff --git a/lib/plausible/stats/fragments.ex b/lib/plausible/stats/fragments.exindex 692233f107..19808288d2 100644--- a/lib/plausible/stats/fragments.ex+++ b/lib/plausible/stats/fragments.ex@@ -35,6 +35,68 @@ defmodule Plausible.Stats.Fragments doendend+ @doc """+ Converts time or date and time to the specified timezone.++ Reference: https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#totimezone+ """+ defmacro to_timezone(date, timezone) do+ quote do+ fragment("toTimeZone(?, ?)", unquote(date), unquote(timezone))+ end+ end++ @doc """+ Returns the weekstart for `date`. If the weekstart is before the `not_before`+ boundary, `not_before` is returned.++ ## Examples++ In this pseudo-code example, the fragment returns the weekstart. The+ `not_before` boundary is set to the past Saturday, which is before the+ weekstart, therefore the cap does not apply.++ iex> this_wednesday = ~D[2022-11-09]+ ...> past_saturday = ~D[2022-11-05]+ ...> weekstart_not_before(this_wednesday, past_saturday)+ ~D[2022-11-07]+++ In this other example, the fragment returns Tuesday and not the weekstart.+ The `not_before` boundary is set to Tuesday, which is past the weekstart,+ therefore the cap applies.++ iex> this_wednesday = ~D[2022-11-09]+ ...> this_tuesday = ~D[2022-11-08]+ ...> weekstart_not_before(this_wednesday, this_tuesday)+ ~D[2022-11-08]++ """+ defmacro weekstart_not_before(date, not_before) do+ quote do+ fragment(+ "if(toMonday(?) < toDate(?), toDate(?), toMonday(?))",+ unquote(date),+ unquote(not_before),+ unquote(not_before),+ unquote(date)+ )+ end+ end++ @doc """+ Same as Plausible.Stats.Fragments.weekstart_not_before/2 but converts dates to+ the specified timezone.+ """+ defmacro weekstart_not_before(date, not_before, timezone) do+ quote do+ weekstart_not_before(+ to_timezone(unquote(date), unquote(timezone)),+ unquote(not_before)+ )+ end+ end+defmacro __using__(_) doquote doimport Plausible.Stats.Fragmentsdiff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.exindex 9888bba5f6..37d1de6753 100644--- a/lib/plausible/stats/imported.ex+++ b/lib/plausible/stats/imported.ex@@ -2,6 +2,8 @@ defmodule Plausible.Stats.Imported douse Plausible.ClickhouseRepoalias Plausible.Stats.Queryimport Ecto.Query+ import Plausible.Stats.Base+ import Plausible.Stats.Fragments@no_ref "Direct / None"@@ -21,7 +23,7 @@ defmodule Plausible.Stats.Imported doselect: %{})|> select_imported_metrics(metrics)- |> apply_interval(query)+ |> apply_interval(query, site.timezone)from(s in Ecto.Query.subquery(native_q),full_join: i in subquery(imported_q),@@ -31,13 +33,21 @@ defmodule Plausible.Stats.Imported do|> select_joined_metrics(metrics)end- defp apply_interval(imported_q, %Plausible.Stats.Query{interval: "month"}) do+ defp apply_interval(imported_q, %Plausible.Stats.Query{interval: "month"}, _timezone) doimported_q|> group_by([i], fragment("toStartOfMonth(?)", i.date))|> select_merge([i], %{date: fragment("toStartOfMonth(?)", i.date)})end- defp apply_interval(imported_q, _query) do+ defp apply_interval(imported_q, %Plausible.Stats.Query{interval: "week"} = query, timezone) do+ {first_datetime, _} = utc_boundaries(query, timezone)++ imported_q+ |> group_by([i], weekstart_not_before(i.date, ^first_datetime))+ |> select_merge([i], %{date: weekstart_not_before(i.date, ^first_datetime)})+ end++ defp apply_interval(imported_q, _query, _timezone) doimported_q|> group_by([i], i.date)|> select_merge([i], %{date: i.date})diff --git a/lib/plausible/stats/interval.ex b/lib/plausible/stats/interval.exnew file mode 100644index 0000000000..75f55f78bc--- /dev/null+++ b/lib/plausible/stats/interval.ex@@ -0,0 +1,78 @@+defmodule Plausible.Stats.Interval do+ @moduledoc """+ Collection of functions to work with intervals.++ The interval of a query defines the granularity of the data. You can think of+ it as a `GROUP BY` clause. Possible values are `minute`, `hour`, `date`,+ `week`, and `month`.+ """++ @type t() :: String.t()+ @typep period() :: String.t()++ @intervals ~w(minute hour date week month)+ @spec list() :: [t()]+ def list, do: @intervals++ @spec valid?(term()) :: boolean()+ def valid?(interval) do+ interval in @intervals+ end++ @spec default_for_period(period()) :: t()+ @doc """+ Returns the suggested interval for the given time period.++ ## Examples++ iex> Plausible.Stats.Interval.default_for_period("7d")+ "date"++ """+ def default_for_period(period) do+ case period do+ "realtime" -> "minute"+ "day" -> "hour"+ period when period in ["custom", "7d", "30d", "month"] -> "date"+ period when period in ["6mo", "12mo", "year"] -> "month"+ end+ end++ @allowed_intervals_for_period %{+ "realtime" => ["minute"],+ "day" => ["minute", "hour"],+ "7d" => ["minute", "hour", "date"],+ "month" => ["minute", "hour", "date", "week"],+ "30d" => ["minute", "hour", "date", "week"],+ "6mo" => ["minute", "hour", "date", "week", "month"],+ "12mo" => ["minute", "hour", "date", "week", "month"],+ "year" => ["minute", "hour", "date", "week", "month"],+ "custom" => ["minute", "hour", "date", "week", "month"],+ "all" => ["minute", "hour", "date", "week", "month"]+ }++ @spec allowed_for_period?(period(), t()) :: boolean()+ @doc """+ Returns whether the given interval is valid for a time period.++ Intervals longer than periods are not supported, e.g. current month stats with+ a month interval, or today stats with a week interval.++ ## Examples+++ iex> Plausible.Stats.Interval.allowed_for_period?("month", "date")+ true++ iex> Plausible.Stats.Interval.allowed_for_period?("30d", "month")+ false++ iex> Plausible.Stats.Interval.allowed_for_period?("realtime", "week")+ false++ """+ def allowed_for_period?(period, interval) do+ allowed = Map.get(@allowed_intervals_for_period, period, [])+ interval in allowed+ end+enddiff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.exindex c645e68af0..a53c689e54 100644--- a/lib/plausible/stats/query.ex+++ b/lib/plausible/stats/query.ex@@ -7,7 +7,7 @@ defmodule Plausible.Stats.Query doinclude_imported: false@default_sample_threshold 20_000_000- alias Plausible.Stats.FilterParser+ alias Plausible.Stats.{FilterParser, Interval}def shift_back(%__MODULE__{period: "year"} = query, site) do# Querying current year to date@@ -67,7 +67,7 @@ defmodule Plausible.Stats.Query do%__MODULE__{period: "realtime",- interval: "minute",+ interval: params["interval"] || Interval.default_for_period(params["period"]),date_range: Date.range(date, date),filters: FilterParser.parse_filters(params["filters"]),sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold),@@ -81,7 +81,7 @@ defmodule Plausible.Stats.Query do%__MODULE__{period: "day",date_range: Date.range(date, date),- interval: "hour",+ interval: params["interval"] || Interval.default_for_period(params["period"]),filters: FilterParser.parse_filters(params["filters"]),sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)}@@ -95,7 +95,7 @@ defmodule Plausible.Stats.Query do%__MODULE__{period: "7d",date_range: Date.range(start_date, end_date),- interval: "date",+ interval: params["interval"] || Interval.default_for_period(params["period"]),filters: FilterParser.parse_filters(params["filters"]),sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)}@@ -109,7 +109,7 @@ defmodule Plausible.Stats.Query do%__MODULE__{period: "30d",date_range: Date.range(start_date, end_date),- interval: "date",+ interval: params["interval"] || Interval.default_for_period(params["period"]),filters: FilterParser.parse_filters(params["filters"]),sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)}@@ -125,7 +125,7 @@ defmodule Plausible.Stats.Query do%__MODULE__{period: "month",date_range: Date.range(start_date, end_date),- interval: "date",+ interval: params["interval"] || Interval.default_for_period(params["period"]),filters: FilterParser.parse_filters(params["filters"]),sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)}@@ -144,7 +144,7 @@ defmodule Plausible.Stats.Query do%__MODULE__{period: "6mo",date_range: Date.range(start_date, end_date),- interval: Map.get(params, "interval", "month"),+ interval: params["interval"] || Interval.default_for_period(params["period"]),filters: FilterParser.parse_filters(params["filters"]),sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)}@@ -163,7 +163,7 @@ defmodule Plausible.Stats.Query do%__MODULE__{period: "12mo",date_range: Date.range(start_date, end_date),- interval: Map.get(params, "interval", "month"),+ interval: params["interval"] || Interval.default_for_period(params["period"]),filters: FilterParser.parse_filters(params["filters"]),sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)}@@ -180,7 +180,7 @@ defmodule Plausible.Stats.Query do%__MODULE__{period: "year",date_range: Date.range(start_date, end_date),- interval: Map.get(params, "interval", "month"),+ interval: params["interval"] || Interval.default_for_period(params["period"]),filters: FilterParser.parse_filters(params["filters"]),sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)}@@ -199,7 +199,7 @@ defmodule Plausible.Stats.Query do"period" => "custom","from" => Date.to_iso8601(start_date),"to" => Date.to_iso8601(now),- "interval" => "month"+ "interval" => params["interval"] || "month"}))|> Map.put(:period, "all")@@ -211,7 +211,7 @@ defmodule Plausible.Stats.Query do"period" => "custom","from" => Date.to_iso8601(start_date),"to" => Date.to_iso8601(now),- "interval" => "date"+ "interval" => params["interval"] || "date"}))|> Map.put(:period, "all")@@ -240,7 +240,7 @@ defmodule Plausible.Stats.Query do%__MODULE__{period: "custom",date_range: Date.range(from_date, to_date),- interval: Map.get(params, "interval", "date"),+ interval: params["interval"] || Interval.default_for_period(params["period"]),filters: FilterParser.parse_filters(params["filters"]),sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)}diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.exindex 694a7dd21a..dfd94d7058 100644--- a/lib/plausible/stats/timeseries.ex+++ b/lib/plausible/stats/timeseries.ex@@ -50,7 +50,7 @@ defmodule Plausible.Stats.Timeseries do|> ClickhouseRepo.all()end- def buckets(%Query{interval: "month"} = query) do+ defp buckets(%Query{interval: "month"} = query) don_buckets = Timex.diff(query.date_range.last, query.date_range.first, :months)Enum.map(n_buckets..0, fn shift ->@@ -60,23 +60,59 @@ defmodule Plausible.Stats.Timeseries doend)end- def buckets(%Query{interval: "date"} = query) do+ defp buckets(%Query{interval: "week"} = query) do+ n_buckets = Timex.diff(query.date_range.last, query.date_range.first, :weeks)++ Enum.map(0..n_buckets, fn shift ->+ query.date_range.first+ |> Timex.shift(weeks: shift)+ |> date_or_weekstart(query)+ end)+ end++ defp buckets(%Query{interval: "date"} = query) doEnum.into(query.date_range, [])end- def buckets(%Query{interval: "hour"} = query) do- Enum.map(0..23, fn step ->- Timex.to_datetime(query.date_range.first)+ @full_day_in_hours 23+ defp buckets(%Query{interval: "hour"} = query) do+ n_buckets =+ if query.date_range.first == query.date_range.last do+ @full_day_in_hours+ else+ Timex.diff(query.date_range.last, query.date_range.first, :hours)+ end++ Enum.map(0..n_buckets, fn step ->+ query.date_range.first+ |> Timex.to_datetime()|> Timex.shift(hours: step)|> Timex.format!("{YYYY}-{0M}-{0D} {h24}:{m}:{s}")end)end- def buckets(%Query{period: "30m", interval: "minute"}) do+ defp buckets(%Query{period: "30m", interval: "minute"}) doEnum.into(-30..-1, [])end- def select_bucket(q, site, %Query{interval: "month"}) do+ @full_day_in_minutes 1439+ defp buckets(%Query{interval: "minute"} = query) do+ n_buckets =+ if query.date_range.first == query.date_range.last do+ @full_day_in_minutes+ else+ Timex.diff(query.date_range.last, query.date_range.first, :minutes)+ end++ Enum.map(0..n_buckets, fn step ->+ query.date_range.first+ |> Timex.to_datetime()+ |> Timex.shift(minutes: step)+ |> Timex.format!("{YYYY}-{0M}-{0D} {h24}:{m}:{s}")+ end)+ end++ defp select_bucket(q, site, %Query{interval: "month"}) dofrom(e in q,group_by: fragment("toStartOfMonth(toTimeZone(?, ?))", e.timestamp, ^site.timezone),@@ -87,7 +123,18 @@ defmodule Plausible.Stats.Timeseries do)end- def select_bucket(q, site, %Query{interval: "date"}) do+ defp select_bucket(q, site, %Query{interval: "week"} = query) do+ {first_datetime, _} = utc_boundaries(query, site.timezone)++ from(+ e in q,+ select_merge: %{date: weekstart_not_before(e.timestamp, ^first_datetime, ^site.timezone)},+ group_by: weekstart_not_before(e.timestamp, ^first_datetime, ^site.timezone),+ order_by: weekstart_not_before(e.timestamp, ^first_datetime, ^site.timezone)+ )+ end++ defp select_bucket(q, site, %Query{interval: "date"}) dofrom(e in q,group_by: fragment("toDate(toTimeZone(?, ?))", e.timestamp, ^site.timezone),@@ -98,7 +145,7 @@ defmodule Plausible.Stats.Timeseries do)end- def select_bucket(q, site, %Query{interval: "hour"}) do+ defp select_bucket(q, site, %Query{interval: "hour"}) dofrom(e in q,group_by: fragment("toStartOfHour(toTimeZone(?, ?))", e.timestamp, ^site.timezone),@@ -109,7 +156,7 @@ defmodule Plausible.Stats.Timeseries do)end- def select_bucket(q, _site, %Query{interval: "minute"}) do+ defp select_bucket(q, _site, %Query{interval: "minute", period: "30m"}) dofrom(e in q,group_by: fragment("dateDiff('minute', now(), ?)", e.timestamp),@@ -120,6 +167,27 @@ defmodule Plausible.Stats.Timeseries do)end+ defp select_bucket(q, site, %Query{interval: "minute"}) do+ from(+ e in q,+ group_by: fragment("toStartOfMinute(toTimeZone(?, ?))", e.timestamp, ^site.timezone),+ order_by: fragment("toStartOfMinute(toTimeZone(?, ?))", e.timestamp, ^site.timezone),+ select_merge: %{+ date: fragment("toStartOfMinute(toTimeZone(?, ?))", e.timestamp, ^site.timezone)+ }+ )+ end++ defp date_or_weekstart(date, query) do+ weekstart = Timex.beginning_of_week(date)++ if Enum.member?(query.date_range, weekstart) do+ weekstart+ else+ date+ end+ end+defp empty_row(date, metrics) doEnum.reduce(metrics, %{date: date}, fn metric, row ->case metric dodiff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex 04cf3237db..cfac6aad84 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -5,6 +5,88 @@ defmodule PlausibleWeb.Api.StatsController doalias Plausible.Statsalias Plausible.Stats.{Query, Filters}+ @doc """+ Returns a time-series based on given parameters.++ ## Parameters++ This API accepts the following parameters:++ * `period` - x-axis of the graph, e.g. `12mo`, `day`, `custom`.++ * `metric` - y-axis of the graph, e.g. `visits`, `visitors`, `pageviews`.+ See the Stats API ["Metrics"](https://plausible.io/docs/stats-api#metrics)+ section for more details. Defaults to `visitors`.++ * `interval` - granularity of the time-series data. You can think of it as+ a `GROUP BY` clause. Possible values are `minute`, `hour`, `date`, `week`,+ and `month`. The default depends on the `period` parameter. Check+ `Plausible.Query.from/2` for each default.++ * `filters` - optional filters to drill down data. See the Stats API+ ["Filtering"](https://plausible.io/docs/stats-api#filtering) section for+ more details.++ * `with_imported` - boolean indicating whether to include Google Analytics+ imported data or not. Defaults to `false`.++ Full example:+ ```elixir+ %{+ "from" => "2021-09-06",+ "interval" => "month",+ "metric" => "visitors",+ "period" => "custom",+ "to" => "2021-12-13"+ }+ ```++ ## Response++ Returns a map with the following keys:++ * `plot` - list of values for the requested metric representing the y-axis+ of the graph.++ * `labels` - list of date times representing the x-axis of the graph.++ * `present_index` - index of the element representing the current date in+ `labels` and `plot` lists.++ * `interval` - the interval used for querying.++ * `with_imported` - boolean indicating whether the Google Analytics data+ was queried or not.++ * `imported_source` - the source of the imported data, when applicable.+ Currently only Google Analytics is supported.++ * `full_intervals` - map of dates indicating whether the interval has been+ cut off by the requested date range or not. For example, if looking at a+ month week-by-week, some weeks may be cut off by the month boundaries.+ It's useful to adjust the graph display slightly in case the interval is+ not 'full' so that the user understands why the numbers might be lower for+ those partial periods.++ Full example:+ ```elixir+ %{+ "full_intervals" => %{+ "2021-09-01" => false,+ "2021-10-01" => true,+ "2021-11-01" => true,+ "2021-12-01" => false+ },+ "imported_source" => nil,+ "interval" => "month",+ "labels" => ["2021-09-01", "2021-10-01", "2021-11-01", "2021-12-01"],+ "plot" => [0, 0, 0, 0],+ "present_index" => nil,+ "with_imported" => false+ }+ ```++ """def main_graph(conn, params) dosite = conn.assigns[:site]@@ -35,6 +117,7 @@ defmodule PlausibleWeb.Api.StatsController dolabels = Enum.map(timeseries_result, fn row -> row[:date] end)present_index = present_index_for(site, query, labels)+ full_intervals = build_full_intervals(query, labels)json(conn, %{plot: plot,@@ -42,14 +125,40 @@ defmodule PlausibleWeb.Api.StatsController dopresent_index: present_index,interval: query.interval,with_imported: query.include_imported,- imported_source: site.imported_data && site.imported_data.source+ imported_source: site.imported_data && site.imported_data.source,+ full_intervals: full_intervals})else- _ ->- bad_request(conn)+ {:error, message} when is_binary(message) -> bad_request(conn, message)+ end+ end++ defp build_full_intervals(%{interval: "week", date_range: range}, labels) do+ for label <- labels, into: %{} do+ interval_start = Timex.beginning_of_week(label)+ interval_end = Timex.end_of_week(label)++ within_interval? = Enum.member?(range, interval_start) && Enum.member?(range, interval_end)++ {label, within_interval?}+ end+ end++ defp build_full_intervals(%{interval: "month", date_range: range}, labels) do+ for label <- labels, into: %{} do+ interval_start = Timex.beginning_of_month(label)+ interval_end = Timex.end_of_month(label)++ within_interval? = Enum.member?(range, interval_start) && Enum.member?(range, interval_end)++ {label, within_interval?}endend+ defp build_full_intervals(_query, _labels) do+ nil+ end+def top_stats(conn, params) dosite = conn.assigns[:site]@@ -66,8 +175,7 @@ defmodule PlausibleWeb.Api.StatsController doimported_source: site.imported_data && site.imported_data.source})else- _ ->- bad_request(conn)+ {:error, message} when is_binary(message) -> bad_request(conn, message)endend@@ -87,6 +195,14 @@ defmodule PlausibleWeb.Api.StatsController doEnum.find_index(dates, &(&1 == current_date))+ "week" ->+ current_date =+ Timex.now(site.timezone)+ |> Timex.to_date()+ |> date_or_weekstart(query)++ Enum.find_index(dates, &(&1 == current_date))+"month" ->current_date =Timex.now(site.timezone)@@ -96,7 +212,21 @@ defmodule PlausibleWeb.Api.StatsController doEnum.find_index(dates, &(&1 == current_date))"minute" ->- nil+ current_date =+ Timex.now(site.timezone)+ |> Timex.format!("{YYYY}-{0M}-{0D} {h24}:{0m}:00")++ Enum.find_index(dates, &(&1 == current_date))+ end+ end++ defp date_or_weekstart(date, query) do+ weekstart = Timex.beginning_of_week(date)++ if Enum.member?(query.date_range, weekstart) do+ weekstart+ else+ dateendend@@ -935,8 +1065,7 @@ defmodule PlausibleWeb.Api.StatsController dojson(conn, Stats.filter_suggestions(site, query, params["filter_name"], params["q"]))else- _ ->- bad_request(conn)+ {:error, message} when is_binary(message) -> bad_request(conn, message)endend@@ -1046,19 +1175,57 @@ defmodule PlausibleWeb.Api.StatsController doendend- defp validate_params(%{"date" => date}) do- with {:ok, _} <- Date.from_iso8601(date) do+ defp validate_params(params) do+ with :ok <- validate_date(params),+ :ok <- validate_interval(params),+ do: validate_interval_granularity(params)+ end++ defp validate_date(params) do+ with %{"date" => date} <- params,+ {:ok, _} <- Date.from_iso8601(date) do+ :ok+ else+ %{} ->+ :ok++ {:error, _reason} ->+ {:error,+ "Failed to parse date argument. Only ISO 8601 dates are allowed, e.g. `2019-09-07`, `2020-01-01`"}+ end+ end++ defp validate_interval(params) do+ with %{"interval" => interval} <- params,+ true <- Plausible.Stats.Interval.valid?(interval) do:ok+ else+ %{} ->+ :ok++ false ->+ values = Enum.join(Plausible.Stats.Interval.list(), ", ")+ {:error, "Invalid value for interval. Accepted values are: #{values}"}endend- defp validate_params(_) do- :ok+ defp validate_interval_granularity(params) do+ with %{"interval" => interval, "period" => period} <- params,+ true <- Plausible.Stats.Interval.allowed_for_period?(period, interval) do+ :ok+ else+ %{} ->+ :ok++ false ->+ {:error,+ "Invalid combination of interval and period. Interval must be smaller than the selected period, e.g. `period=day,interval=minute`"}+ endend- defp bad_request(conn) do+ defp bad_request(conn, message) doconn|> put_status(400)- |> json(%{error: "input validation error"})+ |> json(%{error: message})endenddiff --git a/test/plausible/imported/imported_test.exs b/test/plausible/imported/imported_test.exsindex 68700af95f..3e58aeb1dc 100644--- a/test/plausible/imported/imported_test.exs+++ b/test/plausible/imported/imported_test.exs@@ -60,6 +60,52 @@ defmodule Plausible.ImportedTest doassert Enum.sum(plot) == 4end+ test "returns data grouped by week", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-31 00:00:00])+ ])++ import_data(+ [+ %{+ dimensions: %{"ga:date" => "20210101"},+ metrics: %{+ "ga:users" => "1",+ "ga:pageviews" => "1",+ "ga:bounces" => "0",+ "ga:sessions" => "1",+ "ga:sessionDuration" => "60"+ }+ },+ %{+ dimensions: %{"ga:date" => "20210131"},+ metrics: %{+ "ga:users" => "1",+ "ga:pageviews" => "1",+ "ga:bounces" => "0",+ "ga:sessions" => "1",+ "ga:sessionDuration" => "60"+ }+ }+ ],+ site.id,+ "imported_visitors"+ )++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true&interval=week"+ )++ assert %{"plot" => plot, "imported_source" => "Google Analytics"} = json_response(conn, 200)+ assert Enum.count(plot) == 5+ assert List.first(plot) == 2+ assert List.last(plot) == 2+ assert Enum.sum(plot) == 4+ end+test "Sources are imported", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview,diff --git a/test/plausible/stats/interval_test.exs b/test/plausible/stats/interval_test.exsnew file mode 100644index 0000000000..635c4efa28--- /dev/null+++ b/test/plausible/stats/interval_test.exs@@ -0,0 +1,4 @@+defmodule Plausible.Stats.IntervalTest do+ use ExUnit.Case, async: true+ doctest Plausible.Stats.Interval+enddiff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exsindex 81def8a272..8d80e95afc 100644--- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs@@ -383,4 +383,273 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest doassert List.first(plot) == 200endend++ describe "GET /api/stats/main-graph - varying intervals" do+ setup [:create_user, :log_in, :create_new_site]++ test "displays visitors for a month on an hourly scale", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-01 01:01:00])+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=visitors&interval=hour"+ )++ assert %{"plot" => plot} = json_response(conn, 200)++ assert Enum.count(plot) == 721+ assert List.first(plot) == 1+ assert Enum.at(plot, 1) == 1+ end++ test "displays visitors for a day on a minute scale", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:15:01]),+ build(:pageview, timestamp: ~N[2021-01-01 00:15:02])+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01&metric=visitors&interval=minute"+ )++ assert %{"plot" => plot} = json_response(conn, 200)++ assert Enum.count(plot) == 1440+ assert List.first(plot) == 1+ assert Enum.at(plot, 15) == 2+ end++ test "displays visitors for date range on a minute scale", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:15:01]),+ build(:pageview, timestamp: ~N[2021-01-01 00:15:02]),+ build(:pageview, timestamp: ~N[2021-01-02 00:10:00]),+ build(:pageview, timestamp: ~N[2021-01-02 00:11:01]),+ build(:pageview, timestamp: ~N[2021-01-02 01:00:02]),+ build(:pageview, timestamp: ~N[2021-01-04 03:10:00]),+ build(:pageview, timestamp: ~N[2021-01-04 04:11:01]),+ build(:pageview, timestamp: ~N[2021-01-04 05:00:02])+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=custom&from=2021-01-01&to=2021-01-04&metric=visitors&interval=minute"+ )++ assert %{"plot" => plot} = json_response(conn, 200)++ assert Enum.count(plot) == 4321+ assert List.first(plot) == 1+ assert Enum.at(plot, 15) == 2+ assert Enum.at(plot, 1450) == 1+ end++ test "displays visitors for 6mo on a day scale", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-15 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-15 00:00:00]),+ build(:pageview, timestamp: ~N[2021-02-15 00:00:00]),+ build(:pageview, timestamp: ~N[2021-06-30 01:00:00])+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=6mo&date=2021-06-01&metric=visitors&interval=date"+ )++ assert %{"plot" => plot} = json_response(conn, 200)++ assert Enum.count(plot) == 181+ assert List.first(plot) == 1+ assert Enum.at(plot, 14) == 2+ assert Enum.at(plot, 45) == 1+ assert List.last(plot) == 1+ end++ test "displays visitors for a custom period on a monthly scale", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-15 00:00:00]),+ build(:pageview, timestamp: ~N[2021-02-15 00:00:00]),+ build(:pageview, timestamp: ~N[2021-06-01 00:00:00])+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=custom&from=2021-01-01&to=2021-06-30&metric=visitors&interval=month"+ )++ assert %{"plot" => plot} = json_response(conn, 200)++ assert Enum.count(plot) == 6+ assert List.first(plot) == 2+ assert Enum.at(plot, 1) == 1+ assert List.last(plot) == 1+ end++ test "returns error when requesting an interval longer than the time period", %{+ conn: conn,+ site: site+ } do+ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01&metric=visitors&interval=month"+ )++ assert %{+ "error" =>+ "Invalid combination of interval and period. Interval must be smaller than the selected period, e.g. `period=day,interval=minute`"+ } == json_response(conn, 400)+ end++ test "returns error when the interval is not valid", %{+ conn: conn,+ site: site+ } do+ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01&metric=visitors&interval=biweekly"+ )++ assert %{+ "error" =>+ "Invalid value for interval. Accepted values are: minute, hour, date, week, month"+ } == json_response(conn, 400)+ end++ test "displays visitors for a month on a weekly scale", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:15:01]),+ build(:pageview, timestamp: ~N[2021-01-05 00:15:02])+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=visitors&interval=week"+ )++ assert %{"plot" => plot} = json_response(conn, 200)++ assert Enum.count(plot) == 5+ assert List.first(plot) == 2+ assert Enum.at(plot, 1) == 1+ end++ test "shows imperfect week-split month on week scale with full week indicators", %{+ conn: conn,+ site: site+ } do+ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=month&metric=visitors&interval=week&date=2021-09-01"+ )++ assert %{"labels" => labels, "full_intervals" => full_intervals} = json_response(conn, 200)++ assert labels == ["2021-09-01", "2021-09-06", "2021-09-13", "2021-09-20", "2021-09-27"]++ assert full_intervals == %{+ "2021-09-01" => false,+ "2021-09-06" => true,+ "2021-09-13" => true,+ "2021-09-20" => true,+ "2021-09-27" => false+ }+ end++ test "shows half-perfect week-split month on week scale with full week indicators", %{+ conn: conn,+ site: site+ } do+ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=month&metric=visitors&interval=week&date=2021-10-01"+ )++ assert %{"labels" => labels, "full_intervals" => full_intervals} = json_response(conn, 200)++ assert labels == ["2021-10-01", "2021-10-04", "2021-10-11", "2021-10-18", "2021-10-25"]++ assert full_intervals == %{+ "2021-10-01" => false,+ "2021-10-04" => true,+ "2021-10-11" => true,+ "2021-10-18" => true,+ "2021-10-25" => true+ }+ end++ test "shows perfect week-split range on week scale with full week indicators", %{+ conn: conn,+ site: site+ } do+ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=custom&metric=visitors&interval=week&from=2020-12-21&to=2021-02-07"+ )++ assert %{"labels" => labels, "full_intervals" => full_intervals} = json_response(conn, 200)++ assert labels == [+ "2020-12-21",+ "2020-12-28",+ "2021-01-04",+ "2021-01-11",+ "2021-01-18",+ "2021-01-25",+ "2021-02-01"+ ]++ assert full_intervals == %{+ "2020-12-21" => true,+ "2020-12-28" => true,+ "2021-01-04" => true,+ "2021-01-11" => true,+ "2021-01-18" => true,+ "2021-01-25" => true,+ "2021-02-01" => true+ }+ end++ test "shows imperfect month-split period on month scale with full month indicators", %{+ conn: conn,+ site: site+ } do+ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=custom&metric=visitors&interval=month&from=2021-09-06&to=2021-12-13"+ )++ assert %{"labels" => labels, "full_intervals" => full_intervals} = json_response(conn, 200)++ assert labels == ["2021-09-01", "2021-10-01", "2021-11-01", "2021-12-01"]++ assert full_intervals == %{+ "2021-09-01" => false,+ "2021-10-01" => true,+ "2021-11-01" => true,+ "2021-12-01" => false+ }+ end+ endenddiff --git a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exsindex b5513f84ba..ff0dbaab00 100644--- a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs@@ -299,7 +299,10 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do"/api/stats/#{site.domain}/suggestions/prop_value?period=all&date=CLEVER_SECURITY_RESEARCH&filters=#{filters}")- assert json_response(conn, 400) == %{"error" => "input validation error"}+ assert json_response(conn, 400) == %{+ "error" =>+ "Failed to parse date argument. Only ISO 8601 dates are allowed, e.g. `2019-09-07`, `2020-01-01`"+ }endendend