Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Add comparisons to DashboardQueryParser (#5955)
RobertJoonas9aa6a8aa2327
diff --git a/assets/js/types/query-api.d.ts b/assets/js/types/query-api.d.tsindex f244fba28e..4ccee9bb28 100644--- a/assets/js/types/query-api.d.ts+++ b/assets/js/types/query-api.d.ts@@ -157,6 +157,7 @@ export type OrderByEntry = [Metric | SimpleFilterDimensions | CustomPropertyFilterDimensions | TimeDimensions,"asc" | "desc"];+export type ComparisonMode = "previous_period" | "year_over_year";export interface QueryApiSchema {/**@@ -197,25 +198,14 @@ export interface QueryApiSchema {* If set and using `day`, `month` or `year` date_ranges, the query will be trimmed to the current date*/trim_relative_date_range?: boolean;- comparisons?:- | {- mode: "previous_period" | "year_over_year";- /**- * If set and using time:day dimensions, day-of-week of comparison query is matched- */- match_day_of_week?: boolean;- }- | {- mode: "custom";- /**- * If set and using time:day dimensions, day-of-week of comparison query is matched- */- match_day_of_week?: boolean;- /**- * If custom period. A list of two ISO8601 dates or timestamps to compare against.- */- date_range: DateTimeRange | DateRange;- };+ /**+ * If set, executes the same query but over a comparison date range+ */+ compare?: DateRange | ComparisonMode | never;+ /**+ * With the `compare` option, if set and using time:day dimensions, day-of-week of comparison query is matched+ */+ compare_match_day_of_week?: boolean;};pagination?: {/**diff --git a/extra/lib/plausible/stats/consolidated_view.ex b/extra/lib/plausible/stats/consolidated_view.exindex 376cb49be5..6e42871c75 100644--- a/extra/lib/plausible/stats/consolidated_view.ex+++ b/extra/lib/plausible/stats/consolidated_view.ex@@ -1,5 +1,6 @@defmodule Plausible.Stats.ConsolidatedView doalias Plausible.{Site, Stats}+ alias Plausible.Stats.{ParsedQueryParams, QueryBuilder, QueryInclude}require Logger@spec overview_24h(Site.t(), NaiveDateTime.t()) :: map()@@ -41,29 +42,24 @@ defmodule Plausible.Stats.ConsolidatedView dofrom =NaiveDateTime.shift(now, hour: -24)|> DateTime.from_naive!("Etc/UTC")- |> DateTime.to_iso8601()- to = now |> DateTime.from_naive!("Etc/UTC") |> DateTime.to_iso8601()+ to = now |> DateTime.from_naive!("Etc/UTC")c_from =NaiveDateTime.shift(now, hour: -48)|> DateTime.from_naive!("Etc/UTC")- |> DateTime.to_iso8601()c_to =NaiveDateTime.shift(now, hour: -24)|> DateTime.from_naive!("Etc/UTC")- |> DateTime.to_iso8601()-- stats_query =- Stats.Query.parse_and_build!(view, :internal, %{- "site_id" => view.domain,- "metrics" => ["visitors", "visits", "pageviews", "views_per_visit"],- "include" => %{"comparisons" => %{"mode" => "custom", "date_range" => [c_from, c_to]}},- "date_range" => [- from,- to- ]++ {:ok, stats_query} =+ QueryBuilder.build(view, %ParsedQueryParams{+ input_date_range: {:datetime_range, from, to},+ metrics: [:visitors, :visits, :pageviews, :views_per_visit],+ include: %QueryInclude{+ compare: {:datetime_range, c_from, c_to}+ }})%Stats.QueryResult{diff --git a/lib/plausible/stats/api_query_parser.ex b/lib/plausible/stats/api_query_parser.exindex 60fe483037..e7d4736f21 100644--- a/lib/plausible/stats/api_query_parser.ex+++ b/lib/plausible/stats/api_query_parser.ex@@ -3,15 +3,16 @@ defmodule Plausible.Stats.ApiQueryParser douse Plausible- alias Plausible.Stats.{Filters, Metrics, DateTimeRange, JSONSchema}+ alias Plausible.Stats.{Filters, Metrics, JSONSchema}- @default_include %{+ @default_include %Plausible.Stats.QueryInclude{imports: false,imports_meta: false,time_labels: false,total_rows: false,trim_relative_date_range: false,- comparisons: nil,+ compare: nil,+ compare_match_day_of_week: false,legacy_time_on_page_cutoff: nil}@@ -45,21 +46,6 @@ defmodule Plausible.Stats.ApiQueryParser doendend- def parse_date_range_pair(site, [from, to]) when is_binary(from) and is_binary(to) do- with {:ok, date_range} <- date_range_from_date_strings(site, from, to) do- {:ok, date_range |> DateTimeRange.to_timezone("Etc/UTC")}- end- end-- def parse_date_range_pair(_site, unknown), do: {:error, "Invalid date_range '#{i(unknown)}'."}-- defp date_range_from_date_strings(site, from, to) do- with {:ok, from_date} <- Date.from_iso8601(from),- {:ok, to_date} <- Date.from_iso8601(to) do- {:ok, DateTimeRange.new!(from_date, to_date, site.timezone)}- end- end-defp parse_metrics(metrics) when is_list(metrics) doparse_list(metrics, &parse_metric/1)end@@ -275,35 +261,42 @@ defmodule Plausible.Stats.ApiQueryParser dodefp parse_order_direction(entry), do: {:error, "Invalid order_by entry '#{i(entry)}'."}def parse_include(include) when is_map(include) do- with {:ok, include} <- atomize_include_keys(include),- {:ok, include} <- parse_comparison_date_range(include) do- {:ok, Map.merge(@default_include, include)}+ parsed_include_params_or_error =+ include+ |> Enum.reduce_while({:ok, []}, fn {key, value}, {:ok, acc} ->+ case parse_include_entry(key, value) do+ {:ok, parsed_tuple} -> {:cont, {:ok, acc ++ [parsed_tuple]}}+ {:error, msg} -> {:halt, {:error, msg}}+ end+ end)++ with {:ok, parsed_include_params} <- parsed_include_params_or_error do+ {:ok, struct!(@default_include, parsed_include_params)}endenddef parse_include(nil), do: {:ok, @default_include}def parse_include(include), do: {:error, "Invalid include '#{i(include)}'."}- defp atomize_include_keys(map) do- expected_keys =- @default_include- |> Map.keys()- |> Enum.map(&Atom.to_string/1)+ @allowed_include_keys Enum.map(Map.keys(@default_include), &Atom.to_string/1)- if Map.keys(map) |> Enum.all?(&(&1 in expected_keys)) do- {:ok, atomize_keys(map)}- else- {:error, "Invalid include '#{i(map)}'."}+ defp parse_include_entry("compare", value) do+ with {:ok, parsed_compare} <- parse_compare(value) do+ {:ok, {:compare, parsed_compare}}endend- defp parse_comparison_date_range(%{comparisons: %{date_range: date_range}} = include) do- with {:ok, parsed_date_range} <- parse_input_date_range(date_range) do- {:ok, put_in(include, [:comparisons, :date_range], parsed_date_range)}- end+ defp parse_include_entry(key, value) when key in @allowed_include_keys do+ {:ok, {String.to_existing_atom(key), value}}end- defp parse_comparison_date_range(include), do: {:ok, include}+ defp parse_include_entry(key, _value), do: {:error, "Invalid include key'#{i(key)}'."}++ defp parse_compare(false), do: {:ok, nil}+ defp parse_compare("previous_period"), do: {:ok, :previous_period}+ defp parse_compare("year_over_year"), do: {:ok, :year_over_year}+ defp parse_compare([from_date, to_date]), do: parse_date_strings(from_date, to_date)+ defp parse_compare(compare), do: {:error, "Invalid include.compare '#{i(compare)}'."}defp parse_pagination(pagination) when is_map(pagination) do{:ok, Map.merge(@default_pagination, atomize_keys(pagination))}diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.exindex 2b0334b014..6219d7063a 100644--- a/lib/plausible/stats/breakdown.ex+++ b/lib/plausible/stats/breakdown.ex@@ -49,7 +49,7 @@ defmodule Plausible.Stats.Breakdown dodate_range_label: format_date_range(query)}- if query.include.comparisons do+ if query.include.compare docomparison_date_range_label =query|> Comparisons.get_comparison_query()diff --git a/lib/plausible/stats/comparisons.ex b/lib/plausible/stats/comparisons.exindex 8dede38349..e6097fa6fc 100644--- a/lib/plausible/stats/comparisons.ex+++ b/lib/plausible/stats/comparisons.ex@@ -10,47 +10,59 @@ defmodule Plausible.Stats.Comparisons doalias Plausible.Statsalias Plausible.Stats.{Query, DateTimeRange, Time}- @spec get_comparison_utc_time_range(Stats.Query.t(), map()) :: DateTimeRange.t()+ @spec get_comparison_utc_time_range(Stats.Query.t()) :: DateTimeRange.t()@doc """- Generates a `DateTimeRange` for the comparison period by the given source query- and comparison options.+ Generates a `DateTimeRange` representing the comparison period of a given+ `%Query{}` struct (i.e. the `source_query`).++ There are different modes and options that determine the outcome of the+ resulting DateTimeRange. Those are specified under `source_query.include`.Currently only historical periods are supported for comparisons (not `realtime`and `30m` periods).- ## Options- * `mode` (required) - specifies the type of comparison and can be one of the- following:+ ## Modes (`source_query.include.compare` field)- * `"previous_period"` - shifts back the query by the same number of days the+ * `:previous_period` - shifts back the query by the same number of days thesource query has.- * `"year_over_year"` - shifts back the query by 1 year.+ * `:year_over_year` - shifts back the query by 1 year.++ * `{:date_range, from, to}` - compares the query using a custom date range.++ ## Options++ * `source_query.include.compare_match_day_of_week`- * `"custom"` - compares the query using a custom date range. See `date_range` for- more details.+ Determines whether the comparison query should be adjusted to match the+ day of the week of the source query. When this option is set to true, the+ comparison query is shifted to start on the same day of the week as the+ source query, rather than on the exact same date.- * `:date_range` - a ISO-8601 date string pair used when mode is `"custom"`.+ Example: if the source query starts on Sunday, January 1st, 2023 and the+ `year_over_year` comparison query is configured to `match_day_of_week`, it+ will be shifted to start on Sunday, January 2nd, 2022 instead of January 1st.- * `:match_day_of_week` - determines whether the comparison query should be- adjusted to match the day of the week of the source query. When this option- is set to true, the comparison query is shifted to start on the same day of- the week as the source query, rather than on the exact same date. For- example, if the source query starts on Sunday, January 1st, 2023 and the- `year_over_year` comparison query is configured to `match_day_of_week`,- it will be shifted to start on Sunday, January 2nd, 2022 instead of- January 1st. Defaults to false.+ Note: this option has no effect when custom date range mode is used."""- def get_comparison_utc_time_range(%Stats.Query{} = source_query, options) do- comparison_date_range = get_comparison_date_range(source_query, options)-- DateTimeRange.new!(- comparison_date_range.first,- comparison_date_range.last,- source_query.timezone- )- |> DateTimeRange.to_timezone("Etc/UTC")+ def get_comparison_utc_time_range(%Stats.Query{} = source_query) do+ datetime_range =+ case source_query.include.compare do+ {:datetime_range, from, to} ->+ DateTimeRange.new!(from, to)++ _ ->+ comparison_date_range = get_comparison_date_range(source_query)++ DateTimeRange.new!(+ comparison_date_range.first,+ comparison_date_range.last,+ source_query.timezone+ )+ end++ DateTimeRange.to_timezone(datetime_range, "Etc/UTC")enddef get_comparison_query(@@ -103,7 +115,7 @@ defmodule Plausible.Stats.Comparisons doendend- defp get_comparison_date_range(source_query, %{mode: "year_over_year"} = options) do+ defp get_comparison_date_range(%Query{include: %{compare: :year_over_year}} = source_query) dosource_date_range = Query.date_range(source_query, trim_trailing: true)start_date = source_date_range.first |> Date.shift(year: -1)@@ -111,10 +123,10 @@ defmodule Plausible.Stats.Comparisons doend_date = Date.add(start_date, diff_in_days)Date.range(start_date, end_date)- |> maybe_match_day_of_week(source_date_range, options)+ |> maybe_match_day_of_week(source_date_range, source_query)end- defp get_comparison_date_range(source_query, %{mode: "previous_period"} = options) do+ defp get_comparison_date_range(%Query{include: %{compare: :previous_period}} = source_query) dosource_date_range = Query.date_range(source_query, trim_trailing: true)last = source_date_range.last@@ -124,15 +136,15 @@ defmodule Plausible.Stats.Comparisons donew_last = Date.add(last, diff_in_days)Date.range(new_first, new_last)- |> maybe_match_day_of_week(source_date_range, options)+ |> maybe_match_day_of_week(source_date_range, source_query)end- defp get_comparison_date_range(source_query, %{mode: "custom"} = options) do- DateTimeRange.new!(options.date_range.first, options.date_range.last, source_query.timezone)+ defp get_comparison_date_range(%Query{include: %{compare: {:date_range, from_date, to_date}}}) do+ Date.range(from_date, to_date)end- defp maybe_match_day_of_week(comparison_date_range, source_date_range, options) do- if options[:match_day_of_week] do+ defp maybe_match_day_of_week(comparison_date_range, source_date_range, source_query) do+ if source_query.include.compare_match_day_of_week doday_to_match = Date.day_of_week(source_date_range.first)new_first =diff --git a/lib/plausible/stats/dashboard_query_parser.ex b/lib/plausible/stats/dashboard_query_parser.exindex 3108fb5d81..81a109c8a2 100644--- a/lib/plausible/stats/dashboard_query_parser.ex+++ b/lib/plausible/stats/dashboard_query_parser.ex@@ -5,9 +5,9 @@ defmodule Plausible.Stats.DashboardQueryParser doto be filled in by each specific report."""- alias Plausible.Stats.{ParsedQueryParams}+ alias Plausible.Stats.{ParsedQueryParams, QueryInclude}- @default_include %{+ @default_include %QueryInclude{imports: true,# `include.imports_meta` can be true even when `include.imports`# is false. Even if we don't want to include imported data, we@@ -17,7 +17,8 @@ defmodule Plausible.Stats.DashboardQueryParser dotime_labels: true,total_rows: false,trim_relative_date_range: true,- comparisons: nil,+ compare: nil,+ compare_match_day_of_week: true,legacy_time_on_page_cutoff: nil}@@ -33,14 +34,19 @@ defmodule Plausible.Stats.DashboardQueryParser dowith {:ok, filters} <- parse_filters(query_string),{:ok, relative_date} <- parse_relative_date(params_map) do- include_imports? = parse_include_imports(params_map)+ include =+ Map.merge(@default_include, %{+ imports: parse_include_imports(params_map),+ compare: parse_include_compare(params_map),+ compare_match_day_of_week: parse_include_compare_match_day_of_week(params_map)+ }){:ok,ParsedQueryParams.new!(%{input_date_range: parse_input_date_range(params_map),relative_date: relative_date,filters: filters,- include: Map.merge(@default_include, %{imports: include_imports?})+ include: include})}endend@@ -77,6 +83,20 @@ defmodule Plausible.Stats.DashboardQueryParser dodefp parse_include_imports(%{"with_imported" => "false"}), do: falsedefp parse_include_imports(_), do: true+ defp parse_include_compare(%{"comparison" => "previous_period"}), do: :previous_period+ defp parse_include_compare(%{"comparison" => "year_over_year"}), do: :year_over_year++ defp parse_include_compare(%{"comparison" => "custom"} = params) do+ from_date = Date.from_iso8601!(params["compare_from"])+ to_date = Date.from_iso8601!(params["compare_to"])+ {:date_range, from_date, to_date}+ end++ defp parse_include_compare(_options), do: nil++ defp parse_include_compare_match_day_of_week(%{"match_day_of_week" => "false"}), do: false+ defp parse_include_compare_match_day_of_week(_), do: true+defp parse_filters(query_string) dowith {:ok, filters} <- decode_filters(query_string) doPlausible.Stats.ApiQueryParser.parse_filters(filters)diff --git a/lib/plausible/stats/dashboard_query_serializer.ex b/lib/plausible/stats/dashboard_query_serializer.exindex f04bf60c24..813d60f7e6 100644--- a/lib/plausible/stats/dashboard_query_serializer.ex+++ b/lib/plausible/stats/dashboard_query_serializer.ex@@ -4,7 +4,7 @@ defmodule Plausible.Stats.DashboardQuerySerializer dostring."""- alias Plausible.Stats.{ParsedQueryParams, DashboardQueryParser}+ alias Plausible.Stats.{ParsedQueryParams, DashboardQueryParser, QueryInclude}def serialize(%ParsedQueryParams{} = params) doparams@@ -57,7 +57,18 @@ defmodule Plausible.Stats.DashboardQuerySerializer doend)end- defp get_serialized_fields({:include, include}) do+ defp get_serialized_fields({:include, %QueryInclude{} = include}) do+ [:imports, :compare, :compare_match_day_of_week]+ |> Enum.flat_map(fn include_key ->+ get_serialized_fields_from_include(include_key, include)+ end)+ end++ defp get_serialized_fields(_) do+ []+ end++ defp get_serialized_fields_from_include(:imports, %QueryInclude{} = include) doif include.imports == DashboardQueryParser.default_include().imports do[]else@@ -65,8 +76,30 @@ defmodule Plausible.Stats.DashboardQuerySerializer doendend- defp get_serialized_fields(_) do- []+ defp get_serialized_fields_from_include(:compare, %QueryInclude{} = include) do+ case include.compare do+ nil ->+ []++ mode when mode in [:previous_period, :year_over_year] ->+ [{"comparison", to_string(mode)}]++ {:date_range, from_date, to_date} ->+ [+ {"comparison", "custom"},+ {"compare_from", Date.to_iso8601(from_date)},+ {"compare_to", Date.to_iso8601(to_date)}+ ]+ end+ end++ defp get_serialized_fields_from_include(:compare_match_day_of_week, include) do+ if include.compare_match_day_of_week ==+ DashboardQueryParser.default_include().compare_match_day_of_week do+ []+ else+ [{"match_day_of_week", to_string(include.compare_match_day_of_week)}]+ endend# These characters are not URL encoded to have more readable URLs.diff --git a/lib/plausible/stats/legacy/legacy_query_builder.ex b/lib/plausible/stats/legacy/legacy_query_builder.exindex ff033a267f..89d8dfd710 100644--- a/lib/plausible/stats/legacy/legacy_query_builder.ex+++ b/lib/plausible/stats/legacy/legacy_query_builder.ex@@ -30,7 +30,7 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do|> preload_goals_and_revenue(site)|> put_consolidated_site_ids(site)|> put_order_by(params)- |> put_include(site, params)+ |> put_include(params)|> QueryBuilder.put_comparison_utc_time_range()|> Query.put_imported_opts(site)|> QueryBuilder.set_time_on_page_data(site)@@ -233,12 +233,13 @@ defmodule Plausible.Stats.Legacy.QueryBuilder doendend- defp put_include(query, site, params) do+ defp put_include(query, params) doinclude = parse_include(params["include"])query|> struct!(include: include)- |> Query.set_include(:comparisons, parse_comparison_params(site, params))+ |> Query.set_include(:compare, parse_include_compare(params))+ |> Query.set_include(:compare_match_day_of_week, params["match_day_of_week"] == "true")|> Query.set_include(:imports, params["with_imported"] == "true")end@@ -325,34 +326,20 @@ defmodule Plausible.Stats.Legacy.QueryBuilder doendend- def parse_comparison_params(_site, %{"period" => period}) when period in ~w(realtime all),+ defp parse_include_compare(%{"period" => period}) when period in ~w(realtime all),do: nil- def parse_comparison_params(_site, %{"comparison" => mode} = params)- when mode in ["previous_period", "year_over_year"] do- %{- mode: mode,- match_day_of_week: params["match_day_of_week"] == "true"- }- end+ defp parse_include_compare(%{"comparison" => "previous_period"}), do: :previous_period+ defp parse_include_compare(%{"comparison" => "year_over_year"}), do: :year_over_year- def parse_comparison_params(site, %{"comparison" => "custom"} = params) do- {:ok, date_range} =- ApiQueryParser.parse_date_range_pair(site, [- params["compare_from"],- params["compare_to"]- ])-- %{- mode: "custom",- date_range: date_range,- match_day_of_week: params["match_day_of_week"] == "true"- }- end+ # Stats API v1+ defp parse_include_compare(%{"compare" => "previous_period"}), do: :previous_period- def parse_comparison_params(_site, %{"compare" => "previous_period"}) do- %{mode: "previous_period"}+ defp parse_include_compare(%{"comparison" => "custom"} = params) do+ from_date = Date.from_iso8601!(params["compare_from"])+ to_date = Date.from_iso8601!(params["compare_to"])+ {:date_range, from_date, to_date}end- def parse_comparison_params(_site, _options), do: nil+ defp parse_include_compare(_options), do: nilenddiff --git a/lib/plausible/stats/parsed_query_params.ex b/lib/plausible/stats/parsed_query_params.exindex 09f9152f5d..2d940f0873 100644--- a/lib/plausible/stats/parsed_query_params.ex+++ b/lib/plausible/stats/parsed_query_params.ex@@ -1,23 +1,39 @@defmodule Plausible.Stats.ParsedQueryParams do@moduledoc false- defstruct [- :input_date_range,- # `relative_date` is a convenience currently exclusive to the internal- # dashboard API for constructing datetime ranges. It adds the ability- # to use `day`, `month` and `year` periods relative to a specific date.- # E.g.: `?period=month&date=2021-01-15` will query the entire month of- # January 2021. In the public API it is always today in site.timezone.- :relative_date,- :metrics,- :filters,- :dimensions,- :order_by,- :pagination,- :include- ]+ defstruct input_date_range: nil,+ # `relative_date` is a convenience currently exclusive to the internal+ # dashboard API for constructing datetime ranges. It adds the ability+ # to use `day`, `month` and `year` periods relative to a specific date.+ # E.g.: `?period=month&date=2021-01-15` will query the entire month of+ # January 2021. In the public API it is always today in site.timezone.+ relative_date: nil,+ metrics: [],+ filters: [],+ dimensions: [],+ order_by: nil,+ pagination: nil,+ include: %Plausible.Stats.QueryInclude{}def new!(params) when is_map(params) dostruct!(__MODULE__, Map.to_list(params))end++ @props_prefix "event:props:"++ def add_or_replace_filter(%__MODULE__{filters: filters} = parsed_query_params, new_filter) do+ [_, new_filter_dimension, _] = new_filter++ prop_filter? = String.starts_with?(new_filter_dimension, @props_prefix)++ new_filters =+ filters+ |> Enum.reject(fn [_, existing_filter_dimension, _] ->+ existing_filter_dimension == new_filter_dimension or+ (prop_filter? and String.starts_with?(existing_filter_dimension, @props_prefix))+ end)+ |> Enum.concat([new_filter])++ struct!(parsed_query_params, filters: new_filters)+ endenddiff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.exindex e6482e050e..8b0fbe409e 100644--- a/lib/plausible/stats/query.ex+++ b/lib/plausible/stats/query.ex@@ -107,7 +107,7 @@ defmodule Plausible.Stats.Query doenddef set_include(query, key, value) do- struct!(query, include: Map.put(query.include, key, value))+ struct!(query, include: struct!(query.include, [{key, value}]))enddef add_filter(query, filter) do@@ -168,7 +168,7 @@ defmodule Plausible.Stats.Query doin_range = Plausible.Imported.completed_imports_in_query_range(site, query)in_comparison_range =- if is_map(query.include.comparisons) do+ if query.include.compare docomparison_query = Comparisons.get_comparison_query(query)Plausible.Imported.completed_imports_in_query_range(site, comparison_query)elsediff --git a/lib/plausible/stats/query_builder.ex b/lib/plausible/stats/query_builder.exindex e92893a851..f98195ef0f 100644--- a/lib/plausible/stats/query_builder.ex+++ b/lib/plausible/stats/query_builder.ex@@ -109,23 +109,6 @@ defmodule Plausible.Stats.QueryBuilder doDateTimeRange.new!(from, to)end- defp build_comparison_datetime_range(- %ParsedQueryParams{include: %{comparisons: %{date_range: date_range}}} =- parsed_query_params,- site- ) do- comparison_date_range = build_datetime_range(date_range, site, nil, nil)-- new_include =- put_in(parsed_query_params.include, [:comparisons, :date_range], comparison_date_range)-- struct(parsed_query_params, include: new_include)- end-- defp build_comparison_datetime_range(%ParsedQueryParams{} = parsed_query_params, _site) do- parsed_query_params- end-defp do_build(parsed_query_params, site, debug_metadata) donow = Plausible.Stats.Query.Test.get_fixed_now()@@ -137,8 +120,6 @@ defmodule Plausible.Stats.QueryBuilder dodimensions: dimensions} = parsed_query_params- parsed_query_params = build_comparison_datetime_range(parsed_query_params, site)-relative_date = relative_date || today_from_now(site, now)utc_time_range =@@ -191,10 +172,10 @@ defmodule Plausible.Stats.QueryBuilder do)end- def put_comparison_utc_time_range(%Query{include: %{comparisons: nil}} = query), do: query+ def put_comparison_utc_time_range(%Query{include: %{compare: nil}} = query), do: query- def put_comparison_utc_time_range(%Query{include: %{comparisons: comparison_opts}} = query) do- datetime_range = Comparisons.get_comparison_utc_time_range(query, comparison_opts)+ def put_comparison_utc_time_range(%Query{} = query) do+ datetime_range = Comparisons.get_comparison_utc_time_range(query)struct!(query, comparison_utc_time_range: datetime_range)enddiff --git a/lib/plausible/stats/query_include.ex b/lib/plausible/stats/query_include.exnew file mode 100644index 0000000000..1a128e5ea0--- /dev/null+++ b/lib/plausible/stats/query_include.ex@@ -0,0 +1,27 @@+defmodule Plausible.Stats.QueryInclude do+ @moduledoc false++ defstruct imports: false,+ imports_meta: false,+ time_labels: false,+ total_rows: false,+ trim_relative_date_range: false,+ compare: nil,+ compare_match_day_of_week: false,+ legacy_time_on_page_cutoff: nil++ @type date_range_tuple() :: {:date_range, Date.t(), Date.t()}+ @type datetime_range_tuple() :: {:datetime_range, DateTime.t(), DateTime.t()}++ @type t() :: %__MODULE__{+ imports: boolean(),+ imports_meta: boolean(),+ time_labels: boolean(),+ total_rows: boolean(),+ trim_relative_date_range: boolean(),+ compare:+ nil | :previous_period | :year_over_year | date_range_tuple() | datetime_range_tuple(),+ compare_match_day_of_week: boolean(),+ legacy_time_on_page_cutoff: any()+ }+enddiff --git a/lib/plausible/stats/query_optimizer.ex b/lib/plausible/stats/query_optimizer.exindex 0ce8a18277..b1312f6318 100644--- a/lib/plausible/stats/query_optimizer.ex+++ b/lib/plausible/stats/query_optimizer.ex@@ -285,7 +285,7 @@ defmodule Plausible.Stats.QueryOptimizer dotoday = query.now |> DateTime.shift_zone!(query.timezone) |> DateTime.to_date()date_range = Query.date_range(query)- is_nil(query.include.comparisons) and date_range.first == today and date_range.last == today+ is_nil(query.include.compare) and date_range.first == today and date_range.last == todayenddefp should_trim_date_range?(_query), do: falsediff --git a/lib/plausible/stats/query_result.ex b/lib/plausible/stats/query_result.exindex 47b2460ff7..64c7c4f593 100644--- a/lib/plausible/stats/query_result.ex+++ b/lib/plausible/stats/query_result.ex@@ -8,12 +8,29 @@ defmodule Plausible.Stats.QueryResult do"""use Plausible- alias Plausible.Stats.{DateTimeRange, Query, QueryRunner}+ alias Plausible.Stats.{Query, QueryRunner}defstruct results: [],meta: %{},query: nil+ @imports_warnings %{+ unsupported_query:+ "Imported stats are not included in the results because query parameters are not supported. " <>+ "For more information, see: https://plausible.io/docs/stats-api#filtering-imported-stats",+ unsupported_interval:+ "Imported stats are not included because the time dimension (i.e. the interval) is too short."+ }++ def imports_warnings(), do: @imports_warnings++ @no_imported_scroll_depth_warning %{+ code: :no_imported_scroll_depth,+ warning: "No imports with scroll depth data were found"+ }++ def no_imported_scroll_depth_warning(), do: @no_imported_scroll_depth_warning+@doc """Builds full JSON-serializable query response.@@ -49,14 +66,6 @@ defmodule Plausible.Stats.QueryResult do|> add_total_rows_meta(runner.main_query, runner.total_rows)end- @imports_warnings %{- unsupported_query:- "Imported stats are not included in the results because query parameters are not supported. " <>- "For more information, see: https://plausible.io/docs/stats-api#filtering-imported-stats",- unsupported_interval:- "Imported stats are not included because the time dimension (i.e. the interval) is too short."- }-defp add_imports_meta(meta, %Query{include: include} = query) doif include.imports or include.imports_meta do%{@@ -98,17 +107,14 @@ defmodule Plausible.Stats.QueryResult doenddefp include(query) do- case get_in(query.include, [:comparisons, :date_range]) do- %DateTimeRange{first: first, last: last} ->- query.include- |> put_in([:comparisons, :date_range], [- to_iso8601(first, query.timezone),- to_iso8601(last, query.timezone)- ])+ case query.include.compare do+ {:date_range, first, last} ->+ struct!(query.include, compare: [Date.to_iso8601(first), Date.to_iso8601(last)])- nil ->+ _ ->query.includeend+ |> Map.from_struct()enddefp metric_warnings(%Query{} = query) do@@ -145,14 +151,9 @@ defmodule Plausible.Stats.QueryResult doendend- @no_imported_scroll_depth_metric_warning %{- code: :no_imported_scroll_depth,- warning: "No imports with scroll depth data were found"- }-defp metric_warning(:scroll_depth, %Query{} = query) doif query.include_imported and not Enum.any?(query.imports_in_range, & &1.has_scroll_depth) do- @no_imported_scroll_depth_metric_warning+ @no_imported_scroll_depth_warningendenddiff --git a/lib/plausible/stats/query_runner.ex b/lib/plausible/stats/query_runner.exindex d6db51bb6f..1478d7e1b7 100644--- a/lib/plausible/stats/query_runner.ex+++ b/lib/plausible/stats/query_runner.ex@@ -59,7 +59,7 @@ defmodule Plausible.Stats.QueryRunner doenddefp add_comparison_query(%__MODULE__{main_query: query, main_results: main_results} = runner)- when is_map(query.include.comparisons) do+ when not is_nil(query.include.compare) docomparison_query =query|> Comparisons.get_comparison_query()@@ -181,7 +181,7 @@ defmodule Plausible.Stats.QueryRunner do# Special case: If comparison and single time dimension, add 0 rows - otherwise# comparisons would not be shown for timeseries with 0 values.defp add_empty_timeseries_rows(results_list, %__MODULE__{main_query: query})- when is_map(query.include.comparisons) do+ when not is_nil(query.include.compare) doindexed_results = index_by_dimensions(results_list)empty_timeseries_rows =@@ -210,7 +210,7 @@ defmodule Plausible.Stats.QueryRunner doenddefp add_comparison_results(row, query, comparison_map, time_lookup)- when is_map(query.include.comparisons) do+ when not is_nil(query.include.compare) dodimensions = get_comparison_dimensions(row.dimensions, query, time_lookup)comparison_metrics = get_comparison_metrics(comparison_map, dimensions, query)diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.exindex ff632e5f25..12f662ef17 100644--- a/lib/plausible/stats/timeseries.ex+++ b/lib/plausible/stats/timeseries.ex@@ -29,7 +29,7 @@ defmodule Plausible.Stats.Timeseries do|> QueryOptimizer.optimize()comparison_query =- if(query.include.comparisons,+ if(query.include.compare,do: Comparisons.get_comparison_query(query),else: nil)diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex 12f61120de..318690d22d 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -208,8 +208,8 @@ defmodule PlausibleWeb.Api.StatsController dosample_percent: sample_percent,with_imported_switch: with_imported_switch_info(meta),includes_imported: meta[:imports_included] == true,- comparing_from: query.include.comparisons && Query.date_range(comparison_query).first,- comparing_to: query.include.comparisons && Query.date_range(comparison_query).last,+ comparing_from: query.include.compare && Query.date_range(comparison_query).first,+ comparing_to: query.include.compare && Query.date_range(comparison_query).last,from: Query.date_range(query).first,to: Query.date_range(query).last})@@ -297,7 +297,7 @@ defmodule PlausibleWeb.Api.StatsController doenddefp fetch_goal_realtime_top_stats(site, query) do- query = Query.set_include(query, :comparisons, nil)+ query = Query.set_include(query, :compare, nil)%{results: %{@@ -334,7 +334,7 @@ defmodule PlausibleWeb.Api.StatsController doenddefp fetch_realtime_top_stats(site, query) do- query = Query.set_include(query, :comparisons, nil)+ query = Query.set_include(query, :compare, nil)%{results: %{@@ -1710,7 +1710,7 @@ defmodule PlausibleWeb.Api.StatsController doenddef comparison_query(query) do- if query.include.comparisons do+ if query.include.compare doComparisons.get_comparison_query(query)endenddiff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.exindex 420f03ae7b..4168b8d3d5 100644--- a/lib/plausible_web/router.ex+++ b/lib/plausible_web/router.ex@@ -331,12 +331,6 @@ defmodule PlausibleWeb.Router dopipe_through [:public_api, PlausibleWeb.Plugs.AuthorizePublicAPI]post "/query", ExternalQueryApiController, :query-- if Mix.env() in [:test, :ce_test] do- scope assigns: %{schema_type: :internal} do- post "/query-internal-test", ExternalQueryApiController, :query- end- endendscope "/api/docs", PlausibleWeb.Api dodiff --git a/lib/workers/send_email_report.ex b/lib/workers/send_email_report.exindex 5cfcc7ca1f..333b0752af 100644--- a/lib/workers/send_email_report.ex+++ b/lib/workers/send_email_report.ex@@ -95,7 +95,7 @@ defmodule Plausible.Workers.SendEmailReport do"site_id" => site.domain,"metrics" => ["pageviews", "visitors", "bounce_rate"],"date_range" => date_range,- "include" => %{"comparisons" => %{"mode" => "previous_period"}},+ "include" => %{"compare" => "previous_period"},"pagination" => %{"limit" => 5}})diff --git a/priv/json-schemas/query-api-schema.json b/priv/json-schemas/query-api-schema.jsonindex 7ec2e5187e..d9b7dc483a 100644--- a/priv/json-schemas/query-api-schema.json+++ b/priv/json-schemas/query-api-schema.json@@ -68,47 +68,21 @@"default": false,"description": "If set and using `day`, `month` or `year` date_ranges, the query will be trimmed to the current date"},- "comparisons": {+ "compare": {"$comment": "only :internal",- "type": "object","oneOf": [- {- "properties": {- "mode": {- "type": "string",- "enum": ["previous_period", "year_over_year"]- },- "match_day_of_week": {- "type": "boolean",- "default": false,- "description": "If set and using time:day dimensions, day-of-week of comparison query is matched"- }- },- "required": ["mode"],- "additionalProperties": false- },- {- "properties": {- "mode": {- "const": "custom"- },- "match_day_of_week": {- "type": "boolean",- "default": false,- "description": "If set and using time:day dimensions, day-of-week of comparison query is matched"- },- "date_range": {- "oneOf": [- { "$ref": "#/definitions/date_time_range" },- { "$ref": "#/definitions/date_range" }- ],- "description": "If custom period. A list of two ISO8601 dates or timestamps to compare against."- }- },- "required": ["mode", "date_range"],- "additionalProperties": false- }- ]+ { "$ref": "#/definitions/date_range" },+ { "$ref": "#/definitions/comparison_mode" },+ false+ ],+ "default": false,+ "description": "If set, executes the same query but over a comparison date range"+ },+ "compare_match_day_of_week": {+ "$comment": "only :internal",+ "type": "boolean",+ "default": false,+ "description": "With the `compare` option, if set and using time:day dimensions, day-of-week of comparison query is matched"}}},@@ -232,6 +206,18 @@}]},+ "comparison_mode": {+ "oneOf": [+ {+ "const": "previous_period",+ "description": "Comparison date range shortcut - shift back the original query date range by its length"+ },+ {+ "const": "year_over_year",+ "description": "Comparison date range shortcut - shift back the original query date range by a year"+ }+ ]+ },"metric": {"oneOf": [{diff --git a/test/plausible/stats/comparisons_test.exs b/test/plausible/stats/comparisons_test.exsindex c49d018aa0..9d86a917ef 100644--- a/test/plausible/stats/comparisons_test.exs+++ b/test/plausible/stats/comparisons_test.exs@@ -435,7 +435,7 @@ defmodule Plausible.Stats.ComparisonsTest do"site_id" => site.domain,"metrics" => ["pageviews"],"date_range" => ["2024-01-01", "2024-02-01"],- "include" => %{"comparisons" => %{"mode" => "previous_period"}}+ "include" => %{"compare" => "previous_period"}},params)diff --git a/test/plausible/stats/query/dashboard_query_parser_test.exs b/test/plausible/stats/query/dashboard_query_parser_test.exsindex 8a0dc77852..6fc2d33365 100644--- a/test/plausible/stats/query/dashboard_query_parser_test.exs+++ b/test/plausible/stats/query/dashboard_query_parser_test.exs@@ -3,15 +3,17 @@ defmodule Plausible.Stats.DashboardQueryParserTest doimport Plausible.Stats.DashboardQueryParseralias Plausible.Stats.ParsedQueryParams+ @default_include default_include()+test "parses an empty query string" do{:ok, parsed} = parse("")expected = %Plausible.Stats.ParsedQueryParams{input_date_range: nil,relative_date: nil,- metrics: nil,+ metrics: [],filters: [],- dimensions: nil,+ dimensions: [],order_by: nil,pagination: nil,include: default_include()@@ -24,14 +26,22 @@ defmodule Plausible.Stats.DashboardQueryParserTest dofor period <- [:realtime, :day, :month, :year, :all] dotest "parses #{period} period" do{:ok, parsed} = parse("?period=#{Atom.to_string(unquote(period))}")- assert_matches %ParsedQueryParams{input_date_range: ^unquote(period)} = parsed++ assert_matches %ParsedQueryParams{+ input_date_range: ^unquote(period),+ include: ^@default_include+ } = parsedendendfor i <- [7, 28, 30, 91] dotest "parses #{i}d period" do{:ok, parsed} = parse("?period=#{unquote(i)}d")- assert_matches %ParsedQueryParams{input_date_range: {:last_n_days, ^unquote(i)}} = parsed++ assert_matches %ParsedQueryParams{+ input_date_range: {:last_n_days, ^unquote(i)},+ include: ^@default_include+ } = parsedendend@@ -39,7 +49,10 @@ defmodule Plausible.Stats.DashboardQueryParserTest dotest "parses #{i}mo period" do{:ok, parsed} = parse("?period=#{unquote(i)}mo")- assert_matches %ParsedQueryParams{input_date_range: {:last_n_months, ^unquote(i)}} =+ assert_matches %ParsedQueryParams{+ input_date_range: {:last_n_months, ^unquote(i)},+ include: ^@default_include+ } =parsedendend@@ -48,7 +61,8 @@ defmodule Plausible.Stats.DashboardQueryParserTest do{:ok, parsed} = parse("?period=custom&from=2021-01-01&to=2021-03-05")assert %ParsedQueryParams{- input_date_range: {:date_range, ~D[2021-01-01], ~D[2021-03-05]}+ input_date_range: {:date_range, ~D[2021-01-01], ~D[2021-03-05]},+ include: @default_include} = parsedend@@ -56,7 +70,8 @@ defmodule Plausible.Stats.DashboardQueryParserTest do{:ok, parsed} = parse("?period=abcde")assert %ParsedQueryParams{- input_date_range: nil+ input_date_range: nil,+ include: @default_include} = parsedendend@@ -64,7 +79,7 @@ defmodule Plausible.Stats.DashboardQueryParserTest dodescribe "date -> relative_date" dotest "parses a valid iso8601 date string" do{:ok, parsed} = parse("?date=2021-05-05")- assert %ParsedQueryParams{relative_date: ~D[2021-05-05]} = parsed+ assert %ParsedQueryParams{relative_date: ~D[2021-05-05], include: @default_include} = parsedendtest "errors when invalid date" do@@ -75,17 +90,60 @@ defmodule Plausible.Stats.DashboardQueryParserTest dodescribe "with_imported -> include.imports" dotest "true -> true" do{:ok, parsed} = parse("?with_imported=true")- assert %ParsedQueryParams{include: %{imports: true}} = parsed+ assert %ParsedQueryParams{include: @default_include} = parsedendtest "invalid -> true" do{:ok, parsed} = parse("?with_imported=foo")- assert %ParsedQueryParams{include: %{imports: true}} = parsed+ assert %ParsedQueryParams{include: @default_include} = parsedendtest "false -> false" do{:ok, parsed} = parse("?with_imported=false")- assert %ParsedQueryParams{include: %{imports: false}} = parsed+ expected_include = Map.put(@default_include, :imports, false)+ assert %ParsedQueryParams{include: ^expected_include} = parsed+ end+ end++ describe "comparison -> include.compare" do+ for mode <- [:previous_period, :year_over_year] do+ test "parses #{mode} mode" do+ {:ok, parsed} = parse("?comparison=#{unquote(mode)}")+ expected_include = Map.put(@default_include, :compare, unquote(mode))+ assert_matches %ParsedQueryParams{include: ^expected_include} = parsed+ end+ end++ test "parses custom date range mode" do+ {:ok, parsed} = parse("?comparison=custom&compare_from=2021-01-01&compare_to=2021-04-30")++ expected_include =+ Map.put(@default_include, :compare, {:date_range, ~D[2021-01-01], ~D[2021-04-30]})++ assert_matches %ParsedQueryParams{include: ^expected_include} = parsed+ end++ test "invalid -> nil" do+ {:ok, parsed} = parse("?comparison=invalid_mode")+ assert %ParsedQueryParams{include: @default_include} = parsed+ end+ end++ describe "match_day_of_week -> include.compare_match_day_of_week" do+ test "true -> true" do+ {:ok, parsed} = parse("?match_day_of_week=true")+ assert %ParsedQueryParams{include: @default_include} = parsed+ end++ test "invalid -> true" do+ {:ok, parsed} = parse("?match_day_of_week=foo")+ assert %ParsedQueryParams{include: @default_include} = parsed+ end++ test "false -> false" do+ {:ok, parsed} = parse("?match_day_of_week=false")+ expected_include = Map.put(@default_include, :compare_match_day_of_week, false)+ assert %ParsedQueryParams{include: ^expected_include} = parsedendend@@ -99,7 +157,8 @@ defmodule Plausible.Stats.DashboardQueryParserTest do[:is, "visit:exit_page", ["/:dashboard"]],[:is, "visit:source", ["Bing"]],[:is, "event:props:theme", ["system"]]- ]+ ],+ include: @default_include} = parsedenddiff --git a/test/plausible/stats/query/dashboard_query_serializer_test.exs b/test/plausible/stats/query/dashboard_query_serializer_test.exsindex 8a44861653..bd9fc708e4 100644--- a/test/plausible/stats/query/dashboard_query_serializer_test.exs+++ b/test/plausible/stats/query/dashboard_query_serializer_test.exs@@ -3,6 +3,8 @@ defmodule Plausible.Stats.DashboardQuerySerializerTest doimport Plausible.Stats.{DashboardQuerySerializer, DashboardQueryParser}alias Plausible.Stats.ParsedQueryParams+ @default_include default_include()+describe "parse -> serialize is a reversible transformation" dofor query_string <- ["", "date=2021-07-07&f=is,browser,Chrome,Firefox&period=day"] dotest "with query string being '#{query_string}'" do@@ -30,46 +32,83 @@ defmodule Plausible.Stats.DashboardQuerySerializerTest dodescribe "input_date_range -> period (+ from, to)" dofor input_date_range <- [:realtime, :day, :month, :year, :all] dotest "serializes #{input_date_range} input_date_range" do- serialized = serialize(%ParsedQueryParams{input_date_range: unquote(input_date_range)})- assert serialized == "period=#{Atom.to_string(unquote(input_date_range))}"+ params = %ParsedQueryParams{+ input_date_range: unquote(input_date_range),+ include: @default_include+ }++ assert serialize(params) == "period=#{Atom.to_string(unquote(input_date_range))}"endendfor i <- [7, 28, 30, 91] dotest "serializes {:last_n_days, #{i}} input_date_range" do- serialized = serialize(%ParsedQueryParams{input_date_range: {:last_n_days, unquote(i)}})- assert serialized == "period=#{unquote(i)}d"+ params = %ParsedQueryParams{+ input_date_range: {:last_n_days, unquote(i)},+ include: @default_include+ }++ assert serialize(params) == "period=#{unquote(i)}d"endendfor i <- [6, 12] dotest "serializes {:last_n_months, #{i}} input_date_range" do- serialized = serialize(%ParsedQueryParams{input_date_range: {:last_n_months, unquote(i)}})- assert serialized == "period=#{unquote(i)}mo"+ params = %ParsedQueryParams{+ input_date_range: {:last_n_months, unquote(i)},+ include: @default_include+ }++ assert serialize(params) == "period=#{unquote(i)}mo"endendtest "serlializes custom input_date_range" do- serialized =- serialize(%ParsedQueryParams{- input_date_range: {:date_range, ~D[2021-01-01], ~D[2021-03-05]}- })+ params = %ParsedQueryParams{+ input_date_range: {:date_range, ~D[2021-01-01], ~D[2021-03-05]},+ include: @default_include+ }- assert serialized == "from=2021-01-01&period=custom&to=2021-03-05"+ assert serialize(params) == "from=2021-01-01&period=custom&to=2021-03-05"endenddescribe "relative_date -> date" dotest "serializes a date struct into iso8601" do- serialized = serialize(%ParsedQueryParams{relative_date: ~D[2021-05-05]})- assert serialized == "date=2021-05-05"+ params = %ParsedQueryParams{relative_date: ~D[2021-05-05], include: @default_include}+ assert serialize(params) == "date=2021-05-05"endenddescribe "include.imports -> with_imported" dotest "false -> false" do- serialized = serialize(%ParsedQueryParams{include: %{default_include() | imports: false}})- assert serialized == "with_imported=false"+ params = %ParsedQueryParams{include: %{@default_include | imports: false}}+ assert serialize(params) == "with_imported=false"+ end+ end++ describe "include.compare -> comparison" do+ for mode <- [:previous_period, :year_over_year] do+ test "serializes #{mode} mode" do+ params = %ParsedQueryParams{include: %{@default_include | compare: unquote(mode)}}+ assert serialize(params) == "comparison=#{unquote(mode)}"+ end+ end++ test "serializes custom comparison range" do+ params = %ParsedQueryParams{+ include: %{@default_include | compare: {:date_range, ~D[2021-01-01], ~D[2021-04-30]}}+ }++ assert serialize(params) ==+ "compare_from=2021-01-01&compare_to=2021-04-30&comparison=custom"+ end+ end++ describe "include.compare_match_day_of_week -> match_day_of_week" do+ test "false -> false" do+ params = %ParsedQueryParams{include: %{@default_include | compare_match_day_of_week: false}}+ assert serialize(params) == "match_day_of_week=false"endend@@ -81,14 +120,15 @@ defmodule Plausible.Stats.DashboardQuerySerializerTest do[:is, "visit:exit_page", ["/:dashboard"]],[:is, "visit:source", ["Bing"]],[:is, "event:props:theme", ["system"]]- ]+ ],+ include: @default_include})assert serialized == "f=is,exit_page,/:dashboard&f=is,source,Bing&f=is,props:theme,system"endtest "serializes empty filters" do- serialized = serialize(%ParsedQueryParams{filters: []})+ serialized = serialize(%ParsedQueryParams{filters: [], include: @default_include})assert serialized == ""endenddiff --git a/test/plausible/stats/query/parsed_query_params_test.exs b/test/plausible/stats/query/parsed_query_params_test.exsnew file mode 100644index 0000000000..5ad86beeac--- /dev/null+++ b/test/plausible/stats/query/parsed_query_params_test.exs@@ -0,0 +1,42 @@+defmodule Plausible.Stats.ParsedQueryParamsTest do+ use Plausible.DataCase+ alias Plausible.Stats.ParsedQueryParams++ describe "add_or_replace_filter" do+ test "adds filter" do+ initial_params = %ParsedQueryParams{filters: []}+ new_filter = [:is, "event:page", ["/"]]+ new_params = ParsedQueryParams.add_or_replace_filter(initial_params, new_filter)++ assert new_params.filters == [new_filter]+ end++ test "replaces same dimension filter" do+ initial_params = %ParsedQueryParams{+ filters: [[:contains, "event:page", ["blog", "post"]], [:is, "visit:source", ["Google"]]]+ }++ new_filter = [:is, "event:page", ["/blog/some-post"]]+ new_params = ParsedQueryParams.add_or_replace_filter(initial_params, new_filter)++ assert new_params.filters == [+ [:is, "visit:source", ["Google"]],+ new_filter+ ]+ end++ test "replaces custom prop dimension filter" do+ initial_params = %ParsedQueryParams{+ filters: [[:contains, "event:props:path", ["/path"]], [:is, "visit:source", ["Google"]]]+ }++ new_filter = [:is, "event:props:url", ["https://example.com/path"]]+ new_params = ParsedQueryParams.add_or_replace_filter(initial_params, new_filter)++ assert new_params.filters == [+ [:is, "visit:source", ["Google"]],+ new_filter+ ]+ end+ end+enddiff --git a/test/plausible/stats/query/query_comparisons_test.exs b/test/plausible/stats/query/query_comparisons_test.exsnew file mode 100644index 0000000000..4493694591--- /dev/null+++ b/test/plausible/stats/query/query_comparisons_test.exs@@ -0,0 +1,456 @@+defmodule Plausible.Stats.QueryComparisonsTest do+ use Plausible.DataCase+ alias Plausible.Stats+ alias Plausible.Stats.{ParsedQueryParams, QueryBuilder, QueryInclude}++ setup [:create_user, :create_site, :create_site_import]++ test "aggregates a single metric", %{site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:25:00]),+ build(:pageview, timestamp: ~N[2021-01-02 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-07 00:00:00])+ ])++ assert {:ok, query} =+ QueryBuilder.build(site, %ParsedQueryParams{+ metrics: [:pageviews],+ input_date_range: {:date_range, ~D[2021-01-07], ~D[2021-01-13]},+ include: %QueryInclude{compare: :previous_period}+ })++ assert %Stats.QueryResult{results: results} = Stats.query(site, query)++ assert results == [+ %{+ dimensions: [],+ metrics: [1],+ comparison: %{change: [-67], dimensions: [], metrics: [3]}+ }+ ]+ end++ test "timeseries comparison", %{site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:25:00]),+ build(:pageview, timestamp: ~N[2021-01-06 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-07 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-08 00:00:00])+ ])++ assert {:ok, query} =+ QueryBuilder.build(site, %ParsedQueryParams{+ metrics: [:pageviews],+ dimensions: ["time:day"],+ input_date_range: {:date_range, ~D[2021-01-07], ~D[2021-01-13]},+ include: %QueryInclude{compare: :previous_period}+ })++ assert %Stats.QueryResult{results: results} = Stats.query(site, query)++ assert results == [+ %{+ dimensions: ["2021-01-07"],+ metrics: [1],+ comparison: %{+ dimensions: ["2020-12-31"],+ metrics: [0],+ change: [100]+ }+ },+ %{+ dimensions: ["2021-01-08"],+ metrics: [1],+ comparison: %{+ dimensions: ["2021-01-01"],+ metrics: [2],+ change: [-50]+ }+ },+ %{+ dimensions: ["2021-01-09"],+ metrics: [0],+ comparison: %{+ dimensions: ["2021-01-02"],+ metrics: [0],+ change: [0]+ }+ },+ %{+ dimensions: ["2021-01-10"],+ metrics: [0],+ comparison: %{+ dimensions: ["2021-01-03"],+ metrics: [0],+ change: [0]+ }+ },+ %{+ dimensions: ["2021-01-11"],+ metrics: [0],+ comparison: %{+ dimensions: ["2021-01-04"],+ metrics: [0],+ change: [0]+ }+ },+ %{+ dimensions: ["2021-01-12"],+ metrics: [0],+ comparison: %{+ dimensions: ["2021-01-05"],+ metrics: [0],+ change: [0]+ }+ },+ %{+ dimensions: ["2021-01-13"],+ metrics: [0],+ comparison: %{+ dimensions: ["2021-01-06"],+ metrics: [1],+ change: [-100]+ }+ }+ ]+ end++ test "timeseries last 28d period compares the same period with and without match_day_of_week=true",+ %{+ site: site+ } do+ now = ~U[2021-06-10 12:00:00Z]+ Plausible.Stats.Query.Test.fix_now(now)++ today = DateTime.to_date(now)++ {:ok, query1} =+ QueryBuilder.build(site, %ParsedQueryParams{+ metrics: [:visitors],+ dimensions: ["time"],+ input_date_range: {:last_n_days, 28},+ include: %QueryInclude{time_labels: true, compare: :previous_period}+ })++ query2 = Stats.Query.set_include(query1, :compare_match_day_of_week, true)++ assert %Stats.QueryResult{results: results1} = Stats.query(site, query1)+ assert %Stats.QueryResult{results: results2} = Stats.query(site, query2)++ assert results1 == results2++ expected_first_date = today |> Date.shift(day: -28) |> Date.to_iso8601()+ expected_last_date = today |> Date.shift(day: -1) |> Date.to_iso8601()+ expected_comparison_first_date = today |> Date.shift(day: -56) |> Date.to_iso8601()+ expected_comparison_last_date = today |> Date.shift(day: -29) |> Date.to_iso8601()++ assert %{+ dimensions: [actual_first_date],+ comparison: %{+ dimensions: [actual_comparison_first_date]+ }+ } = List.first(results1)++ assert %{+ dimensions: [actual_last_date],+ comparison: %{+ dimensions: [actual_comparison_last_date]+ }+ } = List.last(results1)++ assert actual_first_date == expected_first_date+ assert actual_last_date == expected_last_date+ assert actual_comparison_first_date == expected_comparison_first_date+ assert actual_comparison_last_date == expected_comparison_last_date+ end++ test "timeseries last 91d period in year_over_year comparison", %{+ site: site+ } do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-04-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-04-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-04-05 00:00:00]),+ build(:pageview, timestamp: ~N[2021-04-05 00:00:00]),+ build(:pageview, timestamp: ~N[2021-06-30 00:00:00]),+ build(:pageview, timestamp: ~N[2022-04-01 00:00:00]),+ build(:pageview, timestamp: ~N[2022-04-05 00:00:00]),+ build(:pageview, timestamp: ~N[2022-06-30 00:00:00]),+ build(:pageview, timestamp: ~N[2022-07-01 00:00:00])+ ])++ Plausible.Stats.Query.Test.fix_now(~U[2022-07-01 14:00:00Z])++ assert {:ok, query} =+ QueryBuilder.build(site, %ParsedQueryParams{+ metrics: [:visitors],+ dimensions: ["time:day"],+ input_date_range: {:last_n_days, 91},+ include: %QueryInclude{time_labels: true, compare: :year_over_year}+ })++ assert %Stats.QueryResult{results: results, meta: meta} = Stats.query(site, query)++ time_labels = meta[:time_labels]++ assert "2022-04-01" = List.first(time_labels)+ assert "2022-04-05" = Enum.at(time_labels, 4)+ assert "2022-06-30" = List.last(time_labels)++ assert %{+ dimensions: ["2022-04-01"],+ metrics: [1],+ comparison: %{+ dimensions: ["2021-04-01"],+ metrics: [2]+ }+ } = Enum.find(results, &(&1[:dimensions] == ["2022-04-01"]))++ assert %{+ dimensions: ["2022-04-05"],+ metrics: [1],+ comparison: %{+ dimensions: ["2021-04-05"],+ metrics: [2]+ }+ } = Enum.find(results, &(&1[:dimensions] == ["2022-04-05"]))++ assert %{+ dimensions: ["2022-06-30"],+ metrics: [1],+ comparison: %{+ dimensions: ["2021-06-30"],+ metrics: [1]+ }+ } = Enum.find(results, &(&1[:dimensions] == ["2022-06-30"]))+ end++ test "dimensional comparison with low limit", %{site: site} do+ populate_stats(site, [+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, browser: "Safari", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, browser: "Safari", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, browser: "Safari", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-07 00:00:00]),+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-07 00:00:00]),+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-07 00:00:00]),+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-07 00:00:00]),+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-07 00:00:00]),+ build(:pageview, browser: "Safari", timestamp: ~N[2021-01-08 00:00:00])+ ])++ {:ok, page_one_query} =+ QueryBuilder.build(site, %ParsedQueryParams{+ metrics: [:visitors, :percentage],+ input_date_range: {:date_range, ~D[2021-01-07], ~D[2021-01-13]},+ dimensions: ["visit:browser"],+ include: %QueryInclude{+ compare: :previous_period+ },+ pagination: %{limit: 2, offset: 0}+ })++ assert %Stats.QueryResult{results: page_one_results} = Stats.query(site, page_one_query)++ assert page_one_results == [+ %{+ dimensions: ["Chrome"],+ metrics: [3, 50.0],+ comparison: %{+ dimensions: ["Chrome"],+ metrics: [1, 12.5],+ change: [200, 300]+ }+ },+ %{+ dimensions: ["Firefox"],+ metrics: [2, 33.33],+ comparison: %{+ dimensions: ["Firefox"],+ metrics: [4, 50.0],+ change: [-50, -33]+ }+ }+ ]++ page_two_query = Stats.Query.set(page_one_query, pagination: %{limit: 2, offset: 2})++ assert %Stats.QueryResult{results: page_two_results} = Stats.query(site, page_two_query)++ assert page_two_results == [+ %{+ dimensions: ["Safari"],+ metrics: [1, 16.67],+ comparison: %{+ dimensions: ["Safari"],+ metrics: [3, 37.5],+ change: [-67, -56]+ }+ }+ ]+ end++ test "dimensional comparison with imported data", %{+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-07 00:00:00]),+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-07 00:00:00]),+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-07 00:00:00]),+ build(:imported_browsers,+ date: ~D[2021-01-01],+ browser: "Firefox",+ browser_version: "121",+ visitors: 50+ ),+ build(:imported_browsers,+ date: ~D[2021-01-01],+ browser: "Chrome",+ browser_version: "99",+ visitors: 39+ ),+ build(:imported_browsers,+ date: ~D[2021-01-01],+ browser: "Safari",+ browser_version: "99",+ visitors: 10+ ),+ build(:imported_visitors, date: ~D[2021-01-01], visitors: 99)+ ])++ assert {:ok, query} =+ QueryBuilder.build(site, %ParsedQueryParams{+ metrics: [:visitors, :percentage],+ input_date_range: {:date_range, ~D[2021-01-07], ~D[2021-01-13]},+ dimensions: ["visit:browser"],+ include: %QueryInclude{+ imports: true,+ compare: :previous_period+ },+ pagination: %{limit: 2, offset: 0}+ })++ assert %Stats.QueryResult{results: results} = Stats.query(site, query)++ assert results == [+ %{+ dimensions: ["Chrome"],+ metrics: [2, 66.67],+ comparison: %{+ dimensions: ["Chrome"],+ metrics: [40, 40.0],+ change: [-95, 67]+ }+ },+ %{+ dimensions: ["Firefox"],+ metrics: [1, 33.33],+ comparison: %{+ dimensions: ["Firefox"],+ metrics: [50, 50.0],+ change: [-98, -33]+ }+ }+ ]+ end++ describe "custom comparison range" do+ test "can use date range for custom comparison", %{site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:25:00]),+ build(:pageview, timestamp: ~N[2021-01-07 00:00:00])+ ])++ assert {:ok, query} =+ QueryBuilder.build(site, %ParsedQueryParams{+ metrics: [:pageviews],+ input_date_range: {:date_range, ~D[2021-01-07], ~D[2021-01-13]},+ include: %QueryInclude{compare: {:date_range, ~D[2021-01-01], ~D[2021-01-06]}}+ })++ assert %Stats.QueryResult{results: results} = Stats.query(site, query)++ assert results == [+ %{+ dimensions: [],+ metrics: [1],+ comparison: %{change: [-50], dimensions: [], metrics: [2]}+ }+ ]+ end++ test "can use datetime range for custom comparison", %{site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 01:00:00]),+ build(:pageview, user_id: 1, timestamp: ~N[2021-01-01 05:25:00]),+ build(:pageview, user_id: 1, timestamp: ~N[2021-01-01 05:26:00]),+ build(:pageview, timestamp: ~N[2021-01-02 04:00:00]),+ build(:pageview, timestamp: ~N[2021-01-03 02:00:00])+ ])++ assert {:ok, query} =+ QueryBuilder.build(site, %ParsedQueryParams{+ metrics: [:visitors, :pageviews],+ input_date_range:+ {:datetime_range, ~U[2021-01-02 03:00:00Z], ~U[2021-01-03 02:59:59Z]},+ include: %QueryInclude{+ compare: {:datetime_range, ~U[2021-01-01 03:00:00Z], ~U[2021-01-02 02:59:59Z]}+ }+ })++ assert %Stats.QueryResult{results: results} = Stats.query(site, query)++ assert results == [+ %{+ dimensions: [],+ metrics: [2, 2],+ comparison: %{change: [100, 0], dimensions: [], metrics: [1, 2]}+ }+ ]+ end++ test "custom datetime range comparison handles timezones correctly", %{user: user} do+ weird_tz_site = new_site(owner: user, timezone: "America/Havana")++ populate_stats(weird_tz_site, [+ # 03:00 America/Havana+ build(:pageview, timestamp: ~N[2021-01-01 08:00:00]),+ # 05:25 America/Havana+ build(:pageview, timestamp: ~N[2021-01-01 10:25:00]),+ # 04:00 America/Havana+ build(:pageview, timestamp: ~N[2021-01-02 09:00:00]),+ # 02:00 America/Havana+ build(:pageview, timestamp: ~N[2021-01-03 08:00:00])+ ])++ assert {:ok, query} =+ QueryBuilder.build(weird_tz_site, %ParsedQueryParams{+ metrics: [:pageviews],+ input_date_range:+ {:datetime_range, ~U[2021-01-02 03:00:00Z], ~U[2021-01-03 02:59:59Z]},+ include: %QueryInclude{+ compare: {:datetime_range, ~U[2021-01-01 03:00:00Z], ~U[2021-01-02 02:59:59Z]}+ }+ })++ assert %Stats.QueryResult{results: results} = Stats.query(weird_tz_site, query)++ assert results == [+ %{+ dimensions: [],+ metrics: [1],+ comparison: %{change: [-50], dimensions: [], metrics: [2]}+ }+ ]+ end+ end+enddiff --git a/test/plausible/stats/query/query_imported_test.exs b/test/plausible/stats/query/query_imported_test.exsnew file mode 100644index 0000000000..40e1b0d60e--- /dev/null+++ b/test/plausible/stats/query/query_imported_test.exs@@ -0,0 +1,128 @@+defmodule Plausible.Stats.QueryImportedTest do+ use Plausible.DataCase+ alias Plausible.Stats+ alias Plausible.Stats.{ParsedQueryParams, QueryBuilder, QueryInclude}++ @unsupported_query_warning Plausible.Stats.QueryResult.imports_warnings().unsupported_query+ @no_imported_scroll_depth_warning Plausible.Stats.QueryResult.no_imported_scroll_depth_warning()++ setup [:create_user, :create_site]++ describe "behavioral filters" do+ setup :create_site_import++ test "imports are skipped when has_done filter is used", %{+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:event, name: "pageview", user_id: 1, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "pageview", user_id: 2, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "Conversion", user_id: 3, timestamp: ~N[2021-01-01 00:00:00]),+ build(:imported_pages,+ page: "/blog",+ pageviews: 5,+ visitors: 3,+ date: ~D[2023-01-01]+ )+ ])++ assert {:ok, query} =+ QueryBuilder.build(site, %ParsedQueryParams{+ metrics: [:visitors],+ input_date_range: :all,+ filters: [[:has_done, [:is, "event:name", ["pageview"]]]],+ include: %QueryInclude{imports: true}+ })++ assert %Stats.QueryResult{results: results, meta: meta} = Stats.query(site, query)++ assert results == [%{dimensions: [], metrics: [2]}]+ refute meta[:imports_included]++ assert meta[:imports_warning] == @unsupported_query_warning+ end++ test "imports are skipped when has_not_done filter is used", %{+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:event, name: "pageview", user_id: 1, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "pageview", user_id: 2, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "Conversion", user_id: 3, timestamp: ~N[2021-01-01 00:00:00]),+ build(:imported_pages,+ page: "/blog",+ pageviews: 5,+ visitors: 3,+ date: ~D[2023-01-01]+ )+ ])++ assert {:ok, query} =+ QueryBuilder.build(site, %ParsedQueryParams{+ metrics: [:visitors],+ input_date_range: :all,+ dimensions: ["event:goal"],+ filters: [[:has_not_done, [:is, "event:name", ["pageview"]]]],+ include: %QueryInclude{imports: true}+ })++ assert %Stats.QueryResult{results: results, meta: meta} = Stats.query(site, query)++ assert results == []+ refute meta[:imports_included]++ assert meta[:imports_warning] == @unsupported_query_warning+ end+ end++ describe "scroll depth metric warnings" do+ test "returns warning when import without scroll depth in comparison range", %{+ site: site+ } do+ site_import =+ insert(:site_import, site: site, start_date: ~D[2021-02-01], end_date: ~D[2021-02-28])++ populate_stats(site, site_import.id, [+ build(:pageview, user_id: 123, pathname: "/", timestamp: ~N[2021-02-01 00:00:00]),+ build(:engagement,+ user_id: 123,+ pathname: "/",+ timestamp: ~N[2021-02-01 00:01:00],+ scroll_depth: 70+ ),+ build(:imported_pages, page: "/", date: ~D[2021-02-01]),+ build(:imported_pages, page: "/", date: ~D[2021-02-28])+ ])++ assert {:ok, query} =+ QueryBuilder.build(site, %ParsedQueryParams{+ metrics: [:visitors, :scroll_depth],+ input_date_range: {:date_range, ~D[2022-01-01], ~D[2022-12-31]},+ filters: [[:is, "event:page", ["/"]]],+ include: %QueryInclude{imports: true, compare: :previous_period}+ })++ assert %Stats.QueryResult{results: results, meta: meta} = Stats.query(site, query)++ assert results == [+ %{+ dimensions: [],+ metrics: [0, nil],+ comparison: %{+ change: [-100, nil],+ dimensions: [],+ metrics: [3, 70]+ }+ }+ ]++ assert meta[:imports_included] == true++ assert meta[:metric_warnings] == %{+ scroll_depth: @no_imported_scroll_depth_warning+ }+ end+ end+enddiff --git a/test/plausible/stats/query/query_parse_and_build_test.exs b/test/plausible/stats/query/query_parse_and_build_test.exsindex 73cd22e470..cfcbcd98d5 100644--- a/test/plausible/stats/query/query_parse_and_build_test.exs+++ b/test/plausible/stats/query/query_parse_and_build_test.exs@@ -50,12 +50,13 @@ defmodule Plausible.Stats.Query.QueryParseAndBuildTest dolast: DateTime.new!(~D[2021-04-30], ~T[23:59:59], "Etc/UTC")}- @default_include %{+ @default_include %Plausible.Stats.QueryInclude{imports: false,imports_meta: false,time_labels: false,total_rows: false,- comparisons: nil,+ compare: nil,+ compare_match_day_of_week: false,legacy_time_on_page_cutoff: nil,trim_relative_date_range: false}@@ -918,7 +919,8 @@ defmodule Plausible.Stats.Query.QueryParseAndBuildTest doimports_meta: false,time_labels: true,total_rows: true,- comparisons: nil,+ compare: nil,+ compare_match_day_of_week: false,legacy_time_on_page_cutoff: nil,trim_relative_date_range: false},@@ -951,25 +953,25 @@ defmodule Plausible.Stats.Query.QueryParseAndBuildTest doendend- describe "include.comparisons" do+ describe "comparisons" dotest "not allowed in public API", %{site: site} doparams = %{"site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all",- "include" => %{"comparisons" => %{"mode" => "previous_period"}}+ "include" => %{"compare" => "previous_period"}}assert {:error, error} = Query.parse_and_build(site, :public, params)- assert error == "#/include/comparisons: Schema does not allow additional properties."+ assert error == "#/include/compare: Schema does not allow additional properties."end- test "mode=previous_period", %{site: site} do+ test "previous_period", %{site: site} doparams = %{"site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all",- "include" => %{"comparisons" => %{"mode" => "previous_period"}}+ "include" => %{"compare" => "previous_period"}}assert {:ok, query} = Query.parse_and_build(site, :internal, params)@@ -982,9 +984,8 @@ defmodule Plausible.Stats.Query.QueryParseAndBuildTest doorder_by: nil,timezone: ^site.timezone,include: %{- comparisons: %{- mode: "previous_period"- },+ compare: :previous_period,+ compare_match_day_of_week: false,imports: false,imports_meta: false,time_labels: false,@@ -996,12 +997,12 @@ defmodule Plausible.Stats.Query.QueryParseAndBuildTest do} = queryend- test "mode=year_over_year", %{site: site} do+ test "year_over_year with match_day_of_week", %{site: site} doparams = %{"site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all",- "include" => %{"comparisons" => %{"mode" => "year_over_year"}}+ "include" => %{"compare" => "year_over_year", "compare_match_day_of_week" => true}}assert {:ok, query} = Query.parse_and_build(site, :internal, params)@@ -1014,9 +1015,8 @@ defmodule Plausible.Stats.Query.QueryParseAndBuildTest doorder_by: nil,timezone: ^site.timezone,include: %{- comparisons: %{- mode: "year_over_year"- },+ compare: :year_over_year,+ compare_match_day_of_week: true,imports: false,imports_meta: false,time_labels: false,@@ -1028,14 +1028,12 @@ defmodule Plausible.Stats.Query.QueryParseAndBuildTest do} = queryend- test "mode=custom", %{site: site} do+ test "with custom date range", %{site: site} doparams = %{"site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all",- "include" => %{- "comparisons" => %{"mode" => "custom", "date_range" => ["2021-04-05", "2021-05-04"]}- }+ "include" => %{"compare" => ["2021-04-05", "2021-05-04"]}}assert {:ok, query} = Query.parse_and_build(site, :internal, params)@@ -1048,10 +1046,8 @@ defmodule Plausible.Stats.Query.QueryParseAndBuildTest doorder_by: nil,timezone: ^site.timezone,include: %{- comparisons: %{- mode: "custom",- date_range: ^@date_range_30d- },+ compare: {:date_range, ~D[2021-04-05], ~D[2021-05-04]},+ compare_match_day_of_week: false,imports_meta: false,imports: false,time_labels: false,@@ -1063,37 +1059,18 @@ defmodule Plausible.Stats.Query.QueryParseAndBuildTest do} = queryend- test "mode=custom without date_range is invalid", %{site: site} do- params = %{- "site_id" => site.domain,- "metrics" => ["visitors"],- "date_range" => "all",- "include" => %{"comparisons" => %{"mode" => "custom"}}- }-- assert {:error, error} = Query.parse_and_build(site, :internal, params)-- assert error ==- "#/include/comparisons: Expected exactly one of the schemata to match, but none of them did."- end-- test "mode=previous_period with date_range is invalid", %{site: site} do+ test "invalid compare", %{site: site} doparams = %{"site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all",- "include" => %{- "comparisons" => %{- "mode" => "previous_period",- "date_range" => ["2024-01-01", "2024-01-31"]- }- }+ "include" => %{"compare" => "2024-01-01"}}assert {:error, error} = Query.parse_and_build(site, :internal, params)assert error ==- "#/include/comparisons: Expected exactly one of the schemata to match, but none of them did."+ "#/include/compare: Expected exactly one of the schemata to match, but none of them did."endenddiff --git a/test/plausible/stats/query/query_special_metrics_test.exs b/test/plausible/stats/query/query_special_metrics_test.exsnew file mode 100644index 0000000000..c6a7ce48a8--- /dev/null+++ b/test/plausible/stats/query/query_special_metrics_test.exs@@ -0,0 +1,389 @@+defmodule Plausible.Stats.QuerySpecialMetricsTest do+ use Plausible.DataCase+ alias Plausible.Stats+ alias Plausible.Stats.{ParsedQueryParams, QueryBuilder, QueryInclude}++ setup [:create_user, :create_site]++ describe "exit_rate" do+ test "in visit:exit_page breakdown without filters", %{site: site} do+ populate_stats(site, [+ build(:pageview, user_id: 1, pathname: "/one", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: 1, pathname: "/two", timestamp: ~N[2021-01-01 00:10:00]),+ build(:pageview, user_id: 3, pathname: "/one", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: 3, pathname: "/never-exit", timestamp: ~N[2021-01-01 00:00:00]),+ build(:event,+ user_id: 3,+ name: "a",+ pathname: "/one",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview, user_id: 3, pathname: "/one", timestamp: ~N[2021-01-01 00:10:00])+ ])++ assert {:ok, query} =+ QueryBuilder.build(site, %ParsedQueryParams{+ metrics: [:exit_rate],+ input_date_range: :all,+ dimensions: ["visit:exit_page"],+ order_by: [{"visit:exit_page", :desc}]+ })++ assert %Stats.QueryResult{results: results} = Stats.query(site, query)++ assert results == [+ %{dimensions: ["/two"], metrics: [100]},+ %{dimensions: ["/one"], metrics: [33.3]}+ ]+ end++ test "in visit:exit_page breakdown filtered by visit:exit_page", %{site: site} do+ populate_stats(site, [+ build(:pageview, pathname: "/one", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, pathname: "/one", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: 1, pathname: "/one", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: 1, pathname: "/two", timestamp: ~N[2021-01-01 00:10:00])+ ])++ assert {:ok, query} =+ QueryBuilder.build(site, %ParsedQueryParams{+ metrics: [:exit_rate],+ input_date_range: :all,+ dimensions: ["visit:exit_page"],+ filters: [[:is, "visit:exit_page", ["/one"]]],+ order_by: [{"visit:exit_page", :desc}]+ })++ assert %Stats.QueryResult{results: results} = Stats.query(site, query)++ assert results == [+ %{dimensions: ["/one"], metrics: [66.7]}+ ]+ end++ test "in visit:exit_page breakdown filtered by visit:exit_page and visit:entry_page", %{+ site: site+ } do+ populate_stats(site, [+ # Bounced sessions: Match both entry- and exit page filters+ build(:pageview, pathname: "/one", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, pathname: "/one", timestamp: ~N[2021-01-01 00:00:00]),+ # Session 1: Matches both entry- and exit page filters+ build(:pageview, user_id: 1, pathname: "/one", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: 1, pathname: "/two", timestamp: ~N[2021-01-01 00:10:00]),+ build(:pageview, user_id: 1, pathname: "/one", timestamp: ~N[2021-01-01 00:00:00]),+ # Session 2: Does not match exit_page filter, BUT the pageview on /one still+ # gets counted towards total pageviews.+ build(:pageview, user_id: 2, pathname: "/one", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: 2, pathname: "/two", timestamp: ~N[2021-01-01 00:10:00]),+ # Session 3: Does not match entry_page filter, should be ignored+ build(:pageview, user_id: 3, pathname: "/two", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: 3, pathname: "/one", timestamp: ~N[2021-01-01 00:20:00])+ ])++ assert {:ok, query} =+ QueryBuilder.build(site, %ParsedQueryParams{+ metrics: [:exit_rate],+ input_date_range: :all,+ dimensions: ["visit:exit_page"],+ filters: [+ [:is, "visit:exit_page", ["/one"]],+ [:is, "visit:entry_page", ["/one"]]+ ]+ })++ assert %Stats.QueryResult{results: results} = Stats.query(site, query)++ assert results == [+ %{dimensions: ["/one"], metrics: [60]}+ ]+ end++ test "in visit:exit_page breakdown filtered by visit:country", %{site: site} do+ populate_stats(site, [+ build(:pageview,+ pathname: "/one",+ country_code: "EE",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/one",+ country_code: "US",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ user_id: 1,+ pathname: "/one",+ country_code: "EE",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ user_id: 1,+ pathname: "/two",+ country_code: "EE",+ timestamp: ~N[2021-01-01 00:10:00]+ )+ ])++ assert {:ok, query} =+ QueryBuilder.build(site, %ParsedQueryParams{+ metrics: [:exit_rate],+ input_date_range: :all,+ dimensions: ["visit:exit_page"],+ filters: [+ [:is, "visit:country", ["EE"]],+ [:is, "visit:entry_page", ["/one"]]+ ],+ order_by: [{:exit_rate, :asc}]+ })++ assert %Stats.QueryResult{results: results} = Stats.query(site, query)++ assert results == [+ %{dimensions: ["/one"], metrics: [50]},+ %{dimensions: ["/two"], metrics: [100.0]}+ ]+ end++ test "sorting and pagination", %{site: site} do+ populate_stats(site, [+ build(:pageview, pathname: "/one", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: 2, pathname: "/two", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: 2, pathname: "/two", timestamp: ~N[2021-01-01 00:01:00]),+ build(:pageview, user_id: 3, pathname: "/three", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: 3, pathname: "/three", timestamp: ~N[2021-01-01 00:01:00]),+ build(:pageview, user_id: 3, pathname: "/three", timestamp: ~N[2021-01-01 00:02:00]),+ build(:pageview, user_id: 4, pathname: "/four", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: 4, pathname: "/four", timestamp: ~N[2021-01-01 00:01:00]),+ build(:pageview, user_id: 4, pathname: "/four", timestamp: ~N[2021-01-01 00:02:00]),+ build(:pageview, user_id: 4, pathname: "/four", timestamp: ~N[2021-01-01 00:03:00])+ ])++ do_query = fn order_by, pagination ->+ {:ok, query} =+ QueryBuilder.build(site, %ParsedQueryParams{+ metrics: [:exit_rate],+ input_date_range: :all,+ dimensions: ["visit:exit_page"],+ order_by: order_by,+ pagination: pagination+ })++ %Stats.QueryResult{results: results} = Stats.query(site, query)+ results+ end++ all_results_asc = do_query.([{:exit_rate, :asc}], %{limit: 4, offset: 0})+ all_results_desc = do_query.([{:exit_rate, :desc}], %{limit: 4, offset: 0})++ assert all_results_asc == Enum.reverse(all_results_desc)++ assert do_query.([{:exit_rate, :desc}], %{limit: 2, offset: 0}) == [+ %{dimensions: ["/one"], metrics: [100]},+ %{dimensions: ["/two"], metrics: [50]}+ ]++ assert do_query.([{:exit_rate, :desc}], %{limit: 2, offset: 2}) == [+ %{dimensions: ["/three"], metrics: [33.3]},+ %{dimensions: ["/four"], metrics: [25]}+ ]++ assert do_query.([{:exit_rate, :asc}], %{limit: 3, offset: 1}) == [+ %{dimensions: ["/three"], metrics: [33.3]},+ %{dimensions: ["/two"], metrics: [50]},+ %{dimensions: ["/one"], metrics: [100]}+ ]+ end++ test "with comparisons", %{site: site} do+ populate_stats(site, [+ build(:pageview, user_id: 1, pathname: "/one", timestamp: ~N[2021-01-09 00:00:00]),+ build(:pageview, user_id: 1, pathname: "/three", timestamp: ~N[2021-01-09 00:00:00]),+ build(:pageview, pathname: "/one", timestamp: ~N[2021-01-09 00:10:00]),+ build(:pageview, user_id: 2, pathname: "/one", timestamp: ~N[2021-01-10 00:00:00]),+ build(:pageview, user_id: 2, pathname: "/two", timestamp: ~N[2021-01-10 00:10:00]),+ build(:pageview, user_id: 3, pathname: "/one", timestamp: ~N[2021-01-10 00:00:00]),+ build(:pageview, user_id: 3, pathname: "/one", timestamp: ~N[2021-01-10 00:10:00])+ ])++ {:ok, query} =+ QueryBuilder.build(site, %ParsedQueryParams{+ metrics: [:exit_rate],+ input_date_range: {:date_range, ~D[2021-01-10], ~D[2021-01-10]},+ dimensions: ["visit:exit_page"],+ order_by: [{:exit_rate, :desc}],+ include: %QueryInclude{compare: :previous_period}+ })++ %Stats.QueryResult{results: results} = Stats.query(site, query)++ assert results == [+ %{+ dimensions: ["/two"],+ metrics: [100],+ comparison: %{+ change: [nil],+ dimensions: ["/two"],+ metrics: [nil]+ }+ },+ %{+ dimensions: ["/one"],+ metrics: [33.3],+ comparison: %{+ change: [-16.7],+ dimensions: ["/one"],+ metrics: [50]+ }+ }+ ]+ end++ test "with imported data", %{site: site} do+ site_import =+ insert(:site_import,+ site: site,+ start_date: ~D[2020-01-01],+ end_date: ~D[2020-12-31]+ )++ populate_stats(site, site_import.id, [+ build(:pageview, user_id: 1, pathname: "/one", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: 1, pathname: "/two", timestamp: ~N[2021-01-01 00:10:00]),+ build(:pageview, user_id: 3, pathname: "/one", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: 3, pathname: "/three", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: 3, pathname: "/one", timestamp: ~N[2021-01-01 00:10:00]),+ build(:imported_pages, page: "/one", visits: 10, pageviews: 20, date: ~D[2020-01-01]),+ build(:imported_exit_pages, exit_page: "/one", exits: 2, date: ~D[2020-01-01])+ ])++ {:ok, query} =+ QueryBuilder.build(site, %ParsedQueryParams{+ metrics: [:exit_rate],+ input_date_range: :all,+ dimensions: ["visit:exit_page"],+ order_by: [{:exit_rate, :desc}],+ include: %QueryInclude{imports: true}+ })++ %Stats.QueryResult{results: results} = Stats.query(site, query)++ assert results == [+ %{dimensions: ["/two"], metrics: [100]},+ %{dimensions: ["/one"], metrics: [13]}+ ]+ end+ end++ describe "time_on_page" do+ test "time_on_page breakdown (with missing data)", %{site: site} do+ populate_stats(site, [+ build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),+ build(:engagement,+ user_id: 12,+ pathname: "/",+ timestamp: ~N[2021-01-01 00:03:00],+ engagement_time: 20_000+ ),+ build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),+ build(:engagement,+ user_id: 12,+ pathname: "/",+ timestamp: ~N[2021-01-01 00:04:00],+ engagement_time: 30_000+ ),+ build(:pageview, pathname: "/blog", timestamp: ~N[2021-01-01 00:00:00])+ ])++ {:ok, query} =+ QueryBuilder.build(site, %ParsedQueryParams{+ metrics: [:visitors, :time_on_page],+ input_date_range: :all,+ dimensions: ["event:page"]+ })++ %Stats.QueryResult{results: results} = Stats.query(site, query)++ assert results == [+ %{dimensions: ["/"], metrics: [1, 50]},+ %{dimensions: ["/blog"], metrics: [1, nil]}+ ]+ end++ test "can use comparisons together with `legacy_time_on_page_cutoff` in timeseries", %{+ site: site+ } do+ populate_stats(site, [+ build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),+ build(:engagement,+ user_id: 12,+ pathname: "/",+ timestamp: ~N[2021-01-01 00:05:00],+ engagement_time: 100_000+ ),+ build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-02 00:00:00]),+ build(:engagement,+ user_id: 12,+ pathname: "/",+ timestamp: ~N[2021-01-02 00:05:00],+ engagement_time: 200_000+ ),+ build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-03 00:00:00]),+ build(:engagement,+ user_id: 12,+ pathname: "/",+ timestamp: ~N[2021-01-03 00:05:00],+ engagement_time: 250_000+ ),+ build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-04 00:00:00]),+ build(:engagement,+ user_id: 12,+ pathname: "/",+ timestamp: ~N[2021-01-04 00:05:00],+ engagement_time: 200_000+ ),+ build(:pageview, user_id: 13, pathname: "/", timestamp: ~N[2021-01-04 00:00:00]),+ build(:engagement,+ user_id: 13,+ pathname: "/",+ timestamp: ~N[2021-01-04 00:05:00],+ engagement_time: 100_000+ )+ ])++ site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[2021-01-02])++ {:ok, query} =+ QueryBuilder.build(site, %ParsedQueryParams{+ metrics: [:time_on_page],+ input_date_range: {:date_range, ~D[2021-01-03], ~D[2021-01-04]},+ filters: [[:is, "event:page", ["/"]]],+ dimensions: ["time:day"],+ include: %QueryInclude{compare: :previous_period}+ })++ %Stats.QueryResult{results: results} = Stats.query(site, query)++ assert results == [+ %{+ dimensions: ["2021-01-03"],+ metrics: [250],+ comparison: %{+ dimensions: ["2021-01-01"],+ metrics: [nil],+ change: [nil]+ }+ },+ %{+ dimensions: ["2021-01-04"],+ metrics: [150],+ comparison: %{+ dimensions: ["2021-01-02"],+ metrics: [200],+ change: [-25]+ }+ }+ ]+ end+ end+enddiff --git a/test/plausible/stats/query/query_test.exs b/test/plausible/stats/query/query_test.exsnew file mode 100644index 0000000000..85fb3c6ba5--- /dev/null+++ b/test/plausible/stats/query/query_test.exs@@ -0,0 +1,129 @@+defmodule Plausible.Stats.QueryTest do+ use Plausible.DataCase+ alias Plausible.Stats+ alias Plausible.Stats.{ParsedQueryParams, QueryBuilder, QueryInclude}++ @user_id 123++ setup [:create_user, :create_site]++ describe "timeseries" do+ test "breakdown by time:minute (internal API), counts visitors and visits in all buckets their session was active in",+ %{site: site} do+ populate_stats(site, [+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:10:00])+ ])++ {:ok, query} =+ QueryBuilder.build(site, %ParsedQueryParams{+ metrics: [:visitors, :visits, :pageviews],+ input_date_range: {:datetime_range, ~U[2021-01-01 00:00:00Z], ~U[2021-01-01 00:10:00Z]},+ dimensions: ["time:minute"]+ })++ %Stats.QueryResult{results: results} = Stats.query(site, query)++ assert results == [+ %{dimensions: ["2021-01-01 00:00:00"], metrics: [1, 1, 1]},+ %{dimensions: ["2021-01-01 00:01:00"], metrics: [1, 1, 0]},+ %{dimensions: ["2021-01-01 00:02:00"], metrics: [1, 1, 0]},+ %{dimensions: ["2021-01-01 00:03:00"], metrics: [1, 1, 0]},+ %{dimensions: ["2021-01-01 00:04:00"], metrics: [1, 1, 0]},+ %{dimensions: ["2021-01-01 00:05:00"], metrics: [1, 1, 0]},+ %{dimensions: ["2021-01-01 00:06:00"], metrics: [1, 1, 0]},+ %{dimensions: ["2021-01-01 00:07:00"], metrics: [1, 1, 0]},+ %{dimensions: ["2021-01-01 00:08:00"], metrics: [1, 1, 0]},+ %{dimensions: ["2021-01-01 00:09:00"], metrics: [1, 1, 0]},+ %{dimensions: ["2021-01-01 00:10:00"], metrics: [1, 1, 1]}+ ]+ end++ test "breakdown by time:hour (internal API), counts visitors and visits in all buckets their session was active in",+ %{site: site} do+ populate_stats(site, [+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:20:00]),+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:40:00]),+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 01:00:00]),+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 01:20:00])+ ])++ {:ok, query} =+ QueryBuilder.build(site, %ParsedQueryParams{+ metrics: [:visitors, :visits, :visit_duration],+ input_date_range: {:date_range, ~D[2021-01-01], ~D[2021-01-02]},+ dimensions: ["time:hour"]+ })++ %Stats.QueryResult{results: results} = Stats.query(site, query)++ assert results == [+ %{dimensions: ["2021-01-01 00:00:00"], metrics: [1, 1, 0]},+ %{dimensions: ["2021-01-01 01:00:00"], metrics: [1, 1, 3600]}+ ]+ end++ test "shows month to date with time labels trimmed", %{site: site} do+ Plausible.Stats.Query.Test.fix_now(~U[2021-01-15 12:00:00Z])++ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-15 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-16 00:00:00])+ ])++ {:ok, query} =+ QueryBuilder.build(site, %ParsedQueryParams{+ metrics: [:visitors],+ input_date_range: :month,+ dimensions: ["time:day"],+ include: %QueryInclude{trim_relative_date_range: true}+ })++ %Stats.QueryResult{results: results, query: query} = Stats.query(site, query)++ assert results == [+ %{dimensions: ["2021-01-01"], metrics: [1]},+ %{dimensions: ["2021-01-15"], metrics: [1]}+ ]++ assert query[:date_range] == [+ "2021-01-01T00:00:00Z",+ "2021-01-15T23:59:59Z"+ ]+ end++ test "visitors and visits are smeared across time:minute buckets but visit_duration is not",+ %{site: site} do+ populate_stats(site, [+ build(:pageview, user_id: 1, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: 1, timestamp: ~N[2021-01-01 00:10:00]),+ build(:pageview, user_id: 2, timestamp: ~N[2021-01-01 00:05:00]),+ build(:pageview, user_id: 2, timestamp: ~N[2021-01-01 00:08:00])+ ])++ {:ok, query} =+ QueryBuilder.build(site, %ParsedQueryParams{+ metrics: [:visitors, :visits, :visit_duration, :pageviews],+ input_date_range: {:datetime_range, ~U[2021-01-01 00:00:00Z], ~U[2021-01-01 00:30:00Z]},+ dimensions: ["time:minute"]+ })++ %Stats.QueryResult{results: results} = Stats.query(site, query)++ assert results == [+ %{dimensions: ["2021-01-01 00:00:00"], metrics: [1, 1, 0, 1]},+ %{dimensions: ["2021-01-01 00:01:00"], metrics: [1, 1, 0, 0]},+ %{dimensions: ["2021-01-01 00:02:00"], metrics: [1, 1, 0, 0]},+ %{dimensions: ["2021-01-01 00:03:00"], metrics: [1, 1, 0, 0]},+ %{dimensions: ["2021-01-01 00:04:00"], metrics: [1, 1, 0, 0]},+ %{dimensions: ["2021-01-01 00:05:00"], metrics: [2, 2, 0, 1]},+ %{dimensions: ["2021-01-01 00:06:00"], metrics: [2, 2, 0, 0]},+ %{dimensions: ["2021-01-01 00:07:00"], metrics: [2, 2, 0, 0]},+ %{dimensions: ["2021-01-01 00:08:00"], metrics: [2, 2, 180, 1]},+ %{dimensions: ["2021-01-01 00:09:00"], metrics: [1, 1, 0, 0]},+ %{dimensions: ["2021-01-01 00:10:00"], metrics: [1, 1, 600, 1]}+ ]+ end+ end+enddiff --git a/test/plausible_web/controllers/api/external_stats_controller/query_comparisons_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_comparisons_test.exsdeleted file mode 100644index 99596e0032..0000000000--- a/test/plausible_web/controllers/api/external_stats_controller/query_comparisons_test.exs+++ /dev/null@@ -1,477 +0,0 @@-defmodule PlausibleWeb.Api.ExternalStatsController.QueryComparisonsTest do- use PlausibleWeb.ConnCase-- setup [:create_user, :create_site, :create_api_key, :use_api_key, :create_site_import]-- test "aggregates a single metric", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, timestamp: ~N[2021-01-01 00:25:00]),- build(:pageview, timestamp: ~N[2021-01-02 00:00:00]),- build(:pageview, timestamp: ~N[2021-01-07 00:00:00])- ])-- conn =- post(conn, "/api/v2/query-internal-test", %{- "site_id" => site.domain,- "metrics" => ["pageviews"],- "date_range" => ["2021-01-07", "2021-01-13"],- "include" => %{"comparisons" => %{"mode" => "previous_period"}}- })-- assert json_response(conn, 200)["results"] == [- %{- "dimensions" => [],- "metrics" => [1],- "comparison" => %{"change" => [-67], "dimensions" => [], "metrics" => [3]}- }- ]- end-- test "timeseries comparison", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, timestamp: ~N[2021-01-01 00:25:00]),- build(:pageview, timestamp: ~N[2021-01-06 00:00:00]),- build(:pageview, timestamp: ~N[2021-01-07 00:00:00]),- build(:pageview, timestamp: ~N[2021-01-08 00:00:00])- ])-- conn =- post(conn, "/api/v2/query-internal-test", %{- "site_id" => site.domain,- "metrics" => ["pageviews"],- "date_range" => ["2021-01-07", "2021-01-13"],- "dimensions" => ["time:day"],- "include" => %{"comparisons" => %{"mode" => "previous_period"}}- })-- assert json_response(conn, 200)["results"] == [- %{- "dimensions" => ["2021-01-07"],- "metrics" => [1],- "comparison" => %{- "dimensions" => ["2020-12-31"],- "metrics" => [0],- "change" => [100]- }- },- %{- "dimensions" => ["2021-01-08"],- "metrics" => [1],- "comparison" => %{- "dimensions" => ["2021-01-01"],- "metrics" => [2],- "change" => [-50]- }- },- %{- "dimensions" => ["2021-01-09"],- "metrics" => [0],- "comparison" => %{- "dimensions" => ["2021-01-02"],- "metrics" => [0],- "change" => [0]- }- },- %{- "dimensions" => ["2021-01-10"],- "metrics" => [0],- "comparison" => %{- "dimensions" => ["2021-01-03"],- "metrics" => [0],- "change" => [0]- }- },- %{- "dimensions" => ["2021-01-11"],- "metrics" => [0],- "comparison" => %{- "dimensions" => ["2021-01-04"],- "metrics" => [0],- "change" => [0]- }- },- %{- "dimensions" => ["2021-01-12"],- "metrics" => [0],- "comparison" => %{- "dimensions" => ["2021-01-05"],- "metrics" => [0],- "change" => [0]- }- },- %{- "dimensions" => ["2021-01-13"],- "metrics" => [0],- "comparison" => %{- "dimensions" => ["2021-01-06"],- "metrics" => [1],- "change" => [-100]- }- }- ]- end-- test "timeseries last 28d period compares the same period with and without match_day_of_week=true",- %{- conn: conn,- site: site- } do- now = ~U[2021-06-10 12:00:00Z]- Plausible.Stats.Query.Test.fix_now(now)-- today = DateTime.to_date(now)-- make_request = fn match_day_of_week ->- conn- |> post("/api/v2/query-internal-test", %{- "site_id" => site.domain,- "metrics" => ["visitors"],- "date_range" => "28d",- "dimensions" => ["time"],- "include" => %{- "time_labels" => true,- "comparisons" => %{- "mode" => "previous_period",- "match_day_of_week" => match_day_of_week- }- }- })- |> json_response(200)- end-- assert %{"results" => results1} = make_request.(false)- assert %{"results" => results2} = make_request.(true)-- assert results1 == results2-- expected_first_date = today |> Date.shift(day: -28) |> Date.to_iso8601()- expected_last_date = today |> Date.shift(day: -1) |> Date.to_iso8601()- expected_comparison_first_date = today |> Date.shift(day: -56) |> Date.to_iso8601()- expected_comparison_last_date = today |> Date.shift(day: -29) |> Date.to_iso8601()-- assert %{- "dimensions" => [actual_first_date],- "comparison" => %{- "dimensions" => [actual_comparison_first_date]- }- } = List.first(results1)-- assert %{- "dimensions" => [actual_last_date],- "comparison" => %{- "dimensions" => [actual_comparison_last_date]- }- } = List.last(results1)-- assert actual_first_date == expected_first_date- assert actual_last_date == expected_last_date- assert actual_comparison_first_date == expected_comparison_first_date- assert actual_comparison_last_date == expected_comparison_last_date- end-- test "timeseries last 91d period in year_over_year comparison", %{- conn: conn,- site: site- } do- populate_stats(site, [- build(:pageview, timestamp: ~N[2021-04-01 00:00:00]),- build(:pageview, timestamp: ~N[2021-04-01 00:00:00]),- build(:pageview, timestamp: ~N[2021-04-05 00:00:00]),- build(:pageview, timestamp: ~N[2021-04-05 00:00:00]),- build(:pageview, timestamp: ~N[2021-06-30 00:00:00]),- build(:pageview, timestamp: ~N[2022-04-01 00:00:00]),- build(:pageview, timestamp: ~N[2022-04-05 00:00:00]),- build(:pageview, timestamp: ~N[2022-06-30 00:00:00]),- build(:pageview, timestamp: ~N[2022-07-01 00:00:00])- ])-- Plausible.Stats.Query.Test.fix_now(~U[2022-07-01 14:00:00Z])-- conn =- post(conn, "/api/v2/query-internal-test", %{- "site_id" => site.domain,- "metrics" => ["visitors"],- "date_range" => "91d",- "dimensions" => ["time:day"],- "include" => %{- "time_labels" => true,- "comparisons" => %{"mode" => "year_over_year"}- }- })-- assert %{- "results" => results,- "meta" => %{"time_labels" => time_labels}- } = json_response(conn, 200)-- assert "2022-04-01" = List.first(time_labels)- assert "2022-04-05" = Enum.at(time_labels, 4)- assert "2022-06-30" = List.last(time_labels)-- assert %{- "dimensions" => ["2022-04-01"],- "metrics" => [1],- "comparison" => %{- "dimensions" => ["2021-04-01"],- "metrics" => [2]- }- } = Enum.find(results, &(&1["dimensions"] == ["2022-04-01"]))-- assert %{- "dimensions" => ["2022-04-05"],- "metrics" => [1],- "comparison" => %{- "dimensions" => ["2021-04-05"],- "metrics" => [2]- }- } = Enum.find(results, &(&1["dimensions"] == ["2022-04-05"]))-- assert %{- "dimensions" => ["2022-06-30"],- "metrics" => [1],- "comparison" => %{- "dimensions" => ["2021-06-30"],- "metrics" => [1]- }- } = Enum.find(results, &(&1["dimensions"] == ["2022-06-30"]))- end-- test "dimensional comparison with low limit", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, browser: "Safari", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, browser: "Safari", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, browser: "Safari", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-07 00:00:00]),- build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-07 00:00:00]),- build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-07 00:00:00]),- build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-07 00:00:00]),- build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-07 00:00:00]),- build(:pageview, browser: "Safari", timestamp: ~N[2021-01-08 00:00:00])- ])-- conn =- post(conn, "/api/v2/query-internal-test", %{- "site_id" => site.domain,- "metrics" => ["visitors", "percentage"],- "date_range" => ["2021-01-07", "2021-01-13"],- "dimensions" => ["visit:browser"],- "include" => %{- "comparisons" => %{"mode" => "previous_period"}- },- "pagination" => %{"limit" => 2}- })-- assert json_response(conn, 200)["results"] == [- %{- "dimensions" => ["Chrome"],- "metrics" => [3, 50.0],- "comparison" => %{- "dimensions" => ["Chrome"],- "metrics" => [1, 12.5],- "change" => [200, 300]- }- },- %{- "dimensions" => ["Firefox"],- "metrics" => [2, 33.33],- "comparison" => %{- "dimensions" => ["Firefox"],- "metrics" => [4, 50.0],- "change" => [-50, -33]- }- }- ]-- conn2 =- post(conn, "/api/v2/query-internal-test", %{- "site_id" => site.domain,- "metrics" => ["visitors", "percentage"],- "date_range" => ["2021-01-07", "2021-01-13"],- "dimensions" => ["visit:browser"],- "include" => %{- "comparisons" => %{"mode" => "previous_period"}- },- "pagination" => %{"limit" => 2, "offset" => 2}- })-- assert json_response(conn2, 200)["results"] == [- %{- "dimensions" => ["Safari"],- "metrics" => [1, 16.67],- "comparison" => %{- "dimensions" => ["Safari"],- "metrics" => [3, 37.5],- "change" => [-67, -56]- }- }- ]- end-- test "dimensional comparison with imported data", %{- conn: conn,- site: site,- site_import: site_import- } do- populate_stats(site, site_import.id, [- build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-07 00:00:00]),- build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-07 00:00:00]),- build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-07 00:00:00]),- build(:imported_browsers,- date: ~D[2021-01-01],- browser: "Firefox",- browser_version: "121",- visitors: 50- ),- build(:imported_browsers,- date: ~D[2021-01-01],- browser: "Chrome",- browser_version: "99",- visitors: 39- ),- build(:imported_browsers,- date: ~D[2021-01-01],- browser: "Safari",- browser_version: "99",- visitors: 10- ),- build(:imported_visitors, date: ~D[2021-01-01], visitors: 99)- ])-- conn =- post(conn, "/api/v2/query-internal-test", %{- "site_id" => site.domain,- "metrics" => ["visitors", "percentage"],- "date_range" => ["2021-01-07", "2021-01-13"],- "dimensions" => ["visit:browser"],- "include" => %{- "imports" => true,- "comparisons" => %{"mode" => "previous_period"}- },- "pagination" => %{"limit" => 2}- })-- assert json_response(conn, 200)["results"] == [- %{- "dimensions" => ["Chrome"],- "metrics" => [2, 66.67],- "comparison" => %{- "dimensions" => ["Chrome"],- "metrics" => [40, 40.0],- "change" => [-95, 67]- }- },- %{- "dimensions" => ["Firefox"],- "metrics" => [1, 33.33],- "comparison" => %{- "dimensions" => ["Firefox"],- "metrics" => [50, 50.0],- "change" => [-98, -33]- }- }- ]- end-- describe "custom comparison range" do- test "can use date range for custom comparison", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, timestamp: ~N[2021-01-01 00:25:00]),- build(:pageview, timestamp: ~N[2021-01-07 00:00:00])- ])-- conn =- post(conn, "/api/v2/query-internal-test", %{- "site_id" => site.domain,- "metrics" => ["pageviews"],- "date_range" => ["2021-01-07", "2021-01-13"],- "include" => %{- "comparisons" => %{"mode" => "custom", "date_range" => ["2021-01-01", "2021-01-06"]}- }- })-- assert json_response(conn, 200)["results"] == [- %{- "dimensions" => [],- "metrics" => [1],- "comparison" => %{"change" => [-50], "dimensions" => [], "metrics" => [2]}- }- ]- end-- test "can use datetime range for custom comparison", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview, timestamp: ~N[2021-01-01 01:00:00]),- build(:pageview, user_id: 1, timestamp: ~N[2021-01-01 05:25:00]),- build(:pageview, user_id: 1, timestamp: ~N[2021-01-01 05:26:00]),- build(:pageview, timestamp: ~N[2021-01-02 04:00:00]),- build(:pageview, timestamp: ~N[2021-01-03 02:00:00])- ])-- conn =- post(conn, "/api/v2/query-internal-test", %{- "site_id" => site.domain,- "metrics" => ["visitors", "pageviews"],- "date_range" => ["2021-01-02T03:00:00Z", "2021-01-03T02:59:59Z"],- "include" => %{- "comparisons" => %{- "mode" => "custom",- "date_range" => ["2021-01-01T03:00:00Z", "2021-01-02T02:59:59Z"]- }- }- })-- assert json_response(conn, 200)["results"] == [- %{- "dimensions" => [],- "metrics" => [2, 2],- "comparison" => %{"change" => [100, 0], "dimensions" => [], "metrics" => [1, 2]}- }- ]- end-- test "custom datetime range comparison handles timezones correctly", %{conn: conn, user: user} do- weird_tz_site = new_site(owner: user, timezone: "America/Havana")-- populate_stats(weird_tz_site, [- # 03:00 America/Havana- build(:pageview, timestamp: ~N[2021-01-01 08:00:00]),- # 05:25 America/Havana- build(:pageview, timestamp: ~N[2021-01-01 10:25:00]),- # 04:00 America/Havana- build(:pageview, timestamp: ~N[2021-01-02 09:00:00]),- # 02:00 America/Havana- build(:pageview, timestamp: ~N[2021-01-03 08:00:00])- ])-- conn =- post(conn, "/api/v2/query-internal-test", %{- "site_id" => weird_tz_site.domain,- "metrics" => ["pageviews"],- "date_range" => ["2021-01-02T03:00:00Z", "2021-01-03T02:59:59Z"],- "include" => %{- "comparisons" => %{- "mode" => "custom",- "date_range" => ["2021-01-01T03:00:00Z", "2021-01-02T02:59:59Z"]- }- }- })-- assert json_response(conn, 200)["results"] == [- %{- "dimensions" => [],- "metrics" => [1],- "comparison" => %{"change" => [-50], "dimensions" => [], "metrics" => [2]}- }- ]- end- end-enddiff --git a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exsindex 98941b1350..9cbaab0273 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs@@ -3,8 +3,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryImportedTest do@user_id Enum.random(1000..9999)- @unsupported_interval_warning "Imported stats are not included because the time dimension (i.e. the interval) is too short."- @unsupported_query_warning "Imported stats are not included in the results because query parameters are not supported. For more information, see: https://plausible.io/docs/stats-api#filtering-imported-stats"+ @unsupported_interval_warning Plausible.Stats.QueryResult.imports_warnings().unsupported_interval+ @unsupported_query_warning Plausible.Stats.QueryResult.imports_warnings().unsupported_query++ def unsupported_query_warning(), do: @unsupported_query_warning@no_imported_scroll_depth_metric_warning %{"code" => "no_imported_scroll_depth",@@ -1294,79 +1296,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryImportedTest doendend- describe "behavioral filters" do- setup :create_site_import-- test "imports are skipped when has_done filter is used", %{- conn: conn,- site: site,- site_import: site_import- } do- populate_stats(site, site_import.id, [- build(:event, name: "pageview", user_id: 1, timestamp: ~N[2021-01-01 00:00:00]),- build(:event, name: "pageview", user_id: 2, timestamp: ~N[2021-01-01 00:00:00]),- build(:event, name: "Conversion", user_id: 3, timestamp: ~N[2021-01-01 00:00:00]),- build(:imported_pages,- page: "/blog",- pageviews: 5,- visitors: 3,- date: ~D[2023-01-01]- )- ])-- conn =- post(conn, "/api/v2/query-internal-test", %{- "site_id" => site.domain,- "metrics" => ["visitors"],- "date_range" => "all",- "filters" => [- ["has_done", ["is", "event:name", ["pageview"]]]- ],- "include" => %{"imports" => true}- })-- assert json_response(conn, 200)["results"] == [%{"dimensions" => [], "metrics" => [2]}]- refute json_response(conn, 200)["meta"]["imports_included"]-- assert json_response(conn, 200)["meta"]["imports_warning"] == @unsupported_query_warning- end-- test "imports are skipped when has_not_done filter is used", %{- conn: conn,- site: site,- site_import: site_import- } do- populate_stats(site, site_import.id, [- build(:event, name: "pageview", user_id: 1, timestamp: ~N[2021-01-01 00:00:00]),- build(:event, name: "pageview", user_id: 2, timestamp: ~N[2021-01-01 00:00:00]),- build(:event, name: "Conversion", user_id: 3, timestamp: ~N[2021-01-01 00:00:00]),- build(:imported_pages,- page: "/blog",- pageviews: 5,- visitors: 3,- date: ~D[2023-01-01]- )- ])-- conn =- post(conn, "/api/v2/query-internal-test", %{- "site_id" => site.domain,- "metrics" => ["visitors"],- "date_range" => "all",- "dimensions" => ["event:goal"],- "filters" => [- ["has_not_done", ["is", "event:name", ["pageview"]]]- ],- "include" => %{"imports" => true}- })-- assert json_response(conn, 200)["results"] == []- refute json_response(conn, 200)["meta"]["imports_included"]-- assert json_response(conn, 200)["meta"]["imports_warning"] == @unsupported_query_warning- end- end-describe "scroll depth metric warnings" dotest "returns warning when a site_import without scroll depth is in queried range", %{conn: conn,@@ -1407,55 +1336,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryImportedTest do}end- test "returns warning when import without scroll depth in comparison range", %{- conn: conn,- site: site- } do- site_import =- insert(:site_import, site: site, start_date: ~D[2021-02-01], end_date: ~D[2021-02-28])-- populate_stats(site, site_import.id, [- build(:pageview, user_id: 123, pathname: "/", timestamp: ~N[2021-02-01 00:00:00]),- build(:engagement,- user_id: 123,- pathname: "/",- timestamp: ~N[2021-02-01 00:01:00],- scroll_depth: 70- ),- build(:imported_pages, page: "/", date: ~D[2021-02-01]),- build(:imported_pages, page: "/", date: ~D[2021-02-28])- ])-- conn =- post(conn, "/api/v2/query-internal-test", %{- "site_id" => site.domain,- "metrics" => ["visitors", "scroll_depth"],- "date_range" => ["2022-01-01", "2022-12-31"],- "filters" => [["is", "event:page", ["/"]]],- "include" => %{"imports" => true, "comparisons" => %{"mode" => "previous_period"}}- })-- assert %{"results" => results, "meta" => meta} = json_response(conn, 200)-- assert results == [- %{- "dimensions" => [],- "metrics" => [0, nil],- "comparison" => %{- "change" => [-100, nil],- "dimensions" => [],- "metrics" => [3, 70]- }- }- ]-- assert meta["imports_included"] == true-- assert meta["metric_warnings"] == %{- "scroll_depth" => @no_imported_scroll_depth_metric_warning- }- end-test "does not return warning when imports requested but rejected", %{conn: conn,site: sitediff --git a/test/plausible_web/controllers/api/external_stats_controller/query_special_metrics_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_special_metrics_test.exsindex d0e0373da4..1b69d1a4a8 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_special_metrics_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_special_metrics_test.exs@@ -216,301 +216,4 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QuerySpecialMetricsTest do%{"dimensions" => ["Mobile"], "metrics" => [25.0]}]end-- describe "exit_rate" do- test "in visit:exit_page breakdown without filters", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview, user_id: 1, pathname: "/one", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, user_id: 1, pathname: "/two", timestamp: ~N[2021-01-01 00:10:00]),- build(:pageview, user_id: 3, pathname: "/one", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, user_id: 3, pathname: "/never-exit", timestamp: ~N[2021-01-01 00:00:00]),- build(:event,- user_id: 3,- name: "a",- pathname: "/one",- timestamp: ~N[2021-01-01 00:00:00]- ),- build(:pageview, user_id: 3, pathname: "/one", timestamp: ~N[2021-01-01 00:10:00])- ])-- conn =- post(- conn,- "/api/v2/query-internal-test",- %{- "site_id" => site.domain,- "metrics" => ["exit_rate"],- "date_range" => "all",- "dimensions" => ["visit:exit_page"],- "order_by" => [["exit_rate", "desc"]]- }- )-- %{"results" => results} = json_response(conn, 200)-- assert results == [- %{"dimensions" => ["/two"], "metrics" => [100]},- %{"dimensions" => ["/one"], "metrics" => [33.3]}- ]- end-- test "in visit:exit_page breakdown filtered by visit:exit_page", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview, pathname: "/one", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, pathname: "/one", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, user_id: 1, pathname: "/one", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, user_id: 1, pathname: "/two", timestamp: ~N[2021-01-01 00:10:00])- ])-- conn =- post(- conn,- "/api/v2/query-internal-test",- %{- "site_id" => site.domain,- "metrics" => ["exit_rate"],- "date_range" => "all",- "dimensions" => ["visit:exit_page"],- "filters" => [["is", "visit:exit_page", ["/one"]]]- }- )-- %{"results" => results} = json_response(conn, 200)-- assert results == [- %{"dimensions" => ["/one"], "metrics" => [66.7]}- ]- end-- test "in visit:exit_page breakdown filtered by visit:exit_page and visit:entry_page", %{- conn: conn,- site: site- } do- populate_stats(site, [- # Bounced sessions: Match both entry- and exit page filters- build(:pageview, pathname: "/one", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, pathname: "/one", timestamp: ~N[2021-01-01 00:00:00]),- # Session 1: Matches both entry- and exit page filters- build(:pageview, user_id: 1, pathname: "/one", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, user_id: 1, pathname: "/two", timestamp: ~N[2021-01-01 00:10:00]),- build(:pageview, user_id: 1, pathname: "/one", timestamp: ~N[2021-01-01 00:00:00]),- # Session 2: Does not match exit_page filter, BUT the pageview on /one still- # gets counted towards total pageviews.- build(:pageview, user_id: 2, pathname: "/one", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, user_id: 2, pathname: "/two", timestamp: ~N[2021-01-01 00:10:00]),- # Session 3: Does not match entry_page filter, should be ignored- build(:pageview, user_id: 3, pathname: "/two", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, user_id: 3, pathname: "/one", timestamp: ~N[2021-01-01 00:20:00])- ])-- conn =- post(- conn,- "/api/v2/query-internal-test",- %{- "site_id" => site.domain,- "metrics" => ["exit_rate"],- "date_range" => "all",- "dimensions" => ["visit:exit_page"],- "filters" => [- ["is", "visit:exit_page", ["/one"]],- ["is", "visit:entry_page", ["/one"]]- ]- }- )-- %{"results" => results} = json_response(conn, 200)-- assert results == [- %{"dimensions" => ["/one"], "metrics" => [60]}- ]- end-- test "in visit:exit_page breakdown filtered by visit:country", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview,- pathname: "/one",- country_code: "EE",- timestamp: ~N[2021-01-01 00:00:00]- ),- build(:pageview,- pathname: "/one",- country_code: "US",- timestamp: ~N[2021-01-01 00:00:00]- ),- build(:pageview,- user_id: 1,- pathname: "/one",- country_code: "EE",- timestamp: ~N[2021-01-01 00:00:00]- ),- build(:pageview,- user_id: 1,- pathname: "/two",- country_code: "EE",- timestamp: ~N[2021-01-01 00:10:00]- )- ])-- conn =- post(- conn,- "/api/v2/query-internal-test",- %{- "site_id" => site.domain,- "metrics" => ["exit_rate"],- "date_range" => "all",- "filters" => [["is", "visit:country", ["EE"]]],- "dimensions" => ["visit:exit_page"],- "order_by" => [["exit_rate", "asc"]]- }- )-- %{"results" => results} = json_response(conn, 200)-- assert results == [- %{"dimensions" => ["/one"], "metrics" => [50]},- %{"dimensions" => ["/two"], "metrics" => [100.0]}- ]- end-- test "sorting and pagination", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview, pathname: "/one", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, user_id: 2, pathname: "/two", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, user_id: 2, pathname: "/two", timestamp: ~N[2021-01-01 00:01:00]),- build(:pageview, user_id: 3, pathname: "/three", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, user_id: 3, pathname: "/three", timestamp: ~N[2021-01-01 00:01:00]),- build(:pageview, user_id: 3, pathname: "/three", timestamp: ~N[2021-01-01 00:02:00]),- build(:pageview, user_id: 4, pathname: "/four", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, user_id: 4, pathname: "/four", timestamp: ~N[2021-01-01 00:01:00]),- build(:pageview, user_id: 4, pathname: "/four", timestamp: ~N[2021-01-01 00:02:00]),- build(:pageview, user_id: 4, pathname: "/four", timestamp: ~N[2021-01-01 00:03:00])- ])-- do_query = fn order_by, pagination ->- conn- |> post("/api/v2/query-internal-test", %{- "site_id" => site.domain,- "metrics" => ["exit_rate"],- "date_range" => "all",- "dimensions" => ["visit:exit_page"],- "order_by" => order_by,- "pagination" => pagination- })- |> json_response(200)- |> Map.get("results")- end-- all_results_asc = do_query.([["exit_rate", "asc"]], %{"limit" => 4})- all_results_desc = do_query.([["exit_rate", "desc"]], %{"limit" => 4})-- assert all_results_asc == Enum.reverse(all_results_desc)-- assert do_query.([["exit_rate", "desc"]], %{"limit" => 2, "offset" => 0}) == [- %{"dimensions" => ["/one"], "metrics" => [100]},- %{"dimensions" => ["/two"], "metrics" => [50]}- ]-- assert do_query.([["exit_rate", "desc"]], %{"limit" => 2, "offset" => 2}) == [- %{"dimensions" => ["/three"], "metrics" => [33.3]},- %{"dimensions" => ["/four"], "metrics" => [25]}- ]-- assert do_query.([["exit_rate", "asc"]], %{"limit" => 3, "offset" => 1}) == [- %{"dimensions" => ["/three"], "metrics" => [33.3]},- %{"dimensions" => ["/two"], "metrics" => [50]},- %{"dimensions" => ["/one"], "metrics" => [100]}- ]- end-- test "with comparisons", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview, user_id: 1, pathname: "/one", timestamp: ~N[2021-01-09 00:00:00]),- build(:pageview, user_id: 1, pathname: "/three", timestamp: ~N[2021-01-09 00:00:00]),- build(:pageview, pathname: "/one", timestamp: ~N[2021-01-09 00:10:00]),- build(:pageview, user_id: 2, pathname: "/one", timestamp: ~N[2021-01-10 00:00:00]),- build(:pageview, user_id: 2, pathname: "/two", timestamp: ~N[2021-01-10 00:10:00]),- build(:pageview, user_id: 3, pathname: "/one", timestamp: ~N[2021-01-10 00:00:00]),- build(:pageview, user_id: 3, pathname: "/one", timestamp: ~N[2021-01-10 00:10:00])- ])-- conn =- post(- conn,- "/api/v2/query-internal-test",- %{- "site_id" => site.domain,- "metrics" => ["exit_rate"],- "date_range" => ["2021-01-10", "2021-01-10"],- "dimensions" => ["visit:exit_page"],- "include" => %{"comparisons" => %{"mode" => "previous_period"}},- "order_by" => [["exit_rate", "desc"]]- }- )-- %{"results" => results} = json_response(conn, 200)-- assert results == [- %{- "dimensions" => ["/two"],- "metrics" => [100],- "comparison" => %{- "change" => [nil],- "dimensions" => ["/two"],- "metrics" => [nil]- }- },- %{- "dimensions" => ["/one"],- "metrics" => [33.3],- "comparison" => %{- "change" => [-16.7],- "dimensions" => ["/one"],- "metrics" => [50]- }- }- ]- end-- test "with imported data", %{conn: conn, site: site} do- site_import =- insert(:site_import,- site: site,- start_date: ~D[2020-01-01],- end_date: ~D[2020-12-31]- )-- populate_stats(site, site_import.id, [- build(:pageview, user_id: 1, pathname: "/one", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, user_id: 1, pathname: "/two", timestamp: ~N[2021-01-01 00:10:00]),- build(:pageview, user_id: 3, pathname: "/one", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, user_id: 3, pathname: "/three", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, user_id: 3, pathname: "/one", timestamp: ~N[2021-01-01 00:10:00]),- build(:imported_pages, page: "/one", visits: 10, pageviews: 20, date: ~D[2020-01-01]),- build(:imported_exit_pages, exit_page: "/one", exits: 2, date: ~D[2020-01-01])- ])-- conn =- post(- conn,- "/api/v2/query-internal-test",- %{- "site_id" => site.domain,- "metrics" => ["exit_rate"],- "date_range" => "all",- "include" => %{"imports" => true},- "dimensions" => ["visit:exit_page"],- "order_by" => [["exit_rate", "desc"]]- }- )-- %{"results" => results} = json_response(conn, 200)-- assert results == [- %{"dimensions" => ["/two"], "metrics" => [100]},- %{"dimensions" => ["/one"], "metrics" => [13]}- ]- end- endenddiff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exsindex 2e565d4838..310875746d 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs@@ -1429,59 +1429,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doenddescribe "timeseries" do- test "breakdown by time:minute (internal API), counts visitors and visits in all buckets their session was active in",- %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:10:00])- ])-- conn =- post(conn, "/api/v2/query-internal-test", %{- "site_id" => site.domain,- "metrics" => ["visitors", "visits", "pageviews"],- "date_range" => ["2021-01-01T00:00:00Z", "2021-01-01T00:10:00Z"],- "dimensions" => ["time:minute"]- })-- assert json_response(conn, 200)["results"] == [- %{"dimensions" => ["2021-01-01 00:00:00"], "metrics" => [1, 1, 1]},- %{"dimensions" => ["2021-01-01 00:01:00"], "metrics" => [1, 1, 0]},- %{"dimensions" => ["2021-01-01 00:02:00"], "metrics" => [1, 1, 0]},- %{"dimensions" => ["2021-01-01 00:03:00"], "metrics" => [1, 1, 0]},- %{"dimensions" => ["2021-01-01 00:04:00"], "metrics" => [1, 1, 0]},- %{"dimensions" => ["2021-01-01 00:05:00"], "metrics" => [1, 1, 0]},- %{"dimensions" => ["2021-01-01 00:06:00"], "metrics" => [1, 1, 0]},- %{"dimensions" => ["2021-01-01 00:07:00"], "metrics" => [1, 1, 0]},- %{"dimensions" => ["2021-01-01 00:08:00"], "metrics" => [1, 1, 0]},- %{"dimensions" => ["2021-01-01 00:09:00"], "metrics" => [1, 1, 0]},- %{"dimensions" => ["2021-01-01 00:10:00"], "metrics" => [1, 1, 1]}- ]- end-- test "breakdown by time:hour (internal API), counts visitors and visits in all buckets their session was active in",- %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:20:00]),- build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:40:00]),- build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 01:00:00]),- build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 01:20:00])- ])-- conn =- post(conn, "/api/v2/query-internal-test", %{- "site_id" => site.domain,- "metrics" => ["visitors", "visits", "visit_duration"],- "date_range" => ["2021-01-01", "2021-01-02"],- "dimensions" => ["time:hour"]- })-- assert json_response(conn, 200)["results"] == [- %{"dimensions" => ["2021-01-01 00:00:00"], "metrics" => [1, 1, 0]},- %{"dimensions" => ["2021-01-01 01:00:00"], "metrics" => [1, 1, 3600]}- ]- end-test "shows hourly data for a certain date with time_labels", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),@@ -1591,35 +1538,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do]end- test "shows month to date with time labels trimmed", %{conn: conn, site: site} do- Plausible.Stats.Query.Test.fix_now(~U[2021-01-15 12:00:00Z])-- populate_stats(site, [- build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, timestamp: ~N[2021-01-15 00:00:00]),- build(:pageview, timestamp: ~N[2021-01-16 00:00:00])- ])-- conn =- post(conn, "/api/v2/query-internal-test", %{- "site_id" => site.domain,- "metrics" => ["visitors"],- "date_range" => "month",- "dimensions" => ["time:day"],- "include" => %{"trim_relative_date_range" => true}- })-- assert json_response(conn, 200)["results"] == [- %{"dimensions" => ["2021-01-01"], "metrics" => [1]},- %{"dimensions" => ["2021-01-15"], "metrics" => [1]}- ]-- assert json_response(conn, 200)["query"]["date_range"] == [- "2021-01-01T00:00:00Z",- "2021-01-15T23:59:59Z"- ]- end-test "shows last 6 months of visitors", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview, timestamp: ~N[2020-08-13 00:00:00]),@@ -1793,38 +1711,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do]end- test "visitors and visits are smeared across time:minute buckets but visit_duration is not",- %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview, user_id: 1, timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, user_id: 1, timestamp: ~N[2021-01-01 00:10:00]),- build(:pageview, user_id: 2, timestamp: ~N[2021-01-01 00:05:00]),- build(:pageview, user_id: 2, timestamp: ~N[2021-01-01 00:08:00])- ])-- conn =- post(conn, "/api/v2/query-internal-test", %{- "site_id" => site.domain,- "metrics" => ["visitors", "visits", "visit_duration", "pageviews"],- "date_range" => ["2021-01-01T00:00:00Z", "2021-01-01T00:30:00Z"],- "dimensions" => ["time:minute"]- })-- assert json_response(conn, 200)["results"] == [- %{"dimensions" => ["2021-01-01 00:00:00"], "metrics" => [1, 1, 0, 1]},- %{"dimensions" => ["2021-01-01 00:01:00"], "metrics" => [1, 1, 0, 0]},- %{"dimensions" => ["2021-01-01 00:02:00"], "metrics" => [1, 1, 0, 0]},- %{"dimensions" => ["2021-01-01 00:03:00"], "metrics" => [1, 1, 0, 0]},- %{"dimensions" => ["2021-01-01 00:04:00"], "metrics" => [1, 1, 0, 0]},- %{"dimensions" => ["2021-01-01 00:05:00"], "metrics" => [2, 2, 0, 1]},- %{"dimensions" => ["2021-01-01 00:06:00"], "metrics" => [2, 2, 0, 0]},- %{"dimensions" => ["2021-01-01 00:07:00"], "metrics" => [2, 2, 0, 0]},- %{"dimensions" => ["2021-01-01 00:08:00"], "metrics" => [2, 2, 180, 1]},- %{"dimensions" => ["2021-01-01 00:09:00"], "metrics" => [1, 1, 0, 0]},- %{"dimensions" => ["2021-01-01 00:10:00"], "metrics" => [1, 1, 600, 1]}- ]- end-test "visitors and visits are smeared across time:hour buckets but visit_duration is not", %{conn: conn,site: sitediff --git a/test/plausible_web/controllers/api/external_stats_controller/query_time_on_page_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_time_on_page_test.exsindex bfbc83f9f7..5e164b1c17 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_time_on_page_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_time_on_page_test.exs@@ -168,41 +168,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do} = json_response(conn, 200)end- test "time_on_page breakdown (with missing data)", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),- build(:engagement,- user_id: 12,- pathname: "/",- timestamp: ~N[2021-01-01 00:03:00],- engagement_time: 20_000- ),- build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),- build(:engagement,- user_id: 12,- pathname: "/",- timestamp: ~N[2021-01-01 00:04:00],- engagement_time: 30_000- ),- build(:pageview, pathname: "/blog", timestamp: ~N[2021-01-01 00:00:00])- ])-- conn =- post(conn, "/api/v2/query-internal-test", %{- "site_id" => site.domain,- "metrics" => ["visitors", "time_on_page"],- "date_range" => "all",- "dimensions" => ["event:page"]- })-- assert_matches %{- "results" => [- %{"dimensions" => ["/"], "metrics" => [1, 50]},- %{"dimensions" => ["/blog"], "metrics" => [1, nil]}- ]- } = json_response(conn, 200)- end-describe "site.legacy_time_on_page_cutoff" dosetup %{site: site, site_import: site_import} = context dopopulate_stats(site, site_import.id, [@@ -673,45 +638,5 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do%{"dimensions" => ["2021-01-04"], "metrics" => [150]}]end-- test "can use comparisons together with `legacy_time_on_page_cutoff`", %{- conn: conn,- site: site- } do- site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[2021-01-02])-- conn =- post(conn, "/api/v2/query-internal-test", %{- "site_id" => site.domain,- "metrics" => ["time_on_page"],- "date_range" => ["2021-01-03", "2021-01-04"],- "filters" => [["is", "event:page", ["/"]]],- "dimensions" => ["time:day"],- "include" => %{- "comparisons" => %{"mode" => "previous_period"}- }- })-- assert json_response(conn, 200)["results"] == [- %{- "dimensions" => ["2021-01-03"],- "metrics" => [250],- "comparison" => %{- "dimensions" => ["2021-01-01"],- "metrics" => [nil],- "change" => [nil]- }- },- %{- "dimensions" => ["2021-01-04"],- "metrics" => [150],- "comparison" => %{- "dimensions" => ["2021-01-02"],- "metrics" => [200],- "change" => [-25]- }- }- ]- endendend