Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Comparisons pagination fixes (#4697)

Karl-Aksel Puulmann472f4f181c45
diff --git a/lib/plausible/stats/comparisons.ex b/lib/plausible/stats/comparisons.ex
index 6e956ac6b8..4dc7d4d1a6 100644
--- a/lib/plausible/stats/comparisons.ex
+++ b/lib/plausible/stats/comparisons.ex
@@ -8,7 +8,7 @@ defmodule Plausible.Stats.Comparisons do
"""
alias Plausible.Stats
- alias Plausible.Stats.{Query, DateTimeRange}
+ alias Plausible.Stats.{Query, DateTimeRange, Time}
@spec get_comparison_query(Stats.Query.t(), map()) :: Stats.Query.t()
@doc """
@@ -57,6 +57,45 @@ defmodule Plausible.Stats.Comparisons do
|> maybe_include_imported(source_query)
end
+ @doc """
+ Builds comparison query that specifically filters for values appearing in the main query results.
+
+ When querying for comparisons with dimensions and pagination, extra
+ filters are added to ensure comparison query returns same set of results
+ as main query.
+ """
+ def add_comparison_filters(comparison_query, main_results_list) do
+ comparison_filters =
+ Enum.flat_map(main_results_list, &build_comparison_filter(&1, comparison_query))
+
+ comparison_query
+ |> add_query_filters(comparison_filters)
+ end
+
+ defp add_query_filters(query, []), do: query
+
+ defp add_query_filters(query, [filter]) do
+ Query.add_filter(query, [:ignore_in_totals_query, filter])
+ end
+
+ defp add_query_filters(query, filters) do
+ Query.add_filter(query, [:ignore_in_totals_query, [:or, filters]])
+ end
+
+ defp build_comparison_filter(%{dimensions: dimension_labels}, query) do
+ query_filters =
+ query.dimensions
+ |> Enum.zip(dimension_labels)
+ |> Enum.reject(fn {dimension, _label} -> Time.time_dimension?(dimension) end)
+ |> Enum.map(fn {dimension, label} -> [:is, dimension, [label]] end)
+
+ case query_filters do
+ [] -> []
+ [filter] -> [filter]
+ filters -> [[:and, filters]]
+ end
+ end
+
defp get_comparison_date_range(source_query, %{mode: "year_over_year"} = options) do
source_date_range = Query.date_range(source_query)
diff --git a/lib/plausible/stats/filters/filters.ex b/lib/plausible/stats/filters/filters.ex
index b296a931f6..9e911ae20b 100644
--- a/lib/plausible/stats/filters/filters.ex
+++ b/lib/plausible/stats/filters/filters.ex
@@ -144,7 +144,7 @@ defmodule Plausible.Stats.Filters do
defp transform_tree(filter, transformer) do
case {transformer.(filter), filter} do
# Transformer did not return that value - transform that subtree
- {nil, [:not, child_filter]} ->
+ {nil, [operation, child_filter]} when operation in [:not, :ignore_in_totals_query] ->
[[:not, transform_tree(child_filter, transformer)]]
{nil, [operation, filters]} when operation in [:and, :or] ->
@@ -167,7 +167,7 @@ defmodule Plausible.Stats.Filters do
defp traverse_tree(filter, root, depth) do
case filter do
- [:not, child_filter] ->
+ [operation, child_filter] when operation in [:not, :ignore_in_totals_query] ->
traverse_tree(child_filter, root, depth + 1)
[operation, filters] when operation in [:and, :or] ->
diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.ex
index d00fe07c70..8b4a7445a4 100644
--- a/lib/plausible/stats/filters/query_parser.ex
+++ b/lib/plausible/stats/filters/query_parser.ex
@@ -3,7 +3,7 @@ defmodule Plausible.Stats.Filters.QueryParser do
use Plausible
- alias Plausible.Stats.{TableDecider, Filters, Metrics, DateTimeRange, JSONSchema}
+ alias Plausible.Stats.{TableDecider, Filters, Metrics, DateTimeRange, JSONSchema, Time}
@default_include %{
imports: false,
@@ -578,7 +578,7 @@ defmodule Plausible.Stats.Filters.QueryParser do
end
defp validate_include(query) do
- time_dimension? = Enum.any?(query.dimensions, &String.starts_with?(&1, "time"))
+ time_dimension? = Enum.any?(query.dimensions, &Time.time_dimension?/1)
if query.include.time_labels and not time_dimension? do
{:error, "Invalid include.time_labels: requires a time dimension."}
diff --git a/lib/plausible/stats/imported/sql/where_builder.ex b/lib/plausible/stats/imported/sql/where_builder.ex
index de3d8cb1e9..638fdf270c 100644
--- a/lib/plausible/stats/imported/sql/where_builder.ex
+++ b/lib/plausible/stats/imported/sql/where_builder.ex
@@ -29,6 +29,10 @@ defmodule Plausible.Stats.Imported.SQL.WhereBuilder do
|> Enum.reduce(fn condition, acc -> dynamic([], ^acc and ^condition) end)
end
+ defp add_filter(query, [:ignore_in_totals_query, filter]) do
+ add_filter(query, filter)
+ end
+
defp add_filter(query, [:not, filter]) do
dynamic([i], not (^add_filter(query, filter)))
end
diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex
index 1624e193e4..0c7430b7c2 100644
--- a/lib/plausible/stats/query.ex
+++ b/lib/plausible/stats/query.ex
@@ -93,7 +93,7 @@ defmodule Plausible.Stats.Query do
def remove_top_level_filters(query, prefixes) do
new_filters =
Enum.reject(query.filters, fn [_, filter_key | _rest] ->
- Enum.any?(prefixes, &String.starts_with?(filter_key, &1))
+ is_binary(filter_key) and Enum.any?(prefixes, &String.starts_with?(filter_key, &1))
end)
query
diff --git a/lib/plausible/stats/query_optimizer.ex b/lib/plausible/stats/query_optimizer.ex
index ce26ec06bb..ed4c5e0b8b 100644
--- a/lib/plausible/stats/query_optimizer.ex
+++ b/lib/plausible/stats/query_optimizer.ex
@@ -4,7 +4,7 @@ defmodule Plausible.Stats.QueryOptimizer do
"""
use Plausible
- alias Plausible.Stats.{DateTimeRange, Filters, Query, TableDecider, Util}
+ alias Plausible.Stats.{DateTimeRange, Filters, Query, TableDecider, Util, Time}
@doc """
This module manipulates an existing query, updating it according to business logic.
@@ -147,7 +147,7 @@ defmodule Plausible.Stats.QueryOptimizer do
end
defp time_dimension(query) do
- Enum.find(query.dimensions, &String.starts_with?(&1, "time"))
+ Enum.find(query.dimensions, &Time.time_dimension?/1)
end
defp split_sessions_query(query, session_metrics) do
diff --git a/lib/plausible/stats/query_runner.ex b/lib/plausible/stats/query_runner.ex
index cb58c22ec2..98e6016b73 100644
--- a/lib/plausible/stats/query_runner.ex
+++ b/lib/plausible/stats/query_runner.ex
@@ -41,9 +41,9 @@ defmodule Plausible.Stats.QueryRunner do
run_results =
%__MODULE__{query: optimized_query, site: site}
+ |> execute_main_query()
|> add_comparison_query()
|> execute_comparison()
- |> execute_main_query()
|> add_meta_extra()
|> add_time_lookup()
|> build_results_list()
@@ -51,9 +51,25 @@ defmodule Plausible.Stats.QueryRunner do
QueryResult.from(run_results.results_list, site, optimized_query, run_results.meta_extra)
end
- defp add_comparison_query(%__MODULE__{query: query} = run_results)
+ defp execute_main_query(%__MODULE__{query: query, site: site} = run_results) do
+ {ch_results, time_on_page} = execute_query(query, site)
+
+ struct!(
+ run_results,
+ main_results_list: build_from_ch(ch_results, query, time_on_page),
+ ch_results: ch_results
+ )
+ end
+
+ defp add_comparison_query(
+ %__MODULE__{query: query, main_results_list: main_results_list} = run_results
+ )
when is_map(query.include.comparisons) do
- comparison_query = Comparisons.get_comparison_query(query, query.include.comparisons)
+ comparison_query =
+ query
+ |> Comparisons.get_comparison_query(query.include.comparisons)
+ |> Comparisons.add_comparison_filters(main_results_list)
+
struct!(run_results, comparison_query: comparison_query)
end
@@ -93,16 +109,6 @@ defmodule Plausible.Stats.QueryRunner do
struct!(run_results, time_lookup: time_lookup)
end
- defp execute_main_query(%__MODULE__{query: query, site: site} = run_results) do
- {ch_results, time_on_page} = execute_query(query, site)
-
- struct!(
- run_results,
- main_results_list: build_from_ch(ch_results, query, time_on_page),
- ch_results: ch_results
- )
- end
-
defp add_meta_extra(%__MODULE__{query: query, ch_results: ch_results} = run_results) do
struct!(run_results,
meta_extra: %{
diff --git a/lib/plausible/stats/sql/special_metrics.ex b/lib/plausible/stats/sql/special_metrics.ex
index 5a1e7b39f7..8f18ae24c5 100644
--- a/lib/plausible/stats/sql/special_metrics.ex
+++ b/lib/plausible/stats/sql/special_metrics.ex
@@ -6,7 +6,7 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do
use Plausible.Stats.SQL.Fragments
- alias Plausible.Stats.{Base, Query, SQL}
+ alias Plausible.Stats.{Base, Query, SQL, Filters}
import Ecto.Query
import Plausible.Stats.Util
@@ -21,7 +21,9 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do
defp maybe_add_percentage_metric(q, site, query) do
if :percentage in query.metrics do
total_query =
- Query.set(query,
+ query
+ |> remove_filters_ignored_in_totals_query()
+ |> Query.set(
dimensions: [],
include_imported: query.include_imported
)
@@ -50,6 +52,7 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do
total_query =
query
|> Query.remove_top_level_filters(["event:goal", "event:props"])
+ |> remove_filters_ignored_in_totals_query()
|> Query.set(
dimensions: [],
include_imported: query.include_imported
@@ -90,6 +93,7 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do
group_totals_query =
query
|> Query.remove_top_level_filters(["event:goal", "event:props"])
+ |> remove_filters_ignored_in_totals_query()
|> Query.set(
metrics: [:visitors],
order_by: [],
@@ -117,13 +121,6 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do
end
end
- defp total_visitors(site, query) do
- Base.base_event_query(site, query)
- |> select([e],
- total_visitors: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.user_id)
- )
- end
-
# `total_visitors_subquery` returns a subquery which selects `total_visitors` -
# the number used as the denominator in the calculation of `conversion_rate` and
# `percentage` metrics.
@@ -149,4 +146,21 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do
total_visitors: subquery(total_visitors(site, query))
})
end
+
+ defp remove_filters_ignored_in_totals_query(query) do
+ totals_query_filters =
+ Filters.transform_filters(query.filters, fn
+ [:ignore_in_totals_query, _] -> []
+ filter -> [filter]
+ end)
+
+ Query.set(query, filters: totals_query_filters)
+ end
+
+ defp total_visitors(site, query) do
+ Base.base_event_query(site, query)
+ |> select([e],
+ total_visitors: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.user_id)
+ )
+ end
end
diff --git a/lib/plausible/stats/sql/where_builder.ex b/lib/plausible/stats/sql/where_builder.ex
index ea33a704c9..cad54c53a2 100644
--- a/lib/plausible/stats/sql/where_builder.ex
+++ b/lib/plausible/stats/sql/where_builder.ex
@@ -75,6 +75,10 @@ defmodule Plausible.Stats.SQL.WhereBuilder do
)
end
+ defp add_filter(table, query, [:ignore_in_totals_query, filter]) do
+ add_filter(table, query, filter)
+ end
+
defp add_filter(table, query, [:not, filter]) do
dynamic([e], not (^add_filter(table, query, filter)))
end
diff --git a/lib/plausible/stats/time.ex b/lib/plausible/stats/time.ex
index f86ed5db0b..cee037e4b0 100644
--- a/lib/plausible/stats/time.ex
+++ b/lib/plausible/stats/time.ex
@@ -33,9 +33,12 @@ defmodule Plausible.Stats.Time do
def format_datetime(other), do: other
def time_dimension(query) do
- Enum.find(query.dimensions, &String.starts_with?(&1, "time"))
+ Enum.find(query.dimensions, &time_dimension?/1)
end
+ def time_dimension?("time" <> _rest), do: true
+ def time_dimension?(_dimension), do: false
+
@doc """
Returns list of time bucket labels for the given query.
"""
diff --git a/lib/plausible_web/controllers/api/external_query_api_controller.ex b/lib/plausible_web/controllers/api/external_query_api_controller.ex
index dabf5daf3f..f64414ee7e 100644
--- a/lib/plausible_web/controllers/api/external_query_api_controller.ex
+++ b/lib/plausible_web/controllers/api/external_query_api_controller.ex
@@ -9,7 +9,7 @@ defmodule PlausibleWeb.Api.ExternalQueryApiController do
def query(conn, params) do
site = Repo.preload(conn.assigns.site, :owner)
- case Query.build(site, :public, params, debug_metadata(conn)) do
+ case Query.build(site, conn.assigns.schema_type, params, debug_metadata(conn)) do
{:ok, query} ->
results = Plausible.Stats.query(site, query)
json(conn, results)
diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.ex
index 7ee5347f67..9ed13fbd21 100644
--- a/lib/plausible_web/router.ex
+++ b/lib/plausible_web/router.ex
@@ -208,16 +208,22 @@ defmodule PlausibleWeb.Router do
get "/timeseries", ExternalStatsController, :timeseries
end
- scope "/api/v2", PlausibleWeb.Api, assigns: %{api_scope: "stats:read:*"} do
+ scope "/api/v2", PlausibleWeb.Api, assigns: %{api_scope: "stats:read:*", schema_type: :public} do
pipe_through [:public_api, PlausibleWeb.Plugs.AuthorizePublicAPI]
post "/query", ExternalQueryApiController, :query
+
+ if Mix.env() in [:test, :ce_test] do
+ scope assigns: %{schema_type: :internal} do
+ post "/query-internal-test", ExternalQueryApiController, :query
+ end
+ end
end
scope "/api/docs", PlausibleWeb.Api do
get "/query/schema.json", ExternalQueryApiController, :schema
- scope [] do
+ scope assigns: %{schema_type: :public} do
pipe_through :docs_stats_api
post "/query", ExternalQueryApiController, :query
diff --git a/test/plausible/stats/comparisons_test.exs b/test/plausible/stats/comparisons_test.exs
index 99392d7501..670bac978f 100644
--- a/test/plausible/stats/comparisons_test.exs
+++ b/test/plausible/stats/comparisons_test.exs
@@ -3,6 +3,8 @@ defmodule Plausible.Stats.ComparisonsTest do
alias Plausible.Stats.{DateTimeRange, Query, Comparisons}
import Plausible.TestUtils
+ setup [:create_user, :create_new_site]
+
def build_query(site, params, now) do
query = Query.from(site, params)
@@ -10,9 +12,7 @@ defmodule Plausible.Stats.ComparisonsTest do
end
describe "with period set to this month" do
- test "shifts back this month period when mode is previous_period" do
- site = insert(:site)
-
+ test "shifts back this month period when mode is previous_period", %{site: site} do
query =
build_query(site, %{"period" => "month", "date" => "2023-03-02"}, ~N[2023-03-02 14:00:00])
@@ -22,9 +22,8 @@ defmodule Plausible.Stats.ComparisonsTest do
assert comparison_query.utc_time_range.last == ~U[2023-02-28 23:59:59Z]
end
- test "shifts back this month period when it's the first day of the month and mode is previous_period" do
- site = insert(:site)
-
+ test "shifts back this month period when it's the first day of the month and mode is previous_period",
+ %{site: site} do
query =
build_query(site, %{"period" => "month", "date" => "2023-03-01"}, ~N[2023-03-01 14:00:00])
@@ -34,9 +33,8 @@ defmodule Plausible.Stats.ComparisonsTest do
assert comparison_query.utc_time_range.last == ~U[2023-02-28 23:59:59Z]
end
- test "matches the day of the week when nearest day is original query start date and mode is previous_period" do
- site = insert(:site)
-
+ test "matches the day of the week when nearest day is original query start date and mode is previous_period",
+ %{site: site} do
query =
build_query(site, %{"period" => "month", "date" => "2023-03-02"}, ~N[2023-03-02 14:00:00])
@@ -64,9 +62,7 @@ defmodule Plausible.Stats.ComparisonsTest do
end
describe "with period set to previous month" do
- test "shifts back using the same number of days when mode is previous_period" do
- site = insert(:site)
-
+ test "shifts back using the same number of days when mode is previous_period", %{site: site} do
query =
build_query(site, %{"period" => "month", "date" => "2023-02-01"}, ~N[2023-03-01 14:00:00])
@@ -76,9 +72,7 @@ defmodule Plausible.Stats.ComparisonsTest do
assert comparison_query.utc_time_range.last == ~U[2023-01-31 23:59:59Z]
end
- test "shifts back the full month when mode is year_over_year" do
- site = insert(:site)
-
+ test "shifts back the full month when mode is year_over_year", %{site: site} do
query =
build_query(site, %{"period" => "month", "date" => "2023-02-01"}, ~N[2023-03-01 14:00:00])
@@ -88,9 +82,9 @@ defmodule Plausible.Stats.ComparisonsTest do
assert comparison_query.utc_time_range.last == ~U[2022-02-28 23:59:59Z]
end
- test "shifts back whole month plus one day when mode is year_over_year and a leap year" do
- site = insert(:site)
-
+ test "shifts back whole month plus one day when mode is year_over_year and a leap year", %{
+ site: site
+ } do
query =
build_query(site, %{"period" => "month", "date" => "2020-02-01"}, ~N[2023-03-01 14:00:00])
@@ -100,9 +94,9 @@ defmodule Plausible.Stats.ComparisonsTest do
assert comparison_query.utc_time_range.last == ~U[2019-03-01 23:59:59Z]
end
- test "matches the day of the week when mode is previous_period keeping the same day" do
- site = insert(:site)
-
+ test "matches the day of the week when mode is previous_period keeping the same day", %{
+ site: site
+ } do
query =
build_query(site, %{"period" => "month", "date" => "2023-02-01"}, ~N[2023-03-01 14:00:00])
@@ -116,9 +110,7 @@ defmodule Plausible.Stats.ComparisonsTest do
assert comparison_query.utc_time_range.last == ~U[2023-01-31 23:59:59Z]
end
- test "matches the day of the week when mode is previous_period" do
- site = insert(:site)
-
+ test "matches the day of the week when mode is previous_period", %{site: site} do
query =
build_query(site, %{"period" => "month", "date" => "2023-01-01"}, ~N[2023-03-01 14:00:00])
@@ -134,9 +126,7 @@ defmodule Plausible.Stats.ComparisonsTest do
end
describe "with period set to year to date" do
- test "shifts back by the same number of days when mode is previous_period" do
- site = insert(:site)
-
+ test "shifts back by the same number of days when mode is previous_period", %{site: site} do
query =
build_query(site, %{"period" => "year", "date" => "2023-03-01"}, ~N[2023-03-01 14:00:00])
@@ -146,9 +136,7 @@ defmodule Plausible.Stats.ComparisonsTest do
assert comparison_query.utc_time_range.last == ~U[2022-12-31 23:59:59Z]
end
- test "shifts back by the same number of days when mode is year_over_year" do
- site = insert(:site)
-
+ test "shifts back by the same number of days when mode is year_over_year", %{site: site} do
query =
build_query(site, %{"period" => "year", "date" => "2023-03-01"}, ~N[2023-03-01 14:00:00])
@@ -158,9 +146,7 @@ defmodule Plausible.Stats.ComparisonsTest do
assert comparison_query.utc_time_range.last == ~U[2022-03-01 23:59:59Z]
end
- test "matches the day of the week when mode is year_over_year" do
- site = insert(:site)
-
+ test "matches the day of the week when mode is year_over_year", %{site: site} do
query =
build_query(site, %{"period" => "year", "date" => "2023-03-01"}, ~N[2023-03-01 14:00:00])
@@ -173,8 +159,7 @@ defmodule Plausible.Stats.ComparisonsTest do
end
describe "with period set to previous year" do
- test "shifts back a whole year when mode is year_over_year" do
- site = insert(:site)
+ test "shifts back a whole year when mode is year_over_year", %{site: site} do
query = Query.from(site, %{"period" => "year", "date" => "2022-03-02"})
comparison_query = Comparisons.get_comparison_query(query, %{mode: "year_over_year"})
@@ -183,8 +168,7 @@ defmodule Plausible.Stats.ComparisonsTest do
assert comparison_query.utc_time_range.last == ~U[2021-12-31 23:59:59Z]
end
- test "shifts back a whole year when mode is previous_period" do
- site = insert(:site)
+ test "shifts back a whole year when mode is previous_period", %{site: site} do
query = Query.from(site, %{"period" => "year", "date" => "2022-03-02"})
comparison_query = Comparisons.get_comparison_query(query, %{mode: "previous_period"})
@@ -195,8 +179,7 @@ defmodule Plausible.Stats.ComparisonsTest do
end
describe "with period set to custom" do
- test "shifts back by the same number of days when mode is previous_period" do
- site = insert(:site)
+ test "shifts back by the same number of days when mode is previous_period", %{site: site} do
query = Query.from(site, %{"period" => "custom", "date" => "2023-01-01,2023-01-07"})
comparison_query = Comparisons.get_comparison_query(query, %{mode: "previous_period"})
@@ -205,8 +188,7 @@ defmodule Plausible.Stats.ComparisonsTest do
assert comparison_query.utc_time_range.last == ~U[2022-12-31 23:59:59Z]
end
- test "shifts back to last year when mode is year_over_year" do
- site = insert(:site)
+ test "shifts back to last year when mode is year_over_year", %{site: site} do
query = Query.from(site, %{"period" => "custom", "date" => "2023-01-01,2023-01-07"})
comparison_query = Comparisons.get_comparison_query(query, %{mode: "year_over_year"})
@@ -217,8 +199,7 @@ defmodule Plausible.Stats.ComparisonsTest do
end
describe "with mode set to custom" do
- test "sets first and last dates" do
- site = insert(:site)
+ test "sets first and last dates", %{site: site} do
query = Query.from(site, %{"period" => "custom", "date" => "2023-01-01,2023-01-07"})
comparison_query =
@@ -233,7 +214,7 @@ defmodule Plausible.Stats.ComparisonsTest do
end
describe "include_imported" do
- setup [:create_user, :create_new_site, :create_site_import]
+ setup [:create_site_import]
test "defaults to source_query.include_imported", %{site: site} do
query = Query.from(site, %{"period" => "day", "date" => "2023-01-01"})
@@ -243,4 +224,113 @@ defmodule Plausible.Stats.ComparisonsTest do
assert comparison_query.include_imported == false
end
end
+
+ describe "add_comparison_filters" do
+ test "no results doesn't update filters", %{site: site} do
+ query = build_comparison_query(site, %{"dimensions" => ["visit:browser"]})
+
+ result_query = Comparisons.add_comparison_filters(query, [])
+
+ assert result_query.filters == []
+ end
+
+ test "no dimensions doesn't update filters", %{site: site} do
+ query = build_comparison_query(site, %{})
+
+ result_query =
+ Comparisons.add_comparison_filters(query, [%{dimensions: [], metrics: [123]}])
+
+ assert result_query.filters == []
+ end
+
+ test "no time dimension doesn't update filters", %{site: site} do
+ query = build_comparison_query(site, %{"dimensions" => ["time:day"]})
+
+ result_query =
+ Comparisons.add_comparison_filters(query, [%{dimensions: ["2024-01-01"], metrics: [123]}])
+
+ assert result_query.filters == []
+ end
+
+ test "updates filters in a single-row case", %{site: site} do
+ query =
+ build_comparison_query(site, %{
+ "dimensions" => ["visit:browser"]
+ })
+
+ result_query =
+ Comparisons.add_comparison_filters(query, [%{dimensions: ["Chrome"], metrics: [123]}])
+
+ assert result_query.filters == [
+ [:ignore_in_totals_query, [:is, "visit:browser", ["Chrome"]]]
+ ]
+ end
+
+ test "updates filters for a complex case", %{site: site} do
+ query =
+ build_comparison_query(site, %{
+ "dimensions" => ["visit:browser", "visit:browser_version", "time:day"],
+ "filters" => [["is", "visit:country_name", ["Estonia"]]]
+ })
+
+ main_query_results = [
+ %{
+ dimensions: ["Chrome", "99.9", "2024-01-01"],
+ metrics: [123]
+ },
+ %{
+ dimensions: ["Firefox", "12.0", "2024-01-01"],
+ metrics: [123]
+ }
+ ]
+
+ result_query = Comparisons.add_comparison_filters(query, main_query_results)
+
+ assert result_query.filters == [
+ [:is, "visit:country_name", ["Estonia"]],
+ [
+ :ignore_in_totals_query,
+ [
+ :or,
+ [
+ [
+ :and,
+ [
+ [:is, "visit:browser", ["Chrome"]],
+ [:is, "visit:browser_version", ["99.9"]]
+ ]
+ ],
+ [
+ :and,
+ [
+ [:is, "visit:browser", ["Firefox"]],
+ [:is, "visit:browser_version", ["12.0"]]
+ ]
+ ]
+ ]
+ ]
+ ]
+ ]
+ end
+ end
+
+ defp build_comparison_query(site, params) do
+ {:ok, query} =
+ Query.build(
+ site,
+ :internal,
+ Map.merge(
+ %{
+ "site_id" => site.domain,
+ "metrics" => ["pageviews"],
+ "date_range" => ["2024-01-01", "2024-02-01"],
+ "include" => %{"comparisons" => %{"mode" => "previous_period"}}
+ },
+ params
+ ),
+ %{}
+ )
+
+ query
+ end
end
diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_comparisons_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_comparisons_test.exs
new file mode 100644
index 0000000000..bf9e746d11
--- /dev/null
+++ b/test/plausible_web/controllers/api/external_stats_controller/query_comparisons_test.exs
@@ -0,0 +1,230 @@
+defmodule PlausibleWeb.Api.ExternalStatsController.QueryComparisonsTest do
+ use PlausibleWeb.ConnCase
+
+ setup [:create_user, :create_new_site, :create_api_key, :use_api_key, :create_site_import]
+
+ test "aggregates a single metric", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-01 00:25:00]),
+ build(:pageview, timestamp: ~N[2021-01-02 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-07 00:00:00])
+ ])
+
+ conn =
+ post(conn, "/api/v2/query-internal-test", %{
+ "site_id" => site.domain,
+ "metrics" => ["pageviews"],
+ "date_range" => ["2021-01-07", "2021-01-13"],
+ "include" => %{"comparisons" => %{"mode" => "previous_period"}}
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{
+ "dimensions" => [],
+ "metrics" => [1],
+ "comparison" => %{"change" => [-67], "dimensions" => [], "metrics" => [3]}
+ }
+ ]
+ end
+
+ test "timeseries comparison", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-01 00:25:00]),
+ build(:pageview, timestamp: ~N[2021-01-06 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-07 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-08 00:00:00])
+ ])
+
+ conn =
+ post(conn, "/api/v2/query-internal-test", %{
+ "site_id" => site.domain,
+ "metrics" => ["pageviews"],
+ "date_range" => ["2021-01-07", "2021-01-13"],
+ "dimensions" => ["time:day"],
+ "include" => %{"comparisons" => %{"mode" => "previous_period"}}
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{
+ "dimensions" => ["2021-01-07"],
+ "metrics" => [1],
+ "comparison" => %{
+ "dimensions" => ["2020-12-31"],
+ "metrics" => [0],
+ "change" => [100]
+ }
+ },
+ %{
+ "dimensions" => ["2021-01-08"],
+ "metrics" => [1],
+ "comparison" => %{
+ "dimensions" => ["2021-01-01"],
+ "metrics" => [2],
+ "change" => [-50]
+ }
+ },
+ %{
+ "dimensions" => ["2021-01-09"],
+ "metrics" => [0],
+ "comparison" => %{
+ "dimensions" => ["2021-01-02"],
+ "metrics" => [0],
+ "change" => [0]
+ }
+ },
+ %{
+ "dimensions" => ["2021-01-10"],
+ "metrics" => [0],
+ "comparison" => %{
+ "dimensions" => ["2021-01-03"],
+ "metrics" => [0],
+ "change" => [0]
+ }
+ },
+ %{
+ "dimensions" => ["2021-01-11"],
+ "metrics" => [0],
+ "comparison" => %{
+ "dimensions" => ["2021-01-04"],
+ "metrics" => [0],
+ "change" => [0]
+ }
+ },
+ %{
+ "dimensions" => ["2021-01-12"],
+ "metrics" => [0],
+ "comparison" => %{
+ "dimensions" => ["2021-01-05"],
+ "metrics" => [0],
+ "change" => [0]
+ }
+ },
+ %{
+ "dimensions" => ["2021-01-13"],
+ "metrics" => [0],
+ "comparison" => %{
+ "dimensions" => ["2021-01-06"],
+ "metrics" => [1],
+ "change" => [-100]
+ }
+ }
+ ]
+ end
+
+ test "dimensional comparison with low limit", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, browser: "Safari", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, browser: "Safari", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, browser: "Safari", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-07 00:00:00]),
+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-07 00:00:00]),
+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-07 00:00:00])
+ ])
+
+ conn =
+ post(conn, "/api/v2/query-internal-test", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors", "percentage"],
+ "date_range" => ["2021-01-07", "2021-01-13"],
+ "dimensions" => ["visit:browser"],
+ "include" => %{
+ "comparisons" => %{"mode" => "previous_period"}
+ },
+ "pagination" => %{"limit" => 2}
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{
+ "dimensions" => ["Chrome"],
+ "metrics" => [2, 66.7],
+ "comparison" => %{
+ "dimensions" => ["Chrome"],
+ "metrics" => [1, 12.5],
+ "change" => [100, 434]
+ }
+ },
+ %{
+ "dimensions" => ["Firefox"],
+ "metrics" => [1, 33.3],
+ "comparison" => %{
+ "dimensions" => ["Firefox"],
+ "metrics" => [4, 50.0],
+ "change" => [-75, -33]
+ }
+ }
+ ]
+ end
+
+ test "dimensional comparison with imported data", %{
+ conn: conn,
+ site: site,
+ site_import: site_import
+ } do
+ populate_stats(site, site_import.id, [
+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-07 00:00:00]),
+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-07 00:00:00]),
+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-07 00:00:00]),
+ build(:imported_browsers,
+ date: ~D[2021-01-01],
+ browser: "Firefox",
+ browser_version: "121",
+ visitors: 50
+ ),
+ build(:imported_browsers,
+ date: ~D[2021-01-01],
+ browser: "Chrome",
+ browser_version: "99",
+ visitors: 39
+ ),
+ build(:imported_browsers,
+ date: ~D[2021-01-01],
+ browser: "Safari",
+ browser_version: "99",
+ visitors: 10
+ ),
+ build(:imported_visitors, date: ~D[2021-01-01], visitors: 99)
+ ])
+
+ conn =
+ post(conn, "/api/v2/query-internal-test", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors", "percentage"],
+ "date_range" => ["2021-01-07", "2021-01-13"],
+ "dimensions" => ["visit:browser"],
+ "include" => %{
+ "imports" => true,
+ "comparisons" => %{"mode" => "previous_period"}
+ },
+ "pagination" => %{"limit" => 2}
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{
+ "dimensions" => ["Chrome"],
+ "metrics" => [2, 66.7],
+ "comparison" => %{
+ "dimensions" => ["Chrome"],
+ "metrics" => [40, 40.0],
+ "change" => [-95, 67]
+ }
+ },
+ %{
+ "dimensions" => ["Firefox"],
+ "metrics" => [1, 33.3],
+ "comparison" => %{
+ "dimensions" => ["Firefox"],
+ "metrics" => [50, 50.0],
+ "change" => [-98, -33]
+ }
+ }
+ ]
+ end
+end
diff --git a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs
index 4020847b87..0bd3d14cc5 100644
--- a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs
@@ -191,6 +191,76 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do
%{"name" => "(not set)", "visitors" => 2, "percentage" => 100.0}
]
end
+
+ test "returns comparisons", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, browser: "Safari", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-07 00:00:00]),
+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-07 00:00:00]),
+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-07 00:00:00])
+ ])
+
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/browsers?period=7d&date=2021-01-13&comparison=previous_period"
+ )
+
+ assert json_response(conn, 200)["results"] == [
+ %{
+ "name" => "Chrome",
+ "visitors" => 2,
+ "percentage" => 66.7,
+ "comparison" => %{
+ "visitors" => 0,
+ "percentage" => 0.0,
+ "change" => %{"percentage" => 100, "visitors" => 100}
+ }
+ },
+ %{
+ "name" => "Firefox",
+ "visitors" => 1,
+ "percentage" => 33.3,
+ "comparison" => %{
+ "visitors" => 1,
+ "percentage" => 50.0,
+ "change" => %{"percentage" => -33, "visitors" => 0}
+ }
+ }
+ ]
+ end
+
+ test "returns comparisons with limit", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-07 00:00:00]),
+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-07 00:00:00]),
+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-07 00:00:00])
+ ])
+
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/browsers?period=7d&date=2021-01-13&comparison=previous_period&limit=1"
+ )
+
+ assert json_response(conn, 200)["results"] == [
+ %{
+ "name" => "Chrome",
+ "visitors" => 2,
+ "percentage" => 66.7,
+ "comparison" => %{
+ "visitors" => 1,
+ "percentage" => 25.0,
+ "change" => %{"percentage" => 167, "visitors" => 100}
+ }
+ }
+ ]
+ end
end
describe "GET /api/stats/:domain/browser-versions" do