Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Comparisons pagination fixes (#4697)
Karl-Aksel Puulmann472f4f181c45
diff --git a/lib/plausible/stats/comparisons.ex b/lib/plausible/stats/comparisons.exindex 6e956ac6b8..4dc7d4d1a6 100644--- a/lib/plausible/stats/comparisons.ex+++ b/lib/plausible/stats/comparisons.ex@@ -8,7 +8,7 @@ defmodule Plausible.Stats.Comparisons do"""alias Plausible.Stats- alias Plausible.Stats.{Query, DateTimeRange}+ alias Plausible.Stats.{Query, DateTimeRange, Time}@spec get_comparison_query(Stats.Query.t(), map()) :: Stats.Query.t()@doc """@@ -57,6 +57,45 @@ defmodule Plausible.Stats.Comparisons do|> maybe_include_imported(source_query)end+ @doc """+ Builds comparison query that specifically filters for values appearing in the main query results.++ When querying for comparisons with dimensions and pagination, extra+ filters are added to ensure comparison query returns same set of results+ as main query.+ """+ def add_comparison_filters(comparison_query, main_results_list) do+ comparison_filters =+ Enum.flat_map(main_results_list, &build_comparison_filter(&1, comparison_query))++ comparison_query+ |> add_query_filters(comparison_filters)+ end++ defp add_query_filters(query, []), do: query++ defp add_query_filters(query, [filter]) do+ Query.add_filter(query, [:ignore_in_totals_query, filter])+ end++ defp add_query_filters(query, filters) do+ Query.add_filter(query, [:ignore_in_totals_query, [:or, filters]])+ end++ defp build_comparison_filter(%{dimensions: dimension_labels}, query) do+ query_filters =+ query.dimensions+ |> Enum.zip(dimension_labels)+ |> Enum.reject(fn {dimension, _label} -> Time.time_dimension?(dimension) end)+ |> Enum.map(fn {dimension, label} -> [:is, dimension, [label]] end)++ case query_filters do+ [] -> []+ [filter] -> [filter]+ filters -> [[:and, filters]]+ end+ end+defp get_comparison_date_range(source_query, %{mode: "year_over_year"} = options) dosource_date_range = Query.date_range(source_query)diff --git a/lib/plausible/stats/filters/filters.ex b/lib/plausible/stats/filters/filters.exindex b296a931f6..9e911ae20b 100644--- a/lib/plausible/stats/filters/filters.ex+++ b/lib/plausible/stats/filters/filters.ex@@ -144,7 +144,7 @@ defmodule Plausible.Stats.Filters dodefp transform_tree(filter, transformer) docase {transformer.(filter), filter} do# Transformer did not return that value - transform that subtree- {nil, [:not, child_filter]} ->+ {nil, [operation, child_filter]} when operation in [:not, :ignore_in_totals_query] ->[[:not, transform_tree(child_filter, transformer)]]{nil, [operation, filters]} when operation in [:and, :or] ->@@ -167,7 +167,7 @@ defmodule Plausible.Stats.Filters dodefp traverse_tree(filter, root, depth) docase filter do- [:not, child_filter] ->+ [operation, child_filter] when operation in [:not, :ignore_in_totals_query] ->traverse_tree(child_filter, root, depth + 1)[operation, filters] when operation in [:and, :or] ->diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.exindex d00fe07c70..8b4a7445a4 100644--- a/lib/plausible/stats/filters/query_parser.ex+++ b/lib/plausible/stats/filters/query_parser.ex@@ -3,7 +3,7 @@ defmodule Plausible.Stats.Filters.QueryParser douse Plausible- alias Plausible.Stats.{TableDecider, Filters, Metrics, DateTimeRange, JSONSchema}+ alias Plausible.Stats.{TableDecider, Filters, Metrics, DateTimeRange, JSONSchema, Time}@default_include %{imports: false,@@ -578,7 +578,7 @@ defmodule Plausible.Stats.Filters.QueryParser doenddefp validate_include(query) do- time_dimension? = Enum.any?(query.dimensions, &String.starts_with?(&1, "time"))+ time_dimension? = Enum.any?(query.dimensions, &Time.time_dimension?/1)if query.include.time_labels and not time_dimension? do{:error, "Invalid include.time_labels: requires a time dimension."}diff --git a/lib/plausible/stats/imported/sql/where_builder.ex b/lib/plausible/stats/imported/sql/where_builder.exindex de3d8cb1e9..638fdf270c 100644--- a/lib/plausible/stats/imported/sql/where_builder.ex+++ b/lib/plausible/stats/imported/sql/where_builder.ex@@ -29,6 +29,10 @@ defmodule Plausible.Stats.Imported.SQL.WhereBuilder do|> Enum.reduce(fn condition, acc -> dynamic([], ^acc and ^condition) end)end+ defp add_filter(query, [:ignore_in_totals_query, filter]) do+ add_filter(query, filter)+ end+defp add_filter(query, [:not, filter]) dodynamic([i], not (^add_filter(query, filter)))enddiff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.exindex 1624e193e4..0c7430b7c2 100644--- a/lib/plausible/stats/query.ex+++ b/lib/plausible/stats/query.ex@@ -93,7 +93,7 @@ defmodule Plausible.Stats.Query dodef remove_top_level_filters(query, prefixes) donew_filters =Enum.reject(query.filters, fn [_, filter_key | _rest] ->- Enum.any?(prefixes, &String.starts_with?(filter_key, &1))+ is_binary(filter_key) and Enum.any?(prefixes, &String.starts_with?(filter_key, &1))end)querydiff --git a/lib/plausible/stats/query_optimizer.ex b/lib/plausible/stats/query_optimizer.exindex ce26ec06bb..ed4c5e0b8b 100644--- a/lib/plausible/stats/query_optimizer.ex+++ b/lib/plausible/stats/query_optimizer.ex@@ -4,7 +4,7 @@ defmodule Plausible.Stats.QueryOptimizer do"""use Plausible- alias Plausible.Stats.{DateTimeRange, Filters, Query, TableDecider, Util}+ alias Plausible.Stats.{DateTimeRange, Filters, Query, TableDecider, Util, Time}@doc """This module manipulates an existing query, updating it according to business logic.@@ -147,7 +147,7 @@ defmodule Plausible.Stats.QueryOptimizer doenddefp time_dimension(query) do- Enum.find(query.dimensions, &String.starts_with?(&1, "time"))+ Enum.find(query.dimensions, &Time.time_dimension?/1)enddefp split_sessions_query(query, session_metrics) dodiff --git a/lib/plausible/stats/query_runner.ex b/lib/plausible/stats/query_runner.exindex cb58c22ec2..98e6016b73 100644--- a/lib/plausible/stats/query_runner.ex+++ b/lib/plausible/stats/query_runner.ex@@ -41,9 +41,9 @@ defmodule Plausible.Stats.QueryRunner dorun_results =%__MODULE__{query: optimized_query, site: site}+ |> execute_main_query()|> add_comparison_query()|> execute_comparison()- |> execute_main_query()|> add_meta_extra()|> add_time_lookup()|> build_results_list()@@ -51,9 +51,25 @@ defmodule Plausible.Stats.QueryRunner doQueryResult.from(run_results.results_list, site, optimized_query, run_results.meta_extra)end- defp add_comparison_query(%__MODULE__{query: query} = run_results)+ defp execute_main_query(%__MODULE__{query: query, site: site} = run_results) do+ {ch_results, time_on_page} = execute_query(query, site)++ struct!(+ run_results,+ main_results_list: build_from_ch(ch_results, query, time_on_page),+ ch_results: ch_results+ )+ end++ defp add_comparison_query(+ %__MODULE__{query: query, main_results_list: main_results_list} = run_results+ )when is_map(query.include.comparisons) do- comparison_query = Comparisons.get_comparison_query(query, query.include.comparisons)+ comparison_query =+ query+ |> Comparisons.get_comparison_query(query.include.comparisons)+ |> Comparisons.add_comparison_filters(main_results_list)+struct!(run_results, comparison_query: comparison_query)end@@ -93,16 +109,6 @@ defmodule Plausible.Stats.QueryRunner dostruct!(run_results, time_lookup: time_lookup)end- defp execute_main_query(%__MODULE__{query: query, site: site} = run_results) do- {ch_results, time_on_page} = execute_query(query, site)-- struct!(- run_results,- main_results_list: build_from_ch(ch_results, query, time_on_page),- ch_results: ch_results- )- end-defp add_meta_extra(%__MODULE__{query: query, ch_results: ch_results} = run_results) dostruct!(run_results,meta_extra: %{diff --git a/lib/plausible/stats/sql/special_metrics.ex b/lib/plausible/stats/sql/special_metrics.exindex 5a1e7b39f7..8f18ae24c5 100644--- a/lib/plausible/stats/sql/special_metrics.ex+++ b/lib/plausible/stats/sql/special_metrics.ex@@ -6,7 +6,7 @@ defmodule Plausible.Stats.SQL.SpecialMetrics douse Plausible.Stats.SQL.Fragments- alias Plausible.Stats.{Base, Query, SQL}+ alias Plausible.Stats.{Base, Query, SQL, Filters}import Ecto.Queryimport Plausible.Stats.Util@@ -21,7 +21,9 @@ defmodule Plausible.Stats.SQL.SpecialMetrics dodefp maybe_add_percentage_metric(q, site, query) doif :percentage in query.metrics dototal_query =- Query.set(query,+ query+ |> remove_filters_ignored_in_totals_query()+ |> Query.set(dimensions: [],include_imported: query.include_imported)@@ -50,6 +52,7 @@ defmodule Plausible.Stats.SQL.SpecialMetrics dototal_query =query|> Query.remove_top_level_filters(["event:goal", "event:props"])+ |> remove_filters_ignored_in_totals_query()|> Query.set(dimensions: [],include_imported: query.include_imported@@ -90,6 +93,7 @@ defmodule Plausible.Stats.SQL.SpecialMetrics dogroup_totals_query =query|> Query.remove_top_level_filters(["event:goal", "event:props"])+ |> remove_filters_ignored_in_totals_query()|> Query.set(metrics: [:visitors],order_by: [],@@ -117,13 +121,6 @@ defmodule Plausible.Stats.SQL.SpecialMetrics doendend- defp total_visitors(site, query) do- Base.base_event_query(site, query)- |> select([e],- total_visitors: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.user_id)- )- end-# `total_visitors_subquery` returns a subquery which selects `total_visitors` -# the number used as the denominator in the calculation of `conversion_rate` and# `percentage` metrics.@@ -149,4 +146,21 @@ defmodule Plausible.Stats.SQL.SpecialMetrics dototal_visitors: subquery(total_visitors(site, query))})end++ defp remove_filters_ignored_in_totals_query(query) do+ totals_query_filters =+ Filters.transform_filters(query.filters, fn+ [:ignore_in_totals_query, _] -> []+ filter -> [filter]+ end)++ Query.set(query, filters: totals_query_filters)+ end++ defp total_visitors(site, query) do+ Base.base_event_query(site, query)+ |> select([e],+ total_visitors: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.user_id)+ )+ endenddiff --git a/lib/plausible/stats/sql/where_builder.ex b/lib/plausible/stats/sql/where_builder.exindex ea33a704c9..cad54c53a2 100644--- a/lib/plausible/stats/sql/where_builder.ex+++ b/lib/plausible/stats/sql/where_builder.ex@@ -75,6 +75,10 @@ defmodule Plausible.Stats.SQL.WhereBuilder do)end+ defp add_filter(table, query, [:ignore_in_totals_query, filter]) do+ add_filter(table, query, filter)+ end+defp add_filter(table, query, [:not, filter]) dodynamic([e], not (^add_filter(table, query, filter)))enddiff --git a/lib/plausible/stats/time.ex b/lib/plausible/stats/time.exindex f86ed5db0b..cee037e4b0 100644--- a/lib/plausible/stats/time.ex+++ b/lib/plausible/stats/time.ex@@ -33,9 +33,12 @@ defmodule Plausible.Stats.Time dodef format_datetime(other), do: otherdef time_dimension(query) do- Enum.find(query.dimensions, &String.starts_with?(&1, "time"))+ Enum.find(query.dimensions, &time_dimension?/1)end+ def time_dimension?("time" <> _rest), do: true+ def time_dimension?(_dimension), do: false+@doc """Returns list of time bucket labels for the given query."""diff --git a/lib/plausible_web/controllers/api/external_query_api_controller.ex b/lib/plausible_web/controllers/api/external_query_api_controller.exindex dabf5daf3f..f64414ee7e 100644--- a/lib/plausible_web/controllers/api/external_query_api_controller.ex+++ b/lib/plausible_web/controllers/api/external_query_api_controller.ex@@ -9,7 +9,7 @@ defmodule PlausibleWeb.Api.ExternalQueryApiController dodef query(conn, params) dosite = Repo.preload(conn.assigns.site, :owner)- case Query.build(site, :public, params, debug_metadata(conn)) do+ case Query.build(site, conn.assigns.schema_type, params, debug_metadata(conn)) do{:ok, query} ->results = Plausible.Stats.query(site, query)json(conn, results)diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.exindex 7ee5347f67..9ed13fbd21 100644--- a/lib/plausible_web/router.ex+++ b/lib/plausible_web/router.ex@@ -208,16 +208,22 @@ defmodule PlausibleWeb.Router doget "/timeseries", ExternalStatsController, :timeseriesend- scope "/api/v2", PlausibleWeb.Api, assigns: %{api_scope: "stats:read:*"} do+ scope "/api/v2", PlausibleWeb.Api, assigns: %{api_scope: "stats:read:*", schema_type: :public} dopipe_through [:public_api, PlausibleWeb.Plugs.AuthorizePublicAPI]post "/query", ExternalQueryApiController, :query++ if Mix.env() in [:test, :ce_test] do+ scope assigns: %{schema_type: :internal} do+ post "/query-internal-test", ExternalQueryApiController, :query+ end+ endendscope "/api/docs", PlausibleWeb.Api doget "/query/schema.json", ExternalQueryApiController, :schema- scope [] do+ scope assigns: %{schema_type: :public} dopipe_through :docs_stats_apipost "/query", ExternalQueryApiController, :querydiff --git a/test/plausible/stats/comparisons_test.exs b/test/plausible/stats/comparisons_test.exsindex 99392d7501..670bac978f 100644--- a/test/plausible/stats/comparisons_test.exs+++ b/test/plausible/stats/comparisons_test.exs@@ -3,6 +3,8 @@ defmodule Plausible.Stats.ComparisonsTest doalias Plausible.Stats.{DateTimeRange, Query, Comparisons}import Plausible.TestUtils+ setup [:create_user, :create_new_site]+def build_query(site, params, now) doquery = Query.from(site, params)@@ -10,9 +12,7 @@ defmodule Plausible.Stats.ComparisonsTest doenddescribe "with period set to this month" do- test "shifts back this month period when mode is previous_period" do- site = insert(:site)-+ test "shifts back this month period when mode is previous_period", %{site: site} doquery =build_query(site, %{"period" => "month", "date" => "2023-03-02"}, ~N[2023-03-02 14:00:00])@@ -22,9 +22,8 @@ defmodule Plausible.Stats.ComparisonsTest doassert comparison_query.utc_time_range.last == ~U[2023-02-28 23:59:59Z]end- test "shifts back this month period when it's the first day of the month and mode is previous_period" do- site = insert(:site)-+ test "shifts back this month period when it's the first day of the month and mode is previous_period",+ %{site: site} doquery =build_query(site, %{"period" => "month", "date" => "2023-03-01"}, ~N[2023-03-01 14:00:00])@@ -34,9 +33,8 @@ defmodule Plausible.Stats.ComparisonsTest doassert comparison_query.utc_time_range.last == ~U[2023-02-28 23:59:59Z]end- test "matches the day of the week when nearest day is original query start date and mode is previous_period" do- site = insert(:site)-+ test "matches the day of the week when nearest day is original query start date and mode is previous_period",+ %{site: site} doquery =build_query(site, %{"period" => "month", "date" => "2023-03-02"}, ~N[2023-03-02 14:00:00])@@ -64,9 +62,7 @@ defmodule Plausible.Stats.ComparisonsTest doenddescribe "with period set to previous month" do- test "shifts back using the same number of days when mode is previous_period" do- site = insert(:site)-+ test "shifts back using the same number of days when mode is previous_period", %{site: site} doquery =build_query(site, %{"period" => "month", "date" => "2023-02-01"}, ~N[2023-03-01 14:00:00])@@ -76,9 +72,7 @@ defmodule Plausible.Stats.ComparisonsTest doassert comparison_query.utc_time_range.last == ~U[2023-01-31 23:59:59Z]end- test "shifts back the full month when mode is year_over_year" do- site = insert(:site)-+ test "shifts back the full month when mode is year_over_year", %{site: site} doquery =build_query(site, %{"period" => "month", "date" => "2023-02-01"}, ~N[2023-03-01 14:00:00])@@ -88,9 +82,9 @@ defmodule Plausible.Stats.ComparisonsTest doassert comparison_query.utc_time_range.last == ~U[2022-02-28 23:59:59Z]end- test "shifts back whole month plus one day when mode is year_over_year and a leap year" do- site = insert(:site)-+ test "shifts back whole month plus one day when mode is year_over_year and a leap year", %{+ site: site+ } doquery =build_query(site, %{"period" => "month", "date" => "2020-02-01"}, ~N[2023-03-01 14:00:00])@@ -100,9 +94,9 @@ defmodule Plausible.Stats.ComparisonsTest doassert comparison_query.utc_time_range.last == ~U[2019-03-01 23:59:59Z]end- test "matches the day of the week when mode is previous_period keeping the same day" do- site = insert(:site)-+ test "matches the day of the week when mode is previous_period keeping the same day", %{+ site: site+ } doquery =build_query(site, %{"period" => "month", "date" => "2023-02-01"}, ~N[2023-03-01 14:00:00])@@ -116,9 +110,7 @@ defmodule Plausible.Stats.ComparisonsTest doassert comparison_query.utc_time_range.last == ~U[2023-01-31 23:59:59Z]end- test "matches the day of the week when mode is previous_period" do- site = insert(:site)-+ test "matches the day of the week when mode is previous_period", %{site: site} doquery =build_query(site, %{"period" => "month", "date" => "2023-01-01"}, ~N[2023-03-01 14:00:00])@@ -134,9 +126,7 @@ defmodule Plausible.Stats.ComparisonsTest doenddescribe "with period set to year to date" do- test "shifts back by the same number of days when mode is previous_period" do- site = insert(:site)-+ test "shifts back by the same number of days when mode is previous_period", %{site: site} doquery =build_query(site, %{"period" => "year", "date" => "2023-03-01"}, ~N[2023-03-01 14:00:00])@@ -146,9 +136,7 @@ defmodule Plausible.Stats.ComparisonsTest doassert comparison_query.utc_time_range.last == ~U[2022-12-31 23:59:59Z]end- test "shifts back by the same number of days when mode is year_over_year" do- site = insert(:site)-+ test "shifts back by the same number of days when mode is year_over_year", %{site: site} doquery =build_query(site, %{"period" => "year", "date" => "2023-03-01"}, ~N[2023-03-01 14:00:00])@@ -158,9 +146,7 @@ defmodule Plausible.Stats.ComparisonsTest doassert comparison_query.utc_time_range.last == ~U[2022-03-01 23:59:59Z]end- test "matches the day of the week when mode is year_over_year" do- site = insert(:site)-+ test "matches the day of the week when mode is year_over_year", %{site: site} doquery =build_query(site, %{"period" => "year", "date" => "2023-03-01"}, ~N[2023-03-01 14:00:00])@@ -173,8 +159,7 @@ defmodule Plausible.Stats.ComparisonsTest doenddescribe "with period set to previous year" do- test "shifts back a whole year when mode is year_over_year" do- site = insert(:site)+ test "shifts back a whole year when mode is year_over_year", %{site: site} doquery = Query.from(site, %{"period" => "year", "date" => "2022-03-02"})comparison_query = Comparisons.get_comparison_query(query, %{mode: "year_over_year"})@@ -183,8 +168,7 @@ defmodule Plausible.Stats.ComparisonsTest doassert comparison_query.utc_time_range.last == ~U[2021-12-31 23:59:59Z]end- test "shifts back a whole year when mode is previous_period" do- site = insert(:site)+ test "shifts back a whole year when mode is previous_period", %{site: site} doquery = Query.from(site, %{"period" => "year", "date" => "2022-03-02"})comparison_query = Comparisons.get_comparison_query(query, %{mode: "previous_period"})@@ -195,8 +179,7 @@ defmodule Plausible.Stats.ComparisonsTest doenddescribe "with period set to custom" do- test "shifts back by the same number of days when mode is previous_period" do- site = insert(:site)+ test "shifts back by the same number of days when mode is previous_period", %{site: site} doquery = Query.from(site, %{"period" => "custom", "date" => "2023-01-01,2023-01-07"})comparison_query = Comparisons.get_comparison_query(query, %{mode: "previous_period"})@@ -205,8 +188,7 @@ defmodule Plausible.Stats.ComparisonsTest doassert comparison_query.utc_time_range.last == ~U[2022-12-31 23:59:59Z]end- test "shifts back to last year when mode is year_over_year" do- site = insert(:site)+ test "shifts back to last year when mode is year_over_year", %{site: site} doquery = Query.from(site, %{"period" => "custom", "date" => "2023-01-01,2023-01-07"})comparison_query = Comparisons.get_comparison_query(query, %{mode: "year_over_year"})@@ -217,8 +199,7 @@ defmodule Plausible.Stats.ComparisonsTest doenddescribe "with mode set to custom" do- test "sets first and last dates" do- site = insert(:site)+ test "sets first and last dates", %{site: site} doquery = Query.from(site, %{"period" => "custom", "date" => "2023-01-01,2023-01-07"})comparison_query =@@ -233,7 +214,7 @@ defmodule Plausible.Stats.ComparisonsTest doenddescribe "include_imported" do- setup [:create_user, :create_new_site, :create_site_import]+ setup [:create_site_import]test "defaults to source_query.include_imported", %{site: site} doquery = Query.from(site, %{"period" => "day", "date" => "2023-01-01"})@@ -243,4 +224,113 @@ defmodule Plausible.Stats.ComparisonsTest doassert comparison_query.include_imported == falseendend++ describe "add_comparison_filters" do+ test "no results doesn't update filters", %{site: site} do+ query = build_comparison_query(site, %{"dimensions" => ["visit:browser"]})++ result_query = Comparisons.add_comparison_filters(query, [])++ assert result_query.filters == []+ end++ test "no dimensions doesn't update filters", %{site: site} do+ query = build_comparison_query(site, %{})++ result_query =+ Comparisons.add_comparison_filters(query, [%{dimensions: [], metrics: [123]}])++ assert result_query.filters == []+ end++ test "no time dimension doesn't update filters", %{site: site} do+ query = build_comparison_query(site, %{"dimensions" => ["time:day"]})++ result_query =+ Comparisons.add_comparison_filters(query, [%{dimensions: ["2024-01-01"], metrics: [123]}])++ assert result_query.filters == []+ end++ test "updates filters in a single-row case", %{site: site} do+ query =+ build_comparison_query(site, %{+ "dimensions" => ["visit:browser"]+ })++ result_query =+ Comparisons.add_comparison_filters(query, [%{dimensions: ["Chrome"], metrics: [123]}])++ assert result_query.filters == [+ [:ignore_in_totals_query, [:is, "visit:browser", ["Chrome"]]]+ ]+ end++ test "updates filters for a complex case", %{site: site} do+ query =+ build_comparison_query(site, %{+ "dimensions" => ["visit:browser", "visit:browser_version", "time:day"],+ "filters" => [["is", "visit:country_name", ["Estonia"]]]+ })++ main_query_results = [+ %{+ dimensions: ["Chrome", "99.9", "2024-01-01"],+ metrics: [123]+ },+ %{+ dimensions: ["Firefox", "12.0", "2024-01-01"],+ metrics: [123]+ }+ ]++ result_query = Comparisons.add_comparison_filters(query, main_query_results)++ assert result_query.filters == [+ [:is, "visit:country_name", ["Estonia"]],+ [+ :ignore_in_totals_query,+ [+ :or,+ [+ [+ :and,+ [+ [:is, "visit:browser", ["Chrome"]],+ [:is, "visit:browser_version", ["99.9"]]+ ]+ ],+ [+ :and,+ [+ [:is, "visit:browser", ["Firefox"]],+ [:is, "visit:browser_version", ["12.0"]]+ ]+ ]+ ]+ ]+ ]+ ]+ end+ end++ defp build_comparison_query(site, params) do+ {:ok, query} =+ Query.build(+ site,+ :internal,+ Map.merge(+ %{+ "site_id" => site.domain,+ "metrics" => ["pageviews"],+ "date_range" => ["2024-01-01", "2024-02-01"],+ "include" => %{"comparisons" => %{"mode" => "previous_period"}}+ },+ params+ ),+ %{}+ )++ query+ endenddiff --git a/test/plausible_web/controllers/api/external_stats_controller/query_comparisons_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_comparisons_test.exsnew file mode 100644index 0000000000..bf9e746d11--- /dev/null+++ b/test/plausible_web/controllers/api/external_stats_controller/query_comparisons_test.exs@@ -0,0 +1,230 @@+defmodule PlausibleWeb.Api.ExternalStatsController.QueryComparisonsTest do+ use PlausibleWeb.ConnCase++ setup [:create_user, :create_new_site, :create_api_key, :use_api_key, :create_site_import]++ test "aggregates a single metric", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:25:00]),+ build(:pageview, timestamp: ~N[2021-01-02 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-07 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["pageviews"],+ "date_range" => ["2021-01-07", "2021-01-13"],+ "include" => %{"comparisons" => %{"mode" => "previous_period"}}+ })++ assert json_response(conn, 200)["results"] == [+ %{+ "dimensions" => [],+ "metrics" => [1],+ "comparison" => %{"change" => [-67], "dimensions" => [], "metrics" => [3]}+ }+ ]+ end++ test "timeseries comparison", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:25:00]),+ build(:pageview, timestamp: ~N[2021-01-06 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-07 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-08 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["pageviews"],+ "date_range" => ["2021-01-07", "2021-01-13"],+ "dimensions" => ["time:day"],+ "include" => %{"comparisons" => %{"mode" => "previous_period"}}+ })++ assert json_response(conn, 200)["results"] == [+ %{+ "dimensions" => ["2021-01-07"],+ "metrics" => [1],+ "comparison" => %{+ "dimensions" => ["2020-12-31"],+ "metrics" => [0],+ "change" => [100]+ }+ },+ %{+ "dimensions" => ["2021-01-08"],+ "metrics" => [1],+ "comparison" => %{+ "dimensions" => ["2021-01-01"],+ "metrics" => [2],+ "change" => [-50]+ }+ },+ %{+ "dimensions" => ["2021-01-09"],+ "metrics" => [0],+ "comparison" => %{+ "dimensions" => ["2021-01-02"],+ "metrics" => [0],+ "change" => [0]+ }+ },+ %{+ "dimensions" => ["2021-01-10"],+ "metrics" => [0],+ "comparison" => %{+ "dimensions" => ["2021-01-03"],+ "metrics" => [0],+ "change" => [0]+ }+ },+ %{+ "dimensions" => ["2021-01-11"],+ "metrics" => [0],+ "comparison" => %{+ "dimensions" => ["2021-01-04"],+ "metrics" => [0],+ "change" => [0]+ }+ },+ %{+ "dimensions" => ["2021-01-12"],+ "metrics" => [0],+ "comparison" => %{+ "dimensions" => ["2021-01-05"],+ "metrics" => [0],+ "change" => [0]+ }+ },+ %{+ "dimensions" => ["2021-01-13"],+ "metrics" => [0],+ "comparison" => %{+ "dimensions" => ["2021-01-06"],+ "metrics" => [1],+ "change" => [-100]+ }+ }+ ]+ end++ test "dimensional comparison with low limit", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, browser: "Safari", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, browser: "Safari", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, browser: "Safari", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-07 00:00:00]),+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-07 00:00:00]),+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-07 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "percentage"],+ "date_range" => ["2021-01-07", "2021-01-13"],+ "dimensions" => ["visit:browser"],+ "include" => %{+ "comparisons" => %{"mode" => "previous_period"}+ },+ "pagination" => %{"limit" => 2}+ })++ assert json_response(conn, 200)["results"] == [+ %{+ "dimensions" => ["Chrome"],+ "metrics" => [2, 66.7],+ "comparison" => %{+ "dimensions" => ["Chrome"],+ "metrics" => [1, 12.5],+ "change" => [100, 434]+ }+ },+ %{+ "dimensions" => ["Firefox"],+ "metrics" => [1, 33.3],+ "comparison" => %{+ "dimensions" => ["Firefox"],+ "metrics" => [4, 50.0],+ "change" => [-75, -33]+ }+ }+ ]+ end++ test "dimensional comparison with imported data", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-07 00:00:00]),+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-07 00:00:00]),+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-07 00:00:00]),+ build(:imported_browsers,+ date: ~D[2021-01-01],+ browser: "Firefox",+ browser_version: "121",+ visitors: 50+ ),+ build(:imported_browsers,+ date: ~D[2021-01-01],+ browser: "Chrome",+ browser_version: "99",+ visitors: 39+ ),+ build(:imported_browsers,+ date: ~D[2021-01-01],+ browser: "Safari",+ browser_version: "99",+ visitors: 10+ ),+ build(:imported_visitors, date: ~D[2021-01-01], visitors: 99)+ ])++ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "percentage"],+ "date_range" => ["2021-01-07", "2021-01-13"],+ "dimensions" => ["visit:browser"],+ "include" => %{+ "imports" => true,+ "comparisons" => %{"mode" => "previous_period"}+ },+ "pagination" => %{"limit" => 2}+ })++ assert json_response(conn, 200)["results"] == [+ %{+ "dimensions" => ["Chrome"],+ "metrics" => [2, 66.7],+ "comparison" => %{+ "dimensions" => ["Chrome"],+ "metrics" => [40, 40.0],+ "change" => [-95, 67]+ }+ },+ %{+ "dimensions" => ["Firefox"],+ "metrics" => [1, 33.3],+ "comparison" => %{+ "dimensions" => ["Firefox"],+ "metrics" => [50, 50.0],+ "change" => [-98, -33]+ }+ }+ ]+ end+enddiff --git a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs b/test/plausible_web/controllers/api/stats_controller/browsers_test.exsindex 4020847b87..0bd3d14cc5 100644--- a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs@@ -191,6 +191,76 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do%{"name" => "(not set)", "visitors" => 2, "percentage" => 100.0}]end++ test "returns comparisons", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, browser: "Safari", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-07 00:00:00]),+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-07 00:00:00]),+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-07 00:00:00])+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/browsers?period=7d&date=2021-01-13&comparison=previous_period"+ )++ assert json_response(conn, 200)["results"] == [+ %{+ "name" => "Chrome",+ "visitors" => 2,+ "percentage" => 66.7,+ "comparison" => %{+ "visitors" => 0,+ "percentage" => 0.0,+ "change" => %{"percentage" => 100, "visitors" => 100}+ }+ },+ %{+ "name" => "Firefox",+ "visitors" => 1,+ "percentage" => 33.3,+ "comparison" => %{+ "visitors" => 1,+ "percentage" => 50.0,+ "change" => %{"percentage" => -33, "visitors" => 0}+ }+ }+ ]+ end++ test "returns comparisons with limit", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-07 00:00:00]),+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-07 00:00:00]),+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-07 00:00:00])+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/browsers?period=7d&date=2021-01-13&comparison=previous_period&limit=1"+ )++ assert json_response(conn, 200)["results"] == [+ %{+ "name" => "Chrome",+ "visitors" => 2,+ "percentage" => 66.7,+ "comparison" => %{+ "visitors" => 1,+ "percentage" => 25.0,+ "change" => %{"percentage" => 167, "visitors" => 100}+ }+ }+ ]+ endenddescribe "GET /api/stats/:domain/browser-versions" do