Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Refactor comparison query & including imported data (#5068)
RobertJoonas51a6ca665466
diff --git a/lib/plausible/imported.ex b/lib/plausible/imported.exindex d2b49f4467..7a67aed616 100644--- a/lib/plausible/imported.ex+++ b/lib/plausible/imported.ex@@ -66,6 +66,11 @@ defmodule Plausible.Imported do@goals_with_pathend+ @spec any_completed_imports?(Site.t()) :: boolean()+ def any_completed_imports?(site) do+ get_completed_imports(site) != []+ end+@spec earliest_import_start_date(Site.t()) :: Date.t() | nildef earliest_import_start_date(site) dosite@@ -74,14 +79,6 @@ defmodule Plausible.Imported do|> Enum.min(Date, fn -> nil end)end- @spec latest_import_end_date(Site.t()) :: Date.t() | nil- def latest_import_end_date(site) do- site- |> get_completed_imports()- |> Enum.map(& &1.end_date)- |> Enum.max(Date, fn -> nil end)- end-@spec complete_import_ids(Site.t()) :: [non_neg_integer()]def complete_import_ids(site) doimports = get_completed_imports(site)@@ -102,8 +99,9 @@ defmodule Plausible.Imported dosite|> get_completed_imports()- |> Enum.filter(fn site_import ->- site_import.start_date <= date_range.last && site_import.end_date >= date_range.first+ |> Enum.reject(fn site_import ->+ Date.after?(site_import.start_date, date_range.last) or+ Date.before?(site_import.end_date, date_range.first)end)enddiff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.exindex ef70183a58..b87eee2a21 100644--- a/lib/plausible/stats/breakdown.ex+++ b/lib/plausible/stats/breakdown.ex@@ -53,7 +53,7 @@ defmodule Plausible.Stats.Breakdown doif query.include.comparisons docomparison_date_range_label =query- |> Comparisons.get_comparison_query(query.include.comparisons)+ |> Comparisons.get_comparison_query()|> format_date_range()Map.put(diff --git a/lib/plausible/stats/comparisons.ex b/lib/plausible/stats/comparisons.exindex e560632f19..1338a21819 100644--- a/lib/plausible/stats/comparisons.ex+++ b/lib/plausible/stats/comparisons.ex@@ -10,9 +10,10 @@ defmodule Plausible.Stats.Comparisons doalias Plausible.Statsalias Plausible.Stats.{Query, DateTimeRange, Time}- @spec get_comparison_query(Stats.Query.t(), map()) :: Stats.Query.t()+ @spec get_comparison_utc_time_range(Stats.Query.t(), map()) :: DateTimeRange.t()@doc """- Generates a comparison query based on the source query and comparison mode.+ Generates a `DateTimeRange` for the comparison period by the given source query+ and comparison options.Currently only historical periods are supported for comparisons (not `realtime`and `30m` periods).@@ -41,20 +42,22 @@ defmodule Plausible.Stats.Comparisons doJanuary 1st. Defaults to false."""- def get_comparison_query(%Stats.Query{} = source_query, options) do+ def get_comparison_utc_time_range(%Stats.Query{} = source_query, options) docomparison_date_range = get_comparison_date_range(source_query, options)- new_range =- DateTimeRange.new!(- comparison_date_range.first,- comparison_date_range.last,- source_query.timezone- )- |> DateTimeRange.to_timezone("Etc/UTC")+ DateTimeRange.new!(+ comparison_date_range.first,+ comparison_date_range.last,+ source_query.timezone+ )+ |> DateTimeRange.to_timezone("Etc/UTC")+ end+ def get_comparison_query(+ %Query{comparison_utc_time_range: %DateTimeRange{} = comparison_range} = source_query+ ) dosource_query- |> Query.set(utc_time_range: new_range)- |> maybe_include_imported(source_query)+ |> Query.set(utc_time_range: comparison_range)end@doc """@@ -172,14 +175,4 @@ defmodule Plausible.Stats.Comparisons doDate.add(date, -days_to_subtract)end-- defp maybe_include_imported(query, source_query) do- requested? = source_query.include.imports- skip_imported_reason = Query.get_skip_imported_reason(query)-- struct!(query,- include_imported: requested? and is_nil(skip_imported_reason),- skip_imported_reason: skip_imported_reason- )- endenddiff --git a/lib/plausible/stats/email_report.ex b/lib/plausible/stats/email_report.exindex fe95d5b64d..ef16d6c5b8 100644--- a/lib/plausible/stats/email_report.ex+++ b/lib/plausible/stats/email_report.ex@@ -14,27 +14,30 @@ defmodule Plausible.Stats.EmailReport do"""alias Plausible.Stats- alias Plausible.Stats.{Query, Compare, Comparisons}+ alias Plausible.Stats.{Query, QueryResult}- def get(site, query) do- metrics = [:pageviews, :visitors, :bounce_rate]-- %{results: results} = Stats.aggregate(site, query, metrics)+ @aggregate_metrics [:pageviews, :visitors, :bounce_rate]- results- |> with_comparisons(site, query, metrics)+ def get(site, query) do+ aggregate_and_compare(site, query)|> put_top_5_pages(site, query)|> put_top_5_sources(site, query)end- defp with_comparisons(stats, site, query, metrics) do- comparison_query = Comparisons.get_comparison_query(query, %{mode: "previous_period"})- %{results: prev_period_stats} = Stats.aggregate(site, comparison_query, metrics)+ defp aggregate_and_compare(site, query) do+ query =+ query+ |> Query.set(metrics: @aggregate_metrics)+ |> Query.set_include(:comparisons, %{mode: "previous_period"})+ |> Query.put_comparison_utc_time_range()++ %QueryResult{results: [result]} = Plausible.Stats.query(site, query)- stats- |> Enum.map(fn {metric, %{value: value}} ->- %{value: prev_value} = Map.fetch!(prev_period_stats, metric)- change = Compare.calculate_change(metric, prev_value, value)+ @aggregate_metrics+ |> Enum.with_index()+ |> Enum.map(fn {metric, idx} ->+ value = Enum.at(result.metrics, idx)+ change = Enum.at(result.comparison.change, idx){metric, %{value: value, change: change}}end)diff --git a/lib/plausible/stats/legacy/legacy_query_builder.ex b/lib/plausible/stats/legacy/legacy_query_builder.exindex 3a2c7730a6..35f03521e4 100644--- a/lib/plausible/stats/legacy/legacy_query_builder.ex+++ b/lib/plausible/stats/legacy/legacy_query_builder.ex@@ -28,6 +28,7 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do|> preload_goals_and_revenue(site)|> put_order_by(params)|> put_include(site, params)+ |> Query.put_comparison_utc_time_range()|> Query.put_imported_opts(site)on_ee do@@ -329,5 +330,9 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do}end+ def parse_comparison_params(_site, %{"compare" => "previous_period"}) do+ %{mode: "previous_period"}+ end+def parse_comparison_params(_site, _options), do: nilenddiff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.exindex 6634919404..f396cf6388 100644--- a/lib/plausible/stats/query.ex+++ b/lib/plausible/stats/query.ex@@ -2,15 +2,17 @@ defmodule Plausible.Stats.Query douse Plausibledefstruct utc_time_range: nil,+ comparison_utc_time_range: nil,interval: nil,period: nil,dimensions: [],filters: [],sample_threshold: 20_000_000,+ imports_exist: false,+ imports_in_range: [],include_imported: false,skip_imported_reason: nil,now: nil,- latest_import_end_date: nil,metrics: [],order_by: nil,timezone: nil,@@ -27,21 +29,22 @@ defmodule Plausible.Stats.Query dosite_native_stats_start_at: nilrequire OpenTelemetry.Tracer, as: Tracer- alias Plausible.Stats.{DateTimeRange, Filters, Imported, Legacy}+ alias Plausible.Stats.{DateTimeRange, Filters, Imported, Legacy, Comparisons}@type t :: %__MODULE__{}def build(site, schema_type, params, debug_metadata) dowith {:ok, query_data} <- Filters.QueryParser.parse(site, schema_type, params) doquery =- struct!(__MODULE__, Map.to_list(query_data))- |> put_imported_opts(site)- |> struct!(+ %__MODULE__{now: DateTime.utc_now(:second),debug_metadata: debug_metadata,site_id: site.id,site_native_stats_start_at: site.native_stats_start_at- )+ }+ |> struct!(Map.to_list(query_data))+ |> put_comparison_utc_time_range()+ |> put_imported_opts(site)on_ee doquery = Plausible.Stats.Sampling.put_threshold(query, site, params)@@ -123,18 +126,26 @@ defmodule Plausible.Stats.Query doput_imported_opts(query, nil)end+ def put_comparison_utc_time_range(%__MODULE__{include: %{comparisons: nil}} = query), do: query++ def put_comparison_utc_time_range(%__MODULE__{include: %{comparisons: comparison_opts}} = query) do+ datetime_range = Comparisons.get_comparison_utc_time_range(query, comparison_opts)+ struct!(query, comparison_utc_time_range: datetime_range)+ end+def put_imported_opts(query, site) dorequested? = query.include.imports- latest_import_end_date =+ query =if site do- Plausible.Imported.latest_import_end_date(site)+ struct!(query,+ imports_exist: Plausible.Imported.any_completed_imports?(site),+ imports_in_range: get_imports_in_range(site, query)+ )else- query.latest_import_end_date+ queryend- query = struct!(query, latest_import_end_date: latest_import_end_date)-skip_imported_reason = get_skip_imported_reason(query)struct!(query,@@ -143,15 +154,33 @@ defmodule Plausible.Stats.Query do)end+ defp get_imports_in_range(_site, %__MODULE__{period: period})+ when period in ["realtime", "30m"] do+ []+ end++ defp get_imports_in_range(site, query) do+ in_range = Plausible.Imported.completed_imports_in_query_range(site, query)++ in_comparison_range =+ if is_map(query.include.comparisons) do+ comparison_query = Comparisons.get_comparison_query(query)+ Plausible.Imported.completed_imports_in_query_range(site, comparison_query)+ else+ []+ end++ in_comparison_range ++ in_range+ end+@spec get_skip_imported_reason(t()) ::nil | :no_imported_data | :out_of_range | :unsupported_querydef get_skip_imported_reason(query) docond do- is_nil(query.latest_import_end_date) ->+ not query.imports_exist ->:no_imported_data- query.period in ["realtime", "30m"] or- Date.after?(date_range(query).first, query.latest_import_end_date) ->+ query.imports_in_range == [] ->:out_of_range"time:minute" in query.dimensions or "time:hour" in query.dimensions ->diff --git a/lib/plausible/stats/query_result.ex b/lib/plausible/stats/query_result.exindex 7e884c7d68..1f9603142c 100644--- a/lib/plausible/stats/query_result.ex+++ b/lib/plausible/stats/query_result.ex@@ -9,7 +9,6 @@ defmodule Plausible.Stats.QueryResult douse Plausiblealias Plausible.Stats.{DateTimeRange, Query, QueryRunner}- alias Plausible.Importeddefstruct results: [],meta: %{},@@ -44,8 +43,8 @@ defmodule Plausible.Stats.QueryResult dodefp meta(%QueryRunner{} = runner) do%{}- |> add_imports_meta(runner)- |> add_metric_warnings_meta(runner)+ |> add_imports_meta(runner.main_query)+ |> add_metric_warnings_meta(runner.main_query)|> add_time_labels_meta(runner.main_query)|> add_total_rows_meta(runner.main_query, runner.total_rows)end@@ -58,30 +57,12 @@ defmodule Plausible.Stats.QueryResult do"Imported stats are not included because the time dimension (i.e. the interval) is too short."}- defp add_imports_meta(meta, %QueryRunner{} = runner) do- %{main_query: %{include: include} = main_query} = runner-- if include.imports or include[:imports_meta] do- comparison_query = Map.get(runner, :comparison_query)-- imports_included =- case comparison_query do- %Query{include_imported: true} -> true- _ -> main_query.include_imported- end-- imports_skip_reason =- case comparison_query do- %Query{skip_imported_reason: nil} -> nil- _ -> main_query.skip_imported_reason- end-- imports_warning = @imports_warnings[imports_skip_reason]-+ defp add_imports_meta(meta, %Query{include: include} = query) do+ if include.imports or include.imports_meta do%{- imports_included: imports_included,- imports_skip_reason: imports_skip_reason,- imports_warning: imports_warning+ imports_included: query.include_imported,+ imports_skip_reason: query.skip_imported_reason,+ imports_warning: @imports_warnings[query.skip_imported_reason]}|> Map.reject(fn {_key, value} -> is_nil(value) end)|> Map.merge(meta)@@ -90,8 +71,8 @@ defmodule Plausible.Stats.QueryResult doendend- defp add_metric_warnings_meta(meta, runner) do- warnings = metric_warnings(meta, runner)+ defp add_metric_warnings_meta(meta, query) do+ warnings = metric_warnings(query)if map_size(warnings) > 0 doMap.put(meta, :metric_warnings, warnings)@@ -130,9 +111,9 @@ defmodule Plausible.Stats.QueryResult doendend- defp metric_warnings(meta, %QueryRunner{main_query: query} = runner) do+ defp metric_warnings(%Query{} = query) doEnum.reduce(query.metrics, %{}, fn metric, acc ->- case metric_warning(metric, meta, runner) do+ case metric_warning(metric, query) donil -> acc%{} = warning -> Map.put(acc, metric, warning)end@@ -151,7 +132,7 @@ defmodule Plausible.Stats.QueryResult do"Revenue metrics are null as there are no matching revenue goals."}- defp metric_warning(metric, _meta, %QueryRunner{main_query: query})+ defp metric_warning(metric, %Query{} = query)when metric in @revenue_metrics doif query.revenue_warning do%{@@ -169,25 +150,13 @@ defmodule Plausible.Stats.QueryResult dowarning: "No imports with scroll depth data were found"}- defp metric_warning(:scroll_depth, %{imports_included: true} = _meta, %QueryRunner{} = runner) do- %QueryRunner{site: site, main_query: main_query, comparison_query: comparison_query} = runner-- imports_in_main_range = Imported.completed_imports_in_query_range(site, main_query)-- imports_in_comparison_range =- case comparison_query do- nil -> []- %Query{} = query -> Imported.completed_imports_in_query_range(site, query)- end-- imports_in_range = imports_in_main_range ++ imports_in_comparison_range-- if not Enum.any?(imports_in_range, & &1.has_scroll_depth) do+ defp metric_warning(:scroll_depth, %Query{} = query) do+ if query.include_imported and not Enum.any?(query.imports_in_range, & &1.has_scroll_depth) do@no_imported_scroll_depth_metric_warningendend- defp metric_warning(_metric, _meta, _query), do: nil+ defp metric_warning(_metric, _query), do: nildefp to_iso8601(datetime, timezone) dodatetimediff --git a/lib/plausible/stats/query_runner.ex b/lib/plausible/stats/query_runner.exindex 6fdd7dacf9..ed51a96dd1 100644--- a/lib/plausible/stats/query_runner.ex+++ b/lib/plausible/stats/query_runner.ex@@ -62,7 +62,7 @@ defmodule Plausible.Stats.QueryRunner dowhen is_map(query.include.comparisons) docomparison_query =query- |> Comparisons.get_comparison_query(query.include.comparisons)+ |> Comparisons.get_comparison_query()|> Comparisons.add_comparison_filters(main_results)struct!(runner, comparison_query: comparison_query)diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.exindex 5de0d3c93d..5a01be0334 100644--- a/lib/plausible/stats/timeseries.ex+++ b/lib/plausible/stats/timeseries.ex@@ -29,7 +29,7 @@ defmodule Plausible.Stats.Timeseries docomparison_query =if(query.include.comparisons,- do: Comparisons.get_comparison_query(query, query.include.comparisons),+ do: Comparisons.get_comparison_query(query),else: nil)diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.exindex a585a71bd2..9bf8edd7c2 100644--- a/lib/plausible_web/controllers/api/external_stats_controller.ex+++ b/lib/plausible_web/controllers/api/external_stats_controller.ex@@ -20,13 +20,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController do:ok <- validate_filters(site, query.filters),{:ok, metrics} <- parse_and_validate_metrics(params, query),:ok <- ensure_custom_props_access(site, query) do- query =- if params["compare"] == "previous_period" do- Query.set_include(query, :comparisons, %{mode: "previous_period"})- else- query- end-%{results: results, meta: meta} = Plausible.Stats.aggregate(site, query, metrics)payload = maybe_add_warning(%{results: results}, meta)diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex 1456c86c76..1399d07f0e 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -1631,7 +1631,7 @@ defmodule PlausibleWeb.Api.StatsController dodef comparison_query(query) doif query.include.comparisons do- Comparisons.get_comparison_query(query, query.include.comparisons)+ Comparisons.get_comparison_query(query)endenddiff --git a/test/plausible/imported_test.exs b/test/plausible/imported_test.exsindex 01669acb79..1c734b8181 100644--- a/test/plausible/imported_test.exs+++ b/test/plausible/imported_test.exs@@ -75,48 +75,6 @@ defmodule Plausible.ImportedTest doendend- describe "latest_import_end_date/1" do- test "returns nil if no site_imports exist" do- site = insert(:site)-- assert is_nil(Imported.latest_import_end_date(site))- end-- test "returns nil when only incomplete or failed imports are present" do- site = insert(:site)-- _import1 = insert(:site_import, site: site, status: :pending)- _import2 = insert(:site_import, site: site, status: :importing)- _import3 = insert(:site_import, site: site, status: :failed)- _rogue_import = insert(:site_import, site: build(:site), status: :completed)-- assert is_nil(Imported.latest_import_end_date(site))- end-- test "returns start and end dates considering all imports" do- site = insert(:site)-- _import1 =- insert(:site_import,- site: site,- start_date: ~D[2020-04-02],- end_date: ~D[2022-06-22],- status: :completed,- legacy: true- )-- _import2 =- insert(:site_import,- site: site,- start_date: ~D[2022-06-22],- end_date: ~D[2024-01-08],- status: :completed- )-- assert Imported.latest_import_end_date(site) == ~D[2024-01-08]- end- end-describe "completed_imports_in_query_range/2" dosetup dosite = insert(:site)diff --git a/test/plausible/stats/comparisons_test.exs b/test/plausible/stats/comparisons_test.exsindex bdacaff230..e3f8058b64 100644--- a/test/plausible/stats/comparisons_test.exs+++ b/test/plausible/stats/comparisons_test.exs@@ -1,14 +1,12 @@defmodule Plausible.Stats.ComparisonsTest douse Plausible.DataCase- alias Plausible.Stats.{DateTimeRange, Query, Comparisons}+ alias Plausible.Stats.{Query, Comparisons}import Plausible.TestUtilssetup [:create_user, :create_site]def build_query(site, params, now) do- query = Query.from(site, params)-- Map.put(query, :now, now)+ Query.from(site, params, %{}, now)enddescribe "with period set to this month" do@@ -16,11 +14,11 @@ defmodule Plausible.Stats.ComparisonsTest doquery =build_query(site,- %{"period" => "month", "date" => "2023-03-02"},+ %{"period" => "month", "date" => "2023-03-02", "comparison" => "previous_period"},~U[2023-03-02 14:00:00Z])- comparison_query = Comparisons.get_comparison_query(query, %{mode: "previous_period"})+ comparison_query = Comparisons.get_comparison_query(query)assert comparison_query.utc_time_range.first == ~U[2023-02-27 00:00:00Z]assert comparison_query.utc_time_range.last == ~U[2023-02-28 23:59:59Z]@@ -31,11 +29,11 @@ defmodule Plausible.Stats.ComparisonsTest doquery =build_query(site,- %{"period" => "month", "date" => "2023-03-01"},+ %{"period" => "month", "date" => "2023-03-01", "comparison" => "previous_period"},~U[2023-03-01 14:00:00Z])- comparison_query = Comparisons.get_comparison_query(query, %{mode: "previous_period"})+ comparison_query = Comparisons.get_comparison_query(query)assert comparison_query.utc_time_range.first == ~U[2023-02-28 00:00:00Z]assert comparison_query.utc_time_range.last == ~U[2023-02-28 23:59:59Z]@@ -46,15 +44,16 @@ defmodule Plausible.Stats.ComparisonsTest doquery =build_query(site,- %{"period" => "month", "date" => "2023-03-02"},+ %{+ "period" => "month",+ "date" => "2023-03-02",+ "comparison" => "previous_period",+ "match_day_of_week" => "true"+ },~U[2023-03-02 14:00:00Z])- comparison_query =- Comparisons.get_comparison_query(query, %{- mode: "previous_period",- match_day_of_week: true- })+ comparison_query = Comparisons.get_comparison_query(query)assert comparison_query.utc_time_range.first == ~U[2023-02-22 00:00:00Z]assert comparison_query.utc_time_range.last == ~U[2023-02-23 23:59:59Z]@@ -66,11 +65,11 @@ defmodule Plausible.Stats.ComparisonsTest doquery =build_query(site,- %{"period" => "month", "date" => "2023-03-02"},+ %{"period" => "month", "date" => "2023-03-02", "comparison" => "previous_period"},~U[2023-03-02 14:00:00Z])- comparison_query = Comparisons.get_comparison_query(query, %{mode: "previous_period"})+ comparison_query = Comparisons.get_comparison_query(query)assert comparison_query.utc_time_range.first == ~U[2023-02-27 05:00:00Z]assert comparison_query.utc_time_range.last == ~U[2023-03-01 04:59:59Z]@@ -82,11 +81,11 @@ defmodule Plausible.Stats.ComparisonsTest doquery =build_query(site,- %{"period" => "month", "date" => "2023-02-01"},+ %{"period" => "month", "date" => "2023-02-01", "comparison" => "previous_period"},~U[2023-03-01 14:00:00Z])- comparison_query = Comparisons.get_comparison_query(query, %{mode: "previous_period"})+ comparison_query = Comparisons.get_comparison_query(query)assert comparison_query.utc_time_range.first == ~U[2023-01-04 00:00:00Z]assert comparison_query.utc_time_range.last == ~U[2023-01-31 23:59:59Z]@@ -96,11 +95,11 @@ defmodule Plausible.Stats.ComparisonsTest doquery =build_query(site,- %{"period" => "month", "date" => "2023-02-01"},+ %{"period" => "month", "date" => "2023-02-01", "comparison" => "year_over_year"},~U[2023-03-01 14:00:00Z])- comparison_query = Comparisons.get_comparison_query(query, %{mode: "year_over_year"})+ comparison_query = Comparisons.get_comparison_query(query)assert comparison_query.utc_time_range.first == ~U[2022-02-01 00:00:00Z]assert comparison_query.utc_time_range.last == ~U[2022-02-28 23:59:59Z]@@ -112,11 +111,11 @@ defmodule Plausible.Stats.ComparisonsTest doquery =build_query(site,- %{"period" => "month", "date" => "2020-02-01"},+ %{"period" => "month", "date" => "2020-02-01", "comparison" => "year_over_year"},~U[2023-03-01 14:00:00Z])- comparison_query = Comparisons.get_comparison_query(query, %{mode: "year_over_year"})+ comparison_query = Comparisons.get_comparison_query(query)assert comparison_query.utc_time_range.first == ~U[2019-02-01 00:00:00Z]assert comparison_query.utc_time_range.last == ~U[2019-03-01 23:59:59Z]@@ -128,15 +127,16 @@ defmodule Plausible.Stats.ComparisonsTest doquery =build_query(site,- %{"period" => "month", "date" => "2023-02-01"},+ %{+ "period" => "month",+ "date" => "2023-02-01",+ "comparison" => "previous_period",+ "match_day_of_week" => "true"+ },~U[2023-03-01 14:00:00Z])- comparison_query =- Comparisons.get_comparison_query(query, %{- mode: "previous_period",- match_day_of_week: true- })+ comparison_query = Comparisons.get_comparison_query(query)assert comparison_query.utc_time_range.first == ~U[2023-01-04 00:00:00Z]assert comparison_query.utc_time_range.last == ~U[2023-01-31 23:59:59Z]@@ -146,15 +146,16 @@ defmodule Plausible.Stats.ComparisonsTest doquery =build_query(site,- %{"period" => "month", "date" => "2023-01-01"},+ %{+ "period" => "month",+ "date" => "2023-01-01",+ "comparison" => "previous_period",+ "match_day_of_week" => "true"+ },~U[2023-03-01 14:00:00Z])- comparison_query =- Comparisons.get_comparison_query(query, %{- mode: "previous_period",- match_day_of_week: true- })+ comparison_query = Comparisons.get_comparison_query(query)assert comparison_query.utc_time_range.first == ~U[2022-12-04 00:00:00Z]assert comparison_query.utc_time_range.last == ~U[2023-01-03 23:59:59Z]@@ -163,8 +164,14 @@ defmodule Plausible.Stats.ComparisonsTest dodescribe "year_over_year, exact dates behavior with leap years" dotest "start of the year matching", %{site: site} do- query = Query.from(site, %{"period" => "7d", "date" => "2021-01-05"})- comparison_query = Comparisons.get_comparison_query(query, %{mode: "year_over_year"})+ query =+ Query.from(site, %{+ "period" => "7d",+ "date" => "2021-01-05",+ "comparison" => "year_over_year"+ })++ comparison_query = Comparisons.get_comparison_query(query)assert comparison_query.utc_time_range.first == ~U[2019-12-30 00:00:00Z]assert comparison_query.utc_time_range.last == ~U[2020-01-05 23:59:59Z]@@ -172,8 +179,14 @@ defmodule Plausible.Stats.ComparisonsTest doendtest "leap day matching", %{site: site} do- query = Query.from(site, %{"period" => "7d", "date" => "2021-03-03"})- comparison_query = Comparisons.get_comparison_query(query, %{mode: "year_over_year"})+ query =+ Query.from(site, %{+ "period" => "7d",+ "date" => "2021-03-03",+ "comparison" => "year_over_year"+ })++ comparison_query = Comparisons.get_comparison_query(query)assert comparison_query.utc_time_range.first == ~U[2020-02-25 00:00:00Z]# :TRICKY: Since dates of the two months don't match precisely we cut off earlier@@ -182,8 +195,14 @@ defmodule Plausible.Stats.ComparisonsTest doendtest "end of the year matching", %{site: site} do- query = Query.from(site, %{"period" => "7d", "date" => "2021-11-25"})- comparison_query = Comparisons.get_comparison_query(query, %{mode: "year_over_year"})+ query =+ Query.from(site, %{+ "period" => "7d",+ "date" => "2021-11-25",+ "comparison" => "year_over_year"+ })++ comparison_query = Comparisons.get_comparison_query(query)assert comparison_query.utc_time_range.first == ~U[2020-11-19 00:00:00Z]assert comparison_query.utc_time_range.last == ~U[2020-11-25 23:59:59Z]@@ -194,9 +213,13 @@ defmodule Plausible.Stats.ComparisonsTest dodescribe "with period set to year to date" dotest "shifts back by the same number of days when mode is previous_period", %{site: site} doquery =- build_query(site, %{"period" => "year", "date" => "2023-03-01"}, ~U[2023-03-01 14:00:00Z])+ build_query(+ site,+ %{"period" => "year", "date" => "2023-03-01", "comparison" => "previous_period"},+ ~U[2023-03-01 14:00:00Z]+ )- comparison_query = Comparisons.get_comparison_query(query, %{mode: "previous_period"})+ comparison_query = Comparisons.get_comparison_query(query)assert comparison_query.utc_time_range.first == ~U[2022-11-02 00:00:00Z]assert comparison_query.utc_time_range.last == ~U[2022-12-31 23:59:59Z]@@ -204,9 +227,13 @@ defmodule Plausible.Stats.ComparisonsTest dotest "shifts back by the same number of days when mode is year_over_year", %{site: site} doquery =- build_query(site, %{"period" => "year", "date" => "2023-03-01"}, ~U[2023-03-01 14:00:00Z])+ build_query(+ site,+ %{"period" => "year", "date" => "2023-03-01", "comparison" => "year_over_year"},+ ~U[2023-03-01 14:00:00Z]+ )- comparison_query = Comparisons.get_comparison_query(query, %{mode: "year_over_year"})+ comparison_query = Comparisons.get_comparison_query(query)assert comparison_query.utc_time_range.first == ~U[2022-01-01 00:00:00Z]assert comparison_query.utc_time_range.last == ~U[2022-03-01 23:59:59Z]@@ -214,10 +241,18 @@ defmodule Plausible.Stats.ComparisonsTest dotest "matches the day of the week when mode is year_over_year", %{site: site} doquery =- build_query(site, %{"period" => "year", "date" => "2023-03-01"}, ~U[2023-03-01 14:00:00Z])+ build_query(+ site,+ %{+ "period" => "year",+ "date" => "2023-03-01",+ "comparison" => "year_over_year",+ "match_day_of_week" => "true"+ },+ ~U[2023-03-01 14:00:00Z]+ )- comparison_query =- Comparisons.get_comparison_query(query, %{mode: "year_over_year", match_day_of_week: true})+ comparison_query = Comparisons.get_comparison_query(query)assert comparison_query.utc_time_range.first == ~U[2022-01-02 00:00:00Z]assert comparison_query.utc_time_range.last == ~U[2022-03-02 23:59:59Z]@@ -226,18 +261,28 @@ defmodule Plausible.Stats.ComparisonsTest dodescribe "with period set to previous year" dotest "shifts back a whole year when mode is year_over_year", %{site: site} do- query = Query.from(site, %{"period" => "year", "date" => "2022-03-02"})+ query =+ Query.from(site, %{+ "period" => "year",+ "date" => "2022-03-02",+ "comparison" => "year_over_year"+ })- comparison_query = Comparisons.get_comparison_query(query, %{mode: "year_over_year"})+ comparison_query = Comparisons.get_comparison_query(query)assert comparison_query.utc_time_range.first == ~U[2021-01-01 00:00:00Z]assert comparison_query.utc_time_range.last == ~U[2021-12-31 23:59:59Z]endtest "shifts back a whole year when mode is previous_period", %{site: site} do- query = Query.from(site, %{"period" => "year", "date" => "2022-03-02"})+ query =+ Query.from(site, %{+ "period" => "year",+ "date" => "2022-03-02",+ "comparison" => "previous_period"+ })- comparison_query = Comparisons.get_comparison_query(query, %{mode: "previous_period"})+ comparison_query = Comparisons.get_comparison_query(query)assert comparison_query.utc_time_range.first == ~U[2021-01-01 00:00:00Z]assert comparison_query.utc_time_range.last == ~U[2021-12-31 23:59:59Z]@@ -246,18 +291,28 @@ defmodule Plausible.Stats.ComparisonsTest dodescribe "with period set to custom" dotest "shifts back by the same number of days when mode is previous_period", %{site: site} do- query = Query.from(site, %{"period" => "custom", "date" => "2023-01-01,2023-01-07"})+ query =+ Query.from(site, %{+ "period" => "custom",+ "date" => "2023-01-01,2023-01-07",+ "comparison" => "previous_period"+ })- comparison_query = Comparisons.get_comparison_query(query, %{mode: "previous_period"})+ comparison_query = Comparisons.get_comparison_query(query)assert comparison_query.utc_time_range.first == ~U[2022-12-25 00:00:00Z]assert comparison_query.utc_time_range.last == ~U[2022-12-31 23:59:59Z]endtest "shifts back to last year when mode is year_over_year", %{site: site} do- query = Query.from(site, %{"period" => "custom", "date" => "2023-01-01,2023-01-07"})+ query =+ Query.from(site, %{+ "period" => "custom",+ "date" => "2023-01-01,2023-01-07",+ "comparison" => "year_over_year"+ })- comparison_query = Comparisons.get_comparison_query(query, %{mode: "year_over_year"})+ comparison_query = Comparisons.get_comparison_query(query)assert comparison_query.utc_time_range.first == ~U[2022-01-01 00:00:00Z]assert comparison_query.utc_time_range.last == ~U[2022-01-07 23:59:59Z]@@ -266,31 +321,22 @@ defmodule Plausible.Stats.ComparisonsTest dodescribe "with mode set to custom" dotest "sets first and last dates", %{site: site} do- query = Query.from(site, %{"period" => "custom", "date" => "2023-01-01,2023-01-07"})-- comparison_query =- Comparisons.get_comparison_query(query, %{- mode: "custom",- date_range: DateTimeRange.new!(~U[2022-05-25 00:00:00Z], ~U[2022-05-30 23:59:59Z])+ query =+ Query.from(site, %{+ "period" => "custom",+ "date" => "2023-01-01,2023-01-07",+ "comparison" => "custom",+ "compare_from" => "2022-05-25",+ "compare_to" => "2022-05-30"})+ comparison_query = Comparisons.get_comparison_query(query)+assert comparison_query.utc_time_range.first == ~U[2022-05-25 00:00:00Z]assert comparison_query.utc_time_range.last == ~U[2022-05-30 23:59:59Z]endend- describe "include_imported" do- setup [:create_site_import]-- test "defaults to source_query.include_imported", %{site: site} do- query = Query.from(site, %{"period" => "day", "date" => "2023-01-01"})- assert query.include_imported == false-- comparison_query = Comparisons.get_comparison_query(query, %{mode: "previous_period"})- assert comparison_query.include_imported == false- end- end-describe "add_comparison_filters" dotest "no results doesn't update filters", %{site: site} doquery = build_comparison_query(site, %{"dimensions" => ["visit:browser"]})diff --git a/test/support/factory.ex b/test/support/factory.exindex 6f8b131cda..8bde583a61 100644--- a/test/support/factory.ex+++ b/test/support/factory.ex@@ -94,7 +94,7 @@ defmodule Plausible.Factory do%Plausible.Imported.SiteImport{site: build(:site),imported_by: build(:user),- start_date: Date.add(today, -200),+ start_date: ~D[2005-01-01],end_date: today,source: :universal_analytics,status: :completed,