Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Refactor including imports in stats queries (#5011)
RobertJoonas00deb05d6438
diff --git a/lib/plausible/imported.ex b/lib/plausible/imported.exindex 6ae034e7ae..2f4674030b 100644--- a/lib/plausible/imported.ex+++ b/lib/plausible/imported.ex@@ -67,20 +67,34 @@ defmodule Plausible.Imported do@goals_with_pathend- @spec load_import_data(Site.t()) :: Site.t()- def load_import_data(%{import_data_loaded: true} = site), do: site+ @spec earliest_import_start_date(Site.t()) :: Date.t() | nil+ def earliest_import_start_date(site) do+ site+ |> get_completed_imports()+ |> Enum.map(& &1.start_date)+ |> Enum.min(Date, fn -> nil end)+ end++ @spec latest_import_end_date(Site.t()) :: Date.t() | nil+ def latest_import_end_date(site) do+ site+ |> get_completed_imports()+ |> Enum.map(& &1.end_date)+ |> Enum.max(Date, fn -> nil end)+ end- def load_import_data(site) do- complete_import_ids = list_complete_import_ids(site)- dates = get_imports_date_range(site)+ @spec complete_import_ids(Site.t()) :: [non_neg_integer()]+ def complete_import_ids(site) do+ imports = get_completed_imports(site)+ has_legacy? = Enum.any?(imports, fn %{legacy: legacy?} -> legacy? end)+ ids = Enum.map(imports, fn %{id: id} -> id end)- %{- site- | import_data_loaded: true,- earliest_import_start_date: dates.start_date,- latest_import_end_date: dates.end_date,- complete_import_ids: complete_import_ids- }+ # account for legacy imports as well+ if has_legacy? do+ [0 | ids]+ else+ ids+ endend@spec get_import(Site.t(), non_neg_integer()) :: SiteImport.t() | nil@@ -88,16 +102,6 @@ defmodule Plausible.Imported doRepo.get_by(SiteImport, id: import_id, site_id: site.id)end- @spec get_legacy_import(Site.t()) :: SiteImport.t() | nil- def get_legacy_import(site) do- from(i in SiteImport,- where: i.site_id == ^site.id and i.legacy == true,- order_by: [desc: i.updated_at],- limit: 1- )- |> Repo.one()- end-defdelegate listen(), to: Imported.Importer@spec list_all_imports(Site.t(), atom()) :: [SiteImport.t()]@@ -123,42 +127,6 @@ defmodule Plausible.Imported dowhere(query, [i], i.status == ^status)end- @spec list_complete_import_ids(Site.t()) :: [non_neg_integer()]- def list_complete_import_ids(site) do- ids =- from(i in SiteImport,- where: i.site_id == ^site.id and i.status == ^SiteImport.completed(),- select: {i.legacy, i.id},- limit: @max_complete_imports- )- |> Repo.all()-- has_legacy? = Enum.any?(ids, fn {legacy?, _} -> legacy? end)- ids = Enum.map(ids, &elem(&1, 1))-- # account for legacy imports as well- if has_legacy? do- [0 | ids]- else- ids- end- end-- @spec get_imports_date_range(Site.t()) :: %{- start_date: Date.t() | nil,- end_date: Date.t() | nil- }- def get_imports_date_range(site) do- dates =- from(i in SiteImport,- where: i.site_id == ^site.id and i.status == ^SiteImport.completed(),- select: %{start_date: min(i.start_date), end_date: max(i.end_date)}- )- |> Repo.one()-- dates || %{start_date: nil, end_date: nil}- end-@spec delete_imports_for_site(Site.t()) :: :okdef delete_imports_for_site(site) doRepo.delete_all(from i in SiteImport, where: i.site_id == ^site.id)@@ -242,4 +210,10 @@ defmodule Plausible.Imported dodefp in_range?(date, range) doDate.before?(range.first, date) && Date.after?(range.last, date)end++ defp get_completed_imports(site) do+ site+ |> Repo.preload(:completed_imports)+ |> Map.fetch!(:completed_imports)+ endenddiff --git a/lib/plausible/site.ex b/lib/plausible/site.exindex 5ea17a189f..37d0cb2187 100644--- a/lib/plausible/site.ex+++ b/lib/plausible/site.ex@@ -62,13 +62,7 @@ defmodule Plausible.Site dofield :invitations, {:array, :map}, virtual: truefield :pinned_at, :naive_datetime, virtual: true- # Used for caching imports data for the duration of the whole request- # to avoid multiple identical fetches. Populated by plugs putting- # `site` in `assigns`.- field :import_data_loaded, :boolean, default: false, virtual: true- field :earliest_import_start_date, :date, virtual: true- field :latest_import_end_date, :date, virtual: true- field :complete_import_ids, {:array, :integer}, default: [], virtual: true+ has_many :completed_imports, Plausible.Imported.SiteImport, where: [status: :completed]timestamps()enddiff --git a/lib/plausible/sites.ex b/lib/plausible/sites.exindex afc531a590..04af80c24e 100644--- a/lib/plausible/sites.ex+++ b/lib/plausible/sites.ex@@ -231,11 +231,9 @@ defmodule Plausible.Sites doenddef stats_start_date(%Site{} = site) do- site = Plausible.Imported.load_import_data(site)-start_date =[- site.earliest_import_start_date,+ Plausible.Imported.earliest_import_start_date(site),native_stats_start_date(site)]|> Enum.reject(&is_nil/1)diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.exindex b7365f9e55..de4a9ba3d3 100644--- a/lib/plausible/stats/aggregate.ex+++ b/lib/plausible/stats/aggregate.ex@@ -6,7 +6,7 @@ defmodule Plausible.Stats.Aggregate do"""use Plausible.ClickhouseRepo- alias Plausible.Stats.{Query, QueryRunner, QueryOptimizer}+ alias Plausible.Stats.{Query, QueryRunner, QueryResult, QueryOptimizer}def aggregate(site, query, metrics) doQuery.trace(query, metrics)@@ -16,19 +16,20 @@ defmodule Plausible.Stats.Aggregate do|> Query.set(metrics: metrics, remove_unavailable_revenue_metrics: true)|> QueryOptimizer.optimize()- query_result = QueryRunner.run(site, query)+ %QueryResult{results: [entry], meta: meta} = QueryRunner.run(site, query)- [entry] = query_result.results+ results =+ query.metrics+ |> Enum.with_index()+ |> Enum.map(fn {metric, index} ->+ {+ metric,+ metric_map(entry, index, metric)+ }+ end)+ |> Enum.into(%{})- query.metrics- |> Enum.with_index()- |> Enum.map(fn {metric, index} ->- {- metric,- metric_map(entry, index, metric)- }- end)- |> Enum.into(%{})+ %{results: results, meta: meta}enddef metric_map(diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.exindex 3e0699b605..ef70183a58 100644--- a/lib/plausible/stats/breakdown.ex+++ b/lib/plausible/stats/breakdown.ex@@ -8,7 +8,7 @@ defmodule Plausible.Stats.Breakdown douse Plausible.ClickhouseRepouse Plausible.Stats.SQL.Fragments- alias Plausible.Stats.{Query, QueryRunner, QueryOptimizer, Comparisons}+ alias Plausible.Stats.{Query, QueryRunner, QueryResult, QueryOptimizer, Comparisons}def breakdown(site,@@ -37,8 +37,12 @@ defmodule Plausible.Stats.Breakdown do)|> QueryOptimizer.optimize()- QueryRunner.run(site, query_with_metrics)- |> build_breakdown_result(query_with_metrics, metrics)+ %QueryResult{results: results, meta: meta} = QueryRunner.run(site, query_with_metrics)++ %{+ results: build_breakdown_result(results, query_with_metrics, metrics),+ meta: meta+ }enddef formatted_date_ranges(query) do@@ -62,10 +66,10 @@ defmodule Plausible.Stats.Breakdown doendend- defp build_breakdown_result(query_result, query, metrics) do+ defp build_breakdown_result(results, query, metrics) dodimension_keys = query.dimensions |> Enum.map(&result_key/1)- query_result.results+ results|> Enum.map(fn entry ->comparison_map =if entry[:comparison] dodiff --git a/lib/plausible/stats/comparisons.ex b/lib/plausible/stats/comparisons.exindex dff2129317..e560632f19 100644--- a/lib/plausible/stats/comparisons.ex+++ b/lib/plausible/stats/comparisons.ex@@ -175,21 +175,11 @@ defmodule Plausible.Stats.Comparisons dodefp maybe_include_imported(query, source_query) dorequested? = source_query.include.imports+ skip_imported_reason = Query.get_skip_imported_reason(query)- case Query.ensure_include_imported(query, requested?) do- :ok ->- struct!(query,- include_imported: true,- skip_imported_reason: nil,- include: Map.put(query.include, :imports, true)- )-- {:error, reason} ->- struct!(query,- include_imported: false,- skip_imported_reason: reason,- include: Map.put(query.include, :imports, requested?)- )- end+ struct!(query,+ include_imported: requested? and is_nil(skip_imported_reason),+ skip_imported_reason: skip_imported_reason+ )endenddiff --git a/lib/plausible/stats/email_report.ex b/lib/plausible/stats/email_report.exindex 4f9a403ab8..fe95d5b64d 100644--- a/lib/plausible/stats/email_report.ex+++ b/lib/plausible/stats/email_report.ex@@ -19,7 +19,9 @@ defmodule Plausible.Stats.EmailReport dodef get(site, query) dometrics = [:pageviews, :visitors, :bounce_rate]- Stats.aggregate(site, query, metrics)+ %{results: results} = Stats.aggregate(site, query, metrics)++ results|> with_comparisons(site, query, metrics)|> put_top_5_pages(site, query)|> put_top_5_sources(site, query)@@ -27,7 +29,7 @@ defmodule Plausible.Stats.EmailReport dodefp with_comparisons(stats, site, query, metrics) docomparison_query = Comparisons.get_comparison_query(query, %{mode: "previous_period"})- prev_period_stats = Stats.aggregate(site, comparison_query, metrics)+ %{results: prev_period_stats} = Stats.aggregate(site, comparison_query, metrics)stats|> Enum.map(fn {metric, %{value: value}} ->@@ -41,7 +43,7 @@ defmodule Plausible.Stats.EmailReport dodefp put_top_5_pages(stats, site, query) doquery = Query.set(query, dimensions: ["event:page"])- pages = Stats.breakdown(site, query, [:visitors], {5, 1})+ %{results: pages} = Stats.breakdown(site, query, [:visitors], {5, 1})Map.put(stats, :pages, pages)end@@ -51,7 +53,7 @@ defmodule Plausible.Stats.EmailReport do|> Query.add_filter([:is_not, "visit:source", ["Direct / None"]])|> Query.set(dimensions: ["visit:source"])- sources = Stats.breakdown(site, query, [:visitors], {5, 1})+ %{results: sources} = Stats.breakdown(site, query, [:visitors], {5, 1})Map.put(stats, :sources, sources)enddiff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.exindex 574cd38362..751761ac45 100644--- a/lib/plausible/stats/filters/query_parser.ex+++ b/lib/plausible/stats/filters/query_parser.ex@@ -7,6 +7,11 @@ defmodule Plausible.Stats.Filters.QueryParser do@default_include %{imports: false,+ # `include.imports_meta` can be true even when `include.imports`+ # is false. Even if we don't want to include imported data, we+ # might still want to know whether imported data can be toggled+ # on/off on the dashboard.+ imports_meta: false,time_labels: false,total_rows: false,comparisons: nildiff --git a/lib/plausible/stats/goal_suggestions.ex b/lib/plausible/stats/goal_suggestions.exindex f0ca5a5eb3..b9cea831e4 100644--- a/lib/plausible/stats/goal_suggestions.ex+++ b/lib/plausible/stats/goal_suggestions.ex@@ -29,7 +29,6 @@ defmodule Plausible.Stats.GoalSuggestions dosite =site|> Repo.preload(:goals)- |> Plausible.Imported.load_import_data()excluded =opts@@ -62,7 +61,7 @@ defmodule Plausible.Stats.GoalSuggestions doimported_q =from(i in "imported_custom_events",where: i.site_id == ^site.id,- where: i.import_id in ^site.complete_import_ids,+ where: i.import_id in ^Plausible.Imported.complete_import_ids(site),where: i.date >= ^date_range.first and i.date <= ^date_range.last,where: i.visitors > 0,where: fragment("? ilike ?", i.name, ^matches),diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.exindex a27d6cb6f3..4346f4d71f 100644--- a/lib/plausible/stats/imported/base.ex+++ b/lib/plausible/stats/imported/base.ex@@ -60,7 +60,7 @@ defmodule Plausible.Stats.Imported.Base doenddef query_imported(table, site, query) do- import_ids = site.complete_import_ids+ import_ids = Imported.complete_import_ids(site)# Assumption: dates in imported table are in user-local timezone.%{first: date_from, last: date_to} = Query.date_range(query)diff --git a/lib/plausible/stats/legacy/legacy_query_builder.ex b/lib/plausible/stats/legacy/legacy_query_builder.exindex fe81589b34..73fd2603bc 100644--- a/lib/plausible/stats/legacy/legacy_query_builder.ex+++ b/lib/plausible/stats/legacy/legacy_query_builder.ex@@ -26,8 +26,8 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do|> put_parsed_filters(params)|> preload_goals_and_revenue(site)|> put_order_by(params)- |> put_include_comparisons(site, params)- |> Query.put_imported_opts(site, params)+ |> put_include(site, params)+ |> Query.put_imported_opts(site)on_ee doquery = Plausible.Stats.Sampling.put_threshold(query, site, params)@@ -205,9 +205,10 @@ defmodule Plausible.Stats.Legacy.QueryBuilder doendend- defp put_include_comparisons(query, site, params) do- comparisons = parse_comparison_params(site, params)- struct!(query, include: Map.put(query.include, :comparisons, comparisons))+ defp put_include(query, site, params) do+ query+ |> Query.set_include(:comparisons, parse_comparison_params(site, params))+ |> Query.set_include(:imports, params["with_imported"] == "true")end@doc """diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.exindex 424fc07aea..6634919404 100644--- a/lib/plausible/stats/query.ex+++ b/lib/plausible/stats/query.ex@@ -35,7 +35,7 @@ defmodule Plausible.Stats.Query dowith {:ok, query_data} <- Filters.QueryParser.parse(site, schema_type, params) doquery =struct!(__MODULE__, Map.to_list(query_data))- |> put_imported_opts(site, %{})+ |> put_imported_opts(site)|> struct!(now: DateTime.utc_now(:second),debug_metadata: debug_metadata,@@ -120,61 +120,48 @@ defmodule Plausible.Stats.Query doenddefp refresh_imported_opts(query) do- put_imported_opts(query, nil, %{})+ put_imported_opts(query, nil)end- def put_imported_opts(query, site, params) do- requested? = params["with_imported"] == "true" || query.include.imports+ def put_imported_opts(query, site) do+ requested? = query.include.importslatest_import_end_date =if site do- site.latest_import_end_date+ Plausible.Imported.latest_import_end_date(site)elsequery.latest_import_end_dateendquery = struct!(query, latest_import_end_date: latest_import_end_date)- case ensure_include_imported(query, requested?) do- :ok ->- struct!(query,- include_imported: true,- include: Map.put(query.include, :imports, true)- )+ skip_imported_reason = get_skip_imported_reason(query)- {:error, reason} ->- struct!(query,- include_imported: false,- skip_imported_reason: reason,- include: Map.put(query.include, :imports, requested?)- )- end+ struct!(query,+ include_imported: requested? and is_nil(skip_imported_reason),+ skip_imported_reason: skip_imported_reason+ )end- @spec ensure_include_imported(t(), boolean()) ::- :ok | {:error, :no_imported_data | :out_of_range | :unsupported_query | :not_requested}- def ensure_include_imported(query, requested?) do+ @spec get_skip_imported_reason(t()) ::+ nil | :no_imported_data | :out_of_range | :unsupported_query+ def get_skip_imported_reason(query) docond do- not requested? ->- {:error, :not_requested}-is_nil(query.latest_import_end_date) ->- {:error, :no_imported_data}+ :no_imported_data- query.period in ["realtime", "30m"] ->- {:error, :unsupported_query}+ query.period in ["realtime", "30m"] or+ Date.after?(date_range(query).first, query.latest_import_end_date) ->+ :out_of_range"time:minute" in query.dimensions or "time:hour" in query.dimensions ->- {:error, :unsupported_interval}-- Date.after?(date_range(query).first, query.latest_import_end_date) ->- {:error, :out_of_range}+ :unsupported_intervalnot Imported.schema_supports_query?(query) ->- {:error, :unsupported_query}+ :unsupported_querytrue ->- :ok+ nilendenddiff --git a/lib/plausible/stats/query_result.ex b/lib/plausible/stats/query_result.exindex 750097ecd7..bc3bc7393c 100644--- a/lib/plausible/stats/query_result.ex+++ b/lib/plausible/stats/query_result.ex@@ -8,7 +8,7 @@ defmodule Plausible.Stats.QueryResult do"""use Plausible- alias Plausible.Stats.DateTimeRange+ alias Plausible.Stats.{DateTimeRange, Query, QueryRunner}defstruct results: [],meta: %{},@@ -19,11 +19,11 @@ defmodule Plausible.Stats.QueryResult do`results` should already-built by Plausible.Stats.QueryRunner"""- def from(results, site, query, meta_extra) do+ def from(%QueryRunner{site: site, main_query: query, results: results} = runner) dostruct!(__MODULE__,results: results,- meta: meta(query, meta_extra),+ meta: meta(runner) |> Enum.sort_by(&elem(&1, 0)) |> Jason.OrderedObject.new(),query:Jason.OrderedObject.new(site_id: site.domain,@@ -41,6 +41,14 @@ defmodule Plausible.Stats.QueryResult do)end+ defp meta(%QueryRunner{} = runner) do+ %{}+ |> add_imports_meta(runner)+ |> add_metric_warnings_meta(runner)+ |> add_time_labels_meta(runner.main_query)+ |> add_total_rows_meta(runner.main_query, runner.total_rows)+ end+@imports_warnings %{unsupported_query:"Imported stats are not included in the results because query parameters are not supported. " <>@@ -49,21 +57,62 @@ defmodule Plausible.Stats.QueryResult do"Imported stats are not included because the time dimension (i.e. the interval) is too short."}- defp meta(query, meta_extra) do- %{- imports_included: if(query.include.imports, do: query.include_imported, else: nil),- imports_skip_reason:- if(query.include.imports and query.skip_imported_reason,- do: to_string(query.skip_imported_reason)- ),- imports_warning: @imports_warnings[query.skip_imported_reason],- metric_warnings: metric_warnings(query),- time_labels:- if(query.include.time_labels, do: Plausible.Stats.Time.time_labels(query), else: nil),- total_rows: if(query.include.total_rows, do: meta_extra.total_rows, else: nil)- }- |> Enum.reject(fn {_, value} -> is_nil(value) end)- |> Map.new()+ defp add_imports_meta(meta, %QueryRunner{} = runner) do+ %{main_query: %{include: include} = main_query} = runner++ if include.imports or include[:imports_meta] do+ comparison_query = Map.get(runner, :comparison_query)++ imports_included =+ case comparison_query do+ %Query{include_imported: true} -> true+ _ -> main_query.include_imported+ end++ imports_skip_reason =+ case comparison_query do+ %Query{skip_imported_reason: nil} -> nil+ _ -> main_query.skip_imported_reason+ end++ imports_warning = @imports_warnings[imports_skip_reason]++ %{+ imports_included: imports_included,+ imports_skip_reason: imports_skip_reason,+ imports_warning: imports_warning+ }+ |> Map.reject(fn {_key, value} -> is_nil(value) end)+ |> Map.merge(meta)+ else+ meta+ end+ end++ defp add_metric_warnings_meta(meta, %QueryRunner{main_query: query}) do+ warnings = metric_warnings(query)++ if map_size(warnings) > 0 do+ Map.put(meta, :metric_warnings, warnings)+ else+ meta+ end+ end++ defp add_time_labels_meta(meta, query) do+ if query.include.time_labels do+ Map.put(meta, :time_labels, Plausible.Stats.Time.time_labels(query))+ else+ meta+ end+ end++ defp add_total_rows_meta(meta, query, total_rows) do+ if query.include.total_rows do+ Map.put(meta, :total_rows, total_rows)+ else+ meta+ endenddefp include(query) do@@ -80,7 +129,18 @@ defmodule Plausible.Stats.QueryResult doendend+ defp metric_warnings(query) do+ Enum.reduce(query.metrics, %{}, fn metric, acc ->+ case metric_warning(metric, query) do+ nil -> acc+ %{} = warning -> Map.put(acc, metric, warning)+ end+ end)+ end+on_ee do+ @revenue_metrics Plausible.Stats.Goal.Revenue.revenue_metrics()+@revenue_metrics_warnings %{revenue_goals_unavailable:"The owner of this site does not have access to the revenue metrics feature.",@@ -90,26 +150,20 @@ defmodule Plausible.Stats.QueryResult do"Revenue metrics are null as there are no matching revenue goals."}- defp metric_warnings(query) do+ defp metric_warning(metric, query) when metric in @revenue_metrics doif query.revenue_warning do- query.metrics- |> Enum.filter(&(&1 in Plausible.Stats.Goal.Revenue.revenue_metrics()))- |> Enum.map(- &{&1,- %{- code: query.revenue_warning,- warning: @revenue_metrics_warnings[query.revenue_warning]- }}- )- |> Map.new()+ %{+ code: query.revenue_warning,+ warning: @revenue_metrics_warnings[query.revenue_warning]+ }elsenilendend- else- defp metric_warnings(_query), do: nilend+ defp metric_warning(_metric, _query), do: nil+defp to_iso8601(datetime, timezone) dodatetime|> DateTime.shift_zone!(timezone)diff --git a/lib/plausible/stats/query_runner.ex b/lib/plausible/stats/query_runner.exindex 6af06f2154..a4abbce93b 100644--- a/lib/plausible/stats/query_runner.ex+++ b/lib/plausible/stats/query_runner.ex@@ -6,7 +6,7 @@ defmodule Plausible.Stats.QueryRunner doSome secondary responsibilities are:1. Dealing with comparison queries and combining results with main2. Dealing with time-on-page- 3. Passing total_rows from clickhouse to QueryResult meta+ 3. Getting total_rows from ClickHouse results"""use Plausible@@ -25,58 +25,54 @@ defmodule Plausible.Stats.QueryRunner do}defstruct [- :query,:site,+ :main_query,+ :main_results,:comparison_query,:comparison_results,- :main_results_list,- :ch_results,- :meta_extra,- :time_lookup,- :results_list+ :total_rows,+ :results]def run(site, query) dooptimized_query = QueryOptimizer.optimize(query)- run_results =- %__MODULE__{query: optimized_query, site: site}- |> execute_main_query()- |> add_comparison_query()- |> execute_comparison()- |> add_meta_extra()- |> add_time_lookup()- |> build_results_list()-- QueryResult.from(run_results.results_list, site, optimized_query, run_results.meta_extra)+ %__MODULE__{main_query: optimized_query, site: site}+ |> execute_main_query()+ |> add_comparison_query()+ |> execute_comparison_query()+ |> build_results_list()+ |> QueryResult.from()end- defp execute_main_query(%__MODULE__{query: query, site: site} = run_results) do+ defp execute_main_query(%__MODULE__{main_query: query, site: site} = runner) do{ch_results, time_on_page} = execute_query(query, site)- struct!(- run_results,- main_results_list: build_from_ch(ch_results, query, time_on_page),- ch_results: ch_results- )+ main_results = build_from_ch(ch_results, query, time_on_page)++ runner = struct!(runner, main_results: main_results)++ if query.include.total_rows do+ struct!(runner, total_rows: total_rows(ch_results))+ else+ runner+ endend- defp add_comparison_query(- %__MODULE__{query: query, main_results_list: main_results_list} = run_results- )+ defp add_comparison_query(%__MODULE__{main_query: query, main_results: main_results} = runner)when is_map(query.include.comparisons) docomparison_query =query|> Comparisons.get_comparison_query(query.include.comparisons)- |> Comparisons.add_comparison_filters(main_results_list)+ |> Comparisons.add_comparison_filters(main_results)- struct!(run_results, comparison_query: comparison_query)+ struct!(runner, comparison_query: comparison_query)end- defp add_comparison_query(run_results), do: run_results+ defp add_comparison_query(runner), do: runner- defp execute_comparison(- %__MODULE__{comparison_query: comparison_query, site: site} = run_results+ defp execute_comparison_query(+ %__MODULE__{comparison_query: comparison_query, site: site} = runner) doif comparison_query do{ch_results, time_on_page} = execute_query(comparison_query, site)@@ -88,46 +84,33 @@ defmodule Plausible.Stats.QueryRunner dotime_on_page)- struct!(run_results, comparison_results: comparison_results)+ struct!(runner, comparison_results: comparison_results)else- run_results+ runnerendend- defp add_time_lookup(run_results) do- time_lookup =- if Time.time_dimension(run_results.query) && run_results.comparison_query do- Enum.zip(- Time.time_labels(run_results.query),- Time.time_labels(run_results.comparison_query)- )- |> Map.new()- else- %{}- end-- struct!(run_results, time_lookup: time_lookup)- end-- defp add_meta_extra(%__MODULE__{query: query, ch_results: ch_results} = run_results) do- struct!(run_results,- meta_extra: %{- total_rows: if(query.include.total_rows, do: total_rows(ch_results), else: nil)- }- )+ defp get_time_lookup(query, comparison_query) do+ if Time.time_dimension(query) && comparison_query do+ Enum.zip(+ Time.time_labels(query),+ Time.time_labels(comparison_query)+ )+ |> Map.new()+ else+ %{}+ endend- defp build_results_list(- %__MODULE__{query: query, main_results_list: main_results_list} = run_results- ) do- results_list =+ defp build_results_list(%__MODULE__{main_query: query, main_results: main_results} = runner) do+ results =case query.dimensions do- ["time:" <> _] -> main_results_list |> add_empty_timeseries_rows(run_results)- _ -> main_results_list+ ["time:" <> _] -> main_results |> add_empty_timeseries_rows(runner)+ _ -> main_resultsend- |> merge_with_comparison_results(run_results)+ |> merge_with_comparison_results(runner)- struct!(run_results, results_list: results_list)+ struct!(runner, results: results)enddefp execute_query(query, site) do@@ -191,7 +174,7 @@ defmodule Plausible.Stats.QueryRunner do# Special case: If comparison and single time dimension, add 0 rows - otherwise# comparisons would not be shown for timeseries with 0 values.- defp add_empty_timeseries_rows(results_list, %__MODULE__{query: query})+ defp add_empty_timeseries_rows(results_list, %__MODULE__{main_query: query})when is_map(query.include.comparisons) doindexed_results = index_by_dimensions(results_list)@@ -210,13 +193,13 @@ defmodule Plausible.Stats.QueryRunner dodefp add_empty_timeseries_rows(results_list, _), do: results_list- defp merge_with_comparison_results(results_list, run_results) do- comparison_map = (run_results.comparison_results || []) |> index_by_dimensions()- time_lookup = run_results.time_lookup || %{}+ defp merge_with_comparison_results(results_list, runner) do+ comparison_map = (runner.comparison_results || []) |> index_by_dimensions()+ time_lookup = get_time_lookup(runner.main_query, runner.comparison_query)Enum.map(results_list,- &add_comparison_results(&1, run_results.query, comparison_map, time_lookup)+ &add_comparison_results(&1, runner.main_query, comparison_map, time_lookup))enddiff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.exindex 8e121b8175..a585a71bd2 100644--- a/lib/plausible_web/controllers/api/external_stats_controller.ex+++ b/lib/plausible_web/controllers/api/external_stats_controller.ex@@ -2,7 +2,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController douse PlausibleWeb, :controlleruse Plausible.Repouse PlausibleWeb.Plugs.ErrorHandler- alias Plausible.Stats.{Query, Compare, Comparisons, Metrics, Filters}+ alias Plausible.Stats.{Query, Metrics, Filters}def realtime_visitors(conn, _params) dosite = conn.assigns.site@@ -20,28 +20,16 @@ defmodule PlausibleWeb.Api.ExternalStatsController do:ok <- validate_filters(site, query.filters),{:ok, metrics} <- parse_and_validate_metrics(params, query),:ok <- ensure_custom_props_access(site, query) do- results =+ query =if params["compare"] == "previous_period" do- comparison_query = Comparisons.get_comparison_query(query, %{mode: "previous_period"})-- [prev_result, curr_result] =- Plausible.ClickhouseRepo.parallel_tasks([- fn -> Plausible.Stats.aggregate(site, comparison_query, metrics) end,- fn -> Plausible.Stats.aggregate(site, query, metrics) end- ])-- Enum.map(curr_result, fn {metric, %{value: current_val}} ->- %{value: prev_val} = prev_result[metric]- change = Compare.calculate_change(metric, prev_val, current_val)-- {metric, %{value: current_val, change: change}}- end)- |> Enum.into(%{})+ Query.set_include(query, :comparisons, %{mode: "previous_period"})else- Plausible.Stats.aggregate(site, query, metrics)+ queryend- payload = maybe_add_warning(%{results: results}, query)+ %{results: results, meta: meta} = Plausible.Stats.aggregate(site, query, metrics)++ payload = maybe_add_warning(%{results: results}, meta)json(conn, payload)else@@ -61,8 +49,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController do{:ok, limit} <- validate_or_default_limit(params),:ok <- ensure_custom_props_access(site, query) dopage = String.to_integer(Map.get(params, "page", "1"))- results = Plausible.Stats.breakdown(site, query, metrics, {limit, page})- payload = maybe_add_warning(%{results: results}, query)++ %{results: results, meta: meta} =+ Plausible.Stats.breakdown(site, query, metrics, {limit, page})++ payload = maybe_add_warning(%{results: results}, meta)json(conn, payload)else@@ -387,18 +378,13 @@ defmodule PlausibleWeb.Api.ExternalStatsController do"The goal `#{goal}` is not configured for this site. "end- defp maybe_add_warning(payload, %{skip_imported_reason: reason})- when reason in [nil, :not_requested, :no_imported_data, :out_of_range, :manual_exclusion] do- payload- end+ @imported_query_unsupported_warning "Imported stats are not included in the results because query parameters are not supported. For more information, see: https://plausible.io/docs/stats-api#filtering-imported-stats"- defp maybe_add_warning(payload, %{skip_imported_reason: :unsupported_query}) do- Map.put(- payload,- :warning,- "Imported stats are not included in the results because query parameters are not supported. " <>- "For more information, see: https://plausible.io/docs/stats-api#filtering-imported-stats"- )+ defp maybe_add_warning(payload, %Jason.OrderedObject{} = meta) do+ case meta[:imports_skip_reason] do+ :unsupported_query -> Map.put(payload, :warning, @imported_query_unsupported_warning)+ _ -> payload+ endenddefp send_json_error_response(conn, {:error, {status, msg}}) dodiff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex 66448d79a8..158da790e9 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -68,9 +68,6 @@ defmodule PlausibleWeb.Api.StatsController do* `includes_imported` - boolean indicating whether imported datawas queried or not.- * `imports_exist` - boolean indicating whether there are any completed- imports for a given site or not.-* `full_intervals` - map of dates indicating whether the interval has beencut off by the requested date range or not. For example, if looking at amonth week-by-week, some weeks may be cut off by the month boundaries.@@ -87,7 +84,6 @@ defmodule PlausibleWeb.Api.StatsController do"2021-11-01" => true,"2021-12-01" => false},- "imports_exist" => false,"interval" => "month","labels" => ["2021-09-01", "2021-10-01", "2021-11-01", "2021-12-01"],"plot" => [0, 0, 0, 0],@@ -196,18 +192,25 @@ defmodule PlausibleWeb.Api.StatsController doparams = realtime_period_to_30m(params)- query = Query.from(site, params, debug_metadata(conn))+ query =+ site+ |> Query.from(params, debug_metadata(conn))+ |> Query.set_include(:imports_meta, true)++ %{+ top_stats: top_stats,+ meta: meta,+ sample_percent: sample_percent+ } = fetch_top_stats(site, query, current_user)- {top_stats, sample_percent} = fetch_top_stats(site, query, current_user)comparison_query = comparison_query(query)json(conn, %{top_stats: top_stats,interval: query.interval,sample_percent: sample_percent,- with_imported_switch: with_imported_switch_info(query, comparison_query),- includes_imported: includes_imported?(query, comparison_query),- imports_exist: site.complete_import_ids != [],+ with_imported_switch: with_imported_switch_info(meta),+ includes_imported: meta[:imports_included] == true,comparing_from: query.include.comparisons && Query.date_range(comparison_query).first,comparing_to: query.include.comparisons && Query.date_range(comparison_query).last,from: Query.date_range(query).first,@@ -215,35 +218,19 @@ defmodule PlausibleWeb.Api.StatsController do})end- defp with_imported_switch_info(%Query{period: "30m"}, _) do- %{visible: false, togglable: false, tooltip_msg: nil}- end-- defp with_imported_switch_info(query, nil) do- with_imported_switch_info(query.skip_imported_reason)- end-- defp with_imported_switch_info(query, comparison_query) do- case {query.skip_imported_reason, comparison_query.skip_imported_reason} do- {:out_of_range, nil} -> with_imported_switch_info(nil)- {:out_of_range, :not_requested} -> with_imported_switch_info(:not_requested)- {reason, _} -> with_imported_switch_info(reason)- end- end+ defp with_imported_switch_info(%Jason.OrderedObject{} = meta) do+ case {meta[:imports_included], meta[:imports_skip_reason]} do+ {true, nil} ->+ %{visible: true, togglable: true, tooltip_msg: "Click to exclude imported data"}- defp with_imported_switch_info(skip_reason) do- case skip_reason do- reason when reason in [:no_imported_data, :out_of_range] ->- %{visible: false, togglable: false, tooltip_msg: nil}+ {false, nil} ->+ %{visible: true, togglable: true, tooltip_msg: "Click to include imported data"}- :unsupported_query ->+ {false, :unsupported_query} ->%{visible: true, togglable: false, tooltip_msg: "Imported data cannot be included"}- :not_requested ->- %{visible: true, togglable: true, tooltip_msg: "Click to include imported data"}-- nil ->- %{visible: true, togglable: true, tooltip_msg: "Click to exclude imported data"}+ {false, reason} when reason in [:no_imported_data, :out_of_range] ->+ %{visible: false, togglable: false, tooltip_msg: nil}endend@@ -315,11 +302,14 @@ defmodule PlausibleWeb.Api.StatsController doquery = Query.set_include(query, :comparisons, nil)%{- visitors: %{value: unique_conversions},- events: %{value: total_conversions}+ results: %{+ visitors: %{value: unique_conversions},+ events: %{value: total_conversions}+ },+ meta: meta} = Stats.aggregate(site, query, [:visitors, :events])- stats = [+ top_stats = [%{name: "Current visitors",graph_metric: :current_visitors,@@ -337,18 +327,21 @@ defmodule PlausibleWeb.Api.StatsController do}]- {stats, 100}+ %{top_stats: top_stats, meta: meta, sample_percent: 100}enddefp fetch_realtime_top_stats(site, query) doquery = Query.set_include(query, :comparisons, nil)%{- visitors: %{value: visitors},- pageviews: %{value: pageviews}+ results: %{+ visitors: %{value: visitors},+ pageviews: %{value: pageviews}+ },+ meta: meta} = Stats.aggregate(site, query, [:visitors, :pageviews])- stats = [+ top_stats = [%{name: "Current visitors",graph_metric: :current_visitors,@@ -366,29 +359,31 @@ defmodule PlausibleWeb.Api.StatsController do}]- {stats, 100}+ %{top_stats: top_stats, meta: meta, sample_percent: 100}enddefp fetch_goal_top_stats(site, query) dometrics =[:total_visitors, :visitors, :events, :conversion_rate] ++ @revenue_metrics- results = Stats.aggregate(site, query, metrics)-- [- top_stats_entry(results, "Unique visitors", :total_visitors),- top_stats_entry(results, "Unique conversions", :visitors),- top_stats_entry(results, "Total conversions", :events),- on_ee do- top_stats_entry(results, "Average revenue", :average_revenue)- end,- on_ee do- top_stats_entry(results, "Total revenue", :total_revenue)- end,- top_stats_entry(results, "Conversion rate", :conversion_rate)- ]- |> Enum.reject(&is_nil/1)- |> then(&{&1, 100})+ %{results: results, meta: meta} = Stats.aggregate(site, query, metrics)++ top_stats =+ [+ top_stats_entry(results, "Unique visitors", :total_visitors),+ top_stats_entry(results, "Unique conversions", :visitors),+ top_stats_entry(results, "Total conversions", :events),+ on_ee do+ top_stats_entry(results, "Average revenue", :average_revenue)+ end,+ on_ee do+ top_stats_entry(results, "Total revenue", :total_revenue)+ end,+ top_stats_entry(results, "Conversion rate", :conversion_rate)+ ]+ |> Enum.reject(&is_nil/1)++ %{top_stats: top_stats, meta: meta, sample_percent: 100}enddefp fetch_other_top_stats(site, query, current_user) do@@ -417,27 +412,29 @@ defmodule PlausibleWeb.Api.StatsController dometrics ++ [:views_per_visit, :bounce_rate, :visit_duration]end- current_results = Stats.aggregate(site, query, metrics)+ %{results: results, meta: meta} = Stats.aggregate(site, query, metrics)- stats =+ top_stats =[- top_stats_entry(current_results, "Unique visitors", :visitors),- top_stats_entry(current_results, "Total visits", :visits),- top_stats_entry(current_results, "Total pageviews", :pageviews),- top_stats_entry(current_results, "Views per visit", :views_per_visit),- top_stats_entry(current_results, "Bounce rate", :bounce_rate),- top_stats_entry(current_results, "Visit duration", :visit_duration),- top_stats_entry(current_results, "Time on page", :time_on_page,+ top_stats_entry(results, "Unique visitors", :visitors),+ top_stats_entry(results, "Total visits", :visits),+ top_stats_entry(results, "Total pageviews", :pageviews),+ top_stats_entry(results, "Views per visit", :views_per_visit),+ top_stats_entry(results, "Bounce rate", :bounce_rate),+ top_stats_entry(results, "Visit duration", :visit_duration),+ top_stats_entry(results, "Time on page", :time_on_page,formatter: fnnil -> 0value -> valueend),- top_stats_entry(current_results, "Scroll depth", :scroll_depth)+ top_stats_entry(results, "Scroll depth", :scroll_depth)]|> Enum.filter(& &1)- {stats, current_results[:sample_percent][:value]}+ sample_percent = results[:sample_percent][:value]++ %{top_stats: top_stats, meta: meta, sample_percent: sample_percent}enddefp top_stats_entry(current_results, name, key, opts \\ []) do@@ -474,8 +471,10 @@ defmodule PlausibleWeb.Api.StatsController dometrics = breakdown_metrics(query, extra_metrics)+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)+res =- Stats.breakdown(site, query, metrics, pagination)+ results|> transform_keys(%{source: :name})if params["csv"] do@@ -490,7 +489,7 @@ defmodule PlausibleWeb.Api.StatsController dojson(conn, %{results: res,meta: Stats.Breakdown.formatted_date_ranges(query),- skip_imported_reason: query.skip_imported_reason+ skip_imported_reason: meta[:imports_skip_reason]})endend@@ -506,8 +505,10 @@ defmodule PlausibleWeb.Api.StatsController dometrics = breakdown_metrics(query, extra_metrics)+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)+res =- Stats.breakdown(site, query, metrics, pagination)+ results|> transform_keys(%{channel: :name})if params["csv"] do@@ -522,7 +523,7 @@ defmodule PlausibleWeb.Api.StatsController dojson(conn, %{results: res,meta: Stats.Breakdown.formatted_date_ranges(query),- skip_imported_reason: query.skip_imported_reason+ skip_imported_reason: meta[:imports_skip_reason]})endend@@ -588,8 +589,10 @@ defmodule PlausibleWeb.Api.StatsController dopagination = parse_pagination(params)metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)+res =- Stats.breakdown(site, query, metrics, pagination)+ results|> transform_keys(%{utm_medium: :name})if params["csv"] do@@ -604,7 +607,7 @@ defmodule PlausibleWeb.Api.StatsController dojson(conn, %{results: res,meta: Stats.Breakdown.formatted_date_ranges(query),- skip_imported_reason: query.skip_imported_reason+ skip_imported_reason: meta[:imports_skip_reason]})endend@@ -616,8 +619,10 @@ defmodule PlausibleWeb.Api.StatsController dopagination = parse_pagination(params)metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)+res =- Stats.breakdown(site, query, metrics, pagination)+ results|> transform_keys(%{utm_campaign: :name})if params["csv"] do@@ -632,7 +637,7 @@ defmodule PlausibleWeb.Api.StatsController dojson(conn, %{results: res,meta: Stats.Breakdown.formatted_date_ranges(query),- skip_imported_reason: query.skip_imported_reason+ skip_imported_reason: meta[:imports_skip_reason]})endend@@ -644,8 +649,10 @@ defmodule PlausibleWeb.Api.StatsController dopagination = parse_pagination(params)metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)+res =- Stats.breakdown(site, query, metrics, pagination)+ results|> transform_keys(%{utm_content: :name})if params["csv"] do@@ -660,7 +667,7 @@ defmodule PlausibleWeb.Api.StatsController dojson(conn, %{results: res,meta: Stats.Breakdown.formatted_date_ranges(query),- skip_imported_reason: query.skip_imported_reason+ skip_imported_reason: meta[:imports_skip_reason]})endend@@ -672,8 +679,10 @@ defmodule PlausibleWeb.Api.StatsController dopagination = parse_pagination(params)metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)+res =- Stats.breakdown(site, query, metrics, pagination)+ results|> transform_keys(%{utm_term: :name})if params["csv"] do@@ -688,7 +697,7 @@ defmodule PlausibleWeb.Api.StatsController dojson(conn, %{results: res,meta: Stats.Breakdown.formatted_date_ranges(query),- skip_imported_reason: query.skip_imported_reason+ skip_imported_reason: meta[:imports_skip_reason]})endend@@ -700,8 +709,10 @@ defmodule PlausibleWeb.Api.StatsController dopagination = parse_pagination(params)metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)+res =- Stats.breakdown(site, query, metrics, pagination)+ results|> transform_keys(%{utm_source: :name})if params["csv"] do@@ -716,7 +727,7 @@ defmodule PlausibleWeb.Api.StatsController dojson(conn, %{results: res,meta: Stats.Breakdown.formatted_date_ranges(query),- skip_imported_reason: query.skip_imported_reason+ skip_imported_reason: meta[:imports_skip_reason]})endend@@ -728,8 +739,10 @@ defmodule PlausibleWeb.Api.StatsController dopagination = parse_pagination(params)metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)+res =- Stats.breakdown(site, query, metrics, pagination)+ results|> transform_keys(%{referrer: :name})if params["csv"] do@@ -744,7 +757,7 @@ defmodule PlausibleWeb.Api.StatsController dojson(conn, %{results: res,meta: Stats.Breakdown.formatted_date_ranges(query),- skip_imported_reason: query.skip_imported_reason+ skip_imported_reason: meta[:imports_skip_reason]})endend@@ -819,14 +832,16 @@ defmodule PlausibleWeb.Api.StatsController dometrics = breakdown_metrics(query, extra_metrics)+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)+referrers =- Stats.breakdown(site, query, metrics, pagination)+ results|> transform_keys(%{referrer: :name})json(conn, %{results: referrers,meta: Stats.Breakdown.formatted_date_ranges(query),- skip_imported_reason: query.skip_imported_reason+ skip_imported_reason: meta[:imports_skip_reason]})end@@ -854,8 +869,10 @@ defmodule PlausibleWeb.Api.StatsController dometrics = breakdown_metrics(query, extra_metrics)pagination = parse_pagination(params)+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)+pages =- Stats.breakdown(site, query, metrics, pagination)+ results|> transform_keys(%{page: :name})if params["csv"] do@@ -875,7 +892,7 @@ defmodule PlausibleWeb.Api.StatsController dojson(conn, %{results: pages,meta: Stats.Breakdown.formatted_date_ranges(query),- skip_imported_reason: query.skip_imported_reason+ skip_imported_reason: meta[:imports_skip_reason]})endend@@ -887,8 +904,10 @@ defmodule PlausibleWeb.Api.StatsController dopagination = parse_pagination(params)metrics = breakdown_metrics(query, [:visits, :visit_duration])+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)+entry_pages =- Stats.breakdown(site, query, metrics, pagination)+ results|> transform_keys(%{entry_page: :name})if params["csv"] do@@ -910,7 +929,7 @@ defmodule PlausibleWeb.Api.StatsController dojson(conn, %{results: entry_pages,meta: Stats.Breakdown.formatted_date_ranges(query),- skip_imported_reason: query.skip_imported_reason+ skip_imported_reason: meta[:imports_skip_reason]})endend@@ -922,8 +941,10 @@ defmodule PlausibleWeb.Api.StatsController do{limit, page} = parse_pagination(params)metrics = breakdown_metrics(query, [:visits])+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, {limit, page})+exit_pages =- Stats.breakdown(site, query, metrics, {limit, page})+ results|> add_exit_rate(site, query, limit)|> transform_keys(%{exit_page: :name})@@ -946,7 +967,7 @@ defmodule PlausibleWeb.Api.StatsController dojson(conn, %{results: exit_pages,meta: Stats.Breakdown.formatted_date_ranges(query),- skip_imported_reason: query.skip_imported_reason+ skip_imported_reason: meta[:imports_skip_reason]})endend@@ -964,7 +985,7 @@ defmodule PlausibleWeb.Api.StatsController do|> Query.add_filter([:is, "event:page", pages])|> Query.set(dimensions: ["event:page"])- total_pageviews =+ %{results: total_pageviews} =Stats.breakdown(site, total_pageviews_query, [:pageviews], {limit, 1})Enum.map(breakdown_results, fn result ->@@ -989,8 +1010,10 @@ defmodule PlausibleWeb.Api.StatsController dopagination = parse_pagination(params)metrics = breakdown_metrics(query, [:percentage])+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)+countries =- Stats.breakdown(site, query, metrics, pagination)+ results|> transform_keys(%{country: :code})if params["csv"] do@@ -1033,7 +1056,7 @@ defmodule PlausibleWeb.Api.StatsController dojson(conn, %{results: countries,meta: Stats.Breakdown.formatted_date_ranges(query),- skip_imported_reason: query.skip_imported_reason+ skip_imported_reason: meta[:imports_skip_reason]})endend@@ -1045,8 +1068,10 @@ defmodule PlausibleWeb.Api.StatsController dopagination = parse_pagination(params)metrics = breakdown_metrics(query)+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)+regions =- Stats.breakdown(site, query, metrics, pagination)+ results|> transform_keys(%{region: :code})|> Enum.map(fn region ->region_entry = Location.get_subdivision(region[:code])@@ -1072,7 +1097,7 @@ defmodule PlausibleWeb.Api.StatsController dojson(conn, %{results: regions,meta: Stats.Breakdown.formatted_date_ranges(query),- skip_imported_reason: query.skip_imported_reason+ skip_imported_reason: meta[:imports_skip_reason]})endend@@ -1084,8 +1109,10 @@ defmodule PlausibleWeb.Api.StatsController dopagination = parse_pagination(params)metrics = breakdown_metrics(query)+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)+cities =- Stats.breakdown(site, query, metrics, pagination)+ results|> transform_keys(%{city: :code})|> Enum.map(fn city ->city_info = Location.get_city(city[:code])@@ -1116,7 +1143,7 @@ defmodule PlausibleWeb.Api.StatsController dojson(conn, %{results: cities,meta: Stats.Breakdown.formatted_date_ranges(query),- skip_imported_reason: query.skip_imported_reason+ skip_imported_reason: meta[:imports_skip_reason]})endend@@ -1132,8 +1159,10 @@ defmodule PlausibleWeb.Api.StatsController dometrics = breakdown_metrics(query, extra_metrics ++ [:percentage])+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)+browsers =- Stats.breakdown(site, query, metrics, pagination)+ results|> transform_keys(%{browser: :name})if params["csv"] do@@ -1148,7 +1177,7 @@ defmodule PlausibleWeb.Api.StatsController dojson(conn, %{results: browsers,meta: Stats.Breakdown.formatted_date_ranges(query),- skip_imported_reason: query.skip_imported_reason+ skip_imported_reason: meta[:imports_skip_reason]})endend@@ -1164,8 +1193,10 @@ defmodule PlausibleWeb.Api.StatsController dometrics = breakdown_metrics(query, extra_metrics ++ [:percentage])+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)+results =- Stats.breakdown(site, query, metrics, pagination)+ results|> transform_keys(%{browser_version: :version})if params["csv"] do@@ -1189,7 +1220,7 @@ defmodule PlausibleWeb.Api.StatsController dojson(conn, %{results: results,meta: Stats.Breakdown.formatted_date_ranges(query),- skip_imported_reason: query.skip_imported_reason+ skip_imported_reason: meta[:imports_skip_reason]})endend@@ -1205,8 +1236,10 @@ defmodule PlausibleWeb.Api.StatsController dometrics = breakdown_metrics(query, extra_metrics ++ [:percentage])+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)+systems =- Stats.breakdown(site, query, metrics, pagination)+ results|> transform_keys(%{os: :name})if params["csv"] do@@ -1221,7 +1254,7 @@ defmodule PlausibleWeb.Api.StatsController dojson(conn, %{results: systems,meta: Stats.Breakdown.formatted_date_ranges(query),- skip_imported_reason: query.skip_imported_reason+ skip_imported_reason: meta[:imports_skip_reason]})endend@@ -1237,8 +1270,10 @@ defmodule PlausibleWeb.Api.StatsController dometrics = breakdown_metrics(query, extra_metrics ++ [:percentage])+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)+results =- Stats.breakdown(site, query, metrics, pagination)+ results|> transform_keys(%{os_version: :version})if params["csv"] do@@ -1262,7 +1297,7 @@ defmodule PlausibleWeb.Api.StatsController dojson(conn, %{results: results,meta: Stats.Breakdown.formatted_date_ranges(query),- skip_imported_reason: query.skip_imported_reason+ skip_imported_reason: meta[:imports_skip_reason]})endend@@ -1278,8 +1313,10 @@ defmodule PlausibleWeb.Api.StatsController dometrics = breakdown_metrics(query, extra_metrics ++ [:percentage])+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)+sizes =- Stats.breakdown(site, query, metrics, pagination)+ results|> transform_keys(%{device: :name})if params["csv"] do@@ -1294,7 +1331,7 @@ defmodule PlausibleWeb.Api.StatsController dojson(conn, %{results: sizes,meta: Stats.Breakdown.formatted_date_ranges(query),- skip_imported_reason: query.skip_imported_reason+ skip_imported_reason: meta[:imports_skip_reason]})endend@@ -1312,9 +1349,10 @@ defmodule PlausibleWeb.Api.StatsController dometrics = [:visitors, :events, :conversion_rate] ++ @revenue_metrics+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)+conversions =- site- |> Stats.breakdown(query, metrics, pagination)+ results|> transform_keys(%{goal: :name})if params["csv"] do@@ -1327,7 +1365,7 @@ defmodule PlausibleWeb.Api.StatsController dojson(conn, %{results: conversions,meta: Stats.Breakdown.formatted_date_ranges(query),- skip_imported_reason: query.skip_imported_reason+ skip_imported_reason: meta[:imports_skip_reason]})endend@@ -1396,14 +1434,16 @@ defmodule PlausibleWeb.Api.StatsController do[:visitors, :events, :percentage] ++ @revenue_metricsend+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)+props =- Stats.breakdown(site, query, metrics, pagination)+ results|> transform_keys(%{prop_key => :name})%{results: props,meta: Stats.Breakdown.formatted_date_ranges(query),- skip_imported_reason: query.skip_imported_reason+ skip_imported_reason: meta[:imports_skip_reason]}end@@ -1595,14 +1635,6 @@ defmodule PlausibleWeb.Api.StatsController doendend- defp includes_imported?(source_query, comparison_query) do- cond do- source_query.include_imported -> true- comparison_query && comparison_query.include_imported -> true- true -> false- end- end-defp breakdown_metrics(query, extra_metrics \\ []) doif toplevel_goal_filter?(query) do[:visitors, :conversion_rate, :total_visitors]diff --git a/lib/plausible_web/live/goal_settings.ex b/lib/plausible_web/live/goal_settings.exindex bb0ed334d8..b2486a533a 100644--- a/lib/plausible_web/live/goal_settings.ex+++ b/lib/plausible_web/live/goal_settings.ex@@ -17,7 +17,6 @@ defmodule PlausibleWeb.Live.GoalSettings do|> assign_new(:site, fn %{current_user: current_user} ->current_user|> Plausible.Sites.get_for_user!(domain, [:owner, :admin, :super_admin])- |> Plausible.Imported.load_import_data()end)|> assign_new(:all_goals, fn %{site: site} ->Goals.for_site(site, preload_funnels?: true)diff --git a/lib/plausible_web/plugs/authorize_public_api.ex b/lib/plausible_web/plugs/authorize_public_api.exindex 90f7acd00d..7cad9beaaa 100644--- a/lib/plausible_web/plugs/authorize_public_api.ex+++ b/lib/plausible_web/plugs/authorize_public_api.ex@@ -65,7 +65,7 @@ defmodule PlausibleWeb.Plugs.AuthorizePublicAPI do{:ok, site} <- find_site(conn.params["site_id"]),:ok <- verify_site_access(api_key, site) doPlausible.OpenTelemetry.add_site_attributes(site)- site = Plausible.Imported.load_import_data(site)+ site = Plausible.Repo.preload(site, :completed_imports){:ok, assign(conn, :site, site)}endenddiff --git a/lib/plausible_web/plugs/authorize_site_access.ex b/lib/plausible_web/plugs/authorize_site_access.exindex fefa92869d..21b4af9a3b 100644--- a/lib/plausible_web/plugs/authorize_site_access.ex+++ b/lib/plausible_web/plugs/authorize_site_access.ex@@ -109,9 +109,9 @@ defmodule PlausibleWeb.Plugs.AuthorizeSiteAccess dosite =site- |> Plausible.Imported.load_import_data()|> Repo.preload([:owner,+ :completed_imports,team: [subscription: Plausible.Teams.last_subscription_query()]])diff --git a/test/plausible/imported_test.exs b/test/plausible/imported_test.exsindex 49666f6523..be396e14d2 100644--- a/test/plausible/imported_test.exs+++ b/test/plausible/imported_test.exs@@ -32,14 +32,56 @@ defmodule Plausible.ImportedTest doendend- describe "get_imports_date_range/1" do- test "returns empty when there are no imports" do+ describe "earliest_import_start_date/1" do+ test "returns nil if no site_imports exist" do+ site = insert(:site)++ assert is_nil(Imported.earliest_import_start_date(site))+ end++ test "returns nil when only incomplete or failed imports are present" do+ site = insert(:site)++ _import1 = insert(:site_import, site: site, status: :pending)+ _import2 = insert(:site_import, site: site, status: :importing)+ _import3 = insert(:site_import, site: site, status: :failed)+ _rogue_import = insert(:site_import, site: build(:site), status: :completed)++ assert is_nil(Imported.earliest_import_start_date(site))+ end++ test "returns start and end dates considering all imports" do+ site = insert(:site)++ _import1 =+ insert(:site_import,+ site: site,+ start_date: ~D[2020-04-02],+ end_date: ~D[2022-06-22],+ status: :completed,+ legacy: true+ )++ _import2 =+ insert(:site_import,+ site: site,+ start_date: ~D[2022-06-22],+ end_date: ~D[2024-01-08],+ status: :completed+ )++ assert Imported.earliest_import_start_date(site) == ~D[2020-04-02]+ end+ end++ describe "latest_import_end_date/1" do+ test "returns nil if no site_imports exist" dosite = insert(:site)- assert %{start_date: nil, end_date: nil} = Imported.get_imports_date_range(site)+ assert is_nil(Imported.latest_import_end_date(site))end- test "returns empty when only incomplete or failed imports are present" do+ test "returns nil when only incomplete or failed imports are present" dosite = insert(:site)_import1 = insert(:site_import, site: site, status: :pending)@@ -47,7 +89,7 @@ defmodule Plausible.ImportedTest do_import3 = insert(:site_import, site: site, status: :failed)_rogue_import = insert(:site_import, site: build(:site), status: :completed)- assert %{start_date: nil, end_date: nil} = Imported.get_imports_date_range(site)+ assert is_nil(Imported.latest_import_end_date(site))endtest "returns start and end dates considering all imports" do@@ -70,8 +112,7 @@ defmodule Plausible.ImportedTest dostatus: :completed)- assert %{start_date: ~D[2020-04-02], end_date: ~D[2024-01-08]} =- Imported.get_imports_date_range(site)+ assert Imported.latest_import_end_date(site) == ~D[2024-01-08]endenddiff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exsindex bde1dfe8e5..a523dc7e87 100644--- a/test/plausible/stats/query_parser_test.exs+++ b/test/plausible/stats/query_parser_test.exs@@ -48,6 +48,14 @@ defmodule Plausible.Stats.Filters.QueryParserTest dolast: DateTime.new!(~D[2021-05-31], ~T[23:59:59], "Etc/UTC")}+ @default_include %{+ imports: false,+ imports_meta: false,+ time_labels: false,+ total_rows: false,+ comparisons: nil+ }+def check_success(params, site, expected_result, schema_type \\ :public) doassert {:ok, result} = parse(site, schema_type, params, @now)@@ -76,7 +84,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}}@@ -111,7 +119,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})end@@ -152,7 +160,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}},:internal@@ -218,7 +226,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}},:internal@@ -345,7 +353,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})end@@ -370,7 +378,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})end@@ -396,7 +404,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})end@@ -462,7 +470,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})@@ -481,7 +489,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})end@@ -541,7 +549,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})end@@ -582,7 +590,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}},:internal@@ -653,7 +661,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})end@@ -702,7 +710,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})end@@ -755,7 +763,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})|> check_goals(@@ -781,7 +789,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})|> check_goals(@@ -807,7 +815,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})|> check_goals(@@ -833,7 +841,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})|> check_goals(@@ -862,7 +870,13 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["time"],order_by: nil,timezone: site.timezone,- include: %{imports: true, time_labels: true, total_rows: true, comparisons: nil},+ include: %{+ imports: true,+ imports_meta: false,+ time_labels: true,+ total_rows: true,+ comparisons: nil+ },pagination: %{limit: 10_000, offset: 0}})end@@ -923,6 +937,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest domode: "previous_period"},imports: false,+ imports_meta: false,time_labels: false,total_rows: false},@@ -953,6 +968,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest domode: "year_over_year"},imports: false,+ imports_meta: false,time_labels: false,total_rows: false},@@ -985,6 +1001,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest domode: "custom",date_range: @date_range_30d},+ imports_meta: false,imports: false,time_labels: false,total_rows: false@@ -1045,7 +1062,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["time"],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 100, offset: 200}})end@@ -1095,7 +1112,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})@@ -1215,7 +1232,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}},:internal@@ -1441,7 +1458,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["event:#{unquote(dimension)}"],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})end@@ -1462,7 +1479,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["visit:#{unquote(dimension)}"],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})end@@ -1482,7 +1499,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["event:props:foobar"],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})end@@ -1543,7 +1560,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: [{:events, :desc}, {:visitors, :asc}],timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})end@@ -1563,7 +1580,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["event:name"],order_by: [{"event:name", :desc}],timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})end@@ -1667,7 +1684,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})|> check_goals(@@ -1696,7 +1713,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["event:goal"],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})|> check_goals(@@ -1725,7 +1742,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["event:goal"],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})end@@ -1789,7 +1806,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}},:internal@@ -1812,7 +1829,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["event:page"],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}},:internal@@ -1837,7 +1854,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})|> check_goals(@@ -1896,7 +1913,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})@@ -1954,7 +1971,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})@@ -1988,7 +2005,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})@@ -2022,7 +2039,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})@@ -2056,7 +2073,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["event:goal"],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})@@ -2092,7 +2109,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["event:goal"],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})@@ -2130,7 +2147,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["event:goal"],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})@@ -2173,7 +2190,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["visit:device"],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})end@@ -2218,7 +2235,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["event:page"],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})end@@ -2237,7 +2254,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}})end@@ -2446,7 +2463,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ include: @default_include,pagination: %{limit: 10_000, offset: 0}},:internaldiff --git a/test/plausible/stats/query_result_test.exs b/test/plausible/stats/query_result_test.exsindex fe61b7f69f..6fc4841205 100644--- a/test/plausible/stats/query_result_test.exs+++ b/test/plausible/stats/query_result_test.exs@@ -1,7 +1,7 @@defmodule Plausible.Stats.QueryResultTest douse Plausible.DataCase, async: trueuse Plausible.Teams.Test- alias Plausible.Stats.{Query, QueryResult, QueryOptimizer}+ alias Plausible.Stats.{Query, QueryRunner, QueryResult, QueryOptimizer}setup douser = insert(:user)@@ -33,7 +33,8 @@ defmodule Plausible.Stats.QueryResultTest doquery = QueryOptimizer.optimize(query)query_result_json =- QueryResult.from([], site, query, %{})+ %QueryRunner{site: site, results: [], main_query: query}+ |> QueryResult.from()|> Jason.encode!(pretty: true)|> String.replace(site.domain, "dummy.site")diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exsindex fc148aa7c8..5834931f8b 100644--- a/test/plausible/stats/query_test.exs+++ b/test/plausible/stats/query_test.exs@@ -266,7 +266,6 @@ defmodule Plausible.Stats.QueryTest dotest "is true when requested via params and imported data exists", %{site: site} doinsert(:site_import, site: site)- site = Plausible.Imported.load_import_data(site)assert %{include_imported: true} =Query.from(site, %{"period" => "day", "with_imported" => "true"})@@ -279,7 +278,6 @@ defmodule Plausible.Stats.QueryTest dotest "is false when imported data exists but is out of the date range", %{site: site} doinsert(:site_import, site: site, start_date: ~D[2021-01-01], end_date: ~D[2022-01-01])- site = Plausible.Imported.load_import_data(site)assert %{include_imported: false, skip_imported_reason: :out_of_range} =Query.from(site, %{"period" => "day", "with_imported" => "true"})@@ -287,15 +285,13 @@ defmodule Plausible.Stats.QueryTest dotest "is false in realtime even when imported data from today exists", %{site: site} doinsert(:site_import, site: site)- site = Plausible.Imported.load_import_data(site)- assert %{include_imported: false, skip_imported_reason: :unsupported_query} =+ assert %{include_imported: false, skip_imported_reason: :out_of_range} =Query.from(site, %{"period" => "realtime", "with_imported" => "true"})endtest "is false when an arbitrary custom property filter is used", %{site: site} doinsert(:site_import, site: site)- site = Plausible.Imported.load_import_data(site)assert %{include_imported: false, skip_imported_reason: :unsupported_query} =Query.from(site, %{@@ -309,7 +305,6 @@ defmodule Plausible.Stats.QueryTest dotest "is true when breaking down by url and filtering by outbound link or file download goal",%{site: site} doinsert(:site_import, site: site)- site = Plausible.Imported.load_import_data(site)Enum.each(["Outbound Link: Click", "File Download"], fn goal_name ->insert(:goal, site: site, event_name: goal_name)@@ -327,7 +322,6 @@ defmodule Plausible.Stats.QueryTest dotest "is false when breaking down by url but without a special goal filter",%{site: site} doinsert(:site_import, site: site)- site = Plausible.Imported.load_import_data(site)assert %{include_imported: false} =Query.from(site, %{@@ -341,7 +335,6 @@ defmodule Plausible.Stats.QueryTest do%{site: site} doinsert(:site_import, site: site)insert(:goal, site: site, event_name: "404")- site = Plausible.Imported.load_import_data(site)assert %{include_imported: false} =Query.from(site, %{@@ -356,7 +349,6 @@ defmodule Plausible.Stats.QueryTest do%{site: site} doinsert(:site_import, site: site)insert(:goal, site: site, event_name: "Outbound Link: Click")- site = Plausible.Imported.load_import_data(site)assert %{include_imported: false} =Query.from(site, %{@@ -376,7 +368,6 @@ defmodule Plausible.Stats.QueryTest do%{site: site} doinsert(:site_import, site: site)insert(:goal, site: site, event_name: "Outbound Link: Click")- site = Plausible.Imported.load_import_data(site)assert %{include_imported: true} =Query.from(site, %{@@ -397,7 +388,6 @@ defmodule Plausible.Stats.QueryTest do} doinsert(:site_import, site: site)insert(:goal, site: site, event_name: "Outbound Link: Click")- site = Plausible.Imported.load_import_data(site)assert %{include_imported: true} =Query.from(site, %{@@ -421,7 +411,6 @@ defmodule Plausible.Stats.QueryTest do} doinsert(:site_import, site: site)insert(:goal, site: site, event_name: "Outbound Link: Click")- site = Plausible.Imported.load_import_data(site)assert %{include_imported: false} =Query.from(site, %{@@ -441,7 +430,6 @@ defmodule Plausible.Stats.QueryTest do%{site: site} doinsert(:site_import, site: site)insert(:goal, site: site, event_name: "404")- site = Plausible.Imported.load_import_data(site)assert %{include_imported: false} =Query.from(site, %{@@ -460,7 +448,6 @@ defmodule Plausible.Stats.QueryTest do%{site: site} doinsert(:site_import, site: site)insert(:goal, site: site, event_name: "Outbound Link: Click")- site = Plausible.Imported.load_import_data(site)assert %{include_imported: false} =Query.from(site, %{@@ -479,7 +466,6 @@ defmodule Plausible.Stats.QueryTest dotest "is false with a custom prop filter and non-matching property", %{site: site} doinsert(:site_import, site: site)insert(:goal, site: site, event_name: "Outbound Link: Click")- site = Plausible.Imported.load_import_data(site)assert %{include_imported: false} =Query.from(site, %{diff --git a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exsindex 8e9c779e49..3fb2558aa3 100644--- a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs@@ -378,10 +378,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do})assert json_response(conn, 200)["results"] == %{- "pageviews" => %{"value" => 3, "change" => 200},- "visitors" => %{"value" => 2, "change" => 100},- "bounce_rate" => %{"value" => 50, "change" => -50},- "visit_duration" => %{"value" => 750, "change" => 100}+ "pageviews" => %{"value" => 3, "change" => 200, "comparison_value" => 1},+ "visitors" => %{"value" => 2, "change" => 100, "comparison_value" => 1},+ "bounce_rate" => %{"value" => 50, "change" => -50, "comparison_value" => 100},+ "visit_duration" => %{"value" => 750, "change" => 100, "comparison_value" => 0}}end@@ -409,10 +409,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do})assert json_response(conn, 200)["results"] == %{- "pageviews" => %{"value" => 4, "change" => 100},- "visitors" => %{"value" => 3, "change" => 100},- "bounce_rate" => %{"value" => 100, "change" => nil},- "visit_duration" => %{"value" => 0, "change" => 0}+ "pageviews" => %{"value" => 4, "change" => 100, "comparison_value" => 0},+ "visitors" => %{"value" => 3, "change" => 100, "comparison_value" => 0},+ "bounce_rate" => %{"value" => 100, "change" => nil, "comparison_value" => 0},+ "visit_duration" => %{"value" => 0, "change" => 0, "comparison_value" => 0}}end@@ -441,7 +441,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do})assert json_response(conn, 200)["results"] == %{- "conversion_rate" => %{"value" => 50.0, "change" => 16.7}+ "conversion_rate" => %{+ "value" => 50.0,+ "change" => 16.7,+ "comparison_value" => 33.3+ }}end@@ -466,7 +470,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do})assert json_response(conn, 200)["results"] == %{- "time_on_page" => %{"value" => 90, "change" => 50.0}+ "time_on_page" => %{"value" => 90, "change" => 50.0, "comparison_value" => 60}}end@@ -491,7 +495,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do})assert json_response(conn, 200)["results"] == %{- "time_on_page" => %{"value" => nil, "change" => nil}+ "time_on_page" => %{"value" => nil, "change" => nil, "comparison_value" => 60}}endend@@ -565,12 +569,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do})assert json_response(conn, 200)["results"] == %{- "visitors" => %{"value" => 2, "change" => 100},- "visits" => %{"value" => 5, "change" => 150},- "pageviews" => %{"value" => 9, "change" => -10},- "bounce_rate" => %{"value" => 40, "change" => -10},- "views_per_visit" => %{"value" => 1.8, "change" => -64},- "visit_duration" => %{"value" => 20, "change" => -80}+ "visitors" => %{"value" => 2, "change" => 100, "comparison_value" => 1},+ "visits" => %{"value" => 5, "change" => 150, "comparison_value" => 2},+ "pageviews" => %{"value" => 9, "change" => -10, "comparison_value" => 10},+ "bounce_rate" => %{"value" => 40, "change" => -10, "comparison_value" => 50},+ "views_per_visit" => %{"value" => 1.8, "change" => -64, "comparison_value" => 5.0},+ "visit_duration" => %{"value" => 20, "change" => -80, "comparison_value" => 100}}end@@ -600,7 +604,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do})assert json_response(conn, 200)["results"] == %{- "visitors" => %{"value" => 1, "change" => -67}+ "visitors" => %{"value" => 1, "change" => -67, "comparison_value" => 3}}end@@ -629,7 +633,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do})assert json_response(conn, 200)["results"] == %{- "visitors" => %{"change" => 100, "value" => 1}+ "visitors" => %{"change" => 100, "value" => 1, "comparison_value" => 0}}assert json_response(conn, 200)["warning"] =~diff --git a/test/plausible_web/controllers/site_controller_test.exs b/test/plausible_web/controllers/site_controller_test.exsindex a55fd1e545..9f3f2a029f 100644--- a/test/plausible_web/controllers/site_controller_test.exs+++ b/test/plausible_web/controllers/site_controller_test.exs@@ -1592,8 +1592,6 @@ defmodule PlausibleWeb.SiteControllerTest doassert Enum.find(imports, &(&1.id == import_id))- site = Plausible.Imported.load_import_data(site)-assert eventually(fn ->count = Plausible.Stats.Clickhouse.imported_pageview_count(site){count == 22, count}