Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Refactor including imports in stats queries (#5011)

RobertJoonas00deb05d6438
diff --git a/lib/plausible/imported.ex b/lib/plausible/imported.ex
index 6ae034e7ae..2f4674030b 100644
--- a/lib/plausible/imported.ex
+++ b/lib/plausible/imported.ex
@@ -67,20 +67,34 @@ defmodule Plausible.Imported do
@goals_with_path
end
- @spec load_import_data(Site.t()) :: Site.t()
- def load_import_data(%{import_data_loaded: true} = site), do: site
+ @spec earliest_import_start_date(Site.t()) :: Date.t() | nil
+ def earliest_import_start_date(site) do
+ site
+ |> get_completed_imports()
+ |> Enum.map(& &1.start_date)
+ |> Enum.min(Date, fn -> nil end)
+ end
+
+ @spec latest_import_end_date(Site.t()) :: Date.t() | nil
+ def latest_import_end_date(site) do
+ site
+ |> get_completed_imports()
+ |> Enum.map(& &1.end_date)
+ |> Enum.max(Date, fn -> nil end)
+ end
- def load_import_data(site) do
- complete_import_ids = list_complete_import_ids(site)
- dates = get_imports_date_range(site)
+ @spec complete_import_ids(Site.t()) :: [non_neg_integer()]
+ def complete_import_ids(site) do
+ imports = get_completed_imports(site)
+ has_legacy? = Enum.any?(imports, fn %{legacy: legacy?} -> legacy? end)
+ ids = Enum.map(imports, fn %{id: id} -> id end)
- %{
- site
- | import_data_loaded: true,
- earliest_import_start_date: dates.start_date,
- latest_import_end_date: dates.end_date,
- complete_import_ids: complete_import_ids
- }
+ # account for legacy imports as well
+ if has_legacy? do
+ [0 | ids]
+ else
+ ids
+ end
end
@spec get_import(Site.t(), non_neg_integer()) :: SiteImport.t() | nil
@@ -88,16 +102,6 @@ defmodule Plausible.Imported do
Repo.get_by(SiteImport, id: import_id, site_id: site.id)
end
- @spec get_legacy_import(Site.t()) :: SiteImport.t() | nil
- def get_legacy_import(site) do
- from(i in SiteImport,
- where: i.site_id == ^site.id and i.legacy == true,
- order_by: [desc: i.updated_at],
- limit: 1
- )
- |> Repo.one()
- end
-
defdelegate listen(), to: Imported.Importer
@spec list_all_imports(Site.t(), atom()) :: [SiteImport.t()]
@@ -123,42 +127,6 @@ defmodule Plausible.Imported do
where(query, [i], i.status == ^status)
end
- @spec list_complete_import_ids(Site.t()) :: [non_neg_integer()]
- def list_complete_import_ids(site) do
- ids =
- from(i in SiteImport,
- where: i.site_id == ^site.id and i.status == ^SiteImport.completed(),
- select: {i.legacy, i.id},
- limit: @max_complete_imports
- )
- |> Repo.all()
-
- has_legacy? = Enum.any?(ids, fn {legacy?, _} -> legacy? end)
- ids = Enum.map(ids, &elem(&1, 1))
-
- # account for legacy imports as well
- if has_legacy? do
- [0 | ids]
- else
- ids
- end
- end
-
- @spec get_imports_date_range(Site.t()) :: %{
- start_date: Date.t() | nil,
- end_date: Date.t() | nil
- }
- def get_imports_date_range(site) do
- dates =
- from(i in SiteImport,
- where: i.site_id == ^site.id and i.status == ^SiteImport.completed(),
- select: %{start_date: min(i.start_date), end_date: max(i.end_date)}
- )
- |> Repo.one()
-
- dates || %{start_date: nil, end_date: nil}
- end
-
@spec delete_imports_for_site(Site.t()) :: :ok
def delete_imports_for_site(site) do
Repo.delete_all(from i in SiteImport, where: i.site_id == ^site.id)
@@ -242,4 +210,10 @@ defmodule Plausible.Imported do
defp in_range?(date, range) do
Date.before?(range.first, date) && Date.after?(range.last, date)
end
+
+ defp get_completed_imports(site) do
+ site
+ |> Repo.preload(:completed_imports)
+ |> Map.fetch!(:completed_imports)
+ end
end
diff --git a/lib/plausible/site.ex b/lib/plausible/site.ex
index 5ea17a189f..37d0cb2187 100644
--- a/lib/plausible/site.ex
+++ b/lib/plausible/site.ex
@@ -62,13 +62,7 @@ defmodule Plausible.Site do
field :invitations, {:array, :map}, virtual: true
field :pinned_at, :naive_datetime, virtual: true
- # Used for caching imports data for the duration of the whole request
- # to avoid multiple identical fetches. Populated by plugs putting
- # `site` in `assigns`.
- field :import_data_loaded, :boolean, default: false, virtual: true
- field :earliest_import_start_date, :date, virtual: true
- field :latest_import_end_date, :date, virtual: true
- field :complete_import_ids, {:array, :integer}, default: [], virtual: true
+ has_many :completed_imports, Plausible.Imported.SiteImport, where: [status: :completed]
timestamps()
end
diff --git a/lib/plausible/sites.ex b/lib/plausible/sites.ex
index afc531a590..04af80c24e 100644
--- a/lib/plausible/sites.ex
+++ b/lib/plausible/sites.ex
@@ -231,11 +231,9 @@ defmodule Plausible.Sites do
end
def stats_start_date(%Site{} = site) do
- site = Plausible.Imported.load_import_data(site)
-
start_date =
[
- site.earliest_import_start_date,
+ Plausible.Imported.earliest_import_start_date(site),
native_stats_start_date(site)
]
|> Enum.reject(&is_nil/1)
diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex
index b7365f9e55..de4a9ba3d3 100644
--- a/lib/plausible/stats/aggregate.ex
+++ b/lib/plausible/stats/aggregate.ex
@@ -6,7 +6,7 @@ defmodule Plausible.Stats.Aggregate do
"""
use Plausible.ClickhouseRepo
- alias Plausible.Stats.{Query, QueryRunner, QueryOptimizer}
+ alias Plausible.Stats.{Query, QueryRunner, QueryResult, QueryOptimizer}
def aggregate(site, query, metrics) do
Query.trace(query, metrics)
@@ -16,19 +16,20 @@ defmodule Plausible.Stats.Aggregate do
|> Query.set(metrics: metrics, remove_unavailable_revenue_metrics: true)
|> QueryOptimizer.optimize()
- query_result = QueryRunner.run(site, query)
+ %QueryResult{results: [entry], meta: meta} = QueryRunner.run(site, query)
- [entry] = query_result.results
+ results =
+ query.metrics
+ |> Enum.with_index()
+ |> Enum.map(fn {metric, index} ->
+ {
+ metric,
+ metric_map(entry, index, metric)
+ }
+ end)
+ |> Enum.into(%{})
- query.metrics
- |> Enum.with_index()
- |> Enum.map(fn {metric, index} ->
- {
- metric,
- metric_map(entry, index, metric)
- }
- end)
- |> Enum.into(%{})
+ %{results: results, meta: meta}
end
def metric_map(
diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex
index 3e0699b605..ef70183a58 100644
--- a/lib/plausible/stats/breakdown.ex
+++ b/lib/plausible/stats/breakdown.ex
@@ -8,7 +8,7 @@ defmodule Plausible.Stats.Breakdown do
use Plausible.ClickhouseRepo
use Plausible.Stats.SQL.Fragments
- alias Plausible.Stats.{Query, QueryRunner, QueryOptimizer, Comparisons}
+ alias Plausible.Stats.{Query, QueryRunner, QueryResult, QueryOptimizer, Comparisons}
def breakdown(
site,
@@ -37,8 +37,12 @@ defmodule Plausible.Stats.Breakdown do
)
|> QueryOptimizer.optimize()
- QueryRunner.run(site, query_with_metrics)
- |> build_breakdown_result(query_with_metrics, metrics)
+ %QueryResult{results: results, meta: meta} = QueryRunner.run(site, query_with_metrics)
+
+ %{
+ results: build_breakdown_result(results, query_with_metrics, metrics),
+ meta: meta
+ }
end
def formatted_date_ranges(query) do
@@ -62,10 +66,10 @@ defmodule Plausible.Stats.Breakdown do
end
end
- defp build_breakdown_result(query_result, query, metrics) do
+ defp build_breakdown_result(results, query, metrics) do
dimension_keys = query.dimensions |> Enum.map(&result_key/1)
- query_result.results
+ results
|> Enum.map(fn entry ->
comparison_map =
if entry[:comparison] do
diff --git a/lib/plausible/stats/comparisons.ex b/lib/plausible/stats/comparisons.ex
index dff2129317..e560632f19 100644
--- a/lib/plausible/stats/comparisons.ex
+++ b/lib/plausible/stats/comparisons.ex
@@ -175,21 +175,11 @@ defmodule Plausible.Stats.Comparisons do
defp maybe_include_imported(query, source_query) do
requested? = source_query.include.imports
+ skip_imported_reason = Query.get_skip_imported_reason(query)
- case Query.ensure_include_imported(query, requested?) do
- :ok ->
- struct!(query,
- include_imported: true,
- skip_imported_reason: nil,
- include: Map.put(query.include, :imports, true)
- )
-
- {:error, reason} ->
- struct!(query,
- include_imported: false,
- skip_imported_reason: reason,
- include: Map.put(query.include, :imports, requested?)
- )
- end
+ struct!(query,
+ include_imported: requested? and is_nil(skip_imported_reason),
+ skip_imported_reason: skip_imported_reason
+ )
end
end
diff --git a/lib/plausible/stats/email_report.ex b/lib/plausible/stats/email_report.ex
index 4f9a403ab8..fe95d5b64d 100644
--- a/lib/plausible/stats/email_report.ex
+++ b/lib/plausible/stats/email_report.ex
@@ -19,7 +19,9 @@ defmodule Plausible.Stats.EmailReport do
def get(site, query) do
metrics = [:pageviews, :visitors, :bounce_rate]
- Stats.aggregate(site, query, metrics)
+ %{results: results} = Stats.aggregate(site, query, metrics)
+
+ results
|> with_comparisons(site, query, metrics)
|> put_top_5_pages(site, query)
|> put_top_5_sources(site, query)
@@ -27,7 +29,7 @@ defmodule Plausible.Stats.EmailReport do
defp with_comparisons(stats, site, query, metrics) do
comparison_query = Comparisons.get_comparison_query(query, %{mode: "previous_period"})
- prev_period_stats = Stats.aggregate(site, comparison_query, metrics)
+ %{results: prev_period_stats} = Stats.aggregate(site, comparison_query, metrics)
stats
|> Enum.map(fn {metric, %{value: value}} ->
@@ -41,7 +43,7 @@ defmodule Plausible.Stats.EmailReport do
defp put_top_5_pages(stats, site, query) do
query = Query.set(query, dimensions: ["event:page"])
- pages = Stats.breakdown(site, query, [:visitors], {5, 1})
+ %{results: pages} = Stats.breakdown(site, query, [:visitors], {5, 1})
Map.put(stats, :pages, pages)
end
@@ -51,7 +53,7 @@ defmodule Plausible.Stats.EmailReport do
|> Query.add_filter([:is_not, "visit:source", ["Direct / None"]])
|> Query.set(dimensions: ["visit:source"])
- sources = Stats.breakdown(site, query, [:visitors], {5, 1})
+ %{results: sources} = Stats.breakdown(site, query, [:visitors], {5, 1})
Map.put(stats, :sources, sources)
end
diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.ex
index 574cd38362..751761ac45 100644
--- a/lib/plausible/stats/filters/query_parser.ex
+++ b/lib/plausible/stats/filters/query_parser.ex
@@ -7,6 +7,11 @@ defmodule Plausible.Stats.Filters.QueryParser do
@default_include %{
imports: false,
+ # `include.imports_meta` can be true even when `include.imports`
+ # is false. Even if we don't want to include imported data, we
+ # might still want to know whether imported data can be toggled
+ # on/off on the dashboard.
+ imports_meta: false,
time_labels: false,
total_rows: false,
comparisons: nil
diff --git a/lib/plausible/stats/goal_suggestions.ex b/lib/plausible/stats/goal_suggestions.ex
index f0ca5a5eb3..b9cea831e4 100644
--- a/lib/plausible/stats/goal_suggestions.ex
+++ b/lib/plausible/stats/goal_suggestions.ex
@@ -29,7 +29,6 @@ defmodule Plausible.Stats.GoalSuggestions do
site =
site
|> Repo.preload(:goals)
- |> Plausible.Imported.load_import_data()
excluded =
opts
@@ -62,7 +61,7 @@ defmodule Plausible.Stats.GoalSuggestions do
imported_q =
from(i in "imported_custom_events",
where: i.site_id == ^site.id,
- where: i.import_id in ^site.complete_import_ids,
+ where: i.import_id in ^Plausible.Imported.complete_import_ids(site),
where: i.date >= ^date_range.first and i.date <= ^date_range.last,
where: i.visitors > 0,
where: fragment("? ilike ?", i.name, ^matches),
diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.ex
index a27d6cb6f3..4346f4d71f 100644
--- a/lib/plausible/stats/imported/base.ex
+++ b/lib/plausible/stats/imported/base.ex
@@ -60,7 +60,7 @@ defmodule Plausible.Stats.Imported.Base do
end
def query_imported(table, site, query) do
- import_ids = site.complete_import_ids
+ import_ids = Imported.complete_import_ids(site)
# Assumption: dates in imported table are in user-local timezone.
%{first: date_from, last: date_to} = Query.date_range(query)
diff --git a/lib/plausible/stats/legacy/legacy_query_builder.ex b/lib/plausible/stats/legacy/legacy_query_builder.ex
index fe81589b34..73fd2603bc 100644
--- a/lib/plausible/stats/legacy/legacy_query_builder.ex
+++ b/lib/plausible/stats/legacy/legacy_query_builder.ex
@@ -26,8 +26,8 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do
|> put_parsed_filters(params)
|> preload_goals_and_revenue(site)
|> put_order_by(params)
- |> put_include_comparisons(site, params)
- |> Query.put_imported_opts(site, params)
+ |> put_include(site, params)
+ |> Query.put_imported_opts(site)
on_ee do
query = Plausible.Stats.Sampling.put_threshold(query, site, params)
@@ -205,9 +205,10 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do
end
end
- defp put_include_comparisons(query, site, params) do
- comparisons = parse_comparison_params(site, params)
- struct!(query, include: Map.put(query.include, :comparisons, comparisons))
+ defp put_include(query, site, params) do
+ query
+ |> Query.set_include(:comparisons, parse_comparison_params(site, params))
+ |> Query.set_include(:imports, params["with_imported"] == "true")
end
@doc """
diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex
index 424fc07aea..6634919404 100644
--- a/lib/plausible/stats/query.ex
+++ b/lib/plausible/stats/query.ex
@@ -35,7 +35,7 @@ defmodule Plausible.Stats.Query do
with {:ok, query_data} <- Filters.QueryParser.parse(site, schema_type, params) do
query =
struct!(__MODULE__, Map.to_list(query_data))
- |> put_imported_opts(site, %{})
+ |> put_imported_opts(site)
|> struct!(
now: DateTime.utc_now(:second),
debug_metadata: debug_metadata,
@@ -120,61 +120,48 @@ defmodule Plausible.Stats.Query do
end
defp refresh_imported_opts(query) do
- put_imported_opts(query, nil, %{})
+ put_imported_opts(query, nil)
end
- def put_imported_opts(query, site, params) do
- requested? = params["with_imported"] == "true" || query.include.imports
+ def put_imported_opts(query, site) do
+ requested? = query.include.imports
latest_import_end_date =
if site do
- site.latest_import_end_date
+ Plausible.Imported.latest_import_end_date(site)
else
query.latest_import_end_date
end
query = struct!(query, latest_import_end_date: latest_import_end_date)
- case ensure_include_imported(query, requested?) do
- :ok ->
- struct!(query,
- include_imported: true,
- include: Map.put(query.include, :imports, true)
- )
+ skip_imported_reason = get_skip_imported_reason(query)
- {:error, reason} ->
- struct!(query,
- include_imported: false,
- skip_imported_reason: reason,
- include: Map.put(query.include, :imports, requested?)
- )
- end
+ struct!(query,
+ include_imported: requested? and is_nil(skip_imported_reason),
+ skip_imported_reason: skip_imported_reason
+ )
end
- @spec ensure_include_imported(t(), boolean()) ::
- :ok | {:error, :no_imported_data | :out_of_range | :unsupported_query | :not_requested}
- def ensure_include_imported(query, requested?) do
+ @spec get_skip_imported_reason(t()) ::
+ nil | :no_imported_data | :out_of_range | :unsupported_query
+ def get_skip_imported_reason(query) do
cond do
- not requested? ->
- {:error, :not_requested}
-
is_nil(query.latest_import_end_date) ->
- {:error, :no_imported_data}
+ :no_imported_data
- query.period in ["realtime", "30m"] ->
- {:error, :unsupported_query}
+ query.period in ["realtime", "30m"] or
+ Date.after?(date_range(query).first, query.latest_import_end_date) ->
+ :out_of_range
"time:minute" in query.dimensions or "time:hour" in query.dimensions ->
- {:error, :unsupported_interval}
-
- Date.after?(date_range(query).first, query.latest_import_end_date) ->
- {:error, :out_of_range}
+ :unsupported_interval
not Imported.schema_supports_query?(query) ->
- {:error, :unsupported_query}
+ :unsupported_query
true ->
- :ok
+ nil
end
end
diff --git a/lib/plausible/stats/query_result.ex b/lib/plausible/stats/query_result.ex
index 750097ecd7..bc3bc7393c 100644
--- a/lib/plausible/stats/query_result.ex
+++ b/lib/plausible/stats/query_result.ex
@@ -8,7 +8,7 @@ defmodule Plausible.Stats.QueryResult do
"""
use Plausible
- alias Plausible.Stats.DateTimeRange
+ alias Plausible.Stats.{DateTimeRange, Query, QueryRunner}
defstruct results: [],
meta: %{},
@@ -19,11 +19,11 @@ defmodule Plausible.Stats.QueryResult do
`results` should already-built by Plausible.Stats.QueryRunner
"""
- def from(results, site, query, meta_extra) do
+ def from(%QueryRunner{site: site, main_query: query, results: results} = runner) do
struct!(
__MODULE__,
results: results,
- meta: meta(query, meta_extra),
+ meta: meta(runner) |> Enum.sort_by(&elem(&1, 0)) |> Jason.OrderedObject.new(),
query:
Jason.OrderedObject.new(
site_id: site.domain,
@@ -41,6 +41,14 @@ defmodule Plausible.Stats.QueryResult do
)
end
+ defp meta(%QueryRunner{} = runner) do
+ %{}
+ |> add_imports_meta(runner)
+ |> add_metric_warnings_meta(runner)
+ |> add_time_labels_meta(runner.main_query)
+ |> add_total_rows_meta(runner.main_query, runner.total_rows)
+ end
+
@imports_warnings %{
unsupported_query:
"Imported stats are not included in the results because query parameters are not supported. " <>
@@ -49,21 +57,62 @@ defmodule Plausible.Stats.QueryResult do
"Imported stats are not included because the time dimension (i.e. the interval) is too short."
}
- defp meta(query, meta_extra) do
- %{
- imports_included: if(query.include.imports, do: query.include_imported, else: nil),
- imports_skip_reason:
- if(query.include.imports and query.skip_imported_reason,
- do: to_string(query.skip_imported_reason)
- ),
- imports_warning: @imports_warnings[query.skip_imported_reason],
- metric_warnings: metric_warnings(query),
- time_labels:
- if(query.include.time_labels, do: Plausible.Stats.Time.time_labels(query), else: nil),
- total_rows: if(query.include.total_rows, do: meta_extra.total_rows, else: nil)
- }
- |> Enum.reject(fn {_, value} -> is_nil(value) end)
- |> Map.new()
+ defp add_imports_meta(meta, %QueryRunner{} = runner) do
+ %{main_query: %{include: include} = main_query} = runner
+
+ if include.imports or include[:imports_meta] do
+ comparison_query = Map.get(runner, :comparison_query)
+
+ imports_included =
+ case comparison_query do
+ %Query{include_imported: true} -> true
+ _ -> main_query.include_imported
+ end
+
+ imports_skip_reason =
+ case comparison_query do
+ %Query{skip_imported_reason: nil} -> nil
+ _ -> main_query.skip_imported_reason
+ end
+
+ imports_warning = @imports_warnings[imports_skip_reason]
+
+ %{
+ imports_included: imports_included,
+ imports_skip_reason: imports_skip_reason,
+ imports_warning: imports_warning
+ }
+ |> Map.reject(fn {_key, value} -> is_nil(value) end)
+ |> Map.merge(meta)
+ else
+ meta
+ end
+ end
+
+ defp add_metric_warnings_meta(meta, %QueryRunner{main_query: query}) do
+ warnings = metric_warnings(query)
+
+ if map_size(warnings) > 0 do
+ Map.put(meta, :metric_warnings, warnings)
+ else
+ meta
+ end
+ end
+
+ defp add_time_labels_meta(meta, query) do
+ if query.include.time_labels do
+ Map.put(meta, :time_labels, Plausible.Stats.Time.time_labels(query))
+ else
+ meta
+ end
+ end
+
+ defp add_total_rows_meta(meta, query, total_rows) do
+ if query.include.total_rows do
+ Map.put(meta, :total_rows, total_rows)
+ else
+ meta
+ end
end
defp include(query) do
@@ -80,7 +129,18 @@ defmodule Plausible.Stats.QueryResult do
end
end
+ defp metric_warnings(query) do
+ Enum.reduce(query.metrics, %{}, fn metric, acc ->
+ case metric_warning(metric, query) do
+ nil -> acc
+ %{} = warning -> Map.put(acc, metric, warning)
+ end
+ end)
+ end
+
on_ee do
+ @revenue_metrics Plausible.Stats.Goal.Revenue.revenue_metrics()
+
@revenue_metrics_warnings %{
revenue_goals_unavailable:
"The owner of this site does not have access to the revenue metrics feature.",
@@ -90,26 +150,20 @@ defmodule Plausible.Stats.QueryResult do
"Revenue metrics are null as there are no matching revenue goals."
}
- defp metric_warnings(query) do
+ defp metric_warning(metric, query) when metric in @revenue_metrics do
if query.revenue_warning do
- query.metrics
- |> Enum.filter(&(&1 in Plausible.Stats.Goal.Revenue.revenue_metrics()))
- |> Enum.map(
- &{&1,
- %{
- code: query.revenue_warning,
- warning: @revenue_metrics_warnings[query.revenue_warning]
- }}
- )
- |> Map.new()
+ %{
+ code: query.revenue_warning,
+ warning: @revenue_metrics_warnings[query.revenue_warning]
+ }
else
nil
end
end
- else
- defp metric_warnings(_query), do: nil
end
+ defp metric_warning(_metric, _query), do: nil
+
defp to_iso8601(datetime, timezone) do
datetime
|> DateTime.shift_zone!(timezone)
diff --git a/lib/plausible/stats/query_runner.ex b/lib/plausible/stats/query_runner.ex
index 6af06f2154..a4abbce93b 100644
--- a/lib/plausible/stats/query_runner.ex
+++ b/lib/plausible/stats/query_runner.ex
@@ -6,7 +6,7 @@ defmodule Plausible.Stats.QueryRunner do
Some secondary responsibilities are:
1. Dealing with comparison queries and combining results with main
2. Dealing with time-on-page
- 3. Passing total_rows from clickhouse to QueryResult meta
+ 3. Getting total_rows from ClickHouse results
"""
use Plausible
@@ -25,58 +25,54 @@ defmodule Plausible.Stats.QueryRunner do
}
defstruct [
- :query,
:site,
+ :main_query,
+ :main_results,
:comparison_query,
:comparison_results,
- :main_results_list,
- :ch_results,
- :meta_extra,
- :time_lookup,
- :results_list
+ :total_rows,
+ :results
]
def run(site, query) do
optimized_query = QueryOptimizer.optimize(query)
- run_results =
- %__MODULE__{query: optimized_query, site: site}
- |> execute_main_query()
- |> add_comparison_query()
- |> execute_comparison()
- |> add_meta_extra()
- |> add_time_lookup()
- |> build_results_list()
-
- QueryResult.from(run_results.results_list, site, optimized_query, run_results.meta_extra)
+ %__MODULE__{main_query: optimized_query, site: site}
+ |> execute_main_query()
+ |> add_comparison_query()
+ |> execute_comparison_query()
+ |> build_results_list()
+ |> QueryResult.from()
end
- defp execute_main_query(%__MODULE__{query: query, site: site} = run_results) do
+ defp execute_main_query(%__MODULE__{main_query: query, site: site} = runner) do
{ch_results, time_on_page} = execute_query(query, site)
- struct!(
- run_results,
- main_results_list: build_from_ch(ch_results, query, time_on_page),
- ch_results: ch_results
- )
+ main_results = build_from_ch(ch_results, query, time_on_page)
+
+ runner = struct!(runner, main_results: main_results)
+
+ if query.include.total_rows do
+ struct!(runner, total_rows: total_rows(ch_results))
+ else
+ runner
+ end
end
- defp add_comparison_query(
- %__MODULE__{query: query, main_results_list: main_results_list} = run_results
- )
+ defp add_comparison_query(%__MODULE__{main_query: query, main_results: main_results} = runner)
when is_map(query.include.comparisons) do
comparison_query =
query
|> Comparisons.get_comparison_query(query.include.comparisons)
- |> Comparisons.add_comparison_filters(main_results_list)
+ |> Comparisons.add_comparison_filters(main_results)
- struct!(run_results, comparison_query: comparison_query)
+ struct!(runner, comparison_query: comparison_query)
end
- defp add_comparison_query(run_results), do: run_results
+ defp add_comparison_query(runner), do: runner
- defp execute_comparison(
- %__MODULE__{comparison_query: comparison_query, site: site} = run_results
+ defp execute_comparison_query(
+ %__MODULE__{comparison_query: comparison_query, site: site} = runner
) do
if comparison_query do
{ch_results, time_on_page} = execute_query(comparison_query, site)
@@ -88,46 +84,33 @@ defmodule Plausible.Stats.QueryRunner do
time_on_page
)
- struct!(run_results, comparison_results: comparison_results)
+ struct!(runner, comparison_results: comparison_results)
else
- run_results
+ runner
end
end
- defp add_time_lookup(run_results) do
- time_lookup =
- if Time.time_dimension(run_results.query) && run_results.comparison_query do
- Enum.zip(
- Time.time_labels(run_results.query),
- Time.time_labels(run_results.comparison_query)
- )
- |> Map.new()
- else
- %{}
- end
-
- struct!(run_results, time_lookup: time_lookup)
- end
-
- defp add_meta_extra(%__MODULE__{query: query, ch_results: ch_results} = run_results) do
- struct!(run_results,
- meta_extra: %{
- total_rows: if(query.include.total_rows, do: total_rows(ch_results), else: nil)
- }
- )
+ defp get_time_lookup(query, comparison_query) do
+ if Time.time_dimension(query) && comparison_query do
+ Enum.zip(
+ Time.time_labels(query),
+ Time.time_labels(comparison_query)
+ )
+ |> Map.new()
+ else
+ %{}
+ end
end
- defp build_results_list(
- %__MODULE__{query: query, main_results_list: main_results_list} = run_results
- ) do
- results_list =
+ defp build_results_list(%__MODULE__{main_query: query, main_results: main_results} = runner) do
+ results =
case query.dimensions do
- ["time:" <> _] -> main_results_list |> add_empty_timeseries_rows(run_results)
- _ -> main_results_list
+ ["time:" <> _] -> main_results |> add_empty_timeseries_rows(runner)
+ _ -> main_results
end
- |> merge_with_comparison_results(run_results)
+ |> merge_with_comparison_results(runner)
- struct!(run_results, results_list: results_list)
+ struct!(runner, results: results)
end
defp execute_query(query, site) do
@@ -191,7 +174,7 @@ defmodule Plausible.Stats.QueryRunner do
# Special case: If comparison and single time dimension, add 0 rows - otherwise
# comparisons would not be shown for timeseries with 0 values.
- defp add_empty_timeseries_rows(results_list, %__MODULE__{query: query})
+ defp add_empty_timeseries_rows(results_list, %__MODULE__{main_query: query})
when is_map(query.include.comparisons) do
indexed_results = index_by_dimensions(results_list)
@@ -210,13 +193,13 @@ defmodule Plausible.Stats.QueryRunner do
defp add_empty_timeseries_rows(results_list, _), do: results_list
- defp merge_with_comparison_results(results_list, run_results) do
- comparison_map = (run_results.comparison_results || []) |> index_by_dimensions()
- time_lookup = run_results.time_lookup || %{}
+ defp merge_with_comparison_results(results_list, runner) do
+ comparison_map = (runner.comparison_results || []) |> index_by_dimensions()
+ time_lookup = get_time_lookup(runner.main_query, runner.comparison_query)
Enum.map(
results_list,
- &add_comparison_results(&1, run_results.query, comparison_map, time_lookup)
+ &add_comparison_results(&1, runner.main_query, comparison_map, time_lookup)
)
end
diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex
index 8e121b8175..a585a71bd2 100644
--- a/lib/plausible_web/controllers/api/external_stats_controller.ex
+++ b/lib/plausible_web/controllers/api/external_stats_controller.ex
@@ -2,7 +2,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
use PlausibleWeb, :controller
use Plausible.Repo
use PlausibleWeb.Plugs.ErrorHandler
- alias Plausible.Stats.{Query, Compare, Comparisons, Metrics, Filters}
+ alias Plausible.Stats.{Query, Metrics, Filters}
def realtime_visitors(conn, _params) do
site = conn.assigns.site
@@ -20,28 +20,16 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
:ok <- validate_filters(site, query.filters),
{:ok, metrics} <- parse_and_validate_metrics(params, query),
:ok <- ensure_custom_props_access(site, query) do
- results =
+ query =
if params["compare"] == "previous_period" do
- comparison_query = Comparisons.get_comparison_query(query, %{mode: "previous_period"})
-
- [prev_result, curr_result] =
- Plausible.ClickhouseRepo.parallel_tasks([
- fn -> Plausible.Stats.aggregate(site, comparison_query, metrics) end,
- fn -> Plausible.Stats.aggregate(site, query, metrics) end
- ])
-
- Enum.map(curr_result, fn {metric, %{value: current_val}} ->
- %{value: prev_val} = prev_result[metric]
- change = Compare.calculate_change(metric, prev_val, current_val)
-
- {metric, %{value: current_val, change: change}}
- end)
- |> Enum.into(%{})
+ Query.set_include(query, :comparisons, %{mode: "previous_period"})
else
- Plausible.Stats.aggregate(site, query, metrics)
+ query
end
- payload = maybe_add_warning(%{results: results}, query)
+ %{results: results, meta: meta} = Plausible.Stats.aggregate(site, query, metrics)
+
+ payload = maybe_add_warning(%{results: results}, meta)
json(conn, payload)
else
@@ -61,8 +49,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
{:ok, limit} <- validate_or_default_limit(params),
:ok <- ensure_custom_props_access(site, query) do
page = String.to_integer(Map.get(params, "page", "1"))
- results = Plausible.Stats.breakdown(site, query, metrics, {limit, page})
- payload = maybe_add_warning(%{results: results}, query)
+
+ %{results: results, meta: meta} =
+ Plausible.Stats.breakdown(site, query, metrics, {limit, page})
+
+ payload = maybe_add_warning(%{results: results}, meta)
json(conn, payload)
else
@@ -387,18 +378,13 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
"The goal `#{goal}` is not configured for this site. "
end
- defp maybe_add_warning(payload, %{skip_imported_reason: reason})
- when reason in [nil, :not_requested, :no_imported_data, :out_of_range, :manual_exclusion] do
- payload
- end
+ @imported_query_unsupported_warning "Imported stats are not included in the results because query parameters are not supported. For more information, see: https://plausible.io/docs/stats-api#filtering-imported-stats"
- defp maybe_add_warning(payload, %{skip_imported_reason: :unsupported_query}) do
- Map.put(
- payload,
- :warning,
- "Imported stats are not included in the results because query parameters are not supported. " <>
- "For more information, see: https://plausible.io/docs/stats-api#filtering-imported-stats"
- )
+ defp maybe_add_warning(payload, %Jason.OrderedObject{} = meta) do
+ case meta[:imports_skip_reason] do
+ :unsupported_query -> Map.put(payload, :warning, @imported_query_unsupported_warning)
+ _ -> payload
+ end
end
defp send_json_error_response(conn, {:error, {status, msg}}) do
diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
index 66448d79a8..158da790e9 100644
--- a/lib/plausible_web/controllers/api/stats_controller.ex
+++ b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -68,9 +68,6 @@ defmodule PlausibleWeb.Api.StatsController do
* `includes_imported` - boolean indicating whether imported data
was queried or not.
- * `imports_exist` - boolean indicating whether there are any completed
- imports for a given site or not.
-
* `full_intervals` - map of dates indicating whether the interval has been
cut off by the requested date range or not. For example, if looking at a
month week-by-week, some weeks may be cut off by the month boundaries.
@@ -87,7 +84,6 @@ defmodule PlausibleWeb.Api.StatsController do
"2021-11-01" => true,
"2021-12-01" => false
},
- "imports_exist" => false,
"interval" => "month",
"labels" => ["2021-09-01", "2021-10-01", "2021-11-01", "2021-12-01"],
"plot" => [0, 0, 0, 0],
@@ -196,18 +192,25 @@ defmodule PlausibleWeb.Api.StatsController do
params = realtime_period_to_30m(params)
- query = Query.from(site, params, debug_metadata(conn))
+ query =
+ site
+ |> Query.from(params, debug_metadata(conn))
+ |> Query.set_include(:imports_meta, true)
+
+ %{
+ top_stats: top_stats,
+ meta: meta,
+ sample_percent: sample_percent
+ } = fetch_top_stats(site, query, current_user)
- {top_stats, sample_percent} = fetch_top_stats(site, query, current_user)
comparison_query = comparison_query(query)
json(conn, %{
top_stats: top_stats,
interval: query.interval,
sample_percent: sample_percent,
- with_imported_switch: with_imported_switch_info(query, comparison_query),
- includes_imported: includes_imported?(query, comparison_query),
- imports_exist: site.complete_import_ids != [],
+ with_imported_switch: with_imported_switch_info(meta),
+ includes_imported: meta[:imports_included] == true,
comparing_from: query.include.comparisons && Query.date_range(comparison_query).first,
comparing_to: query.include.comparisons && Query.date_range(comparison_query).last,
from: Query.date_range(query).first,
@@ -215,35 +218,19 @@ defmodule PlausibleWeb.Api.StatsController do
})
end
- defp with_imported_switch_info(%Query{period: "30m"}, _) do
- %{visible: false, togglable: false, tooltip_msg: nil}
- end
-
- defp with_imported_switch_info(query, nil) do
- with_imported_switch_info(query.skip_imported_reason)
- end
-
- defp with_imported_switch_info(query, comparison_query) do
- case {query.skip_imported_reason, comparison_query.skip_imported_reason} do
- {:out_of_range, nil} -> with_imported_switch_info(nil)
- {:out_of_range, :not_requested} -> with_imported_switch_info(:not_requested)
- {reason, _} -> with_imported_switch_info(reason)
- end
- end
+ defp with_imported_switch_info(%Jason.OrderedObject{} = meta) do
+ case {meta[:imports_included], meta[:imports_skip_reason]} do
+ {true, nil} ->
+ %{visible: true, togglable: true, tooltip_msg: "Click to exclude imported data"}
- defp with_imported_switch_info(skip_reason) do
- case skip_reason do
- reason when reason in [:no_imported_data, :out_of_range] ->
- %{visible: false, togglable: false, tooltip_msg: nil}
+ {false, nil} ->
+ %{visible: true, togglable: true, tooltip_msg: "Click to include imported data"}
- :unsupported_query ->
+ {false, :unsupported_query} ->
%{visible: true, togglable: false, tooltip_msg: "Imported data cannot be included"}
- :not_requested ->
- %{visible: true, togglable: true, tooltip_msg: "Click to include imported data"}
-
- nil ->
- %{visible: true, togglable: true, tooltip_msg: "Click to exclude imported data"}
+ {false, reason} when reason in [:no_imported_data, :out_of_range] ->
+ %{visible: false, togglable: false, tooltip_msg: nil}
end
end
@@ -315,11 +302,14 @@ defmodule PlausibleWeb.Api.StatsController do
query = Query.set_include(query, :comparisons, nil)
%{
- visitors: %{value: unique_conversions},
- events: %{value: total_conversions}
+ results: %{
+ visitors: %{value: unique_conversions},
+ events: %{value: total_conversions}
+ },
+ meta: meta
} = Stats.aggregate(site, query, [:visitors, :events])
- stats = [
+ top_stats = [
%{
name: "Current visitors",
graph_metric: :current_visitors,
@@ -337,18 +327,21 @@ defmodule PlausibleWeb.Api.StatsController do
}
]
- {stats, 100}
+ %{top_stats: top_stats, meta: meta, sample_percent: 100}
end
defp fetch_realtime_top_stats(site, query) do
query = Query.set_include(query, :comparisons, nil)
%{
- visitors: %{value: visitors},
- pageviews: %{value: pageviews}
+ results: %{
+ visitors: %{value: visitors},
+ pageviews: %{value: pageviews}
+ },
+ meta: meta
} = Stats.aggregate(site, query, [:visitors, :pageviews])
- stats = [
+ top_stats = [
%{
name: "Current visitors",
graph_metric: :current_visitors,
@@ -366,29 +359,31 @@ defmodule PlausibleWeb.Api.StatsController do
}
]
- {stats, 100}
+ %{top_stats: top_stats, meta: meta, sample_percent: 100}
end
defp fetch_goal_top_stats(site, query) do
metrics =
[:total_visitors, :visitors, :events, :conversion_rate] ++ @revenue_metrics
- results = Stats.aggregate(site, query, metrics)
-
- [
- top_stats_entry(results, "Unique visitors", :total_visitors),
- top_stats_entry(results, "Unique conversions", :visitors),
- top_stats_entry(results, "Total conversions", :events),
- on_ee do
- top_stats_entry(results, "Average revenue", :average_revenue)
- end,
- on_ee do
- top_stats_entry(results, "Total revenue", :total_revenue)
- end,
- top_stats_entry(results, "Conversion rate", :conversion_rate)
- ]
- |> Enum.reject(&is_nil/1)
- |> then(&{&1, 100})
+ %{results: results, meta: meta} = Stats.aggregate(site, query, metrics)
+
+ top_stats =
+ [
+ top_stats_entry(results, "Unique visitors", :total_visitors),
+ top_stats_entry(results, "Unique conversions", :visitors),
+ top_stats_entry(results, "Total conversions", :events),
+ on_ee do
+ top_stats_entry(results, "Average revenue", :average_revenue)
+ end,
+ on_ee do
+ top_stats_entry(results, "Total revenue", :total_revenue)
+ end,
+ top_stats_entry(results, "Conversion rate", :conversion_rate)
+ ]
+ |> Enum.reject(&is_nil/1)
+
+ %{top_stats: top_stats, meta: meta, sample_percent: 100}
end
defp fetch_other_top_stats(site, query, current_user) do
@@ -417,27 +412,29 @@ defmodule PlausibleWeb.Api.StatsController do
metrics ++ [:views_per_visit, :bounce_rate, :visit_duration]
end
- current_results = Stats.aggregate(site, query, metrics)
+ %{results: results, meta: meta} = Stats.aggregate(site, query, metrics)
- stats =
+ top_stats =
[
- top_stats_entry(current_results, "Unique visitors", :visitors),
- top_stats_entry(current_results, "Total visits", :visits),
- top_stats_entry(current_results, "Total pageviews", :pageviews),
- top_stats_entry(current_results, "Views per visit", :views_per_visit),
- top_stats_entry(current_results, "Bounce rate", :bounce_rate),
- top_stats_entry(current_results, "Visit duration", :visit_duration),
- top_stats_entry(current_results, "Time on page", :time_on_page,
+ top_stats_entry(results, "Unique visitors", :visitors),
+ top_stats_entry(results, "Total visits", :visits),
+ top_stats_entry(results, "Total pageviews", :pageviews),
+ top_stats_entry(results, "Views per visit", :views_per_visit),
+ top_stats_entry(results, "Bounce rate", :bounce_rate),
+ top_stats_entry(results, "Visit duration", :visit_duration),
+ top_stats_entry(results, "Time on page", :time_on_page,
formatter: fn
nil -> 0
value -> value
end
),
- top_stats_entry(current_results, "Scroll depth", :scroll_depth)
+ top_stats_entry(results, "Scroll depth", :scroll_depth)
]
|> Enum.filter(& &1)
- {stats, current_results[:sample_percent][:value]}
+ sample_percent = results[:sample_percent][:value]
+
+ %{top_stats: top_stats, meta: meta, sample_percent: sample_percent}
end
defp top_stats_entry(current_results, name, key, opts \\ []) do
@@ -474,8 +471,10 @@ defmodule PlausibleWeb.Api.StatsController do
metrics = breakdown_metrics(query, extra_metrics)
+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)
+
res =
- Stats.breakdown(site, query, metrics, pagination)
+ results
|> transform_keys(%{source: :name})
if params["csv"] do
@@ -490,7 +489,7 @@ defmodule PlausibleWeb.Api.StatsController do
json(conn, %{
results: res,
meta: Stats.Breakdown.formatted_date_ranges(query),
- skip_imported_reason: query.skip_imported_reason
+ skip_imported_reason: meta[:imports_skip_reason]
})
end
end
@@ -506,8 +505,10 @@ defmodule PlausibleWeb.Api.StatsController do
metrics = breakdown_metrics(query, extra_metrics)
+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)
+
res =
- Stats.breakdown(site, query, metrics, pagination)
+ results
|> transform_keys(%{channel: :name})
if params["csv"] do
@@ -522,7 +523,7 @@ defmodule PlausibleWeb.Api.StatsController do
json(conn, %{
results: res,
meta: Stats.Breakdown.formatted_date_ranges(query),
- skip_imported_reason: query.skip_imported_reason
+ skip_imported_reason: meta[:imports_skip_reason]
})
end
end
@@ -588,8 +589,10 @@ defmodule PlausibleWeb.Api.StatsController do
pagination = parse_pagination(params)
metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])
+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)
+
res =
- Stats.breakdown(site, query, metrics, pagination)
+ results
|> transform_keys(%{utm_medium: :name})
if params["csv"] do
@@ -604,7 +607,7 @@ defmodule PlausibleWeb.Api.StatsController do
json(conn, %{
results: res,
meta: Stats.Breakdown.formatted_date_ranges(query),
- skip_imported_reason: query.skip_imported_reason
+ skip_imported_reason: meta[:imports_skip_reason]
})
end
end
@@ -616,8 +619,10 @@ defmodule PlausibleWeb.Api.StatsController do
pagination = parse_pagination(params)
metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])
+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)
+
res =
- Stats.breakdown(site, query, metrics, pagination)
+ results
|> transform_keys(%{utm_campaign: :name})
if params["csv"] do
@@ -632,7 +637,7 @@ defmodule PlausibleWeb.Api.StatsController do
json(conn, %{
results: res,
meta: Stats.Breakdown.formatted_date_ranges(query),
- skip_imported_reason: query.skip_imported_reason
+ skip_imported_reason: meta[:imports_skip_reason]
})
end
end
@@ -644,8 +649,10 @@ defmodule PlausibleWeb.Api.StatsController do
pagination = parse_pagination(params)
metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])
+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)
+
res =
- Stats.breakdown(site, query, metrics, pagination)
+ results
|> transform_keys(%{utm_content: :name})
if params["csv"] do
@@ -660,7 +667,7 @@ defmodule PlausibleWeb.Api.StatsController do
json(conn, %{
results: res,
meta: Stats.Breakdown.formatted_date_ranges(query),
- skip_imported_reason: query.skip_imported_reason
+ skip_imported_reason: meta[:imports_skip_reason]
})
end
end
@@ -672,8 +679,10 @@ defmodule PlausibleWeb.Api.StatsController do
pagination = parse_pagination(params)
metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])
+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)
+
res =
- Stats.breakdown(site, query, metrics, pagination)
+ results
|> transform_keys(%{utm_term: :name})
if params["csv"] do
@@ -688,7 +697,7 @@ defmodule PlausibleWeb.Api.StatsController do
json(conn, %{
results: res,
meta: Stats.Breakdown.formatted_date_ranges(query),
- skip_imported_reason: query.skip_imported_reason
+ skip_imported_reason: meta[:imports_skip_reason]
})
end
end
@@ -700,8 +709,10 @@ defmodule PlausibleWeb.Api.StatsController do
pagination = parse_pagination(params)
metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])
+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)
+
res =
- Stats.breakdown(site, query, metrics, pagination)
+ results
|> transform_keys(%{utm_source: :name})
if params["csv"] do
@@ -716,7 +727,7 @@ defmodule PlausibleWeb.Api.StatsController do
json(conn, %{
results: res,
meta: Stats.Breakdown.formatted_date_ranges(query),
- skip_imported_reason: query.skip_imported_reason
+ skip_imported_reason: meta[:imports_skip_reason]
})
end
end
@@ -728,8 +739,10 @@ defmodule PlausibleWeb.Api.StatsController do
pagination = parse_pagination(params)
metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])
+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)
+
res =
- Stats.breakdown(site, query, metrics, pagination)
+ results
|> transform_keys(%{referrer: :name})
if params["csv"] do
@@ -744,7 +757,7 @@ defmodule PlausibleWeb.Api.StatsController do
json(conn, %{
results: res,
meta: Stats.Breakdown.formatted_date_ranges(query),
- skip_imported_reason: query.skip_imported_reason
+ skip_imported_reason: meta[:imports_skip_reason]
})
end
end
@@ -819,14 +832,16 @@ defmodule PlausibleWeb.Api.StatsController do
metrics = breakdown_metrics(query, extra_metrics)
+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)
+
referrers =
- Stats.breakdown(site, query, metrics, pagination)
+ results
|> transform_keys(%{referrer: :name})
json(conn, %{
results: referrers,
meta: Stats.Breakdown.formatted_date_ranges(query),
- skip_imported_reason: query.skip_imported_reason
+ skip_imported_reason: meta[:imports_skip_reason]
})
end
@@ -854,8 +869,10 @@ defmodule PlausibleWeb.Api.StatsController do
metrics = breakdown_metrics(query, extra_metrics)
pagination = parse_pagination(params)
+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)
+
pages =
- Stats.breakdown(site, query, metrics, pagination)
+ results
|> transform_keys(%{page: :name})
if params["csv"] do
@@ -875,7 +892,7 @@ defmodule PlausibleWeb.Api.StatsController do
json(conn, %{
results: pages,
meta: Stats.Breakdown.formatted_date_ranges(query),
- skip_imported_reason: query.skip_imported_reason
+ skip_imported_reason: meta[:imports_skip_reason]
})
end
end
@@ -887,8 +904,10 @@ defmodule PlausibleWeb.Api.StatsController do
pagination = parse_pagination(params)
metrics = breakdown_metrics(query, [:visits, :visit_duration])
+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)
+
entry_pages =
- Stats.breakdown(site, query, metrics, pagination)
+ results
|> transform_keys(%{entry_page: :name})
if params["csv"] do
@@ -910,7 +929,7 @@ defmodule PlausibleWeb.Api.StatsController do
json(conn, %{
results: entry_pages,
meta: Stats.Breakdown.formatted_date_ranges(query),
- skip_imported_reason: query.skip_imported_reason
+ skip_imported_reason: meta[:imports_skip_reason]
})
end
end
@@ -922,8 +941,10 @@ defmodule PlausibleWeb.Api.StatsController do
{limit, page} = parse_pagination(params)
metrics = breakdown_metrics(query, [:visits])
+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, {limit, page})
+
exit_pages =
- Stats.breakdown(site, query, metrics, {limit, page})
+ results
|> add_exit_rate(site, query, limit)
|> transform_keys(%{exit_page: :name})
@@ -946,7 +967,7 @@ defmodule PlausibleWeb.Api.StatsController do
json(conn, %{
results: exit_pages,
meta: Stats.Breakdown.formatted_date_ranges(query),
- skip_imported_reason: query.skip_imported_reason
+ skip_imported_reason: meta[:imports_skip_reason]
})
end
end
@@ -964,7 +985,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> Query.add_filter([:is, "event:page", pages])
|> Query.set(dimensions: ["event:page"])
- total_pageviews =
+ %{results: total_pageviews} =
Stats.breakdown(site, total_pageviews_query, [:pageviews], {limit, 1})
Enum.map(breakdown_results, fn result ->
@@ -989,8 +1010,10 @@ defmodule PlausibleWeb.Api.StatsController do
pagination = parse_pagination(params)
metrics = breakdown_metrics(query, [:percentage])
+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)
+
countries =
- Stats.breakdown(site, query, metrics, pagination)
+ results
|> transform_keys(%{country: :code})
if params["csv"] do
@@ -1033,7 +1056,7 @@ defmodule PlausibleWeb.Api.StatsController do
json(conn, %{
results: countries,
meta: Stats.Breakdown.formatted_date_ranges(query),
- skip_imported_reason: query.skip_imported_reason
+ skip_imported_reason: meta[:imports_skip_reason]
})
end
end
@@ -1045,8 +1068,10 @@ defmodule PlausibleWeb.Api.StatsController do
pagination = parse_pagination(params)
metrics = breakdown_metrics(query)
+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)
+
regions =
- Stats.breakdown(site, query, metrics, pagination)
+ results
|> transform_keys(%{region: :code})
|> Enum.map(fn region ->
region_entry = Location.get_subdivision(region[:code])
@@ -1072,7 +1097,7 @@ defmodule PlausibleWeb.Api.StatsController do
json(conn, %{
results: regions,
meta: Stats.Breakdown.formatted_date_ranges(query),
- skip_imported_reason: query.skip_imported_reason
+ skip_imported_reason: meta[:imports_skip_reason]
})
end
end
@@ -1084,8 +1109,10 @@ defmodule PlausibleWeb.Api.StatsController do
pagination = parse_pagination(params)
metrics = breakdown_metrics(query)
+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)
+
cities =
- Stats.breakdown(site, query, metrics, pagination)
+ results
|> transform_keys(%{city: :code})
|> Enum.map(fn city ->
city_info = Location.get_city(city[:code])
@@ -1116,7 +1143,7 @@ defmodule PlausibleWeb.Api.StatsController do
json(conn, %{
results: cities,
meta: Stats.Breakdown.formatted_date_ranges(query),
- skip_imported_reason: query.skip_imported_reason
+ skip_imported_reason: meta[:imports_skip_reason]
})
end
end
@@ -1132,8 +1159,10 @@ defmodule PlausibleWeb.Api.StatsController do
metrics = breakdown_metrics(query, extra_metrics ++ [:percentage])
+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)
+
browsers =
- Stats.breakdown(site, query, metrics, pagination)
+ results
|> transform_keys(%{browser: :name})
if params["csv"] do
@@ -1148,7 +1177,7 @@ defmodule PlausibleWeb.Api.StatsController do
json(conn, %{
results: browsers,
meta: Stats.Breakdown.formatted_date_ranges(query),
- skip_imported_reason: query.skip_imported_reason
+ skip_imported_reason: meta[:imports_skip_reason]
})
end
end
@@ -1164,8 +1193,10 @@ defmodule PlausibleWeb.Api.StatsController do
metrics = breakdown_metrics(query, extra_metrics ++ [:percentage])
+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)
+
results =
- Stats.breakdown(site, query, metrics, pagination)
+ results
|> transform_keys(%{browser_version: :version})
if params["csv"] do
@@ -1189,7 +1220,7 @@ defmodule PlausibleWeb.Api.StatsController do
json(conn, %{
results: results,
meta: Stats.Breakdown.formatted_date_ranges(query),
- skip_imported_reason: query.skip_imported_reason
+ skip_imported_reason: meta[:imports_skip_reason]
})
end
end
@@ -1205,8 +1236,10 @@ defmodule PlausibleWeb.Api.StatsController do
metrics = breakdown_metrics(query, extra_metrics ++ [:percentage])
+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)
+
systems =
- Stats.breakdown(site, query, metrics, pagination)
+ results
|> transform_keys(%{os: :name})
if params["csv"] do
@@ -1221,7 +1254,7 @@ defmodule PlausibleWeb.Api.StatsController do
json(conn, %{
results: systems,
meta: Stats.Breakdown.formatted_date_ranges(query),
- skip_imported_reason: query.skip_imported_reason
+ skip_imported_reason: meta[:imports_skip_reason]
})
end
end
@@ -1237,8 +1270,10 @@ defmodule PlausibleWeb.Api.StatsController do
metrics = breakdown_metrics(query, extra_metrics ++ [:percentage])
+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)
+
results =
- Stats.breakdown(site, query, metrics, pagination)
+ results
|> transform_keys(%{os_version: :version})
if params["csv"] do
@@ -1262,7 +1297,7 @@ defmodule PlausibleWeb.Api.StatsController do
json(conn, %{
results: results,
meta: Stats.Breakdown.formatted_date_ranges(query),
- skip_imported_reason: query.skip_imported_reason
+ skip_imported_reason: meta[:imports_skip_reason]
})
end
end
@@ -1278,8 +1313,10 @@ defmodule PlausibleWeb.Api.StatsController do
metrics = breakdown_metrics(query, extra_metrics ++ [:percentage])
+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)
+
sizes =
- Stats.breakdown(site, query, metrics, pagination)
+ results
|> transform_keys(%{device: :name})
if params["csv"] do
@@ -1294,7 +1331,7 @@ defmodule PlausibleWeb.Api.StatsController do
json(conn, %{
results: sizes,
meta: Stats.Breakdown.formatted_date_ranges(query),
- skip_imported_reason: query.skip_imported_reason
+ skip_imported_reason: meta[:imports_skip_reason]
})
end
end
@@ -1312,9 +1349,10 @@ defmodule PlausibleWeb.Api.StatsController do
metrics = [:visitors, :events, :conversion_rate] ++ @revenue_metrics
+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)
+
conversions =
- site
- |> Stats.breakdown(query, metrics, pagination)
+ results
|> transform_keys(%{goal: :name})
if params["csv"] do
@@ -1327,7 +1365,7 @@ defmodule PlausibleWeb.Api.StatsController do
json(conn, %{
results: conversions,
meta: Stats.Breakdown.formatted_date_ranges(query),
- skip_imported_reason: query.skip_imported_reason
+ skip_imported_reason: meta[:imports_skip_reason]
})
end
end
@@ -1396,14 +1434,16 @@ defmodule PlausibleWeb.Api.StatsController do
[:visitors, :events, :percentage] ++ @revenue_metrics
end
+ %{results: results, meta: meta} = Stats.breakdown(site, query, metrics, pagination)
+
props =
- Stats.breakdown(site, query, metrics, pagination)
+ results
|> transform_keys(%{prop_key => :name})
%{
results: props,
meta: Stats.Breakdown.formatted_date_ranges(query),
- skip_imported_reason: query.skip_imported_reason
+ skip_imported_reason: meta[:imports_skip_reason]
}
end
@@ -1595,14 +1635,6 @@ defmodule PlausibleWeb.Api.StatsController do
end
end
- defp includes_imported?(source_query, comparison_query) do
- cond do
- source_query.include_imported -> true
- comparison_query && comparison_query.include_imported -> true
- true -> false
- end
- end
-
defp breakdown_metrics(query, extra_metrics \\ []) do
if toplevel_goal_filter?(query) do
[:visitors, :conversion_rate, :total_visitors]
diff --git a/lib/plausible_web/live/goal_settings.ex b/lib/plausible_web/live/goal_settings.ex
index bb0ed334d8..b2486a533a 100644
--- a/lib/plausible_web/live/goal_settings.ex
+++ b/lib/plausible_web/live/goal_settings.ex
@@ -17,7 +17,6 @@ defmodule PlausibleWeb.Live.GoalSettings do
|> assign_new(:site, fn %{current_user: current_user} ->
current_user
|> Plausible.Sites.get_for_user!(domain, [:owner, :admin, :super_admin])
- |> Plausible.Imported.load_import_data()
end)
|> assign_new(:all_goals, fn %{site: site} ->
Goals.for_site(site, preload_funnels?: true)
diff --git a/lib/plausible_web/plugs/authorize_public_api.ex b/lib/plausible_web/plugs/authorize_public_api.ex
index 90f7acd00d..7cad9beaaa 100644
--- a/lib/plausible_web/plugs/authorize_public_api.ex
+++ b/lib/plausible_web/plugs/authorize_public_api.ex
@@ -65,7 +65,7 @@ defmodule PlausibleWeb.Plugs.AuthorizePublicAPI do
{:ok, site} <- find_site(conn.params["site_id"]),
:ok <- verify_site_access(api_key, site) do
Plausible.OpenTelemetry.add_site_attributes(site)
- site = Plausible.Imported.load_import_data(site)
+ site = Plausible.Repo.preload(site, :completed_imports)
{:ok, assign(conn, :site, site)}
end
end
diff --git a/lib/plausible_web/plugs/authorize_site_access.ex b/lib/plausible_web/plugs/authorize_site_access.ex
index fefa92869d..21b4af9a3b 100644
--- a/lib/plausible_web/plugs/authorize_site_access.ex
+++ b/lib/plausible_web/plugs/authorize_site_access.ex
@@ -109,9 +109,9 @@ defmodule PlausibleWeb.Plugs.AuthorizeSiteAccess do
site =
site
- |> Plausible.Imported.load_import_data()
|> Repo.preload([
:owner,
+ :completed_imports,
team: [subscription: Plausible.Teams.last_subscription_query()]
])
diff --git a/test/plausible/imported_test.exs b/test/plausible/imported_test.exs
index 49666f6523..be396e14d2 100644
--- a/test/plausible/imported_test.exs
+++ b/test/plausible/imported_test.exs
@@ -32,14 +32,56 @@ defmodule Plausible.ImportedTest do
end
end
- describe "get_imports_date_range/1" do
- test "returns empty when there are no imports" do
+ describe "earliest_import_start_date/1" do
+ test "returns nil if no site_imports exist" do
+ site = insert(:site)
+
+ assert is_nil(Imported.earliest_import_start_date(site))
+ end
+
+ test "returns nil when only incomplete or failed imports are present" do
+ site = insert(:site)
+
+ _import1 = insert(:site_import, site: site, status: :pending)
+ _import2 = insert(:site_import, site: site, status: :importing)
+ _import3 = insert(:site_import, site: site, status: :failed)
+ _rogue_import = insert(:site_import, site: build(:site), status: :completed)
+
+ assert is_nil(Imported.earliest_import_start_date(site))
+ end
+
+ test "returns start and end dates considering all imports" do
+ site = insert(:site)
+
+ _import1 =
+ insert(:site_import,
+ site: site,
+ start_date: ~D[2020-04-02],
+ end_date: ~D[2022-06-22],
+ status: :completed,
+ legacy: true
+ )
+
+ _import2 =
+ insert(:site_import,
+ site: site,
+ start_date: ~D[2022-06-22],
+ end_date: ~D[2024-01-08],
+ status: :completed
+ )
+
+ assert Imported.earliest_import_start_date(site) == ~D[2020-04-02]
+ end
+ end
+
+ describe "latest_import_end_date/1" do
+ test "returns nil if no site_imports exist" do
site = insert(:site)
- assert %{start_date: nil, end_date: nil} = Imported.get_imports_date_range(site)
+ assert is_nil(Imported.latest_import_end_date(site))
end
- test "returns empty when only incomplete or failed imports are present" do
+ test "returns nil when only incomplete or failed imports are present" do
site = insert(:site)
_import1 = insert(:site_import, site: site, status: :pending)
@@ -47,7 +89,7 @@ defmodule Plausible.ImportedTest do
_import3 = insert(:site_import, site: site, status: :failed)
_rogue_import = insert(:site_import, site: build(:site), status: :completed)
- assert %{start_date: nil, end_date: nil} = Imported.get_imports_date_range(site)
+ assert is_nil(Imported.latest_import_end_date(site))
end
test "returns start and end dates considering all imports" do
@@ -70,8 +112,7 @@ defmodule Plausible.ImportedTest do
status: :completed
)
- assert %{start_date: ~D[2020-04-02], end_date: ~D[2024-01-08]} =
- Imported.get_imports_date_range(site)
+ assert Imported.latest_import_end_date(site) == ~D[2024-01-08]
end
end
diff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exs
index bde1dfe8e5..a523dc7e87 100644
--- a/test/plausible/stats/query_parser_test.exs
+++ b/test/plausible/stats/query_parser_test.exs
@@ -48,6 +48,14 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
last: DateTime.new!(~D[2021-05-31], ~T[23:59:59], "Etc/UTC")
}
+ @default_include %{
+ imports: false,
+ imports_meta: false,
+ time_labels: false,
+ total_rows: false,
+ comparisons: nil
+ }
+
def check_success(params, site, expected_result, schema_type \\ :public) do
assert {:ok, result} = parse(site, schema_type, params, @now)
@@ -76,7 +84,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
}
@@ -111,7 +119,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
})
end
@@ -152,7 +160,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
},
:internal
@@ -218,7 +226,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
},
:internal
@@ -345,7 +353,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
})
end
@@ -370,7 +378,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
})
end
@@ -396,7 +404,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
})
end
@@ -462,7 +470,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
})
@@ -481,7 +489,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
})
end
@@ -541,7 +549,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
})
end
@@ -582,7 +590,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
},
:internal
@@ -653,7 +661,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
})
end
@@ -702,7 +710,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
})
end
@@ -755,7 +763,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
})
|> check_goals(
@@ -781,7 +789,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
})
|> check_goals(
@@ -807,7 +815,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
})
|> check_goals(
@@ -833,7 +841,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
})
|> check_goals(
@@ -862,7 +870,13 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["time"],
order_by: nil,
timezone: site.timezone,
- include: %{imports: true, time_labels: true, total_rows: true, comparisons: nil},
+ include: %{
+ imports: true,
+ imports_meta: false,
+ time_labels: true,
+ total_rows: true,
+ comparisons: nil
+ },
pagination: %{limit: 10_000, offset: 0}
})
end
@@ -923,6 +937,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
mode: "previous_period"
},
imports: false,
+ imports_meta: false,
time_labels: false,
total_rows: false
},
@@ -953,6 +968,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
mode: "year_over_year"
},
imports: false,
+ imports_meta: false,
time_labels: false,
total_rows: false
},
@@ -985,6 +1001,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
mode: "custom",
date_range: @date_range_30d
},
+ imports_meta: false,
imports: false,
time_labels: false,
total_rows: false
@@ -1045,7 +1062,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["time"],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 100, offset: 200}
})
end
@@ -1095,7 +1112,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
}
)
@@ -1215,7 +1232,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
},
:internal
@@ -1441,7 +1458,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["event:#{unquote(dimension)}"],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
})
end
@@ -1462,7 +1479,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["visit:#{unquote(dimension)}"],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
})
end
@@ -1482,7 +1499,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["event:props:foobar"],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
})
end
@@ -1543,7 +1560,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: [{:events, :desc}, {:visitors, :asc}],
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
})
end
@@ -1563,7 +1580,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["event:name"],
order_by: [{"event:name", :desc}],
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
})
end
@@ -1667,7 +1684,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
})
|> check_goals(
@@ -1696,7 +1713,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["event:goal"],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
})
|> check_goals(
@@ -1725,7 +1742,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["event:goal"],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
})
end
@@ -1789,7 +1806,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
},
:internal
@@ -1812,7 +1829,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["event:page"],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
},
:internal
@@ -1837,7 +1854,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
})
|> check_goals(
@@ -1896,7 +1913,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
}
)
@@ -1954,7 +1971,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
}
)
@@ -1988,7 +2005,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
}
)
@@ -2022,7 +2039,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
}
)
@@ -2056,7 +2073,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["event:goal"],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
}
)
@@ -2092,7 +2109,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["event:goal"],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
}
)
@@ -2130,7 +2147,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["event:goal"],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
}
)
@@ -2173,7 +2190,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["visit:device"],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
})
end
@@ -2218,7 +2235,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["event:page"],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
})
end
@@ -2237,7 +2254,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
})
end
@@ -2446,7 +2463,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ include: @default_include,
pagination: %{limit: 10_000, offset: 0}
},
:internal
diff --git a/test/plausible/stats/query_result_test.exs b/test/plausible/stats/query_result_test.exs
index fe61b7f69f..6fc4841205 100644
--- a/test/plausible/stats/query_result_test.exs
+++ b/test/plausible/stats/query_result_test.exs
@@ -1,7 +1,7 @@
defmodule Plausible.Stats.QueryResultTest do
use Plausible.DataCase, async: true
use Plausible.Teams.Test
- alias Plausible.Stats.{Query, QueryResult, QueryOptimizer}
+ alias Plausible.Stats.{Query, QueryRunner, QueryResult, QueryOptimizer}
setup do
user = insert(:user)
@@ -33,7 +33,8 @@ defmodule Plausible.Stats.QueryResultTest do
query = QueryOptimizer.optimize(query)
query_result_json =
- QueryResult.from([], site, query, %{})
+ %QueryRunner{site: site, results: [], main_query: query}
+ |> QueryResult.from()
|> Jason.encode!(pretty: true)
|> String.replace(site.domain, "dummy.site")
diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exs
index fc148aa7c8..5834931f8b 100644
--- a/test/plausible/stats/query_test.exs
+++ b/test/plausible/stats/query_test.exs
@@ -266,7 +266,6 @@ defmodule Plausible.Stats.QueryTest do
test "is true when requested via params and imported data exists", %{site: site} do
insert(:site_import, site: site)
- site = Plausible.Imported.load_import_data(site)
assert %{include_imported: true} =
Query.from(site, %{"period" => "day", "with_imported" => "true"})
@@ -279,7 +278,6 @@ defmodule Plausible.Stats.QueryTest do
test "is false when imported data exists but is out of the date range", %{site: site} do
insert(:site_import, site: site, start_date: ~D[2021-01-01], end_date: ~D[2022-01-01])
- site = Plausible.Imported.load_import_data(site)
assert %{include_imported: false, skip_imported_reason: :out_of_range} =
Query.from(site, %{"period" => "day", "with_imported" => "true"})
@@ -287,15 +285,13 @@ defmodule Plausible.Stats.QueryTest do
test "is false in realtime even when imported data from today exists", %{site: site} do
insert(:site_import, site: site)
- site = Plausible.Imported.load_import_data(site)
- assert %{include_imported: false, skip_imported_reason: :unsupported_query} =
+ assert %{include_imported: false, skip_imported_reason: :out_of_range} =
Query.from(site, %{"period" => "realtime", "with_imported" => "true"})
end
test "is false when an arbitrary custom property filter is used", %{site: site} do
insert(:site_import, site: site)
- site = Plausible.Imported.load_import_data(site)
assert %{include_imported: false, skip_imported_reason: :unsupported_query} =
Query.from(site, %{
@@ -309,7 +305,6 @@ defmodule Plausible.Stats.QueryTest do
test "is true when breaking down by url and filtering by outbound link or file download goal",
%{site: site} do
insert(:site_import, site: site)
- site = Plausible.Imported.load_import_data(site)
Enum.each(["Outbound Link: Click", "File Download"], fn goal_name ->
insert(:goal, site: site, event_name: goal_name)
@@ -327,7 +322,6 @@ defmodule Plausible.Stats.QueryTest do
test "is false when breaking down by url but without a special goal filter",
%{site: site} do
insert(:site_import, site: site)
- site = Plausible.Imported.load_import_data(site)
assert %{include_imported: false} =
Query.from(site, %{
@@ -341,7 +335,6 @@ defmodule Plausible.Stats.QueryTest do
%{site: site} do
insert(:site_import, site: site)
insert(:goal, site: site, event_name: "404")
- site = Plausible.Imported.load_import_data(site)
assert %{include_imported: false} =
Query.from(site, %{
@@ -356,7 +349,6 @@ defmodule Plausible.Stats.QueryTest do
%{site: site} do
insert(:site_import, site: site)
insert(:goal, site: site, event_name: "Outbound Link: Click")
- site = Plausible.Imported.load_import_data(site)
assert %{include_imported: false} =
Query.from(site, %{
@@ -376,7 +368,6 @@ defmodule Plausible.Stats.QueryTest do
%{site: site} do
insert(:site_import, site: site)
insert(:goal, site: site, event_name: "Outbound Link: Click")
- site = Plausible.Imported.load_import_data(site)
assert %{include_imported: true} =
Query.from(site, %{
@@ -397,7 +388,6 @@ defmodule Plausible.Stats.QueryTest do
} do
insert(:site_import, site: site)
insert(:goal, site: site, event_name: "Outbound Link: Click")
- site = Plausible.Imported.load_import_data(site)
assert %{include_imported: true} =
Query.from(site, %{
@@ -421,7 +411,6 @@ defmodule Plausible.Stats.QueryTest do
} do
insert(:site_import, site: site)
insert(:goal, site: site, event_name: "Outbound Link: Click")
- site = Plausible.Imported.load_import_data(site)
assert %{include_imported: false} =
Query.from(site, %{
@@ -441,7 +430,6 @@ defmodule Plausible.Stats.QueryTest do
%{site: site} do
insert(:site_import, site: site)
insert(:goal, site: site, event_name: "404")
- site = Plausible.Imported.load_import_data(site)
assert %{include_imported: false} =
Query.from(site, %{
@@ -460,7 +448,6 @@ defmodule Plausible.Stats.QueryTest do
%{site: site} do
insert(:site_import, site: site)
insert(:goal, site: site, event_name: "Outbound Link: Click")
- site = Plausible.Imported.load_import_data(site)
assert %{include_imported: false} =
Query.from(site, %{
@@ -479,7 +466,6 @@ defmodule Plausible.Stats.QueryTest do
test "is false with a custom prop filter and non-matching property", %{site: site} do
insert(:site_import, site: site)
insert(:goal, site: site, event_name: "Outbound Link: Click")
- site = Plausible.Imported.load_import_data(site)
assert %{include_imported: false} =
Query.from(site, %{
diff --git a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
index 8e9c779e49..3fb2558aa3 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
@@ -378,10 +378,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
})
assert json_response(conn, 200)["results"] == %{
- "pageviews" => %{"value" => 3, "change" => 200},
- "visitors" => %{"value" => 2, "change" => 100},
- "bounce_rate" => %{"value" => 50, "change" => -50},
- "visit_duration" => %{"value" => 750, "change" => 100}
+ "pageviews" => %{"value" => 3, "change" => 200, "comparison_value" => 1},
+ "visitors" => %{"value" => 2, "change" => 100, "comparison_value" => 1},
+ "bounce_rate" => %{"value" => 50, "change" => -50, "comparison_value" => 100},
+ "visit_duration" => %{"value" => 750, "change" => 100, "comparison_value" => 0}
}
end
@@ -409,10 +409,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
})
assert json_response(conn, 200)["results"] == %{
- "pageviews" => %{"value" => 4, "change" => 100},
- "visitors" => %{"value" => 3, "change" => 100},
- "bounce_rate" => %{"value" => 100, "change" => nil},
- "visit_duration" => %{"value" => 0, "change" => 0}
+ "pageviews" => %{"value" => 4, "change" => 100, "comparison_value" => 0},
+ "visitors" => %{"value" => 3, "change" => 100, "comparison_value" => 0},
+ "bounce_rate" => %{"value" => 100, "change" => nil, "comparison_value" => 0},
+ "visit_duration" => %{"value" => 0, "change" => 0, "comparison_value" => 0}
}
end
@@ -441,7 +441,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
})
assert json_response(conn, 200)["results"] == %{
- "conversion_rate" => %{"value" => 50.0, "change" => 16.7}
+ "conversion_rate" => %{
+ "value" => 50.0,
+ "change" => 16.7,
+ "comparison_value" => 33.3
+ }
}
end
@@ -466,7 +470,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
})
assert json_response(conn, 200)["results"] == %{
- "time_on_page" => %{"value" => 90, "change" => 50.0}
+ "time_on_page" => %{"value" => 90, "change" => 50.0, "comparison_value" => 60}
}
end
@@ -491,7 +495,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
})
assert json_response(conn, 200)["results"] == %{
- "time_on_page" => %{"value" => nil, "change" => nil}
+ "time_on_page" => %{"value" => nil, "change" => nil, "comparison_value" => 60}
}
end
end
@@ -565,12 +569,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
})
assert json_response(conn, 200)["results"] == %{
- "visitors" => %{"value" => 2, "change" => 100},
- "visits" => %{"value" => 5, "change" => 150},
- "pageviews" => %{"value" => 9, "change" => -10},
- "bounce_rate" => %{"value" => 40, "change" => -10},
- "views_per_visit" => %{"value" => 1.8, "change" => -64},
- "visit_duration" => %{"value" => 20, "change" => -80}
+ "visitors" => %{"value" => 2, "change" => 100, "comparison_value" => 1},
+ "visits" => %{"value" => 5, "change" => 150, "comparison_value" => 2},
+ "pageviews" => %{"value" => 9, "change" => -10, "comparison_value" => 10},
+ "bounce_rate" => %{"value" => 40, "change" => -10, "comparison_value" => 50},
+ "views_per_visit" => %{"value" => 1.8, "change" => -64, "comparison_value" => 5.0},
+ "visit_duration" => %{"value" => 20, "change" => -80, "comparison_value" => 100}
}
end
@@ -600,7 +604,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
})
assert json_response(conn, 200)["results"] == %{
- "visitors" => %{"value" => 1, "change" => -67}
+ "visitors" => %{"value" => 1, "change" => -67, "comparison_value" => 3}
}
end
@@ -629,7 +633,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
})
assert json_response(conn, 200)["results"] == %{
- "visitors" => %{"change" => 100, "value" => 1}
+ "visitors" => %{"change" => 100, "value" => 1, "comparison_value" => 0}
}
assert json_response(conn, 200)["warning"] =~
diff --git a/test/plausible_web/controllers/site_controller_test.exs b/test/plausible_web/controllers/site_controller_test.exs
index a55fd1e545..9f3f2a029f 100644
--- a/test/plausible_web/controllers/site_controller_test.exs
+++ b/test/plausible_web/controllers/site_controller_test.exs
@@ -1592,8 +1592,6 @@ defmodule PlausibleWeb.SiteControllerTest do
assert Enum.find(imports, &(&1.id == import_id))
- site = Plausible.Imported.load_import_data(site)
-
assert eventually(fn ->
count = Plausible.Stats.Clickhouse.imported_pageview_count(site)
{count == 22, count}