Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Support `with_imported=true` in Stats API aggregate (#2919)

RobertJoonasfa54efbc6d2c
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2791742a15..f3b37ff6e9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,7 @@ All notable changes to this project will be documented in this file.
## Unreleased
### Added
+- Add support for `with_imported=true` in Stats API aggregate endpoint
- Ability to use '--' instead of '=' sign in the `tagged-events` classnames
- 'Last updated X seconds ago' info to 'current visitors' tooltips
- Add support for more Bamboo adapters, i.e. `Bamboo.MailgunAdapter`, `Bamboo.MandrillAdapter`, `Bamboo.SendGridAdapter` plausible/analytics#2649
diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex
index 1c0dcc775d..d628d76128 100644
--- a/lib/plausible/stats/aggregate.ex
+++ b/lib/plausible/stats/aggregate.ex
@@ -1,7 +1,7 @@
defmodule Plausible.Stats.Aggregate do
alias Plausible.Stats.Query
use Plausible.ClickhouseRepo
- import Plausible.Stats.{Base, Imported}
+ import Plausible.Stats.{Base, Imported, Util}
@event_metrics [:visitors, :pageviews, :events, :sample_percent]
@session_metrics [:visits, :bounce_rate, :visit_duration, :views_per_visit, :sample_percent]
@@ -43,6 +43,7 @@ defmodule Plausible.Stats.Aggregate do
|> select_session_metrics(metrics, query)
|> merge_imported(site, query, :aggregate, metrics)
|> ClickhouseRepo.one()
+ |> remove_internal_visits_metric()
end
defp aggregate_time_on_page(site, query) do
diff --git a/lib/plausible/stats/comparisons.ex b/lib/plausible/stats/comparisons.ex
index 4e50d408b0..cc15ecd477 100644
--- a/lib/plausible/stats/comparisons.ex
+++ b/lib/plausible/stats/comparisons.ex
@@ -53,20 +53,16 @@ defmodule Plausible.Stats.Comparisons do
it will be shifted to start on Sunday, January 2nd, 2022 instead of
January 1st. Defaults to false.
- * `:include_imported?` - includes imported data in the generated comparison
- query if there is imported data. Defaults to true.
-
"""
def compare(%Plausible.Site{} = site, %Stats.Query{} = source_query, mode, opts \\ []) do
opts =
opts
|> Keyword.put_new(:now, Timex.now(site.timezone))
|> Keyword.put_new(:match_day_of_week?, false)
- |> Keyword.put_new(:include_imported?, true)
with :ok <- validate_mode(source_query, mode),
{:ok, comparison_query} <- do_compare(source_query, mode, opts),
- comparison_query <- maybe_include_imported(comparison_query, site, opts),
+ comparison_query <- maybe_include_imported(comparison_query, source_query, site),
do: {:ok, comparison_query}
end
@@ -165,8 +161,9 @@ defmodule Plausible.Stats.Comparisons do
Date.add(date, -days_to_subtract)
end
- defp maybe_include_imported(query, site, opts) do
- include_imported? = Stats.Query.include_imported?(query, site, opts[:include_imported?])
+ defp maybe_include_imported(query, source_query, site) do
+ requested? = source_query.imported_data_requested
+ include_imported? = Stats.Query.include_imported?(query, site, requested?)
%Stats.Query{query | include_imported: include_imported?}
end
diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex
index 12221a126f..4dcb6401af 100644
--- a/lib/plausible/stats/query.ex
+++ b/lib/plausible/stats/query.ex
@@ -4,6 +4,7 @@ defmodule Plausible.Stats.Query do
period: nil,
filters: %{},
sample_threshold: 20_000_000,
+ imported_data_requested: false,
include_imported: false
@default_sample_threshold 20_000_000
@@ -20,8 +21,7 @@ defmodule Plausible.Stats.Query do
interval: params["interval"] || Interval.default_for_period(params["period"]),
date_range: Date.range(date, date),
filters: FilterParser.parse_filters(params["filters"]),
- sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold),
- include_imported: false
+ sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)
}
end
@@ -35,7 +35,7 @@ defmodule Plausible.Stats.Query do
filters: FilterParser.parse_filters(params["filters"]),
sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)
}
- |> maybe_include_imported(site, params)
+ |> put_imported_opts(site, params)
end
def from(site, %{"period" => "7d"} = params) do
@@ -49,7 +49,7 @@ defmodule Plausible.Stats.Query do
filters: FilterParser.parse_filters(params["filters"]),
sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)
}
- |> maybe_include_imported(site, params)
+ |> put_imported_opts(site, params)
end
def from(site, %{"period" => "30d"} = params) do
@@ -63,7 +63,7 @@ defmodule Plausible.Stats.Query do
filters: FilterParser.parse_filters(params["filters"]),
sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)
}
- |> maybe_include_imported(site, params)
+ |> put_imported_opts(site, params)
end
def from(site, %{"period" => "month"} = params) do
@@ -79,7 +79,7 @@ defmodule Plausible.Stats.Query do
filters: FilterParser.parse_filters(params["filters"]),
sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)
}
- |> maybe_include_imported(site, params)
+ |> put_imported_opts(site, params)
end
def from(site, %{"period" => "6mo"} = params) do
@@ -98,7 +98,7 @@ defmodule Plausible.Stats.Query do
filters: FilterParser.parse_filters(params["filters"]),
sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)
}
- |> maybe_include_imported(site, params)
+ |> put_imported_opts(site, params)
end
def from(site, %{"period" => "12mo"} = params) do
@@ -117,7 +117,7 @@ defmodule Plausible.Stats.Query do
filters: FilterParser.parse_filters(params["filters"]),
sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)
}
- |> maybe_include_imported(site, params)
+ |> put_imported_opts(site, params)
end
def from(site, %{"period" => "year"} = params) do
@@ -134,7 +134,7 @@ defmodule Plausible.Stats.Query do
filters: FilterParser.parse_filters(params["filters"]),
sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)
}
- |> maybe_include_imported(site, params)
+ |> put_imported_opts(site, params)
end
def from(site, %{"period" => "all"} = params) do
@@ -194,7 +194,7 @@ defmodule Plausible.Stats.Query do
filters: FilterParser.parse_filters(params["filters"]),
sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)
}
- |> maybe_include_imported(site, params)
+ |> put_imported_opts(site, params)
end
def from(tz, params) do
@@ -248,9 +248,12 @@ defmodule Plausible.Stats.Query do
end
end
- defp maybe_include_imported(query, site, params) do
+ defp put_imported_opts(query, site, params) do
requested? = params["with_imported"] == "true"
- %{query | include_imported: include_imported?(query, site, requested?)}
+
+ query
+ |> Map.put(:imported_data_requested, requested?)
+ |> Map.put(:include_imported, include_imported?(query, site, requested?))
end
@spec include_imported?(t(), Plausible.Site.t(), boolean()) :: boolean()
diff --git a/lib/plausible/stats/util.ex b/lib/plausible/stats/util.ex
index 71a82d4f99..0465506608 100644
--- a/lib/plausible/stats/util.ex
+++ b/lib/plausible/stats/util.ex
@@ -8,12 +8,16 @@ defmodule Plausible.Stats.Util do
is needed to calculate these from imported data. This function removes that metric
from all entries in the results list.
"""
- def remove_internal_visits_metric(results, metrics) do
+ def remove_internal_visits_metric(results, metrics) when is_list(results) do
if :bounce_rate in metrics or :visit_duration in metrics do
results
- |> Enum.map(&Map.delete(&1, :__internal_visits))
+ |> Enum.map(&remove_internal_visits_metric/1)
else
results
end
end
+
+ def remove_internal_visits_metric(result) when is_map(result) do
+ Map.delete(result, :__internal_visits)
+ end
end
diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex
index 6256c057e2..227543fc63 100644
--- a/lib/plausible_web/controllers/api/external_stats_controller.ex
+++ b/lib/plausible_web/controllers/api/external_stats_controller.ex
@@ -42,10 +42,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
Plausible.Stats.aggregate(site, query, metrics)
end
- results =
- results
- |> Map.take(metrics)
-
json(conn, %{results: results})
else
{:error, msg} ->
@@ -141,6 +137,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
end)
end
+ defp validate_metric("events", nil, %{include_imported: true}) do
+ {:error, "Metric `events` cannot be queried with imported data"}
+ end
+
defp validate_metric(metric, _, _) when metric in @event_metrics, do: {:ok, metric}
defp validate_metric(metric, property, query) when metric in @session_metrics do
diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
index 93d63ff822..bba59a5f2e 100644
--- a/lib/plausible_web/controllers/api/stats_controller.ex
+++ b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -1304,8 +1304,7 @@ defmodule PlausibleWeb.Api.StatsController do
[
from: params["compare_from"],
to: params["compare_to"],
- match_day_of_week?: params["match_day_of_week"] == "true",
- include_imported?: params["with_imported"] == "true"
+ match_day_of_week?: params["match_day_of_week"] == "true"
]
end
diff --git a/test/plausible/stats/comparisons_test.exs b/test/plausible/stats/comparisons_test.exs
index 657c1894f2..c0a8e8d0e4 100644
--- a/test/plausible/stats/comparisons_test.exs
+++ b/test/plausible/stats/comparisons_test.exs
@@ -1,6 +1,7 @@
defmodule Plausible.Stats.ComparisonsTest do
use Plausible.DataCase
alias Plausible.Stats.{Query, Comparisons}
+ import Plausible.TestUtils
describe "with period set to this month" do
test "shifts back this month period when mode is previous_period" do
@@ -200,4 +201,16 @@ defmodule Plausible.Stats.ComparisonsTest do
Comparisons.compare(site, query, "custom", from: "2022-05-30", to: "2022-05-25")
end
end
+
+ describe "include_imported" do
+ setup [:create_user, :create_new_site, :add_imported_data]
+
+ test "defaults to source_query.include_imported", %{site: site} do
+ query = Query.from(site, %{"period" => "day", "date" => "2023-01-01"})
+ assert query.include_imported == false
+
+ {:ok, comparison_query} = Comparisons.compare(site, query, "previous_period")
+ assert comparison_query.include_imported == false
+ end
+ end
end
diff --git a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
index 2236e324a4..8f7ff2ec78 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
@@ -1,5 +1,6 @@
defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
use PlausibleWeb.ConnCase
+ import Plausible.TestUtils
setup [:create_user, :create_new_site, :create_api_key, :use_api_key]
@user_id 123
@@ -250,6 +251,116 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
end
+ describe "with imported data" do
+ setup :add_imported_data
+
+ test "does not count imported stats unless specified", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:imported_visitors, date: ~D[2023-01-01]),
+ build(:pageview, timestamp: ~N[2023-01-01 00:00:00])
+ ])
+
+ query_params = %{
+ "site_id" => site.domain,
+ "period" => "day",
+ "date" => "2023-01-01",
+ "metrics" => "pageviews"
+ }
+
+ conn1 = get(conn, "/api/v1/stats/aggregate", query_params)
+
+ assert json_response(conn1, 200)["results"] == %{
+ "pageviews" => %{"value" => 1}
+ }
+
+ conn2 = get(conn, "/api/v1/stats/aggregate", Map.put(query_params, "with_imported", "true"))
+
+ assert json_response(conn2, 200)["results"] == %{
+ "pageviews" => %{"value" => 2}
+ }
+ end
+
+ test "counts imported stats when comparing with previous period", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:imported_visitors,
+ visits: 2,
+ bounces: 1,
+ visit_duration: 200,
+ pageviews: 10,
+ date: ~D[2023-01-01]
+ ),
+ build(:imported_visitors,
+ visits: 4,
+ bounces: 1,
+ visit_duration: 100,
+ pageviews: 8,
+ date: ~D[2023-01-02]
+ ),
+ build(:pageview, timestamp: ~N[2023-01-02 00:10:00])
+ ])
+
+ conn =
+ get(conn, "/api/v1/stats/aggregate", %{
+ "site_id" => site.domain,
+ "period" => "custom",
+ "date" => "2023-01-02,2023-01-02",
+ "metrics" => "visitors,visits,pageviews,views_per_visit,bounce_rate,visit_duration",
+ "compare" => "previous_period",
+ "with_imported" => "true"
+ })
+
+ assert json_response(conn, 200)["results"] == %{
+ "visitors" => %{"value" => 2, "change" => 100},
+ "visits" => %{"value" => 5, "change" => 150},
+ "pageviews" => %{"value" => 9, "change" => -10},
+ "bounce_rate" => %{"value" => 40, "change" => -20},
+ "views_per_visit" => %{"value" => 1.0, "change" => 100},
+ "visit_duration" => %{"value" => 20, "change" => -80}
+ }
+ end
+
+ test "ignores imported data when filters are applied", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:imported_visitors, date: ~D[2023-01-01]),
+ build(:imported_sources, date: ~D[2023-01-01]),
+ build(:pageview, referrer_source: "Google", timestamp: ~N[2023-01-02 00:10:00])
+ ])
+
+ conn =
+ get(conn, "/api/v1/stats/aggregate", %{
+ "site_id" => site.domain,
+ "period" => "day",
+ "date" => "2023-01-02",
+ "metrics" => "visitors",
+ "filters" => "visit:source==Google",
+ "compare" => "previous_period",
+ "with_imported" => "true"
+ })
+
+ assert json_response(conn, 200)["results"] == %{
+ "visitors" => %{"value" => 1, "change" => 100}
+ }
+ end
+
+ test "events metric with imported data is disallowed", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:imported_visitors, date: ~D[2023-01-01])
+ ])
+
+ conn =
+ get(conn, "/api/v1/stats/aggregate", %{
+ "site_id" => site.domain,
+ "period" => "day",
+ "date" => "2023-01-02",
+ "metrics" => "events",
+ "with_imported" => "true"
+ })
+
+ assert %{"error" => msg} = json_response(conn, 400)
+ assert msg == "Metric `events` cannot be queried with imported data"
+ end
+ end
+
describe "filters" do
test "can filter by source", %{conn: conn, site: site} do
populate_stats(site, [