Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Support `with_imported=true` in Stats API aggregate (#2919)
RobertJoonasfa54efbc6d2c
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 2791742a15..f3b37ff6e9 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -4,6 +4,7 @@ All notable changes to this project will be documented in this file.## Unreleased### Added+- Add support for `with_imported=true` in Stats API aggregate endpoint- Ability to use '--' instead of '=' sign in the `tagged-events` classnames- 'Last updated X seconds ago' info to 'current visitors' tooltips- Add support for more Bamboo adapters, i.e. `Bamboo.MailgunAdapter`, `Bamboo.MandrillAdapter`, `Bamboo.SendGridAdapter` plausible/analytics#2649diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.exindex 1c0dcc775d..d628d76128 100644--- a/lib/plausible/stats/aggregate.ex+++ b/lib/plausible/stats/aggregate.ex@@ -1,7 +1,7 @@defmodule Plausible.Stats.Aggregate doalias Plausible.Stats.Queryuse Plausible.ClickhouseRepo- import Plausible.Stats.{Base, Imported}+ import Plausible.Stats.{Base, Imported, Util}@event_metrics [:visitors, :pageviews, :events, :sample_percent]@session_metrics [:visits, :bounce_rate, :visit_duration, :views_per_visit, :sample_percent]@@ -43,6 +43,7 @@ defmodule Plausible.Stats.Aggregate do|> select_session_metrics(metrics, query)|> merge_imported(site, query, :aggregate, metrics)|> ClickhouseRepo.one()+ |> remove_internal_visits_metric()enddefp aggregate_time_on_page(site, query) dodiff --git a/lib/plausible/stats/comparisons.ex b/lib/plausible/stats/comparisons.exindex 4e50d408b0..cc15ecd477 100644--- a/lib/plausible/stats/comparisons.ex+++ b/lib/plausible/stats/comparisons.ex@@ -53,20 +53,16 @@ defmodule Plausible.Stats.Comparisons doit will be shifted to start on Sunday, January 2nd, 2022 instead ofJanuary 1st. Defaults to false.- * `:include_imported?` - includes imported data in the generated comparison- query if there is imported data. Defaults to true.-"""def compare(%Plausible.Site{} = site, %Stats.Query{} = source_query, mode, opts \\ []) doopts =opts|> Keyword.put_new(:now, Timex.now(site.timezone))|> Keyword.put_new(:match_day_of_week?, false)- |> Keyword.put_new(:include_imported?, true)with :ok <- validate_mode(source_query, mode),{:ok, comparison_query} <- do_compare(source_query, mode, opts),- comparison_query <- maybe_include_imported(comparison_query, site, opts),+ comparison_query <- maybe_include_imported(comparison_query, source_query, site),do: {:ok, comparison_query}end@@ -165,8 +161,9 @@ defmodule Plausible.Stats.Comparisons doDate.add(date, -days_to_subtract)end- defp maybe_include_imported(query, site, opts) do- include_imported? = Stats.Query.include_imported?(query, site, opts[:include_imported?])+ defp maybe_include_imported(query, source_query, site) do+ requested? = source_query.imported_data_requested+ include_imported? = Stats.Query.include_imported?(query, site, requested?)%Stats.Query{query | include_imported: include_imported?}enddiff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.exindex 12221a126f..4dcb6401af 100644--- a/lib/plausible/stats/query.ex+++ b/lib/plausible/stats/query.ex@@ -4,6 +4,7 @@ defmodule Plausible.Stats.Query doperiod: nil,filters: %{},sample_threshold: 20_000_000,+ imported_data_requested: false,include_imported: false@default_sample_threshold 20_000_000@@ -20,8 +21,7 @@ defmodule Plausible.Stats.Query dointerval: params["interval"] || Interval.default_for_period(params["period"]),date_range: Date.range(date, date),filters: FilterParser.parse_filters(params["filters"]),- sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold),- include_imported: false+ sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)}end@@ -35,7 +35,7 @@ defmodule Plausible.Stats.Query dofilters: FilterParser.parse_filters(params["filters"]),sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)}- |> maybe_include_imported(site, params)+ |> put_imported_opts(site, params)enddef from(site, %{"period" => "7d"} = params) do@@ -49,7 +49,7 @@ defmodule Plausible.Stats.Query dofilters: FilterParser.parse_filters(params["filters"]),sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)}- |> maybe_include_imported(site, params)+ |> put_imported_opts(site, params)enddef from(site, %{"period" => "30d"} = params) do@@ -63,7 +63,7 @@ defmodule Plausible.Stats.Query dofilters: FilterParser.parse_filters(params["filters"]),sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)}- |> maybe_include_imported(site, params)+ |> put_imported_opts(site, params)enddef from(site, %{"period" => "month"} = params) do@@ -79,7 +79,7 @@ defmodule Plausible.Stats.Query dofilters: FilterParser.parse_filters(params["filters"]),sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)}- |> maybe_include_imported(site, params)+ |> put_imported_opts(site, params)enddef from(site, %{"period" => "6mo"} = params) do@@ -98,7 +98,7 @@ defmodule Plausible.Stats.Query dofilters: FilterParser.parse_filters(params["filters"]),sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)}- |> maybe_include_imported(site, params)+ |> put_imported_opts(site, params)enddef from(site, %{"period" => "12mo"} = params) do@@ -117,7 +117,7 @@ defmodule Plausible.Stats.Query dofilters: FilterParser.parse_filters(params["filters"]),sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)}- |> maybe_include_imported(site, params)+ |> put_imported_opts(site, params)enddef from(site, %{"period" => "year"} = params) do@@ -134,7 +134,7 @@ defmodule Plausible.Stats.Query dofilters: FilterParser.parse_filters(params["filters"]),sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)}- |> maybe_include_imported(site, params)+ |> put_imported_opts(site, params)enddef from(site, %{"period" => "all"} = params) do@@ -194,7 +194,7 @@ defmodule Plausible.Stats.Query dofilters: FilterParser.parse_filters(params["filters"]),sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)}- |> maybe_include_imported(site, params)+ |> put_imported_opts(site, params)enddef from(tz, params) do@@ -248,9 +248,12 @@ defmodule Plausible.Stats.Query doendend- defp maybe_include_imported(query, site, params) do+ defp put_imported_opts(query, site, params) dorequested? = params["with_imported"] == "true"- %{query | include_imported: include_imported?(query, site, requested?)}++ query+ |> Map.put(:imported_data_requested, requested?)+ |> Map.put(:include_imported, include_imported?(query, site, requested?))end@spec include_imported?(t(), Plausible.Site.t(), boolean()) :: boolean()diff --git a/lib/plausible/stats/util.ex b/lib/plausible/stats/util.exindex 71a82d4f99..0465506608 100644--- a/lib/plausible/stats/util.ex+++ b/lib/plausible/stats/util.ex@@ -8,12 +8,16 @@ defmodule Plausible.Stats.Util dois needed to calculate these from imported data. This function removes that metricfrom all entries in the results list."""- def remove_internal_visits_metric(results, metrics) do+ def remove_internal_visits_metric(results, metrics) when is_list(results) doif :bounce_rate in metrics or :visit_duration in metrics doresults- |> Enum.map(&Map.delete(&1, :__internal_visits))+ |> Enum.map(&remove_internal_visits_metric/1)elseresultsendend++ def remove_internal_visits_metric(result) when is_map(result) do+ Map.delete(result, :__internal_visits)+ endenddiff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.exindex 6256c057e2..227543fc63 100644--- a/lib/plausible_web/controllers/api/external_stats_controller.ex+++ b/lib/plausible_web/controllers/api/external_stats_controller.ex@@ -42,10 +42,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController doPlausible.Stats.aggregate(site, query, metrics)end- results =- results- |> Map.take(metrics)-json(conn, %{results: results})else{:error, msg} ->@@ -141,6 +137,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController doend)end+ defp validate_metric("events", nil, %{include_imported: true}) do+ {:error, "Metric `events` cannot be queried with imported data"}+ end+defp validate_metric(metric, _, _) when metric in @event_metrics, do: {:ok, metric}defp validate_metric(metric, property, query) when metric in @session_metrics dodiff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex 93d63ff822..bba59a5f2e 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -1304,8 +1304,7 @@ defmodule PlausibleWeb.Api.StatsController do[from: params["compare_from"],to: params["compare_to"],- match_day_of_week?: params["match_day_of_week"] == "true",- include_imported?: params["with_imported"] == "true"+ match_day_of_week?: params["match_day_of_week"] == "true"]enddiff --git a/test/plausible/stats/comparisons_test.exs b/test/plausible/stats/comparisons_test.exsindex 657c1894f2..c0a8e8d0e4 100644--- a/test/plausible/stats/comparisons_test.exs+++ b/test/plausible/stats/comparisons_test.exs@@ -1,6 +1,7 @@defmodule Plausible.Stats.ComparisonsTest douse Plausible.DataCasealias Plausible.Stats.{Query, Comparisons}+ import Plausible.TestUtilsdescribe "with period set to this month" dotest "shifts back this month period when mode is previous_period" do@@ -200,4 +201,16 @@ defmodule Plausible.Stats.ComparisonsTest doComparisons.compare(site, query, "custom", from: "2022-05-30", to: "2022-05-25")endend++ describe "include_imported" do+ setup [:create_user, :create_new_site, :add_imported_data]++ test "defaults to source_query.include_imported", %{site: site} do+ query = Query.from(site, %{"period" => "day", "date" => "2023-01-01"})+ assert query.include_imported == false++ {:ok, comparison_query} = Comparisons.compare(site, query, "previous_period")+ assert comparison_query.include_imported == false+ end+ endenddiff --git a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exsindex 2236e324a4..8f7ff2ec78 100644--- a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs@@ -1,5 +1,6 @@defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest douse PlausibleWeb.ConnCase+ import Plausible.TestUtilssetup [:create_user, :create_new_site, :create_api_key, :use_api_key]@user_id 123@@ -250,6 +251,116 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest doendend+ describe "with imported data" do+ setup :add_imported_data++ test "does not count imported stats unless specified", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:imported_visitors, date: ~D[2023-01-01]),+ build(:pageview, timestamp: ~N[2023-01-01 00:00:00])+ ])++ query_params = %{+ "site_id" => site.domain,+ "period" => "day",+ "date" => "2023-01-01",+ "metrics" => "pageviews"+ }++ conn1 = get(conn, "/api/v1/stats/aggregate", query_params)++ assert json_response(conn1, 200)["results"] == %{+ "pageviews" => %{"value" => 1}+ }++ conn2 = get(conn, "/api/v1/stats/aggregate", Map.put(query_params, "with_imported", "true"))++ assert json_response(conn2, 200)["results"] == %{+ "pageviews" => %{"value" => 2}+ }+ end++ test "counts imported stats when comparing with previous period", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:imported_visitors,+ visits: 2,+ bounces: 1,+ visit_duration: 200,+ pageviews: 10,+ date: ~D[2023-01-01]+ ),+ build(:imported_visitors,+ visits: 4,+ bounces: 1,+ visit_duration: 100,+ pageviews: 8,+ date: ~D[2023-01-02]+ ),+ build(:pageview, timestamp: ~N[2023-01-02 00:10:00])+ ])++ conn =+ get(conn, "/api/v1/stats/aggregate", %{+ "site_id" => site.domain,+ "period" => "custom",+ "date" => "2023-01-02,2023-01-02",+ "metrics" => "visitors,visits,pageviews,views_per_visit,bounce_rate,visit_duration",+ "compare" => "previous_period",+ "with_imported" => "true"+ })++ assert json_response(conn, 200)["results"] == %{+ "visitors" => %{"value" => 2, "change" => 100},+ "visits" => %{"value" => 5, "change" => 150},+ "pageviews" => %{"value" => 9, "change" => -10},+ "bounce_rate" => %{"value" => 40, "change" => -20},+ "views_per_visit" => %{"value" => 1.0, "change" => 100},+ "visit_duration" => %{"value" => 20, "change" => -80}+ }+ end++ test "ignores imported data when filters are applied", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:imported_visitors, date: ~D[2023-01-01]),+ build(:imported_sources, date: ~D[2023-01-01]),+ build(:pageview, referrer_source: "Google", timestamp: ~N[2023-01-02 00:10:00])+ ])++ conn =+ get(conn, "/api/v1/stats/aggregate", %{+ "site_id" => site.domain,+ "period" => "day",+ "date" => "2023-01-02",+ "metrics" => "visitors",+ "filters" => "visit:source==Google",+ "compare" => "previous_period",+ "with_imported" => "true"+ })++ assert json_response(conn, 200)["results"] == %{+ "visitors" => %{"value" => 1, "change" => 100}+ }+ end++ test "events metric with imported data is disallowed", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:imported_visitors, date: ~D[2023-01-01])+ ])++ conn =+ get(conn, "/api/v1/stats/aggregate", %{+ "site_id" => site.domain,+ "period" => "day",+ "date" => "2023-01-02",+ "metrics" => "events",+ "with_imported" => "true"+ })++ assert %{"error" => msg} = json_response(conn, 400)+ assert msg == "Metric `events` cannot be queried with imported data"+ end+ end+describe "filters" dotest "can filter by source", %{conn: conn, site: site} dopopulate_stats(site, [