Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

A few more refactoring cleanups (#6471)

RobertJoonas1c539f6bf022
diff --git a/lib/plausible/stats.ex b/lib/plausible/stats.ex
index 495c020491..2dd9229dd2 100644
--- a/lib/plausible/stats.ex
+++ b/lib/plausible/stats.ex
@@ -1,11 +1,10 @@
defmodule Plausible.Stats do
+ @moduledoc false
+
use Plausible
use Plausible.ClickhouseRepo
alias Plausible.Stats.{
- Breakdown,
- Aggregate,
- Timeseries,
CurrentVisitors,
FilterSuggestions,
QueryRunner
@@ -15,18 +14,6 @@ defmodule Plausible.Stats do
QueryRunner.run(site, query)
end
- def breakdown(site, query, metrics, pagination) do
- Breakdown.breakdown(site, query, metrics, pagination)
- end
-
- def aggregate(site, query, metrics) do
- Aggregate.aggregate(site, query, metrics)
- end
-
- def timeseries(site, query, metrics) do
- Timeseries.timeseries(site, query, metrics)
- end
-
def current_visitors(site, duration \\ Duration.new!(minute: -5)) do
CurrentVisitors.current_visitors(site, duration)
end
diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/legacy/aggregate.ex
similarity index 89%
rename from lib/plausible/stats/aggregate.ex
rename to lib/plausible/stats/legacy/aggregate.ex
index ae2e80e7c9..cc633af968 100644
--- a/lib/plausible/stats/aggregate.ex
+++ b/lib/plausible/stats/legacy/aggregate.ex
@@ -1,8 +1,6 @@
-defmodule Plausible.Stats.Aggregate do
+defmodule Plausible.Stats.Legacy.Aggregate do
@moduledoc """
- Builds aggregate results for v1 of our stats API and dashboards.
-
- Avoid adding new logic here - update QueryBuilder etc instead.
+ Builds aggregate results for the Stats API v1. Avoid adding new logic here.
"""
use Plausible.ClickhouseRepo
diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/legacy/breakdown.ex
similarity index 96%
rename from lib/plausible/stats/breakdown.ex
rename to lib/plausible/stats/legacy/breakdown.ex
index e636640272..364a5e05df 100644
--- a/lib/plausible/stats/breakdown.ex
+++ b/lib/plausible/stats/legacy/breakdown.ex
@@ -1,8 +1,6 @@
-defmodule Plausible.Stats.Breakdown do
+defmodule Plausible.Stats.Legacy.Breakdown do
@moduledoc """
- Builds breakdown results for v1 of our stats API and dashboards.
-
- Avoid adding new logic here - update QueryBuilder etc instead.
+ Builds breakdown results for the Stats API v1. Avoid adding new logic here.
"""
use Plausible.ClickhouseRepo
diff --git a/lib/plausible/stats/legacy/legacy_query_builder.ex b/lib/plausible/stats/legacy/legacy_query_builder.ex
index aa61521080..f942d27d1b 100644
--- a/lib/plausible/stats/legacy/legacy_query_builder.ex
+++ b/lib/plausible/stats/legacy/legacy_query_builder.ex
@@ -8,7 +8,7 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do
use Plausible
- alias Plausible.Stats.{Filters, Interval, Query, ApiQueryParser, QueryBuilder, DateTimeRange}
+ alias Plausible.Stats.{Filters, Query, ApiQueryParser, QueryBuilder, DateTimeRange}
alias Plausible.Times
def from(site, params, debug_metadata, now \\ nil) do
@@ -25,7 +25,6 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do
|> put_input_date_range(site, params)
|> put_timezone(site)
|> put_dimensions(params)
- |> put_interval(params)
|> put_parsed_filters(params)
|> resolve_segments(site)
|> preload_goals_and_revenue(site)
@@ -317,11 +316,6 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do
struct!(query, order_by: parse_order_by(params["order_by"]))
end
- defp put_interval(query, params) do
- interval = Map.get(params, "interval", Interval.default_for_query(query))
- struct!(query, interval: interval)
- end
-
defp put_parsed_filters(query, params) do
struct!(query, filters: Filters.parse(params["filters"]))
end
diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/legacy/timeseries.ex
similarity index 81%
rename from lib/plausible/stats/timeseries.ex
rename to lib/plausible/stats/legacy/timeseries.ex
index fdf1e48e61..93eb1ba512 100644
--- a/lib/plausible/stats/timeseries.ex
+++ b/lib/plausible/stats/legacy/timeseries.ex
@@ -1,29 +1,20 @@
-defmodule Plausible.Stats.Timeseries do
+defmodule Plausible.Stats.Legacy.Timeseries do
@moduledoc """
- Builds timeseries results for v1 of our stats API and dashboards.
-
- Avoid adding new logic here - update QueryBuilder etc instead.
+ Builds timeseries results for the Stats API v1. Avoid adding new logic here.
"""
use Plausible
use Plausible.ClickhouseRepo
alias Plausible.Stats.{Query, QueryRunner, Metrics, Time, QueryOptimizer}
- @time_dimension %{
- "month" => "time:month",
- "week" => "time:week",
- "day" => "time:day",
- "hour" => "time:hour",
- "minute" => "time:minute"
- }
-
def timeseries(site, query, metrics) do
+ [time_dimension] = query.dimensions
+
query =
query
|> Query.set(
metrics: transform_metrics(metrics, %{conversion_rate: :group_conversion_rate}),
- dimensions: [time_dimension(query)],
- order_by: [{time_dimension(query), :asc}]
+ order_by: [{time_dimension, :asc}]
)
|> Query.set_include(:drop_unavailable_revenue_metrics, true)
|> QueryOptimizer.optimize()
@@ -36,8 +27,6 @@ defmodule Plausible.Stats.Timeseries do
}
end
- defp time_dimension(query), do: Map.fetch!(@time_dimension, query.interval)
-
# Given a query result, build a legacy timeseries result
# Format is %{ date => %{ date: date_string, [metric] => value } } with a bunch of special cases for the UI
defp build_result(query_result, %Query{} = query) do
diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex
index b12802d6d8..cf4fdfffed 100644
--- a/lib/plausible/stats/query.ex
+++ b/lib/plausible/stats/query.ex
@@ -1,13 +1,13 @@
defmodule Plausible.Stats.Query do
+ @moduledoc false
use Plausible
defstruct utc_time_range: nil,
comparison_utc_time_range: nil,
- interval: nil,
input_date_range: nil,
dimensions: [],
filters: [],
- sample_threshold: 20_000_000,
+ sample_threshold: nil,
imports_exist: false,
imports_in_range: [],
include_imported: false,
@@ -221,7 +221,6 @@ defmodule Plausible.Stats.Query do
metrics = metrics |> Enum.sort() |> Enum.join(";")
Tracer.set_attributes([
- {"plausible.query.interval", query.interval},
{"plausible.query.dimensions", query.dimensions |> Enum.join(";")},
{"plausible.query.include_imported", query.include_imported},
{"plausible.query.filter_keys", filter_dimensions},
diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex
index 2ca65f3178..50cebddd70 100644
--- a/lib/plausible_web/controllers/api/external_stats_controller.ex
+++ b/lib/plausible_web/controllers/api/external_stats_controller.ex
@@ -2,7 +2,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
use PlausibleWeb, :controller
use Plausible.Repo
use PlausibleWeb.Plugs.ErrorHandler
- alias Plausible.Stats.{Query, Metrics, Filters}
+ alias Plausible.Stats.{Query, Metrics, Filters, Interval, Legacy}
def realtime_visitors(conn, _params) do
site = conn.assigns.site
@@ -20,7 +20,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
:ok <- validate_filters(site, query.filters),
{:ok, metrics} <- parse_and_validate_metrics(params, query),
:ok <- ensure_custom_props_access(site, query) do
- %{results: results, meta: meta} = Plausible.Stats.aggregate(site, query, metrics)
+ %{results: results, meta: meta} = Legacy.Aggregate.aggregate(site, query, metrics)
payload = maybe_add_warning(%{results: results}, meta)
@@ -44,7 +44,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
page = String.to_integer(Map.get(params, "page", "1"))
%{results: results, meta: meta} =
- Plausible.Stats.breakdown(site, query, metrics, {limit, page})
+ Legacy.Breakdown.breakdown(site, query, metrics, {limit, page})
payload = maybe_add_warning(%{results: results}, meta)
@@ -258,7 +258,9 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
:ok <- validate_filters(site, query.filters),
{:ok, metrics} <- parse_and_validate_metrics(params, query),
:ok <- ensure_custom_props_access(site, query) do
- {results, meta} = Plausible.Stats.timeseries(site, query, metrics)
+ time_dimension = interval_to_time_dimension(Map.get(params, "interval"), query)
+ query = Query.set(query, dimensions: [time_dimension])
+ {results, meta} = Legacy.Timeseries.timeseries(site, query, metrics)
payload =
case meta[:imports_warning] do
@@ -327,6 +329,19 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
defp validate_interval(_), do: :ok
+ @time_dimensions %{
+ "minute" => "time:minute",
+ "hour" => "time:hour",
+ "day" => "time:day",
+ "week" => "time:week",
+ "month" => "time:month"
+ }
+
+ defp interval_to_time_dimension(interval, query) do
+ interval = interval || Interval.default_for_query(query)
+ Map.fetch!(@time_dimensions, interval)
+ end
+
defp validate_filters(site, filters) do
Enum.reduce_while(filters, :ok, fn filter, _ ->
case validate_filter(site, filter) do
diff --git a/test/plausible/stats/query/query_from_test.exs b/test/plausible/stats/query/query_from_test.exs
index e03a4ff966..4d4ddb8437 100644
--- a/test/plausible/stats/query/query_from_test.exs
+++ b/test/plausible/stats/query/query_from_test.exs
@@ -40,7 +40,6 @@ defmodule Plausible.Stats.Query.QueryFromTest do
assert q.utc_time_range.first == ~U[2019-01-01 05:00:00Z]
assert q.utc_time_range.last == ~U[2019-01-02 04:59:59Z]
- assert q.interval == "hour"
end
test "day format defaults to today", %{site: site} do
@@ -48,7 +47,6 @@ defmodule Plausible.Stats.Query.QueryFromTest do
assert q.utc_time_range.first == ~U[2024-05-03 04:00:00Z]
assert q.utc_time_range.last == @now
- assert q.interval == "hour"
end
test "parses realtime format", %{site: site} do
@@ -64,7 +62,6 @@ defmodule Plausible.Stats.Query.QueryFromTest do
assert q.utc_time_range.first == ~U[2019-01-01 05:00:00Z]
assert q.utc_time_range.last == ~U[2019-02-01 04:59:59Z]
- assert q.interval == "day"
end
test "parses 6 month format", %{site: site} do
@@ -72,7 +69,6 @@ defmodule Plausible.Stats.Query.QueryFromTest do
assert q.utc_time_range.first == ~U[2023-11-01 04:00:00Z]
assert q.utc_time_range.last == ~U[2024-05-01 03:59:59Z]
- assert q.interval == "month"
end
test "parses 12 month format", %{site: site} do
@@ -80,7 +76,6 @@ defmodule Plausible.Stats.Query.QueryFromTest do
assert q.utc_time_range.first == ~U[2023-05-01 04:00:00Z]
assert q.utc_time_range.last == ~U[2024-05-01 03:59:59Z]
- assert q.interval == "month"
end
test "parses year to date format", %{site: site} do
@@ -88,7 +83,6 @@ defmodule Plausible.Stats.Query.QueryFromTest do
assert q.utc_time_range.first == ~U[2024-01-01 05:00:00Z]
assert q.utc_time_range.last == ~U[2025-01-01 04:59:59Z]
- assert q.interval == "month"
end
test "parses all time", %{site: site} do
@@ -97,7 +91,6 @@ defmodule Plausible.Stats.Query.QueryFromTest do
assert q.utc_time_range.first == ~U[2020-01-01 05:00:00Z]
assert q.utc_time_range.last == ~U[2024-05-04 03:59:59Z]
assert q.input_date_range == :all
- assert q.interval == "month"
end
test "parses all time in GMT+12 timezone", %{site: site} do
@@ -115,7 +108,6 @@ defmodule Plausible.Stats.Query.QueryFromTest do
assert q.utc_time_range.first == ~U[2024-05-03 04:00:00Z]
assert q.utc_time_range.last == ~U[2024-05-04 03:59:59Z]
assert q.input_date_range == :all
- assert q.interval == "hour"
end
test "all time shows hourly if site is completely new", %{site: site} do
@@ -125,7 +117,6 @@ defmodule Plausible.Stats.Query.QueryFromTest do
assert q.utc_time_range.first == ~U[2024-05-03 04:00:00Z]
assert q.utc_time_range.last == ~U[2024-05-04 03:59:59Z]
assert q.input_date_range == :all
- assert q.interval == "hour"
end
test "all time shows daily if site is more than a day old", %{site: site} do
@@ -137,7 +128,6 @@ defmodule Plausible.Stats.Query.QueryFromTest do
assert q.utc_time_range.first == ~U[2024-05-02 04:00:00Z]
assert q.utc_time_range.last == ~U[2024-05-04 03:59:59Z]
assert q.input_date_range == :all
- assert q.interval == "day"
end
test "all time shows monthly if site is more than a month old", %{site: site} do
@@ -149,7 +139,6 @@ defmodule Plausible.Stats.Query.QueryFromTest do
assert q.utc_time_range.first == ~U[2024-04-03 04:00:00Z]
assert q.utc_time_range.last == ~U[2024-05-04 03:59:59Z]
assert q.input_date_range == :all
- assert q.interval == "month"
end
test "all time uses passed interval different from the default interval", %{site: site} do
@@ -161,7 +150,6 @@ defmodule Plausible.Stats.Query.QueryFromTest do
assert q.utc_time_range.first == ~U[2024-04-03 04:00:00Z]
assert q.utc_time_range.last == ~U[2024-05-04 03:59:59Z]
assert q.input_date_range == :all
- assert q.interval == "week"
end
test "defaults to 30 days format", %{site: site} do
@@ -181,7 +169,6 @@ defmodule Plausible.Stats.Query.QueryFromTest do
assert q.utc_time_range.first == ~U[2019-01-01 05:00:00Z]
assert q.utc_time_range.last == ~U[2019-01-16 04:59:59Z]
- assert q.interval == "day"
end
@tag :ee_only
diff --git a/test/plausible_web/controllers/api/stats_controller/internal_query_api_test.exs b/test/plausible_web/controllers/api/stats_controller/internal_query_api_test.exs
deleted file mode 100644
index 63f815b3b7..0000000000
--- a/test/plausible_web/controllers/api/stats_controller/internal_query_api_test.exs
+++ /dev/null
@@ -1,108 +0,0 @@
-defmodule PlausibleWeb.Api.StatsController.InternalQueryApiTest do
- use PlausibleWeb.ConnCase
-
- defp do_query_success(conn, site, params) do
- conn
- |> post("/api/stats/#{site.domain}/query", params)
- |> json_response(200)
- end
-
- setup [:create_user, :log_in, :create_site]
-
- describe "aggregates (e.g. top stats)" do
- on_ee do
- test "drops revenue metrics when they are not allowed", %{
- conn: conn,
- user: user,
- site: site
- } do
- {:ok, team} = Plausible.Teams.get_or_create(user)
- insert(:goal, site: site, event_name: "Purchase", currency: "EUR")
- subscribe_to_growth_plan(team)
-
- params = %{
- "date_range" => "day",
- "filters" => [["is", "event:goal", ["Purchase"]]],
- "metrics" => [
- "visitors",
- "events",
- "conversion_rate",
- "total_revenue",
- "average_revenue"
- ]
- }
-
- %{"results" => results, "query" => query} = do_query_success(conn, site, params)
-
- assert query["metrics"] == ["visitors", "events", "conversion_rate"]
- assert results == [%{"dimensions" => [], "metrics" => [0, 0, 0.0]}]
- end
- end
-
- test "returns empty metrics when no data", %{conn: conn, site: site} do
- requested_metrics = [
- "visitors",
- "visits",
- "pageviews",
- "views_per_visit",
- "bounce_rate",
- "visit_duration"
- ]
-
- params = %{
- "date_range" => "all",
- "filters" => [],
- "metrics" => requested_metrics
- }
-
- response = do_query_success(conn, site, params)
-
- assert response["query"]["metrics"] == requested_metrics
- assert response["results"] == [%{"dimensions" => [], "metrics" => [0, 0, 0, 0.0, 0, 0]}]
- end
-
- test "returns information about imports with the imports_meta option", %{
- conn: conn,
- site: site
- } do
- params = %{
- "date_range" => "all",
- "filters" => [],
- "metrics" => ["visitors"],
- "include" => %{"imports" => false, "imports_meta" => true}
- }
-
- response = do_query_success(conn, site, params)
-
- assert response["meta"] == %{
- "imports_included" => false,
- "imports_skip_reason" => "no_imported_data"
- }
- end
-
- test "drops time on page if unavailable", %{conn: conn, site: site} do
- site_import =
- insert(:site_import, site: site, start_date: ~D[2021-01-01], end_date: ~D[2022-01-01])
-
- site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[2023-01-01])
-
- populate_stats(site, site_import.id, [
- build(:pageview, pathname: "/"),
- build(:imported_pages, page: "/", date: ~D[2021-01-01])
- ])
-
- params = %{
- "date_range" => "all",
- "filters" => [["is", "event:page", ["/"]]],
- "metrics" => ["visitors", "time_on_page"],
- "include" => %{"imports" => true}
- }
-
- response = do_query_success(conn, site, params)
-
- assert response["meta"]["imports_included"]
- assert response["query"]["metrics"] == ["visitors"]
- assert response["results"] == [%{"dimensions" => [], "metrics" => [2]}]
- end
- end
-end
diff --git a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
index b1434c80c7..bdf444257f 100644
--- a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
@@ -373,6 +373,72 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
assert response["results"] == [%{"dimensions" => [], "metrics" => [nil]}]
end
+
+ test "returns empty metrics when no data", %{conn: conn, site: site} do
+ requested_metrics = [
+ "visitors",
+ "visits",
+ "pageviews",
+ "views_per_visit",
+ "bounce_rate",
+ "visit_duration"
+ ]
+
+ params = %{
+ "date_range" => "all",
+ "filters" => [],
+ "metrics" => requested_metrics
+ }
+
+ response = do_query_success(conn, site, params)
+
+ assert response["query"]["metrics"] == requested_metrics
+ assert response["results"] == [%{"dimensions" => [], "metrics" => [0, 0, 0, 0.0, 0, 0]}]
+ end
+
+ test "returns information about imports with the imports_meta option", %{
+ conn: conn,
+ site: site
+ } do
+ params = %{
+ "date_range" => "all",
+ "filters" => [],
+ "metrics" => ["visitors"],
+ "include" => %{"imports" => false, "imports_meta" => true}
+ }
+
+ response = do_query_success(conn, site, params)
+
+ assert response["meta"] == %{
+ "imports_included" => false,
+ "imports_skip_reason" => "no_imported_data"
+ }
+ end
+
+ test "drops time on page if unavailable", %{conn: conn, site: site} do
+ site_import =
+ insert(:site_import, site: site, start_date: ~D[2021-01-01], end_date: ~D[2022-01-01])
+
+ site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[2023-01-01])
+
+ populate_stats(site, site_import.id, [
+ build(:pageview, pathname: "/"),
+ build(:imported_pages, page: "/", date: ~D[2021-01-01])
+ ])
+
+ params = %{
+ "date_range" => "all",
+ "filters" => [["is", "event:page", ["/"]]],
+ "metrics" => ["visitors", "time_on_page"],
+ "include" => %{"imports" => true}
+ }
+
+ response = do_query_success(conn, site, params)
+
+ assert response["meta"]["imports_included"]
+ assert response["query"]["metrics"] == ["visitors"]
+ assert response["results"] == [%{"dimensions" => [], "metrics" => [2]}]
+ end
end
describe "imported data" do
@@ -1105,6 +1171,34 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
assert response["results"] == [%{"dimensions" => [], "metrics" => [2, 3, 50.0]}]
end
+ @tag :ee_only
+ test "drops revenue metrics when they are not allowed", %{
+ conn: conn,
+ user: user,
+ site: site
+ } do
+ {:ok, team} = Plausible.Teams.get_or_create(user)
+ insert(:goal, site: site, event_name: "Purchase", currency: "EUR")
+ subscribe_to_growth_plan(team)
+
+ params = %{
+ "date_range" => "day",
+ "filters" => [["is", "event:goal", ["Purchase"]]],
+ "metrics" => [
+ "visitors",
+ "events",
+ "conversion_rate",
+ "total_revenue",
+ "average_revenue"
+ ]
+ }
+
+ %{"results" => results, "query" => query} = do_query_success(conn, site, params)
+
+ assert query["metrics"] == ["visitors", "events", "conversion_rate"]
+ assert results == [%{"dimensions" => [], "metrics" => [0, 0, 0.0]}]
+ end
+
@tag :ee_only
test "returns average and total when filtering by a revenue goal", %{conn: conn, site: site} do
insert(:goal,