Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
A few more refactoring cleanups (#6471)
RobertJoonas1c539f6bf022
diff --git a/lib/plausible/stats.ex b/lib/plausible/stats.exindex 495c020491..2dd9229dd2 100644--- a/lib/plausible/stats.ex+++ b/lib/plausible/stats.ex@@ -1,11 +1,10 @@defmodule Plausible.Stats do+ @moduledoc false+use Plausibleuse Plausible.ClickhouseRepoalias Plausible.Stats.{- Breakdown,- Aggregate,- Timeseries,CurrentVisitors,FilterSuggestions,QueryRunner@@ -15,18 +14,6 @@ defmodule Plausible.Stats doQueryRunner.run(site, query)end- def breakdown(site, query, metrics, pagination) do- Breakdown.breakdown(site, query, metrics, pagination)- end-- def aggregate(site, query, metrics) do- Aggregate.aggregate(site, query, metrics)- end-- def timeseries(site, query, metrics) do- Timeseries.timeseries(site, query, metrics)- end-def current_visitors(site, duration \\ Duration.new!(minute: -5)) doCurrentVisitors.current_visitors(site, duration)enddiff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/legacy/aggregate.exsimilarity index 89%rename from lib/plausible/stats/aggregate.exrename to lib/plausible/stats/legacy/aggregate.exindex ae2e80e7c9..cc633af968 100644--- a/lib/plausible/stats/aggregate.ex+++ b/lib/plausible/stats/legacy/aggregate.ex@@ -1,8 +1,6 @@-defmodule Plausible.Stats.Aggregate do+defmodule Plausible.Stats.Legacy.Aggregate do@moduledoc """- Builds aggregate results for v1 of our stats API and dashboards.-- Avoid adding new logic here - update QueryBuilder etc instead.+ Builds aggregate results for the Stats API v1. Avoid adding new logic here."""use Plausible.ClickhouseRepodiff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/legacy/breakdown.exsimilarity index 96%rename from lib/plausible/stats/breakdown.exrename to lib/plausible/stats/legacy/breakdown.exindex e636640272..364a5e05df 100644--- a/lib/plausible/stats/breakdown.ex+++ b/lib/plausible/stats/legacy/breakdown.ex@@ -1,8 +1,6 @@-defmodule Plausible.Stats.Breakdown do+defmodule Plausible.Stats.Legacy.Breakdown do@moduledoc """- Builds breakdown results for v1 of our stats API and dashboards.-- Avoid adding new logic here - update QueryBuilder etc instead.+ Builds breakdown results for the Stats API v1. Avoid adding new logic here."""use Plausible.ClickhouseRepodiff --git a/lib/plausible/stats/legacy/legacy_query_builder.ex b/lib/plausible/stats/legacy/legacy_query_builder.exindex aa61521080..f942d27d1b 100644--- a/lib/plausible/stats/legacy/legacy_query_builder.ex+++ b/lib/plausible/stats/legacy/legacy_query_builder.ex@@ -8,7 +8,7 @@ defmodule Plausible.Stats.Legacy.QueryBuilder douse Plausible- alias Plausible.Stats.{Filters, Interval, Query, ApiQueryParser, QueryBuilder, DateTimeRange}+ alias Plausible.Stats.{Filters, Query, ApiQueryParser, QueryBuilder, DateTimeRange}alias Plausible.Timesdef from(site, params, debug_metadata, now \\ nil) do@@ -25,7 +25,6 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do|> put_input_date_range(site, params)|> put_timezone(site)|> put_dimensions(params)- |> put_interval(params)|> put_parsed_filters(params)|> resolve_segments(site)|> preload_goals_and_revenue(site)@@ -317,11 +316,6 @@ defmodule Plausible.Stats.Legacy.QueryBuilder dostruct!(query, order_by: parse_order_by(params["order_by"]))end- defp put_interval(query, params) do- interval = Map.get(params, "interval", Interval.default_for_query(query))- struct!(query, interval: interval)- end-defp put_parsed_filters(query, params) dostruct!(query, filters: Filters.parse(params["filters"]))enddiff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/legacy/timeseries.exsimilarity index 81%rename from lib/plausible/stats/timeseries.exrename to lib/plausible/stats/legacy/timeseries.exindex fdf1e48e61..93eb1ba512 100644--- a/lib/plausible/stats/timeseries.ex+++ b/lib/plausible/stats/legacy/timeseries.ex@@ -1,29 +1,20 @@-defmodule Plausible.Stats.Timeseries do+defmodule Plausible.Stats.Legacy.Timeseries do@moduledoc """- Builds timeseries results for v1 of our stats API and dashboards.-- Avoid adding new logic here - update QueryBuilder etc instead.+ Builds timeseries results for the Stats API v1. Avoid adding new logic here."""use Plausibleuse Plausible.ClickhouseRepoalias Plausible.Stats.{Query, QueryRunner, Metrics, Time, QueryOptimizer}- @time_dimension %{- "month" => "time:month",- "week" => "time:week",- "day" => "time:day",- "hour" => "time:hour",- "minute" => "time:minute"- }-def timeseries(site, query, metrics) do+ [time_dimension] = query.dimensions+query =query|> Query.set(metrics: transform_metrics(metrics, %{conversion_rate: :group_conversion_rate}),- dimensions: [time_dimension(query)],- order_by: [{time_dimension(query), :asc}]+ order_by: [{time_dimension, :asc}])|> Query.set_include(:drop_unavailable_revenue_metrics, true)|> QueryOptimizer.optimize()@@ -36,8 +27,6 @@ defmodule Plausible.Stats.Timeseries do}end- defp time_dimension(query), do: Map.fetch!(@time_dimension, query.interval)-# Given a query result, build a legacy timeseries result# Format is %{ date => %{ date: date_string, [metric] => value } } with a bunch of special cases for the UIdefp build_result(query_result, %Query{} = query) dodiff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.exindex b12802d6d8..cf4fdfffed 100644--- a/lib/plausible/stats/query.ex+++ b/lib/plausible/stats/query.ex@@ -1,13 +1,13 @@defmodule Plausible.Stats.Query do+ @moduledoc falseuse Plausibledefstruct utc_time_range: nil,comparison_utc_time_range: nil,- interval: nil,input_date_range: nil,dimensions: [],filters: [],- sample_threshold: 20_000_000,+ sample_threshold: nil,imports_exist: false,imports_in_range: [],include_imported: false,@@ -221,7 +221,6 @@ defmodule Plausible.Stats.Query dometrics = metrics |> Enum.sort() |> Enum.join(";")Tracer.set_attributes([- {"plausible.query.interval", query.interval},{"plausible.query.dimensions", query.dimensions |> Enum.join(";")},{"plausible.query.include_imported", query.include_imported},{"plausible.query.filter_keys", filter_dimensions},diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.exindex 2ca65f3178..50cebddd70 100644--- a/lib/plausible_web/controllers/api/external_stats_controller.ex+++ b/lib/plausible_web/controllers/api/external_stats_controller.ex@@ -2,7 +2,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController douse PlausibleWeb, :controlleruse Plausible.Repouse PlausibleWeb.Plugs.ErrorHandler- alias Plausible.Stats.{Query, Metrics, Filters}+ alias Plausible.Stats.{Query, Metrics, Filters, Interval, Legacy}def realtime_visitors(conn, _params) dosite = conn.assigns.site@@ -20,7 +20,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do:ok <- validate_filters(site, query.filters),{:ok, metrics} <- parse_and_validate_metrics(params, query),:ok <- ensure_custom_props_access(site, query) do- %{results: results, meta: meta} = Plausible.Stats.aggregate(site, query, metrics)+ %{results: results, meta: meta} = Legacy.Aggregate.aggregate(site, query, metrics)payload = maybe_add_warning(%{results: results}, meta)@@ -44,7 +44,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController dopage = String.to_integer(Map.get(params, "page", "1"))%{results: results, meta: meta} =- Plausible.Stats.breakdown(site, query, metrics, {limit, page})+ Legacy.Breakdown.breakdown(site, query, metrics, {limit, page})payload = maybe_add_warning(%{results: results}, meta)@@ -258,7 +258,9 @@ defmodule PlausibleWeb.Api.ExternalStatsController do:ok <- validate_filters(site, query.filters),{:ok, metrics} <- parse_and_validate_metrics(params, query),:ok <- ensure_custom_props_access(site, query) do- {results, meta} = Plausible.Stats.timeseries(site, query, metrics)+ time_dimension = interval_to_time_dimension(Map.get(params, "interval"), query)+ query = Query.set(query, dimensions: [time_dimension])+ {results, meta} = Legacy.Timeseries.timeseries(site, query, metrics)payload =case meta[:imports_warning] do@@ -327,6 +329,19 @@ defmodule PlausibleWeb.Api.ExternalStatsController dodefp validate_interval(_), do: :ok+ @time_dimensions %{+ "minute" => "time:minute",+ "hour" => "time:hour",+ "day" => "time:day",+ "week" => "time:week",+ "month" => "time:month"+ }++ defp interval_to_time_dimension(interval, query) do+ interval = interval || Interval.default_for_query(query)+ Map.fetch!(@time_dimensions, interval)+ end+defp validate_filters(site, filters) doEnum.reduce_while(filters, :ok, fn filter, _ ->case validate_filter(site, filter) dodiff --git a/test/plausible/stats/query/query_from_test.exs b/test/plausible/stats/query/query_from_test.exsindex e03a4ff966..4d4ddb8437 100644--- a/test/plausible/stats/query/query_from_test.exs+++ b/test/plausible/stats/query/query_from_test.exs@@ -40,7 +40,6 @@ defmodule Plausible.Stats.Query.QueryFromTest doassert q.utc_time_range.first == ~U[2019-01-01 05:00:00Z]assert q.utc_time_range.last == ~U[2019-01-02 04:59:59Z]- assert q.interval == "hour"endtest "day format defaults to today", %{site: site} do@@ -48,7 +47,6 @@ defmodule Plausible.Stats.Query.QueryFromTest doassert q.utc_time_range.first == ~U[2024-05-03 04:00:00Z]assert q.utc_time_range.last == @now- assert q.interval == "hour"endtest "parses realtime format", %{site: site} do@@ -64,7 +62,6 @@ defmodule Plausible.Stats.Query.QueryFromTest doassert q.utc_time_range.first == ~U[2019-01-01 05:00:00Z]assert q.utc_time_range.last == ~U[2019-02-01 04:59:59Z]- assert q.interval == "day"endtest "parses 6 month format", %{site: site} do@@ -72,7 +69,6 @@ defmodule Plausible.Stats.Query.QueryFromTest doassert q.utc_time_range.first == ~U[2023-11-01 04:00:00Z]assert q.utc_time_range.last == ~U[2024-05-01 03:59:59Z]- assert q.interval == "month"endtest "parses 12 month format", %{site: site} do@@ -80,7 +76,6 @@ defmodule Plausible.Stats.Query.QueryFromTest doassert q.utc_time_range.first == ~U[2023-05-01 04:00:00Z]assert q.utc_time_range.last == ~U[2024-05-01 03:59:59Z]- assert q.interval == "month"endtest "parses year to date format", %{site: site} do@@ -88,7 +83,6 @@ defmodule Plausible.Stats.Query.QueryFromTest doassert q.utc_time_range.first == ~U[2024-01-01 05:00:00Z]assert q.utc_time_range.last == ~U[2025-01-01 04:59:59Z]- assert q.interval == "month"endtest "parses all time", %{site: site} do@@ -97,7 +91,6 @@ defmodule Plausible.Stats.Query.QueryFromTest doassert q.utc_time_range.first == ~U[2020-01-01 05:00:00Z]assert q.utc_time_range.last == ~U[2024-05-04 03:59:59Z]assert q.input_date_range == :all- assert q.interval == "month"endtest "parses all time in GMT+12 timezone", %{site: site} do@@ -115,7 +108,6 @@ defmodule Plausible.Stats.Query.QueryFromTest doassert q.utc_time_range.first == ~U[2024-05-03 04:00:00Z]assert q.utc_time_range.last == ~U[2024-05-04 03:59:59Z]assert q.input_date_range == :all- assert q.interval == "hour"endtest "all time shows hourly if site is completely new", %{site: site} do@@ -125,7 +117,6 @@ defmodule Plausible.Stats.Query.QueryFromTest doassert q.utc_time_range.first == ~U[2024-05-03 04:00:00Z]assert q.utc_time_range.last == ~U[2024-05-04 03:59:59Z]assert q.input_date_range == :all- assert q.interval == "hour"endtest "all time shows daily if site is more than a day old", %{site: site} do@@ -137,7 +128,6 @@ defmodule Plausible.Stats.Query.QueryFromTest doassert q.utc_time_range.first == ~U[2024-05-02 04:00:00Z]assert q.utc_time_range.last == ~U[2024-05-04 03:59:59Z]assert q.input_date_range == :all- assert q.interval == "day"endtest "all time shows monthly if site is more than a month old", %{site: site} do@@ -149,7 +139,6 @@ defmodule Plausible.Stats.Query.QueryFromTest doassert q.utc_time_range.first == ~U[2024-04-03 04:00:00Z]assert q.utc_time_range.last == ~U[2024-05-04 03:59:59Z]assert q.input_date_range == :all- assert q.interval == "month"endtest "all time uses passed interval different from the default interval", %{site: site} do@@ -161,7 +150,6 @@ defmodule Plausible.Stats.Query.QueryFromTest doassert q.utc_time_range.first == ~U[2024-04-03 04:00:00Z]assert q.utc_time_range.last == ~U[2024-05-04 03:59:59Z]assert q.input_date_range == :all- assert q.interval == "week"endtest "defaults to 30 days format", %{site: site} do@@ -181,7 +169,6 @@ defmodule Plausible.Stats.Query.QueryFromTest doassert q.utc_time_range.first == ~U[2019-01-01 05:00:00Z]assert q.utc_time_range.last == ~U[2019-01-16 04:59:59Z]- assert q.interval == "day"end@tag :ee_onlydiff --git a/test/plausible_web/controllers/api/stats_controller/internal_query_api_test.exs b/test/plausible_web/controllers/api/stats_controller/internal_query_api_test.exsdeleted file mode 100644index 63f815b3b7..0000000000--- a/test/plausible_web/controllers/api/stats_controller/internal_query_api_test.exs+++ /dev/null@@ -1,108 +0,0 @@-defmodule PlausibleWeb.Api.StatsController.InternalQueryApiTest do- use PlausibleWeb.ConnCase-- defp do_query_success(conn, site, params) do- conn- |> post("/api/stats/#{site.domain}/query", params)- |> json_response(200)- end-- setup [:create_user, :log_in, :create_site]-- describe "aggregates (e.g. top stats)" do- on_ee do- test "drops revenue metrics when they are not allowed", %{- conn: conn,- user: user,- site: site- } do- {:ok, team} = Plausible.Teams.get_or_create(user)- insert(:goal, site: site, event_name: "Purchase", currency: "EUR")- subscribe_to_growth_plan(team)-- params = %{- "date_range" => "day",- "filters" => [["is", "event:goal", ["Purchase"]]],- "metrics" => [- "visitors",- "events",- "conversion_rate",- "total_revenue",- "average_revenue"- ]- }-- %{"results" => results, "query" => query} = do_query_success(conn, site, params)-- assert query["metrics"] == ["visitors", "events", "conversion_rate"]- assert results == [%{"dimensions" => [], "metrics" => [0, 0, 0.0]}]- end- end-- test "returns empty metrics when no data", %{conn: conn, site: site} do- requested_metrics = [- "visitors",- "visits",- "pageviews",- "views_per_visit",- "bounce_rate",- "visit_duration"- ]-- params = %{- "date_range" => "all",- "filters" => [],- "metrics" => requested_metrics- }-- response = do_query_success(conn, site, params)-- assert response["query"]["metrics"] == requested_metrics- assert response["results"] == [%{"dimensions" => [], "metrics" => [0, 0, 0, 0.0, 0, 0]}]- end-- test "returns information about imports with the imports_meta option", %{- conn: conn,- site: site- } do- params = %{- "date_range" => "all",- "filters" => [],- "metrics" => ["visitors"],- "include" => %{"imports" => false, "imports_meta" => true}- }-- response = do_query_success(conn, site, params)-- assert response["meta"] == %{- "imports_included" => false,- "imports_skip_reason" => "no_imported_data"- }- end-- test "drops time on page if unavailable", %{conn: conn, site: site} do- site_import =- insert(:site_import, site: site, start_date: ~D[2021-01-01], end_date: ~D[2022-01-01])-- site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[2023-01-01])-- populate_stats(site, site_import.id, [- build(:pageview, pathname: "/"),- build(:imported_pages, page: "/", date: ~D[2021-01-01])- ])-- params = %{- "date_range" => "all",- "filters" => [["is", "event:page", ["/"]]],- "metrics" => ["visitors", "time_on_page"],- "include" => %{"imports" => true}- }-- response = do_query_success(conn, site, params)-- assert response["meta"]["imports_included"]- assert response["query"]["metrics"] == ["visitors"]- assert response["results"] == [%{"dimensions" => [], "metrics" => [2]}]- end- end-enddiff --git a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exsindex b1434c80c7..bdf444257f 100644--- a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs@@ -373,6 +373,72 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest doassert response["results"] == [%{"dimensions" => [], "metrics" => [nil]}]end++ test "returns empty metrics when no data", %{conn: conn, site: site} do+ requested_metrics = [+ "visitors",+ "visits",+ "pageviews",+ "views_per_visit",+ "bounce_rate",+ "visit_duration"+ ]++ params = %{+ "date_range" => "all",+ "filters" => [],+ "metrics" => requested_metrics+ }++ response = do_query_success(conn, site, params)++ assert response["query"]["metrics"] == requested_metrics+ assert response["results"] == [%{"dimensions" => [], "metrics" => [0, 0, 0, 0.0, 0, 0]}]+ end++ test "returns information about imports with the imports_meta option", %{+ conn: conn,+ site: site+ } do+ params = %{+ "date_range" => "all",+ "filters" => [],+ "metrics" => ["visitors"],+ "include" => %{"imports" => false, "imports_meta" => true}+ }++ response = do_query_success(conn, site, params)++ assert response["meta"] == %{+ "imports_included" => false,+ "imports_skip_reason" => "no_imported_data"+ }+ end++ test "drops time on page if unavailable", %{conn: conn, site: site} do+ site_import =+ insert(:site_import, site: site, start_date: ~D[2021-01-01], end_date: ~D[2022-01-01])++ site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[2023-01-01])++ populate_stats(site, site_import.id, [+ build(:pageview, pathname: "/"),+ build(:imported_pages, page: "/", date: ~D[2021-01-01])+ ])++ params = %{+ "date_range" => "all",+ "filters" => [["is", "event:page", ["/"]]],+ "metrics" => ["visitors", "time_on_page"],+ "include" => %{"imports" => true}+ }++ response = do_query_success(conn, site, params)++ assert response["meta"]["imports_included"]+ assert response["query"]["metrics"] == ["visitors"]+ assert response["results"] == [%{"dimensions" => [], "metrics" => [2]}]+ endenddescribe "imported data" do@@ -1105,6 +1171,34 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest doassert response["results"] == [%{"dimensions" => [], "metrics" => [2, 3, 50.0]}]end+ @tag :ee_only+ test "drops revenue metrics when they are not allowed", %{+ conn: conn,+ user: user,+ site: site+ } do+ {:ok, team} = Plausible.Teams.get_or_create(user)+ insert(:goal, site: site, event_name: "Purchase", currency: "EUR")+ subscribe_to_growth_plan(team)++ params = %{+ "date_range" => "day",+ "filters" => [["is", "event:goal", ["Purchase"]]],+ "metrics" => [+ "visitors",+ "events",+ "conversion_rate",+ "total_revenue",+ "average_revenue"+ ]+ }++ %{"results" => results, "query" => query} = do_query_success(conn, site, params)++ assert query["metrics"] == ["visitors", "events", "conversion_rate"]+ assert results == [%{"dimensions" => [], "metrics" => [0, 0, 0.0]}]+ end+@tag :ee_onlytest "returns average and total when filtering by a revenue goal", %{conn: conn, site: site} doinsert(:goal,