Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Add new `Pages / Visit` metric to Stats API Aggregate and Timeseries (#2714)
RobertJoonas0d6c72d50f4b
diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.exindex d0c3299df5..9121922546 100644--- a/lib/plausible/stats/aggregate.ex+++ b/lib/plausible/stats/aggregate.ex@@ -4,7 +4,7 @@ defmodule Plausible.Stats.Aggregate doimport Plausible.Stats.{Base, Imported}@event_metrics [:visitors, :pageviews, :events, :sample_percent]- @session_metrics [:visits, :bounce_rate, :visit_duration, :sample_percent]+ @session_metrics [:visits, :bounce_rate, :visit_duration, :pages_per_visit, :sample_percent]def aggregate(site, query, metrics) doevent_metrics = Enum.filter(metrics, &(&1 in @event_metrics))@@ -21,9 +21,8 @@ defmodule Plausible.Stats.Aggregate doPlausible.ClickhouseRepo.parallel_tasks([session_task, event_task, time_on_page_task])|> Enum.reduce(%{}, fn aggregate, task_result -> Map.merge(aggregate, task_result) end)- |> Enum.map(fn {metric, value} ->- {metric, %{value: round(value || 0)}}- end)+ |> Enum.map(&maybe_round_value/1)+ |> Enum.map(fn {metric, value} -> {metric, %{value: value}} end)|> Enum.into(%{})end@@ -109,4 +108,14 @@ defmodule Plausible.Stats.Aggregate do[[time_on_page]] = res.rows%{time_on_page: time_on_page}end++ @metrics_to_round [:bounce_rate, :time_on_page, :visit_duration, :sample_percent]++ defp maybe_round_value({metric, nil}), do: {metric, 0}++ defp maybe_round_value({metric, value}) when metric in @metrics_to_round do+ {metric, round(value)}+ end++ defp maybe_round_value(entry), do: entryenddiff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.exindex 7bba222dc9..413914209a 100644--- a/lib/plausible/stats/base.ex+++ b/lib/plausible/stats/base.ex@@ -314,7 +314,7 @@ defmodule Plausible.Stats.Base dodef select_session_metrics(q, [:visits | rest]) dofrom(s in q,select_merge: %{- visits: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", s.session_id)+ visits: fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign)})|> select_session_metrics(rest)@@ -359,6 +359,16 @@ defmodule Plausible.Stats.Base do|> select_session_metrics(rest)end+ def select_session_metrics(q, [:pages_per_visit | rest]) do+ from(s in q,+ select_merge: %{+ pages_per_visit:+ fragment("ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)", s.sign, s.pageviews, s.sign)+ }+ )+ |> select_session_metrics(rest)+ end+def select_session_metrics(q, [:sample_percent | rest]) dofrom(e in q,select_merge: %{diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.exindex d8f154620e..7768479476 100644--- a/lib/plausible/stats/breakdown.ex+++ b/lib/plausible/stats/breakdown.ex@@ -1,6 +1,6 @@defmodule Plausible.Stats.Breakdown douse Plausible.ClickhouseRepo- import Plausible.Stats.{Base, Imported}+ import Plausible.Stats.{Base, Imported, Util}require OpenTelemetry.Tracer, as: Traceralias Plausible.Stats.Queryalias Plausible.Goals@@ -584,18 +584,6 @@ defmodule Plausible.Stats.Breakdown doend)end- defp remove_internal_visits_metric(results, metrics) do- # "__internal_visits" is fetched when querying bounce rate and visit duration, as it- # is needed to calculate these from imported data. Let's remove it from the- # result if it wasn't requested.- if :bounce_rate in metrics or :visit_duration in metrics do- results- |> Enum.map(&Map.delete(&1, :__internal_visits))- else- results- end- end-defp apply_pagination(q, {limit, page}) dooffset = (page - 1) * limitdiff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.exindex 6b330d6bee..4bbaf361ad 100644--- a/lib/plausible/stats/timeseries.ex+++ b/lib/plausible/stats/timeseries.ex@@ -1,11 +1,11 @@defmodule Plausible.Stats.Timeseries douse Plausible.ClickhouseRepoalias Plausible.Stats.Query- import Plausible.Stats.Base+ import Plausible.Stats.{Base, Util}use Plausible.Stats.Fragments@event_metrics [:visitors, :pageviews]- @session_metrics [:visits, :bounce_rate, :visit_duration]+ @session_metrics [:visits, :bounce_rate, :visit_duration, :pages_per_visit]def timeseries(site, query, metrics) dosteps = buckets(query)@@ -45,6 +45,7 @@ defmodule Plausible.Stats.Timeseries do|> select_session_metrics(metrics)|> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics)|> ClickhouseRepo.all()+ |> remove_internal_visits_metric(metrics)enddefp buckets(%Query{interval: "month"} = query) do@@ -191,6 +192,7 @@ defmodule Plausible.Stats.Timeseries do:pageviews -> Map.merge(row, %{pageviews: 0}):visitors -> Map.merge(row, %{visitors: 0}):visits -> Map.merge(row, %{visits: 0})+ :pages_per_visit -> Map.merge(row, %{pages_per_visit: 0.0}):bounce_rate -> Map.merge(row, %{bounce_rate: nil}):visit_duration -> Map.merge(row, %{:visit_duration => nil})enddiff --git a/lib/plausible/stats/util.ex b/lib/plausible/stats/util.exnew file mode 100644index 0000000000..71a82d4f99--- /dev/null+++ b/lib/plausible/stats/util.ex@@ -0,0 +1,19 @@+defmodule Plausible.Stats.Util do+ @moduledoc """+ Utilities for modifying stat results+ """++ @doc """+ `__internal_visits` is fetched when querying bounce rate and visit duration, as it+ is needed to calculate these from imported data. This function removes that metric+ from all entries in the results list.+ """+ def remove_internal_visits_metric(results, metrics) do+ if :bounce_rate in metrics or :visit_duration in metrics do+ results+ |> Enum.map(&Map.delete(&1, :__internal_visits))+ else+ results+ end+ end+enddiff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.exindex 2f8b324fda..b656963ff0 100644--- a/lib/plausible_web/controllers/api/external_stats_controller.ex+++ b/lib/plausible_web/controllers/api/external_stats_controller.ex@@ -17,7 +17,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController dowith :ok <- validate_period(params),:ok <- validate_date(params),query <- Query.from(site, params),- {:ok, metrics} <- parse_metrics(params, nil, query) do+ {:ok, metrics} <- parse_and_validate_metrics(params, nil, query) doresults =if params["compare"] == "previous_period" doprev_query = Query.shift_back(query, site)@@ -42,7 +42,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController doPlausible.Stats.aggregate(site, query, metrics)end- json(conn, %{results: Map.take(results, metrics)})+ results =+ results+ |> Map.take(metrics)++ json(conn, %{results: results})else{:error, msg} ->conn@@ -59,7 +63,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do:ok <- validate_date(params),{:ok, property} <- validate_property(params),query <- Query.from(site, params),- {:ok, metrics} <- parse_metrics(params, property, query),+ {:ok, metrics} <- parse_and_validate_metrics(params, property, query),{:ok, limit} <- validate_or_default_limit(params) dopage = String.to_integer(Map.get(params, "page", "1"))results = Plausible.Stats.breakdown(site, query, property, metrics, {limit, page})@@ -116,42 +120,53 @@ defmodule PlausibleWeb.Api.ExternalStatsController dodefp event_only_property?(_), do: false@event_metrics ["visitors", "pageviews", "events"]- @session_metrics ["visits", "bounce_rate", "visit_duration"]- defp parse_metrics(params, property, query) do+ @session_metrics ["visits", "bounce_rate", "visit_duration", "pages_per_visit"]+ defp parse_and_validate_metrics(params, property, query) dometrics =Map.get(params, "metrics", "visitors")|> String.split(",")- event_only_filter = Map.keys(query.filters) |> Enum.find(&event_only_property?/1)+ case validate_all_metrics(metrics, property, query) do+ {:error, reason} -> {:error, reason}+ metrics -> {:ok, Enum.map(metrics, &String.to_atom/1)}+ end+ end- valid_metrics =- if event_only_property?(property) || event_only_filter do- @event_metrics- else- @event_metrics ++ @session_metrics+ defp validate_all_metrics(metrics, property, query) do+ Enum.reduce_while(metrics, [], fn metric, acc ->+ case validate_metric(metric, property, query) do+ {:ok, metric} -> {:cont, acc ++ [metric]}+ {:error, reason} -> {:halt, {:error, reason}}end+ end)+ end- invalid_metric = Enum.find(metrics, fn metric -> metric not in valid_metrics end)+ defp validate_metric(metric, _, _) when metric in @event_metrics, do: {:ok, metric}- if invalid_metric do- cond do- event_only_property?(property) && invalid_metric in @session_metrics ->- {:error,- "Session metric `#{invalid_metric}` cannot be queried for breakdown by `#{property}`."}+ defp validate_metric(metric, property, query) when metric in @session_metrics do+ event_only_filter = Map.keys(query.filters) |> Enum.find(&event_only_property?/1)- event_only_filter && invalid_metric in @session_metrics ->- {:error,- "Session metric `#{invalid_metric}` cannot be queried when using a filter on `#{event_only_filter}`."}+ cond do+ metric == "pages_per_visit" && property != nil ->+ {:error, "Metric `#{metric}` is not supported in breakdown queries"}- true ->- {:error,- "The metric `#{invalid_metric}` is not recognized. Find valid metrics from the documentation: https://plausible.io/docs/stats-api#get-apiv1statsbreakdown"}- end- else- {:ok, Enum.map(metrics, &String.to_atom/1)}+ event_only_property?(property) ->+ {:error, "Session metric `#{metric}` cannot be queried for breakdown by `#{property}`."}++ event_only_filter ->+ {:error,+ "Session metric `#{metric}` cannot be queried when using a filter on `#{event_only_filter}`."}++ true ->+ {:ok, metric}endend+ defp validate_metric(metric, _, _) do+ {:error,+ "The metric `#{metric}` is not recognized. Find valid metrics from the documentation: https://plausible.io/docs/stats-api#metrics"}+ end+def timeseries(conn, params) dosite = conn.assigns[:site]params = Map.put(params, "sample_threshold", "infinite")@@ -160,10 +175,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController do:ok <- validate_date(params),:ok <- validate_interval(params),query <- Query.from(site, params),- {:ok, metrics} <- parse_metrics(params, nil, query) do+ {:ok, metrics} <- parse_and_validate_metrics(params, nil, query) dograph = Plausible.Stats.timeseries(site, query, metrics)- metrics = metrics ++ [:date]- json(conn, %{results: Enum.map(graph, &Map.take(&1, metrics))})++ json(conn, %{results: graph})else{:error, msg} ->conndiff --git a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exsindex 76f83201ea..15456ccdd5 100644--- a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs@@ -73,7 +73,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest doassert json_response(conn, 400) == %{"error" =>- "The metric `led_zeppelin` is not recognized. Find valid metrics from the documentation: https://plausible.io/docs/stats-api#get-apiv1statsbreakdown"+ "The metric `led_zeppelin` is not recognized. Find valid metrics from the documentation: https://plausible.io/docs/stats-api#metrics"}end@@ -116,13 +116,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do}end- test "aggregates visitors, pageviews, visits, bounce rate and visit duration", %{+ test "rounds pages_per_visit to two decimal places", %{conn: conn,site: site} dopopulate_stats([build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),+ build(:pageview, user_id: 456, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: 456, domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])])@@ -131,13 +133,37 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do"site_id" => site.domain,"period" => "day","date" => "2021-01-01",- "metrics" => "pageviews,visits,visitors,bounce_rate,visit_duration"+ "metrics" => "pages_per_visit"+ })++ assert json_response(conn, 200)["results"] == %{+ "pages_per_visit" => %{"value" => 1.67}+ }+ end++ test "aggregates all metrics in a single query", %{+ conn: conn,+ site: site+ } do+ populate_stats([+ build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),+ build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn =+ get(conn, "/api/v1/stats/aggregate", %{+ "site_id" => site.domain,+ "period" => "day",+ "date" => "2021-01-01",+ "metrics" => "pageviews,visits,pages_per_visit,visitors,bounce_rate,visit_duration"})assert json_response(conn, 200)["results"] == %{"pageviews" => %{"value" => 3},"visitors" => %{"value" => 2},"visits" => %{"value" => 2},+ "pages_per_visit" => %{"value" => 1.5},"bounce_rate" => %{"value" => 50},"visit_duration" => %{"value" => 750}}diff --git a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exsindex 948a7558d1..60c697c3cb 100644--- a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs@@ -67,7 +67,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doassert json_response(conn, 400) == %{"error" =>- "The metric `baa` is not recognized. Find valid metrics from the documentation: https://plausible.io/docs/stats-api#get-apiv1statsbreakdown"+ "The metric `baa` is not recognized. Find valid metrics from the documentation: https://plausible.io/docs/stats-api#metrics"}enddiff --git a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exsindex 300e8b7df2..00dc07c86c 100644--- a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs@@ -797,4 +797,129 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest doassert second == %{"date" => "2021-01-02", "visitors" => 1}endend++ describe "metrics" do+ test "shows pageviews,visits,pages_per_visit for last 7d", %{conn: conn, site: site} do+ populate_stats([+ build(:pageview,+ user_id: @user_id,+ domain: site.domain,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ user_id: @user_id,+ domain: site.domain,+ timestamp: ~N[2021-01-01 00:05:00]+ ),+ build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, domain: site.domain, timestamp: ~N[2021-01-07 23:59:00])+ ])++ conn =+ get(conn, "/api/v1/stats/timeseries", %{+ "site_id" => site.domain,+ "period" => "7d",+ "metrics" => "pageviews,visits,pages_per_visit",+ "date" => "2021-01-07"+ })++ assert json_response(conn, 200) == %{+ "results" => [+ %{+ "date" => "2021-01-01",+ "pageviews" => 3,+ "visits" => 2,+ "pages_per_visit" => 1.5+ },+ %{+ "date" => "2021-01-02",+ "pageviews" => 0,+ "visits" => 0,+ "pages_per_visit" => 0.0+ },+ %{+ "date" => "2021-01-03",+ "pageviews" => 0,+ "visits" => 0,+ "pages_per_visit" => 0.0+ },+ %{+ "date" => "2021-01-04",+ "pageviews" => 0,+ "visits" => 0,+ "pages_per_visit" => 0.0+ },+ %{+ "date" => "2021-01-05",+ "pageviews" => 0,+ "visits" => 0,+ "pages_per_visit" => 0.0+ },+ %{+ "date" => "2021-01-06",+ "pageviews" => 0,+ "visits" => 0,+ "pages_per_visit" => 0.0+ },+ %{+ "date" => "2021-01-07",+ "pageviews" => 1,+ "visits" => 1,+ "pages_per_visit" => 1.0+ }+ ]+ }+ end++ test "rounds pages_per_visit to two decimal places", %{+ conn: conn,+ site: site+ } do+ populate_stats([+ build(:pageview,+ user_id: @user_id,+ domain: site.domain,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ user_id: @user_id,+ domain: site.domain,+ timestamp: ~N[2021-01-01 00:05:00]+ ),+ build(:pageview,+ user_id: @user_id,+ domain: site.domain,+ timestamp: ~N[2021-01-03 00:00:00]+ ),+ build(:pageview,+ user_id: @user_id,+ domain: site.domain,+ timestamp: ~N[2021-01-03 00:01:00]+ ),+ build(:pageview, domain: site.domain, timestamp: ~N[2021-01-03 00:00:00]),+ build(:pageview, domain: site.domain, timestamp: ~N[2021-01-03 00:00:00]),+ build(:pageview, domain: site.domain, timestamp: ~N[2021-01-07 23:59:00])+ ])++ conn =+ get(conn, "/api/v1/stats/timeseries", %{+ "site_id" => site.domain,+ "period" => "7d",+ "metrics" => "pages_per_visit",+ "date" => "2021-01-07"+ })++ assert json_response(conn, 200) == %{+ "results" => [+ %{"date" => "2021-01-01", "pages_per_visit" => 2.0},+ %{"date" => "2021-01-02", "pages_per_visit" => 0.0},+ %{"date" => "2021-01-03", "pages_per_visit" => 1.33},+ %{"date" => "2021-01-04", "pages_per_visit" => 0.0},+ %{"date" => "2021-01-05", "pages_per_visit" => 0.0},+ %{"date" => "2021-01-06", "pages_per_visit" => 0.0},+ %{"date" => "2021-01-07", "pages_per_visit" => 1.0}+ ]+ }+ end+ endend