Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Treat page filter as entry page filter only for bounce rate (#2874)
RobertJoonas7d935b79bf51
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 05669be754..2791742a15 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -25,6 +25,7 @@ All notable changes to this project will be documented in this file.- Fix bug when combining goal and prop filters plausible/analytics#2654### Changed+- Treat page filter as entry page filter for `bounce_rate`- Reject events with long URIs and data URIs plausible/analytics#2536- Always show direct traffic in sources reports plausible/analytics#2531- Stop recording XX and T1 country codes plausible/analytics#2556diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.exindex e7d79081c2..1c0dcc775d 100644--- a/lib/plausible/stats/aggregate.ex+++ b/lib/plausible/stats/aggregate.ex@@ -40,7 +40,7 @@ defmodule Plausible.Stats.Aggregate dodefp aggregate_sessions(site, query, metrics) dofrom(e in query_sessions(site, query), select: %{})|> filter_converted_sessions(site, query)- |> select_session_metrics(metrics)+ |> select_session_metrics(metrics, query)|> merge_imported(site, query, :aggregate, metrics)|> ClickhouseRepo.one()enddiff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.exindex 8fa3dc524a..901428888e 100644--- a/lib/plausible/stats/base.ex+++ b/lib/plausible/stats/base.ex@@ -42,42 +42,7 @@ defmodule Plausible.Stats.Base do)|> add_sample_hint(query)- q =- case query.filters["event:page"] do- {:is, page} ->- from(e in q, where: e.pathname == ^page)-- {:is_not, page} ->- from(e in q, where: e.pathname != ^page)-- {:matches_member, glob_exprs} ->- page_regexes = Enum.map(glob_exprs, &page_regex/1)- from(e in q, where: fragment("multiMatchAny(?, ?)", e.pathname, ^page_regexes))-- {:not_matches_member, glob_exprs} ->- page_regexes = Enum.map(glob_exprs, &page_regex/1)-- from(e in q,- where: fragment("not(multiMatchAny(?, ?))", e.pathname, ^page_regexes)- )-- {:matches, glob_expr} ->- regex = page_regex(glob_expr)- from(e in q, where: fragment("match(?, ?)", e.pathname, ^regex))-- {:does_not_match, glob_expr} ->- regex = page_regex(glob_expr)- from(e in q, where: fragment("not(match(?, ?))", e.pathname, ^regex))-- {:member, list} ->- from(e in q, where: e.pathname in ^list)-- {:not_member, list} ->- from(e in q, where: e.pathname not in ^list)-- nil ->- q- end+ q = from(e in q, where: ^dynamic_filter_condition(query, "event:page", :pathname))q =case query.filters["event:name"] do@@ -213,7 +178,6 @@ defmodule Plausible.Stats.Base do"city" => "city_geoname_id"}- # credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexitydef query_sessions(site, query) do{first_datetime, last_datetime} = utc_boundaries(query, site)@@ -227,57 +191,15 @@ defmodule Plausible.Stats.Base do|> filter_by_entry_props(query)Enum.reduce(Filters.visit_props(), sessions_q, fn prop_name, sessions_q ->- filter = query.filters["visit:" <> prop_name]+ filter_key = "visit:" <> prop_name- prop_name =+ db_field =Map.get(@api_prop_name_to_db, prop_name, prop_name)|> String.to_existing_atom()- case filter do- {:is, value} ->- value = db_prop_val(prop_name, value)- from(s in sessions_q, where: fragment("? = ?", field(s, ^prop_name), ^value))-- {:is_not, value} ->- value = db_prop_val(prop_name, value)- from(s in sessions_q, where: fragment("? != ?", field(s, ^prop_name), ^value))-- {:member, values} ->- list = Enum.map(values, &db_prop_val(prop_name, &1))- from(s in sessions_q, where: field(s, ^prop_name) in ^list)-- {:not_member, values} ->- list = Enum.map(values, &db_prop_val(prop_name, &1))- from(s in sessions_q, where: fragment("? not in ?", field(s, ^prop_name), ^list))-- {:matches, expr} ->- regex = page_regex(expr)- from(s in sessions_q, where: fragment("match(?, ?)", field(s, ^prop_name), ^regex))-- {:matches_member, exprs} ->- page_regexes = Enum.map(exprs, &page_regex/1)-- from(s in sessions_q,- where: fragment("multiMatchAny(?, ?)", field(s, ^prop_name), ^page_regexes)- )-- {:not_matches_member, exprs} ->- page_regexes = Enum.map(exprs, &page_regex/1)-- from(s in sessions_q,- where: fragment("not(multiMatchAny(?, ?))", field(s, ^prop_name), ^page_regexes)- )-- {:does_not_match, expr} ->- regex = page_regex(expr)- from(s in sessions_q, where: fragment("not(match(?, ?))", field(s, ^prop_name), ^regex))-- nil ->- sessions_q-- _ ->- raise "Unknown filter type"- end+ from(s in sessions_q,+ where: ^dynamic_filter_condition(query, filter_key, db_field)+ )end)end@@ -367,57 +289,67 @@ defmodule Plausible.Stats.Base dodef select_event_metrics(_, [unknown | _]), do: raise("Unknown metric " <> unknown)- def select_session_metrics(q, []), do: q+ def select_session_metrics(q, [], _query), do: q++ def select_session_metrics(q, [:bounce_rate | rest], query) do+ condition = dynamic_filter_condition(query, "event:page", :entry_page)- def select_session_metrics(q, [:bounce_rate | rest]) dofrom(s in q,- select_merge: %{- bounce_rate:- fragment("toUInt32(ifNotFinite(round(sum(is_bounce * sign) / sum(sign) * 100), 0))"),- __internal_visits: fragment("toUInt32(sum(sign))")- }+ select_merge:+ ^%{+ bounce_rate:+ dynamic(+ [],+ fragment(+ "toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))",+ ^condition,+ ^condition+ )+ ),+ __internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))+ })- |> select_session_metrics(rest)+ |> select_session_metrics(rest, query)end- def select_session_metrics(q, [:visits | rest]) do+ def select_session_metrics(q, [:visits | rest], query) dofrom(s in q,select_merge: %{visits: fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign)})- |> select_session_metrics(rest)+ |> select_session_metrics(rest, query)end- def select_session_metrics(q, [:pageviews | rest]) do+ def select_session_metrics(q, [:pageviews | rest], query) dofrom(s in q,select_merge: %{pageviews:fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews)})- |> select_session_metrics(rest)+ |> select_session_metrics(rest, query)end- def select_session_metrics(q, [:events | rest]) do+ def select_session_metrics(q, [:events | rest], query) dofrom(s in q,select_merge: %{events: fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events)})- |> select_session_metrics(rest)+ |> select_session_metrics(rest, query)end- def select_session_metrics(q, [:visitors | rest]) do+ def select_session_metrics(q, [:visitors | rest], query) dofrom(s in q,select_merge: %{visitors: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", s.user_id)})- |> select_session_metrics(rest)+ |> select_session_metrics(rest, query)end- def select_session_metrics(q, [:visit_duration | rest]) do+ def select_session_metrics(q, [:visit_duration | rest], query) dofrom(s in q,select_merge: %{:visit_duration =>@@ -425,27 +357,67 @@ defmodule Plausible.Stats.Base do__internal_visits: fragment("toUInt32(sum(sign))")})- |> select_session_metrics(rest)+ |> select_session_metrics(rest, query)end- def select_session_metrics(q, [:views_per_visit | rest]) do+ def select_session_metrics(q, [:views_per_visit | rest], query) dofrom(s in q,select_merge: %{views_per_visit:fragment("ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)", s.sign, s.pageviews, s.sign)})- |> select_session_metrics(rest)+ |> select_session_metrics(rest, query)end- def select_session_metrics(q, [:sample_percent | rest]) do+ def select_session_metrics(q, [:sample_percent | rest], query) dofrom(e in q,select_merge: %{sample_percent:fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)")})- |> select_event_metrics(rest)+ |> select_session_metrics(rest, query)+ end++ # credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity+ defp dynamic_filter_condition(query, filter_key, db_field) do+ case query && query.filters && query.filters[filter_key] do+ {:is, value} ->+ value = db_field_val(db_field, value)+ dynamic([x], field(x, ^db_field) == ^value)++ {:is_not, value} ->+ value = db_field_val(db_field, value)+ dynamic([x], field(x, ^db_field) != ^value)++ {:matches_member, glob_exprs} ->+ page_regexes = Enum.map(glob_exprs, &page_regex/1)+ dynamic([x], fragment("multiMatchAny(?, ?)", field(x, ^db_field), ^page_regexes))++ {:not_matches_member, glob_exprs} ->+ page_regexes = Enum.map(glob_exprs, &page_regex/1)+ dynamic([x], fragment("not(multiMatchAny(?, ?))", field(x, ^db_field), ^page_regexes))++ {:matches, glob_expr} ->+ regex = page_regex(glob_expr)+ dynamic([x], fragment("match(?, ?)", field(x, ^db_field), ^regex))++ {:does_not_match, glob_expr} ->+ regex = page_regex(glob_expr)+ dynamic([x], fragment("not(match(?, ?))", field(x, ^db_field), ^regex))++ {:member, list} ->+ list = Enum.map(list, &db_field_val(db_field, &1))+ dynamic([x], field(x, ^db_field) in ^list)++ {:not_member, list} ->+ list = Enum.map(list, &db_field_val(db_field, &1))+ dynamic([x], field(x, ^db_field) not in ^list)++ nil ->+ true+ endenddef filter_converted_sessions(db_query, site, query) do@@ -464,15 +436,15 @@ defmodule Plausible.Stats.Base doendend- defp db_prop_val(:referrer_source, @no_ref), do: ""- defp db_prop_val(:referrer, @no_ref), do: ""- defp db_prop_val(:utm_medium, @no_ref), do: ""- defp db_prop_val(:utm_source, @no_ref), do: ""- defp db_prop_val(:utm_campaign, @no_ref), do: ""- defp db_prop_val(:utm_content, @no_ref), do: ""- defp db_prop_val(:utm_term, @no_ref), do: ""- defp db_prop_val(_, @not_set), do: ""- defp db_prop_val(_, val), do: val+ defp db_field_val(:referrer_source, @no_ref), do: ""+ defp db_field_val(:referrer, @no_ref), do: ""+ defp db_field_val(:utm_medium, @no_ref), do: ""+ defp db_field_val(:utm_source, @no_ref), do: ""+ defp db_field_val(:utm_campaign, @no_ref), do: ""+ defp db_field_val(:utm_content, @no_ref), do: ""+ defp db_field_val(:utm_term, @no_ref), do: ""+ defp db_field_val(_, @not_set), do: ""+ defp db_field_val(_, val), do: valdefp beginning_of_time(candidate, native_stats_start_at) doif Timex.after?(native_stats_start_at, candidate) dodiff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.exindex 91db6b8b59..a33823e297 100644--- a/lib/plausible/stats/breakdown.ex+++ b/lib/plausible/stats/breakdown.ex@@ -189,7 +189,7 @@ defmodule Plausible.Stats.Breakdown do)|> filter_converted_sessions(site, query)|> do_group_by(property)- |> select_session_metrics(metrics)+ |> select_session_metrics(metrics, query)|> merge_imported(site, query, property, metrics)|> apply_pagination(pagination)|> ClickhouseRepo.all()diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.exindex 1125f39881..6a95f5d389 100644--- a/lib/plausible/stats/timeseries.ex+++ b/lib/plausible/stats/timeseries.ex@@ -46,7 +46,7 @@ defmodule Plausible.Stats.Timeseries dofrom(e in query_sessions(site, query), select: %{})|> filter_converted_sessions(site, query)|> select_bucket(site, query)- |> select_session_metrics(metrics)+ |> select_session_metrics(metrics, query)|> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics)|> ClickhouseRepo.all()|> remove_internal_visits_metric(metrics)diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.exindex 58f7226dac..6256c057e2 100644--- a/lib/plausible_web/controllers/api/external_stats_controller.ex+++ b/lib/plausible_web/controllers/api/external_stats_controller.ex@@ -147,8 +147,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController doevent_only_filter = Map.keys(query.filters) |> Enum.find(&event_only_property?/1)cond do+ metric == "views_per_visit" && query.filters["event:page"] ->+ {:error, "Metric `#{metric}` cannot be queried with a filter on `event:page`."}+metric == "views_per_visit" && property != nil ->- {:error, "Metric `#{metric}` is not supported in breakdown queries"}+ {:error, "Metric `#{metric}` is not supported in breakdown queries."}event_only_property?(property) ->{:error, "Session metric `#{metric}` cannot be queried for breakdown by `#{property}`."}diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex 92cf9cea59..bb0c6dfa66 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -385,7 +385,6 @@ defmodule PlausibleWeb.Api.StatsController do:visitors,:visits,:pageviews,- :views_per_visit,:bounce_rate,:time_on_page,:sample_percentdiff --git a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exsindex 908cfc9683..2236e324a4 100644--- a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs@@ -94,6 +94,23 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do"Session metric `visit_duration` cannot be queried when using a filter on `event:name`."}end++ test "validates that views_per_visit cannot be used with event:page filter", %{+ conn: conn,+ site: site+ } do+ conn =+ get(conn, "/api/v1/stats/aggregate", %{+ "site_id" => site.domain,+ "filters" => "event:page==/something",+ "metrics" => "views_per_visit"+ })++ assert json_response(conn, 400) == %{+ "error" =>+ "Metric `views_per_visit` cannot be queried with a filter on `event:page`."+ }+ endendtest "aggregates a single metric", %{conn: conn, site: site} do@@ -642,7 +659,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest doassert json_response(conn, 200)["results"] == %{"pageviews" => %{"value" => 2},"visitors" => %{"value" => 2},- "bounce_rate" => %{"value" => 50},+ "bounce_rate" => %{"value" => 100},"visit_duration" => %{"value" => 750}}enddiff --git a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exsindex fdc9047a3c..8d2b052ced 100644--- a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs@@ -1204,6 +1204,63 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendend+ test "event:page filter is interpreted as entry_page filter only for bounce_rate", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview,+ browser: "Chrome",+ user_id: @user_id,+ pathname: "/ignore",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ browser: "Chrome",+ user_id: @user_id,+ pathname: "/plausible.io",+ timestamp: ~N[2021-01-01 00:01:00]+ ),+ build(:pageview,+ browser: "Chrome",+ user_id: 456,+ pathname: "/important-page",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ browser: "Chrome",+ user_id: 456,+ pathname: "/",+ timestamp: ~N[2021-01-01 00:01:00]+ ),+ build(:pageview,+ browser: "Chrome",+ pathname: "/plausible.io",+ timestamp: ~N[2021-01-01 00:01:00]+ )+ ])++ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "site_id" => site.domain,+ "period" => "day",+ "date" => "2021-01-01",+ "metrics" => "visitors,bounce_rate",+ "property" => "visit:browser",+ "filters" => "event:page == /plausible.io|/important-page"+ })++ assert json_response(conn, 200) == %{+ "results" => [+ %{+ "browser" => "Chrome",+ "bounce_rate" => 50,+ "visitors" => 3+ }+ ]+ }+ end+test "event:goal pageview filter for breakdown by visit source", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview,diff --git a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exsindex b0e1f6109b..2581a7a18f 100644--- a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs@@ -697,7 +697,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do"period" => "month","date" => "2021-01-01","filters" => "event:page==/hello",- "metrics" => "visitors,visits,pageviews,views_per_visit,bounce_rate,visit_duration"+ "metrics" => "visitors,visits,pageviews,bounce_rate,visit_duration"})res = json_response(conn, 200)["results"]@@ -707,8 +707,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do"visitors" => 2,"visits" => 2,"pageviews" => 2,- "views_per_visit" => 1.5,- "bounce_rate" => 50,+ "bounce_rate" => 100,"visit_duration" => 150}enddiff --git a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exsindex 8711a482e3..991e6552e8 100644--- a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs@@ -575,6 +575,30 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest doassert %{"name" => "Total visits", "value" => 2} in res["top_stats"]end++ test "does not return the views_per_visit metric when a page filter is applied", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview, pathname: "/")+ ])++ filters = Jason.encode!(%{page: "/"})++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/top-stats?period=day&filters=#{filters}"+ )++ res = json_response(conn, 200)++ refute Enum.any?(res["top_stats"], fn+ %{"name" => "Views per visit"} -> true+ _ -> false+ end)+ endenddescribe "GET /api/stats/top-stats - filtered for goal" do