Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
time-on-page: query (#5159)
Karl-Aksel Puulmanne9b30e0ba553
diff --git a/assets/js/types/query-api.d.ts b/assets/js/types/query-api.d.tsindex a52c60da4a..e568c0c31a 100644--- a/assets/js/types/query-api.d.ts+++ b/assets/js/types/query-api.d.ts@@ -201,6 +201,7 @@ export interface QueryApiSchema {*/date_range: [string, string];};+ legacy_time_on_page_cutoff?: string;};pagination?: {/**diff --git a/extra/lib/plausible/stats/funnel.ex b/extra/lib/plausible/stats/funnel.exindex 409f226c2a..a9fae4ca90 100644--- a/extra/lib/plausible/stats/funnel.ex+++ b/extra/lib/plausible/stats/funnel.ex@@ -27,10 +27,10 @@ defmodule Plausible.Stats.Funnel doendend- def funnel(site, query, %Funnel{} = funnel) do+ def funnel(_site, query, %Funnel{} = funnel) dofunnel_data =- site- |> Base.base_event_query(query)+ query+ |> Base.base_event_query()|> query_funnel(funnel)# Funnel definition steps are 1-indexed, if there's index 0 in the resulting query,diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.exindex d3ad7bffa0..bc80949539 100644--- a/lib/plausible/imported/csv_importer.ex+++ b/lib/plausible/imported/csv_importer.ex@@ -182,7 +182,7 @@ defmodule Plausible.Imported.CSVImporter do"imported_operating_systems" =>"date Date, operating_system String, operating_system_version String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64","imported_pages" =>- "date Date, hostname String, page String, visits UInt64, visitors UInt64, pageviews UInt64, total_scroll_depth UInt64, total_scroll_depth_visits UInt64",+ "date Date, hostname String, page String, visits UInt64, visitors UInt64, pageviews UInt64, total_scroll_depth UInt64, total_scroll_depth_visits UInt64, total_time_on_page UInt64, total_time_on_page_visits UInt64","imported_sources" =>"date Date, source String, referrer String, utm_source String, utm_medium String, utm_campaign String, utm_content String, utm_term String, pageviews UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32","imported_visitors" =>diff --git a/lib/plausible/imported/google_analytics4.ex b/lib/plausible/imported/google_analytics4.exindex 90709f17d2..cc03cda51b 100644--- a/lib/plausible/imported/google_analytics4.ex+++ b/lib/plausible/imported/google_analytics4.ex@@ -205,7 +205,8 @@ defmodule Plausible.Imported.GoogleAnalytics4 dopageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),# NOTE: no exits metric in GA4 API currentlyexits: 0,- time_on_page: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()+ total_time_on_page: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number(),+ total_time_on_page_visits: row.metrics |> Map.fetch!("sessions") |> parse_number()}enddiff --git a/lib/plausible/imported/page.ex b/lib/plausible/imported/page.exindex 6e27c1bd89..bc4a9edba7 100644--- a/lib/plausible/imported/page.ex+++ b/lib/plausible/imported/page.ex@@ -14,8 +14,9 @@ defmodule Plausible.Imported.Page dofield :active_visitors, Ch, type: "UInt64"field :pageviews, Ch, type: "UInt64"field :exits, Ch, type: "UInt64"- field :time_on_page, Ch, type: "UInt64"field :total_scroll_depth, Ch, type: "UInt64"field :total_scroll_depth_visits, Ch, type: "UInt64"+ field :total_time_on_page, Ch, type: "UInt64"+ field :total_time_on_page_visits, Ch, type: "UInt64"endenddiff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.exindex de4a9ba3d3..5b77b8c9a5 100644--- a/lib/plausible/stats/aggregate.ex+++ b/lib/plausible/stats/aggregate.ex@@ -56,7 +56,7 @@ defmodule Plausible.Stats.Aggregate do|> maybe_round_value(metric)end- @metrics_to_round [:bounce_rate, :time_on_page, :visit_duration, :sample_percent]+ @metrics_to_round [:bounce_rate, :visit_duration, :sample_percent]defp maybe_round_value(nil, _metric), do: nildefp maybe_round_value(value, metric) when metric in @metrics_to_round, do: round(value)diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.exindex e9c2a48b63..efed82b903 100644--- a/lib/plausible/stats/base.ex+++ b/lib/plausible/stats/base.ex@@ -4,13 +4,13 @@ defmodule Plausible.Stats.Base doalias Plausible.Stats.{TableDecider, SQL}import Ecto.Query- def base_event_query(site, query) do- events_q = query_events(site, query)+ def base_event_query(query) do+ events_q = query_events(query)if TableDecider.events_join_sessions?(query) dosessions_q =from(- s in query_sessions(site, query),+ s in query_sessions(query),select: %{session_id: s.session_id},where: s.sign == 1,group_by: s.session_id@@ -26,7 +26,7 @@ defmodule Plausible.Stats.Base doendend- defp query_events(_site, query) do+ defp query_events(query) doq = from(e in "events_v2", where: ^SQL.WhereBuilder.build(:events, query))on_ee do@@ -36,7 +36,7 @@ defmodule Plausible.Stats.Base doqend- def query_sessions(_site, query) do+ def query_sessions(query) doq = from(s in "sessions_v2", where: ^SQL.WhereBuilder.build(:sessions, query))on_ee dodiff --git a/lib/plausible/stats/custom_props.ex b/lib/plausible/stats/custom_props.exindex 932e5079f6..8190d9caec 100644--- a/lib/plausible/stats/custom_props.ex+++ b/lib/plausible/stats/custom_props.ex@@ -13,7 +13,7 @@ defmodule Plausible.Stats.CustomProps do[key]_ ->- from(e in base_event_query(site, query),+ from(e in base_event_query(query),join: meta in fragment("meta"),hints: "ARRAY",on: true,diff --git a/lib/plausible/stats/filter_suggestions.ex b/lib/plausible/stats/filter_suggestions.exindex f3134f198f..91e9149780 100644--- a/lib/plausible/stats/filter_suggestions.ex+++ b/lib/plausible/stats/filter_suggestions.ex@@ -14,7 +14,7 @@ defmodule Plausible.Stats.FilterSuggestions doq =from(- e in query_sessions(site, query),+ e in query_sessions(query),group_by: e.country_code,order_by: [desc: fragment("count(*)")],select: e.country_code@@ -35,7 +35,7 @@ defmodule Plausible.Stats.FilterSuggestions dodef filter_suggestions(site, query, "region", "") dofrom(- e in query_sessions(site, query),+ e in query_sessions(query),group_by: e.subdivision1_code,order_by: [desc: fragment("count(*)")],select: e.subdivision1_code,@@ -67,7 +67,7 @@ defmodule Plausible.Stats.FilterSuggestions doq =from(- e in query_sessions(site, query),+ e in query_sessions(query),group_by: e.subdivision1_code,order_by: [desc: fragment("count(*)")],select: e.subdivision1_code,@@ -105,7 +105,7 @@ defmodule Plausible.Stats.FilterSuggestions dodef filter_suggestions(site, query, "city", "") dofrom(- e in query_sessions(site, query),+ e in query_sessions(query),group_by: e.city_geoname_id,order_by: [desc: fragment("count(*)")],select: e.city_geoname_id,@@ -129,7 +129,7 @@ defmodule Plausible.Stats.FilterSuggestions doq =from(- e in query_sessions(site, query),+ e in query_sessions(query),group_by: e.city_geoname_id,order_by: [desc: fragment("count(*)")],select: e.city_geoname_id,@@ -163,7 +163,7 @@ defmodule Plausible.Stats.FilterSuggestions dodef filter_suggestions(site, query, "prop_key", filter_search) dofilter_query = if filter_search == nil, do: "%", else: "%#{filter_search}%"- from(e in base_event_query(site, query),+ from(e in base_event_query(query),join: meta in "meta",hints: "ARRAY",on: true,@@ -213,9 +213,9 @@ defmodule Plausible.Stats.FilterSuggestions dobase_q =if filter_name in [:pathname, :hostname] do- base_event_query(site, query)+ base_event_query(query)else- query_sessions(site, query)+ query_sessions(query)endfrom(e in base_q,@@ -237,22 +237,19 @@ defmodule Plausible.Stats.FilterSuggestions do|> wrap_suggestions()end- def custom_prop_value_filter_suggestions(site, query, prop_key, filter_search) do+ def custom_prop_value_filter_suggestions(_site, query, prop_key, filter_search) dofilter_query = if filter_search == nil, do: "%", else: "%#{filter_search}%"none_q =from(- e in base_event_query(- site,- Query.remove_top_level_filters(query, ["event:props:#{prop_key}"])- ),+ e in base_event_query(Query.remove_top_level_filters(query, ["event:props:#{prop_key}"])),select: "(none)",where: not has_key(e, :meta, ^prop_key),limit: 1)search_q =- from(e in base_event_query(site, query),+ from(e in base_event_query(query),select: get_by_key(e, :meta, ^prop_key),where:has_key(e, :meta, ^prop_key) anddiff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.exindex 392e375287..b8af4e9bf9 100644--- a/lib/plausible/stats/filters/query_parser.ex+++ b/lib/plausible/stats/filters/query_parser.ex@@ -14,7 +14,8 @@ defmodule Plausible.Stats.Filters.QueryParser doimports_meta: false,time_labels: false,total_rows: false,- comparisons: nil+ comparisons: nil,+ legacy_time_on_page_cutoff: nil}@default_pagination %{diff --git a/lib/plausible/stats/goal_suggestions.ex b/lib/plausible/stats/goal_suggestions.exindex e0843cf1eb..2cc20e0808 100644--- a/lib/plausible/stats/goal_suggestions.ex+++ b/lib/plausible/stats/goal_suggestions.ex@@ -41,7 +41,7 @@ defmodule Plausible.Stats.GoalSuggestions doquery = Query.from(site, params)native_q =- from(e in base_event_query(site, query),+ from(e in base_event_query(query),where: fragment("? ilike ?", e.name, ^matches),where: e.name not in ["pageview", "engagement"],where: fragment("trim(?)", e.name) != "",diff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.exindex af7f874be3..6948879252 100644--- a/lib/plausible/stats/imported/imported.ex+++ b/lib/plausible/stats/imported/imported.ex@@ -226,7 +226,7 @@ defmodule Plausible.Stats.Imported doimported_q =site|> Imported.Base.query_imported(query)- |> select_imported_metrics(query.metrics)+ |> select_imported_metrics(query)|> paginate_optimization(query)from(@@ -234,7 +234,7 @@ defmodule Plausible.Stats.Imported docross_join: i in subquery(imported_q),select: %{})- |> select_joined_metrics(query.metrics)+ |> select_joined_metrics(query)enddef merge_imported(q, site, %Query{dimensions: ["event:goal"]} = query) do@@ -253,7 +253,7 @@ defmodule Plausible.Stats.Imported doi.name)})- |> select_imported_metrics(query.metrics)+ |> select_imported_metrics(query)|> group_by([], selected_as(:dim0))|> where([], selected_as(:dim0) != 0)@@ -282,10 +282,10 @@ defmodule Plausible.Stats.Imported do|> select_merge_as([_i, index], %{dim0: fragment("CAST(?, 'UInt64')", index)})- |> select_imported_metrics(query.metrics)+ |> select_imported_metrics(query)end)|> Enum.reduce(q, fn imports_q, q ->- naive_dimension_join(q, imports_q, query.metrics)+ naive_dimension_join(q, imports_q, query)end)end@@ -298,7 +298,7 @@ defmodule Plausible.Stats.Imported do|> Imported.Base.query_imported(query)|> where([i], i.visitors > 0)|> group_imported_by(query)- |> select_imported_metrics(query.metrics)+ |> select_imported_metrics(query)|> paginate_optimization(query)from(s in subquery(q),@@ -307,7 +307,7 @@ defmodule Plausible.Stats.Imported doselect: %{})|> select_joined_dimensions(query)- |> select_joined_metrics(query.metrics)+ |> select_joined_metrics(query)elseqend@@ -319,7 +319,7 @@ defmodule Plausible.Stats.Imported do|> select_merge([i], %{total_visitors: fragment("sum(?)", i.visitors)})end- defp naive_dimension_join(q1, q2, metrics) do+ defp naive_dimension_join(q1, q2, query) dofrom(a in subquery(q1),full_join: b in subquery(q2),on: a.dim0 == b.dim0,@@ -328,7 +328,7 @@ defmodule Plausible.Stats.Imported do|> select_merge_as([a, b], %{dim0: fragment("if(? != 0, ?, ?)", a.dim0, a.dim0, b.dim0)})- |> select_joined_metrics(metrics)+ |> select_joined_metrics(query)end# Optimization for cases when grouping by a very high cardinality column.diff --git a/lib/plausible/stats/imported/sql/expression.ex b/lib/plausible/stats/imported/sql/expression.exindex 95d005e676..b9719677dc 100644--- a/lib/plausible/stats/imported/sql/expression.ex+++ b/lib/plausible/stats/imported/sql/expression.ex@@ -17,16 +17,16 @@ defmodule Plausible.Stats.Imported.SQL.Expression dodef select_imported_metrics(%Ecto.Query{from: %Ecto.Query.FromExpr{source: {table, _}}} = q,- metrics+ query) doselect_clause =- metrics- |> Enum.map(&select_metric(&1, table))+ query.metrics+ |> Enum.map(&select_metric(&1, table, query))|> Enum.reduce(%{}, &Map.merge/2)q|> select_merge(q, ^select_clause)- |> filter_pageviews(metrics, table)+ |> filter_pageviews(query.metrics, table)enddefp filter_pageviews(q, metrics, table) do@@ -39,90 +39,116 @@ defmodule Plausible.Stats.Imported.SQL.Expression doendend- defp select_metric(:visitors, _table) do+ defp select_metric(:visitors, _table, _query) dowrap_alias([i], %{visitors: sum(i.visitors)})end- defp select_metric(:events, "imported_custom_events") do+ defp select_metric(:events, "imported_custom_events", _query) dowrap_alias([i], %{events: sum(i.events)})end- defp select_metric(:events, _table) do+ defp select_metric(:events, _table, _query) dowrap_alias([i], %{events: sum(i.pageviews)})end- defp select_metric(:visits, "imported_exit_pages") do+ defp select_metric(:visits, "imported_exit_pages", _query) dowrap_alias([i], %{visits: sum(i.exits)})end- defp select_metric(:visits, "imported_entry_pages") do+ defp select_metric(:visits, "imported_entry_pages", _query) dowrap_alias([i], %{visits: sum(i.entrances)})end- defp select_metric(:visits, _table) do+ defp select_metric(:visits, _table, _query) dowrap_alias([i], %{visits: sum(i.visits)})end- defp select_metric(:pageviews, "imported_custom_events") do+ defp select_metric(:pageviews, "imported_custom_events", _query) dowrap_alias([i], %{pageviews: 0})end- defp select_metric(:pageviews, _table) do+ defp select_metric(:pageviews, _table, _query) dowrap_alias([i], %{pageviews: sum(i.pageviews)})end- defp select_metric(:bounce_rate, "imported_pages") do+ defp select_metric(:bounce_rate, "imported_pages", _query) dowrap_alias([i], %{bounces: 0, __internal_visits: 0})end- defp select_metric(:bounce_rate, "imported_exit_pages") do+ defp select_metric(:bounce_rate, "imported_exit_pages", _query) dowrap_alias([i], %{bounces: sum(i.bounces), __internal_visits: sum(i.exits)})end- defp select_metric(:bounce_rate, "imported_entry_pages") do+ defp select_metric(:bounce_rate, "imported_entry_pages", _query) dowrap_alias([i], %{bounces: sum(i.bounces), __internal_visits: sum(i.entrances)})end- defp select_metric(:bounce_rate, _table) do+ defp select_metric(:bounce_rate, _table, _query) dowrap_alias([i], %{bounces: sum(i.bounces), __internal_visits: sum(i.visits)})end- defp select_metric(:visit_duration, "imported_pages") do+ defp select_metric(:visit_duration, "imported_pages", _query) dowrap_alias([i], %{visit_duration: 0})end- defp select_metric(:visit_duration, "imported_exit_pages") do+ defp select_metric(:visit_duration, "imported_exit_pages", _query) dowrap_alias([i], %{visit_duration: sum(i.visit_duration), __internal_visits: sum(i.exits)})end- defp select_metric(:visit_duration, "imported_entry_pages") do+ defp select_metric(:visit_duration, "imported_entry_pages", _query) dowrap_alias([i], %{visit_duration: sum(i.visit_duration), __internal_visits: sum(i.entrances)})end- defp select_metric(:visit_duration, _table) do+ defp select_metric(:visit_duration, _table, _query) dowrap_alias([i], %{visit_duration: sum(i.visit_duration), __internal_visits: sum(i.visits)})end- defp select_metric(:views_per_visit, "imported_exit_pages") do+ defp select_metric(:views_per_visit, "imported_exit_pages", _query) dowrap_alias([i], %{pageviews: sum(i.pageviews), __internal_visits: sum(i.exits)})end- defp select_metric(:views_per_visit, "imported_entry_pages") do+ defp select_metric(:views_per_visit, "imported_entry_pages", _query) dowrap_alias([i], %{pageviews: sum(i.pageviews), __internal_visits: sum(i.entrances)})end- defp select_metric(:views_per_visit, _table) do+ defp select_metric(:views_per_visit, _table, _query) dowrap_alias([i], %{pageviews: sum(i.pageviews), __internal_visits: sum(i.visits)})end- defp select_metric(:scroll_depth, "imported_pages") do+ defp select_metric(:scroll_depth, "imported_pages", _query) dowrap_alias([i], %{total_scroll_depth: sum(i.total_scroll_depth),total_scroll_depth_visits: sum(i.total_scroll_depth_visits)})end- defp select_metric(_metric, _table), do: %{}+ defp select_metric(:time_on_page, "imported_pages", query) do+ case query.time_on_page_data do+ %{include_new_metric: false} ->+ wrap_alias([i], %{+ total_time_on_page: 0,+ total_time_on_page_visits: 0+ })++ %{include_new_metric: true, cutoff: nil} ->+ wrap_alias([i], %{+ total_time_on_page: sum(i.total_time_on_page),+ total_time_on_page_visits: sum(i.total_time_on_page_visits)+ })++ %{include_new_metric: true, cutoff: cutoff} ->+ cutoff_date = cutoff |> DateTime.shift_zone!(query.timezone) |> DateTime.to_date()++ wrap_alias([i], %{+ total_time_on_page:+ fragment("sumIf(?, ? >= ?)", i.total_time_on_page, i.date, ^cutoff_date),+ total_time_on_page_visits:+ fragment("sumIf(?, ? >= ?)", i.total_time_on_page_visits, i.date, ^cutoff_date)+ })+ end+ end++ defp select_metric(_metric, _table, _query), do: %{}def group_imported_by(q, query) doEnum.reduce(query.dimensions, q, fn dimension, q ->@@ -147,7 +173,7 @@ defmodule Plausible.Stats.Imported.SQL.Expression doenddefp select_group_fields(q, "event:page", key, _query) do- select_merge_as(q, [i], %{key => i.page, time_on_page: sum(i.time_on_page)})+ select_merge_as(q, [i], %{key => i.page})enddefp select_group_fields(q, dimension, key, _query)@@ -267,40 +293,39 @@ defmodule Plausible.Stats.Imported.SQL.Expression do})end- def select_joined_metrics(q, []), do: q+ def select_joined_metrics(q, query) do+ select_metrics =+ query.metrics+ |> Enum.map(&joined_metric(&1, query))+ |> Enum.reduce(%{}, &Map.merge/2)++ select_merge(q, ^select_metrics)+ end+# NOTE: Reverse-engineering the native data bounces and total visit# durations to combine with imported data is inefficient. Instead both# queries should fetch bounces/total_visit_duration and visits and be# used as subqueries to a main query that then find the bounce rate/avg# visit_duration.- def select_joined_metrics(q, [:visits | rest]) do- q- |> select_merge_as([s, i], %{visits: s.visits + i.visits})- |> select_joined_metrics(rest)+ defp joined_metric(:visits, _query) do+ wrap_alias([s, i], %{visits: s.visits + i.visits})end- def select_joined_metrics(q, [:visitors | rest]) do- q- |> select_merge_as([s, i], %{visitors: s.visitors + i.visitors})- |> select_joined_metrics(rest)+ defp joined_metric(:visitors, _query) do+ wrap_alias([s, i], %{visitors: s.visitors + i.visitors})end- def select_joined_metrics(q, [:events | rest]) do- q- |> select_merge_as([s, i], %{events: s.events + i.events})- |> select_joined_metrics(rest)+ defp joined_metric(:events, _query) do+ wrap_alias([s, i], %{events: s.events + i.events})end- def select_joined_metrics(q, [:pageviews | rest]) do- q- |> select_merge_as([s, i], %{pageviews: s.pageviews + i.pageviews})- |> select_joined_metrics(rest)+ defp joined_metric(:pageviews, _query) do+ wrap_alias([s, i], %{pageviews: s.pageviews + i.pageviews})end- def select_joined_metrics(q, [:views_per_visit | rest]) do- q- |> select_merge_as([s, i], %{+ defp joined_metric(:views_per_visit, _query) do+ wrap_alias([s, i], %{views_per_visit:fragment("if(? + ? > 0, round((? + ? * ?) / (? + ?), 2), 0)",@@ -313,12 +338,10 @@ defmodule Plausible.Stats.Imported.SQL.Expression dos.__internal_visits)})- |> select_joined_metrics(rest)end- def select_joined_metrics(q, [:bounce_rate | rest]) do- q- |> select_merge_as([s, i], %{+ defp joined_metric(:bounce_rate, _query) do+ wrap_alias([s, i], %{bounce_rate:fragment("if(? + ? > 0, round(100 * (? + (? * ? / 100)) / (? + ?)), 0)",@@ -331,12 +354,10 @@ defmodule Plausible.Stats.Imported.SQL.Expression dos.__internal_visits)})- |> select_joined_metrics(rest)end- def select_joined_metrics(q, [:visit_duration | rest]) do- q- |> select_merge_as([s, i], %{+ defp joined_metric(:visit_duration, _query) do+ wrap_alias([s, i], %{visit_duration:fragment("""@@ -355,34 +376,51 @@ defmodule Plausible.Stats.Imported.SQL.Expression doi.__internal_visits)})- |> select_joined_metrics(rest)end# The final `scroll_depth` gets selected at a later querybuilding step# (in `Plausible.Stats.SQL.SpecialMetrics.add/3`). But in order to avoid# having to join with imported data there again, we select the required# information from imported data here already.- def select_joined_metrics(q, [:scroll_depth | rest]) do- q- |> select_merge_as([s, i], %{+ defp joined_metric(:scroll_depth, _query) do+ wrap_alias([s, i], %{__imported_total_scroll_depth: i.total_scroll_depth,__imported_total_scroll_depth_visits: i.total_scroll_depth_visits})- |> select_joined_metrics(rest)+ end++ defp joined_metric(:time_on_page, query) do+ wrap_alias([s, i], %{+ __internal_total_time_on_page: s.__internal_total_time_on_page + i.total_time_on_page,+ __internal_total_time_on_page_visits:+ s.__internal_total_time_on_page_visits + i.total_time_on_page_visits+ })+ |> Map.merge(time_on_page_metric(query))end# Ignored as it's calculated separately- def select_joined_metrics(q, [metric | rest])- when metric in [:conversion_rate, :group_conversion_rate, :percentage] do- q- |> select_joined_metrics(rest)+ defp joined_metric(metric, _query)+ when metric in [:conversion_rate, :group_conversion_rate, :percentage] do+ %{}end- def select_joined_metrics(q, [metric | rest]) do- q- |> select_merge_as([s, i], %{metric => field(s, ^metric)})- |> select_joined_metrics(rest)+ defp joined_metric(metric, _query) do+ wrap_alias([s, i], %{metric => field(s, ^metric)})enddefp dim(dimension), do: Plausible.Stats.Filters.without_prefix(dimension)++ defp time_on_page_metric(query) do+ if query.time_on_page_data.include_legacy_metric do+ %{}+ else+ wrap_alias([], %{+ time_on_page:+ time_on_page(+ selected_as(:__internal_total_time_on_page),+ selected_as(:__internal_total_time_on_page_visits)+ )+ })+ end+ endenddiff --git a/lib/plausible/stats/legacy/time_on_page.ex b/lib/plausible/stats/legacy/time_on_page.exindex c004b331dc..488df28e1e 100644--- a/lib/plausible/stats/legacy/time_on_page.ex+++ b/lib/plausible/stats/legacy/time_on_page.ex@@ -1,36 +1,74 @@defmodule Plausible.Stats.Legacy.TimeOnPage do@moduledoc """- Calculation methods for legacy time_on_page metric. Note the metric- has its own limitations and quirks.+ Calculation methods for `legacy` time_on_page metric. `Legacy` calculation methods+ are used when site does not have engagement data for the requested dates.- Closely coupled with Plausible.Stats.QueryRunner.+ Query `include.legacy_time_on_page_cutoff` is used to determine what time range+ to use for the legacy time_on_page calculations.++ Legacy metric is not exposed in the public API."""use Plausible.ClickhouseRepo+ use Plausible.Stats.SQL.Fragments+import Ecto.Query+ import Plausible.Stats.Util++ alias Plausible.Stats.{Base, Filters, Query, SQL}- alias Plausible.Stats.{Base, Filters, Query, SQL, Util}+ def can_merge_legacy_time_on_page?(query) do+ :time_on_page in query.metrics and query.dimensions in [[], ["event:page"]]+ end- def calculate(site, query, ch_results) do- case {:time_on_page in query.metrics, query.dimensions} do- {true, []} ->- aggregate_time_on_page(site, query)+ def merge_legacy_time_on_page(q, query) do+ if :time_on_page in query.metrics and query.time_on_page_data.include_legacy_metric do+ q |> merge_legacy_time_on_page(query, query.dimensions)+ else+ q+ end+ end- {true, ["event:page"]} ->- pages =- Enum.map(ch_results, fn entry -> Map.get(entry, Util.shortname(query, "event:page")) end)+ defp merge_legacy_time_on_page(q, query, []) do+ from(+ e in subquery(q),+ inner_join: t in subquery(aggregate_time_on_page_q(query)),+ on: true+ )+ |> select_metrics_and_dimensions(query)+ end- breakdown_time_on_page(site, query, pages)+ defp merge_legacy_time_on_page(q, query, ["event:page"]) do+ from(+ e in subquery(q),+ left_join: t in subquery(breakdown_q(query)),+ on: e.dim0 == t.pathname+ )+ |> select_metrics_and_dimensions(query)+ end- _ ->- %{}- end+ defp merge_legacy_time_on_page(q, _query, _dimensions) do+ qend- defp aggregate_time_on_page(site, query) do+ defp select_metrics_and_dimensions(q, query) do+ q+ |> select_join_fields(query, List.delete(query.metrics, :time_on_page), e)+ |> select_join_fields(query, query.dimensions, e)+ |> select_merge_as([e, t], %{+ time_on_page:+ time_on_page(+ e.__internal_total_time_on_page + t.total_time_on_page,+ e.__internal_total_time_on_page_visits + t.transition_count+ )+ })+ end++ defp aggregate_time_on_page_q(query) dowindowed_pages_q =- from e in Base.base_event_query(site, Query.remove_top_level_filters(query, ["event:page"])),+ from(e in Base.base_event_query(Query.remove_top_level_filters(query, ["event:page"])),where: e.name != "engagement",+ where: ^filter_by_cutoff(query),select: %{next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon),next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon),@@ -45,6 +83,7 @@ defmodule Plausible.Stats.Legacy.TimeOnPage doframe: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")]]+ )event_page_filter = Filters.get_toplevel_filter(query, "event:page")@@ -61,54 +100,56 @@ defmodule Plausible.Stats.Legacy.TimeOnPage doavg_time_per_page_transition_q =from e in Ecto.Query.subquery(timed_page_transitions_q),- select: %{avg: fragment("sum(?)/countIf(?)", e.duration, e.transition)},+ select: %{+ avg: fragment("sum(?)/countIf(?)", e.duration, e.transition),+ duration: fragment("sum(?)", e.duration),+ transition_count: fragment("countIf(?)", e.transition)+ },group_by: e.pathname- time_on_page_q =- from e in Ecto.Query.subquery(avg_time_per_page_transition_q),- select: fragment("avg(ifNotFinite(?,NULL))", e.avg)-- %{[] => ClickhouseRepo.one(time_on_page_q, query: query)}- end-- defp breakdown_time_on_page(_site, _query, []) do- %{}+ from(+ e in subquery(avg_time_per_page_transition_q),+ select: %{+ total_time_on_page: fragment("sum(?)", e.duration),+ transition_count: fragment("sum(?)", e.transition_count)+ }+ )end- defp breakdown_time_on_page(site, query, pages) do- import Ecto.Query-+ defp breakdown_q(query) dowindowed_pages_q =- from e in Base.base_event_query(- site,- Query.remove_top_level_filters(query, ["event:page", "event:props"])- ),- where: e.name != "engagement",- select: %{- next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon),- next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon),- timestamp: e.timestamp,- pathname: e.pathname,- session_id: e.session_id- },- windows: [- event_horizon: [- partition_by: e.session_id,- order_by: e.timestamp,- frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")- ]- ]+ from(+ e in Base.base_event_query(+ Query.remove_top_level_filters(query, ["event:page", "event:props"])+ ),+ where: e.name != "engagement",+ where: ^filter_by_cutoff(query),+ select: %{+ next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon),+ next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon),+ timestamp: e.timestamp,+ pathname: e.pathname,+ session_id: e.session_id+ },+ windows: [+ event_horizon: [+ partition_by: e.session_id,+ order_by: e.timestamp,+ frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")+ ]+ ]+ )timed_page_transitions_q =- from e in subquery(windowed_pages_q),+ from(e in subquery(windowed_pages_q),group_by: [e.pathname, e.next_pathname, e.session_id],- where: e.pathname in ^pages,where: e.next_timestamp != 0,select: %{pathname: e.pathname,transition: e.next_pathname != e.pathname,duration: sum(e.next_timestamp - e.timestamp)}+ )no_select_timed_pages_q =from e in subquery(timed_page_transitions_q),@@ -116,47 +157,58 @@ defmodule Plausible.Stats.Legacy.TimeOnPage dodate_range = Query.date_range(query)- timed_pages_q =- if query.include_imported do- # Imported page views have pre-calculated values- imported_timed_pages_q =- from i in "imported_pages",- group_by: i.page,- where: i.site_id == ^site.id,- where: i.date >= ^date_range.first and i.date <= ^date_range.last,- where: i.page in ^pages,- select: %{- page: i.page,- time_on_page: sum(i.time_on_page),- visits: sum(i.pageviews) - sum(i.exits)- }-- timed_pages_q =- from e in no_select_timed_pages_q,- select: %{- page: e.pathname,- time_on_page: sum(e.duration),- visits: fragment("countIf(?)", e.transition)- }-- "timed_pages"- |> with_cte("timed_pages", as: ^timed_pages_q)- |> with_cte("imported_timed_pages", as: ^imported_timed_pages_q)- |> join(:full, [t], i in "imported_timed_pages", on: t.page == i.page)- |> select(- [t, i],- {- fragment("if(empty(?),?,?)", t.page, i.page, t.page),- (t.time_on_page + i.time_on_page) / (t.visits + i.visits)+ if query.include_imported do+ # Imported page views have pre-calculated values+ imported_timed_pages_q =+ from(i in "imported_pages",+ group_by: i.page,+ where: i.site_id == ^query.site_id,+ where: i.date >= ^date_range.first and i.date <= ^date_range.last,+ select: %{+ page: i.page,+ time_on_page: sum(i.total_time_on_page),+ visits: sum(i.pageviews) - sum(i.exits)})- else++ timed_pages_q =from e in no_select_timed_pages_q,- select: {e.pathname, fragment("sum(?)/countIf(?)", e.duration, e.transition)}- end+ select: %{+ page: e.pathname,+ time_on_page: fragment("sum(?)", e.duration),+ visits: fragment("countIf(?)", e.transition)+ }- timed_pages_q- |> Plausible.ClickhouseRepo.all(query: query)- |> Map.new(fn {path, value} -> {[path], value} end)+ "timed_pages"+ |> with_cte("timed_pages", as: ^timed_pages_q)+ |> with_cte("imported_timed_pages", as: ^imported_timed_pages_q)+ |> join(:full, [t], i in "imported_timed_pages", on: t.page == i.page)+ |> select(+ [t, i],+ %{+ pathname: fragment("if(empty(?),?,?)", t.page, i.page, t.page),+ total_time_on_page: t.time_on_page + i.time_on_page,+ transition_count: t.visits + i.visits+ }+ )+ else+ from(e in no_select_timed_pages_q,+ select: %{+ pathname: e.pathname,+ total_time_on_page: fragment("sum(?)", e.duration),+ transition_count: fragment("countIf(?)", e.transition)+ }+ )+ end+ end++ defp filter_by_cutoff(query) do+ case query.time_on_page_data do+ %{include_legacy_metric: true, include_new_metric: true, cutoff: cutoff} ->+ dynamic([e], e.timestamp < ^cutoff)++ _ ->+ true+ endendenddiff --git a/lib/plausible/stats/metrics.ex b/lib/plausible/stats/metrics.exindex 11f54827f0..e9d250283c 100644--- a/lib/plausible/stats/metrics.ex+++ b/lib/plausible/stats/metrics.ex@@ -34,6 +34,7 @@ defmodule Plausible.Stats.Metrics dodef default_value(:visit_duration, _query, _dimensions), do: nildef default_value(:scroll_depth, _query, _dimensions), do: nil+ def default_value(:time_on_page, _query, _dimensions), do: nil@float_metrics [:views_per_visit,diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.exindex f396cf6388..2bbe5e406c 100644--- a/lib/plausible/stats/query.ex+++ b/lib/plausible/stats/query.ex@@ -26,7 +26,14 @@ defmodule Plausible.Stats.Query dorevenue_warning: nil,remove_unavailable_revenue_metrics: false,site_id: nil,- site_native_stats_start_at: nil+ site_native_stats_start_at: nil,+ # Contains information to determine how to combine legacy and new time on page metrics+ # Old stats API and dashboard uses legacy metric by default, new API uses new metric by default+ time_on_page_data: %{+ include_new_metric: false,+ include_legacy_metric: true,+ cutoff: ~U[2100-01-01 00:00:00Z]+ }require OpenTelemetry.Tracer, as: Traceralias Plausible.Stats.{DateTimeRange, Filters, Imported, Legacy, Comparisons}diff --git a/lib/plausible/stats/query_optimizer.ex b/lib/plausible/stats/query_optimizer.exindex e3d8798d30..6e590d866d 100644--- a/lib/plausible/stats/query_optimizer.ex+++ b/lib/plausible/stats/query_optimizer.ex@@ -4,7 +4,7 @@ defmodule Plausible.Stats.QueryOptimizer do"""use Plausible- alias Plausible.Stats.{DateTimeRange, Filters, Query, TableDecider, Util, Time}+ alias Plausible.Stats.{DateTimeRange, Filters, Query, TableDecider, Util, Time, Legacy}@doc """This module manipulates an existing query, updating it according to business logic.@@ -49,7 +49,8 @@ defmodule Plausible.Stats.QueryOptimizer do&add_missing_order_by/1,&update_time_in_order_by/1,&extend_hostname_filters_to_visit/1,- &remove_revenue_metrics_if_unavailable/1+ &remove_revenue_metrics_if_unavailable/1,+ &set_time_on_page_data/1]end@@ -185,4 +186,30 @@ defmodule Plausible.Stats.QueryOptimizer doelsedefp remove_revenue_metrics_if_unavailable(query), do: queryend++ defp set_time_on_page_data(query) do+ if :time_on_page in query.metrics and query.include.legacy_time_on_page_cutoff do+ {:ok, cutoff, _} =+ query.include.legacy_time_on_page_cutoff+ |> DateTime.from_iso8601()++ cutoff =+ cutoff+ |> DateTime.shift_zone!("Etc/UTC")+ |> DateTime.truncate(:second)++ Query.set(+ query,+ time_on_page_data: %{+ include_new_metric: DateTime.before?(cutoff, query.utc_time_range.last),+ include_legacy_metric:+ DateTime.after?(cutoff, query.utc_time_range.first) and+ Legacy.TimeOnPage.can_merge_legacy_time_on_page?(query),+ cutoff: if(DateTime.after?(cutoff, query.utc_time_range.first), do: cutoff, else: nil)+ }+ )+ else+ query+ end+ endenddiff --git a/lib/plausible/stats/query_runner.ex b/lib/plausible/stats/query_runner.exindex 41d04bf2d4..fd148efef8 100644--- a/lib/plausible/stats/query_runner.ex+++ b/lib/plausible/stats/query_runner.ex@@ -17,7 +17,6 @@ defmodule Plausible.Stats.QueryRunner doCompare,QueryOptimizer,QueryResult,- Legacy,Metrics,SQL,Util,@@ -46,9 +45,9 @@ defmodule Plausible.Stats.QueryRunner doenddefp execute_main_query(%__MODULE__{main_query: query, site: site} = runner) do- {ch_results, time_on_page} = execute_query(query, site)+ ch_results = execute_query(query, site)- main_results = build_from_ch(ch_results, query, time_on_page)+ main_results = build_from_ch(ch_results, query)runner = struct!(runner, main_results: main_results)@@ -75,14 +74,8 @@ defmodule Plausible.Stats.QueryRunner do%__MODULE__{comparison_query: comparison_query, site: site} = runner) doif comparison_query do- {ch_results, time_on_page} = execute_query(comparison_query, site)-- comparison_results =- build_from_ch(- ch_results,- comparison_query,- time_on_page- )+ ch_results = execute_query(comparison_query, site)+ comparison_results = build_from_ch(ch_results, comparison_query)struct!(runner, comparison_results: comparison_results)else@@ -114,25 +107,19 @@ defmodule Plausible.Stats.QueryRunner doenddefp execute_query(query, site) do- ch_results =- query- |> SQL.QueryBuilder.build(site)- |> ClickhouseRepo.all(query: query)-- time_on_page = Legacy.TimeOnPage.calculate(site, query, ch_results)-- {ch_results, time_on_page}+ query+ |> SQL.QueryBuilder.build(site)+ |> ClickhouseRepo.all(query: query)end- defp build_from_ch(ch_results, query, time_on_page) do+ defp build_from_ch(ch_results, query) doch_results|> Enum.map(fn entry ->dimension_labels = Enum.map(query.dimensions, &dimension_label(&1, entry, query))%{dimensions: dimension_labels,- metrics:- Enum.map(query.metrics, &get_metric(entry, &1, dimension_labels, query, time_on_page))+ metrics: Enum.map(query.metrics, &get_metric(entry, &1, dimension_labels, query))}end)end@@ -153,7 +140,7 @@ defmodule Plausible.Stats.QueryRunner doendon_ee do- defp get_metric(entry, metric, dimensions, query, _time_on_page)+ defp get_metric(entry, metric, dimensions, query)when metric in [:average_revenue, :total_revenue] dovalue = Map.get(entry, metric)@@ -161,10 +148,7 @@ defmodule Plausible.Stats.QueryRunner doendend- defp get_metric(_entry, :time_on_page, dimensions, _query, time_on_page),- do: Map.get(time_on_page, dimensions)-- defp get_metric(entry, :events, _dimensions, query, _time_on_page) do+ defp get_metric(entry, :events, _dimensions, query) docond do"event:goal" in query.dimensions ->goal = get_dimension_goal(entry, query)@@ -184,7 +168,7 @@ defmodule Plausible.Stats.QueryRunner doendend- defp get_metric(entry, metric, _dimensions, _query, _time_on_page), do: Map.get(entry, metric)+ defp get_metric(entry, metric, _dimensions, _query), do: Map.get(entry, metric)defp get_dimension_goal(entry, query) dogoal_index = Map.get(entry, Util.shortname(query, "event:goal"))diff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.exindex 4708d8027f..22d8af1dc3 100644--- a/lib/plausible/stats/sql/expression.ex+++ b/lib/plausible/stats/sql/expression.ex@@ -189,32 +189,32 @@ defmodule Plausible.Stats.SQL.Expression dodef select_dimension(q, key, "visit:city_name", _table, _query),do: select_merge_as(q, [t], %{key => t.city_name})- def event_metric(:pageviews) do+ def event_metric(:pageviews, _query) dowrap_alias([e], %{pageviews: scale_sample(fragment("countIf(? = 'pageview')", e.name))})end- def event_metric(:events) do+ def event_metric(:events, _query) dowrap_alias([e], %{events: scale_sample(fragment("countIf(? != 'engagement')", e.name))})end- def event_metric(:visitors) do+ def event_metric(:visitors, _query) dowrap_alias([e], %{visitors: scale_sample(fragment("uniq(?)", e.user_id))})end- def event_metric(:visits) do+ def event_metric(:visits, _query) dowrap_alias([e], %{visits: scale_sample(fragment("uniq(?)", e.session_id))})endon_ee do- def event_metric(:total_revenue) do+ def event_metric(:total_revenue, _query) dowrap_alias([e],%{@@ -224,7 +224,7 @@ defmodule Plausible.Stats.SQL.Expression do)end- def event_metric(:average_revenue) do+ def event_metric(:average_revenue, _query) dowrap_alias([e],%{@@ -235,20 +235,81 @@ defmodule Plausible.Stats.SQL.Expression doendend- def event_metric(:sample_percent) do+ def event_metric(:sample_percent, _query) dowrap_alias([], %{sample_percent:fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)")})end- def event_metric(:percentage), do: %{}- def event_metric(:conversion_rate), do: %{}- def event_metric(:scroll_depth), do: %{}- def event_metric(:group_conversion_rate), do: %{}- def event_metric(:total_visitors), do: %{}+ def event_metric(:percentage, _query), do: %{}+ def event_metric(:conversion_rate, _query), do: %{}+ def event_metric(:scroll_depth, _query), do: %{}+ def event_metric(:group_conversion_rate, _query), do: %{}+ def event_metric(:total_visitors, _query), do: %{}++ def event_metric(:time_on_page, query) do+ selected =+ case query.time_on_page_data do+ %{include_new_metric: false} ->+ wrap_alias(+ [e],+ %{+ __internal_total_time_on_page: 0,+ __internal_total_time_on_page_visits: 0+ }+ )++ %{include_new_metric: true, cutoff: nil} ->+ wrap_alias(+ [e],+ %{+ __internal_total_time_on_page: fragment("sum(?) / 1000", e.engagement_time),+ __internal_total_time_on_page_visits:+ fragment("uniqIf(?, ? = 'engagement')", e.session_id, e.name)+ }+ )++ %{include_new_metric: true, cutoff: cutoff} ->+ wrap_alias(+ [e],+ %{+ __internal_total_time_on_page:+ fragment(+ "sumIf(?, ? >= ?) / 1000",+ e.engagement_time,+ e.timestamp,+ ^cutoff+ ),+ __internal_total_time_on_page_visits:+ fragment(+ "uniqIf(?, ? = 'engagement' and ? >= ?)",+ e.session_id,+ e.name,+ e.timestamp,+ ^cutoff+ )+ }+ )+ end++ if query.time_on_page_data.include_legacy_metric do+ selected+ else+ Map.merge(+ selected,+ wrap_alias([e], %{+ time_on_page:+ time_on_page(+ selected_as(:__internal_total_time_on_page),+ selected_as(:__internal_total_time_on_page_visits)+ )+ })+ )+ end+ end- def event_metric(unknown), do: raise("Unknown metric: #{unknown}")+ def event_metric(unknown, _query), do: raise("Unknown metric: #{unknown}")def session_metric(:bounce_rate, query) do# :TRICKY: If page is passed to query, we only count bounce rate where users _entered_ at page.diff --git a/lib/plausible/stats/sql/fragments.ex b/lib/plausible/stats/sql/fragments.exindex 8e39fb823f..36ce9d7990 100644--- a/lib/plausible/stats/sql/fragments.ex+++ b/lib/plausible/stats/sql/fragments.ex@@ -154,6 +154,17 @@ defmodule Plausible.Stats.SQL.Fragments dodef meta_value_column(:meta), do: :"meta.value"def meta_value_column(:entry_meta), do: :"entry_meta.value"+ defmacro time_on_page(total_time_on_page, total_time_on_page_visits) do+ quote do+ fragment(+ "if(? > 0, toInt32(round((?) / (?))), NULL)",+ unquote(total_time_on_page_visits),+ unquote(total_time_on_page),+ unquote(total_time_on_page_visits)+ )+ end+ end+@doc """Convenience Ecto macro for wrapping a map passed to select_merge_as such that eachexpression gets wrapped in dynamic and set as selected_as.diff --git a/lib/plausible/stats/sql/query_builder.ex b/lib/plausible/stats/sql/query_builder.exindex 6696f19197..1449447e08 100644--- a/lib/plausible/stats/sql/query_builder.ex+++ b/lib/plausible/stats/sql/query_builder.ex@@ -10,6 +10,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder doalias Plausible.Stats.{Query, QueryOptimizer, TableDecider, SQL}alias Plausible.Stats.SQL.Expression+ alias Plausible.Stats.Legacy.TimeOnPagerequire Plausible.Stats.SQL.Expression@@ -50,6 +51,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do|> build_group_by(:events, events_query)|> merge_imported(site, events_query)|> SQL.SpecialMetrics.add(site, events_query)+ |> TimeOnPage.merge_legacy_time_on_page(events_query)enddefp join_sessions_if_needed(q, query) do@@ -124,7 +126,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder dodefp select_event_metrics(query) doquery.metrics- |> Enum.map(&SQL.Expression.event_metric/1)+ |> Enum.map(&SQL.Expression.event_metric(&1, query))|> Enum.reduce(%{}, &Map.merge/2)enddiff --git a/lib/plausible/stats/sql/special_metrics.ex b/lib/plausible/stats/sql/special_metrics.exindex 8ade1c18a5..4951cc0944 100644--- a/lib/plausible/stats/sql/special_metrics.ex+++ b/lib/plausible/stats/sql/special_metrics.ex@@ -16,7 +16,7 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do|> maybe_add_percentage_metric(site, query)|> maybe_add_global_conversion_rate(site, query)|> maybe_add_group_conversion_rate(site, query)- |> maybe_add_scroll_depth(site, query)+ |> maybe_add_scroll_depth(query)enddefp maybe_add_percentage_metric(q, site, query) do@@ -127,10 +127,10 @@ defmodule Plausible.Stats.SQL.SpecialMetrics doendend- def maybe_add_scroll_depth(q, site, query) do+ def maybe_add_scroll_depth(q, query) doif :scroll_depth in query.metrics domax_per_session_q =- Base.base_event_query(site, query)+ Base.base_event_query(query)|> where([e], e.name == "engagement" and e.scroll_depth <= 100)|> select([e], %{session_id: e.session_id,@@ -222,14 +222,14 @@ defmodule Plausible.Stats.SQL.SpecialMetrics dodefp total_visitors_subquery(site, query, true = _include_imported) dowrap_alias([], %{total_visitors:- subquery(total_visitors(site, query)) ++ subquery(total_visitors(query)) +subquery(Plausible.Stats.Imported.total_imported_visitors(site, query))})end- defp total_visitors_subquery(site, query, false = _include_imported) do+ defp total_visitors_subquery(_site, query, false = _include_imported) dowrap_alias([], %{- total_visitors: subquery(total_visitors(site, query))+ total_visitors: subquery(total_visitors(query))})end@@ -243,8 +243,8 @@ defmodule Plausible.Stats.SQL.SpecialMetrics doQuery.set(query, filters: totals_query_filters)end- defp total_visitors(site, query) do- Base.base_event_query(site, query)+ defp total_visitors(query) do+ Base.base_event_query(query)|> select([e],total_visitors: scale_sample(fragment("uniq(?)", e.user_id)))diff --git a/lib/plausible/stats/table_decider.ex b/lib/plausible/stats/table_decider.exindex 5f40a32645..7a936630f1 100644--- a/lib/plausible/stats/table_decider.ex+++ b/lib/plausible/stats/table_decider.ex@@ -111,11 +111,11 @@ defmodule Plausible.Stats.TableDecider dodefp metric_partitioner(_, :pageviews), do: :eventdefp metric_partitioner(_, :events), do: :eventdefp metric_partitioner(_, :bounce_rate), do: :session+ defp metric_partitioner(_, :time_on_page), do: :eventdefp metric_partitioner(_, :visit_duration), do: :sessiondefp metric_partitioner(_, :views_per_visit), do: :session# Calculated metrics - handled on callsite separately from other metrics.- defp metric_partitioner(_, :time_on_page), do: :otherdefp metric_partitioner(_, :total_visitors), do: :other# Sample percentage is included in both tables if queried.defp metric_partitioner(_, :sample_percent), do: :sample_percentdiff --git a/lib/plausible_web/controllers/api/external_query_api_controller.ex b/lib/plausible_web/controllers/api/external_query_api_controller.exindex 7fc17e0e64..417f42a643 100644--- a/lib/plausible_web/controllers/api/external_query_api_controller.ex+++ b/lib/plausible_web/controllers/api/external_query_api_controller.ex@@ -11,6 +11,8 @@ defmodule PlausibleWeb.Api.ExternalQueryApiController docase Query.build(site, conn.assigns.schema_type, params, debug_metadata(conn)) do{:ok, query} ->+ query = update_time_on_page_query_data(query)+results = Plausible.Stats.query(site, query)json(conn, results)@@ -24,4 +26,18 @@ defmodule PlausibleWeb.Api.ExternalQueryApiController dodef schema(conn, _params) dojson(conn, Plausible.Stats.JSONSchema.raw_public_schema())end++ defp update_time_on_page_query_data(query) do+ if is_nil(query.include.legacy_time_on_page_cutoff) do+ Query.set(query,+ time_on_page_data: %{+ include_new_metric: true,+ include_legacy_metric: false,+ cutoff: nil+ }+ )+ else+ query+ end+ endenddiff --git a/priv/json-schemas/query-api-schema.json b/priv/json-schemas/query-api-schema.jsonindex 46ae2613d6..5a24d82afb 100644--- a/priv/json-schemas/query-api-schema.json+++ b/priv/json-schemas/query-api-schema.json@@ -109,6 +109,11 @@"additionalProperties": false}]+ },+ "legacy_time_on_page_cutoff": {+ "$comment": "only :internal",+ "type": "string",+ "format": "datetime"}}},diff --git a/priv/repo/seeds.exs b/priv/repo/seeds.exsindex dbec23c141..3baa8be38e 100644--- a/priv/repo/seeds.exs+++ b/priv/repo/seeds.exs@@ -228,12 +228,12 @@ native_stats_rangeevent = Keyword.merge(event, timestamp: timestamp)- event =+ to_insert =cond doevent_index > 0 && :rand.uniform() < 0.1 ->event|> Keyword.merge(name: outbound.event_name)- |> then(&Plausible.Factory.build(:event, &1))+ |> then(&[Plausible.Factory.build(:event, &1)])event_index > 0 && :rand.uniform() < 0.05 ->amount = Decimal.new(:rand.uniform(100))@@ -244,14 +244,22 @@ native_stats_range|> Keyword.merge(revenue_source_amount: amount)|> Keyword.merge(revenue_reporting_currency: "USD")|> Keyword.merge(revenue_reporting_amount: amount)- |> then(&Plausible.Factory.build(:event, &1))+ |> then(&[Plausible.Factory.build(:event, &1)])true ->- event- |> then(&Plausible.Factory.build(:pageview, &1))+ pageview = Plausible.Factory.build(:pageview, event)++ engagement =+ Map.merge(pageview, %{+ name: "engagement",+ engagement_time: Enum.random(300..10000),+ scroll_depth: Enum.random(1..100)+ })++ [engagement, pageview]end- [event | events]+ to_insert ++ eventsend)|> Enum.reverse()end)@@ -272,6 +280,8 @@ site_import =imported_stats_range|> Enum.flat_map(fn date ->Enum.flat_map(0..Enum.random(1..50), fn _ ->+ pages_visits = Enum.random(1..15)+[Plausible.Factory.build(:imported_visitors,date: date,@@ -294,10 +304,11 @@ imported_stats_rangedate: date,page: Enum.random(long_random_paths),visitors: Enum.random(1..10),- visits: Enum.random(1..15),+ visits: pages_visits,pageviews: Enum.random(1..50),exits: Enum.random(1..10),- time_on_page: Enum.random(1000..10000)+ total_time_on_page: Enum.random(1000..10000),+ total_time_on_page_visits: pages_visits)]end)diff --git a/test/plausible/imported/google_analytics4_test.exs b/test/plausible/imported/google_analytics4_test.exsindex 81b857d1f0..11317f7fcb 100644--- a/test/plausible/imported/google_analytics4_test.exs+++ b/test/plausible/imported/google_analytics4_test.exs@@ -809,7 +809,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do# it will allow us to assert on the session metrics as well.assert Enum.at(results, 2) == %{"page" => "/",- "time_on_page" => 17.677262055264585,+ "time_on_page" => 18,"visitors" => 371,"visits" => 212,"bounce_rate" => 54.0,@@ -821,7 +821,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert List.last(results) == %{"bounce_rate" => 0.0,"page" => "/znamenitosti-rima-koje-treba-vidjeti/",- "time_on_page" => 40.0,+ "time_on_page" => 40,"visit_duration" => 0.0,"visitors" => 1,"visits" => 1diff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exsindex a359d829a0..14b3297fa3 100644--- a/test/plausible/stats/query_parser_test.exs+++ b/test/plausible/stats/query_parser_test.exs@@ -53,7 +53,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest doimports_meta: false,time_labels: false,total_rows: false,- comparisons: nil+ comparisons: nil,+ legacy_time_on_page_cutoff: nil}def check_success(params, site, expected_result, schema_type \\ :public) do@@ -875,7 +876,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest doimports_meta: false,time_labels: true,total_rows: true,- comparisons: nil+ comparisons: nil,+ legacy_time_on_page_cutoff: nil},pagination: %{limit: 10_000, offset: 0}})@@ -939,7 +941,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest doimports: false,imports_meta: false,time_labels: false,- total_rows: false+ total_rows: false,+ legacy_time_on_page_cutoff: nil},pagination: %{limit: 10_000, offset: 0}},@@ -970,7 +973,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest doimports: false,imports_meta: false,time_labels: false,- total_rows: false+ total_rows: false,+ legacy_time_on_page_cutoff: nil},pagination: %{limit: 10_000, offset: 0}},@@ -1004,7 +1008,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest doimports_meta: false,imports: false,time_labels: false,- total_rows: false+ total_rows: false,+ legacy_time_on_page_cutoff: nil},pagination: %{limit: 10_000, offset: 0}},diff --git a/test/plausible/stats/table_decider_test.exs b/test/plausible/stats/table_decider_test.exsindex 81445df176..1ba7768f9b 100644--- a/test/plausible/stats/table_decider_test.exs+++ b/test/plausible/stats/table_decider_test.exs@@ -79,8 +79,8 @@ defmodule Plausible.Stats.TableDeciderTest dotest "other metrics put in its own result" doquery = make_query([])- assert partition_metrics([:time_on_page, :percentage, :total_visitors], query) ==- {[], [:percentage], [:time_on_page, :total_visitors]}+ assert partition_metrics([:percentage, :total_visitors], query) ==+ {[], [:percentage], [:total_visitors]}endtest "metrics that can be calculated on either when event-only metrics" dodiff --git a/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv b/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csvindex ae0e05a64a..2ff1477f78 100644--- a/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv+++ b/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv@@ -1,2 +1,2 @@name,visitors,pageviews,bounce_rate,time_on_page,scroll_depth-/some-other-page,1,1,0,60.0,+/some-other-page,1,1,0,60,diff --git a/test/plausible_web/controllers/CSVs/30d/pages.csv b/test/plausible_web/controllers/CSVs/30d/pages.csvindex c04e0e7d7c..30fdeb67a1 100644--- a/test/plausible_web/controllers/CSVs/30d/pages.csv+++ b/test/plausible_web/controllers/CSVs/30d/pages.csv@@ -1,4 +1,4 @@name,visitors,pageviews,bounce_rate,time_on_page,scroll_depth/,4,3,67,,-/signup,1,1,0,60.0,-/some-other-page,1,1,0,60.0,+/signup,1,1,0,60,+/some-other-page,1,1,0,60,diff --git a/test/plausible_web/controllers/CSVs/6m/pages.csv b/test/plausible_web/controllers/CSVs/6m/pages.csvindex 294a95100a..3e7aadc91d 100644--- a/test/plausible_web/controllers/CSVs/6m/pages.csv+++ b/test/plausible_web/controllers/CSVs/6m/pages.csv@@ -1,4 +1,4 @@name,visitors,pageviews,bounce_rate,time_on_page,scroll_depth/,5,4,75,,-/signup,1,1,0,60.0,-/some-other-page,1,1,0,60.0,+/signup,1,1,0,60,+/some-other-page,1,1,0,60,diff --git a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exsindex 92d4e177d0..9281309050 100644--- a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs@@ -2526,9 +2526,9 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do)populate_stats(site, site_import.id, [- build(:imported_pages, page: "/A", time_on_page: 40, date: ~D[2021-01-01]),- build(:imported_pages, page: "/A", time_on_page: 110, date: ~D[2021-01-01]),- build(:imported_pages, page: "/B", time_on_page: 499, date: ~D[2021-01-01]),+ build(:imported_pages, page: "/A", total_time_on_page: 40, date: ~D[2021-01-01]),+ build(:imported_pages, page: "/A", total_time_on_page: 110, date: ~D[2021-01-01]),+ build(:imported_pages, page: "/B", total_time_on_page: 499, date: ~D[2021-01-01]),build(:pageview, pathname: "/A", user_id: 4, timestamp: ~N[2021-01-01 00:00:00]),build(:pageview, pathname: "/B", user_id: 4, timestamp: ~N[2021-01-01 00:01:00])])diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_time_on_page_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_time_on_page_test.exsnew file mode 100644index 0000000000..455be7b9a3--- /dev/null+++ b/test/plausible_web/controllers/api/external_stats_controller/query_time_on_page_test.exs@@ -0,0 +1,412 @@+defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do+ use PlausibleWeb.ConnCase++ setup [:create_user, :create_site, :create_api_key, :use_api_key, :create_site_import]++ test "aggregated time_on_page metric based on engagement data", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview, user_id: 12, pathname: "/blog", timestamp: ~N[2021-01-01 00:00:00]),+ build(:engagement,+ user_id: 12,+ pathname: "/blog",+ timestamp: ~N[2021-01-01 00:10:00],+ engagement_time: 120_000+ ),+ build(:engagement,+ user_id: 12,+ pathname: "/blog",+ timestamp: ~N[2021-01-01 00:11:00],+ engagement_time: 20_000+ ),+ build(:pageview, user_id: 13, pathname: "/blog", timestamp: ~N[2021-01-01 00:10:00]),+ build(:engagement,+ user_id: 13,+ pathname: "/blog",+ timestamp: ~N[2021-01-01 00:10:00],+ engagement_time: 60_000+ )+ ])++ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["time_on_page"],+ "date_range" => "all",+ "dimensions" => ["event:page"],+ "include" => %{"imports" => true}+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["/blog"], "metrics" => [100]}+ ]+ end++ test "aggregated time_on_page metric with imported data", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:pageview, user_id: 12, pathname: "/blog", timestamp: ~N[2021-01-01 00:00:00]),+ build(:engagement,+ user_id: 12,+ pathname: "/blog",+ timestamp: ~N[2021-01-01 00:10:00],+ engagement_time: 120_000+ ),+ build(:imported_pages,+ page: "/blog",+ date: ~D[2021-01-01],+ visitors: 9,+ total_time_on_page: 9 * 20,+ total_time_on_page_visits: 9+ )+ ])++ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["time_on_page"],+ "date_range" => "all",+ "dimensions" => ["event:page"],+ "include" => %{"imports" => true}+ })++ assert_matches %{+ "results" => [+ %{"dimensions" => ["/blog"], "metrics" => [30]}+ ],+ "meta" =>+ ^strict_map(%{+ "imports_included" => true+ })+ } = json_response(conn, 200)+ end++ test "time_on_page time series", %{conn: conn, site: site, site_import: site_import} do+ populate_stats(site, site_import.id, [+ build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),+ build(:engagement,+ user_id: 12,+ pathname: "/",+ timestamp: ~N[2021-01-01 00:03:00],+ engagement_time: 100_000+ ),+ build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-02 00:00:00]),+ build(:engagement,+ user_id: 12,+ pathname: "/",+ timestamp: ~N[2021-01-02 00:03:00],+ engagement_time: 100_000+ ),+ build(:pageview, user_id: 13, pathname: "/", timestamp: ~N[2021-01-02 00:00:00]),+ build(:engagement,+ user_id: 13,+ pathname: "/",+ timestamp: ~N[2021-01-02 00:10:00],+ engagement_time: 300_000+ ),+ build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-03 00:00:00]),+ build(:engagement,+ user_id: 12,+ pathname: "/",+ timestamp: ~N[2021-01-03 00:03:00],+ engagement_time: 100_000+ ),+ build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-04 00:00:00]),+ build(:engagement,+ user_id: 12,+ pathname: "/",+ timestamp: ~N[2021-01-04 00:03:00],+ engagement_time: 100_000+ ),+ build(:imported_pages,+ page: "/blog",+ date: ~D[2021-01-01],+ visitors: 9,+ total_time_on_page: 9 * 20,+ total_time_on_page_visits: 9+ )+ ])++ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["time_on_page"],+ "date_range" => ["2021-01-01", "2021-01-04"],+ "dimensions" => ["time:day", "event:page"],+ "include" => %{"imports" => true}+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["2021-01-01", "/"], "metrics" => [100]},+ %{"dimensions" => ["2021-01-01", "/blog"], "metrics" => [20]},+ %{"dimensions" => ["2021-01-02", "/"], "metrics" => [200]},+ %{"dimensions" => ["2021-01-03", "/"], "metrics" => [100]},+ %{"dimensions" => ["2021-01-04", "/"], "metrics" => [100]}+ ]+ end++ describe "include.legacy_time_on_page_cutoff" do+ setup %{site: site, site_import: site_import} = context do+ populate_stats(site, site_import.id, [+ build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),+ build(:engagement,+ user_id: 12,+ pathname: "/",+ timestamp: ~N[2021-01-01 00:02:00],+ engagement_time: 100_000+ ),+ build(:pageview, user_id: 12, pathname: "/blog", timestamp: ~N[2021-01-01 00:02:00]),+ build(:engagement,+ user_id: 12,+ pathname: "/blog",+ timestamp: ~N[2021-01-01 00:05:00],+ engagement_time: 100_000+ ),+ build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-01 00:05:00]),+ build(:engagement,+ user_id: 12,+ pathname: "/",+ timestamp: ~N[2021-01-01 00:10:00],+ engagement_time: 100_000+ ),+ build(:pageview, user_id: 12, pathname: "/blog", timestamp: ~N[2021-01-01 00:10:00]),+ build(:engagement,+ user_id: 12,+ pathname: "/blog",+ timestamp: ~N[2021-01-01 00:15:00],+ engagement_time: 100_000+ ),+ build(:pageview, user_id: 13, pathname: "/pricing", timestamp: ~N[2021-01-02 01:00:00]),+ build(:engagement,+ user_id: 13,+ pathname: "/pricing",+ timestamp: ~N[2021-01-02 01:02:00],+ engagement_time: 30_000+ ),+ build(:pageview, user_id: 14, pathname: "/", timestamp: ~N[2021-01-02 01:00:00]),+ build(:engagement,+ user_id: 14,+ pathname: "/",+ timestamp: ~N[2021-01-02 01:02:00],+ engagement_time: 30_000+ ),+ build(:imported_pages,+ page: "/blog",+ date: ~D[2021-01-01],+ visitors: 9,+ total_time_on_page: 9 * 20,+ total_time_on_page_visits: 9+ )+ ])++ context+ end++ test "breakdown with cutoff being before data (new time-on-page query used)", %{+ conn: conn,+ site: site+ } do+ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["time_on_page"],+ "date_range" => "all",+ "dimensions" => ["event:page"],+ "include" => %{+ "legacy_time_on_page_cutoff" => "1970-01-01T00:00:00Z",+ "imports" => true+ }+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["/"], "metrics" => [115]},+ # (2 * 100s + 9 * 20s) / 10 = 38+ %{"dimensions" => ["/blog"], "metrics" => [38]},+ %{"dimensions" => ["/pricing"], "metrics" => [30]}+ ]+ end++ test "breakdown with cutoff being after data (legacy query used)", %{conn: conn, site: site} do+ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["time_on_page"],+ "date_range" => "all",+ "dimensions" => ["event:page"],+ "include" => %{+ "legacy_time_on_page_cutoff" => "2021-01-05T00:00:00Z",+ "imports" => true+ }+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["/"], "metrics" => [420]},+ %{"dimensions" => ["/blog"], "metrics" => [180]},+ %{"dimensions" => ["/pricing"], "metrics" => [nil]}+ ]+ end++ test "breakdown with cutoff being mid-data (two queries joined)", %{conn: conn, site: site} do+ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["time_on_page"],+ "date_range" => "all",+ "dimensions" => ["event:page"],+ "include" => %{+ "legacy_time_on_page_cutoff" => "2021-01-02T00:00:00Z",+ "imports" => true+ }+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["/"], "metrics" => [225]},+ %{"dimensions" => ["/blog"], "metrics" => [180]},+ %{"dimensions" => ["/pricing"], "metrics" => [30]}+ ]+ end++ test "breakdown with cutoff being mid-data (two queries joined), no imports", %{+ conn: conn,+ site: site+ } do+ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["time_on_page"],+ "date_range" => "all",+ "dimensions" => ["event:page"],+ "include" => %{+ "legacy_time_on_page_cutoff" => "2021-01-02T00:00:00Z"+ }+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["/"], "metrics" => [225]},+ %{"dimensions" => ["/blog"], "metrics" => [180]},+ %{"dimensions" => ["/pricing"], "metrics" => [30]}+ ]+ end++ test "aggregation with cutoff being after data (legacy query used)", %{conn: conn, site: site} do+ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["time_on_page"],+ "date_range" => "all",+ "filters" => [["is", "event:page", ["/blog"]]],+ "include" => %{+ "legacy_time_on_page_cutoff" => "2021-01-05T00:00:00Z",+ "imports" => true+ }+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => [], "metrics" => [180]}+ ]+ end++ test "aggregation with cutoff being mid-data (two queries joined)", %{conn: conn, site: site} do+ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["time_on_page"],+ "date_range" => "all",+ "filters" => [["is", "event:page", ["/blog"]]],+ "include" => %{+ "legacy_time_on_page_cutoff" => "2021-01-02T00:00:00Z",+ "imports" => true+ }+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => [], "metrics" => [180]}+ ]+ end++ test "aggregation with cutoff being mid-data (two queries joined), no imports", %{+ conn: conn,+ site: site+ } do+ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["time_on_page"],+ "date_range" => "all",+ "filters" => [["is", "event:page", ["/blog"]]],+ "include" => %{+ "legacy_time_on_page_cutoff" => "2021-01-02T00:00:00Z"+ }+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => [], "metrics" => [180]}+ ]+ end+ end++ describe "legacy time_on_page metric" do+ test "aggregated", %{conn: conn, site: site, site_import: site_import} do+ populate_stats(site, site_import.id, [+ build(:pageview, pathname: "/A", user_id: 111, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, pathname: "/B", user_id: 111, timestamp: ~N[2021-01-01 00:01:00]),+ build(:pageview, pathname: "/A", user_id: 999, timestamp: ~N[2021-01-02 00:00:00]),+ build(:pageview, pathname: "/B", user_id: 999, timestamp: ~N[2021-01-02 00:01:30]),+ # These are ignored for time_on_page metric+ build(:imported_pages, page: "/A", total_time_on_page: 40, date: ~D[2021-01-01]),+ build(:imported_pages, page: "/B", total_time_on_page: 499, date: ~D[2021-01-01])+ ])++ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "time_on_page"],+ "date_range" => "all",+ "filters" => [["is", "event:page", ["/A"]]],+ "include" => %{+ "imports" => true,+ "legacy_time_on_page_cutoff" => "2100-01-01T00:00:00Z"+ }+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => [], "metrics" => [3, 75]}+ ]+ end++ test "breakdown", %{conn: conn, site: site, site_import: site_import} do+ populate_stats(site, site_import.id, [+ build(:pageview, pathname: "/A", user_id: 111, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, pathname: "/B", user_id: 111, timestamp: ~N[2021-01-01 00:01:00]),+ build(:pageview, pathname: "/A", user_id: 999, timestamp: ~N[2021-01-02 00:00:00]),+ build(:pageview, pathname: "/B", user_id: 999, timestamp: ~N[2021-01-02 00:01:30]),+ build(:pageview, pathname: "/C", user_id: 999, timestamp: ~N[2021-01-02 00:02:00]),+ build(:imported_pages, page: "/A", total_time_on_page: 40, date: ~D[2021-01-01]),+ build(:imported_pages, page: "/B", total_time_on_page: 499, date: ~D[2021-01-01])+ ])++ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "time_on_page"],+ "date_range" => "all",+ "dimensions" => ["event:page"],+ "include" => %{+ "imports" => true,+ "legacy_time_on_page_cutoff" => "2100-01-01T00:00:00Z"+ }+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["/A"], "metrics" => [3, 63]},+ %{"dimensions" => ["/B"], "metrics" => [3, 264]},+ %{"dimensions" => ["/C"], "metrics" => [1, nil]}+ ]+ end+ end+enddiff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exsindex 673b091655..58a3562927 100644--- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs@@ -692,7 +692,8 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest dodate: ~D[2020-01-01],visitors: 3,pageviews: 3,- time_on_page: 90,+ total_time_on_page: 90,+ total_time_on_page_visits: 3,page: "/blog",total_scroll_depth: 120,total_scroll_depth_visits: 3@@ -748,7 +749,8 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest dodate: ~D[2020-01-01],visitors: 4,pageviews: 4,- time_on_page: 180,+ total_time_on_page: 180,+ total_time_on_page_visits: 4,page: "/native-and-imported",total_scroll_depth: 120,total_scroll_depth_visits: 3@@ -757,7 +759,8 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest dodate: ~D[2020-01-01],visitors: 20,pageviews: 30,- time_on_page: 300,+ total_time_on_page: 300,+ total_time_on_page_visits: 10,page: "/imported-only",total_scroll_depth: 100,total_scroll_depth_visits: 10@@ -833,7 +836,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"visitors" => 110,"pageviews" => 160,"bounce_rate" => 0,- "time_on_page" => 0.125,+ "time_on_page" => 0,"scroll_depth" => 10}]@@ -1374,7 +1377,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do])assert [- %{"name" => "/", "time_on_page" => _three_minutes = 180.0},+ %{"name" => "/", "time_on_page" => _three_minutes = 180},%{"name" => "/exit", "time_on_page" => nil}] =conn@@ -1407,9 +1410,9 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do])assert [- %{"name" => "/a", "time_on_page" => 100.0},+ %{"name" => "/a", "time_on_page" => 100},%{"name" => "/b", "time_on_page" => nil},- %{"name" => "/d", "time_on_page" => +0.0}+ %{"name" => "/d", "time_on_page" => 0}] =conn|> get("/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true")@@ -1439,7 +1442,8 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest dobuild(:imported_pages,page: "/",date: ~D[2021-01-01],- time_on_page: 700+ total_time_on_page: 700,+ total_time_on_page_visits: 3),build(:imported_entry_pages,entry_page: "/",@@ -1450,7 +1454,8 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest dobuild(:imported_pages,page: "/some-other-page",date: ~D[2021-01-01],- time_on_page: 60+ total_time_on_page: 60,+ total_time_on_page_visits: 1)])@@ -1529,7 +1534,8 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest dopage: "/",visitors: 3,pageviews: 3,- time_on_page: 300,+ total_time_on_page: 300,+ total_time_on_page_visits: 3,date: ~D[2021-01-01]),build(:imported_pages, page: "/ignored", visitors: 10, date: ~D[2021-01-01])@@ -1574,7 +1580,8 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest dopage: "/",visitors: 3,pageviews: 3,- time_on_page: 300,+ total_time_on_page: 300,+ total_time_on_page_visits: 3,date: ~D[2021-01-01]),build(:imported_pages,@@ -1632,7 +1639,8 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest dopage: "/aaa",visitors: 3,pageviews: 3,- time_on_page: 300,+ total_time_on_page: 300,+ total_time_on_page_visits: 3,date: ~D[2021-01-01]),build(:imported_pages,@@ -1700,7 +1708,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"comparison" => %{"bounce_rate" => 0,"pageviews" => 0,- "time_on_page" => 0,+ "time_on_page" => nil,"visitors" => 0,"scroll_depth" => nil,"change" => %{diff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exsindex 99481f4592..b60c276705 100644--- a/test/plausible_web/controllers/stats_controller_test.exs+++ b/test/plausible_web/controllers/stats_controller_test.exs@@ -276,8 +276,8 @@ defmodule PlausibleWeb.StatsControllerTest doassert pages == [["name", "visitors", "pageviews", "bounce_rate", "time_on_page", "scroll_depth"],- ["/blog", "3", "4", "33", "60.0", "60"],- ["/another", "2", "2", "0", "60.0", "25"],+ ["/blog", "3", "4", "33", "60", "60"],+ ["/another", "2", "2", "0", "60", "25"],[""]]end@@ -587,7 +587,7 @@ defmodule PlausibleWeb.StatsControllerTest do"time_on_page","scroll_depth"],- ["/test", "1", "1", "0.0", "10.0", ""],+ ["/test", "1", "1", "0.0", "10", ""],[""]]diff --git a/test/support/factory.ex b/test/support/factory.exindex 2e1383a6b7..c4478e1b54 100644--- a/test/support/factory.ex+++ b/test/support/factory.ex@@ -277,7 +277,8 @@ defmodule Plausible.Factory dovisitors: 1,pageviews: 1,exits: 0,- time_on_page: 10+ total_time_on_page: 10,+ total_time_on_page_visits: 1}end