Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
time-on-page: dashboard + timeseries support (#5174)
Karl-Aksel Puulmann0514a6db1866
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex bc270234c9..ea09d22f68 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -14,6 +14,7 @@ All notable changes to this project will be documented in this file.- Add filter `is not` for goals in dashboard plausible/analytics#4983- Add Segments feature- Support `["is", "segment", [<segment ID>]]` filter in Stats API+- Time on page metric is now sortable in reports### Removed@@ -41,7 +42,7 @@ All notable changes to this project will be documented in this file.- Breakdown modals now display correct comparison values instead of 0 after pagination- Fix database mismatch between event and session user_ids after rotating salts- `/api/v2/query` no longer returns a 500 when querying percentage metric without `visitors`-- Fix current visitors loading when viewing a dashboard with a shared link+- Fix current visitors loading when viewing a dashboard with a shared link- Fix Conversion Rate graph being unselectable when "Goal is ..." filter is within a segment- Fix Channels filter input appearing when clicking Sources in filter menu or clicking an applied "Channel is..." filterdiff --git a/assets/js/dashboard/api.ts b/assets/js/dashboard/api.tsindex 3a22b36348..85aebbfe48 100644--- a/assets/js/dashboard/api.ts+++ b/assets/js/dashboard/api.ts@@ -74,11 +74,25 @@ export function queryToSearchParams(queryObj.auth = sharedLinkParams.auth}+ if (+ query.legacy_time_on_page_cutoff &&+ validDate(query.legacy_time_on_page_cutoff)+ ) {+ queryObj.include = JSON.stringify({+ legacy_time_on_page_cutoff: query.legacy_time_on_page_cutoff+ })+ }+Object.assign(queryObj, ...extraQuery)return serializeUrlParams(queryObj)}+function validDate(dateString: string) {+ const date = new Date(dateString)+ return isFinite(date.getTime())+}+function getHeaders(): Record<string, string> {return SHARED_LINK_AUTH ? { 'X-Shared-Link-Auth': SHARED_LINK_AUTH } : {}}diff --git a/assets/js/dashboard/query-context.tsx b/assets/js/dashboard/query-context.tsxindex d4987ff10d..e9cb8a6841 100644--- a/assets/js/dashboard/query-context.tsx+++ b/assets/js/dashboard/query-context.tsx@@ -60,6 +60,7 @@ export default function QueryContextProvider({period,to,with_imported,+ legacy_time_on_page_cutoff,...otherSearch} = useMemo(() => parseSearch(location.search), [location.search])@@ -105,7 +106,9 @@ export default function QueryContextProvider({filters: Array.isArray(filters)? postProcessFilters(filters as Filter[]): defaultValues.filters,- labels: (labels as FilterClauseLabels) || defaultValues.labels+ labels: (labels as FilterClauseLabels) || defaultValues.labels,+ legacy_time_on_page_cutoff:+ (legacy_time_on_page_cutoff as string) || site.legacyTimeOnPageCutoff}}, [compare_from,@@ -119,6 +122,7 @@ export default function QueryContextProvider({period,to,with_imported,+ legacy_time_on_page_cutoff,site,expandedSegment])diff --git a/assets/js/dashboard/query.ts b/assets/js/dashboard/query.tsindex 68859c482a..5c4005df32 100644--- a/assets/js/dashboard/query.ts+++ b/assets/js/dashboard/query.ts@@ -44,7 +44,8 @@ export const queryDefaultValue = {compare_to: null as Dayjs | null,filters: [] as Filter[],labels: {} as FilterClauseLabels,- with_imported: true+ with_imported: true,+ legacy_time_on_page_cutoff: undefined as string | undefined}export type DashboardQuery = typeof queryDefaultValuediff --git a/assets/js/dashboard/site-context.test.tsx b/assets/js/dashboard/site-context.test.tsxindex b089b8831f..3014793430 100644--- a/assets/js/dashboard/site-context.test.tsx+++ b/assets/js/dashboard/site-context.test.tsx@@ -25,6 +25,7 @@ describe('parseSiteFromDataset', () => {data-logged-in="true"data-stats-begin="2021-09-07"data-native-stats-begin="2022-09-02"+ data-legacy-time-on-page-cutoff="2022-01-01T00:00:00Z"data-embedded=""data-is-dbip="false"data-current-user-role="owner"@@ -66,7 +67,8 @@ describe('parseSiteFromDataset', () => {realtime: ['minute'],year: ['day', 'week', 'month']},- shared: false+ shared: false,+ legacyTimeOnPageCutoff: '2022-01-01T00:00:00Z'}it('parses from dom string map correctly', () => {diff --git a/assets/js/dashboard/site-context.tsx b/assets/js/dashboard/site-context.tsxindex 2fcc8da52d..e251ce2322 100644--- a/assets/js/dashboard/site-context.tsx+++ b/assets/js/dashboard/site-context.tsx@@ -14,6 +14,7 @@ export function parseSiteFromDataset(dataset: DOMStringMap): PlausibleSite {conversionsOptedOut: dataset.conversionsOptedOut === 'true',funnelsOptedOut: dataset.funnelsOptedOut === 'true',propsOptedOut: dataset.propsOptedOut === 'true',+ legacyTimeOnPageCutoff: dataset.legacyTimeOnPageCutoff,revenueGoals: JSON.parse(dataset.revenueGoals!),funnels: JSON.parse(dataset.funnels!),statsBegin: dataset.statsBegin!,@@ -30,6 +31,7 @@ export function parseSiteFromDataset(dataset: DOMStringMap): PlausibleSite {type FeatureFlags = {saved_segments?: booleansaved_segments_fe?: boolean+ new_time_on_page?: boolean}const siteContextDefaultValue = {@@ -56,7 +58,8 @@ const siteContextDefaultValue = {isDbip: false,flags: {} as FeatureFlags,validIntervalsByPeriod: {} as Record<string, Array<string>>,- shared: false+ shared: false,+ legacyTimeOnPageCutoff: undefined as string | undefined}export type PlausibleSite = typeof siteContextDefaultValuediff --git a/assets/js/dashboard/stats/graph/graph-util.js b/assets/js/dashboard/stats/graph/graph-util.jsindex 743ab00c39..cdac657387 100644--- a/assets/js/dashboard/stats/graph/graph-util.js+++ b/assets/js/dashboard/stats/graph/graph-util.js@@ -11,6 +11,7 @@ export const METRIC_LABELS = {'average_revenue': 'Average Revenue','total_revenue': 'Total Revenue','scroll_depth': 'Scroll Depth',+ 'time_on_page': 'Time on Page',}function plottable(dataArray) {diff --git a/assets/js/dashboard/stats/graph/top-stats.js b/assets/js/dashboard/stats/graph/top-stats.jsindex 576102ab35..e5ba985112 100644--- a/assets/js/dashboard/stats/graph/top-stats.js+++ b/assets/js/dashboard/stats/graph/top-stats.js@@ -39,6 +39,11 @@ export default function TopStats({function tooltip(stat) {let statName = stat.name.toLowerCase()+ const warning = warningText(+ stat.graph_metric,+ data.meta.metric_warnings?.[stat.graph_metric],+ site+ )statName = stat.value === 1 ? statName.slice(0, -1) : statNamereturn (@@ -69,17 +74,32 @@ export default function TopStats({</p>)}- {stat.name === 'Scroll depth' &&- data.meta.metric_warnings?.scroll_depth?.code ===- 'no_imported_scroll_depth' && (- <p className="font-normal text-xs whitespace-nowrap">- * Does not include imported data- </p>- )}+ {warning ? (+ <p className="font-normal text-xs whitespace-nowrap">* {warning}</p>+ ) : null}</div>)}+ function warningText(metric, warning, site) {+ if (!warning) {+ return null+ }++ if (+ metric === 'scroll_depth' &&+ warning.code === 'no_imported_scroll_depth'+ ) {+ return 'Does not include imported data'+ }++ if (metric === 'time_on_page' && site.flags.new_time_on_page) {+ return warning.message+ }++ return null+ }+function canMetricBeGraphed(stat) {return graphableMetrics.includes(stat.graph_metric)}diff --git a/assets/js/dashboard/stats/modals/breakdown-modal.tsx b/assets/js/dashboard/stats/modals/breakdown-modal.tsxindex 946d14db52..f950c2b5df 100644--- a/assets/js/dashboard/stats/modals/breakdown-modal.tsx+++ b/assets/js/dashboard/stats/modals/breakdown-modal.tsx@@ -17,7 +17,7 @@ import { Metric } from '../reports/metrics'import { BreakdownResultMeta, DashboardQuery } from '../../query'import { ColumnConfiguraton } from '../../components/table'import { BreakdownTable } from './breakdown-table'-import { useSiteContext } from '../../site-context'+import { PlausibleSite, useSiteContext } from '../../site-context'export type ReportInfo = {/** Title of the report to render on the top left. */@@ -152,7 +152,7 @@ export default function BreakdownModal<TListItem extends { name: string }>({key: m.key,width: m.width,align: 'right',- metricWarning: getMetricWarning(m, meta),+ metricWarning: getMetricWarning(m, meta, site),renderValue: (item) => m.renderValue(item, meta),onSort: m.sortable ? () => toggleSortByMetric(m) : undefined,sortDirection: orderByDictionary[m.key]@@ -235,14 +235,22 @@ const ExternalLinkIcon = ({ url }: { url?: string }) =></a>) : null-const getMetricWarning = (metric: Metric, meta: BreakdownResultMeta | null) => {+const getMetricWarning = (+ metric: Metric,+ meta: BreakdownResultMeta | null,+ site: PlausibleSite+) => {const warnings = meta?.metric_warnings- if (warnings) {- const code = warnings[metric.key]?.code+ if (warnings && warnings[metric.key]) {+ const { code, message } = warnings[metric.key]- if (code == 'no_imported_scroll_depth') {+ if (metric.key == 'scroll_depth' && code == 'no_imported_scroll_depth') {return 'Does not include imported data'}++ if (metric.key == 'time_on_page' && code && site.flags.new_time_on_page) {+ return message+ }}}diff --git a/assets/js/dashboard/stats/reports/metrics.js b/assets/js/dashboard/stats/reports/metrics.jsindex b906e5018a..8ee043a746 100644--- a/assets/js/dashboard/stats/reports/metrics.js+++ b/assets/js/dashboard/stats/reports/metrics.js@@ -198,7 +198,7 @@ export const createTimeOnPage = (props) => {...props,key: 'time_on_page',renderLabel,- sortable: false+ sortable: true})}diff --git a/assets/test-utils/app-context-providers.tsx b/assets/test-utils/app-context-providers.tsxindex be2b47b107..7bfd0339b5 100644--- a/assets/test-utils/app-context-providers.tsx+++ b/assets/test-utils/app-context-providers.tsx@@ -43,7 +43,8 @@ export const TestContextProviders = ({isDbip: false,flags: {},validIntervalsByPeriod: {},- shared: false+ shared: false,+ legacyTimeOnPageCutoff: undefined}const site = { ...defaultSite, ...siteOptions }diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.exindex b8af4e9bf9..ff5411873c 100644--- a/lib/plausible/stats/filters/query_parser.ex+++ b/lib/plausible/stats/filters/query_parser.ex@@ -46,7 +46,7 @@ defmodule Plausible.Stats.Filters.QueryParser doPlausible.Segments.Filters.resolve_segments(filters, preloaded_segments),{:ok, dimensions} <- parse_dimensions(Map.get(params, "dimensions", [])),{:ok, order_by} <- parse_order_by(Map.get(params, "order_by")),- {:ok, include} <- parse_include(site, Map.get(params, "include", %{})),+ {:ok, include} <- parse_include(Map.get(params, "include", %{}), site),{:ok, pagination} <- parse_pagination(Map.get(params, "pagination", %{})),{preloaded_goals, revenue_warning, revenue_currencies} <-preload_goals_and_revenue(site, metrics, filters, dimensions),@@ -310,12 +310,12 @@ defmodule Plausible.Stats.Filters.QueryParser do)end- defp parse_order_by(order_by) when is_list(order_by) do+ def parse_order_by(order_by) when is_list(order_by) doparse_list(order_by, &parse_order_by_entry/1)end- defp parse_order_by(nil), do: {:ok, nil}- defp parse_order_by(order_by), do: {:error, "Invalid order_by '#{i(order_by)}'."}+ def parse_order_by(nil), do: {:ok, nil}+ def parse_order_by(order_by), do: {:error, "Invalid order_by '#{i(order_by)}'."}defp parse_order_by_entry(entry) dowith {:ok, value} <- parse_metric_or_dimension(entry),@@ -359,17 +359,25 @@ defmodule Plausible.Stats.Filters.QueryParser dodefp parse_order_direction([_, "desc"]), do: {:ok, :desc}defp parse_order_direction(entry), do: {:error, "Invalid order_by entry '#{i(entry)}'."}- defp parse_include(site, include) when is_map(include) do- parsed =- include- |> atomize_keys()- |> update_comparisons_date_range(site)-- with {:ok, include} <- parsed do+ def parse_include(include, site) when is_map(include) do+ with {:ok, include} <- atomize_include_keys(include),+ {:ok, include} <- update_comparisons_date_range(include, site) do{:ok, Map.merge(@default_include, include)}endend+ def parse_include(include, _site), do: {:error, "Invalid include '#{i(include)}'."}++ defp atomize_include_keys(map) do+ expected_keys = @default_include |> Map.keys() |> Enum.map(&Atom.to_string/1)++ if Map.keys(map) |> Enum.all?(&(&1 in expected_keys)) do+ {:ok, atomize_keys(map)}+ else+ {:error, "Invalid include '#{i(map)}'."}+ end+ end+defp update_comparisons_date_range(%{comparisons: %{date_range: date_range}} = include, site) dowith {:ok, parsed_date_range} <- parse_date_range_pair(site, date_range) do{:ok, put_in(include, [:comparisons, :date_range], parsed_date_range)}diff --git a/lib/plausible/stats/legacy/legacy_query_builder.ex b/lib/plausible/stats/legacy/legacy_query_builder.exindex 35f03521e4..7323f27a34 100644--- a/lib/plausible/stats/legacy/legacy_query_builder.ex+++ b/lib/plausible/stats/legacy/legacy_query_builder.ex@@ -6,7 +6,7 @@ defmodule Plausible.Stats.Legacy.QueryBuilder douse Plausible- alias Plausible.Stats.{Filters, Interval, Query, DateTimeRange, Metrics}+ alias Plausible.Stats.{Filters, Interval, Query, DateTimeRange}def from(site, params, debug_metadata, now \\ nil) donow = now || DateTime.utc_now(:second)@@ -219,7 +219,10 @@ defmodule Plausible.Stats.Legacy.QueryBuilder doenddefp put_include(query, site, params) do+ include = parse_include(site, params["include"])+query+ |> struct!(include: include)|> Query.set_include(:comparisons, parse_comparison_params(site, params))|> Query.set_include(:imports, params["with_imported"] == "true")end@@ -250,31 +253,37 @@ defmodule Plausible.Stats.Legacy.QueryBuilder doiex> QueryBuilder.parse_order_by(~s([["visitors","asc"],["visit:source","desc"]]))[{:visitors, :asc}, {"visit:source", :desc}]"""- def parse_order_by(order_by) when is_binary(order_by) do- case Jason.decode(order_by) do- {:ok, parsed} when is_list(parsed) ->- Enum.flat_map(parsed, &parse_order_by_pair/1)-- _ ->- []- end+ def parse_order_by(order_by) do+ json_decode(order_by)+ |> unwrap([])+ |> Filters.QueryParser.parse_order_by()+ |> unwrap([])end- def parse_order_by(_) do- []- end+ @doc """+ ### Examples:+ iex> QueryBuilder.parse_include(%{}, nil)+ QueryParser.default_include()- defp parse_order_by_pair([metric_or_dimension, direction]) when direction in ["asc", "desc"] do- case Metrics.from_string(metric_or_dimension) do- {:ok, metric} -> [{metric, String.to_existing_atom(direction)}]- :error -> [{metric_or_dimension, String.to_existing_atom(direction)}]- end+ iex> QueryBuilder.parse_include(%{}, ~s({"total_rows": true}))+ Map.merge(QueryParser.default_include(), %{total_rows: true})+ """+ def parse_include(site, include) do+ json_decode(include)+ |> unwrap(%{})+ |> Filters.QueryParser.parse_include(site)+ |> unwrap(Filters.QueryParser.default_include())end- defp parse_order_by_pair(_) do- []+ defp json_decode(string) when is_binary(string) do+ Jason.decode(string)end+ defp json_decode(_other), do: :error++ defp unwrap({:ok, result}, _default), do: result+ defp unwrap(_, default), do: default+defp put_order_by(query, %{} = params) dostruct!(query, order_by: parse_order_by(params["order_by"]))enddiff --git a/lib/plausible/stats/query_result.ex b/lib/plausible/stats/query_result.exindex 1f9603142c..f272dd647c 100644--- a/lib/plausible/stats/query_result.ex+++ b/lib/plausible/stats/query_result.ex@@ -156,6 +156,22 @@ defmodule Plausible.Stats.QueryResult doendend+ defp metric_warning(:time_on_page, %Query{} = query) do+ case query.time_on_page_data do+ %{include_legacy_metric: true, cutoff: cutoff} ->+ cutoff_time =+ cutoff |> DateTime.shift_zone!(query.timezone) |> Calendar.strftime("%Y-%m-%d %H:%M:%S")++ %{+ code: :legacy_time_on_page_used,+ message: "Contains less accurate legacy time-on-page data up to #{cutoff_time}"+ }++ _ ->+ nil+ end+ end+defp metric_warning(_metric, _query), do: nildefp to_iso8601(datetime, timezone) dodiff --git a/lib/plausible/stats/query_runner.ex b/lib/plausible/stats/query_runner.exindex fd148efef8..d6db51bb6f 100644--- a/lib/plausible/stats/query_runner.ex+++ b/lib/plausible/stats/query_runner.ex@@ -64,6 +64,7 @@ defmodule Plausible.Stats.QueryRunner doquery|> Comparisons.get_comparison_query()|> Comparisons.add_comparison_filters(main_results)+ |> QueryOptimizer.optimize()struct!(runner, comparison_query: comparison_query)enddiff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.exindex 22d8af1dc3..3de6cc86b1 100644--- a/lib/plausible/stats/sql/expression.ex+++ b/lib/plausible/stats/sql/expression.ex@@ -255,8 +255,10 @@ defmodule Plausible.Stats.SQL.Expression dowrap_alias([e],%{- __internal_total_time_on_page: 0,- __internal_total_time_on_page_visits: 0+ # :KLUDGE: We would like to but can't use constant 0 here as it leads to cyclic aliases if there's no other+ # metrics selected other than time_on_page.+ __internal_total_time_on_page: fragment("sumArray([0])"),+ __internal_total_time_on_page_visits: fragment("sumArray([0])")})diff --git a/lib/plausible/stats/time_on_page.ex b/lib/plausible/stats/time_on_page.exnew file mode 100644index 0000000000..38ca7a983a--- /dev/null+++ b/lib/plausible/stats/time_on_page.ex@@ -0,0 +1,16 @@+defmodule Plausible.Stats.TimeOnPage do+ @moduledoc """+ Module to check whether the new time on page metric is available.+ """++ def new_time_on_page_enabled?(site, user) do+ FunWithFlags.enabled?(:new_time_on_page, for: user) ||+ FunWithFlags.enabled?(:new_time_on_page, for: site)+ end++ def legacy_time_on_page_cutoff() do+ # Placeholder until we implement a more sophisticated way to determine the cutoff+ # Only used when `new_time_on_page` flag is enabled+ DateTime.utc_now() |> DateTime.shift(day: -4) |> DateTime.to_iso8601()+ end+enddiff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex c773948b1b..4ceca38a1b 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -5,7 +5,7 @@ defmodule PlausibleWeb.Api.StatsController douse PlausibleWeb.Plugs.ErrorHandleralias Plausible.Stats- alias Plausible.Stats.{Query, Comparisons, Filters, Time, TableDecider}+ alias Plausible.Stats.{Query, Comparisons, Filters, Time, TableDecider, TimeOnPage}alias PlausibleWeb.Api.Helpers, as: Hrequire Logger@@ -444,7 +444,11 @@ defmodule PlausibleWeb.Api.StatsController do%{top_stats: top_stats,meta: meta,- graphable_metrics: metrics -- [:time_on_page],+ graphable_metrics:+ if(TimeOnPage.new_time_on_page_enabled?(site, current_user),+ do: metrics,+ else: metrics -- [:time_on_page]+ ),sample_percent: sample_percent}enddiff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.exindex 8601a24c3c..f8bd1082de 100644--- a/lib/plausible_web/controllers/stats_controller.ex+++ b/lib/plausible_web/controllers/stats_controller.ex@@ -45,7 +45,7 @@ defmodule PlausibleWeb.StatsController douse Plausible.Repoalias Plausible.Sites- alias Plausible.Stats.{Filters, Query}+ alias Plausible.Stats.{Filters, Query, TimeOnPage}alias PlausibleWeb.Apiplug(PlausibleWeb.Plugs.AuthorizeSiteAccess when action in [:stats, :csv_export])@@ -77,6 +77,7 @@ defmodule PlausibleWeb.StatsController doscroll_depth_visible: scroll_depth_visible?,stats_start_date: stats_start_date,native_stats_start_date: NaiveDateTime.to_date(site.native_stats_start_at),+ legacy_time_on_page_cutoff: TimeOnPage.legacy_time_on_page_cutoff(),title: title(conn, site),demo: demo,flags: flags,@@ -361,6 +362,7 @@ defmodule PlausibleWeb.StatsController doscroll_depth_visible: scroll_depth_visible?,stats_start_date: stats_start_date,native_stats_start_date: NaiveDateTime.to_date(shared_link.site.native_stats_start_at),+ legacy_time_on_page_cutoff: TimeOnPage.legacy_time_on_page_cutoff(),title: title(conn, shared_link.site),demo: false,dogfood_page_path: "/share/:dashboard",@@ -388,7 +390,7 @@ defmodule PlausibleWeb.StatsController dodefp get_flags(user, site),do:- [:saved_segments, :saved_segments_fe, :scroll_depth]+ [:saved_segments, :saved_segments_fe, :scroll_depth, :new_time_on_page]|> Enum.map(fn flag ->{flag, FunWithFlags.enabled?(flag, for: user) || FunWithFlags.enabled?(flag, for: site)}end)diff --git a/lib/plausible_web/templates/stats/stats.html.heex b/lib/plausible_web/templates/stats/stats.html.heexindex abd70d64da..ff8b5811bb 100644--- a/lib/plausible_web/templates/stats/stats.html.heex+++ b/lib/plausible_web/templates/stats/stats.html.heex@@ -37,6 +37,7 @@data-logged-in={to_string([email protected][:current_user])}data-stats-begin={@stats_start_date}data-native-stats-begin={@native_stats_start_date}+ data-legacy-time-on-page-cutoff={@legacy_time_on_page_cutoff}data-shared-link-auth={assigns[:shared_link_auth]}data-embedded={to_string(@conn.assigns[:embedded])}data-background={@conn.assigns[:background]}diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exsindex 5834931f8b..1461aaab7e 100644--- a/test/plausible/stats/query_test.exs+++ b/test/plausible/stats/query_test.exs@@ -3,6 +3,7 @@ defmodule Plausible.Stats.QueryTest douse Plausible.Teams.Testalias Plausible.Stats.Queryalias Plausible.Stats.Legacy.QueryBuilder+ alias Plausible.Stats.Filters.QueryParseralias Plausible.Stats.DateTimeRangedoctest Plausible.Stats.Legacy.QueryBuilderdiff --git a/test/plausible_web/controllers/api/external_stats_controller/query_time_on_page_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_time_on_page_test.exsindex 455be7b9a3..8fcc054cca 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_time_on_page_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_time_on_page_test.exs@@ -39,9 +39,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do"include" => %{"imports" => true}})- assert json_response(conn, 200)["results"] == [- %{"dimensions" => ["/blog"], "metrics" => [100]}- ]+ assert_matches %{+ "results" => [+ %{"dimensions" => ["/blog"], "metrics" => [100]}+ ],+ "meta" =>+ ^strict_map(%{+ "imports_included" => true+ })+ } = json_response(conn, 200)endtest "aggregated time_on_page metric with imported data", %{@@ -141,13 +147,19 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do"include" => %{"imports" => true}})- assert json_response(conn, 200)["results"] == [- %{"dimensions" => ["2021-01-01", "/"], "metrics" => [100]},- %{"dimensions" => ["2021-01-01", "/blog"], "metrics" => [20]},- %{"dimensions" => ["2021-01-02", "/"], "metrics" => [200]},- %{"dimensions" => ["2021-01-03", "/"], "metrics" => [100]},- %{"dimensions" => ["2021-01-04", "/"], "metrics" => [100]}- ]+ assert_matches %{+ "results" => [+ %{"dimensions" => ["2021-01-01", "/"], "metrics" => [100]},+ %{"dimensions" => ["2021-01-01", "/blog"], "metrics" => [20]},+ %{"dimensions" => ["2021-01-02", "/"], "metrics" => [200]},+ %{"dimensions" => ["2021-01-03", "/"], "metrics" => [100]},+ %{"dimensions" => ["2021-01-04", "/"], "metrics" => [100]}+ ],+ "meta" =>+ ^strict_map(%{+ "imports_included" => true+ })+ } = json_response(conn, 200)enddescribe "include.legacy_time_on_page_cutoff" do@@ -223,12 +235,18 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do}})- assert json_response(conn, 200)["results"] == [- %{"dimensions" => ["/"], "metrics" => [115]},- # (2 * 100s + 9 * 20s) / 10 = 38- %{"dimensions" => ["/blog"], "metrics" => [38]},- %{"dimensions" => ["/pricing"], "metrics" => [30]}- ]+ assert_matches %{+ "results" => [+ %{"dimensions" => ["/"], "metrics" => [115]},+ # (2 * 100s + 9 * 20s) / 10 = 38+ %{"dimensions" => ["/blog"], "metrics" => [38]},+ %{"dimensions" => ["/pricing"], "metrics" => [30]}+ ],+ "meta" =>+ ^strict_map(%{+ "imports_included" => true+ })+ } = json_response(conn, 200)endtest "breakdown with cutoff being after data (legacy query used)", %{conn: conn, site: site} do@@ -244,11 +262,24 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do}})- assert json_response(conn, 200)["results"] == [- %{"dimensions" => ["/"], "metrics" => [420]},- %{"dimensions" => ["/blog"], "metrics" => [180]},- %{"dimensions" => ["/pricing"], "metrics" => [nil]}- ]+ assert_matches %{+ "results" => [+ %{"dimensions" => ["/"], "metrics" => [420]},+ %{"dimensions" => ["/blog"], "metrics" => [180]},+ %{"dimensions" => ["/pricing"], "metrics" => [nil]}+ ],+ "meta" =>+ ^strict_map(%{+ "imports_included" => true,+ "metric_warnings" => %{+ "time_on_page" => %{+ "code" => "legacy_time_on_page_used",+ "message" =>+ "Contains less accurate legacy time-on-page data up to 2021-01-05 00:00:00"+ }+ }+ })+ } = json_response(conn, 200)endtest "breakdown with cutoff being mid-data (two queries joined)", %{conn: conn, site: site} do@@ -264,11 +295,24 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do}})- assert json_response(conn, 200)["results"] == [- %{"dimensions" => ["/"], "metrics" => [225]},- %{"dimensions" => ["/blog"], "metrics" => [180]},- %{"dimensions" => ["/pricing"], "metrics" => [30]}- ]+ assert_matches %{+ "results" => [+ %{"dimensions" => ["/"], "metrics" => [225]},+ %{"dimensions" => ["/blog"], "metrics" => [180]},+ %{"dimensions" => ["/pricing"], "metrics" => [30]}+ ],+ "meta" =>+ ^strict_map(%{+ "imports_included" => true,+ "metric_warnings" => %{+ "time_on_page" => %{+ "code" => "legacy_time_on_page_used",+ "message" =>+ "Contains less accurate legacy time-on-page data up to 2021-01-02 00:00:00"+ }+ }+ })+ } = json_response(conn, 200)endtest "breakdown with cutoff being mid-data (two queries joined), no imports", %{@@ -286,11 +330,23 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do}})- assert json_response(conn, 200)["results"] == [- %{"dimensions" => ["/"], "metrics" => [225]},- %{"dimensions" => ["/blog"], "metrics" => [180]},- %{"dimensions" => ["/pricing"], "metrics" => [30]}- ]+ assert_matches %{+ "results" => [+ %{"dimensions" => ["/"], "metrics" => [225]},+ %{"dimensions" => ["/blog"], "metrics" => [180]},+ %{"dimensions" => ["/pricing"], "metrics" => [30]}+ ],+ "meta" =>+ ^strict_map(%{+ "metric_warnings" => %{+ "time_on_page" => %{+ "code" => "legacy_time_on_page_used",+ "message" =>+ "Contains less accurate legacy time-on-page data up to 2021-01-02 00:00:00"+ }+ }+ })+ } = json_response(conn, 200)endtest "aggregation with cutoff being after data (legacy query used)", %{conn: conn, site: site} do@@ -306,9 +362,22 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do}})- assert json_response(conn, 200)["results"] == [- %{"dimensions" => [], "metrics" => [180]}- ]+ assert_matches %{+ "results" => [+ %{"dimensions" => [], "metrics" => [180]}+ ],+ "meta" =>+ ^strict_map(%{+ "imports_included" => true,+ "metric_warnings" => %{+ "time_on_page" => %{+ "code" => "legacy_time_on_page_used",+ "message" =>+ "Contains less accurate legacy time-on-page data up to 2021-01-05 00:00:00"+ }+ }+ })+ } = json_response(conn, 200)endtest "aggregation with cutoff being mid-data (two queries joined)", %{conn: conn, site: site} do@@ -324,9 +393,22 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do}})- assert json_response(conn, 200)["results"] == [- %{"dimensions" => [], "metrics" => [180]}- ]+ assert_matches %{+ "results" => [+ %{"dimensions" => [], "metrics" => [180]}+ ],+ "meta" =>+ ^strict_map(%{+ "imports_included" => true,+ "metric_warnings" => %{+ "time_on_page" => %{+ "code" => "legacy_time_on_page_used",+ "message" =>+ "Contains less accurate legacy time-on-page data up to 2021-01-02 00:00:00"+ }+ }+ })+ } = json_response(conn, 200)endtest "aggregation with cutoff being mid-data (two queries joined), no imports", %{@@ -344,9 +426,21 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do}})- assert json_response(conn, 200)["results"] == [- %{"dimensions" => [], "metrics" => [180]}- ]+ assert_matches %{+ "results" => [+ %{"dimensions" => [], "metrics" => [180]}+ ],+ "meta" =>+ ^strict_map(%{+ "metric_warnings" => %{+ "time_on_page" => %{+ "code" => "legacy_time_on_page_used",+ "message" =>+ "Contains less accurate legacy time-on-page data up to 2021-01-02 00:00:00"+ }+ }+ })+ } = json_response(conn, 200)endend@@ -409,4 +503,143 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do]endend++ describe "timeseries" do+ setup %{site: site} = context do+ populate_stats(site, [+ build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),+ build(:engagement,+ user_id: 12,+ pathname: "/",+ timestamp: ~N[2021-01-01 00:05:00],+ engagement_time: 100_000+ ),+ build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-02 00:00:00]),+ build(:engagement,+ user_id: 12,+ pathname: "/",+ timestamp: ~N[2021-01-02 00:05:00],+ engagement_time: 200_000+ ),+ build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-03 00:00:00]),+ build(:engagement,+ user_id: 12,+ pathname: "/",+ timestamp: ~N[2021-01-03 00:05:00],+ engagement_time: 250_000+ ),+ build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-04 00:00:00]),+ build(:engagement,+ user_id: 12,+ pathname: "/",+ timestamp: ~N[2021-01-04 00:05:00],+ engagement_time: 200_000+ ),+ build(:pageview, user_id: 13, pathname: "/", timestamp: ~N[2021-01-04 00:00:00]),+ build(:engagement,+ user_id: 13,+ pathname: "/",+ timestamp: ~N[2021-01-04 00:05:00],+ engagement_time: 100_000+ )+ ])++ context+ end++ test "reports average new time-on-page per day", %{conn: conn, site: site} do+ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["time_on_page"],+ "date_range" => ["2021-01-01", "2021-01-04"],+ "filters" => [["is", "event:page", ["/"]]],+ "dimensions" => ["time:day"]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["2021-01-01"], "metrics" => [100]},+ %{"dimensions" => ["2021-01-02"], "metrics" => [200]},+ %{"dimensions" => ["2021-01-03"], "metrics" => [250]},+ %{"dimensions" => ["2021-01-04"], "metrics" => [150]}+ ]+ end++ test "reports legacy time-on-page as nulls per day", %{conn: conn, site: site} do+ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["time_on_page"],+ "date_range" => ["2021-01-01", "2021-01-04"],+ "filters" => [["is", "event:page", ["/"]]],+ "dimensions" => ["time:day"],+ "include" => %{"legacy_time_on_page_cutoff" => "2100-01-01T00:00:00Z"}+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["2021-01-01"], "metrics" => [nil]},+ %{"dimensions" => ["2021-01-02"], "metrics" => [nil]},+ %{"dimensions" => ["2021-01-03"], "metrics" => [nil]},+ %{"dimensions" => ["2021-01-04"], "metrics" => [nil]}+ ]+ end++ test "respects `legacy_time_on_page_cutoff`", %{conn: conn, site: site} do+ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["time_on_page"],+ "date_range" => ["2021-01-01", "2021-01-04"],+ "filters" => [["is", "event:page", ["/"]]],+ "dimensions" => ["time:day"],+ "include" => %{"legacy_time_on_page_cutoff" => "2021-01-02T23:59:59Z"}+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["2021-01-01"], "metrics" => [nil]},+ %{"dimensions" => ["2021-01-02"], "metrics" => [nil]},+ %{"dimensions" => ["2021-01-03"], "metrics" => [250]},+ %{"dimensions" => ["2021-01-04"], "metrics" => [150]}+ ]+ end++ test "can use comparisons together with `legacy_time_on_page_cutoff`", %{+ conn: conn,+ site: site+ } do+ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["time_on_page"],+ "date_range" => ["2021-01-03", "2021-01-04"],+ "filters" => [["is", "event:page", ["/"]]],+ "dimensions" => ["time:day"],+ "include" => %{+ "legacy_time_on_page_cutoff" => "2021-01-01T23:59:59Z",+ "comparisons" => %{"mode" => "previous_period"}+ }+ })++ assert json_response(conn, 200)["results"] == [+ %{+ "dimensions" => ["2021-01-03"],+ "metrics" => [250],+ "comparison" => %{+ "dimensions" => ["2021-01-01"],+ "metrics" => [nil],+ "change" => [nil]+ }+ },+ %{+ "dimensions" => ["2021-01-04"],+ "metrics" => [150],+ "comparison" => %{+ "dimensions" => ["2021-01-02"],+ "metrics" => [200],+ "change" => [-25]+ }+ }+ ]+ end+ endend