Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

time-on-page: dashboard + timeseries support (#5174)

Karl-Aksel Puulmann0514a6db1866
diff --git a/CHANGELOG.md b/CHANGELOG.md
index bc270234c9..ea09d22f68 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,7 @@ All notable changes to this project will be documented in this file.
- Add filter `is not` for goals in dashboard plausible/analytics#4983
- Add Segments feature
- Support `["is", "segment", [<segment ID>]]` filter in Stats API
+- Time on page metric is now sortable in reports
### Removed
@@ -41,7 +42,7 @@ All notable changes to this project will be documented in this file.
- Breakdown modals now display correct comparison values instead of 0 after pagination
- Fix database mismatch between event and session user_ids after rotating salts
- `/api/v2/query` no longer returns a 500 when querying percentage metric without `visitors`
-- Fix current visitors loading when viewing a dashboard with a shared link
+- Fix current visitors loading when viewing a dashboard with a shared link
- Fix Conversion Rate graph being unselectable when "Goal is ..." filter is within a segment
- Fix Channels filter input appearing when clicking Sources in filter menu or clicking an applied "Channel is..." filter
diff --git a/assets/js/dashboard/api.ts b/assets/js/dashboard/api.ts
index 3a22b36348..85aebbfe48 100644
--- a/assets/js/dashboard/api.ts
+++ b/assets/js/dashboard/api.ts
@@ -74,11 +74,25 @@ export function queryToSearchParams(
queryObj.auth = sharedLinkParams.auth
}
+ if (
+ query.legacy_time_on_page_cutoff &&
+ validDate(query.legacy_time_on_page_cutoff)
+ ) {
+ queryObj.include = JSON.stringify({
+ legacy_time_on_page_cutoff: query.legacy_time_on_page_cutoff
+ })
+ }
+
Object.assign(queryObj, ...extraQuery)
return serializeUrlParams(queryObj)
}
+function validDate(dateString: string) {
+ const date = new Date(dateString)
+ return isFinite(date.getTime())
+}
+
function getHeaders(): Record<string, string> {
return SHARED_LINK_AUTH ? { 'X-Shared-Link-Auth': SHARED_LINK_AUTH } : {}
}
diff --git a/assets/js/dashboard/query-context.tsx b/assets/js/dashboard/query-context.tsx
index d4987ff10d..e9cb8a6841 100644
--- a/assets/js/dashboard/query-context.tsx
+++ b/assets/js/dashboard/query-context.tsx
@@ -60,6 +60,7 @@ export default function QueryContextProvider({
period,
to,
with_imported,
+ legacy_time_on_page_cutoff,
...otherSearch
} = useMemo(() => parseSearch(location.search), [location.search])
@@ -105,7 +106,9 @@ export default function QueryContextProvider({
filters: Array.isArray(filters)
? postProcessFilters(filters as Filter[])
: defaultValues.filters,
- labels: (labels as FilterClauseLabels) || defaultValues.labels
+ labels: (labels as FilterClauseLabels) || defaultValues.labels,
+ legacy_time_on_page_cutoff:
+ (legacy_time_on_page_cutoff as string) || site.legacyTimeOnPageCutoff
}
}, [
compare_from,
@@ -119,6 +122,7 @@ export default function QueryContextProvider({
period,
to,
with_imported,
+ legacy_time_on_page_cutoff,
site,
expandedSegment
])
diff --git a/assets/js/dashboard/query.ts b/assets/js/dashboard/query.ts
index 68859c482a..5c4005df32 100644
--- a/assets/js/dashboard/query.ts
+++ b/assets/js/dashboard/query.ts
@@ -44,7 +44,8 @@ export const queryDefaultValue = {
compare_to: null as Dayjs | null,
filters: [] as Filter[],
labels: {} as FilterClauseLabels,
- with_imported: true
+ with_imported: true,
+ legacy_time_on_page_cutoff: undefined as string | undefined
}
export type DashboardQuery = typeof queryDefaultValue
diff --git a/assets/js/dashboard/site-context.test.tsx b/assets/js/dashboard/site-context.test.tsx
index b089b8831f..3014793430 100644
--- a/assets/js/dashboard/site-context.test.tsx
+++ b/assets/js/dashboard/site-context.test.tsx
@@ -25,6 +25,7 @@ describe('parseSiteFromDataset', () => {
data-logged-in="true"
data-stats-begin="2021-09-07"
data-native-stats-begin="2022-09-02"
+ data-legacy-time-on-page-cutoff="2022-01-01T00:00:00Z"
data-embedded=""
data-is-dbip="false"
data-current-user-role="owner"
@@ -66,7 +67,8 @@ describe('parseSiteFromDataset', () => {
realtime: ['minute'],
year: ['day', 'week', 'month']
},
- shared: false
+ shared: false,
+ legacyTimeOnPageCutoff: '2022-01-01T00:00:00Z'
}
it('parses from dom string map correctly', () => {
diff --git a/assets/js/dashboard/site-context.tsx b/assets/js/dashboard/site-context.tsx
index 2fcc8da52d..e251ce2322 100644
--- a/assets/js/dashboard/site-context.tsx
+++ b/assets/js/dashboard/site-context.tsx
@@ -14,6 +14,7 @@ export function parseSiteFromDataset(dataset: DOMStringMap): PlausibleSite {
conversionsOptedOut: dataset.conversionsOptedOut === 'true',
funnelsOptedOut: dataset.funnelsOptedOut === 'true',
propsOptedOut: dataset.propsOptedOut === 'true',
+ legacyTimeOnPageCutoff: dataset.legacyTimeOnPageCutoff,
revenueGoals: JSON.parse(dataset.revenueGoals!),
funnels: JSON.parse(dataset.funnels!),
statsBegin: dataset.statsBegin!,
@@ -30,6 +31,7 @@ export function parseSiteFromDataset(dataset: DOMStringMap): PlausibleSite {
type FeatureFlags = {
saved_segments?: boolean
saved_segments_fe?: boolean
+ new_time_on_page?: boolean
}
const siteContextDefaultValue = {
@@ -56,7 +58,8 @@ const siteContextDefaultValue = {
isDbip: false,
flags: {} as FeatureFlags,
validIntervalsByPeriod: {} as Record<string, Array<string>>,
- shared: false
+ shared: false,
+ legacyTimeOnPageCutoff: undefined as string | undefined
}
export type PlausibleSite = typeof siteContextDefaultValue
diff --git a/assets/js/dashboard/stats/graph/graph-util.js b/assets/js/dashboard/stats/graph/graph-util.js
index 743ab00c39..cdac657387 100644
--- a/assets/js/dashboard/stats/graph/graph-util.js
+++ b/assets/js/dashboard/stats/graph/graph-util.js
@@ -11,6 +11,7 @@ export const METRIC_LABELS = {
'average_revenue': 'Average Revenue',
'total_revenue': 'Total Revenue',
'scroll_depth': 'Scroll Depth',
+ 'time_on_page': 'Time on Page',
}
function plottable(dataArray) {
diff --git a/assets/js/dashboard/stats/graph/top-stats.js b/assets/js/dashboard/stats/graph/top-stats.js
index 576102ab35..e5ba985112 100644
--- a/assets/js/dashboard/stats/graph/top-stats.js
+++ b/assets/js/dashboard/stats/graph/top-stats.js
@@ -39,6 +39,11 @@ export default function TopStats({
function tooltip(stat) {
let statName = stat.name.toLowerCase()
+ const warning = warningText(
+ stat.graph_metric,
+ data.meta.metric_warnings?.[stat.graph_metric],
+ site
+ )
statName = stat.value === 1 ? statName.slice(0, -1) : statName
return (
@@ -69,17 +74,32 @@ export default function TopStats({
</p>
)}
- {stat.name === 'Scroll depth' &&
- data.meta.metric_warnings?.scroll_depth?.code ===
- 'no_imported_scroll_depth' && (
- <p className="font-normal text-xs whitespace-nowrap">
- * Does not include imported data
- </p>
- )}
+ {warning ? (
+ <p className="font-normal text-xs whitespace-nowrap">* {warning}</p>
+ ) : null}
</div>
)
}
+ function warningText(metric, warning, site) {
+ if (!warning) {
+ return null
+ }
+
+ if (
+ metric === 'scroll_depth' &&
+ warning.code === 'no_imported_scroll_depth'
+ ) {
+ return 'Does not include imported data'
+ }
+
+ if (metric === 'time_on_page' && site.flags.new_time_on_page) {
+ return warning.message
+ }
+
+ return null
+ }
+
function canMetricBeGraphed(stat) {
return graphableMetrics.includes(stat.graph_metric)
}
diff --git a/assets/js/dashboard/stats/modals/breakdown-modal.tsx b/assets/js/dashboard/stats/modals/breakdown-modal.tsx
index 946d14db52..f950c2b5df 100644
--- a/assets/js/dashboard/stats/modals/breakdown-modal.tsx
+++ b/assets/js/dashboard/stats/modals/breakdown-modal.tsx
@@ -17,7 +17,7 @@ import { Metric } from '../reports/metrics'
import { BreakdownResultMeta, DashboardQuery } from '../../query'
import { ColumnConfiguraton } from '../../components/table'
import { BreakdownTable } from './breakdown-table'
-import { useSiteContext } from '../../site-context'
+import { PlausibleSite, useSiteContext } from '../../site-context'
export type ReportInfo = {
/** Title of the report to render on the top left. */
@@ -152,7 +152,7 @@ export default function BreakdownModal<TListItem extends { name: string }>({
key: m.key,
width: m.width,
align: 'right',
- metricWarning: getMetricWarning(m, meta),
+ metricWarning: getMetricWarning(m, meta, site),
renderValue: (item) => m.renderValue(item, meta),
onSort: m.sortable ? () => toggleSortByMetric(m) : undefined,
sortDirection: orderByDictionary[m.key]
@@ -235,14 +235,22 @@ const ExternalLinkIcon = ({ url }: { url?: string }) =>
</a>
) : null
-const getMetricWarning = (metric: Metric, meta: BreakdownResultMeta | null) => {
+const getMetricWarning = (
+ metric: Metric,
+ meta: BreakdownResultMeta | null,
+ site: PlausibleSite
+) => {
const warnings = meta?.metric_warnings
- if (warnings) {
- const code = warnings[metric.key]?.code
+ if (warnings && warnings[metric.key]) {
+ const { code, message } = warnings[metric.key]
- if (code == 'no_imported_scroll_depth') {
+ if (metric.key == 'scroll_depth' && code == 'no_imported_scroll_depth') {
return 'Does not include imported data'
}
+
+ if (metric.key == 'time_on_page' && code && site.flags.new_time_on_page) {
+ return message
+ }
}
}
diff --git a/assets/js/dashboard/stats/reports/metrics.js b/assets/js/dashboard/stats/reports/metrics.js
index b906e5018a..8ee043a746 100644
--- a/assets/js/dashboard/stats/reports/metrics.js
+++ b/assets/js/dashboard/stats/reports/metrics.js
@@ -198,7 +198,7 @@ export const createTimeOnPage = (props) => {
...props,
key: 'time_on_page',
renderLabel,
- sortable: false
+ sortable: true
})
}
diff --git a/assets/test-utils/app-context-providers.tsx b/assets/test-utils/app-context-providers.tsx
index be2b47b107..7bfd0339b5 100644
--- a/assets/test-utils/app-context-providers.tsx
+++ b/assets/test-utils/app-context-providers.tsx
@@ -43,7 +43,8 @@ export const TestContextProviders = ({
isDbip: false,
flags: {},
validIntervalsByPeriod: {},
- shared: false
+ shared: false,
+ legacyTimeOnPageCutoff: undefined
}
const site = { ...defaultSite, ...siteOptions }
diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.ex
index b8af4e9bf9..ff5411873c 100644
--- a/lib/plausible/stats/filters/query_parser.ex
+++ b/lib/plausible/stats/filters/query_parser.ex
@@ -46,7 +46,7 @@ defmodule Plausible.Stats.Filters.QueryParser do
Plausible.Segments.Filters.resolve_segments(filters, preloaded_segments),
{:ok, dimensions} <- parse_dimensions(Map.get(params, "dimensions", [])),
{:ok, order_by} <- parse_order_by(Map.get(params, "order_by")),
- {:ok, include} <- parse_include(site, Map.get(params, "include", %{})),
+ {:ok, include} <- parse_include(Map.get(params, "include", %{}), site),
{:ok, pagination} <- parse_pagination(Map.get(params, "pagination", %{})),
{preloaded_goals, revenue_warning, revenue_currencies} <-
preload_goals_and_revenue(site, metrics, filters, dimensions),
@@ -310,12 +310,12 @@ defmodule Plausible.Stats.Filters.QueryParser do
)
end
- defp parse_order_by(order_by) when is_list(order_by) do
+ def parse_order_by(order_by) when is_list(order_by) do
parse_list(order_by, &parse_order_by_entry/1)
end
- defp parse_order_by(nil), do: {:ok, nil}
- defp parse_order_by(order_by), do: {:error, "Invalid order_by '#{i(order_by)}'."}
+ def parse_order_by(nil), do: {:ok, nil}
+ def parse_order_by(order_by), do: {:error, "Invalid order_by '#{i(order_by)}'."}
defp parse_order_by_entry(entry) do
with {:ok, value} <- parse_metric_or_dimension(entry),
@@ -359,17 +359,25 @@ defmodule Plausible.Stats.Filters.QueryParser do
defp parse_order_direction([_, "desc"]), do: {:ok, :desc}
defp parse_order_direction(entry), do: {:error, "Invalid order_by entry '#{i(entry)}'."}
- defp parse_include(site, include) when is_map(include) do
- parsed =
- include
- |> atomize_keys()
- |> update_comparisons_date_range(site)
-
- with {:ok, include} <- parsed do
+ def parse_include(include, site) when is_map(include) do
+ with {:ok, include} <- atomize_include_keys(include),
+ {:ok, include} <- update_comparisons_date_range(include, site) do
{:ok, Map.merge(@default_include, include)}
end
end
+ def parse_include(include, _site), do: {:error, "Invalid include '#{i(include)}'."}
+
+ defp atomize_include_keys(map) do
+ expected_keys = @default_include |> Map.keys() |> Enum.map(&Atom.to_string/1)
+
+ if Map.keys(map) |> Enum.all?(&(&1 in expected_keys)) do
+ {:ok, atomize_keys(map)}
+ else
+ {:error, "Invalid include '#{i(map)}'."}
+ end
+ end
+
defp update_comparisons_date_range(%{comparisons: %{date_range: date_range}} = include, site) do
with {:ok, parsed_date_range} <- parse_date_range_pair(site, date_range) do
{:ok, put_in(include, [:comparisons, :date_range], parsed_date_range)}
diff --git a/lib/plausible/stats/legacy/legacy_query_builder.ex b/lib/plausible/stats/legacy/legacy_query_builder.ex
index 35f03521e4..7323f27a34 100644
--- a/lib/plausible/stats/legacy/legacy_query_builder.ex
+++ b/lib/plausible/stats/legacy/legacy_query_builder.ex
@@ -6,7 +6,7 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do
use Plausible
- alias Plausible.Stats.{Filters, Interval, Query, DateTimeRange, Metrics}
+ alias Plausible.Stats.{Filters, Interval, Query, DateTimeRange}
def from(site, params, debug_metadata, now \\ nil) do
now = now || DateTime.utc_now(:second)
@@ -219,7 +219,10 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do
end
defp put_include(query, site, params) do
+ include = parse_include(site, params["include"])
+
query
+ |> struct!(include: include)
|> Query.set_include(:comparisons, parse_comparison_params(site, params))
|> Query.set_include(:imports, params["with_imported"] == "true")
end
@@ -250,31 +253,37 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do
iex> QueryBuilder.parse_order_by(~s([["visitors","asc"],["visit:source","desc"]]))
[{:visitors, :asc}, {"visit:source", :desc}]
"""
- def parse_order_by(order_by) when is_binary(order_by) do
- case Jason.decode(order_by) do
- {:ok, parsed} when is_list(parsed) ->
- Enum.flat_map(parsed, &parse_order_by_pair/1)
-
- _ ->
- []
- end
+ def parse_order_by(order_by) do
+ json_decode(order_by)
+ |> unwrap([])
+ |> Filters.QueryParser.parse_order_by()
+ |> unwrap([])
end
- def parse_order_by(_) do
- []
- end
+ @doc """
+ ### Examples:
+ iex> QueryBuilder.parse_include(%{}, nil)
+ QueryParser.default_include()
- defp parse_order_by_pair([metric_or_dimension, direction]) when direction in ["asc", "desc"] do
- case Metrics.from_string(metric_or_dimension) do
- {:ok, metric} -> [{metric, String.to_existing_atom(direction)}]
- :error -> [{metric_or_dimension, String.to_existing_atom(direction)}]
- end
+ iex> QueryBuilder.parse_include(%{}, ~s({"total_rows": true}))
+ Map.merge(QueryParser.default_include(), %{total_rows: true})
+ """
+ def parse_include(site, include) do
+ json_decode(include)
+ |> unwrap(%{})
+ |> Filters.QueryParser.parse_include(site)
+ |> unwrap(Filters.QueryParser.default_include())
end
- defp parse_order_by_pair(_) do
- []
+ defp json_decode(string) when is_binary(string) do
+ Jason.decode(string)
end
+ defp json_decode(_other), do: :error
+
+ defp unwrap({:ok, result}, _default), do: result
+ defp unwrap(_, default), do: default
+
defp put_order_by(query, %{} = params) do
struct!(query, order_by: parse_order_by(params["order_by"]))
end
diff --git a/lib/plausible/stats/query_result.ex b/lib/plausible/stats/query_result.ex
index 1f9603142c..f272dd647c 100644
--- a/lib/plausible/stats/query_result.ex
+++ b/lib/plausible/stats/query_result.ex
@@ -156,6 +156,22 @@ defmodule Plausible.Stats.QueryResult do
end
end
+ defp metric_warning(:time_on_page, %Query{} = query) do
+ case query.time_on_page_data do
+ %{include_legacy_metric: true, cutoff: cutoff} ->
+ cutoff_time =
+ cutoff |> DateTime.shift_zone!(query.timezone) |> Calendar.strftime("%Y-%m-%d %H:%M:%S")
+
+ %{
+ code: :legacy_time_on_page_used,
+ message: "Contains less accurate legacy time-on-page data up to #{cutoff_time}"
+ }
+
+ _ ->
+ nil
+ end
+ end
+
defp metric_warning(_metric, _query), do: nil
defp to_iso8601(datetime, timezone) do
diff --git a/lib/plausible/stats/query_runner.ex b/lib/plausible/stats/query_runner.ex
index fd148efef8..d6db51bb6f 100644
--- a/lib/plausible/stats/query_runner.ex
+++ b/lib/plausible/stats/query_runner.ex
@@ -64,6 +64,7 @@ defmodule Plausible.Stats.QueryRunner do
query
|> Comparisons.get_comparison_query()
|> Comparisons.add_comparison_filters(main_results)
+ |> QueryOptimizer.optimize()
struct!(runner, comparison_query: comparison_query)
end
diff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.ex
index 22d8af1dc3..3de6cc86b1 100644
--- a/lib/plausible/stats/sql/expression.ex
+++ b/lib/plausible/stats/sql/expression.ex
@@ -255,8 +255,10 @@ defmodule Plausible.Stats.SQL.Expression do
wrap_alias(
[e],
%{
- __internal_total_time_on_page: 0,
- __internal_total_time_on_page_visits: 0
+ # :KLUDGE: We would like to but can't use constant 0 here as it leads to cyclic aliases if there's no other
+ # metrics selected other than time_on_page.
+ __internal_total_time_on_page: fragment("sumArray([0])"),
+ __internal_total_time_on_page_visits: fragment("sumArray([0])")
}
)
diff --git a/lib/plausible/stats/time_on_page.ex b/lib/plausible/stats/time_on_page.ex
new file mode 100644
index 0000000000..38ca7a983a
--- /dev/null
+++ b/lib/plausible/stats/time_on_page.ex
@@ -0,0 +1,16 @@
+defmodule Plausible.Stats.TimeOnPage do
+ @moduledoc """
+ Module to check whether the new time on page metric is available.
+ """
+
+ def new_time_on_page_enabled?(site, user) do
+ FunWithFlags.enabled?(:new_time_on_page, for: user) ||
+ FunWithFlags.enabled?(:new_time_on_page, for: site)
+ end
+
+ def legacy_time_on_page_cutoff() do
+ # Placeholder until we implement a more sophisticated way to determine the cutoff
+ # Only used when `new_time_on_page` flag is enabled
+ DateTime.utc_now() |> DateTime.shift(day: -4) |> DateTime.to_iso8601()
+ end
+end
diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
index c773948b1b..4ceca38a1b 100644
--- a/lib/plausible_web/controllers/api/stats_controller.ex
+++ b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -5,7 +5,7 @@ defmodule PlausibleWeb.Api.StatsController do
use PlausibleWeb.Plugs.ErrorHandler
alias Plausible.Stats
- alias Plausible.Stats.{Query, Comparisons, Filters, Time, TableDecider}
+ alias Plausible.Stats.{Query, Comparisons, Filters, Time, TableDecider, TimeOnPage}
alias PlausibleWeb.Api.Helpers, as: H
require Logger
@@ -444,7 +444,11 @@ defmodule PlausibleWeb.Api.StatsController do
%{
top_stats: top_stats,
meta: meta,
- graphable_metrics: metrics -- [:time_on_page],
+ graphable_metrics:
+ if(TimeOnPage.new_time_on_page_enabled?(site, current_user),
+ do: metrics,
+ else: metrics -- [:time_on_page]
+ ),
sample_percent: sample_percent
}
end
diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex
index 8601a24c3c..f8bd1082de 100644
--- a/lib/plausible_web/controllers/stats_controller.ex
+++ b/lib/plausible_web/controllers/stats_controller.ex
@@ -45,7 +45,7 @@ defmodule PlausibleWeb.StatsController do
use Plausible.Repo
alias Plausible.Sites
- alias Plausible.Stats.{Filters, Query}
+ alias Plausible.Stats.{Filters, Query, TimeOnPage}
alias PlausibleWeb.Api
plug(PlausibleWeb.Plugs.AuthorizeSiteAccess when action in [:stats, :csv_export])
@@ -77,6 +77,7 @@ defmodule PlausibleWeb.StatsController do
scroll_depth_visible: scroll_depth_visible?,
stats_start_date: stats_start_date,
native_stats_start_date: NaiveDateTime.to_date(site.native_stats_start_at),
+ legacy_time_on_page_cutoff: TimeOnPage.legacy_time_on_page_cutoff(),
title: title(conn, site),
demo: demo,
flags: flags,
@@ -361,6 +362,7 @@ defmodule PlausibleWeb.StatsController do
scroll_depth_visible: scroll_depth_visible?,
stats_start_date: stats_start_date,
native_stats_start_date: NaiveDateTime.to_date(shared_link.site.native_stats_start_at),
+ legacy_time_on_page_cutoff: TimeOnPage.legacy_time_on_page_cutoff(),
title: title(conn, shared_link.site),
demo: false,
dogfood_page_path: "/share/:dashboard",
@@ -388,7 +390,7 @@ defmodule PlausibleWeb.StatsController do
defp get_flags(user, site),
do:
- [:saved_segments, :saved_segments_fe, :scroll_depth]
+ [:saved_segments, :saved_segments_fe, :scroll_depth, :new_time_on_page]
|> Enum.map(fn flag ->
{flag, FunWithFlags.enabled?(flag, for: user) || FunWithFlags.enabled?(flag, for: site)}
end)
diff --git a/lib/plausible_web/templates/stats/stats.html.heex b/lib/plausible_web/templates/stats/stats.html.heex
index abd70d64da..ff8b5811bb 100644
--- a/lib/plausible_web/templates/stats/stats.html.heex
+++ b/lib/plausible_web/templates/stats/stats.html.heex
@@ -37,6 +37,7 @@
data-logged-in={to_string([email protected][:current_user])}
data-stats-begin={@stats_start_date}
data-native-stats-begin={@native_stats_start_date}
+ data-legacy-time-on-page-cutoff={@legacy_time_on_page_cutoff}
data-shared-link-auth={assigns[:shared_link_auth]}
data-embedded={to_string(@conn.assigns[:embedded])}
data-background={@conn.assigns[:background]}
diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exs
index 5834931f8b..1461aaab7e 100644
--- a/test/plausible/stats/query_test.exs
+++ b/test/plausible/stats/query_test.exs
@@ -3,6 +3,7 @@ defmodule Plausible.Stats.QueryTest do
use Plausible.Teams.Test
alias Plausible.Stats.Query
alias Plausible.Stats.Legacy.QueryBuilder
+ alias Plausible.Stats.Filters.QueryParser
alias Plausible.Stats.DateTimeRange
doctest Plausible.Stats.Legacy.QueryBuilder
diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_time_on_page_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_time_on_page_test.exs
index 455be7b9a3..8fcc054cca 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/query_time_on_page_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/query_time_on_page_test.exs
@@ -39,9 +39,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
"include" => %{"imports" => true}
})
- assert json_response(conn, 200)["results"] == [
- %{"dimensions" => ["/blog"], "metrics" => [100]}
- ]
+ assert_matches %{
+ "results" => [
+ %{"dimensions" => ["/blog"], "metrics" => [100]}
+ ],
+ "meta" =>
+ ^strict_map(%{
+ "imports_included" => true
+ })
+ } = json_response(conn, 200)
end
test "aggregated time_on_page metric with imported data", %{
@@ -141,13 +147,19 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
"include" => %{"imports" => true}
})
- assert json_response(conn, 200)["results"] == [
- %{"dimensions" => ["2021-01-01", "/"], "metrics" => [100]},
- %{"dimensions" => ["2021-01-01", "/blog"], "metrics" => [20]},
- %{"dimensions" => ["2021-01-02", "/"], "metrics" => [200]},
- %{"dimensions" => ["2021-01-03", "/"], "metrics" => [100]},
- %{"dimensions" => ["2021-01-04", "/"], "metrics" => [100]}
- ]
+ assert_matches %{
+ "results" => [
+ %{"dimensions" => ["2021-01-01", "/"], "metrics" => [100]},
+ %{"dimensions" => ["2021-01-01", "/blog"], "metrics" => [20]},
+ %{"dimensions" => ["2021-01-02", "/"], "metrics" => [200]},
+ %{"dimensions" => ["2021-01-03", "/"], "metrics" => [100]},
+ %{"dimensions" => ["2021-01-04", "/"], "metrics" => [100]}
+ ],
+ "meta" =>
+ ^strict_map(%{
+ "imports_included" => true
+ })
+ } = json_response(conn, 200)
end
describe "include.legacy_time_on_page_cutoff" do
@@ -223,12 +235,18 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
}
})
- assert json_response(conn, 200)["results"] == [
- %{"dimensions" => ["/"], "metrics" => [115]},
- # (2 * 100s + 9 * 20s) / 10 = 38
- %{"dimensions" => ["/blog"], "metrics" => [38]},
- %{"dimensions" => ["/pricing"], "metrics" => [30]}
- ]
+ assert_matches %{
+ "results" => [
+ %{"dimensions" => ["/"], "metrics" => [115]},
+ # (2 * 100s + 9 * 20s) / 10 = 38
+ %{"dimensions" => ["/blog"], "metrics" => [38]},
+ %{"dimensions" => ["/pricing"], "metrics" => [30]}
+ ],
+ "meta" =>
+ ^strict_map(%{
+ "imports_included" => true
+ })
+ } = json_response(conn, 200)
end
test "breakdown with cutoff being after data (legacy query used)", %{conn: conn, site: site} do
@@ -244,11 +262,24 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
}
})
- assert json_response(conn, 200)["results"] == [
- %{"dimensions" => ["/"], "metrics" => [420]},
- %{"dimensions" => ["/blog"], "metrics" => [180]},
- %{"dimensions" => ["/pricing"], "metrics" => [nil]}
- ]
+ assert_matches %{
+ "results" => [
+ %{"dimensions" => ["/"], "metrics" => [420]},
+ %{"dimensions" => ["/blog"], "metrics" => [180]},
+ %{"dimensions" => ["/pricing"], "metrics" => [nil]}
+ ],
+ "meta" =>
+ ^strict_map(%{
+ "imports_included" => true,
+ "metric_warnings" => %{
+ "time_on_page" => %{
+ "code" => "legacy_time_on_page_used",
+ "message" =>
+ "Contains less accurate legacy time-on-page data up to 2021-01-05 00:00:00"
+ }
+ }
+ })
+ } = json_response(conn, 200)
end
test "breakdown with cutoff being mid-data (two queries joined)", %{conn: conn, site: site} do
@@ -264,11 +295,24 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
}
})
- assert json_response(conn, 200)["results"] == [
- %{"dimensions" => ["/"], "metrics" => [225]},
- %{"dimensions" => ["/blog"], "metrics" => [180]},
- %{"dimensions" => ["/pricing"], "metrics" => [30]}
- ]
+ assert_matches %{
+ "results" => [
+ %{"dimensions" => ["/"], "metrics" => [225]},
+ %{"dimensions" => ["/blog"], "metrics" => [180]},
+ %{"dimensions" => ["/pricing"], "metrics" => [30]}
+ ],
+ "meta" =>
+ ^strict_map(%{
+ "imports_included" => true,
+ "metric_warnings" => %{
+ "time_on_page" => %{
+ "code" => "legacy_time_on_page_used",
+ "message" =>
+ "Contains less accurate legacy time-on-page data up to 2021-01-02 00:00:00"
+ }
+ }
+ })
+ } = json_response(conn, 200)
end
test "breakdown with cutoff being mid-data (two queries joined), no imports", %{
@@ -286,11 +330,23 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
}
})
- assert json_response(conn, 200)["results"] == [
- %{"dimensions" => ["/"], "metrics" => [225]},
- %{"dimensions" => ["/blog"], "metrics" => [180]},
- %{"dimensions" => ["/pricing"], "metrics" => [30]}
- ]
+ assert_matches %{
+ "results" => [
+ %{"dimensions" => ["/"], "metrics" => [225]},
+ %{"dimensions" => ["/blog"], "metrics" => [180]},
+ %{"dimensions" => ["/pricing"], "metrics" => [30]}
+ ],
+ "meta" =>
+ ^strict_map(%{
+ "metric_warnings" => %{
+ "time_on_page" => %{
+ "code" => "legacy_time_on_page_used",
+ "message" =>
+ "Contains less accurate legacy time-on-page data up to 2021-01-02 00:00:00"
+ }
+ }
+ })
+ } = json_response(conn, 200)
end
test "aggregation with cutoff being after data (legacy query used)", %{conn: conn, site: site} do
@@ -306,9 +362,22 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
}
})
- assert json_response(conn, 200)["results"] == [
- %{"dimensions" => [], "metrics" => [180]}
- ]
+ assert_matches %{
+ "results" => [
+ %{"dimensions" => [], "metrics" => [180]}
+ ],
+ "meta" =>
+ ^strict_map(%{
+ "imports_included" => true,
+ "metric_warnings" => %{
+ "time_on_page" => %{
+ "code" => "legacy_time_on_page_used",
+ "message" =>
+ "Contains less accurate legacy time-on-page data up to 2021-01-05 00:00:00"
+ }
+ }
+ })
+ } = json_response(conn, 200)
end
test "aggregation with cutoff being mid-data (two queries joined)", %{conn: conn, site: site} do
@@ -324,9 +393,22 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
}
})
- assert json_response(conn, 200)["results"] == [
- %{"dimensions" => [], "metrics" => [180]}
- ]
+ assert_matches %{
+ "results" => [
+ %{"dimensions" => [], "metrics" => [180]}
+ ],
+ "meta" =>
+ ^strict_map(%{
+ "imports_included" => true,
+ "metric_warnings" => %{
+ "time_on_page" => %{
+ "code" => "legacy_time_on_page_used",
+ "message" =>
+ "Contains less accurate legacy time-on-page data up to 2021-01-02 00:00:00"
+ }
+ }
+ })
+ } = json_response(conn, 200)
end
test "aggregation with cutoff being mid-data (two queries joined), no imports", %{
@@ -344,9 +426,21 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
}
})
- assert json_response(conn, 200)["results"] == [
- %{"dimensions" => [], "metrics" => [180]}
- ]
+ assert_matches %{
+ "results" => [
+ %{"dimensions" => [], "metrics" => [180]}
+ ],
+ "meta" =>
+ ^strict_map(%{
+ "metric_warnings" => %{
+ "time_on_page" => %{
+ "code" => "legacy_time_on_page_used",
+ "message" =>
+ "Contains less accurate legacy time-on-page data up to 2021-01-02 00:00:00"
+ }
+ }
+ })
+ } = json_response(conn, 200)
end
end
@@ -409,4 +503,143 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
]
end
end
+
+ describe "timeseries" do
+ setup %{site: site} = context do
+ populate_stats(site, [
+ build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:engagement,
+ user_id: 12,
+ pathname: "/",
+ timestamp: ~N[2021-01-01 00:05:00],
+ engagement_time: 100_000
+ ),
+ build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-02 00:00:00]),
+ build(:engagement,
+ user_id: 12,
+ pathname: "/",
+ timestamp: ~N[2021-01-02 00:05:00],
+ engagement_time: 200_000
+ ),
+ build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-03 00:00:00]),
+ build(:engagement,
+ user_id: 12,
+ pathname: "/",
+ timestamp: ~N[2021-01-03 00:05:00],
+ engagement_time: 250_000
+ ),
+ build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-04 00:00:00]),
+ build(:engagement,
+ user_id: 12,
+ pathname: "/",
+ timestamp: ~N[2021-01-04 00:05:00],
+ engagement_time: 200_000
+ ),
+ build(:pageview, user_id: 13, pathname: "/", timestamp: ~N[2021-01-04 00:00:00]),
+ build(:engagement,
+ user_id: 13,
+ pathname: "/",
+ timestamp: ~N[2021-01-04 00:05:00],
+ engagement_time: 100_000
+ )
+ ])
+
+ context
+ end
+
+ test "reports average new time-on-page per day", %{conn: conn, site: site} do
+ conn =
+ post(conn, "/api/v2/query-internal-test", %{
+ "site_id" => site.domain,
+ "metrics" => ["time_on_page"],
+ "date_range" => ["2021-01-01", "2021-01-04"],
+ "filters" => [["is", "event:page", ["/"]]],
+ "dimensions" => ["time:day"]
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["2021-01-01"], "metrics" => [100]},
+ %{"dimensions" => ["2021-01-02"], "metrics" => [200]},
+ %{"dimensions" => ["2021-01-03"], "metrics" => [250]},
+ %{"dimensions" => ["2021-01-04"], "metrics" => [150]}
+ ]
+ end
+
+ test "reports legacy time-on-page as nulls per day", %{conn: conn, site: site} do
+ conn =
+ post(conn, "/api/v2/query-internal-test", %{
+ "site_id" => site.domain,
+ "metrics" => ["time_on_page"],
+ "date_range" => ["2021-01-01", "2021-01-04"],
+ "filters" => [["is", "event:page", ["/"]]],
+ "dimensions" => ["time:day"],
+ "include" => %{"legacy_time_on_page_cutoff" => "2100-01-01T00:00:00Z"}
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["2021-01-01"], "metrics" => [nil]},
+ %{"dimensions" => ["2021-01-02"], "metrics" => [nil]},
+ %{"dimensions" => ["2021-01-03"], "metrics" => [nil]},
+ %{"dimensions" => ["2021-01-04"], "metrics" => [nil]}
+ ]
+ end
+
+ test "respects `legacy_time_on_page_cutoff`", %{conn: conn, site: site} do
+ conn =
+ post(conn, "/api/v2/query-internal-test", %{
+ "site_id" => site.domain,
+ "metrics" => ["time_on_page"],
+ "date_range" => ["2021-01-01", "2021-01-04"],
+ "filters" => [["is", "event:page", ["/"]]],
+ "dimensions" => ["time:day"],
+ "include" => %{"legacy_time_on_page_cutoff" => "2021-01-02T23:59:59Z"}
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["2021-01-01"], "metrics" => [nil]},
+ %{"dimensions" => ["2021-01-02"], "metrics" => [nil]},
+ %{"dimensions" => ["2021-01-03"], "metrics" => [250]},
+ %{"dimensions" => ["2021-01-04"], "metrics" => [150]}
+ ]
+ end
+
+ test "can use comparisons together with `legacy_time_on_page_cutoff`", %{
+ conn: conn,
+ site: site
+ } do
+ conn =
+ post(conn, "/api/v2/query-internal-test", %{
+ "site_id" => site.domain,
+ "metrics" => ["time_on_page"],
+ "date_range" => ["2021-01-03", "2021-01-04"],
+ "filters" => [["is", "event:page", ["/"]]],
+ "dimensions" => ["time:day"],
+ "include" => %{
+ "legacy_time_on_page_cutoff" => "2021-01-01T23:59:59Z",
+ "comparisons" => %{"mode" => "previous_period"}
+ }
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{
+ "dimensions" => ["2021-01-03"],
+ "metrics" => [250],
+ "comparison" => %{
+ "dimensions" => ["2021-01-01"],
+ "metrics" => [nil],
+ "change" => [nil]
+ }
+ },
+ %{
+ "dimensions" => ["2021-01-04"],
+ "metrics" => [150],
+ "comparison" => %{
+ "dimensions" => ["2021-01-02"],
+ "metrics" => [200],
+ "change" => [-25]
+ }
+ }
+ ]
+ end
+ end
end