Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

time-on-page: csv import/export support, preparation for release (#5274)

Karl-Aksel Puulmann0fcbcc3a8c25
diff --git a/assets/js/dashboard/api.ts b/assets/js/dashboard/api.ts
index 0f183b2580..fa407178e4 100644
--- a/assets/js/dashboard/api.ts
+++ b/assets/js/dashboard/api.ts
@@ -72,25 +72,11 @@ export function queryToSearchParams(
queryObj.auth = sharedLinkParams.auth
}
- if (
- query.legacy_time_on_page_cutoff &&
- validDate(query.legacy_time_on_page_cutoff)
- ) {
- queryObj.include = JSON.stringify({
- legacy_time_on_page_cutoff: query.legacy_time_on_page_cutoff
- })
- }
-
Object.assign(queryObj, ...extraQuery)
return serializeUrlParams(queryObj)
}
-function validDate(dateString: string) {
- const date = new Date(dateString)
- return isFinite(date.getTime())
-}
-
function getHeaders(): Record<string, string> {
return SHARED_LINK_AUTH ? { 'X-Shared-Link-Auth': SHARED_LINK_AUTH } : {}
}
diff --git a/assets/js/dashboard/query-context.tsx b/assets/js/dashboard/query-context.tsx
index 27144258db..6f9d869106 100644
--- a/assets/js/dashboard/query-context.tsx
+++ b/assets/js/dashboard/query-context.tsx
@@ -61,7 +61,6 @@ export default function QueryContextProvider({
period,
to,
with_imported,
- legacy_time_on_page_cutoff,
...otherSearch
} = useMemo(() => parseSearch(location.search), [location.search])
@@ -112,10 +111,7 @@ export default function QueryContextProvider({
: defaultValues.with_imported,
filters,
resolvedFilters,
- labels: (labels as FilterClauseLabels) || defaultValues.labels,
- legacy_time_on_page_cutoff: site.flags.new_time_on_page
- ? (legacy_time_on_page_cutoff as string) || site.legacyTimeOnPageCutoff
- : defaultValues.legacy_time_on_page_cutoff
+ labels: (labels as FilterClauseLabels) || defaultValues.labels
}
}, [
compare_from,
@@ -129,7 +125,6 @@ export default function QueryContextProvider({
period,
to,
with_imported,
- legacy_time_on_page_cutoff,
site,
expandedSegment,
segmentsContext.segments
diff --git a/assets/js/dashboard/query.ts b/assets/js/dashboard/query.ts
index ff4fc7018f..5b2d2b71b8 100644
--- a/assets/js/dashboard/query.ts
+++ b/assets/js/dashboard/query.ts
@@ -50,7 +50,6 @@ export type DashboardQuery = {
resolvedFilters: Filter[]
labels: FilterClauseLabels
with_imported: boolean
- legacy_time_on_page_cutoff: string | undefined
}
export const queryDefaultValue: DashboardQuery = {
@@ -65,8 +64,7 @@ export const queryDefaultValue: DashboardQuery = {
filters: [],
resolvedFilters: [],
labels: {},
- with_imported: true,
- legacy_time_on_page_cutoff: undefined
+ with_imported: true
}
export type BreakdownResultMeta = {
diff --git a/assets/js/dashboard/site-context.test.tsx b/assets/js/dashboard/site-context.test.tsx
index d82cfe7128..a73d846cba 100644
--- a/assets/js/dashboard/site-context.test.tsx
+++ b/assets/js/dashboard/site-context.test.tsx
@@ -67,8 +67,7 @@ describe('parseSiteFromDataset', () => {
realtime: ['minute'],
year: ['day', 'week', 'month']
},
- shared: false,
- legacyTimeOnPageCutoff: '2022-01-01T00:00:00Z'
+ shared: false
}
it('parses from dom string map correctly', () => {
diff --git a/assets/js/dashboard/site-context.tsx b/assets/js/dashboard/site-context.tsx
index 56c97dc6dd..2e8d4cd804 100644
--- a/assets/js/dashboard/site-context.tsx
+++ b/assets/js/dashboard/site-context.tsx
@@ -12,7 +12,6 @@ export function parseSiteFromDataset(dataset: DOMStringMap): PlausibleSite {
conversionsOptedOut: dataset.conversionsOptedOut === 'true',
funnelsOptedOut: dataset.funnelsOptedOut === 'true',
propsOptedOut: dataset.propsOptedOut === 'true',
- legacyTimeOnPageCutoff: dataset.legacyTimeOnPageCutoff,
revenueGoals: JSON.parse(dataset.revenueGoals!),
funnels: JSON.parse(dataset.funnels!),
statsBegin: dataset.statsBegin!,
@@ -53,8 +52,7 @@ const siteContextDefaultValue = {
isDbip: false,
flags: {} as FeatureFlags,
validIntervalsByPeriod: {} as Record<string, Array<string>>,
- shared: false,
- legacyTimeOnPageCutoff: undefined as string | undefined
+ shared: false
}
export type PlausibleSite = typeof siteContextDefaultValue
diff --git a/assets/js/types/query-api.d.ts b/assets/js/types/query-api.d.ts
index 0c274ede0b..e56b773c42 100644
--- a/assets/js/types/query-api.d.ts
+++ b/assets/js/types/query-api.d.ts
@@ -208,7 +208,6 @@ export interface QueryApiSchema {
*/
date_range: [string, string];
};
- legacy_time_on_page_cutoff?: string;
};
pagination?: {
/**
diff --git a/assets/test-utils/app-context-providers.tsx b/assets/test-utils/app-context-providers.tsx
index 6a3d86b3a9..cb73c60726 100644
--- a/assets/test-utils/app-context-providers.tsx
+++ b/assets/test-utils/app-context-providers.tsx
@@ -49,8 +49,7 @@ export const TestContextProviders = ({
isDbip: false,
flags: {},
validIntervalsByPeriod: {},
- shared: false,
- legacyTimeOnPageCutoff: undefined
+ shared: false
}
const site = { ...defaultSite, ...siteOptions }
diff --git a/lib/plausible/exports.ex b/lib/plausible/exports.ex
index 91657cd6d5..0cd6eebf5f 100644
--- a/lib/plausible/exports.ex
+++ b/lib/plausible/exports.ex
@@ -8,15 +8,14 @@ defmodule Plausible.Exports do
import Ecto.Query
@doc "Schedules CSV export job to S3 storage"
- @spec schedule_s3_export(pos_integer, pos_integer | nil, String.t()) ::
+ @spec schedule_s3_export(pos_integer, String.t()) ::
{:ok, Oban.Job.t()} | {:error, :no_data}
- def schedule_s3_export(site_id, current_user_id, email_to) do
+ def schedule_s3_export(site_id, email_to) do
with :ok <- ensure_has_data(site_id) do
args = %{
"storage" => "s3",
"site_id" => site_id,
"email_to" => email_to,
- "current_user_id" => current_user_id,
"s3_bucket" => Plausible.S3.exports_bucket(),
"s3_path" => s3_export_key(site_id)
}
@@ -209,13 +208,13 @@ defmodule Plausible.Exports do
Builds Ecto queries to export data from `events_v2` and `sessions_v2`
tables into the format of `imported_*` tables for a website.
"""
- @spec export_queries(pos_integer, pos_integer | nil,
+ @spec export_queries(pos_integer,
extname: String.t(),
date_range: Date.Range.t(),
timezone: String.t()
) ::
%{String.t() => Ecto.Query.t()}
- def export_queries(site_id, current_user_id, opts \\ []) do
+ def export_queries(site_id, opts \\ []) do
extname = opts[:extname] || ".csv"
date_range = opts[:date_range]
timezone = opts[:timezone] || "UTC"
@@ -234,8 +233,7 @@ defmodule Plausible.Exports do
%{
filename.("imported_visitors") => export_visitors_q(site_id, timezone, date_range),
filename.("imported_sources") => export_sources_q(site_id, timezone, date_range),
- filename.("imported_pages") =>
- export_pages_q(site_id, current_user_id, timezone, date_range),
+ filename.("imported_pages") => export_pages_q(site_id, timezone, date_range),
filename.("imported_entry_pages") => export_entry_pages_q(site_id, timezone, date_range),
filename.("imported_exit_pages") => export_exit_pages_q(site_id, timezone, date_range),
filename.("imported_custom_events") =>
@@ -414,9 +412,7 @@ defmodule Plausible.Exports do
]
end
- defp export_pages_q(site_id, current_user_id, timezone, date_range) do
- _will_be_used_for_time_on_page_flag = current_user_id
-
+ defp export_pages_q(site_id, timezone, date_range) do
base_q =
from(e in sampled("events_v2"),
where: ^export_filter(site_id, date_range),
@@ -452,20 +448,64 @@ defmodule Plausible.Exports do
from(e in base_q,
left_join: s in subquery(scroll_depth_q),
on: s.date == selected_as(:date) and s.page == selected_as(:page),
- select: [
- date(e.timestamp, ^timezone),
- selected_as(fragment("any(?)", e.hostname), :hostname),
- selected_as(e.pathname, :page),
- selected_as(
- scale_sample(fragment("uniq(?)", e.session_id)),
- :visits
- ),
- visitors(e),
- selected_as(scale_sample(fragment("count()")), :pageviews),
- selected_as(fragment("any(?)", s.total_scroll_depth), :total_scroll_depth),
- selected_as(fragment("any(?)", s.total_scroll_depth_visits), :total_scroll_depth_visits)
- ]
+ select: %{
+ date: date(e.timestamp, ^timezone),
+ hostname: selected_as(fragment("any(?)", e.hostname), :hostname),
+ page: selected_as(e.pathname, :page),
+ visits:
+ selected_as(
+ scale_sample(fragment("uniq(?)", e.session_id)),
+ :visits
+ ),
+ visitors: visitors(e),
+ pageviews: selected_as(scale_sample(fragment("count()")), :pageviews),
+ total_scroll_depth:
+ selected_as(fragment("any(?)", s.total_scroll_depth), :total_scroll_depth),
+ total_scroll_depth_visits:
+ selected_as(fragment("any(?)", s.total_scroll_depth_visits), :total_scroll_depth_visits)
+ }
)
+ |> add_time_on_page_columns(site_id, timezone, date_range)
+ end
+
+ defp add_time_on_page_columns(q, site_id, timezone, date_range) do
+ site = Plausible.Repo.get(Plausible.Site, site_id)
+
+ if Plausible.Stats.TimeOnPage.new_time_on_page_visible?(site) do
+ cutoff = Plausible.Stats.TimeOnPage.legacy_time_on_page_cutoff(site)
+
+ engagements_q =
+ from(e in sampled("events_v2"),
+ where: ^export_filter(site_id, date_range),
+ where: e.name == "engagement",
+ group_by: [selected_as(:date), selected_as(:page)],
+ order_by: selected_as(:date),
+ select: %{
+ date: date(e.timestamp, ^timezone),
+ page: selected_as(e.pathname, :page),
+ total_time_on_page:
+ fragment(
+ "toUInt64(round(sumIf(?, ? >= ?) / 1000))",
+ e.engagement_time,
+ e.timestamp,
+ ^cutoff
+ ),
+ total_time_on_page_visits:
+ fragment("uniqIf(?, ? >= ?)", e.session_id, e.timestamp, ^cutoff)
+ }
+ )
+
+ q
+ |> join(:left, [], s in subquery(engagements_q),
+ on: s.date == selected_as(:date) and s.page == selected_as(:page)
+ )
+ |> select_merge_as([..., s], %{
+ total_time_on_page: fragment("any(?)", s.total_time_on_page),
+ total_time_on_page_visits: fragment("any(?)", s.total_time_on_page_visits)
+ })
+ else
+ q
+ end
end
defp export_entry_pages_q(site_id, timezone, date_range) do
diff --git a/lib/plausible/sites.ex b/lib/plausible/sites.ex
index e4777a5619..a2349bcb92 100644
--- a/lib/plausible/sites.ex
+++ b/lib/plausible/sites.ex
@@ -352,6 +352,13 @@ defmodule Plausible.Sites do
|> Repo.update!()
end
+ def update_legacy_time_on_page_cutoff!(site, cutoff) do
+ site
+ |> Ecto.Changeset.change()
+ |> Ecto.Changeset.put_change(:legacy_time_on_page_cutoff, cutoff)
+ |> Repo.update!()
+ end
+
def has_goals?(site) do
Repo.exists?(
from(g in Plausible.Goal,
diff --git a/lib/plausible/stats/legacy/legacy_query_builder.ex b/lib/plausible/stats/legacy/legacy_query_builder.ex
index eba5dc6c3c..9ee542650f 100644
--- a/lib/plausible/stats/legacy/legacy_query_builder.ex
+++ b/lib/plausible/stats/legacy/legacy_query_builder.ex
@@ -30,6 +30,7 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do
|> put_include(site, params)
|> Query.put_comparison_utc_time_range()
|> Query.put_imported_opts(site)
+ |> Query.set_time_on_page_data(site)
on_ee do
query = Plausible.Stats.Sampling.put_threshold(query, site, params)
diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex
index 2bbe5e406c..a560b9bbdf 100644
--- a/lib/plausible/stats/query.ex
+++ b/lib/plausible/stats/query.ex
@@ -28,12 +28,7 @@ defmodule Plausible.Stats.Query do
site_id: nil,
site_native_stats_start_at: nil,
# Contains information to determine how to combine legacy and new time on page metrics
- # Old stats API and dashboard uses legacy metric by default, new API uses new metric by default
- time_on_page_data: %{
- include_new_metric: false,
- include_legacy_metric: true,
- cutoff: ~U[2100-01-01 00:00:00Z]
- }
+ time_on_page_data: %{}
require OpenTelemetry.Tracer, as: Tracer
alias Plausible.Stats.{DateTimeRange, Filters, Imported, Legacy, Comparisons}
@@ -50,6 +45,7 @@ defmodule Plausible.Stats.Query do
site_native_stats_start_at: site.native_stats_start_at
}
|> struct!(Map.to_list(query_data))
+ |> set_time_on_page_data(site)
|> put_comparison_utc_time_range()
|> put_imported_opts(site)
@@ -180,6 +176,15 @@ defmodule Plausible.Stats.Query do
in_comparison_range ++ in_range
end
+ def set_time_on_page_data(query, site) do
+ struct!(query,
+ time_on_page_data: %{
+ new_metric_visible: Plausible.Stats.TimeOnPage.new_time_on_page_visible?(site),
+ cutoff_date: site.legacy_time_on_page_cutoff
+ }
+ )
+ end
+
@spec get_skip_imported_reason(t()) ::
nil | :no_imported_data | :out_of_range | :unsupported_query
def get_skip_imported_reason(query) do
diff --git a/lib/plausible/stats/query_optimizer.ex b/lib/plausible/stats/query_optimizer.ex
index 6e590d866d..c9f7a7c82c 100644
--- a/lib/plausible/stats/query_optimizer.ex
+++ b/lib/plausible/stats/query_optimizer.ex
@@ -4,7 +4,17 @@ defmodule Plausible.Stats.QueryOptimizer do
"""
use Plausible
- alias Plausible.Stats.{DateTimeRange, Filters, Query, TableDecider, Util, Time, Legacy}
+
+ alias Plausible.Stats.{
+ DateTimeRange,
+ Filters,
+ Query,
+ TableDecider,
+ TimeOnPage,
+ Util,
+ Time,
+ Legacy
+ }
@doc """
This module manipulates an existing query, updating it according to business logic.
@@ -188,28 +198,37 @@ defmodule Plausible.Stats.QueryOptimizer do
end
defp set_time_on_page_data(query) do
- if :time_on_page in query.metrics and query.include.legacy_time_on_page_cutoff do
- {:ok, cutoff, _} =
- query.include.legacy_time_on_page_cutoff
- |> DateTime.from_iso8601()
-
- cutoff =
- cutoff
- |> DateTime.shift_zone!("Etc/UTC")
- |> DateTime.truncate(:second)
-
- Query.set(
- query,
- time_on_page_data: %{
- include_new_metric: DateTime.before?(cutoff, query.utc_time_range.last),
- include_legacy_metric:
- DateTime.after?(cutoff, query.utc_time_range.first) and
- Legacy.TimeOnPage.can_merge_legacy_time_on_page?(query),
- cutoff: if(DateTime.after?(cutoff, query.utc_time_range.first), do: cutoff, else: nil)
- }
- )
- else
- query
+ case {:time_on_page in query.metrics, query.time_on_page_data} do
+ {true, %{new_metric_visible: true, cutoff_date: cutoff_date}} ->
+ cutoff =
+ cutoff_date
+ |> TimeOnPage.cutoff_datetime(query.timezone)
+ |> DateTime.shift_zone!("Etc/UTC")
+ |> DateTime.truncate(:second)
+
+ Query.set(
+ query,
+ time_on_page_data:
+ Map.merge(query.time_on_page_data, %{
+ include_new_metric: DateTime.before?(cutoff, query.utc_time_range.last),
+ include_legacy_metric:
+ DateTime.after?(cutoff, query.utc_time_range.first) and
+ Legacy.TimeOnPage.can_merge_legacy_time_on_page?(query),
+ cutoff:
+ if(DateTime.after?(cutoff, query.utc_time_range.first), do: cutoff, else: nil)
+ })
+ )
+
+ _ ->
+ Query.set(
+ query,
+ time_on_page_data:
+ Map.merge(query.time_on_page_data, %{
+ include_new_metric: false,
+ include_legacy_metric: true,
+ cutoff: nil
+ })
+ )
end
end
end
diff --git a/lib/plausible/stats/query_result.ex b/lib/plausible/stats/query_result.ex
index 75c8bed891..47b2460ff7 100644
--- a/lib/plausible/stats/query_result.ex
+++ b/lib/plausible/stats/query_result.ex
@@ -158,7 +158,7 @@ defmodule Plausible.Stats.QueryResult do
defp metric_warning(:time_on_page, %Query{} = query) do
case query.time_on_page_data do
- %{include_legacy_metric: true, cutoff: cutoff} ->
+ %{new_metric_visible: true, include_legacy_metric: true, cutoff: cutoff} ->
cutoff_date =
cutoff |> DateTime.shift_zone!(query.timezone) |> Calendar.strftime("%Y-%m-%d")
diff --git a/lib/plausible/stats/time_on_page.ex b/lib/plausible/stats/time_on_page.ex
index d5305ced8e..c6d9c1822e 100644
--- a/lib/plausible/stats/time_on_page.ex
+++ b/lib/plausible/stats/time_on_page.ex
@@ -3,23 +3,29 @@ defmodule Plausible.Stats.TimeOnPage do
Module to check whether the new time on page metric is available.
"""
- def new_time_on_page_enabled?(site, user) do
- FunWithFlags.enabled?(:new_time_on_page, for: user) ||
- FunWithFlags.enabled?(:new_time_on_page, for: site)
+ def new_time_on_page_visible?(site) do
+ new_time_on_page_enabled?(site) && not is_nil(site.legacy_time_on_page_cutoff)
end
def legacy_time_on_page_cutoff(site) do
- cutoff(site.legacy_time_on_page_cutoff, site.timezone)
+ cutoff_datetime(site.legacy_time_on_page_cutoff, site.timezone)
end
- defp cutoff(nil, _timezone), do: ""
+ def cutoff_datetime(nil, _timezone), do: nil
- defp cutoff(date, timezone) do
+ # Workaround for a case where casting unix epoch to DateTime fails for sites that should
+ # always have the new time-on-page only. Affects CSV imports.
+ def cutoff_datetime(~D[1970-01-01], _timezone), do: ~U[2000-01-01 00:00:00Z]
+
+ def cutoff_datetime(date, timezone) do
case DateTime.new(date, ~T[00:00:00], timezone) do
{:ok, datetime} -> datetime
{:gap, just_before, _just_after} -> just_before
{:ambiguous, first_datetime, _second_datetime} -> first_datetime
end
- |> DateTime.to_iso8601()
+ end
+
+ defp new_time_on_page_enabled?(site) do
+ FunWithFlags.enabled?(:new_time_on_page, for: site)
end
end
diff --git a/lib/plausible_web/controllers/api/external_query_api_controller.ex b/lib/plausible_web/controllers/api/external_query_api_controller.ex
index 417f42a643..7fc17e0e64 100644
--- a/lib/plausible_web/controllers/api/external_query_api_controller.ex
+++ b/lib/plausible_web/controllers/api/external_query_api_controller.ex
@@ -11,8 +11,6 @@ defmodule PlausibleWeb.Api.ExternalQueryApiController do
case Query.build(site, conn.assigns.schema_type, params, debug_metadata(conn)) do
{:ok, query} ->
- query = update_time_on_page_query_data(query)
-
results = Plausible.Stats.query(site, query)
json(conn, results)
@@ -26,18 +24,4 @@ defmodule PlausibleWeb.Api.ExternalQueryApiController do
def schema(conn, _params) do
json(conn, Plausible.Stats.JSONSchema.raw_public_schema())
end
-
- defp update_time_on_page_query_data(query) do
- if is_nil(query.include.legacy_time_on_page_cutoff) do
- Query.set(query,
- time_on_page_data: %{
- include_new_metric: true,
- include_legacy_metric: false,
- cutoff: nil
- }
- )
- else
- query
- end
- end
end
diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
index 644690c4fb..8002b037bb 100644
--- a/lib/plausible_web/controllers/api/stats_controller.ex
+++ b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -182,7 +182,6 @@ defmodule PlausibleWeb.Api.StatsController do
def top_stats(conn, params) do
site = conn.assigns[:site]
- current_user = conn.assigns[:current_user]
params = realtime_period_to_30m(params)
@@ -196,7 +195,7 @@ defmodule PlausibleWeb.Api.StatsController do
meta: meta,
sample_percent: sample_percent,
graphable_metrics: graphable_metrics
- } = fetch_top_stats(site, query, current_user)
+ } = fetch_top_stats(site, query)
comparison_query = comparison_query(query)
@@ -277,7 +276,7 @@ defmodule PlausibleWeb.Api.StatsController do
end
end
- defp fetch_top_stats(site, query, current_user) do
+ defp fetch_top_stats(site, query) do
goal_filter? =
toplevel_goal_filter?(query)
@@ -292,7 +291,7 @@ defmodule PlausibleWeb.Api.StatsController do
fetch_goal_top_stats(site, query)
true ->
- fetch_other_top_stats(site, query, current_user)
+ fetch_other_top_stats(site, query)
end
end
@@ -393,7 +392,7 @@ defmodule PlausibleWeb.Api.StatsController do
%{top_stats: top_stats, meta: meta, graphable_metrics: metrics, sample_percent: 100}
end
- defp fetch_other_top_stats(site, query, current_user) do
+ defp fetch_other_top_stats(site, query) do
page_filter? =
Filters.filtering_on_dimension?(query, "event:page", behavioral_filters: :ignore)
@@ -437,7 +436,7 @@ defmodule PlausibleWeb.Api.StatsController do
top_stats: top_stats,
meta: meta,
graphable_metrics:
- if(TimeOnPage.new_time_on_page_enabled?(site, current_user),
+ if(TimeOnPage.new_time_on_page_visible?(site),
do: metrics,
else: metrics -- [:time_on_page]
),
diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex
index 00b7273c19..9d28241dd7 100644
--- a/lib/plausible_web/controllers/stats_controller.ex
+++ b/lib/plausible_web/controllers/stats_controller.ex
@@ -45,7 +45,7 @@ defmodule PlausibleWeb.StatsController do
use Plausible.Repo
alias Plausible.Sites
- alias Plausible.Stats.{Filters, Query, TimeOnPage}
+ alias Plausible.Stats.{Filters, Query}
alias PlausibleWeb.Api
plug(PlausibleWeb.Plugs.AuthorizeSiteAccess when action in [:stats, :csv_export])
@@ -77,7 +77,6 @@ defmodule PlausibleWeb.StatsController do
has_props: Plausible.Props.configured?(site),
stats_start_date: stats_start_date,
native_stats_start_date: NaiveDateTime.to_date(site.native_stats_start_at),
- legacy_time_on_page_cutoff: TimeOnPage.legacy_time_on_page_cutoff(site),
title: title(conn, site),
demo: demo,
flags: flags,
@@ -362,7 +361,6 @@ defmodule PlausibleWeb.StatsController do
has_props: Plausible.Props.configured?(shared_link.site),
stats_start_date: stats_start_date,
native_stats_start_date: NaiveDateTime.to_date(shared_link.site.native_stats_start_at),
- legacy_time_on_page_cutoff: TimeOnPage.legacy_time_on_page_cutoff(shared_link.site),
title: title(conn, shared_link.site),
demo: false,
dogfood_page_path: "/share/:dashboard",
diff --git a/lib/plausible_web/live/csv_export.ex b/lib/plausible_web/live/csv_export.ex
index 4be300fb90..21eeb69154 100644
--- a/lib/plausible_web/live/csv_export.ex
+++ b/lib/plausible_web/live/csv_export.ex
@@ -210,12 +210,12 @@ defmodule PlausibleWeb.Live.CSVExport do
@impl true
def handle_event("export", _params, socket) do
- %{storage: storage, site_id: site_id, email_to: email_to, current_user: current_user} =
+ %{storage: storage, site_id: site_id, email_to: email_to} =
socket.assigns
schedule_result =
case storage do
- "s3" -> Exports.schedule_s3_export(site_id, current_user.id, email_to)
+ "s3" -> Exports.schedule_s3_export(site_id, email_to)
"local" -> Exports.schedule_local_export(site_id, email_to)
end
diff --git a/lib/plausible_web/templates/stats/stats.html.heex b/lib/plausible_web/templates/stats/stats.html.heex
index 2454b68d63..b1537e19f3 100644
--- a/lib/plausible_web/templates/stats/stats.html.heex
+++ b/lib/plausible_web/templates/stats/stats.html.heex
@@ -36,7 +36,6 @@
data-logged-in={to_string([email protected][:current_user])}
data-stats-begin={@stats_start_date}
data-native-stats-begin={@native_stats_start_date}
- data-legacy-time-on-page-cutoff={@legacy_time_on_page_cutoff}
data-shared-link-auth={assigns[:shared_link_auth]}
data-embedded={to_string(@conn.assigns[:embedded])}
data-background={@conn.assigns[:background]}
diff --git a/lib/workers/export_analytics.ex b/lib/workers/export_analytics.ex
index d5da9f05d0..51a1c0af06 100644
--- a/lib/workers/export_analytics.ex
+++ b/lib/workers/export_analytics.ex
@@ -29,13 +29,11 @@ defmodule Plausible.Workers.ExportAnalytics do
"site_id" => site_id
} = args
- current_user_id = args["current_user_id"]
-
site = Plausible.Repo.get!(Plausible.Site, site_id)
%Date.Range{} = date_range = Exports.date_range(site.id, site.timezone)
queries =
- Exports.export_queries(site_id, current_user_id,
+ Exports.export_queries(site_id,
date_range: date_range,
timezone: site.timezone,
extname: ".csv"
diff --git a/priv/json-schemas/query-api-schema.json b/priv/json-schemas/query-api-schema.json
index d511d58d41..7d1403a82a 100644
--- a/priv/json-schemas/query-api-schema.json
+++ b/priv/json-schemas/query-api-schema.json
@@ -109,11 +109,6 @@
"additionalProperties": false
}
]
- },
- "legacy_time_on_page_cutoff": {
- "$comment": "only :internal",
- "type": "string",
- "format": "datetime"
}
}
},
diff --git a/test/plausible/exports_test.exs b/test/plausible/exports_test.exs
index da28d17b3e..d6016916fc 100644
--- a/test/plausible/exports_test.exs
+++ b/test/plausible/exports_test.exs
@@ -9,7 +9,7 @@ defmodule Plausible.ExportsTest do
setup [:create_user, :create_site]
test "returns named ecto queries", %{site: site} do
- queries = Plausible.Exports.export_queries(site.id, nil)
+ queries = Plausible.Exports.export_queries(site.id)
assert queries |> Map.values() |> Enum.all?(&match?(%Ecto.Query{}, &1))
assert Map.keys(queries) == [
@@ -28,7 +28,7 @@ defmodule Plausible.ExportsTest do
test "with date range", %{site: site} do
queries =
- Plausible.Exports.export_queries(site.id, nil,
+ Plausible.Exports.export_queries(site.id,
date_range: Date.range(~D[2023-01-01], ~D[2024-03-12])
)
@@ -47,7 +47,7 @@ defmodule Plausible.ExportsTest do
end
test "with custom extension", %{site: site} do
- queries = Plausible.Exports.export_queries(site.id, nil, extname: ".ch")
+ queries = Plausible.Exports.export_queries(site.id, extname: ".ch")
assert Map.keys(queries) == [
"imported_browsers.ch",
diff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exs
index a7ccffd182..ef9f13fdc9 100644
--- a/test/plausible/imported/csv_importer_test.exs
+++ b/test/plausible/imported/csv_importer_test.exs
@@ -1059,9 +1059,10 @@ defmodule Plausible.Imported.CSVImporterTest do
end
@tag :tmp_dir
- test "scroll_depth", %{conn: conn, user: user, tmp_dir: tmp_dir} do
+ test "scroll_depth and time_on_page", %{conn: conn, user: user, tmp_dir: tmp_dir} do
exported_site = new_site(owner: user)
imported_site = new_site(owner: user)
+ FunWithFlags.enable(:new_time_on_page, for_actor: exported_site)
t0 =
NaiveDateTime.utc_now(:second)
@@ -1073,19 +1074,61 @@ defmodule Plausible.Imported.CSVImporterTest do
stats =
[
build(:pageview, user_id: 12, pathname: "/blog", timestamp: t0),
- build(:engagement, user_id: 12, pathname: "/blog", timestamp: t1, scroll_depth: 20),
+ build(:engagement,
+ user_id: 12,
+ pathname: "/blog",
+ timestamp: t1,
+ scroll_depth: 20,
+ engagement_time: 50_000
+ ),
build(:pageview, user_id: 12, pathname: "/another", timestamp: t1),
- build(:engagement, user_id: 12, pathname: "/another", timestamp: t2, scroll_depth: 24),
+ build(:engagement,
+ user_id: 12,
+ pathname: "/another",
+ timestamp: t2,
+ scroll_depth: 24,
+ engagement_time: 30_000
+ ),
build(:pageview, user_id: 34, pathname: "/blog", timestamp: t0),
- build(:engagement, user_id: 34, pathname: "/blog", timestamp: t1, scroll_depth: 17),
+ build(:engagement,
+ user_id: 34,
+ pathname: "/blog",
+ timestamp: t1,
+ scroll_depth: 17,
+ engagement_time: 10_000
+ ),
build(:pageview, user_id: 34, pathname: "/another", timestamp: t1),
- build(:engagement, user_id: 34, pathname: "/another", timestamp: t2, scroll_depth: 26),
+ build(:engagement,
+ user_id: 34,
+ pathname: "/another",
+ timestamp: t2,
+ scroll_depth: 26,
+ engagement_time: 20_000
+ ),
build(:pageview, user_id: 34, pathname: "/blog", timestamp: t2),
- build(:engagement, user_id: 34, pathname: "/blog", timestamp: t3, scroll_depth: 60),
+ build(:engagement,
+ user_id: 34,
+ pathname: "/blog",
+ timestamp: t3,
+ scroll_depth: 60,
+ engagement_time: 30_000
+ ),
build(:pageview, user_id: 56, pathname: "/blog", timestamp: t0),
- build(:engagement, user_id: 56, pathname: "/blog", timestamp: t1, scroll_depth: 100),
+ build(:engagement,
+ user_id: 56,
+ pathname: "/blog",
+ timestamp: t1,
+ scroll_depth: 100,
+ engagement_time: 40_000
+ ),
build(:pageview, user_id: 78, pathname: "/", timestamp: t0),
- build(:engagement, user_id: 78, pathname: "/", timestamp: t1, scroll_depth: 20),
+ build(:engagement,
+ user_id: 78,
+ pathname: "/",
+ timestamp: t1,
+ scroll_depth: 20,
+ engagement_time: 10_000
+ ),
build(:pageview, pathname: "/", timestamp: t1),
build(:pageview, pathname: "/blog", timestamp: NaiveDateTime.add(t0, 1, :day))
]
@@ -1132,7 +1175,9 @@ defmodule Plausible.Imported.CSVImporterTest do
date: i.date,
page: i.page,
total_scroll_depth: i.total_scroll_depth,
- total_scroll_depth_visits: i.total_scroll_depth_visits
+ total_scroll_depth_visits: i.total_scroll_depth_visits,
+ total_time_on_page: i.total_time_on_page,
+ total_time_on_page_visits: i.total_time_on_page_visits
}
)
|> Plausible.IngestRepo.all()
@@ -1141,28 +1186,36 @@ defmodule Plausible.Imported.CSVImporterTest do
date: expected_start_date,
page: "/",
total_scroll_depth: 20,
- total_scroll_depth_visits: 1
+ total_scroll_depth_visits: 1,
+ total_time_on_page: 10,
+ total_time_on_page_visits: 1
} in imported_data
assert %{
date: expected_start_date,
page: "/another",
total_scroll_depth: 50,
- total_scroll_depth_visits: 2
+ total_scroll_depth_visits: 2,
+ total_time_on_page: 50,
+ total_time_on_page_visits: 2
} in imported_data
assert %{
date: expected_start_date,
page: "/blog",
total_scroll_depth: 180,
- total_scroll_depth_visits: 3
+ total_scroll_depth_visits: 3,
+ total_time_on_page: 130,
+ total_time_on_page_visits: 3
} in imported_data
assert %{
date: expected_end_date,
page: "/blog",
total_scroll_depth: 0,
- total_scroll_depth_visits: 0
+ total_scroll_depth_visits: 0,
+ total_time_on_page: 0,
+ total_time_on_page_visits: 0
} in imported_data
# assert via stats queries that scroll_depth from imported
@@ -1195,7 +1248,7 @@ defmodule Plausible.Imported.CSVImporterTest do
context =
on_ee do
assert {:ok, _job} =
- Plausible.Exports.schedule_s3_export(exported_site.id, nil, user.email)
+ Plausible.Exports.schedule_s3_export(exported_site.id, user.email)
context
else
diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_time_on_page_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_time_on_page_test.exs
index cd5f568f74..0ebbea846a 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/query_time_on_page_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/query_time_on_page_test.exs
@@ -3,6 +3,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
setup [:create_user, :create_site, :create_api_key, :use_api_key, :create_site_import]
+ setup %{site: site} = context do
+ FunWithFlags.enable(:new_time_on_page, for_actor: site)
+
+ context
+ end
+
test "aggregated time_on_page metric based on engagement data", %{
conn: conn,
site: site
@@ -162,7 +168,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
} = json_response(conn, 200)
end
- describe "include.legacy_time_on_page_cutoff" do
+ describe "site.legacy_time_on_page_cutoff" do
setup %{site: site, site_import: site_import} = context do
populate_stats(site, site_import.id, [
build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),
@@ -223,6 +229,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
conn: conn,
site: site
} do
+ site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[1970-01-01])
+
conn =
post(conn, "/api/v2/query-internal-test", %{
"site_id" => site.domain,
@@ -230,7 +238,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
"date_range" => "all",
"dimensions" => ["event:page"],
"include" => %{
- "legacy_time_on_page_cutoff" => "1970-01-01T00:00:00Z",
"imports" => true
}
})
@@ -250,6 +257,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
end
test "breakdown with cutoff being after data (legacy query used)", %{conn: conn, site: site} do
+ site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[2021-01-05])
+
conn =
post(conn, "/api/v2/query-internal-test", %{
"site_id" => site.domain,
@@ -257,7 +266,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
"date_range" => "all",
"dimensions" => ["event:page"],
"include" => %{
- "legacy_time_on_page_cutoff" => "2021-01-05T00:00:00Z",
"imports" => true
}
})
@@ -283,6 +291,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
end
test "breakdown with cutoff being mid-data (two queries joined)", %{conn: conn, site: site} do
+ site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[2021-01-02])
+
conn =
post(conn, "/api/v2/query-internal-test", %{
"site_id" => site.domain,
@@ -290,7 +300,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
"date_range" => "all",
"dimensions" => ["event:page"],
"include" => %{
- "legacy_time_on_page_cutoff" => "2021-01-02T00:00:00Z",
"imports" => true
}
})
@@ -319,15 +328,14 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
conn: conn,
site: site
} do
+ site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[2021-01-02])
+
conn =
post(conn, "/api/v2/query-internal-test", %{
"site_id" => site.domain,
"metrics" => ["time_on_page"],
"date_range" => "all",
- "dimensions" => ["event:page"],
- "include" => %{
- "legacy_time_on_page_cutoff" => "2021-01-02T00:00:00Z"
- }
+ "dimensions" => ["event:page"]
})
assert_matches %{
@@ -350,6 +358,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
end
test "aggregation with cutoff being after data (legacy query used)", %{conn: conn, site: site} do
+ site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[2021-01-05])
+
conn =
post(conn, "/api/v2/query-internal-test", %{
"site_id" => site.domain,
@@ -357,7 +367,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
"date_range" => "all",
"filters" => [["is", "event:page", ["/blog"]]],
"include" => %{
- "legacy_time_on_page_cutoff" => "2021-01-05T00:00:00Z",
"imports" => true
}
})
@@ -381,6 +390,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
end
test "aggregation with cutoff being mid-data (two queries joined)", %{conn: conn, site: site} do
+ site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[2021-01-02])
+
conn =
post(conn, "/api/v2/query-internal-test", %{
"site_id" => site.domain,
@@ -388,7 +399,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
"date_range" => "all",
"filters" => [["is", "event:page", ["/blog"]]],
"include" => %{
- "legacy_time_on_page_cutoff" => "2021-01-02T00:00:00Z",
"imports" => true
}
})
@@ -415,15 +425,14 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
conn: conn,
site: site
} do
+ site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[2021-01-02])
+
conn =
post(conn, "/api/v2/query-internal-test", %{
"site_id" => site.domain,
"metrics" => ["time_on_page"],
"date_range" => "all",
- "filters" => [["is", "event:page", ["/blog"]]],
- "include" => %{
- "legacy_time_on_page_cutoff" => "2021-01-02T00:00:00Z"
- }
+ "filters" => [["is", "event:page", ["/blog"]]]
})
assert_matches %{
@@ -446,6 +455,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
describe "legacy time_on_page metric" do
test "aggregated", %{conn: conn, site: site, site_import: site_import} do
+ site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[2100-01-01])
+
populate_stats(site, site_import.id, [
build(:pageview, pathname: "/A", user_id: 111, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, pathname: "/B", user_id: 111, timestamp: ~N[2021-01-01 00:01:00]),
@@ -463,8 +474,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
"date_range" => "all",
"filters" => [["is", "event:page", ["/A"]]],
"include" => %{
- "imports" => true,
- "legacy_time_on_page_cutoff" => "2100-01-01T00:00:00Z"
+ "imports" => true
}
})
@@ -474,6 +484,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
end
test "breakdown", %{conn: conn, site: site, site_import: site_import} do
+ site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[2100-01-01])
+
populate_stats(site, site_import.id, [
build(:pageview, pathname: "/A", user_id: 111, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, pathname: "/B", user_id: 111, timestamp: ~N[2021-01-01 00:01:00]),
@@ -491,8 +503,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
"date_range" => "all",
"dimensions" => ["event:page"],
"include" => %{
- "imports" => true,
- "legacy_time_on_page_cutoff" => "2100-01-01T00:00:00Z"
+ "imports" => true
}
})
@@ -566,14 +577,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
end
test "reports legacy time-on-page as nulls per day", %{conn: conn, site: site} do
+ site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[2100-01-01])
+
conn =
post(conn, "/api/v2/query-internal-test", %{
"site_id" => site.domain,
"metrics" => ["time_on_page"],
"date_range" => ["2021-01-01", "2021-01-04"],
"filters" => [["is", "event:page", ["/"]]],
- "dimensions" => ["time:day"],
- "include" => %{"legacy_time_on_page_cutoff" => "2100-01-01T00:00:00Z"}
+ "dimensions" => ["time:day"]
})
assert json_response(conn, 200)["results"] == [
@@ -585,14 +597,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
end
test "respects `legacy_time_on_page_cutoff`", %{conn: conn, site: site} do
+ site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[2021-01-03])
+
conn =
post(conn, "/api/v2/query-internal-test", %{
"site_id" => site.domain,
"metrics" => ["time_on_page"],
"date_range" => ["2021-01-01", "2021-01-04"],
"filters" => [["is", "event:page", ["/"]]],
- "dimensions" => ["time:day"],
- "include" => %{"legacy_time_on_page_cutoff" => "2021-01-02T23:59:59Z"}
+ "dimensions" => ["time:day"]
})
assert json_response(conn, 200)["results"] == [
@@ -607,6 +620,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
conn: conn,
site: site
} do
+ site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[2021-01-02])
+
conn =
post(conn, "/api/v2/query-internal-test", %{
"site_id" => site.domain,
@@ -615,7 +630,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do
"filters" => [["is", "event:page", ["/"]]],
"dimensions" => ["time:day"],
"include" => %{
- "legacy_time_on_page_cutoff" => "2021-01-01T23:59:59Z",
"comparisons" => %{"mode" => "previous_period"}
}
})