Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
time-on-page: csv import/export support, preparation for release (#5274)
Karl-Aksel Puulmann0fcbcc3a8c25
diff --git a/assets/js/dashboard/api.ts b/assets/js/dashboard/api.tsindex 0f183b2580..fa407178e4 100644--- a/assets/js/dashboard/api.ts+++ b/assets/js/dashboard/api.ts@@ -72,25 +72,11 @@ export function queryToSearchParams(queryObj.auth = sharedLinkParams.auth}- if (- query.legacy_time_on_page_cutoff &&- validDate(query.legacy_time_on_page_cutoff)- ) {- queryObj.include = JSON.stringify({- legacy_time_on_page_cutoff: query.legacy_time_on_page_cutoff- })- }-Object.assign(queryObj, ...extraQuery)return serializeUrlParams(queryObj)}-function validDate(dateString: string) {- const date = new Date(dateString)- return isFinite(date.getTime())-}-function getHeaders(): Record<string, string> {return SHARED_LINK_AUTH ? { 'X-Shared-Link-Auth': SHARED_LINK_AUTH } : {}}diff --git a/assets/js/dashboard/query-context.tsx b/assets/js/dashboard/query-context.tsxindex 27144258db..6f9d869106 100644--- a/assets/js/dashboard/query-context.tsx+++ b/assets/js/dashboard/query-context.tsx@@ -61,7 +61,6 @@ export default function QueryContextProvider({period,to,with_imported,- legacy_time_on_page_cutoff,...otherSearch} = useMemo(() => parseSearch(location.search), [location.search])@@ -112,10 +111,7 @@ export default function QueryContextProvider({: defaultValues.with_imported,filters,resolvedFilters,- labels: (labels as FilterClauseLabels) || defaultValues.labels,- legacy_time_on_page_cutoff: site.flags.new_time_on_page- ? (legacy_time_on_page_cutoff as string) || site.legacyTimeOnPageCutoff- : defaultValues.legacy_time_on_page_cutoff+ labels: (labels as FilterClauseLabels) || defaultValues.labels}}, [compare_from,@@ -129,7 +125,6 @@ export default function QueryContextProvider({period,to,with_imported,- legacy_time_on_page_cutoff,site,expandedSegment,segmentsContext.segmentsdiff --git a/assets/js/dashboard/query.ts b/assets/js/dashboard/query.tsindex ff4fc7018f..5b2d2b71b8 100644--- a/assets/js/dashboard/query.ts+++ b/assets/js/dashboard/query.ts@@ -50,7 +50,6 @@ export type DashboardQuery = {resolvedFilters: Filter[]labels: FilterClauseLabelswith_imported: boolean- legacy_time_on_page_cutoff: string | undefined}export const queryDefaultValue: DashboardQuery = {@@ -65,8 +64,7 @@ export const queryDefaultValue: DashboardQuery = {filters: [],resolvedFilters: [],labels: {},- with_imported: true,- legacy_time_on_page_cutoff: undefined+ with_imported: true}export type BreakdownResultMeta = {diff --git a/assets/js/dashboard/site-context.test.tsx b/assets/js/dashboard/site-context.test.tsxindex d82cfe7128..a73d846cba 100644--- a/assets/js/dashboard/site-context.test.tsx+++ b/assets/js/dashboard/site-context.test.tsx@@ -67,8 +67,7 @@ describe('parseSiteFromDataset', () => {realtime: ['minute'],year: ['day', 'week', 'month']},- shared: false,- legacyTimeOnPageCutoff: '2022-01-01T00:00:00Z'+ shared: false}it('parses from dom string map correctly', () => {diff --git a/assets/js/dashboard/site-context.tsx b/assets/js/dashboard/site-context.tsxindex 56c97dc6dd..2e8d4cd804 100644--- a/assets/js/dashboard/site-context.tsx+++ b/assets/js/dashboard/site-context.tsx@@ -12,7 +12,6 @@ export function parseSiteFromDataset(dataset: DOMStringMap): PlausibleSite {conversionsOptedOut: dataset.conversionsOptedOut === 'true',funnelsOptedOut: dataset.funnelsOptedOut === 'true',propsOptedOut: dataset.propsOptedOut === 'true',- legacyTimeOnPageCutoff: dataset.legacyTimeOnPageCutoff,revenueGoals: JSON.parse(dataset.revenueGoals!),funnels: JSON.parse(dataset.funnels!),statsBegin: dataset.statsBegin!,@@ -53,8 +52,7 @@ const siteContextDefaultValue = {isDbip: false,flags: {} as FeatureFlags,validIntervalsByPeriod: {} as Record<string, Array<string>>,- shared: false,- legacyTimeOnPageCutoff: undefined as string | undefined+ shared: false}export type PlausibleSite = typeof siteContextDefaultValuediff --git a/assets/js/types/query-api.d.ts b/assets/js/types/query-api.d.tsindex 0c274ede0b..e56b773c42 100644--- a/assets/js/types/query-api.d.ts+++ b/assets/js/types/query-api.d.ts@@ -208,7 +208,6 @@ export interface QueryApiSchema {*/date_range: [string, string];};- legacy_time_on_page_cutoff?: string;};pagination?: {/**diff --git a/assets/test-utils/app-context-providers.tsx b/assets/test-utils/app-context-providers.tsxindex 6a3d86b3a9..cb73c60726 100644--- a/assets/test-utils/app-context-providers.tsx+++ b/assets/test-utils/app-context-providers.tsx@@ -49,8 +49,7 @@ export const TestContextProviders = ({isDbip: false,flags: {},validIntervalsByPeriod: {},- shared: false,- legacyTimeOnPageCutoff: undefined+ shared: false}const site = { ...defaultSite, ...siteOptions }diff --git a/lib/plausible/exports.ex b/lib/plausible/exports.exindex 91657cd6d5..0cd6eebf5f 100644--- a/lib/plausible/exports.ex+++ b/lib/plausible/exports.ex@@ -8,15 +8,14 @@ defmodule Plausible.Exports doimport Ecto.Query@doc "Schedules CSV export job to S3 storage"- @spec schedule_s3_export(pos_integer, pos_integer | nil, String.t()) ::+ @spec schedule_s3_export(pos_integer, String.t()) ::{:ok, Oban.Job.t()} | {:error, :no_data}- def schedule_s3_export(site_id, current_user_id, email_to) do+ def schedule_s3_export(site_id, email_to) dowith :ok <- ensure_has_data(site_id) doargs = %{"storage" => "s3","site_id" => site_id,"email_to" => email_to,- "current_user_id" => current_user_id,"s3_bucket" => Plausible.S3.exports_bucket(),"s3_path" => s3_export_key(site_id)}@@ -209,13 +208,13 @@ defmodule Plausible.Exports doBuilds Ecto queries to export data from `events_v2` and `sessions_v2`tables into the format of `imported_*` tables for a website."""- @spec export_queries(pos_integer, pos_integer | nil,+ @spec export_queries(pos_integer,extname: String.t(),date_range: Date.Range.t(),timezone: String.t()) ::%{String.t() => Ecto.Query.t()}- def export_queries(site_id, current_user_id, opts \\ []) do+ def export_queries(site_id, opts \\ []) doextname = opts[:extname] || ".csv"date_range = opts[:date_range]timezone = opts[:timezone] || "UTC"@@ -234,8 +233,7 @@ defmodule Plausible.Exports do%{filename.("imported_visitors") => export_visitors_q(site_id, timezone, date_range),filename.("imported_sources") => export_sources_q(site_id, timezone, date_range),- filename.("imported_pages") =>- export_pages_q(site_id, current_user_id, timezone, date_range),+ filename.("imported_pages") => export_pages_q(site_id, timezone, date_range),filename.("imported_entry_pages") => export_entry_pages_q(site_id, timezone, date_range),filename.("imported_exit_pages") => export_exit_pages_q(site_id, timezone, date_range),filename.("imported_custom_events") =>@@ -414,9 +412,7 @@ defmodule Plausible.Exports do]end- defp export_pages_q(site_id, current_user_id, timezone, date_range) do- _will_be_used_for_time_on_page_flag = current_user_id-+ defp export_pages_q(site_id, timezone, date_range) dobase_q =from(e in sampled("events_v2"),where: ^export_filter(site_id, date_range),@@ -452,20 +448,64 @@ defmodule Plausible.Exports dofrom(e in base_q,left_join: s in subquery(scroll_depth_q),on: s.date == selected_as(:date) and s.page == selected_as(:page),- select: [- date(e.timestamp, ^timezone),- selected_as(fragment("any(?)", e.hostname), :hostname),- selected_as(e.pathname, :page),- selected_as(- scale_sample(fragment("uniq(?)", e.session_id)),- :visits- ),- visitors(e),- selected_as(scale_sample(fragment("count()")), :pageviews),- selected_as(fragment("any(?)", s.total_scroll_depth), :total_scroll_depth),- selected_as(fragment("any(?)", s.total_scroll_depth_visits), :total_scroll_depth_visits)- ]+ select: %{+ date: date(e.timestamp, ^timezone),+ hostname: selected_as(fragment("any(?)", e.hostname), :hostname),+ page: selected_as(e.pathname, :page),+ visits:+ selected_as(+ scale_sample(fragment("uniq(?)", e.session_id)),+ :visits+ ),+ visitors: visitors(e),+ pageviews: selected_as(scale_sample(fragment("count()")), :pageviews),+ total_scroll_depth:+ selected_as(fragment("any(?)", s.total_scroll_depth), :total_scroll_depth),+ total_scroll_depth_visits:+ selected_as(fragment("any(?)", s.total_scroll_depth_visits), :total_scroll_depth_visits)+ })+ |> add_time_on_page_columns(site_id, timezone, date_range)+ end++ defp add_time_on_page_columns(q, site_id, timezone, date_range) do+ site = Plausible.Repo.get(Plausible.Site, site_id)++ if Plausible.Stats.TimeOnPage.new_time_on_page_visible?(site) do+ cutoff = Plausible.Stats.TimeOnPage.legacy_time_on_page_cutoff(site)++ engagements_q =+ from(e in sampled("events_v2"),+ where: ^export_filter(site_id, date_range),+ where: e.name == "engagement",+ group_by: [selected_as(:date), selected_as(:page)],+ order_by: selected_as(:date),+ select: %{+ date: date(e.timestamp, ^timezone),+ page: selected_as(e.pathname, :page),+ total_time_on_page:+ fragment(+ "toUInt64(round(sumIf(?, ? >= ?) / 1000))",+ e.engagement_time,+ e.timestamp,+ ^cutoff+ ),+ total_time_on_page_visits:+ fragment("uniqIf(?, ? >= ?)", e.session_id, e.timestamp, ^cutoff)+ }+ )++ q+ |> join(:left, [], s in subquery(engagements_q),+ on: s.date == selected_as(:date) and s.page == selected_as(:page)+ )+ |> select_merge_as([..., s], %{+ total_time_on_page: fragment("any(?)", s.total_time_on_page),+ total_time_on_page_visits: fragment("any(?)", s.total_time_on_page_visits)+ })+ else+ q+ endenddefp export_entry_pages_q(site_id, timezone, date_range) dodiff --git a/lib/plausible/sites.ex b/lib/plausible/sites.exindex e4777a5619..a2349bcb92 100644--- a/lib/plausible/sites.ex+++ b/lib/plausible/sites.ex@@ -352,6 +352,13 @@ defmodule Plausible.Sites do|> Repo.update!()end+ def update_legacy_time_on_page_cutoff!(site, cutoff) do+ site+ |> Ecto.Changeset.change()+ |> Ecto.Changeset.put_change(:legacy_time_on_page_cutoff, cutoff)+ |> Repo.update!()+ end+def has_goals?(site) doRepo.exists?(from(g in Plausible.Goal,diff --git a/lib/plausible/stats/legacy/legacy_query_builder.ex b/lib/plausible/stats/legacy/legacy_query_builder.exindex eba5dc6c3c..9ee542650f 100644--- a/lib/plausible/stats/legacy/legacy_query_builder.ex+++ b/lib/plausible/stats/legacy/legacy_query_builder.ex@@ -30,6 +30,7 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do|> put_include(site, params)|> Query.put_comparison_utc_time_range()|> Query.put_imported_opts(site)+ |> Query.set_time_on_page_data(site)on_ee doquery = Plausible.Stats.Sampling.put_threshold(query, site, params)diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.exindex 2bbe5e406c..a560b9bbdf 100644--- a/lib/plausible/stats/query.ex+++ b/lib/plausible/stats/query.ex@@ -28,12 +28,7 @@ defmodule Plausible.Stats.Query dosite_id: nil,site_native_stats_start_at: nil,# Contains information to determine how to combine legacy and new time on page metrics- # Old stats API and dashboard uses legacy metric by default, new API uses new metric by default- time_on_page_data: %{- include_new_metric: false,- include_legacy_metric: true,- cutoff: ~U[2100-01-01 00:00:00Z]- }+ time_on_page_data: %{}require OpenTelemetry.Tracer, as: Traceralias Plausible.Stats.{DateTimeRange, Filters, Imported, Legacy, Comparisons}@@ -50,6 +45,7 @@ defmodule Plausible.Stats.Query dosite_native_stats_start_at: site.native_stats_start_at}|> struct!(Map.to_list(query_data))+ |> set_time_on_page_data(site)|> put_comparison_utc_time_range()|> put_imported_opts(site)@@ -180,6 +176,15 @@ defmodule Plausible.Stats.Query doin_comparison_range ++ in_rangeend+ def set_time_on_page_data(query, site) do+ struct!(query,+ time_on_page_data: %{+ new_metric_visible: Plausible.Stats.TimeOnPage.new_time_on_page_visible?(site),+ cutoff_date: site.legacy_time_on_page_cutoff+ }+ )+ end+@spec get_skip_imported_reason(t()) ::nil | :no_imported_data | :out_of_range | :unsupported_querydef get_skip_imported_reason(query) dodiff --git a/lib/plausible/stats/query_optimizer.ex b/lib/plausible/stats/query_optimizer.exindex 6e590d866d..c9f7a7c82c 100644--- a/lib/plausible/stats/query_optimizer.ex+++ b/lib/plausible/stats/query_optimizer.ex@@ -4,7 +4,17 @@ defmodule Plausible.Stats.QueryOptimizer do"""use Plausible- alias Plausible.Stats.{DateTimeRange, Filters, Query, TableDecider, Util, Time, Legacy}++ alias Plausible.Stats.{+ DateTimeRange,+ Filters,+ Query,+ TableDecider,+ TimeOnPage,+ Util,+ Time,+ Legacy+ }@doc """This module manipulates an existing query, updating it according to business logic.@@ -188,28 +198,37 @@ defmodule Plausible.Stats.QueryOptimizer doenddefp set_time_on_page_data(query) do- if :time_on_page in query.metrics and query.include.legacy_time_on_page_cutoff do- {:ok, cutoff, _} =- query.include.legacy_time_on_page_cutoff- |> DateTime.from_iso8601()-- cutoff =- cutoff- |> DateTime.shift_zone!("Etc/UTC")- |> DateTime.truncate(:second)-- Query.set(- query,- time_on_page_data: %{- include_new_metric: DateTime.before?(cutoff, query.utc_time_range.last),- include_legacy_metric:- DateTime.after?(cutoff, query.utc_time_range.first) and- Legacy.TimeOnPage.can_merge_legacy_time_on_page?(query),- cutoff: if(DateTime.after?(cutoff, query.utc_time_range.first), do: cutoff, else: nil)- }- )- else- query+ case {:time_on_page in query.metrics, query.time_on_page_data} do+ {true, %{new_metric_visible: true, cutoff_date: cutoff_date}} ->+ cutoff =+ cutoff_date+ |> TimeOnPage.cutoff_datetime(query.timezone)+ |> DateTime.shift_zone!("Etc/UTC")+ |> DateTime.truncate(:second)++ Query.set(+ query,+ time_on_page_data:+ Map.merge(query.time_on_page_data, %{+ include_new_metric: DateTime.before?(cutoff, query.utc_time_range.last),+ include_legacy_metric:+ DateTime.after?(cutoff, query.utc_time_range.first) and+ Legacy.TimeOnPage.can_merge_legacy_time_on_page?(query),+ cutoff:+ if(DateTime.after?(cutoff, query.utc_time_range.first), do: cutoff, else: nil)+ })+ )++ _ ->+ Query.set(+ query,+ time_on_page_data:+ Map.merge(query.time_on_page_data, %{+ include_new_metric: false,+ include_legacy_metric: true,+ cutoff: nil+ })+ )endendenddiff --git a/lib/plausible/stats/query_result.ex b/lib/plausible/stats/query_result.exindex 75c8bed891..47b2460ff7 100644--- a/lib/plausible/stats/query_result.ex+++ b/lib/plausible/stats/query_result.ex@@ -158,7 +158,7 @@ defmodule Plausible.Stats.QueryResult dodefp metric_warning(:time_on_page, %Query{} = query) docase query.time_on_page_data do- %{include_legacy_metric: true, cutoff: cutoff} ->+ %{new_metric_visible: true, include_legacy_metric: true, cutoff: cutoff} ->cutoff_date =cutoff |> DateTime.shift_zone!(query.timezone) |> Calendar.strftime("%Y-%m-%d")diff --git a/lib/plausible/stats/time_on_page.ex b/lib/plausible/stats/time_on_page.exindex d5305ced8e..c6d9c1822e 100644--- a/lib/plausible/stats/time_on_page.ex+++ b/lib/plausible/stats/time_on_page.ex@@ -3,23 +3,29 @@ defmodule Plausible.Stats.TimeOnPage doModule to check whether the new time on page metric is available."""- def new_time_on_page_enabled?(site, user) do- FunWithFlags.enabled?(:new_time_on_page, for: user) ||- FunWithFlags.enabled?(:new_time_on_page, for: site)+ def new_time_on_page_visible?(site) do+ new_time_on_page_enabled?(site) && not is_nil(site.legacy_time_on_page_cutoff)enddef legacy_time_on_page_cutoff(site) do- cutoff(site.legacy_time_on_page_cutoff, site.timezone)+ cutoff_datetime(site.legacy_time_on_page_cutoff, site.timezone)end- defp cutoff(nil, _timezone), do: ""+ def cutoff_datetime(nil, _timezone), do: nil- defp cutoff(date, timezone) do+ # Workaround for a case where casting unix epoch to DateTime fails for sites that should+ # always have the new time-on-page only. Affects CSV imports.+ def cutoff_datetime(~D[1970-01-01], _timezone), do: ~U[2000-01-01 00:00:00Z]++ def cutoff_datetime(date, timezone) docase DateTime.new(date, ~T[00:00:00], timezone) do{:ok, datetime} -> datetime{:gap, just_before, _just_after} -> just_before{:ambiguous, first_datetime, _second_datetime} -> first_datetimeend- |> DateTime.to_iso8601()+ end++ defp new_time_on_page_enabled?(site) do+ FunWithFlags.enabled?(:new_time_on_page, for: site)endenddiff --git a/lib/plausible_web/controllers/api/external_query_api_controller.ex b/lib/plausible_web/controllers/api/external_query_api_controller.exindex 417f42a643..7fc17e0e64 100644--- a/lib/plausible_web/controllers/api/external_query_api_controller.ex+++ b/lib/plausible_web/controllers/api/external_query_api_controller.ex@@ -11,8 +11,6 @@ defmodule PlausibleWeb.Api.ExternalQueryApiController docase Query.build(site, conn.assigns.schema_type, params, debug_metadata(conn)) do{:ok, query} ->- query = update_time_on_page_query_data(query)-results = Plausible.Stats.query(site, query)json(conn, results)@@ -26,18 +24,4 @@ defmodule PlausibleWeb.Api.ExternalQueryApiController dodef schema(conn, _params) dojson(conn, Plausible.Stats.JSONSchema.raw_public_schema())end-- defp update_time_on_page_query_data(query) do- if is_nil(query.include.legacy_time_on_page_cutoff) do- Query.set(query,- time_on_page_data: %{- include_new_metric: true,- include_legacy_metric: false,- cutoff: nil- }- )- else- query- end- endenddiff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex 644690c4fb..8002b037bb 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -182,7 +182,6 @@ defmodule PlausibleWeb.Api.StatsController dodef top_stats(conn, params) dosite = conn.assigns[:site]- current_user = conn.assigns[:current_user]params = realtime_period_to_30m(params)@@ -196,7 +195,7 @@ defmodule PlausibleWeb.Api.StatsController dometa: meta,sample_percent: sample_percent,graphable_metrics: graphable_metrics- } = fetch_top_stats(site, query, current_user)+ } = fetch_top_stats(site, query)comparison_query = comparison_query(query)@@ -277,7 +276,7 @@ defmodule PlausibleWeb.Api.StatsController doendend- defp fetch_top_stats(site, query, current_user) do+ defp fetch_top_stats(site, query) dogoal_filter? =toplevel_goal_filter?(query)@@ -292,7 +291,7 @@ defmodule PlausibleWeb.Api.StatsController dofetch_goal_top_stats(site, query)true ->- fetch_other_top_stats(site, query, current_user)+ fetch_other_top_stats(site, query)endend@@ -393,7 +392,7 @@ defmodule PlausibleWeb.Api.StatsController do%{top_stats: top_stats, meta: meta, graphable_metrics: metrics, sample_percent: 100}end- defp fetch_other_top_stats(site, query, current_user) do+ defp fetch_other_top_stats(site, query) dopage_filter? =Filters.filtering_on_dimension?(query, "event:page", behavioral_filters: :ignore)@@ -437,7 +436,7 @@ defmodule PlausibleWeb.Api.StatsController dotop_stats: top_stats,meta: meta,graphable_metrics:- if(TimeOnPage.new_time_on_page_enabled?(site, current_user),+ if(TimeOnPage.new_time_on_page_visible?(site),do: metrics,else: metrics -- [:time_on_page]),diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.exindex 00b7273c19..9d28241dd7 100644--- a/lib/plausible_web/controllers/stats_controller.ex+++ b/lib/plausible_web/controllers/stats_controller.ex@@ -45,7 +45,7 @@ defmodule PlausibleWeb.StatsController douse Plausible.Repoalias Plausible.Sites- alias Plausible.Stats.{Filters, Query, TimeOnPage}+ alias Plausible.Stats.{Filters, Query}alias PlausibleWeb.Apiplug(PlausibleWeb.Plugs.AuthorizeSiteAccess when action in [:stats, :csv_export])@@ -77,7 +77,6 @@ defmodule PlausibleWeb.StatsController dohas_props: Plausible.Props.configured?(site),stats_start_date: stats_start_date,native_stats_start_date: NaiveDateTime.to_date(site.native_stats_start_at),- legacy_time_on_page_cutoff: TimeOnPage.legacy_time_on_page_cutoff(site),title: title(conn, site),demo: demo,flags: flags,@@ -362,7 +361,6 @@ defmodule PlausibleWeb.StatsController dohas_props: Plausible.Props.configured?(shared_link.site),stats_start_date: stats_start_date,native_stats_start_date: NaiveDateTime.to_date(shared_link.site.native_stats_start_at),- legacy_time_on_page_cutoff: TimeOnPage.legacy_time_on_page_cutoff(shared_link.site),title: title(conn, shared_link.site),demo: false,dogfood_page_path: "/share/:dashboard",diff --git a/lib/plausible_web/live/csv_export.ex b/lib/plausible_web/live/csv_export.exindex 4be300fb90..21eeb69154 100644--- a/lib/plausible_web/live/csv_export.ex+++ b/lib/plausible_web/live/csv_export.ex@@ -210,12 +210,12 @@ defmodule PlausibleWeb.Live.CSVExport do@impl truedef handle_event("export", _params, socket) do- %{storage: storage, site_id: site_id, email_to: email_to, current_user: current_user} =+ %{storage: storage, site_id: site_id, email_to: email_to} =socket.assignsschedule_result =case storage do- "s3" -> Exports.schedule_s3_export(site_id, current_user.id, email_to)+ "s3" -> Exports.schedule_s3_export(site_id, email_to)"local" -> Exports.schedule_local_export(site_id, email_to)enddiff --git a/lib/plausible_web/templates/stats/stats.html.heex b/lib/plausible_web/templates/stats/stats.html.heexindex 2454b68d63..b1537e19f3 100644--- a/lib/plausible_web/templates/stats/stats.html.heex+++ b/lib/plausible_web/templates/stats/stats.html.heex@@ -36,7 +36,6 @@data-logged-in={to_string([email protected][:current_user])}data-stats-begin={@stats_start_date}data-native-stats-begin={@native_stats_start_date}- data-legacy-time-on-page-cutoff={@legacy_time_on_page_cutoff}data-shared-link-auth={assigns[:shared_link_auth]}data-embedded={to_string(@conn.assigns[:embedded])}data-background={@conn.assigns[:background]}diff --git a/lib/workers/export_analytics.ex b/lib/workers/export_analytics.exindex d5da9f05d0..51a1c0af06 100644--- a/lib/workers/export_analytics.ex+++ b/lib/workers/export_analytics.ex@@ -29,13 +29,11 @@ defmodule Plausible.Workers.ExportAnalytics do"site_id" => site_id} = args- current_user_id = args["current_user_id"]-site = Plausible.Repo.get!(Plausible.Site, site_id)%Date.Range{} = date_range = Exports.date_range(site.id, site.timezone)queries =- Exports.export_queries(site_id, current_user_id,+ Exports.export_queries(site_id,date_range: date_range,timezone: site.timezone,extname: ".csv"diff --git a/priv/json-schemas/query-api-schema.json b/priv/json-schemas/query-api-schema.jsonindex d511d58d41..7d1403a82a 100644--- a/priv/json-schemas/query-api-schema.json+++ b/priv/json-schemas/query-api-schema.json@@ -109,11 +109,6 @@"additionalProperties": false}]- },- "legacy_time_on_page_cutoff": {- "$comment": "only :internal",- "type": "string",- "format": "datetime"}}},diff --git a/test/plausible/exports_test.exs b/test/plausible/exports_test.exsindex da28d17b3e..d6016916fc 100644--- a/test/plausible/exports_test.exs+++ b/test/plausible/exports_test.exs@@ -9,7 +9,7 @@ defmodule Plausible.ExportsTest dosetup [:create_user, :create_site]test "returns named ecto queries", %{site: site} do- queries = Plausible.Exports.export_queries(site.id, nil)+ queries = Plausible.Exports.export_queries(site.id)assert queries |> Map.values() |> Enum.all?(&match?(%Ecto.Query{}, &1))assert Map.keys(queries) == [@@ -28,7 +28,7 @@ defmodule Plausible.ExportsTest dotest "with date range", %{site: site} doqueries =- Plausible.Exports.export_queries(site.id, nil,+ Plausible.Exports.export_queries(site.id,date_range: Date.range(~D[2023-01-01], ~D[2024-03-12]))@@ -47,7 +47,7 @@ defmodule Plausible.ExportsTest doendtest "with custom extension", %{site: site} do- queries = Plausible.Exports.export_queries(site.id, nil, extname: ".ch")+ queries = Plausible.Exports.export_queries(site.id, extname: ".ch")assert Map.keys(queries) == ["imported_browsers.ch",diff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exsindex a7ccffd182..ef9f13fdc9 100644--- a/test/plausible/imported/csv_importer_test.exs+++ b/test/plausible/imported/csv_importer_test.exs@@ -1059,9 +1059,10 @@ defmodule Plausible.Imported.CSVImporterTest doend@tag :tmp_dir- test "scroll_depth", %{conn: conn, user: user, tmp_dir: tmp_dir} do+ test "scroll_depth and time_on_page", %{conn: conn, user: user, tmp_dir: tmp_dir} doexported_site = new_site(owner: user)imported_site = new_site(owner: user)+ FunWithFlags.enable(:new_time_on_page, for_actor: exported_site)t0 =NaiveDateTime.utc_now(:second)@@ -1073,19 +1074,61 @@ defmodule Plausible.Imported.CSVImporterTest dostats =[build(:pageview, user_id: 12, pathname: "/blog", timestamp: t0),- build(:engagement, user_id: 12, pathname: "/blog", timestamp: t1, scroll_depth: 20),+ build(:engagement,+ user_id: 12,+ pathname: "/blog",+ timestamp: t1,+ scroll_depth: 20,+ engagement_time: 50_000+ ),build(:pageview, user_id: 12, pathname: "/another", timestamp: t1),- build(:engagement, user_id: 12, pathname: "/another", timestamp: t2, scroll_depth: 24),+ build(:engagement,+ user_id: 12,+ pathname: "/another",+ timestamp: t2,+ scroll_depth: 24,+ engagement_time: 30_000+ ),build(:pageview, user_id: 34, pathname: "/blog", timestamp: t0),- build(:engagement, user_id: 34, pathname: "/blog", timestamp: t1, scroll_depth: 17),+ build(:engagement,+ user_id: 34,+ pathname: "/blog",+ timestamp: t1,+ scroll_depth: 17,+ engagement_time: 10_000+ ),build(:pageview, user_id: 34, pathname: "/another", timestamp: t1),- build(:engagement, user_id: 34, pathname: "/another", timestamp: t2, scroll_depth: 26),+ build(:engagement,+ user_id: 34,+ pathname: "/another",+ timestamp: t2,+ scroll_depth: 26,+ engagement_time: 20_000+ ),build(:pageview, user_id: 34, pathname: "/blog", timestamp: t2),- build(:engagement, user_id: 34, pathname: "/blog", timestamp: t3, scroll_depth: 60),+ build(:engagement,+ user_id: 34,+ pathname: "/blog",+ timestamp: t3,+ scroll_depth: 60,+ engagement_time: 30_000+ ),build(:pageview, user_id: 56, pathname: "/blog", timestamp: t0),- build(:engagement, user_id: 56, pathname: "/blog", timestamp: t1, scroll_depth: 100),+ build(:engagement,+ user_id: 56,+ pathname: "/blog",+ timestamp: t1,+ scroll_depth: 100,+ engagement_time: 40_000+ ),build(:pageview, user_id: 78, pathname: "/", timestamp: t0),- build(:engagement, user_id: 78, pathname: "/", timestamp: t1, scroll_depth: 20),+ build(:engagement,+ user_id: 78,+ pathname: "/",+ timestamp: t1,+ scroll_depth: 20,+ engagement_time: 10_000+ ),build(:pageview, pathname: "/", timestamp: t1),build(:pageview, pathname: "/blog", timestamp: NaiveDateTime.add(t0, 1, :day))]@@ -1132,7 +1175,9 @@ defmodule Plausible.Imported.CSVImporterTest dodate: i.date,page: i.page,total_scroll_depth: i.total_scroll_depth,- total_scroll_depth_visits: i.total_scroll_depth_visits+ total_scroll_depth_visits: i.total_scroll_depth_visits,+ total_time_on_page: i.total_time_on_page,+ total_time_on_page_visits: i.total_time_on_page_visits})|> Plausible.IngestRepo.all()@@ -1141,28 +1186,36 @@ defmodule Plausible.Imported.CSVImporterTest dodate: expected_start_date,page: "/",total_scroll_depth: 20,- total_scroll_depth_visits: 1+ total_scroll_depth_visits: 1,+ total_time_on_page: 10,+ total_time_on_page_visits: 1} in imported_dataassert %{date: expected_start_date,page: "/another",total_scroll_depth: 50,- total_scroll_depth_visits: 2+ total_scroll_depth_visits: 2,+ total_time_on_page: 50,+ total_time_on_page_visits: 2} in imported_dataassert %{date: expected_start_date,page: "/blog",total_scroll_depth: 180,- total_scroll_depth_visits: 3+ total_scroll_depth_visits: 3,+ total_time_on_page: 130,+ total_time_on_page_visits: 3} in imported_dataassert %{date: expected_end_date,page: "/blog",total_scroll_depth: 0,- total_scroll_depth_visits: 0+ total_scroll_depth_visits: 0,+ total_time_on_page: 0,+ total_time_on_page_visits: 0} in imported_data# assert via stats queries that scroll_depth from imported@@ -1195,7 +1248,7 @@ defmodule Plausible.Imported.CSVImporterTest docontext =on_ee doassert {:ok, _job} =- Plausible.Exports.schedule_s3_export(exported_site.id, nil, user.email)+ Plausible.Exports.schedule_s3_export(exported_site.id, user.email)contextelsediff --git a/test/plausible_web/controllers/api/external_stats_controller/query_time_on_page_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_time_on_page_test.exsindex cd5f568f74..0ebbea846a 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_time_on_page_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_time_on_page_test.exs@@ -3,6 +3,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest dosetup [:create_user, :create_site, :create_api_key, :use_api_key, :create_site_import]+ setup %{site: site} = context do+ FunWithFlags.enable(:new_time_on_page, for_actor: site)++ context+ end+test "aggregated time_on_page metric based on engagement data", %{conn: conn,site: site@@ -162,7 +168,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do} = json_response(conn, 200)end- describe "include.legacy_time_on_page_cutoff" do+ describe "site.legacy_time_on_page_cutoff" dosetup %{site: site, site_import: site_import} = context dopopulate_stats(site, site_import.id, [build(:pageview, user_id: 12, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),@@ -223,6 +229,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest doconn: conn,site: site} do+ site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[1970-01-01])+conn =post(conn, "/api/v2/query-internal-test", %{"site_id" => site.domain,@@ -230,7 +238,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do"date_range" => "all","dimensions" => ["event:page"],"include" => %{- "legacy_time_on_page_cutoff" => "1970-01-01T00:00:00Z","imports" => true}})@@ -250,6 +257,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest doendtest "breakdown with cutoff being after data (legacy query used)", %{conn: conn, site: site} do+ site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[2021-01-05])+conn =post(conn, "/api/v2/query-internal-test", %{"site_id" => site.domain,@@ -257,7 +266,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do"date_range" => "all","dimensions" => ["event:page"],"include" => %{- "legacy_time_on_page_cutoff" => "2021-01-05T00:00:00Z","imports" => true}})@@ -283,6 +291,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest doendtest "breakdown with cutoff being mid-data (two queries joined)", %{conn: conn, site: site} do+ site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[2021-01-02])+conn =post(conn, "/api/v2/query-internal-test", %{"site_id" => site.domain,@@ -290,7 +300,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do"date_range" => "all","dimensions" => ["event:page"],"include" => %{- "legacy_time_on_page_cutoff" => "2021-01-02T00:00:00Z","imports" => true}})@@ -319,15 +328,14 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest doconn: conn,site: site} do+ site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[2021-01-02])+conn =post(conn, "/api/v2/query-internal-test", %{"site_id" => site.domain,"metrics" => ["time_on_page"],"date_range" => "all",- "dimensions" => ["event:page"],- "include" => %{- "legacy_time_on_page_cutoff" => "2021-01-02T00:00:00Z"- }+ "dimensions" => ["event:page"]})assert_matches %{@@ -350,6 +358,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest doendtest "aggregation with cutoff being after data (legacy query used)", %{conn: conn, site: site} do+ site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[2021-01-05])+conn =post(conn, "/api/v2/query-internal-test", %{"site_id" => site.domain,@@ -357,7 +367,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do"date_range" => "all","filters" => [["is", "event:page", ["/blog"]]],"include" => %{- "legacy_time_on_page_cutoff" => "2021-01-05T00:00:00Z","imports" => true}})@@ -381,6 +390,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest doendtest "aggregation with cutoff being mid-data (two queries joined)", %{conn: conn, site: site} do+ site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[2021-01-02])+conn =post(conn, "/api/v2/query-internal-test", %{"site_id" => site.domain,@@ -388,7 +399,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do"date_range" => "all","filters" => [["is", "event:page", ["/blog"]]],"include" => %{- "legacy_time_on_page_cutoff" => "2021-01-02T00:00:00Z","imports" => true}})@@ -415,15 +425,14 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest doconn: conn,site: site} do+ site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[2021-01-02])+conn =post(conn, "/api/v2/query-internal-test", %{"site_id" => site.domain,"metrics" => ["time_on_page"],"date_range" => "all",- "filters" => [["is", "event:page", ["/blog"]]],- "include" => %{- "legacy_time_on_page_cutoff" => "2021-01-02T00:00:00Z"- }+ "filters" => [["is", "event:page", ["/blog"]]]})assert_matches %{@@ -446,6 +455,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest dodescribe "legacy time_on_page metric" dotest "aggregated", %{conn: conn, site: site, site_import: site_import} do+ site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[2100-01-01])+populate_stats(site, site_import.id, [build(:pageview, pathname: "/A", user_id: 111, timestamp: ~N[2021-01-01 00:00:00]),build(:pageview, pathname: "/B", user_id: 111, timestamp: ~N[2021-01-01 00:01:00]),@@ -463,8 +474,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do"date_range" => "all","filters" => [["is", "event:page", ["/A"]]],"include" => %{- "imports" => true,- "legacy_time_on_page_cutoff" => "2100-01-01T00:00:00Z"+ "imports" => true}})@@ -474,6 +484,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest doendtest "breakdown", %{conn: conn, site: site, site_import: site_import} do+ site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[2100-01-01])+populate_stats(site, site_import.id, [build(:pageview, pathname: "/A", user_id: 111, timestamp: ~N[2021-01-01 00:00:00]),build(:pageview, pathname: "/B", user_id: 111, timestamp: ~N[2021-01-01 00:01:00]),@@ -491,8 +503,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do"date_range" => "all","dimensions" => ["event:page"],"include" => %{- "imports" => true,- "legacy_time_on_page_cutoff" => "2100-01-01T00:00:00Z"+ "imports" => true}})@@ -566,14 +577,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest doendtest "reports legacy time-on-page as nulls per day", %{conn: conn, site: site} do+ site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[2100-01-01])+conn =post(conn, "/api/v2/query-internal-test", %{"site_id" => site.domain,"metrics" => ["time_on_page"],"date_range" => ["2021-01-01", "2021-01-04"],"filters" => [["is", "event:page", ["/"]]],- "dimensions" => ["time:day"],- "include" => %{"legacy_time_on_page_cutoff" => "2100-01-01T00:00:00Z"}+ "dimensions" => ["time:day"]})assert json_response(conn, 200)["results"] == [@@ -585,14 +597,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest doendtest "respects `legacy_time_on_page_cutoff`", %{conn: conn, site: site} do+ site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[2021-01-03])+conn =post(conn, "/api/v2/query-internal-test", %{"site_id" => site.domain,"metrics" => ["time_on_page"],"date_range" => ["2021-01-01", "2021-01-04"],"filters" => [["is", "event:page", ["/"]]],- "dimensions" => ["time:day"],- "include" => %{"legacy_time_on_page_cutoff" => "2021-01-02T23:59:59Z"}+ "dimensions" => ["time:day"]})assert json_response(conn, 200)["results"] == [@@ -607,6 +620,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest doconn: conn,site: site} do+ site = Plausible.Sites.update_legacy_time_on_page_cutoff!(site, ~D[2021-01-02])+conn =post(conn, "/api/v2/query-internal-test", %{"site_id" => site.domain,@@ -615,7 +630,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTimeOnPageTest do"filters" => [["is", "event:page", ["/"]]],"dimensions" => ["time:day"],"include" => %{- "legacy_time_on_page_cutoff" => "2021-01-01T23:59:59Z","comparisons" => %{"mode" => "previous_period"}}})