Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

time-on-page: `site.legacy_time_on_page` cronjob (#5215)

Karl-Aksel Puulmann42cea1d88984
diff --git a/config/runtime.exs b/config/runtime.exs
index d81477423e..e6428b43b1 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -713,7 +713,9 @@ cloud_cron = [
# Daily at 8
{"0 8 * * *", Plausible.Workers.AcceptTrafficUntil},
# First sunday of the month, 4:00 UTC
- {"0 4 1-7 * SUN", Plausible.Workers.ClickhouseCleanSites}
+ {"0 4 1-7 * SUN", Plausible.Workers.ClickhouseCleanSites},
+ # Daily at 4:00 UTC
+ {"0 4 * * *", Plausible.Workers.SetLegacyTimeOnPageCutoff}
]
crontab = if(is_selfhost, do: base_cron, else: base_cron ++ cloud_cron)
@@ -740,7 +742,8 @@ cloud_queues = [
trial_notification_emails: 1,
check_usage: 1,
notify_annual_renewal: 1,
- lock_sites: 1
+ lock_sites: 1,
+ legacy_time_on_page_cutoff: 1
]
queues = if(is_selfhost, do: base_queues, else: base_queues ++ cloud_queues)
diff --git a/lib/plausible/site.ex b/lib/plausible/site.ex
index e3cfa0e7af..375a9d773a 100644
--- a/lib/plausible/site.ex
+++ b/lib/plausible/site.ex
@@ -21,6 +21,7 @@ defmodule Plausible.Site do
field :conversions_enabled, :boolean, default: true
field :props_enabled, :boolean, default: true
field :funnels_enabled, :boolean, default: true
+ field :legacy_time_on_page_cutoff, :date, default: ~D[1970-01-01]
field :ingest_rate_limit_scale_seconds, :integer, default: 60
# default is set via changeset/2
@@ -87,7 +88,7 @@ defmodule Plausible.Site do
def changeset(site, attrs \\ %{}) do
site
- |> cast(attrs, [:domain, :timezone])
+ |> cast(attrs, [:domain, :timezone, :legacy_time_on_page_cutoff])
|> clean_domain()
|> validate_required([:domain, :timezone])
|> validate_timezone()
diff --git a/lib/plausible/stats/time_on_page.ex b/lib/plausible/stats/time_on_page.ex
index 38ca7a983a..d5305ced8e 100644
--- a/lib/plausible/stats/time_on_page.ex
+++ b/lib/plausible/stats/time_on_page.ex
@@ -8,9 +8,18 @@ defmodule Plausible.Stats.TimeOnPage do
FunWithFlags.enabled?(:new_time_on_page, for: site)
end
- def legacy_time_on_page_cutoff() do
- # Placeholder until we implement a more sophisticated way to determine the cutoff
- # Only used when `new_time_on_page` flag is enabled
- DateTime.utc_now() |> DateTime.shift(day: -4) |> DateTime.to_iso8601()
+ def legacy_time_on_page_cutoff(site) do
+ cutoff(site.legacy_time_on_page_cutoff, site.timezone)
+ end
+
+ defp cutoff(nil, _timezone), do: ""
+
+ defp cutoff(date, timezone) do
+ case DateTime.new(date, ~T[00:00:00], timezone) do
+ {:ok, datetime} -> datetime
+ {:gap, just_before, _just_after} -> just_before
+ {:ambiguous, first_datetime, _second_datetime} -> first_datetime
+ end
+ |> DateTime.to_iso8601()
end
end
diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex
index 4e671f8784..00b7273c19 100644
--- a/lib/plausible_web/controllers/stats_controller.ex
+++ b/lib/plausible_web/controllers/stats_controller.ex
@@ -77,7 +77,7 @@ defmodule PlausibleWeb.StatsController do
has_props: Plausible.Props.configured?(site),
stats_start_date: stats_start_date,
native_stats_start_date: NaiveDateTime.to_date(site.native_stats_start_at),
- legacy_time_on_page_cutoff: TimeOnPage.legacy_time_on_page_cutoff(),
+ legacy_time_on_page_cutoff: TimeOnPage.legacy_time_on_page_cutoff(site),
title: title(conn, site),
demo: demo,
flags: flags,
@@ -362,7 +362,7 @@ defmodule PlausibleWeb.StatsController do
has_props: Plausible.Props.configured?(shared_link.site),
stats_start_date: stats_start_date,
native_stats_start_date: NaiveDateTime.to_date(shared_link.site.native_stats_start_at),
- legacy_time_on_page_cutoff: TimeOnPage.legacy_time_on_page_cutoff(),
+ legacy_time_on_page_cutoff: TimeOnPage.legacy_time_on_page_cutoff(shared_link.site),
title: title(conn, shared_link.site),
demo: false,
dogfood_page_path: "/share/:dashboard",
diff --git a/lib/workers/set_legacy_time_on_page_cutoff.ex b/lib/workers/set_legacy_time_on_page_cutoff.ex
new file mode 100644
index 0000000000..b188df2c50
--- /dev/null
+++ b/lib/workers/set_legacy_time_on_page_cutoff.ex
@@ -0,0 +1,94 @@
+defmodule Plausible.Workers.SetLegacyTimeOnPageCutoff do
+ @moduledoc """
+ Sets sites `legacy_time_on_page_cutoff` depending on whether they have
+ sent us engagement data in the past.
+ """
+
+ use Plausible.Repo
+ use Plausible.ClickhouseRepo
+ use Oban.Worker, queue: :legacy_time_on_page_cutoff
+
+ import Ecto.Query
+
+ require Logger
+
+ @impl Oban.Worker
+ def perform(%Oban.Job{args: args}) do
+ cutoff_date = Map.get(args, "cutoff_date", Date.utc_today())
+
+ {small_sites, large_sites} =
+ sites_with_engagement_data(cutoff_date)
+ |> filter_sites_needing_update()
+
+ to_update = small_sites ++ large_sites
+
+ if length(to_update) > 0 do
+ Logger.info(
+ "Setting legacy_time_on_page_cutoff for #{length(to_update)} sites (#{length(small_sites)} small, #{length(large_sites)} large)"
+ )
+
+ {count, _} =
+ Repo.update_all(
+ from(s in Plausible.Site,
+ where: s.id in ^to_update,
+ where: is_nil(s.legacy_time_on_page_cutoff)
+ ),
+ set: [legacy_time_on_page_cutoff: cutoff_date]
+ )
+
+ Logger.info("Successfully set legacy_time_on_page_cutoff=#{cutoff_date} for #{count} sites")
+ else
+ Logger.debug("No sites legacy_time_on_page_cutoff needs updating")
+ end
+
+ :ok
+ end
+
+ defp sites_with_engagement_data(cutoff_date) do
+ site_info_q =
+ from(
+ e in "events_v2",
+ where: e.timestamp >= fragment("toStartOfHour(? - toIntervalHour(48))", ^cutoff_date),
+ where: e.timestamp < fragment("toStartOfHour(? - toIntervalHour(24))", ^cutoff_date),
+ group_by: e.site_id,
+ select: %{
+ site_id: e.site_id,
+ hours_with_engagement:
+ fragment(
+ "uniqIf(toStartOfHour(timestamp), name = 'engagement' AND engagement_time > 0)"
+ ),
+ is_small_site: fragment("count() < 2000")
+ }
+ )
+
+ q =
+ from(
+ s in subquery(site_info_q),
+ select: %{
+ small_sites:
+ fragment("groupArrayIf(site_id, (is_small_site and hours_with_engagement > 0))"),
+ large_sites:
+ fragment("groupArrayIf(site_id, (not is_small_site and hours_with_engagement = 24))")
+ }
+ )
+
+ result = ClickhouseRepo.one(q)
+
+ {MapSet.new(result.small_sites), MapSet.new(result.large_sites)}
+ end
+
+ defp filter_sites_needing_update({small_sites, large_sites}) do
+ needing_update =
+ from(s in Plausible.Site,
+ where: is_nil(s.legacy_time_on_page_cutoff),
+ select: s.id
+ )
+ |> Repo.all()
+ |> MapSet.new()
+
+ {
+ small_sites |> MapSet.intersection(needing_update) |> MapSet.to_list(),
+ large_sites |> MapSet.intersection(needing_update) |> MapSet.to_list()
+ }
+ end
+end
diff --git a/test/plausible/site/sites_test.exs b/test/plausible/site/sites_test.exs
index 98b70a7d2e..4485ad411a 100644
--- a/test/plausible/site/sites_test.exs
+++ b/test/plausible/site/sites_test.exs
@@ -14,6 +14,15 @@ defmodule Plausible.SitesTest do
Sites.create(user, params)
end
+ test "creating a site sets `legacy_time_on_page_cutoff`" do
+ user = new_user()
+
+ params = %{"domain" => "example.com", "timezone" => "Europe/London"}
+
+ assert {:ok, %{site: %{legacy_time_on_page_cutoff: ~D[1970-01-01]}}} =
+ Sites.create(user, params)
+ end
+
test "does not start a trial for pre-teams guest users without trial expiry date" do
user = new_user() |> subscribe_to_growth_plan()
new_site(owner: user)
diff --git a/test/workers/set_legacy_time_on_page_cutoff_test.exs b/test/workers/set_legacy_time_on_page_cutoff_test.exs
new file mode 100644
index 0000000000..4b1689600e
--- /dev/null
+++ b/test/workers/set_legacy_time_on_page_cutoff_test.exs
@@ -0,0 +1,105 @@
+defmodule Plausible.Workers.SetLegacyTimeOnPageCutoffTest do
+ use Plausible.DataCase
+ use Plausible.TestUtils
+ use Plausible
+
+ alias Plausible.Workers.SetLegacyTimeOnPageCutoff
+
+ test "sets cutoff for small site with engagement" do
+ site = create_site_with_cutoff(nil)
+
+ populate_stats(site, [
+ build(:pageview, user_id: 13, timestamp: ~U[2025-03-03 23:05:00Z]),
+ build(:engagement, user_id: 13, engagement_time: 10, timestamp: ~U[2025-03-03 23:05:00Z])
+ ])
+
+ perform(~D[2025-03-05])
+ assert Repo.reload!(site).legacy_time_on_page_cutoff == ~D[2025-03-05]
+ end
+
+ test "does not update an already set cutoff" do
+ site = create_site_with_cutoff(~D[1970-01-01])
+
+ populate_stats(site, [
+ build(:pageview, user_id: 13, timestamp: ~U[2025-03-03 23:05:00Z]),
+ build(:engagement, user_id: 13, engagement_time: 10, timestamp: ~U[2025-03-03 23:05:00Z])
+ ])
+
+ perform(~D[2025-03-05])
+ assert Repo.reload!(site).legacy_time_on_page_cutoff == ~D[1970-01-01]
+ end
+
+ test "sets cutoff for large site with complete engagement" do
+ site = create_site_with_cutoff(nil)
+
+ 1..2500
+ |> Enum.flat_map(fn i ->
+ hour = 25 + rem(i, 24)
+ timestamp = ~U[2025-03-05 00:00:00Z] |> DateTime.add(-hour, :hour)
+
+ [
+ build(:pageview, user_id: i, timestamp: timestamp),
+ build(:engagement, user_id: i, engagement_time: 10, timestamp: timestamp)
+ ]
+ end)
+ |> then(&populate_stats(site, &1))
+
+ perform(~D[2025-03-05])
+ assert Repo.reload!(site).legacy_time_on_page_cutoff == ~D[2025-03-05]
+ end
+
+ test "does not update large site with incomplete engagement" do
+ site = create_site_with_cutoff(nil)
+
+ 1..2500
+ |> Enum.flat_map(fn i ->
+ [
+ build(:pageview, user_id: i, timestamp: ~U[2025-03-03 23:05:00Z]),
+ build(:engagement, user_id: i, engagement_time: 10, timestamp: ~U[2025-03-03 23:05:00Z])
+ ]
+ end)
+ |> then(&populate_stats(site, &1))
+
+ perform(~D[2025-03-05])
+ assert Repo.reload!(site).legacy_time_on_page_cutoff == nil
+ end
+
+ test "does not update site that already has cutoff set" do
+ site = create_site_with_cutoff(~D[2025-01-01])
+
+ populate_stats(site, [
+ build(:pageview, user_id: 13, timestamp: ~U[2025-03-03 23:05:00Z]),
+ build(:engagement, user_id: 13, engagement_time: 10, timestamp: ~U[2025-03-03 23:05:00Z])
+ ])
+
+ perform(~D[2025-03-05])
+ assert Repo.reload!(site).legacy_time_on_page_cutoff == ~D[2025-01-01]
+ end
+
+ test "ignores sites without relevant data" do
+ site = create_site_with_cutoff(nil)
+
+ populate_stats(site, [
+ build(:pageview, user_id: 13, timestamp: ~U[2025-03-04 22:05:00Z]),
+ build(:engagement, user_id: 13, engagement_time: 10, timestamp: ~U[2025-03-04 22:05:00Z])
+ ])
+
+ perform(~D[2025-03-05])
+ assert Repo.reload!(site).legacy_time_on_page_cutoff == nil
+ end
+
+ defp perform(date) do
+ SetLegacyTimeOnPageCutoff.perform(%Oban.Job{args: %{"cutoff_date" => date}})
+ end
+
+ defp create_site_with_cutoff(cutoff) do
+ site = insert(:site)
+
+ # Work around model defaults
+ site
+ |> Plausible.Site.changeset(%{legacy_time_on_page_cutoff: cutoff})
+ |> Plausible.Repo.update()
+
+ site
+ end
+end