Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
time-on-page: `site.legacy_time_on_page` cronjob (#5215)
Karl-Aksel Puulmann42cea1d88984
diff --git a/config/runtime.exs b/config/runtime.exsindex d81477423e..e6428b43b1 100644--- a/config/runtime.exs+++ b/config/runtime.exs@@ -713,7 +713,9 @@ cloud_cron = [# Daily at 8{"0 8 * * *", Plausible.Workers.AcceptTrafficUntil},# First sunday of the month, 4:00 UTC- {"0 4 1-7 * SUN", Plausible.Workers.ClickhouseCleanSites}+ {"0 4 1-7 * SUN", Plausible.Workers.ClickhouseCleanSites},+ # Daily at 4:00 UTC+ {"0 4 * * *", Plausible.Workers.SetLegacyTimeOnPageCutoff}]crontab = if(is_selfhost, do: base_cron, else: base_cron ++ cloud_cron)@@ -740,7 +742,8 @@ cloud_queues = [trial_notification_emails: 1,check_usage: 1,notify_annual_renewal: 1,- lock_sites: 1+ lock_sites: 1,+ legacy_time_on_page_cutoff: 1]queues = if(is_selfhost, do: base_queues, else: base_queues ++ cloud_queues)diff --git a/lib/plausible/site.ex b/lib/plausible/site.exindex e3cfa0e7af..375a9d773a 100644--- a/lib/plausible/site.ex+++ b/lib/plausible/site.ex@@ -21,6 +21,7 @@ defmodule Plausible.Site dofield :conversions_enabled, :boolean, default: truefield :props_enabled, :boolean, default: truefield :funnels_enabled, :boolean, default: true+ field :legacy_time_on_page_cutoff, :date, default: ~D[1970-01-01]field :ingest_rate_limit_scale_seconds, :integer, default: 60# default is set via changeset/2@@ -87,7 +88,7 @@ defmodule Plausible.Site dodef changeset(site, attrs \\ %{}) dosite- |> cast(attrs, [:domain, :timezone])+ |> cast(attrs, [:domain, :timezone, :legacy_time_on_page_cutoff])|> clean_domain()|> validate_required([:domain, :timezone])|> validate_timezone()diff --git a/lib/plausible/stats/time_on_page.ex b/lib/plausible/stats/time_on_page.exindex 38ca7a983a..d5305ced8e 100644--- a/lib/plausible/stats/time_on_page.ex+++ b/lib/plausible/stats/time_on_page.ex@@ -8,9 +8,18 @@ defmodule Plausible.Stats.TimeOnPage doFunWithFlags.enabled?(:new_time_on_page, for: site)end- def legacy_time_on_page_cutoff() do- # Placeholder until we implement a more sophisticated way to determine the cutoff- # Only used when `new_time_on_page` flag is enabled- DateTime.utc_now() |> DateTime.shift(day: -4) |> DateTime.to_iso8601()+ def legacy_time_on_page_cutoff(site) do+ cutoff(site.legacy_time_on_page_cutoff, site.timezone)+ end++ defp cutoff(nil, _timezone), do: ""++ defp cutoff(date, timezone) do+ case DateTime.new(date, ~T[00:00:00], timezone) do+ {:ok, datetime} -> datetime+ {:gap, just_before, _just_after} -> just_before+ {:ambiguous, first_datetime, _second_datetime} -> first_datetime+ end+ |> DateTime.to_iso8601()endenddiff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.exindex 4e671f8784..00b7273c19 100644--- a/lib/plausible_web/controllers/stats_controller.ex+++ b/lib/plausible_web/controllers/stats_controller.ex@@ -77,7 +77,7 @@ defmodule PlausibleWeb.StatsController dohas_props: Plausible.Props.configured?(site),stats_start_date: stats_start_date,native_stats_start_date: NaiveDateTime.to_date(site.native_stats_start_at),- legacy_time_on_page_cutoff: TimeOnPage.legacy_time_on_page_cutoff(),+ legacy_time_on_page_cutoff: TimeOnPage.legacy_time_on_page_cutoff(site),title: title(conn, site),demo: demo,flags: flags,@@ -362,7 +362,7 @@ defmodule PlausibleWeb.StatsController dohas_props: Plausible.Props.configured?(shared_link.site),stats_start_date: stats_start_date,native_stats_start_date: NaiveDateTime.to_date(shared_link.site.native_stats_start_at),- legacy_time_on_page_cutoff: TimeOnPage.legacy_time_on_page_cutoff(),+ legacy_time_on_page_cutoff: TimeOnPage.legacy_time_on_page_cutoff(shared_link.site),title: title(conn, shared_link.site),demo: false,dogfood_page_path: "/share/:dashboard",diff --git a/lib/workers/set_legacy_time_on_page_cutoff.ex b/lib/workers/set_legacy_time_on_page_cutoff.exnew file mode 100644index 0000000000..b188df2c50--- /dev/null+++ b/lib/workers/set_legacy_time_on_page_cutoff.ex@@ -0,0 +1,94 @@+defmodule Plausible.Workers.SetLegacyTimeOnPageCutoff do+ @moduledoc """+ Sets sites `legacy_time_on_page_cutoff` depending on whether they have+ sent us engagement data in the past.+ """++ use Plausible.Repo+ use Plausible.ClickhouseRepo+ use Oban.Worker, queue: :legacy_time_on_page_cutoff++ import Ecto.Query++ require Logger++ @impl Oban.Worker+ def perform(%Oban.Job{args: args}) do+ cutoff_date = Map.get(args, "cutoff_date", Date.utc_today())++ {small_sites, large_sites} =+ sites_with_engagement_data(cutoff_date)+ |> filter_sites_needing_update()++ to_update = small_sites ++ large_sites++ if length(to_update) > 0 do+ Logger.info(+ "Setting legacy_time_on_page_cutoff for #{length(to_update)} sites (#{length(small_sites)} small, #{length(large_sites)} large)"+ )++ {count, _} =+ Repo.update_all(+ from(s in Plausible.Site,+ where: s.id in ^to_update,+ where: is_nil(s.legacy_time_on_page_cutoff)+ ),+ set: [legacy_time_on_page_cutoff: cutoff_date]+ )++ Logger.info("Successfully set legacy_time_on_page_cutoff=#{cutoff_date} for #{count} sites")+ else+ Logger.debug("No sites legacy_time_on_page_cutoff needs updating")+ end++ :ok+ end++ defp sites_with_engagement_data(cutoff_date) do+ site_info_q =+ from(+ e in "events_v2",+ where: e.timestamp >= fragment("toStartOfHour(? - toIntervalHour(48))", ^cutoff_date),+ where: e.timestamp < fragment("toStartOfHour(? - toIntervalHour(24))", ^cutoff_date),+ group_by: e.site_id,+ select: %{+ site_id: e.site_id,+ hours_with_engagement:+ fragment(+ "uniqIf(toStartOfHour(timestamp), name = 'engagement' AND engagement_time > 0)"+ ),+ is_small_site: fragment("count() < 2000")+ }+ )++ q =+ from(+ s in subquery(site_info_q),+ select: %{+ small_sites:+ fragment("groupArrayIf(site_id, (is_small_site and hours_with_engagement > 0))"),+ large_sites:+ fragment("groupArrayIf(site_id, (not is_small_site and hours_with_engagement = 24))")+ }+ )++ result = ClickhouseRepo.one(q)++ {MapSet.new(result.small_sites), MapSet.new(result.large_sites)}+ end++ defp filter_sites_needing_update({small_sites, large_sites}) do+ needing_update =+ from(s in Plausible.Site,+ where: is_nil(s.legacy_time_on_page_cutoff),+ select: s.id+ )+ |> Repo.all()+ |> MapSet.new()++ {+ small_sites |> MapSet.intersection(needing_update) |> MapSet.to_list(),+ large_sites |> MapSet.intersection(needing_update) |> MapSet.to_list()+ }+ end+enddiff --git a/test/plausible/site/sites_test.exs b/test/plausible/site/sites_test.exsindex 98b70a7d2e..4485ad411a 100644--- a/test/plausible/site/sites_test.exs+++ b/test/plausible/site/sites_test.exs@@ -14,6 +14,15 @@ defmodule Plausible.SitesTest doSites.create(user, params)end+ test "creating a site sets `legacy_time_on_page_cutoff`" do+ user = new_user()++ params = %{"domain" => "example.com", "timezone" => "Europe/London"}++ assert {:ok, %{site: %{legacy_time_on_page_cutoff: ~D[1970-01-01]}}} =+ Sites.create(user, params)+ end+test "does not start a trial for pre-teams guest users without trial expiry date" douser = new_user() |> subscribe_to_growth_plan()new_site(owner: user)diff --git a/test/workers/set_legacy_time_on_page_cutoff_test.exs b/test/workers/set_legacy_time_on_page_cutoff_test.exsnew file mode 100644index 0000000000..4b1689600e--- /dev/null+++ b/test/workers/set_legacy_time_on_page_cutoff_test.exs@@ -0,0 +1,105 @@+defmodule Plausible.Workers.SetLegacyTimeOnPageCutoffTest do+ use Plausible.DataCase+ use Plausible.TestUtils+ use Plausible++ alias Plausible.Workers.SetLegacyTimeOnPageCutoff++ test "sets cutoff for small site with engagement" do+ site = create_site_with_cutoff(nil)++ populate_stats(site, [+ build(:pageview, user_id: 13, timestamp: ~U[2025-03-03 23:05:00Z]),+ build(:engagement, user_id: 13, engagement_time: 10, timestamp: ~U[2025-03-03 23:05:00Z])+ ])++ perform(~D[2025-03-05])+ assert Repo.reload!(site).legacy_time_on_page_cutoff == ~D[2025-03-05]+ end++ test "does not update an already set cutoff" do+ site = create_site_with_cutoff(~D[1970-01-01])++ populate_stats(site, [+ build(:pageview, user_id: 13, timestamp: ~U[2025-03-03 23:05:00Z]),+ build(:engagement, user_id: 13, engagement_time: 10, timestamp: ~U[2025-03-03 23:05:00Z])+ ])++ perform(~D[2025-03-05])+ assert Repo.reload!(site).legacy_time_on_page_cutoff == ~D[1970-01-01]+ end++ test "sets cutoff for large site with complete engagement" do+ site = create_site_with_cutoff(nil)++ 1..2500+ |> Enum.flat_map(fn i ->+ hour = 25 + rem(i, 24)+ timestamp = ~U[2025-03-05 00:00:00Z] |> DateTime.add(-hour, :hour)++ [+ build(:pageview, user_id: i, timestamp: timestamp),+ build(:engagement, user_id: i, engagement_time: 10, timestamp: timestamp)+ ]+ end)+ |> then(&populate_stats(site, &1))++ perform(~D[2025-03-05])+ assert Repo.reload!(site).legacy_time_on_page_cutoff == ~D[2025-03-05]+ end++ test "does not update large site with incomplete engagement" do+ site = create_site_with_cutoff(nil)++ 1..2500+ |> Enum.flat_map(fn i ->+ [+ build(:pageview, user_id: i, timestamp: ~U[2025-03-03 23:05:00Z]),+ build(:engagement, user_id: i, engagement_time: 10, timestamp: ~U[2025-03-03 23:05:00Z])+ ]+ end)+ |> then(&populate_stats(site, &1))++ perform(~D[2025-03-05])+ assert Repo.reload!(site).legacy_time_on_page_cutoff == nil+ end++ test "does not update site that already has cutoff set" do+ site = create_site_with_cutoff(~D[2025-01-01])++ populate_stats(site, [+ build(:pageview, user_id: 13, timestamp: ~U[2025-03-03 23:05:00Z]),+ build(:engagement, user_id: 13, engagement_time: 10, timestamp: ~U[2025-03-03 23:05:00Z])+ ])++ perform(~D[2025-03-05])+ assert Repo.reload!(site).legacy_time_on_page_cutoff == ~D[2025-01-01]+ end++ test "ignores sites without relevant data" do+ site = create_site_with_cutoff(nil)++ populate_stats(site, [+ build(:pageview, user_id: 13, timestamp: ~U[2025-03-04 22:05:00Z]),+ build(:engagement, user_id: 13, engagement_time: 10, timestamp: ~U[2025-03-04 22:05:00Z])+ ])++ perform(~D[2025-03-05])+ assert Repo.reload!(site).legacy_time_on_page_cutoff == nil+ end++ defp perform(date) do+ SetLegacyTimeOnPageCutoff.perform(%Oban.Job{args: %{"cutoff_date" => date}})+ end++ defp create_site_with_cutoff(cutoff) do+ site = insert(:site)++ # Work around model defaults+ site+ |> Plausible.Site.changeset(%{legacy_time_on_page_cutoff: cutoff})+ |> Plausible.Repo.update()++ site+ end+end