Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Add top 3 pages to traffic spike email (#5251)
RobertJoonas167ff65a6961
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex d851806a3a..8a5648e2ac 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -4,6 +4,7 @@ All notable changes to this project will be documented in this file.## Unreleased### Added+- Add top 3 pages into the traffic spike email- Two new shorthand time periods `28d` and `90d` available on both dashboard and in public API- Average scroll depth metric- Scroll Depth goalsdiff --git a/lib/plausible/site/traffic_change_notification.ex b/lib/plausible/site/traffic_change_notification.exindex 9963c2a41d..d84af9787f 100644--- a/lib/plausible/site/traffic_change_notification.ex+++ b/lib/plausible/site/traffic_change_notification.ex@@ -34,8 +34,7 @@ defmodule Plausible.Site.TrafficChangeNotification do|> change(recipients: List.delete(schema.recipients, recipient))end- def was_sent(schema) do- schema- |> change(last_sent: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second))+ def was_sent(schema, now) do+ schema |> change(last_sent: now)endenddiff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.exindex b21f7b0ae6..c8895378e0 100644--- a/lib/plausible/stats/clickhouse.ex+++ b/lib/plausible/stats/clickhouse.ex@@ -75,34 +75,6 @@ defmodule Plausible.Stats.Clickhouse dodef usage_breakdown([], _date_range), do: {0, 0}- def top_sources_for_spike(site, query, limit, page) do- offset = (page - 1) * limit-- {first_datetime, last_datetime} = Plausible.Stats.Time.utc_boundaries(query)-- referrers =- from(s in "sessions_v2",- select: %{- name: s.referrer_source,- count: uniq(s.user_id)- },- where: s.site_id == ^site.id,- # Note: This query intentionally uses session end timestamp to get currently active users- where: s.timestamp >= ^first_datetime and s.start < ^last_datetime,- where: s.referrer_source != "",- group_by: s.referrer_source,- order_by: [desc: uniq(s.user_id), asc: s.referrer_source],- limit: ^limit,- offset: ^offset- )-- on_ee do- referrers = Plausible.Stats.Sampling.add_query_hint(referrers, 10_000_000)- end-- ClickhouseRepo.all(referrers)- end-def current_visitors(site) doPlausible.Stats.current_visitors(site)enddiff --git a/lib/plausible_web/email.ex b/lib/plausible_web/email.exindex ad14eadef2..3314c67d64 100644--- a/lib/plausible_web/email.ex+++ b/lib/plausible_web/email.ex@@ -127,15 +127,16 @@ defmodule PlausibleWeb.Email do|> html_body(PlausibleWeb.MJML.StatsReport.render(assigns))end- def spike_notification(email, site, current_visitors, sources, dashboard_link) do+ def spike_notification(email, site, stats, dashboard_link) dobase_email()|> to(email)|> tag("spike-notification")|> subject("Traffic Spike on #{site.domain}")|> render("spike_notification.html", %{site: site,- current_visitors: current_visitors,- sources: sources,+ current_visitors: stats.current_visitors,+ sources: stats.sources,+ pages: stats.pages,link: dashboard_link})enddiff --git a/lib/plausible_web/templates/email/spike_notification.html.heex b/lib/plausible_web/templates/email/spike_notification.html.heexindex 295e016043..8f37e0faa7 100644--- a/lib/plausible_web/templates/email/spike_notification.html.heex+++ b/lib/plausible_web/templates/email/spike_notification.html.heex@@ -1,15 +1,27 @@-There are currently {@current_visitors} visitors on-<a href={"https://" <> @site.domain}><%= @site.domain %></a>.+There are currently <b>{@current_visitors}</b>+visitors on <a href={"https://" <> @site.domain}><%= @site.domain %></a>. <br /><%= if Enum.count(@sources) > 0 do %>- <br /><br /> The top sources for current visitors:<br />- <%= for %{name: source, count: visitors} <- @sources do %>- {source} - {visitors} visitor{if visitors > 1, do: "s"}<br />- <% end %>+ <ul>+ <%= for %{dimensions: [source], metrics: [visitors]} <- @sources do %>+ <li>+ <b>{visitors}</b> visitor{if visitors > 1, do: "s"} from <b>{source}</b>+ </li>+ <% end %>+ </ul><% end %>+<br /> Your top pages being visited:<br />+<ul>+ <%= for %{dimensions: [page], metrics: [visitors]} <- @pages do %>+ <li>+ <b>{visitors}</b> visitor{if visitors > 1, do: "s"} on <b>{page}</b>+ </li>+ <% end %>+</ul>+<%= if @link do %>- <br /><br /> View dashboard: <a href={@link}>{@link}</a>+ <br /> View dashboard: <a href={@link}>{@link}</a><% end %><br /><br /> Congrats on the spike in traffic!<%= if Plausible.ce? do %>diff --git a/lib/workers/traffic_change_notifier.ex b/lib/workers/traffic_change_notifier.exindex 98ba573ff6..313e4f75d0 100644--- a/lib/workers/traffic_change_notifier.ex+++ b/lib/workers/traffic_change_notifier.ex@@ -3,23 +3,24 @@ defmodule Plausible.Workers.TrafficChangeNotifier doOban service sending out traffic drop/spike notifications"""use Plausible.Repo- alias Plausible.Stats.Query+ alias Plausible.Stats.{Query, Clickhouse}alias Plausible.Site.TrafficChangeNotificationalias PlausibleWeb.Router.Helpers, as: Routesuse Oban.Worker, queue: :spike_notifications- @at_most_every "12 hours"+ @min_interval_hours 12@impl Oban.Worker- def perform(_job, clickhouse \\ Plausible.Stats.Clickhouse) do- today = Date.utc_today()+ def perform(_job, now \\ NaiveDateTime.utc_now(:second)) do+ today = NaiveDateTime.to_date(now)notifications =Repo.all(from sn in TrafficChangeNotification,where:- is_nil(sn.last_sent) or sn.last_sent < fragment("now() - INTERVAL ?", @at_most_every),+ is_nil(sn.last_sent) or+ sn.last_sent < ^NaiveDateTime.add(now, -@min_interval_hours, :hour),inner_join: s in assoc(sn, :site),inner_join: t in assoc(s, :team),where: not s.locked,@@ -30,20 +31,22 @@ defmodule Plausible.Workers.TrafficChangeNotifier dofor notification <- notifications docase notification.type do:spike ->- current_visitors = clickhouse.current_visitors(notification.site)+ current_visitors = Clickhouse.current_visitors(notification.site)if current_visitors >= notification.threshold do- query = Query.from(notification.site, %{"period" => "realtime"})- sources = clickhouse.top_sources_for_spike(notification.site, query, 3, 1)+ stats =+ notification.site+ |> get_traffic_spike_stats()+ |> Map.put(:current_visitors, current_visitors)- notify_spike(notification, current_visitors, sources)+ notify_spike(notification, stats, now)end:drop ->- current_visitors = clickhouse.current_visitors_12h(notification.site)+ current_visitors = Clickhouse.current_visitors_12h(notification.site)if current_visitors < notification.threshold do- notify_drop(notification, current_visitors)+ notify_drop(notification, current_visitors, now)endendend@@ -51,27 +54,27 @@ defmodule Plausible.Workers.TrafficChangeNotifier do:okend- defp notify_spike(notification, current_visitors, sources) do+ defp notify_spike(notification, stats, now) dofor recipient <- notification.recipients do- send_spike_notification(recipient, notification.site, current_visitors, sources)+ send_spike_notification(recipient, notification.site, stats)endnotification- |> TrafficChangeNotification.was_sent()+ |> TrafficChangeNotification.was_sent(now)|> Repo.update()end- defp notify_drop(notification, current_visitors) do+ defp notify_drop(notification, current_visitors, now) dofor recipient <- notification.recipients dosend_drop_notification(recipient, notification.site, current_visitors)endnotification- |> TrafficChangeNotification.was_sent()+ |> TrafficChangeNotification.was_sent(now)|> Repo.update()end- defp send_spike_notification(recipient, site, current_visitors, sources) do+ defp send_spike_notification(recipient, site, stats) dodashboard_link =if Repo.exists?(email_match_query(site, recipient)) doRoutes.stats_url(PlausibleWeb.Endpoint, :stats, site.domain, [])@@ -81,8 +84,7 @@ defmodule Plausible.Workers.TrafficChangeNotifier doPlausibleWeb.Email.spike_notification(recipient,site,- current_visitors,- sources,+ stats,dashboard_link)@@ -114,6 +116,44 @@ defmodule Plausible.Workers.TrafficChangeNotifier doPlausible.Mailer.send(template)end+ defp get_traffic_spike_stats(site) do+ {:ok, query} =+ Query.build(+ site,+ :internal,+ %{+ "site_id" => "#{site.id}",+ "metrics" => ["visitors"],+ "pagination" => %{"limit" => 3},+ "date_range" => "realtime"+ },+ %{}+ )++ %{}+ |> put_sources(site, query)+ |> put_pages(site, query)+ end++ defp put_sources(stats, site, query) do+ query =+ query+ |> Query.set(dimensions: ["visit:source"])+ |> Query.add_filter([:is_not, "visit:source", ["Direct / None"]])++ %{results: sources} = Plausible.Stats.query(site, query)++ Map.put(stats, :sources, sources)+ end++ defp put_pages(stats, site, query) do+ query = Query.set(query, dimensions: ["event:page"])++ %{results: pages} = Plausible.Stats.query(site, query)++ Map.put(stats, :pages, pages)+ end+defp email_match_query(site, recipient) dofrom tm in Plausible.Teams.Membership,inner_join: u in assoc(tm, :user),diff --git a/test/plausible/stats/clickhouse_test.exs b/test/plausible/stats/clickhouse_test.exsindex 6dfd9f1990..5d682727f5 100644--- a/test/plausible/stats/clickhouse_test.exs+++ b/test/plausible/stats/clickhouse_test.exs@@ -307,41 +307,6 @@ defmodule Plausible.Stats.ClickhouseTest doendend- describe "top_sources_for_spike/4" do- test "gets named sources" do- site = insert(:site)- query = Plausible.Stats.Query.from(site, %{"period" => "all"})-- populate_stats(site, [- build(:pageview,- pathname: "/",- referrer_source: "Twitter"- ),- build(:pageview,- pathname: "/plausible.io"- ),- build(:pageview,- pathname: "/plausible.io",- referrer_source: "Google"- ),- build(:pageview,- pathname: "/plausible.io",- referrer_source: "Google"- ),- build(:pageview,- pathname: "/plausible.io",- referrer_source: "Bing"- )- ])-- assert [- %{count: 2, name: "Google"},- %{count: 1, name: "Bing"},- %{count: 1, name: "Twitter"}- ] = Clickhouse.top_sources_for_spike(site, query, 5, 1)- end- end-describe "imported_pageview_counts/1" dotest "gets pageview counts for each of sites' imports" dosite = insert(:site)diff --git a/test/workers/traffic_change_notifier_test.exs b/test/workers/traffic_change_notifier_test.exsindex 2f7aa706c3..1a66f26c40 100644--- a/test/workers/traffic_change_notifier_test.exs+++ b/test/workers/traffic_change_notifier_test.exs@@ -2,7 +2,6 @@ defmodule Plausible.Workers.TrafficChangeNotifierTest douse Plausible.DataCase, async: trueuse Bamboo.Testuse Plausible.Teams.Test- import Doublealias Plausible.Workers.TrafficChangeNotifierdescribe "drops" do@@ -16,10 +15,7 @@ defmodule Plausible.Workers.TrafficChangeNotifierTest dorecipients: ["[email protected]", "[email protected]"])- clickhouse_stub =- stub(Plausible.Stats.Clickhouse, :current_visitors_12h, fn _site -> 1 end)-- TrafficChangeNotifier.perform(nil, clickhouse_stub)+ TrafficChangeNotifier.perform(nil)assert_no_emails_delivered()end@@ -34,10 +30,7 @@ defmodule Plausible.Workers.TrafficChangeNotifierTest dorecipients: ["[email protected]"])- clickhouse_stub =- stub(Plausible.Stats.Clickhouse, :current_visitors_12h, fn _site -> 1 end)-- TrafficChangeNotifier.perform(nil, clickhouse_stub)+ TrafficChangeNotifier.perform(nil)assert_email_delivered_with(subject: "Traffic Drop on #{site.domain}",@@ -50,14 +43,16 @@ defmodule Plausible.Workers.TrafficChangeNotifierTest doinsert(:drop_notification,site: site,- threshold: 10,+ threshold: 2,recipients: ["[email protected]", "[email protected]"])- clickhouse_stub =- stub(Plausible.Stats.Clickhouse, :current_visitors_12h, fn _site -> 11 end)+ populate_stats(site, [+ build(:pageview, timestamp: minutes_ago(2)),+ build(:pageview, timestamp: minutes_ago(1))+ ])- TrafficChangeNotifier.perform(nil, clickhouse_stub)+ TrafficChangeNotifier.perform(nil)assert_no_emails_delivered()end@@ -71,10 +66,7 @@ defmodule Plausible.Workers.TrafficChangeNotifierTest dorecipients: ["[email protected]", "[email protected]"])- clickhouse_stub =- stub(Plausible.Stats.Clickhouse, :current_visitors_12h, fn _site -> 7 end)-- TrafficChangeNotifier.perform(nil, clickhouse_stub)+ TrafficChangeNotifier.perform(nil)assert_email_delivered_with(subject: "Traffic Drop on #{site.domain}",@@ -87,28 +79,32 @@ defmodule Plausible.Workers.TrafficChangeNotifierTest do)end- test "does not notify anyone if a notification already went out in the last 12 hours" do+ test "does not send notifications more than once every 12 hours" dosite = new_site()insert(:drop_notification,site: site,- threshold: 10,+ threshold: 1,recipients: ["[email protected]"])- clickhouse_stub =- stub(Plausible.Stats.Clickhouse, :current_visitors_12h, fn _site -> 4 end)-- TrafficChangeNotifier.perform(nil, clickhouse_stub)+ TrafficChangeNotifier.perform(nil, ~N[2021-01-01 00:00:00])assert_email_delivered_with(subject: "Traffic Drop on #{site.domain}",to: [nil: "[email protected]"])- TrafficChangeNotifier.perform(nil, clickhouse_stub)+ TrafficChangeNotifier.perform(nil, ~N[2021-01-01 11:59:59])assert_no_emails_delivered()++ TrafficChangeNotifier.perform(nil, ~N[2021-01-01 12:00:01])++ assert_email_delivered_with(+ subject: "Traffic Drop on #{site.domain}",+ to: [nil: "[email protected]"]+ )endtest "adds settings link if recipient has access to the site" do@@ -121,10 +117,7 @@ defmodule Plausible.Workers.TrafficChangeNotifierTest dorecipients: ["[email protected]"])- clickhouse_stub =- stub(Plausible.Stats.Clickhouse, :current_visitors_12h, fn _site -> 6 end)-- TrafficChangeNotifier.perform(nil, clickhouse_stub)+ TrafficChangeNotifier.perform(nil)assert_email_delivered_with(html_body: ~r|http://localhost:8000/example.com/installation\?flow=review|@@ -138,15 +131,13 @@ defmodule Plausible.Workers.TrafficChangeNotifierTest doinsert(:spike_notification,site: site,- threshold: 10,+ threshold: 2,recipients: ["[email protected]", "[email protected]"])- clickhouse_stub =- stub(Plausible.Stats.Clickhouse, :current_visitors, fn _site -> 5 end)- |> stub(:top_sources_for_spike, fn _site, _query, _limit, _page -> [] end)+ populate_stats(site, [build(:pageview, timestamp: minutes_ago(1))])- TrafficChangeNotifier.perform(nil, clickhouse_stub)+ TrafficChangeNotifier.perform(nil)assert_no_emails_delivered()end@@ -156,15 +147,17 @@ defmodule Plausible.Workers.TrafficChangeNotifierTest doinsert(:spike_notification,site: site,- threshold: 10,+ threshold: 2,recipients: ["[email protected]", "[email protected]"])- clickhouse_stub =- stub(Plausible.Stats.Clickhouse, :current_visitors, fn _site -> 10 end)- |> stub(:top_sources_for_spike, fn _site, _query, _limit, _page -> [] end)+ populate_stats(site, [+ build(:pageview, timestamp: minutes_ago(3)),+ build(:pageview, timestamp: minutes_ago(2)),+ build(:pageview, timestamp: minutes_ago(1))+ ])- TrafficChangeNotifier.perform(nil, clickhouse_stub)+ TrafficChangeNotifier.perform(nil)assert_email_delivered_with(subject: "Traffic Spike on #{site.domain}",@@ -177,40 +170,140 @@ defmodule Plausible.Workers.TrafficChangeNotifierTest do)end+ test "ignores 'Direct / None' source" do+ site = new_site()++ insert(:spike_notification,+ site: site,+ threshold: 5,+ recipients: ["[email protected]"]+ )++ populate_stats(site, [+ build(:pageview, referrer_source: "A", timestamp: minutes_ago(1)),+ build(:pageview, referrer_source: "A", timestamp: minutes_ago(1)),+ build(:pageview, referrer_source: "A", timestamp: minutes_ago(1)),+ build(:pageview, referrer_source: "B", timestamp: minutes_ago(1)),+ build(:pageview, referrer_source: "B", timestamp: minutes_ago(1)),+ build(:pageview, referrer_source: "C", timestamp: minutes_ago(1)),+ build(:pageview, timestamp: minutes_ago(1)),+ build(:pageview, timestamp: minutes_ago(1))+ ])++ TrafficChangeNotifier.perform(nil)++ assert_delivered_email_matches(%{+ to: [nil: "[email protected]"],+ html_body: html_body+ })++ assert html_body =~ "The top sources for current visitors:"+ assert html_body =~ "<b>3</b> visitors from <b>A</b>"+ assert html_body =~ "<b>2</b> visitors from <b>B</b>"+ assert html_body =~ "<b>1</b> visitor from <b>C</b>"+ assert html_body =~ "There are currently <b>8</b>"+ end++ test "does not list sources at all when everything is 'Direct / None'" do+ site = new_site()++ insert(:spike_notification,+ site: site,+ threshold: 1,+ recipients: ["[email protected]"]+ )++ populate_stats(site, [+ build(:pageview, timestamp: minutes_ago(1)),+ build(:pageview, timestamp: minutes_ago(1))+ ])++ TrafficChangeNotifier.perform(nil)++ assert_delivered_email_matches(%{+ to: [nil: "[email protected]"],+ html_body: html_body+ })++ refute html_body =~ "The top sources for current visitors:"+ assert html_body =~ "There are currently <b>2</b>"+ end++ test "includes top 3 pages" do+ site = new_site()++ insert(:spike_notification,+ site: site,+ threshold: 10,+ recipients: ["[email protected]"]+ )++ populate_stats(site, [+ build(:pageview, pathname: "/one", timestamp: minutes_ago(1)),+ build(:pageview, pathname: "/one", timestamp: minutes_ago(1)),+ build(:pageview, pathname: "/one", timestamp: minutes_ago(1)),+ build(:pageview, pathname: "/one", timestamp: minutes_ago(1)),+ build(:pageview, pathname: "/two", timestamp: minutes_ago(1)),+ build(:pageview, pathname: "/two", timestamp: minutes_ago(1)),+ build(:pageview, pathname: "/two", timestamp: minutes_ago(1)),+ build(:pageview, timestamp: minutes_ago(1)),+ build(:pageview, timestamp: minutes_ago(1)),+ build(:pageview, pathname: "/not-this-one", timestamp: minutes_ago(1))+ ])++ TrafficChangeNotifier.perform(nil)++ assert_delivered_email_matches(%{+ to: [nil: "[email protected]"],+ html_body: html_body+ })++ assert html_body =~ "There are currently <b>10</b>"++ assert html_body =~ "Your top pages being visited:"+ assert html_body =~ "<b>4</b> visitors on <b>/one</b>"+ assert html_body =~ "<b>3</b> visitors on <b>/two</b>"+ assert html_body =~ "<b>2</b> visitors on <b>/</b>"++ refute html_body =~ "/not-this-one"+ end+test "does not check site if it is locked" dosite = insert(:site, locked: true)insert(:spike_notification,site: site,- threshold: 10,+ threshold: 1,recipients: ["[email protected]"])- clickhouse_stub =- stub(Plausible.Stats.Clickhouse, :current_visitors, fn _site -> 10 end)- |> stub(:top_sources_for_spike, fn _site, _query, _limit, _page -> [] end)+ populate_stats(site, [+ build(:pageview, timestamp: minutes_ago(1)),+ build(:pageview, timestamp: minutes_ago(1))+ ])- TrafficChangeNotifier.perform(nil, clickhouse_stub)+ TrafficChangeNotifier.perform(nil)assert_no_emails_delivered()end- test "does not notify anyone if a notification already went out in the last 12 hours" do+ test "does not send notifications more than once every 12 hours" dosite = new_site()- insert(:spike_notification, site: site, threshold: 10, recipients: ["[email protected]"])+ insert(:spike_notification, site: site, threshold: 1, recipients: ["[email protected]"])- clickhouse_stub =- stub(Plausible.Stats.Clickhouse, :current_visitors, fn _site -> 10 end)- |> stub(:top_sources_for_spike, fn _site, _query, _limit, _page -> [] end)+ populate_stats(site, [+ build(:pageview, timestamp: minutes_ago(1)),+ build(:pageview, timestamp: minutes_ago(1))+ ])- TrafficChangeNotifier.perform(nil, clickhouse_stub)+ TrafficChangeNotifier.perform(nil)assert_email_delivered_with(subject: "Traffic Spike on #{site.domain}",to: [nil: "[email protected]"])- TrafficChangeNotifier.perform(nil, clickhouse_stub)+ TrafficChangeNotifier.perform(nil)assert_no_emails_delivered()end@@ -218,15 +311,20 @@ defmodule Plausible.Workers.TrafficChangeNotifierTest dotest "adds a dashboard link if recipient has access to the site" douser = new_user(email: "[email protected]")site = new_site(domain: "example.com", owner: user)- insert(:spike_notification, site: site, threshold: 10, recipients: ["[email protected]"])+ insert(:spike_notification, site: site, threshold: 1, recipients: ["[email protected]"])- clickhouse_stub =- stub(Plausible.Stats.Clickhouse, :current_visitors, fn _site -> 10 end)- |> stub(:top_sources_for_spike, fn _site, _query, _limit, _page -> [] end)+ populate_stats(site, [+ build(:pageview, timestamp: minutes_ago(1)),+ build(:pageview, timestamp: minutes_ago(1))+ ])- TrafficChangeNotifier.perform(nil, clickhouse_stub)+ TrafficChangeNotifier.perform(nil)assert_email_delivered_with(html_body: ~r/View dashboard:\s+<a href=\"http.+\/example.com/)endend++ def minutes_ago(min) do+ NaiveDateTime.utc_now(:second) |> NaiveDateTime.add(-min, :minute)+ endend