Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Add top 3 pages to traffic spike email (#5251)

RobertJoonas167ff65a6961
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d851806a3a..8a5648e2ac 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,7 @@ All notable changes to this project will be documented in this file.
## Unreleased
### Added
+- Add top 3 pages into the traffic spike email
- Two new shorthand time periods `28d` and `90d` available on both dashboard and in public API
- Average scroll depth metric
- Scroll Depth goals
diff --git a/lib/plausible/site/traffic_change_notification.ex b/lib/plausible/site/traffic_change_notification.ex
index 9963c2a41d..d84af9787f 100644
--- a/lib/plausible/site/traffic_change_notification.ex
+++ b/lib/plausible/site/traffic_change_notification.ex
@@ -34,8 +34,7 @@ defmodule Plausible.Site.TrafficChangeNotification do
|> change(recipients: List.delete(schema.recipients, recipient))
end
- def was_sent(schema) do
- schema
- |> change(last_sent: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second))
+ def was_sent(schema, now) do
+ schema |> change(last_sent: now)
end
end
diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex
index b21f7b0ae6..c8895378e0 100644
--- a/lib/plausible/stats/clickhouse.ex
+++ b/lib/plausible/stats/clickhouse.ex
@@ -75,34 +75,6 @@ defmodule Plausible.Stats.Clickhouse do
def usage_breakdown([], _date_range), do: {0, 0}
- def top_sources_for_spike(site, query, limit, page) do
- offset = (page - 1) * limit
-
- {first_datetime, last_datetime} = Plausible.Stats.Time.utc_boundaries(query)
-
- referrers =
- from(s in "sessions_v2",
- select: %{
- name: s.referrer_source,
- count: uniq(s.user_id)
- },
- where: s.site_id == ^site.id,
- # Note: This query intentionally uses session end timestamp to get currently active users
- where: s.timestamp >= ^first_datetime and s.start < ^last_datetime,
- where: s.referrer_source != "",
- group_by: s.referrer_source,
- order_by: [desc: uniq(s.user_id), asc: s.referrer_source],
- limit: ^limit,
- offset: ^offset
- )
-
- on_ee do
- referrers = Plausible.Stats.Sampling.add_query_hint(referrers, 10_000_000)
- end
-
- ClickhouseRepo.all(referrers)
- end
-
def current_visitors(site) do
Plausible.Stats.current_visitors(site)
end
diff --git a/lib/plausible_web/email.ex b/lib/plausible_web/email.ex
index ad14eadef2..3314c67d64 100644
--- a/lib/plausible_web/email.ex
+++ b/lib/plausible_web/email.ex
@@ -127,15 +127,16 @@ defmodule PlausibleWeb.Email do
|> html_body(PlausibleWeb.MJML.StatsReport.render(assigns))
end
- def spike_notification(email, site, current_visitors, sources, dashboard_link) do
+ def spike_notification(email, site, stats, dashboard_link) do
base_email()
|> to(email)
|> tag("spike-notification")
|> subject("Traffic Spike on #{site.domain}")
|> render("spike_notification.html", %{
site: site,
- current_visitors: current_visitors,
- sources: sources,
+ current_visitors: stats.current_visitors,
+ sources: stats.sources,
+ pages: stats.pages,
link: dashboard_link
})
end
diff --git a/lib/plausible_web/templates/email/spike_notification.html.heex b/lib/plausible_web/templates/email/spike_notification.html.heex
index 295e016043..8f37e0faa7 100644
--- a/lib/plausible_web/templates/email/spike_notification.html.heex
+++ b/lib/plausible_web/templates/email/spike_notification.html.heex
@@ -1,15 +1,27 @@
-There are currently {@current_visitors} visitors on
-<a href={"https://" <> @site.domain}><%= @site.domain %></a>.
+There are currently <b>{@current_visitors}</b>
+visitors on <a href={"https://" <> @site.domain}><%= @site.domain %></a>. <br />
<%= if Enum.count(@sources) > 0 do %>
- <br />
<br /> The top sources for current visitors:<br />
- <%= for %{name: source, count: visitors} <- @sources do %>
- {source} - {visitors} visitor{if visitors > 1, do: "s"}<br />
- <% end %>
+ <ul>
+ <%= for %{dimensions: [source], metrics: [visitors]} <- @sources do %>
+ <li>
+ <b>{visitors}</b> visitor{if visitors > 1, do: "s"} from <b>{source}</b>
+ </li>
+ <% end %>
+ </ul>
<% end %>
+<br /> Your top pages being visited:<br />
+<ul>
+ <%= for %{dimensions: [page], metrics: [visitors]} <- @pages do %>
+ <li>
+ <b>{visitors}</b> visitor{if visitors > 1, do: "s"} on <b>{page}</b>
+ </li>
+ <% end %>
+</ul>
+
<%= if @link do %>
- <br /><br /> View dashboard: <a href={@link}>{@link}</a>
+ <br /> View dashboard: <a href={@link}>{@link}</a>
<% end %>
<br /><br /> Congrats on the spike in traffic!
<%= if Plausible.ce? do %>
diff --git a/lib/workers/traffic_change_notifier.ex b/lib/workers/traffic_change_notifier.ex
index 98ba573ff6..313e4f75d0 100644
--- a/lib/workers/traffic_change_notifier.ex
+++ b/lib/workers/traffic_change_notifier.ex
@@ -3,23 +3,24 @@ defmodule Plausible.Workers.TrafficChangeNotifier do
Oban service sending out traffic drop/spike notifications
"""
use Plausible.Repo
- alias Plausible.Stats.Query
+ alias Plausible.Stats.{Query, Clickhouse}
alias Plausible.Site.TrafficChangeNotification
alias PlausibleWeb.Router.Helpers, as: Routes
use Oban.Worker, queue: :spike_notifications
- @at_most_every "12 hours"
+ @min_interval_hours 12
@impl Oban.Worker
- def perform(_job, clickhouse \\ Plausible.Stats.Clickhouse) do
- today = Date.utc_today()
+ def perform(_job, now \\ NaiveDateTime.utc_now(:second)) do
+ today = NaiveDateTime.to_date(now)
notifications =
Repo.all(
from sn in TrafficChangeNotification,
where:
- is_nil(sn.last_sent) or sn.last_sent < fragment("now() - INTERVAL ?", @at_most_every),
+ is_nil(sn.last_sent) or
+ sn.last_sent < ^NaiveDateTime.add(now, -@min_interval_hours, :hour),
inner_join: s in assoc(sn, :site),
inner_join: t in assoc(s, :team),
where: not s.locked,
@@ -30,20 +31,22 @@ defmodule Plausible.Workers.TrafficChangeNotifier do
for notification <- notifications do
case notification.type do
:spike ->
- current_visitors = clickhouse.current_visitors(notification.site)
+ current_visitors = Clickhouse.current_visitors(notification.site)
if current_visitors >= notification.threshold do
- query = Query.from(notification.site, %{"period" => "realtime"})
- sources = clickhouse.top_sources_for_spike(notification.site, query, 3, 1)
+ stats =
+ notification.site
+ |> get_traffic_spike_stats()
+ |> Map.put(:current_visitors, current_visitors)
- notify_spike(notification, current_visitors, sources)
+ notify_spike(notification, stats, now)
end
:drop ->
- current_visitors = clickhouse.current_visitors_12h(notification.site)
+ current_visitors = Clickhouse.current_visitors_12h(notification.site)
if current_visitors < notification.threshold do
- notify_drop(notification, current_visitors)
+ notify_drop(notification, current_visitors, now)
end
end
end
@@ -51,27 +54,27 @@ defmodule Plausible.Workers.TrafficChangeNotifier do
:ok
end
- defp notify_spike(notification, current_visitors, sources) do
+ defp notify_spike(notification, stats, now) do
for recipient <- notification.recipients do
- send_spike_notification(recipient, notification.site, current_visitors, sources)
+ send_spike_notification(recipient, notification.site, stats)
end
notification
- |> TrafficChangeNotification.was_sent()
+ |> TrafficChangeNotification.was_sent(now)
|> Repo.update()
end
- defp notify_drop(notification, current_visitors) do
+ defp notify_drop(notification, current_visitors, now) do
for recipient <- notification.recipients do
send_drop_notification(recipient, notification.site, current_visitors)
end
notification
- |> TrafficChangeNotification.was_sent()
+ |> TrafficChangeNotification.was_sent(now)
|> Repo.update()
end
- defp send_spike_notification(recipient, site, current_visitors, sources) do
+ defp send_spike_notification(recipient, site, stats) do
dashboard_link =
if Repo.exists?(email_match_query(site, recipient)) do
Routes.stats_url(PlausibleWeb.Endpoint, :stats, site.domain, [])
@@ -81,8 +84,7 @@ defmodule Plausible.Workers.TrafficChangeNotifier do
PlausibleWeb.Email.spike_notification(
recipient,
site,
- current_visitors,
- sources,
+ stats,
dashboard_link
)
@@ -114,6 +116,44 @@ defmodule Plausible.Workers.TrafficChangeNotifier do
Plausible.Mailer.send(template)
end
+ defp get_traffic_spike_stats(site) do
+ {:ok, query} =
+ Query.build(
+ site,
+ :internal,
+ %{
+ "site_id" => "#{site.id}",
+ "metrics" => ["visitors"],
+ "pagination" => %{"limit" => 3},
+ "date_range" => "realtime"
+ },
+ %{}
+ )
+
+ %{}
+ |> put_sources(site, query)
+ |> put_pages(site, query)
+ end
+
+ defp put_sources(stats, site, query) do
+ query =
+ query
+ |> Query.set(dimensions: ["visit:source"])
+ |> Query.add_filter([:is_not, "visit:source", ["Direct / None"]])
+
+ %{results: sources} = Plausible.Stats.query(site, query)
+
+ Map.put(stats, :sources, sources)
+ end
+
+ defp put_pages(stats, site, query) do
+ query = Query.set(query, dimensions: ["event:page"])
+
+ %{results: pages} = Plausible.Stats.query(site, query)
+
+ Map.put(stats, :pages, pages)
+ end
+
defp email_match_query(site, recipient) do
from tm in Plausible.Teams.Membership,
inner_join: u in assoc(tm, :user),
diff --git a/test/plausible/stats/clickhouse_test.exs b/test/plausible/stats/clickhouse_test.exs
index 6dfd9f1990..5d682727f5 100644
--- a/test/plausible/stats/clickhouse_test.exs
+++ b/test/plausible/stats/clickhouse_test.exs
@@ -307,41 +307,6 @@ defmodule Plausible.Stats.ClickhouseTest do
end
end
- describe "top_sources_for_spike/4" do
- test "gets named sources" do
- site = insert(:site)
- query = Plausible.Stats.Query.from(site, %{"period" => "all"})
-
- populate_stats(site, [
- build(:pageview,
- pathname: "/",
- referrer_source: "Twitter"
- ),
- build(:pageview,
- pathname: "/plausible.io"
- ),
- build(:pageview,
- pathname: "/plausible.io",
- referrer_source: "Google"
- ),
- build(:pageview,
- pathname: "/plausible.io",
- referrer_source: "Google"
- ),
- build(:pageview,
- pathname: "/plausible.io",
- referrer_source: "Bing"
- )
- ])
-
- assert [
- %{count: 2, name: "Google"},
- %{count: 1, name: "Bing"},
- %{count: 1, name: "Twitter"}
- ] = Clickhouse.top_sources_for_spike(site, query, 5, 1)
- end
- end
-
describe "imported_pageview_counts/1" do
test "gets pageview counts for each of sites' imports" do
site = insert(:site)
diff --git a/test/workers/traffic_change_notifier_test.exs b/test/workers/traffic_change_notifier_test.exs
index 2f7aa706c3..1a66f26c40 100644
--- a/test/workers/traffic_change_notifier_test.exs
+++ b/test/workers/traffic_change_notifier_test.exs
@@ -2,7 +2,6 @@ defmodule Plausible.Workers.TrafficChangeNotifierTest do
use Plausible.DataCase, async: true
use Bamboo.Test
use Plausible.Teams.Test
- import Double
alias Plausible.Workers.TrafficChangeNotifier
describe "drops" do
@@ -16,10 +15,7 @@ defmodule Plausible.Workers.TrafficChangeNotifierTest do
)
- clickhouse_stub =
- stub(Plausible.Stats.Clickhouse, :current_visitors_12h, fn _site -> 1 end)
-
- TrafficChangeNotifier.perform(nil, clickhouse_stub)
+ TrafficChangeNotifier.perform(nil)
assert_no_emails_delivered()
end
@@ -34,10 +30,7 @@ defmodule Plausible.Workers.TrafficChangeNotifierTest do
recipients: ["[email protected]"]
)
- clickhouse_stub =
- stub(Plausible.Stats.Clickhouse, :current_visitors_12h, fn _site -> 1 end)
-
- TrafficChangeNotifier.perform(nil, clickhouse_stub)
+ TrafficChangeNotifier.perform(nil)
assert_email_delivered_with(
subject: "Traffic Drop on #{site.domain}",
@@ -50,14 +43,16 @@ defmodule Plausible.Workers.TrafficChangeNotifierTest do
insert(:drop_notification,
site: site,
- threshold: 10,
+ threshold: 2,
)
- clickhouse_stub =
- stub(Plausible.Stats.Clickhouse, :current_visitors_12h, fn _site -> 11 end)
+ populate_stats(site, [
+ build(:pageview, timestamp: minutes_ago(2)),
+ build(:pageview, timestamp: minutes_ago(1))
+ ])
- TrafficChangeNotifier.perform(nil, clickhouse_stub)
+ TrafficChangeNotifier.perform(nil)
assert_no_emails_delivered()
end
@@ -71,10 +66,7 @@ defmodule Plausible.Workers.TrafficChangeNotifierTest do
)
- clickhouse_stub =
- stub(Plausible.Stats.Clickhouse, :current_visitors_12h, fn _site -> 7 end)
-
- TrafficChangeNotifier.perform(nil, clickhouse_stub)
+ TrafficChangeNotifier.perform(nil)
assert_email_delivered_with(
subject: "Traffic Drop on #{site.domain}",
@@ -87,28 +79,32 @@ defmodule Plausible.Workers.TrafficChangeNotifierTest do
)
end
- test "does not notify anyone if a notification already went out in the last 12 hours" do
+ test "does not send notifications more than once every 12 hours" do
site = new_site()
insert(:drop_notification,
site: site,
- threshold: 10,
+ threshold: 1,
recipients: ["[email protected]"]
)
- clickhouse_stub =
- stub(Plausible.Stats.Clickhouse, :current_visitors_12h, fn _site -> 4 end)
-
- TrafficChangeNotifier.perform(nil, clickhouse_stub)
+ TrafficChangeNotifier.perform(nil, ~N[2021-01-01 00:00:00])
assert_email_delivered_with(
subject: "Traffic Drop on #{site.domain}",
to: [nil: "[email protected]"]
)
- TrafficChangeNotifier.perform(nil, clickhouse_stub)
+ TrafficChangeNotifier.perform(nil, ~N[2021-01-01 11:59:59])
assert_no_emails_delivered()
+
+ TrafficChangeNotifier.perform(nil, ~N[2021-01-01 12:00:01])
+
+ assert_email_delivered_with(
+ subject: "Traffic Drop on #{site.domain}",
+ to: [nil: "[email protected]"]
+ )
end
test "adds settings link if recipient has access to the site" do
@@ -121,10 +117,7 @@ defmodule Plausible.Workers.TrafficChangeNotifierTest do
recipients: ["[email protected]"]
)
- clickhouse_stub =
- stub(Plausible.Stats.Clickhouse, :current_visitors_12h, fn _site -> 6 end)
-
- TrafficChangeNotifier.perform(nil, clickhouse_stub)
+ TrafficChangeNotifier.perform(nil)
assert_email_delivered_with(
html_body: ~r|http://localhost:8000/example.com/installation\?flow=review|
@@ -138,15 +131,13 @@ defmodule Plausible.Workers.TrafficChangeNotifierTest do
insert(:spike_notification,
site: site,
- threshold: 10,
+ threshold: 2,
)
- clickhouse_stub =
- stub(Plausible.Stats.Clickhouse, :current_visitors, fn _site -> 5 end)
- |> stub(:top_sources_for_spike, fn _site, _query, _limit, _page -> [] end)
+ populate_stats(site, [build(:pageview, timestamp: minutes_ago(1))])
- TrafficChangeNotifier.perform(nil, clickhouse_stub)
+ TrafficChangeNotifier.perform(nil)
assert_no_emails_delivered()
end
@@ -156,15 +147,17 @@ defmodule Plausible.Workers.TrafficChangeNotifierTest do
insert(:spike_notification,
site: site,
- threshold: 10,
+ threshold: 2,
)
- clickhouse_stub =
- stub(Plausible.Stats.Clickhouse, :current_visitors, fn _site -> 10 end)
- |> stub(:top_sources_for_spike, fn _site, _query, _limit, _page -> [] end)
+ populate_stats(site, [
+ build(:pageview, timestamp: minutes_ago(3)),
+ build(:pageview, timestamp: minutes_ago(2)),
+ build(:pageview, timestamp: minutes_ago(1))
+ ])
- TrafficChangeNotifier.perform(nil, clickhouse_stub)
+ TrafficChangeNotifier.perform(nil)
assert_email_delivered_with(
subject: "Traffic Spike on #{site.domain}",
@@ -177,40 +170,140 @@ defmodule Plausible.Workers.TrafficChangeNotifierTest do
)
end
+ test "ignores 'Direct / None' source" do
+ site = new_site()
+
+ insert(:spike_notification,
+ site: site,
+ threshold: 5,
+ recipients: ["[email protected]"]
+ )
+
+ populate_stats(site, [
+ build(:pageview, referrer_source: "A", timestamp: minutes_ago(1)),
+ build(:pageview, referrer_source: "A", timestamp: minutes_ago(1)),
+ build(:pageview, referrer_source: "A", timestamp: minutes_ago(1)),
+ build(:pageview, referrer_source: "B", timestamp: minutes_ago(1)),
+ build(:pageview, referrer_source: "B", timestamp: minutes_ago(1)),
+ build(:pageview, referrer_source: "C", timestamp: minutes_ago(1)),
+ build(:pageview, timestamp: minutes_ago(1)),
+ build(:pageview, timestamp: minutes_ago(1))
+ ])
+
+ TrafficChangeNotifier.perform(nil)
+
+ assert_delivered_email_matches(%{
+ to: [nil: "[email protected]"],
+ html_body: html_body
+ })
+
+ assert html_body =~ "The top sources for current visitors:"
+ assert html_body =~ "<b>3</b> visitors from <b>A</b>"
+ assert html_body =~ "<b>2</b> visitors from <b>B</b>"
+ assert html_body =~ "<b>1</b> visitor from <b>C</b>"
+ assert html_body =~ "There are currently <b>8</b>"
+ end
+
+ test "does not list sources at all when everything is 'Direct / None'" do
+ site = new_site()
+
+ insert(:spike_notification,
+ site: site,
+ threshold: 1,
+ recipients: ["[email protected]"]
+ )
+
+ populate_stats(site, [
+ build(:pageview, timestamp: minutes_ago(1)),
+ build(:pageview, timestamp: minutes_ago(1))
+ ])
+
+ TrafficChangeNotifier.perform(nil)
+
+ assert_delivered_email_matches(%{
+ to: [nil: "[email protected]"],
+ html_body: html_body
+ })
+
+ refute html_body =~ "The top sources for current visitors:"
+ assert html_body =~ "There are currently <b>2</b>"
+ end
+
+ test "includes top 3 pages" do
+ site = new_site()
+
+ insert(:spike_notification,
+ site: site,
+ threshold: 10,
+ recipients: ["[email protected]"]
+ )
+
+ populate_stats(site, [
+ build(:pageview, pathname: "/one", timestamp: minutes_ago(1)),
+ build(:pageview, pathname: "/one", timestamp: minutes_ago(1)),
+ build(:pageview, pathname: "/one", timestamp: minutes_ago(1)),
+ build(:pageview, pathname: "/one", timestamp: minutes_ago(1)),
+ build(:pageview, pathname: "/two", timestamp: minutes_ago(1)),
+ build(:pageview, pathname: "/two", timestamp: minutes_ago(1)),
+ build(:pageview, pathname: "/two", timestamp: minutes_ago(1)),
+ build(:pageview, timestamp: minutes_ago(1)),
+ build(:pageview, timestamp: minutes_ago(1)),
+ build(:pageview, pathname: "/not-this-one", timestamp: minutes_ago(1))
+ ])
+
+ TrafficChangeNotifier.perform(nil)
+
+ assert_delivered_email_matches(%{
+ to: [nil: "[email protected]"],
+ html_body: html_body
+ })
+
+ assert html_body =~ "There are currently <b>10</b>"
+
+ assert html_body =~ "Your top pages being visited:"
+ assert html_body =~ "<b>4</b> visitors on <b>/one</b>"
+ assert html_body =~ "<b>3</b> visitors on <b>/two</b>"
+ assert html_body =~ "<b>2</b> visitors on <b>/</b>"
+
+ refute html_body =~ "/not-this-one"
+ end
+
test "does not check site if it is locked" do
site = insert(:site, locked: true)
insert(:spike_notification,
site: site,
- threshold: 10,
+ threshold: 1,
recipients: ["[email protected]"]
)
- clickhouse_stub =
- stub(Plausible.Stats.Clickhouse, :current_visitors, fn _site -> 10 end)
- |> stub(:top_sources_for_spike, fn _site, _query, _limit, _page -> [] end)
+ populate_stats(site, [
+ build(:pageview, timestamp: minutes_ago(1)),
+ build(:pageview, timestamp: minutes_ago(1))
+ ])
- TrafficChangeNotifier.perform(nil, clickhouse_stub)
+ TrafficChangeNotifier.perform(nil)
assert_no_emails_delivered()
end
- test "does not notify anyone if a notification already went out in the last 12 hours" do
+ test "does not send notifications more than once every 12 hours" do
site = new_site()
- insert(:spike_notification, site: site, threshold: 10, recipients: ["[email protected]"])
+ insert(:spike_notification, site: site, threshold: 1, recipients: ["[email protected]"])
- clickhouse_stub =
- stub(Plausible.Stats.Clickhouse, :current_visitors, fn _site -> 10 end)
- |> stub(:top_sources_for_spike, fn _site, _query, _limit, _page -> [] end)
+ populate_stats(site, [
+ build(:pageview, timestamp: minutes_ago(1)),
+ build(:pageview, timestamp: minutes_ago(1))
+ ])
- TrafficChangeNotifier.perform(nil, clickhouse_stub)
+ TrafficChangeNotifier.perform(nil)
assert_email_delivered_with(
subject: "Traffic Spike on #{site.domain}",
to: [nil: "[email protected]"]
)
- TrafficChangeNotifier.perform(nil, clickhouse_stub)
+ TrafficChangeNotifier.perform(nil)
assert_no_emails_delivered()
end
@@ -218,15 +311,20 @@ defmodule Plausible.Workers.TrafficChangeNotifierTest do
test "adds a dashboard link if recipient has access to the site" do
user = new_user(email: "[email protected]")
site = new_site(domain: "example.com", owner: user)
- insert(:spike_notification, site: site, threshold: 10, recipients: ["[email protected]"])
+ insert(:spike_notification, site: site, threshold: 1, recipients: ["[email protected]"])
- clickhouse_stub =
- stub(Plausible.Stats.Clickhouse, :current_visitors, fn _site -> 10 end)
- |> stub(:top_sources_for_spike, fn _site, _query, _limit, _page -> [] end)
+ populate_stats(site, [
+ build(:pageview, timestamp: minutes_ago(1)),
+ build(:pageview, timestamp: minutes_ago(1))
+ ])
- TrafficChangeNotifier.perform(nil, clickhouse_stub)
+ TrafficChangeNotifier.perform(nil)
assert_email_delivered_with(html_body: ~r/View dashboard:\s+<a href=\"http.+\/example.com/)
end
end
+
+ def minutes_ago(min) do
+ NaiveDateTime.utc_now(:second) |> NaiveDateTime.add(-min, :minute)
+ end
end