Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Refactor spike detection top sources query (#3770)
Karl-Aksel Puulmannf3509f2a17c1
diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.exindex 4c0d34c246..782b868320 100644--- a/lib/plausible/stats/clickhouse.ex+++ b/lib/plausible/stats/clickhouse.ex@@ -9,8 +9,6 @@ defmodule Plausible.Stats.Clickhouse doalias Plausible.Stats.Queryalias Plausible.Timezones- @no_ref "Direct / None"-@spec pageview_start_date_local(Plausible.Site.t()) :: Date.t() | nildef pageview_start_date_local(site) dodatetime =@@ -83,89 +81,32 @@ defmodule Plausible.Stats.Clickhouse dodef usage_breakdown([], _date_range), do: {0, 0}- def top_sources(site, query, limit, page, show_noref \\ false, include_details) do+ def top_sources_for_spike(site, query, limit, page) dooffset = (page - 1) * limit+ {first_datetime, last_datetime} = utc_boundaries(query, site)+referrers =- from(s in base_session_query(site, query),+ from(s in "sessions_v2",+ select: %{+ name: s.referrer_source,+ count: uniq(s.user_id)+ },+ where: s.site_id == ^site.id,+ # Note: This query intentionally uses session end timestamp to get currently active users+ where: s.timestamp >= ^first_datetime and s.start < ^last_datetime,+ where: s.referrer_source != "",group_by: s.referrer_source,- order_by: [desc: uniq(s.user_id), asc: fragment("min(start)")],+ order_by: [desc: uniq(s.user_id), asc: s.referrer_source],limit: ^limit,offset: ^offset)- |> filter_converted_sessions(site, query)-- referrers =- if show_noref do- referrers- else- from(s in referrers, where: s.referrer_source != "")- end-- referrers = apply_page_as_entry_page(referrers, site, query)-- referrers =- if include_details do- from(- s in referrers,- select: %{- name:- fragment(- "if(empty(?), ?, ?) as name",- s.referrer_source,- @no_ref,- s.referrer_source- ),- url: fragment("any(?)", s.referrer),- count: uniq(s.user_id),- bounce_rate: bounce_rate(),- visit_duration: visit_duration()- }- )- else- from(- s in referrers,- select: %{- name:- fragment(- "if(empty(?), ?, ?) as name",- s.referrer_source,- @no_ref,- s.referrer_source- ),- url: fragment("any(?)", s.referrer),- count: uniq(s.user_id)- }- )- end-- ClickhouseRepo.all(referrers)- |> Enum.map(fn ref ->- Map.update(ref, :url, nil, fn url -> url && URI.parse("http://" <> url).host end)- end)- end-- defp filter_converted_sessions(db_query, site, query) do- goal = query.filters["goal"]- page = query.filters[:page]- if is_binary(goal) || is_binary(page) do- converted_sessions =- from(e in base_query(site, query),- select: %{session_id: fragment("DISTINCT ?", e.session_id)}- )-- from(s in db_query,- join: cs in subquery(converted_sessions),- on: s.session_id == cs.session_id- )- else- db_query+ on_full_build do+ referrers = Plausible.Stats.Sampling.add_query_hint(referrers, 10_000_000)end- end- defp apply_page_as_entry_page(db_query, _site, query) do- include_path_filter_entry(db_query, query.filters[:page])+ ClickhouseRepo.all(referrers)enddef current_visitors(site, query) do@@ -299,128 +240,6 @@ defmodule Plausible.Stats.Clickhouse doendend- defp base_session_query(site, query) do- {first_datetime, last_datetime} = utc_boundaries(query, site)-- q =- from(s in "sessions_v2",- where: s.site_id == ^site.id,- where: s.timestamp >= ^first_datetime and s.start < ^last_datetime- )-- on_full_build do- q = Plausible.Stats.Sampling.add_query_hint(q, 10_000_000)- end-- q =- if query.filters["source"] do- source = query.filters["source"]- source = if source == @no_ref, do: "", else: source- from(s in q, where: s.referrer_source == ^source)- else- q- end-- q =- if query.filters["screen"] do- size = query.filters["screen"]- from(s in q, where: s.screen_size == ^size)- else- q- end-- q =- if query.filters["browser"] do- browser = query.filters["browser"]- from(s in q, where: s.browser == ^browser)- else- q- end-- q =- if query.filters["browser_version"] do- version = query.filters["browser_version"]- from(s in q, where: s.browser_version == ^version)- else- q- end-- q =- if query.filters["os"] do- os = query.filters["os"]- from(s in q, where: s.operating_system == ^os)- else- q- end-- q =- if query.filters["os_version"] do- version = query.filters["os_version"]- from(s in q, where: s.operating_system_version == ^version)- else- q- end-- q =- if query.filters["country"] do- country = query.filters["country"]- from(s in q, where: s.country_code == ^country)- else- q- end-- q =- if query.filters["utm_medium"] do- utm_medium = query.filters["utm_medium"]- from(s in q, where: s.utm_medium == ^utm_medium)- else- q- end-- q =- if query.filters["utm_source"] do- utm_source = query.filters["utm_source"]- from(s in q, where: s.utm_source == ^utm_source)- else- q- end-- q =- if query.filters["utm_campaign"] do- utm_campaign = query.filters["utm_campaign"]- from(s in q, where: s.utm_campaign == ^utm_campaign)- else- q- end-- q =- if query.filters["utm_content"] do- utm_content = query.filters["utm_content"]- from(s in q, where: s.utm_content == ^utm_content)- else- q- end-- q =- if query.filters["utm_term"] do- utm_term = query.filters["utm_term"]- from(s in q, where: s.utm_term == ^utm_term)- else- q- end-- q = include_path_filter_entry(q, query.filters["entry_page"])-- q = include_path_filter_exit(q, query.filters["exit_page"])-- if query.filters["referrer"] do- ref = query.filters["referrer"]- from(s in q, where: s.referrer == ^ref)- else- q- end- end-defp base_query_bare(site, query) do{first_datetime, last_datetime} = utc_boundaries(query, site)@@ -672,52 +491,6 @@ defmodule Plausible.Stats.Clickhouse doendend- defp include_path_filter_entry(db_query, path) do- if path do- {negated, path} = check_negated_filter(path)- {contains_regex, path_regex} = convert_path_regex(path)-- if contains_regex do- if negated do- from(e in db_query, where: fragment("not(match(?, ?))", e.entry_page, ^path_regex))- else- from(e in db_query, where: fragment("match(?, ?)", e.entry_page, ^path_regex))- end- else- if negated do- from(e in db_query, where: e.entry_page != ^path)- else- from(e in db_query, where: e.entry_page == ^path)- end- end- else- db_query- end- end-- defp include_path_filter_exit(db_query, path) do- if path do- {negated, path} = check_negated_filter(path)- {contains_regex, path_regex} = convert_path_regex(path)-- if contains_regex do- if negated do- from(e in db_query, where: fragment("not(match(?, ?))", e.exit_page, ^path_regex))- else- from(e in db_query, where: fragment("match(?, ?)", e.exit_page, ^path_regex))- end- else- if negated do- from(e in db_query, where: e.exit_page != ^path)- else- from(e in db_query, where: e.exit_page == ^path)- end- end- else- db_query- end- end-defp beginning_of_time(candidate, site_creation_date) doif Timex.after?(site_creation_date, candidate) dosite_creation_datediff --git a/lib/workers/spike_notifier.ex b/lib/workers/spike_notifier.exindex 4e0b0c4100..3505f5564f 100644--- a/lib/workers/spike_notifier.ex+++ b/lib/workers/spike_notifier.ex@@ -23,7 +23,7 @@ defmodule Plausible.Workers.SpikeNotifier docurrent_visitors = clickhouse.current_visitors(notification.site, query)if current_visitors >= notification.threshold do- sources = clickhouse.top_sources(notification.site, query, 3, 1, true)+ sources = clickhouse.top_sources_for_spike(notification.site, query, 3, 1)notify(notification, current_visitors, sources)endenddiff --git a/test/plausible/stats/clickhouse_test.exs b/test/plausible/stats/clickhouse_test.exsindex 4c6c5dfc7d..3e36dbd7a1 100644--- a/test/plausible/stats/clickhouse_test.exs+++ b/test/plausible/stats/clickhouse_test.exs@@ -219,4 +219,39 @@ defmodule Plausible.Stats.ClickhouseTest do} = Clickhouse.last_24h_visitors_hourly_intervals([site], fixed_now)[site.domain]endend++ describe "top_sources_for_spike/4" do+ test "gets named sources" do+ site = insert(:site)+ query = Plausible.Stats.Query.from(site, %{"period" => "all"})++ populate_stats(site, [+ build(:pageview,+ pathname: "/",+ referrer_source: "Twitter"+ ),+ build(:pageview,+ pathname: "/plausible.io"+ ),+ build(:pageview,+ pathname: "/plausible.io",+ referrer_source: "Google"+ ),+ build(:pageview,+ pathname: "/plausible.io",+ referrer_source: "Google"+ ),+ build(:pageview,+ pathname: "/plausible.io",+ referrer_source: "Bing"+ )+ ])++ assert [+ %{count: 2, name: "Google"},+ %{count: 1, name: "Bing"},+ %{count: 1, name: "Twitter"}+ ] = Clickhouse.top_sources_for_spike(site, query, 5, 1)+ end+ endenddiff --git a/test/workers/spike_notifier_test.exs b/test/workers/spike_notifier_test.exsindex 35857e64f3..522f05d7ed 100644--- a/test/workers/spike_notifier_test.exs+++ b/test/workers/spike_notifier_test.exs@@ -15,7 +15,7 @@ defmodule Plausible.Workers.SpikeNotifierTest doclickhouse_stub =stub(Plausible.Stats.Clickhouse, :current_visitors, fn _site, _query -> 5 end)- |> stub(:top_sources, fn _site, _query, _limit, _page, _show_noref -> [] end)+ |> stub(:top_sources_for_spike, fn _site, _query, _limit, _page -> [] end)SpikeNotifier.perform(nil, clickhouse_stub)@@ -33,7 +33,7 @@ defmodule Plausible.Workers.SpikeNotifierTest doclickhouse_stub =stub(Plausible.Stats.Clickhouse, :current_visitors, fn _site, _query -> 10 end)- |> stub(:top_sources, fn _site, _query, _limit, _page, _show_noref -> [] end)+ |> stub(:top_sources_for_spike, fn _site, _query, _limit, _page -> [] end)SpikeNotifier.perform(nil, clickhouse_stub)@@ -59,7 +59,7 @@ defmodule Plausible.Workers.SpikeNotifierTest doclickhouse_stub =stub(Plausible.Stats.Clickhouse, :current_visitors, fn _site, _query -> 10 end)- |> stub(:top_sources, fn _site, _query, _limit, _page, _show_noref -> [] end)+ |> stub(:top_sources_for_spike, fn _site, _query, _limit, _page -> [] end)SpikeNotifier.perform(nil, clickhouse_stub)@@ -72,7 +72,7 @@ defmodule Plausible.Workers.SpikeNotifierTest doclickhouse_stub =stub(Plausible.Stats.Clickhouse, :current_visitors, fn _site, _query -> 10 end)- |> stub(:top_sources, fn _site, _query, _limit, _page, _show_noref -> [] end)+ |> stub(:top_sources_for_spike, fn _site, _query, _limit, _page -> [] end)SpikeNotifier.perform(nil, clickhouse_stub)@@ -93,7 +93,7 @@ defmodule Plausible.Workers.SpikeNotifierTest doclickhouse_stub =stub(Plausible.Stats.Clickhouse, :current_visitors, fn _site, _query -> 10 end)- |> stub(:top_sources, fn _site, _query, _limit, _page, _show_noref -> [] end)+ |> stub(:top_sources_for_spike, fn _site, _query, _limit, _page -> [] end)SpikeNotifier.perform(nil, clickhouse_stub)