Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Remove dead code
Uku Taht01706b7590df
diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.exindex 71ac9d3bef..098c93a32f 100644--- a/lib/plausible/stats/clickhouse.ex+++ b/lib/plausible/stats/clickhouse.ex@@ -5,175 +5,6 @@ defmodule Plausible.Stats.Clickhouse douse Plausible.Stats.Fragments@no_ref "Direct / None"- def compare_pageviews_and_visitors(site, query, {pageviews, visitors}) do- query = Query.shift_back(query, site)- {old_pageviews, old_visitors} = pageviews_and_visitors(site, query)-- cond do- old_pageviews == 0 and pageviews > 0 ->- {100, 100}-- old_pageviews == 0 and pageviews == 0 ->- {0, 0}-- true ->- {- round((pageviews - old_pageviews) / old_pageviews * 100),- round((visitors - old_visitors) / old_visitors * 100)- }- end- end-- def calculate_plot(site, %Query{interval: "month"} = query) do- n_steps = Timex.diff(query.date_range.last, query.date_range.first, :months)-- steps =- Enum.map(n_steps..0, fn shift ->- Timex.now(site.timezone)- |> Timex.beginning_of_month()- |> Timex.shift(months: -shift)- |> DateTime.to_date()- end)-- groups =- ClickhouseRepo.all(- from e in base_query_w_sessions(site, query),- select:- {fragment("toStartOfMonth(toTimeZone(?, ?)) as month", e.timestamp, ^site.timezone),- uniq(e.user_id)},- group_by: fragment("month"),- order_by: fragment("month")- )- |> Enum.into(%{})-- present_index =- Enum.find_index(steps, fn step ->- step == Timex.now(site.timezone) |> Timex.to_date() |> Timex.beginning_of_month()- end)-- plot = Enum.map(steps, fn step -> groups[step] || 0 end)- labels = Enum.map(steps, fn step -> Timex.format!(step, "{ISOdate}") end)-- {plot, labels, present_index}- end-- def calculate_plot(site, %Query{interval: "date"} = query) do- steps = Enum.into(query.date_range, [])-- groups =- ClickhouseRepo.all(- from e in base_query_w_sessions(site, query),- select:- {fragment("toDate(toTimeZone(?, ?)) as day", e.timestamp, ^site.timezone),- uniq(e.user_id)},- group_by: fragment("day"),- order_by: fragment("day")- )- |> Enum.into(%{})-- present_index =- Enum.find_index(steps, fn step -> step == Timex.now(site.timezone) |> Timex.to_date() end)-- steps_to_show = if present_index, do: present_index + 1, else: Enum.count(steps)- plot = Enum.map(steps, fn step -> groups[step] || 0 end) |> Enum.take(steps_to_show)- labels = Enum.map(steps, fn step -> Timex.format!(step, "{ISOdate}") end)-- {plot, labels, present_index}- end-- def calculate_plot(site, %Query{interval: "hour"} = query) do- steps = 0..23-- groups =- ClickhouseRepo.all(- from e in base_query_w_sessions(site, query),- select:- {fragment("toHour(toTimeZone(?, ?)) as hour", e.timestamp, ^site.timezone),- uniq(e.user_id)},- group_by: fragment("hour"),- order_by: fragment("hour")- )- |> Enum.into(%{})-- now = Timex.now(site.timezone)- is_today = Timex.to_date(now) == query.date_range.first- present_index = is_today && Enum.find_index(steps, fn step -> step == now.hour end)- steps_to_show = if present_index, do: present_index + 1, else: Enum.count(steps)-- labels =- Enum.map(steps, fn step ->- Timex.to_datetime(query.date_range.first)- |> Timex.shift(hours: step)- |> NaiveDateTime.to_iso8601()- end)-- plot = Enum.map(steps, fn step -> groups[step] || 0 end) |> Enum.take(steps_to_show)- {plot, labels, present_index}- end-- def calculate_plot(site, %Query{period: "realtime"} = query) do- query = %Query{query | period: "30m"}-- groups =- ClickhouseRepo.all(- from e in base_query_w_sessions(site, query),- select: {- fragment("dateDiff('minute', now(), ?) as relativeMinute", e.timestamp),- total()- },- group_by: fragment("relativeMinute"),- order_by: fragment("relativeMinute")- )- |> Enum.into(%{})-- labels = Enum.into(-30..-1, [])- plot = Enum.map(labels, fn label -> groups[label] || 0 end)- {plot, labels, nil}- end-- def bounce_rate(site, query) do- q = base_session_query(site, query) |> apply_page_as_entry_page(site, query)-- ClickhouseRepo.one(- from s in q,- select: bounce_rate()- ) || 0- end-- def visit_duration(site, query) do- q = base_session_query(site, query) |> apply_page_as_entry_page(site, query)-- ClickhouseRepo.one(- from s in q,- select: visit_duration()- ) || 0- end-- def total_pageviews(site, %Query{period: "realtime"} = query) do- query = %Query{query | period: "30m"}-- q = base_session_query(site, query) |> apply_page_as_entry_page(site, query)-- ClickhouseRepo.one(- from e in q,- select: fragment("sum(sign * pageviews)")- )- end-- def total_pageviews(site, query) do- ClickhouseRepo.one(- from e in base_query_w_sessions(site, query),- select: total()- )- end-- def total_events(site, query) do- ClickhouseRepo.one(- from e in base_query_w_sessions(site, query),- select: total()- )- end-def usage_breakdown(domains) dorange =Date.range(@@ -203,53 +34,6 @@ defmodule Plausible.Stats.Clickhouse doend)end- def pageviews_and_visitors(site, query) do- {pageviews, visitors, _} = pageviews_and_visitors_with_sample_percent(site, query)- {pageviews, visitors}- end-- def pageviews_and_visitors_with_sample_percent(site, query) do- ClickhouseRepo.one(- from e in base_query_w_sessions(site, query),- select: {total(), uniq(e.user_id), sample_percent()}- )- end-- def unique_visitors(site, query) do- {visitors, _} = unique_visitors_with_sample_percent(site, query)- visitors- end-- def unique_visitors_with_sample_percent(site, query) do- query = if query.period == "realtime", do: %Query{query | period: "30m"}, else: query-- ClickhouseRepo.one(- from e in base_query_w_sessions(site, query),- select: {uniq(e.user_id), sample_percent()}- )- end-- def top_referrers_for_goal(site, query, limit, page) do- offset = (page - 1) * limit-- ClickhouseRepo.all(- from s in base_query_w_sessions(site, query),- where: s.referrer_source != "",- group_by: s.referrer_source,- order_by: [desc: uniq(s.user_id)],- limit: ^limit,- offset: ^offset,- select: %{- name: s.referrer_source,- url: fragment("any(?)", s.referrer),- count: uniq(s.user_id)- }- )- |> Enum.map(fn ref ->- Map.update(ref, :url, nil, fn url -> url && URI.parse("http://" <> url).host end)- end)- end-def top_sources(site, query, limit, page, show_noref \\ false, include_details) dooffset = (page - 1) * limit@@ -335,539 +119,6 @@ defmodule Plausible.Stats.Clickhouse doinclude_path_filter_entry(db_query, query.filters["page"])end- def utm_mediums(site, query, limit \\ 9, page \\ 1, show_noref \\ false) do- offset = (page - 1) * limit-- q =- from(- s in base_session_query(site, query),- group_by: s.utm_medium,- order_by: [desc: uniq(s.user_id), asc: min(s.start)],- limit: ^limit,- offset: ^offset,- select: %{- name: coalesce_string(s.utm_medium, @no_ref),- count: uniq(s.user_id),- bounce_rate: bounce_rate(),- visit_duration: visit_duration()- }- )-- q =- if show_noref do- q- else- from(s in q, where: s.utm_medium != "")- end-- q- |> filter_converted_sessions(site, query)- |> ClickhouseRepo.all()- end-- def utm_campaigns(site, query, limit \\ 9, page \\ 1, show_noref \\ false) do- offset = (page - 1) * limit-- q =- from(- s in base_session_query(site, query),- group_by: s.utm_campaign,- order_by: [desc: uniq(s.user_id), asc: min(s.start)],- limit: ^limit,- offset: ^offset,- select: %{- name: coalesce_string(s.utm_campaign, @no_ref),- count: uniq(s.user_id),- bounce_rate: bounce_rate(),- visit_duration: visit_duration()- }- )-- q =- if show_noref do- q- else- from(s in q, where: s.utm_campaign != "")- end-- q- |> filter_converted_sessions(site, query)- |> ClickhouseRepo.all()- end-- def utm_sources(site, query, limit \\ 9, page \\ 1, show_noref \\ false) do- offset = (page - 1) * limit-- q =- from(- s in base_session_query(site, query),- group_by: s.utm_source,- order_by: [desc: uniq(s.user_id), asc: min(s.start)],- limit: ^limit,- offset: ^offset,- select: %{- name: coalesce_string(s.utm_source, @no_ref),- count: uniq(s.user_id),- bounce_rate: bounce_rate(),- visit_duration: visit_duration()- }- )-- q =- if show_noref do- q- else- from(s in q, where: s.utm_source != "")- end-- q- |> filter_converted_sessions(site, query)- |> ClickhouseRepo.all()- end-- def conversions_from_referrer(site, query, referrer) do- converted_sessions =- from(- from e in base_query(site, query),- select: %{session_id: e.session_id}- )-- ClickhouseRepo.one(- from s in Plausible.ClickhouseSession,- join: cs in subquery(converted_sessions),- on: s.session_id == cs.session_id,- where: s.referrer_source == ^referrer,- select: uniq(s.user_id)- )- end-- def referrer_drilldown(site, query, referrer, include_details, limit) do- referrer = if referrer == @no_ref, do: "", else: referrer-- q =- from(- s in base_session_query(site, query),- group_by: s.referrer,- where: s.referrer_source == ^referrer,- order_by: [desc: uniq(s.user_id)],- limit: ^limit- )- |> filter_converted_sessions(site, query)-- q =- if include_details do- from(- s in q,- select: %{- name: coalesce_string(s.referrer, @no_ref),- count: uniq(s.user_id),- bounce_rate: bounce_rate(),- visit_duration: visit_duration()- }- )- else- from(s in q,- select: %{- name: coalesce_string(s.referrer, @no_ref),- count: uniq(s.user_id)- }- )- end-- ClickhouseRepo.all(q)- |> Enum.map(fn ref ->- url = if ref[:name] !== "", do: URI.parse("http://" <> ref[:name]).host- Map.put(ref, :url, url)- end)- end-- def referrer_drilldown_for_goal(site, query, referrer) do- Plausible.ClickhouseRepo.all(- from s in base_query_w_sessions(site, query),- where: s.referrer_source == ^referrer,- group_by: s.referrer,- order_by: [desc: uniq(s.user_id)],- limit: 100,- select: %{- name: s.referrer,- count: uniq(s.user_id)- }- )- end-- def entry_pages(site, query, limit, page \\ 1) do- offset = (page - 1) * limit-- q =- from(- s in base_session_query(site, query),- group_by: s.entry_page,- order_by: [desc: uniq(s.user_id)],- limit: ^limit,- offset: ^offset,- select: %{- name: s.entry_page,- count: uniq(s.user_id),- entries: uniq(s.session_id),- visit_duration: visit_duration()- }- )- |> filter_converted_sessions(site, query)-- ClickhouseRepo.all(q)- end-- def exit_pages(site, query, limit, page \\ 1) do- offset = (page - 1) * limit-- q =- from(- s in base_session_query(site, query),- group_by: s.exit_page,- order_by: [desc: uniq(s.user_id)],- limit: ^limit,- offset: ^offset,- where: s.exit_page != "",- select: %{- name: s.exit_page,- count: uniq(s.user_id),- exits: uniq(s.session_id)- }- )- |> filter_converted_sessions(site, query)-- result = ClickhouseRepo.all(q)-- if Enum.count(result) > 0 do- pages = Enum.map(result, fn r -> r[:name] end)-- event_q =- from(e in base_query_w_session_based_pageviews(site, query),- group_by: e.pathname,- where: fragment("? IN tuple(?)", e.pathname, ^pages),- where: e.name == "pageview",- select: {- e.pathname,- total()- }- )-- total_pageviews = ClickhouseRepo.all(event_q) |> Enum.into(%{})-- Enum.map(result, fn r ->- if Map.get(total_pageviews, r[:name]) do- exit_rate = r[:exits] / Map.get(total_pageviews, r[:name]) * 100- Map.put(r, :exit_rate, Float.floor(exit_rate))- else- Map.put(r, :exit_rate, nil)- end- end)- else- result- end- end-- def top_pages(site, %Query{period: "realtime"} = query, limit, page, _include_details) do- offset = (page - 1) * limit-- q = base_session_query(site, query) |> apply_page_as_entry_page(site, query)-- ClickhouseRepo.all(- from s in q,- group_by: s.exit_page,- order_by: [desc: uniq(s.user_id)],- limit: ^limit,- offset: ^offset,- select: %{- name: s.exit_page,- count: uniq(s.user_id)- }- )- end-- def top_pages(site, query, limit, page, include_details) do- offset = (page - 1) * limit-- q =- from(- e in base_query_w_sessions(site, query),- group_by: e.pathname,- order_by: [desc: uniq(e.user_id)],- limit: ^limit,- offset: ^offset,- select: %{- name: e.pathname,- count: uniq(e.user_id),- pageviews: total()- }- )-- pages = ClickhouseRepo.all(q)-- if include_details do- [{bounce_state, bounce_result}, {time_state, time_result}] =- Task.yield_many(- [- Task.async(fn -> bounce_rates_by_page_url(site, query) end),- Task.async(fn ->- {:ok, page_times} =- page_times_by_page_url(site, query, Enum.map(pages, fn p -> p.name end))-- page_times.rows |> Enum.map(fn [a, b] -> {a, b} end) |> Enum.into(%{})- end)- ],- 15000- )- |> Enum.map(fn {task, response} ->- case response do- nil ->- Task.shutdown(task, :brutal_kill)- {nil, nil}-- {:ok, result} ->- {:ok, result}-- _ ->- response- end- end)-- Enum.map(pages, fn page ->- if bounce_state == :ok,- do: Map.put(page, :bounce_rate, bounce_result[page[:name]]),- else: page- end)- |> Enum.map(fn page ->- if time_state == :ok do- time = time_result[page[:name]]-- Map.put(- page,- :time_on_page,- if(time, do: round(time), else: nil)- )- else- page- end- end)- else- pages- end- end-- defp bounce_rates_by_page_url(site, query) do- q = base_session_query(site, query) |> apply_page_as_entry_page(site, query)-- ClickhouseRepo.all(- from s in q,- group_by: s.entry_page,- order_by: [desc: total()],- limit: 100,- select: %{- entry_page: s.entry_page,- total: total(),- bounce_rate: bounce_rate()- }- )- |> Enum.map(fn row -> {row[:entry_page], row[:bounce_rate]} end)- |> Enum.into(%{})- end-- def page_times_by_page_url(site, query, page_list \\ nil)-- def page_times_by_page_url(site, query, _page_list = nil) do- {negated, updated_page_selection} = query.filters["page"] |> check_negated_filter()- {_, page_regex} = updated_page_selection |> convert_path_regex()-- q =- from(- e in base_query_w_sessions(site, %Query{- query- | filters: Map.delete(query.filters, "page")- }),- select: {- fragment("? as p", e.pathname),- fragment("? as t", e.timestamp),- fragment("? as s", e.session_id)- },- order_by: [e.session_id, e.timestamp]- )-- {base_query_raw, base_query_raw_params} = ClickhouseRepo.to_sql(:all, q)-- time_query = "- SELECT- avg(ifNotFinite(avgTime, null))- FROM- (SELECT- p,- sum(td)/count(case when p2 != p then 1 end) as avgTime- FROM- (SELECT- p,- p2,- sum(t2-t) as td- FROM- (SELECT- *,- neighbor(t, 1) as t2,- neighbor(p, 1) as p2,- neighbor(s, 1) as s2- FROM (#{base_query_raw}))- WHERE s=s2 AND- #{if negated, do: "not(match(p, ?))", else: "match(p, ?)"}- GROUP BY p,p2,s)- GROUP BY p)"-- time_query |> ClickhouseRepo.query(base_query_raw_params ++ [page_regex])- end-- def page_times_by_page_url(site, query, page_list) do- q =- from(- e in base_query_w_sessions(site, %Query{- query- | filters: Map.delete(query.filters, "page")- }),- select: {- fragment("? as p", e.pathname),- fragment("? as t", e.timestamp),- fragment("? as s", e.session_id)- },- order_by: [e.session_id, e.timestamp]- )-- {base_query_raw, base_query_raw_params} = ClickhouseRepo.to_sql(:all, q)-- time_query = "- SELECT- p,- sum(td)/count(case when p2 != p then 1 end) as avgTime- FROM- (SELECT- p,- p2,- sum(t2-t) as td- FROM- (SELECT- *,- neighbor(t, 1) as t2,- neighbor(p, 1) as p2,- neighbor(s, 1) as s2- FROM (#{base_query_raw}))- WHERE s=s2 AND p IN tuple(?)- GROUP BY p,p2,s)- GROUP BY p"-- time_query- |> ClickhouseRepo.query(- base_query_raw_params ++ [(Enum.count(page_list) > 0 && page_list) || ["/"]]- )- end-- defp add_percentages(stat_list) do- total = Enum.reduce(stat_list, 0, fn %{count: count}, total -> total + count end)-- Enum.map(stat_list, fn stat ->- Map.put(stat, :percentage, round(stat[:count] / total * 100))- end)- end-- def top_screen_sizes(site, query) do- ClickhouseRepo.all(- from e in base_query_w_sessions(site, query),- group_by: e.screen_size,- where: e.screen_size != "",- order_by: [desc: uniq(e.user_id)],- select: %{- name: e.screen_size,- count: uniq(e.user_id)- }- )- |> add_percentages- end-- def countries(site, query) do- ClickhouseRepo.all(- from e in base_query_w_sessions(site, query),- group_by: e.country_code,- where: e.country_code != "\0\0",- order_by: [desc: uniq(e.user_id)],- select: %{- name: e.country_code,- count: uniq(e.user_id)- }- )- |> Enum.map(fn stat ->- two_letter_code = stat[:name]-- stat- |> Map.put(:name, Plausible.Stats.CountryName.to_alpha3(two_letter_code))- |> Map.put(:full_country_name, Plausible.Stats.CountryName.from_iso3166(two_letter_code))- end)- |> add_percentages- end-- def browsers(site, query, limit \\ 5) do- ClickhouseRepo.all(- from e in base_query_w_sessions(site, query),- group_by: e.browser,- where: e.browser != "",- order_by: [desc: uniq(e.user_id)],- select: %{- name: e.browser,- count: uniq(e.user_id)- }- )- |> add_percentages- |> Enum.take(limit)- end-- def browser_versions(site, query, limit \\ 5) do- ClickhouseRepo.all(- from e in base_query_w_sessions(site, query),- group_by: e.browser_version,- where: e.browser_version != "",- order_by: [desc: uniq(e.user_id)],- select: %{- name: e.browser_version,- count: uniq(e.user_id)- }- )- |> add_percentages- |> Enum.take(limit)- end-- def operating_systems(site, query, limit \\ 5) do- ClickhouseRepo.all(- from e in base_query_w_sessions(site, query),- group_by: e.operating_system,- where: e.operating_system != "",- order_by: [desc: uniq(e.user_id)],- select: %{- name: e.operating_system,- count: uniq(e.user_id)- }- )- |> add_percentages- |> Enum.take(limit)- end-- def operating_system_versions(site, query, limit \\ 5) do- ClickhouseRepo.all(- from e in base_query_w_sessions(site, query),- group_by: e.operating_system_version,- where: e.operating_system_version != "",- order_by: [desc: uniq(e.user_id)],- select: %{- name: e.operating_system_version,- count: uniq(e.user_id)- }- )- |> add_percentages- |> Enum.take(limit)- end-def current_visitors(site, query) doPlausible.ClickhouseRepo.one(from e in base_query(site, query),@@ -891,105 +142,6 @@ defmodule Plausible.Stats.Clickhouse do)end- def all_props(site, %Query{filters: %{"props" => meta}} = query) when is_map(meta) do- [{key, val}] = meta |> Enum.into([])-- if val == "(none)" do- goal = query.filters["goal"]- %{goal => [key]}- else- ClickhouseRepo.all(- from [e, meta: meta] in base_query_w_sessions_bare(site, query),- select: {e.name, meta.key},- distinct: true- )- |> Enum.reduce(%{}, fn {goal_name, meta_key}, acc ->- Map.update(acc, goal_name, [meta_key], fn list -> [meta_key | list] end)- end)- end- end-- def all_props(site, query) do- ClickhouseRepo.all(- from e in base_query_w_sessions_bare(site, query),- inner_lateral_join: meta in fragment("meta as m"),- select: {e.name, meta.key},- distinct: true- )- |> Enum.reduce(%{}, fn {goal_name, meta_key}, acc ->- Map.update(acc, goal_name, [meta_key], fn list -> [meta_key | list] end)- end)- end-- def property_breakdown(site, %Query{filters: %{"props" => meta}} = query, key)- when is_map(meta) do- [{_key, val}] = meta |> Enum.into([])-- if val == "(none)" do- ClickhouseRepo.all(- from e in base_query_w_sessions(site, query),- where: fragment("not has(meta.key, ?)", ^key),- order_by: [desc: uniq(e.user_id)],- select: %{- name: "(none)",- count: uniq(e.user_id),- total_count: total()- }- )- else- ClickhouseRepo.all(- from [e, meta: meta] in base_query_w_sessions(site, query),- group_by: meta.value,- order_by: [desc: uniq(e.user_id)],- select: %{- name: meta.value,- count: uniq(e.user_id),- total_count: total()- }- )- end- end-- def property_breakdown(site, query, key) do- none =- ClickhouseRepo.all(- from e in base_query_w_sessions(site, query),- where: fragment("not has(?.key, ?)", e.meta, ^key),- select: %{- name: "(none)",- count: uniq(e.user_id),- total_count: total()- }- )-- values =- ClickhouseRepo.all(- from e in base_query_w_sessions(site, query),- inner_lateral_join: meta in fragment("meta as m"),- where: meta.key == ^key,- group_by: meta.value,- order_by: [desc: uniq(e.user_id)],- select: %{- name: meta.value,- count: uniq(e.user_id),- total_count: total()- }- )-- (values ++ none)- |> Enum.sort(fn row1, row2 -> row1[:count] >= row2[:count] end)- |> Enum.filter(fn row -> row[:count] > 0 end)- |> Enum.map(fn row ->- uri = URI.parse(row[:name])-- if uri.host && uri.scheme do- Map.put(row, :is_url, true)- else- row- end- end)- end-def last_24h_visitors([]), do: %{}def last_24h_visitors(sites) do@@ -1005,93 +157,6 @@ defmodule Plausible.Stats.Clickhouse do|> Enum.into(%{})end- def goal_conversions(site, %Query{filters: %{"goal" => goal}} = query) when is_binary(goal) do- ClickhouseRepo.all(- from e in base_query_w_sessions(site, query),- group_by: e.name,- order_by: [desc: uniq(e.user_id)],- select: %{- name: ^goal,- count: uniq(e.user_id),- total_count: total()- }- )- end-- def goal_conversions(site, query) do- goals = Repo.all(from g in Plausible.Goal, where: g.domain == ^site.domain)- query = if query.period == "realtime", do: %Query{query | period: "30m"}, else: query-- (fetch_pageview_goals(goals, site, query) ++- fetch_event_goals(goals, site, query))- |> sort_conversions()- end-- defp fetch_event_goals(goals, site, query) do- events =- Enum.map(goals, fn goal -> goal.event_name end)- |> Enum.filter(& &1)-- if Enum.count(events) > 0 do- q =- from(- e in base_query_w_sessions_bare(site, query),- where: fragment("? IN tuple(?)", e.name, ^events),- group_by: e.name,- select: %{- name: e.name,- count: uniq(e.user_id),- total_count: total()- }- )-- ClickhouseRepo.all(q)- else- []- end- end-- defp fetch_pageview_goals(goals, site, query) do- goals =- Enum.map(goals, fn goal -> goal.page_path end)- |> Enum.filter(& &1)-- if Enum.count(goals) > 0 do- regex_goals =- Enum.map(goals, fn g ->- "^#{g}\/?$"- |> String.replace(~r/\*\*/, ".*")- |> String.replace(~r/(?<!\.)\*/, "[^/]*")- end)-- from(- e in base_query_w_sessions(site, query),- where:- fragment(- "notEmpty(multiMatchAllIndices(?, array(?)) as indices)",- e.pathname,- ^regex_goals- ),- select: %{- index: fragment("arrayJoin(indices) as index"),- count: uniq(e.user_id),- total_count: total()- },- group_by: fragment("index")- )- |> ClickhouseRepo.all()- |> Enum.map(fn x ->- Map.put(x, :name, "Visit #{Enum.at(goals, x[:index] - 1)}") |> Map.delete(:index)- end)- else- []- end- end-- defp sort_conversions(conversions) do- Enum.sort_by(conversions, fn conversion -> -conversion[:count] end)- end-defp base_query_w_sessions_bare(site, query) do{first_datetime, last_datetime} = utc_boundaries(query, site.timezone)@@ -1241,38 +306,6 @@ defmodule Plausible.Stats.Clickhouse doendend- defp base_query_w_session_based_pageviews(site, query) do- {first_datetime, last_datetime} = utc_boundaries(query, site.timezone)-- sessions_q = base_session_query(site, query) |> filter_converted_sessions(site, query)-- sessions_q =- from(s in sessions_q,- select: %{session_id: s.session_id}- )-- q =- from(e in "events",- hints: ["SAMPLE 10000000"],- where: e.domain == ^site.domain,- where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime- )-- if query.filters["source"] || query.filters["referrer"] || query.filters["utm_medium"] ||- query.filters["utm_source"] || query.filters["utm_campaign"] || query.filters["screen"] ||- query.filters["browser"] || query.filters["browser_version"] || query.filters["os"] ||- query.filters["os_version"] || query.filters["country"] || query.filters["entry_page"] ||- query.filters["exit_page"] || query.filters["page"] || query.filters["goal"] do- from(- e in q,- join: sq in subquery(sessions_q),- on: e.session_id == sq.session_id- )- else- q- end- end-defp base_query_w_sessions(site, query) dobase_query_w_sessions_bare(site, query) |> include_goal_conversions(query)enddiff --git a/lib/plausible/stats/compare.ex b/lib/plausible/stats/compare.exnew file mode 100644index 0000000000..2e98d908c5--- /dev/null+++ b/lib/plausible/stats/compare.ex@@ -0,0 +1,28 @@+defmodule Plausible.Stats.Compare do+ def calculate_change("bounce_rate", old_stats, new_stats) do+ old_count = old_stats["bounce_rate"]["value"]+ new_count = new_stats["bounce_rate"]["value"]++ if old_count > 0, do: new_count - old_count+ end++ def calculate_change(metric, old_stats, new_stats) do+ old_count = old_stats[metric]["value"]+ new_count = new_stats[metric]["value"]++ percent_change(old_count, new_count)+ end++ defp percent_change(old_count, new_count) do+ cond do+ old_count == 0 and new_count > 0 ->+ 100++ old_count == 0 and new_count == 0 ->+ 0++ true ->+ round((new_count - old_count) / old_count * 100)+ end+ end+enddiff --git a/lib/plausible_web/templates/email/weekly_report.html.eex b/lib/plausible_web/templates/email/weekly_report.html.eexindex 5de9cd5f62..6756d75669 100644--- a/lib/plausible_web/templates/email/weekly_report.html.eex+++ b/lib/plausible_web/templates/email/weekly_report.html.eex@@ -265,7 +265,7 @@ body {<!--[if mso]><table width="100%" cellpadding="0" cellspacing="0" border="0"><tr><td style="padding-right: 5px; padding-left: 5px; padding-top: 0px; padding-bottom: 5px; font-family: Arial, sans-serif"><![endif]--><div style="color:#555555;font-family:'Helvetica Neue', Helvetica, Arial, sans-serif;line-height:120%;padding-top:0px;padding-right:5px;padding-bottom:5px;padding-left:5px;"><div style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 12px; line-height: 14px; color: #555555;">- <p style="font-size: 12px; line-height: 14px; text-align: left; margin: 0;"><strong><span style="font-size: 20px; line-height: 24px;"><%= PlausibleWeb.StatsView.large_number_format(@pageviews) %></span></strong></p>+ <p style="font-size: 12px; line-height: 14px; text-align: left; margin: 0;"><strong><span id="pageviews" style="font-size: 20px; line-height: 24px;"><%= PlausibleWeb.StatsView.large_number_format(@pageviews) %></span></strong></p></div></div><!--[if mso]></td></tr></table><![endif]-->@@ -307,7 +307,7 @@ body {<!--[if mso]><table width="100%" cellpadding="0" cellspacing="0" border="0"><tr><td style="padding-right: 5px; padding-left: 5px; padding-top: 0px; padding-bottom: 5px; font-family: Arial, sans-serif"><![endif]--><div style="color:#555555;font-family:'Helvetica Neue', Helvetica, Arial, sans-serif;line-height:120%;padding-top:0px;padding-right:5px;padding-bottom:5px;padding-left:5px;"><div style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 12px; line-height: 14px; color: #555555;">- <p style="font-size: 12px; line-height: 14px; text-align: left; margin: 0;"><strong><span style="font-size: 20px; line-height: 24px;"><%= @bounce_rate %>%</span></strong></p>+ <p style="font-size: 12px; line-height: 14px; text-align: left; margin: 0;"><strong><span id="bounce_rate" style="font-size: 20px; line-height: 24px;"><%= @bounce_rate %>%</span></strong></p></div></div><!--[if mso]></td></tr></table><![endif]-->@@ -420,8 +420,8 @@ body {</div></div></div>- <%= for referrer <- @referrers do %>- <div style="background-color:transparent;">+ <%= for source <- @sources do %>+ <div class="referrer" style="background-color:transparent;"><div class="block-grid mixed-two-up" style="Margin: 0 auto; min-width: 320px; max-width: 480px; overflow-wrap: break-word; word-wrap: break-word; word-break: break-word; background-color: transparent;"><div style="border-collapse: collapse;display: table;width: 100%;background-color:transparent;"><!--[if (mso)|(IE)]><table width="100%" cellpadding="0" cellspacing="0" border="0" style="background-color:transparent;"><tr><td align="center"><table cellpadding="0" cellspacing="0" border="0" style="width:480px"><tr class="layout-full-width" style="background-color:transparent"><![endif]-->@@ -434,7 +434,7 @@ body {<!--[if mso]><table width="100%" cellpadding="0" cellspacing="0" border="0"><tr><td style="padding-right: 5px; padding-left: 5px; padding-top: 5px; padding-bottom: 5px; font-family: Arial, sans-serif"><![endif]--><div style="color:#555555;font-family:'Helvetica Neue', Helvetica, Arial, sans-serif;line-height:120%;padding-top:5px;padding-right:5px;padding-bottom:5px;padding-left:5px;"><div style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 12px; line-height: 14px; color: #555555;">- <p style="font-size: 14px; line-height: 16px; margin: 0;"><%= referrer[:name] %></p>+ <p id="referrer-name" style="font-size: 14px; line-height: 16px; margin: 0;"><%= source["source"] %></p></div></div><!--[if mso]></td></tr></table><![endif]-->@@ -453,7 +453,7 @@ body {<!--[if mso]><table width="100%" cellpadding="0" cellspacing="0" border="0"><tr><td style="padding-right: 5px; padding-left: 5px; padding-top: 5px; padding-bottom: 5px; font-family: Arial, sans-serif"><![endif]--><div style="color:#555555;font-family:'Helvetica Neue', Helvetica, Arial, sans-serif;line-height:120%;padding-top:5px;padding-right:5px;padding-bottom:5px;padding-left:5px;"><div style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 12px; line-height: 14px; color: #555555;">- <p style="font-size: 12px; line-height: 16px; text-align: right; margin: 0;"><span style="font-size: 14px;"><%= PlausibleWeb.StatsView.large_number_format(referrer[:count]) %></span></p>+ <p style="font-size: 12px; line-height: 16px; text-align: right; margin: 0;"><span id="referrer-count" style="font-size: 14px;"><%= PlausibleWeb.StatsView.large_number_format(source["visitors"]) %></span></p></div></div><!--[if mso]></td></tr></table><![endif]-->@@ -550,7 +550,7 @@ body {</div></div><%= for page <- @pages do %>- <div style="background-color:transparent;">+ <div class="page" style="background-color:transparent;"><div class="block-grid mixed-two-up" style="Margin: 0 auto; min-width: 320px; max-width: 480px; overflow-wrap: break-word; word-wrap: break-word; word-break: break-word; background-color: transparent;"><div style="border-collapse: collapse;display: table;width: 100%;background-color:transparent;"><!--[if (mso)|(IE)]><table width="100%" cellpadding="0" cellspacing="0" border="0" style="background-color:transparent;"><tr><td align="center"><table cellpadding="0" cellspacing="0" border="0" style="width:480px"><tr class="layout-full-width" style="background-color:transparent"><![endif]-->@@ -563,7 +563,7 @@ body {<!--[if mso]><table width="100%" cellpadding="0" cellspacing="0" border="0"><tr><td style="padding-right: 5px; padding-left: 5px; padding-top: 5px; padding-bottom: 5px; font-family: Arial, sans-serif"><![endif]--><div style="color:#555555;font-family:'Helvetica Neue', Helvetica, Arial, sans-serif;line-height:120%;padding-top:5px;padding-right:5px;padding-bottom:5px;padding-left:5px;"><div style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 12px; line-height: 14px; color: #555555;">- <p style="font-size: 14px; line-height: 16px; margin: 0;"><%= page[:name] %></p>+ <p id="page-name" style="font-size: 14px; line-height: 16px; margin: 0;"><%= page["page"] %></p></div></div><!--[if mso]></td></tr></table><![endif]-->@@ -582,7 +582,7 @@ body {<!--[if mso]><table width="100%" cellpadding="0" cellspacing="0" border="0"><tr><td style="padding-right: 5px; padding-left: 5px; padding-top: 5px; padding-bottom: 5px; font-family: Arial, sans-serif"><![endif]--><div style="color:#555555;font-family:'Helvetica Neue', Helvetica, Arial, sans-serif;line-height:120%;padding-top:5px;padding-right:5px;padding-bottom:5px;padding-left:5px;"><div style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 12px; line-height: 14px; color: #555555;">- <p style="font-size: 12px; line-height: 16px; text-align: right; margin: 0;"><span style="font-size: 14px;"><%= PlausibleWeb.StatsView.large_number_format(page[:count]) %></span></p>+ <p style="font-size: 12px; line-height: 16px; text-align: right; margin: 0;"><span id="page-count" style="font-size: 14px;"><%= PlausibleWeb.StatsView.large_number_format(page["visitors"]) %></span></p></div></div><!--[if mso]></td></tr></table><![endif]-->diff --git a/lib/workers/send_email_report.ex b/lib/workers/send_email_report.exindex ca4f15cbb9..f53ab48b1a 100644--- a/lib/workers/send_email_report.ex+++ b/lib/workers/send_email_report.ex@@ -2,7 +2,7 @@ defmodule Plausible.Workers.SendEmailReport douse Plausible.Repouse Oban.Worker, queue: :send_email_reports, max_attempts: 1alias Plausible.Stats.Query- alias Plausible.Stats.Clickhouse, as: Stats+ alias Plausible.Stats@impl Oban.Workerdef perform(%Oban.Job{args: %{"interval" => "weekly", "site_id" => site_id}}) do@@ -49,28 +49,29 @@ defmodule Plausible.Workers.SendEmailReport doenddefp send_report(email, site, name, unsubscribe_link, query) do- {pageviews, unique_visitors} = Stats.pageviews_and_visitors(site, query)+ prev_query = Query.shift_back(query, site)+ curr_period = Stats.aggregate(site, query, ["pageviews", "visitors", "bounce_rate"])+ prev_period = Stats.aggregate(site, prev_query, ["pageviews", "visitors", "bounce_rate"])- {change_pageviews, change_visitors} =- Stats.compare_pageviews_and_visitors(site, query, {pageviews, unique_visitors})+ change_pageviews = Stats.Compare.calculate_change("pageviews", prev_period, curr_period)+ change_visitors = Stats.Compare.calculate_change("visitors", prev_period, curr_period)+ change_bounce_rate = Stats.Compare.calculate_change("bounce_rate", prev_period, curr_period)- bounce_rate = Stats.bounce_rate(site, query)- prev_bounce_rate = Stats.bounce_rate(site, Query.shift_back(query, site))- change_bounce_rate = if prev_bounce_rate > 0, do: bounce_rate - prev_bounce_rate- referrers = Stats.top_sources(site, query, 5, 1, [])- pages = Stats.top_pages(site, query, 5, 1, [])+ source_query = Query.put_filter(query, "visit:source", {:is_not, "Direct / None"})+ sources = Stats.breakdown(site, source_query, "visit:source", ["visitors"], {5, 1})+ pages = Stats.breakdown(site, query, "event:page", ["visitors"], {5, 1})user = Plausible.Auth.find_user_by(email: email)login_link = user && Plausible.Sites.is_member?(user.id, site)template =PlausibleWeb.Email.weekly_report(email, site,- unique_visitors: unique_visitors,+ unique_visitors: curr_period["visitors"]["value"],change_visitors: change_visitors,- pageviews: pageviews,+ pageviews: curr_period["pageviews"]["value"],change_pageviews: change_pageviews,- bounce_rate: bounce_rate,+ bounce_rate: curr_period["bounce_rate"]["value"],change_bounce_rate: change_bounce_rate,- referrers: referrers,+ sources: sources,unsubscribe_link: unsubscribe_link,login_link: login_link,pages: pages,diff --git a/mix.exs b/mix.exsindex aea4c9e052..f8cb2e03a3 100644--- a/mix.exs+++ b/mix.exs@@ -101,7 +101,8 @@ defmodule Plausible.MixProject do{:opentelemetry_exporter, "1.0.0-rc.3"},{:opentelemetry_phoenix, "1.0.0-rc.5"},{:opentelemetry_ecto, "1.0.0-rc.3"},- {:opentelemetry_oban, "~> 0.2.0-rc.2"}+ {:opentelemetry_oban, "~> 0.2.0-rc.2"},+ {:floki, "~> 0.32.0", only: :test}]enddiff --git a/mix.lock b/mix.lockindex 5aac48a2b5..b76bf89c16 100644--- a/mix.lock+++ b/mix.lock@@ -37,6 +37,7 @@"ex_machina": {:hex, :ex_machina, "2.7.0", "b792cc3127fd0680fecdb6299235b4727a4944a09ff0fa904cc639272cd92dc7", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm", "419aa7a39bde11894c87a615c4ecaa52d8f107bbdd81d810465186f783245bf8"},"excoveralls": {:hex, :excoveralls, "0.14.4", "295498f1ae47bdc6dce59af9a585c381e1aefc63298d48172efaaa90c3d251db", [:mix], [{:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "e3ab02f2df4c1c7a519728a6f0a747e71d7d6e846020aae338173619217931c1"},"file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"},+ "floki": {:hex, :floki, "0.32.0", "f915dc15258bc997d49be1f5ef7d3992f8834d6f5695270acad17b41f5bcc8e2", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "1c5a91cae1fd8931c26a4826b5e2372c284813904c8bacb468b5de39c7ececbd"},"gen_smtp": {:hex, :gen_smtp, "1.1.1", "bf9303c31735100631b1d708d629e4c65944319d1143b5c9952054f4a1311d85", [:rebar3], [{:hut, "1.3.0", [hex: :hut, repo: "hexpm", optional: false]}, {:ranch, ">= 1.7.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "51bc50cc017efd4a4248cbc39ea30fb60efa7d4a49688986fafad84434ff9ab7"},"geolix": {:hex, :geolix, "1.1.0", "8b0fe847fef486d9e8b7c21eae6cbc2d998fb249e61d3f4f136f8016b9c1c833", [:mix], [{:poolboy, "~> 1.0", [hex: :poolboy, repo: "hexpm", optional: false]}], "hexpm", "980854f2aef30c288dc79e86c5267806d704c4525fde1b75de9a92f67fb16300"},"geolix_adapter_mmdb2": {:hex, :geolix_adapter_mmdb2, "0.5.0", "5912723d9538ecddc6b29b1d8041b917b735a78fd3c122bfea8c44aa782e3369", [:mix], [{:geolix, "~> 1.1", [hex: :geolix, repo: "hexpm", optional: false]}, {:mmdb2_decoder, "~> 3.0", [hex: :mmdb2_decoder, repo: "hexpm", optional: false]}], "hexpm", "cb1485b6a0a2d3e541949207428a245718dbf1258453a0df0e5fdd925bcecd3e"},@@ -46,6 +47,7 @@"hackney": {:hex, :hackney, "1.18.0", "c4443d960bb9fba6d01161d01cd81173089686717d9490e5d3606644c48d121f", [:rebar3], [{:certifi, "~>2.8.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "9afcda620704d720db8c6a3123e9848d09c87586dc1c10479c42627b905b5c5e"},"hammer": {:hex, :hammer, "6.0.0", "72ec6fff10e9d63856968988a22ee04c4d6d5248071ddccfbda50aa6c455c1d7", [:mix], [{:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: false]}], "hexpm", "d8e1ec2e534c4aae508b906759e077c3c1eb3e2b9425235d4b7bbab0b016210a"},"hpack": {:hex, :hpack_erl, "0.2.3", "17670f83ff984ae6cd74b1c456edde906d27ff013740ee4d9efaa4f1bf999633", [:rebar3], [], "hexpm", "06f580167c4b8b8a6429040df36cc93bba6d571faeaec1b28816523379cbb23a"},+ "html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"},"httpoison": {:hex, :httpoison, "1.8.0", "6b85dea15820b7804ef607ff78406ab449dd78bed923a49c7160e1886e987a3d", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "28089eaa98cf90c66265b6b5ad87c59a3729bea2e74e9d08f9b51eb9729b3c3a"},"hut": {:hex, :hut, "1.3.0", "71f2f054e657c03f959cf1acc43f436ea87580696528ca2a55c8afb1b06c85e7", [:"erlang.mk", :rebar, :rebar3], [], "hexpm", "7e15d28555d8a1f2b5a3a931ec120af0753e4853a4c66053db354f35bf9ab563"},"idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},diff --git a/test/workers/send_email_report_test.exs b/test/workers/send_email_report_test.exsindex 09f45e33e9..fb8b626784 100644--- a/test/workers/send_email_report_test.exs+++ b/test/workers/send_email_report_test.exs@@ -60,6 +60,51 @@ defmodule Plausible.Workers.SendEmailReportTest doassert html_body =~~s(<span id="visitors" style="line-height: 24px; font-size: 20px;">2</span>)end++ test "includes the correct stats" do+ site = insert(:site, domain: "test-site.com")+ insert(:weekly_report, site: site, recipients: ["[email protected]"])+ now = NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second)++ populate_stats(site, [+ build(:pageview,+ referrer_source: "Google",+ user_id: 123,+ timestamp: Timex.shift(now, days: -7)+ ),+ build(:pageview, user_id: 123, timestamp: Timex.shift(now, days: -7)),+ build(:pageview, timestamp: Timex.shift(now, days: -7))+ ])++ perform_job(SendEmailReport, %{"site_id" => site.id, "interval" => "weekly"})++ assert_delivered_email_matches(%{+ to: [nil: "[email protected]"],+ html_body: html_body+ })++ {:ok, document} = Floki.parse_document(html_body)++ visitors = Floki.find(document, "#visitors") |> Floki.text()+ assert visitors == "2"++ pageviews = Floki.find(document, "#pageviews") |> Floki.text()+ assert pageviews == "3"++ referrer = Floki.find(document, ".referrer") |> List.first()+ referrer_name = referrer |> Floki.find("#referrer-name") |> Floki.text()+ referrer_count = referrer |> Floki.find("#referrer-count") |> Floki.text()++ assert referrer_name == "Google"+ assert referrer_count == "1"++ page = Floki.find(document, ".page") |> List.first()+ page_name = page |> Floki.find("#page-name") |> Floki.text()+ page_count = page |> Floki.find("#page-count") |> Floki.text()++ assert page_name == "/"+ assert page_count == "2"+ endenddescribe "monthly_reports" do