Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Implement dashboard with new stats module
Uku Taht7d1b1214e201
diff --git a/assets/js/dashboard/stats/modals/countries.js b/assets/js/dashboard/stats/modals/countries.jsindex 82da8faa20..0e833389a8 100644--- a/assets/js/dashboard/stats/modals/countries.js+++ b/assets/js/dashboard/stats/modals/countries.js@@ -1,5 +1,6 @@import React from "react";import { Link, withRouter } from 'react-router-dom'+import Datamap from 'datamaps'import Modal from './modal'import * as api from '../../api'@@ -25,11 +26,15 @@ class CountriesModal extends React.Component {const query = new URLSearchParams(window.location.search)query.set('country', country.name)+ const allCountries = Datamap.prototype.worldTopo.objects.world.geometries;+ const thisCountry = allCountries.find((c) => c.id === country.name) || {properties: {name: country.name}};+ const countryFullName = thisCountry.properties.name+return (<tr className="text-sm dark:text-gray-200" key={country.name}><td className="p-2"><Link className="hover:underline" to={{search: query.toString(), pathname: '/' + encodeURIComponent(this.props.site.domain)}}>- {country.full_country_name}+ {countryFullName}</Link></td><td className="p-2 w-32 font-medium" align="right">diff --git a/assets/js/dashboard/stats/visitor-graph.js b/assets/js/dashboard/stats/visitor-graph.jsindex 9a44b06d0c..ae60d43d5c 100644--- a/assets/js/dashboard/stats/visitor-graph.js+++ b/assets/js/dashboard/stats/visitor-graph.js@@ -260,12 +260,12 @@ class LineGraph extends React.Component {}topStatNumberShort(stat) {- if (typeof(stat.duration) == 'number') {- return durationFormatter(stat.duration)- } else if (typeof(stat.count) == 'number') {- return numberFormatter(stat.count)+ if (['visit duration', 'time on page'].includes(stat.name.toLowerCase())) {+ return durationFormatter(stat.value)+ } else if (['bounce rate', 'conversion rate'].includes(stat.name.toLowerCase())) {+ return stat.value + '%'} else {- return stat.percentage + '%'+ return numberFormatter(stat.value)}}diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.exindex 5678f569ff..724827b1f9 100644--- a/lib/plausible/stats/aggregate.ex+++ b/lib/plausible/stats/aggregate.ex@@ -1,9 +1,10 @@defmodule Plausible.Stats.Aggregate do+ alias Plausible.Stats.Queryuse Plausible.ClickhouseRepoimport Plausible.Stats.Base- @event_metrics ["visitors", "pageviews"]- @session_metrics ["bounce_rate", "visit_duration"]+ @event_metrics ["visitors", "pageviews", "events"]+ @session_metrics ["visits", "bounce_rate", "visit_duration"]def aggregate(site, query, metrics) doevent_metrics = Enum.filter(metrics, &(&1 in @event_metrics))@@ -11,12 +12,18 @@ defmodule Plausible.Stats.Aggregate dosession_metrics = Enum.filter(metrics, &(&1 in @session_metrics))session_task = Task.async(fn -> aggregate_sessions(site, query, session_metrics) end)- Map.merge(- Task.await(event_task),- Task.await(session_task)- )+ time_on_page_task =+ if "time_on_page" in metrics do+ Task.async(fn -> aggregate_time_on_page(site, query) end)+ else+ Task.async(fn -> %{} end)+ end++ Task.await(event_task)+ |> Map.merge(Task.await(session_task))+ |> Map.merge(Task.await(time_on_page_task))|> Enum.map(fn {metric, value} ->- {metric, %{value: value || 0}}+ {metric, %{value: round(value || 0)}}end)|> Enum.into(%{})end@@ -36,6 +43,12 @@ defmodule Plausible.Stats.Aggregate do)end+ defp select_event_metric("events", q) do+ from(e in q,+ select_merge: %{events: fragment("count(*)")}+ )+ end+defp select_event_metric("visitors", q) dofrom(e in q, select_merge: %{visitors: fragment("uniq(?)", e.user_id)})end@@ -55,7 +68,68 @@ defmodule Plausible.Stats.Aggregate do)end+ defp select_session_metric("visits", q) do+ from(s in q,+ select_merge: %{visits: fragment("sum(?)", s.sign)}+ )+ end+defp select_session_metric("visit_duration", q) dofrom(s in q, select_merge: %{visit_duration: fragment("round(avg(duration * sign))")})end++ defp aggregate_time_on_page(site, query) do+ q =+ from(+ e in base_event_query(site, %Query{+ query+ | filters: Map.delete(query.filters, "event:page")+ }),+ select: {+ fragment("? as p", e.pathname),+ fragment("? as t", e.timestamp),+ fragment("? as s", e.session_id)+ },+ order_by: [e.session_id, e.timestamp]+ )++ {base_query_raw, base_query_raw_params} = ClickhouseRepo.to_sql(:all, q)++ {where_clause, where_arg} =+ case query.filters["event:page"] do+ {:is, page} ->+ {"p=?", page}++ {:matches, expr} ->+ regex = page_regex(expr)+ {"match(p, ?)", regex}+ end++ time_query = "+ SELECT+ avg(ifNotFinite(avgTime, null))+ FROM+ (SELECT+ p,+ sum(td)/count(case when p2 != p then 1 end) as avgTime+ FROM+ (SELECT+ p,+ p2,+ sum(t2-t) as td+ FROM+ (SELECT+ *,+ neighbor(t, 1) as t2,+ neighbor(p, 1) as p2,+ neighbor(s, 1) as s2+ FROM (#{base_query_raw}))+ WHERE s=s2 AND #{where_clause}+ GROUP BY p,p2,s)+ GROUP BY p)"++ {:ok, res} = ClickhouseRepo.query(time_query, base_query_raw_params ++ [where_arg])+ [[time_on_page]] = res.rows+ %{time_on_page: time_on_page}+ endenddiff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.exindex 311766b83c..5db177499c 100644--- a/lib/plausible/stats/base.ex+++ b/lib/plausible/stats/base.ex@@ -1,26 +1,13 @@defmodule Plausible.Stats.Base douse Plausible.ClickhouseRepo- alias Plausible.Stats.Query+ alias Plausible.Stats.{Query, Filters}@no_ref "Direct / None"- @session_props [- "source",- "referrer",- "utm_medium",- "utm_source",- "utm_campaign",- "device",- "browser",- "browser_version",- "os",- "os_version",- "country"- ]def base_event_query(site, query) doevents_q = query_events(site, query)- if Enum.any?(@session_props, &query.filters["visit:" <> &1]) do+ if Enum.any?(Filters.visit_props(), &query.filters["visit:" <> &1]) dosessions_q =from(s in query_sessions(site, query),@@ -48,9 +35,18 @@ defmodule Plausible.Stats.Base doq =case query.filters["event:page"] do- {:is, page} -> from(e in q, where: e.pathname == ^page)- {:member, list} -> from(e in q, where: e.pathname in ^list)- _ -> q+ {:is, page} ->+ from(e in q, where: e.pathname == ^page)++ {:matches, glob_expr} ->+ regex = page_regex(glob_expr)+ from(e in q, where: fragment("match(?, ?)", e.pathname, ^regex))++ {:member, list} ->+ from(e in q, where: e.pathname in ^list)++ _ ->+ qendq =@@ -60,6 +56,18 @@ defmodule Plausible.Stats.Base do_ -> qend+ q =+ case goal_type(query) do+ {:is, :page, path} ->+ from(e in q, where: e.pathname == ^path)++ {:is, :event, event} ->+ from(e in q, where: e.name == ^event)++ _ ->+ q+ end+Enum.reduce(query.filters, q, fn {filter_key, filter_value}, query ->case filter_key do"event:props:" <> prop_name ->@@ -88,6 +96,7 @@ defmodule Plausible.Stats.Base do@api_prop_name_to_db %{"source" => "referrer_source","device" => "screen_size",+ "screen" => "screen_size","os" => "operating_system","os_version" => "operating_system_version","country" => "country_code"@@ -103,19 +112,22 @@ defmodule Plausible.Stats.Base do)sessions_q =- if query.filters["event:page"] do- case query.filters["event:page"] do- {:is, page} ->- from(e in sessions_q, where: e.entry_page == ^page)-- {:member, list} ->- from(e in sessions_q, where: e.entry_page in ^list)- end- else- sessions_q+ case query.filters["event:page"] do+ {:is, page} ->+ from(e in sessions_q, where: e.entry_page == ^page)++ {:matches, glob_expr} ->+ regex = page_regex(glob_expr)+ from(s in sessions_q, where: fragment("match(?, ?)", s.entry_page, ^regex))++ {:member, list} ->+ from(e in sessions_q, where: e.entry_page in ^list)++ _ ->+ sessions_qend- Enum.reduce(@session_props, sessions_q, fn prop_name, sessions_q ->+ Enum.reduce(Filters.visit_props(), sessions_q, fn prop_name, sessions_q ->filter = query.filters["visit:" <> prop_name]prop_name = Map.get(@api_prop_name_to_db, prop_name, prop_name)@@ -124,6 +136,13 @@ defmodule Plausible.Stats.Base dowhere_target = [{String.to_existing_atom(prop_name), db_prop_val(prop_name, value)}]from(s in sessions_q, where: ^where_target)+ {:is_not, value} ->+ fragment_data = [+ {String.to_existing_atom(prop_name), {:!=, db_prop_val(prop_name, value)}}+ ]++ from(s in sessions_q, where: fragment(^fragment_data))+{:member, values} ->list = Enum.map(values, &db_prop_val(prop_name, &1))fragment_data = [{String.to_existing_atom(prop_name), {:in, list}}]@@ -138,6 +157,21 @@ defmodule Plausible.Stats.Base dodefp db_prop_val("referrer_source", @no_ref), do: ""defp db_prop_val(_, val), do: val+ defp goal_type(query) do+ case query.filters["visit:goal"] do+ {:is, "Visit " <> page} -> {:is, :page, page}+ {:is, event} -> {:is, :event, event}+ _ -> nil+ end+ end++ defp utc_boundaries(%Query{period: "30m"}, _timezone) do+ last_datetime = NaiveDateTime.utc_now() |> Timex.shift(seconds: 5)+ first_datetime = NaiveDateTime.utc_now() |> Timex.shift(minutes: -30)++ {first_datetime, last_datetime}+ end+defp utc_boundaries(%Query{date_range: date_range}, timezone) do{:ok, first} = NaiveDateTime.new(date_range.first, ~T[00:00:00])@@ -153,4 +187,10 @@ defmodule Plausible.Stats.Base do{first_datetime, last_datetime}end++ def page_regex(expr) do+ "^#{expr}\/?$"+ |> String.replace(~r/\*\*/, ".*")+ |> String.replace(~r/(?<!\.)\*/, "[^/]*")+ endenddiff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.exindex bb335c77af..7e66a8eae1 100644--- a/lib/plausible/stats/breakdown.ex+++ b/lib/plausible/stats/breakdown.ex@@ -1,37 +1,137 @@defmodule Plausible.Stats.Breakdown douse Plausible.ClickhouseRepoimport Plausible.Stats.Base+ alias Plausible.Stats.Query@no_ref "Direct / None"- @event_metrics ["visitors", "pageviews"]- @session_metrics ["bounce_rate", "visit_duration"]- @event_props ["event:page", "event:name"]+ @event_metrics ["visitors", "pageviews", "events"]+ @session_metrics ["visits", "bounce_rate", "visit_duration"]+ @event_props ["event:page", "event:page_match", "event:name"]- def breakdown(site, query, property, metrics, pagination) do- if property in @event_props || String.starts_with?(property, "event:props:") do- event_metrics = Enum.filter(metrics, &(&1 in @event_metrics))- session_metrics = Enum.filter(metrics, &(&1 in @session_metrics))+ def breakdown(site, query, "visit:goal", metrics, pagination) do+ {event_goals, pageview_goals} =+ Plausible.Repo.all(from g in Plausible.Goal, where: g.domain == ^site.domain)+ |> Enum.split_with(fn goal -> goal.event_name end)- event_task =- Task.async(fn -> breakdown_events(site, query, property, event_metrics, pagination) end)+ events = Enum.map(event_goals, & &1.event_name)+ event_query = %Query{query | filters: Map.put(query.filters, "event:name", {:member, events})}- session_task =- Task.async(fn ->- breakdown_sessions(site, query, property, session_metrics, pagination)- end)+ event_goals =+ breakdown(site, event_query, "event:name", metrics, pagination)+ |> transform_keys(%{"name" => "goal"})++ page_exprs = Enum.map(pageview_goals, & &1.page_path)+ page_regexes = Enum.map(page_exprs, &page_regex/1)++ {limit, page} = pagination+ offset = (page - 1) * limit- zip_results(- Task.await(event_task),- Task.await(session_task),- property,- metrics+ page_goals =+ from(e in base_event_query(site, query),+ order_by: [desc: fragment("uniq(?)", e.user_id)],+ limit: ^limit,+ offset: ^offset,+ where:+ fragment(+ "notEmpty(multiMatchAllIndices(?, array(?)) as indices)",+ e.pathname,+ ^page_regexes+ ),+ group_by: fragment("index"),+ select: %{+ "index" => fragment("arrayJoin(indices) as index"),+ "goal" => fragment("concat('Visit ', array(?)[index])", ^page_exprs)+ })- else- breakdown_sessions(site, query, property, metrics, pagination)- end+ |> select_event_metrics(metrics)+ |> ClickhouseRepo.all()+ |> Enum.map(fn row -> Map.delete(row, "index") end)++ zip_results(event_goals, page_goals, "visit:goal", metrics)+ end++ def breakdown(site, query, "event:props:" <> custom_prop, metrics, pagination) do+ none_result =+ if !Enum.any?(query.filters, fn {key, _} -> String.starts_with?(key, "event:props") end) do+ none_filters = Map.put(query.filters, "event:props:" <> custom_prop, {:is, "(none)"})+ none_query = %Query{query | filters: none_filters}++ {limit, page} = pagination+ offset = (page - 1) * limit++ from(e in base_event_query(site, none_query),+ order_by: [desc: fragment("uniq(?)", e.user_id)],+ limit: ^limit,+ offset: ^offset,+ select: %{},+ select_merge: %{^custom_prop => "(none)"},+ having: fragment("uniq(?)", e.user_id) > 0+ )+ |> select_event_metrics(metrics)+ |> ClickhouseRepo.all()+ else+ []+ end++ results = breakdown_events(site, query, "event:props:" <> custom_prop, metrics, pagination)+ zip_results(none_result, results, custom_prop, metrics)+ end++ def breakdown(site, query, "event:page", metrics, pagination) do+ event_metrics = Enum.filter(metrics, &(&1 in @event_metrics))+ session_metrics = Enum.filter(metrics, &(&1 in @session_metrics))++ event_result = breakdown_events(site, query, "event:page", event_metrics, pagination)++ event_result =+ if "time_on_page" in metrics do+ pages = Enum.map(event_result, & &1["page"])+ time_on_page_result = breakdown_time_on_page(site, query, pages)++ Enum.map(event_result, fn row ->+ Map.put(row, "time_on_page", time_on_page_result[row["page"]])+ end)+ else+ event_result+ end++ new_query =+ case event_result do+ [] ->+ query++ pages ->+ new_filters =+ Map.put(query.filters, "event:page", {:member, Enum.map(pages, & &1["page"])})++ %Query{query | filters: new_filters}+ end++ {limit, _page} = pagination++ session_result =+ breakdown_sessions(site, new_query, "visit:entry_page", session_metrics, {limit, 1})+ |> transform_keys(%{"entry_page" => "page"})++ zip_results(+ event_result,+ session_result,+ "event:page",+ metrics+ )+ end++ def breakdown(site, query, property, metrics, pagination) when property in @event_props do+ breakdown_events(site, query, property, metrics, pagination)+ end++ def breakdown(site, query, property, metrics, pagination) do+ breakdown_sessions(site, query, property, metrics, pagination)enddefp zip_results(event_result, session_result, property, metrics) do+ sort_by = if Enum.member?(metrics, "visitors"), do: "visitors", else: List.first(metrics)+property =property|> String.trim_leading("event:")@@ -51,6 +151,7 @@ defmodule Plausible.Stats.Breakdown doMap.merge(null_row, event_row)|> Map.merge(session_row)end)+ |> Enum.sort_by(fn row -> row[sort_by] end, :desc)enddefp breakdown_sessions(_, _, _, [], _), do: []@@ -71,7 +172,7 @@ defmodule Plausible.Stats.Breakdown doenddefp filter_converted_sessions(db_query, site, query) do- if query.filters["event:name"] do+ if query.filters["event:name"] || query.filters["event:page"] || query.filters["visit:goal"] doconverted_sessions =from(e in query_events(site, query),select: %{session_id: fragment("DISTINCT ?", e.session_id)}@@ -102,6 +203,47 @@ defmodule Plausible.Stats.Breakdown do|> ClickhouseRepo.all()end+ defp breakdown_time_on_page(site, query, pages) do+ q =+ from(+ e in base_event_query(site, %Query{+ query+ | filters: Map.delete(query.filters, "event:page")+ }),+ select: {+ fragment("? as p", e.pathname),+ fragment("? as t", e.timestamp),+ fragment("? as s", e.session_id)+ },+ order_by: [e.session_id, e.timestamp]+ )++ {base_query_raw, base_query_raw_params} = ClickhouseRepo.to_sql(:all, q)++ time_query = "+ SELECT+ p,+ sum(td)/count(case when p2 != p then 1 end) as avgTime+ FROM+ (SELECT+ p,+ p2,+ sum(t2-t) as td+ FROM+ (SELECT+ *,+ neighbor(t, 1) as t2,+ neighbor(p, 1) as p2,+ neighbor(s, 1) as s2+ FROM (#{base_query_raw}))+ WHERE s=s2 AND p IN tuple(?)+ GROUP BY p,p2,s)+ GROUP BY p"++ {:ok, res} = ClickhouseRepo.query(time_query, base_query_raw_params ++ [pages])+ res.rows |> Enum.map(fn [page, time] -> {page, time} end) |> Enum.into(%{})+ end+defp do_group_by(%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events", _}},@@ -154,14 +296,20 @@ defmodule Plausible.Stats.Breakdown doenddefp do_group_by(- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"sessions", _}}} = q,- "event:page"+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events", _}}} = q,+ "event:page_match") do- from(- s in q,- group_by: s.entry_page,- select_merge: %{"page" => s.entry_page}- )+ case Map.get(q, :__private_match_sources__) do+ match_exprs when is_list(match_exprs) ->+ from(+ e in q,+ group_by: fragment("index"),+ select_merge: %{+ "index" => fragment("arrayJoin(indices) as index"),+ "page_match" => fragment("array(?)[index]", ^match_exprs)+ }+ )+ endenddefp do_group_by(q, "visit:source") do@@ -191,6 +339,14 @@ defmodule Plausible.Stats.Breakdown do)end+ defp do_group_by(q, "visit:exit_page") do+ from(+ s in q,+ group_by: s.exit_page,+ select_merge: %{"exit_page" => s.exit_page}+ )+ end+defp do_group_by(q, "visit:referrer") dofrom(s in q,@@ -287,6 +443,13 @@ defmodule Plausible.Stats.Breakdown do|> select_event_metrics(rest)end+ defp select_event_metrics(q, ["events" | rest]) do+ from(e in q,+ select_merge: %{"events" => fragment("count(*)")}+ )+ |> select_event_metrics(rest)+ end+defp select_metrics(q, []), do: qdefp select_metrics(q, ["pageviews" | rest]) do@@ -303,6 +466,15 @@ defmodule Plausible.Stats.Breakdown do|> select_metrics(rest)end+ defp select_metrics(q, ["visits" | rest]) do+ from(s in q,+ select_merge: %{+ "visits" => fragment("sum(?)", s.sign)+ }+ )+ |> select_metrics(rest)+ end+defp select_metrics(q, ["bounce_rate" | rest]) dofrom(s in q,select_merge: %{@@ -318,4 +490,13 @@ defmodule Plausible.Stats.Breakdown do)|> select_metrics(rest)end++ defp transform_keys(results, keys_to_replace) do+ Enum.map(results, fn map ->+ Enum.map(map, fn {key, val} ->+ {Map.get(keys_to_replace, key, key), val}+ end)+ |> Enum.into(%{})+ end)+ endenddiff --git a/lib/plausible/stats/current_visitors.ex b/lib/plausible/stats/current_visitors.exnew file mode 100644index 0000000000..225c0d8a58--- /dev/null+++ b/lib/plausible/stats/current_visitors.ex@@ -0,0 +1,17 @@+defmodule Plausible.Stats.CurrentVisitors do+ use Plausible.ClickhouseRepo+ use Plausible.Stats.Fragments++ def current_visitors(site) do+ first_datetime =+ NaiveDateTime.utc_now()+ |> Timex.shift(minutes: -5)++ ClickhouseRepo.one(+ from e in "events",+ where: e.domain == ^site.domain,+ where: e.timestamp >= ^first_datetime,+ select: uniq(e.user_id)+ )+ end+enddiff --git a/lib/plausible/stats/filter_suggestions.ex b/lib/plausible/stats/filter_suggestions.exnew file mode 100644index 0000000000..538647dc2d--- /dev/null+++ b/lib/plausible/stats/filter_suggestions.ex@@ -0,0 +1,148 @@+defmodule Plausible.Stats.FilterSuggestions do+ use Plausible.Repo+ use Plausible.ClickhouseRepo+ import Plausible.Stats.Base++ def filter_suggestions(site, query, "country", filter_search) do+ filter_search =+ String.split(filter_search, ",")+ |> Enum.map(fn c -> Plausible.Stats.CountryName.to_alpha2(c) end)++ q =+ from(+ e in query_sessions(site, query),+ group_by: e.country_code,+ order_by: [desc: fragment("count(*)")],+ select: {e.country_code}+ )++ ClickhouseRepo.all(q)+ |> Enum.map(fn {x} -> x end)+ |> Enum.filter(fn c -> Enum.find(filter_search, false, fn x -> x == c end) end)+ |> Enum.map(fn c -> Plausible.Stats.CountryName.to_alpha3(c) end)+ |> Enum.slice(0..24)+ end++ def filter_suggestions(site, _query, "goal", filter_search) do+ Repo.all(from g in Plausible.Goal, where: g.domain == ^site.domain)+ |> Enum.map(fn x -> if x.event_name, do: x.event_name, else: "Visit #{x.page_path}" end)+ |> Enum.filter(fn goal ->+ String.contains?(+ String.downcase(goal),+ String.downcase(filter_search)+ )+ end)+ end++ def filter_suggestions(site, query, filter_name, filter_search) do+ filter_search = if filter_search == nil, do: "", else: filter_search++ filter_query =+ if Enum.member?(["entry_page", "page", "exit_page"], filter_name),+ do: "%#{String.replace(filter_search, "*", "")}%",+ else: "%#{filter_search}%"++ filter_name =+ case filter_name do+ "page" -> "pathname"+ "source" -> "referrer_source"+ "os" -> "operating_system"+ "os_version" -> "operating_system_version"+ "screen" -> "screen_size"+ _ -> filter_name+ end++ q =+ if(filter_name == "pathname",+ do: base_event_query(site, query),+ else: query_sessions(site, query)+ )+ |> from(+ group_by: ^String.to_atom(filter_name),+ order_by: [desc: fragment("count(*)")],+ limit: 25+ )++ q =+ case filter_name do+ "pathname" ->+ from(e in q,+ select: {e.pathname},+ where: fragment("? ilike ?", e.pathname, ^filter_query)+ )++ "entry_page" ->+ from(e in q,+ select: {e.entry_page},+ where: fragment("? ilike ?", e.entry_page, ^filter_query)+ )++ "exit_page" ->+ from(e in q,+ select: {e.exit_page},+ where: fragment("? ilike ?", e.exit_page, ^filter_query)+ )++ "referrer_source" ->+ from(e in q,+ select: {e.referrer_source},+ where: fragment("? ilike ?", e.referrer_source, ^filter_query)+ )++ "utm_medium" ->+ from(e in q,+ select: {e.utm_medium},+ where: fragment("? ilike ?", e.utm_medium, ^filter_query)+ )++ "utm_source" ->+ from(e in q,+ select: {e.utm_source},+ where: fragment("? ilike ?", e.utm_source, ^filter_query)+ )++ "utm_campaign" ->+ from(e in q,+ select: {e.utm_campaign},+ where: fragment("? ilike ?", e.utm_campaign, ^filter_query)+ )++ "referrer" ->+ from(e in q,+ select: {e.referrer},+ where: fragment("? ilike ?", e.referrer, ^filter_query)+ )++ "browser" ->+ from(e in q, select: {e.browser}, where: fragment("? ilike ?", e.browser, ^filter_query))++ "browser_version" ->+ from(e in q,+ select: {e.browser_version},+ where: fragment("? ilike ?", e.browser_version, ^filter_query)+ )++ "operating_system" ->+ from(e in q,+ select: {e.operating_system},+ where: fragment("? ilike ?", e.operating_system, ^filter_query)+ )++ "operating_system_version" ->+ from(e in q,+ select: {e.operating_system_version},+ where: fragment("? ilike ?", e.operating_system_version, ^filter_query)+ )++ "screen_size" ->+ from(e in q,+ select: {e.screen_size},+ where: fragment("? ilike ?", e.screen_size, ^filter_query)+ )+ end++ ClickhouseRepo.all(q)+ |> Enum.map(fn {suggestion} -> suggestion end)+ |> Enum.filter(fn suggestion -> suggestion != "" end)+ end+enddiff --git a/lib/plausible/stats/filters.ex b/lib/plausible/stats/filters.exnew file mode 100644index 0000000000..3dbce01ac6--- /dev/null+++ b/lib/plausible/stats/filters.ex@@ -0,0 +1,59 @@+defmodule Plausible.Stats.Filters do+ @visit_props [+ "source",+ "referrer",+ "utm_medium",+ "utm_source",+ "utm_campaign",+ "screen",+ "device",+ "browser",+ "browser_version",+ "os",+ "os_version",+ "country"+ ]++ @event_props [+ "name",+ "page"+ ]++ def visit_props() do+ @visit_props+ end++ def add_prefix(query) do+ new_filters =+ Enum.reduce(query.filters, %{}, fn {name, val}, new_filters ->+ cond do+ name == "country" ->+ new_val = Plausible.Stats.CountryName.to_alpha2(val)+ Map.put(new_filters, "visit:country", {:is, new_val})++ name == "page" ->+ if String.match?(val, ~r/\*/) do+ Map.put(new_filters, "event:page", {:matches, val})+ else+ Map.put(new_filters, "event:page", {:is, val})+ end++ name in (@visit_props ++ ["goal"]) ->+ Map.put(new_filters, "visit:" <> name, {:is, val})++ name in @event_props ->+ Map.put(new_filters, "event:" <> name, {:is, val})++ name == "props" ->+ Enum.reduce(val, new_filters, fn {prop_key, prop_val}, new_filters ->+ Map.put(new_filters, "event:props:" <> prop_key, {:is, prop_val})+ end)++ true ->+ Map.put(new_filters, name, {:is, val})+ end+ end)++ %Plausible.Stats.Query{query | filters: new_filters}+ end+enddiff --git a/lib/plausible/stats/mod.ex b/lib/plausible/stats/mod.exindex 88fc12818c..97f329a377 100644--- a/lib/plausible/stats/mod.ex+++ b/lib/plausible/stats/mod.ex@@ -2,4 +2,9 @@ defmodule Plausible.Stats dodefdelegate breakdown(site, query, prop, metrics, pagination), to: Plausible.Stats.Breakdowndefdelegate aggregate(site, query, metrics), to: Plausible.Stats.Aggregatedefdelegate timeseries(site, query, metrics), to: Plausible.Stats.Timeseries+ defdelegate current_visitors(site), to: Plausible.Stats.CurrentVisitors+ defdelegate props(site, query), to: Plausible.Stats.Props++ defdelegate filter_suggestions(site, query, filter_name, filter_search),+ to: Plausible.Stats.FilterSuggestionsenddiff --git a/lib/plausible/stats/props.ex b/lib/plausible/stats/props.exnew file mode 100644index 0000000000..68b9d8c4bf--- /dev/null+++ b/lib/plausible/stats/props.ex@@ -0,0 +1,35 @@+defmodule Plausible.Stats.Props do+ use Plausible.ClickhouseRepo+ alias Plausible.Stats.Query+ import Plausible.Stats.Base++ def props(site, %Query{filters: %{"props" => meta}} = query) when is_map(meta) do+ [{key, val}] = meta |> Enum.into([])++ if val == "(none)" do+ goal = query.filters["goal"]+ %{goal => [key]}+ else+ ClickhouseRepo.all(+ from [e, meta: meta] in base_event_query(site, query),+ select: {e.name, meta.key},+ distinct: true+ )+ |> Enum.reduce(%{}, fn {goal_name, meta_key}, acc ->+ Map.update(acc, goal_name, [meta_key], fn list -> [meta_key | list] end)+ end)+ end+ end++ def props(site, query) do+ ClickhouseRepo.all(+ from e in base_event_query(site, query),+ inner_lateral_join: meta in fragment("meta as m"),+ select: {e.name, meta.key},+ distinct: true+ )+ |> Enum.reduce(%{}, fn {goal_name, meta_key}, acc ->+ Map.update(acc, goal_name, [meta_key], fn list -> [meta_key | list] end)+ end)+ end+enddiff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.exindex acac183b5c..2649a426df 100644--- a/lib/plausible/stats/query.ex+++ b/lib/plausible/stats/query.ex@@ -35,7 +35,7 @@ defmodule Plausible.Stats.Query dodate = today(tz)%__MODULE__{- period: "realtime",+ period: "30m",interval: "minute",date_range: Date.range(date, date),filters: parse_filters(params)@@ -152,6 +152,13 @@ defmodule Plausible.Stats.Query do__MODULE__.from(tz, Map.merge(params, %{"period" => "30d"}))end+ def put_filter(query, key, val) do+ %__MODULE__{+ query+ | filters: Map.put(query.filters, key, val)+ }+ end+defp today(tz) doTimex.now(tz) |> Timex.to_date()end@@ -184,12 +191,19 @@ defmodule Plausible.Stats.Query dodefp parse_single_filter(str) do[key, val] =String.trim(str)- |> String.split("==")+ |> String.split(["==", "!="], trim: true)|> Enum.map(&String.trim/1)- case String.split(val, "|") do- [single_value] -> {key, {:is, single_value}}- list -> {key, {:member, list}}+ is_negated = String.contains?(str, "!=")+ is_list = String.contains?(val, "|")+ is_glob = String.contains?(val, "*")++ cond do+ is_list && is_glob -> raise "Not implemented"+ is_list -> {key, {:member, String.split(val, "|")}}+ is_glob -> {key, {:matches, val}}+ is_negated -> {key, {:is_not, val}}+ true -> {key, {:is, val}}endendenddiff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.exindex 74eceb3afe..8d52c6f4a4 100644--- a/lib/plausible/stats/timeseries.ex+++ b/lib/plausible/stats/timeseries.ex@@ -5,7 +5,7 @@ defmodule Plausible.Stats.Timeseries douse Plausible.Stats.Fragments@event_metrics ["visitors", "pageviews"]- @session_metrics ["bounce_rate", "visit_duration"]+ @session_metrics ["visits", "bounce_rate", "visit_duration"]def timeseries(site, query, metrics) dosteps = buckets(query)@@ -69,6 +69,10 @@ defmodule Plausible.Stats.Timeseries doend)end+ defp buckets(%Query{period: "30m", interval: "minute"}) do+ Enum.into(-30..-1, [])+ end+defp select_bucket(q, site, %Query{interval: "month"}) dofrom(e in q,@@ -97,6 +101,15 @@ defmodule Plausible.Stats.Timeseries do)end+ defp select_bucket(q, _site, %Query{interval: "minute"}) do+ from(+ e in q,+ select_merge: %{+ "date" => fragment("dateDiff('minute', now(), ?) as date", e.timestamp)+ }+ )+ end+defp select_event_metrics(q, []), do: qdefp select_event_metrics(q, ["pageviews" | rest]) do@@ -124,6 +137,15 @@ defmodule Plausible.Stats.Timeseries do|> select_session_metrics(rest)end+ defp select_session_metrics(q, ["visits" | rest]) do+ from(s in q,+ select_merge: %{+ "visits" => fragment("sum(?)", s.sign)+ }+ )+ |> select_session_metrics(rest)+ end+defp select_session_metrics(q, ["visit_duration" | rest]) dofrom(s in q,select_merge: %{"visit_duration" => visit_duration()}@@ -136,6 +158,7 @@ defmodule Plausible.Stats.Timeseries docase metric do"pageviews" -> Map.merge(row, %{"pageviews" => 0})"visitors" -> Map.merge(row, %{"visitors" => 0})+ "visits" -> Map.merge(row, %{"visits" => 0})"bounce_rate" -> Map.merge(row, %{"bounce_rate" => nil})"visit_duration" -> Map.merge(row, %{"visit_duration" => nil})enddiff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.exindex f50c5ca596..8a752fb3ee 100644--- a/lib/plausible_web/controllers/api/external_stats_controller.ex+++ b/lib/plausible_web/controllers/api/external_stats_controller.ex@@ -79,15 +79,21 @@ defmodule PlausibleWeb.Api.ExternalStatsController do"The `property` parameter is required. Please provide at least one property to show a breakdown by."}end+ defp event_only_property?("event:name"), do: true+ defp event_only_property?("event:props:" <> _), do: true+ defp event_only_property?(_), do: false+@event_metrics ["visitors", "pageviews"]- @session_metrics ["bounce_rate", "visit_duration"]+ @session_metrics ["visits", "bounce_rate", "visit_duration"]defp parse_metrics(params, property, query) dometrics =Map.get(params, "metrics", "visitors")|> String.split(",")+ event_only_filter = Map.keys(query.filters) |> Enum.find(&event_only_property?/1)+valid_metrics =- if property == "event:name" || query.filters["event:name"] do+ if event_only_property?(property) || event_only_filter do@event_metricselse@event_metrics ++ @session_metrics@@ -97,13 +103,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController doif invalid_metric docond do- property == "event:name" && invalid_metric in @session_metrics ->+ event_only_property?(property) && invalid_metric in @session_metrics ->{:error,- "Session metric `#{invalid_metric}` cannot be queried for breakdown by `event:name`."}+ "Session metric `#{invalid_metric}` cannot be queried for breakdown by `#{property}`."}- query.filters["event:name"] && invalid_metric in @session_metrics ->+ event_only_filter && invalid_metric in @session_metrics ->{:error,- "Session metric `#{invalid_metric}` cannot be queried when using a filter on `event:name`."}+ "Session metric `#{invalid_metric}` cannot be queried when using a filter on `#{+ event_only_filter+ }`."}true ->{:error,diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex afaa9e0bb7..4b4b44bc77 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -2,16 +2,20 @@ defmodule PlausibleWeb.Api.StatsController douse PlausibleWeb, :controlleruse Plausible.Repouse Plug.ErrorHandler- alias Plausible.Stats.Clickhouse, as: Stats- alias Plausible.Stats.Query+ alias Plausible.Stats+ alias Plausible.Stats.{Query, Filters}def main_graph(conn, params) dosite = conn.assigns[:site]- query = Query.from(site.timezone, params)+ query = Query.from(site.timezone, params) |> Filters.add_prefix()- plot_task = Task.async(fn -> Stats.calculate_plot(site, query) end)+ timeseries = Task.async(fn -> Stats.timeseries(site, query, ["visitors"]) end){top_stats, sample_percent} = fetch_top_stats(site, query)- {plot, labels, present_index} = Task.await(plot_task)++ timeseries_result = Task.await(timeseries)+ plot = Enum.map(timeseries_result, fn row -> row["visitors"] end)+ labels = Enum.map(timeseries_result, fn row -> row["date"] end)+ present_index = present_index_for(site, query, labels)json(conn, %{plot: plot,@@ -23,34 +27,72 @@ defmodule PlausibleWeb.Api.StatsController do})end- defp fetch_top_stats(site, %Query{period: "realtime"} = query) do+ defp present_index_for(site, query, dates) do+ case query.interval do+ "date" ->+ current_date =+ Timex.now(site.timezone)+ |> Timex.to_date()++ Enum.find_index(dates, &(&1 == current_date))++ "hour" ->+ current_date =+ Timex.now(site.timezone)+ |> Timex.format!("{YYYY}-{0M}-{0D} {h24}:{m}:{s}")++ Enum.find_index(dates, &(&1 == current_date))++ "minute" ->+ nil+ end+ end++ defp fetch_top_stats(site, %Query{period: "30m"} = query) do+ %{+ visitors: %{value: visitors},+ pageviews: %{value: pageviews}+ } = Stats.aggregate(site, query, ["visitors", "pageviews"])+stats = [%{name: "Current visitors",- count: Stats.current_visitors(site, query)+ value: Stats.current_visitors(site)},%{name: "Unique visitors (last 30 min)",- count: Stats.unique_visitors(site, query)+ value: visitors},%{name: "Pageviews (last 30 min)",- count: Stats.total_pageviews(site, query)+ value: pageviews}]{stats, 100}end- defp fetch_top_stats(site, %Query{filters: %{"goal" => goal}} = query) when is_binary(goal) do- total_filter = Map.merge(query.filters, %{"goal" => nil, "props" => nil})+ defp fetch_top_stats(site, %Query{filters: %{"visit:goal" => _goal}} = query) do+ total_filter = Map.merge(query.filters, %{"visit:goal" => nil})prev_query = Query.shift_back(query, site)- unique_visitors = Stats.unique_visitors(site, %{query | filters: total_filter})- prev_unique_visitors = Stats.unique_visitors(site, %{prev_query | filters: total_filter})- {converted_visitors, sample_percent} = Stats.unique_visitors_with_sample_percent(site, query)- prev_converted_visitors = Stats.unique_visitors(site, prev_query)- completions = Stats.total_events(site, query)- prev_completions = Stats.total_events(site, prev_query)++ %{+ visitors: %{value: unique_visitors}+ } = Stats.aggregate(site, %{query | filters: total_filter}, ["visitors"])++ %{+ visitors: %{value: prev_unique_visitors}+ } = Stats.aggregate(site, %{prev_query | filters: total_filter}, ["visitors"])++ %{+ visitors: %{value: converted_visitors},+ events: %{value: completions}+ } = Stats.aggregate(site, query, ["visitors", "events"])++ %{+ visitors: %{value: prev_converted_visitors},+ events: %{value: prev_completions}+ } = Stats.aggregate(site, prev_query, ["visitors", "events"])conversion_rate = calculate_cr(unique_visitors, converted_visitors)prev_conversion_rate = calculate_cr(prev_unique_visitors, prev_converted_visitors)@@ -58,114 +100,71 @@ defmodule PlausibleWeb.Api.StatsController dostats = [%{name: "Unique visitors",- count: unique_visitors,+ value: unique_visitors,change: percent_change(prev_unique_visitors, unique_visitors)},%{name: "Unique conversions",- count: converted_visitors,+ value: converted_visitors,change: percent_change(prev_converted_visitors, converted_visitors)},%{name: "Total conversions",- count: completions,+ value: completions,change: percent_change(prev_completions, completions)},%{name: "Conversion rate",- percentage: conversion_rate,+ value: conversion_rate,change: percent_change(prev_conversion_rate, conversion_rate)}]- {stats, sample_percent}+ {stats, 0}enddefp fetch_top_stats(site, query) doprev_query = Query.shift_back(query, site)- {pageviews, visitors, sample_percent} =- Stats.pageviews_and_visitors_with_sample_percent(site, query)-- {prev_pageviews, prev_visitors} = Stats.pageviews_and_visitors(site, prev_query)- bounce_rate = Stats.bounce_rate(site, query)- prev_bounce_rate = Stats.bounce_rate(site, prev_query)- change_bounce_rate = if prev_bounce_rate > 0, do: bounce_rate - prev_bounce_rate-- visit_duration =- if !query.filters["page"] do- duration = Stats.visit_duration(site, query)- prev_duration = Stats.visit_duration(site, prev_query)-- %{- name: "Visit duration",- duration: duration,- change: percent_change(prev_duration, duration)- }+ metrics =+ if query.filters["event:page"] do+ ["visitors", "pageviews", "bounce_rate", "time_on_page"]+ else+ ["visitors", "pageviews", "bounce_rate", "visit_duration"]end- time_on_page =- if query.filters["page"] do- [{success, duration}, {prev_success, prev_duration}] =- Task.yield_many(- [- Task.async(fn ->- {:ok, page_times} = Stats.page_times_by_page_url(site, query)-- page_times- end),- Task.async(fn ->- {:ok, page_times} = Stats.page_times_by_page_url(site, prev_query)-- page_times- end)- ],- 5000- )- |> Enum.map(fn {task, response} ->- case response do- nil ->- Task.shutdown(task, :brutal_kill)- {nil, nil}-- {:ok, page_times} ->- result = Enum.at(page_times.rows, 0)- result = if result, do: Enum.at(result, 0), else: nil- if result, do: {:ok, round(result)}, else: {nil, nil}-- _ ->- response- end- end)-- if success == :ok && prev_success == :ok do- %{- name: "Time on Page",- duration: duration,- change: percent_change(prev_duration, duration)- }- end- end+ current_results = Stats.aggregate(site, query, metrics)+ prev_results = Stats.aggregate(site, prev_query, metrics)stats =[- %{- name: "Unique visitors",- count: visitors,- change: percent_change(prev_visitors, visitors)- },- %{- name: "Total pageviews",- count: pageviews,- change: percent_change(prev_pageviews, pageviews)- },- %{name: "Bounce rate", percentage: bounce_rate, change: change_bounce_rate},- visit_duration,- time_on_page+ top_stats_entry(current_results, prev_results, "Unique visitors", :visitors),+ top_stats_entry(current_results, prev_results, "Total pageviews", :pageviews),+ top_stats_entry(current_results, prev_results, "Bounce rate", :bounce_rate),+ top_stats_entry(current_results, prev_results, "Visit duration", :visit_duration),+ top_stats_entry(current_results, prev_results, "Time on page", :time_on_page)]|> Enum.filter(& &1)- {stats, sample_percent}+ {stats, 0}+ end++ defp top_stats_entry(current_results, prev_results, name, key) do+ if current_results[key] do+ %{+ name: name,+ value: current_results[key][:value],+ change: calculate_change(key, prev_results[key][:value], current_results[key][:value])+ }+ end+ end++ defp calculate_change(:bounce_rate, old_count, new_count) do+ if old_count > 0, do: new_count - old_count+ end++ defp calculate_change(_metric, old_count, new_count) do+ percent_change(old_count, new_count)enddefp percent_change(old_count, new_count) do@@ -183,39 +182,64 @@ defmodule PlausibleWeb.Api.StatsController dodef sources(conn, params) dosite = conn.assigns[:site]- query = Query.from(site.timezone, params)- include_details = params["detailed"] == "true"- limit = if params["limit"], do: String.to_integer(params["limit"])- page = if params["page"], do: String.to_integer(params["page"])- show_noref = params["show_noref"] == "true"- json(conn, Stats.top_sources(site, query, limit || 9, page || 1, show_noref, include_details))++ query =+ Query.from(site.timezone, params)+ |> Filters.add_prefix()+ |> maybe_hide_noref(params)++ pagination = parse_pagination(params)++ metrics =+ if params["detailed"], do: ["visitors", "bounce_rate", "visit_duration"], else: ["visitors"]++ res =+ Stats.breakdown(site, query, "visit:source", metrics, pagination)+ |> transform_keys(%{"source" => "name", "visitors" => "count"})++ json(conn, res)enddef utm_mediums(conn, params) dosite = conn.assigns[:site]query = Query.from(site.timezone, params)- limit = if params["limit"], do: String.to_integer(params["limit"])- page = if params["page"], do: String.to_integer(params["page"])+ pagination = parse_pagination(params)show_noref = params["show_noref"] == "true"- json(conn, Stats.utm_mediums(site, query, limit || 9, page || 1, show_noref))+ metrics = ["visitors", "bounce_rate", "visit_duration"]++ res =+ Stats.breakdown(site, query, "visit:utm_medium", metrics, pagination)+ |> transform_keys(%{"utm_medium" => "name", "visitors" => "count"})++ json(conn, res)enddef utm_campaigns(conn, params) dosite = conn.assigns[:site]query = Query.from(site.timezone, params)- limit = if params["limit"], do: String.to_integer(params["limit"])- page = if params["page"], do: String.to_integer(params["page"])+ pagination = parse_pagination(params)show_noref = params["show_noref"] == "true"- json(conn, Stats.utm_campaigns(site, query, limit || 9, page || 1, show_noref))+ metrics = ["visitors", "bounce_rate", "visit_duration"]++ res =+ Stats.breakdown(site, query, "visit:utm_campaign", metrics, pagination)+ |> transform_keys(%{"utm_campaign" => "name", "visitors" => "count"})++ json(conn, res)enddef utm_sources(conn, params) dosite = conn.assigns[:site]query = Query.from(site.timezone, params)- limit = if params["limit"], do: String.to_integer(params["limit"])- page = if params["page"], do: String.to_integer(params["page"])+ pagination = parse_pagination(params)show_noref = params["show_noref"] == "true"- json(conn, Stats.utm_sources(site, query, limit || 9, page || 1, show_noref))+ metrics = ["visitors", "bounce_rate", "visit_duration"]++ res =+ Stats.breakdown(site, query, "visit:utm_source", metrics, pagination)+ |> transform_keys(%{"utm_source" => "name", "visitors" => "count"})++ json(conn, res)enddef referrer_drilldown(conn, %{"referrer" => "Google"} = params) do@@ -227,15 +251,15 @@ defmodule PlausibleWeb.Api.StatsController dogoogle_api().fetch_stats(site, query, params["limit"] || 9)end+ %{visitors: %{value: total_visitors}} = Stats.aggregate(site, query, ["visitors"])+case search_terms donil ->- {_, total_visitors} = Stats.pageviews_and_visitors(site, query)user_id = get_session(conn, :current_user_id)is_admin = user_id && Plausible.Sites.has_admin_access?(user_id, site)json(conn, %{not_configured: true, is_admin: is_admin, total_visitors: total_visitors}){:ok, terms} ->- {_, total_visitors} = Stats.pageviews_and_visitors(site, query)json(conn, %{search_terms: terms, total_visitors: total_visitors}){:error, e} ->@@ -246,92 +270,168 @@ defmodule PlausibleWeb.Api.StatsController dodef referrer_drilldown(conn, %{"referrer" => referrer} = params) dosite = conn.assigns[:site]- query = Query.from(site.timezone, params)- include_details = params["detailed"] == "true"- limit = params["limit"] || 9- referrers = Stats.referrer_drilldown(site, query, referrer, include_details, limit)- {_, total_visitors} = Stats.pageviews_and_visitors(site, query)- json(conn, %{referrers: referrers, total_visitors: total_visitors})- end+ query =+ Query.from(site.timezone, params)+ |> Query.put_filter("source", referrer)+ |> Filters.add_prefix()- def referrer_drilldown_for_goal(conn, %{"referrer" => referrer} = params) do- site = conn.assigns[:site]- query = Query.from(site.timezone, params)+ pagination = parse_pagination(params)++ metrics =+ if params["detailed"], do: ["visitors", "bounce_rate", "visit_duration"], else: ["visitors"]++ referrers =+ Stats.breakdown(site, query, "visit:referrer", metrics, pagination)+ |> transform_keys(%{"referrer" => "name", "visitors" => "count"})- referrers = Stats.referrer_drilldown_for_goal(site, query, referrer)- total_visitors = Stats.conversions_from_referrer(site, query, referrer)+ %{visitors: %{value: total_visitors}} = Stats.aggregate(site, query, ["visitors"])json(conn, %{referrers: referrers, total_visitors: total_visitors})enddef pages(conn, params) dosite = conn.assigns[:site]- query = Query.from(site.timezone, params)- include_details = params["detailed"] == "true"- limit = if params["limit"], do: String.to_integer(params["limit"])- page = if params["page"], do: String.to_integer(params["page"])+ query = Query.from(site.timezone, params) |> Filters.add_prefix()++ metrics =+ if params["detailed"],+ do: ["visitors", "pageviews", "bounce_rate", "time_on_page"],+ else: ["visitors"]++ pagination = parse_pagination(params)- json(conn, Stats.top_pages(site, query, limit || 9, page || 1, include_details))+ pages =+ Stats.breakdown(site, query, "event:page", metrics, pagination)+ |> transform_keys(%{"page" => "name", "visitors" => "count"})++ json(conn, pages)enddef entry_pages(conn, params) dosite = conn.assigns[:site]- query = Query.from(site.timezone, params)- limit = if params["limit"], do: String.to_integer(params["limit"])- page = if params["page"], do: String.to_integer(params["page"])+ query = Query.from(site.timezone, params) |> Filters.add_prefix()+ pagination = parse_pagination(params)+ metrics = ["visitors", "visits", "visit_duration"]++ entry_pages =+ Stats.breakdown(site, query, "event:page", metrics, pagination)+ |> transform_keys(%{"page" => "name", "visits" => "entries", "visitors" => "count"})- json(conn, Stats.entry_pages(site, query, limit || 9, page || 1))+ json(conn, entry_pages)enddef exit_pages(conn, params) dosite = conn.assigns[:site]- query = Query.from(site.timezone, params)- limit = if params["limit"], do: String.to_integer(params["limit"])- page = if params["page"], do: String.to_integer(params["page"])+ query = Query.from(site.timezone, params) |> Filters.add_prefix()+ {limit, page} = parse_pagination(params)+ metrics = ["visitors", "visits"]++ exit_pages =+ Stats.breakdown(site, query, "visit:exit_page", metrics, {limit, page})+ |> transform_keys(%{"exit_page" => "name", "visits" => "exits", "visitors" => "count"})++ page_filter_expr = Enum.map(exit_pages, & &1["page"]) |> Enum.join("|")+ total_visits_query = Query.put_filter(query, "page", page_filter_expr)++ total_pageviews =+ Stats.breakdown(site, total_visits_query, "event:page", ["pageviews"], {limit, 1})++ exit_pages =+ Enum.map(exit_pages, fn exit_page ->+ exit_rate =+ case Enum.find(total_pageviews, &(&1["page"] == exit_page["name"])) do+ %{"pageviews" => pageviews} ->+ round(exit_page["exits"] / pageviews * 100)++ nil ->+ nil+ end- json(conn, Stats.exit_pages(site, query, limit || 9, page || 1))+ Map.put(exit_page, "exit_rate", exit_rate)+ end)++ json(conn, exit_pages)enddef countries(conn, params) dosite = conn.assigns[:site]- query = Query.from(site.timezone, params)+ query = Query.from(site.timezone, params) |> Filters.add_prefix()++ countries =+ Stats.breakdown(site, query, "visit:country", ["visitors"], {300, 1})+ |> transform_keys(%{"country" => "name", "visitors" => "count"})+ |> Enum.map(fn country ->+ alpha3 = Stats.CountryName.to_alpha3(country["name"])+ Map.put(country, "name", alpha3)+ end)+ |> add_percentages- json(conn, Stats.countries(site, query))+ json(conn, countries)enddef browsers(conn, params) dosite = conn.assigns[:site]- query = Query.from(site.timezone, params)+ query = Query.from(site.timezone, params) |> Filters.add_prefix()+ pagination = parse_pagination(params)++ browsers =+ Stats.breakdown(site, query, "visit:browser", ["visitors"], pagination)+ |> transform_keys(%{"browser" => "name", "visitors" => "count"})+ |> add_percentages- json(conn, Stats.browsers(site, query, params["limit"] || 9))+ json(conn, browsers)enddef browser_versions(conn, params) dosite = conn.assigns[:site]- query = Query.from(site.timezone, params)+ query = Query.from(site.timezone, params) |> Filters.add_prefix()+ pagination = parse_pagination(params)++ versions =+ Stats.breakdown(site, query, "visit:browser_version", ["visitors"], pagination)+ |> transform_keys(%{"browser_version" => "name", "visitors" => "count"})+ |> add_percentages- json(conn, Stats.browser_versions(site, query, params["limit"] || 9))+ json(conn, versions)enddef operating_systems(conn, params) dosite = conn.assigns[:site]- query = Query.from(site.timezone, params)+ query = Query.from(site.timezone, params) |> Filters.add_prefix()+ pagination = parse_pagination(params)- json(conn, Stats.operating_systems(site, query, params["limit"] || 9))+ systems =+ Stats.breakdown(site, query, "visit:os", ["visitors"], pagination)+ |> transform_keys(%{"os" => "name", "visitors" => "count"})+ |> add_percentages++ json(conn, systems)enddef operating_system_versions(conn, params) dosite = conn.assigns[:site]- query = Query.from(site.timezone, params)+ query = Query.from(site.timezone, params) |> Filters.add_prefix()+ pagination = parse_pagination(params)++ versions =+ Stats.breakdown(site, query, "visit:os_version", ["visitors"], pagination)+ |> transform_keys(%{"os_version" => "name", "visitors" => "count"})+ |> add_percentages- json(conn, Stats.operating_system_versions(site, query, params["limit"] || 9))+ json(conn, versions)enddef screen_sizes(conn, params) dosite = conn.assigns[:site]- query = Query.from(site.timezone, params)+ query = Query.from(site.timezone, params) |> Filters.add_prefix()+ pagination = parse_pagination(params)++ sizes =+ Stats.breakdown(site, query, "visit:device", ["visitors"], pagination)+ |> transform_keys(%{"device" => "name", "visitors" => "count"})+ |> add_percentages- json(conn, Stats.top_screen_sizes(site, query))+ json(conn, sizes)enddefp calculate_cr(unique_visitors, converted_visitors) do@@ -342,17 +442,23 @@ defmodule PlausibleWeb.Api.StatsController dodef conversions(conn, params) dosite = conn.assigns[:site]- query = Query.from(site.timezone, params)- total_filter = Map.merge(query.filters, %{"goal" => nil, "props" => nil})- unique_visitors = Stats.unique_visitors(site, %{query | filters: total_filter})- prop_names = Stats.all_props(site, query)+ query = Query.from(site.timezone, params) |> Filters.add_prefix()+ pagination = parse_pagination(params)++ total_filter = Map.merge(query.filters, %{"visit:goal" => nil})++ %{visitors: %{value: total_visitors}} =+ Stats.aggregate(site, %{query | filters: total_filter}, ["visitors"])++ prop_names = Stats.props(site, query)conversions =- Stats.goal_conversions(site, query)+ Stats.breakdown(site, query, "visit:goal", ["visitors", "events"], pagination)+ |> transform_keys(%{"goal" => "name", "visitors" => "count", "events" => "total_count"})|> Enum.map(fn goal ->goal- |> Map.put(:prop_names, prop_names[goal[:name]])- |> Map.put(:conversion_rate, calculate_cr(unique_visitors, goal[:count]))+ |> Map.put(:prop_names, prop_names[goal["name"]])+ |> Map.put(:conversion_rate, calculate_cr(total_visitors, goal["count"]))end)json(conn, conversions)@@ -360,14 +466,25 @@ defmodule PlausibleWeb.Api.StatsController dodef prop_breakdown(conn, params) dosite = conn.assigns[:site]- query = Query.from(site.timezone, params)- total_filter = Map.merge(query.filters, %{"goal" => nil, "props" => nil})- unique_visitors = Stats.unique_visitors(site, %{query | filters: total_filter})+ query = Query.from(site.timezone, params) |> Filters.add_prefix()+ pagination = parse_pagination(params)++ total_filter = Map.merge(query.filters, %{"visit:goal" => nil})++ %{visitors: %{value: unique_visitors}} =+ Stats.aggregate(site, %{query | filters: total_filter}, ["visitors"])++ prop_name = "event:props:" <> params["prop_name"]props =- Stats.property_breakdown(site, query, params["prop_name"])+ Stats.breakdown(site, query, prop_name, ["visitors", "events"], pagination)+ |> transform_keys(%{+ params["prop_name"] => "name",+ "visitors" => "count",+ "events" => "total_count"+ })|> Enum.map(fn prop ->- Map.put(prop, :conversion_rate, calculate_cr(unique_visitors, prop[:count]))+ Map.put(prop, "conversion_rate", calculate_cr(unique_visitors, prop["count"]))end)json(conn, props)@@ -375,8 +492,7 @@ defmodule PlausibleWeb.Api.StatsController dodef current_visitors(conn, _) dosite = conn.assigns[:site]- query = Query.from(site.timezone, %{"period" => "realtime"})- json(conn, Stats.current_visitors(site, query))+ json(conn, Stats.current_visitors(site))enddefp google_api(), do: Application.fetch_env!(:plausible, :google_api)@@ -389,6 +505,40 @@ defmodule PlausibleWeb.Api.StatsController dosite = conn.assigns[:site]query = Query.from(site.timezone, params)- json(conn, Stats.make_suggestions(site, query, params["filter_name"], params["q"]))+ json(conn, Stats.filter_suggestions(site, query, params["filter_name"], params["q"]))+ end++ defp transform_keys(results, keys_to_replace) do+ Enum.map(results, fn map ->+ Enum.map(map, fn {key, val} ->+ {Map.get(keys_to_replace, key, key), val}+ end)+ |> Enum.into(%{})+ end)+ end++ defp parse_pagination(params) do+ limit = if params["limit"], do: String.to_integer(params["limit"]), else: 9+ page = if params["page"], do: String.to_integer(params["page"]), else: 1+ {limit, page}+ end++ defp add_percentages(stat_list) do+ total = Enum.reduce(stat_list, 0, fn %{"count" => count}, total -> total + count end)++ Enum.map(stat_list, fn stat ->+ Map.put(stat, "percentage", round(stat["count"] / total * 100))+ end)+ end++ defp maybe_hide_noref(query, params) do+ cond do+ is_nil(query.filters["visit:source"]) and params["show_noref"] != "true" ->+ new_filters = Map.put(query.filters, "visit:source", {:is_not, "Direct / None"})+ %Query{query | filters: new_filters}++ true ->+ query+ endendenddiff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.exindex ab5f7ee87b..158da85013 100644--- a/lib/plausible_web/router.ex+++ b/lib/plausible_web/router.ex@@ -57,7 +57,6 @@ defmodule PlausibleWeb.Router doget "/:domain/utm_sources", StatsController, :utm_sourcesget "/:domain/utm_campaigns", StatsController, :utm_campaignsget "/:domain/referrers/:referrer", StatsController, :referrer_drilldown- get "/:domain/goal/referrers/:referrer", StatsController, :referrer_drilldown_for_goalget "/:domain/pages", StatsController, :pagesget "/:domain/entry-pages", StatsController, :entry_pagesget "/:domain/exit-pages", StatsController, :exit_pagesdiff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exsindex 71830ea8b2..7d26c49db6 100644--- a/test/plausible/stats/query_test.exs+++ b/test/plausible/stats/query_test.exs@@ -25,7 +25,7 @@ defmodule Plausible.Stats.QueryTest doassert q.date_range.first == Timex.today()assert q.date_range.last == Timex.today()- assert q.period == "realtime"+ assert q.period == "30m"endtest "parses month format" dodiff --git a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exsindex 0be9dad971..01bab4da48 100644--- a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs@@ -117,7 +117,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do}end- test "aggregates visitors, pageviews, bounce rate and visit duration", %{conn: conn, site: site} do+ test "aggregates visitors, pageviews, visits, bounce rate and visit duration", %{+ conn: conn,+ site: site+ } dopopulate_stats([build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),@@ -129,12 +132,13 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do"site_id" => site.domain,"period" => "day","date" => "2021-01-01",- "metrics" => "pageviews,visitors,bounce_rate,visit_duration"+ "metrics" => "pageviews,visits,visitors,bounce_rate,visit_duration"})assert json_response(conn, 200)["results"] == %{"pageviews" => %{"value" => 3},"visitors" => %{"value" => 2},+ "visits" => %{"value" => 2},"bounce_rate" => %{"value" => 50},"visit_duration" => %{"value" => 750}}@@ -650,6 +654,35 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do}end+ test "can filter by page regex", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ pathname: "/"+ ),+ build(:pageview,+ pathname: "/blog/post1"+ ),+ build(:pageview,+ pathname: "/blog/post1"+ ),+ build(:pageview,+ pathname: "/blog/post2"+ )+ ])++ conn =+ get(conn, "/api/v1/stats/aggregate", %{+ "site_id" => site.domain,+ "period" => "day",+ "metrics" => "visitors",+ "filters" => "event:page==/blog/**"+ })++ assert json_response(conn, 200)["results"] == %{+ "visitors" => %{"value" => 3}+ }+ end+test "filtering by event:name", %{conn: conn, site: site} dopopulate_stats([build(:event,diff --git a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exsindex 6688ae1b09..81612b2eb2 100644--- a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs@@ -59,6 +59,20 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do}end+ test "session metrics cannot be used with event:props:* property", %{conn: conn, site: site} do+ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "property" => "event:props:url",+ "metrics" => "visitors,bounce_rate",+ "site_id" => site.domain+ })++ assert json_response(conn, 400) == %{+ "error" =>+ "Session metric `bounce_rate` cannot be queried for breakdown by `event:props:url`."+ }+ end+test "session metrics cannot be used with event:name filter", %{conn: conn, site: site} doconn =get(conn, "/api/v1/stats/breakdown", %{@@ -73,6 +87,21 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do"Session metric `bounce_rate` cannot be queried when using a filter on `event:name`."}end++ test "session metrics cannot be used with event:props:* filter", %{conn: conn, site: site} do+ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "property" => "event:page",+ "filters" => "event:props:url==google.com",+ "metrics" => "visitors,bounce_rate",+ "site_id" => site.domain+ })++ assert json_response(conn, 400) == %{+ "error" =>+ "Session metric `bounce_rate` cannot be queried when using a filter on `event:props:url`."+ }+ endendtest "breakdown by visit:source", %{conn: conn, site: site} do@@ -777,6 +806,141 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do}end+ test "event:page filter shows traffic sources directly to that page", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview,+ pathname: "/ignore",+ referrer_source: "Should not show up",+ user_id: @user_id+ ),+ build(:pageview,+ pathname: "/plausible.io",+ user_id: @user_id+ ),+ build(:pageview,+ pathname: "/plausible.io",+ referrer_source: "Google"+ )+ ])++ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "site_id" => site.domain,+ "period" => "day",+ "property" => "visit:source",+ "filters" => "event:page==/plausible.io"+ })++ assert json_response(conn, 200) == %{+ "results" => [+ %{"source" => "Google", "visitors" => 1}+ ]+ }+ end++ test "visit:goal pageview filter for breakdown by visit source", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ referrer_source: "Bing",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ referrer_source: "Google",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/plausible.io",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])++ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "site_id" => site.domain,+ "period" => "day",+ "date" => "2021-01-01",+ "property" => "visit:source",+ "filters" => "visit:goal == Visit /plausible.io"+ })++ assert json_response(conn, 200) == %{+ "results" => [+ %{"source" => "Google", "visitors" => 1}+ ]+ }+ end++ test "visit:goal custom event filter for breakdown by visit source", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ referrer_source: "Bing",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ referrer_source: "Google",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Register",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])++ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "site_id" => site.domain,+ "period" => "day",+ "date" => "2021-01-01",+ "property" => "visit:source",+ "filters" => "visit:goal == Register"+ })++ assert json_response(conn, 200) == %{+ "results" => [+ %{"source" => "Google", "visitors" => 1}+ ]+ }+ end++ test "visit:goal custom event filter for breakdown by event page", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event,+ pathname: "/en/register",+ name: "Register"+ ),+ build(:event,+ pathname: "/en/register",+ name: "Register"+ ),+ build(:event,+ pathname: "/it/register",+ name: "Register"+ )+ ])++ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "site_id" => site.domain,+ "period" => "day",+ "property" => "event:page",+ "filters" => "visit:goal == Register"+ })++ assert json_response(conn, 200) == %{+ "results" => [+ %{"page" => "/en/register", "visitors" => 2},+ %{"page" => "/it/register", "visitors" => 1}+ ]+ }+ end+test "IN filter for event:page", %{conn: conn, site: site} dopopulate_stats([build(:pageview,@@ -945,6 +1109,29 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do]}end++ test "can use a is_not filter", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, browser: "Chrome"),+ build(:pageview, browser: "Safari"),+ build(:pageview, browser: "Safari"),+ build(:pageview, browser: "Edge")+ ])++ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "site_id" => site.domain,+ "filters" => "visit:browser != Chrome",+ "property" => "visit:browser"+ })++ assert json_response(conn, 200) == %{+ "results" => [+ %{"browser" => "Safari", "visitors" => 2},+ %{"browser" => "Edge", "visitors" => 1}+ ]+ }+ endenddescribe "pagination" do@@ -992,27 +1179,23 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest dodescribe "metrics" dotest "all metrics for breakdown by visit prop", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,user_id: 1,referrer_source: "Google",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,user_id: 1,referrer_source: "Google",- domain: site.domain,timestamp: ~N[2021-01-01 00:10:00]),build(:pageview,referrer_source: "Google",- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),build(:pageview,referrer_source: "Twitter",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00])])@@ -1023,7 +1206,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do"period" => "day","date" => "2021-01-01","property" => "visit:source",- "metrics" => "visitors,pageviews,bounce_rate,visit_duration"+ "metrics" => "visitors,visits,pageviews,bounce_rate,visit_duration"})assert json_response(conn, 200) == %{@@ -1031,6 +1214,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do%{"source" => "Google","visitors" => 2,+ "visits" => 2,"bounce_rate" => 50,"visit_duration" => 300,"pageviews" => 3@@ -1038,6 +1222,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do%{"source" => "Twitter","visitors" => 1,+ "visits" => 1,"bounce_rate" => 100,"visit_duration" => 0,"pageviews" => 1diff --git a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exsindex ec75776be1..23048bd4bb 100644--- a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs@@ -49,10 +49,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do@user_id 123test "shows hourly data for a certain date", %{conn: conn, site: site} do- populate_stats([- build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:10:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 23:59:00])+ populate_stats(site, [+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:10:00]),+ build(:pageview, timestamp: ~N[2021-01-01 23:59:00])])conn =@@ -60,7 +60,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do"site_id" => site.domain,"period" => "day","date" => "2021-01-01",- "metrics" => "visitors,pageviews,visit_duration,bounce_rate"+ "metrics" => "visitors,pageviews,visits,visit_duration,bounce_rate"})assert json_response(conn, 200) == %{@@ -68,6 +68,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do%{"date" => "2021-01-01 00:00:00","visitors" => 1,+ "visits" => 1,"pageviews" => 2,"visit_duration" => 600,"bounce_rate" => 0@@ -75,6 +76,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do%{"date" => "2021-01-01 01:00:00","visitors" => 0,+ "visits" => 0,"pageviews" => 0,"visit_duration" => nil,"bounce_rate" => nil@@ -82,6 +84,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do%{"date" => "2021-01-01 02:00:00","visitors" => 0,+ "visits" => 0,"pageviews" => 0,"visit_duration" => nil,"bounce_rate" => nil@@ -89,6 +92,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do%{"date" => "2021-01-01 03:00:00","visitors" => 0,+ "visits" => 0,"pageviews" => 0,"visit_duration" => nil,"bounce_rate" => nil@@ -96,6 +100,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do%{"date" => "2021-01-01 04:00:00","visitors" => 0,+ "visits" => 0,"pageviews" => 0,"visit_duration" => nil,"bounce_rate" => nil@@ -103,6 +108,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do%{"date" => "2021-01-01 05:00:00","visitors" => 0,+ "visits" => 0,"pageviews" => 0,"visit_duration" => nil,"bounce_rate" => nil@@ -110,6 +116,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do%{"date" => "2021-01-01 06:00:00","visitors" => 0,+ "visits" => 0,"pageviews" => 0,"visit_duration" => nil,"bounce_rate" => nil@@ -117,6 +124,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do%{"date" => "2021-01-01 07:00:00","visitors" => 0,+ "visits" => 0,"pageviews" => 0,"visit_duration" => nil,"bounce_rate" => nil@@ -124,6 +132,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do%{"date" => "2021-01-01 08:00:00","visitors" => 0,+ "visits" => 0,"pageviews" => 0,"visit_duration" => nil,"bounce_rate" => nil@@ -131,6 +140,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do%{"date" => "2021-01-01 09:00:00","visitors" => 0,+ "visits" => 0,"pageviews" => 0,"visit_duration" => nil,"bounce_rate" => nil@@ -138,6 +148,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do%{"date" => "2021-01-01 10:00:00","visitors" => 0,+ "visits" => 0,"pageviews" => 0,"visit_duration" => nil,"bounce_rate" => nil@@ -145,6 +156,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do%{"date" => "2021-01-01 11:00:00","visitors" => 0,+ "visits" => 0,"pageviews" => 0,"visit_duration" => nil,"bounce_rate" => nil@@ -152,6 +164,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do%{"date" => "2021-01-01 12:00:00","visitors" => 0,+ "visits" => 0,"pageviews" => 0,"visit_duration" => nil,"bounce_rate" => nil@@ -159,6 +172,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do%{"date" => "2021-01-01 13:00:00","visitors" => 0,+ "visits" => 0,"pageviews" => 0,"visit_duration" => nil,"bounce_rate" => nil@@ -166,6 +180,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do%{"date" => "2021-01-01 14:00:00","visitors" => 0,+ "visits" => 0,"pageviews" => 0,"visit_duration" => nil,"bounce_rate" => nil@@ -173,6 +188,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do%{"date" => "2021-01-01 15:00:00","visitors" => 0,+ "visits" => 0,"pageviews" => 0,"visit_duration" => nil,"bounce_rate" => nil@@ -180,6 +196,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do%{"date" => "2021-01-01 16:00:00","visitors" => 0,+ "visits" => 0,"pageviews" => 0,"visit_duration" => nil,"bounce_rate" => nil@@ -187,6 +204,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do%{"date" => "2021-01-01 17:00:00","visitors" => 0,+ "visits" => 0,"pageviews" => 0,"visit_duration" => nil,"bounce_rate" => nil@@ -194,6 +212,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do%{"date" => "2021-01-01 18:00:00","visitors" => 0,+ "visits" => 0,"pageviews" => 0,"visit_duration" => nil,"bounce_rate" => nil@@ -201,6 +220,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do%{"date" => "2021-01-01 19:00:00","visitors" => 0,+ "visits" => 0,"pageviews" => 0,"visit_duration" => nil,"bounce_rate" => nil@@ -208,6 +228,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do%{"date" => "2021-01-01 20:00:00","visitors" => 0,+ "visits" => 0,"pageviews" => 0,"visit_duration" => nil,"bounce_rate" => nil@@ -215,6 +236,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do%{"date" => "2021-01-01 21:00:00","visitors" => 0,+ "visits" => 0,"pageviews" => 0,"visit_duration" => nil,"bounce_rate" => nil@@ -222,6 +244,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do%{"date" => "2021-01-01 22:00:00","visitors" => 0,+ "visits" => 0,"pageviews" => 0,"visit_duration" => nil,"bounce_rate" => nil@@ -229,6 +252,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do%{"date" => "2021-01-01 23:00:00","visitors" => 1,+ "visits" => 1,"pageviews" => 1,"visit_duration" => 0,"bounce_rate" => 100diff --git a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs b/test/plausible_web/controllers/api/stats_controller/browsers_test.exsindex ee6d7b4269..7f540ade4e 100644--- a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs@@ -3,34 +3,46 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest doimport Plausible.TestUtilsdescribe "GET /api/stats/:domain/browsers" do- setup [:create_user, :log_in, :create_site]+ setup [:create_user, :log_in, :create_new_site]- test "returns top browsers by new visitors", %{conn: conn, site: site} do- conn = get(conn, "/api/stats/#{site.domain}/browsers?period=day&date=2019-01-01")+ test "returns top browsers by unique visitors", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, browser: "Chrome"),+ build(:pageview, browser: "Chrome"),+ build(:pageview, browser: "Firefox")+ ])++ conn = get(conn, "/api/stats/#{site.domain}/browsers?period=day")assert json_response(conn, 200) == [- %{"name" => "Chrome", "count" => 3, "percentage" => 75},- %{"name" => "Firefox", "count" => 1, "percentage" => 25}+ %{"name" => "Chrome", "count" => 2, "percentage" => 67},+ %{"name" => "Firefox", "count" => 1, "percentage" => 33}]endenddescribe "GET /api/stats/:domain/browser-versions" do- setup [:create_user, :log_in, :create_site]+ setup [:create_user, :log_in, :create_new_site]test "returns top browser versions by unique visitors", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, browser: "Chrome", browser_version: "78.0"),+ build(:pageview, browser: "Chrome", browser_version: "78.0"),+ build(:pageview, browser: "Chrome", browser_version: "77.0"),+ build(:pageview, browser: "Firefox", browser_version: "88.0")+ ])+filters = Jason.encode!(%{browser: "Chrome"})conn =get(conn,- "/api/stats/#{site.domain}/browser-versions?period=day&date=2019-01-01&filters=#{- filters- }"+ "/api/stats/#{site.domain}/browser-versions?period=day&filters=#{filters}")assert json_response(conn, 200) == [- %{"name" => "78.0", "count" => 1, "percentage" => 100}+ %{"name" => "78.0", "count" => 2, "percentage" => 67},+ %{"name" => "77.0", "count" => 1, "percentage" => 33}]endenddiff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exsindex b9e9e54548..66bd2392e4 100644--- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs@@ -1,39 +1,69 @@defmodule PlausibleWeb.Api.StatsController.ConversionsTest douse PlausibleWeb.ConnCaseimport Plausible.TestUtils+ @user_id 123describe "GET /api/stats/:domain/conversions" do- setup [:create_user, :log_in, :create_site]+ setup [:create_user, :log_in, :create_new_site]test "returns mixed conversions in ordered by count", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, pathname: "/"),+ build(:pageview, pathname: "/"),+ build(:pageview, pathname: "/register"),+ build(:pageview, pathname: "/register"),+ build(:event, name: "Signup", "meta.key": ["variant"], "meta.value": ["A"]),+ build(:event,+ user_id: @user_id,+ name: "Signup",+ "meta.key": ["variant"],+ "meta.value": ["A"]+ ),+ build(:event,+ user_id: @user_id,+ name: "Signup",+ "meta.key": ["variant"],+ "meta.value": ["B"]+ )+ ])+insert(:goal, %{domain: site.domain, page_path: "/register"})insert(:goal, %{domain: site.domain, event_name: "Signup"})- conn = get(conn, "/api/stats/#{site.domain}/conversions?period=day&date=2019-01-01")+ conn = get(conn, "/api/stats/#{site.domain}/conversions?period=day")assert json_response(conn, 200) == [%{"name" => "Signup",- "count" => 3,+ "count" => 2,"total_count" => 3,"prop_names" => ["variant"],- "conversion_rate" => 42.9+ "conversion_rate" => 33.3},%{"name" => "Visit /register","count" => 2,"total_count" => 2,"prop_names" => nil,- "conversion_rate" => 28.6+ "conversion_rate" => 33.3}]endenddescribe "GET /api/stats/:domain/conversions - with goal filter" do- setup [:create_user, :log_in, :create_site]+ setup [:create_user, :log_in, :create_new_site]test "returns only the conversion tha is filtered for", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, pathname: "/"),+ build(:pageview, pathname: "/"),+ build(:pageview, pathname: "/register"),+ build(:event, name: "Signup", "meta.key": ["variant"], "meta.value": ["A"]),+ build(:event, name: "Signup", "meta.key": ["variant"], "meta.value": ["A"]),+ build(:event, name: "Signup", "meta.key": ["variant"], "meta.value": ["B"])+ ])+insert(:goal, %{domain: site.domain, page_path: "/register"})insert(:goal, %{domain: site.domain, event_name: "Signup"})@@ -42,7 +72,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest doconn =get(conn,- "/api/stats/#{site.domain}/conversions?period=day&date=2019-01-01&filters=#{filters}"+ "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}")assert json_response(conn, 200) == [@@ -51,16 +81,25 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do"count" => 3,"total_count" => 3,"prop_names" => ["variant"],- "conversion_rate" => 42.9+ "conversion_rate" => 50}]endenddescribe "GET /api/stats/:domain/property/:key" do- setup [:create_user, :log_in, :create_site]+ setup [:create_user, :log_in, :create_new_site]++ test "returns property breakdown for goal", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, pathname: "/"),+ build(:pageview, pathname: "/"),+ build(:pageview, pathname: "/register"),+ build(:event, name: "Signup", "meta.key": ["variant"], "meta.value": ["A"]),+ build(:event, name: "Signup", "meta.key": ["variant"], "meta.value": ["B"]),+ build(:event, name: "Signup", "meta.key": ["variant"], "meta.value": ["B"])+ ])- test "returns metadata breakdown for goal", %{conn: conn, site: site} doinsert(:goal, %{domain: site.domain, event_name: "Signup"})filters = Jason.encode!(%{goal: "Signup"})prop_key = "variant"@@ -68,14 +107,64 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest doconn =get(conn,- "/api/stats/#{site.domain}/property/#{prop_key}?period=day&date=2019-01-01&filters=#{- filters- }"+ "/api/stats/#{site.domain}/property/#{prop_key}?period=day&filters=#{filters}"+ )++ assert json_response(conn, 200) == [+ %{"count" => 2, "name" => "B", "total_count" => 2, "conversion_rate" => 33.3},+ %{"count" => 1, "name" => "A", "total_count" => 1, "conversion_rate" => 16.7}+ ]+ end++ test "returns (none) values in property breakdown for goal", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event, name: "Signup"),+ build(:event, name: "Signup"),+ build(:event, name: "Signup", "meta.key": ["variant"], "meta.value": ["A"]),+ build(:event, name: "Signup", "meta.key": ["variant"], "meta.value": ["B"]),+ build(:event, name: "Signup", "meta.key": ["variant"], "meta.value": ["B"]),+ build(:event, name: "Signup", "meta.key": ["variant"], "meta.value": ["B"])+ ])++ insert(:goal, %{domain: site.domain, event_name: "Signup"})+ filters = Jason.encode!(%{goal: "Signup"})+ prop_key = "variant"++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/property/#{prop_key}?period=day&filters=#{filters}"+ )++ assert json_response(conn, 200) == [+ %{"count" => 3, "name" => "B", "total_count" => 3, "conversion_rate" => 50.0},+ %{"count" => 2, "name" => "(none)", "total_count" => 2, "conversion_rate" => 33.3},+ %{"count" => 1, "name" => "A", "total_count" => 1, "conversion_rate" => 16.7}+ ]+ end++ test "property breakdown with prop filter", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event, name: "Signup"),+ build(:event, name: "Signup"),+ build(:event, name: "Signup", "meta.key": ["variant"], "meta.value": ["A"]),+ build(:event, name: "Signup", "meta.key": ["variant"], "meta.value": ["B"]),+ build(:event, name: "Signup", "meta.key": ["variant"], "meta.value": ["B"]),+ build(:event, name: "Signup", "meta.key": ["variant"], "meta.value": ["B"])+ ])++ insert(:goal, %{domain: site.domain, event_name: "Signup"})+ filters = Jason.encode!(%{goal: "Signup", props: %{"variant" => "B"}})+ prop_key = "variant"++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/property/#{prop_key}?period=day&filters=#{filters}")assert json_response(conn, 200) == [- %{"count" => 2, "name" => "B", "total_count" => 2, "conversion_rate" => 28.6},- %{"count" => 1, "name" => "A", "total_count" => 1, "conversion_rate" => 14.3}+ %{"count" => 3, "name" => "B", "total_count" => 3, "conversion_rate" => 100.0}]endenddiff --git a/test/plausible_web/controllers/api/stats_controller/countries_test.exs b/test/plausible_web/controllers/api/stats_controller/countries_test.exsindex f64862c014..c7dc27cca4 100644--- a/test/plausible_web/controllers/api/stats_controller/countries_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/countries_test.exs@@ -3,29 +3,33 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest doimport Plausible.TestUtilsdescribe "GET /api/stats/:domain/countries" do- setup [:create_user, :log_in, :create_site]+ setup [:create_user, :log_in, :create_new_site]test "returns top countries by new visitors", %{conn: conn, site: site} do- conn = get(conn, "/api/stats/#{site.domain}/countries?period=day&date=2019-01-01")+ populate_stats(site, [+ build(:pageview,+ country_code: "EE"+ ),+ build(:pageview,+ country_code: "EE"+ ),+ build(:pageview,+ country_code: "GB"+ )+ ])++ conn = get(conn, "/api/stats/#{site.domain}/countries?period=day")assert json_response(conn, 200) == [%{"name" => "EST",- "full_country_name" => "Estonia","count" => 2,- "percentage" => 50+ "percentage" => 67},%{"name" => "GBR",- "full_country_name" => "United Kingdom",- "count" => 1,- "percentage" => 25- },- %{- "name" => "USA",- "full_country_name" => "United States","count" => 1,- "percentage" => 25+ "percentage" => 33}]enddiff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exsindex 83940b9151..68d44fd7c5 100644--- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs@@ -1,11 +1,16 @@defmodule PlausibleWeb.Api.StatsController.MainGraphTest douse PlausibleWeb.ConnCaseimport Plausible.TestUtils+ @user_id 123describe "GET /api/stats/main-graph - plot" do- setup [:create_user, :log_in, :create_site]+ setup [:create_user, :log_in, :create_new_site]test "displays pageviews for the last 30 minutes in realtime graph", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: relative_time(minutes: -5))+ ])+conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=realtime")assert %{"plot" => plot} = json_response(conn, 200)@@ -15,21 +20,30 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest doendtest "displays visitors for a day", %{conn: conn, site: site} do- conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=day&date=2019-01-01")+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-01 23:00:00])+ ])++ conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01")assert %{"plot" => plot} = json_response(conn, 200)zeroes = Stream.repeatedly(fn -> 0 end) |> Stream.take(22) |> Enum.into([])assert Enum.count(plot) == 24- assert plot == [4] ++ zeroes ++ [3]+ assert plot == [1] ++ zeroes ++ [1]endtest "displays hourly stats in configured timezone", %{conn: conn, user: user} do# UTC+1site = insert(:site, domain: "tz-test.com", members: [user], timezone: "CET")- conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=day&date=2019-01-01")+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01")assert %{"plot" => plot} = json_response(conn, 200)@@ -40,12 +54,17 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest doendtest "displays visitors for a month", %{conn: conn, site: site} do- conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=month&date=2019-01-01")+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-31 00:00:00])+ ])++ conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01")assert %{"plot" => plot} = json_response(conn, 200)assert Enum.count(plot) == 31- assert List.first(plot) == 7+ assert List.first(plot) == 1assert List.last(plot) == 1end@@ -77,136 +96,304 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest doenddescribe "GET /api/stats/main-graph - top stats" do- setup [:create_user, :log_in, :create_site]+ setup [:create_user, :log_in, :create_new_site]test "counts distinct user ids", %{conn: conn, site: site} do- conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=day&date=2019-01-01")+ populate_stats(site, [+ build(:pageview, user_id: @user_id),+ build(:pageview, user_id: @user_id),+ build(:pageview)+ ])++ conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=day")res = json_response(conn, 200)- assert %{"name" => "Unique visitors", "count" => 7, "change" => 100} in res["top_stats"]+ assert %{"name" => "Unique visitors", "value" => 2, "change" => 100} in res["top_stats"]endtest "counts total pageviews", %{conn: conn, site: site} do- conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=day&date=2019-01-01")+ populate_stats(site, [+ build(:pageview, user_id: @user_id),+ build(:pageview, user_id: @user_id),+ build(:pageview)+ ])++ conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=day")res = json_response(conn, 200)- assert %{"name" => "Total pageviews", "count" => 7, "change" => 100} in res["top_stats"]+ assert %{"name" => "Total pageviews", "value" => 3, "change" => 100} in res["top_stats"]endtest "calculates bounce rate", %{conn: conn, site: site} do- conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=day&date=2019-01-01")+ populate_stats(site, [+ build(:pageview, user_id: @user_id),+ build(:pageview, user_id: @user_id),+ build(:pageview)+ ])++ conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=day")res = json_response(conn, 200)- assert %{"name" => "Bounce rate", "percentage" => 33, "change" => nil} in res["top_stats"]+ assert %{"name" => "Bounce rate", "value" => 50, "change" => nil} in res["top_stats"]endtest "calculates average visit duration", %{conn: conn, site: site} do- conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=day&date=2019-01-01")+ populate_stats(site, [+ build(:pageview,+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:15:00]+ ),+ build(:pageview,+ timestamp: ~N[2021-01-01 00:15:00]+ )+ ])++ conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01")res = json_response(conn, 200)- assert %{"name" => "Visit duration", "duration" => 67, "change" => 100} in res["top_stats"]+ assert %{"name" => "Visit duration", "value" => 450, "change" => 100} in res["top_stats"]+ end++ test "calculates time on page instead when filtered for page", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ pathname: "/pageA",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/pageB",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:15:00]+ ),+ build(:pageview,+ pathname: "/pageA",+ timestamp: ~N[2021-01-01 00:15:00]+ )+ ])++ filters = Jason.encode!(%{page: "/pageA"})++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01&filters=#{filters}"+ )++ res = json_response(conn, 200)+ assert %{"name" => "Time on page", "value" => 900, "change" => 100} in res["top_stats"]+ end+ end++ describe "GET /api/stats/main-graph - realtime top stats" do+ setup [:create_user, :log_in, :create_new_site]++ test "shows current visitors (last 5 minutes)", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: relative_time(minutes: -10)),+ build(:pageview, timestamp: relative_time(minutes: -4)),+ build(:pageview, timestamp: relative_time(minutes: -1))+ ])++ conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=realtime")++ res = json_response(conn, 200)+ assert %{"name" => "Current visitors", "value" => 2} in res["top_stats"]+ end++ test "shows unique visitors (last 30 minutes)", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: relative_time(minutes: -45)),+ build(:pageview, timestamp: relative_time(minutes: -25)),+ build(:pageview, timestamp: relative_time(minutes: -1))+ ])++ conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=realtime")++ res = json_response(conn, 200)+ assert %{"name" => "Unique visitors (last 30 min)", "value" => 2} in res["top_stats"]+ end++ test "shows pageviews (last 30 minutes)", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, user_id: @user_id, timestamp: relative_time(minutes: -45)),+ build(:pageview, user_id: @user_id, timestamp: relative_time(minutes: -25)),+ build(:pageview, user_id: @user_id, timestamp: relative_time(minutes: -20)),+ build(:pageview, timestamp: relative_time(minutes: -1))+ ])++ conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=realtime")++ res = json_response(conn, 200)+ assert %{"name" => "Pageviews (last 30 min)", "value" => 3} in res["top_stats"]endenddescribe "GET /api/stats/main-graph - filtered for goal" do- setup [:create_user, :log_in, :create_site]+ setup [:create_user, :log_in, :create_new_site]test "returns total unique visitors", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, user_id: @user_id),+ build(:pageview, user_id: @user_id),+ build(:pageview),+ build(:event, name: "Signup")+ ])+filters = Jason.encode!(%{goal: "Signup"})conn =get(conn,- "/api/stats/#{site.domain}/main-graph?period=day&date=2019-01-01&filters=#{filters}"+ "/api/stats/#{site.domain}/main-graph?period=day&filters=#{filters}")res = json_response(conn, 200)- assert %{"name" => "Unique visitors", "count" => 7, "change" => 100} in res["top_stats"]+ assert %{"name" => "Unique visitors", "value" => 3, "change" => 100} in res["top_stats"]endtest "returns converted visitors", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, user_id: @user_id),+ build(:pageview, user_id: @user_id),+ build(:pageview),+ build(:event, name: "Signup")+ ])+filters = Jason.encode!(%{goal: "Signup"})conn =get(conn,- "/api/stats/#{site.domain}/main-graph?period=month&date=2019-01-01&filters=#{filters}"+ "/api/stats/#{site.domain}/main-graph?period=month&filters=#{filters}")res = json_response(conn, 200)- assert %{"name" => "Unique conversions", "count" => 3, "change" => 100} in res["top_stats"]+ assert %{"name" => "Unique conversions", "value" => 1, "change" => 100} in res["top_stats"]endtest "returns conversion rate", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, user_id: @user_id),+ build(:pageview, user_id: @user_id),+ build(:pageview),+ build(:event, name: "Signup")+ ])+filters = Jason.encode!(%{goal: "Signup"})conn =get(conn,- "/api/stats/#{site.domain}/main-graph?period=day&date=2019-01-01&filters=#{filters}"+ "/api/stats/#{site.domain}/main-graph?period=day&filters=#{filters}")res = json_response(conn, 200)- assert %{"name" => "Conversion rate", "percentage" => 42.9, "change" => 100} in res[- "top_stats"- ]+ assert %{"name" => "Conversion rate", "value" => 33.3, "change" => 100} in res["top_stats"]endenddescribe "GET /api/stats/main-graph - top stats - filters" do- setup [:create_user, :log_in, :create_site]+ setup [:create_user, :log_in, :create_new_site]test "returns only visitors from a country based on alpha3 code", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, country_code: "US"),+ build(:pageview, country_code: "US"),+ build(:pageview, country_code: "EE")+ ])+filters = Jason.encode!(%{country: "USA"})conn =get(conn,- "/api/stats/#{site.domain}/main-graph?period=month&date=2019-01-01&filters=#{filters}"+ "/api/stats/#{site.domain}/main-graph?period=month&filters=#{filters}")res = json_response(conn, 200)- assert %{"name" => "Unique visitors", "count" => 3, "change" => 100} in res["top_stats"]+ assert %{"name" => "Unique visitors", "value" => 2, "change" => 100} in res["top_stats"]+ end++ test "page glob filter", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, pathname: "/index"),+ build(:pageview, pathname: "/blog/post1"),+ build(:pageview, pathname: "/blog/post2")+ ])++ filters = Jason.encode!(%{page: "/blog/**"})++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=month&filters=#{filters}"+ )++ res = json_response(conn, 200)+ assert %{"name" => "Unique visitors", "value" => 2, "change" => 100} in res["top_stats"]endtest "returns only visitors with specific screen size", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, screen_size: "Desktop"),+ build(:pageview, screen_size: "Desktop"),+ build(:pageview, screen_size: "Mobile")+ ])+filters = Jason.encode!(%{screen: "Desktop"})conn =get(conn,- "/api/stats/#{site.domain}/main-graph?period=month&date=2019-01-01&filters=#{filters}"+ "/api/stats/#{site.domain}/main-graph?period=month&filters=#{filters}")res = json_response(conn, 200)- assert %{"name" => "Unique visitors", "count" => 3, "change" => 100} in res["top_stats"]+ assert %{"name" => "Unique visitors", "value" => 2, "change" => 100} in res["top_stats"]endtest "returns only visitors with specific browser", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, browser: "Chrome"),+ build(:pageview, browser: "Chrome"),+ build(:pageview, browser: "Safari")+ ])+filters = Jason.encode!(%{browser: "Chrome"})conn =get(conn,- "/api/stats/#{site.domain}/main-graph?period=month&date=2019-01-01&filters=#{filters}"+ "/api/stats/#{site.domain}/main-graph?period=month&filters=#{filters}")res = json_response(conn, 200)- assert %{"name" => "Unique visitors", "count" => 3, "change" => 100} in res["top_stats"]+ assert %{"name" => "Unique visitors", "value" => 2, "change" => 100} in res["top_stats"]endtest "returns only visitors with specific operating system", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, operating_system: "Mac"),+ build(:pageview, operating_system: "Mac"),+ build(:pageview, operating_system: "Windows")+ ])+filters = Jason.encode!(%{os: "Mac"})conn =get(conn,- "/api/stats/#{site.domain}/main-graph?period=month&date=2019-01-01&filters=#{filters}"+ "/api/stats/#{site.domain}/main-graph?period=month&filters=#{filters}")res = json_response(conn, 200)- assert %{"name" => "Unique visitors", "count" => 3, "change" => 100} in res["top_stats"]+ assert %{"name" => "Unique visitors", "value" => 2, "change" => 100} in res["top_stats"]endendenddiff --git a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exsindex b5e691d6da..1eb612bb9f 100644--- a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs@@ -3,34 +3,46 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest doimport Plausible.TestUtilsdescribe "GET /api/stats/:domain/operating_systems" do- setup [:create_user, :log_in, :create_site]+ setup [:create_user, :log_in, :create_new_site]- test "returns operating systems by new visitors", %{conn: conn, site: site} do- conn = get(conn, "/api/stats/#{site.domain}/operating-systems?period=day&date=2019-01-01")+ test "returns operating systems by unique visitors", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, operating_system: "Mac"),+ build(:pageview, operating_system: "Mac"),+ build(:pageview, operating_system: "Android")+ ])++ conn = get(conn, "/api/stats/#{site.domain}/operating-systems?period=day")assert json_response(conn, 200) == [- %{"name" => "Mac", "count" => 3, "percentage" => 75},- %{"name" => "Android", "count" => 1, "percentage" => 25}+ %{"name" => "Mac", "count" => 2, "percentage" => 67},+ %{"name" => "Android", "count" => 1, "percentage" => 33}]endenddescribe "GET /api/stats/:domain/operating-system-versions" do- setup [:create_user, :log_in, :create_site]+ setup [:create_user, :log_in, :create_new_site]test "returns top OS versions by unique visitors", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, operating_system: "Mac", operating_system_version: "10.15"),+ build(:pageview, operating_system: "Mac", operating_system_version: "10.16"),+ build(:pageview, operating_system: "Mac", operating_system_version: "10.16"),+ build(:pageview, operating_system: "Android", operating_system_version: "4")+ ])+filters = Jason.encode!(%{os: "Mac"})conn =get(conn,- "/api/stats/#{site.domain}/operating-system-versions?period=day&date=2019-01-01&filters=#{- filters- }"+ "/api/stats/#{site.domain}/operating-system-versions?period=day&filters=#{filters}")assert json_response(conn, 200) == [- %{"name" => "10.15", "count" => 1, "percentage" => 100}+ %{"name" => "10.16", "count" => 2, "percentage" => 67},+ %{"name" => "10.15", "count" => 1, "percentage" => 33}]endenddiff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exsindex 4c14d16e4f..a5bbeeeabe 100644--- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs@@ -1,112 +1,170 @@defmodule PlausibleWeb.Api.StatsController.PagesTest douse PlausibleWeb.ConnCaseimport Plausible.TestUtils+ @user_id 123describe "GET /api/stats/:domain/pages" do- setup [:create_user, :log_in, :create_site]+ setup [:create_user, :log_in, :create_new_site]test "returns top pages by visitors", %{conn: conn, site: site} do- conn = get(conn, "/api/stats/#{site.domain}/pages?period=day&date=2019-01-01")+ populate_stats(site, [+ build(:pageview, pathname: "/"),+ build(:pageview, pathname: "/"),+ build(:pageview, pathname: "/"),+ build(:pageview, pathname: "/register"),+ build(:pageview, pathname: "/register"),+ build(:pageview, pathname: "/contact")+ ])++ conn = get(conn, "/api/stats/#{site.domain}/pages?period=day")assert json_response(conn, 200) == [- %{"count" => 3, "pageviews" => 3, "name" => "/"},- %{"count" => 2, "pageviews" => 2, "name" => "/register"},- %{"count" => 1, "pageviews" => 1, "name" => "/contact"},- %{"count" => 1, "pageviews" => 1, "name" => "/irrelevant"}+ %{"count" => 3, "name" => "/"},+ %{"count" => 2, "name" => "/register"},+ %{"count" => 1, "name" => "/contact"}]endtest "calculates bounce rate and time on page for pages", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ pathname: "/",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/some-other-page",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:15:00]+ ),+ build(:pageview,+ pathname: "/",+ timestamp: ~N[2021-01-01 00:15:00]+ )+ ])+conn =get(conn,- "/api/stats/#{site.domain}/pages?period=day&date=2019-01-01&detailed=true"+ "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true")assert json_response(conn, 200) == [%{- "time_on_page" => 82800,- "bounce_rate" => 33.0,- "count" => 3,- "pageviews" => 3,- "name" => "/"- },- %{- "time_on_page" => 1,- "bounce_rate" => nil,+ "bounce_rate" => 50.0,+ "time_on_page" => 900.0,"count" => 2,"pageviews" => 2,- "name" => "/register"+ "name" => "/"},%{- "time_on_page" => nil,"bounce_rate" => nil,- "count" => 1,- "pageviews" => 1,- "name" => "/contact"- },- %{"time_on_page" => nil,- "bounce_rate" => nil,"count" => 1,"pageviews" => 1,- "name" => "/irrelevant"+ "name" => "/some-other-page"}]endtest "returns top pages in realtime report", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, pathname: "/page1"),+ build(:pageview, pathname: "/page2"),+ build(:pageview, pathname: "/page1")+ ])+conn = get(conn, "/api/stats/#{site.domain}/pages?period=realtime")assert json_response(conn, 200) == [- %{"count" => 2, "name" => "/exit"},- %{"count" => 1, "name" => "/"}+ %{"count" => 2, "name" => "/page1"},+ %{"count" => 1, "name" => "/page2"}]endenddescribe "GET /api/stats/:domain/entry-pages" do- setup [:create_user, :log_in, :create_site]+ setup [:create_user, :log_in, :create_new_site]test "returns top entry pages by visitors", %{conn: conn, site: site} do- conn = get(conn, "/api/stats/#{site.domain}/entry-pages?period=day&date=2019-01-01")-- assert json_response(conn, 200) == [- %{- "count" => 3,- "entries" => 3,- "name" => "/",- "visit_duration" => 67- }- ]- end+ populate_stats(site, [+ build(:pageview,+ pathname: "/page1",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/page1",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/page2",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/page2",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:15:00]+ )+ ])- test "calculates visit duration for entry pages", %{conn: conn, site: site} do- conn =- get(- conn,- "/api/stats/#{site.domain}/entry-pages?period=day&date=2019-01-01"+ populate_stats(site, [+ build(:pageview,+ pathname: "/page2",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 23:15:00])+ ])++ conn = get(conn, "/api/stats/#{site.domain}/entry-pages?period=day&date=2021-01-01")assert json_response(conn, 200) == [%{- "visit_duration" => 67,- "count" => 3,- "entries" => 3,- "name" => "/"+ "count" => 2,+ "entries" => 2,+ "name" => "/page1",+ "visit_duration" => 0+ },+ %{+ "count" => 1,+ "entries" => 2,+ "name" => "/page2",+ "visit_duration" => 450}]endenddescribe "GET /api/stats/:domain/exit-pages" do- setup [:create_user, :log_in, :create_site]+ setup [:create_user, :log_in, :create_new_site]test "returns top exit pages by visitors", %{conn: conn, site: site} do- conn = get(conn, "/api/stats/#{site.domain}/exit-pages?period=day&date=2019-01-01")+ populate_stats(site, [+ build(:pageview,+ pathname: "/page1",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/page1",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/page1",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/page2",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:15:00]+ )+ ])++ conn = get(conn, "/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01")assert json_response(conn, 200) == [- %{"count" => 3, "exits" => 3, "name" => "/", "exit_rate" => 100.0}+ %{"name" => "/page1", "count" => 2, "exits" => 2, "exit_rate" => 67},+ %{"name" => "/page2", "count" => 1, "exits" => 1, "exit_rate" => 100}]endenddiff --git a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exsindex fc2fc574cf..0510dfab85 100644--- a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs@@ -3,15 +3,20 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest doimport Plausible.TestUtilsdescribe "GET /api/stats/:domain/browsers" do- setup [:create_user, :log_in, :create_site]+ setup [:create_user, :log_in, :create_new_site]test "returns screen sizes by new visitors", %{conn: conn, site: site} do- conn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day&date=2019-01-01")+ populate_stats(site, [+ build(:pageview, screen_size: "Desktop"),+ build(:pageview, screen_size: "Desktop"),+ build(:pageview, screen_size: "Laptop")+ ])++ conn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day")assert json_response(conn, 200) == [- %{"name" => "Desktop", "count" => 2, "percentage" => 50},- %{"name" => "Laptop", "count" => 1, "percentage" => 25},- %{"name" => "Mobile", "count" => 1, "percentage" => 25}+ %{"name" => "Desktop", "count" => 2, "percentage" => 67},+ %{"name" => "Laptop", "count" => 1, "percentage" => 33}]endenddiff --git a/test/plausible_web/controllers/api/stats_controller/sources_test.exs b/test/plausible_web/controllers/api/stats_controller/sources_test.exsindex 9759ce91cb..0cb9da473c 100644--- a/test/plausible_web/controllers/api/stats_controller/sources_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/sources_test.exs@@ -1,160 +1,407 @@defmodule PlausibleWeb.Api.StatsController.SourcesTest douse PlausibleWeb.ConnCaseimport Plausible.TestUtils+ @user_id 123describe "GET /api/stats/:domain/sources" do- setup [:create_user, :log_in, :create_site]+ setup [:create_user, :log_in, :create_new_site]test "returns top sources by unique user ids", %{conn: conn, site: site} do- conn = get(conn, "/api/stats/#{site.domain}/sources?period=day&date=2019-01-01")+ populate_stats(site, [+ build(:pageview,+ referrer_source: "Google",+ referrer: "google.com"+ ),+ build(:pageview,+ referrer_source: "Google",+ referrer: "google.com"+ ),+ build(:pageview,+ referrer_source: "DuckDuckGo",+ referrer: "duckduckgo.com"+ )+ ])++ conn = get(conn, "/api/stats/#{site.domain}/sources")assert json_response(conn, 200) == [- %{"name" => "10words", "count" => 2, "url" => "10words.com"},- %{"name" => "Bing", "count" => 1, "url" => ""}+ %{"name" => "Google", "count" => 2},+ %{"name" => "DuckDuckGo", "count" => 1}]endtest "calculates bounce rate and visit duration for sources", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ referrer_source: "Google",+ referrer: "google.com",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ referrer_source: "Google",+ referrer: "google.com",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:15:00]+ ),+ build(:pageview,+ referrer_source: "DuckDuckGo",+ referrer: "duckduckgo.com",+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])+conn =get(conn,- "/api/stats/#{site.domain}/sources?period=day&date=2019-01-01&detailed=true"+ "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&detailed=true")assert json_response(conn, 200) == [%{- "name" => "10words",- "count" => 2,- "bounce_rate" => 50.0,- "visit_duration" => 50,- "url" => "10words.com"+ "name" => "Google",+ "count" => 1,+ "bounce_rate" => 0,+ "visit_duration" => 900},%{- "name" => "Bing",+ "name" => "DuckDuckGo","count" => 1,- "bounce_rate" => 0,- "visit_duration" => 100,- "url" => ""+ "bounce_rate" => 100,+ "visit_duration" => 0}]endtest "returns top sources in realtime report", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ referrer_source: "Google",+ referrer: "google.com",+ timestamp: relative_time(minutes: -3)+ ),+ build(:pageview,+ referrer_source: "Google",+ referrer: "google.com",+ timestamp: relative_time(minutes: -2)+ ),+ build(:pageview,+ referrer_source: "DuckDuckGo",+ referrer: "duckduckgo.com",+ timestamp: relative_time(minutes: -1)+ )+ ])+conn = get(conn, "/api/stats/#{site.domain}/sources?period=realtime")assert json_response(conn, 200) == [- %{"name" => "10words", "count" => 2, "url" => "10words.com"},- %{"name" => "Bing", "count" => 1, "url" => "bing.com"}+ %{"name" => "Google", "count" => 2},+ %{"name" => "DuckDuckGo", "count" => 1}]endtest "can paginate the results", %{conn: conn, site: site} do- conn =- get(conn, "/api/stats/#{site.domain}/sources?period=day&date=2019-01-01&limit=1&page=2")+ populate_stats(site, [+ build(:pageview,+ referrer_source: "Google",+ referrer: "google.com"+ ),+ build(:pageview,+ referrer_source: "Google",+ referrer: "google.com"+ ),+ build(:pageview,+ referrer_source: "DuckDuckGo",+ referrer: "duckduckgo.com"+ )+ ])++ conn = get(conn, "/api/stats/#{site.domain}/sources?limit=1&page=2")assert json_response(conn, 200) == [- %{"name" => "Bing", "count" => 1, "url" => ""}+ %{"name" => "DuckDuckGo", "count" => 1}]endenddescribe "GET /api/stats/:domain/utm_mediums" do- setup [:create_user, :log_in, :create_site]+ setup [:create_user, :log_in, :create_new_site]test "returns top utm_mediums by unique user ids", %{conn: conn, site: site} do- conn = get(conn, "/api/stats/#{site.domain}/utm_mediums?period=day&date=2019-01-01")+ populate_stats(site, [+ build(:pageview,+ utm_medium: "social",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ utm_medium: "social",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:15:00]+ ),+ build(:pageview,+ utm_medium: "email",+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/utm_mediums?period=day&date=2021-01-01"+ )++ assert json_response(conn, 200) == [+ %{+ "name" => "social",+ "count" => 1,+ "bounce_rate" => 0,+ "visit_duration" => 900+ },+ %{+ "name" => "email",+ "count" => 1,+ "bounce_rate" => 100,+ "visit_duration" => 0+ }+ ]+ end+ end++ describe "GET /api/stats/:domain/utm_campaigns" do+ setup [:create_user, :log_in, :create_new_site]++ test "returns top utm_campaigns by unique user ids", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ utm_campaign: "profile",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ utm_campaign: "profile",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:15:00]+ ),+ build(:pageview,+ utm_campaign: "august",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ utm_campaign: "august",+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/utm_campaigns?period=day&date=2021-01-01"+ )assert json_response(conn, 200) == [%{- "name" => "listing",+ "name" => "august","count" => 2,- "bounce_rate" => 50.0,- "visit_duration" => 50.0+ "bounce_rate" => 100,+ "visit_duration" => 0},%{- "name" => "search",+ "name" => "profile","count" => 1,- "bounce_rate" => 0.0,- "visit_duration" => 100.0+ "bounce_rate" => 0,+ "visit_duration" => 900}]endend- describe "GET /api/stats/:domain/goal/referrers" do- setup [:create_user, :log_in, :create_site]+ describe "GET /api/stats/:domain/utm_sources" do+ setup [:create_user, :log_in, :create_new_site]++ test "returns top utm_sources by unique user ids", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ utm_source: "Twitter",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ utm_source: "Twitter",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:15:00]+ ),+ build(:pageview,+ utm_source: "newsletter",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ utm_source: "newsletter",+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/utm_sources?period=day&date=2021-01-01"+ )++ assert json_response(conn, 200) == [+ %{+ "name" => "newsletter",+ "count" => 2,+ "bounce_rate" => 100,+ "visit_duration" => 0+ },+ %{+ "name" => "Twitter",+ "count" => 1,+ "bounce_rate" => 0,+ "visit_duration" => 900+ }+ ]+ end+ end++ describe "GET /api/stats/:domain/sources - with goal filter" do+ setup [:create_user, :log_in, :create_new_site]test "returns top referrers for a custom goal", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ referrer_source: "Twitter",+ user_id: @user_id+ ),+ build(:event,+ name: "Signup",+ user_id: @user_id+ ),+ build(:pageview,+ referrer_source: "Twitter"+ )+ ])+filters = Jason.encode!(%{goal: "Signup"})conn =get(conn,- "/api/stats/#{site.domain}/sources?period=day&date=2019-01-01&filters=#{filters}"+ "/api/stats/#{site.domain}/sources?period=day&filters=#{filters}")assert json_response(conn, 200) == [- %{"name" => "10words", "count" => 2, "url" => "10words.com"}+ %{"name" => "Twitter", "count" => 1}]endtest "returns top referrers for a pageview goal", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ referrer_source: "Twitter",+ user_id: @user_id+ ),+ build(:pageview,+ pathname: "/register",+ user_id: @user_id+ ),+ build(:pageview,+ referrer_source: "Twitter"+ )+ ])+filters = Jason.encode!(%{goal: "Visit /register"})conn =get(conn,- "/api/stats/#{site.domain}/sources?period=day&date=2019-01-01&filters=#{filters}"+ "/api/stats/#{site.domain}/sources?period=day&filters=#{filters}")assert json_response(conn, 200) == [- %{"name" => "10words", "count" => 2, "url" => "10words.com"}+ %{"name" => "Twitter", "count" => 1}]endenddescribe "GET /api/stats/:domain/referrer-drilldown" do- setup [:create_user, :log_in, :create_site]+ setup [:create_user, :log_in, :create_new_site]test "returns top referrers for a particular source", %{conn: conn, site: site} do- filters = Jason.encode!(%{source: "10words"})+ populate_stats(site, [+ build(:pageview,+ referrer_source: "10words",+ referrer: "10words.com"+ ),+ build(:pageview,+ referrer_source: "10words",+ referrer: "10words.com"+ ),+ build(:pageview,+ referrer_source: "10words",+ referrer: "10words.com/page1"+ ),+ build(:pageview,+ referrer_source: "ignored",+ referrer: "ignored"+ )+ ])conn =get(conn,- "/api/stats/#{site.domain}/referrers/10words?period=day&date=2019-01-01&filters=#{- filters- }"+ "/api/stats/#{site.domain}/referrers/10words?period=day")assert json_response(conn, 200) == %{- "total_visitors" => 4,+ "total_visitors" => 3,"referrers" => [- %{"name" => "10words.com/page1", "url" => "10words.com", "count" => 2}+ %{"name" => "10words.com", "count" => 2},+ %{"name" => "10words.com/page1", "count" => 1}]}endtest "calculates bounce rate and visit duration for referrer urls", %{conn: conn, site: site} do- filters = Jason.encode!(%{source: "10words"})+ populate_stats(site, [+ build(:pageview,+ referrer_source: "10words",+ referrer: "10words.com",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ referrer_source: "10words",+ referrer: "10words.com",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:15:00]+ ),+ build(:pageview,+ referrer_source: "10words",+ referrer: "10words.com",+ timestamp: ~N[2021-01-01 00:15:00]+ ),+ build(:pageview,+ referrer_source: "ignored",+ referrer: "ignored",+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])conn =get(conn,- "/api/stats/#{site.domain}/referrers/10words?period=day&date=2019-01-01&filters=#{- filters- }&detailed=true"+ "/api/stats/#{site.domain}/referrers/10words?period=day&date=2021-01-01&detailed=true")assert json_response(conn, 200) == %{- "total_visitors" => 4,+ "total_visitors" => 2,"referrers" => [%{- "name" => "10words.com/page1",- "url" => "10words.com",+ "name" => "10words.com","count" => 2,"bounce_rate" => 50.0,- "visit_duration" => 50.0+ "visit_duration" => 450}]}@@ -162,7 +409,19 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest dotest "gets keywords from Google", %{conn: conn, user: user, site: site} doinsert(:google_auth, user: user, user: user, site: site, property: "sc-domain:example.com")- conn = get(conn, "/api/stats/#{site.domain}/referrers/Google?period=day&date=2019-02-01")++ populate_stats(site, [+ build(:pageview,+ referrer_source: "Google",+ referrer: "google.com"+ ),+ build(:pageview,+ referrer_source: "Google",+ referrer: "google.com"+ )+ ])++ conn = get(conn, "/api/stats/#{site.domain}/referrers/Google?period=day"){:ok, terms} = Plausible.Google.Api.Mock.fetch_stats(nil, nil, nil)assert json_response(conn, 200) == %{@@ -171,62 +430,74 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do}end- test "enriches twitter referrers with tweets if available", %{conn: conn, site: site} do- insert(:tweet, link: "t.co/some-link", text: "important tweet")-- conn = get(conn, "/api/stats/#{site.domain}/referrers/Twitter?period=day&date=2019-03-01")-- res = json_response(conn, 200)- assert res["total_visitors"] == 3- assert [tweet1, tweet2] = res["referrers"]-- assert %{- "name" => "t.co/some-link",- "count" => 2,- "tweets" => [%{"text" => "important tweet"}]- } = tweet1-- assert %{"name" => "t.co/nonexistent-link", "count" => 1, "tweets" => nil} = tweet2- end- end-- describe "GET /api/stats/:domain/goal/referrers/:referrer" do- setup [:create_user, :log_in, :create_site]-test "returns top referring urls for a custom goal", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ referrer_source: "10words",+ referrer: "10words.com"+ ),+ build(:pageview,+ referrer_source: "10words",+ referrer: "10words.com",+ user_id: @user_id+ ),+ build(:event,+ name: "Signup",+ user_id: @user_id+ ),+ build(:event,+ name: "Signup"+ )+ ])+filters = Jason.encode!(%{goal: "Signup"})conn =get(conn,- "/api/stats/#{site.domain}/referrers/10words?period=day&date=2019-01-01&filters=#{- filters- }"+ "/api/stats/#{site.domain}/referrers/10words?period=day&filters=#{filters}")assert json_response(conn, 200) == %{- "total_visitors" => 3,+ "total_visitors" => 1,"referrers" => [- %{"name" => "10words.com/page1", "url" => "10words.com", "count" => 2}+ %{"name" => "10words.com", "count" => 1}]}endtest "returns top referring urls for a pageview goal", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ referrer_source: "10words",+ referrer: "10words.com"+ ),+ build(:pageview,+ referrer_source: "10words",+ referrer: "10words.com",+ user_id: @user_id+ ),+ build(:pageview,+ pathname: "/register",+ user_id: @user_id+ ),+ build(:pageview,+ pathname: "/register"+ )+ ])+filters = Jason.encode!(%{goal: "Visit /register"})conn =get(conn,- "/api/stats/#{site.domain}/referrers/10words?period=day&date=2019-01-01&filters=#{- filters- }"+ "/api/stats/#{site.domain}/referrers/10words?period=day&filters=#{filters}")assert json_response(conn, 200) == %{- "total_visitors" => 2,+ "total_visitors" => 1,"referrers" => [- %{"name" => "10words.com/page1", "url" => "10words.com", "count" => 2}+ %{"name" => "10words.com", "count" => 1}]}enddiff --git a/test/support/factory.ex b/test/support/factory.exindex 05fa273ab2..3e6e11dae2 100644--- a/test/support/factory.ex+++ b/test/support/factory.ex@@ -80,7 +80,7 @@ defmodule Plausible.Factory dohostname: hostname,domain: hostname,pathname: "/",- timestamp: Timex.now(),+ timestamp: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second),user_id: SipHash.hash!(hash_key(), UUID.uuid4()),session_id: SipHash.hash!(hash_key(), UUID.uuid4()),referrer: "",diff --git a/test/support/test_utils.ex b/test/support/test_utils.exindex f306c7e008..616846e7dd 100644--- a/test/support/test_utils.ex+++ b/test/support/test_utils.ex@@ -79,6 +79,13 @@ defmodule Plausible.TestUtils do|> Plug.Conn.fetch_session()end+ def populate_stats(site, events) do+ Enum.map(events, fn event ->+ Map.put(event, :domain, site.domain)+ end)+ |> populate_stats+ end+def populate_stats(events) dosessions =Enum.reduce(events, %{}, fn event, sessions ->@@ -101,6 +108,12 @@ defmodule Plausible.TestUtils do)end+ def relative_time(shifts) do+ NaiveDateTime.utc_now()+ |> Timex.shift(shifts)+ |> NaiveDateTime.truncate(:second)+ end+defp schema_to_map(schema) doMap.from_struct(schema)|> Map.delete(:__meta__)