Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
APIv2: macros, SQL cleanup (#4286)
Karl-Aksel Puulmann05ac840078cf
diff --git a/extra/lib/plausible/stats/funnel.ex b/extra/lib/plausible/stats/funnel.exindex d9c497cee6..121a699fae 100644--- a/extra/lib/plausible/stats/funnel.ex+++ b/extra/lib/plausible/stats/funnel.ex@@ -10,7 +10,7 @@ defmodule Plausible.Stats.Funnel doalias Plausible.Funnelsimport Ecto.Query- import Plausible.Stats.Fragments+ import Plausible.Stats.SQL.Fragmentsalias Plausible.ClickhouseRepoalias Plausible.Stats.Basediff --git a/extra/lib/plausible/stats/goal/revenue.ex b/extra/lib/plausible/stats/goal/revenue.exindex 7443a1758d..1ba564e3ff 100644--- a/extra/lib/plausible/stats/goal/revenue.ex+++ b/extra/lib/plausible/stats/goal/revenue.ex@@ -12,26 +12,6 @@ defmodule Plausible.Stats.Goal.Revenue do@revenue_metricsend- def total_revenue_query() do- dynamic(- [e],- selected_as(- fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount),- :total_revenue- )- )- end-- def average_revenue_query() do- dynamic(- [e],- selected_as(- fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount),- :average_revenue- )- )- end-@spec get_revenue_tracking_currency(Plausible.Site.t(), Plausible.Stats.Query.t(), [atom()]) ::{atom() | nil, [atom()]}@doc """diff --git a/lib/plausible/exports.ex b/lib/plausible/exports.exindex d5cb20ada2..04e6a42021 100644--- a/lib/plausible/exports.ex+++ b/lib/plausible/exports.ex@@ -4,7 +4,7 @@ defmodule Plausible.Exports do"""use Plausible- use Plausible.Stats.Fragments+ use Plausible.Stats.SQL.Fragmentsimport Ecto.Query@doc "Schedules CSV export job to S3 storage"diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.exindex 9a2e65bab6..8c5cdc4f7b 100644--- a/lib/plausible/stats/aggregate.ex+++ b/lib/plausible/stats/aggregate.ex@@ -3,7 +3,7 @@ defmodule Plausible.Stats.Aggregate douse Plausibleimport Plausible.Stats.Baseimport Ecto.Query- alias Plausible.Stats.{Query, Util}+ alias Plausible.Stats.{Query, Util, SQL}def aggregate(site, query, metrics) do{currency, metrics} =@@ -64,8 +64,7 @@ defmodule Plausible.Stats.Aggregate dotimed_page_transitions_q =from e in Ecto.Query.subquery(windowed_pages_q),group_by: [e.pathname, e.next_pathname, e.session_id],- where:- ^Plausible.Stats.Filters.WhereBuilder.build_condition(:pathname, event_page_filter),+ where: ^SQL.WhereBuilder.build_condition(:pathname, event_page_filter),where: e.next_timestamp != 0,select: %{pathname: e.pathname,diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.exindex 5354833a1f..8335308cce 100644--- a/lib/plausible/stats/base.ex+++ b/lib/plausible/stats/base.ex@@ -1,14 +1,12 @@defmodule Plausible.Stats.Base douse Plausible.ClickhouseRepouse Plausible- use Plausible.Stats.Fragments+ use Plausible.Stats.SQL.Fragments- alias Plausible.Stats.{Query, Filters, TableDecider}+ alias Plausible.Stats.{Query, TableDecider, SQL}alias Plausible.Timezonesimport Ecto.Query- @uniq_users_expression "toUInt64(round(uniq(?) * any(_sample_factor)))"-def base_event_query(site, query) doevents_q = query_events(site, query)@@ -32,7 +30,7 @@ defmodule Plausible.Stats.Base doenddef query_events(site, query) do- q = from(e in "events_v2", where: ^Filters.WhereBuilder.build(:events, site, query))+ q = from(e in "events_v2", where: ^SQL.WhereBuilder.build(:events, site, query))on_ee doq = Plausible.Stats.Sampling.add_query_hint(q, query)@@ -42,7 +40,7 @@ defmodule Plausible.Stats.Base doenddef query_sessions(site, query) do- q = from(s in "sessions_v2", where: ^Filters.WhereBuilder.build(:sessions, site, query))+ q = from(s in "sessions_v2", where: ^SQL.WhereBuilder.build(:sessions, site, query))on_ee doq = Plausible.Stats.Sampling.add_query_hint(q, query)@@ -53,206 +51,16 @@ defmodule Plausible.Stats.Base dodef select_event_metrics(metrics) dometrics- |> Enum.map(&select_event_metric/1)+ |> Enum.map(&SQL.Expression.event_metric/1)|> Enum.reduce(%{}, &Map.merge/2)end- defp select_event_metric(:pageviews) do- %{- pageviews:- dynamic(- [e],- selected_as(- fragment("toUInt64(round(countIf(? = 'pageview') * any(_sample_factor)))", e.name),- :pageviews- )- )- }- end-- defp select_event_metric(:events) do- %{- events:- dynamic(- [],- selected_as(fragment("toUInt64(round(count(*) * any(_sample_factor)))"), :events)- )- }- end-- defp select_event_metric(:visitors) do- %{- visitors: dynamic([e], selected_as(fragment(@uniq_users_expression, e.user_id), :visitors))- }- end-- defp select_event_metric(:visits) do- %{- visits:- dynamic(- [e],- selected_as(- fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.session_id),- :visits- )- )- }- end-- on_ee do- defp select_event_metric(:total_revenue) do- %{total_revenue: Plausible.Stats.Goal.Revenue.total_revenue_query()}- end-- defp select_event_metric(:average_revenue) do- %{average_revenue: Plausible.Stats.Goal.Revenue.average_revenue_query()}- end- end-- defp select_event_metric(:sample_percent) do- %{- sample_percent:- dynamic(- [],- fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)")- )- }- end-- defp select_event_metric(:percentage), do: %{}- defp select_event_metric(:conversion_rate), do: %{}- defp select_event_metric(:group_conversion_rate), do: %{}- defp select_event_metric(:total_visitors), do: %{}-- defp select_event_metric(unknown), do: raise("Unknown metric: #{unknown}")-def select_session_metrics(metrics, query) dometrics- |> Enum.map(&select_session_metric(&1, query))+ |> Enum.map(&SQL.Expression.session_metric(&1, query))|> Enum.reduce(%{}, &Map.merge/2)end- defp select_session_metric(:bounce_rate, query) do- # :TRICKY: If page is passed to query, we only count bounce rate where users _entered_ at page.- event_page_filter = Query.get_filter(query, "event:page")- condition = Filters.WhereBuilder.build_condition(:entry_page, event_page_filter)-- %{- bounce_rate:- dynamic(- [],- selected_as(- fragment(- "toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))",- ^condition,- ^condition- ),- :bounce_rate- )- ),- __internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))- }- end-- defp select_session_metric(:visits, _query) do- %{- visits:- dynamic(- [s],- selected_as(- fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign),- :visits- )- )- }- end-- defp select_session_metric(:pageviews, _query) do- %{- pageviews:- dynamic(- [s],- selected_as(- fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews),- :pageviews- )- )- }- end-- defp select_session_metric(:events, _query) do- %{- events:- dynamic(- [s],- selected_as(- fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events),- :events- )- )- }- end-- defp select_session_metric(:visitors, _query) do- %{- visitors:- dynamic(- [s],- selected_as(- fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", s.user_id),- :visitors- )- )- }- end-- defp select_session_metric(:visit_duration, _query) do- %{- visit_duration:- dynamic(- [],- selected_as(- fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))"),- :visit_duration- )- ),- __internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))- }- end-- defp select_session_metric(:views_per_visit, _query) do- %{- views_per_visit:- dynamic(- [s],- selected_as(- fragment(- "ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)",- s.sign,- s.pageviews,- s.sign- ),- :views_per_visit- )- ),- __internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))- }- end-- defp select_session_metric(:sample_percent, _query) do- %{- sample_percent:- dynamic(- [],- fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)")- )- }- end-- defp select_session_metric(:percentage, _query), do: %{}- defp select_session_metric(:conversion_rate, _query), do: %{}- defp select_session_metric(:group_conversion_rate, _query), do: %{}-def filter_converted_sessions(db_query, site, query) doif Query.has_event_filters?(query) doconverted_sessions =@@ -334,7 +142,9 @@ defmodule Plausible.Stats.Base dodefp total_visitors(site, query) dobase_event_query(site, query)- |> select([e], total_visitors: fragment(@uniq_users_expression, e.user_id))+ |> select([e],+ total_visitors: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.user_id)+ )end# `total_visitors_subquery` returns a subquery which selects `total_visitors` -@@ -350,18 +160,17 @@ defmodule Plausible.Stats.Base dodef total_visitors_subquery(site, query, include_imported)def total_visitors_subquery(site, query, true = _include_imported) do- dynamic(- [e],- selected_as(+ wrap_alias([], %{+ total_visitors:subquery(total_visitors(site, query)) +- subquery(Plausible.Stats.Imported.total_imported_visitors(site, query)),- :__total_visitors- )- )+ subquery(Plausible.Stats.Imported.total_imported_visitors(site, query))+ })enddef total_visitors_subquery(site, query, false = _include_imported) do- dynamic([e], selected_as(subquery(total_visitors(site, query)), :__total_visitors))+ wrap_alias([], %{+ total_visitors: subquery(total_visitors(site, query))+ })enddef add_percentage_metric(q, site, query, metrics) do@@ -369,19 +178,14 @@ defmodule Plausible.Stats.Base dototal_query = Query.set_dimensions(query, [])q- |> select_merge(- ^%{__total_visitors: total_visitors_subquery(site, total_query, query.include_imported)}- )- |> select_merge(%{+ |> select_merge_as([], total_visitors_subquery(site, total_query, query.include_imported))+ |> select_merge_as([], %{percentage:- selected_as(- fragment(- "if(? > 0, round(? / ? * 100, 1), null)",- selected_as(:__total_visitors),- selected_as(:visitors),- selected_as(:__total_visitors)- ),- :percentage+ fragment(+ "if(? > 0, round(? / ? * 100, 1), null)",+ selected_as(:total_visitors),+ selected_as(:visitors),+ selected_as(:total_visitors))})else@@ -401,19 +205,14 @@ defmodule Plausible.Stats.Base do# :TRICKY: Subquery is used due to event:goal breakdown above doing an UNION ALLsubquery(q)- |> select_merge(- ^%{total_visitors: total_visitors_subquery(site, total_query, query.include_imported)}- )- |> select_merge([e], %{+ |> select_merge_as([], total_visitors_subquery(site, total_query, query.include_imported))+ |> select_merge_as([e], %{conversion_rate:- selected_as(- fragment(- "if(? > 0, round(? / ? * 100, 1), 0)",- selected_as(:__total_visitors),- e.visitors,- selected_as(:__total_visitors)- ),- :conversion_rate+ fragment(+ "if(? > 0, round(? / ? * 100, 1), 0)",+ selected_as(:total_visitors),+ e.visitors,+ selected_as(:total_visitors))})elsediff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.exindex a0bdaa1509..abf78beac3 100644--- a/lib/plausible/stats/breakdown.ex+++ b/lib/plausible/stats/breakdown.ex@@ -1,7 +1,7 @@defmodule Plausible.Stats.Breakdown douse Plausible.ClickhouseRepouse Plausible- use Plausible.Stats.Fragments+ use Plausible.Stats.SQL.Fragmentsimport Plausible.Stats.Baseimport Ecto.Querydiff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.exindex ed112b9a93..d22a4bb0c5 100644--- a/lib/plausible/stats/clickhouse.ex+++ b/lib/plausible/stats/clickhouse.ex@@ -2,7 +2,7 @@ defmodule Plausible.Stats.Clickhouse douse Plausibleuse Plausible.Repouse Plausible.ClickhouseRepo- use Plausible.Stats.Fragments+ use Plausible.Stats.SQL.Fragmentsimport Ecto.Query, only: [from: 2]diff --git a/lib/plausible/stats/current_visitors.ex b/lib/plausible/stats/current_visitors.exindex 4af1466912..3249e510d7 100644--- a/lib/plausible/stats/current_visitors.ex+++ b/lib/plausible/stats/current_visitors.ex@@ -1,6 +1,6 @@defmodule Plausible.Stats.CurrentVisitors douse Plausible.ClickhouseRepo- use Plausible.Stats.Fragments+ use Plausible.Stats.SQL.Fragmentsdef current_visitors(site) dofirst_datetime =diff --git a/lib/plausible/stats/filter_suggestions.ex b/lib/plausible/stats/filter_suggestions.exindex 8713919e3e..c4c59a3736 100644--- a/lib/plausible/stats/filter_suggestions.ex+++ b/lib/plausible/stats/filter_suggestions.ex@@ -1,7 +1,7 @@defmodule Plausible.Stats.FilterSuggestions douse Plausible.Repouse Plausible.ClickhouseRepo- use Plausible.Stats.Fragments+ use Plausible.Stats.SQL.Fragmentsimport Plausible.Stats.Baseimport Ecto.Querydiff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.exindex 5b774d9ea9..72a888d186 100644--- a/lib/plausible/stats/imported/base.ex+++ b/lib/plausible/stats/imported/base.ex@@ -6,8 +6,7 @@ defmodule Plausible.Stats.Imported.Base doimport Ecto.Queryalias Plausible.Imported- alias Plausible.Stats.Filters- alias Plausible.Stats.Query+ alias Plausible.Stats.{Filters, Query, SQL}@property_to_table_mappings %{"visit:source" => "imported_sources",@@ -213,9 +212,9 @@ defmodule Plausible.Stats.Imported.Base dodefp apply_filter(q, %Query{filters: filters}) doEnum.reduce(filters, q, fn [_, filter_key | _] = filter, q ->- db_field = Plausible.Stats.Filters.without_prefix(filter_key)+ db_field = Filters.without_prefix(filter_key)mapped_db_field = Map.get(@db_field_mappings, db_field, db_field)- condition = Filters.WhereBuilder.build_condition(mapped_db_field, filter)+ condition = SQL.WhereBuilder.build_condition(mapped_db_field, filter)where(q, ^condition)end)diff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.exindex 57cd6737bb..4fcdd04a66 100644--- a/lib/plausible/stats/imported/imported.ex+++ b/lib/plausible/stats/imported/imported.ex@@ -1,11 +1,11 @@defmodule Plausible.Stats.Imported do- alias Plausible.Stats.Filtersuse Plausible.ClickhouseRepo+ use Plausible.Stats.SQL.Fragmentsimport Ecto.Query- import Plausible.Stats.Fragmentsimport Plausible.Stats.Util, only: [shortname: 2]+ alias Plausible.Stats.Filtersalias Plausible.Stats.Importedalias Plausible.Stats.Queryalias Plausible.Stats.SQL.QueryBuilder@@ -290,12 +290,8 @@ defmodule Plausible.Stats.Imported do"imported_custom_events" ->Imported.Base.query_imported("imported_custom_events", site, query)|> where([i], i.visitors > 0)- |> select_merge([i], %{- dim0:- selected_as(- fragment("-indexOf(?, ?)", type(^events, {:array, :string}), i.name),- :dim0- )+ |> select_merge_as([i], %{+ dim0: fragment("-indexOf(?, ?)", type(^events, {:array, :string}), i.name)})|> select_imported_metrics(metrics)|> group_by([], selected_as(:dim0))@@ -314,8 +310,8 @@ defmodule Plausible.Stats.Imported do)|> join(:array, index in fragment("indices"))|> group_by([_i, index], index)- |> select_merge([_i, index], %{- dim0: selected_as(type(fragment("?", index), :integer), :dim0)+ |> select_merge_as([_i, index], %{+ dim0: type(fragment("?", index), :integer)})|> select_imported_metrics(metrics)end)@@ -563,17 +559,8 @@ defmodule Plausible.Stats.Imported dodefp group_imported_by(q, dim, key) when dim in [:source, :referrer] doq|> group_by([i], field(i, ^dim))- |> select_merge([i], %{- ^key =>- selected_as(- fragment(- "if(empty(?), ?, ?)",- field(i, ^dim),- @no_ref,- field(i, ^dim)- ),- ^key- )+ |> select_merge_as([i], %{+ key => fragment("if(empty(?), ?, ?)", field(i, ^dim), @no_ref, field(i, ^dim))})end@@ -582,90 +569,70 @@ defmodule Plausible.Stats.Imported doq|> group_by([i], field(i, ^dim))|> where([i], fragment("not empty(?)", field(i, ^dim)))- |> select_merge([i], %{^key => selected_as(field(i, ^dim), ^key)})+ |> select_merge_as([i], %{key => field(i, ^dim)})enddefp group_imported_by(q, :page, key) doq|> group_by([i], i.page)- |> select_merge([i], %{^key => selected_as(i.page, ^key), time_on_page: sum(i.time_on_page)})+ |> select_merge_as([i], %{key => i.page, time_on_page: sum(i.time_on_page)})enddefp group_imported_by(q, :country, key) doq|> group_by([i], i.country)|> where([i], i.country != "ZZ")- |> select_merge([i], %{^key => selected_as(i.country, ^key)})+ |> select_merge_as([i], %{key => i.country})enddefp group_imported_by(q, :region, key) doq|> group_by([i], i.region)|> where([i], i.region != "")- |> select_merge([i], %{^key => selected_as(i.region, ^key)})+ |> select_merge_as([i], %{key => i.region})enddefp group_imported_by(q, :city, key) doq|> group_by([i], i.city)|> where([i], i.city != 0 and not is_nil(i.city))- |> select_merge([i], %{^key => selected_as(i.city, ^key)})+ |> select_merge_as([i], %{key => i.city})enddefp group_imported_by(q, dim, key) when dim in [:device, :browser] doq|> group_by([i], field(i, ^dim))- |> select_merge([i], %{- ^key =>- selected_as(- fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim)),- ^key- )+ |> select_merge_as([i], %{+ key => fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim))})enddefp group_imported_by(q, :browser_version, key) doq|> group_by([i], [i.browser_version])- |> select_merge([i], %{- ^key =>- selected_as(- fragment(- "if(empty(?), ?, ?)",- i.browser_version,- @not_set,- i.browser_version- ),- ^key- )+ |> select_merge_as([i], %{+ key => fragment("if(empty(?), ?, ?)", i.browser_version, @not_set, i.browser_version)})enddefp group_imported_by(q, :os, key) doq|> group_by([i], i.operating_system)- |> select_merge([i], %{- ^key =>- selected_as(- fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system),- ^key- )+ |> select_merge_as([i], %{+ key => fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system)})enddefp group_imported_by(q, :os_version, key) doq|> group_by([i], [i.operating_system_version])- |> select_merge([i], %{- ^key =>- selected_as(- fragment(- "if(empty(?), ?, ?)",- i.operating_system_version,- @not_set,- i.operating_system_version- ),- ^key+ |> select_merge_as([i], %{+ key =>+ fragment(+ "if(empty(?), ?, ?)",+ i.operating_system_version,+ @not_set,+ i.operating_system_version)})end@@ -673,28 +640,28 @@ defmodule Plausible.Stats.Imported dodefp group_imported_by(q, dim, key) when dim in [:entry_page, :exit_page] doq|> group_by([i], field(i, ^dim))- |> select_merge([i], %{^key => selected_as(field(i, ^dim), ^key)})+ |> select_merge_as([i], %{key => field(i, ^dim)})enddefp group_imported_by(q, :name, key) doq|> group_by([i], i.name)- |> select_merge([i], %{^key => selected_as(i.name, ^key)})+ |> select_merge_as([i], %{key => i.name})enddefp group_imported_by(q, :url, key) doq|> group_by([i], i.link_url)- |> select_merge([i], %{- ^key => selected_as(fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none), ^key)+ |> select_merge_as([i], %{+ key => fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none)})enddefp group_imported_by(q, :path, key) doq|> group_by([i], i.path)- |> select_merge([i], %{- ^key => selected_as(fragment("if(not empty(?), ?, ?)", i.path, i.path, @none), ^key)+ |> select_merge_as([i], %{+ key => fragment("if(not empty(?), ?, ?)", i.path, i.path, @none)})end@@ -705,23 +672,14 @@ defmodule Plausible.Stats.Imported doenddefp select_joined_dimension(q, "visit:city", key) do- select_merge(q, [s, i], %{- ^key => selected_as(fragment("greatest(?,?)", field(i, ^key), field(s, ^key)), ^key)+ select_merge_as(q, [s, i], %{+ key => fragment("greatest(?,?)", field(i, ^key), field(s, ^key))})enddefp select_joined_dimension(q, _dimension, key) do- select_merge(q, [s, i], %{- ^key =>- selected_as(- fragment(- "if(empty(?), ?, ?)",- field(s, ^key),- field(i, ^key),- field(s, ^key)- ),- ^key- )+ select_merge_as(q, [s, i], %{+ key => fragment("if(empty(?), ?, ?)", field(s, ^key), field(i, ^key), field(s, ^key))})end@@ -734,31 +692,31 @@ defmodule Plausible.Stats.Imported dodefp select_joined_metrics(q, [:visits | rest]) doq- |> select_merge([s, i], %{visits: selected_as(s.visits + i.visits, :visits)})+ |> select_merge_as([s, i], %{visits: s.visits + i.visits})|> select_joined_metrics(rest)enddefp select_joined_metrics(q, [:visitors | rest]) doq- |> select_merge([s, i], %{visitors: selected_as(s.visitors + i.visitors, :visitors)})+ |> select_merge_as([s, i], %{visitors: s.visitors + i.visitors})|> select_joined_metrics(rest)enddefp select_joined_metrics(q, [:events | rest]) doq- |> select_merge([s, i], %{events: selected_as(s.events + i.events, :events)})+ |> select_merge_as([s, i], %{events: s.events + i.events})|> select_joined_metrics(rest)enddefp select_joined_metrics(q, [:pageviews | rest]) doq- |> select_merge([s, i], %{pageviews: selected_as(s.pageviews + i.pageviews, :pageviews)})+ |> select_merge_as([s, i], %{pageviews: s.pageviews + i.pageviews})|> select_joined_metrics(rest)enddefp select_joined_metrics(q, [:views_per_visit | rest]) doq- |> select_merge([s, i], %{+ |> select_merge_as([s, i], %{views_per_visit:fragment("if(? + ? > 0, round((? + ? * ?) / (? + ?), 2), 0)",@@ -776,7 +734,7 @@ defmodule Plausible.Stats.Imported dodefp select_joined_metrics(q, [:bounce_rate | rest]) doq- |> select_merge([s, i], %{+ |> select_merge_as([s, i], %{bounce_rate:fragment("if(? + ? > 0, round(100 * (? + (? * ? / 100)) / (? + ?)), 0)",@@ -794,7 +752,7 @@ defmodule Plausible.Stats.Imported dodefp select_joined_metrics(q, [:visit_duration | rest]) doq- |> select_merge([s, i], %{+ |> select_merge_as([s, i], %{visit_duration:fragment("""@@ -818,7 +776,7 @@ defmodule Plausible.Stats.Imported dodefp select_joined_metrics(q, [:sample_percent | rest]) doq- |> select_merge([s, i], %{sample_percent: s.sample_percent})+ |> select_merge_as([s, i], %{sample_percent: s.sample_percent})|> select_joined_metrics(rest)end@@ -831,10 +789,11 @@ defmodule Plausible.Stats.Imported dofrom(a in subquery(q1),full_join: b in subquery(q2),on: a.dim0 == b.dim0,- select: %{- dim0: selected_as(fragment("if(? != 0, ?, ?)", a.dim0, a.dim0, b.dim0), :dim0)- }+ select: %{})+ |> select_merge_as([a, b], %{+ dim0: fragment("if(? != 0, ?, ?)", a.dim0, a.dim0, b.dim0)+ })|> select_joined_metrics(metrics)endenddiff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.exindex d3f38a1428..daea6791f4 100644--- a/lib/plausible/stats/sql/expression.ex+++ b/lib/plausible/stats/sql/expression.ex@@ -1,131 +1,253 @@defmodule Plausible.Stats.SQL.Expression do@moduledoc """This module is responsible for generating SQL/Ecto expressions- for dimensions used in query select, group_by and order_by.+ for dimensions and metrics used in query SELECT statement.++ Each dimension and metric is tagged with with selected_as for easier+ usage down the line."""+ use Plausible+ use Plausible.Stats.SQL.Fragments+import Ecto.Query- use Plausible.Stats.Fragments+ alias Plausible.Stats.{Query, SQL}@no_ref "Direct / None"@not_set "(not set)"- defmacrop field_or_blank_value(expr, empty_value, select_alias) do+ defmacrop field_or_blank_value(key, expr, empty_value) doquote do- dynamic(- [t],- selected_as(- fragment("if(empty(?), ?, ?)", unquote(expr), unquote(empty_value), unquote(expr)),- ^unquote(select_alias)- )- )+ wrap_alias([t], %{+ unquote(key) =>+ fragment("if(empty(?), ?, ?)", unquote(expr), unquote(empty_value), unquote(expr))+ })endend- def dimension("time:hour", query, select_alias) do- dynamic(- [t],- selected_as(- fragment("toStartOfHour(toTimeZone(?, ?))", t.timestamp, ^query.timezone),- ^select_alias- )- )+ def dimension(key, "time:hour", query) do+ wrap_alias([t], %{+ key => fragment("toStartOfHour(toTimeZone(?, ?))", t.timestamp, ^query.timezone)+ })end- def dimension("time:day", query, select_alias) do- dynamic(- [t],- selected_as(- fragment("toDate(toTimeZone(?, ?))", t.timestamp, ^query.timezone),- ^select_alias- )- )+ def dimension(key, "time:day", query) do+ wrap_alias([t], %{+ key => fragment("toDate(toTimeZone(?, ?))", t.timestamp, ^query.timezone)+ })end- def dimension("time:month", query, select_alias) do- dynamic(- [t],- selected_as(- fragment("toStartOfMonth(toTimeZone(?, ?))", t.timestamp, ^query.timezone),- ^select_alias- )- )+ def dimension(key, "time:month", query) do+ wrap_alias([t], %{+ key => fragment("toStartOfMonth(toTimeZone(?, ?))", t.timestamp, ^query.timezone)+ })end- def dimension("event:name", _query, select_alias),- do: dynamic([t], selected_as(t.name, ^select_alias))+ def dimension(key, "event:name", _query),+ do: wrap_alias([t], %{key => t.name})- def dimension("event:page", _query, select_alias),- do: dynamic([t], selected_as(t.pathname, ^select_alias))+ def dimension(key, "event:page", _query),+ do: wrap_alias([t], %{key => t.pathname})- def dimension("event:hostname", _query, select_alias),- do: dynamic([t], selected_as(t.hostname, ^select_alias))+ def dimension(key, "event:hostname", _query),+ do: wrap_alias([t], %{key => t.hostname})- def dimension("event:props:" <> property_name, _query, select_alias) do- dynamic(- [t],- selected_as(+ def dimension(key, "event:props:" <> property_name, _query) do+ wrap_alias([t], %{+ key =>fragment("if(not empty(?), ?, '(none)')",get_by_key(t, :meta, ^property_name),get_by_key(t, :meta, ^property_name)- ),- ^select_alias- )- )+ )+ })end- def dimension("visit:entry_page", _query, select_alias),- do: dynamic([t], selected_as(t.entry_page, ^select_alias))+ def dimension(key, "visit:entry_page", _query),+ do: wrap_alias([t], %{key => t.entry_page})++ def dimension(key, "visit:exit_page", _query),+ do: wrap_alias([t], %{key => t.exit_page})++ def dimension(key, "visit:utm_medium", _query),+ do: field_or_blank_value(key, t.utm_medium, @not_set)++ def dimension(key, "visit:utm_source", _query),+ do: field_or_blank_value(key, t.utm_source, @not_set)- def dimension("visit:exit_page", _query, select_alias),- do: dynamic([t], selected_as(t.exit_page, ^select_alias))+ def dimension(key, "visit:utm_campaign", _query),+ do: field_or_blank_value(key, t.utm_campaign, @not_set)- def dimension("visit:utm_medium", _query, select_alias),- do: field_or_blank_value(t.utm_medium, @not_set, select_alias)+ def dimension(key, "visit:utm_content", _query),+ do: field_or_blank_value(key, t.utm_content, @not_set)- def dimension("visit:utm_source", _query, select_alias),- do: field_or_blank_value(t.utm_source, @not_set, select_alias)+ def dimension(key, "visit:utm_term", _query),+ do: field_or_blank_value(key, t.utm_term, @not_set)- def dimension("visit:utm_campaign", _query, select_alias),- do: field_or_blank_value(t.utm_campaign, @not_set, select_alias)+ def dimension(key, "visit:source", _query),+ do: field_or_blank_value(key, t.source, @no_ref)- def dimension("visit:utm_content", _query, select_alias),- do: field_or_blank_value(t.utm_content, @not_set, select_alias)+ def dimension(key, "visit:referrer", _query),+ do: field_or_blank_value(key, t.referrer, @no_ref)- def dimension("visit:utm_term", _query, select_alias),- do: field_or_blank_value(t.utm_term, @not_set, select_alias)+ def dimension(key, "visit:device", _query),+ do: field_or_blank_value(key, t.device, @not_set)- def dimension("visit:source", _query, select_alias),- do: field_or_blank_value(t.source, @no_ref, select_alias)+ def dimension(key, "visit:os", _query),+ do: field_or_blank_value(key, t.os, @not_set)- def dimension("visit:referrer", _query, select_alias),- do: field_or_blank_value(t.referrer, @no_ref, select_alias)+ def dimension(key, "visit:os_version", _query),+ do: field_or_blank_value(key, t.os_version, @not_set)- def dimension("visit:device", _query, select_alias),- do: field_or_blank_value(t.device, @not_set, select_alias)+ def dimension(key, "visit:browser", _query),+ do: field_or_blank_value(key, t.browser, @not_set)- def dimension("visit:os", _query, select_alias),- do: field_or_blank_value(t.os, @not_set, select_alias)+ def dimension(key, "visit:browser_version", _query),+ do: field_or_blank_value(key, t.browser_version, @not_set)- def dimension("visit:os_version", _query, select_alias),- do: field_or_blank_value(t.os_version, @not_set, select_alias)+ def dimension(key, "visit:country", _query),+ do: wrap_alias([t], %{key => t.country})- def dimension("visit:browser", _query, select_alias),- do: field_or_blank_value(t.browser, @not_set, select_alias)+ def dimension(key, "visit:region", _query),+ do: wrap_alias([t], %{key => t.region})- def dimension("visit:browser_version", _query, select_alias),- do: field_or_blank_value(t.browser_version, @not_set, select_alias)+ def dimension(key, "visit:city", _query),+ do: wrap_alias([t], %{key => t.city})- def dimension("visit:country", _query, select_alias),- do: dynamic([t], selected_as(t.country, ^select_alias))+ def event_metric(:pageviews) do+ wrap_alias([e], %{+ pageviews:+ fragment("toUInt64(round(countIf(? = 'pageview') * any(_sample_factor)))", e.name)+ })+ end++ def event_metric(:events) do+ wrap_alias([], %{+ events: fragment("toUInt64(round(count(*) * any(_sample_factor)))")+ })+ end++ def event_metric(:visitors) do+ wrap_alias([e], %{+ visitors: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.user_id)+ })+ end++ def event_metric(:visits) do+ wrap_alias([e], %{+ visits: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.session_id)+ })+ end- def dimension("visit:region", _query, select_alias),- do: dynamic([t], selected_as(t.region, ^select_alias))+ on_ee do+ def event_metric(:total_revenue) do+ wrap_alias(+ [e],+ %{+ total_revenue:+ fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)+ }+ )+ end++ def event_metric(:average_revenue) do+ wrap_alias(+ [e],+ %{+ average_revenue:+ fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)+ }+ )+ end+ end++ def event_metric(:sample_percent) do+ wrap_alias([], %{+ sample_percent:+ fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)")+ })+ end++ def event_metric(:percentage), do: %{}+ def event_metric(:conversion_rate), do: %{}+ def event_metric(:group_conversion_rate), do: %{}+ def event_metric(:total_visitors), do: %{}++ def event_metric(unknown), do: raise("Unknown metric: #{unknown}")++ def session_metric(:bounce_rate, query) do+ # :TRICKY: If page is passed to query, we only count bounce rate where users _entered_ at page.+ event_page_filter = Query.get_filter(query, "event:page")+ condition = SQL.WhereBuilder.build_condition(:entry_page, event_page_filter)++ wrap_alias([], %{+ bounce_rate:+ fragment(+ "toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))",+ ^condition,+ ^condition+ ),+ __internal_visits: fragment("toUInt32(sum(sign))")+ })+ end++ def session_metric(:visits, _query) do+ wrap_alias([s], %{+ visits: fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign)+ })+ end++ def session_metric(:pageviews, _query) do+ wrap_alias([s], %{+ pageviews:+ fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews)+ })+ end++ def session_metric(:events, _query) do+ wrap_alias([s], %{+ events: fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events)+ })+ end++ def session_metric(:visitors, _query) do+ wrap_alias([s], %{+ visitors: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", s.user_id)+ })+ end++ def session_metric(:visit_duration, _query) do+ wrap_alias([], %{+ visit_duration:+ fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))"),+ __internal_visits: fragment("toUInt32(sum(sign))")+ })+ end++ def session_metric(:views_per_visit, _query) do+ wrap_alias([s], %{+ views_per_visit:+ fragment(+ "ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)",+ s.sign,+ s.pageviews,+ s.sign+ ),+ __internal_visits: fragment("toUInt32(sum(sign))")+ })+ end++ def session_metric(:sample_percent, _query) do+ wrap_alias([], %{+ sample_percent:+ fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)")+ })+ end- def dimension("visit:city", _query, select_alias),- do: dynamic([t], selected_as(t.city, ^select_alias))+ def session_metric(:percentage, _query), do: %{}+ def session_metric(:conversion_rate, _query), do: %{}+ def session_metric(:group_conversion_rate, _query), do: %{}defmacro event_goal_join(events, page_regexes) doquote dodiff --git a/lib/plausible/stats/fragments.ex b/lib/plausible/stats/sql/fragments.exsimilarity index 65%rename from lib/plausible/stats/fragments.exrename to lib/plausible/stats/sql/fragments.exindex 27f49839f5..526682f9ef 100644--- a/lib/plausible/stats/fragments.ex+++ b/lib/plausible/stats/sql/fragments.ex@@ -1,4 +1,15 @@-defmodule Plausible.Stats.Fragments do+defmodule Plausible.Stats.SQL.Fragments do+ @moduledoc """+ Various macros and common SQL fragments used in Stats code.+ """++ defmacro __using__(_) do+ quote do+ import Plausible.Stats.SQL.Fragments+ require Plausible.Stats.SQL.Fragments+ end+ end+defmacro uniq(user_id) doquote dofragment("toUInt64(round(uniq(?) * any(_sample_factor)))", unquote(user_id))@@ -56,21 +67,23 @@ defmodule Plausible.Stats.Fragments do`not_before` boundary is set to the past Saturday, which is before theweekstart, therefore the cap does not apply.- iex> this_wednesday = ~D[2022-11-09]- ...> past_saturday = ~D[2022-11-05]- ...> weekstart_not_before(this_wednesday, past_saturday)+ ```+ > this_wednesday = ~D[2022-11-09]+ > past_saturday = ~D[2022-11-05]+ > weekstart_not_before(this_wednesday, past_saturday)~D[2022-11-07]-+ ```In this other example, the fragment returns Tuesday and not the weekstart.The `not_before` boundary is set to Tuesday, which is past the weekstart,therefore the cap applies.- iex> this_wednesday = ~D[2022-11-09]- ...> this_tuesday = ~D[2022-11-08]- ...> weekstart_not_before(this_wednesday, this_tuesday)+ ```+ > this_wednesday = ~D[2022-11-09]+ > this_tuesday = ~D[2022-11-08]+ > weekstart_not_before(this_wednesday, this_tuesday)~D[2022-11-08]-+ ```"""defmacro weekstart_not_before(date, not_before) doquote do@@ -85,7 +98,7 @@ defmodule Plausible.Stats.Fragments doend@doc """- Same as Plausible.Stats.Fragments.weekstart_not_before/2 but converts dates to+ Same as Plausible.Stats.SQL.Fragments.weekstart_not_before/2 but converts dates tothe specified timezone."""defmacro weekstart_not_before(date, not_before, timezone) do@@ -143,9 +156,51 @@ defmodule Plausible.Stats.Fragments dodef meta_value_column(:meta), do: :"meta.value"def meta_value_column(:entry_meta), do: :"entry_meta.value"- defmacro __using__(_) do+ @doc """+ Convenience Ecto macro for wrapping a map passed to select_merge_as such that each+ expression gets wrapped in dynamic and set as selected_as.++ ### Examples++ iex> wrap_alias([t], %{ foo: t.column }) |> expand_macro_once+ "%{foo: dynamic([t], selected_as(t.column, :foo))}"+ """+ defmacro wrap_alias(binding, map_literal) do+ update_literal_map_values(map_literal, fn {key, expr} ->+ key_expr =+ if Macro.quoted_literal?(key) do+ key+ else+ quote(do: ^unquote(key))+ end++ quote(do: dynamic(unquote(binding), selected_as(unquote(expr), unquote(key_expr))))+ end)+ end++ @doc """+ Convenience Ecto macro for wrapping select_merge where each value gets in turn passed to selected_as.++ ### Examples++ iex> select_merge_as(q, [t], %{ foo: t.column }) |> expand_macro_once+ "select_merge(q, [], ^wrap_alias([t], %{foo: t.column}))"+ """+ defmacro select_merge_as(q, binding, map_literal) doquote do- import Plausible.Stats.Fragments+ select_merge(unquote(q), [], ^wrap_alias(unquote(binding), unquote(map_literal)))endend++ defp update_literal_map_values({:%{}, ctx, keyword_list}, mapper_fn) do+ {+ :%{},+ ctx,+ Enum.map(keyword_list, fn {key, expr} ->+ {key, mapper_fn.({key, expr})}+ end)+ }+ end++ defp update_literal_map_values(ast, _), do: astenddiff --git a/lib/plausible/stats/sql/query_builder.ex b/lib/plausible/stats/sql/query_builder.exindex 7e5e49a887..8cf92725bc 100644--- a/lib/plausible/stats/sql/query_builder.ex+++ b/lib/plausible/stats/sql/query_builder.ex@@ -2,12 +2,13 @@ defmodule Plausible.Stats.SQL.QueryBuilder do@moduledoc falseuse Plausible+ use Plausible.Stats.SQL.Fragmentsimport Ecto.Queryimport Plausible.Stats.Importedimport Plausible.Stats.Util- alias Plausible.Stats.{Base, Query, QueryOptimizer, TableDecider, Filters}+ alias Plausible.Stats.{Base, Filters, Query, QueryOptimizer, TableDecider, SQL}alias Plausible.Stats.SQL.Expressionrequire Plausible.Stats.SQL.Expression@@ -30,7 +31,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder doq =from(e in "events_v2",- where: ^Filters.WhereBuilder.build(:events, site, events_query),+ where: ^SQL.WhereBuilder.build(:events, site, events_query),select: ^Base.select_event_metrics(events_query.metrics))@@ -73,7 +74,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder doq =from(e in "sessions_v2",- where: ^Filters.WhereBuilder.build(:sessions, site, sessions_query),+ where: ^SQL.WhereBuilder.build(:sessions, site, sessions_query),select: ^Base.select_session_metrics(sessions_query.metrics, sessions_query))@@ -94,7 +95,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder doif Query.has_event_filters?(query) doevents_q =from(e in "events_v2",- where: ^Filters.WhereBuilder.build(:events, site, query),+ where: ^SQL.WhereBuilder.build(:events, site, query),select: %{session_id: fragment("DISTINCT ?", e.session_id),_sample_factor: fragment("_sample_factor")@@ -135,7 +136,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder dokey = shortname(query, dimension)q- |> select_merge(^%{key => Expression.dimension(dimension, query, key)})+ |> select_merge_as([], Expression.dimension(key, dimension, query))|> group_by([], selected_as(^key))end@@ -147,9 +148,9 @@ defmodule Plausible.Stats.SQL.QueryBuilder doorder_by(q,[t],- ^{- order_direction,- dynamic([], selected_as(^shortname(query, metric_or_dimension)))+ {+ ^order_direction,+ selected_as(^shortname(query, metric_or_dimension))})end@@ -157,19 +158,11 @@ defmodule Plausible.Stats.SQL.QueryBuilder dodefmacrop select_join_fields(q, query, list, table_name) doquote doEnum.reduce(unquote(list), unquote(q), fn metric_or_dimension, q ->- select_merge(- q,- ^%{- shortname(unquote(query), metric_or_dimension) =>- dynamic(- [e, s],- selected_as(- field(unquote(table_name), ^shortname(unquote(query), metric_or_dimension)),- ^shortname(unquote(query), metric_or_dimension)- )- )- }- )+ key = shortname(unquote(query), metric_or_dimension)++ select_merge_as(q, [e, s], %{+ key => field(unquote(table_name), ^key)+ })end)endend@@ -185,21 +178,17 @@ defmodule Plausible.Stats.SQL.QueryBuilder do|> Query.set_dimensions([])q- |> select_merge(- ^%{- total_visitors: Base.total_visitors_subquery(site, total_query, query.include_imported)- }+ |> select_merge_as(+ [],+ Base.total_visitors_subquery(site, total_query, query.include_imported))- |> select_merge([e], %{+ |> select_merge_as([e], %{conversion_rate:- selected_as(- fragment(- "if(? > 0, round(? / ? * 100, 1), 0)",- selected_as(:__total_visitors),- selected_as(:visitors),- selected_as(:__total_visitors)- ),- :conversion_rate+ fragment(+ "if(? > 0, round(? / ? * 100, 1), 0)",+ selected_as(:total_visitors),+ selected_as(:visitors),+ selected_as(:total_visitors))})else@@ -228,21 +217,18 @@ defmodule Plausible.Stats.SQL.QueryBuilder dofrom(e in subquery(q),left_join: c in subquery(build(group_totals_query, site)),- on: ^build_group_by_join(query),- select_merge: %{- total_visitors: c.visitors,- group_conversion_rate:- selected_as(- fragment(- "if(? > 0, round(? / ? * 100, 1), 0)",- c.visitors,- e.visitors,- c.visitors- ),- :group_conversion_rate- )- }+ on: ^build_group_by_join(query))+ |> select_merge_as([e, c], %{+ total_visitors: c.visitors,+ group_conversion_rate:+ fragment(+ "if(? > 0, round(? / ? * 100, 1), 0)",+ c.visitors,+ e.visitors,+ c.visitors+ )+ })|> select_join_fields(query, query.dimensions, e)|> select_join_fields(query, List.delete(query.metrics, :group_conversion_rate), e)elsediff --git a/lib/plausible/stats/filters/where_builder.ex b/lib/plausible/stats/sql/where_builder.exsimilarity index 99%rename from lib/plausible/stats/filters/where_builder.exrename to lib/plausible/stats/sql/where_builder.exindex 257d4257a0..71bcdad495 100644--- a/lib/plausible/stats/filters/where_builder.ex+++ b/lib/plausible/stats/sql/where_builder.ex@@ -1,4 +1,4 @@-defmodule Plausible.Stats.Filters.WhereBuilder do+defmodule Plausible.Stats.SQL.WhereBuilder do@moduledoc """A module for building am ecto where clause of a query out of a query."""@@ -8,7 +8,7 @@ defmodule Plausible.Stats.Filters.WhereBuilder doalias Plausible.Stats.Query- use Plausible.Stats.Fragments+ use Plausible.Stats.SQL.Fragmentsrequire Loggerdiff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.exindex 69e3dc528f..4914023bfc 100644--- a/lib/plausible/stats/timeseries.ex+++ b/lib/plausible/stats/timeseries.ex@@ -4,7 +4,7 @@ defmodule Plausible.Stats.Timeseries doalias Plausible.Stats.{Query, Util, Imported}import Plausible.Stats.{Base}import Ecto.Query- use Plausible.Stats.Fragments+ use Plausible.Stats.SQL.Fragments@typep metric :::pageviewsdiff --git a/lib/plausible/stats/util.ex b/lib/plausible/stats/util.exindex 60bdd7a931..de1411f637 100644--- a/lib/plausible/stats/util.ex+++ b/lib/plausible/stats/util.ex@@ -6,7 +6,6 @@ defmodule Plausible.Stats.Util do@manually_removable_metrics [:__internal_visits,:visitors,- :__total_visitors,:__breakdown_value,:total_visitors]diff --git a/test/plausible/stats/sql/fragments_test.exs b/test/plausible/stats/sql/fragments_test.exsnew file mode 100644index 0000000000..0932d7e598--- /dev/null+++ b/test/plausible/stats/sql/fragments_test.exs@@ -0,0 +1,10 @@+defmodule Plausible.Stats.SQL.FragmentsTest do+ use ExUnit.Case, async: true+ use Plausible.Stats.SQL.Fragments++ defmacro expand_macro_once(ast) do+ ast |> Macro.expand_once(__ENV__) |> Macro.to_string()+ end++ doctest Plausible.Stats.SQL.Fragments+end