Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
APIv2: Cleanup (#4308)
Karl-Aksel Puulmanna9676546dcb8
diff --git a/extra/lib/plausible/stats/funnel.ex b/extra/lib/plausible/stats/funnel.exindex 121a699fae..a365f36107 100644--- a/extra/lib/plausible/stats/funnel.ex+++ b/extra/lib/plausible/stats/funnel.ex@@ -110,7 +110,7 @@ defmodule Plausible.Stats.Funnel do%Plausible.Goal{page_path: pathname} when is_binary(pathname) ->if String.contains?(pathname, "*") do- regex = Plausible.Stats.Base.page_regex(pathname)+ regex = Plausible.Stats.Filters.Utils.page_regex(pathname)dynamic([], fragment("match(pathname, ?)", ^regex))elsedynamic([], fragment("pathname = ?", ^pathname))diff --git a/lib/plausible/google/search_console/filters.ex b/lib/plausible/google/search_console/filters.exindex ee9bc21578..43e10e1c87 100644--- a/lib/plausible/google/search_console/filters.ex+++ b/lib/plausible/google/search_console/filters.ex@@ -1,6 +1,6 @@defmodule Plausible.Google.SearchConsole.Filters do@moduledoc false- import Plausible.Stats.Base, only: [page_regex: 1]+ import Plausible.Stats.Filters.Utils, only: [page_regex: 1]def transform(property, plausible_filters) dosearch_console_filters =diff --git a/lib/plausible/stats.ex b/lib/plausible/stats.exindex ffa59389ea..f22f1b069c 100644--- a/lib/plausible/stats.ex+++ b/lib/plausible/stats.ex@@ -15,6 +15,17 @@ defmodule Plausible.Stats douse Plausible.DebugReplayInfo+ def query(site, query) do+ include_sentry_replay_info()++ optimized_query = QueryOptimizer.optimize(query)++ optimized_query+ |> SQL.QueryBuilder.build(site)+ |> ClickhouseRepo.all()+ |> QueryResult.from(optimized_query)+ end+def breakdown(site, query, metrics, pagination) doinclude_sentry_replay_info()Breakdown.breakdown(site, query, metrics, pagination)@@ -35,15 +46,6 @@ defmodule Plausible.Stats doCurrentVisitors.current_visitors(site)end- def query(site, query) do- optimized_query = QueryOptimizer.optimize(query)-- optimized_query- |> SQL.QueryBuilder.build(site)- |> ClickhouseRepo.all()- |> QueryResult.from(optimized_query)- end-on_ee dodef funnel(site, query, funnel) doinclude_sentry_replay_info()diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.exindex 8c5cdc4f7b..790b5ed719 100644--- a/lib/plausible/stats/aggregate.ex+++ b/lib/plausible/stats/aggregate.ex@@ -1,4 +1,10 @@defmodule Plausible.Stats.Aggregate do+ @moduledoc """+ Builds aggregate results for v1 of our stats API and dashboards.++ Avoid adding new logic here - update QueryBuilder etc instead.+ """+use Plausible.ClickhouseRepouse Plausibleimport Plausible.Stats.Basediff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.exindex cb104ffc76..c5304eaacd 100644--- a/lib/plausible/stats/base.ex+++ b/lib/plausible/stats/base.ex@@ -1,10 +1,7 @@defmodule Plausible.Stats.Base do- use Plausible.ClickhouseRepouse Plausible- use Plausible.Stats.SQL.Fragments- alias Plausible.Stats.{Query, TableDecider, SQL}- alias Plausible.Timezones+ alias Plausible.Stats.{TableDecider, SQL}import Ecto.Querydef base_event_query(site, query) do@@ -29,7 +26,7 @@ defmodule Plausible.Stats.Base doendend- def query_events(site, query) do+ defp query_events(site, query) doq = from(e in "events_v2", where: ^SQL.WhereBuilder.build(:events, site, query))on_ee do@@ -48,95 +45,4 @@ defmodule Plausible.Stats.Base doqend-- def select_event_metrics(metrics) do- metrics- |> Enum.map(&SQL.Expression.event_metric/1)- |> Enum.reduce(%{}, &Map.merge/2)- end-- def select_session_metrics(metrics, query) do- metrics- |> Enum.map(&SQL.Expression.session_metric(&1, query))- |> Enum.reduce(%{}, &Map.merge/2)- end-- def filter_converted_sessions(db_query, site, query) do- if Query.has_event_filters?(query) do- converted_sessions =- from(e in query_events(site, query),- select: %{- session_id: fragment("DISTINCT ?", e.session_id),- _sample_factor: fragment("_sample_factor")- }- )-- from(s in db_query,- join: cs in subquery(converted_sessions),- on: s.session_id == cs.session_id- )- else- db_query- end- end-- defp beginning_of_time(candidate, native_stats_start_at) do- if Timex.after?(native_stats_start_at, candidate) do- native_stats_start_at- else- candidate- end- end-- def utc_boundaries(%Query{period: "realtime", now: now}, site) do- last_datetime =- now- |> Timex.shift(seconds: 5)- |> beginning_of_time(site.native_stats_start_at)- |> NaiveDateTime.truncate(:second)-- first_datetime =- now |> Timex.shift(minutes: -5) |> NaiveDateTime.truncate(:second)-- {first_datetime, last_datetime}- end-- def utc_boundaries(%Query{period: "30m", now: now}, site) do- last_datetime =- now- |> Timex.shift(seconds: 5)- |> beginning_of_time(site.native_stats_start_at)- |> NaiveDateTime.truncate(:second)-- first_datetime =- now |> Timex.shift(minutes: -30) |> NaiveDateTime.truncate(:second)-- {first_datetime, last_datetime}- end-- def utc_boundaries(%Query{date_range: date_range}, site) do- {:ok, first} = NaiveDateTime.new(date_range.first, ~T[00:00:00])-- first_datetime =- first- |> Timezones.to_utc_datetime(site.timezone)- |> beginning_of_time(site.native_stats_start_at)-- {:ok, last} = NaiveDateTime.new(date_range.last |> Timex.shift(days: 1), ~T[00:00:00])-- last_datetime = Timezones.to_utc_datetime(last, site.timezone)-- {first_datetime, last_datetime}- end-- def page_regex(expr) do- escaped =- expr- |> Regex.escape()- |> String.replace("\\|", "|")- |> String.replace("\\*\\*", ".*")- |> String.replace("\\*", ".*")-- "^#{escaped}$"- endenddiff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.exindex abf78beac3..f4f1d40d26 100644--- a/lib/plausible/stats/breakdown.ex+++ b/lib/plausible/stats/breakdown.ex@@ -1,4 +1,10 @@defmodule Plausible.Stats.Breakdown do+ @moduledoc """+ Builds breakdown results for v1 of our stats API and dashboards.++ Avoid adding new logic here - update QueryBuilder etc instead.+ """+use Plausible.ClickhouseRepouse Plausibleuse Plausible.Stats.SQL.Fragmentsdiff --git a/lib/plausible/stats/email_report.ex b/lib/plausible/stats/email_report.exindex 1dc26465bb..d745ed7971 100644--- a/lib/plausible/stats/email_report.ex+++ b/lib/plausible/stats/email_report.ex@@ -40,7 +40,7 @@ defmodule Plausible.Stats.EmailReport doenddefp put_top_5_pages(stats, site, query) do- query = Query.set_dimensions(query, ["event:page"])+ query = Query.set(query, dimensions: ["event:page"])pages = Stats.breakdown(site, query, [:visitors], {5, 1})Map.put(stats, :pages, pages)end@@ -48,8 +48,8 @@ defmodule Plausible.Stats.EmailReport dodefp put_top_5_sources(stats, site, query) doquery =query- |> Query.put_filter([:is_not, "visit:source", ["Direct / None"]])- |> Query.set_dimensions(["visit:source"])+ |> Query.add_filter([:is_not, "visit:source", ["Direct / None"]])+ |> Query.set(dimensions: ["visit:source"])sources = Stats.breakdown(site, query, [:visitors], {5, 1})diff --git a/lib/plausible/stats/filters/utils.ex b/lib/plausible/stats/filters/utils.exindex 96606c1d02..4869286fa4 100644--- a/lib/plausible/stats/filters/utils.ex+++ b/lib/plausible/stats/filters/utils.ex@@ -76,8 +76,19 @@ defmodule Plausible.Stats.Filters.Utils doevents = Enum.map(event_goals, fn {_, event} -> event end)page_regexes =- Enum.map(pageview_goals, fn {_, path} -> Plausible.Stats.Base.page_regex(path) end)+ Enum.map(pageview_goals, fn {_, path} -> page_regex(path) end){events, page_regexes}end++ def page_regex(expr) do+ escaped =+ expr+ |> Regex.escape()+ |> String.replace("\\|", "|")+ |> String.replace("\\*\\*", ".*")+ |> String.replace("\\*", ".*")++ "^#{escaped}$"+ endenddiff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.exindex a1e817e484..ead5a5db80 100644--- a/lib/plausible/stats/imported/imported.ex+++ b/lib/plausible/stats/imported/imported.ex@@ -3,17 +3,13 @@ defmodule Plausible.Stats.Imported douse Plausible.Stats.SQL.Fragmentsimport Ecto.Query- import Plausible.Stats.Util, only: [shortname: 2]+ import Plausible.Stats.Imported.SQL.Builderalias Plausible.Stats.Filtersalias Plausible.Stats.Importedalias Plausible.Stats.Queryalias Plausible.Stats.SQL.QueryBuilder- @no_ref "Direct / None"- @not_set "(not set)"- @none "(none)"-@property_to_table_mappings Imported.Base.property_to_table_mappings()@goals_with_url Plausible.Imported.goals_with_url()@@ -302,479 +298,4 @@ defmodule Plausible.Stats.Imported do|> Imported.Base.query_imported(query)|> select_merge([i], %{total_visitors: fragment("sum(?)", i.visitors)})end-- defp select_imported_metrics(q, []), do: q-- defp select_imported_metrics(q, [:visitors | rest]) do- q- |> select_merge([i], %{visitors: sum(i.visitors)})- |> select_imported_metrics(rest)- end-- defp select_imported_metrics(- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_custom_events", _}}} = q,- [:events | rest]- ) do- q- |> select_merge([i], %{events: sum(i.events)})- |> select_imported_metrics(rest)- end-- defp select_imported_metrics(q, [:events | rest]) do- q- |> select_merge([i], %{events: sum(i.pageviews)})- |> select_imported_metrics(rest)- end-- defp select_imported_metrics(- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q,- [:visits | rest]- ) do- q- |> select_merge([i], %{visits: sum(i.exits)})- |> select_imported_metrics(rest)- end-- defp select_imported_metrics(- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,- [:visits | rest]- ) do- q- |> select_merge([i], %{visits: sum(i.entrances)})- |> select_imported_metrics(rest)- end-- defp select_imported_metrics(q, [:visits | rest]) do- q- |> select_merge([i], %{visits: sum(i.visits)})- |> select_imported_metrics(rest)- end-- defp select_imported_metrics(- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_custom_events", _}}} = q,- [:pageviews | rest]- ) do- q- |> select_merge([i], %{pageviews: 0})- |> select_imported_metrics(rest)- end-- defp select_imported_metrics(q, [:pageviews | rest]) do- q- |> where([i], i.pageviews > 0)- |> select_merge([i], %{pageviews: sum(i.pageviews)})- |> select_imported_metrics(rest)- end-- defp select_imported_metrics(- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_pages", _}}} = q,- [:bounce_rate | rest]- ) do- q- |> select_merge([i], %{- bounces: 0,- __internal_visits: 0- })- |> select_imported_metrics(rest)- end-- defp select_imported_metrics(- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,- [:bounce_rate | rest]- ) do- q- |> select_merge([i], %{- bounces: sum(i.bounces),- __internal_visits: sum(i.entrances)- })- |> select_imported_metrics(rest)- end-- defp select_imported_metrics(- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q,- [:bounce_rate | rest]- ) do- q- |> select_merge([i], %{- bounces: sum(i.bounces),- __internal_visits: sum(i.exits)- })- |> select_imported_metrics(rest)- end-- defp select_imported_metrics(q, [:bounce_rate | rest]) do- q- |> select_merge([i], %{- bounces: sum(i.bounces),- __internal_visits: sum(i.visits)- })- |> select_imported_metrics(rest)- end-- defp select_imported_metrics(- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_pages", _}}} = q,- [:visit_duration | rest]- ) do- q- |> select_merge([i], %{- visit_duration: 0,- __internal_visits: 0- })- |> select_imported_metrics(rest)- end-- defp select_imported_metrics(- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,- [:visit_duration | rest]- ) do- q- |> select_merge([i], %{- visit_duration: sum(i.visit_duration),- __internal_visits: sum(i.entrances)- })- |> select_imported_metrics(rest)- end-- defp select_imported_metrics(- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q,- [:visit_duration | rest]- ) do- q- |> select_merge([i], %{- visit_duration: sum(i.visit_duration),- __internal_visits: sum(i.exits)- })- |> select_imported_metrics(rest)- end-- defp select_imported_metrics(q, [:visit_duration | rest]) do- q- |> select_merge([i], %{- visit_duration: sum(i.visit_duration),- __internal_visits: sum(i.visits)- })- |> select_imported_metrics(rest)- end-- defp select_imported_metrics(- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,- [:views_per_visit | rest]- ) do- q- |> where([i], i.pageviews > 0)- |> select_merge([i], %{- pageviews: sum(i.pageviews),- __internal_visits: sum(i.entrances)- })- |> select_imported_metrics(rest)- end-- defp select_imported_metrics(- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q,- [:views_per_visit | rest]- ) do- q- |> where([i], i.pageviews > 0)- |> select_merge([i], %{- pageviews: sum(i.pageviews),- __internal_visits: sum(i.exits)- })- |> select_imported_metrics(rest)- end-- defp select_imported_metrics(q, [:views_per_visit | rest]) do- q- |> where([i], i.pageviews > 0)- |> select_merge([i], %{- pageviews: sum(i.pageviews),- __internal_visits: sum(i.visits)- })- |> select_imported_metrics(rest)- end-- defp select_imported_metrics(q, [_ | rest]) do- q- |> select_imported_metrics(rest)- end-- defp group_imported_by(q, query) do- Enum.reduce(query.dimensions, q, fn dimension, q ->- dim = Plausible.Stats.Filters.without_prefix(dimension)-- group_imported_by(q, dim, shortname(query, dimension), query)- end)- end-- defp group_imported_by(q, dim, key, _query) when dim in [:source, :referrer] do- q- |> group_by([i], field(i, ^dim))- |> select_merge_as([i], %{- key => fragment("if(empty(?), ?, ?)", field(i, ^dim), @no_ref, field(i, ^dim))- })- end-- defp group_imported_by(q, dim, key, _query)- when dim in [:utm_source, :utm_medium, :utm_campaign, :utm_term, :utm_content] do- q- |> group_by([i], field(i, ^dim))- |> where([i], fragment("not empty(?)", field(i, ^dim)))- |> select_merge_as([i], %{key => field(i, ^dim)})- end-- defp group_imported_by(q, :page, key, _query) do- q- |> group_by([i], i.page)- |> select_merge_as([i], %{key => i.page, time_on_page: sum(i.time_on_page)})- end-- defp group_imported_by(q, :country, key, _query) do- q- |> group_by([i], i.country)- |> where([i], i.country != "ZZ")- |> select_merge_as([i], %{key => i.country})- end-- defp group_imported_by(q, :region, key, _query) do- q- |> group_by([i], i.region)- |> where([i], i.region != "")- |> select_merge_as([i], %{key => i.region})- end-- defp group_imported_by(q, :city, key, _query) do- q- |> group_by([i], i.city)- |> where([i], i.city != 0 and not is_nil(i.city))- |> select_merge_as([i], %{key => i.city})- end-- defp group_imported_by(q, dim, key, _query) when dim in [:device, :browser] do- q- |> group_by([i], field(i, ^dim))- |> select_merge_as([i], %{- key => fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim))- })- end-- defp group_imported_by(q, :browser_version, key, _query) do- q- |> group_by([i], [i.browser_version])- |> select_merge_as([i], %{- key => fragment("if(empty(?), ?, ?)", i.browser_version, @not_set, i.browser_version)- })- end-- defp group_imported_by(q, :os, key, _query) do- q- |> group_by([i], i.operating_system)- |> select_merge_as([i], %{- key => fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system)- })- end-- defp group_imported_by(q, :os_version, key, _query) do- q- |> group_by([i], [i.operating_system_version])- |> select_merge_as([i], %{- key =>- fragment(- "if(empty(?), ?, ?)",- i.operating_system_version,- @not_set,- i.operating_system_version- )- })- end-- defp group_imported_by(q, dim, key, _query) when dim in [:entry_page, :exit_page] do- q- |> group_by([i], field(i, ^dim))- |> select_merge_as([i], %{key => field(i, ^dim)})- end-- defp group_imported_by(q, :name, key, _query) do- q- |> group_by([i], i.name)- |> select_merge_as([i], %{key => i.name})- end-- defp group_imported_by(q, :url, key, _query) do- q- |> group_by([i], i.link_url)- |> select_merge_as([i], %{- key => fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none)- })- end-- defp group_imported_by(q, :path, key, _query) do- q- |> group_by([i], i.path)- |> select_merge_as([i], %{- key => fragment("if(not empty(?), ?, ?)", i.path, i.path, @none)- })- end-- defp group_imported_by(q, :month, key, _query) do- q- |> group_by([i], fragment("toStartOfMonth(?)", i.date))- |> select_merge_as([i], %{key => fragment("toStartOfMonth(?)", i.date)})- end-- defp group_imported_by(q, :hour, key, _query) do- q- |> group_by([i], i.date)- |> select_merge_as([i], %{key => i.date})- end-- defp group_imported_by(q, :week, key, query) do- q- |> group_by([i], weekstart_not_before(i.date, ^query.date_range.first))- |> select_merge_as([i], %{- key => weekstart_not_before(i.date, ^query.date_range.first)- })- end-- defp group_imported_by(q, :day, key, _query) do- q- |> group_by([i], i.date)- |> select_merge_as([i], %{key => i.date})- end-- defp select_joined_dimensions(q, query) do- Enum.reduce(query.dimensions, q, fn dimension, q ->- select_joined_dimension(q, dimension, shortname(query, dimension))- end)- end-- defp select_joined_dimension(q, "visit:city", key) do- select_merge_as(q, [s, i], %{- key => fragment("greatest(?,?)", field(i, ^key), field(s, ^key))- })- end-- defp select_joined_dimension(q, "time:" <> _, key) do- select_merge_as(q, [s, i], %{- key => fragment("greatest(?, ?)", field(i, ^key), field(s, ^key))- })- end-- defp select_joined_dimension(q, _dimension, key) do- select_merge_as(q, [s, i], %{- key => fragment("if(empty(?), ?, ?)", field(s, ^key), field(i, ^key), field(s, ^key))- })- end-- defp select_joined_metrics(q, []), do: q- # NOTE: Reverse-engineering the native data bounces and total visit- # durations to combine with imported data is inefficient. Instead both- # queries should fetch bounces/total_visit_duration and visits and be- # used as subqueries to a main query that then find the bounce rate/avg- # visit_duration.-- defp select_joined_metrics(q, [:visits | rest]) do- q- |> select_merge_as([s, i], %{visits: s.visits + i.visits})- |> select_joined_metrics(rest)- end-- defp select_joined_metrics(q, [:visitors | rest]) do- q- |> select_merge_as([s, i], %{visitors: s.visitors + i.visitors})- |> select_joined_metrics(rest)- end-- defp select_joined_metrics(q, [:events | rest]) do- q- |> select_merge_as([s, i], %{events: s.events + i.events})- |> select_joined_metrics(rest)- end-- defp select_joined_metrics(q, [:pageviews | rest]) do- q- |> select_merge_as([s, i], %{pageviews: s.pageviews + i.pageviews})- |> select_joined_metrics(rest)- end-- defp select_joined_metrics(q, [:views_per_visit | rest]) do- q- |> select_merge_as([s, i], %{- views_per_visit:- fragment(- "if(? + ? > 0, round((? + ? * ?) / (? + ?), 2), 0)",- s.__internal_visits,- i.__internal_visits,- i.pageviews,- s.views_per_visit,- s.__internal_visits,- i.__internal_visits,- s.__internal_visits- )- })- |> select_joined_metrics(rest)- end-- defp select_joined_metrics(q, [:bounce_rate | rest]) do- q- |> select_merge_as([s, i], %{- bounce_rate:- fragment(- "if(? + ? > 0, round(100 * (? + (? * ? / 100)) / (? + ?)), 0)",- s.__internal_visits,- i.__internal_visits,- i.bounces,- s.bounce_rate,- s.__internal_visits,- i.__internal_visits,- s.__internal_visits- )- })- |> select_joined_metrics(rest)- end-- defp select_joined_metrics(q, [:visit_duration | rest]) do- q- |> select_merge_as([s, i], %{- visit_duration:- fragment(- """- if(- ? + ? > 0,- round((? + ? * ?) / (? + ?), 0),- 0- )- """,- s.__internal_visits,- i.__internal_visits,- i.visit_duration,- s.visit_duration,- s.__internal_visits,- s.__internal_visits,- i.__internal_visits- )- })- |> select_joined_metrics(rest)- end-- defp select_joined_metrics(q, [:sample_percent | rest]) do- q- |> select_merge_as([s, i], %{sample_percent: s.sample_percent})- |> select_joined_metrics(rest)- end-- defp select_joined_metrics(q, [_ | rest]) do- q- |> select_joined_metrics(rest)- end-- defp naive_dimension_join(q1, q2, metrics) do- from(a in subquery(q1),- full_join: b in subquery(q2),- on: a.dim0 == b.dim0,- select: %{}- )- |> select_merge_as([a, b], %{- dim0: fragment("if(? != 0, ?, ?)", a.dim0, a.dim0, b.dim0)- })- |> select_joined_metrics(metrics)- endenddiff --git a/lib/plausible/stats/imported/sql/builder.ex b/lib/plausible/stats/imported/sql/builder.exnew file mode 100644index 0000000000..cf81959d3f--- /dev/null+++ b/lib/plausible/stats/imported/sql/builder.ex@@ -0,0 +1,462 @@+defmodule Plausible.Stats.Imported.SQL.Builder do+ @moduledoc """+ This module is responsible for generating SQL/Ecto expressions+ for dimensions, filters and metrics used in import table queries+ """++ use Plausible.Stats.SQL.Fragments++ import Plausible.Stats.Util, only: [shortname: 2]+ import Ecto.Query++ @no_ref "Direct / None"+ @not_set "(not set)"+ @none "(none)"++ def select_imported_metrics(q, []), do: q++ def select_imported_metrics(q, [:visitors | rest]) do+ q+ |> select_merge([i], %{visitors: sum(i.visitors)})+ |> select_imported_metrics(rest)+ end++ def select_imported_metrics(+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_custom_events", _}}} = q,+ [:events | rest]+ ) do+ q+ |> select_merge([i], %{events: sum(i.events)})+ |> select_imported_metrics(rest)+ end++ def select_imported_metrics(q, [:events | rest]) do+ q+ |> select_merge([i], %{events: sum(i.pageviews)})+ |> select_imported_metrics(rest)+ end++ def select_imported_metrics(+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q,+ [:visits | rest]+ ) do+ q+ |> select_merge([i], %{visits: sum(i.exits)})+ |> select_imported_metrics(rest)+ end++ def select_imported_metrics(+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,+ [:visits | rest]+ ) do+ q+ |> select_merge([i], %{visits: sum(i.entrances)})+ |> select_imported_metrics(rest)+ end++ def select_imported_metrics(q, [:visits | rest]) do+ q+ |> select_merge([i], %{visits: sum(i.visits)})+ |> select_imported_metrics(rest)+ end++ def select_imported_metrics(+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_custom_events", _}}} = q,+ [:pageviews | rest]+ ) do+ q+ |> select_merge([i], %{pageviews: 0})+ |> select_imported_metrics(rest)+ end++ def select_imported_metrics(q, [:pageviews | rest]) do+ q+ |> where([i], i.pageviews > 0)+ |> select_merge([i], %{pageviews: sum(i.pageviews)})+ |> select_imported_metrics(rest)+ end++ def select_imported_metrics(+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_pages", _}}} = q,+ [:bounce_rate | rest]+ ) do+ q+ |> select_merge([i], %{+ bounces: 0,+ __internal_visits: 0+ })+ |> select_imported_metrics(rest)+ end++ def select_imported_metrics(+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,+ [:bounce_rate | rest]+ ) do+ q+ |> select_merge([i], %{+ bounces: sum(i.bounces),+ __internal_visits: sum(i.entrances)+ })+ |> select_imported_metrics(rest)+ end++ def select_imported_metrics(+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q,+ [:bounce_rate | rest]+ ) do+ q+ |> select_merge([i], %{+ bounces: sum(i.bounces),+ __internal_visits: sum(i.exits)+ })+ |> select_imported_metrics(rest)+ end++ def select_imported_metrics(q, [:bounce_rate | rest]) do+ q+ |> select_merge([i], %{+ bounces: sum(i.bounces),+ __internal_visits: sum(i.visits)+ })+ |> select_imported_metrics(rest)+ end++ def select_imported_metrics(+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_pages", _}}} = q,+ [:visit_duration | rest]+ ) do+ q+ |> select_merge([i], %{+ visit_duration: 0,+ __internal_visits: 0+ })+ |> select_imported_metrics(rest)+ end++ def select_imported_metrics(+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,+ [:visit_duration | rest]+ ) do+ q+ |> select_merge([i], %{+ visit_duration: sum(i.visit_duration),+ __internal_visits: sum(i.entrances)+ })+ |> select_imported_metrics(rest)+ end++ def select_imported_metrics(+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q,+ [:visit_duration | rest]+ ) do+ q+ |> select_merge([i], %{+ visit_duration: sum(i.visit_duration),+ __internal_visits: sum(i.exits)+ })+ |> select_imported_metrics(rest)+ end++ def select_imported_metrics(q, [:visit_duration | rest]) do+ q+ |> select_merge([i], %{+ visit_duration: sum(i.visit_duration),+ __internal_visits: sum(i.visits)+ })+ |> select_imported_metrics(rest)+ end++ def select_imported_metrics(+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,+ [:views_per_visit | rest]+ ) do+ q+ |> where([i], i.pageviews > 0)+ |> select_merge([i], %{+ pageviews: sum(i.pageviews),+ __internal_visits: sum(i.entrances)+ })+ |> select_imported_metrics(rest)+ end++ def select_imported_metrics(+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q,+ [:views_per_visit | rest]+ ) do+ q+ |> where([i], i.pageviews > 0)+ |> select_merge([i], %{+ pageviews: sum(i.pageviews),+ __internal_visits: sum(i.exits)+ })+ |> select_imported_metrics(rest)+ end++ def select_imported_metrics(q, [:views_per_visit | rest]) do+ q+ |> where([i], i.pageviews > 0)+ |> select_merge([i], %{+ pageviews: sum(i.pageviews),+ __internal_visits: sum(i.visits)+ })+ |> select_imported_metrics(rest)+ end++ def select_imported_metrics(q, [_ | rest]) do+ q+ |> select_imported_metrics(rest)+ end++ def group_imported_by(q, query) do+ Enum.reduce(query.dimensions, q, fn dimension, q ->+ q+ |> select_group_fields(dimension, shortname(query, dimension), query)+ |> filter_group_values(dimension)+ |> group_by([], selected_as(^shortname(query, dimension)))+ end)+ end++ defp select_group_fields(q, dimension, key, _query)+ when dimension in ["visit:source", "visit:referrer"] do+ select_merge_as(q, [i], %{+ key =>+ fragment(+ "if(empty(?), ?, ?)",+ field(i, ^dim(dimension)),+ @no_ref,+ field(i, ^dim(dimension))+ )+ })+ end++ defp select_group_fields(q, "event:page", key, _query) do+ select_merge_as(q, [i], %{key => i.page, time_on_page: sum(i.time_on_page)})+ end++ defp select_group_fields(q, dimension, key, _query)+ when dimension in ["visit:device", "visit:browser"] do+ select_merge_as(q, [i], %{+ key =>+ fragment(+ "if(empty(?), ?, ?)",+ field(i, ^dim(dimension)),+ @not_set,+ field(i, ^dim(dimension))+ )+ })+ end++ defp select_group_fields(q, "visit:browser_version", key, _query) do+ select_merge_as(q, [i], %{+ key => fragment("if(empty(?), ?, ?)", i.browser_version, @not_set, i.browser_version)+ })+ end++ defp select_group_fields(q, "visit:os", key, _query) do+ select_merge_as(q, [i], %{+ key => fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system)+ })+ end++ defp select_group_fields(q, "visit:os_version", key, _query) do+ select_merge_as(q, [i], %{+ key =>+ fragment(+ "if(empty(?), ?, ?)",+ i.operating_system_version,+ @not_set,+ i.operating_system_version+ )+ })+ end++ defp select_group_fields(q, "event:props:url", key, _query) do+ select_merge_as(q, [i], %{+ key => fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none)+ })+ end++ defp select_group_fields(q, "event:props:path", key, _query) do+ select_merge_as(q, [i], %{+ key => fragment("if(not empty(?), ?, ?)", i.path, i.path, @none)+ })+ end++ defp select_group_fields(q, "time:month", key, _query) do+ select_merge_as(q, [i], %{key => fragment("toStartOfMonth(?)", i.date)})+ end++ defp select_group_fields(q, dimension, key, _query)+ when dimension in ["time:hour", "time:day"] do+ select_merge_as(q, [i], %{key => i.date})+ end++ defp select_group_fields(q, "time:week", key, query) do+ select_merge_as(q, [i], %{+ key => weekstart_not_before(i.date, ^query.date_range.first)+ })+ end++ defp select_group_fields(q, dimension, key, _query) do+ select_merge_as(q, [i], %{key => field(i, ^dim(dimension))})+ end++ @utm_dimensions [+ "visit:utm_source",+ "visit:utm_medium",+ "visit:utm_campaign",+ "visit:utm_term",+ "visit:utm_content"+ ]+ defp filter_group_values(q, dimension) when dimension in @utm_dimensions do+ dim = Plausible.Stats.Filters.without_prefix(dimension)++ where(q, [i], fragment("not empty(?)", field(i, ^dim)))+ end++ defp filter_group_values(q, "visit:country"), do: where(q, [i], i.country != "ZZ")+ defp filter_group_values(q, "visit:region"), do: where(q, [i], i.region != "")+ defp filter_group_values(q, "visit:city"), do: where(q, [i], i.city != 0 and not is_nil(i.city))++ defp filter_group_values(q, _dimension), do: q++ def select_joined_dimensions(q, query) do+ Enum.reduce(query.dimensions, q, fn dimension, q ->+ select_joined_dimension(q, dimension, shortname(query, dimension))+ end)+ end++ defp select_joined_dimension(q, "visit:city", key) do+ select_merge_as(q, [s, i], %{+ key => fragment("greatest(?,?)", field(i, ^key), field(s, ^key))+ })+ end++ defp select_joined_dimension(q, "time:" <> _, key) do+ select_merge_as(q, [s, i], %{+ key => fragment("greatest(?, ?)", field(i, ^key), field(s, ^key))+ })+ end++ defp select_joined_dimension(q, _dimension, key) do+ select_merge_as(q, [s, i], %{+ key => fragment("if(empty(?), ?, ?)", field(s, ^key), field(i, ^key), field(s, ^key))+ })+ end++ def select_joined_metrics(q, []), do: q+ # NOTE: Reverse-engineering the native data bounces and total visit+ # durations to combine with imported data is inefficient. Instead both+ # queries should fetch bounces/total_visit_duration and visits and be+ # used as subqueries to a main query that then find the bounce rate/avg+ # visit_duration.++ def select_joined_metrics(q, [:visits | rest]) do+ q+ |> select_merge_as([s, i], %{visits: s.visits + i.visits})+ |> select_joined_metrics(rest)+ end++ def select_joined_metrics(q, [:visitors | rest]) do+ q+ |> select_merge_as([s, i], %{visitors: s.visitors + i.visitors})+ |> select_joined_metrics(rest)+ end++ def select_joined_metrics(q, [:events | rest]) do+ q+ |> select_merge_as([s, i], %{events: s.events + i.events})+ |> select_joined_metrics(rest)+ end++ def select_joined_metrics(q, [:pageviews | rest]) do+ q+ |> select_merge_as([s, i], %{pageviews: s.pageviews + i.pageviews})+ |> select_joined_metrics(rest)+ end++ def select_joined_metrics(q, [:views_per_visit | rest]) do+ q+ |> select_merge_as([s, i], %{+ views_per_visit:+ fragment(+ "if(? + ? > 0, round((? + ? * ?) / (? + ?), 2), 0)",+ s.__internal_visits,+ i.__internal_visits,+ i.pageviews,+ s.views_per_visit,+ s.__internal_visits,+ i.__internal_visits,+ s.__internal_visits+ )+ })+ |> select_joined_metrics(rest)+ end++ def select_joined_metrics(q, [:bounce_rate | rest]) do+ q+ |> select_merge_as([s, i], %{+ bounce_rate:+ fragment(+ "if(? + ? > 0, round(100 * (? + (? * ? / 100)) / (? + ?)), 0)",+ s.__internal_visits,+ i.__internal_visits,+ i.bounces,+ s.bounce_rate,+ s.__internal_visits,+ i.__internal_visits,+ s.__internal_visits+ )+ })+ |> select_joined_metrics(rest)+ end++ def select_joined_metrics(q, [:visit_duration | rest]) do+ q+ |> select_merge_as([s, i], %{+ visit_duration:+ fragment(+ """+ if(+ ? + ? > 0,+ round((? + ? * ?) / (? + ?), 0),+ 0+ )+ """,+ s.__internal_visits,+ i.__internal_visits,+ i.visit_duration,+ s.visit_duration,+ s.__internal_visits,+ s.__internal_visits,+ i.__internal_visits+ )+ })+ |> select_joined_metrics(rest)+ end++ def select_joined_metrics(q, [:sample_percent | rest]) do+ q+ |> select_merge_as([s, i], %{sample_percent: s.sample_percent})+ |> select_joined_metrics(rest)+ end++ def select_joined_metrics(q, [_ | rest]) do+ q+ |> select_joined_metrics(rest)+ end++ def naive_dimension_join(q1, q2, metrics) do+ from(a in subquery(q1),+ full_join: b in subquery(q2),+ on: a.dim0 == b.dim0,+ select: %{}+ )+ |> select_merge_as([a, b], %{+ dim0: fragment("if(? != 0, ?, ?)", a.dim0, a.dim0, b.dim0)+ })+ |> select_joined_metrics(metrics)+ end++ defp dim(dimension), do: Plausible.Stats.Filters.without_prefix(dimension)+enddiff --git a/lib/plausible/stats/props.ex b/lib/plausible/stats/legacy/legacy_dimensions.exsimilarity index 58%rename from lib/plausible/stats/props.exrename to lib/plausible/stats/legacy/legacy_dimensions.exindex c0f1bff9dd..3de9c023af 100644--- a/lib/plausible/stats/props.ex+++ b/lib/plausible/stats/legacy/legacy_dimensions.ex@@ -1,4 +1,7 @@-defmodule Plausible.Stats.Props do+defmodule Plausible.Stats.Legacy.Dimensions do+ @moduledoc """+ Deprecated module. See QueryParser for list of valid dimensions+ """@event_props ["event:page", "event:name", "event:goal", "event:hostname"]@session_props ["visit:source",@@ -20,8 +23,8 @@ defmodule Plausible.Stats.Props do"visit:browser_version"]- def valid_prop?(prop) when prop in @event_props, do: true- def valid_prop?(prop) when prop in @session_props, do: true- def valid_prop?("event:props:" <> prop) when byte_size(prop) > 0, do: true- def valid_prop?(_), do: false+ def valid?(prop) when prop in @event_props, do: true+ def valid?(prop) when prop in @session_props, do: true+ def valid?("event:props:" <> prop) when byte_size(prop) > 0, do: true+ def valid?(_), do: falseenddiff --git a/lib/plausible/stats/legacy/legacy_query_builder.ex b/lib/plausible/stats/legacy/legacy_query_builder.exnew file mode 100644index 0000000000..9ec14a713d--- /dev/null+++ b/lib/plausible/stats/legacy/legacy_query_builder.ex@@ -0,0 +1,180 @@+defmodule Plausible.Stats.Legacy.QueryBuilder do+ @moduledoc false++ use Plausible++ alias Plausible.Stats.{Filters, Interval, Query}++ def from(site, params) do+ now = NaiveDateTime.utc_now(:second)++ query =+ Query+ |> struct!(now: now, timezone: site.timezone)+ |> put_period(site, params)+ |> put_dimensions(params)+ |> put_interval(params)+ |> put_parsed_filters(params)+ |> Query.put_experimental_reduced_joins(site, params)+ |> Query.put_imported_opts(site, params)++ on_ee do+ query = Plausible.Stats.Sampling.put_threshold(query, params)+ end++ query+ end++ defp put_period(query, site, %{"period" => "realtime"}) do+ date = today(site.timezone)++ struct!(query, period: "realtime", date_range: Date.range(date, date))+ end++ defp put_period(query, site, %{"period" => "day"} = params) do+ date = parse_single_date(site.timezone, params)++ struct!(query, period: "day", date_range: Date.range(date, date))+ end++ defp put_period(query, site, %{"period" => "7d"} = params) do+ end_date = parse_single_date(site.timezone, params)+ start_date = end_date |> Timex.shift(days: -6)++ struct!(+ query,+ period: "7d",+ date_range: Date.range(start_date, end_date)+ )+ end++ defp put_period(query, site, %{"period" => "30d"} = params) do+ end_date = parse_single_date(site.timezone, params)+ start_date = end_date |> Timex.shift(days: -30)++ struct!(query, period: "30d", date_range: Date.range(start_date, end_date))+ end++ defp put_period(query, site, %{"period" => "month"} = params) do+ date = parse_single_date(site.timezone, params)++ start_date = Timex.beginning_of_month(date)+ end_date = Timex.end_of_month(date)++ struct!(query,+ period: "month",+ date_range: Date.range(start_date, end_date)+ )+ end++ defp put_period(query, site, %{"period" => "6mo"} = params) do+ end_date =+ parse_single_date(site.timezone, params)+ |> Timex.end_of_month()++ start_date =+ Timex.shift(end_date, months: -5)+ |> Timex.beginning_of_month()++ struct!(query,+ period: "6mo",+ date_range: Date.range(start_date, end_date)+ )+ end++ defp put_period(query, site, %{"period" => "12mo"} = params) do+ end_date =+ parse_single_date(site.timezone, params)+ |> Timex.end_of_month()++ start_date =+ Timex.shift(end_date, months: -11)+ |> Timex.beginning_of_month()++ struct!(query,+ period: "12mo",+ date_range: Date.range(start_date, end_date)+ )+ end++ defp put_period(query, site, %{"period" => "year"} = params) do+ end_date =+ parse_single_date(site.timezone, params)+ |> Timex.end_of_year()++ start_date = Timex.beginning_of_year(end_date)++ struct!(query,+ period: "year",+ date_range: Date.range(start_date, end_date)+ )+ end++ defp put_period(query, site, %{"period" => "all"}) do+ now = today(site.timezone)+ start_date = Plausible.Sites.stats_start_date(site) || now++ struct!(query,+ period: "all",+ date_range: Date.range(start_date, now)+ )+ end++ defp put_period(query, site, %{"period" => "custom", "from" => from, "to" => to} = params) do+ new_params =+ params+ |> Map.drop(["from", "to"])+ |> Map.put("date", Enum.join([from, to], ","))++ put_period(query, site, new_params)+ end++ defp put_period(query, _site, %{"period" => "custom", "date" => date}) do+ [from, to] = String.split(date, ",")+ from_date = Date.from_iso8601!(String.trim(from))+ to_date = Date.from_iso8601!(String.trim(to))++ struct!(query,+ period: "custom",+ date_range: Date.range(from_date, to_date)+ )+ end++ defp put_period(query, site, params) do+ put_period(query, site, Map.merge(params, %{"period" => "30d"}))+ end++ defp put_dimensions(query, params) do+ if not is_nil(params["property"]) do+ struct!(query, dimensions: [params["property"]])+ else+ struct!(query, dimensions: Map.get(params, "dimensions", []))+ end+ end++ defp put_interval(%{:period => "all"} = query, params) do+ interval = Map.get(params, "interval", Interval.default_for_date_range(query.date_range))+ struct!(query, interval: interval)+ end++ defp put_interval(query, params) do+ interval = Map.get(params, "interval", Interval.default_for_period(query.period))+ struct!(query, interval: interval)+ end++ defp put_parsed_filters(query, params) do+ struct!(query, filters: Filters.parse(params["filters"]))+ end++ defp today(tz) do+ Timex.now(tz) |> Timex.to_date()+ end++ defp parse_single_date(tz, params) do+ case params["date"] do+ "today" -> Timex.now(tz) |> Timex.to_date()+ date when is_binary(date) -> Date.from_iso8601!(date)+ _ -> today(tz)+ end+ end+enddiff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.exindex 9d2eb0d667..7de4e931b4 100644--- a/lib/plausible/stats/query.ex+++ b/lib/plausible/stats/query.ex@@ -24,44 +24,31 @@ defmodule Plausible.Stats.Query do}require OpenTelemetry.Tracer, as: Tracer- alias Plausible.Stats.{Filters, Interval, Imported}+ alias Plausible.Stats.{Filters, Imported, Legacy}@type t :: %__MODULE__{}- @spec from(Plausible.Site.t(), map()) :: t()- def from(site, params) do- now = NaiveDateTime.utc_now(:second)-- query =- __MODULE__- |> struct!(now: now, timezone: site.timezone)- |> put_experimental_reduced_joins(site, params)- |> put_period(site, params)- |> put_dimensions(params)- |> put_interval(params)- |> put_parsed_filters(params)- |> put_imported_opts(site, params)-- on_ee do- query = Plausible.Stats.Sampling.put_threshold(query, params)- end-- query- end-def build(site, params) dowith {:ok, query_data} <- Filters.QueryParser.parse(site, params) doquery =struct!(__MODULE__, Map.to_list(query_data))|> put_imported_opts(site, %{})|> put_experimental_reduced_joins(site, params)- |> struct!(v2: true)+ |> struct!(v2: true, now: NaiveDateTime.utc_now(:second)){:ok, query}endend- defp put_experimental_reduced_joins(query, site, params) do+ @doc """+ Builds query from old-style params. New code should prefer Query.build+ """+ @spec from(Plausible.Site.t(), map()) :: t()+ def from(site, params) do+ Legacy.QueryBuilder.from(site, params)+ end++ def put_experimental_reduced_joins(query, site, params) doif Map.has_key?(params, "experimental_reduced_joins") dostruct!(query,experimental_reduced_joins?: Map.get(params, "experimental_reduced_joins") == "true"@@ -73,147 +60,6 @@ defmodule Plausible.Stats.Query doendend- defp put_period(query, site, %{"period" => "realtime"}) do- date = today(site.timezone)-- struct!(query, period: "realtime", date_range: Date.range(date, date))- end-- defp put_period(query, site, %{"period" => "day"} = params) do- date = parse_single_date(site.timezone, params)-- struct!(query, period: "day", date_range: Date.range(date, date))- end-- defp put_period(query, site, %{"period" => "7d"} = params) do- end_date = parse_single_date(site.timezone, params)- start_date = end_date |> Timex.shift(days: -6)-- struct!(- query,- period: "7d",- date_range: Date.range(start_date, end_date)- )- end-- defp put_period(query, site, %{"period" => "30d"} = params) do- end_date = parse_single_date(site.timezone, params)- start_date = end_date |> Timex.shift(days: -30)-- struct!(query, period: "30d", date_range: Date.range(start_date, end_date))- end-- defp put_period(query, site, %{"period" => "month"} = params) do- date = parse_single_date(site.timezone, params)-- start_date = Timex.beginning_of_month(date)- end_date = Timex.end_of_month(date)-- struct!(query,- period: "month",- date_range: Date.range(start_date, end_date)- )- end-- defp put_period(query, site, %{"period" => "6mo"} = params) do- end_date =- parse_single_date(site.timezone, params)- |> Timex.end_of_month()-- start_date =- Timex.shift(end_date, months: -5)- |> Timex.beginning_of_month()-- struct!(query,- period: "6mo",- date_range: Date.range(start_date, end_date)- )- end-- defp put_period(query, site, %{"period" => "12mo"} = params) do- end_date =- parse_single_date(site.timezone, params)- |> Timex.end_of_month()-- start_date =- Timex.shift(end_date, months: -11)- |> Timex.beginning_of_month()-- struct!(query,- period: "12mo",- date_range: Date.range(start_date, end_date)- )- end-- defp put_period(query, site, %{"period" => "year"} = params) do- end_date =- parse_single_date(site.timezone, params)- |> Timex.end_of_year()-- start_date = Timex.beginning_of_year(end_date)-- struct!(query,- period: "year",- date_range: Date.range(start_date, end_date)- )- end-- defp put_period(query, site, %{"period" => "all"}) do- now = today(site.timezone)- start_date = Plausible.Sites.stats_start_date(site) || now-- struct!(query,- period: "all",- date_range: Date.range(start_date, now)- )- end-- defp put_period(query, site, %{"period" => "custom", "from" => from, "to" => to} = params) do- new_params =- params- |> Map.drop(["from", "to"])- |> Map.put("date", Enum.join([from, to], ","))-- put_period(query, site, new_params)- end-- defp put_period(query, _site, %{"period" => "custom", "date" => date}) do- [from, to] = String.split(date, ",")- from_date = Date.from_iso8601!(String.trim(from))- to_date = Date.from_iso8601!(String.trim(to))-- struct!(query,- period: "custom",- date_range: Date.range(from_date, to_date)- )- end-- defp put_period(query, site, params) do- put_period(query, site, Map.merge(params, %{"period" => "30d"}))- end-- defp put_dimensions(query, params) do- if not is_nil(params["property"]) do- struct!(query, dimensions: [params["property"]])- else- struct!(query, dimensions: Map.get(params, "dimensions", []))- end- end-- defp put_interval(%{:period => "all"} = query, params) do- interval = Map.get(params, "interval", Interval.default_for_date_range(query.date_range))- struct!(query, interval: interval)- end-- defp put_interval(query, params) do- interval = Map.get(params, "interval", Interval.default_for_period(query.period))- struct!(query, interval: interval)- end-- defp put_parsed_filters(query, params) do- struct!(query, filters: Filters.parse(params["filters"]))- end-def set(query, keywords) donew_query = struct!(query, keywords)@@ -224,26 +70,7 @@ defmodule Plausible.Stats.Query doendend- @spec set_dimensions(t(), list(String.t())) :: t()- def set_dimensions(query, dimensions) do- query- |> struct!(dimensions: dimensions)- |> refresh_imported_opts()- end-- def set_metrics(query, metrics) do- query- |> struct!(metrics: metrics)- |> refresh_imported_opts()- end-- def set_order_by(query, order_by) do- query- |> struct!(order_by: order_by)- |> refresh_imported_opts()- end-- def put_filter(query, filter) do+ def add_filter(query, filter) doquery|> struct!(filters: query.filters ++ [filter])|> refresh_imported_opts()@@ -260,13 +87,6 @@ defmodule Plausible.Stats.Query do|> refresh_imported_opts()end- def exclude_imported(query) do- struct!(query,- include_imported: false,- skip_imported_reason: :manual_exclusion- )- end-defp refresh_imported_opts(query) doput_imported_opts(query, nil, %{})end@@ -289,19 +109,7 @@ defmodule Plausible.Stats.Query doend)end- defp today(tz) do- Timex.now(tz) |> Timex.to_date()- end-- defp parse_single_date(tz, params) do- case params["date"] do- "today" -> Timex.now(tz) |> Timex.to_date()- date when is_binary(date) -> Date.from_iso8601!(date)- _ -> today(tz)- end- end-- defp put_imported_opts(query, site, params) do+ def put_imported_opts(query, site, params) dorequested? = params["with_imported"] == "true" || query.include.importslatest_import_end_date =diff --git a/lib/plausible/stats/sql/query_builder.ex b/lib/plausible/stats/sql/query_builder.exindex ef61a59758..aaf3eb8c9d 100644--- a/lib/plausible/stats/sql/query_builder.ex+++ b/lib/plausible/stats/sql/query_builder.ex@@ -32,7 +32,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder dofrom(e in "events_v2",where: ^SQL.WhereBuilder.build(:events, site, events_query),- select: ^Base.select_event_metrics(events_query.metrics)+ select: ^select_event_metrics(events_query))on_ee do@@ -73,7 +73,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder dofrom(e in "sessions_v2",where: ^SQL.WhereBuilder.build(:sessions, site, sessions_query),- select: ^Base.select_session_metrics(sessions_query.metrics, sessions_query)+ select: ^select_session_metrics(sessions_query))on_ee do@@ -111,6 +111,18 @@ defmodule Plausible.Stats.SQL.QueryBuilder doendend+ defp select_event_metrics(query) do+ query.metrics+ |> Enum.map(&SQL.Expression.event_metric/1)+ |> Enum.reduce(%{}, &Map.merge/2)+ end++ defp select_session_metrics(query) do+ query.metrics+ |> Enum.map(&SQL.Expression.session_metric(&1, query))+ |> Enum.reduce(%{}, &Map.merge/2)+ end+defp build_group_by(q, table, query) doEnum.reduce(query.dimensions, q, &dimension_group_by(&2, table, query, &1))enddiff --git a/lib/plausible/stats/sql/where_builder.ex b/lib/plausible/stats/sql/where_builder.exindex 73eeecaeef..5f882e0609 100644--- a/lib/plausible/stats/sql/where_builder.ex+++ b/lib/plausible/stats/sql/where_builder.ex@@ -4,7 +4,8 @@ defmodule Plausible.Stats.SQL.WhereBuilder do"""import Ecto.Query- import Plausible.Stats.Base, only: [page_regex: 1, utc_boundaries: 2]+ import Plausible.Stats.Time, only: [utc_boundaries: 2]+ import Plausible.Stats.Filters.Utils, only: [page_regex: 1]alias Plausible.Stats.Querydiff --git a/lib/plausible/stats/time.ex b/lib/plausible/stats/time.exindex c8c0d7a71f..83296bffaa 100644--- a/lib/plausible/stats/time.ex+++ b/lib/plausible/stats/time.ex@@ -4,6 +4,57 @@ defmodule Plausible.Stats.Time do"""alias Plausible.Stats.Query+ alias Plausible.Timezones++ def utc_boundaries(%Query{period: "realtime", now: now}, site) do+ last_datetime =+ now+ |> Timex.shift(seconds: 5)+ |> beginning_of_time(site.native_stats_start_at)+ |> NaiveDateTime.truncate(:second)++ first_datetime =+ now |> Timex.shift(minutes: -5) |> NaiveDateTime.truncate(:second)++ {first_datetime, last_datetime}+ end++ def utc_boundaries(%Query{period: "30m", now: now}, site) do+ last_datetime =+ now+ |> Timex.shift(seconds: 5)+ |> beginning_of_time(site.native_stats_start_at)+ |> NaiveDateTime.truncate(:second)++ first_datetime =+ now |> Timex.shift(minutes: -30) |> NaiveDateTime.truncate(:second)++ {first_datetime, last_datetime}+ end++ def utc_boundaries(%Query{date_range: date_range}, site) do+ {:ok, first} = NaiveDateTime.new(date_range.first, ~T[00:00:00])++ first_datetime =+ first+ |> Timezones.to_utc_datetime(site.timezone)+ |> beginning_of_time(site.native_stats_start_at)++ {:ok, last} = NaiveDateTime.new(date_range.last |> Timex.shift(days: 1), ~T[00:00:00])++ last_datetime = Timezones.to_utc_datetime(last, site.timezone)++ {first_datetime, last_datetime}+ end++ defp beginning_of_time(candidate, native_stats_start_at) do+ if Timex.after?(native_stats_start_at, candidate) do+ native_stats_start_at+ else+ candidate+ end+ end+def format_datetime(%Date{} = date), do: Date.to_string(date)def format_datetime(%DateTime{} = datetime),diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.exindex 5fc3938ed0..513debc597 100644--- a/lib/plausible/stats/timeseries.ex+++ b/lib/plausible/stats/timeseries.ex@@ -1,4 +1,10 @@defmodule Plausible.Stats.Timeseries do+ @moduledoc """+ Builds timeseries results for v1 of our stats API and dashboards.++ Avoid adding new logic here - update QueryBuilder etc instead.+ """+use Plausibleuse Plausible.ClickhouseRepoalias Plausible.Stats.{Query, QueryOptimizer, QueryResult, SQL}diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.exindex 248f709baf..e282ca97b4 100644--- a/lib/plausible_web/controllers/api/external_stats_controller.ex+++ b/lib/plausible_web/controllers/api/external_stats_controller.ex@@ -76,7 +76,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do{:error,"Property 'event:hostname' is currently not supported for breakdowns. Please provide a valid property for the breakdown endpoint: https://plausible.io/docs/stats-api#properties"}- Plausible.Stats.Props.valid_prop?(property) ->+ Plausible.Stats.Legacy.Dimensions.valid?(property) ->:oktrue ->@@ -363,7 +363,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController doenddefp validate_filter(_site, [_, property | _]) do- if Plausible.Stats.Props.valid_prop?(property) do+ if Plausible.Stats.Legacy.Dimensions.valid?(property) do:okelse{:error,diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex e5f01eda35..b3beef0e2b 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -777,7 +777,7 @@ defmodule PlausibleWeb.Api.StatsController doquery =Query.from(site, params)- |> Query.put_filter(referrer_filter)+ |> Query.add_filter(referrer_filter)pagination = parse_pagination(params)@@ -907,9 +907,9 @@ defmodule PlausibleWeb.Api.StatsController dototal_pageviews_query =query|> Query.remove_filters(["visit:exit_page"])- |> Query.put_filter([:is, "event:page", pages])- |> Query.put_filter([:is, "event:name", ["pageview"]])- |> Query.set_dimensions(["event:page"])+ |> Query.add_filter([:is, "event:page", pages])+ |> Query.add_filter([:is, "event:name", ["pageview"]])+ |> Query.set(dimensions: ["event:page"])total_pageviews =Stats.breakdown(site, total_pageviews_query, [:pageviews], {limit, 1})diff --git a/test/plausible/stats/legacy_dashboard_filter_parser_test.exs b/test/plausible/stats/legacy_dashboard_filter_parser_test.exsindex 8a3e94053a..ffdcc558a4 100644--- a/test/plausible/stats/legacy_dashboard_filter_parser_test.exs+++ b/test/plausible/stats/legacy_dashboard_filter_parser_test.exs@@ -1,4 +1,4 @@-defmodule Plausible.Stats.LegacyDashboardFilterParserTest do+defmodule Plausible.Stats.Legacy.DashboardFilterParserTest douse ExUnit.Case, async: truealias Plausible.Stats.Filters.LegacyDashboardFilterParserdiff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exsindex 62c919319f..cd6a606688 100644--- a/test/plausible/stats/query_test.exs+++ b/test/plausible/stats/query_test.exs@@ -21,11 +21,11 @@ defmodule Plausible.Stats.QueryTest do} doq1 = %{now: %NaiveDateTime{}} = Query.from(site, %{"period" => "realtime"})q2 = %{now: %NaiveDateTime{}} = Query.from(site, %{"period" => "30m"})- boundaries1 = Plausible.Stats.Base.utc_boundaries(q1, site)- boundaries2 = Plausible.Stats.Base.utc_boundaries(q2, site)+ boundaries1 = Plausible.Stats.Time.utc_boundaries(q1, site)+ boundaries2 = Plausible.Stats.Time.utc_boundaries(q2, site):timer.sleep(1500)- assert ^boundaries1 = Plausible.Stats.Base.utc_boundaries(q1, site)- assert ^boundaries2 = Plausible.Stats.Base.utc_boundaries(q2, site)+ assert ^boundaries1 = Plausible.Stats.Time.utc_boundaries(q1, site)+ assert ^boundaries2 = Plausible.Stats.Time.utc_boundaries(q2, site)endtest "parses day format", %{site: site} dodiff --git a/test/plausible_web/controllers/api/external_stats_controller/query_goal_dimension_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_goal_dimension_test.exsnew file mode 100644index 0000000000..643394e4f5--- /dev/null+++ b/test/plausible_web/controllers/api/external_stats_controller/query_goal_dimension_test.exs@@ -0,0 +1,133 @@+defmodule PlausibleWeb.Api.ExternalStatsController.QueryGoalDimensionTest do+ use PlausibleWeb.ConnCase++ @user_id 1231++ setup [:create_user, :create_new_site, :create_api_key, :use_api_key]++ describe "breakdown by event:goal" do+ test "returns custom event goals and pageview goals", %{conn: conn, site: site} do+ insert(:goal, %{site: site, event_name: "Purchase"})+ insert(:goal, %{site: site, page_path: "/test"})++ populate_stats(site, [+ build(:pageview,+ timestamp: ~N[2021-01-01 00:00:01],+ pathname: "/test"+ ),+ build(:event,+ name: "Purchase",+ timestamp: ~N[2021-01-01 00:00:03]+ ),+ build(:event,+ name: "Purchase",+ timestamp: ~N[2021-01-01 00:00:03]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors"],+ "dimensions" => ["event:goal"]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["Purchase"], "metrics" => [2]},+ %{"dimensions" => ["Visit /test"], "metrics" => [1]}+ ]+ end++ test "returns pageview goals containing wildcards", %{conn: conn, site: site} do+ insert(:goal, %{site: site, page_path: "/**/post"})+ insert(:goal, %{site: site, page_path: "/blog**"})++ populate_stats(site, [+ build(:pageview, pathname: "/blog", user_id: @user_id),+ build(:pageview, pathname: "/blog/post-1", user_id: @user_id),+ build(:pageview, pathname: "/blog/post-2", user_id: @user_id),+ build(:pageview, pathname: "/blog/something/post"),+ build(:pageview, pathname: "/different/page/post")+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors", "pageviews"],+ "dimensions" => ["event:goal"],+ "order_by" => [["pageviews", "desc"]]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["Visit /blog**"], "metrics" => [2, 4]},+ %{"dimensions" => ["Visit /**/post"], "metrics" => [2, 2]}+ ]+ end++ test "does not return goals that are not configured for the site", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, pathname: "/register"),+ build(:event, name: "Signup")+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors", "pageviews"],+ "dimensions" => ["event:goal"]+ })++ assert json_response(conn, 200)["results"] == []+ end++ test "returns conversion_rate in an event:goal breakdown", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event, name: "Signup", user_id: 1),+ build(:event, name: "Signup", user_id: 1),+ build(:pageview, pathname: "/blog"),+ build(:pageview, pathname: "/blog/post"),+ build(:pageview)+ ])++ insert(:goal, %{site: site, event_name: "Signup"})+ insert(:goal, %{site: site, page_path: "/blog**"})++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors", "events", "conversion_rate"],+ "dimensions" => ["event:goal"]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["Visit /blog**"], "metrics" => [2, 2, 50.0]},+ %{"dimensions" => ["Signup"], "metrics" => [1, 2, 25.0]}+ ]+ end++ test "returns conversion_rate alone in an event:goal breakdown", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event, name: "Signup", user_id: 1),+ build(:pageview)+ ])++ insert(:goal, %{site: site, event_name: "Signup"})++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["conversion_rate"],+ "dimensions" => ["event:goal"]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["Signup"], "metrics" => [50.0]}+ ]+ end+ end+enddiff --git a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exsnew file mode 100644index 0000000000..2e395a6acd--- /dev/null+++ b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs@@ -0,0 +1,695 @@+defmodule PlausibleWeb.Api.ExternalStatsController.QueryImportedTest do+ use PlausibleWeb.ConnCase++ @user_id 1231++ setup [:create_user, :create_new_site, :create_api_key, :use_api_key]++ describe "aggregation with imported data" do+ setup :create_site_import++ test "does not count imported stats unless specified", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:imported_visitors, date: ~D[2023-01-01]),+ build(:pageview, timestamp: ~N[2023-01-01 00:00:00])+ ])++ query_params = %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["pageviews"]+ }++ conn1 = post(conn, "/api/v2/query", query_params)++ assert json_response(conn1, 200)["results"] == [%{"metrics" => [1], "dimensions" => []}]++ conn2 = post(conn, "/api/v2/query", Map.put(query_params, "include", %{"imports" => true}))++ assert json_response(conn2, 200)["results"] == [%{"metrics" => [2], "dimensions" => []}]+ refute json_response(conn2, 200)["meta"]["warning"]+ end+ end++ test "breaks down all metrics by visit:referrer with imported data", %{conn: conn, site: site} do+ site_import =+ insert(:site_import,+ site: site,+ start_date: ~D[2005-01-01],+ end_date: Timex.today(),+ source: :universal_analytics+ )++ populate_stats(site, site_import.id, [+ build(:pageview, referrer: "site.com", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, referrer: "site.com/1", timestamp: ~N[2021-01-01 00:00:00]),+ build(:imported_sources,+ referrer: "site.com",+ date: ~D[2021-01-01],+ visitors: 2,+ visits: 2,+ pageviews: 2,+ bounces: 1,+ visit_duration: 120+ ),+ build(:imported_sources,+ referrer: "site.com/2",+ date: ~D[2021-01-01],+ visitors: 2,+ visits: 2,+ pageviews: 2,+ bounces: 2,+ visit_duration: 0+ ),+ build(:imported_sources,+ date: ~D[2021-01-01],+ visitors: 10,+ visits: 11,+ pageviews: 50,+ bounces: 0,+ visit_duration: 1100+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "visits", "pageviews", "bounce_rate", "visit_duration"],+ "date_range" => "all",+ "dimensions" => ["visit:referrer"],+ "include" => %{"imports" => true}+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["Direct / None"], "metrics" => [10, 11, 50, 0.0, 100.0]},+ %{"dimensions" => ["site.com"], "metrics" => [3, 3, 3, 67.0, 40.0]},+ %{"dimensions" => ["site.com/2"], "metrics" => [2, 2, 2, 100.0, 0.0]},+ %{"dimensions" => ["site.com/1"], "metrics" => [1, 1, 1, 100.0, 0.0]}+ ]+ end++ test "breaks down all metrics by visit:utm_source with imported data", %{conn: conn, site: site} do+ site_import =+ insert(:site_import,+ site: site,+ start_date: ~D[2005-01-01],+ end_date: Timex.today(),+ source: :universal_analytics+ )++ populate_stats(site, site_import.id, [+ build(:pageview, utm_source: "SomeUTMSource", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, utm_source: "SomeUTMSource-1", timestamp: ~N[2021-01-01 00:00:00]),+ build(:imported_sources,+ utm_source: "SomeUTMSource",+ date: ~D[2021-01-01],+ visitors: 2,+ visits: 2,+ pageviews: 2,+ bounces: 1,+ visit_duration: 120+ ),+ build(:imported_sources,+ utm_source: "SomeUTMSource-2",+ date: ~D[2021-01-01],+ visitors: 2,+ visits: 2,+ pageviews: 2,+ bounces: 2,+ visit_duration: 0+ ),+ build(:imported_sources,+ date: ~D[2021-01-01],+ visitors: 10,+ visits: 11,+ pageviews: 50,+ bounces: 0,+ visit_duration: 1100+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "visits", "pageviews", "bounce_rate", "visit_duration"],+ "date_range" => "all",+ "dimensions" => ["visit:utm_source"],+ "include" => %{"imports" => true}+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["SomeUTMSource"], "metrics" => [3, 3, 3, 67.0, 40.0]},+ %{"dimensions" => ["SomeUTMSource-2"], "metrics" => [2, 2, 2, 100.0, 0.0]},+ %{"dimensions" => ["SomeUTMSource-1"], "metrics" => [1, 1, 1, 100.0, 0.0]}+ ]+ end++ test "pageviews breakdown by event:page - imported data having pageviews=0 and visitors=n should be bypassed",+ %{conn: conn, site: site} do+ site_import =+ insert(:site_import,+ site: site,+ start_date: ~D[2005-01-01],+ end_date: Timex.today(),+ source: :universal_analytics+ )++ populate_stats(site, site_import.id, [+ build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),+ build(:pageview,+ pathname: "/plausible.io",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:imported_pages,+ page: "/skip-me",+ date: ~D[2021-01-01],+ visitors: 1,+ pageviews: 0+ ),+ build(:imported_pages,+ page: "/include-me",+ date: ~D[2021-01-01],+ visitors: 1,+ pageviews: 1+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["pageviews"],+ "date_range" => "all",+ "dimensions" => ["event:page"],+ "include" => %{"imports" => true}+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["/"], "metrics" => [2]},+ %{"dimensions" => ["/plausible.io"], "metrics" => [1]},+ %{"dimensions" => ["/include-me"], "metrics" => [1]}+ ]+ end++ describe "breakdown by visit:exit_page with" do+ setup %{site: site} do+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:pageview,+ pathname: "/a",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ user_id: @user_id,+ pathname: "/a",+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:pageview,+ user_id: @user_id,+ pathname: "/b",+ timestamp: ~N[2021-01-01 00:35:00]+ ),+ build(:imported_exit_pages,+ exit_page: "/b",+ exits: 3,+ visitors: 2,+ pageviews: 5,+ date: ~D[2021-01-01]+ )+ ])+ end++ test "can query with visit:exit_page dimension", %{conn: conn, site: site} do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visits"],+ "date_range" => "all",+ "dimensions" => ["visit:exit_page"],+ "include" => %{"imports" => true}+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["/b"], "metrics" => [4]},+ %{"dimensions" => ["/a"], "metrics" => [1]}+ ]+ end+ end++ describe "imported data" do+ test "returns screen sizes breakdown when filtering by screen size", %{conn: conn, site: site} do+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:pageview,+ timestamp: ~N[2021-01-01 00:00:01],+ screen_size: "Mobile"+ ),+ build(:imported_devices,+ device: "Mobile",+ visitors: 3,+ pageviews: 5,+ date: ~D[2021-01-01]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "pageviews"],+ "date_range" => "all",+ "dimensions" => ["visit:device"],+ "filters" => [+ ["is", "visit:device", ["Mobile"]]+ ],+ "include" => %{"imports" => true}+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [%{"dimensions" => ["Mobile"], "metrics" => [4, 6]}]+ end++ test "returns custom event goals and pageview goals", %{conn: conn, site: site} do+ insert(:goal, site: site, event_name: "Purchase")+ insert(:goal, site: site, page_path: "/test")++ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:pageview,+ timestamp: ~N[2021-01-01 00:00:01],+ pathname: "/test"+ ),+ build(:event,+ name: "Purchase",+ timestamp: ~N[2021-01-01 00:00:03]+ ),+ build(:event,+ name: "Purchase",+ timestamp: ~N[2021-01-01 00:00:03]+ ),+ build(:imported_custom_events,+ name: "Purchase",+ visitors: 3,+ events: 5,+ date: ~D[2021-01-01]+ ),+ build(:imported_pages,+ page: "/test",+ visitors: 2,+ pageviews: 2,+ date: ~D[2021-01-01]+ ),+ build(:imported_visitors, visitors: 5, date: ~D[2021-01-01])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "dimensions" => ["event:goal"],+ "metrics" => ["visitors", "events", "pageviews", "conversion_rate"],+ "include" => %{"imports" => true}+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["Purchase"], "metrics" => [5, 7, 0, 62.5]},+ %{"dimensions" => ["Visit /test"], "metrics" => [3, 3, 3, 37.5]}+ ]+ end++ test "pageviews are returned as events for breakdown reports other than custom events", %{+ conn: conn,+ site: site+ } do+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:imported_browsers, browser: "Chrome", pageviews: 1, date: ~D[2021-01-01]),+ build(:imported_devices, device: "Desktop", pageviews: 1, date: ~D[2021-01-01]),+ build(:imported_entry_pages, entry_page: "/test", pageviews: 1, date: ~D[2021-01-01]),+ build(:imported_exit_pages, exit_page: "/test", pageviews: 1, date: ~D[2021-01-01]),+ build(:imported_locations, country: "EE", pageviews: 1, date: ~D[2021-01-01]),+ build(:imported_operating_systems,+ operating_system: "Mac",+ pageviews: 1,+ date: ~D[2021-01-01]+ ),+ build(:imported_pages, page: "/test", pageviews: 1, date: ~D[2021-01-01]),+ build(:imported_sources, source: "Google", pageviews: 1, date: ~D[2021-01-01])+ ])++ breakdown_and_first = fn dimension ->+ conn+ |> post("/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["events"],+ "date_range" => ["2021-01-01", "2021-01-01"],+ "dimensions" => [dimension],+ "include" => %{"imports" => true}+ })+ |> json_response(200)+ |> Map.get("results")+ |> List.first()+ end++ assert %{"dimensions" => ["Chrome"], "metrics" => [1]} =+ breakdown_and_first.("visit:browser")++ assert %{"dimensions" => ["Desktop"], "metrics" => [1]} =+ breakdown_and_first.("visit:device")++ assert %{"dimensions" => ["EE"], "metrics" => [1]} = breakdown_and_first.("visit:country")+ assert %{"dimensions" => ["Mac"], "metrics" => [1]} = breakdown_and_first.("visit:os")+ assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("event:page")++ assert %{"dimensions" => ["Google"], "metrics" => [1]} =+ breakdown_and_first.("visit:source")+ end++ for goal_name <- Plausible.Imported.goals_with_url() do+ test "returns url breakdown for #{goal_name} goal", %{conn: conn, site: site} do+ insert(:goal, event_name: unquote(goal_name), site: site)+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:event,+ name: unquote(goal_name),+ "meta.key": ["url"],+ "meta.value": ["https://one.com"]+ ),+ build(:imported_custom_events,+ name: unquote(goal_name),+ visitors: 2,+ events: 5,+ link_url: "https://one.com"+ ),+ build(:imported_custom_events,+ name: unquote(goal_name),+ visitors: 5,+ events: 10,+ link_url: "https://two.com"+ ),+ build(:imported_custom_events,+ name: "some goal",+ visitors: 5,+ events: 10+ ),+ build(:imported_visitors, visitors: 9)+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "events", "conversion_rate"],+ "date_range" => "all",+ "dimensions" => ["event:props:url"],+ "filters" => [+ ["is", "event:goal", [unquote(goal_name)]]+ ],+ "include" => %{"imports" => true}+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["https://two.com"], "metrics" => [5, 10, 50]},+ %{"dimensions" => ["https://one.com"], "metrics" => [3, 6, 30]}+ ]++ refute json_response(conn, 200)["meta"]["warning"]+ end+ end++ for goal_name <- Plausible.Imported.goals_with_path() do+ test "returns path breakdown for #{goal_name} goal", %{conn: conn, site: site} do+ insert(:goal, event_name: unquote(goal_name), site: site)+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:event,+ name: unquote(goal_name),+ "meta.key": ["path"],+ "meta.value": ["/one"]+ ),+ build(:imported_custom_events,+ name: unquote(goal_name),+ visitors: 2,+ events: 5,+ path: "/one"+ ),+ build(:imported_custom_events,+ name: unquote(goal_name),+ visitors: 5,+ events: 10,+ path: "/two"+ ),+ build(:imported_custom_events,+ name: "some goal",+ visitors: 5,+ events: 10+ ),+ build(:imported_visitors, visitors: 9)+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "events", "conversion_rate"],+ "date_range" => "all",+ "dimensions" => ["event:props:path"],+ "filters" => [+ ["is", "event:goal", [unquote(goal_name)]]+ ],+ "include" => %{"imports" => true}+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["/two"], "metrics" => [5, 10, 50]},+ %{"dimensions" => ["/one"], "metrics" => [3, 6, 30]}+ ]++ refute json_response(conn, 200)["meta"]["warning"]+ end+ end++ test "adds a warning when query params are not supported for imported data", %{+ conn: conn,+ site: site+ } do+ site_import = insert(:site_import, site: site)++ insert(:goal, event_name: "Signup", site: site)++ populate_stats(site, site_import.id, [+ build(:event,+ name: "Signup",+ "meta.key": ["package"],+ "meta.value": ["large"]+ ),+ build(:imported_visitors, visitors: 9)+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["event:props:package"],+ "filters" => [+ ["is", "event:goal", ["Signup"]]+ ],+ "include" => %{"imports" => true}+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["large"], "metrics" => [1]}+ ]++ assert json_response(conn, 200)["meta"]["warning"] =~+ "Imported stats are not included in the results because query parameters are not supported."+ end++ test "does not add a warning when there are no site imports", %{conn: conn, site: site} do+ insert(:goal, event_name: "Signup", site: site)++ populate_stats(site, [+ build(:event,+ name: "Signup",+ "meta.key": ["package"],+ "meta.value": ["large"]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["event:props:package"],+ "filters" => [+ ["is", "event:goal", ["Signup"]]+ ],+ "include" => %{"imports" => true}+ })++ refute json_response(conn, 200)["meta"]["warning"]+ end++ test "does not add a warning when import is out of queried date range", %{+ conn: conn,+ site: site+ } do+ site_import = insert(:site_import, site: site, end_date: Date.add(Date.utc_today(), -3))++ insert(:goal, event_name: "Signup", site: site)++ populate_stats(site, site_import.id, [+ build(:event,+ name: "Signup",+ "meta.key": ["package"],+ "meta.value": ["large"]+ ),+ build(:imported_visitors, visitors: 9)+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "day",+ "dimensions" => ["event:props:package"],+ "filters" => [+ ["is", "event:goal", ["Signup"]]+ ],+ "include" => %{"imports" => true}+ })++ refute json_response(conn, 200)["meta"]["warning"]+ end++ test "applies multiple filters if the properties belong to the same table", %{+ conn: conn,+ site: site+ } do+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:imported_sources, source: "Google", utm_medium: "organic", utm_term: "one"),+ build(:imported_sources, source: "Twitter", utm_medium: "organic", utm_term: "two"),+ build(:imported_sources,+ source: "Facebook",+ utm_medium: "something_else",+ utm_term: "one"+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "day",+ "dimensions" => ["visit:source"],+ "filters" => [+ ["is", "visit:utm_medium", ["organic"]],+ ["is", "visit:utm_term", ["one"]]+ ],+ "include" => %{"imports" => true}+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["Google"], "metrics" => [1]}+ ]+ end++ test "ignores imported data if filtered property belongs to a different table than the breakdown property",+ %{+ conn: conn,+ site: site+ } do+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:imported_sources, source: "Google"),+ build(:imported_devices, device: "Desktop")+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "day",+ "dimensions" => ["visit:source"],+ "filters" => [+ ["is", "visit:device", ["Desktop"]]+ ],+ "include" => %{"imports" => true}+ })++ assert %{+ "results" => [],+ "meta" => meta+ } = json_response(conn, 200)++ assert meta["warning"] =~ "Imported stats are not included in the results"+ end++ test "imported country, region and city data",+ %{+ conn: conn,+ site: site+ } do+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:pageview,+ timestamp: ~N[2021-01-01 00:15:00],+ country_code: "DE",+ subdivision1_code: "DE-BE",+ city_geoname_id: 2_950_159+ ),+ build(:pageview,+ timestamp: ~N[2021-01-01 00:15:00],+ country_code: "DE",+ subdivision1_code: "DE-BE",+ city_geoname_id: 2_950_159+ ),+ build(:pageview,+ timestamp: ~N[2021-01-01 00:15:00],+ country_code: "EE",+ subdivision1_code: "EE-37",+ city_geoname_id: 588_409+ ),+ build(:imported_locations, country: "EE", region: "EE-37", city: 588_409, visitors: 33)+ ])++ for {dimension, stats_value, imports_value} <- [+ {"visit:country", "DE", "EE"},+ {"visit:region", "DE-BE", "EE-37"},+ {"visit:city", 2_950_159, 588_409}+ ] do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => [dimension],+ "include" => %{"imports" => true}+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => [imports_value], "metrics" => [34]},+ %{"dimensions" => [stats_value], "metrics" => [2]}+ ]+ end+ end+ end+enddiff --git a/test/plausible_web/controllers/api/external_stats_controller/query_special_metrics_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_special_metrics_test.exsnew file mode 100644index 0000000000..de403aa6b2--- /dev/null+++ b/test/plausible_web/controllers/api/external_stats_controller/query_special_metrics_test.exs@@ -0,0 +1,193 @@+defmodule PlausibleWeb.Api.ExternalStatsController.QuerySpecialMetricsTest do+ use PlausibleWeb.ConnCase++ setup [:create_user, :create_new_site, :create_api_key, :use_api_key]++ test "returns conversion_rate in a goal filtered custom prop breakdown", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]),+ build(:pageview, pathname: "/blog/2", "meta.key": ["author"], "meta.value": ["Uku"]),+ build(:pageview, pathname: "/blog/3", "meta.key": ["author"], "meta.value": ["Uku"]),+ build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Marko"]),+ build(:pageview,+ pathname: "/blog/2",+ "meta.key": ["author"],+ "meta.value": ["Marko"],+ user_id: 1+ ),+ build(:pageview,+ pathname: "/blog/3",+ "meta.key": ["author"],+ "meta.value": ["Marko"],+ user_id: 1+ ),+ build(:pageview, pathname: "/blog"),+ build(:pageview, "meta.key": ["author"], "meta.value": ["Marko"]),+ build(:pageview)+ ])++ insert(:goal, %{site: site, page_path: "/blog**"})++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "filters" => [["matches", "event:goal", ["Visit /blog**"]]],+ "metrics" => ["visitors", "events", "conversion_rate"],+ "dimensions" => ["event:props:author"]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["Uku"], "metrics" => [3, 3, 37.5]},+ %{"dimensions" => ["Marko"], "metrics" => [2, 3, 25.0]},+ %{"dimensions" => ["(none)"], "metrics" => [1, 1, 12.5]}+ ]+ end++ test "returns conversion_rate alone in a goal filtered custom prop breakdown", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]),+ build(:pageview)+ ])++ insert(:goal, %{site: site, page_path: "/blog**"})++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["conversion_rate"],+ "date_range" => "all",+ "dimensions" => ["event:props:author"],+ "filters" => [["matches", "event:goal", ["Visit /blog**"]]]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["Uku"], "metrics" => [50]}+ ]+ end++ test "returns conversion_rate in a goal filtered event:page breakdown", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:event, pathname: "/en/register", name: "pageview"),+ build(:event, pathname: "/en/register", name: "Signup"),+ build(:event, pathname: "/en/register", name: "Signup"),+ build(:event, pathname: "/it/register", name: "Signup", user_id: 1),+ build(:event, pathname: "/it/register", name: "Signup", user_id: 1),+ build(:event, pathname: "/it/register", name: "pageview")+ ])++ insert(:goal, %{site: site, event_name: "Signup"})++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "dimensions" => ["event:page"],+ "filters" => [["is", "event:goal", ["Signup"]]],+ "metrics" => ["visitors", "events", "group_conversion_rate"]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["/en/register"], "metrics" => [2, 2, 66.7]},+ %{"dimensions" => ["/it/register"], "metrics" => [1, 2, 50.0]}+ ]+ end++ test "returns conversion_rate alone in a goal filtered event:page breakdown", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:event, pathname: "/en/register", name: "pageview"),+ build(:event, pathname: "/en/register", name: "Signup")+ ])++ insert(:goal, %{site: site, event_name: "Signup"})++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "filters" => [["is", "event:goal", ["Signup"]]],+ "metrics" => ["group_conversion_rate"],+ "dimensions" => ["event:page"]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["/en/register"], "metrics" => [50.0]}+ ]+ end++ test "returns conversion_rate in a multi-goal filtered visit:screen_size breakdown", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:event, screen_size: "Mobile", name: "pageview"),+ build(:event, screen_size: "Mobile", name: "AddToCart"),+ build(:event, screen_size: "Mobile", name: "AddToCart"),+ build(:event, screen_size: "Desktop", name: "AddToCart", user_id: 1),+ build(:event, screen_size: "Desktop", name: "Purchase", user_id: 1),+ build(:event, screen_size: "Desktop", name: "pageview")+ ])++ # Make sure that revenue goals are treated the same+ # way as regular custom event goals+ insert(:goal, %{site: site, event_name: "Purchase", currency: :EUR})+ insert(:goal, %{site: site, event_name: "AddToCart"})++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "events", "group_conversion_rate"],+ "date_range" => "all",+ "dimensions" => ["visit:device"],+ "filters" => [["is", "event:goal", ["AddToCart", "Purchase"]]]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["Mobile"], "metrics" => [2, 2, 66.7]},+ %{"dimensions" => ["Desktop"], "metrics" => [1, 2, 50]}+ ]+ end++ test "returns conversion_rate alone in a goal filtered visit:screen_size breakdown", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:event, screen_size: "Mobile", name: "pageview"),+ build(:event, screen_size: "Mobile", name: "AddToCart")+ ])++ insert(:goal, %{site: site, event_name: "AddToCart"})++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["conversion_rate"],+ "date_range" => "all",+ "dimensions" => ["visit:device"],+ "filters" => [["is", "event:goal", ["AddToCart"]]]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["Mobile"], "metrics" => [50]}+ ]+ end+enddiff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exsindex 9454196398..8bcd82a1ab 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs@@ -1,239 +1,10 @@defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest douse PlausibleWeb.ConnCase- alias Plausible.Billing.Feature@user_id 1231setup [:create_user, :create_new_site, :create_api_key, :use_api_key]- describe "feature access" do- test "cannot break down by a custom prop without access to the props feature", %{- conn: conn,- user: user,- site: site- } do- ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)- insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["visitors"],- "date_range" => "all",- "dimensions" => ["event:props:author"]- })-- assert json_response(conn, 400)["error"] ==- "The owner of this site does not have access to the custom properties feature"- end-- test "can break down by an internal prop key without access to the props feature", %{- conn: conn,- user: user,- site: site- } do- ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)- insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["visitors"],- "date_range" => "all",- "dimensions" => ["event:props:path"]- })-- assert json_response(conn, 200)["results"]- end-- test "cannot filter by a custom prop without access to the props feature", %{- conn: conn,- user: user,- site: site- } do- ep =- insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)-- insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["visitors"],- "date_range" => "all",- "dimensions" => ["visit:source"],- "filters" => [["is", "event:props:author", ["Uku"]]]- })-- assert json_response(conn, 400)["error"] ==- "The owner of this site does not have access to the custom properties feature"- end-- test "can filter by an internal prop key without access to the props feature", %{- conn: conn,- user: user,- site: site- } do- ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)- insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["visitors"],- "date_range" => "all",- "dimensions" => ["visit:source"],- "filters" => [["is", "event:props:url", ["whatever"]]]- })-- assert json_response(conn, 200)["results"]- end- end-- describe "param validation" do- test "does not allow querying conversion_rate without a goal filter", %{- conn: conn,- site: site- } do- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["conversion_rate"],- "date_range" => "all",- "dimensions" => ["event:page"],- "filters" => [["is", "event:props:author", ["Uku"]]]- })-- assert json_response(conn, 400)["error"] ==- "Metric `conversion_rate` can only be queried with event:goal filters or dimensions"- end-- test "validates that dimensions are valid", %{conn: conn, site: site} do- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["visitors"],- "date_range" => "all",- "dimensions" => ["badproperty"]- })-- assert json_response(conn, 400)["error"] =~ "Invalid dimensions"- end-- test "empty custom property is invalid", %{conn: conn, site: site} do- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["visitors"],- "date_range" => "all",- "dimensions" => ["event:props:"]- })-- assert json_response(conn, 400)["error"] =~ "Invalid dimensions"- end-- test "validates that correct date range is used", %{conn: conn, site: site} do- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["visitors"],- "date_range" => "bad_period",- "dimensions" => ["event:name"]- })-- assert json_response(conn, 400)["error"] =~ "Invalid date_range"- end-- test "fails when an invalid metric is provided", %{conn: conn, site: site} do- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["visitors", "baa"],- "date_range" => "all",- "dimensions" => ["event:name"]- })-- assert json_response(conn, 400)["error"] =~ "Unknown metric '\"baa\"'"- end-- test "session metrics cannot be used with event:name dimension", %{conn: conn, site: site} do- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["visitors", "bounce_rate"],- "date_range" => "all",- "dimensions" => ["event:name"]- })-- assert json_response(conn, 400)["error"] =~- "Session metric(s) `bounce_rate` cannot be queried along with event dimensions"- end-- test "session metrics cannot be used with event:props:* dimension", %{conn: conn, site: site} do- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["visitors", "bounce_rate"],- "date_range" => "all",- "dimensions" => ["event:props:url"]- })-- assert json_response(conn, 400)["error"] =~- "Session metric(s) `bounce_rate` cannot be queried along with event dimensions"- end-- test "validates that metric views_per_visit cannot be used with event:page filter", %{- conn: conn,- site: site- } do- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "date_range" => "all",- "metrics" => ["views_per_visit"],- "filters" => [["is", "event:page", ["/something"]]]- })-- assert json_response(conn, 400) == %{- "error" =>- "Metric `views_per_visit` cannot be queried with a filter on `event:page`"- }- end-- test "validates that metric views_per_visit cannot be used together with dimensions", %{- conn: conn,- site: site- } do- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "date_range" => "all",- "metrics" => ["views_per_visit"],- "dimensions" => ["event:name"]- })-- assert json_response(conn, 400) == %{- "error" => "Metric `views_per_visit` cannot be queried with `dimensions`"- }- end-- test "validates a metric can't be asked multiple times", %{- conn: conn,- site: site- } do- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "date_range" => "all",- "metrics" => ["views_per_visit", "visitors", "visitors"]- })-- assert json_response(conn, 400) == %{- "error" => "Metrics cannot be queried multiple times"- }- end- end-test "aggregates a single metric", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),@@ -1065,36 +836,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doendend- describe "aggregation with imported data" do- setup :create_site_import-- test "does not count imported stats unless specified", %{- conn: conn,- site: site,- site_import: site_import- } do- populate_stats(site, site_import.id, [- build(:imported_visitors, date: ~D[2023-01-01]),- build(:pageview, timestamp: ~N[2023-01-01 00:00:00])- ])-- query_params = %{- "site_id" => site.domain,- "date_range" => "all",- "metrics" => ["pageviews"]- }-- conn1 = post(conn, "/api/v2/query", query_params)-- assert json_response(conn1, 200)["results"] == [%{"metrics" => [1], "dimensions" => []}]-- conn2 = post(conn, "/api/v2/query", Map.put(query_params, "include", %{"imports" => true}))-- assert json_response(conn2, 200)["results"] == [%{"metrics" => [2], "dimensions" => []}]- refute json_response(conn2, 200)["meta"]["warning"]- end- end-describe "timeseries" dotest "shows hourly data for a certain date with time_labels", %{conn: conn, site: site} dopopulate_stats(site, [@@ -1405,63 +1146,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do]end- test "breaks down all metrics by visit:referrer with imported data", %{conn: conn, site: site} do- site_import =- insert(:site_import,- site: site,- start_date: ~D[2005-01-01],- end_date: Timex.today(),- source: :universal_analytics- )-- populate_stats(site, site_import.id, [- build(:pageview, referrer: "site.com", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, referrer: "site.com/1", timestamp: ~N[2021-01-01 00:00:00]),- build(:imported_sources,- referrer: "site.com",- date: ~D[2021-01-01],- visitors: 2,- visits: 2,- pageviews: 2,- bounces: 1,- visit_duration: 120- ),- build(:imported_sources,- referrer: "site.com/2",- date: ~D[2021-01-01],- visitors: 2,- visits: 2,- pageviews: 2,- bounces: 2,- visit_duration: 0- ),- build(:imported_sources,- date: ~D[2021-01-01],- visitors: 10,- visits: 11,- pageviews: 50,- bounces: 0,- visit_duration: 1100- )- ])-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["visitors", "visits", "pageviews", "bounce_rate", "visit_duration"],- "date_range" => "all",- "dimensions" => ["visit:referrer"],- "include" => %{"imports" => true}- })-- assert json_response(conn, 200)["results"] == [- %{"dimensions" => ["Direct / None"], "metrics" => [10, 11, 50, 0.0, 100.0]},- %{"dimensions" => ["site.com"], "metrics" => [3, 3, 3, 67.0, 40.0]},- %{"dimensions" => ["site.com/2"], "metrics" => [2, 2, 2, 100.0, 0.0]},- %{"dimensions" => ["site.com/1"], "metrics" => [1, 1, 1, 100.0, 0.0]}- ]- end-for {dimension, attr} <- [{"visit:utm_campaign", :utm_campaign},{"visit:utm_source", :utm_source},@@ -1657,111 +1341,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do]end- test "breaks down all metrics by visit:utm_source with imported data", %{conn: conn, site: site} do- site_import =- insert(:site_import,- site: site,- start_date: ~D[2005-01-01],- end_date: Timex.today(),- source: :universal_analytics- )-- populate_stats(site, site_import.id, [- build(:pageview, utm_source: "SomeUTMSource", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, utm_source: "SomeUTMSource-1", timestamp: ~N[2021-01-01 00:00:00]),- build(:imported_sources,- utm_source: "SomeUTMSource",- date: ~D[2021-01-01],- visitors: 2,- visits: 2,- pageviews: 2,- bounces: 1,- visit_duration: 120- ),- build(:imported_sources,- utm_source: "SomeUTMSource-2",- date: ~D[2021-01-01],- visitors: 2,- visits: 2,- pageviews: 2,- bounces: 2,- visit_duration: 0- ),- build(:imported_sources,- date: ~D[2021-01-01],- visitors: 10,- visits: 11,- pageviews: 50,- bounces: 0,- visit_duration: 1100- )- ])-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["visitors", "visits", "pageviews", "bounce_rate", "visit_duration"],- "date_range" => "all",- "dimensions" => ["visit:utm_source"],- "include" => %{"imports" => true}- })-- %{"results" => results} = json_response(conn, 200)-- assert results == [- %{"dimensions" => ["SomeUTMSource"], "metrics" => [3, 3, 3, 67.0, 40.0]},- %{"dimensions" => ["SomeUTMSource-2"], "metrics" => [2, 2, 2, 100.0, 0.0]},- %{"dimensions" => ["SomeUTMSource-1"], "metrics" => [1, 1, 1, 100.0, 0.0]}- ]- end-- test "pageviews breakdown by event:page - imported data having pageviews=0 and visitors=n should be bypassed",- %{conn: conn, site: site} do- site_import =- insert(:site_import,- site: site,- start_date: ~D[2005-01-01],- end_date: Timex.today(),- source: :universal_analytics- )-- populate_stats(site, site_import.id, [- build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),- build(:pageview,- pathname: "/plausible.io",- timestamp: ~N[2021-01-01 00:00:00]- ),- build(:imported_pages,- page: "/skip-me",- date: ~D[2021-01-01],- visitors: 1,- pageviews: 0- ),- build(:imported_pages,- page: "/include-me",- date: ~D[2021-01-01],- visitors: 1,- pageviews: 1- )- ])-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["pageviews"],- "date_range" => "all",- "dimensions" => ["event:page"],- "include" => %{"imports" => true}- })-- assert json_response(conn, 200)["results"] == [- %{"dimensions" => ["/"], "metrics" => [2]},- %{"dimensions" => ["/plausible.io"], "metrics" => [1]},- %{"dimensions" => ["/include-me"], "metrics" => [1]}- ]- end-test "breakdown by event:page", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),@@ -1811,87 +1390,39 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do]end- describe "breakdown by visit:exit_page" do- setup %{site: site} do- site_import = insert(:site_import, site: site)-- populate_stats(site, site_import.id, [- build(:pageview,- pathname: "/a",+ describe "custom events" do+ test "can breakdown by event:name", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event,+ name: "Signup",timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview,- user_id: @user_id,- pathname: "/a",- timestamp: ~N[2021-01-01 00:25:00]+ build(:event,+ name: "Signup",+ timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,- user_id: @user_id,- pathname: "/b",- timestamp: ~N[2021-01-01 00:35:00]- ),- build(:imported_exit_pages,- exit_page: "/b",- exits: 3,- visitors: 2,- pageviews: 5,- date: ~D[2021-01-01]+ timestamp: ~N[2021-01-01 00:25:00])])- end- test "can query with visit:exit_page dimension", %{conn: conn, site: site} doconn =post(conn, "/api/v2/query", %{"site_id" => site.domain,- "metrics" => ["visits"],+ "metrics" => ["visitors"],"date_range" => "all",- "dimensions" => ["visit:exit_page"],- "include" => %{"imports" => true}+ "dimensions" => ["event:name"]})%{"results" => results} = json_response(conn, 200)assert results == [- %{"dimensions" => ["/b"], "metrics" => [4]},- %{"dimensions" => ["/a"], "metrics" => [1]}+ %{"dimensions" => ["Signup"], "metrics" => [2]},+ %{"dimensions" => ["pageview"], "metrics" => [1]}]end- end- describe "custom events" do- test "can breakdown by event:name", %{conn: conn, site: site} do- populate_stats(site, [- build(:event,- name: "Signup",- timestamp: ~N[2021-01-01 00:00:00]- ),- build(:event,- name: "Signup",- timestamp: ~N[2021-01-01 00:00:00]- ),- build(:pageview,- timestamp: ~N[2021-01-01 00:25:00]- )- ])-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["visitors"],- "date_range" => "all",- "dimensions" => ["event:name"]- })-- %{"results" => results} = json_response(conn, 200)-- assert results == [- %{"dimensions" => ["Signup"], "metrics" => [2]},- %{"dimensions" => ["pageview"], "metrics" => [1]}- ]- end-- test "can breakdown by event:name with visitors and events metrics", %{conn: conn, site: site} do+ test "can breakdown by event:name with visitors and events metrics", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview,pathname: "/non-existing",@@ -2248,8 +1779,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do"site_id" => site.domain,"metrics" => ["pageviews"],"date_range" => "all",- "dimensions" => ["event:props:package"],- "filters" => []+ "dimensions" => ["event:props:package"]})%{"results" => results} = json_response(conn, 200)@@ -2317,85 +1847,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doendend- describe "breakdown by event:goal" do- test "returns custom event goals and pageview goals", %{conn: conn, site: site} do- insert(:goal, %{site: site, event_name: "Purchase"})- insert(:goal, %{site: site, page_path: "/test"})-- populate_stats(site, [- build(:pageview,- timestamp: ~N[2021-01-01 00:00:01],- pathname: "/test"- ),- build(:event,- name: "Purchase",- timestamp: ~N[2021-01-01 00:00:03]- ),- build(:event,- name: "Purchase",- timestamp: ~N[2021-01-01 00:00:03]- )- ])-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "date_range" => "all",- "metrics" => ["visitors"],- "dimensions" => ["event:goal"]- })-- assert json_response(conn, 200)["results"] == [- %{"dimensions" => ["Purchase"], "metrics" => [2]},- %{"dimensions" => ["Visit /test"], "metrics" => [1]}- ]- end-- test "returns pageview goals containing wildcards", %{conn: conn, site: site} do- insert(:goal, %{site: site, page_path: "/**/post"})- insert(:goal, %{site: site, page_path: "/blog**"})-- populate_stats(site, [- build(:pageview, pathname: "/blog", user_id: @user_id),- build(:pageview, pathname: "/blog/post-1", user_id: @user_id),- build(:pageview, pathname: "/blog/post-2", user_id: @user_id),- build(:pageview, pathname: "/blog/something/post"),- build(:pageview, pathname: "/different/page/post")- ])-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "date_range" => "all",- "metrics" => ["visitors", "pageviews"],- "dimensions" => ["event:goal"],- "order_by" => [["pageviews", "desc"]]- })-- assert json_response(conn, 200)["results"] == [- %{"dimensions" => ["Visit /blog**"], "metrics" => [2, 4]},- %{"dimensions" => ["Visit /**/post"], "metrics" => [2, 2]}- ]- end-- test "does not return goals that are not configured for the site", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview, pathname: "/register"),- build(:event, name: "Signup")- ])-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "date_range" => "all",- "metrics" => ["visitors", "pageviews"],- "dimensions" => ["event:goal"]- })-- assert json_response(conn, 200)["results"] == []- end- end-test "event:goal filter returns 400 when goal not configured", %{conn: conn, site: site} doconn =post(conn, "/api/v2/query", %{@@ -2802,21 +2253,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do)])- conn =- get(conn, "/api/v1/stats/breakdown", %{- "site_id" => site.domain,- "period" => "day",- "property" => "event:page",- "filters" => "event:goal == Register"- })-- assert json_response(conn, 200) == %{- "results" => [- %{"page" => "/en/register", "visitors" => 2},- %{"page" => "/it/register", "visitors" => 1}- ]- }-conn =post(conn, "/api/v2/query", %{"site_id" => site.domain,@@ -3159,241 +2595,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doenddescribe "metrics" do- test "returns conversion_rate in an event:goal breakdown", %{conn: conn, site: site} do- populate_stats(site, [- build(:event, name: "Signup", user_id: 1),- build(:event, name: "Signup", user_id: 1),- build(:pageview, pathname: "/blog"),- build(:pageview, pathname: "/blog/post"),- build(:pageview)- ])-- insert(:goal, %{site: site, event_name: "Signup"})- insert(:goal, %{site: site, page_path: "/blog**"})-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "date_range" => "all",- "metrics" => ["visitors", "events", "conversion_rate"],- "dimensions" => ["event:goal"]- })-- assert json_response(conn, 200)["results"] == [- %{"dimensions" => ["Visit /blog**"], "metrics" => [2, 2, 50.0]},- %{"dimensions" => ["Signup"], "metrics" => [1, 2, 25.0]}- ]- end-- test "returns conversion_rate alone in an event:goal breakdown", %{conn: conn, site: site} do- populate_stats(site, [- build(:event, name: "Signup", user_id: 1),- build(:pageview)- ])-- insert(:goal, %{site: site, event_name: "Signup"})-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "date_range" => "all",- "metrics" => ["conversion_rate"],- "dimensions" => ["event:goal"]- })-- assert json_response(conn, 200)["results"] == [- %{"dimensions" => ["Signup"], "metrics" => [50.0]}- ]- end-- test "returns conversion_rate in a goal filtered custom prop breakdown", %{- conn: conn,- site: site- } do- populate_stats(site, [- build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]),- build(:pageview, pathname: "/blog/2", "meta.key": ["author"], "meta.value": ["Uku"]),- build(:pageview, pathname: "/blog/3", "meta.key": ["author"], "meta.value": ["Uku"]),- build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Marko"]),- build(:pageview,- pathname: "/blog/2",- "meta.key": ["author"],- "meta.value": ["Marko"],- user_id: 1- ),- build(:pageview,- pathname: "/blog/3",- "meta.key": ["author"],- "meta.value": ["Marko"],- user_id: 1- ),- build(:pageview, pathname: "/blog"),- build(:pageview, "meta.key": ["author"], "meta.value": ["Marko"]),- build(:pageview)- ])-- insert(:goal, %{site: site, page_path: "/blog**"})-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "date_range" => "all",- "filters" => [["matches", "event:goal", ["Visit /blog**"]]],- "metrics" => ["visitors", "events", "conversion_rate"],- "dimensions" => ["event:props:author"]- })-- assert json_response(conn, 200)["results"] == [- %{"dimensions" => ["Uku"], "metrics" => [3, 3, 37.5]},- %{"dimensions" => ["Marko"], "metrics" => [2, 3, 25.0]},- %{"dimensions" => ["(none)"], "metrics" => [1, 1, 12.5]}- ]- end-- test "returns conversion_rate alone in a goal filtered custom prop breakdown", %{- conn: conn,- site: site- } do- populate_stats(site, [- build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]),- build(:pageview)- ])-- insert(:goal, %{site: site, page_path: "/blog**"})-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["conversion_rate"],- "date_range" => "all",- "dimensions" => ["event:props:author"],- "filters" => [["matches", "event:goal", ["Visit /blog**"]]]- })-- %{"results" => results} = json_response(conn, 200)-- assert results == [- %{"dimensions" => ["Uku"], "metrics" => [50]}- ]- end-- test "returns conversion_rate in a goal filtered event:page breakdown", %{- conn: conn,- site: site- } do- populate_stats(site, [- build(:event, pathname: "/en/register", name: "pageview"),- build(:event, pathname: "/en/register", name: "Signup"),- build(:event, pathname: "/en/register", name: "Signup"),- build(:event, pathname: "/it/register", name: "Signup", user_id: 1),- build(:event, pathname: "/it/register", name: "Signup", user_id: 1),- build(:event, pathname: "/it/register", name: "pageview")- ])-- insert(:goal, %{site: site, event_name: "Signup"})-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "date_range" => "all",- "dimensions" => ["event:page"],- "filters" => [["is", "event:goal", ["Signup"]]],- "metrics" => ["visitors", "events", "group_conversion_rate"]- })-- assert json_response(conn, 200)["results"] == [- %{"dimensions" => ["/en/register"], "metrics" => [2, 2, 66.7]},- %{"dimensions" => ["/it/register"], "metrics" => [1, 2, 50.0]}- ]- end-- test "returns conversion_rate alone in a goal filtered event:page breakdown", %{- conn: conn,- site: site- } do- populate_stats(site, [- build(:event, pathname: "/en/register", name: "pageview"),- build(:event, pathname: "/en/register", name: "Signup")- ])-- insert(:goal, %{site: site, event_name: "Signup"})-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "date_range" => "all",- "filters" => [["is", "event:goal", ["Signup"]]],- "metrics" => ["group_conversion_rate"],- "dimensions" => ["event:page"]- })-- assert json_response(conn, 200)["results"] == [- %{"dimensions" => ["/en/register"], "metrics" => [50.0]}- ]- end-- test "returns conversion_rate in a multi-goal filtered visit:screen_size breakdown", %{- conn: conn,- site: site- } do- populate_stats(site, [- build(:event, screen_size: "Mobile", name: "pageview"),- build(:event, screen_size: "Mobile", name: "AddToCart"),- build(:event, screen_size: "Mobile", name: "AddToCart"),- build(:event, screen_size: "Desktop", name: "AddToCart", user_id: 1),- build(:event, screen_size: "Desktop", name: "Purchase", user_id: 1),- build(:event, screen_size: "Desktop", name: "pageview")- ])-- # Make sure that revenue goals are treated the same- # way as regular custom event goals- insert(:goal, %{site: site, event_name: "Purchase", currency: :EUR})- insert(:goal, %{site: site, event_name: "AddToCart"})-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["visitors", "events", "group_conversion_rate"],- "date_range" => "all",- "dimensions" => ["visit:device"],- "filters" => [["is", "event:goal", ["AddToCart", "Purchase"]]]- })-- %{"results" => results} = json_response(conn, 200)-- assert results == [- %{"dimensions" => ["Mobile"], "metrics" => [2, 2, 66.7]},- %{"dimensions" => ["Desktop"], "metrics" => [1, 2, 50]}- ]- end-- test "returns conversion_rate alone in a goal filtered visit:screen_size breakdown", %{- conn: conn,- site: site- } do- populate_stats(site, [- build(:event, screen_size: "Mobile", name: "pageview"),- build(:event, screen_size: "Mobile", name: "AddToCart")- ])-- insert(:goal, %{site: site, event_name: "AddToCart"})-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["conversion_rate"],- "date_range" => "all",- "dimensions" => ["visit:device"],- "filters" => [["is", "event:goal", ["AddToCart"]]]- })-- %{"results" => results} = json_response(conn, 200)-- assert results == [- %{"dimensions" => ["Mobile"], "metrics" => [50]}- ]- end-test "all metrics for breakdown by visit prop", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview,@@ -3434,8 +2635,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do"visit_duration"],"date_range" => "all",- "dimensions" => ["visit:source"],- "filters" => []+ "dimensions" => ["visit:source"]})%{"results" => results} = json_response(conn, 200)@@ -3483,58 +2683,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do]end- test "filter by custom event property", %{conn: conn, site: site} do- populate_stats(site, [- build(:event,- name: "Purchase",- "meta.key": ["package"],- "meta.value": ["business"],- browser: "Chrome",- timestamp: ~N[2021-01-01 00:00:00]- ),- build(:event,- name: "Purchase",- "meta.key": ["package"],- "meta.value": ["business"],- browser: "Safari",- timestamp: ~N[2021-01-01 00:00:00]- ),- build(:event,- name: "Purchase",- "meta.key": ["package"],- "meta.value": ["business"],- browser: "Safari",- timestamp: ~N[2021-01-01 00:25:00]- ),- build(:event,- name: "Purchase",- "meta.key": ["package"],- "meta.value": ["personal"],- browser: "IE",- timestamp: ~N[2021-01-01 00:25:00]- )- ])-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["visitors"],- "date_range" => "all",- "dimensions" => ["visit:browser"],- "filters" => [- ["is", "event:name", ["Purchase"]],- ["is", "event:props:package", ["business"]]- ]- })-- %{"results" => results} = json_response(conn, 200)-- assert results == [- %{"dimensions" => ["Safari"], "metrics" => [2]},- %{"dimensions" => ["Chrome"], "metrics" => [1]}- ]- end-test "all metrics for breakdown by event prop", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview,@@ -3578,452 +2726,56 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doendend- describe "imported data" do- test "returns screen sizes breakdown when filtering by screen size", %{conn: conn, site: site} do- site_import = insert(:site_import, site: site)-- populate_stats(site, site_import.id, [- build(:pageview,- timestamp: ~N[2021-01-01 00:00:01],- screen_size: "Mobile"- ),- build(:imported_devices,- device: "Mobile",- visitors: 3,- pageviews: 5,- date: ~D[2021-01-01]- )- ])-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["visitors", "pageviews"],- "date_range" => "all",- "dimensions" => ["visit:device"],- "filters" => [- ["is", "visit:device", ["Mobile"]]- ],- "include" => %{"imports" => true}- })-- %{"results" => results} = json_response(conn, 200)-- assert results == [%{"dimensions" => ["Mobile"], "metrics" => [4, 6]}]- end-- test "returns custom event goals and pageview goals", %{conn: conn, site: site} do- insert(:goal, site: site, event_name: "Purchase")- insert(:goal, site: site, page_path: "/test")-- site_import = insert(:site_import, site: site)-- populate_stats(site, site_import.id, [- build(:pageview,- timestamp: ~N[2021-01-01 00:00:01],- pathname: "/test"- ),- build(:event,- name: "Purchase",- timestamp: ~N[2021-01-01 00:00:03]- ),- build(:event,- name: "Purchase",- timestamp: ~N[2021-01-01 00:00:03]- ),- build(:imported_custom_events,- name: "Purchase",- visitors: 3,- events: 5,- date: ~D[2021-01-01]- ),- build(:imported_pages,- page: "/test",- visitors: 2,- pageviews: 2,- date: ~D[2021-01-01]- ),- build(:imported_visitors, visitors: 5, date: ~D[2021-01-01])- ])-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "date_range" => "all",- "dimensions" => ["event:goal"],- "metrics" => ["visitors", "events", "pageviews", "conversion_rate"],- "include" => %{"imports" => true}- })-- assert json_response(conn, 200)["results"] == [- %{"dimensions" => ["Purchase"], "metrics" => [5, 7, 0, 62.5]},- %{"dimensions" => ["Visit /test"], "metrics" => [3, 3, 3, 37.5]}- ]- end-- test "pageviews are returned as events for breakdown reports other than custom events", %{- conn: conn,- site: site- } do- site_import = insert(:site_import, site: site)-- populate_stats(site, site_import.id, [- build(:imported_browsers, browser: "Chrome", pageviews: 1, date: ~D[2021-01-01]),- build(:imported_devices, device: "Desktop", pageviews: 1, date: ~D[2021-01-01]),- build(:imported_entry_pages, entry_page: "/test", pageviews: 1, date: ~D[2021-01-01]),- build(:imported_exit_pages, exit_page: "/test", pageviews: 1, date: ~D[2021-01-01]),- build(:imported_locations, country: "EE", pageviews: 1, date: ~D[2021-01-01]),- build(:imported_operating_systems,- operating_system: "Mac",- pageviews: 1,- date: ~D[2021-01-01]- ),- build(:imported_pages, page: "/test", pageviews: 1, date: ~D[2021-01-01]),- build(:imported_sources, source: "Google", pageviews: 1, date: ~D[2021-01-01])- ])-- breakdown_and_first = fn dimension ->- conn- |> post("/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["events"],- "date_range" => ["2021-01-01", "2021-01-01"],- "dimensions" => [dimension],- "include" => %{"imports" => true}- })- |> json_response(200)- |> Map.get("results")- |> List.first()- end-- assert %{"dimensions" => ["Chrome"], "metrics" => [1]} =- breakdown_and_first.("visit:browser")-- assert %{"dimensions" => ["Desktop"], "metrics" => [1]} =- breakdown_and_first.("visit:device")-- assert %{"dimensions" => ["EE"], "metrics" => [1]} = breakdown_and_first.("visit:country")- assert %{"dimensions" => ["Mac"], "metrics" => [1]} = breakdown_and_first.("visit:os")- assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("event:page")-- assert %{"dimensions" => ["Google"], "metrics" => [1]} =- breakdown_and_first.("visit:source")- end-- for goal_name <- Plausible.Imported.goals_with_url() do- test "returns url breakdown for #{goal_name} goal", %{conn: conn, site: site} do- insert(:goal, event_name: unquote(goal_name), site: site)- site_import = insert(:site_import, site: site)-- populate_stats(site, site_import.id, [- build(:event,- name: unquote(goal_name),- "meta.key": ["url"],- "meta.value": ["https://one.com"]- ),- build(:imported_custom_events,- name: unquote(goal_name),- visitors: 2,- events: 5,- link_url: "https://one.com"- ),- build(:imported_custom_events,- name: unquote(goal_name),- visitors: 5,- events: 10,- link_url: "https://two.com"- ),- build(:imported_custom_events,- name: "some goal",- visitors: 5,- events: 10- ),- build(:imported_visitors, visitors: 9)- ])-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["visitors", "events", "conversion_rate"],- "date_range" => "all",- "dimensions" => ["event:props:url"],- "filters" => [- ["is", "event:goal", [unquote(goal_name)]]- ],- "include" => %{"imports" => true}- })-- assert json_response(conn, 200)["results"] == [- %{"dimensions" => ["https://two.com"], "metrics" => [5, 10, 50]},- %{"dimensions" => ["https://one.com"], "metrics" => [3, 6, 30]}- ]-- refute json_response(conn, 200)["meta"]["warning"]- end- end-- for goal_name <- Plausible.Imported.goals_with_path() do- test "returns path breakdown for #{goal_name} goal", %{conn: conn, site: site} do- insert(:goal, event_name: unquote(goal_name), site: site)- site_import = insert(:site_import, site: site)-- populate_stats(site, site_import.id, [- build(:event,- name: unquote(goal_name),- "meta.key": ["path"],- "meta.value": ["/one"]- ),- build(:imported_custom_events,- name: unquote(goal_name),- visitors: 2,- events: 5,- path: "/one"- ),- build(:imported_custom_events,- name: unquote(goal_name),- visitors: 5,- events: 10,- path: "/two"- ),- build(:imported_custom_events,- name: "some goal",- visitors: 5,- events: 10- ),- build(:imported_visitors, visitors: 9)- ])-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["visitors", "events", "conversion_rate"],- "date_range" => "all",- "dimensions" => ["event:props:path"],- "filters" => [- ["is", "event:goal", [unquote(goal_name)]]- ],- "include" => %{"imports" => true}- })-- assert json_response(conn, 200)["results"] == [- %{"dimensions" => ["/two"], "metrics" => [5, 10, 50]},- %{"dimensions" => ["/one"], "metrics" => [3, 6, 30]}- ]-- refute json_response(conn, 200)["meta"]["warning"]- end- end-- test "adds a warning when query params are not supported for imported data", %{- conn: conn,- site: site- } do- site_import = insert(:site_import, site: site)-- insert(:goal, event_name: "Signup", site: site)-- populate_stats(site, site_import.id, [- build(:event,- name: "Signup",- "meta.key": ["package"],- "meta.value": ["large"]- ),- build(:imported_visitors, visitors: 9)- ])-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["visitors"],- "date_range" => "all",- "dimensions" => ["event:props:package"],- "filters" => [- ["is", "event:goal", ["Signup"]]- ],- "include" => %{"imports" => true}- })-- assert json_response(conn, 200)["results"] == [- %{"dimensions" => ["large"], "metrics" => [1]}- ]-- assert json_response(conn, 200)["meta"]["warning"] =~- "Imported stats are not included in the results because query parameters are not supported."- end-- test "does not add a warning when there are no site imports", %{conn: conn, site: site} do- insert(:goal, event_name: "Signup", site: site)-- populate_stats(site, [- build(:event,- name: "Signup",- "meta.key": ["package"],- "meta.value": ["large"]- )- ])-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["visitors"],- "date_range" => "all",- "dimensions" => ["event:props:package"],- "filters" => [- ["is", "event:goal", ["Signup"]]- ],- "include" => %{"imports" => true}- })-- refute json_response(conn, 200)["meta"]["warning"]- end-- test "does not add a warning when import is out of queried date range", %{- conn: conn,- site: site- } do- site_import = insert(:site_import, site: site, end_date: Date.add(Date.utc_today(), -3))-- insert(:goal, event_name: "Signup", site: site)-- populate_stats(site, site_import.id, [- build(:event,- name: "Signup",- "meta.key": ["package"],- "meta.value": ["large"]- ),- build(:imported_visitors, visitors: 9)- ])-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["visitors"],- "date_range" => "day",- "dimensions" => ["event:props:package"],- "filters" => [- ["is", "event:goal", ["Signup"]]- ],- "include" => %{"imports" => true}- })-- refute json_response(conn, 200)["meta"]["warning"]- end-- test "applies multiple filters if the properties belong to the same table", %{- conn: conn,- site: site- } do- site_import = insert(:site_import, site: site)-- populate_stats(site, site_import.id, [- build(:imported_sources, source: "Google", utm_medium: "organic", utm_term: "one"),- build(:imported_sources, source: "Twitter", utm_medium: "organic", utm_term: "two"),- build(:imported_sources,- source: "Facebook",- utm_medium: "something_else",- utm_term: "one"- )- ])-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["visitors"],- "date_range" => "day",- "dimensions" => ["visit:source"],- "filters" => [- ["is", "visit:utm_medium", ["organic"]],- ["is", "visit:utm_term", ["one"]]- ],- "include" => %{"imports" => true}- })-- assert json_response(conn, 200)["results"] == [- %{"dimensions" => ["Google"], "metrics" => [1]}- ]- end-- test "ignores imported data if filtered property belongs to a different table than the breakdown property",- %{- conn: conn,- site: site- } do- site_import = insert(:site_import, site: site)-- populate_stats(site, site_import.id, [- build(:imported_sources, source: "Google"),- build(:imported_devices, device: "Desktop")- ])-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["visitors"],- "date_range" => "day",- "dimensions" => ["visit:source"],- "filters" => [- ["is", "visit:device", ["Desktop"]]- ],- "include" => %{"imports" => true}- })-- assert %{- "results" => [],- "meta" => meta- } = json_response(conn, 200)-- assert meta["warning"] =~ "Imported stats are not included in the results"- end+ test "filtering by custom event property", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event,+ name: "Purchase",+ "meta.key": ["package"],+ "meta.value": ["business"],+ browser: "Chrome",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Purchase",+ "meta.key": ["package"],+ "meta.value": ["business"],+ browser: "Safari",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Purchase",+ "meta.key": ["package"],+ "meta.value": ["business"],+ browser: "Safari",+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:event,+ name: "Purchase",+ "meta.key": ["package"],+ "meta.value": ["personal"],+ browser: "IE",+ timestamp: ~N[2021-01-01 00:25:00]+ )+ ])- test "imported country, region and city data",- %{- conn: conn,- site: site- } do- site_import = insert(:site_import, site: site)+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["visit:browser"],+ "filters" => [+ ["is", "event:name", ["Purchase"]],+ ["is", "event:props:package", ["business"]]+ ]+ })- populate_stats(site, site_import.id, [- build(:pageview,- timestamp: ~N[2021-01-01 00:15:00],- country_code: "DE",- subdivision1_code: "DE-BE",- city_geoname_id: 2_950_159- ),- build(:pageview,- timestamp: ~N[2021-01-01 00:15:00],- country_code: "DE",- subdivision1_code: "DE-BE",- city_geoname_id: 2_950_159- ),- build(:pageview,- timestamp: ~N[2021-01-01 00:15:00],- country_code: "EE",- subdivision1_code: "EE-37",- city_geoname_id: 588_409- ),- build(:imported_locations, country: "EE", region: "EE-37", city: 588_409, visitors: 33)- ])+ %{"results" => results} = json_response(conn, 200)- for {dimension, stats_value, imports_value} <- [- {"visit:country", "DE", "EE"},- {"visit:region", "DE-BE", "EE-37"},- {"visit:city", 2_950_159, 588_409}- ] do- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["visitors"],- "date_range" => "all",- "dimensions" => [dimension],- "include" => %{"imports" => true}- })-- assert json_response(conn, 200)["results"] == [- %{"dimensions" => [imports_value], "metrics" => [34]},- %{"dimensions" => [stats_value], "metrics" => [2]}- ]- end- end+ assert results == [+ %{"dimensions" => ["Safari"], "metrics" => [2]},+ %{"dimensions" => ["Chrome"], "metrics" => [1]}+ ]endtest "multiple breakdown timeseries with sources", %{conn: conn, site: site} dodiff --git a/test/plausible_web/controllers/api/external_stats_controller/query_validations_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_validations_test.exsnew file mode 100644index 0000000000..27c745950b--- /dev/null+++ b/test/plausible_web/controllers/api/external_stats_controller/query_validations_test.exs@@ -0,0 +1,234 @@+defmodule PlausibleWeb.Api.ExternalStatsController.QueryValidationsTest do+ use PlausibleWeb.ConnCase+ alias Plausible.Billing.Feature++ setup [:create_user, :create_new_site, :create_api_key, :use_api_key]++ describe "feature access" do+ test "cannot break down by a custom prop without access to the props feature", %{+ conn: conn,+ user: user,+ site: site+ } do+ ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)+ insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["event:props:author"]+ })++ assert json_response(conn, 400)["error"] ==+ "The owner of this site does not have access to the custom properties feature"+ end++ test "can break down by an internal prop key without access to the props feature", %{+ conn: conn,+ user: user,+ site: site+ } do+ ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)+ insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["event:props:path"]+ })++ assert json_response(conn, 200)["results"]+ end++ test "cannot filter by a custom prop without access to the props feature", %{+ conn: conn,+ user: user,+ site: site+ } do+ ep =+ insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)++ insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["visit:source"],+ "filters" => [["is", "event:props:author", ["Uku"]]]+ })++ assert json_response(conn, 400)["error"] ==+ "The owner of this site does not have access to the custom properties feature"+ end++ test "can filter by an internal prop key without access to the props feature", %{+ conn: conn,+ user: user,+ site: site+ } do+ ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)+ insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["visit:source"],+ "filters" => [["is", "event:props:url", ["whatever"]]]+ })++ assert json_response(conn, 200)["results"]+ end+ end++ describe "param validation" do+ test "does not allow querying conversion_rate without a goal filter", %{+ conn: conn,+ site: site+ } do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["conversion_rate"],+ "date_range" => "all",+ "dimensions" => ["event:page"],+ "filters" => [["is", "event:props:author", ["Uku"]]]+ })++ assert json_response(conn, 400)["error"] ==+ "Metric `conversion_rate` can only be queried with event:goal filters or dimensions"+ end++ test "validates that dimensions are valid", %{conn: conn, site: site} do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["badproperty"]+ })++ assert json_response(conn, 400)["error"] =~ "Invalid dimensions"+ end++ test "empty custom property is invalid", %{conn: conn, site: site} do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["event:props:"]+ })++ assert json_response(conn, 400)["error"] =~ "Invalid dimensions"+ end++ test "validates that correct date range is used", %{conn: conn, site: site} do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "bad_period",+ "dimensions" => ["event:name"]+ })++ assert json_response(conn, 400)["error"] =~ "Invalid date_range"+ end++ test "fails when an invalid metric is provided", %{conn: conn, site: site} do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "baa"],+ "date_range" => "all",+ "dimensions" => ["event:name"]+ })++ assert json_response(conn, 400)["error"] =~ "Unknown metric '\"baa\"'"+ end++ test "session metrics cannot be used with event:name dimension", %{conn: conn, site: site} do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "bounce_rate"],+ "date_range" => "all",+ "dimensions" => ["event:name"]+ })++ assert json_response(conn, 400)["error"] =~+ "Session metric(s) `bounce_rate` cannot be queried along with event dimensions"+ end++ test "session metrics cannot be used with event:props:* dimension", %{conn: conn, site: site} do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "bounce_rate"],+ "date_range" => "all",+ "dimensions" => ["event:props:url"]+ })++ assert json_response(conn, 400)["error"] =~+ "Session metric(s) `bounce_rate` cannot be queried along with event dimensions"+ end++ test "validates that metric views_per_visit cannot be used with event:page filter", %{+ conn: conn,+ site: site+ } do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["views_per_visit"],+ "filters" => [["is", "event:page", ["/something"]]]+ })++ assert json_response(conn, 400) == %{+ "error" =>+ "Metric `views_per_visit` cannot be queried with a filter on `event:page`"+ }+ end++ test "validates that metric views_per_visit cannot be used together with dimensions", %{+ conn: conn,+ site: site+ } do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["views_per_visit"],+ "dimensions" => ["event:name"]+ })++ assert json_response(conn, 400) == %{+ "error" => "Metric `views_per_visit` cannot be queried with `dimensions`"+ }+ end++ test "validates a metric can't be asked multiple times", %{+ conn: conn,+ site: site+ } do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["views_per_visit", "visitors", "visitors"]+ })++ assert json_response(conn, 400) == %{+ "error" => "Metrics cannot be queried multiple times"+ }+ end+ end+end