Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

APIv2: Cleanup (#4308)

Karl-Aksel Puulmanna9676546dcb8
diff --git a/extra/lib/plausible/stats/funnel.ex b/extra/lib/plausible/stats/funnel.ex
index 121a699fae..a365f36107 100644
--- a/extra/lib/plausible/stats/funnel.ex
+++ b/extra/lib/plausible/stats/funnel.ex
@@ -110,7 +110,7 @@ defmodule Plausible.Stats.Funnel do
%Plausible.Goal{page_path: pathname} when is_binary(pathname) ->
if String.contains?(pathname, "*") do
- regex = Plausible.Stats.Base.page_regex(pathname)
+ regex = Plausible.Stats.Filters.Utils.page_regex(pathname)
dynamic([], fragment("match(pathname, ?)", ^regex))
else
dynamic([], fragment("pathname = ?", ^pathname))
diff --git a/lib/plausible/google/search_console/filters.ex b/lib/plausible/google/search_console/filters.ex
index ee9bc21578..43e10e1c87 100644
--- a/lib/plausible/google/search_console/filters.ex
+++ b/lib/plausible/google/search_console/filters.ex
@@ -1,6 +1,6 @@
defmodule Plausible.Google.SearchConsole.Filters do
@moduledoc false
- import Plausible.Stats.Base, only: [page_regex: 1]
+ import Plausible.Stats.Filters.Utils, only: [page_regex: 1]
def transform(property, plausible_filters) do
search_console_filters =
diff --git a/lib/plausible/stats.ex b/lib/plausible/stats.ex
index ffa59389ea..f22f1b069c 100644
--- a/lib/plausible/stats.ex
+++ b/lib/plausible/stats.ex
@@ -15,6 +15,17 @@ defmodule Plausible.Stats do
use Plausible.DebugReplayInfo
+ def query(site, query) do
+ include_sentry_replay_info()
+
+ optimized_query = QueryOptimizer.optimize(query)
+
+ optimized_query
+ |> SQL.QueryBuilder.build(site)
+ |> ClickhouseRepo.all()
+ |> QueryResult.from(optimized_query)
+ end
+
def breakdown(site, query, metrics, pagination) do
include_sentry_replay_info()
Breakdown.breakdown(site, query, metrics, pagination)
@@ -35,15 +46,6 @@ defmodule Plausible.Stats do
CurrentVisitors.current_visitors(site)
end
- def query(site, query) do
- optimized_query = QueryOptimizer.optimize(query)
-
- optimized_query
- |> SQL.QueryBuilder.build(site)
- |> ClickhouseRepo.all()
- |> QueryResult.from(optimized_query)
- end
-
on_ee do
def funnel(site, query, funnel) do
include_sentry_replay_info()
diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex
index 8c5cdc4f7b..790b5ed719 100644
--- a/lib/plausible/stats/aggregate.ex
+++ b/lib/plausible/stats/aggregate.ex
@@ -1,4 +1,10 @@
defmodule Plausible.Stats.Aggregate do
+ @moduledoc """
+ Builds aggregate results for v1 of our stats API and dashboards.
+
+ Avoid adding new logic here - update QueryBuilder etc instead.
+ """
+
use Plausible.ClickhouseRepo
use Plausible
import Plausible.Stats.Base
diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex
index cb104ffc76..c5304eaacd 100644
--- a/lib/plausible/stats/base.ex
+++ b/lib/plausible/stats/base.ex
@@ -1,10 +1,7 @@
defmodule Plausible.Stats.Base do
- use Plausible.ClickhouseRepo
use Plausible
- use Plausible.Stats.SQL.Fragments
- alias Plausible.Stats.{Query, TableDecider, SQL}
- alias Plausible.Timezones
+ alias Plausible.Stats.{TableDecider, SQL}
import Ecto.Query
def base_event_query(site, query) do
@@ -29,7 +26,7 @@ defmodule Plausible.Stats.Base do
end
end
- def query_events(site, query) do
+ defp query_events(site, query) do
q = from(e in "events_v2", where: ^SQL.WhereBuilder.build(:events, site, query))
on_ee do
@@ -48,95 +45,4 @@ defmodule Plausible.Stats.Base do
q
end
-
- def select_event_metrics(metrics) do
- metrics
- |> Enum.map(&SQL.Expression.event_metric/1)
- |> Enum.reduce(%{}, &Map.merge/2)
- end
-
- def select_session_metrics(metrics, query) do
- metrics
- |> Enum.map(&SQL.Expression.session_metric(&1, query))
- |> Enum.reduce(%{}, &Map.merge/2)
- end
-
- def filter_converted_sessions(db_query, site, query) do
- if Query.has_event_filters?(query) do
- converted_sessions =
- from(e in query_events(site, query),
- select: %{
- session_id: fragment("DISTINCT ?", e.session_id),
- _sample_factor: fragment("_sample_factor")
- }
- )
-
- from(s in db_query,
- join: cs in subquery(converted_sessions),
- on: s.session_id == cs.session_id
- )
- else
- db_query
- end
- end
-
- defp beginning_of_time(candidate, native_stats_start_at) do
- if Timex.after?(native_stats_start_at, candidate) do
- native_stats_start_at
- else
- candidate
- end
- end
-
- def utc_boundaries(%Query{period: "realtime", now: now}, site) do
- last_datetime =
- now
- |> Timex.shift(seconds: 5)
- |> beginning_of_time(site.native_stats_start_at)
- |> NaiveDateTime.truncate(:second)
-
- first_datetime =
- now |> Timex.shift(minutes: -5) |> NaiveDateTime.truncate(:second)
-
- {first_datetime, last_datetime}
- end
-
- def utc_boundaries(%Query{period: "30m", now: now}, site) do
- last_datetime =
- now
- |> Timex.shift(seconds: 5)
- |> beginning_of_time(site.native_stats_start_at)
- |> NaiveDateTime.truncate(:second)
-
- first_datetime =
- now |> Timex.shift(minutes: -30) |> NaiveDateTime.truncate(:second)
-
- {first_datetime, last_datetime}
- end
-
- def utc_boundaries(%Query{date_range: date_range}, site) do
- {:ok, first} = NaiveDateTime.new(date_range.first, ~T[00:00:00])
-
- first_datetime =
- first
- |> Timezones.to_utc_datetime(site.timezone)
- |> beginning_of_time(site.native_stats_start_at)
-
- {:ok, last} = NaiveDateTime.new(date_range.last |> Timex.shift(days: 1), ~T[00:00:00])
-
- last_datetime = Timezones.to_utc_datetime(last, site.timezone)
-
- {first_datetime, last_datetime}
- end
-
- def page_regex(expr) do
- escaped =
- expr
- |> Regex.escape()
- |> String.replace("\\|", "|")
- |> String.replace("\\*\\*", ".*")
- |> String.replace("\\*", ".*")
-
- "^#{escaped}$"
- end
end
diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex
index abf78beac3..f4f1d40d26 100644
--- a/lib/plausible/stats/breakdown.ex
+++ b/lib/plausible/stats/breakdown.ex
@@ -1,4 +1,10 @@
defmodule Plausible.Stats.Breakdown do
+ @moduledoc """
+ Builds breakdown results for v1 of our stats API and dashboards.
+
+ Avoid adding new logic here - update QueryBuilder etc instead.
+ """
+
use Plausible.ClickhouseRepo
use Plausible
use Plausible.Stats.SQL.Fragments
diff --git a/lib/plausible/stats/email_report.ex b/lib/plausible/stats/email_report.ex
index 1dc26465bb..d745ed7971 100644
--- a/lib/plausible/stats/email_report.ex
+++ b/lib/plausible/stats/email_report.ex
@@ -40,7 +40,7 @@ defmodule Plausible.Stats.EmailReport do
end
defp put_top_5_pages(stats, site, query) do
- query = Query.set_dimensions(query, ["event:page"])
+ query = Query.set(query, dimensions: ["event:page"])
pages = Stats.breakdown(site, query, [:visitors], {5, 1})
Map.put(stats, :pages, pages)
end
@@ -48,8 +48,8 @@ defmodule Plausible.Stats.EmailReport do
defp put_top_5_sources(stats, site, query) do
query =
query
- |> Query.put_filter([:is_not, "visit:source", ["Direct / None"]])
- |> Query.set_dimensions(["visit:source"])
+ |> Query.add_filter([:is_not, "visit:source", ["Direct / None"]])
+ |> Query.set(dimensions: ["visit:source"])
sources = Stats.breakdown(site, query, [:visitors], {5, 1})
diff --git a/lib/plausible/stats/filters/utils.ex b/lib/plausible/stats/filters/utils.ex
index 96606c1d02..4869286fa4 100644
--- a/lib/plausible/stats/filters/utils.ex
+++ b/lib/plausible/stats/filters/utils.ex
@@ -76,8 +76,19 @@ defmodule Plausible.Stats.Filters.Utils do
events = Enum.map(event_goals, fn {_, event} -> event end)
page_regexes =
- Enum.map(pageview_goals, fn {_, path} -> Plausible.Stats.Base.page_regex(path) end)
+ Enum.map(pageview_goals, fn {_, path} -> page_regex(path) end)
{events, page_regexes}
end
+
+ def page_regex(expr) do
+ escaped =
+ expr
+ |> Regex.escape()
+ |> String.replace("\\|", "|")
+ |> String.replace("\\*\\*", ".*")
+ |> String.replace("\\*", ".*")
+
+ "^#{escaped}$"
+ end
end
diff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.ex
index a1e817e484..ead5a5db80 100644
--- a/lib/plausible/stats/imported/imported.ex
+++ b/lib/plausible/stats/imported/imported.ex
@@ -3,17 +3,13 @@ defmodule Plausible.Stats.Imported do
use Plausible.Stats.SQL.Fragments
import Ecto.Query
- import Plausible.Stats.Util, only: [shortname: 2]
+ import Plausible.Stats.Imported.SQL.Builder
alias Plausible.Stats.Filters
alias Plausible.Stats.Imported
alias Plausible.Stats.Query
alias Plausible.Stats.SQL.QueryBuilder
- @no_ref "Direct / None"
- @not_set "(not set)"
- @none "(none)"
-
@property_to_table_mappings Imported.Base.property_to_table_mappings()
@goals_with_url Plausible.Imported.goals_with_url()
@@ -302,479 +298,4 @@ defmodule Plausible.Stats.Imported do
|> Imported.Base.query_imported(query)
|> select_merge([i], %{total_visitors: fragment("sum(?)", i.visitors)})
end
-
- defp select_imported_metrics(q, []), do: q
-
- defp select_imported_metrics(q, [:visitors | rest]) do
- q
- |> select_merge([i], %{visitors: sum(i.visitors)})
- |> select_imported_metrics(rest)
- end
-
- defp select_imported_metrics(
- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_custom_events", _}}} = q,
- [:events | rest]
- ) do
- q
- |> select_merge([i], %{events: sum(i.events)})
- |> select_imported_metrics(rest)
- end
-
- defp select_imported_metrics(q, [:events | rest]) do
- q
- |> select_merge([i], %{events: sum(i.pageviews)})
- |> select_imported_metrics(rest)
- end
-
- defp select_imported_metrics(
- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q,
- [:visits | rest]
- ) do
- q
- |> select_merge([i], %{visits: sum(i.exits)})
- |> select_imported_metrics(rest)
- end
-
- defp select_imported_metrics(
- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,
- [:visits | rest]
- ) do
- q
- |> select_merge([i], %{visits: sum(i.entrances)})
- |> select_imported_metrics(rest)
- end
-
- defp select_imported_metrics(q, [:visits | rest]) do
- q
- |> select_merge([i], %{visits: sum(i.visits)})
- |> select_imported_metrics(rest)
- end
-
- defp select_imported_metrics(
- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_custom_events", _}}} = q,
- [:pageviews | rest]
- ) do
- q
- |> select_merge([i], %{pageviews: 0})
- |> select_imported_metrics(rest)
- end
-
- defp select_imported_metrics(q, [:pageviews | rest]) do
- q
- |> where([i], i.pageviews > 0)
- |> select_merge([i], %{pageviews: sum(i.pageviews)})
- |> select_imported_metrics(rest)
- end
-
- defp select_imported_metrics(
- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_pages", _}}} = q,
- [:bounce_rate | rest]
- ) do
- q
- |> select_merge([i], %{
- bounces: 0,
- __internal_visits: 0
- })
- |> select_imported_metrics(rest)
- end
-
- defp select_imported_metrics(
- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,
- [:bounce_rate | rest]
- ) do
- q
- |> select_merge([i], %{
- bounces: sum(i.bounces),
- __internal_visits: sum(i.entrances)
- })
- |> select_imported_metrics(rest)
- end
-
- defp select_imported_metrics(
- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q,
- [:bounce_rate | rest]
- ) do
- q
- |> select_merge([i], %{
- bounces: sum(i.bounces),
- __internal_visits: sum(i.exits)
- })
- |> select_imported_metrics(rest)
- end
-
- defp select_imported_metrics(q, [:bounce_rate | rest]) do
- q
- |> select_merge([i], %{
- bounces: sum(i.bounces),
- __internal_visits: sum(i.visits)
- })
- |> select_imported_metrics(rest)
- end
-
- defp select_imported_metrics(
- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_pages", _}}} = q,
- [:visit_duration | rest]
- ) do
- q
- |> select_merge([i], %{
- visit_duration: 0,
- __internal_visits: 0
- })
- |> select_imported_metrics(rest)
- end
-
- defp select_imported_metrics(
- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,
- [:visit_duration | rest]
- ) do
- q
- |> select_merge([i], %{
- visit_duration: sum(i.visit_duration),
- __internal_visits: sum(i.entrances)
- })
- |> select_imported_metrics(rest)
- end
-
- defp select_imported_metrics(
- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q,
- [:visit_duration | rest]
- ) do
- q
- |> select_merge([i], %{
- visit_duration: sum(i.visit_duration),
- __internal_visits: sum(i.exits)
- })
- |> select_imported_metrics(rest)
- end
-
- defp select_imported_metrics(q, [:visit_duration | rest]) do
- q
- |> select_merge([i], %{
- visit_duration: sum(i.visit_duration),
- __internal_visits: sum(i.visits)
- })
- |> select_imported_metrics(rest)
- end
-
- defp select_imported_metrics(
- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,
- [:views_per_visit | rest]
- ) do
- q
- |> where([i], i.pageviews > 0)
- |> select_merge([i], %{
- pageviews: sum(i.pageviews),
- __internal_visits: sum(i.entrances)
- })
- |> select_imported_metrics(rest)
- end
-
- defp select_imported_metrics(
- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q,
- [:views_per_visit | rest]
- ) do
- q
- |> where([i], i.pageviews > 0)
- |> select_merge([i], %{
- pageviews: sum(i.pageviews),
- __internal_visits: sum(i.exits)
- })
- |> select_imported_metrics(rest)
- end
-
- defp select_imported_metrics(q, [:views_per_visit | rest]) do
- q
- |> where([i], i.pageviews > 0)
- |> select_merge([i], %{
- pageviews: sum(i.pageviews),
- __internal_visits: sum(i.visits)
- })
- |> select_imported_metrics(rest)
- end
-
- defp select_imported_metrics(q, [_ | rest]) do
- q
- |> select_imported_metrics(rest)
- end
-
- defp group_imported_by(q, query) do
- Enum.reduce(query.dimensions, q, fn dimension, q ->
- dim = Plausible.Stats.Filters.without_prefix(dimension)
-
- group_imported_by(q, dim, shortname(query, dimension), query)
- end)
- end
-
- defp group_imported_by(q, dim, key, _query) when dim in [:source, :referrer] do
- q
- |> group_by([i], field(i, ^dim))
- |> select_merge_as([i], %{
- key => fragment("if(empty(?), ?, ?)", field(i, ^dim), @no_ref, field(i, ^dim))
- })
- end
-
- defp group_imported_by(q, dim, key, _query)
- when dim in [:utm_source, :utm_medium, :utm_campaign, :utm_term, :utm_content] do
- q
- |> group_by([i], field(i, ^dim))
- |> where([i], fragment("not empty(?)", field(i, ^dim)))
- |> select_merge_as([i], %{key => field(i, ^dim)})
- end
-
- defp group_imported_by(q, :page, key, _query) do
- q
- |> group_by([i], i.page)
- |> select_merge_as([i], %{key => i.page, time_on_page: sum(i.time_on_page)})
- end
-
- defp group_imported_by(q, :country, key, _query) do
- q
- |> group_by([i], i.country)
- |> where([i], i.country != "ZZ")
- |> select_merge_as([i], %{key => i.country})
- end
-
- defp group_imported_by(q, :region, key, _query) do
- q
- |> group_by([i], i.region)
- |> where([i], i.region != "")
- |> select_merge_as([i], %{key => i.region})
- end
-
- defp group_imported_by(q, :city, key, _query) do
- q
- |> group_by([i], i.city)
- |> where([i], i.city != 0 and not is_nil(i.city))
- |> select_merge_as([i], %{key => i.city})
- end
-
- defp group_imported_by(q, dim, key, _query) when dim in [:device, :browser] do
- q
- |> group_by([i], field(i, ^dim))
- |> select_merge_as([i], %{
- key => fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim))
- })
- end
-
- defp group_imported_by(q, :browser_version, key, _query) do
- q
- |> group_by([i], [i.browser_version])
- |> select_merge_as([i], %{
- key => fragment("if(empty(?), ?, ?)", i.browser_version, @not_set, i.browser_version)
- })
- end
-
- defp group_imported_by(q, :os, key, _query) do
- q
- |> group_by([i], i.operating_system)
- |> select_merge_as([i], %{
- key => fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system)
- })
- end
-
- defp group_imported_by(q, :os_version, key, _query) do
- q
- |> group_by([i], [i.operating_system_version])
- |> select_merge_as([i], %{
- key =>
- fragment(
- "if(empty(?), ?, ?)",
- i.operating_system_version,
- @not_set,
- i.operating_system_version
- )
- })
- end
-
- defp group_imported_by(q, dim, key, _query) when dim in [:entry_page, :exit_page] do
- q
- |> group_by([i], field(i, ^dim))
- |> select_merge_as([i], %{key => field(i, ^dim)})
- end
-
- defp group_imported_by(q, :name, key, _query) do
- q
- |> group_by([i], i.name)
- |> select_merge_as([i], %{key => i.name})
- end
-
- defp group_imported_by(q, :url, key, _query) do
- q
- |> group_by([i], i.link_url)
- |> select_merge_as([i], %{
- key => fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none)
- })
- end
-
- defp group_imported_by(q, :path, key, _query) do
- q
- |> group_by([i], i.path)
- |> select_merge_as([i], %{
- key => fragment("if(not empty(?), ?, ?)", i.path, i.path, @none)
- })
- end
-
- defp group_imported_by(q, :month, key, _query) do
- q
- |> group_by([i], fragment("toStartOfMonth(?)", i.date))
- |> select_merge_as([i], %{key => fragment("toStartOfMonth(?)", i.date)})
- end
-
- defp group_imported_by(q, :hour, key, _query) do
- q
- |> group_by([i], i.date)
- |> select_merge_as([i], %{key => i.date})
- end
-
- defp group_imported_by(q, :week, key, query) do
- q
- |> group_by([i], weekstart_not_before(i.date, ^query.date_range.first))
- |> select_merge_as([i], %{
- key => weekstart_not_before(i.date, ^query.date_range.first)
- })
- end
-
- defp group_imported_by(q, :day, key, _query) do
- q
- |> group_by([i], i.date)
- |> select_merge_as([i], %{key => i.date})
- end
-
- defp select_joined_dimensions(q, query) do
- Enum.reduce(query.dimensions, q, fn dimension, q ->
- select_joined_dimension(q, dimension, shortname(query, dimension))
- end)
- end
-
- defp select_joined_dimension(q, "visit:city", key) do
- select_merge_as(q, [s, i], %{
- key => fragment("greatest(?,?)", field(i, ^key), field(s, ^key))
- })
- end
-
- defp select_joined_dimension(q, "time:" <> _, key) do
- select_merge_as(q, [s, i], %{
- key => fragment("greatest(?, ?)", field(i, ^key), field(s, ^key))
- })
- end
-
- defp select_joined_dimension(q, _dimension, key) do
- select_merge_as(q, [s, i], %{
- key => fragment("if(empty(?), ?, ?)", field(s, ^key), field(i, ^key), field(s, ^key))
- })
- end
-
- defp select_joined_metrics(q, []), do: q
- # NOTE: Reverse-engineering the native data bounces and total visit
- # durations to combine with imported data is inefficient. Instead both
- # queries should fetch bounces/total_visit_duration and visits and be
- # used as subqueries to a main query that then find the bounce rate/avg
- # visit_duration.
-
- defp select_joined_metrics(q, [:visits | rest]) do
- q
- |> select_merge_as([s, i], %{visits: s.visits + i.visits})
- |> select_joined_metrics(rest)
- end
-
- defp select_joined_metrics(q, [:visitors | rest]) do
- q
- |> select_merge_as([s, i], %{visitors: s.visitors + i.visitors})
- |> select_joined_metrics(rest)
- end
-
- defp select_joined_metrics(q, [:events | rest]) do
- q
- |> select_merge_as([s, i], %{events: s.events + i.events})
- |> select_joined_metrics(rest)
- end
-
- defp select_joined_metrics(q, [:pageviews | rest]) do
- q
- |> select_merge_as([s, i], %{pageviews: s.pageviews + i.pageviews})
- |> select_joined_metrics(rest)
- end
-
- defp select_joined_metrics(q, [:views_per_visit | rest]) do
- q
- |> select_merge_as([s, i], %{
- views_per_visit:
- fragment(
- "if(? + ? > 0, round((? + ? * ?) / (? + ?), 2), 0)",
- s.__internal_visits,
- i.__internal_visits,
- i.pageviews,
- s.views_per_visit,
- s.__internal_visits,
- i.__internal_visits,
- s.__internal_visits
- )
- })
- |> select_joined_metrics(rest)
- end
-
- defp select_joined_metrics(q, [:bounce_rate | rest]) do
- q
- |> select_merge_as([s, i], %{
- bounce_rate:
- fragment(
- "if(? + ? > 0, round(100 * (? + (? * ? / 100)) / (? + ?)), 0)",
- s.__internal_visits,
- i.__internal_visits,
- i.bounces,
- s.bounce_rate,
- s.__internal_visits,
- i.__internal_visits,
- s.__internal_visits
- )
- })
- |> select_joined_metrics(rest)
- end
-
- defp select_joined_metrics(q, [:visit_duration | rest]) do
- q
- |> select_merge_as([s, i], %{
- visit_duration:
- fragment(
- """
- if(
- ? + ? > 0,
- round((? + ? * ?) / (? + ?), 0),
- 0
- )
- """,
- s.__internal_visits,
- i.__internal_visits,
- i.visit_duration,
- s.visit_duration,
- s.__internal_visits,
- s.__internal_visits,
- i.__internal_visits
- )
- })
- |> select_joined_metrics(rest)
- end
-
- defp select_joined_metrics(q, [:sample_percent | rest]) do
- q
- |> select_merge_as([s, i], %{sample_percent: s.sample_percent})
- |> select_joined_metrics(rest)
- end
-
- defp select_joined_metrics(q, [_ | rest]) do
- q
- |> select_joined_metrics(rest)
- end
-
- defp naive_dimension_join(q1, q2, metrics) do
- from(a in subquery(q1),
- full_join: b in subquery(q2),
- on: a.dim0 == b.dim0,
- select: %{}
- )
- |> select_merge_as([a, b], %{
- dim0: fragment("if(? != 0, ?, ?)", a.dim0, a.dim0, b.dim0)
- })
- |> select_joined_metrics(metrics)
- end
end
diff --git a/lib/plausible/stats/imported/sql/builder.ex b/lib/plausible/stats/imported/sql/builder.ex
new file mode 100644
index 0000000000..cf81959d3f
--- /dev/null
+++ b/lib/plausible/stats/imported/sql/builder.ex
@@ -0,0 +1,462 @@
+defmodule Plausible.Stats.Imported.SQL.Builder do
+ @moduledoc """
+ This module is responsible for generating SQL/Ecto expressions
+ for dimensions, filters and metrics used in import table queries
+ """
+
+ use Plausible.Stats.SQL.Fragments
+
+ import Plausible.Stats.Util, only: [shortname: 2]
+ import Ecto.Query
+
+ @no_ref "Direct / None"
+ @not_set "(not set)"
+ @none "(none)"
+
+ def select_imported_metrics(q, []), do: q
+
+ def select_imported_metrics(q, [:visitors | rest]) do
+ q
+ |> select_merge([i], %{visitors: sum(i.visitors)})
+ |> select_imported_metrics(rest)
+ end
+
+ def select_imported_metrics(
+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_custom_events", _}}} = q,
+ [:events | rest]
+ ) do
+ q
+ |> select_merge([i], %{events: sum(i.events)})
+ |> select_imported_metrics(rest)
+ end
+
+ def select_imported_metrics(q, [:events | rest]) do
+ q
+ |> select_merge([i], %{events: sum(i.pageviews)})
+ |> select_imported_metrics(rest)
+ end
+
+ def select_imported_metrics(
+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q,
+ [:visits | rest]
+ ) do
+ q
+ |> select_merge([i], %{visits: sum(i.exits)})
+ |> select_imported_metrics(rest)
+ end
+
+ def select_imported_metrics(
+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,
+ [:visits | rest]
+ ) do
+ q
+ |> select_merge([i], %{visits: sum(i.entrances)})
+ |> select_imported_metrics(rest)
+ end
+
+ def select_imported_metrics(q, [:visits | rest]) do
+ q
+ |> select_merge([i], %{visits: sum(i.visits)})
+ |> select_imported_metrics(rest)
+ end
+
+ def select_imported_metrics(
+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_custom_events", _}}} = q,
+ [:pageviews | rest]
+ ) do
+ q
+ |> select_merge([i], %{pageviews: 0})
+ |> select_imported_metrics(rest)
+ end
+
+ def select_imported_metrics(q, [:pageviews | rest]) do
+ q
+ |> where([i], i.pageviews > 0)
+ |> select_merge([i], %{pageviews: sum(i.pageviews)})
+ |> select_imported_metrics(rest)
+ end
+
+ def select_imported_metrics(
+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_pages", _}}} = q,
+ [:bounce_rate | rest]
+ ) do
+ q
+ |> select_merge([i], %{
+ bounces: 0,
+ __internal_visits: 0
+ })
+ |> select_imported_metrics(rest)
+ end
+
+ def select_imported_metrics(
+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,
+ [:bounce_rate | rest]
+ ) do
+ q
+ |> select_merge([i], %{
+ bounces: sum(i.bounces),
+ __internal_visits: sum(i.entrances)
+ })
+ |> select_imported_metrics(rest)
+ end
+
+ def select_imported_metrics(
+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q,
+ [:bounce_rate | rest]
+ ) do
+ q
+ |> select_merge([i], %{
+ bounces: sum(i.bounces),
+ __internal_visits: sum(i.exits)
+ })
+ |> select_imported_metrics(rest)
+ end
+
+ def select_imported_metrics(q, [:bounce_rate | rest]) do
+ q
+ |> select_merge([i], %{
+ bounces: sum(i.bounces),
+ __internal_visits: sum(i.visits)
+ })
+ |> select_imported_metrics(rest)
+ end
+
+ def select_imported_metrics(
+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_pages", _}}} = q,
+ [:visit_duration | rest]
+ ) do
+ q
+ |> select_merge([i], %{
+ visit_duration: 0,
+ __internal_visits: 0
+ })
+ |> select_imported_metrics(rest)
+ end
+
+ def select_imported_metrics(
+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,
+ [:visit_duration | rest]
+ ) do
+ q
+ |> select_merge([i], %{
+ visit_duration: sum(i.visit_duration),
+ __internal_visits: sum(i.entrances)
+ })
+ |> select_imported_metrics(rest)
+ end
+
+ def select_imported_metrics(
+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q,
+ [:visit_duration | rest]
+ ) do
+ q
+ |> select_merge([i], %{
+ visit_duration: sum(i.visit_duration),
+ __internal_visits: sum(i.exits)
+ })
+ |> select_imported_metrics(rest)
+ end
+
+ def select_imported_metrics(q, [:visit_duration | rest]) do
+ q
+ |> select_merge([i], %{
+ visit_duration: sum(i.visit_duration),
+ __internal_visits: sum(i.visits)
+ })
+ |> select_imported_metrics(rest)
+ end
+
+ def select_imported_metrics(
+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,
+ [:views_per_visit | rest]
+ ) do
+ q
+ |> where([i], i.pageviews > 0)
+ |> select_merge([i], %{
+ pageviews: sum(i.pageviews),
+ __internal_visits: sum(i.entrances)
+ })
+ |> select_imported_metrics(rest)
+ end
+
+ def select_imported_metrics(
+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q,
+ [:views_per_visit | rest]
+ ) do
+ q
+ |> where([i], i.pageviews > 0)
+ |> select_merge([i], %{
+ pageviews: sum(i.pageviews),
+ __internal_visits: sum(i.exits)
+ })
+ |> select_imported_metrics(rest)
+ end
+
+ def select_imported_metrics(q, [:views_per_visit | rest]) do
+ q
+ |> where([i], i.pageviews > 0)
+ |> select_merge([i], %{
+ pageviews: sum(i.pageviews),
+ __internal_visits: sum(i.visits)
+ })
+ |> select_imported_metrics(rest)
+ end
+
+ def select_imported_metrics(q, [_ | rest]) do
+ q
+ |> select_imported_metrics(rest)
+ end
+
+ def group_imported_by(q, query) do
+ Enum.reduce(query.dimensions, q, fn dimension, q ->
+ q
+ |> select_group_fields(dimension, shortname(query, dimension), query)
+ |> filter_group_values(dimension)
+ |> group_by([], selected_as(^shortname(query, dimension)))
+ end)
+ end
+
+ defp select_group_fields(q, dimension, key, _query)
+ when dimension in ["visit:source", "visit:referrer"] do
+ select_merge_as(q, [i], %{
+ key =>
+ fragment(
+ "if(empty(?), ?, ?)",
+ field(i, ^dim(dimension)),
+ @no_ref,
+ field(i, ^dim(dimension))
+ )
+ })
+ end
+
+ defp select_group_fields(q, "event:page", key, _query) do
+ select_merge_as(q, [i], %{key => i.page, time_on_page: sum(i.time_on_page)})
+ end
+
+ defp select_group_fields(q, dimension, key, _query)
+ when dimension in ["visit:device", "visit:browser"] do
+ select_merge_as(q, [i], %{
+ key =>
+ fragment(
+ "if(empty(?), ?, ?)",
+ field(i, ^dim(dimension)),
+ @not_set,
+ field(i, ^dim(dimension))
+ )
+ })
+ end
+
+ defp select_group_fields(q, "visit:browser_version", key, _query) do
+ select_merge_as(q, [i], %{
+ key => fragment("if(empty(?), ?, ?)", i.browser_version, @not_set, i.browser_version)
+ })
+ end
+
+ defp select_group_fields(q, "visit:os", key, _query) do
+ select_merge_as(q, [i], %{
+ key => fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system)
+ })
+ end
+
+ defp select_group_fields(q, "visit:os_version", key, _query) do
+ select_merge_as(q, [i], %{
+ key =>
+ fragment(
+ "if(empty(?), ?, ?)",
+ i.operating_system_version,
+ @not_set,
+ i.operating_system_version
+ )
+ })
+ end
+
+ defp select_group_fields(q, "event:props:url", key, _query) do
+ select_merge_as(q, [i], %{
+ key => fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none)
+ })
+ end
+
+ defp select_group_fields(q, "event:props:path", key, _query) do
+ select_merge_as(q, [i], %{
+ key => fragment("if(not empty(?), ?, ?)", i.path, i.path, @none)
+ })
+ end
+
+ defp select_group_fields(q, "time:month", key, _query) do
+ select_merge_as(q, [i], %{key => fragment("toStartOfMonth(?)", i.date)})
+ end
+
+ defp select_group_fields(q, dimension, key, _query)
+ when dimension in ["time:hour", "time:day"] do
+ select_merge_as(q, [i], %{key => i.date})
+ end
+
+ defp select_group_fields(q, "time:week", key, query) do
+ select_merge_as(q, [i], %{
+ key => weekstart_not_before(i.date, ^query.date_range.first)
+ })
+ end
+
+ defp select_group_fields(q, dimension, key, _query) do
+ select_merge_as(q, [i], %{key => field(i, ^dim(dimension))})
+ end
+
+ @utm_dimensions [
+ "visit:utm_source",
+ "visit:utm_medium",
+ "visit:utm_campaign",
+ "visit:utm_term",
+ "visit:utm_content"
+ ]
+ defp filter_group_values(q, dimension) when dimension in @utm_dimensions do
+ dim = Plausible.Stats.Filters.without_prefix(dimension)
+
+ where(q, [i], fragment("not empty(?)", field(i, ^dim)))
+ end
+
+ defp filter_group_values(q, "visit:country"), do: where(q, [i], i.country != "ZZ")
+ defp filter_group_values(q, "visit:region"), do: where(q, [i], i.region != "")
+ defp filter_group_values(q, "visit:city"), do: where(q, [i], i.city != 0 and not is_nil(i.city))
+
+ defp filter_group_values(q, _dimension), do: q
+
+ def select_joined_dimensions(q, query) do
+ Enum.reduce(query.dimensions, q, fn dimension, q ->
+ select_joined_dimension(q, dimension, shortname(query, dimension))
+ end)
+ end
+
+ defp select_joined_dimension(q, "visit:city", key) do
+ select_merge_as(q, [s, i], %{
+ key => fragment("greatest(?,?)", field(i, ^key), field(s, ^key))
+ })
+ end
+
+ defp select_joined_dimension(q, "time:" <> _, key) do
+ select_merge_as(q, [s, i], %{
+ key => fragment("greatest(?, ?)", field(i, ^key), field(s, ^key))
+ })
+ end
+
+ defp select_joined_dimension(q, _dimension, key) do
+ select_merge_as(q, [s, i], %{
+ key => fragment("if(empty(?), ?, ?)", field(s, ^key), field(i, ^key), field(s, ^key))
+ })
+ end
+
+ def select_joined_metrics(q, []), do: q
+ # NOTE: Reverse-engineering the native data bounces and total visit
+ # durations to combine with imported data is inefficient. Instead both
+ # queries should fetch bounces/total_visit_duration and visits and be
+ # used as subqueries to a main query that then find the bounce rate/avg
+ # visit_duration.
+
+ def select_joined_metrics(q, [:visits | rest]) do
+ q
+ |> select_merge_as([s, i], %{visits: s.visits + i.visits})
+ |> select_joined_metrics(rest)
+ end
+
+ def select_joined_metrics(q, [:visitors | rest]) do
+ q
+ |> select_merge_as([s, i], %{visitors: s.visitors + i.visitors})
+ |> select_joined_metrics(rest)
+ end
+
+ def select_joined_metrics(q, [:events | rest]) do
+ q
+ |> select_merge_as([s, i], %{events: s.events + i.events})
+ |> select_joined_metrics(rest)
+ end
+
+ def select_joined_metrics(q, [:pageviews | rest]) do
+ q
+ |> select_merge_as([s, i], %{pageviews: s.pageviews + i.pageviews})
+ |> select_joined_metrics(rest)
+ end
+
+ def select_joined_metrics(q, [:views_per_visit | rest]) do
+ q
+ |> select_merge_as([s, i], %{
+ views_per_visit:
+ fragment(
+ "if(? + ? > 0, round((? + ? * ?) / (? + ?), 2), 0)",
+ s.__internal_visits,
+ i.__internal_visits,
+ i.pageviews,
+ s.views_per_visit,
+ s.__internal_visits,
+ i.__internal_visits,
+ s.__internal_visits
+ )
+ })
+ |> select_joined_metrics(rest)
+ end
+
+ def select_joined_metrics(q, [:bounce_rate | rest]) do
+ q
+ |> select_merge_as([s, i], %{
+ bounce_rate:
+ fragment(
+ "if(? + ? > 0, round(100 * (? + (? * ? / 100)) / (? + ?)), 0)",
+ s.__internal_visits,
+ i.__internal_visits,
+ i.bounces,
+ s.bounce_rate,
+ s.__internal_visits,
+ i.__internal_visits,
+ s.__internal_visits
+ )
+ })
+ |> select_joined_metrics(rest)
+ end
+
+ def select_joined_metrics(q, [:visit_duration | rest]) do
+ q
+ |> select_merge_as([s, i], %{
+ visit_duration:
+ fragment(
+ """
+ if(
+ ? + ? > 0,
+ round((? + ? * ?) / (? + ?), 0),
+ 0
+ )
+ """,
+ s.__internal_visits,
+ i.__internal_visits,
+ i.visit_duration,
+ s.visit_duration,
+ s.__internal_visits,
+ s.__internal_visits,
+ i.__internal_visits
+ )
+ })
+ |> select_joined_metrics(rest)
+ end
+
+ def select_joined_metrics(q, [:sample_percent | rest]) do
+ q
+ |> select_merge_as([s, i], %{sample_percent: s.sample_percent})
+ |> select_joined_metrics(rest)
+ end
+
+ def select_joined_metrics(q, [_ | rest]) do
+ q
+ |> select_joined_metrics(rest)
+ end
+
+ def naive_dimension_join(q1, q2, metrics) do
+ from(a in subquery(q1),
+ full_join: b in subquery(q2),
+ on: a.dim0 == b.dim0,
+ select: %{}
+ )
+ |> select_merge_as([a, b], %{
+ dim0: fragment("if(? != 0, ?, ?)", a.dim0, a.dim0, b.dim0)
+ })
+ |> select_joined_metrics(metrics)
+ end
+
+ defp dim(dimension), do: Plausible.Stats.Filters.without_prefix(dimension)
+end
diff --git a/lib/plausible/stats/props.ex b/lib/plausible/stats/legacy/legacy_dimensions.ex
similarity index 58%
rename from lib/plausible/stats/props.ex
rename to lib/plausible/stats/legacy/legacy_dimensions.ex
index c0f1bff9dd..3de9c023af 100644
--- a/lib/plausible/stats/props.ex
+++ b/lib/plausible/stats/legacy/legacy_dimensions.ex
@@ -1,4 +1,7 @@
-defmodule Plausible.Stats.Props do
+defmodule Plausible.Stats.Legacy.Dimensions do
+ @moduledoc """
+ Deprecated module. See QueryParser for list of valid dimensions
+ """
@event_props ["event:page", "event:name", "event:goal", "event:hostname"]
@session_props [
"visit:source",
@@ -20,8 +23,8 @@ defmodule Plausible.Stats.Props do
"visit:browser_version"
]
- def valid_prop?(prop) when prop in @event_props, do: true
- def valid_prop?(prop) when prop in @session_props, do: true
- def valid_prop?("event:props:" <> prop) when byte_size(prop) > 0, do: true
- def valid_prop?(_), do: false
+ def valid?(prop) when prop in @event_props, do: true
+ def valid?(prop) when prop in @session_props, do: true
+ def valid?("event:props:" <> prop) when byte_size(prop) > 0, do: true
+ def valid?(_), do: false
end
diff --git a/lib/plausible/stats/legacy/legacy_query_builder.ex b/lib/plausible/stats/legacy/legacy_query_builder.ex
new file mode 100644
index 0000000000..9ec14a713d
--- /dev/null
+++ b/lib/plausible/stats/legacy/legacy_query_builder.ex
@@ -0,0 +1,180 @@
+defmodule Plausible.Stats.Legacy.QueryBuilder do
+ @moduledoc false
+
+ use Plausible
+
+ alias Plausible.Stats.{Filters, Interval, Query}
+
+ def from(site, params) do
+ now = NaiveDateTime.utc_now(:second)
+
+ query =
+ Query
+ |> struct!(now: now, timezone: site.timezone)
+ |> put_period(site, params)
+ |> put_dimensions(params)
+ |> put_interval(params)
+ |> put_parsed_filters(params)
+ |> Query.put_experimental_reduced_joins(site, params)
+ |> Query.put_imported_opts(site, params)
+
+ on_ee do
+ query = Plausible.Stats.Sampling.put_threshold(query, params)
+ end
+
+ query
+ end
+
+ defp put_period(query, site, %{"period" => "realtime"}) do
+ date = today(site.timezone)
+
+ struct!(query, period: "realtime", date_range: Date.range(date, date))
+ end
+
+ defp put_period(query, site, %{"period" => "day"} = params) do
+ date = parse_single_date(site.timezone, params)
+
+ struct!(query, period: "day", date_range: Date.range(date, date))
+ end
+
+ defp put_period(query, site, %{"period" => "7d"} = params) do
+ end_date = parse_single_date(site.timezone, params)
+ start_date = end_date |> Timex.shift(days: -6)
+
+ struct!(
+ query,
+ period: "7d",
+ date_range: Date.range(start_date, end_date)
+ )
+ end
+
+ defp put_period(query, site, %{"period" => "30d"} = params) do
+ end_date = parse_single_date(site.timezone, params)
+ start_date = end_date |> Timex.shift(days: -30)
+
+ struct!(query, period: "30d", date_range: Date.range(start_date, end_date))
+ end
+
+ defp put_period(query, site, %{"period" => "month"} = params) do
+ date = parse_single_date(site.timezone, params)
+
+ start_date = Timex.beginning_of_month(date)
+ end_date = Timex.end_of_month(date)
+
+ struct!(query,
+ period: "month",
+ date_range: Date.range(start_date, end_date)
+ )
+ end
+
+ defp put_period(query, site, %{"period" => "6mo"} = params) do
+ end_date =
+ parse_single_date(site.timezone, params)
+ |> Timex.end_of_month()
+
+ start_date =
+ Timex.shift(end_date, months: -5)
+ |> Timex.beginning_of_month()
+
+ struct!(query,
+ period: "6mo",
+ date_range: Date.range(start_date, end_date)
+ )
+ end
+
+ defp put_period(query, site, %{"period" => "12mo"} = params) do
+ end_date =
+ parse_single_date(site.timezone, params)
+ |> Timex.end_of_month()
+
+ start_date =
+ Timex.shift(end_date, months: -11)
+ |> Timex.beginning_of_month()
+
+ struct!(query,
+ period: "12mo",
+ date_range: Date.range(start_date, end_date)
+ )
+ end
+
+ defp put_period(query, site, %{"period" => "year"} = params) do
+ end_date =
+ parse_single_date(site.timezone, params)
+ |> Timex.end_of_year()
+
+ start_date = Timex.beginning_of_year(end_date)
+
+ struct!(query,
+ period: "year",
+ date_range: Date.range(start_date, end_date)
+ )
+ end
+
+ defp put_period(query, site, %{"period" => "all"}) do
+ now = today(site.timezone)
+ start_date = Plausible.Sites.stats_start_date(site) || now
+
+ struct!(query,
+ period: "all",
+ date_range: Date.range(start_date, now)
+ )
+ end
+
+ defp put_period(query, site, %{"period" => "custom", "from" => from, "to" => to} = params) do
+ new_params =
+ params
+ |> Map.drop(["from", "to"])
+ |> Map.put("date", Enum.join([from, to], ","))
+
+ put_period(query, site, new_params)
+ end
+
+ defp put_period(query, _site, %{"period" => "custom", "date" => date}) do
+ [from, to] = String.split(date, ",")
+ from_date = Date.from_iso8601!(String.trim(from))
+ to_date = Date.from_iso8601!(String.trim(to))
+
+ struct!(query,
+ period: "custom",
+ date_range: Date.range(from_date, to_date)
+ )
+ end
+
+ defp put_period(query, site, params) do
+ put_period(query, site, Map.merge(params, %{"period" => "30d"}))
+ end
+
+ defp put_dimensions(query, params) do
+ if not is_nil(params["property"]) do
+ struct!(query, dimensions: [params["property"]])
+ else
+ struct!(query, dimensions: Map.get(params, "dimensions", []))
+ end
+ end
+
+ defp put_interval(%{:period => "all"} = query, params) do
+ interval = Map.get(params, "interval", Interval.default_for_date_range(query.date_range))
+ struct!(query, interval: interval)
+ end
+
+ defp put_interval(query, params) do
+ interval = Map.get(params, "interval", Interval.default_for_period(query.period))
+ struct!(query, interval: interval)
+ end
+
+ defp put_parsed_filters(query, params) do
+ struct!(query, filters: Filters.parse(params["filters"]))
+ end
+
+ defp today(tz) do
+ Timex.now(tz) |> Timex.to_date()
+ end
+
+ defp parse_single_date(tz, params) do
+ case params["date"] do
+ "today" -> Timex.now(tz) |> Timex.to_date()
+ date when is_binary(date) -> Date.from_iso8601!(date)
+ _ -> today(tz)
+ end
+ end
+end
diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex
index 9d2eb0d667..7de4e931b4 100644
--- a/lib/plausible/stats/query.ex
+++ b/lib/plausible/stats/query.ex
@@ -24,44 +24,31 @@ defmodule Plausible.Stats.Query do
}
require OpenTelemetry.Tracer, as: Tracer
- alias Plausible.Stats.{Filters, Interval, Imported}
+ alias Plausible.Stats.{Filters, Imported, Legacy}
@type t :: %__MODULE__{}
- @spec from(Plausible.Site.t(), map()) :: t()
- def from(site, params) do
- now = NaiveDateTime.utc_now(:second)
-
- query =
- __MODULE__
- |> struct!(now: now, timezone: site.timezone)
- |> put_experimental_reduced_joins(site, params)
- |> put_period(site, params)
- |> put_dimensions(params)
- |> put_interval(params)
- |> put_parsed_filters(params)
- |> put_imported_opts(site, params)
-
- on_ee do
- query = Plausible.Stats.Sampling.put_threshold(query, params)
- end
-
- query
- end
-
def build(site, params) do
with {:ok, query_data} <- Filters.QueryParser.parse(site, params) do
query =
struct!(__MODULE__, Map.to_list(query_data))
|> put_imported_opts(site, %{})
|> put_experimental_reduced_joins(site, params)
- |> struct!(v2: true)
+ |> struct!(v2: true, now: NaiveDateTime.utc_now(:second))
{:ok, query}
end
end
- defp put_experimental_reduced_joins(query, site, params) do
+ @doc """
+ Builds query from old-style params. New code should prefer Query.build
+ """
+ @spec from(Plausible.Site.t(), map()) :: t()
+ def from(site, params) do
+ Legacy.QueryBuilder.from(site, params)
+ end
+
+ def put_experimental_reduced_joins(query, site, params) do
if Map.has_key?(params, "experimental_reduced_joins") do
struct!(query,
experimental_reduced_joins?: Map.get(params, "experimental_reduced_joins") == "true"
@@ -73,147 +60,6 @@ defmodule Plausible.Stats.Query do
end
end
- defp put_period(query, site, %{"period" => "realtime"}) do
- date = today(site.timezone)
-
- struct!(query, period: "realtime", date_range: Date.range(date, date))
- end
-
- defp put_period(query, site, %{"period" => "day"} = params) do
- date = parse_single_date(site.timezone, params)
-
- struct!(query, period: "day", date_range: Date.range(date, date))
- end
-
- defp put_period(query, site, %{"period" => "7d"} = params) do
- end_date = parse_single_date(site.timezone, params)
- start_date = end_date |> Timex.shift(days: -6)
-
- struct!(
- query,
- period: "7d",
- date_range: Date.range(start_date, end_date)
- )
- end
-
- defp put_period(query, site, %{"period" => "30d"} = params) do
- end_date = parse_single_date(site.timezone, params)
- start_date = end_date |> Timex.shift(days: -30)
-
- struct!(query, period: "30d", date_range: Date.range(start_date, end_date))
- end
-
- defp put_period(query, site, %{"period" => "month"} = params) do
- date = parse_single_date(site.timezone, params)
-
- start_date = Timex.beginning_of_month(date)
- end_date = Timex.end_of_month(date)
-
- struct!(query,
- period: "month",
- date_range: Date.range(start_date, end_date)
- )
- end
-
- defp put_period(query, site, %{"period" => "6mo"} = params) do
- end_date =
- parse_single_date(site.timezone, params)
- |> Timex.end_of_month()
-
- start_date =
- Timex.shift(end_date, months: -5)
- |> Timex.beginning_of_month()
-
- struct!(query,
- period: "6mo",
- date_range: Date.range(start_date, end_date)
- )
- end
-
- defp put_period(query, site, %{"period" => "12mo"} = params) do
- end_date =
- parse_single_date(site.timezone, params)
- |> Timex.end_of_month()
-
- start_date =
- Timex.shift(end_date, months: -11)
- |> Timex.beginning_of_month()
-
- struct!(query,
- period: "12mo",
- date_range: Date.range(start_date, end_date)
- )
- end
-
- defp put_period(query, site, %{"period" => "year"} = params) do
- end_date =
- parse_single_date(site.timezone, params)
- |> Timex.end_of_year()
-
- start_date = Timex.beginning_of_year(end_date)
-
- struct!(query,
- period: "year",
- date_range: Date.range(start_date, end_date)
- )
- end
-
- defp put_period(query, site, %{"period" => "all"}) do
- now = today(site.timezone)
- start_date = Plausible.Sites.stats_start_date(site) || now
-
- struct!(query,
- period: "all",
- date_range: Date.range(start_date, now)
- )
- end
-
- defp put_period(query, site, %{"period" => "custom", "from" => from, "to" => to} = params) do
- new_params =
- params
- |> Map.drop(["from", "to"])
- |> Map.put("date", Enum.join([from, to], ","))
-
- put_period(query, site, new_params)
- end
-
- defp put_period(query, _site, %{"period" => "custom", "date" => date}) do
- [from, to] = String.split(date, ",")
- from_date = Date.from_iso8601!(String.trim(from))
- to_date = Date.from_iso8601!(String.trim(to))
-
- struct!(query,
- period: "custom",
- date_range: Date.range(from_date, to_date)
- )
- end
-
- defp put_period(query, site, params) do
- put_period(query, site, Map.merge(params, %{"period" => "30d"}))
- end
-
- defp put_dimensions(query, params) do
- if not is_nil(params["property"]) do
- struct!(query, dimensions: [params["property"]])
- else
- struct!(query, dimensions: Map.get(params, "dimensions", []))
- end
- end
-
- defp put_interval(%{:period => "all"} = query, params) do
- interval = Map.get(params, "interval", Interval.default_for_date_range(query.date_range))
- struct!(query, interval: interval)
- end
-
- defp put_interval(query, params) do
- interval = Map.get(params, "interval", Interval.default_for_period(query.period))
- struct!(query, interval: interval)
- end
-
- defp put_parsed_filters(query, params) do
- struct!(query, filters: Filters.parse(params["filters"]))
- end
-
def set(query, keywords) do
new_query = struct!(query, keywords)
@@ -224,26 +70,7 @@ defmodule Plausible.Stats.Query do
end
end
- @spec set_dimensions(t(), list(String.t())) :: t()
- def set_dimensions(query, dimensions) do
- query
- |> struct!(dimensions: dimensions)
- |> refresh_imported_opts()
- end
-
- def set_metrics(query, metrics) do
- query
- |> struct!(metrics: metrics)
- |> refresh_imported_opts()
- end
-
- def set_order_by(query, order_by) do
- query
- |> struct!(order_by: order_by)
- |> refresh_imported_opts()
- end
-
- def put_filter(query, filter) do
+ def add_filter(query, filter) do
query
|> struct!(filters: query.filters ++ [filter])
|> refresh_imported_opts()
@@ -260,13 +87,6 @@ defmodule Plausible.Stats.Query do
|> refresh_imported_opts()
end
- def exclude_imported(query) do
- struct!(query,
- include_imported: false,
- skip_imported_reason: :manual_exclusion
- )
- end
-
defp refresh_imported_opts(query) do
put_imported_opts(query, nil, %{})
end
@@ -289,19 +109,7 @@ defmodule Plausible.Stats.Query do
end)
end
- defp today(tz) do
- Timex.now(tz) |> Timex.to_date()
- end
-
- defp parse_single_date(tz, params) do
- case params["date"] do
- "today" -> Timex.now(tz) |> Timex.to_date()
- date when is_binary(date) -> Date.from_iso8601!(date)
- _ -> today(tz)
- end
- end
-
- defp put_imported_opts(query, site, params) do
+ def put_imported_opts(query, site, params) do
requested? = params["with_imported"] == "true" || query.include.imports
latest_import_end_date =
diff --git a/lib/plausible/stats/sql/query_builder.ex b/lib/plausible/stats/sql/query_builder.ex
index ef61a59758..aaf3eb8c9d 100644
--- a/lib/plausible/stats/sql/query_builder.ex
+++ b/lib/plausible/stats/sql/query_builder.ex
@@ -32,7 +32,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
from(
e in "events_v2",
where: ^SQL.WhereBuilder.build(:events, site, events_query),
- select: ^Base.select_event_metrics(events_query.metrics)
+ select: ^select_event_metrics(events_query)
)
on_ee do
@@ -73,7 +73,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
from(
e in "sessions_v2",
where: ^SQL.WhereBuilder.build(:sessions, site, sessions_query),
- select: ^Base.select_session_metrics(sessions_query.metrics, sessions_query)
+ select: ^select_session_metrics(sessions_query)
)
on_ee do
@@ -111,6 +111,18 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
end
end
+ defp select_event_metrics(query) do
+ query.metrics
+ |> Enum.map(&SQL.Expression.event_metric/1)
+ |> Enum.reduce(%{}, &Map.merge/2)
+ end
+
+ defp select_session_metrics(query) do
+ query.metrics
+ |> Enum.map(&SQL.Expression.session_metric(&1, query))
+ |> Enum.reduce(%{}, &Map.merge/2)
+ end
+
defp build_group_by(q, table, query) do
Enum.reduce(query.dimensions, q, &dimension_group_by(&2, table, query, &1))
end
diff --git a/lib/plausible/stats/sql/where_builder.ex b/lib/plausible/stats/sql/where_builder.ex
index 73eeecaeef..5f882e0609 100644
--- a/lib/plausible/stats/sql/where_builder.ex
+++ b/lib/plausible/stats/sql/where_builder.ex
@@ -4,7 +4,8 @@ defmodule Plausible.Stats.SQL.WhereBuilder do
"""
import Ecto.Query
- import Plausible.Stats.Base, only: [page_regex: 1, utc_boundaries: 2]
+ import Plausible.Stats.Time, only: [utc_boundaries: 2]
+ import Plausible.Stats.Filters.Utils, only: [page_regex: 1]
alias Plausible.Stats.Query
diff --git a/lib/plausible/stats/time.ex b/lib/plausible/stats/time.ex
index c8c0d7a71f..83296bffaa 100644
--- a/lib/plausible/stats/time.ex
+++ b/lib/plausible/stats/time.ex
@@ -4,6 +4,57 @@ defmodule Plausible.Stats.Time do
"""
alias Plausible.Stats.Query
+ alias Plausible.Timezones
+
+ def utc_boundaries(%Query{period: "realtime", now: now}, site) do
+ last_datetime =
+ now
+ |> Timex.shift(seconds: 5)
+ |> beginning_of_time(site.native_stats_start_at)
+ |> NaiveDateTime.truncate(:second)
+
+ first_datetime =
+ now |> Timex.shift(minutes: -5) |> NaiveDateTime.truncate(:second)
+
+ {first_datetime, last_datetime}
+ end
+
+ def utc_boundaries(%Query{period: "30m", now: now}, site) do
+ last_datetime =
+ now
+ |> Timex.shift(seconds: 5)
+ |> beginning_of_time(site.native_stats_start_at)
+ |> NaiveDateTime.truncate(:second)
+
+ first_datetime =
+ now |> Timex.shift(minutes: -30) |> NaiveDateTime.truncate(:second)
+
+ {first_datetime, last_datetime}
+ end
+
+ def utc_boundaries(%Query{date_range: date_range}, site) do
+ {:ok, first} = NaiveDateTime.new(date_range.first, ~T[00:00:00])
+
+ first_datetime =
+ first
+ |> Timezones.to_utc_datetime(site.timezone)
+ |> beginning_of_time(site.native_stats_start_at)
+
+ {:ok, last} = NaiveDateTime.new(date_range.last |> Timex.shift(days: 1), ~T[00:00:00])
+
+ last_datetime = Timezones.to_utc_datetime(last, site.timezone)
+
+ {first_datetime, last_datetime}
+ end
+
+ defp beginning_of_time(candidate, native_stats_start_at) do
+ if Timex.after?(native_stats_start_at, candidate) do
+ native_stats_start_at
+ else
+ candidate
+ end
+ end
+
def format_datetime(%Date{} = date), do: Date.to_string(date)
def format_datetime(%DateTime{} = datetime),
diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex
index 5fc3938ed0..513debc597 100644
--- a/lib/plausible/stats/timeseries.ex
+++ b/lib/plausible/stats/timeseries.ex
@@ -1,4 +1,10 @@
defmodule Plausible.Stats.Timeseries do
+ @moduledoc """
+ Builds timeseries results for v1 of our stats API and dashboards.
+
+ Avoid adding new logic here - update QueryBuilder etc instead.
+ """
+
use Plausible
use Plausible.ClickhouseRepo
alias Plausible.Stats.{Query, QueryOptimizer, QueryResult, SQL}
diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex
index 248f709baf..e282ca97b4 100644
--- a/lib/plausible_web/controllers/api/external_stats_controller.ex
+++ b/lib/plausible_web/controllers/api/external_stats_controller.ex
@@ -76,7 +76,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
{:error,
"Property 'event:hostname' is currently not supported for breakdowns. Please provide a valid property for the breakdown endpoint: https://plausible.io/docs/stats-api#properties"}
- Plausible.Stats.Props.valid_prop?(property) ->
+ Plausible.Stats.Legacy.Dimensions.valid?(property) ->
:ok
true ->
@@ -363,7 +363,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
end
defp validate_filter(_site, [_, property | _]) do
- if Plausible.Stats.Props.valid_prop?(property) do
+ if Plausible.Stats.Legacy.Dimensions.valid?(property) do
:ok
else
{:error,
diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
index e5f01eda35..b3beef0e2b 100644
--- a/lib/plausible_web/controllers/api/stats_controller.ex
+++ b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -777,7 +777,7 @@ defmodule PlausibleWeb.Api.StatsController do
query =
Query.from(site, params)
- |> Query.put_filter(referrer_filter)
+ |> Query.add_filter(referrer_filter)
pagination = parse_pagination(params)
@@ -907,9 +907,9 @@ defmodule PlausibleWeb.Api.StatsController do
total_pageviews_query =
query
|> Query.remove_filters(["visit:exit_page"])
- |> Query.put_filter([:is, "event:page", pages])
- |> Query.put_filter([:is, "event:name", ["pageview"]])
- |> Query.set_dimensions(["event:page"])
+ |> Query.add_filter([:is, "event:page", pages])
+ |> Query.add_filter([:is, "event:name", ["pageview"]])
+ |> Query.set(dimensions: ["event:page"])
total_pageviews =
Stats.breakdown(site, total_pageviews_query, [:pageviews], {limit, 1})
diff --git a/test/plausible/stats/legacy_dashboard_filter_parser_test.exs b/test/plausible/stats/legacy_dashboard_filter_parser_test.exs
index 8a3e94053a..ffdcc558a4 100644
--- a/test/plausible/stats/legacy_dashboard_filter_parser_test.exs
+++ b/test/plausible/stats/legacy_dashboard_filter_parser_test.exs
@@ -1,4 +1,4 @@
-defmodule Plausible.Stats.LegacyDashboardFilterParserTest do
+defmodule Plausible.Stats.Legacy.DashboardFilterParserTest do
use ExUnit.Case, async: true
alias Plausible.Stats.Filters.LegacyDashboardFilterParser
diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exs
index 62c919319f..cd6a606688 100644
--- a/test/plausible/stats/query_test.exs
+++ b/test/plausible/stats/query_test.exs
@@ -21,11 +21,11 @@ defmodule Plausible.Stats.QueryTest do
} do
q1 = %{now: %NaiveDateTime{}} = Query.from(site, %{"period" => "realtime"})
q2 = %{now: %NaiveDateTime{}} = Query.from(site, %{"period" => "30m"})
- boundaries1 = Plausible.Stats.Base.utc_boundaries(q1, site)
- boundaries2 = Plausible.Stats.Base.utc_boundaries(q2, site)
+ boundaries1 = Plausible.Stats.Time.utc_boundaries(q1, site)
+ boundaries2 = Plausible.Stats.Time.utc_boundaries(q2, site)
:timer.sleep(1500)
- assert ^boundaries1 = Plausible.Stats.Base.utc_boundaries(q1, site)
- assert ^boundaries2 = Plausible.Stats.Base.utc_boundaries(q2, site)
+ assert ^boundaries1 = Plausible.Stats.Time.utc_boundaries(q1, site)
+ assert ^boundaries2 = Plausible.Stats.Time.utc_boundaries(q2, site)
end
test "parses day format", %{site: site} do
diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_goal_dimension_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_goal_dimension_test.exs
new file mode 100644
index 0000000000..643394e4f5
--- /dev/null
+++ b/test/plausible_web/controllers/api/external_stats_controller/query_goal_dimension_test.exs
@@ -0,0 +1,133 @@
+defmodule PlausibleWeb.Api.ExternalStatsController.QueryGoalDimensionTest do
+ use PlausibleWeb.ConnCase
+
+ @user_id 1231
+
+ setup [:create_user, :create_new_site, :create_api_key, :use_api_key]
+
+ describe "breakdown by event:goal" do
+ test "returns custom event goals and pageview goals", %{conn: conn, site: site} do
+ insert(:goal, %{site: site, event_name: "Purchase"})
+ insert(:goal, %{site: site, page_path: "/test"})
+
+ populate_stats(site, [
+ build(:pageview,
+ timestamp: ~N[2021-01-01 00:00:01],
+ pathname: "/test"
+ ),
+ build(:event,
+ name: "Purchase",
+ timestamp: ~N[2021-01-01 00:00:03]
+ ),
+ build(:event,
+ name: "Purchase",
+ timestamp: ~N[2021-01-01 00:00:03]
+ )
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["visitors"],
+ "dimensions" => ["event:goal"]
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["Purchase"], "metrics" => [2]},
+ %{"dimensions" => ["Visit /test"], "metrics" => [1]}
+ ]
+ end
+
+ test "returns pageview goals containing wildcards", %{conn: conn, site: site} do
+ insert(:goal, %{site: site, page_path: "/**/post"})
+ insert(:goal, %{site: site, page_path: "/blog**"})
+
+ populate_stats(site, [
+ build(:pageview, pathname: "/blog", user_id: @user_id),
+ build(:pageview, pathname: "/blog/post-1", user_id: @user_id),
+ build(:pageview, pathname: "/blog/post-2", user_id: @user_id),
+ build(:pageview, pathname: "/blog/something/post"),
+ build(:pageview, pathname: "/different/page/post")
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["visitors", "pageviews"],
+ "dimensions" => ["event:goal"],
+ "order_by" => [["pageviews", "desc"]]
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["Visit /blog**"], "metrics" => [2, 4]},
+ %{"dimensions" => ["Visit /**/post"], "metrics" => [2, 2]}
+ ]
+ end
+
+ test "does not return goals that are not configured for the site", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, pathname: "/register"),
+ build(:event, name: "Signup")
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["visitors", "pageviews"],
+ "dimensions" => ["event:goal"]
+ })
+
+ assert json_response(conn, 200)["results"] == []
+ end
+
+ test "returns conversion_rate in an event:goal breakdown", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:event, name: "Signup", user_id: 1),
+ build(:event, name: "Signup", user_id: 1),
+ build(:pageview, pathname: "/blog"),
+ build(:pageview, pathname: "/blog/post"),
+ build(:pageview)
+ ])
+
+ insert(:goal, %{site: site, event_name: "Signup"})
+ insert(:goal, %{site: site, page_path: "/blog**"})
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["visitors", "events", "conversion_rate"],
+ "dimensions" => ["event:goal"]
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["Visit /blog**"], "metrics" => [2, 2, 50.0]},
+ %{"dimensions" => ["Signup"], "metrics" => [1, 2, 25.0]}
+ ]
+ end
+
+ test "returns conversion_rate alone in an event:goal breakdown", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:event, name: "Signup", user_id: 1),
+ build(:pageview)
+ ])
+
+ insert(:goal, %{site: site, event_name: "Signup"})
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["conversion_rate"],
+ "dimensions" => ["event:goal"]
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["Signup"], "metrics" => [50.0]}
+ ]
+ end
+ end
+end
diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs
new file mode 100644
index 0000000000..2e395a6acd
--- /dev/null
+++ b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs
@@ -0,0 +1,695 @@
+defmodule PlausibleWeb.Api.ExternalStatsController.QueryImportedTest do
+ use PlausibleWeb.ConnCase
+
+ @user_id 1231
+
+ setup [:create_user, :create_new_site, :create_api_key, :use_api_key]
+
+ describe "aggregation with imported data" do
+ setup :create_site_import
+
+ test "does not count imported stats unless specified", %{
+ conn: conn,
+ site: site,
+ site_import: site_import
+ } do
+ populate_stats(site, site_import.id, [
+ build(:imported_visitors, date: ~D[2023-01-01]),
+ build(:pageview, timestamp: ~N[2023-01-01 00:00:00])
+ ])
+
+ query_params = %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["pageviews"]
+ }
+
+ conn1 = post(conn, "/api/v2/query", query_params)
+
+ assert json_response(conn1, 200)["results"] == [%{"metrics" => [1], "dimensions" => []}]
+
+ conn2 = post(conn, "/api/v2/query", Map.put(query_params, "include", %{"imports" => true}))
+
+ assert json_response(conn2, 200)["results"] == [%{"metrics" => [2], "dimensions" => []}]
+ refute json_response(conn2, 200)["meta"]["warning"]
+ end
+ end
+
+ test "breaks down all metrics by visit:referrer with imported data", %{conn: conn, site: site} do
+ site_import =
+ insert(:site_import,
+ site: site,
+ start_date: ~D[2005-01-01],
+ end_date: Timex.today(),
+ source: :universal_analytics
+ )
+
+ populate_stats(site, site_import.id, [
+ build(:pageview, referrer: "site.com", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, referrer: "site.com/1", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:imported_sources,
+ referrer: "site.com",
+ date: ~D[2021-01-01],
+ visitors: 2,
+ visits: 2,
+ pageviews: 2,
+ bounces: 1,
+ visit_duration: 120
+ ),
+ build(:imported_sources,
+ referrer: "site.com/2",
+ date: ~D[2021-01-01],
+ visitors: 2,
+ visits: 2,
+ pageviews: 2,
+ bounces: 2,
+ visit_duration: 0
+ ),
+ build(:imported_sources,
+ date: ~D[2021-01-01],
+ visitors: 10,
+ visits: 11,
+ pageviews: 50,
+ bounces: 0,
+ visit_duration: 1100
+ )
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors", "visits", "pageviews", "bounce_rate", "visit_duration"],
+ "date_range" => "all",
+ "dimensions" => ["visit:referrer"],
+ "include" => %{"imports" => true}
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["Direct / None"], "metrics" => [10, 11, 50, 0.0, 100.0]},
+ %{"dimensions" => ["site.com"], "metrics" => [3, 3, 3, 67.0, 40.0]},
+ %{"dimensions" => ["site.com/2"], "metrics" => [2, 2, 2, 100.0, 0.0]},
+ %{"dimensions" => ["site.com/1"], "metrics" => [1, 1, 1, 100.0, 0.0]}
+ ]
+ end
+
+ test "breaks down all metrics by visit:utm_source with imported data", %{conn: conn, site: site} do
+ site_import =
+ insert(:site_import,
+ site: site,
+ start_date: ~D[2005-01-01],
+ end_date: Timex.today(),
+ source: :universal_analytics
+ )
+
+ populate_stats(site, site_import.id, [
+ build(:pageview, utm_source: "SomeUTMSource", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, utm_source: "SomeUTMSource-1", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:imported_sources,
+ utm_source: "SomeUTMSource",
+ date: ~D[2021-01-01],
+ visitors: 2,
+ visits: 2,
+ pageviews: 2,
+ bounces: 1,
+ visit_duration: 120
+ ),
+ build(:imported_sources,
+ utm_source: "SomeUTMSource-2",
+ date: ~D[2021-01-01],
+ visitors: 2,
+ visits: 2,
+ pageviews: 2,
+ bounces: 2,
+ visit_duration: 0
+ ),
+ build(:imported_sources,
+ date: ~D[2021-01-01],
+ visitors: 10,
+ visits: 11,
+ pageviews: 50,
+ bounces: 0,
+ visit_duration: 1100
+ )
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors", "visits", "pageviews", "bounce_rate", "visit_duration"],
+ "date_range" => "all",
+ "dimensions" => ["visit:utm_source"],
+ "include" => %{"imports" => true}
+ })
+
+ %{"results" => results} = json_response(conn, 200)
+
+ assert results == [
+ %{"dimensions" => ["SomeUTMSource"], "metrics" => [3, 3, 3, 67.0, 40.0]},
+ %{"dimensions" => ["SomeUTMSource-2"], "metrics" => [2, 2, 2, 100.0, 0.0]},
+ %{"dimensions" => ["SomeUTMSource-1"], "metrics" => [1, 1, 1, 100.0, 0.0]}
+ ]
+ end
+
+ test "pageviews breakdown by event:page - imported data having pageviews=0 and visitors=n should be bypassed",
+ %{conn: conn, site: site} do
+ site_import =
+ insert(:site_import,
+ site: site,
+ start_date: ~D[2005-01-01],
+ end_date: Timex.today(),
+ source: :universal_analytics
+ )
+
+ populate_stats(site, site_import.id, [
+ build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),
+ build(:pageview,
+ pathname: "/plausible.io",
+ timestamp: ~N[2021-01-01 00:00:00]
+ ),
+ build(:imported_pages,
+ page: "/skip-me",
+ date: ~D[2021-01-01],
+ visitors: 1,
+ pageviews: 0
+ ),
+ build(:imported_pages,
+ page: "/include-me",
+ date: ~D[2021-01-01],
+ visitors: 1,
+ pageviews: 1
+ )
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["pageviews"],
+ "date_range" => "all",
+ "dimensions" => ["event:page"],
+ "include" => %{"imports" => true}
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["/"], "metrics" => [2]},
+ %{"dimensions" => ["/plausible.io"], "metrics" => [1]},
+ %{"dimensions" => ["/include-me"], "metrics" => [1]}
+ ]
+ end
+
+ describe "breakdown by visit:exit_page with" do
+ setup %{site: site} do
+ site_import = insert(:site_import, site: site)
+
+ populate_stats(site, site_import.id, [
+ build(:pageview,
+ pathname: "/a",
+ timestamp: ~N[2021-01-01 00:00:00]
+ ),
+ build(:pageview,
+ user_id: @user_id,
+ pathname: "/a",
+ timestamp: ~N[2021-01-01 00:25:00]
+ ),
+ build(:pageview,
+ user_id: @user_id,
+ pathname: "/b",
+ timestamp: ~N[2021-01-01 00:35:00]
+ ),
+ build(:imported_exit_pages,
+ exit_page: "/b",
+ exits: 3,
+ visitors: 2,
+ pageviews: 5,
+ date: ~D[2021-01-01]
+ )
+ ])
+ end
+
+ test "can query with visit:exit_page dimension", %{conn: conn, site: site} do
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["visits"],
+ "date_range" => "all",
+ "dimensions" => ["visit:exit_page"],
+ "include" => %{"imports" => true}
+ })
+
+ %{"results" => results} = json_response(conn, 200)
+
+ assert results == [
+ %{"dimensions" => ["/b"], "metrics" => [4]},
+ %{"dimensions" => ["/a"], "metrics" => [1]}
+ ]
+ end
+ end
+
+ describe "imported data" do
+ test "returns screen sizes breakdown when filtering by screen size", %{conn: conn, site: site} do
+ site_import = insert(:site_import, site: site)
+
+ populate_stats(site, site_import.id, [
+ build(:pageview,
+ timestamp: ~N[2021-01-01 00:00:01],
+ screen_size: "Mobile"
+ ),
+ build(:imported_devices,
+ device: "Mobile",
+ visitors: 3,
+ pageviews: 5,
+ date: ~D[2021-01-01]
+ )
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors", "pageviews"],
+ "date_range" => "all",
+ "dimensions" => ["visit:device"],
+ "filters" => [
+ ["is", "visit:device", ["Mobile"]]
+ ],
+ "include" => %{"imports" => true}
+ })
+
+ %{"results" => results} = json_response(conn, 200)
+
+ assert results == [%{"dimensions" => ["Mobile"], "metrics" => [4, 6]}]
+ end
+
+ test "returns custom event goals and pageview goals", %{conn: conn, site: site} do
+ insert(:goal, site: site, event_name: "Purchase")
+ insert(:goal, site: site, page_path: "/test")
+
+ site_import = insert(:site_import, site: site)
+
+ populate_stats(site, site_import.id, [
+ build(:pageview,
+ timestamp: ~N[2021-01-01 00:00:01],
+ pathname: "/test"
+ ),
+ build(:event,
+ name: "Purchase",
+ timestamp: ~N[2021-01-01 00:00:03]
+ ),
+ build(:event,
+ name: "Purchase",
+ timestamp: ~N[2021-01-01 00:00:03]
+ ),
+ build(:imported_custom_events,
+ name: "Purchase",
+ visitors: 3,
+ events: 5,
+ date: ~D[2021-01-01]
+ ),
+ build(:imported_pages,
+ page: "/test",
+ visitors: 2,
+ pageviews: 2,
+ date: ~D[2021-01-01]
+ ),
+ build(:imported_visitors, visitors: 5, date: ~D[2021-01-01])
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "dimensions" => ["event:goal"],
+ "metrics" => ["visitors", "events", "pageviews", "conversion_rate"],
+ "include" => %{"imports" => true}
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["Purchase"], "metrics" => [5, 7, 0, 62.5]},
+ %{"dimensions" => ["Visit /test"], "metrics" => [3, 3, 3, 37.5]}
+ ]
+ end
+
+ test "pageviews are returned as events for breakdown reports other than custom events", %{
+ conn: conn,
+ site: site
+ } do
+ site_import = insert(:site_import, site: site)
+
+ populate_stats(site, site_import.id, [
+ build(:imported_browsers, browser: "Chrome", pageviews: 1, date: ~D[2021-01-01]),
+ build(:imported_devices, device: "Desktop", pageviews: 1, date: ~D[2021-01-01]),
+ build(:imported_entry_pages, entry_page: "/test", pageviews: 1, date: ~D[2021-01-01]),
+ build(:imported_exit_pages, exit_page: "/test", pageviews: 1, date: ~D[2021-01-01]),
+ build(:imported_locations, country: "EE", pageviews: 1, date: ~D[2021-01-01]),
+ build(:imported_operating_systems,
+ operating_system: "Mac",
+ pageviews: 1,
+ date: ~D[2021-01-01]
+ ),
+ build(:imported_pages, page: "/test", pageviews: 1, date: ~D[2021-01-01]),
+ build(:imported_sources, source: "Google", pageviews: 1, date: ~D[2021-01-01])
+ ])
+
+ breakdown_and_first = fn dimension ->
+ conn
+ |> post("/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["events"],
+ "date_range" => ["2021-01-01", "2021-01-01"],
+ "dimensions" => [dimension],
+ "include" => %{"imports" => true}
+ })
+ |> json_response(200)
+ |> Map.get("results")
+ |> List.first()
+ end
+
+ assert %{"dimensions" => ["Chrome"], "metrics" => [1]} =
+ breakdown_and_first.("visit:browser")
+
+ assert %{"dimensions" => ["Desktop"], "metrics" => [1]} =
+ breakdown_and_first.("visit:device")
+
+ assert %{"dimensions" => ["EE"], "metrics" => [1]} = breakdown_and_first.("visit:country")
+ assert %{"dimensions" => ["Mac"], "metrics" => [1]} = breakdown_and_first.("visit:os")
+ assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("event:page")
+
+ assert %{"dimensions" => ["Google"], "metrics" => [1]} =
+ breakdown_and_first.("visit:source")
+ end
+
+ for goal_name <- Plausible.Imported.goals_with_url() do
+ test "returns url breakdown for #{goal_name} goal", %{conn: conn, site: site} do
+ insert(:goal, event_name: unquote(goal_name), site: site)
+ site_import = insert(:site_import, site: site)
+
+ populate_stats(site, site_import.id, [
+ build(:event,
+ name: unquote(goal_name),
+ "meta.key": ["url"],
+ "meta.value": ["https://one.com"]
+ ),
+ build(:imported_custom_events,
+ name: unquote(goal_name),
+ visitors: 2,
+ events: 5,
+ link_url: "https://one.com"
+ ),
+ build(:imported_custom_events,
+ name: unquote(goal_name),
+ visitors: 5,
+ events: 10,
+ link_url: "https://two.com"
+ ),
+ build(:imported_custom_events,
+ name: "some goal",
+ visitors: 5,
+ events: 10
+ ),
+ build(:imported_visitors, visitors: 9)
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors", "events", "conversion_rate"],
+ "date_range" => "all",
+ "dimensions" => ["event:props:url"],
+ "filters" => [
+ ["is", "event:goal", [unquote(goal_name)]]
+ ],
+ "include" => %{"imports" => true}
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["https://two.com"], "metrics" => [5, 10, 50]},
+ %{"dimensions" => ["https://one.com"], "metrics" => [3, 6, 30]}
+ ]
+
+ refute json_response(conn, 200)["meta"]["warning"]
+ end
+ end
+
+ for goal_name <- Plausible.Imported.goals_with_path() do
+ test "returns path breakdown for #{goal_name} goal", %{conn: conn, site: site} do
+ insert(:goal, event_name: unquote(goal_name), site: site)
+ site_import = insert(:site_import, site: site)
+
+ populate_stats(site, site_import.id, [
+ build(:event,
+ name: unquote(goal_name),
+ "meta.key": ["path"],
+ "meta.value": ["/one"]
+ ),
+ build(:imported_custom_events,
+ name: unquote(goal_name),
+ visitors: 2,
+ events: 5,
+ path: "/one"
+ ),
+ build(:imported_custom_events,
+ name: unquote(goal_name),
+ visitors: 5,
+ events: 10,
+ path: "/two"
+ ),
+ build(:imported_custom_events,
+ name: "some goal",
+ visitors: 5,
+ events: 10
+ ),
+ build(:imported_visitors, visitors: 9)
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors", "events", "conversion_rate"],
+ "date_range" => "all",
+ "dimensions" => ["event:props:path"],
+ "filters" => [
+ ["is", "event:goal", [unquote(goal_name)]]
+ ],
+ "include" => %{"imports" => true}
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["/two"], "metrics" => [5, 10, 50]},
+ %{"dimensions" => ["/one"], "metrics" => [3, 6, 30]}
+ ]
+
+ refute json_response(conn, 200)["meta"]["warning"]
+ end
+ end
+
+ test "adds a warning when query params are not supported for imported data", %{
+ conn: conn,
+ site: site
+ } do
+ site_import = insert(:site_import, site: site)
+
+ insert(:goal, event_name: "Signup", site: site)
+
+ populate_stats(site, site_import.id, [
+ build(:event,
+ name: "Signup",
+ "meta.key": ["package"],
+ "meta.value": ["large"]
+ ),
+ build(:imported_visitors, visitors: 9)
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "dimensions" => ["event:props:package"],
+ "filters" => [
+ ["is", "event:goal", ["Signup"]]
+ ],
+ "include" => %{"imports" => true}
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["large"], "metrics" => [1]}
+ ]
+
+ assert json_response(conn, 200)["meta"]["warning"] =~
+ "Imported stats are not included in the results because query parameters are not supported."
+ end
+
+ test "does not add a warning when there are no site imports", %{conn: conn, site: site} do
+ insert(:goal, event_name: "Signup", site: site)
+
+ populate_stats(site, [
+ build(:event,
+ name: "Signup",
+ "meta.key": ["package"],
+ "meta.value": ["large"]
+ )
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "dimensions" => ["event:props:package"],
+ "filters" => [
+ ["is", "event:goal", ["Signup"]]
+ ],
+ "include" => %{"imports" => true}
+ })
+
+ refute json_response(conn, 200)["meta"]["warning"]
+ end
+
+ test "does not add a warning when import is out of queried date range", %{
+ conn: conn,
+ site: site
+ } do
+ site_import = insert(:site_import, site: site, end_date: Date.add(Date.utc_today(), -3))
+
+ insert(:goal, event_name: "Signup", site: site)
+
+ populate_stats(site, site_import.id, [
+ build(:event,
+ name: "Signup",
+ "meta.key": ["package"],
+ "meta.value": ["large"]
+ ),
+ build(:imported_visitors, visitors: 9)
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "day",
+ "dimensions" => ["event:props:package"],
+ "filters" => [
+ ["is", "event:goal", ["Signup"]]
+ ],
+ "include" => %{"imports" => true}
+ })
+
+ refute json_response(conn, 200)["meta"]["warning"]
+ end
+
+ test "applies multiple filters if the properties belong to the same table", %{
+ conn: conn,
+ site: site
+ } do
+ site_import = insert(:site_import, site: site)
+
+ populate_stats(site, site_import.id, [
+ build(:imported_sources, source: "Google", utm_medium: "organic", utm_term: "one"),
+ build(:imported_sources, source: "Twitter", utm_medium: "organic", utm_term: "two"),
+ build(:imported_sources,
+ source: "Facebook",
+ utm_medium: "something_else",
+ utm_term: "one"
+ )
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "day",
+ "dimensions" => ["visit:source"],
+ "filters" => [
+ ["is", "visit:utm_medium", ["organic"]],
+ ["is", "visit:utm_term", ["one"]]
+ ],
+ "include" => %{"imports" => true}
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["Google"], "metrics" => [1]}
+ ]
+ end
+
+ test "ignores imported data if filtered property belongs to a different table than the breakdown property",
+ %{
+ conn: conn,
+ site: site
+ } do
+ site_import = insert(:site_import, site: site)
+
+ populate_stats(site, site_import.id, [
+ build(:imported_sources, source: "Google"),
+ build(:imported_devices, device: "Desktop")
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "day",
+ "dimensions" => ["visit:source"],
+ "filters" => [
+ ["is", "visit:device", ["Desktop"]]
+ ],
+ "include" => %{"imports" => true}
+ })
+
+ assert %{
+ "results" => [],
+ "meta" => meta
+ } = json_response(conn, 200)
+
+ assert meta["warning"] =~ "Imported stats are not included in the results"
+ end
+
+ test "imported country, region and city data",
+ %{
+ conn: conn,
+ site: site
+ } do
+ site_import = insert(:site_import, site: site)
+
+ populate_stats(site, site_import.id, [
+ build(:pageview,
+ timestamp: ~N[2021-01-01 00:15:00],
+ country_code: "DE",
+ subdivision1_code: "DE-BE",
+ city_geoname_id: 2_950_159
+ ),
+ build(:pageview,
+ timestamp: ~N[2021-01-01 00:15:00],
+ country_code: "DE",
+ subdivision1_code: "DE-BE",
+ city_geoname_id: 2_950_159
+ ),
+ build(:pageview,
+ timestamp: ~N[2021-01-01 00:15:00],
+ country_code: "EE",
+ subdivision1_code: "EE-37",
+ city_geoname_id: 588_409
+ ),
+ build(:imported_locations, country: "EE", region: "EE-37", city: 588_409, visitors: 33)
+ ])
+
+ for {dimension, stats_value, imports_value} <- [
+ {"visit:country", "DE", "EE"},
+ {"visit:region", "DE-BE", "EE-37"},
+ {"visit:city", 2_950_159, 588_409}
+ ] do
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "dimensions" => [dimension],
+ "include" => %{"imports" => true}
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => [imports_value], "metrics" => [34]},
+ %{"dimensions" => [stats_value], "metrics" => [2]}
+ ]
+ end
+ end
+ end
+end
diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_special_metrics_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_special_metrics_test.exs
new file mode 100644
index 0000000000..de403aa6b2
--- /dev/null
+++ b/test/plausible_web/controllers/api/external_stats_controller/query_special_metrics_test.exs
@@ -0,0 +1,193 @@
+defmodule PlausibleWeb.Api.ExternalStatsController.QuerySpecialMetricsTest do
+ use PlausibleWeb.ConnCase
+
+ setup [:create_user, :create_new_site, :create_api_key, :use_api_key]
+
+ test "returns conversion_rate in a goal filtered custom prop breakdown", %{
+ conn: conn,
+ site: site
+ } do
+ populate_stats(site, [
+ build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]),
+ build(:pageview, pathname: "/blog/2", "meta.key": ["author"], "meta.value": ["Uku"]),
+ build(:pageview, pathname: "/blog/3", "meta.key": ["author"], "meta.value": ["Uku"]),
+ build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Marko"]),
+ build(:pageview,
+ pathname: "/blog/2",
+ "meta.key": ["author"],
+ "meta.value": ["Marko"],
+ user_id: 1
+ ),
+ build(:pageview,
+ pathname: "/blog/3",
+ "meta.key": ["author"],
+ "meta.value": ["Marko"],
+ user_id: 1
+ ),
+ build(:pageview, pathname: "/blog"),
+ build(:pageview, "meta.key": ["author"], "meta.value": ["Marko"]),
+ build(:pageview)
+ ])
+
+ insert(:goal, %{site: site, page_path: "/blog**"})
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "filters" => [["matches", "event:goal", ["Visit /blog**"]]],
+ "metrics" => ["visitors", "events", "conversion_rate"],
+ "dimensions" => ["event:props:author"]
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["Uku"], "metrics" => [3, 3, 37.5]},
+ %{"dimensions" => ["Marko"], "metrics" => [2, 3, 25.0]},
+ %{"dimensions" => ["(none)"], "metrics" => [1, 1, 12.5]}
+ ]
+ end
+
+ test "returns conversion_rate alone in a goal filtered custom prop breakdown", %{
+ conn: conn,
+ site: site
+ } do
+ populate_stats(site, [
+ build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]),
+ build(:pageview)
+ ])
+
+ insert(:goal, %{site: site, page_path: "/blog**"})
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["conversion_rate"],
+ "date_range" => "all",
+ "dimensions" => ["event:props:author"],
+ "filters" => [["matches", "event:goal", ["Visit /blog**"]]]
+ })
+
+ %{"results" => results} = json_response(conn, 200)
+
+ assert results == [
+ %{"dimensions" => ["Uku"], "metrics" => [50]}
+ ]
+ end
+
+ test "returns conversion_rate in a goal filtered event:page breakdown", %{
+ conn: conn,
+ site: site
+ } do
+ populate_stats(site, [
+ build(:event, pathname: "/en/register", name: "pageview"),
+ build(:event, pathname: "/en/register", name: "Signup"),
+ build(:event, pathname: "/en/register", name: "Signup"),
+ build(:event, pathname: "/it/register", name: "Signup", user_id: 1),
+ build(:event, pathname: "/it/register", name: "Signup", user_id: 1),
+ build(:event, pathname: "/it/register", name: "pageview")
+ ])
+
+ insert(:goal, %{site: site, event_name: "Signup"})
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "dimensions" => ["event:page"],
+ "filters" => [["is", "event:goal", ["Signup"]]],
+ "metrics" => ["visitors", "events", "group_conversion_rate"]
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["/en/register"], "metrics" => [2, 2, 66.7]},
+ %{"dimensions" => ["/it/register"], "metrics" => [1, 2, 50.0]}
+ ]
+ end
+
+ test "returns conversion_rate alone in a goal filtered event:page breakdown", %{
+ conn: conn,
+ site: site
+ } do
+ populate_stats(site, [
+ build(:event, pathname: "/en/register", name: "pageview"),
+ build(:event, pathname: "/en/register", name: "Signup")
+ ])
+
+ insert(:goal, %{site: site, event_name: "Signup"})
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "filters" => [["is", "event:goal", ["Signup"]]],
+ "metrics" => ["group_conversion_rate"],
+ "dimensions" => ["event:page"]
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["/en/register"], "metrics" => [50.0]}
+ ]
+ end
+
+ test "returns conversion_rate in a multi-goal filtered visit:screen_size breakdown", %{
+ conn: conn,
+ site: site
+ } do
+ populate_stats(site, [
+ build(:event, screen_size: "Mobile", name: "pageview"),
+ build(:event, screen_size: "Mobile", name: "AddToCart"),
+ build(:event, screen_size: "Mobile", name: "AddToCart"),
+ build(:event, screen_size: "Desktop", name: "AddToCart", user_id: 1),
+ build(:event, screen_size: "Desktop", name: "Purchase", user_id: 1),
+ build(:event, screen_size: "Desktop", name: "pageview")
+ ])
+
+ # Make sure that revenue goals are treated the same
+ # way as regular custom event goals
+ insert(:goal, %{site: site, event_name: "Purchase", currency: :EUR})
+ insert(:goal, %{site: site, event_name: "AddToCart"})
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors", "events", "group_conversion_rate"],
+ "date_range" => "all",
+ "dimensions" => ["visit:device"],
+ "filters" => [["is", "event:goal", ["AddToCart", "Purchase"]]]
+ })
+
+ %{"results" => results} = json_response(conn, 200)
+
+ assert results == [
+ %{"dimensions" => ["Mobile"], "metrics" => [2, 2, 66.7]},
+ %{"dimensions" => ["Desktop"], "metrics" => [1, 2, 50]}
+ ]
+ end
+
+ test "returns conversion_rate alone in a goal filtered visit:screen_size breakdown", %{
+ conn: conn,
+ site: site
+ } do
+ populate_stats(site, [
+ build(:event, screen_size: "Mobile", name: "pageview"),
+ build(:event, screen_size: "Mobile", name: "AddToCart")
+ ])
+
+ insert(:goal, %{site: site, event_name: "AddToCart"})
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["conversion_rate"],
+ "date_range" => "all",
+ "dimensions" => ["visit:device"],
+ "filters" => [["is", "event:goal", ["AddToCart"]]]
+ })
+
+ %{"results" => results} = json_response(conn, 200)
+
+ assert results == [
+ %{"dimensions" => ["Mobile"], "metrics" => [50]}
+ ]
+ end
+end
diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs
index 9454196398..8bcd82a1ab 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs
@@ -1,239 +1,10 @@
defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
use PlausibleWeb.ConnCase
- alias Plausible.Billing.Feature
@user_id 1231
setup [:create_user, :create_new_site, :create_api_key, :use_api_key]
- describe "feature access" do
- test "cannot break down by a custom prop without access to the props feature", %{
- conn: conn,
- user: user,
- site: site
- } do
- ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)
- insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)
-
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "metrics" => ["visitors"],
- "date_range" => "all",
- "dimensions" => ["event:props:author"]
- })
-
- assert json_response(conn, 400)["error"] ==
- "The owner of this site does not have access to the custom properties feature"
- end
-
- test "can break down by an internal prop key without access to the props feature", %{
- conn: conn,
- user: user,
- site: site
- } do
- ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)
- insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)
-
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "metrics" => ["visitors"],
- "date_range" => "all",
- "dimensions" => ["event:props:path"]
- })
-
- assert json_response(conn, 200)["results"]
- end
-
- test "cannot filter by a custom prop without access to the props feature", %{
- conn: conn,
- user: user,
- site: site
- } do
- ep =
- insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)
-
- insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)
-
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "metrics" => ["visitors"],
- "date_range" => "all",
- "dimensions" => ["visit:source"],
- "filters" => [["is", "event:props:author", ["Uku"]]]
- })
-
- assert json_response(conn, 400)["error"] ==
- "The owner of this site does not have access to the custom properties feature"
- end
-
- test "can filter by an internal prop key without access to the props feature", %{
- conn: conn,
- user: user,
- site: site
- } do
- ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)
- insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)
-
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "metrics" => ["visitors"],
- "date_range" => "all",
- "dimensions" => ["visit:source"],
- "filters" => [["is", "event:props:url", ["whatever"]]]
- })
-
- assert json_response(conn, 200)["results"]
- end
- end
-
- describe "param validation" do
- test "does not allow querying conversion_rate without a goal filter", %{
- conn: conn,
- site: site
- } do
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "metrics" => ["conversion_rate"],
- "date_range" => "all",
- "dimensions" => ["event:page"],
- "filters" => [["is", "event:props:author", ["Uku"]]]
- })
-
- assert json_response(conn, 400)["error"] ==
- "Metric `conversion_rate` can only be queried with event:goal filters or dimensions"
- end
-
- test "validates that dimensions are valid", %{conn: conn, site: site} do
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "metrics" => ["visitors"],
- "date_range" => "all",
- "dimensions" => ["badproperty"]
- })
-
- assert json_response(conn, 400)["error"] =~ "Invalid dimensions"
- end
-
- test "empty custom property is invalid", %{conn: conn, site: site} do
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "metrics" => ["visitors"],
- "date_range" => "all",
- "dimensions" => ["event:props:"]
- })
-
- assert json_response(conn, 400)["error"] =~ "Invalid dimensions"
- end
-
- test "validates that correct date range is used", %{conn: conn, site: site} do
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "metrics" => ["visitors"],
- "date_range" => "bad_period",
- "dimensions" => ["event:name"]
- })
-
- assert json_response(conn, 400)["error"] =~ "Invalid date_range"
- end
-
- test "fails when an invalid metric is provided", %{conn: conn, site: site} do
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "metrics" => ["visitors", "baa"],
- "date_range" => "all",
- "dimensions" => ["event:name"]
- })
-
- assert json_response(conn, 400)["error"] =~ "Unknown metric '\"baa\"'"
- end
-
- test "session metrics cannot be used with event:name dimension", %{conn: conn, site: site} do
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "metrics" => ["visitors", "bounce_rate"],
- "date_range" => "all",
- "dimensions" => ["event:name"]
- })
-
- assert json_response(conn, 400)["error"] =~
- "Session metric(s) `bounce_rate` cannot be queried along with event dimensions"
- end
-
- test "session metrics cannot be used with event:props:* dimension", %{conn: conn, site: site} do
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "metrics" => ["visitors", "bounce_rate"],
- "date_range" => "all",
- "dimensions" => ["event:props:url"]
- })
-
- assert json_response(conn, 400)["error"] =~
- "Session metric(s) `bounce_rate` cannot be queried along with event dimensions"
- end
-
- test "validates that metric views_per_visit cannot be used with event:page filter", %{
- conn: conn,
- site: site
- } do
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "date_range" => "all",
- "metrics" => ["views_per_visit"],
- "filters" => [["is", "event:page", ["/something"]]]
- })
-
- assert json_response(conn, 400) == %{
- "error" =>
- "Metric `views_per_visit` cannot be queried with a filter on `event:page`"
- }
- end
-
- test "validates that metric views_per_visit cannot be used together with dimensions", %{
- conn: conn,
- site: site
- } do
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "date_range" => "all",
- "metrics" => ["views_per_visit"],
- "dimensions" => ["event:name"]
- })
-
- assert json_response(conn, 400) == %{
- "error" => "Metric `views_per_visit` cannot be queried with `dimensions`"
- }
- end
-
- test "validates a metric can't be asked multiple times", %{
- conn: conn,
- site: site
- } do
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "date_range" => "all",
- "metrics" => ["views_per_visit", "visitors", "visitors"]
- })
-
- assert json_response(conn, 400) == %{
- "error" => "Metrics cannot be queried multiple times"
- }
- end
- end
-
test "aggregates a single metric", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),
@@ -1065,36 +836,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
end
end
- describe "aggregation with imported data" do
- setup :create_site_import
-
- test "does not count imported stats unless specified", %{
- conn: conn,
- site: site,
- site_import: site_import
- } do
- populate_stats(site, site_import.id, [
- build(:imported_visitors, date: ~D[2023-01-01]),
- build(:pageview, timestamp: ~N[2023-01-01 00:00:00])
- ])
-
- query_params = %{
- "site_id" => site.domain,
- "date_range" => "all",
- "metrics" => ["pageviews"]
- }
-
- conn1 = post(conn, "/api/v2/query", query_params)
-
- assert json_response(conn1, 200)["results"] == [%{"metrics" => [1], "dimensions" => []}]
-
- conn2 = post(conn, "/api/v2/query", Map.put(query_params, "include", %{"imports" => true}))
-
- assert json_response(conn2, 200)["results"] == [%{"metrics" => [2], "dimensions" => []}]
- refute json_response(conn2, 200)["meta"]["warning"]
- end
- end
-
describe "timeseries" do
test "shows hourly data for a certain date with time_labels", %{conn: conn, site: site} do
populate_stats(site, [
@@ -1405,63 +1146,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
]
end
- test "breaks down all metrics by visit:referrer with imported data", %{conn: conn, site: site} do
- site_import =
- insert(:site_import,
- site: site,
- start_date: ~D[2005-01-01],
- end_date: Timex.today(),
- source: :universal_analytics
- )
-
- populate_stats(site, site_import.id, [
- build(:pageview, referrer: "site.com", timestamp: ~N[2021-01-01 00:00:00]),
- build(:pageview, referrer: "site.com/1", timestamp: ~N[2021-01-01 00:00:00]),
- build(:imported_sources,
- referrer: "site.com",
- date: ~D[2021-01-01],
- visitors: 2,
- visits: 2,
- pageviews: 2,
- bounces: 1,
- visit_duration: 120
- ),
- build(:imported_sources,
- referrer: "site.com/2",
- date: ~D[2021-01-01],
- visitors: 2,
- visits: 2,
- pageviews: 2,
- bounces: 2,
- visit_duration: 0
- ),
- build(:imported_sources,
- date: ~D[2021-01-01],
- visitors: 10,
- visits: 11,
- pageviews: 50,
- bounces: 0,
- visit_duration: 1100
- )
- ])
-
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "metrics" => ["visitors", "visits", "pageviews", "bounce_rate", "visit_duration"],
- "date_range" => "all",
- "dimensions" => ["visit:referrer"],
- "include" => %{"imports" => true}
- })
-
- assert json_response(conn, 200)["results"] == [
- %{"dimensions" => ["Direct / None"], "metrics" => [10, 11, 50, 0.0, 100.0]},
- %{"dimensions" => ["site.com"], "metrics" => [3, 3, 3, 67.0, 40.0]},
- %{"dimensions" => ["site.com/2"], "metrics" => [2, 2, 2, 100.0, 0.0]},
- %{"dimensions" => ["site.com/1"], "metrics" => [1, 1, 1, 100.0, 0.0]}
- ]
- end
-
for {dimension, attr} <- [
{"visit:utm_campaign", :utm_campaign},
{"visit:utm_source", :utm_source},
@@ -1657,111 +1341,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
]
end
- test "breaks down all metrics by visit:utm_source with imported data", %{conn: conn, site: site} do
- site_import =
- insert(:site_import,
- site: site,
- start_date: ~D[2005-01-01],
- end_date: Timex.today(),
- source: :universal_analytics
- )
-
- populate_stats(site, site_import.id, [
- build(:pageview, utm_source: "SomeUTMSource", timestamp: ~N[2021-01-01 00:00:00]),
- build(:pageview, utm_source: "SomeUTMSource-1", timestamp: ~N[2021-01-01 00:00:00]),
- build(:imported_sources,
- utm_source: "SomeUTMSource",
- date: ~D[2021-01-01],
- visitors: 2,
- visits: 2,
- pageviews: 2,
- bounces: 1,
- visit_duration: 120
- ),
- build(:imported_sources,
- utm_source: "SomeUTMSource-2",
- date: ~D[2021-01-01],
- visitors: 2,
- visits: 2,
- pageviews: 2,
- bounces: 2,
- visit_duration: 0
- ),
- build(:imported_sources,
- date: ~D[2021-01-01],
- visitors: 10,
- visits: 11,
- pageviews: 50,
- bounces: 0,
- visit_duration: 1100
- )
- ])
-
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "metrics" => ["visitors", "visits", "pageviews", "bounce_rate", "visit_duration"],
- "date_range" => "all",
- "dimensions" => ["visit:utm_source"],
- "include" => %{"imports" => true}
- })
-
- %{"results" => results} = json_response(conn, 200)
-
- assert results == [
- %{"dimensions" => ["SomeUTMSource"], "metrics" => [3, 3, 3, 67.0, 40.0]},
- %{"dimensions" => ["SomeUTMSource-2"], "metrics" => [2, 2, 2, 100.0, 0.0]},
- %{"dimensions" => ["SomeUTMSource-1"], "metrics" => [1, 1, 1, 100.0, 0.0]}
- ]
- end
-
- test "pageviews breakdown by event:page - imported data having pageviews=0 and visitors=n should be bypassed",
- %{conn: conn, site: site} do
- site_import =
- insert(:site_import,
- site: site,
- start_date: ~D[2005-01-01],
- end_date: Timex.today(),
- source: :universal_analytics
- )
-
- populate_stats(site, site_import.id, [
- build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),
- build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),
- build(:pageview,
- pathname: "/plausible.io",
- timestamp: ~N[2021-01-01 00:00:00]
- ),
- build(:imported_pages,
- page: "/skip-me",
- date: ~D[2021-01-01],
- visitors: 1,
- pageviews: 0
- ),
- build(:imported_pages,
- page: "/include-me",
- date: ~D[2021-01-01],
- visitors: 1,
- pageviews: 1
- )
- ])
-
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "metrics" => ["pageviews"],
- "date_range" => "all",
- "dimensions" => ["event:page"],
- "include" => %{"imports" => true}
- })
-
- assert json_response(conn, 200)["results"] == [
- %{"dimensions" => ["/"], "metrics" => [2]},
- %{"dimensions" => ["/plausible.io"], "metrics" => [1]},
- %{"dimensions" => ["/include-me"], "metrics" => [1]}
- ]
- end
-
test "breakdown by event:page", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),
@@ -1811,87 +1390,39 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
]
end
- describe "breakdown by visit:exit_page" do
- setup %{site: site} do
- site_import = insert(:site_import, site: site)
-
- populate_stats(site, site_import.id, [
- build(:pageview,
- pathname: "/a",
+ describe "custom events" do
+ test "can breakdown by event:name", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:event,
+ name: "Signup",
timestamp: ~N[2021-01-01 00:00:00]
),
- build(:pageview,
- user_id: @user_id,
- pathname: "/a",
- timestamp: ~N[2021-01-01 00:25:00]
+ build(:event,
+ name: "Signup",
+ timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
- user_id: @user_id,
- pathname: "/b",
- timestamp: ~N[2021-01-01 00:35:00]
- ),
- build(:imported_exit_pages,
- exit_page: "/b",
- exits: 3,
- visitors: 2,
- pageviews: 5,
- date: ~D[2021-01-01]
+ timestamp: ~N[2021-01-01 00:25:00]
)
])
- end
- test "can query with visit:exit_page dimension", %{conn: conn, site: site} do
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
- "metrics" => ["visits"],
+ "metrics" => ["visitors"],
"date_range" => "all",
- "dimensions" => ["visit:exit_page"],
- "include" => %{"imports" => true}
+ "dimensions" => ["event:name"]
})
%{"results" => results} = json_response(conn, 200)
assert results == [
- %{"dimensions" => ["/b"], "metrics" => [4]},
- %{"dimensions" => ["/a"], "metrics" => [1]}
+ %{"dimensions" => ["Signup"], "metrics" => [2]},
+ %{"dimensions" => ["pageview"], "metrics" => [1]}
]
end
- end
- describe "custom events" do
- test "can breakdown by event:name", %{conn: conn, site: site} do
- populate_stats(site, [
- build(:event,
- name: "Signup",
- timestamp: ~N[2021-01-01 00:00:00]
- ),
- build(:event,
- name: "Signup",
- timestamp: ~N[2021-01-01 00:00:00]
- ),
- build(:pageview,
- timestamp: ~N[2021-01-01 00:25:00]
- )
- ])
-
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "metrics" => ["visitors"],
- "date_range" => "all",
- "dimensions" => ["event:name"]
- })
-
- %{"results" => results} = json_response(conn, 200)
-
- assert results == [
- %{"dimensions" => ["Signup"], "metrics" => [2]},
- %{"dimensions" => ["pageview"], "metrics" => [1]}
- ]
- end
-
- test "can breakdown by event:name with visitors and events metrics", %{conn: conn, site: site} do
+ test "can breakdown by event:name with visitors and events metrics", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
pathname: "/non-existing",
@@ -2248,8 +1779,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
"site_id" => site.domain,
"metrics" => ["pageviews"],
"date_range" => "all",
- "dimensions" => ["event:props:package"],
- "filters" => []
+ "dimensions" => ["event:props:package"]
})
%{"results" => results} = json_response(conn, 200)
@@ -2317,85 +1847,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
end
end
- describe "breakdown by event:goal" do
- test "returns custom event goals and pageview goals", %{conn: conn, site: site} do
- insert(:goal, %{site: site, event_name: "Purchase"})
- insert(:goal, %{site: site, page_path: "/test"})
-
- populate_stats(site, [
- build(:pageview,
- timestamp: ~N[2021-01-01 00:00:01],
- pathname: "/test"
- ),
- build(:event,
- name: "Purchase",
- timestamp: ~N[2021-01-01 00:00:03]
- ),
- build(:event,
- name: "Purchase",
- timestamp: ~N[2021-01-01 00:00:03]
- )
- ])
-
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "date_range" => "all",
- "metrics" => ["visitors"],
- "dimensions" => ["event:goal"]
- })
-
- assert json_response(conn, 200)["results"] == [
- %{"dimensions" => ["Purchase"], "metrics" => [2]},
- %{"dimensions" => ["Visit /test"], "metrics" => [1]}
- ]
- end
-
- test "returns pageview goals containing wildcards", %{conn: conn, site: site} do
- insert(:goal, %{site: site, page_path: "/**/post"})
- insert(:goal, %{site: site, page_path: "/blog**"})
-
- populate_stats(site, [
- build(:pageview, pathname: "/blog", user_id: @user_id),
- build(:pageview, pathname: "/blog/post-1", user_id: @user_id),
- build(:pageview, pathname: "/blog/post-2", user_id: @user_id),
- build(:pageview, pathname: "/blog/something/post"),
- build(:pageview, pathname: "/different/page/post")
- ])
-
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "date_range" => "all",
- "metrics" => ["visitors", "pageviews"],
- "dimensions" => ["event:goal"],
- "order_by" => [["pageviews", "desc"]]
- })
-
- assert json_response(conn, 200)["results"] == [
- %{"dimensions" => ["Visit /blog**"], "metrics" => [2, 4]},
- %{"dimensions" => ["Visit /**/post"], "metrics" => [2, 2]}
- ]
- end
-
- test "does not return goals that are not configured for the site", %{conn: conn, site: site} do
- populate_stats(site, [
- build(:pageview, pathname: "/register"),
- build(:event, name: "Signup")
- ])
-
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "date_range" => "all",
- "metrics" => ["visitors", "pageviews"],
- "dimensions" => ["event:goal"]
- })
-
- assert json_response(conn, 200)["results"] == []
- end
- end
-
test "event:goal filter returns 400 when goal not configured", %{conn: conn, site: site} do
conn =
post(conn, "/api/v2/query", %{
@@ -2802,21 +2253,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
)
])
- conn =
- get(conn, "/api/v1/stats/breakdown", %{
- "site_id" => site.domain,
- "period" => "day",
- "property" => "event:page",
- "filters" => "event:goal == Register"
- })
-
- assert json_response(conn, 200) == %{
- "results" => [
- %{"page" => "/en/register", "visitors" => 2},
- %{"page" => "/it/register", "visitors" => 1}
- ]
- }
-
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
@@ -3159,241 +2595,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
end
describe "metrics" do
- test "returns conversion_rate in an event:goal breakdown", %{conn: conn, site: site} do
- populate_stats(site, [
- build(:event, name: "Signup", user_id: 1),
- build(:event, name: "Signup", user_id: 1),
- build(:pageview, pathname: "/blog"),
- build(:pageview, pathname: "/blog/post"),
- build(:pageview)
- ])
-
- insert(:goal, %{site: site, event_name: "Signup"})
- insert(:goal, %{site: site, page_path: "/blog**"})
-
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "date_range" => "all",
- "metrics" => ["visitors", "events", "conversion_rate"],
- "dimensions" => ["event:goal"]
- })
-
- assert json_response(conn, 200)["results"] == [
- %{"dimensions" => ["Visit /blog**"], "metrics" => [2, 2, 50.0]},
- %{"dimensions" => ["Signup"], "metrics" => [1, 2, 25.0]}
- ]
- end
-
- test "returns conversion_rate alone in an event:goal breakdown", %{conn: conn, site: site} do
- populate_stats(site, [
- build(:event, name: "Signup", user_id: 1),
- build(:pageview)
- ])
-
- insert(:goal, %{site: site, event_name: "Signup"})
-
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "date_range" => "all",
- "metrics" => ["conversion_rate"],
- "dimensions" => ["event:goal"]
- })
-
- assert json_response(conn, 200)["results"] == [
- %{"dimensions" => ["Signup"], "metrics" => [50.0]}
- ]
- end
-
- test "returns conversion_rate in a goal filtered custom prop breakdown", %{
- conn: conn,
- site: site
- } do
- populate_stats(site, [
- build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]),
- build(:pageview, pathname: "/blog/2", "meta.key": ["author"], "meta.value": ["Uku"]),
- build(:pageview, pathname: "/blog/3", "meta.key": ["author"], "meta.value": ["Uku"]),
- build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Marko"]),
- build(:pageview,
- pathname: "/blog/2",
- "meta.key": ["author"],
- "meta.value": ["Marko"],
- user_id: 1
- ),
- build(:pageview,
- pathname: "/blog/3",
- "meta.key": ["author"],
- "meta.value": ["Marko"],
- user_id: 1
- ),
- build(:pageview, pathname: "/blog"),
- build(:pageview, "meta.key": ["author"], "meta.value": ["Marko"]),
- build(:pageview)
- ])
-
- insert(:goal, %{site: site, page_path: "/blog**"})
-
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "date_range" => "all",
- "filters" => [["matches", "event:goal", ["Visit /blog**"]]],
- "metrics" => ["visitors", "events", "conversion_rate"],
- "dimensions" => ["event:props:author"]
- })
-
- assert json_response(conn, 200)["results"] == [
- %{"dimensions" => ["Uku"], "metrics" => [3, 3, 37.5]},
- %{"dimensions" => ["Marko"], "metrics" => [2, 3, 25.0]},
- %{"dimensions" => ["(none)"], "metrics" => [1, 1, 12.5]}
- ]
- end
-
- test "returns conversion_rate alone in a goal filtered custom prop breakdown", %{
- conn: conn,
- site: site
- } do
- populate_stats(site, [
- build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]),
- build(:pageview)
- ])
-
- insert(:goal, %{site: site, page_path: "/blog**"})
-
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "metrics" => ["conversion_rate"],
- "date_range" => "all",
- "dimensions" => ["event:props:author"],
- "filters" => [["matches", "event:goal", ["Visit /blog**"]]]
- })
-
- %{"results" => results} = json_response(conn, 200)
-
- assert results == [
- %{"dimensions" => ["Uku"], "metrics" => [50]}
- ]
- end
-
- test "returns conversion_rate in a goal filtered event:page breakdown", %{
- conn: conn,
- site: site
- } do
- populate_stats(site, [
- build(:event, pathname: "/en/register", name: "pageview"),
- build(:event, pathname: "/en/register", name: "Signup"),
- build(:event, pathname: "/en/register", name: "Signup"),
- build(:event, pathname: "/it/register", name: "Signup", user_id: 1),
- build(:event, pathname: "/it/register", name: "Signup", user_id: 1),
- build(:event, pathname: "/it/register", name: "pageview")
- ])
-
- insert(:goal, %{site: site, event_name: "Signup"})
-
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "date_range" => "all",
- "dimensions" => ["event:page"],
- "filters" => [["is", "event:goal", ["Signup"]]],
- "metrics" => ["visitors", "events", "group_conversion_rate"]
- })
-
- assert json_response(conn, 200)["results"] == [
- %{"dimensions" => ["/en/register"], "metrics" => [2, 2, 66.7]},
- %{"dimensions" => ["/it/register"], "metrics" => [1, 2, 50.0]}
- ]
- end
-
- test "returns conversion_rate alone in a goal filtered event:page breakdown", %{
- conn: conn,
- site: site
- } do
- populate_stats(site, [
- build(:event, pathname: "/en/register", name: "pageview"),
- build(:event, pathname: "/en/register", name: "Signup")
- ])
-
- insert(:goal, %{site: site, event_name: "Signup"})
-
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "date_range" => "all",
- "filters" => [["is", "event:goal", ["Signup"]]],
- "metrics" => ["group_conversion_rate"],
- "dimensions" => ["event:page"]
- })
-
- assert json_response(conn, 200)["results"] == [
- %{"dimensions" => ["/en/register"], "metrics" => [50.0]}
- ]
- end
-
- test "returns conversion_rate in a multi-goal filtered visit:screen_size breakdown", %{
- conn: conn,
- site: site
- } do
- populate_stats(site, [
- build(:event, screen_size: "Mobile", name: "pageview"),
- build(:event, screen_size: "Mobile", name: "AddToCart"),
- build(:event, screen_size: "Mobile", name: "AddToCart"),
- build(:event, screen_size: "Desktop", name: "AddToCart", user_id: 1),
- build(:event, screen_size: "Desktop", name: "Purchase", user_id: 1),
- build(:event, screen_size: "Desktop", name: "pageview")
- ])
-
- # Make sure that revenue goals are treated the same
- # way as regular custom event goals
- insert(:goal, %{site: site, event_name: "Purchase", currency: :EUR})
- insert(:goal, %{site: site, event_name: "AddToCart"})
-
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "metrics" => ["visitors", "events", "group_conversion_rate"],
- "date_range" => "all",
- "dimensions" => ["visit:device"],
- "filters" => [["is", "event:goal", ["AddToCart", "Purchase"]]]
- })
-
- %{"results" => results} = json_response(conn, 200)
-
- assert results == [
- %{"dimensions" => ["Mobile"], "metrics" => [2, 2, 66.7]},
- %{"dimensions" => ["Desktop"], "metrics" => [1, 2, 50]}
- ]
- end
-
- test "returns conversion_rate alone in a goal filtered visit:screen_size breakdown", %{
- conn: conn,
- site: site
- } do
- populate_stats(site, [
- build(:event, screen_size: "Mobile", name: "pageview"),
- build(:event, screen_size: "Mobile", name: "AddToCart")
- ])
-
- insert(:goal, %{site: site, event_name: "AddToCart"})
-
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "metrics" => ["conversion_rate"],
- "date_range" => "all",
- "dimensions" => ["visit:device"],
- "filters" => [["is", "event:goal", ["AddToCart"]]]
- })
-
- %{"results" => results} = json_response(conn, 200)
-
- assert results == [
- %{"dimensions" => ["Mobile"], "metrics" => [50]}
- ]
- end
-
test "all metrics for breakdown by visit prop", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
@@ -3434,8 +2635,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
"visit_duration"
],
"date_range" => "all",
- "dimensions" => ["visit:source"],
- "filters" => []
+ "dimensions" => ["visit:source"]
})
%{"results" => results} = json_response(conn, 200)
@@ -3483,58 +2683,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
]
end
- test "filter by custom event property", %{conn: conn, site: site} do
- populate_stats(site, [
- build(:event,
- name: "Purchase",
- "meta.key": ["package"],
- "meta.value": ["business"],
- browser: "Chrome",
- timestamp: ~N[2021-01-01 00:00:00]
- ),
- build(:event,
- name: "Purchase",
- "meta.key": ["package"],
- "meta.value": ["business"],
- browser: "Safari",
- timestamp: ~N[2021-01-01 00:00:00]
- ),
- build(:event,
- name: "Purchase",
- "meta.key": ["package"],
- "meta.value": ["business"],
- browser: "Safari",
- timestamp: ~N[2021-01-01 00:25:00]
- ),
- build(:event,
- name: "Purchase",
- "meta.key": ["package"],
- "meta.value": ["personal"],
- browser: "IE",
- timestamp: ~N[2021-01-01 00:25:00]
- )
- ])
-
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "metrics" => ["visitors"],
- "date_range" => "all",
- "dimensions" => ["visit:browser"],
- "filters" => [
- ["is", "event:name", ["Purchase"]],
- ["is", "event:props:package", ["business"]]
- ]
- })
-
- %{"results" => results} = json_response(conn, 200)
-
- assert results == [
- %{"dimensions" => ["Safari"], "metrics" => [2]},
- %{"dimensions" => ["Chrome"], "metrics" => [1]}
- ]
- end
-
test "all metrics for breakdown by event prop", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
@@ -3578,452 +2726,56 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
end
end
- describe "imported data" do
- test "returns screen sizes breakdown when filtering by screen size", %{conn: conn, site: site} do
- site_import = insert(:site_import, site: site)
-
- populate_stats(site, site_import.id, [
- build(:pageview,
- timestamp: ~N[2021-01-01 00:00:01],
- screen_size: "Mobile"
- ),
- build(:imported_devices,
- device: "Mobile",
- visitors: 3,
- pageviews: 5,
- date: ~D[2021-01-01]
- )
- ])
-
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "metrics" => ["visitors", "pageviews"],
- "date_range" => "all",
- "dimensions" => ["visit:device"],
- "filters" => [
- ["is", "visit:device", ["Mobile"]]
- ],
- "include" => %{"imports" => true}
- })
-
- %{"results" => results} = json_response(conn, 200)
-
- assert results == [%{"dimensions" => ["Mobile"], "metrics" => [4, 6]}]
- end
-
- test "returns custom event goals and pageview goals", %{conn: conn, site: site} do
- insert(:goal, site: site, event_name: "Purchase")
- insert(:goal, site: site, page_path: "/test")
-
- site_import = insert(:site_import, site: site)
-
- populate_stats(site, site_import.id, [
- build(:pageview,
- timestamp: ~N[2021-01-01 00:00:01],
- pathname: "/test"
- ),
- build(:event,
- name: "Purchase",
- timestamp: ~N[2021-01-01 00:00:03]
- ),
- build(:event,
- name: "Purchase",
- timestamp: ~N[2021-01-01 00:00:03]
- ),
- build(:imported_custom_events,
- name: "Purchase",
- visitors: 3,
- events: 5,
- date: ~D[2021-01-01]
- ),
- build(:imported_pages,
- page: "/test",
- visitors: 2,
- pageviews: 2,
- date: ~D[2021-01-01]
- ),
- build(:imported_visitors, visitors: 5, date: ~D[2021-01-01])
- ])
-
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "date_range" => "all",
- "dimensions" => ["event:goal"],
- "metrics" => ["visitors", "events", "pageviews", "conversion_rate"],
- "include" => %{"imports" => true}
- })
-
- assert json_response(conn, 200)["results"] == [
- %{"dimensions" => ["Purchase"], "metrics" => [5, 7, 0, 62.5]},
- %{"dimensions" => ["Visit /test"], "metrics" => [3, 3, 3, 37.5]}
- ]
- end
-
- test "pageviews are returned as events for breakdown reports other than custom events", %{
- conn: conn,
- site: site
- } do
- site_import = insert(:site_import, site: site)
-
- populate_stats(site, site_import.id, [
- build(:imported_browsers, browser: "Chrome", pageviews: 1, date: ~D[2021-01-01]),
- build(:imported_devices, device: "Desktop", pageviews: 1, date: ~D[2021-01-01]),
- build(:imported_entry_pages, entry_page: "/test", pageviews: 1, date: ~D[2021-01-01]),
- build(:imported_exit_pages, exit_page: "/test", pageviews: 1, date: ~D[2021-01-01]),
- build(:imported_locations, country: "EE", pageviews: 1, date: ~D[2021-01-01]),
- build(:imported_operating_systems,
- operating_system: "Mac",
- pageviews: 1,
- date: ~D[2021-01-01]
- ),
- build(:imported_pages, page: "/test", pageviews: 1, date: ~D[2021-01-01]),
- build(:imported_sources, source: "Google", pageviews: 1, date: ~D[2021-01-01])
- ])
-
- breakdown_and_first = fn dimension ->
- conn
- |> post("/api/v2/query", %{
- "site_id" => site.domain,
- "metrics" => ["events"],
- "date_range" => ["2021-01-01", "2021-01-01"],
- "dimensions" => [dimension],
- "include" => %{"imports" => true}
- })
- |> json_response(200)
- |> Map.get("results")
- |> List.first()
- end
-
- assert %{"dimensions" => ["Chrome"], "metrics" => [1]} =
- breakdown_and_first.("visit:browser")
-
- assert %{"dimensions" => ["Desktop"], "metrics" => [1]} =
- breakdown_and_first.("visit:device")
-
- assert %{"dimensions" => ["EE"], "metrics" => [1]} = breakdown_and_first.("visit:country")
- assert %{"dimensions" => ["Mac"], "metrics" => [1]} = breakdown_and_first.("visit:os")
- assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("event:page")
-
- assert %{"dimensions" => ["Google"], "metrics" => [1]} =
- breakdown_and_first.("visit:source")
- end
-
- for goal_name <- Plausible.Imported.goals_with_url() do
- test "returns url breakdown for #{goal_name} goal", %{conn: conn, site: site} do
- insert(:goal, event_name: unquote(goal_name), site: site)
- site_import = insert(:site_import, site: site)
-
- populate_stats(site, site_import.id, [
- build(:event,
- name: unquote(goal_name),
- "meta.key": ["url"],
- "meta.value": ["https://one.com"]
- ),
- build(:imported_custom_events,
- name: unquote(goal_name),
- visitors: 2,
- events: 5,
- link_url: "https://one.com"
- ),
- build(:imported_custom_events,
- name: unquote(goal_name),
- visitors: 5,
- events: 10,
- link_url: "https://two.com"
- ),
- build(:imported_custom_events,
- name: "some goal",
- visitors: 5,
- events: 10
- ),
- build(:imported_visitors, visitors: 9)
- ])
-
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "metrics" => ["visitors", "events", "conversion_rate"],
- "date_range" => "all",
- "dimensions" => ["event:props:url"],
- "filters" => [
- ["is", "event:goal", [unquote(goal_name)]]
- ],
- "include" => %{"imports" => true}
- })
-
- assert json_response(conn, 200)["results"] == [
- %{"dimensions" => ["https://two.com"], "metrics" => [5, 10, 50]},
- %{"dimensions" => ["https://one.com"], "metrics" => [3, 6, 30]}
- ]
-
- refute json_response(conn, 200)["meta"]["warning"]
- end
- end
-
- for goal_name <- Plausible.Imported.goals_with_path() do
- test "returns path breakdown for #{goal_name} goal", %{conn: conn, site: site} do
- insert(:goal, event_name: unquote(goal_name), site: site)
- site_import = insert(:site_import, site: site)
-
- populate_stats(site, site_import.id, [
- build(:event,
- name: unquote(goal_name),
- "meta.key": ["path"],
- "meta.value": ["/one"]
- ),
- build(:imported_custom_events,
- name: unquote(goal_name),
- visitors: 2,
- events: 5,
- path: "/one"
- ),
- build(:imported_custom_events,
- name: unquote(goal_name),
- visitors: 5,
- events: 10,
- path: "/two"
- ),
- build(:imported_custom_events,
- name: "some goal",
- visitors: 5,
- events: 10
- ),
- build(:imported_visitors, visitors: 9)
- ])
-
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "metrics" => ["visitors", "events", "conversion_rate"],
- "date_range" => "all",
- "dimensions" => ["event:props:path"],
- "filters" => [
- ["is", "event:goal", [unquote(goal_name)]]
- ],
- "include" => %{"imports" => true}
- })
-
- assert json_response(conn, 200)["results"] == [
- %{"dimensions" => ["/two"], "metrics" => [5, 10, 50]},
- %{"dimensions" => ["/one"], "metrics" => [3, 6, 30]}
- ]
-
- refute json_response(conn, 200)["meta"]["warning"]
- end
- end
-
- test "adds a warning when query params are not supported for imported data", %{
- conn: conn,
- site: site
- } do
- site_import = insert(:site_import, site: site)
-
- insert(:goal, event_name: "Signup", site: site)
-
- populate_stats(site, site_import.id, [
- build(:event,
- name: "Signup",
- "meta.key": ["package"],
- "meta.value": ["large"]
- ),
- build(:imported_visitors, visitors: 9)
- ])
-
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "metrics" => ["visitors"],
- "date_range" => "all",
- "dimensions" => ["event:props:package"],
- "filters" => [
- ["is", "event:goal", ["Signup"]]
- ],
- "include" => %{"imports" => true}
- })
-
- assert json_response(conn, 200)["results"] == [
- %{"dimensions" => ["large"], "metrics" => [1]}
- ]
-
- assert json_response(conn, 200)["meta"]["warning"] =~
- "Imported stats are not included in the results because query parameters are not supported."
- end
-
- test "does not add a warning when there are no site imports", %{conn: conn, site: site} do
- insert(:goal, event_name: "Signup", site: site)
-
- populate_stats(site, [
- build(:event,
- name: "Signup",
- "meta.key": ["package"],
- "meta.value": ["large"]
- )
- ])
-
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "metrics" => ["visitors"],
- "date_range" => "all",
- "dimensions" => ["event:props:package"],
- "filters" => [
- ["is", "event:goal", ["Signup"]]
- ],
- "include" => %{"imports" => true}
- })
-
- refute json_response(conn, 200)["meta"]["warning"]
- end
-
- test "does not add a warning when import is out of queried date range", %{
- conn: conn,
- site: site
- } do
- site_import = insert(:site_import, site: site, end_date: Date.add(Date.utc_today(), -3))
-
- insert(:goal, event_name: "Signup", site: site)
-
- populate_stats(site, site_import.id, [
- build(:event,
- name: "Signup",
- "meta.key": ["package"],
- "meta.value": ["large"]
- ),
- build(:imported_visitors, visitors: 9)
- ])
-
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "metrics" => ["visitors"],
- "date_range" => "day",
- "dimensions" => ["event:props:package"],
- "filters" => [
- ["is", "event:goal", ["Signup"]]
- ],
- "include" => %{"imports" => true}
- })
-
- refute json_response(conn, 200)["meta"]["warning"]
- end
-
- test "applies multiple filters if the properties belong to the same table", %{
- conn: conn,
- site: site
- } do
- site_import = insert(:site_import, site: site)
-
- populate_stats(site, site_import.id, [
- build(:imported_sources, source: "Google", utm_medium: "organic", utm_term: "one"),
- build(:imported_sources, source: "Twitter", utm_medium: "organic", utm_term: "two"),
- build(:imported_sources,
- source: "Facebook",
- utm_medium: "something_else",
- utm_term: "one"
- )
- ])
-
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "metrics" => ["visitors"],
- "date_range" => "day",
- "dimensions" => ["visit:source"],
- "filters" => [
- ["is", "visit:utm_medium", ["organic"]],
- ["is", "visit:utm_term", ["one"]]
- ],
- "include" => %{"imports" => true}
- })
-
- assert json_response(conn, 200)["results"] == [
- %{"dimensions" => ["Google"], "metrics" => [1]}
- ]
- end
-
- test "ignores imported data if filtered property belongs to a different table than the breakdown property",
- %{
- conn: conn,
- site: site
- } do
- site_import = insert(:site_import, site: site)
-
- populate_stats(site, site_import.id, [
- build(:imported_sources, source: "Google"),
- build(:imported_devices, device: "Desktop")
- ])
-
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "metrics" => ["visitors"],
- "date_range" => "day",
- "dimensions" => ["visit:source"],
- "filters" => [
- ["is", "visit:device", ["Desktop"]]
- ],
- "include" => %{"imports" => true}
- })
-
- assert %{
- "results" => [],
- "meta" => meta
- } = json_response(conn, 200)
-
- assert meta["warning"] =~ "Imported stats are not included in the results"
- end
+ test "filtering by custom event property", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:event,
+ name: "Purchase",
+ "meta.key": ["package"],
+ "meta.value": ["business"],
+ browser: "Chrome",
+ timestamp: ~N[2021-01-01 00:00:00]
+ ),
+ build(:event,
+ name: "Purchase",
+ "meta.key": ["package"],
+ "meta.value": ["business"],
+ browser: "Safari",
+ timestamp: ~N[2021-01-01 00:00:00]
+ ),
+ build(:event,
+ name: "Purchase",
+ "meta.key": ["package"],
+ "meta.value": ["business"],
+ browser: "Safari",
+ timestamp: ~N[2021-01-01 00:25:00]
+ ),
+ build(:event,
+ name: "Purchase",
+ "meta.key": ["package"],
+ "meta.value": ["personal"],
+ browser: "IE",
+ timestamp: ~N[2021-01-01 00:25:00]
+ )
+ ])
- test "imported country, region and city data",
- %{
- conn: conn,
- site: site
- } do
- site_import = insert(:site_import, site: site)
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "dimensions" => ["visit:browser"],
+ "filters" => [
+ ["is", "event:name", ["Purchase"]],
+ ["is", "event:props:package", ["business"]]
+ ]
+ })
- populate_stats(site, site_import.id, [
- build(:pageview,
- timestamp: ~N[2021-01-01 00:15:00],
- country_code: "DE",
- subdivision1_code: "DE-BE",
- city_geoname_id: 2_950_159
- ),
- build(:pageview,
- timestamp: ~N[2021-01-01 00:15:00],
- country_code: "DE",
- subdivision1_code: "DE-BE",
- city_geoname_id: 2_950_159
- ),
- build(:pageview,
- timestamp: ~N[2021-01-01 00:15:00],
- country_code: "EE",
- subdivision1_code: "EE-37",
- city_geoname_id: 588_409
- ),
- build(:imported_locations, country: "EE", region: "EE-37", city: 588_409, visitors: 33)
- ])
+ %{"results" => results} = json_response(conn, 200)
- for {dimension, stats_value, imports_value} <- [
- {"visit:country", "DE", "EE"},
- {"visit:region", "DE-BE", "EE-37"},
- {"visit:city", 2_950_159, 588_409}
- ] do
- conn =
- post(conn, "/api/v2/query", %{
- "site_id" => site.domain,
- "metrics" => ["visitors"],
- "date_range" => "all",
- "dimensions" => [dimension],
- "include" => %{"imports" => true}
- })
-
- assert json_response(conn, 200)["results"] == [
- %{"dimensions" => [imports_value], "metrics" => [34]},
- %{"dimensions" => [stats_value], "metrics" => [2]}
- ]
- end
- end
+ assert results == [
+ %{"dimensions" => ["Safari"], "metrics" => [2]},
+ %{"dimensions" => ["Chrome"], "metrics" => [1]}
+ ]
end
test "multiple breakdown timeseries with sources", %{conn: conn, site: site} do
diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_validations_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_validations_test.exs
new file mode 100644
index 0000000000..27c745950b
--- /dev/null
+++ b/test/plausible_web/controllers/api/external_stats_controller/query_validations_test.exs
@@ -0,0 +1,234 @@
+defmodule PlausibleWeb.Api.ExternalStatsController.QueryValidationsTest do
+ use PlausibleWeb.ConnCase
+ alias Plausible.Billing.Feature
+
+ setup [:create_user, :create_new_site, :create_api_key, :use_api_key]
+
+ describe "feature access" do
+ test "cannot break down by a custom prop without access to the props feature", %{
+ conn: conn,
+ user: user,
+ site: site
+ } do
+ ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)
+ insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "dimensions" => ["event:props:author"]
+ })
+
+ assert json_response(conn, 400)["error"] ==
+ "The owner of this site does not have access to the custom properties feature"
+ end
+
+ test "can break down by an internal prop key without access to the props feature", %{
+ conn: conn,
+ user: user,
+ site: site
+ } do
+ ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)
+ insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "dimensions" => ["event:props:path"]
+ })
+
+ assert json_response(conn, 200)["results"]
+ end
+
+ test "cannot filter by a custom prop without access to the props feature", %{
+ conn: conn,
+ user: user,
+ site: site
+ } do
+ ep =
+ insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)
+
+ insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "dimensions" => ["visit:source"],
+ "filters" => [["is", "event:props:author", ["Uku"]]]
+ })
+
+ assert json_response(conn, 400)["error"] ==
+ "The owner of this site does not have access to the custom properties feature"
+ end
+
+ test "can filter by an internal prop key without access to the props feature", %{
+ conn: conn,
+ user: user,
+ site: site
+ } do
+ ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)
+ insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "dimensions" => ["visit:source"],
+ "filters" => [["is", "event:props:url", ["whatever"]]]
+ })
+
+ assert json_response(conn, 200)["results"]
+ end
+ end
+
+ describe "param validation" do
+ test "does not allow querying conversion_rate without a goal filter", %{
+ conn: conn,
+ site: site
+ } do
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["conversion_rate"],
+ "date_range" => "all",
+ "dimensions" => ["event:page"],
+ "filters" => [["is", "event:props:author", ["Uku"]]]
+ })
+
+ assert json_response(conn, 400)["error"] ==
+ "Metric `conversion_rate` can only be queried with event:goal filters or dimensions"
+ end
+
+ test "validates that dimensions are valid", %{conn: conn, site: site} do
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "dimensions" => ["badproperty"]
+ })
+
+ assert json_response(conn, 400)["error"] =~ "Invalid dimensions"
+ end
+
+ test "empty custom property is invalid", %{conn: conn, site: site} do
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "dimensions" => ["event:props:"]
+ })
+
+ assert json_response(conn, 400)["error"] =~ "Invalid dimensions"
+ end
+
+ test "validates that correct date range is used", %{conn: conn, site: site} do
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "bad_period",
+ "dimensions" => ["event:name"]
+ })
+
+ assert json_response(conn, 400)["error"] =~ "Invalid date_range"
+ end
+
+ test "fails when an invalid metric is provided", %{conn: conn, site: site} do
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors", "baa"],
+ "date_range" => "all",
+ "dimensions" => ["event:name"]
+ })
+
+ assert json_response(conn, 400)["error"] =~ "Unknown metric '\"baa\"'"
+ end
+
+ test "session metrics cannot be used with event:name dimension", %{conn: conn, site: site} do
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors", "bounce_rate"],
+ "date_range" => "all",
+ "dimensions" => ["event:name"]
+ })
+
+ assert json_response(conn, 400)["error"] =~
+ "Session metric(s) `bounce_rate` cannot be queried along with event dimensions"
+ end
+
+ test "session metrics cannot be used with event:props:* dimension", %{conn: conn, site: site} do
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors", "bounce_rate"],
+ "date_range" => "all",
+ "dimensions" => ["event:props:url"]
+ })
+
+ assert json_response(conn, 400)["error"] =~
+ "Session metric(s) `bounce_rate` cannot be queried along with event dimensions"
+ end
+
+ test "validates that metric views_per_visit cannot be used with event:page filter", %{
+ conn: conn,
+ site: site
+ } do
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["views_per_visit"],
+ "filters" => [["is", "event:page", ["/something"]]]
+ })
+
+ assert json_response(conn, 400) == %{
+ "error" =>
+ "Metric `views_per_visit` cannot be queried with a filter on `event:page`"
+ }
+ end
+
+ test "validates that metric views_per_visit cannot be used together with dimensions", %{
+ conn: conn,
+ site: site
+ } do
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["views_per_visit"],
+ "dimensions" => ["event:name"]
+ })
+
+ assert json_response(conn, 400) == %{
+ "error" => "Metric `views_per_visit` cannot be queried with `dimensions`"
+ }
+ end
+
+ test "validates a metric can't be asked multiple times", %{
+ conn: conn,
+ site: site
+ } do
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["views_per_visit", "visitors", "visitors"]
+ })
+
+ assert json_response(conn, 400) == %{
+ "error" => "Metrics cannot be queried multiple times"
+ }
+ end
+ end
+end