Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Support multiple filters - backend (#2749)
Uku Tahta154a4214bfa
diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.exindex 4b913c4d30..e7d79081c2 100644--- a/lib/plausible/stats/aggregate.ex+++ b/lib/plausible/stats/aggregate.ex@@ -62,23 +62,36 @@ defmodule Plausible.Stats.Aggregate do{base_query_raw, base_query_raw_params} = ClickhouseRepo.to_sql(:all, q)where_param_idx = length(base_query_raw_params)- where_param = "{$#{where_param_idx}:String}"{where_clause, where_arg} =case query.filters["event:page"] do{:is, page} ->- {"p = #{where_param}", page}+ {"p = {$#{where_param_idx}:String}", page}{:is_not, page} ->- {"p != #{where_param}", page}+ {"p != {$#{where_param_idx}:String}", page}++ {:member, page} ->+ {"p IN {$#{where_param_idx}:Array(String)}", page}++ {:not_member, page} ->+ {"p NOT IN {$#{where_param_idx}:Array(String)}", page}{:matches, expr} ->regex = page_regex(expr)- {"match(p, #{where_param})", regex}+ {"match(p, {$#{where_param_idx}:String})", regex}++ {:matches_member, exprs} ->+ page_regexes = Enum.map(exprs, &page_regex/1)+ {"multiMatchAny(p, {$#{where_param_idx}:Array(String)})", page_regexes}++ {:not_matches_member, exprs} ->+ page_regexes = Enum.map(exprs, &page_regex/1)+ {"not(multiMatchAny(p, {$#{where_param_idx}:Array(String)}))", page_regexes}{:does_not_match, expr} ->regex = page_regex(expr)- {"not(match(p, #{where_param}))", regex}+ {"not(match(p, {$#{where_param_idx}:String}))", regex}endparams = base_query_raw_params ++ [where_arg]diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.exindex 244f576401..225fc30294 100644--- a/lib/plausible/stats/base.ex+++ b/lib/plausible/stats/base.ex@@ -30,6 +30,7 @@ defmodule Plausible.Stats.Base doendend+ # credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexitydef query_events(site, query) do{first_datetime, last_datetime} = utc_boundaries(query, site)@@ -49,6 +50,17 @@ defmodule Plausible.Stats.Base do{:is_not, page} ->from(e in q, where: e.pathname != ^page)+ {:matches_member, glob_exprs} ->+ page_regexes = Enum.map(glob_exprs, &page_regex/1)+ from(e in q, where: fragment("multiMatchAny(?, ?)", e.pathname, ^page_regexes))++ {:not_matches_member, glob_exprs} ->+ page_regexes = Enum.map(glob_exprs, &page_regex/1)++ from(e in q,+ where: fragment("not(multiMatchAny(?, ?))", e.pathname, ^page_regexes)+ )+{:matches, glob_expr} ->regex = page_regex(glob_expr)from(e in q, where: fragment("match(?, ?)", e.pathname, ^regex))@@ -60,6 +72,9 @@ defmodule Plausible.Stats.Base do{:member, list} ->from(e in q, where: e.pathname in ^list)+ {:not_member, list} ->+ from(e in q, where: e.pathname not in ^list)+nil ->qend@@ -78,16 +93,42 @@ defmodule Plausible.Stats.Base doq =case query.filters["event:goal"] do- {:is, :page, path} ->+ {:is, {:page, path}} ->from(e in q, where: e.pathname == ^path)- {:matches, :page, expr} ->+ {:matches, {:page, expr}} ->regex = page_regex(expr)from(e in q, where: fragment("match(?, ?)", e.pathname, ^regex))- {:is, :event, event} ->+ {:is, {:event, event}} ->from(e in q, where: e.name == ^event)+ {:member, clauses} ->+ {events, pages} = split_goals(clauses)+ from(e in q, where: e.pathname in ^pages or e.name in ^events)++ {:matches_member, clauses} ->+ {events, pages} = split_goals(clauses, &page_regex/1)++ from(e in q,+ where:+ fragment("multiMatchAny(?, ?)", e.pathname, ^pages) or+ fragment("multiMatchAny(?, ?)", e.name, ^events)+ )++ {:not_matches_member, clauses} ->+ {events, pages} = split_goals(clauses, &page_regex/1)++ from(e in q,+ where:+ fragment("not(multiMatchAny(?, ?))", e.pathname, ^pages) and+ fragment("not(multiMatchAny(?, ?))", e.name, ^events)+ )++ {:not_member, clauses} ->+ {events, pages} = split_goals(clauses)+ from(e in q, where: e.pathname not in ^pages and e.name not in ^events)+nil ->qend@@ -144,6 +185,7 @@ defmodule Plausible.Stats.Base do"city" => "city_geoname_id"}+ # credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexitydef query_sessions(site, query) do{first_datetime, last_datetime} = utc_boundaries(query, site)@@ -176,10 +218,28 @@ defmodule Plausible.Stats.Base dolist = Enum.map(values, &db_prop_val(prop_name, &1))from(s in sessions_q, where: field(s, ^prop_name) in ^list)+ {:not_member, values} ->+ list = Enum.map(values, &db_prop_val(prop_name, &1))+ from(s in sessions_q, where: fragment("? not in ?", field(s, ^prop_name), ^list))+{:matches, expr} ->regex = page_regex(expr)from(s in sessions_q, where: fragment("match(?, ?)", field(s, ^prop_name), ^regex))+ {:matches_member, exprs} ->+ page_regexes = Enum.map(exprs, &page_regex/1)++ from(s in sessions_q,+ where: fragment("multiMatchAny(?, ?)", field(s, ^prop_name), ^page_regexes)+ )++ {:not_matches_member, exprs} ->+ page_regexes = Enum.map(exprs, &page_regex/1)++ from(s in sessions_q,+ where: fragment("not(multiMatchAny(?, ?))", field(s, ^prop_name), ^page_regexes)+ )+{:does_not_match, expr} ->regex = page_regex(expr)from(s in sessions_q, where: fragment("not(match(?, ?))", field(s, ^prop_name), ^regex))@@ -452,4 +512,14 @@ defmodule Plausible.Stats.Base dofrom(e in db_q, hints: [sample: threshold])endend++ defp split_goals(clauses, map_fn \\ &Function.identity/1) do+ groups =+ Enum.group_by(clauses, fn {goal_type, _v} -> goal_type end, fn {_k, val} -> map_fn.(val) end)++ {+ Map.get(groups, :event, []),+ Map.get(groups, :page, [])+ }+ endenddiff --git a/lib/plausible/stats/filter_parser.ex b/lib/plausible/stats/filter_parser.exindex c0b4faceb7..160638893e 100644--- a/lib/plausible/stats/filter_parser.ex+++ b/lib/plausible/stats/filter_parser.ex@@ -74,8 +74,8 @@ defmodule Plausible.Stats.FilterParser do|> Enum.map(&String.trim/1)end- defp parse_goal_filter("Visit " <> page), do: {:is, :page, page}- defp parse_goal_filter(event), do: {:is, :event, event}+ defp parse_goal_filter("Visit " <> page), do: {:is, {:page, page}}+ defp parse_goal_filter(event), do: {:is, {:event, event}}defp remove_escape_chars(value) doString.replace(value, "\\|", "|")diff --git a/lib/plausible/stats/filters.ex b/lib/plausible/stats/filters.exindex d65274f53b..bb7af2cb72 100644--- a/lib/plausible/stats/filters.ex+++ b/lib/plausible/stats/filters.ex@@ -22,7 +22,8 @@ defmodule Plausible.Stats.Filters do@event_props ["name",- "page"+ "page",+ "goal"]def visit_props() do@@ -33,20 +34,7 @@ defmodule Plausible.Stats.Filters donew_filters =Enum.reduce(query.filters, %{}, fn {name, val}, new_filters ->cond do- name == "goal" ->- filter =- case val do- "Visit " <> page ->- {filter_type, filter_val} = filter_value(name, page)- {filter_type, :page, filter_val}-- event ->- {:is, :event, event}- end-- Map.put(new_filters, "event:goal", filter)-- name in (@visit_props ++ ["goal"]) ->+ name in @visit_props ->Map.put(new_filters, "visit:" <> name, filter_value(name, val))name in @event_props ->@@ -62,23 +50,49 @@ defmodule Plausible.Stats.Filters do%Plausible.Stats.Query{query | filters: new_filters}end- defp filter_value(key, "!" <> val) do- if String.contains?(key, ["page", "goal"]) && String.match?(val, ~r/\*/) do- {:does_not_match, val}- else- {:is_not, val}+ @non_escaped_pipe_regex ~r/(?<!\\)\|/+ # credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity+ defp filter_value(key, val) do+ {is_negated, val} = parse_negated_prefix(val)+ {is_contains, val} = parse_contains_prefix(val)+ is_list = Regex.match?(@non_escaped_pipe_regex, val)+ is_wildcard = String.contains?(key, ["page", "goal"]) && String.match?(val, ~r/\*/)+ val = if is_list, do: parse_member_list(val), else: val+ val = if key == "goal", do: wrap_goal_value(val), else: val++ cond do+ is_negated && is_wildcard && is_list -> {:not_matches_member, val}+ is_negated && is_contains && is_list -> {:not_matches_member, Enum.map(val, &"**#{&1}**")}+ is_negated && is_wildcard -> {:does_not_match, val}+ is_negated && is_list -> {:not_member, val}+ is_negated && is_contains -> {:does_not_match, "**" <> val <> "**"}+ is_contains && is_list -> {:matches_member, Enum.map(val, &"**#{&1}**")}+ is_wildcard && is_list -> {:matches_member, val}+ is_negated -> {:is_not, val}+ is_list -> {:member, val}+ is_contains -> {:matches, "**" <> val <> "**"}+ is_wildcard -> {:matches, val}+ true -> {:is, val}endend- defp filter_value(_, "~" <> val) do- {:matches, "**" <> val <> "**"}+ defp parse_negated_prefix("!" <> val), do: {true, val}+ defp parse_negated_prefix(val), do: {false, val}++ defp parse_contains_prefix("~" <> val), do: {true, val}+ defp parse_contains_prefix(val), do: {false, val}++ defp parse_member_list(raw_value) do+ raw_value+ |> String.split(@non_escaped_pipe_regex)+ |> Enum.map(&remove_escape_chars/1)end- defp filter_value(key, val) do- if String.contains?(key, ["page", "goal"]) && String.match?(val, ~r/\*/) do- {:matches, val}- else- {:is, val}- end+ defp remove_escape_chars(value) do+ String.replace(value, "\\|", "|")end++ defp wrap_goal_value(goals) when is_list(goals), do: Enum.map(goals, &wrap_goal_value/1)+ defp wrap_goal_value("Visit " <> page), do: {:page, page}+ defp wrap_goal_value(event), do: {:event, event}enddiff --git a/lib/plausible/stats/props.ex b/lib/plausible/stats/props.exindex 7d4a264772..cc8b6856d7 100644--- a/lib/plausible/stats/props.ex+++ b/lib/plausible/stats/props.ex@@ -35,7 +35,7 @@ defmodule Plausible.Stats.Props docase prop_filter do{"event:props:" <> key, {_, "(none)"}} ->- {_, _, goal_name} = query.filters["event:goal"]+ {_, {_, goal_name}} = query.filters["event:goal"]%{goal_name => [key]}{"event:props:" <> _key, _} ->diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex 26fefc22ef..7df5e1a854 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -1098,7 +1098,7 @@ defmodule PlausibleWeb.Api.StatsController doprop_names =if query.filters["event:goal"] do- {_, _, goal} = query.filters["event:goal"]+ {_, {_, goal}} = query.filters["event:goal"]Stats.props(site, query)|> Map.get(goal, [])diff --git a/test/plausible/stats/filter_parser_test.exs b/test/plausible/stats/filter_parser_test.exsindex 2f6007bdf3..9e7c09866c 100644--- a/test/plausible/stats/filter_parser_test.exs+++ b/test/plausible/stats/filter_parser_test.exs@@ -36,12 +36,12 @@ defmodule Plausible.Stats.FilterParserTest dotest "custom event goal" do"event:goal==Signup"- |> assert_parsed(%{"event:goal" => {:is, :event, "Signup"}})+ |> assert_parsed(%{"event:goal" => {:is, {:event, "Signup"}}})endtest "pageview goal" do"event:goal==Visit /blog"- |> assert_parsed(%{"event:goal" => {:is, :page, "/blog"}})+ |> assert_parsed(%{"event:goal" => {:is, {:page, "/blog"}}})endtest "member" dodiff --git a/test/plausible/stats/filters_test.exs b/test/plausible/stats/filters_test.exsnew file mode 100644index 0000000000..69fe6d4dfb--- /dev/null+++ b/test/plausible/stats/filters_test.exs@@ -0,0 +1,196 @@+defmodule Plausible.Stats.FiltersTest do+ use ExUnit.Case, async: true+ alias Plausible.Stats.{Query, Filters}++ def assert_parsed(filters, expected_output) do+ new_query =+ %Query{filters: filters}+ |> Filters.add_prefix()++ assert new_query.filters == expected_output+ end++ describe "adding prefix" do+ test "adds appropriate prefix to filter" do+ %{"page" => "/"}+ |> assert_parsed(%{"event:page" => {:is, "/"}})++ %{"goal" => "Signup"}+ |> assert_parsed(%{"event:goal" => {:is, {:event, "Signup"}}})++ %{"goal" => "Visit /blog"}+ |> assert_parsed(%{"event:goal" => {:is, {:page, "/blog"}}})++ %{"source" => "Google"}+ |> assert_parsed(%{"visit:source" => {:is, "Google"}})++ %{"referrer" => "cnn.com"}+ |> assert_parsed(%{"visit:referrer" => {:is, "cnn.com"}})++ %{"utm_medium" => "search"}+ |> assert_parsed(%{"visit:utm_medium" => {:is, "search"}})++ %{"utm_source" => "bing"}+ |> assert_parsed(%{"visit:utm_source" => {:is, "bing"}})++ %{"utm_content" => "content"}+ |> assert_parsed(%{"visit:utm_content" => {:is, "content"}})++ %{"utm_term" => "term"}+ |> assert_parsed(%{"visit:utm_term" => {:is, "term"}})++ %{"screen" => "Desktop"}+ |> assert_parsed(%{"visit:screen" => {:is, "Desktop"}})++ %{"browser" => "Opera"}+ |> assert_parsed(%{"visit:browser" => {:is, "Opera"}})++ %{"browser_version" => "10.1"}+ |> assert_parsed(%{"visit:browser_version" => {:is, "10.1"}})++ %{"os" => "Linux"}+ |> assert_parsed(%{"visit:os" => {:is, "Linux"}})++ %{"os_version" => "13.0"}+ |> assert_parsed(%{"visit:os_version" => {:is, "13.0"}})++ %{"country" => "EE"}+ |> assert_parsed(%{"visit:country" => {:is, "EE"}})++ %{"region" => "EE-12"}+ |> assert_parsed(%{"visit:region" => {:is, "EE-12"}})++ %{"city" => "123"}+ |> assert_parsed(%{"visit:city" => {:is, "123"}})++ %{"entry_page" => "/blog"}+ |> assert_parsed(%{"visit:entry_page" => {:is, "/blog"}})++ %{"exit_page" => "/blog"}+ |> assert_parsed(%{"visit:exit_page" => {:is, "/blog"}})++ %{"props" => %{"cta" => "Top"}}+ |> assert_parsed(%{"event:props:cta" => {:is, "Top"}})+ end+ end++ describe "is not filter type" do+ test "simple is not filter" do+ %{"page" => "!/"}+ |> assert_parsed(%{"event:page" => {:is_not, "/"}})++ %{"props" => %{"cta" => "!Top"}}+ |> assert_parsed(%{"event:props:cta" => {:is_not, "Top"}})+ end+ end++ describe "member filter type" do+ test "simple member filter" do+ %{"page" => "/|/blog"}+ |> assert_parsed(%{"event:page" => {:member, ["/", "/blog"]}})+ end++ test "escaping pipe character" do+ %{"page" => "/|\\|"}+ |> assert_parsed(%{"event:page" => {:member, ["/", "|"]}})+ end++ test "mixed goals" do+ %{"goal" => "Signup|Visit /thank-you"}+ |> assert_parsed(%{"event:goal" => {:member, [{:event, "Signup"}, {:page, "/thank-you"}]}})++ %{"goal" => "Visit /thank-you|Signup"}+ |> assert_parsed(%{"event:goal" => {:member, [{:page, "/thank-you"}, {:event, "Signup"}]}})+ end+ end++ describe "matches_member filter type" do+ test "parses matches_member filter type" do+ %{"page" => "/|/blog**"}+ |> assert_parsed(%{"event:page" => {:matches_member, ["/", "/blog**"]}})+ end++ test "parses not_matches_member filter type" do+ %{"page" => "!/|/blog**"}+ |> assert_parsed(%{"event:page" => {:not_matches_member, ["/", "/blog**"]}})+ end+ end++ describe "contains filter type" do+ test "single contains" do+ %{"page" => "~blog"}+ |> assert_parsed(%{"event:page" => {:matches, "**blog**"}})+ end++ test "negated contains" do+ %{"page" => "!~articles"}+ |> assert_parsed(%{"event:page" => {:does_not_match, "**articles**"}})+ end++ test "contains member" do+ %{"page" => "~articles|blog"}+ |> assert_parsed(%{"event:page" => {:matches_member, ["**articles**", "**blog**"]}})+ end++ test "not contains member" do+ %{"page" => "!~articles|blog"}+ |> assert_parsed(%{"event:page" => {:not_matches_member, ["**articles**", "**blog**"]}})+ end+ end++ describe "not_member filter type" do+ test "simple not_member filter" do+ %{"page" => "!/|/blog"}+ |> assert_parsed(%{"event:page" => {:not_member, ["/", "/blog"]}})+ end++ test "mixed goals" do+ %{"goal" => "!Signup|Visit /thank-you"}+ |> assert_parsed(%{+ "event:goal" => {:not_member, [{:event, "Signup"}, {:page, "/thank-you"}]}+ })++ %{"goal" => "!Visit /thank-you|Signup"}+ |> assert_parsed(%{+ "event:goal" => {:not_member, [{:page, "/thank-you"}, {:event, "Signup"}]}+ })+ end+ end++ describe "matches filter type" do+ test "can be used with `goal` or `page` filters" do+ %{"page" => "/blog/post-*"}+ |> assert_parsed(%{"event:page" => {:matches, "/blog/post-*"}})++ %{"goal" => "Visit /blog/post-*"}+ |> assert_parsed(%{"event:goal" => {:matches, {:page, "/blog/post-*"}}})+ end++ test "other filters default to `is` even when wildcard is present" do+ %{"country" => "Germa**"}+ |> assert_parsed(%{"visit:country" => {:is, "Germa**"}})+ end+ end++ describe "does_not_match filter type" do+ test "can be used with `page` filter" do+ %{"page" => "!/blog/post-*"}+ |> assert_parsed(%{"event:page" => {:does_not_match, "/blog/post-*"}})+ end++ test "other filters default to is_not even when wildcard is present" do+ %{"country" => "!Germa**"}+ |> assert_parsed(%{"visit:country" => {:is_not, "Germa**"}})+ end+ end++ describe "contains prefix filter type" do+ test "can be used with any filter" do+ %{"page" => "~/blog/post"}+ |> assert_parsed(%{"event:page" => {:matches, "**/blog/post**"}})++ %{"source" => "~facebook"}+ |> assert_parsed(%{"visit:source" => {:matches, "**facebook**"}})+ end+ end+enddiff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exsindex 6cc20f9519..ffc38b7a12 100644--- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs@@ -248,6 +248,167 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do}]end++ test "can filter by multiple mixed goals", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, pathname: "/"),+ build(:pageview, pathname: "/"),+ build(:pageview, pathname: "/another"),+ build(:pageview, pathname: "/register"),+ build(:event, name: "Signup"),+ build(:event, name: "Signup")+ ])++ insert(:goal, %{domain: site.domain, page_path: "/register"})+ insert(:goal, %{domain: site.domain, event_name: "Signup"})++ filters = Jason.encode!(%{goal: "Signup|Visit /register"})++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}"+ )++ assert json_response(conn, 200) == [+ %{+ "name" => "Signup",+ "unique_conversions" => 2,+ "total_conversions" => 2,+ "prop_names" => nil,+ "conversion_rate" => 33.3+ },+ %{+ "name" => "Visit /register",+ "unique_conversions" => 1,+ "total_conversions" => 1,+ "prop_names" => nil,+ "conversion_rate" => 16.7+ }+ ]+ end++ test "can filter by multiple negated mixed goals", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, pathname: "/"),+ build(:pageview, pathname: "/"),+ build(:pageview, pathname: "/another"),+ build(:pageview, pathname: "/register"),+ build(:event, name: "CTA"),+ build(:event, name: "Signup")+ ])++ insert(:goal, %{domain: site.domain, page_path: "/register"})+ insert(:goal, %{domain: site.domain, page_path: "/another"})+ insert(:goal, %{domain: site.domain, event_name: "CTA"})+ insert(:goal, %{domain: site.domain, event_name: "Signup"})++ filters = Jason.encode!(%{goal: "!Signup|Visit /another"})++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}"+ )++ assert json_response(conn, 200) == [+ %{+ "name" => "CTA",+ "unique_conversions" => 1,+ "total_conversions" => 1,+ "prop_names" => nil,+ "conversion_rate" => 16.7+ },+ %{+ "name" => "Visit /register",+ "unique_conversions" => 1,+ "total_conversions" => 1,+ "prop_names" => nil,+ "conversion_rate" => 16.7+ }+ ]+ end++ test "can filter by matches_member filter type on goals", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, pathname: "/"),+ build(:pageview, pathname: "/another"),+ build(:pageview, pathname: "/blog/post-1"),+ build(:pageview, pathname: "/blog/post-2"),+ build(:event, name: "CTA"),+ build(:event, name: "Signup")+ ])++ insert(:goal, %{domain: site.domain, page_path: "/blog**"})+ insert(:goal, %{domain: site.domain, event_name: "CTA"})+ insert(:goal, %{domain: site.domain, event_name: "Signup"})++ filters = Jason.encode!(%{goal: "Signup|Visit /blog**"})++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}"+ )++ assert json_response(conn, 200) == [+ %{+ "name" => "Visit /blog**",+ "unique_conversions" => 2,+ "total_conversions" => 2,+ "prop_names" => nil,+ "conversion_rate" => 33.3+ },+ %{+ "name" => "Signup",+ "unique_conversions" => 1,+ "total_conversions" => 1,+ "prop_names" => nil,+ "conversion_rate" => 16.7+ }+ ]+ end++ test "can filter by not_matches_member filter type on goals", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, pathname: "/another"),+ build(:pageview, pathname: "/another"),+ build(:pageview, pathname: "/blog/post-1"),+ build(:pageview, pathname: "/blog/post-2"),+ build(:event, name: "CTA"),+ build(:event, name: "Signup")+ ])++ insert(:goal, %{domain: site.domain, page_path: "/blog**"})+ insert(:goal, %{domain: site.domain, page_path: "/ano**"})+ insert(:goal, %{domain: site.domain, event_name: "CTA"})+ insert(:goal, %{domain: site.domain, event_name: "Signup"})++ filters = Jason.encode!(%{goal: "!Signup|Visit /blog**"})++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}"+ )++ assert json_response(conn, 200) == [+ %{+ "name" => "Visit /ano**",+ "unique_conversions" => 2,+ "total_conversions" => 2,+ "prop_names" => nil,+ "conversion_rate" => 33.3+ },+ %{+ "name" => "CTA",+ "unique_conversions" => 1,+ "total_conversions" => 1,+ "prop_names" => nil,+ "conversion_rate" => 16.7+ }+ ]+ endenddescribe "GET /api/stats/:domain/conversions - with goal and prop=(none) filter" dodiff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exsindex ab8a95ce4b..4cfe4f9830 100644--- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs@@ -354,6 +354,220 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do]end+ test "can filter using the | (OR) filter",+ %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ pathname: "/",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/irrelevant",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:01:00]+ ),+ build(:pageview,+ pathname: "/",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:02:00]+ ),+ build(:pageview,+ pathname: "/",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/about",+ timestamp: ~N[2021-01-01 00:10:00]+ )+ ])++ filters = Jason.encode!(%{page: "/about|/"})++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}&detailed=true"+ )++ assert json_response(conn, 200) == [+ %{+ "name" => "/",+ "visitors" => 2,+ "pageviews" => 3,+ "bounce_rate" => 50,+ "time_on_page" => 60+ },+ %{+ "name" => "/about",+ "visitors" => 1,+ "pageviews" => 1,+ "bounce_rate" => 100,+ "time_on_page" => nil+ }+ ]+ end++ test "can filter using the not_member filter type",+ %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ pathname: "/",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/irrelevant",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:01:00]+ ),+ build(:pageview,+ pathname: "/",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:02:00]+ ),+ build(:pageview,+ pathname: "/",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/about",+ timestamp: ~N[2021-01-01 00:10:00]+ )+ ])++ filters = Jason.encode!(%{page: "!/irrelevant|/about"})++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}&detailed=true"+ )++ assert json_response(conn, 200) == [+ %{+ "name" => "/",+ "visitors" => 2,+ "pageviews" => 3,+ "bounce_rate" => 50,+ "time_on_page" => 60+ }+ ]+ end++ test "can filter using the matches_member filter type",+ %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ pathname: "/blog/post-1",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/blog/post-2",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:01:00]+ ),+ build(:pageview,+ pathname: "/",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/articles/post-1",+ timestamp: ~N[2021-01-01 00:10:00]+ ),+ build(:pageview,+ pathname: "/articles/post-1",+ timestamp: ~N[2021-01-01 00:10:00]+ )+ ])++ filters = Jason.encode!(%{page: "/blog/**|/articles/**"})++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}&detailed=true"+ )++ assert json_response(conn, 200) == [+ %{+ "name" => "/articles/post-1",+ "visitors" => 2,+ "pageviews" => 2,+ "bounce_rate" => 100,+ "time_on_page" => nil+ },+ %{+ "name" => "/blog/post-1",+ "visitors" => 1,+ "pageviews" => 1,+ "bounce_rate" => 0,+ "time_on_page" => 60+ },+ %{+ "name" => "/blog/post-2",+ "visitors" => 1,+ "pageviews" => 1,+ "bounce_rate" => nil,+ "time_on_page" => nil+ }+ ]+ end++ test "can filter using the not_matches_member filter type",+ %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ pathname: "/blog/post-1",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/about",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:10:00]+ ),+ build(:pageview,+ pathname: "/",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/articles/post-1",+ timestamp: ~N[2021-01-01 00:10:00]+ )+ ])++ filters = Jason.encode!(%{page: "!/blog/**|/articles/**"})++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}&detailed=true"+ )++ assert json_response(conn, 200) == [+ %{+ "name" => "/",+ "visitors" => 2,+ "pageviews" => 2,+ "bounce_rate" => 50,+ "time_on_page" => 600+ },+ %{+ "name" => "/about",+ "visitors" => 1,+ "pageviews" => 1,+ "bounce_rate" => nil,+ "time_on_page" => nil+ }+ ]+ end+test "returns top pages by visitors with imported data", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview, pathname: "/"),diff --git a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exsindex 22f0d837bd..048c775932 100644--- a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs@@ -163,5 +163,24 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do}]end++ test "returns screen sizes with not_member filter type", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, referrer_source: "Google", screen_size: "Desktop"),+ build(:pageview, referrer_source: "Bad source", screen_size: "Desktop"),+ build(:pageview, referrer_source: "Google", screen_size: "Desktop"),+ build(:pageview, referrer_source: "Twitter", screen_size: "Mobile"),+ build(:pageview, referrer_source: "Second bad source", screen_size: "Mobile")+ ])++ filters = Jason.encode!(%{"source" => "!Bad source|Second bad source"})++ conn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day&filters=#{filters}")++ assert json_response(conn, 200) == [+ %{"name" => "Desktop", "visitors" => 2, "percentage" => 67},+ %{"name" => "Mobile", "visitors" => 1, "percentage" => 33}+ ]+ endendenddiff --git a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exsindex ff3e7c8a67..78ca014876 100644--- a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs@@ -123,6 +123,162 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest dores = json_response(conn, 200)assert %{"name" => "Time on page", "value" => 900, "change" => 100} in res["top_stats"]end++ test "calculates time on page when filtered for multiple pages", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview,+ pathname: "/pageA",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/pageB",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:15:00]+ ),+ build(:pageview,+ pathname: "/pageC",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:16:00]+ ),+ build(:pageview,+ pathname: "/pageA",+ timestamp: ~N[2021-01-01 00:15:00]+ )+ ])++ filters = Jason.encode!(%{page: "/pageA|/pageB"})++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/top-stats?period=day&date=2021-01-01&filters=#{filters}"+ )++ res = json_response(conn, 200)+ assert %{"name" => "Time on page", "value" => 480, "change" => 100} in res["top_stats"]+ end++ test "calculates time on page when filtered for multiple negated pages", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview,+ pathname: "/pageA",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/pageB",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:15:00]+ ),+ build(:pageview,+ pathname: "/pageC",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:16:00]+ ),+ build(:pageview,+ pathname: "/pageA",+ timestamp: ~N[2021-01-01 00:15:00]+ )+ ])++ filters = Jason.encode!(%{page: "!/pageA|/pageC"})++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/top-stats?period=day&date=2021-01-01&filters=#{filters}"+ )++ res = json_response(conn, 200)+ assert %{"name" => "Time on page", "value" => 60, "change" => 100} in res["top_stats"]+ end++ test "calculates time on page when filtered for multiple wildcard pages", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview,+ pathname: "/blog/post-1",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/about",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:15:00]+ ),+ build(:pageview,+ pathname: "/articles/post-1",+ user_id: 321,+ timestamp: ~N[2021-01-01 00:15:00]+ ),+ build(:pageview,+ pathname: "/",+ user_id: 321,+ timestamp: ~N[2021-01-01 00:16:00]+ )+ ])++ filters = Jason.encode!(%{page: "/blog/**|/articles/**"})++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/top-stats?period=day&date=2021-01-01&filters=#{filters}"+ )++ res = json_response(conn, 200)+ assert %{"name" => "Time on page", "value" => 480, "change" => 100} in res["top_stats"]+ end++ test "calculates time on page when filtered for multiple negated wildcard pages", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview,+ pathname: "/blog/post-1",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/about",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:10:00]+ ),+ build(:pageview,+ pathname: "/",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/articles/post-1",+ timestamp: ~N[2021-01-01 00:10:00]+ )+ ])++ filters = Jason.encode!(%{page: "!/blog/**|/articles/**"})++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/top-stats?period=day&date=2021-01-01&filters=#{filters}"+ )++ res = json_response(conn, 200)+ assert %{"name" => "Time on page", "value" => 600, "change" => 100} in res["top_stats"]+ endenddescribe "GET /api/stats/top-stats - with imported data" do