Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
APIv2: Regex operations, consistent operators (#4488)
Karl-Aksel Puulmann604dde99fd3b
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 0af8054c5d..665cca033a 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -11,7 +11,7 @@ All notable changes to this project will be documented in this file.- Add a search functionality in all Details views- Icons for browsers plausible/analytics#4239- Automatic custom property selection in the dashboard Properties report-- Add `does_not_contain` filter support to dashboard+- Add `contains_not` filter support to dashboard- Traffic drop notifications plausible/analytics#4300- Add search and pagination functionality into Google Keywords > Details modal- ClickHouse system.query_log table log_comment column now contains information about source of queries. Useful for debuggingdiff --git a/assets/js/dashboard/components/filter-operator-selector.js b/assets/js/dashboard/components/filter-operator-selector.jsindex fb898a0fca..155e0164bc 100644--- a/assets/js/dashboard/components/filter-operator-selector.js+++ b/assets/js/dashboard/components/filter-operator-selector.js@@ -63,7 +63,7 @@ export default function FilterOperatorSelector(props) {{renderTypeItem(FILTER_OPERATIONS.is, true)}{renderTypeItem(FILTER_OPERATIONS.isNot, supportsIsNot(filterName))}{renderTypeItem(FILTER_OPERATIONS.contains, isFreeChoiceFilter(filterName))}- {renderTypeItem(FILTER_OPERATIONS.does_not_contain, isFreeChoiceFilter(filterName) && supportsIsNot(filterName))}+ {renderTypeItem(FILTER_OPERATIONS.contains_not, isFreeChoiceFilter(filterName) && supportsIsNot(filterName))}</div></Menu.Items></Transition>diff --git a/assets/js/dashboard/query-context.tsx b/assets/js/dashboard/query-context.tsxindex 25056ebbfa..498f8b9d2b 100644--- a/assets/js/dashboard/query-context.tsx+++ b/assets/js/dashboard/query-context.tsx@@ -21,7 +21,12 @@ import {QueryPeriod,useSaveTimePreferencesToStorage} from './query-time-periods'-import { Filter, FilterClauseLabels, queryDefaultValue } from './query'+import {+ Filter,+ FilterClauseLabels,+ queryDefaultValue,+ postProcessFilters+} from './query'const queryContextDefaultValue = {query: queryDefaultValue,@@ -99,7 +104,7 @@ export default function QueryContextProvider({? (with_imported as boolean): defaultValues.with_imported,filters: Array.isArray(filters)- ? (filters as Filter[])+ ? postProcessFilters(filters as Filter[]): defaultValues.filters,labels: (labels as FilterClauseLabels) || defaultValues.labels}diff --git a/assets/js/dashboard/query.ts b/assets/js/dashboard/query.tsindex 5ca08d0fd3..2054e5893d 100644--- a/assets/js/dashboard/query.ts+++ b/assets/js/dashboard/query.ts@@ -11,6 +11,7 @@ import {isAfter} from './util/date'import {+ FILTER_OPERATIONS,getFiltersByKeyPrefix,parseLegacyFilter,parseLegacyPropsFilter@@ -84,6 +85,16 @@ const LEGACY_URL_PARAMETERS = {exit_page: null}+export function postProcessFilters(filters: Array<Filter>): Array<Filter> {+ return filters.map(([operation, dimension, clauses]) => {+ // Rename old name of the operation+ if (operation === 'does_not_contain') {+ operation = FILTER_OPERATIONS.contains_not+ }+ return [operation, dimension, clauses]+ })+}+// Called once when dashboard is loaded load. Checks whether old filter style is used and if so,// updates the filters and updates locationexport function filtersBackwardsCompatibilityRedirect(diff --git a/assets/js/dashboard/stats/modals/filter-modal-props-row.js b/assets/js/dashboard/stats/modals/filter-modal-props-row.jsindex 486d0d469c..d8fd123ba8 100644--- a/assets/js/dashboard/stats/modals/filter-modal-props-row.js+++ b/assets/js/dashboard/stats/modals/filter-modal-props-row.js@@ -35,7 +35,7 @@ export default function FilterModalPropsRow({}function fetchPropValueOptions(input) {- if ([FILTER_OPERATIONS.contains, FILTER_OPERATIONS.does_not_contain].includes(operation) || propKey == "") {+ if ([FILTER_OPERATIONS.contains, FILTER_OPERATIONS.contains_not].includes(operation) || propKey == "") {return Promise.resolve([])}return fetchSuggestions(apiPath(site, `/suggestions/prop_value`), query, input, [diff --git a/assets/js/dashboard/stats/modals/filter-modal-row.js b/assets/js/dashboard/stats/modals/filter-modal-row.jsindex e984a13e41..adcc2ad50d 100644--- a/assets/js/dashboard/stats/modals/filter-modal-row.js+++ b/assets/js/dashboard/stats/modals/filter-modal-row.js@@ -34,7 +34,7 @@ export default function FilterModalRow({}function fetchOptions(input) {- if ([FILTER_OPERATIONS.contains, FILTER_OPERATIONS.does_not_contain].includes(operation)) {+ if ([FILTER_OPERATIONS.contains, FILTER_OPERATIONS.contains_not].includes(operation)) {return Promise.resolve([])}diff --git a/assets/js/dashboard/util/filters.js b/assets/js/dashboard/util/filters.jsindex 6c5fc6f0c5..31f96a0b19 100644--- a/assets/js/dashboard/util/filters.js+++ b/assets/js/dashboard/util/filters.js@@ -28,14 +28,14 @@ export const FILTER_OPERATIONS = {is: 'is',isNot: 'is_not',contains: 'contains',- does_not_contain: 'does_not_contain'+ contains_not: 'contains_not'};export const FILTER_OPERATIONS_DISPLAY_NAMES = {[FILTER_OPERATIONS.is]: 'is',[FILTER_OPERATIONS.isNot]: 'is not',[FILTER_OPERATIONS.contains]: 'contains',- [FILTER_OPERATIONS.does_not_contain]: 'does not contain'+ [FILTER_OPERATIONS.contains_not]: 'does not contain'}const OPERATION_PREFIX = {diff --git a/lib/plausible/google/search_console/filters.ex b/lib/plausible/google/search_console/filters.exindex beca3923dc..d010e013b5 100644--- a/lib/plausible/google/search_console/filters.ex+++ b/lib/plausible/google/search_console/filters.ex@@ -31,7 +31,7 @@ defmodule Plausible.Google.SearchConsole.Filters do%{dimension: "page", operator: "includingRegex", expression: expression}end- defp transform_filter(property, [:matches, "visit:entry_page", pages])+ defp transform_filter(property, [:matches_wildcard, "visit:entry_page", pages])when is_list(pages) doexpression =Enum.map_join(pages, "|", fn page -> page_regex(property_url(property, page)) end)diff --git a/lib/plausible/stats/filters/filters.ex b/lib/plausible/stats/filters/filters.exindex 62b15cedab..d7650c2b2c 100644--- a/lib/plausible/stats/filters/filters.ex+++ b/lib/plausible/stats/filters/filters.ex@@ -60,7 +60,7 @@ defmodule Plausible.Stats.Filters do### Examples:iex> Filters.parse("{\\"page\\":\\"/blog/**\\"}")- [[:matches, "event:page", ["/blog/**"]]]+ [[:matches_wildcard, "event:page", ["/blog/**"]]]iex> Filters.parse("visit:browser!=Chrome")[[:is_not, "visit:browser", ["Chrome"]]]diff --git a/lib/plausible/stats/filters/legacy_dashboard_filter_parser.ex b/lib/plausible/stats/filters/legacy_dashboard_filter_parser.exindex c6fef54726..8e3ef1ccd2 100644--- a/lib/plausible/stats/filters/legacy_dashboard_filter_parser.ex+++ b/lib/plausible/stats/filters/legacy_dashboard_filter_parser.ex@@ -35,23 +35,23 @@ defmodule Plausible.Stats.Filters.LegacyDashboardFilterParser docond dois_negated && is_wildcard && is_list ->- [:does_not_match, key, val]+ [:matches_wildcard_not, key, val]# TODOis_negated && is_contains && is_list ->- [:does_not_match, key, Enum.map(val, &"**#{&1}**")]+ [:matches_wildcard_not, key, Enum.map(val, &"**#{&1}**")]is_wildcard && is_list ->- [:matches, key, val]+ [:matches_wildcard, key, val]is_negated && is_wildcard ->- [:does_not_match, key, [val]]+ [:matches_wildcard_not, key, [val]]is_negated && is_list ->[:is_not, key, val]is_negated && is_contains ->- [:does_not_match, key, ["**" <> val <> "**"]]+ [:matches_wildcard_not, key, ["**" <> val <> "**"]]is_negated ->[:is_not, key, [val]]@@ -66,7 +66,7 @@ defmodule Plausible.Stats.Filters.LegacyDashboardFilterParser do[:contains, key, [val]]is_wildcard ->- [:matches, key, [val]]+ [:matches_wildcard, key, [val]]true ->[:is, key, [val]]diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.exindex d79cbacc6e..87873a1dd0 100644--- a/lib/plausible/stats/filters/query_parser.ex+++ b/lib/plausible/stats/filters/query_parser.ex@@ -74,9 +74,13 @@ defmodule Plausible.Stats.Filters.QueryParser dodefp parse_operator(["is" | _rest]), do: {:ok, :is}defp parse_operator(["is_not" | _rest]), do: {:ok, :is_not}defp parse_operator(["matches" | _rest]), do: {:ok, :matches}- defp parse_operator(["does_not_match" | _rest]), do: {:ok, :does_not_match}+ defp parse_operator(["matches_not" | _rest]), do: {:ok, :matches_not}+ defp parse_operator(["matches_wildcard" | _rest]), do: {:ok, :matches_wildcard}+ defp parse_operator(["matches_wildcard_not" | _rest]), do: {:ok, :matches_wildcard_not}defp parse_operator(["contains" | _rest]), do: {:ok, :contains}- defp parse_operator(["does_not_contain" | _rest]), do: {:ok, :does_not_contain}+ defp parse_operator(["contains_not" | _rest]), do: {:ok, :contains_not}+ # :TODO: Remove this once frontend support is gone+ defp parse_operator(["does_not_contain" | _rest]), do: {:ok, :contains_not}defp parse_operator(["not" | _rest]), do: {:ok, :not}defp parse_operator(["and" | _rest]), do: {:ok, :and}defp parse_operator(["or" | _rest]), do: {:ok, :or}@@ -96,7 +100,16 @@ defmodule Plausible.Stats.Filters.QueryParser dodefp parse_filter_key(filter), do: {:error, "Invalid filter '#{i(filter)}'."}defp parse_filter_rest(operator, filter)- when operator in [:is, :is_not, :matches, :does_not_match, :contains, :does_not_contain],+ when operator in [+ :is,+ :is_not,+ :matches,+ :matches_not,+ :matches_wildcard,+ :matches_wildcard_not,+ :contains,+ :contains_not+ ],do: parse_clauses_list(filter)defp parse_filter_rest(operator, _filter)diff --git a/lib/plausible/stats/filters/stats_api_filter_parser.ex b/lib/plausible/stats/filters/stats_api_filter_parser.exindex 1985f32c5b..f7168a6741 100644--- a/lib/plausible/stats/filters/stats_api_filter_parser.ex+++ b/lib/plausible/stats/filters/stats_api_filter_parser.ex@@ -27,8 +27,8 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser dofinal_value = remove_escape_chars(raw_value)cond do- is_wildcard? && is_negated? -> [:does_not_match, key, [raw_value]]- is_wildcard? -> [:matches, key, [raw_value]]+ is_wildcard? && is_negated? -> [:matches_wildcard_not, key, [raw_value]]+ is_wildcard? -> [:matches_wildcard, key, [raw_value]]is_list? -> [:is, key, parse_member_list(raw_value)]is_negated? -> [:is_not, key, [final_value]]true -> [:is, key, [final_value]]diff --git a/lib/plausible/stats/json_schema.ex b/lib/plausible/stats/json_schema.exindex c1f9beedef..81297dc76d 100644--- a/lib/plausible/stats/json_schema.ex+++ b/lib/plausible/stats/json_schema.ex@@ -17,12 +17,12 @@ defmodule Plausible.Stats.JSONSchema do@internal_query_schema @raw_public_schema# Add overrides for things allowed in the internal API|> JSONPointer.add!(- "#/definitions/filter_entry/oneOf/0/items/0/enum/0",- "matches"+ "#/definitions/filter_entry/oneOf/0/items/0/oneOf/0/enum/0",+ "matches_wildcard")|> JSONPointer.add!(- "#/definitions/filter_entry/oneOf/0/items/0/enum/0",- "does_not_match"+ "#/definitions/filter_entry/oneOf/0/items/0/oneOf/0/enum/0",+ "matches_wildcard_not")|> JSONPointer.add!("#/definitions/metric/oneOf/0", %{"const" => "time_on_page"diff --git a/lib/plausible/stats/sql/where_builder.ex b/lib/plausible/stats/sql/where_builder.exindex fbf7e33f63..ea33a704c9 100644--- a/lib/plausible/stats/sql/where_builder.ex+++ b/lib/plausible/stats/sql/where_builder.ex@@ -156,59 +156,67 @@ defmodule Plausible.Stats.SQL.WhereBuilder dofalseend- defp filter_custom_prop(prop_name, column_name, [:is, _, values]) do- none_value_included = Enum.member?(values, "(none)")+ defp filter_custom_prop(prop_name, column_name, [:is, _, clauses]) do+ none_value_included = Enum.member?(clauses, "(none)")dynamic([t],- (has_key(t, column_name, ^prop_name) and get_by_key(t, column_name, ^prop_name) in ^values) or+ (has_key(t, column_name, ^prop_name) and get_by_key(t, column_name, ^prop_name) in ^clauses) or(^none_value_included and not has_key(t, column_name, ^prop_name)))end- defp filter_custom_prop(prop_name, column_name, [:is_not, _, values]) do- none_value_included = Enum.member?(values, "(none)")+ defp filter_custom_prop(prop_name, column_name, [:is_not, _, clauses]) do+ none_value_included = Enum.member?(clauses, "(none)")dynamic([t],(has_key(t, column_name, ^prop_name) and- get_by_key(t, column_name, ^prop_name) not in ^values) or+ get_by_key(t, column_name, ^prop_name) not in ^clauses) or(^none_value_included andhas_key(t, column_name, ^prop_name) and- get_by_key(t, column_name, ^prop_name) not in ^values) or+ get_by_key(t, column_name, ^prop_name) not in ^clauses) or(not (^none_value_included) and not has_key(t, column_name, ^prop_name)))end- defp filter_custom_prop(prop_name, column_name, [:matches, _, clauses]) do+ defp filter_custom_prop(prop_name, column_name, [:matches_wildcard, dimension, clauses]) doregexes = Enum.map(clauses, &page_regex/1)+ filter_custom_prop(prop_name, column_name, [:matches, dimension, regexes])+ end++ defp filter_custom_prop(prop_name, column_name, [:matches_wildcard_not, dimension, clauses]) do+ regexes = Enum.map(clauses, &page_regex/1)++ filter_custom_prop(prop_name, column_name, [:matches_not, dimension, regexes])+ end++ defp filter_custom_prop(prop_name, column_name, [:matches, _dimension, clauses]) dodynamic([t],has_key(t, column_name, ^prop_name) andfragment("arrayExists(k -> match(?, k), ?)",get_by_key(t, column_name, ^prop_name),- ^regexes+ ^clauses))end- defp filter_custom_prop(prop_name, column_name, [:does_not_match, _, clauses]) do- regexes = Enum.map(clauses, &page_regex/1)-+ defp filter_custom_prop(prop_name, column_name, [:matches_not, _dimension, clauses]) dodynamic([t],has_key(t, column_name, ^prop_name) andfragment("not(arrayExists(k -> match(?, k), ?))",get_by_key(t, column_name, ^prop_name),- ^regexes+ ^clauses))end- defp filter_custom_prop(prop_name, column_name, [:contains, _, clauses]) do+ defp filter_custom_prop(prop_name, column_name, [:contains, _dimension, clauses]) dodynamic([t],has_key(t, column_name, ^prop_name) and@@ -220,7 +228,7 @@ defmodule Plausible.Stats.SQL.WhereBuilder do)end- defp filter_custom_prop(prop_name, column_name, [:does_not_contain, _, clauses]) do+ defp filter_custom_prop(prop_name, column_name, [:contains_not, _dimension, clauses]) dodynamic([t],has_key(t, column_name, ^prop_name) and@@ -232,7 +240,7 @@ defmodule Plausible.Stats.SQL.WhereBuilder do)end- defp filter_field(db_field, [:matches, _key, glob_exprs]) do+ defp filter_field(db_field, [:matches_wildcard, _dimension, glob_exprs]) dopage_regexes = Enum.map(glob_exprs, &page_regex/1)dynamic(@@ -241,34 +249,36 @@ defmodule Plausible.Stats.SQL.WhereBuilder do)end- defp filter_field(db_field, [:does_not_match, _key, glob_exprs]) do- page_regexes = Enum.map(glob_exprs, &page_regex/1)-- dynamic(- [x],- fragment("not(multiMatchAny(?, ?))", type(field(x, ^db_field), :string), ^page_regexes)- )+ defp filter_field(db_field, [:matches_wildcard_not, dimension, clauses]) do+ dynamic([], not (^filter_field(db_field, [:matches_wildcard, dimension, clauses])))end- defp filter_field(db_field, [:contains, _key, values]) do+ defp filter_field(db_field, [:contains, _dimension, values]) dodynamic([x], fragment("multiSearchAny(?, ?)", type(field(x, ^db_field), :string), ^values))end- defp filter_field(db_field, [:does_not_contain, _key, values]) do+ defp filter_field(db_field, [:contains_not, dimension, clauses]) do+ dynamic([], not (^filter_field(db_field, [:contains, dimension, clauses])))+ end++ defp filter_field(db_field, [:matches, _dimension, clauses]) dodynamic([x],- fragment("not(multiSearchAny(?, ?))", type(field(x, ^db_field), :string), ^values)+ fragment("multiMatchAny(?, ?)", type(field(x, ^db_field), :string), ^clauses))end- defp filter_field(db_field, [:is, _key, list]) do- list = Enum.map(list, &db_field_val(db_field, &1))+ defp filter_field(db_field, [:matches_not, dimension, clauses]) do+ dynamic([], not (^filter_field(db_field, [:matches, dimension, clauses])))+ end++ defp filter_field(db_field, [:is, _dimension, clauses]) do+ list = Enum.map(clauses, &db_field_val(db_field, &1))dynamic([x], field(x, ^db_field) in ^list)end- defp filter_field(db_field, [:is_not, _key, list]) do- list = Enum.map(list, &db_field_val(db_field, &1))- dynamic([x], field(x, ^db_field) not in ^list)+ defp filter_field(db_field, [:is_not, dimension, clauses]) do+ dynamic([], not (^filter_field(db_field, [:is, dimension, clauses])))end@no_ref "Direct / None"diff --git a/priv/json-schemas/query-api-schema.json b/priv/json-schemas/query-api-schema.jsonindex e12f299710..61258f88ae 100644--- a/priv/json-schemas/query-api-schema.json+++ b/priv/json-schemas/query-api-schema.json@@ -243,10 +243,21 @@"type": "array","items": [{- "type": "string",- "enum": [- "is_not",- "does_not_contain"+ "oneOf": [+ {+ "type": "string",+ "enum": [+ "is_not",+ "contains_not",+ "matches",+ "matches_not"+ ]+ },+ {+ "const": "does_not_contain",+ "deprecationMessage": "Legacy support, will be removed next week",+ "deprecated": true+ }],"description": "filter operation"},diff --git a/test/plausible/google/search_console/filters_test.exs b/test/plausible/google/search_console/filters_test.exsindex 83b0102cc6..d056dfb2e6 100644--- a/test/plausible/google/search_console/filters_test.exs+++ b/test/plausible/google/search_console/filters_test.exs@@ -22,9 +22,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do]end- test "transforms matches page filter" do+ test "transforms matches_wildcard page filter" dofilters = [- [:matches, "visit:entry_page", ["*page*"]]+ [:matches_wildcard, "visit:entry_page", ["*page*"]]]{:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters, "")@@ -64,7 +64,7 @@ defmodule Plausible.Google.SearchConsole.FiltersTest dotest "transforms matches multiple page filter" dofilters = [- [:matches, "visit:entry_page", ["/pageA*", "/pageB*"]]+ [:matches_wildcard, "visit:entry_page", ["/pageA*", "/pageB*"]]]{:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters, "")@@ -84,7 +84,7 @@ defmodule Plausible.Google.SearchConsole.FiltersTest dotest "transforms event:page exactly like visit:entry_page" dofilters = [- [:matches, "event:page", ["/pageA*", "/pageB*"]]+ [:matches_wildcard, "event:page", ["/pageA*", "/pageB*"]]]{:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters, "")@@ -165,7 +165,7 @@ defmodule Plausible.Google.SearchConsole.FiltersTest dotest "filters can be combined" dofilters = [[:is, "visit:country", ["EE", "PL"]],- [:matches, "visit:entry_page", ["*web-analytics*"]],+ [:matches_wildcard, "visit:entry_page", ["*web-analytics*"]],[:is, "visit:screen", ["Desktop"]]]@@ -188,7 +188,7 @@ defmodule Plausible.Google.SearchConsole.FiltersTest dotest "when unsupported filter is included the whole set becomes invalid" dofilters = [- [:matches, "visit:entry_page", "*web-analytics*"],+ [:matches_wildcard, "visit:entry_page", "*web-analytics*"],[:is, "visit:screen", "Desktop"],[:member, "visit:country", ["EE", "PL"]],[:is, "visit:utm_medium", "facebook"]diff --git a/test/plausible/stats/filters_test.exs b/test/plausible/stats/filters_test.exsindex 9a4b39f0ae..c239bfa156 100644--- a/test/plausible/stats/filters_test.exs+++ b/test/plausible/stats/filters_test.exs@@ -27,12 +27,12 @@ defmodule Plausible.Stats.FiltersTest dotest "wildcard" do"event:page==/blog/post-*"- |> assert_parsed([[:matches, "event:page", ["/blog/post-*"]]])+ |> assert_parsed([[:matches_wildcard, "event:page", ["/blog/post-*"]]])endtest "negative wildcard" do"event:page!=/blog/post-*"- |> assert_parsed([[:does_not_match, "event:page", ["/blog/post-*"]]])+ |> assert_parsed([[:matches_wildcard_not, "event:page", ["/blog/post-*"]]])endtest "custom event goal" do@@ -52,13 +52,13 @@ defmodule Plausible.Stats.FiltersTest dotest "member + wildcard" do"event:page==/blog**|/newsletter|/*/"- |> assert_parsed([[:matches, "event:page", ["/blog**|/newsletter|/*/"]]])+ |> assert_parsed([[:matches_wildcard, "event:page", ["/blog**|/newsletter|/*/"]]])endtest "combined with \";\"" do"event:page==/blog**|/newsletter|/*/ ; visit:country==FR|GB|DE"|> assert_parsed([- [:matches, "event:page", ["/blog**|/newsletter|/*/"]],+ [:matches_wildcard, "event:page", ["/blog**|/newsletter|/*/"]],[:is, "visit:country", ["FR", "GB", "DE"]]])end@@ -75,7 +75,7 @@ defmodule Plausible.Stats.FiltersTest dotest "keeps escape characters in is + wildcard filter" do"event:page==/**\\|page|/other/page"- |> assert_parsed([[:matches, "event:page", ["/**\\|page|/other/page"]]])+ |> assert_parsed([[:matches_wildcard, "event:page", ["/**\\|page|/other/page"]]])endtest "gracefully fails to parse garbage" dodiff --git a/test/plausible/stats/legacy_dashboard_filter_parser_test.exs b/test/plausible/stats/legacy_dashboard_filter_parser_test.exsindex dbc4a18472..89002036f8 100644--- a/test/plausible/stats/legacy_dashboard_filter_parser_test.exs+++ b/test/plausible/stats/legacy_dashboard_filter_parser_test.exs@@ -115,15 +115,15 @@ defmodule Plausible.Stats.Legacy.DashboardFilterParserTest doendend- describe "matches filter type" do+ describe "matches_wildcard filter type" dotest "parses matches filter type" do%{"page" => "/|/blog**"}- |> assert_parsed([[:matches, "event:page", ["/", "/blog**"]]])+ |> assert_parsed([[:matches_wildcard, "event:page", ["/", "/blog**"]]])endtest "parses not_matches filter type" do%{"page" => "!/|/blog**"}- |> assert_parsed([[:does_not_match, "event:page", ["/", "/blog**"]]])+ |> assert_parsed([[:matches_wildcard_not, "event:page", ["/", "/blog**"]]])endtest "single matches" do@@ -133,7 +133,7 @@ defmodule Plausible.Stats.Legacy.DashboardFilterParserTest dotest "negated matches" do%{"page" => "!~articles"}- |> assert_parsed([[:does_not_match, "event:page", ["**articles**"]]])+ |> assert_parsed([[:matches_wildcard_not, "event:page", ["**articles**"]]])endtest "matches member" do@@ -143,7 +143,7 @@ defmodule Plausible.Stats.Legacy.DashboardFilterParserTest dotest "not matches member" do%{"page" => "!~articles|blog"}- |> assert_parsed([[:does_not_match, "event:page", ["**articles**", "**blog**"]]])+ |> assert_parsed([[:matches_wildcard_not, "event:page", ["**articles**", "**blog**"]]])endtest "other filters default to `is` even when wildcard is present" do@@ -153,7 +153,7 @@ defmodule Plausible.Stats.Legacy.DashboardFilterParserTest dotest "can be used with `page` filter" do%{"page" => "!/blog/post-*"}- |> assert_parsed([[:does_not_match, "event:page", ["/blog/post-*"]]])+ |> assert_parsed([[:matches_wildcard_not, "event:page", ["/blog/post-*"]]])endtest "other filters default to is_not even when wildcard is present" dodiff --git a/test/plausible/stats/query_optimizer_test.exs b/test/plausible/stats/query_optimizer_test.exsindex 64d526edda..259d540287 100644--- a/test/plausible/stats/query_optimizer_test.exs+++ b/test/plausible/stats/query_optimizer_test.exs@@ -127,16 +127,16 @@ defmodule Plausible.Stats.QueryOptimizerTest dodate_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]),filters: [[:is, "event:hostname", ["example.com"]],- [:matches, "event:hostname", ["*.com"]]+ [:matches_wildcard, "event:hostname", ["*.com"]]],dimensions: ["visit:referrer", "visit:exit_page"]}).filters == [[:is, "event:hostname", ["example.com"]],- [:matches, "event:hostname", ["*.com"]],+ [:matches_wildcard, "event:hostname", ["*.com"]],[:is, "visit:entry_page_hostname", ["example.com"]],- [:matches, "visit:entry_page_hostname", ["*.com"]],+ [:matches_wildcard, "visit:entry_page_hostname", ["*.com"]],[:is, "visit:exit_page_hostname", ["example.com"]],- [:matches, "visit:exit_page_hostname", ["*.com"]]+ [:matches_wildcard, "visit:exit_page_hostname", ["*.com"]]]enddiff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exsindex 5aaf631461..657c017d79 100644--- a/test/plausible/stats/query_parser_test.exs+++ b/test/plausible/stats/query_parser_test.exs@@ -167,7 +167,16 @@ defmodule Plausible.Stats.Filters.QueryParserTest doenddescribe "filters validation" do- for operation <- [:is, :is_not, :matches, :does_not_match, :contains, :does_not_contain] do+ for operation <- [+ :is,+ :is_not,+ :matches_wildcard,+ :matches_wildcard_not,+ :matches,+ :matches_not,+ :contains,+ :contains_not+ ] dotest "#{operation} filter", %{site: site} do%{"site_id" => site.domain,@@ -212,7 +221,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest doendend- for operation <- [:matches, :does_not_match] do+ for operation <- [:matches_wildcard, :matches_wildcard_not] dotest "#{operation} is not a valid filter operation in public API", %{site: site} do%{"site_id" => site.domain,diff --git a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exsindex abb00b3f75..1ef25bfb73 100644--- a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs@@ -1852,7 +1852,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do"site_id" => site.domain,"metrics" => "visitors","filters" => [- ["does_not_contain", "event:page", ["/en*"]]+ ["contains_not", "event:page", ["/en*"]]]})@@ -1884,14 +1884,14 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do"site_id" => site.domain,"metrics" => "visitors","filters" => [- ["matches", "event:props:tier", ["small*"]]+ ["matches_wildcard", "event:props:tier", ["small*"]]]})assert json_response(conn, 200)["results"] == %{"visitors" => %{"value" => 3}}end- test "does_not_match custom event property", %{conn: conn, site: site} do+ test "not matches_wildcard custom event property", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview,"meta.key": ["tier"],@@ -1908,7 +1908,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest dobuild(:pageview,"meta.key": ["tier"],"meta.value": ["small-2"]- )+ ),+ build(:pageview)])conn =@@ -1916,7 +1917,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do"site_id" => site.domain,"metrics" => "visitors","filters" => [- ["does_not_match", "event:props:tier", ["small*"]]+ ["matches_wildcard_not", "event:props:tier", ["small*"]]]})@@ -1955,7 +1956,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest doassert json_response(conn, 200)["results"] == %{"visitors" => %{"value" => 3}}end- test "does_not_contain custom event property", %{conn: conn, site: site} do+ test "contains_not custom event property", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview,"meta.key": ["tier"],@@ -1980,7 +1981,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do"site_id" => site.domain,"metrics" => "visitors","filters" => [- ["does_not_contain", "event:props:tier", ["small"]]+ ["contains_not", "event:props:tier", ["small"]]]})diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exsindex cb820d9c7c..db78c9bcf8 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs@@ -752,7 +752,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doassert json_response(conn, 200)["results"] == [%{"metrics" => [2], "dimensions" => []}]end- test "does_not_contain page filter", %{conn: conn, site: site} do+ test "contains_not page filter", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview, pathname: "/en/page1"),build(:pageview, pathname: "/en/page2"),@@ -765,7 +765,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do"date_range" => "all","metrics" => ["visitors"],"filters" => [- ["does_not_contain", "event:page", ["/en/"]]+ ["contains_not", "event:page", ["/en/"]]]})@@ -848,6 +848,150 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doassert json_response(conn, 200)["results"] == [%{"metrics" => [3], "dimensions" => []}]end+ test "`matches` operator", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, pathname: "/user/1234"),+ build(:pageview, pathname: "/user/789/contributions"),+ build(:pageview, pathname: "/blog/user/1234"),+ build(:pageview, pathname: "/user/ef/contributions"),+ build(:pageview, pathname: "/other/path")+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors"],+ "filters" => [+ ["matches", "event:page", ["^/user/[0-9]+"]]+ ]+ })++ assert json_response(conn, 200)["results"] == [%{"metrics" => [2], "dimensions" => []}]+ end++ test "`matches_not` operator", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, pathname: "/user/1234"),+ build(:pageview, pathname: "/user/789/contributions"),+ build(:pageview, pathname: "/blog/user/1234"),+ build(:pageview, pathname: "/user/ef/contributions"),+ build(:pageview, pathname: "/other/path")+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors"],+ "filters" => [+ ["matches_not", "event:page", ["^/user/[0-9]+"]]+ ]+ })++ assert json_response(conn, 200)["results"] == [%{"metrics" => [3], "dimensions" => []}]+ end++ test "`contains` and `contains_not` operator with custom properties", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview,+ "meta.key": ["tier", "value"],+ "meta.value": ["large-1", "ax"]+ ),+ build(:pageview,+ "meta.key": ["tier", "value"],+ "meta.value": ["small-1", "bx"]+ ),+ build(:pageview,+ "meta.key": ["tier", "value"],+ "meta.value": ["small-1", "ax"]+ ),+ build(:pageview,+ "meta.key": ["tier", "value"],+ "meta.value": ["small-2", "bx"]+ ),+ build(:pageview,+ "meta.key": ["tier", "value"],+ "meta.value": ["small-2", "cx"]+ ),+ build(:pageview,+ "meta.key": ["tier"],+ "meta.value": ["small-3"]+ ),+ build(:pageview,+ "meta.key": ["value"],+ "meta.value": ["ax"]+ ),+ build(:pageview)+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors"],+ "filters" => [+ ["contains", "event:props:tier", ["small"]],+ ["contains_not", "event:props:value", ["b", "c"]]+ ]+ })++ assert json_response(conn, 200)["results"] == [%{"metrics" => [1], "dimensions" => []}]+ end++ test "`matches` and `matches_not` operator with custom properties", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview,+ "meta.key": ["tier", "value"],+ "meta.value": ["large-1", "a"]+ ),+ build(:pageview,+ "meta.key": ["tier", "value"],+ "meta.value": ["small-1", "b"]+ ),+ build(:pageview,+ "meta.key": ["tier", "value"],+ "meta.value": ["small-1", "a"]+ ),+ build(:pageview,+ "meta.key": ["tier", "value"],+ "meta.value": ["small-2", "b"]+ ),+ build(:pageview,+ "meta.key": ["tier", "value"],+ "meta.value": ["small-2", "c"]+ ),+ build(:pageview,+ "meta.key": ["tier"],+ "meta.value": ["small-3"]+ ),+ build(:pageview,+ "meta.key": ["value"],+ "meta.value": ["a"]+ ),+ build(:pageview)+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors"],+ "filters" => [+ ["matches", "event:props:tier", ["small.+"]],+ ["matches_not", "event:props:value", ["b|c"]]+ ]+ })++ assert json_response(conn, 200)["results"] == [%{"metrics" => [1], "dimensions" => []}]+ end+test "handles filtering by visit:country", %{conn: conn,site: sitediff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exsindex 9f384d693c..b4d8994ef9 100644--- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs@@ -106,7 +106,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do]end- test "returns top pages with :matches filter on custom pageview props", %{+ test "returns top pages with :matches_wildcard filter on custom pageview props", %{conn: conn,site: site} do@@ -1328,7 +1328,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do]end- test "filter by :matches page with imported data", %{conn: conn, site: site} do+ test "filter by :matches_wildcard page with imported data", %{conn: conn, site: site} dosite_import = insert(:site_import, site: site)populate_stats(site, site_import.id, [