Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
APIv2: Case insensitive search (#4863)
Karl-Aksel Puulmanna38eacfed54a
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex dc0f3056f8..00bb32b5c9 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -6,9 +6,13 @@ All notable changes to this project will be documented in this file.### Added- Dashboard shows comparisons for all reports- UTM Medium report and API shows (gclid) and (msclkid) for paid searches when no explicit utm medium present.+- Support for `case_sensitive: false` modifiers in Stats API V2 filters for case-insensitive searches.### Removed### Changed++- Details modal search inputs are now case-insensitive.+### Fixed- Fix returning filter suggestions for multiple custom property values in the dashboard Filter modaldiff --git a/assets/js/dashboard/stats/modals/conversions.js b/assets/js/dashboard/stats/modals/conversions.jsindex 3514bd852d..cc1a00d947 100644--- a/assets/js/dashboard/stats/modals/conversions.js+++ b/assets/js/dashboard/stats/modals/conversions.js@@ -28,7 +28,7 @@ function ConversionsModal() {}, [reportInfo.dimension])const addSearchFilter = useCallback((query, searchString) => {- return addFilter(query, ['contains', reportInfo.dimension, [searchString]])+ return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }])}, [reportInfo.dimension])function chooseMetrics() {diff --git a/assets/js/dashboard/stats/modals/devices/browser-versions-modal.js b/assets/js/dashboard/stats/modals/devices/browser-versions-modal.jsindex 369b4dce9f..d0e11e299b 100644--- a/assets/js/dashboard/stats/modals/devices/browser-versions-modal.js+++ b/assets/js/dashboard/stats/modals/devices/browser-versions-modal.js@@ -29,7 +29,7 @@ function BrowserVersionsModal() {}, [reportInfo.dimension])const addSearchFilter = useCallback((query, searchString) => {- return addFilter(query, ['contains', reportInfo.dimension, [searchString]])+ return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }])}, [reportInfo.dimension])const renderIcon = useCallback((listItem) => browserIconFor(listItem.browser), [])diff --git a/assets/js/dashboard/stats/modals/devices/browsers-modal.js b/assets/js/dashboard/stats/modals/devices/browsers-modal.jsindex 1160b8ebf5..9b600514bd 100644--- a/assets/js/dashboard/stats/modals/devices/browsers-modal.js+++ b/assets/js/dashboard/stats/modals/devices/browsers-modal.js@@ -29,7 +29,7 @@ function BrowsersModal() {}, [reportInfo.dimension])const addSearchFilter = useCallback((query, searchString) => {- return addFilter(query, ['contains', reportInfo.dimension, [searchString]])+ return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }])}, [reportInfo.dimension])const renderIcon = useCallback((listItem) => browserIconFor(listItem.name), [])diff --git a/assets/js/dashboard/stats/modals/devices/operating-system-versions-modal.js b/assets/js/dashboard/stats/modals/devices/operating-system-versions-modal.jsindex 6c7d5c72cc..9c5622b48d 100644--- a/assets/js/dashboard/stats/modals/devices/operating-system-versions-modal.js+++ b/assets/js/dashboard/stats/modals/devices/operating-system-versions-modal.js@@ -29,7 +29,7 @@ function OperatingSystemVersionsModal() {}, [reportInfo.dimension])const addSearchFilter = useCallback((query, searchString) => {- return addFilter(query, ['contains', reportInfo.dimension, [searchString]])+ return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }])}, [reportInfo.dimension])const renderIcon = useCallback((listItem) => osIconFor(listItem.os), [])diff --git a/assets/js/dashboard/stats/modals/devices/operating-systems-modal.js b/assets/js/dashboard/stats/modals/devices/operating-systems-modal.jsindex 2a92795648..9df8a6c7c7 100644--- a/assets/js/dashboard/stats/modals/devices/operating-systems-modal.js+++ b/assets/js/dashboard/stats/modals/devices/operating-systems-modal.js@@ -29,7 +29,7 @@ function OperatingSystemsModal() {}, [reportInfo.dimension])const addSearchFilter = useCallback((query, searchString) => {- return addFilter(query, ['contains', reportInfo.dimension, [searchString]])+ return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }])}, [reportInfo.dimension])const renderIcon = useCallback((listItem) => osIconFor(listItem.name), [])diff --git a/assets/js/dashboard/stats/modals/entry-pages.js b/assets/js/dashboard/stats/modals/entry-pages.jsindex 6d5a72d693..79963046fb 100644--- a/assets/js/dashboard/stats/modals/entry-pages.js+++ b/assets/js/dashboard/stats/modals/entry-pages.js@@ -29,7 +29,7 @@ function EntryPagesModal() {}, [reportInfo.dimension])const addSearchFilter = useCallback((query, searchString) => {- return addFilter(query, ['contains', reportInfo.dimension, [searchString]])+ return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }])}, [reportInfo.dimension])function chooseMetrics() {diff --git a/assets/js/dashboard/stats/modals/exit-pages.js b/assets/js/dashboard/stats/modals/exit-pages.jsindex 2be05f0168..413c890568 100644--- a/assets/js/dashboard/stats/modals/exit-pages.js+++ b/assets/js/dashboard/stats/modals/exit-pages.js@@ -29,7 +29,7 @@ function ExitPagesModal() {}, [reportInfo.dimension])const addSearchFilter = useCallback((query, searchString) => {- return addFilter(query, ['contains', reportInfo.dimension, [searchString]])+ return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }])}, [reportInfo.dimension])function chooseMetrics() {diff --git a/assets/js/dashboard/stats/modals/locations-modal.js b/assets/js/dashboard/stats/modals/locations-modal.jsindex 615f6e9e3d..bfdd471b97 100644--- a/assets/js/dashboard/stats/modals/locations-modal.js+++ b/assets/js/dashboard/stats/modals/locations-modal.js@@ -32,7 +32,7 @@ function LocationsModal({ currentView }) {}, [reportInfo.dimension])const addSearchFilter = useCallback((query, searchString) => {- return addFilter(query, ['contains', `${reportInfo.dimension}_name`, [searchString]])+ return addFilter(query, ['contains', `${reportInfo.dimension}_name`, [searchString], { case_sensitive: false }])}, [reportInfo.dimension])function chooseMetrics() {diff --git a/assets/js/dashboard/stats/modals/pages.js b/assets/js/dashboard/stats/modals/pages.jsindex 26bf287c6c..e9521f9774 100644--- a/assets/js/dashboard/stats/modals/pages.js+++ b/assets/js/dashboard/stats/modals/pages.js@@ -29,7 +29,7 @@ function PagesModal() {}, [reportInfo.dimension])const addSearchFilter = useCallback((query, searchString) => {- return addFilter(query, ['contains', reportInfo.dimension, [searchString]])+ return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }])}, [reportInfo.dimension])function chooseMetrics() {@@ -46,14 +46,14 @@ function PagesModal() {metrics.createVisitors({renderLabel: (_query) => 'Current visitors', width: 'w-36'})]}-+const defaultMetrics = [metrics.createVisitors({renderLabel: (_query) => "Visitors" }),metrics.createPageviews(),metrics.createBounceRate(),metrics.createTimeOnPage()]-+return site.flags.scroll_depth ? [...defaultMetrics, metrics.createScrollDepth()] : defaultMetrics}diff --git a/assets/js/dashboard/stats/modals/props.js b/assets/js/dashboard/stats/modals/props.jsindex 0d691abe8d..a131e8a438 100644--- a/assets/js/dashboard/stats/modals/props.js+++ b/assets/js/dashboard/stats/modals/props.js@@ -36,7 +36,7 @@ function PropsModal() {}, [propKey])const addSearchFilter = useCallback((query, searchString) => {- return addFilter(query, ['contains', `${EVENT_PROPS_PREFIX}${propKey}`, [searchString]])+ return addFilter(query, ['contains', `${EVENT_PROPS_PREFIX}${propKey}`, [searchString], { case_sensitive: false }])}, [propKey])function chooseMetrics() {diff --git a/assets/js/dashboard/stats/modals/referrer-drilldown.js b/assets/js/dashboard/stats/modals/referrer-drilldown.jsindex 6077bb737e..264e1d5454 100644--- a/assets/js/dashboard/stats/modals/referrer-drilldown.js+++ b/assets/js/dashboard/stats/modals/referrer-drilldown.js@@ -33,7 +33,7 @@ function ReferrerDrilldownModal() {}, [reportInfo.dimension])const addSearchFilter = useCallback((query, searchString) => {- return addFilter(query, ['contains', reportInfo.dimension, [searchString]])+ return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }])}, [reportInfo.dimension])function chooseMetrics() {diff --git a/assets/js/dashboard/stats/modals/sources.js b/assets/js/dashboard/stats/modals/sources.jsindex d4be4b30b9..3fd0006c71 100644--- a/assets/js/dashboard/stats/modals/sources.js+++ b/assets/js/dashboard/stats/modals/sources.js@@ -57,7 +57,7 @@ function SourcesModal({ currentView }) {}, [reportInfo.dimension])const addSearchFilter = useCallback((query, searchString) => {- return addFilter(query, ['contains', reportInfo.dimension, [searchString]])+ return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }])}, [reportInfo.dimension])function chooseMetrics() {diff --git a/assets/js/dashboard/util/filters.js b/assets/js/dashboard/util/filters.jsindex d8d13a70da..12e9a60c60 100644--- a/assets/js/dashboard/util/filters.js+++ b/assets/js/dashboard/util/filters.js@@ -229,16 +229,18 @@ export function cleanLabels(filters, labels, mergedFilterKey, mergedLabels) {const EVENT_FILTER_KEYS = new Set(['name', 'page', 'goal', 'hostname'])export function serializeApiFilters(filters) {- const apiFilters = filters.map(([operation, filterKey, clauses]) => {- let apiFilterKey = `visit:${filterKey}`- if (- filterKey.startsWith(EVENT_PROPS_PREFIX) ||- EVENT_FILTER_KEYS.has(filterKey)- ) {- apiFilterKey = `event:${filterKey}`+ const apiFilters = filters.map(+ ([operation, filterKey, clauses, ...modifiers]) => {+ let apiFilterKey = `visit:${filterKey}`+ if (+ filterKey.startsWith(EVENT_PROPS_PREFIX) ||+ EVENT_FILTER_KEYS.has(filterKey)+ ) {+ apiFilterKey = `event:${filterKey}`+ }+ return [operation, apiFilterKey, clauses, ...modifiers]}- return [operation, apiFilterKey, clauses]- })+ )return JSON.stringify(apiFilters)}diff --git a/assets/js/types/query-api.d.ts b/assets/js/types/query-api.d.tsindex 06121a8166..2b77acf192 100644--- a/assets/js/types/query-api.d.ts+++ b/assets/js/types/query-api.d.ts@@ -64,34 +64,57 @@ export type CustomPropertyFilterDimensions = string;export type GoalDimension = "event:goal";export type TimeDimensions = "time" | "time:month" | "time:week" | "time:day" | "time:hour";export type FilterTree = FilterEntry | FilterAndOr | FilterNot;-export type FilterEntry = FilterWithoutGoals | FilterWithGoals;+export type FilterEntry = FilterWithoutGoals | FilterWithGoals | FilterWithPattern;/*** @minItems 3- * @maxItems 3+ * @maxItems 4*/-export type FilterWithoutGoals = [- FilterOperationWithoutGoals | ("matches_wildcard" | "matches_wildcard_not"),- SimpleFilterDimensions | CustomPropertyFilterDimensions,- Clauses-];+export type FilterWithoutGoals =+ | [FilterOperationWithoutGoals, SimpleFilterDimensions | CustomPropertyFilterDimensions, Clauses]+ | [+ FilterOperationWithoutGoals,+ SimpleFilterDimensions | CustomPropertyFilterDimensions,+ Clauses,+ {+ case_sensitive?: boolean;+ }+ ];/*** filter operation*/-export type FilterOperationWithoutGoals = "is_not" | "contains_not" | "matches" | "matches_not";+export type FilterOperationWithoutGoals = "is_not" | "contains_not";export type Clauses = (string | number)[];+/**+ * @minItems 3+ * @maxItems 4+ */+export type FilterWithGoals =+ | [FilterOperationContains, GoalDimension | SimpleFilterDimensions | CustomPropertyFilterDimensions, Clauses]+ | [+ FilterOperationContains,+ GoalDimension | SimpleFilterDimensions | CustomPropertyFilterDimensions,+ Clauses,+ {+ case_sensitive?: boolean;+ }+ ];+/**+ * filter operation+ */+export type FilterOperationContains = "is" | "contains";/*** @minItems 3* @maxItems 3*/-export type FilterWithGoals = [- FilterOperationWithGoals,- GoalDimension | SimpleFilterDimensions | CustomPropertyFilterDimensions,+export type FilterWithPattern = [+ FilterOperationRegex | ("matches_wildcard" | "matches_wildcard_not"),+ SimpleFilterDimensions | CustomPropertyFilterDimensions,Clauses];/*** filter operation*/-export type FilterOperationWithGoals = "is" | "contains";+export type FilterOperationRegex = "matches" | "matches_not";/*** @minItems 2* @maxItems 2diff --git a/lib/plausible/goals/filters.ex b/lib/plausible/goals/filters.exindex 142095e048..b9d7d5a749 100644--- a/lib/plausible/goals/filters.ex+++ b/lib/plausible/goals/filters.ex@@ -20,14 +20,14 @@ defmodule Plausible.Goals.Filters do* `imported?` - when `true`, builds conditions on the `page` db field rather than`pathname`, and also skips the `e.name == "pageview"` check."""- def add_filter(query, [operation, "event:goal", clauses], opts \\ [])+ def add_filter(query, [operation, "event:goal", clauses | _] = filter, opts \\ [])when operation in [:is, :contains] doimported? = Keyword.get(opts, :imported?, false)Enum.reduce(clauses, false, fn clause, dynamic_statement ->condition =query.preloaded_goals- |> filter_preloaded(operation, clause)+ |> filter_preloaded(filter, clause)|> build_condition(imported?)dynamic([e], ^condition or ^dynamic_statement)@@ -38,32 +38,46 @@ defmodule Plausible.Goals.Filters dogoals = Plausible.Goals.for_site(site)Enum.reduce(filters, goals, fn- [operation, "event:goal", clauses], goals ->- goals_matching_any_clause(goals, operation, clauses)+ [_, "event:goal" | _] = filter, goals ->+ goals_matching_any_clause(goals, filter)_filter, goals ->goalsend)end- def filter_preloaded(preloaded_goals, operation, clause) when operation in [:is, :contains] do- Enum.filter(preloaded_goals, fn goal -> matches?(goal, operation, clause) end)+ defp filter_preloaded(preloaded_goals, filter, clause) do+ Enum.filter(preloaded_goals, fn goal -> matches?(goal, filter, clause) end)end- defp goals_matching_any_clause(goals, operation, clauses) do+ defp goals_matching_any_clause(goals, [_, _, clauses | _] = filter) dogoals|> Enum.filter(fn goal ->- Enum.any?(clauses, fn clause -> matches?(goal, operation, clause) end)+ Enum.any?(clauses, fn clause -> matches?(goal, filter, clause) end)end)end- defp matches?(goal, operation, clause) do+ defp matches?(goal, [operation | _rest] = filter, clause) do+ goal_name =+ goal+ |> Plausible.Goal.display_name()+ |> mod(filter)++ clause = mod(clause, filter)+case operation do:is ->- Plausible.Goal.display_name(goal) == clause+ goal_name == clause:contains ->- String.contains?(Plausible.Goal.display_name(goal), clause)+ String.contains?(goal_name, clause)+ end+ end++ defp mod(str, filter) do+ case filter do+ [_, _, _, %{case_sensitive: false}] -> String.downcase(str)+ _ -> strendenddiff --git a/lib/plausible/google/search_console/filters.ex b/lib/plausible/google/search_console/filters.exindex d010e013b5..1874e16403 100644--- a/lib/plausible/google/search_console/filters.ex+++ b/lib/plausible/google/search_console/filters.ex@@ -24,14 +24,15 @@ defmodule Plausible.Google.SearchConsole.Filters dotransform_filter(property, [op, "visit:entry_page" | rest])end- defp transform_filter(property, [:is, "visit:entry_page", pages]) when is_list(pages) do+ # :TODO: Should also work case-insensitive, if not, blacklist.+ defp transform_filter(property, [:is, "visit:entry_page", pages | _]) when is_list(pages) doexpression =Enum.map_join(pages, "|", fn page -> property_url(property, Regex.escape(page)) end)%{dimension: "page", operator: "includingRegex", expression: expression}end- defp transform_filter(property, [:matches_wildcard, "visit:entry_page", pages])+ defp transform_filter(property, [:matches_wildcard, "visit:entry_page", pages | _])when is_list(pages) doexpression =Enum.map_join(pages, "|", fn page -> page_regex(property_url(property, page)) end)@@ -39,12 +40,12 @@ defmodule Plausible.Google.SearchConsole.Filters do%{dimension: "page", operator: "includingRegex", expression: expression}end- defp transform_filter(_property, [:is, "visit:screen", devices]) when is_list(devices) do+ defp transform_filter(_property, [:is, "visit:screen", devices | _]) when is_list(devices) doexpression = Enum.map_join(devices, "|", &search_console_device/1)%{dimension: "device", operator: "includingRegex", expression: expression}end- defp transform_filter(_property, [:is, "visit:country", countries])+ defp transform_filter(_property, [:is, "visit:country", countries | _])when is_list(countries) doexpression = Enum.map_join(countries, "|", &search_console_country/1)%{dimension: "country", operator: "includingRegex", expression: expression}diff --git a/lib/plausible/stats/filters/filters.ex b/lib/plausible/stats/filters/filters.exindex c68e2b3f86..0c16ccb3f2 100644--- a/lib/plausible/stats/filters/filters.ex+++ b/lib/plausible/stats/filters/filters.ex@@ -121,8 +121,8 @@ defmodule Plausible.Stats.Filters dodef rename_dimensions_used_in_filter(filters, renames) dotransform_filters(filters, fn- [operation, dimension, clauses] ->- [[operation, Map.get(renames, dimension, dimension), clauses]]+ [operation, dimension | rest] ->+ [[operation, Map.get(renames, dimension, dimension) | rest]]_subtree ->nildiff --git a/lib/plausible/stats/filters/legacy_dashboard_filter_parser.ex b/lib/plausible/stats/filters/legacy_dashboard_filter_parser.exindex 8e3ef1ccd2..a0a6cf0401 100644--- a/lib/plausible/stats/filters/legacy_dashboard_filter_parser.ex+++ b/lib/plausible/stats/filters/legacy_dashboard_filter_parser.ex@@ -37,7 +37,6 @@ defmodule Plausible.Stats.Filters.LegacyDashboardFilterParser dois_negated && is_wildcard && is_list ->[:matches_wildcard_not, key, val]- # TODOis_negated && is_contains && is_list ->[:matches_wildcard_not, key, Enum.map(val, &"**#{&1}**")]diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.exindex b7410964b8..684d6e1310 100644--- a/lib/plausible/stats/filters/query_parser.ex+++ b/lib/plausible/stats/filters/query_parser.ex@@ -133,31 +133,35 @@ defmodule Plausible.Stats.Filters.QueryParser do:matches_wildcard_not,:contains,:contains_not- ],- do: parse_clauses_list(filter)+ ] do+ with {:ok, clauses} <- parse_clauses_list(filter),+ {:ok, modifiers} <- parse_filter_modifiers(Enum.at(filter, 3)) do+ {:ok, [clauses | modifiers]}+ end+ enddefp parse_filter_rest(operator, _filter)when operator in [:not, :and, :or],do: {:ok, []}- defp parse_clauses_list([operation, filter_key, list] = filter) when is_list(list) do+ defp parse_clauses_list([operator, filter_key, list | _rest] = filter) when is_list(list) doall_strings? = Enum.all?(list, &is_binary/1)all_integers? = Enum.all?(list, &is_integer/1)case {filter_key, all_strings?} do{"visit:city", false} when all_integers? ->- {:ok, [list]}+ {:ok, list}- {"visit:country", true} when operation in ["is", "is_not"] ->+ {"visit:country", true} when operator in ["is", "is_not"] ->if Enum.all?(list, &(String.length(&1) == 2)) do- {:ok, [list]}+ {:ok, list}else{:error,"Invalid visit:country filter, visit:country needs to be a valid 2-letter country code."}end{_, true} ->- {:ok, [list]}+ {:ok, list}_ ->{:error, "Invalid filter '#{i(filter)}'."}@@ -166,6 +170,14 @@ defmodule Plausible.Stats.Filters.QueryParser dodefp parse_clauses_list(filter), do: {:error, "Invalid filter '#{i(filter)}'"}+ defp parse_filter_modifiers(modifiers) when is_map(modifiers) do+ {:ok, [atomize_keys(modifiers)]}+ end++ defp parse_filter_modifiers(nil) do+ {:ok, []}+ end+defp parse_date(_site, date_string, _date) when is_binary(date_string) docase Date.from_iso8601(date_string) do{:ok, date} -> {:ok, date}diff --git a/lib/plausible/stats/filters/stats_api_filter_parser.ex b/lib/plausible/stats/filters/stats_api_filter_parser.exindex f7168a6741..ef3b03cdaa 100644--- a/lib/plausible/stats/filters/stats_api_filter_parser.ex+++ b/lib/plausible/stats/filters/stats_api_filter_parser.ex@@ -40,7 +40,7 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser doendend- defp reject_invalid_country_codes([_op, "visit:country", code_or_codes] = filter) do+ defp reject_invalid_country_codes([_op, "visit:country", code_or_codes | _rest] = filter) docode_or_codes|> List.wrap()|> Enum.reduce_while(filter, fndiff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.exindex 831314e6dc..2328a4497f 100644--- a/lib/plausible/stats/imported/base.ex+++ b/lib/plausible/stats/imported/base.ex@@ -117,8 +117,8 @@ defmodule Plausible.Stats.Imported.Base dohas_required_name_filter? =query.filters|> Enum.flat_map(fn- [:is, "event:name", names] -> names- [:is, "event:goal", names] -> names+ [:is, "event:name", names | _rest] -> names+ [:is, "event:goal", names | _rest] -> names_ -> []end)|> Enum.any?(&(&1 in special_goals_for(property)))@@ -144,7 +144,7 @@ defmodule Plausible.Stats.Imported.Base dodefp do_decide_tables(%Query{dimensions: ["event:goal"]} = query) dofilter_dimensions = dimensions_used_in_filters(query.filters)- filter_goals = get_filter_goals(query)+ filter_goals = query.preloaded_goalsany_event_goals? = Enum.any?(filter_goals, fn goal -> Plausible.Goal.type(goal) == :event end)@@ -177,8 +177,7 @@ defmodule Plausible.Stats.Imported.Base do|> Enum.map(&@property_to_table_mappings[&1])filter_goal_table_candidates =- query- |> get_filter_goals()+ query.preloaded_goals|> Enum.map(&Plausible.Goal.type/1)|> Enum.map(fn:event -> "imported_custom_events"@@ -193,17 +192,6 @@ defmodule Plausible.Stats.Imported.Base doendend- defp get_filter_goals(query) do- query.filters- |> Enum.filter(fn [_, dimension | _rest] -> dimension == "event:goal" end)- |> Enum.flat_map(fn [operation, _dimension, clauses] ->- Enum.flat_map(clauses, fn clause ->- query.preloaded_goals- |> Plausible.Goals.Filters.filter_preloaded(operation, clause)- end)- end)- end-def special_goals_for("event:props:url"), do: Imported.goals_with_url()def special_goals_for("event:props:path"), do: Imported.goals_with_path()enddiff --git a/lib/plausible/stats/imported/sql/where_builder.ex b/lib/plausible/stats/imported/sql/where_builder.exindex 638fdf270c..93d8476210 100644--- a/lib/plausible/stats/imported/sql/where_builder.ex+++ b/lib/plausible/stats/imported/sql/where_builder.ex@@ -49,7 +49,7 @@ defmodule Plausible.Stats.Imported.SQL.WhereBuilder do|> Enum.reduce(fn condition, acc -> dynamic([], ^acc or ^condition) end)end- defp add_filter(query, [_operation, dimension, _clauses] = filter) do+ defp add_filter(query, [_operation, dimension, _clauses | _rest] = filter) dodb_field = Plausible.Stats.Filters.without_prefix(dimension)if db_field == :goal dodiff --git a/lib/plausible/stats/sql/special_metrics.ex b/lib/plausible/stats/sql/special_metrics.exindex 1bf77d50b6..e6b3aeba90 100644--- a/lib/plausible/stats/sql/special_metrics.ex+++ b/lib/plausible/stats/sql/special_metrics.ex@@ -56,7 +56,8 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do|> remove_filters_ignored_in_totals_query()|> Query.set(dimensions: [],- include_imported: query.include_imported+ include_imported: query.include_imported,+ preloaded_goals: [])q@@ -98,7 +99,8 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do|> Query.set(metrics: [:visitors],order_by: [],- include_imported: query.include_imported+ include_imported: query.include_imported,+ preloaded_goals: [])from(e in subquery(q),diff --git a/lib/plausible/stats/sql/where_builder.ex b/lib/plausible/stats/sql/where_builder.exindex 2463a889e2..443983521a 100644--- a/lib/plausible/stats/sql/where_builder.ex+++ b/lib/plausible/stats/sql/where_builder.ex@@ -93,11 +93,11 @@ defmodule Plausible.Stats.SQL.WhereBuilder do|> Enum.reduce(fn condition, acc -> dynamic([], ^acc or ^condition) end)end- defp add_filter(:events, _query, [:is, "event:name", list]) do- dynamic([e], e.name in ^list)+ defp add_filter(:events, _query, [:is, "event:name" | _rest] = filter) do+ in_clause(col_value(:name), filter)end- defp add_filter(:events, query, [_, "event:goal", _] = filter) do+ defp add_filter(:events, query, [_, "event:goal" | _rest] = filter) doPlausible.Goals.Filters.add_filter(query, filter)end@@ -154,43 +154,49 @@ defmodule Plausible.Stats.SQL.WhereBuilder dofalseend- defp filter_custom_prop(prop_name, column_name, [:is, _, clauses]) do+ defp filter_custom_prop(prop_name, column_name, [:is, _, clauses | _rest] = filter) donone_value_included = Enum.member?(clauses, "(none)")+ prop_value_expr = custom_prop_value(column_name, prop_name)dynamic([t],- (has_key(t, column_name, ^prop_name) and get_by_key(t, column_name, ^prop_name) in ^clauses) or+ (has_key(t, column_name, ^prop_name) and ^in_clause(prop_value_expr, filter)) or(^none_value_included and not has_key(t, column_name, ^prop_name)))end- defp filter_custom_prop(prop_name, column_name, [:is_not, _, clauses]) do+ defp filter_custom_prop(prop_name, column_name, [:is_not, _, clauses | _rest] = filter) donone_value_included = Enum.member?(clauses, "(none)")+ prop_value_expr = custom_prop_value(column_name, prop_name)dynamic([t],(has_key(t, column_name, ^prop_name) and- get_by_key(t, column_name, ^prop_name) not in ^clauses) or+ not (^in_clause(prop_value_expr, filter))) or(^none_value_included andhas_key(t, column_name, ^prop_name) and- get_by_key(t, column_name, ^prop_name) not in ^clauses) or+ not (^in_clause(prop_value_expr, filter))) or(not (^none_value_included) and not has_key(t, column_name, ^prop_name)))end- defp filter_custom_prop(prop_name, column_name, [:matches_wildcard, dimension, clauses]) do+ defp filter_custom_prop(prop_name, column_name, [:matches_wildcard, dimension, clauses | rest]) doregexes = Enum.map(clauses, &page_regex/1)- filter_custom_prop(prop_name, column_name, [:matches, dimension, regexes])+ filter_custom_prop(prop_name, column_name, [:matches, dimension, regexes | rest])end- defp filter_custom_prop(prop_name, column_name, [:matches_wildcard_not, dimension, clauses]) do+ defp filter_custom_prop(prop_name, column_name, [+ :matches_wildcard_not,+ dimension,+ clauses | rest+ ]) doregexes = Enum.map(clauses, &page_regex/1)- filter_custom_prop(prop_name, column_name, [:matches_not, dimension, regexes])+ filter_custom_prop(prop_name, column_name, [:matches_not, dimension, regexes | rest])end- defp filter_custom_prop(prop_name, column_name, [:matches, _dimension, clauses]) do+ defp filter_custom_prop(prop_name, column_name, [:matches, _dimension, clauses | _rest]) dodynamic([t],has_key(t, column_name, ^prop_name) and@@ -202,7 +208,7 @@ defmodule Plausible.Stats.SQL.WhereBuilder do)end- defp filter_custom_prop(prop_name, column_name, [:matches_not, _dimension, clauses]) do+ defp filter_custom_prop(prop_name, column_name, [:matches_not, _dimension, clauses | _rest]) dodynamic([t],has_key(t, column_name, ^prop_name) and@@ -214,31 +220,23 @@ defmodule Plausible.Stats.SQL.WhereBuilder do)end- defp filter_custom_prop(prop_name, column_name, [:contains, _dimension, clauses]) do+ defp filter_custom_prop(prop_name, column_name, [:contains | _rest] = filter) dodynamic([t],has_key(t, column_name, ^prop_name) and- fragment(- "multiSearchAny(?, ?)",- get_by_key(t, column_name, ^prop_name),- ^clauses- )+ ^contains_clause(custom_prop_value(column_name, prop_name), filter))end- defp filter_custom_prop(prop_name, column_name, [:contains_not, _dimension, clauses]) do+ defp filter_custom_prop(prop_name, column_name, [:contains_not | _] = filter) dodynamic([t],has_key(t, column_name, ^prop_name) and- fragment(- "not(multiSearchAny(?, ?))",- get_by_key(t, column_name, ^prop_name),- ^clauses- )+ not (^contains_clause(custom_prop_value(column_name, prop_name), filter)))end- defp filter_field(db_field, [:matches_wildcard, _dimension, glob_exprs]) do+ defp filter_field(db_field, [:matches_wildcard, _dimension, glob_exprs | _rest]) dopage_regexes = Enum.map(glob_exprs, &page_regex/1)dynamic(@@ -247,36 +245,36 @@ defmodule Plausible.Stats.SQL.WhereBuilder do)end- defp filter_field(db_field, [:matches_wildcard_not, dimension, clauses]) do- dynamic([], not (^filter_field(db_field, [:matches_wildcard, dimension, clauses])))+ defp filter_field(db_field, [:matches_wildcard_not | rest]) do+ dynamic([], not (^filter_field(db_field, [:matches_wildcard | rest])))end- defp filter_field(db_field, [:contains, _dimension, values]) do- dynamic([x], fragment("multiSearchAny(?, ?)", type(field(x, ^db_field), :string), ^values))+ defp filter_field(db_field, [:contains | _rest] = filter) do+ contains_clause(col_value_string(db_field), filter)end- defp filter_field(db_field, [:contains_not, dimension, clauses]) do- dynamic([], not (^filter_field(db_field, [:contains, dimension, clauses])))+ defp filter_field(db_field, [:contains_not | rest]) do+ dynamic([], not (^filter_field(db_field, [:contains | rest])))end- defp filter_field(db_field, [:matches, _dimension, clauses]) do+ defp filter_field(db_field, [:matches, _dimension, clauses | _rest]) dodynamic([x],fragment("multiMatchAny(?, ?)", type(field(x, ^db_field), :string), ^clauses))end- defp filter_field(db_field, [:matches_not, dimension, clauses]) do- dynamic([], not (^filter_field(db_field, [:matches, dimension, clauses])))+ defp filter_field(db_field, [:matches_not | rest]) do+ dynamic([], not (^filter_field(db_field, [:matches | rest])))end- defp filter_field(db_field, [:is, _dimension, clauses]) do- list = Enum.map(clauses, &db_field_val(db_field, &1))- dynamic([x], field(x, ^db_field) in ^list)+ defp filter_field(db_field, [:is, _dimension, clauses | _rest] = filter) do+ list = clauses |> Enum.map(&db_field_val(db_field, &1))+ in_clause(col_value(db_field), filter, list)end- defp filter_field(db_field, [:is_not, dimension, clauses]) do- dynamic([], not (^filter_field(db_field, [:is, dimension, clauses])))+ defp filter_field(db_field, [:is_not | rest]) do+ dynamic([], not (^filter_field(db_field, [:is | rest])))end@no_ref "Direct / None"@@ -294,4 +292,45 @@ defmodule Plausible.Stats.SQL.WhereBuilder dodefp db_field_val(:utm_term, @no_ref), do: ""defp db_field_val(_, @not_set), do: ""defp db_field_val(_, val), do: val++ defp col_value(column_name) do+ dynamic([t], field(t, ^column_name))+ end++ # Needed for string functions to work properly+ defp col_value_string(column_name) do+ dynamic([t], type(field(t, ^column_name), :string))+ end++ defp custom_prop_value(column_name, prop_name) do+ dynamic([t], get_by_key(t, column_name, ^prop_name))+ end++ defp in_clause(value_expression, [_, _, clauses | _] = filter, values \\ nil) do+ values = values || clauses++ if case_sensitive?(filter) do+ dynamic([t], ^value_expression in ^values)+ else+ values = values |> Enum.map(&String.downcase/1)+ dynamic([t], fragment("lower(?)", ^value_expression) in ^values)+ end+ end++ defp contains_clause(value_expression, [_, _, clauses | _] = filter) do+ if case_sensitive?(filter) do+ dynamic(+ [x],+ fragment("multiSearchAny(?, ?)", ^value_expression, ^clauses)+ )+ else+ dynamic(+ [x],+ fragment("multiSearchAnyCaseInsensitive(?, ?)", ^value_expression, ^clauses)+ )+ end+ end++ defp case_sensitive?([_, _, _, %{case_sensitive: false}]), do: false+ defp case_sensitive?(_), do: trueenddiff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.exindex 0adebbd506..8e121b8175 100644--- a/lib/plausible_web/controllers/api/external_stats_controller.ex+++ b/lib/plausible_web/controllers/api/external_stats_controller.ex@@ -351,7 +351,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController doend)end- defp validate_filter(site, [_type, "event:goal", goal_filter]) do+ defp validate_filter(site, [_type, "event:goal", goal_filter | _rest]) doconfigured_goals =site|> Plausible.Goals.for_site()diff --git a/priv/json-schemas/query-api-schema.json b/priv/json-schemas/query-api-schema.jsonindex e715c59571..593544d1ec 100644--- a/priv/json-schemas/query-api-schema.json+++ b/priv/json-schemas/query-api-schema.json@@ -341,17 +341,22 @@"enum": ["matches_wildcard", "matches_wildcard_not"],"description": "filter operation"},+ "filter_operation_regex": {+ "type": "string",+ "enum": ["matches", "matches_not"],+ "description": "filter operation"+ },"filter_operation_without_goals": {"type": "string",- "enum": ["is_not", "contains_not", "matches", "matches_not"],+ "enum": ["is_not", "contains_not"],"description": "filter operation"},- "filter_operation_with_goals": {+ "filter_operation_contains": {"type": "string","enum": ["is", "contains"],"description": "filter operation"},- "filter_without_goals": {+ "filter_with_pattern": {"type": "array","additionalItems": false,"minItems": 3,@@ -359,7 +364,7 @@"items": [{"oneOf": [- { "$ref": "#/definitions/filter_operation_without_goals" },+ { "$ref": "#/definitions/filter_operation_regex" },{"$ref": "#/definitions/filter_operation_wildcard","$comment": "only :internal"@@ -375,14 +380,40 @@{ "$ref": "#/definitions/clauses" }]},+ "filter_without_goals": {+ "type": "array",+ "additionalItems": false,+ "minItems": 3,+ "maxItems": 4,+ "items": [+ { "$ref": "#/definitions/filter_operation_without_goals" },+ {+ "oneOf": [+ { "$ref": "#/definitions/simple_filter_dimensions" },+ { "$ref": "#/definitions/custom_property_filter_dimensions" }+ ]+ },+ { "$ref": "#/definitions/clauses" },+ {+ "type": "object",+ "additionalProperties": false,+ "properties": {+ "case_sensitive": {+ "type": "boolean",+ "default": true+ }+ }+ }+ ]+ },"filter_with_goals": {"type": "array","additionalItems": false,"minItems": 3,- "maxItems": 3,+ "maxItems": 4,"items": [{- "$ref": "#/definitions/filter_operation_with_goals"+ "$ref": "#/definitions/filter_operation_contains"},{"oneOf": [@@ -393,13 +424,24 @@},{"$ref": "#/definitions/clauses"+ },+ {+ "type": "object",+ "additionalProperties": false,+ "properties": {+ "case_sensitive": {+ "type": "boolean",+ "default": true+ }+ }}]},"filter_entry": {"oneOf": [{ "$ref": "#/definitions/filter_without_goals" },- { "$ref": "#/definitions/filter_with_goals" }+ { "$ref": "#/definitions/filter_with_goals" },+ { "$ref": "#/definitions/filter_with_pattern" }]},"filter_tree": {diff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exsindex ef990f5ce4..f8c2f536a2 100644--- a/test/plausible/stats/query_parser_test.exs+++ b/test/plausible/stats/query_parser_test.exs@@ -598,6 +598,156 @@ defmodule Plausible.Stats.Filters.QueryParserTest do"Invalid filters. Dimension `event:hostname` can only be filtered at the top level.")end++ for operation <- [:is, :contains, :is_not, :contains_not] do+ test "#{operation} allows case_sensitive modifier", %{site: site} do+ %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "filters" => [+ [+ Atom.to_string(unquote(operation)),+ "event:page",+ ["/foo"],+ %{"case_sensitive" => false}+ ],+ [+ Atom.to_string(unquote(operation)),+ "event:name",+ ["/foo"],+ %{"case_sensitive" => true}+ ]+ ]+ }+ |> check_success(site, %{+ metrics: [:visitors],+ utc_time_range: @date_range_day,+ filters: [+ [unquote(operation), "event:page", ["/foo"], %{case_sensitive: false}],+ [unquote(operation), "event:name", ["/foo"], %{case_sensitive: true}]+ ],+ dimensions: [],+ order_by: nil,+ timezone: site.timezone,+ include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ pagination: %{limit: 10_000, offset: 0}+ })+ end+ end++ for operation <- [:matches, :matches_not, :matches_wildcard, :matches_wildcard_not] do+ test "case_sensitive modifier is not valid for #{operation}", %{site: site} do+ %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "filters" => [+ [+ Atom.to_string(unquote(operation)),+ "event:hostname",+ ["a.plausible.io"],+ %{"case_sensitive" => false}+ ]+ ]+ }+ |> check_error(+ site,+ "#/filters/0: Invalid filter [\"#{unquote(operation)}\", \"event:hostname\", [\"a.plausible.io\"], %{\"case_sensitive\" => false}]",+ :internal+ )+ end+ end+ end++ describe "preloading goals" do+ setup %{site: site} do+ insert(:goal, %{site: site, event_name: "Signup"})+ insert(:goal, %{site: site, event_name: "Purchase"})+ insert(:goal, %{site: site, event_name: "Contact"})++ :ok+ end++ test "with exact match", %{site: site} do+ %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "filters" => [["is", "event:goal", ["Signup", "Purchase"]]]+ }+ |> check_success(site, %{+ metrics: [:visitors],+ utc_time_range: @date_range_day,+ filters: [[:is, "event:goal", ["Signup", "Purchase"]]],+ dimensions: [],+ order_by: nil,+ timezone: site.timezone,+ include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ pagination: %{limit: 10_000, offset: 0}+ })+ |> check_goals(preloaded_goals: ["Purchase", "Signup"], revenue_currencies: %{})+ end++ test "with case insensitive match", %{site: site} do+ %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "filters" => [["is", "event:goal", ["signup", "purchase"], %{"case_sensitive" => false}]]+ }+ |> check_success(site, %{+ metrics: [:visitors],+ utc_time_range: @date_range_day,+ filters: [[:is, "event:goal", ["signup", "purchase"], %{case_sensitive: false}]],+ dimensions: [],+ order_by: nil,+ timezone: site.timezone,+ include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ pagination: %{limit: 10_000, offset: 0}+ })+ |> check_goals(preloaded_goals: ["Purchase", "Signup"], revenue_currencies: %{})+ end++ test "with contains match", %{site: site} do+ %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "filters" => [["contains", "event:goal", ["Sign", "pur"]]]+ }+ |> check_success(site, %{+ metrics: [:visitors],+ utc_time_range: @date_range_day,+ filters: [[:contains, "event:goal", ["Sign", "pur"]]],+ dimensions: [],+ order_by: nil,+ timezone: site.timezone,+ include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ pagination: %{limit: 10_000, offset: 0}+ })+ |> check_goals(preloaded_goals: ["Signup"], revenue_currencies: %{})+ end++ test "with case insensitive contains match", %{site: site} do+ %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "filters" => [["contains", "event:goal", ["sign", "CONT"], %{"case_sensitive" => false}]]+ }+ |> check_success(site, %{+ metrics: [:visitors],+ utc_time_range: @date_range_day,+ filters: [[:contains, "event:goal", ["sign", "CONT"], %{case_sensitive: false}]],+ dimensions: [],+ order_by: nil,+ timezone: site.timezone,+ include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ pagination: %{limit: 10_000, offset: 0}+ })+ |> check_goals(preloaded_goals: ["Contact", "Signup"], revenue_currencies: %{})+ endenddescribe "include validation" dodiff --git a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exsindex 3c7c1a41c3..cd887fb741 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs@@ -34,6 +34,301 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryImportedTest doassert json_response(conn2, 200)["meta"]["imports_included"]refute json_response(conn2, 200)["meta"]["imports_warning"]end++ test "filters correctly with 'is' operator", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:pageview, pathname: "/blog", timestamp: ~N[2023-01-01 00:00:00]),+ build(:pageview, pathname: "/blog", timestamp: ~N[2023-01-01 00:00:00]),+ build(:pageview, pathname: "/blog/post/1", timestamp: ~N[2023-01-01 00:00:00]),+ build(:pageview, pathname: "/about", timestamp: ~N[2023-01-01 00:00:00]),+ build(:imported_pages,+ page: "/blog",+ pageviews: 5,+ visitors: 3,+ date: ~D[2023-01-01]+ ),+ build(:imported_pages,+ page: "/blog/post/1",+ pageviews: 2,+ visitors: 2,+ date: ~D[2023-01-01]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors", "pageviews"],+ "filters" => [+ ["is", "event:page", ["/blog"]]+ ],+ "include" => %{"imports" => true}+ })++ assert json_response(conn, 200)["results"] == [+ %{"metrics" => [5, 7], "dimensions" => []}+ ]+ end++ test "filters correctly with 'is' operator (case insensitive)", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:pageview, pathname: "/BLOG", timestamp: ~N[2023-01-01 00:00:00]),+ build(:pageview, pathname: "/blog", timestamp: ~N[2023-01-01 00:00:00]),+ build(:pageview, pathname: "/blog/post/1", timestamp: ~N[2023-01-01 00:00:00]),+ build(:pageview, pathname: "/about", timestamp: ~N[2023-01-01 00:00:00]),+ build(:imported_pages,+ page: "/BLOG",+ pageviews: 5,+ visitors: 3,+ date: ~D[2023-01-01]+ ),+ build(:imported_pages,+ page: "/blog/post/1",+ pageviews: 2,+ visitors: 2,+ date: ~D[2023-01-01]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors", "pageviews"],+ "filters" => [+ ["is", "event:page", ["/blOG"], %{"case_sensitive" => false}]+ ],+ "include" => %{"imports" => true}+ })++ assert json_response(conn, 200)["results"] == [+ %{"metrics" => [5, 7], "dimensions" => []}+ ]+ end++ test "filters correctly with 'contains' operator", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:pageview, pathname: "/blog", timestamp: ~N[2023-01-01 00:00:00]),+ build(:pageview, pathname: "/blog/post/1", timestamp: ~N[2023-01-01 00:00:00]),+ build(:pageview, pathname: "/blog/post/2", timestamp: ~N[2023-01-01 00:00:00]),+ build(:pageview, pathname: "/about", timestamp: ~N[2023-01-01 00:00:00]),+ build(:imported_pages,+ page: "/blog",+ pageviews: 5,+ visitors: 3,+ date: ~D[2023-01-01]+ ),+ build(:imported_pages,+ page: "/blog/post/1",+ pageviews: 2,+ visitors: 2,+ date: ~D[2023-01-01]+ ),+ build(:imported_pages,+ page: "/blog/POST/2",+ pageviews: 3,+ visitors: 1,+ date: ~D[2023-01-01]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors", "pageviews"],+ "filters" => [+ ["contains", "event:page", ["blog/post"]]+ ],+ "include" => %{"imports" => true}+ })++ assert json_response(conn, 200)["results"] == [+ %{"metrics" => [4, 4], "dimensions" => []}+ ]+ end++ test "filters correctly with 'contains' operator (case insensitive)", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:pageview, pathname: "/BLOG/post/1", timestamp: ~N[2023-01-01 00:00:00]),+ build(:pageview, pathname: "/blog/POST/2", timestamp: ~N[2023-01-01 00:00:00]),+ build(:pageview, pathname: "/about", timestamp: ~N[2023-01-01 00:00:00]),+ build(:imported_pages,+ page: "/BLOG/POST/1",+ pageviews: 5,+ visitors: 3,+ date: ~D[2023-01-01]+ ),+ build(:imported_pages,+ page: "/blog/post/2",+ pageviews: 2,+ visitors: 2,+ date: ~D[2023-01-01]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors", "pageviews"],+ "filters" => [+ ["contains", "event:page", ["blog/POST"], %{"case_sensitive" => false}]+ ],+ "include" => %{"imports" => true}+ })++ assert json_response(conn, 200)["results"] == [+ %{"metrics" => [7, 9], "dimensions" => []}+ ]+ end++ test "aggregates custom event goals with 'is' and 'contains' operators", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ insert(:goal, event_name: "Purchase", site: site)++ populate_stats(site, site_import.id, [+ build(:event,+ name: "Purchase",+ timestamp: ~N[2023-01-01 00:00:00]+ ),+ build(:event,+ name: "Purchase",+ timestamp: ~N[2023-01-01 00:00:00]+ ),+ build(:event,+ name: "Signup",+ timestamp: ~N[2023-01-01 00:00:00]+ ),+ build(:imported_custom_events,+ name: "Purchase",+ visitors: 3,+ events: 5,+ date: ~D[2023-01-01]+ ),+ build(:imported_custom_events,+ name: "Signup",+ visitors: 2,+ events: 2,+ date: ~D[2023-01-01]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors", "events"],+ "filters" => [+ ["is", "event:goal", ["Purchase"]]+ ],+ "include" => %{"imports" => true}+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => [], "metrics" => [5, 7]}+ ]++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors", "events"],+ "filters" => [+ ["contains", "event:goal", ["Purch", "sign"]]+ ],+ "include" => %{"imports" => true}+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => [], "metrics" => [5, 7]}+ ]+ end++ test "aggregates custom event goals with 'is' and 'contains' operators (case insensitive)", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ insert(:goal, event_name: "Purchase", site: site)++ populate_stats(site, site_import.id, [+ build(:event,+ name: "Purchase",+ timestamp: ~N[2023-01-01 00:00:00]+ ),+ build(:event,+ name: "Purchase",+ timestamp: ~N[2023-01-01 00:00:00]+ ),+ build(:event,+ name: "Signup",+ timestamp: ~N[2023-01-01 00:00:00]+ ),+ build(:imported_custom_events,+ name: "Purchase",+ visitors: 3,+ events: 5,+ date: ~D[2023-01-01]+ ),+ build(:imported_custom_events,+ name: "Signup",+ visitors: 2,+ events: 2,+ date: ~D[2023-01-01]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors", "events"],+ "filters" => [+ ["is", "event:goal", ["purchase"], %{"case_sensitive" => false}]+ ],+ "include" => %{"imports" => true}+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => [], "metrics" => [5, 7]}+ ]++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors", "events"],+ "filters" => [+ ["contains", "event:goal", ["PURCH"], %{"case_sensitive" => false}]+ ],+ "include" => %{"imports" => true}+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => [], "metrics" => [5, 7]}+ ]+ endendtest "breaks down all metrics by visit:referrer with imported data", %{conn: conn, site: site} dodiff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exsindex 8966046a19..f22b2c52a8 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs@@ -272,6 +272,62 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do]end+ test "can filter by utm_medium case insensitively", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ utm_medium: "Social",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ utm_medium: "SOCIAL",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"],+ "filters" => [["is", "visit:utm_medium", ["sociaL"], %{"case_sensitive" => false}]]+ })++ assert json_response(conn, 200)["results"] == [+ %{"metrics" => [2, 1, 0, 1500], "dimensions" => []}+ ]+ end++ test "can filter by is_not utm_medium case insensitively", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ utm_medium: "Social",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ utm_medium: "SOCIAL",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["pageviews"],+ "filters" => [["is_not", "visit:utm_medium", ["sociaL"], %{"case_sensitive" => false}]]+ })++ assert json_response(conn, 200)["results"] == [+ %{"metrics" => [1], "dimensions" => []}+ ]+ end+test "can filter by utm_source", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview,@@ -622,6 +678,43 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doassert json_response(conn, 200)["results"] == [%{"metrics" => [2, 3], "dimensions" => []}]end+ test "filtering by a custom event goal (case insensitive)", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event,+ name: "Signup",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Signup",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:event,+ name: "Signup",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:event,+ name: "NotConfigured",+ timestamp: ~N[2021-01-01 00:25:00]+ )+ ])++ insert(:goal, %{site: site, event_name: "Signup"})++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors", "events"],+ "filters" => [+ ["is", "event:goal", ["signup"], %{"case_sensitive" => false}]+ ]+ })++ assert json_response(conn, 200)["results"] == [%{"metrics" => [2, 3], "dimensions" => []}]+ end+test "filtering by a revenue goal", %{conn: conn, site: site} dopopulate_stats(site, [build(:event,@@ -820,6 +913,46 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doassert json_response(conn, 200)["results"] == [%{"metrics" => [2], "dimensions" => []}]end+ test "contains page filter case insensitive", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, pathname: "/en/page1"),+ build(:pageview, pathname: "/EN/page2"),+ build(:pageview, pathname: "/pl/page1")+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors"],+ "filters" => [+ ["contains", "event:page", ["/En/"], %{"case_sensitive" => false}]+ ]+ })++ assert json_response(conn, 200)["results"] == [%{"metrics" => [2], "dimensions" => []}]+ end++ test "contains_not page filter case insensitive", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, pathname: "/en/page1"),+ build(:pageview, pathname: "/EN/page2"),+ build(:pageview, pathname: "/pl/page1")+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors"],+ "filters" => [+ ["contains_not", "event:page", ["/En/"], %{"case_sensitive" => false}]+ ]+ })++ assert json_response(conn, 200)["results"] == [%{"metrics" => [1], "dimensions" => []}]+ end+test "contains_not page filter", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview, pathname: "/en/page1"),@@ -1010,6 +1143,88 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doassert json_response(conn, 200)["results"] == [%{"metrics" => [1], "dimensions" => []}]end+ test "`contains` operator with custom properties case insensitive", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview,+ "meta.key": ["name"],+ "meta.value": ["large-1"]+ ),+ build(:pageview,+ "meta.key": ["name"],+ "meta.value": ["Small-1"]+ ),+ build(:pageview,+ "meta.key": ["name"],+ "meta.value": ["mall-1"]+ ),+ build(:pageview,+ "meta.key": ["name"],+ "meta.value": ["SMALL-2"]+ ),+ build(:pageview,+ "meta.key": ["name"],+ "meta.value": ["small-2"]+ ),+ build(:pageview)+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors"],+ "filters" => [+ ["contains", "event:props:name", ["maLL"], %{"case_sensitive" => false}]+ ]+ })++ assert json_response(conn, 200)["results"] == [%{"metrics" => [4], "dimensions" => []}]+ end++ test "`contains_not` operator with custom properties case insensitive", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview,+ "meta.key": ["name"],+ "meta.value": ["large-1"]+ ),+ build(:pageview,+ "meta.key": ["name"],+ "meta.value": ["Small-1"]+ ),+ build(:pageview,+ "meta.key": ["name"],+ "meta.value": ["mall-1"]+ ),+ build(:pageview,+ "meta.key": ["name"],+ "meta.value": ["SMALL-2"]+ ),+ build(:pageview,+ "meta.key": ["name"],+ "meta.value": ["small-2"]+ ),+ build(:pageview)+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors"],+ "filters" => [+ ["contains_not", "event:props:name", ["maLL"], %{"case_sensitive" => false}]+ ]+ })++ assert json_response(conn, 200)["results"] == [%{"metrics" => [1], "dimensions" => []}]+ end+test "`matches` and `matches_not` operator with custom properties", %{conn: conn,site: site@@ -2588,6 +2803,39 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do]end+ test "goal contains filter for goal breakdown (case insensitive)", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event, name: "Onboarding conversion: Step 1"),+ build(:event, name: "Onboarding conversion: Step 1"),+ build(:event, name: "Onboarding conversion: Step 2"),+ build(:event, name: "Unrelated"),+ build(:pageview, pathname: "/conversion")+ ])++ insert(:goal, site: site, event_name: "Onboarding conversion: Step 1")+ insert(:goal, site: site, event_name: "Onboarding conversion: Step 2")+ insert(:goal, site: site, event_name: "Unrelated")+ insert(:goal, site: site, page_path: "/conversion")++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["event:goal"],+ "filters" => [+ ["contains", "event:goal", ["step"], %{"case_sensitive" => false}]+ ]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["Onboarding conversion: Step 1"], "metrics" => [2]},+ %{"dimensions" => ["Onboarding conversion: Step 2"], "metrics" => [1]}+ ]+ end+test "mixed multi-goal filter for breakdown by visit:country", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview, country_code: "EE", pathname: "/en/register"),@@ -2814,6 +3062,43 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do]end+ test "IN filter for event:name case insensitive", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event,+ name: "Signup",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Signup",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Login",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Irrelevant",+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "filters" => [+ ["is", "event:name", ["signup", "LOGIN"], %{"case_sensitive" => false}]+ ]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => [], "metrics" => [3]}+ ]+ end+test "IN filter for event:props:*", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview,@@ -2881,6 +3166,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do"meta.value": ["Safari", "target_value"],timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview,+ browser: "Chrome",+ "meta.key": ["browser", "prop"],+ "meta.value": ["Chrome", "target_value"],+ timestamp: ~N[2021-01-01 00:00:00]+ ),build(:pageview,browser: "Firefox","meta.key": ["browser", "prop"],@@ -2896,7 +3187,9 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do"date_range" => "all","dimensions" => ["visit:browser"],"filters" => [- ["is", "event:props:browser", ["Chrome", "Safari"]],+ ["is", "event:props:browser", ["CHROME", "sAFari"], %{"case_sensitive" => false}],+ # Negate a previously set filter+ ["is_not", "event:props:browser", ["Chrome"], %{"case_sensitive" => false}],["is", "event:props:prop", ["target_value"]]]})@@ -3721,4 +4014,32 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do]endend++ test "can filter by utm_medium case insensitively", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ utm_medium: "Social",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ utm_medium: "SOCIAL",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"],+ "filters" => [["is", "visit:utm_medium", ["social"], %{"case_sensitive" => false}]]+ })++ assert json_response(conn, 200)["results"] == [+ %{"metrics" => [2, 1, 0, 1500], "dimensions" => []}+ ]+ endend