Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

APIv2: Case insensitive search (#4863)

Karl-Aksel Puulmanna38eacfed54a
diff --git a/CHANGELOG.md b/CHANGELOG.md
index dc0f3056f8..00bb32b5c9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,9 +6,13 @@ All notable changes to this project will be documented in this file.
### Added
- Dashboard shows comparisons for all reports
- UTM Medium report and API shows (gclid) and (msclkid) for paid searches when no explicit utm medium present.
+- Support for `case_sensitive: false` modifiers in Stats API V2 filters for case-insensitive searches.
### Removed
### Changed
+
+- Details modal search inputs are now case-insensitive.
+
### Fixed
- Fix returning filter suggestions for multiple custom property values in the dashboard Filter modal
diff --git a/assets/js/dashboard/stats/modals/conversions.js b/assets/js/dashboard/stats/modals/conversions.js
index 3514bd852d..cc1a00d947 100644
--- a/assets/js/dashboard/stats/modals/conversions.js
+++ b/assets/js/dashboard/stats/modals/conversions.js
@@ -28,7 +28,7 @@ function ConversionsModal() {
}, [reportInfo.dimension])
const addSearchFilter = useCallback((query, searchString) => {
- return addFilter(query, ['contains', reportInfo.dimension, [searchString]])
+ return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }])
}, [reportInfo.dimension])
function chooseMetrics() {
diff --git a/assets/js/dashboard/stats/modals/devices/browser-versions-modal.js b/assets/js/dashboard/stats/modals/devices/browser-versions-modal.js
index 369b4dce9f..d0e11e299b 100644
--- a/assets/js/dashboard/stats/modals/devices/browser-versions-modal.js
+++ b/assets/js/dashboard/stats/modals/devices/browser-versions-modal.js
@@ -29,7 +29,7 @@ function BrowserVersionsModal() {
}, [reportInfo.dimension])
const addSearchFilter = useCallback((query, searchString) => {
- return addFilter(query, ['contains', reportInfo.dimension, [searchString]])
+ return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }])
}, [reportInfo.dimension])
const renderIcon = useCallback((listItem) => browserIconFor(listItem.browser), [])
diff --git a/assets/js/dashboard/stats/modals/devices/browsers-modal.js b/assets/js/dashboard/stats/modals/devices/browsers-modal.js
index 1160b8ebf5..9b600514bd 100644
--- a/assets/js/dashboard/stats/modals/devices/browsers-modal.js
+++ b/assets/js/dashboard/stats/modals/devices/browsers-modal.js
@@ -29,7 +29,7 @@ function BrowsersModal() {
}, [reportInfo.dimension])
const addSearchFilter = useCallback((query, searchString) => {
- return addFilter(query, ['contains', reportInfo.dimension, [searchString]])
+ return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }])
}, [reportInfo.dimension])
const renderIcon = useCallback((listItem) => browserIconFor(listItem.name), [])
diff --git a/assets/js/dashboard/stats/modals/devices/operating-system-versions-modal.js b/assets/js/dashboard/stats/modals/devices/operating-system-versions-modal.js
index 6c7d5c72cc..9c5622b48d 100644
--- a/assets/js/dashboard/stats/modals/devices/operating-system-versions-modal.js
+++ b/assets/js/dashboard/stats/modals/devices/operating-system-versions-modal.js
@@ -29,7 +29,7 @@ function OperatingSystemVersionsModal() {
}, [reportInfo.dimension])
const addSearchFilter = useCallback((query, searchString) => {
- return addFilter(query, ['contains', reportInfo.dimension, [searchString]])
+ return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }])
}, [reportInfo.dimension])
const renderIcon = useCallback((listItem) => osIconFor(listItem.os), [])
diff --git a/assets/js/dashboard/stats/modals/devices/operating-systems-modal.js b/assets/js/dashboard/stats/modals/devices/operating-systems-modal.js
index 2a92795648..9df8a6c7c7 100644
--- a/assets/js/dashboard/stats/modals/devices/operating-systems-modal.js
+++ b/assets/js/dashboard/stats/modals/devices/operating-systems-modal.js
@@ -29,7 +29,7 @@ function OperatingSystemsModal() {
}, [reportInfo.dimension])
const addSearchFilter = useCallback((query, searchString) => {
- return addFilter(query, ['contains', reportInfo.dimension, [searchString]])
+ return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }])
}, [reportInfo.dimension])
const renderIcon = useCallback((listItem) => osIconFor(listItem.name), [])
diff --git a/assets/js/dashboard/stats/modals/entry-pages.js b/assets/js/dashboard/stats/modals/entry-pages.js
index 6d5a72d693..79963046fb 100644
--- a/assets/js/dashboard/stats/modals/entry-pages.js
+++ b/assets/js/dashboard/stats/modals/entry-pages.js
@@ -29,7 +29,7 @@ function EntryPagesModal() {
}, [reportInfo.dimension])
const addSearchFilter = useCallback((query, searchString) => {
- return addFilter(query, ['contains', reportInfo.dimension, [searchString]])
+ return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }])
}, [reportInfo.dimension])
function chooseMetrics() {
diff --git a/assets/js/dashboard/stats/modals/exit-pages.js b/assets/js/dashboard/stats/modals/exit-pages.js
index 2be05f0168..413c890568 100644
--- a/assets/js/dashboard/stats/modals/exit-pages.js
+++ b/assets/js/dashboard/stats/modals/exit-pages.js
@@ -29,7 +29,7 @@ function ExitPagesModal() {
}, [reportInfo.dimension])
const addSearchFilter = useCallback((query, searchString) => {
- return addFilter(query, ['contains', reportInfo.dimension, [searchString]])
+ return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }])
}, [reportInfo.dimension])
function chooseMetrics() {
diff --git a/assets/js/dashboard/stats/modals/locations-modal.js b/assets/js/dashboard/stats/modals/locations-modal.js
index 615f6e9e3d..bfdd471b97 100644
--- a/assets/js/dashboard/stats/modals/locations-modal.js
+++ b/assets/js/dashboard/stats/modals/locations-modal.js
@@ -32,7 +32,7 @@ function LocationsModal({ currentView }) {
}, [reportInfo.dimension])
const addSearchFilter = useCallback((query, searchString) => {
- return addFilter(query, ['contains', `${reportInfo.dimension}_name`, [searchString]])
+ return addFilter(query, ['contains', `${reportInfo.dimension}_name`, [searchString], { case_sensitive: false }])
}, [reportInfo.dimension])
function chooseMetrics() {
diff --git a/assets/js/dashboard/stats/modals/pages.js b/assets/js/dashboard/stats/modals/pages.js
index 26bf287c6c..e9521f9774 100644
--- a/assets/js/dashboard/stats/modals/pages.js
+++ b/assets/js/dashboard/stats/modals/pages.js
@@ -29,7 +29,7 @@ function PagesModal() {
}, [reportInfo.dimension])
const addSearchFilter = useCallback((query, searchString) => {
- return addFilter(query, ['contains', reportInfo.dimension, [searchString]])
+ return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }])
}, [reportInfo.dimension])
function chooseMetrics() {
@@ -46,14 +46,14 @@ function PagesModal() {
metrics.createVisitors({renderLabel: (_query) => 'Current visitors', width: 'w-36'})
]
}
-
+
const defaultMetrics = [
metrics.createVisitors({renderLabel: (_query) => "Visitors" }),
metrics.createPageviews(),
metrics.createBounceRate(),
metrics.createTimeOnPage()
]
-
+
return site.flags.scroll_depth ? [...defaultMetrics, metrics.createScrollDepth()] : defaultMetrics
}
diff --git a/assets/js/dashboard/stats/modals/props.js b/assets/js/dashboard/stats/modals/props.js
index 0d691abe8d..a131e8a438 100644
--- a/assets/js/dashboard/stats/modals/props.js
+++ b/assets/js/dashboard/stats/modals/props.js
@@ -36,7 +36,7 @@ function PropsModal() {
}, [propKey])
const addSearchFilter = useCallback((query, searchString) => {
- return addFilter(query, ['contains', `${EVENT_PROPS_PREFIX}${propKey}`, [searchString]])
+ return addFilter(query, ['contains', `${EVENT_PROPS_PREFIX}${propKey}`, [searchString], { case_sensitive: false }])
}, [propKey])
function chooseMetrics() {
diff --git a/assets/js/dashboard/stats/modals/referrer-drilldown.js b/assets/js/dashboard/stats/modals/referrer-drilldown.js
index 6077bb737e..264e1d5454 100644
--- a/assets/js/dashboard/stats/modals/referrer-drilldown.js
+++ b/assets/js/dashboard/stats/modals/referrer-drilldown.js
@@ -33,7 +33,7 @@ function ReferrerDrilldownModal() {
}, [reportInfo.dimension])
const addSearchFilter = useCallback((query, searchString) => {
- return addFilter(query, ['contains', reportInfo.dimension, [searchString]])
+ return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }])
}, [reportInfo.dimension])
function chooseMetrics() {
diff --git a/assets/js/dashboard/stats/modals/sources.js b/assets/js/dashboard/stats/modals/sources.js
index d4be4b30b9..3fd0006c71 100644
--- a/assets/js/dashboard/stats/modals/sources.js
+++ b/assets/js/dashboard/stats/modals/sources.js
@@ -57,7 +57,7 @@ function SourcesModal({ currentView }) {
}, [reportInfo.dimension])
const addSearchFilter = useCallback((query, searchString) => {
- return addFilter(query, ['contains', reportInfo.dimension, [searchString]])
+ return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }])
}, [reportInfo.dimension])
function chooseMetrics() {
diff --git a/assets/js/dashboard/util/filters.js b/assets/js/dashboard/util/filters.js
index d8d13a70da..12e9a60c60 100644
--- a/assets/js/dashboard/util/filters.js
+++ b/assets/js/dashboard/util/filters.js
@@ -229,16 +229,18 @@ export function cleanLabels(filters, labels, mergedFilterKey, mergedLabels) {
const EVENT_FILTER_KEYS = new Set(['name', 'page', 'goal', 'hostname'])
export function serializeApiFilters(filters) {
- const apiFilters = filters.map(([operation, filterKey, clauses]) => {
- let apiFilterKey = `visit:${filterKey}`
- if (
- filterKey.startsWith(EVENT_PROPS_PREFIX) ||
- EVENT_FILTER_KEYS.has(filterKey)
- ) {
- apiFilterKey = `event:${filterKey}`
+ const apiFilters = filters.map(
+ ([operation, filterKey, clauses, ...modifiers]) => {
+ let apiFilterKey = `visit:${filterKey}`
+ if (
+ filterKey.startsWith(EVENT_PROPS_PREFIX) ||
+ EVENT_FILTER_KEYS.has(filterKey)
+ ) {
+ apiFilterKey = `event:${filterKey}`
+ }
+ return [operation, apiFilterKey, clauses, ...modifiers]
}
- return [operation, apiFilterKey, clauses]
- })
+ )
return JSON.stringify(apiFilters)
}
diff --git a/assets/js/types/query-api.d.ts b/assets/js/types/query-api.d.ts
index 06121a8166..2b77acf192 100644
--- a/assets/js/types/query-api.d.ts
+++ b/assets/js/types/query-api.d.ts
@@ -64,34 +64,57 @@ export type CustomPropertyFilterDimensions = string;
export type GoalDimension = "event:goal";
export type TimeDimensions = "time" | "time:month" | "time:week" | "time:day" | "time:hour";
export type FilterTree = FilterEntry | FilterAndOr | FilterNot;
-export type FilterEntry = FilterWithoutGoals | FilterWithGoals;
+export type FilterEntry = FilterWithoutGoals | FilterWithGoals | FilterWithPattern;
/**
* @minItems 3
- * @maxItems 3
+ * @maxItems 4
*/
-export type FilterWithoutGoals = [
- FilterOperationWithoutGoals | ("matches_wildcard" | "matches_wildcard_not"),
- SimpleFilterDimensions | CustomPropertyFilterDimensions,
- Clauses
-];
+export type FilterWithoutGoals =
+ | [FilterOperationWithoutGoals, SimpleFilterDimensions | CustomPropertyFilterDimensions, Clauses]
+ | [
+ FilterOperationWithoutGoals,
+ SimpleFilterDimensions | CustomPropertyFilterDimensions,
+ Clauses,
+ {
+ case_sensitive?: boolean;
+ }
+ ];
/**
* filter operation
*/
-export type FilterOperationWithoutGoals = "is_not" | "contains_not" | "matches" | "matches_not";
+export type FilterOperationWithoutGoals = "is_not" | "contains_not";
export type Clauses = (string | number)[];
+/**
+ * @minItems 3
+ * @maxItems 4
+ */
+export type FilterWithGoals =
+ | [FilterOperationContains, GoalDimension | SimpleFilterDimensions | CustomPropertyFilterDimensions, Clauses]
+ | [
+ FilterOperationContains,
+ GoalDimension | SimpleFilterDimensions | CustomPropertyFilterDimensions,
+ Clauses,
+ {
+ case_sensitive?: boolean;
+ }
+ ];
+/**
+ * filter operation
+ */
+export type FilterOperationContains = "is" | "contains";
/**
* @minItems 3
* @maxItems 3
*/
-export type FilterWithGoals = [
- FilterOperationWithGoals,
- GoalDimension | SimpleFilterDimensions | CustomPropertyFilterDimensions,
+export type FilterWithPattern = [
+ FilterOperationRegex | ("matches_wildcard" | "matches_wildcard_not"),
+ SimpleFilterDimensions | CustomPropertyFilterDimensions,
Clauses
];
/**
* filter operation
*/
-export type FilterOperationWithGoals = "is" | "contains";
+export type FilterOperationRegex = "matches" | "matches_not";
/**
* @minItems 2
* @maxItems 2
diff --git a/lib/plausible/goals/filters.ex b/lib/plausible/goals/filters.ex
index 142095e048..b9d7d5a749 100644
--- a/lib/plausible/goals/filters.ex
+++ b/lib/plausible/goals/filters.ex
@@ -20,14 +20,14 @@ defmodule Plausible.Goals.Filters do
* `imported?` - when `true`, builds conditions on the `page` db field rather than
`pathname`, and also skips the `e.name == "pageview"` check.
"""
- def add_filter(query, [operation, "event:goal", clauses], opts \\ [])
+ def add_filter(query, [operation, "event:goal", clauses | _] = filter, opts \\ [])
when operation in [:is, :contains] do
imported? = Keyword.get(opts, :imported?, false)
Enum.reduce(clauses, false, fn clause, dynamic_statement ->
condition =
query.preloaded_goals
- |> filter_preloaded(operation, clause)
+ |> filter_preloaded(filter, clause)
|> build_condition(imported?)
dynamic([e], ^condition or ^dynamic_statement)
@@ -38,32 +38,46 @@ defmodule Plausible.Goals.Filters do
goals = Plausible.Goals.for_site(site)
Enum.reduce(filters, goals, fn
- [operation, "event:goal", clauses], goals ->
- goals_matching_any_clause(goals, operation, clauses)
+ [_, "event:goal" | _] = filter, goals ->
+ goals_matching_any_clause(goals, filter)
_filter, goals ->
goals
end)
end
- def filter_preloaded(preloaded_goals, operation, clause) when operation in [:is, :contains] do
- Enum.filter(preloaded_goals, fn goal -> matches?(goal, operation, clause) end)
+ defp filter_preloaded(preloaded_goals, filter, clause) do
+ Enum.filter(preloaded_goals, fn goal -> matches?(goal, filter, clause) end)
end
- defp goals_matching_any_clause(goals, operation, clauses) do
+ defp goals_matching_any_clause(goals, [_, _, clauses | _] = filter) do
goals
|> Enum.filter(fn goal ->
- Enum.any?(clauses, fn clause -> matches?(goal, operation, clause) end)
+ Enum.any?(clauses, fn clause -> matches?(goal, filter, clause) end)
end)
end
- defp matches?(goal, operation, clause) do
+ defp matches?(goal, [operation | _rest] = filter, clause) do
+ goal_name =
+ goal
+ |> Plausible.Goal.display_name()
+ |> mod(filter)
+
+ clause = mod(clause, filter)
+
case operation do
:is ->
- Plausible.Goal.display_name(goal) == clause
+ goal_name == clause
:contains ->
- String.contains?(Plausible.Goal.display_name(goal), clause)
+ String.contains?(goal_name, clause)
+ end
+ end
+
+ defp mod(str, filter) do
+ case filter do
+ [_, _, _, %{case_sensitive: false}] -> String.downcase(str)
+ _ -> str
end
end
diff --git a/lib/plausible/google/search_console/filters.ex b/lib/plausible/google/search_console/filters.ex
index d010e013b5..1874e16403 100644
--- a/lib/plausible/google/search_console/filters.ex
+++ b/lib/plausible/google/search_console/filters.ex
@@ -24,14 +24,15 @@ defmodule Plausible.Google.SearchConsole.Filters do
transform_filter(property, [op, "visit:entry_page" | rest])
end
- defp transform_filter(property, [:is, "visit:entry_page", pages]) when is_list(pages) do
+ # :TODO: Should also work case-insensitive, if not, blacklist.
+ defp transform_filter(property, [:is, "visit:entry_page", pages | _]) when is_list(pages) do
expression =
Enum.map_join(pages, "|", fn page -> property_url(property, Regex.escape(page)) end)
%{dimension: "page", operator: "includingRegex", expression: expression}
end
- defp transform_filter(property, [:matches_wildcard, "visit:entry_page", pages])
+ defp transform_filter(property, [:matches_wildcard, "visit:entry_page", pages | _])
when is_list(pages) do
expression =
Enum.map_join(pages, "|", fn page -> page_regex(property_url(property, page)) end)
@@ -39,12 +40,12 @@ defmodule Plausible.Google.SearchConsole.Filters do
%{dimension: "page", operator: "includingRegex", expression: expression}
end
- defp transform_filter(_property, [:is, "visit:screen", devices]) when is_list(devices) do
+ defp transform_filter(_property, [:is, "visit:screen", devices | _]) when is_list(devices) do
expression = Enum.map_join(devices, "|", &search_console_device/1)
%{dimension: "device", operator: "includingRegex", expression: expression}
end
- defp transform_filter(_property, [:is, "visit:country", countries])
+ defp transform_filter(_property, [:is, "visit:country", countries | _])
when is_list(countries) do
expression = Enum.map_join(countries, "|", &search_console_country/1)
%{dimension: "country", operator: "includingRegex", expression: expression}
diff --git a/lib/plausible/stats/filters/filters.ex b/lib/plausible/stats/filters/filters.ex
index c68e2b3f86..0c16ccb3f2 100644
--- a/lib/plausible/stats/filters/filters.ex
+++ b/lib/plausible/stats/filters/filters.ex
@@ -121,8 +121,8 @@ defmodule Plausible.Stats.Filters do
def rename_dimensions_used_in_filter(filters, renames) do
transform_filters(filters, fn
- [operation, dimension, clauses] ->
- [[operation, Map.get(renames, dimension, dimension), clauses]]
+ [operation, dimension | rest] ->
+ [[operation, Map.get(renames, dimension, dimension) | rest]]
_subtree ->
nil
diff --git a/lib/plausible/stats/filters/legacy_dashboard_filter_parser.ex b/lib/plausible/stats/filters/legacy_dashboard_filter_parser.ex
index 8e3ef1ccd2..a0a6cf0401 100644
--- a/lib/plausible/stats/filters/legacy_dashboard_filter_parser.ex
+++ b/lib/plausible/stats/filters/legacy_dashboard_filter_parser.ex
@@ -37,7 +37,6 @@ defmodule Plausible.Stats.Filters.LegacyDashboardFilterParser do
is_negated && is_wildcard && is_list ->
[:matches_wildcard_not, key, val]
- # TODO
is_negated && is_contains && is_list ->
[:matches_wildcard_not, key, Enum.map(val, &"**#{&1}**")]
diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.ex
index b7410964b8..684d6e1310 100644
--- a/lib/plausible/stats/filters/query_parser.ex
+++ b/lib/plausible/stats/filters/query_parser.ex
@@ -133,31 +133,35 @@ defmodule Plausible.Stats.Filters.QueryParser do
:matches_wildcard_not,
:contains,
:contains_not
- ],
- do: parse_clauses_list(filter)
+ ] do
+ with {:ok, clauses} <- parse_clauses_list(filter),
+ {:ok, modifiers} <- parse_filter_modifiers(Enum.at(filter, 3)) do
+ {:ok, [clauses | modifiers]}
+ end
+ end
defp parse_filter_rest(operator, _filter)
when operator in [:not, :and, :or],
do: {:ok, []}
- defp parse_clauses_list([operation, filter_key, list] = filter) when is_list(list) do
+ defp parse_clauses_list([operator, filter_key, list | _rest] = filter) when is_list(list) do
all_strings? = Enum.all?(list, &is_binary/1)
all_integers? = Enum.all?(list, &is_integer/1)
case {filter_key, all_strings?} do
{"visit:city", false} when all_integers? ->
- {:ok, [list]}
+ {:ok, list}
- {"visit:country", true} when operation in ["is", "is_not"] ->
+ {"visit:country", true} when operator in ["is", "is_not"] ->
if Enum.all?(list, &(String.length(&1) == 2)) do
- {:ok, [list]}
+ {:ok, list}
else
{:error,
"Invalid visit:country filter, visit:country needs to be a valid 2-letter country code."}
end
{_, true} ->
- {:ok, [list]}
+ {:ok, list}
_ ->
{:error, "Invalid filter '#{i(filter)}'."}
@@ -166,6 +170,14 @@ defmodule Plausible.Stats.Filters.QueryParser do
defp parse_clauses_list(filter), do: {:error, "Invalid filter '#{i(filter)}'"}
+ defp parse_filter_modifiers(modifiers) when is_map(modifiers) do
+ {:ok, [atomize_keys(modifiers)]}
+ end
+
+ defp parse_filter_modifiers(nil) do
+ {:ok, []}
+ end
+
defp parse_date(_site, date_string, _date) when is_binary(date_string) do
case Date.from_iso8601(date_string) do
{:ok, date} -> {:ok, date}
diff --git a/lib/plausible/stats/filters/stats_api_filter_parser.ex b/lib/plausible/stats/filters/stats_api_filter_parser.ex
index f7168a6741..ef3b03cdaa 100644
--- a/lib/plausible/stats/filters/stats_api_filter_parser.ex
+++ b/lib/plausible/stats/filters/stats_api_filter_parser.ex
@@ -40,7 +40,7 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser do
end
end
- defp reject_invalid_country_codes([_op, "visit:country", code_or_codes] = filter) do
+ defp reject_invalid_country_codes([_op, "visit:country", code_or_codes | _rest] = filter) do
code_or_codes
|> List.wrap()
|> Enum.reduce_while(filter, fn
diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.ex
index 831314e6dc..2328a4497f 100644
--- a/lib/plausible/stats/imported/base.ex
+++ b/lib/plausible/stats/imported/base.ex
@@ -117,8 +117,8 @@ defmodule Plausible.Stats.Imported.Base do
has_required_name_filter? =
query.filters
|> Enum.flat_map(fn
- [:is, "event:name", names] -> names
- [:is, "event:goal", names] -> names
+ [:is, "event:name", names | _rest] -> names
+ [:is, "event:goal", names | _rest] -> names
_ -> []
end)
|> Enum.any?(&(&1 in special_goals_for(property)))
@@ -144,7 +144,7 @@ defmodule Plausible.Stats.Imported.Base do
defp do_decide_tables(%Query{dimensions: ["event:goal"]} = query) do
filter_dimensions = dimensions_used_in_filters(query.filters)
- filter_goals = get_filter_goals(query)
+ filter_goals = query.preloaded_goals
any_event_goals? = Enum.any?(filter_goals, fn goal -> Plausible.Goal.type(goal) == :event end)
@@ -177,8 +177,7 @@ defmodule Plausible.Stats.Imported.Base do
|> Enum.map(&@property_to_table_mappings[&1])
filter_goal_table_candidates =
- query
- |> get_filter_goals()
+ query.preloaded_goals
|> Enum.map(&Plausible.Goal.type/1)
|> Enum.map(fn
:event -> "imported_custom_events"
@@ -193,17 +192,6 @@ defmodule Plausible.Stats.Imported.Base do
end
end
- defp get_filter_goals(query) do
- query.filters
- |> Enum.filter(fn [_, dimension | _rest] -> dimension == "event:goal" end)
- |> Enum.flat_map(fn [operation, _dimension, clauses] ->
- Enum.flat_map(clauses, fn clause ->
- query.preloaded_goals
- |> Plausible.Goals.Filters.filter_preloaded(operation, clause)
- end)
- end)
- end
-
def special_goals_for("event:props:url"), do: Imported.goals_with_url()
def special_goals_for("event:props:path"), do: Imported.goals_with_path()
end
diff --git a/lib/plausible/stats/imported/sql/where_builder.ex b/lib/plausible/stats/imported/sql/where_builder.ex
index 638fdf270c..93d8476210 100644
--- a/lib/plausible/stats/imported/sql/where_builder.ex
+++ b/lib/plausible/stats/imported/sql/where_builder.ex
@@ -49,7 +49,7 @@ defmodule Plausible.Stats.Imported.SQL.WhereBuilder do
|> Enum.reduce(fn condition, acc -> dynamic([], ^acc or ^condition) end)
end
- defp add_filter(query, [_operation, dimension, _clauses] = filter) do
+ defp add_filter(query, [_operation, dimension, _clauses | _rest] = filter) do
db_field = Plausible.Stats.Filters.without_prefix(dimension)
if db_field == :goal do
diff --git a/lib/plausible/stats/sql/special_metrics.ex b/lib/plausible/stats/sql/special_metrics.ex
index 1bf77d50b6..e6b3aeba90 100644
--- a/lib/plausible/stats/sql/special_metrics.ex
+++ b/lib/plausible/stats/sql/special_metrics.ex
@@ -56,7 +56,8 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do
|> remove_filters_ignored_in_totals_query()
|> Query.set(
dimensions: [],
- include_imported: query.include_imported
+ include_imported: query.include_imported,
+ preloaded_goals: []
)
q
@@ -98,7 +99,8 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do
|> Query.set(
metrics: [:visitors],
order_by: [],
- include_imported: query.include_imported
+ include_imported: query.include_imported,
+ preloaded_goals: []
)
from(e in subquery(q),
diff --git a/lib/plausible/stats/sql/where_builder.ex b/lib/plausible/stats/sql/where_builder.ex
index 2463a889e2..443983521a 100644
--- a/lib/plausible/stats/sql/where_builder.ex
+++ b/lib/plausible/stats/sql/where_builder.ex
@@ -93,11 +93,11 @@ defmodule Plausible.Stats.SQL.WhereBuilder do
|> Enum.reduce(fn condition, acc -> dynamic([], ^acc or ^condition) end)
end
- defp add_filter(:events, _query, [:is, "event:name", list]) do
- dynamic([e], e.name in ^list)
+ defp add_filter(:events, _query, [:is, "event:name" | _rest] = filter) do
+ in_clause(col_value(:name), filter)
end
- defp add_filter(:events, query, [_, "event:goal", _] = filter) do
+ defp add_filter(:events, query, [_, "event:goal" | _rest] = filter) do
Plausible.Goals.Filters.add_filter(query, filter)
end
@@ -154,43 +154,49 @@ defmodule Plausible.Stats.SQL.WhereBuilder do
false
end
- defp filter_custom_prop(prop_name, column_name, [:is, _, clauses]) do
+ defp filter_custom_prop(prop_name, column_name, [:is, _, clauses | _rest] = filter) do
none_value_included = Enum.member?(clauses, "(none)")
+ prop_value_expr = custom_prop_value(column_name, prop_name)
dynamic(
[t],
- (has_key(t, column_name, ^prop_name) and get_by_key(t, column_name, ^prop_name) in ^clauses) or
+ (has_key(t, column_name, ^prop_name) and ^in_clause(prop_value_expr, filter)) or
(^none_value_included and not has_key(t, column_name, ^prop_name))
)
end
- defp filter_custom_prop(prop_name, column_name, [:is_not, _, clauses]) do
+ defp filter_custom_prop(prop_name, column_name, [:is_not, _, clauses | _rest] = filter) do
none_value_included = Enum.member?(clauses, "(none)")
+ prop_value_expr = custom_prop_value(column_name, prop_name)
dynamic(
[t],
(has_key(t, column_name, ^prop_name) and
- get_by_key(t, column_name, ^prop_name) not in ^clauses) or
+ not (^in_clause(prop_value_expr, filter))) or
(^none_value_included and
has_key(t, column_name, ^prop_name) and
- get_by_key(t, column_name, ^prop_name) not in ^clauses) or
+ not (^in_clause(prop_value_expr, filter))) or
(not (^none_value_included) and not has_key(t, column_name, ^prop_name))
)
end
- defp filter_custom_prop(prop_name, column_name, [:matches_wildcard, dimension, clauses]) do
+ defp filter_custom_prop(prop_name, column_name, [:matches_wildcard, dimension, clauses | rest]) do
regexes = Enum.map(clauses, &page_regex/1)
- filter_custom_prop(prop_name, column_name, [:matches, dimension, regexes])
+ filter_custom_prop(prop_name, column_name, [:matches, dimension, regexes | rest])
end
- defp filter_custom_prop(prop_name, column_name, [:matches_wildcard_not, dimension, clauses]) do
+ defp filter_custom_prop(prop_name, column_name, [
+ :matches_wildcard_not,
+ dimension,
+ clauses | rest
+ ]) do
regexes = Enum.map(clauses, &page_regex/1)
- filter_custom_prop(prop_name, column_name, [:matches_not, dimension, regexes])
+ filter_custom_prop(prop_name, column_name, [:matches_not, dimension, regexes | rest])
end
- defp filter_custom_prop(prop_name, column_name, [:matches, _dimension, clauses]) do
+ defp filter_custom_prop(prop_name, column_name, [:matches, _dimension, clauses | _rest]) do
dynamic(
[t],
has_key(t, column_name, ^prop_name) and
@@ -202,7 +208,7 @@ defmodule Plausible.Stats.SQL.WhereBuilder do
)
end
- defp filter_custom_prop(prop_name, column_name, [:matches_not, _dimension, clauses]) do
+ defp filter_custom_prop(prop_name, column_name, [:matches_not, _dimension, clauses | _rest]) do
dynamic(
[t],
has_key(t, column_name, ^prop_name) and
@@ -214,31 +220,23 @@ defmodule Plausible.Stats.SQL.WhereBuilder do
)
end
- defp filter_custom_prop(prop_name, column_name, [:contains, _dimension, clauses]) do
+ defp filter_custom_prop(prop_name, column_name, [:contains | _rest] = filter) do
dynamic(
[t],
has_key(t, column_name, ^prop_name) and
- fragment(
- "multiSearchAny(?, ?)",
- get_by_key(t, column_name, ^prop_name),
- ^clauses
- )
+ ^contains_clause(custom_prop_value(column_name, prop_name), filter)
)
end
- defp filter_custom_prop(prop_name, column_name, [:contains_not, _dimension, clauses]) do
+ defp filter_custom_prop(prop_name, column_name, [:contains_not | _] = filter) do
dynamic(
[t],
has_key(t, column_name, ^prop_name) and
- fragment(
- "not(multiSearchAny(?, ?))",
- get_by_key(t, column_name, ^prop_name),
- ^clauses
- )
+ not (^contains_clause(custom_prop_value(column_name, prop_name), filter))
)
end
- defp filter_field(db_field, [:matches_wildcard, _dimension, glob_exprs]) do
+ defp filter_field(db_field, [:matches_wildcard, _dimension, glob_exprs | _rest]) do
page_regexes = Enum.map(glob_exprs, &page_regex/1)
dynamic(
@@ -247,36 +245,36 @@ defmodule Plausible.Stats.SQL.WhereBuilder do
)
end
- defp filter_field(db_field, [:matches_wildcard_not, dimension, clauses]) do
- dynamic([], not (^filter_field(db_field, [:matches_wildcard, dimension, clauses])))
+ defp filter_field(db_field, [:matches_wildcard_not | rest]) do
+ dynamic([], not (^filter_field(db_field, [:matches_wildcard | rest])))
end
- defp filter_field(db_field, [:contains, _dimension, values]) do
- dynamic([x], fragment("multiSearchAny(?, ?)", type(field(x, ^db_field), :string), ^values))
+ defp filter_field(db_field, [:contains | _rest] = filter) do
+ contains_clause(col_value_string(db_field), filter)
end
- defp filter_field(db_field, [:contains_not, dimension, clauses]) do
- dynamic([], not (^filter_field(db_field, [:contains, dimension, clauses])))
+ defp filter_field(db_field, [:contains_not | rest]) do
+ dynamic([], not (^filter_field(db_field, [:contains | rest])))
end
- defp filter_field(db_field, [:matches, _dimension, clauses]) do
+ defp filter_field(db_field, [:matches, _dimension, clauses | _rest]) do
dynamic(
[x],
fragment("multiMatchAny(?, ?)", type(field(x, ^db_field), :string), ^clauses)
)
end
- defp filter_field(db_field, [:matches_not, dimension, clauses]) do
- dynamic([], not (^filter_field(db_field, [:matches, dimension, clauses])))
+ defp filter_field(db_field, [:matches_not | rest]) do
+ dynamic([], not (^filter_field(db_field, [:matches | rest])))
end
- defp filter_field(db_field, [:is, _dimension, clauses]) do
- list = Enum.map(clauses, &db_field_val(db_field, &1))
- dynamic([x], field(x, ^db_field) in ^list)
+ defp filter_field(db_field, [:is, _dimension, clauses | _rest] = filter) do
+ list = clauses |> Enum.map(&db_field_val(db_field, &1))
+ in_clause(col_value(db_field), filter, list)
end
- defp filter_field(db_field, [:is_not, dimension, clauses]) do
- dynamic([], not (^filter_field(db_field, [:is, dimension, clauses])))
+ defp filter_field(db_field, [:is_not | rest]) do
+ dynamic([], not (^filter_field(db_field, [:is | rest])))
end
@no_ref "Direct / None"
@@ -294,4 +292,45 @@ defmodule Plausible.Stats.SQL.WhereBuilder do
defp db_field_val(:utm_term, @no_ref), do: ""
defp db_field_val(_, @not_set), do: ""
defp db_field_val(_, val), do: val
+
+ defp col_value(column_name) do
+ dynamic([t], field(t, ^column_name))
+ end
+
+ # Needed for string functions to work properly
+ defp col_value_string(column_name) do
+ dynamic([t], type(field(t, ^column_name), :string))
+ end
+
+ defp custom_prop_value(column_name, prop_name) do
+ dynamic([t], get_by_key(t, column_name, ^prop_name))
+ end
+
+ defp in_clause(value_expression, [_, _, clauses | _] = filter, values \\ nil) do
+ values = values || clauses
+
+ if case_sensitive?(filter) do
+ dynamic([t], ^value_expression in ^values)
+ else
+ values = values |> Enum.map(&String.downcase/1)
+ dynamic([t], fragment("lower(?)", ^value_expression) in ^values)
+ end
+ end
+
+ defp contains_clause(value_expression, [_, _, clauses | _] = filter) do
+ if case_sensitive?(filter) do
+ dynamic(
+ [x],
+ fragment("multiSearchAny(?, ?)", ^value_expression, ^clauses)
+ )
+ else
+ dynamic(
+ [x],
+ fragment("multiSearchAnyCaseInsensitive(?, ?)", ^value_expression, ^clauses)
+ )
+ end
+ end
+
+ defp case_sensitive?([_, _, _, %{case_sensitive: false}]), do: false
+ defp case_sensitive?(_), do: true
end
diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex
index 0adebbd506..8e121b8175 100644
--- a/lib/plausible_web/controllers/api/external_stats_controller.ex
+++ b/lib/plausible_web/controllers/api/external_stats_controller.ex
@@ -351,7 +351,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
end)
end
- defp validate_filter(site, [_type, "event:goal", goal_filter]) do
+ defp validate_filter(site, [_type, "event:goal", goal_filter | _rest]) do
configured_goals =
site
|> Plausible.Goals.for_site()
diff --git a/priv/json-schemas/query-api-schema.json b/priv/json-schemas/query-api-schema.json
index e715c59571..593544d1ec 100644
--- a/priv/json-schemas/query-api-schema.json
+++ b/priv/json-schemas/query-api-schema.json
@@ -341,17 +341,22 @@
"enum": ["matches_wildcard", "matches_wildcard_not"],
"description": "filter operation"
},
+ "filter_operation_regex": {
+ "type": "string",
+ "enum": ["matches", "matches_not"],
+ "description": "filter operation"
+ },
"filter_operation_without_goals": {
"type": "string",
- "enum": ["is_not", "contains_not", "matches", "matches_not"],
+ "enum": ["is_not", "contains_not"],
"description": "filter operation"
},
- "filter_operation_with_goals": {
+ "filter_operation_contains": {
"type": "string",
"enum": ["is", "contains"],
"description": "filter operation"
},
- "filter_without_goals": {
+ "filter_with_pattern": {
"type": "array",
"additionalItems": false,
"minItems": 3,
@@ -359,7 +364,7 @@
"items": [
{
"oneOf": [
- { "$ref": "#/definitions/filter_operation_without_goals" },
+ { "$ref": "#/definitions/filter_operation_regex" },
{
"$ref": "#/definitions/filter_operation_wildcard",
"$comment": "only :internal"
@@ -375,14 +380,40 @@
{ "$ref": "#/definitions/clauses" }
]
},
+ "filter_without_goals": {
+ "type": "array",
+ "additionalItems": false,
+ "minItems": 3,
+ "maxItems": 4,
+ "items": [
+ { "$ref": "#/definitions/filter_operation_without_goals" },
+ {
+ "oneOf": [
+ { "$ref": "#/definitions/simple_filter_dimensions" },
+ { "$ref": "#/definitions/custom_property_filter_dimensions" }
+ ]
+ },
+ { "$ref": "#/definitions/clauses" },
+ {
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "case_sensitive": {
+ "type": "boolean",
+ "default": true
+ }
+ }
+ }
+ ]
+ },
"filter_with_goals": {
"type": "array",
"additionalItems": false,
"minItems": 3,
- "maxItems": 3,
+ "maxItems": 4,
"items": [
{
- "$ref": "#/definitions/filter_operation_with_goals"
+ "$ref": "#/definitions/filter_operation_contains"
},
{
"oneOf": [
@@ -393,13 +424,24 @@
},
{
"$ref": "#/definitions/clauses"
+ },
+ {
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "case_sensitive": {
+ "type": "boolean",
+ "default": true
+ }
+ }
}
]
},
"filter_entry": {
"oneOf": [
{ "$ref": "#/definitions/filter_without_goals" },
- { "$ref": "#/definitions/filter_with_goals" }
+ { "$ref": "#/definitions/filter_with_goals" },
+ { "$ref": "#/definitions/filter_with_pattern" }
]
},
"filter_tree": {
diff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exs
index ef990f5ce4..f8c2f536a2 100644
--- a/test/plausible/stats/query_parser_test.exs
+++ b/test/plausible/stats/query_parser_test.exs
@@ -598,6 +598,156 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
"Invalid filters. Dimension `event:hostname` can only be filtered at the top level."
)
end
+
+ for operation <- [:is, :contains, :is_not, :contains_not] do
+ test "#{operation} allows case_sensitive modifier", %{site: site} do
+ %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "filters" => [
+ [
+ Atom.to_string(unquote(operation)),
+ "event:page",
+ ["/foo"],
+ %{"case_sensitive" => false}
+ ],
+ [
+ Atom.to_string(unquote(operation)),
+ "event:name",
+ ["/foo"],
+ %{"case_sensitive" => true}
+ ]
+ ]
+ }
+ |> check_success(site, %{
+ metrics: [:visitors],
+ utc_time_range: @date_range_day,
+ filters: [
+ [unquote(operation), "event:page", ["/foo"], %{case_sensitive: false}],
+ [unquote(operation), "event:name", ["/foo"], %{case_sensitive: true}]
+ ],
+ dimensions: [],
+ order_by: nil,
+ timezone: site.timezone,
+ include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ pagination: %{limit: 10_000, offset: 0}
+ })
+ end
+ end
+
+ for operation <- [:matches, :matches_not, :matches_wildcard, :matches_wildcard_not] do
+ test "case_sensitive modifier is not valid for #{operation}", %{site: site} do
+ %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "filters" => [
+ [
+ Atom.to_string(unquote(operation)),
+ "event:hostname",
+ ["a.plausible.io"],
+ %{"case_sensitive" => false}
+ ]
+ ]
+ }
+ |> check_error(
+ site,
+ "#/filters/0: Invalid filter [\"#{unquote(operation)}\", \"event:hostname\", [\"a.plausible.io\"], %{\"case_sensitive\" => false}]",
+ :internal
+ )
+ end
+ end
+ end
+
+ describe "preloading goals" do
+ setup %{site: site} do
+ insert(:goal, %{site: site, event_name: "Signup"})
+ insert(:goal, %{site: site, event_name: "Purchase"})
+ insert(:goal, %{site: site, event_name: "Contact"})
+
+ :ok
+ end
+
+ test "with exact match", %{site: site} do
+ %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "filters" => [["is", "event:goal", ["Signup", "Purchase"]]]
+ }
+ |> check_success(site, %{
+ metrics: [:visitors],
+ utc_time_range: @date_range_day,
+ filters: [[:is, "event:goal", ["Signup", "Purchase"]]],
+ dimensions: [],
+ order_by: nil,
+ timezone: site.timezone,
+ include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ pagination: %{limit: 10_000, offset: 0}
+ })
+ |> check_goals(preloaded_goals: ["Purchase", "Signup"], revenue_currencies: %{})
+ end
+
+ test "with case insensitive match", %{site: site} do
+ %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "filters" => [["is", "event:goal", ["signup", "purchase"], %{"case_sensitive" => false}]]
+ }
+ |> check_success(site, %{
+ metrics: [:visitors],
+ utc_time_range: @date_range_day,
+ filters: [[:is, "event:goal", ["signup", "purchase"], %{case_sensitive: false}]],
+ dimensions: [],
+ order_by: nil,
+ timezone: site.timezone,
+ include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ pagination: %{limit: 10_000, offset: 0}
+ })
+ |> check_goals(preloaded_goals: ["Purchase", "Signup"], revenue_currencies: %{})
+ end
+
+ test "with contains match", %{site: site} do
+ %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "filters" => [["contains", "event:goal", ["Sign", "pur"]]]
+ }
+ |> check_success(site, %{
+ metrics: [:visitors],
+ utc_time_range: @date_range_day,
+ filters: [[:contains, "event:goal", ["Sign", "pur"]]],
+ dimensions: [],
+ order_by: nil,
+ timezone: site.timezone,
+ include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ pagination: %{limit: 10_000, offset: 0}
+ })
+ |> check_goals(preloaded_goals: ["Signup"], revenue_currencies: %{})
+ end
+
+ test "with case insensitive contains match", %{site: site} do
+ %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "filters" => [["contains", "event:goal", ["sign", "CONT"], %{"case_sensitive" => false}]]
+ }
+ |> check_success(site, %{
+ metrics: [:visitors],
+ utc_time_range: @date_range_day,
+ filters: [[:contains, "event:goal", ["sign", "CONT"], %{case_sensitive: false}]],
+ dimensions: [],
+ order_by: nil,
+ timezone: site.timezone,
+ include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ pagination: %{limit: 10_000, offset: 0}
+ })
+ |> check_goals(preloaded_goals: ["Contact", "Signup"], revenue_currencies: %{})
+ end
end
describe "include validation" do
diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs
index 3c7c1a41c3..cd887fb741 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs
@@ -34,6 +34,301 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryImportedTest do
assert json_response(conn2, 200)["meta"]["imports_included"]
refute json_response(conn2, 200)["meta"]["imports_warning"]
end
+
+ test "filters correctly with 'is' operator", %{
+ conn: conn,
+ site: site,
+ site_import: site_import
+ } do
+ populate_stats(site, site_import.id, [
+ build(:pageview, pathname: "/blog", timestamp: ~N[2023-01-01 00:00:00]),
+ build(:pageview, pathname: "/blog", timestamp: ~N[2023-01-01 00:00:00]),
+ build(:pageview, pathname: "/blog/post/1", timestamp: ~N[2023-01-01 00:00:00]),
+ build(:pageview, pathname: "/about", timestamp: ~N[2023-01-01 00:00:00]),
+ build(:imported_pages,
+ page: "/blog",
+ pageviews: 5,
+ visitors: 3,
+ date: ~D[2023-01-01]
+ ),
+ build(:imported_pages,
+ page: "/blog/post/1",
+ pageviews: 2,
+ visitors: 2,
+ date: ~D[2023-01-01]
+ )
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["visitors", "pageviews"],
+ "filters" => [
+ ["is", "event:page", ["/blog"]]
+ ],
+ "include" => %{"imports" => true}
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"metrics" => [5, 7], "dimensions" => []}
+ ]
+ end
+
+ test "filters correctly with 'is' operator (case insensitive)", %{
+ conn: conn,
+ site: site,
+ site_import: site_import
+ } do
+ populate_stats(site, site_import.id, [
+ build(:pageview, pathname: "/BLOG", timestamp: ~N[2023-01-01 00:00:00]),
+ build(:pageview, pathname: "/blog", timestamp: ~N[2023-01-01 00:00:00]),
+ build(:pageview, pathname: "/blog/post/1", timestamp: ~N[2023-01-01 00:00:00]),
+ build(:pageview, pathname: "/about", timestamp: ~N[2023-01-01 00:00:00]),
+ build(:imported_pages,
+ page: "/BLOG",
+ pageviews: 5,
+ visitors: 3,
+ date: ~D[2023-01-01]
+ ),
+ build(:imported_pages,
+ page: "/blog/post/1",
+ pageviews: 2,
+ visitors: 2,
+ date: ~D[2023-01-01]
+ )
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["visitors", "pageviews"],
+ "filters" => [
+ ["is", "event:page", ["/blOG"], %{"case_sensitive" => false}]
+ ],
+ "include" => %{"imports" => true}
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"metrics" => [5, 7], "dimensions" => []}
+ ]
+ end
+
+ test "filters correctly with 'contains' operator", %{
+ conn: conn,
+ site: site,
+ site_import: site_import
+ } do
+ populate_stats(site, site_import.id, [
+ build(:pageview, pathname: "/blog", timestamp: ~N[2023-01-01 00:00:00]),
+ build(:pageview, pathname: "/blog/post/1", timestamp: ~N[2023-01-01 00:00:00]),
+ build(:pageview, pathname: "/blog/post/2", timestamp: ~N[2023-01-01 00:00:00]),
+ build(:pageview, pathname: "/about", timestamp: ~N[2023-01-01 00:00:00]),
+ build(:imported_pages,
+ page: "/blog",
+ pageviews: 5,
+ visitors: 3,
+ date: ~D[2023-01-01]
+ ),
+ build(:imported_pages,
+ page: "/blog/post/1",
+ pageviews: 2,
+ visitors: 2,
+ date: ~D[2023-01-01]
+ ),
+ build(:imported_pages,
+ page: "/blog/POST/2",
+ pageviews: 3,
+ visitors: 1,
+ date: ~D[2023-01-01]
+ )
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["visitors", "pageviews"],
+ "filters" => [
+ ["contains", "event:page", ["blog/post"]]
+ ],
+ "include" => %{"imports" => true}
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"metrics" => [4, 4], "dimensions" => []}
+ ]
+ end
+
+ test "filters correctly with 'contains' operator (case insensitive)", %{
+ conn: conn,
+ site: site,
+ site_import: site_import
+ } do
+ populate_stats(site, site_import.id, [
+ build(:pageview, pathname: "/BLOG/post/1", timestamp: ~N[2023-01-01 00:00:00]),
+ build(:pageview, pathname: "/blog/POST/2", timestamp: ~N[2023-01-01 00:00:00]),
+ build(:pageview, pathname: "/about", timestamp: ~N[2023-01-01 00:00:00]),
+ build(:imported_pages,
+ page: "/BLOG/POST/1",
+ pageviews: 5,
+ visitors: 3,
+ date: ~D[2023-01-01]
+ ),
+ build(:imported_pages,
+ page: "/blog/post/2",
+ pageviews: 2,
+ visitors: 2,
+ date: ~D[2023-01-01]
+ )
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["visitors", "pageviews"],
+ "filters" => [
+ ["contains", "event:page", ["blog/POST"], %{"case_sensitive" => false}]
+ ],
+ "include" => %{"imports" => true}
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"metrics" => [7, 9], "dimensions" => []}
+ ]
+ end
+
+ test "aggregates custom event goals with 'is' and 'contains' operators", %{
+ conn: conn,
+ site: site,
+ site_import: site_import
+ } do
+ insert(:goal, event_name: "Purchase", site: site)
+
+ populate_stats(site, site_import.id, [
+ build(:event,
+ name: "Purchase",
+ timestamp: ~N[2023-01-01 00:00:00]
+ ),
+ build(:event,
+ name: "Purchase",
+ timestamp: ~N[2023-01-01 00:00:00]
+ ),
+ build(:event,
+ name: "Signup",
+ timestamp: ~N[2023-01-01 00:00:00]
+ ),
+ build(:imported_custom_events,
+ name: "Purchase",
+ visitors: 3,
+ events: 5,
+ date: ~D[2023-01-01]
+ ),
+ build(:imported_custom_events,
+ name: "Signup",
+ visitors: 2,
+ events: 2,
+ date: ~D[2023-01-01]
+ )
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["visitors", "events"],
+ "filters" => [
+ ["is", "event:goal", ["Purchase"]]
+ ],
+ "include" => %{"imports" => true}
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => [], "metrics" => [5, 7]}
+ ]
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["visitors", "events"],
+ "filters" => [
+ ["contains", "event:goal", ["Purch", "sign"]]
+ ],
+ "include" => %{"imports" => true}
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => [], "metrics" => [5, 7]}
+ ]
+ end
+
+ test "aggregates custom event goals with 'is' and 'contains' operators (case insensitive)", %{
+ conn: conn,
+ site: site,
+ site_import: site_import
+ } do
+ insert(:goal, event_name: "Purchase", site: site)
+
+ populate_stats(site, site_import.id, [
+ build(:event,
+ name: "Purchase",
+ timestamp: ~N[2023-01-01 00:00:00]
+ ),
+ build(:event,
+ name: "Purchase",
+ timestamp: ~N[2023-01-01 00:00:00]
+ ),
+ build(:event,
+ name: "Signup",
+ timestamp: ~N[2023-01-01 00:00:00]
+ ),
+ build(:imported_custom_events,
+ name: "Purchase",
+ visitors: 3,
+ events: 5,
+ date: ~D[2023-01-01]
+ ),
+ build(:imported_custom_events,
+ name: "Signup",
+ visitors: 2,
+ events: 2,
+ date: ~D[2023-01-01]
+ )
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["visitors", "events"],
+ "filters" => [
+ ["is", "event:goal", ["purchase"], %{"case_sensitive" => false}]
+ ],
+ "include" => %{"imports" => true}
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => [], "metrics" => [5, 7]}
+ ]
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["visitors", "events"],
+ "filters" => [
+ ["contains", "event:goal", ["PURCH"], %{"case_sensitive" => false}]
+ ],
+ "include" => %{"imports" => true}
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => [], "metrics" => [5, 7]}
+ ]
+ end
end
test "breaks down all metrics by visit:referrer with imported data", %{conn: conn, site: site} do
diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs
index 8966046a19..f22b2c52a8 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs
@@ -272,6 +272,62 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
]
end
+ test "can filter by utm_medium case insensitively", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview,
+ utm_medium: "Social",
+ user_id: @user_id,
+ timestamp: ~N[2021-01-01 00:00:00]
+ ),
+ build(:pageview,
+ utm_medium: "SOCIAL",
+ user_id: @user_id,
+ timestamp: ~N[2021-01-01 00:25:00]
+ ),
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"],
+ "filters" => [["is", "visit:utm_medium", ["sociaL"], %{"case_sensitive" => false}]]
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"metrics" => [2, 1, 0, 1500], "dimensions" => []}
+ ]
+ end
+
+ test "can filter by is_not utm_medium case insensitively", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview,
+ utm_medium: "Social",
+ user_id: @user_id,
+ timestamp: ~N[2021-01-01 00:00:00]
+ ),
+ build(:pageview,
+ utm_medium: "SOCIAL",
+ user_id: @user_id,
+ timestamp: ~N[2021-01-01 00:25:00]
+ ),
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["pageviews"],
+ "filters" => [["is_not", "visit:utm_medium", ["sociaL"], %{"case_sensitive" => false}]]
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"metrics" => [1], "dimensions" => []}
+ ]
+ end
+
test "can filter by utm_source", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
@@ -622,6 +678,43 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
assert json_response(conn, 200)["results"] == [%{"metrics" => [2, 3], "dimensions" => []}]
end
+ test "filtering by a custom event goal (case insensitive)", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:event,
+ name: "Signup",
+ timestamp: ~N[2021-01-01 00:00:00]
+ ),
+ build(:event,
+ name: "Signup",
+ user_id: @user_id,
+ timestamp: ~N[2021-01-01 00:25:00]
+ ),
+ build(:event,
+ name: "Signup",
+ user_id: @user_id,
+ timestamp: ~N[2021-01-01 00:25:00]
+ ),
+ build(:event,
+ name: "NotConfigured",
+ timestamp: ~N[2021-01-01 00:25:00]
+ )
+ ])
+
+ insert(:goal, %{site: site, event_name: "Signup"})
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["visitors", "events"],
+ "filters" => [
+ ["is", "event:goal", ["signup"], %{"case_sensitive" => false}]
+ ]
+ })
+
+ assert json_response(conn, 200)["results"] == [%{"metrics" => [2, 3], "dimensions" => []}]
+ end
+
test "filtering by a revenue goal", %{conn: conn, site: site} do
populate_stats(site, [
build(:event,
@@ -820,6 +913,46 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
assert json_response(conn, 200)["results"] == [%{"metrics" => [2], "dimensions" => []}]
end
+ test "contains page filter case insensitive", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, pathname: "/en/page1"),
+ build(:pageview, pathname: "/EN/page2"),
+ build(:pageview, pathname: "/pl/page1")
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["visitors"],
+ "filters" => [
+ ["contains", "event:page", ["/En/"], %{"case_sensitive" => false}]
+ ]
+ })
+
+ assert json_response(conn, 200)["results"] == [%{"metrics" => [2], "dimensions" => []}]
+ end
+
+ test "contains_not page filter case insensitive", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, pathname: "/en/page1"),
+ build(:pageview, pathname: "/EN/page2"),
+ build(:pageview, pathname: "/pl/page1")
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["visitors"],
+ "filters" => [
+ ["contains_not", "event:page", ["/En/"], %{"case_sensitive" => false}]
+ ]
+ })
+
+ assert json_response(conn, 200)["results"] == [%{"metrics" => [1], "dimensions" => []}]
+ end
+
test "contains_not page filter", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, pathname: "/en/page1"),
@@ -1010,6 +1143,88 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
assert json_response(conn, 200)["results"] == [%{"metrics" => [1], "dimensions" => []}]
end
+ test "`contains` operator with custom properties case insensitive", %{
+ conn: conn,
+ site: site
+ } do
+ populate_stats(site, [
+ build(:pageview,
+ "meta.key": ["name"],
+ "meta.value": ["large-1"]
+ ),
+ build(:pageview,
+ "meta.key": ["name"],
+ "meta.value": ["Small-1"]
+ ),
+ build(:pageview,
+ "meta.key": ["name"],
+ "meta.value": ["mall-1"]
+ ),
+ build(:pageview,
+ "meta.key": ["name"],
+ "meta.value": ["SMALL-2"]
+ ),
+ build(:pageview,
+ "meta.key": ["name"],
+ "meta.value": ["small-2"]
+ ),
+ build(:pageview)
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["visitors"],
+ "filters" => [
+ ["contains", "event:props:name", ["maLL"], %{"case_sensitive" => false}]
+ ]
+ })
+
+ assert json_response(conn, 200)["results"] == [%{"metrics" => [4], "dimensions" => []}]
+ end
+
+ test "`contains_not` operator with custom properties case insensitive", %{
+ conn: conn,
+ site: site
+ } do
+ populate_stats(site, [
+ build(:pageview,
+ "meta.key": ["name"],
+ "meta.value": ["large-1"]
+ ),
+ build(:pageview,
+ "meta.key": ["name"],
+ "meta.value": ["Small-1"]
+ ),
+ build(:pageview,
+ "meta.key": ["name"],
+ "meta.value": ["mall-1"]
+ ),
+ build(:pageview,
+ "meta.key": ["name"],
+ "meta.value": ["SMALL-2"]
+ ),
+ build(:pageview,
+ "meta.key": ["name"],
+ "meta.value": ["small-2"]
+ ),
+ build(:pageview)
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["visitors"],
+ "filters" => [
+ ["contains_not", "event:props:name", ["maLL"], %{"case_sensitive" => false}]
+ ]
+ })
+
+ assert json_response(conn, 200)["results"] == [%{"metrics" => [1], "dimensions" => []}]
+ end
+
test "`matches` and `matches_not` operator with custom properties", %{
conn: conn,
site: site
@@ -2588,6 +2803,39 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
]
end
+ test "goal contains filter for goal breakdown (case insensitive)", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:event, name: "Onboarding conversion: Step 1"),
+ build(:event, name: "Onboarding conversion: Step 1"),
+ build(:event, name: "Onboarding conversion: Step 2"),
+ build(:event, name: "Unrelated"),
+ build(:pageview, pathname: "/conversion")
+ ])
+
+ insert(:goal, site: site, event_name: "Onboarding conversion: Step 1")
+ insert(:goal, site: site, event_name: "Onboarding conversion: Step 2")
+ insert(:goal, site: site, event_name: "Unrelated")
+ insert(:goal, site: site, page_path: "/conversion")
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "dimensions" => ["event:goal"],
+ "filters" => [
+ ["contains", "event:goal", ["step"], %{"case_sensitive" => false}]
+ ]
+ })
+
+ %{"results" => results} = json_response(conn, 200)
+
+ assert results == [
+ %{"dimensions" => ["Onboarding conversion: Step 1"], "metrics" => [2]},
+ %{"dimensions" => ["Onboarding conversion: Step 2"], "metrics" => [1]}
+ ]
+ end
+
test "mixed multi-goal filter for breakdown by visit:country", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, country_code: "EE", pathname: "/en/register"),
@@ -2814,6 +3062,43 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
]
end
+ test "IN filter for event:name case insensitive", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:event,
+ name: "Signup",
+ timestamp: ~N[2021-01-01 00:00:00]
+ ),
+ build(:event,
+ name: "Signup",
+ timestamp: ~N[2021-01-01 00:00:00]
+ ),
+ build(:event,
+ name: "Login",
+ timestamp: ~N[2021-01-01 00:00:00]
+ ),
+ build(:event,
+ name: "Irrelevant",
+ timestamp: ~N[2021-01-01 00:00:00]
+ )
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "filters" => [
+ ["is", "event:name", ["signup", "LOGIN"], %{"case_sensitive" => false}]
+ ]
+ })
+
+ %{"results" => results} = json_response(conn, 200)
+
+ assert results == [
+ %{"dimensions" => [], "metrics" => [3]}
+ ]
+ end
+
test "IN filter for event:props:*", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
@@ -2881,6 +3166,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
"meta.value": ["Safari", "target_value"],
timestamp: ~N[2021-01-01 00:00:00]
),
+ build(:pageview,
+ browser: "Chrome",
+ "meta.key": ["browser", "prop"],
+ "meta.value": ["Chrome", "target_value"],
+ timestamp: ~N[2021-01-01 00:00:00]
+ ),
build(:pageview,
browser: "Firefox",
"meta.key": ["browser", "prop"],
@@ -2896,7 +3187,9 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
"date_range" => "all",
"dimensions" => ["visit:browser"],
"filters" => [
- ["is", "event:props:browser", ["Chrome", "Safari"]],
+ ["is", "event:props:browser", ["CHROME", "sAFari"], %{"case_sensitive" => false}],
+ # Negate a previously set filter
+ ["is_not", "event:props:browser", ["Chrome"], %{"case_sensitive" => false}],
["is", "event:props:prop", ["target_value"]]
]
})
@@ -3721,4 +4014,32 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
]
end
end
+
+ test "can filter by utm_medium case insensitively", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview,
+ utm_medium: "Social",
+ user_id: @user_id,
+ timestamp: ~N[2021-01-01 00:00:00]
+ ),
+ build(:pageview,
+ utm_medium: "SOCIAL",
+ user_id: @user_id,
+ timestamp: ~N[2021-01-01 00:25:00]
+ ),
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"],
+ "filters" => [["is", "visit:utm_medium", ["social"], %{"case_sensitive" => false}]]
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"metrics" => [2, 1, 0, 1500], "dimensions" => []}
+ ]
+ end
end