Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Expand segments in queries (#4982)

Artur Pataf0104cb6e7ba
diff --git a/assets/js/types/query-api.d.ts b/assets/js/types/query-api.d.ts
index 2b77acf192..51b52f5050 100644
--- a/assets/js/types/query-api.d.ts
+++ b/assets/js/types/query-api.d.ts
@@ -64,7 +64,7 @@ export type CustomPropertyFilterDimensions = string;
export type GoalDimension = "event:goal";
export type TimeDimensions = "time" | "time:month" | "time:week" | "time:day" | "time:hour";
export type FilterTree = FilterEntry | FilterAndOr | FilterNot;
-export type FilterEntry = FilterWithoutGoals | FilterWithGoals | FilterWithPattern;
+export type FilterEntry = FilterWithoutGoals | FilterWithGoals | FilterWithPattern | FilterForSegment;
/**
* @minItems 3
* @maxItems 4
@@ -115,6 +115,11 @@ export type FilterWithPattern = [
* filter operation
*/
export type FilterOperationRegex = "matches" | "matches_not";
+/**
+ * @minItems 3
+ * @maxItems 3
+ */
+export type FilterForSegment = ["is", "segment", number[]];
/**
* @minItems 2
* @maxItems 2
diff --git a/lib/plausible/segments/filters.ex b/lib/plausible/segments/filters.ex
new file mode 100644
index 0000000000..a18cf33e78
--- /dev/null
+++ b/lib/plausible/segments/filters.ex
@@ -0,0 +1,108 @@
+defmodule Plausible.Segments.Filters do
+ @moduledoc """
+ This module contains functions that enable resolving segments in filters.
+ """
+ alias Plausible.Segments
+ alias Plausible.Stats.Filters
+
+ @max_segment_filters_count 10
+
+ @doc """
+ Finds unique segment IDs used in query filters.
+
+ ## Examples
+ iex> get_segment_ids([[:not, [:is, "segment", [10, 20]]], [:contains, "visit:entry_page", ["blog"]]])
+ {:ok, [10, 20]}
+
+ iex> get_segment_ids([[:and, [[:is, "segment", Enum.to_list(1..6)], [:is, "segment", Enum.to_list(1..6)]]]])
+ {:error, "Invalid filters. You can only use up to 10 segment filters in a query."}
+ """
+ def get_segment_ids(filters) do
+ ids =
+ filters
+ |> Filters.traverse()
+ |> Enum.flat_map(fn
+ {[_operation, "segment", clauses], _depth} -> clauses
+ _ -> []
+ end)
+
+ if length(ids) > @max_segment_filters_count do
+ {:error,
+ "Invalid filters. You can only use up to #{@max_segment_filters_count} segment filters in a query."}
+ else
+ {:ok, Enum.uniq(ids)}
+ end
+ end
+
+ def preload_needed_segments(%Plausible.Site{} = site, filters) do
+ with {:ok, segment_ids} <- get_segment_ids(filters),
+ {:ok, segments} <-
+ Segments.get_many(
+ site,
+ segment_ids,
+ fields: [:id, :segment_data]
+ ),
+ {:ok, segments_by_id} <-
+ {:ok,
+ Enum.into(
+ segments,
+ %{},
+ fn %Segments.Segment{id: id, segment_data: segment_data} ->
+ case Filters.QueryParser.parse_filters(segment_data["filters"]) do
+ {:ok, filters} -> {id, filters}
+ _ -> {id, nil}
+ end
+ end
+ )},
+ :ok <-
+ if(Enum.any?(segment_ids, fn id -> is_nil(Map.get(segments_by_id, id)) end),
+ do: {:error, "Invalid filters. Some segments don't exist or aren't accessible."},
+ else: :ok
+ ) do
+ {:ok, segments_by_id}
+ end
+ end
+
+ defp replace_segment_with_filter_tree([_, "segment", clauses], preloaded_segments) do
+ if length(clauses) === 1 do
+ [[:and, Map.get(preloaded_segments, Enum.at(clauses, 0))]]
+ else
+ [[:or, Enum.map(clauses, fn id -> [:and, Map.get(preloaded_segments, id)] end)]]
+ end
+ end
+
+ defp replace_segment_with_filter_tree(_filter, _preloaded_segments) do
+ nil
+ end
+
+ @doc """
+ ## Examples
+
+ iex> resolve_segments([[:is, "visit:entry_page", ["/home"]]], %{})
+ {:ok, [[:is, "visit:entry_page", ["/home"]]]}
+
+ iex> resolve_segments([[:is, "visit:entry_page", ["/home"]], [:is, "segment", [1]]], %{1 => [[:contains, "visit:entry_page", ["blog"]], [:is, "visit:country", ["PL"]]]})
+ {:ok, [
+ [:is, "visit:entry_page", ["/home"]],
+ [:and, [[:contains, "visit:entry_page", ["blog"]], [:is, "visit:country", ["PL"]]]]
+ ]}
+
+ iex> resolve_segments([[:is, "segment", [1, 2]]], %{1 => [[:contains, "event:goal", ["Singup"]], [:is, "visit:country", ["PL"]]], 2 => [[:contains, "event:goal", ["Sauna"]], [:is, "visit:country", ["EE"]]]})
+ {:ok, [
+ [:or, [
+ [:and, [[:contains, "event:goal", ["Singup"]], [:is, "visit:country", ["PL"]]]],
+ [:and, [[:contains, "event:goal", ["Sauna"]], [:is, "visit:country", ["EE"]]]]]
+ ]
+ ]}
+ """
+ def resolve_segments(original_filters, preloaded_segments) do
+ if map_size(preloaded_segments) > 0 do
+ {:ok,
+ Filters.transform_filters(original_filters, fn f ->
+ replace_segment_with_filter_tree(f, preloaded_segments)
+ end)}
+ else
+ {:ok, original_filters}
+ end
+ end
+end
diff --git a/lib/plausible/segments/segments.ex b/lib/plausible/segments/segments.ex
index 6284bd6345..37f420e6d6 100644
--- a/lib/plausible/segments/segments.ex
+++ b/lib/plausible/segments/segments.ex
@@ -37,6 +37,21 @@ defmodule Plausible.Segments do
end
end
+ @spec get_many(Plausible.Site.t(), list(pos_integer()), Keyword.t()) ::
+ {:ok, [Segment.t()]}
+ def get_many(%Plausible.Site{} = site, segment_ids, opts) when is_list(segment_ids) do
+ fields = Keyword.get(opts, :fields, [:id])
+
+ query =
+ from(segment in Segment,
+ select: ^fields,
+ where: segment.site_id == ^site.id,
+ where: segment.id in ^segment_ids
+ )
+
+ {:ok, Repo.all(query)}
+ end
+
@spec get_one(pos_integer(), Plausible.Site.t(), atom(), pos_integer() | nil) ::
{:ok, Segment.t()}
| error_not_enough_permissions()
diff --git a/lib/plausible/stats/filters/filters.ex b/lib/plausible/stats/filters/filters.ex
index 9f4116f2c8..4c828fef1e 100644
--- a/lib/plausible/stats/filters/filters.ex
+++ b/lib/plausible/stats/filters/filters.ex
@@ -57,7 +57,7 @@ defmodule Plausible.Stats.Filters do
Returns an empty list when argument type is unexpected (e.g. `nil`).
- ### Examples:
+ ## Examples:
iex> Filters.parse("visit:browser!=Chrome")
[[:is_not, "visit:browser", ["Chrome"]]]
@@ -128,6 +128,13 @@ defmodule Plausible.Stats.Filters do
Transformer will receive each node (filter, and/or/not subtree) of
query and must return a list of nodes to replace it with or nil
to ignore and look deeper.
+
+ ## Examples
+ iex> Filters.transform_filters([[:is, "visit:os", ["Linux"]], [:and, [[:is, "segment", [1]], [:is, "segment", [2]]]]], fn
+ ...> [_, "segment", _] -> [[:is, "segment", ["changed"]]]
+ ...> _ -> nil
+ ...> end)
+ [[:is, "visit:os", ["Linux"]], [:and, [[:is, "segment", ["changed"]], [:is, "segment", ["changed"]]]]]
"""
def transform_filters(filters, transformer) do
filters
@@ -146,7 +153,7 @@ defmodule Plausible.Stats.Filters do
# Reached a leaf node, return existing value
{nil, filter} ->
- [[filter]]
+ [filter]
# Transformer returned a value - don't transform that subtree
{transformed_filters, _filter} ->
@@ -154,7 +161,7 @@ defmodule Plausible.Stats.Filters do
end
end
- defp traverse(filters, depth \\ -1) do
+ def traverse(filters, depth \\ -1) do
filters
|> Enum.flat_map(&traverse_tree(&1, depth + 1))
end
diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.ex
index 048e61f988..0dd6a976c8 100644
--- a/lib/plausible/stats/filters/query_parser.ex
+++ b/lib/plausible/stats/filters/query_parser.ex
@@ -34,6 +34,10 @@ defmodule Plausible.Stats.Filters.QueryParser do
utc_time_range = raw_time_range |> DateTimeRange.to_timezone("Etc/UTC"),
{:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])),
{:ok, filters} <- parse_filters(Map.get(params, "filters", [])),
+ {:ok, preloaded_segments} <-
+ Plausible.Segments.Filters.preload_needed_segments(site, filters),
+ {:ok, filters} <-
+ Plausible.Segments.Filters.resolve_segments(filters, preloaded_segments),
{:ok, dimensions} <- parse_dimensions(Map.get(params, "dimensions", [])),
{:ok, order_by} <- parse_order_by(Map.get(params, "order_by")),
{:ok, include} <- parse_include(site, Map.get(params, "include", %{})),
@@ -161,7 +165,10 @@ defmodule Plausible.Stats.Filters.QueryParser do
"Invalid visit:country filter, visit:country needs to be a valid 2-letter country code."}
end
- {_, true} ->
+ {"segment", _} when all_integers? ->
+ {:ok, list}
+
+ {_, true} when filter_key !== "segment" ->
{:ok, list}
_ ->
@@ -396,6 +403,9 @@ defmodule Plausible.Stats.Filters.QueryParser do
{:error, error_message}
end
+ "segment" ->
+ {:ok, filter_key}
+
_ ->
{:error, error_message}
end
diff --git a/priv/json-schemas/query-api-schema.json b/priv/json-schemas/query-api-schema.json
index df18503283..6a325c8d8e 100644
--- a/priv/json-schemas/query-api-schema.json
+++ b/priv/json-schemas/query-api-schema.json
@@ -98,13 +98,11 @@
"minItems": 2,
"maxItems": 2,
"items": {
- "type": "string",
- "format": "date"
+ "type": "string",
+ "format": "date"
},
"description": "If custom period. A list of two ISO8601 dates or timestamps to compare against.",
- "examples": [
- ["2024-01-01", "2024-01-31"]
- ]
+ "examples": [["2024-01-01", "2024-01-31"]]
}
},
"required": ["mode", "date_range"],
@@ -439,11 +437,35 @@
}
]
},
+ "filter_for_segment": {
+ "type": "array",
+ "additionalItems": false,
+ "minItems": 3,
+ "maxItems": 3,
+ "items": [
+ {
+ "const": "is"
+ },
+ {
+ "const": "segment"
+ },
+ {
+ "type": "array",
+ "items": {
+ "type": ["integer"]
+ }
+ }
+ ]
+ },
"filter_entry": {
"oneOf": [
{ "$ref": "#/definitions/filter_without_goals" },
{ "$ref": "#/definitions/filter_with_goals" },
- { "$ref": "#/definitions/filter_with_pattern" }
+ { "$ref": "#/definitions/filter_with_pattern" },
+ {
+ "$ref": "#/definitions/filter_for_segment",
+ "$comment": "only :internal"
+ }
]
},
"filter_tree": {
diff --git a/test/plausible/segments/filters_test.exs b/test/plausible/segments/filters_test.exs
new file mode 100644
index 0000000000..e19b58df73
--- /dev/null
+++ b/test/plausible/segments/filters_test.exs
@@ -0,0 +1,5 @@
+defmodule Plausible.Segments.FiltersTest do
+ use ExUnit.Case, async: true
+
+ doctest Plausible.Segments.Filters, import: true
+end
diff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exs
index ba7c8e8b81..968d04fa0f 100644
--- a/test/plausible/stats/query_parser_test.exs
+++ b/test/plausible/stats/query_parser_test.exs
@@ -2,10 +2,11 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
use Plausible
use Plausible.DataCase
use Plausible.Teams.Test
+ import Plausible.Stats.Filters.QueryParser
+ doctest Plausible.Stats.Filters.QueryParser
alias Plausible.Stats.DateTimeRange
alias Plausible.Stats.Filters
- import Plausible.Stats.Filters.QueryParser
setup [:create_user, :create_site]
@@ -2028,4 +2029,215 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
})
end
end
+
+ describe "filtering with segments" do
+ test "parsing fails when too many segments in query", %{
+ user: user,
+ site: site
+ } do
+ segments =
+ insert_list(11, :segment,
+ type: :site,
+ owner: user,
+ site: site,
+ name: "any"
+ )
+
+ %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "filters" => [
+ ["and", segments |> Enum.map(fn segment -> ["is", "segment", [segment.id]] end)]
+ ]
+ }
+ |> check_error(
+ site,
+ "Invalid filters. You can only use up to 10 segment filters in a query.",
+ :internal
+ )
+ end
+
+ test "parsing fails when segment filter is used, but segment is from another site", %{
+ site: site
+ } do
+ other_user = new_user()
+ other_site = new_site(owner: other_user)
+
+ segment =
+ insert(:segment,
+ type: :site,
+ owner: other_user,
+ site: other_site,
+ name: "any"
+ )
+
+ %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "filters" => [["is", "segment", [segment.id]]]
+ }
+ |> check_error(
+ site,
+ "Invalid filters. Some segments don't exist or aren't accessible.",
+ :internal
+ )
+ end
+
+ test "hiding custom properties filters in segments doesn't allow bypasssing feature check",
+ %{
+ site: site,
+ user: user
+ } do
+ subscribe_to_enterprise_plan(user, features: [Plausible.Billing.Feature.StatsAPI])
+
+ segment =
+ insert(:segment,
+ type: :site,
+ owner: user,
+ site: site,
+ name: "segment with custom props filter",
+ segment_data: %{"filters" => [["is", "event:props:foobar", ["foo"]]]}
+ )
+
+ %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "filters" => [["is", "segment", [segment.id]]]
+ }
+ |> check_error(
+ site,
+ "The owner of this site does not have access to the custom properties feature.",
+ :internal
+ )
+ end
+
+ test "querying conversion rate is illegal if the complex event:goal filter is within a segment",
+ %{
+ site: site,
+ user: user
+ } do
+ segment =
+ insert(:segment,
+ type: :site,
+ owner: user,
+ site: site,
+ name: "any",
+ segment_data: %{
+ "filters" => [
+ [
+ "or",
+ [
+ ["is", "event:goal", ["Signup"]],
+ ["contains", "event:page", ["/"]]
+ ]
+ ]
+ ]
+ }
+ )
+
+ %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors", "conversion_rate"],
+ "date_range" => "all",
+ "filters" => [["is", "segment", [segment.id]]]
+ }
+ |> check_error(
+ site,
+ "Invalid filters. Dimension `event:goal` can only be filtered at the top level.",
+ :internal
+ )
+ end
+
+ test "resolves segments correctly", %{site: site, user: user} do
+ emea_segment =
+ insert(:segment,
+ type: :site,
+ owner: user,
+ site: site,
+ name: "EMEA",
+ segment_data: %{
+ "filters" => [["is", "visit:country", ["FR", "DE"]]],
+ "labels" => %{"FR" => "France", "DE" => "Germany"}
+ }
+ )
+
+ apac_segment =
+ insert(:segment,
+ type: :site,
+ owner: user,
+ site: site,
+ name: "APAC",
+ segment_data: %{
+ "filters" => [["is", "visit:country", ["AU", "NZ"]]],
+ "labels" => %{"AU" => "Australia", "NZ" => "New Zealand"}
+ }
+ )
+
+ firefox_segment =
+ insert(:segment,
+ type: :site,
+ owner: user,
+ site: site,
+ name: "APAC",
+ segment_data: %{
+ "filters" => [
+ ["is", "visit:browser", ["Firefox"]],
+ ["is", "visit:os", ["Linux"]]
+ ]
+ }
+ )
+
+ %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors", "events"],
+ "date_range" => "all",
+ "filters" => [
+ [
+ "and",
+ [
+ ["is", "segment", [apac_segment.id, emea_segment.id]],
+ ["is", "segment", [firefox_segment.id]]
+ ]
+ ]
+ ]
+ }
+ |> check_success(
+ site,
+ %{
+ metrics: [:visitors, :events],
+ utc_time_range: @date_range_day,
+ filters: [
+ [
+ :and,
+ [
+ [
+ :or,
+ [
+ [:and, [[:is, "visit:country", ["AU", "NZ"]]]],
+ [:and, [[:is, "visit:country", ["FR", "DE"]]]]
+ ]
+ ],
+ [
+ :and,
+ [
+ [:is, "visit:browser", ["Firefox"]],
+ [:is, "visit:os", ["Linux"]]
+ ]
+ ]
+ ]
+ ]
+ ],
+ dimensions: [],
+ order_by: nil,
+ timezone: site.timezone,
+ include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},
+ pagination: %{limit: 10_000, offset: 0}
+ },
+ :internal
+ )
+ end
+ end
end
diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs
index 1681bb208f..4697b94c71 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs
@@ -4577,4 +4577,110 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
assert "metric_warnings" not in json_response(conn, 200)["meta"]
end
end
+
+ describe "segment filters" do
+ setup [:create_user, :create_site, :create_api_key, :use_api_key]
+
+ test "segment filters are (not yet) available in public API", %{conn: conn, site: site} do
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "filters" => [["is", "segment", [1]]],
+ "date_range" => "all",
+ "metrics" => ["visitors"]
+ })
+
+ assert json_response(conn, 400) == %{
+ "error" => "#/filters/0: Invalid filter [\"is\", \"segment\", [1]]"
+ }
+ end
+
+ test "site segments of other sites don't resolve", %{
+ conn: conn,
+ site: site
+ } do
+ other_site = new_site()
+ other_user = add_guest(other_site, role: :editor)
+
+ segment =
+ insert(:segment,
+ name: "Segment of another site",
+ type: :site,
+ owner: other_user,
+ site: other_site,
+ segment_data: %{
+ "filters" => [["is", "event:page", ["/blog"]]]
+ }
+ )
+
+ conn =
+ post(conn, "/api/v2/query-internal-test", %{
+ "site_id" => site.domain,
+ "filters" => [["is", "segment", [segment.id]]],
+ "date_range" => "all",
+ "metrics" => ["events"]
+ })
+
+ assert json_response(conn, 400) == %{
+ "error" => "Invalid filters. Some segments don't exist or aren't accessible."
+ }
+ end
+
+ test "even personal segments of other users of the same site resolve to filters, with segments expanded in response",
+ %{
+ conn: conn,
+ site: site
+ } do
+ other_user = add_guest(site, role: :editor)
+
+ segment =
+ insert(:segment,
+ type: :personal,
+ owner: other_user,
+ site: site,
+ name: "Signups",
+ segment_data: %{
+ "filters" => [["is", "event:name", ["Signup"]]]
+ }
+ )
+
+ insert(:goal, %{site: site, event_name: "Signup"})
+
+ populate_stats(site, [
+ build(:event,
+ name: "Signup",
+ timestamp: ~N[2021-01-01 00:00:00]
+ ),
+ build(:event,
+ name: "Signup",
+ user_id: @user_id,
+ timestamp: ~N[2021-01-01 00:00:00]
+ ),
+ build(:event,
+ name: "Signup",
+ user_id: @user_id,
+ timestamp: ~N[2021-01-01 00:00:00]
+ ),
+ build(:event,
+ name: "AnyOtherEvent",
+ timestamp: ~N[2021-01-01 00:00:00]
+ )
+ ])
+
+ conn =
+ post(conn, "/api/v2/query-internal-test", %{
+ "site_id" => site.domain,
+ "filters" => [["is", "segment", [segment.id]]],
+ "date_range" => "all",
+ "metrics" => ["events"]
+ })
+
+ assert json_response(conn, 200)["results"] == [%{"dimensions" => [], "metrics" => [3]}]
+
+ # response shows what filters the segment was resolved to
+ assert json_response(conn, 200)["query"]["filters"] == [
+ ["and", [["is", "event:name", ["Signup"]]]]
+ ]
+ end
+ end
end