Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Expand segments in queries (#4982)
Artur Pataf0104cb6e7ba
diff --git a/assets/js/types/query-api.d.ts b/assets/js/types/query-api.d.tsindex 2b77acf192..51b52f5050 100644--- a/assets/js/types/query-api.d.ts+++ b/assets/js/types/query-api.d.ts@@ -64,7 +64,7 @@ export type CustomPropertyFilterDimensions = string;export type GoalDimension = "event:goal";export type TimeDimensions = "time" | "time:month" | "time:week" | "time:day" | "time:hour";export type FilterTree = FilterEntry | FilterAndOr | FilterNot;-export type FilterEntry = FilterWithoutGoals | FilterWithGoals | FilterWithPattern;+export type FilterEntry = FilterWithoutGoals | FilterWithGoals | FilterWithPattern | FilterForSegment;/*** @minItems 3* @maxItems 4@@ -115,6 +115,11 @@ export type FilterWithPattern = [* filter operation*/export type FilterOperationRegex = "matches" | "matches_not";+/**+ * @minItems 3+ * @maxItems 3+ */+export type FilterForSegment = ["is", "segment", number[]];/*** @minItems 2* @maxItems 2diff --git a/lib/plausible/segments/filters.ex b/lib/plausible/segments/filters.exnew file mode 100644index 0000000000..a18cf33e78--- /dev/null+++ b/lib/plausible/segments/filters.ex@@ -0,0 +1,108 @@+defmodule Plausible.Segments.Filters do+ @moduledoc """+ This module contains functions that enable resolving segments in filters.+ """+ alias Plausible.Segments+ alias Plausible.Stats.Filters++ @max_segment_filters_count 10++ @doc """+ Finds unique segment IDs used in query filters.++ ## Examples+ iex> get_segment_ids([[:not, [:is, "segment", [10, 20]]], [:contains, "visit:entry_page", ["blog"]]])+ {:ok, [10, 20]}++ iex> get_segment_ids([[:and, [[:is, "segment", Enum.to_list(1..6)], [:is, "segment", Enum.to_list(1..6)]]]])+ {:error, "Invalid filters. You can only use up to 10 segment filters in a query."}+ """+ def get_segment_ids(filters) do+ ids =+ filters+ |> Filters.traverse()+ |> Enum.flat_map(fn+ {[_operation, "segment", clauses], _depth} -> clauses+ _ -> []+ end)++ if length(ids) > @max_segment_filters_count do+ {:error,+ "Invalid filters. You can only use up to #{@max_segment_filters_count} segment filters in a query."}+ else+ {:ok, Enum.uniq(ids)}+ end+ end++ def preload_needed_segments(%Plausible.Site{} = site, filters) do+ with {:ok, segment_ids} <- get_segment_ids(filters),+ {:ok, segments} <-+ Segments.get_many(+ site,+ segment_ids,+ fields: [:id, :segment_data]+ ),+ {:ok, segments_by_id} <-+ {:ok,+ Enum.into(+ segments,+ %{},+ fn %Segments.Segment{id: id, segment_data: segment_data} ->+ case Filters.QueryParser.parse_filters(segment_data["filters"]) do+ {:ok, filters} -> {id, filters}+ _ -> {id, nil}+ end+ end+ )},+ :ok <-+ if(Enum.any?(segment_ids, fn id -> is_nil(Map.get(segments_by_id, id)) end),+ do: {:error, "Invalid filters. Some segments don't exist or aren't accessible."},+ else: :ok+ ) do+ {:ok, segments_by_id}+ end+ end++ defp replace_segment_with_filter_tree([_, "segment", clauses], preloaded_segments) do+ if length(clauses) === 1 do+ [[:and, Map.get(preloaded_segments, Enum.at(clauses, 0))]]+ else+ [[:or, Enum.map(clauses, fn id -> [:and, Map.get(preloaded_segments, id)] end)]]+ end+ end++ defp replace_segment_with_filter_tree(_filter, _preloaded_segments) do+ nil+ end++ @doc """+ ## Examples++ iex> resolve_segments([[:is, "visit:entry_page", ["/home"]]], %{})+ {:ok, [[:is, "visit:entry_page", ["/home"]]]}++ iex> resolve_segments([[:is, "visit:entry_page", ["/home"]], [:is, "segment", [1]]], %{1 => [[:contains, "visit:entry_page", ["blog"]], [:is, "visit:country", ["PL"]]]})+ {:ok, [+ [:is, "visit:entry_page", ["/home"]],+ [:and, [[:contains, "visit:entry_page", ["blog"]], [:is, "visit:country", ["PL"]]]]+ ]}++ iex> resolve_segments([[:is, "segment", [1, 2]]], %{1 => [[:contains, "event:goal", ["Singup"]], [:is, "visit:country", ["PL"]]], 2 => [[:contains, "event:goal", ["Sauna"]], [:is, "visit:country", ["EE"]]]})+ {:ok, [+ [:or, [+ [:and, [[:contains, "event:goal", ["Singup"]], [:is, "visit:country", ["PL"]]]],+ [:and, [[:contains, "event:goal", ["Sauna"]], [:is, "visit:country", ["EE"]]]]]+ ]+ ]}+ """+ def resolve_segments(original_filters, preloaded_segments) do+ if map_size(preloaded_segments) > 0 do+ {:ok,+ Filters.transform_filters(original_filters, fn f ->+ replace_segment_with_filter_tree(f, preloaded_segments)+ end)}+ else+ {:ok, original_filters}+ end+ end+enddiff --git a/lib/plausible/segments/segments.ex b/lib/plausible/segments/segments.exindex 6284bd6345..37f420e6d6 100644--- a/lib/plausible/segments/segments.ex+++ b/lib/plausible/segments/segments.ex@@ -37,6 +37,21 @@ defmodule Plausible.Segments doendend+ @spec get_many(Plausible.Site.t(), list(pos_integer()), Keyword.t()) ::+ {:ok, [Segment.t()]}+ def get_many(%Plausible.Site{} = site, segment_ids, opts) when is_list(segment_ids) do+ fields = Keyword.get(opts, :fields, [:id])++ query =+ from(segment in Segment,+ select: ^fields,+ where: segment.site_id == ^site.id,+ where: segment.id in ^segment_ids+ )++ {:ok, Repo.all(query)}+ end+@spec get_one(pos_integer(), Plausible.Site.t(), atom(), pos_integer() | nil) ::{:ok, Segment.t()}| error_not_enough_permissions()diff --git a/lib/plausible/stats/filters/filters.ex b/lib/plausible/stats/filters/filters.exindex 9f4116f2c8..4c828fef1e 100644--- a/lib/plausible/stats/filters/filters.ex+++ b/lib/plausible/stats/filters/filters.ex@@ -57,7 +57,7 @@ defmodule Plausible.Stats.Filters doReturns an empty list when argument type is unexpected (e.g. `nil`).- ### Examples:+ ## Examples:iex> Filters.parse("visit:browser!=Chrome")[[:is_not, "visit:browser", ["Chrome"]]]@@ -128,6 +128,13 @@ defmodule Plausible.Stats.Filters doTransformer will receive each node (filter, and/or/not subtree) ofquery and must return a list of nodes to replace it with or nilto ignore and look deeper.++ ## Examples+ iex> Filters.transform_filters([[:is, "visit:os", ["Linux"]], [:and, [[:is, "segment", [1]], [:is, "segment", [2]]]]], fn+ ...> [_, "segment", _] -> [[:is, "segment", ["changed"]]]+ ...> _ -> nil+ ...> end)+ [[:is, "visit:os", ["Linux"]], [:and, [[:is, "segment", ["changed"]], [:is, "segment", ["changed"]]]]]"""def transform_filters(filters, transformer) dofilters@@ -146,7 +153,7 @@ defmodule Plausible.Stats.Filters do# Reached a leaf node, return existing value{nil, filter} ->- [[filter]]+ [filter]# Transformer returned a value - don't transform that subtree{transformed_filters, _filter} ->@@ -154,7 +161,7 @@ defmodule Plausible.Stats.Filters doendend- defp traverse(filters, depth \\ -1) do+ def traverse(filters, depth \\ -1) dofilters|> Enum.flat_map(&traverse_tree(&1, depth + 1))enddiff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.exindex 048e61f988..0dd6a976c8 100644--- a/lib/plausible/stats/filters/query_parser.ex+++ b/lib/plausible/stats/filters/query_parser.ex@@ -34,6 +34,10 @@ defmodule Plausible.Stats.Filters.QueryParser doutc_time_range = raw_time_range |> DateTimeRange.to_timezone("Etc/UTC"),{:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])),{:ok, filters} <- parse_filters(Map.get(params, "filters", [])),+ {:ok, preloaded_segments} <-+ Plausible.Segments.Filters.preload_needed_segments(site, filters),+ {:ok, filters} <-+ Plausible.Segments.Filters.resolve_segments(filters, preloaded_segments),{:ok, dimensions} <- parse_dimensions(Map.get(params, "dimensions", [])),{:ok, order_by} <- parse_order_by(Map.get(params, "order_by")),{:ok, include} <- parse_include(site, Map.get(params, "include", %{})),@@ -161,7 +165,10 @@ defmodule Plausible.Stats.Filters.QueryParser do"Invalid visit:country filter, visit:country needs to be a valid 2-letter country code."}end- {_, true} ->+ {"segment", _} when all_integers? ->+ {:ok, list}++ {_, true} when filter_key !== "segment" ->{:ok, list}_ ->@@ -396,6 +403,9 @@ defmodule Plausible.Stats.Filters.QueryParser do{:error, error_message}end+ "segment" ->+ {:ok, filter_key}+_ ->{:error, error_message}enddiff --git a/priv/json-schemas/query-api-schema.json b/priv/json-schemas/query-api-schema.jsonindex df18503283..6a325c8d8e 100644--- a/priv/json-schemas/query-api-schema.json+++ b/priv/json-schemas/query-api-schema.json@@ -98,13 +98,11 @@"minItems": 2,"maxItems": 2,"items": {- "type": "string",- "format": "date"+ "type": "string",+ "format": "date"},"description": "If custom period. A list of two ISO8601 dates or timestamps to compare against.",- "examples": [- ["2024-01-01", "2024-01-31"]- ]+ "examples": [["2024-01-01", "2024-01-31"]]}},"required": ["mode", "date_range"],@@ -439,11 +437,35 @@}]},+ "filter_for_segment": {+ "type": "array",+ "additionalItems": false,+ "minItems": 3,+ "maxItems": 3,+ "items": [+ {+ "const": "is"+ },+ {+ "const": "segment"+ },+ {+ "type": "array",+ "items": {+ "type": ["integer"]+ }+ }+ ]+ },"filter_entry": {"oneOf": [{ "$ref": "#/definitions/filter_without_goals" },{ "$ref": "#/definitions/filter_with_goals" },- { "$ref": "#/definitions/filter_with_pattern" }+ { "$ref": "#/definitions/filter_with_pattern" },+ {+ "$ref": "#/definitions/filter_for_segment",+ "$comment": "only :internal"+ }]},"filter_tree": {diff --git a/test/plausible/segments/filters_test.exs b/test/plausible/segments/filters_test.exsnew file mode 100644index 0000000000..e19b58df73--- /dev/null+++ b/test/plausible/segments/filters_test.exs@@ -0,0 +1,5 @@+defmodule Plausible.Segments.FiltersTest do+ use ExUnit.Case, async: true++ doctest Plausible.Segments.Filters, import: true+enddiff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exsindex ba7c8e8b81..968d04fa0f 100644--- a/test/plausible/stats/query_parser_test.exs+++ b/test/plausible/stats/query_parser_test.exs@@ -2,10 +2,11 @@ defmodule Plausible.Stats.Filters.QueryParserTest douse Plausibleuse Plausible.DataCaseuse Plausible.Teams.Test+ import Plausible.Stats.Filters.QueryParser+ doctest Plausible.Stats.Filters.QueryParseralias Plausible.Stats.DateTimeRangealias Plausible.Stats.Filters- import Plausible.Stats.Filters.QueryParsersetup [:create_user, :create_site]@@ -2028,4 +2029,215 @@ defmodule Plausible.Stats.Filters.QueryParserTest do})endend++ describe "filtering with segments" do+ test "parsing fails when too many segments in query", %{+ user: user,+ site: site+ } do+ segments =+ insert_list(11, :segment,+ type: :site,+ owner: user,+ site: site,+ name: "any"+ )++ %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "filters" => [+ ["and", segments |> Enum.map(fn segment -> ["is", "segment", [segment.id]] end)]+ ]+ }+ |> check_error(+ site,+ "Invalid filters. You can only use up to 10 segment filters in a query.",+ :internal+ )+ end++ test "parsing fails when segment filter is used, but segment is from another site", %{+ site: site+ } do+ other_user = new_user()+ other_site = new_site(owner: other_user)++ segment =+ insert(:segment,+ type: :site,+ owner: other_user,+ site: other_site,+ name: "any"+ )++ %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "filters" => [["is", "segment", [segment.id]]]+ }+ |> check_error(+ site,+ "Invalid filters. Some segments don't exist or aren't accessible.",+ :internal+ )+ end++ test "hiding custom properties filters in segments doesn't allow bypasssing feature check",+ %{+ site: site,+ user: user+ } do+ subscribe_to_enterprise_plan(user, features: [Plausible.Billing.Feature.StatsAPI])++ segment =+ insert(:segment,+ type: :site,+ owner: user,+ site: site,+ name: "segment with custom props filter",+ segment_data: %{"filters" => [["is", "event:props:foobar", ["foo"]]]}+ )++ %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "filters" => [["is", "segment", [segment.id]]]+ }+ |> check_error(+ site,+ "The owner of this site does not have access to the custom properties feature.",+ :internal+ )+ end++ test "querying conversion rate is illegal if the complex event:goal filter is within a segment",+ %{+ site: site,+ user: user+ } do+ segment =+ insert(:segment,+ type: :site,+ owner: user,+ site: site,+ name: "any",+ segment_data: %{+ "filters" => [+ [+ "or",+ [+ ["is", "event:goal", ["Signup"]],+ ["contains", "event:page", ["/"]]+ ]+ ]+ ]+ }+ )++ %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "conversion_rate"],+ "date_range" => "all",+ "filters" => [["is", "segment", [segment.id]]]+ }+ |> check_error(+ site,+ "Invalid filters. Dimension `event:goal` can only be filtered at the top level.",+ :internal+ )+ end++ test "resolves segments correctly", %{site: site, user: user} do+ emea_segment =+ insert(:segment,+ type: :site,+ owner: user,+ site: site,+ name: "EMEA",+ segment_data: %{+ "filters" => [["is", "visit:country", ["FR", "DE"]]],+ "labels" => %{"FR" => "France", "DE" => "Germany"}+ }+ )++ apac_segment =+ insert(:segment,+ type: :site,+ owner: user,+ site: site,+ name: "APAC",+ segment_data: %{+ "filters" => [["is", "visit:country", ["AU", "NZ"]]],+ "labels" => %{"AU" => "Australia", "NZ" => "New Zealand"}+ }+ )++ firefox_segment =+ insert(:segment,+ type: :site,+ owner: user,+ site: site,+ name: "APAC",+ segment_data: %{+ "filters" => [+ ["is", "visit:browser", ["Firefox"]],+ ["is", "visit:os", ["Linux"]]+ ]+ }+ )++ %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "events"],+ "date_range" => "all",+ "filters" => [+ [+ "and",+ [+ ["is", "segment", [apac_segment.id, emea_segment.id]],+ ["is", "segment", [firefox_segment.id]]+ ]+ ]+ ]+ }+ |> check_success(+ site,+ %{+ metrics: [:visitors, :events],+ utc_time_range: @date_range_day,+ filters: [+ [+ :and,+ [+ [+ :or,+ [+ [:and, [[:is, "visit:country", ["AU", "NZ"]]]],+ [:and, [[:is, "visit:country", ["FR", "DE"]]]]+ ]+ ],+ [+ :and,+ [+ [:is, "visit:browser", ["Firefox"]],+ [:is, "visit:os", ["Linux"]]+ ]+ ]+ ]+ ]+ ],+ dimensions: [],+ order_by: nil,+ timezone: site.timezone,+ include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ pagination: %{limit: 10_000, offset: 0}+ },+ :internal+ )+ end+ endenddiff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exsindex 1681bb208f..4697b94c71 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs@@ -4577,4 +4577,110 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doassert "metric_warnings" not in json_response(conn, 200)["meta"]endend++ describe "segment filters" do+ setup [:create_user, :create_site, :create_api_key, :use_api_key]++ test "segment filters are (not yet) available in public API", %{conn: conn, site: site} do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "filters" => [["is", "segment", [1]]],+ "date_range" => "all",+ "metrics" => ["visitors"]+ })++ assert json_response(conn, 400) == %{+ "error" => "#/filters/0: Invalid filter [\"is\", \"segment\", [1]]"+ }+ end++ test "site segments of other sites don't resolve", %{+ conn: conn,+ site: site+ } do+ other_site = new_site()+ other_user = add_guest(other_site, role: :editor)++ segment =+ insert(:segment,+ name: "Segment of another site",+ type: :site,+ owner: other_user,+ site: other_site,+ segment_data: %{+ "filters" => [["is", "event:page", ["/blog"]]]+ }+ )++ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "filters" => [["is", "segment", [segment.id]]],+ "date_range" => "all",+ "metrics" => ["events"]+ })++ assert json_response(conn, 400) == %{+ "error" => "Invalid filters. Some segments don't exist or aren't accessible."+ }+ end++ test "even personal segments of other users of the same site resolve to filters, with segments expanded in response",+ %{+ conn: conn,+ site: site+ } do+ other_user = add_guest(site, role: :editor)++ segment =+ insert(:segment,+ type: :personal,+ owner: other_user,+ site: site,+ name: "Signups",+ segment_data: %{+ "filters" => [["is", "event:name", ["Signup"]]]+ }+ )++ insert(:goal, %{site: site, event_name: "Signup"})++ populate_stats(site, [+ build(:event,+ name: "Signup",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Signup",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Signup",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "AnyOtherEvent",+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "filters" => [["is", "segment", [segment.id]]],+ "date_range" => "all",+ "metrics" => ["events"]+ })++ assert json_response(conn, 200)["results"] == [%{"dimensions" => [], "metrics" => [3]}]++ # response shows what filters the segment was resolved to+ assert json_response(conn, 200)["query"]["filters"] == [+ ["and", [["is", "event:name", ["Signup"]]]]+ ]+ end+ endend