Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
fix: ignore imported data when querying goals with custom props (#6441)
Uku Taht37e352c73dc5
diff --git a/lib/plausible/stats/goals.ex b/lib/plausible/stats/goals.exindex 806635f350..7733233793 100644--- a/lib/plausible/stats/goals.ex+++ b/lib/plausible/stats/goals.ex@@ -65,8 +65,6 @@ defmodule Plausible.Stats.Goals do@type goal_join_data() :: %{indices: [non_neg_integer()],- types: [String.t()],- event_names_imports: [String.t()],event_names_by_type: [String.t()],page_regexes: [String.t()],scroll_thresholds: [non_neg_integer()],@@ -74,47 +72,61 @@ defmodule Plausible.Stats.Goals docustom_props_values: [[String.t()]]}+ @type goal_join_imported_data() :: %{+ indices: [non_neg_integer()],+ types: [String.t()],+ event_names: [String.t()],+ page_regexes: [String.t()]+ }+@doc """Returns data needed to perform a GROUP BY on goals in an ecto query."""@spec goal_join_data(Plausible.Stats.Query.t()) :: goal_join_data()def goal_join_data(query) do- goals = query.preloaded_goals.matching_toplevel_filters+ {goals, indices} =+ query.preloaded_goals.matching_toplevel_filters+ |> Enum.with_index(1)+ |> Enum.unzip()++ {custom_props_keys, custom_props_values} =+ goals+ |> Enum.map(fn goal -> Enum.unzip(goal.custom_props) end)+ |> Enum.unzip()++ %{+ indices: indices,+ event_names_by_type:+ Enum.map(goals, fn goal -> event_name_by_type(Plausible.Goal.type(goal), goal) end),+ # Event goals are considered to match everything for the sake of efficient queries in query_builder.ex+ # See also Plausible.Stats.SQL.Expression.event_goal_join/1+ page_regexes:+ Enum.map(goals, fn goal -> page_regex_for_goal(Plausible.Goal.type(goal), goal) end),+ scroll_thresholds: Enum.map(goals, fn goal -> goal.scroll_threshold end),+ custom_props_keys: custom_props_keys,+ custom_props_values: custom_props_values+ }+ end- goals- |> Enum.with_index(1)- |> Enum.reduce(- %{- indices: [],- types: [],- event_names_imports: [],- event_names_by_type: [],- page_regexes: [],- scroll_thresholds: [],- custom_props_keys: [],- custom_props_values: []- },- fn {goal, idx}, acc ->- goal_type = Plausible.Goal.type(goal)- {prop_keys, prop_values} = Enum.unzip(goal.custom_props)-- %{- indices: [idx | acc.indices],- types: [to_string(goal_type) | acc.types],- # This will contain "" for non-event goals- event_names_imports: [to_string(goal.event_name) | acc.event_names_imports],- event_names_by_type: [event_name_by_type(goal_type, goal) | acc.event_names_by_type],- # Event goals are considered to match everything for the sake of efficient queries in query_builder.ex- # See also Plausible.Stats.SQL.Expression.event_goal_join/1- page_regexes: [page_regex_for_goal(goal_type, goal) | acc.page_regexes],- scroll_thresholds: [goal.scroll_threshold | acc.scroll_thresholds],- custom_props_keys: [prop_keys | acc.custom_props_keys],- custom_props_values: [prop_values | acc.custom_props_values]- }- end- )- |> Enum.map(fn {key, list} -> {key, Enum.reverse(list)} end)- |> Map.new()+ @doc """+ Returns data needed to perform a GROUP BY on goals in an imported data query.+ """+ @spec goal_join_imported_data(Plausible.Stats.Query.t()) :: goal_join_imported_data()+ def goal_join_imported_data(query) do+ {goals, indices} =+ query.preloaded_goals.matching_toplevel_filters+ # Keep indices aligned with the complete preloaded goal list.+ |> Enum.with_index(1)+ |> Enum.reject(fn {goal, _idx} -> Plausible.Goal.has_custom_props?(goal) end)+ |> Enum.unzip()++ %{+ indices: indices,+ types: Enum.map(goals, fn goal -> goal |> Plausible.Goal.type() |> to_string() end),+ event_names: Enum.map(goals, fn goal -> to_string(goal.event_name) end),+ page_regexes:+ Enum.map(goals, fn goal -> page_regex_for_goal(Plausible.Goal.type(goal), goal) end)+ }enddefp event_name_by_type(:event, goal), do: goal.event_namediff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.exindex a8b485a252..ec0eda67a6 100644--- a/lib/plausible/stats/imported/base.ex+++ b/lib/plausible/stats/imported/base.ex@@ -142,13 +142,21 @@ defmodule Plausible.Stats.Imported.Base do|> Enum.any?(&(&1 not in [property, "event:name", "event:goal"]))if has_required_event_or_goal_name_filter? and- not has_unsupported_filters? do+ not has_unsupported_filters? and+ not any_custom_prop_goal_filters?(query) do["imported_custom_events"]else[]endend+ # Goals with custom props cannot be queried from imported data, as imported+ # tables are aggregated without custom properties.+ defp any_custom_prop_goal_filters?(query) do+ query.preloaded_goals.matching_toplevel_filters+ |> Enum.any?(&Plausible.Goal.has_custom_props?/1)+ end+defp do_decide_tables(%Query{filters: [], dimensions: []}), do: ["imported_visitors"]defp do_decide_tables(%Query{filters: [], dimensions: ["event:goal"]}) do@@ -172,6 +180,7 @@ defmodule Plausible.Stats.Imported.Base doEnum.any?(filter_dimensions, &(&1 not in ["event:page", "event:name", "event:goal"]))cond do+ any_custom_prop_goal_filters?(query) -> []any_other_filters? -> []any_event_name_filters? and not any_page_filters? -> ["imported_custom_events"]any_page_filters? and not any_event_name_filters? -> ["imported_pages"]@@ -192,11 +201,13 @@ defmodule Plausible.Stats.Imported.Base dofilter_goal_table_candidates =query.preloaded_goals.matching_toplevel_filters- |> Enum.map(&Plausible.Goal.type/1)- |> Enum.map(fn- :event -> "imported_custom_events"- :page -> "imported_pages"- :scroll -> nil+ |> Enum.map(fn goal ->+ case {Plausible.Goal.has_custom_props?(goal), Plausible.Goal.type(goal)} do+ {true, _} -> nil+ {false, :event} -> "imported_custom_events"+ {false, :page} -> "imported_pages"+ {false, :scroll} -> nil+ endend)case Enum.uniq(table_candidates ++ filter_goal_table_candidates) dodiff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.exindex caf3e5ae80..7b08db1e11 100644--- a/lib/plausible/stats/imported/imported.ex+++ b/lib/plausible/stats/imported/imported.ex@@ -240,7 +240,7 @@ defmodule Plausible.Stats.Imported doenddef merge_imported(q, site, %Query{dimensions: ["event:goal"]} = query) do- goal_join_data = Plausible.Stats.Goals.goal_join_data(query)+ goal_join_data = Plausible.Stats.Goals.goal_join_imported_data(query)Imported.Base.decide_tables(query)|> Enum.map(fn@@ -250,8 +250,9 @@ defmodule Plausible.Stats.Imported do|> select_merge_as([i], %{dim0:fragment(- "indexOf(?, ?)",- type(^goal_join_data.event_names_imports, {:array, :string}),+ "CAST(?[indexOf(?, ?)], 'UInt64')",+ type(^goal_join_data.indices, {:array, :integer}),+ type(^goal_join_data.event_names, {:array, :string}),i.name)})@@ -268,15 +269,17 @@ defmodule Plausible.Stats.Imported do"""notEmpty(arrayFilter(- goal_idx -> ?[goal_idx] = 'page' AND match(?, ?[goal_idx]),+ (goal_idx, type, page_regex) -> type = 'page' AND match(?, page_regex),+ ?,+ ?,?) as indices)""",- type(^goal_join_data.types, {:array, :string}),i.page,- type(^goal_join_data.page_regexes, {:array, :string}),- type(^goal_join_data.indices, {:array, :integer})+ type(^goal_join_data.indices, {:array, :integer}),+ type(^goal_join_data.types, {:array, :string}),+ type(^goal_join_data.page_regexes, {:array, :string})))|> join(:inner, [_i], index in fragment("indices"), hints: "ARRAY", on: true)diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_goal_custom_props_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_goal_custom_props_test.exsindex 360e00a642..9201731780 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_goal_custom_props_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_goal_custom_props_test.exs@@ -367,4 +367,235 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryGoalCustomPropsTest dorefute Enum.find(results, &(&1["dimensions"] == ["Button Click"]))endend++ describe "goals with custom props and imported data" do+ setup :create_site_import++ test "filtering by a goal with custom props excludes imported data instead of crashing", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ {:ok, _goal} =+ Goals.create(site, %{+ "event_name" => "Purchase",+ "custom_props" => %{"variant" => "A"}+ })++ populate_stats(site, site_import.id, [+ build(:event,+ name: "Purchase",+ "meta.key": ["variant"],+ "meta.value": ["A"],+ timestamp: ~N[2023-01-01 00:00:00]+ ),+ build(:imported_custom_events,+ name: "Purchase",+ visitors: 3,+ events: 5,+ date: ~D[2023-01-01]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors", "events"],+ "filters" => [["is", "event:goal", ["Purchase"]]],+ "include" => %{"imports" => true}+ })++ resp = json_response(conn, 200)++ # Imported data cannot be filtered by the goal's custom props,+ # so it must be excluded from the query.+ assert resp["results"] == [%{"dimensions" => [], "metrics" => [1, 1]}]+ refute resp["meta"]["imports_included"]+ end++ test "breakdown by event:goal excludes custom prop goals from imported data",+ %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ {:ok, _goal} = Goals.create(site, %{"event_name" => "Signup"})++ {:ok, _goal} =+ Goals.create(site, %{+ "event_name" => "Purchase",+ "custom_props" => %{"variant" => "A"}+ })++ populate_stats(site, site_import.id, [+ build(:event,+ name: "Purchase",+ "meta.key": ["variant"],+ "meta.value": ["A"],+ timestamp: ~N[2023-01-01 00:00:00]+ ),+ build(:imported_custom_events,+ name: "Purchase",+ visitors: 3,+ events: 5,+ date: ~D[2023-01-01]+ ),+ build(:imported_custom_events,+ name: "Signup",+ visitors: 3,+ events: 5,+ date: ~D[2023-01-01]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors", "events"],+ "dimensions" => ["event:goal"],+ "include" => %{"imports" => true}+ })++ resp = json_response(conn, 200)++ assert resp["results"] == [+ %{"dimensions" => ["Signup"], "metrics" => [3, 5]},+ %{"dimensions" => ["Purchase"], "metrics" => [1, 1]}+ ]+ end++ test "breakdown by event:goal filtered by a goal with custom props excludes imported data instead of crashing",+ %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ {:ok, _goal} =+ Goals.create(site, %{+ "event_name" => "Purchase",+ "custom_props" => %{"variant" => "A"}+ })++ populate_stats(site, site_import.id, [+ build(:event,+ name: "Purchase",+ "meta.key": ["variant"],+ "meta.value": ["A"],+ timestamp: ~N[2023-01-01 00:00:00]+ ),+ build(:imported_custom_events,+ name: "Purchase",+ visitors: 3,+ events: 5,+ date: ~D[2023-01-01]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors", "events"],+ "dimensions" => ["event:goal"],+ "filters" => [["is", "event:goal", ["Purchase"]]],+ "include" => %{"imports" => true}+ })++ resp = json_response(conn, 200)++ assert resp["results"] == [%{"dimensions" => ["Purchase"], "metrics" => [1, 1]}]+ refute resp["meta"]["imports_included"]+ end++ test "filtering by a custom property and a special goal with custom props excludes imported data instead of crashing",+ %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ {:ok, _goal} =+ Goals.create(site, %{+ "event_name" => "Outbound Link: Click",+ "custom_props" => %{"page_theme" => "dark"}+ })++ populate_stats(site, site_import.id, [+ build(:event,+ name: "Outbound Link: Click",+ "meta.key": ["url", "page_theme"],+ "meta.value": ["https://example.com", "dark"],+ timestamp: ~N[2023-01-01 00:00:00]+ ),+ build(:imported_custom_events,+ name: "Outbound Link: Click",+ link_url: "https://example.com",+ visitors: 3,+ events: 5,+ date: ~D[2023-01-01]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors", "events"],+ "filters" => [+ ["is", "event:goal", ["Outbound Link: Click"]],+ ["is", "event:props:url", ["https://example.com"]]+ ],+ "include" => %{"imports" => true}+ })++ resp = json_response(conn, 200)++ assert resp["results"] == [%{"dimensions" => [], "metrics" => [1, 1]}]+ refute resp["meta"]["imports_included"]+ end++ test "breakdown by event:goal does not attribute imported pageviews to a page goal with custom props",+ %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ {:ok, _goal} =+ Goals.create(site, %{+ "page_path" => "/checkout",+ "custom_props" => %{"variant" => "A"}+ })++ populate_stats(site, site_import.id, [+ build(:pageview,+ pathname: "/checkout",+ "meta.key": ["variant"],+ "meta.value": ["A"],+ timestamp: ~N[2023-01-01 00:00:00]+ ),+ build(:imported_pages,+ page: "/checkout",+ visitors: 3,+ pageviews: 5,+ date: ~D[2023-01-01]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors"],+ "dimensions" => ["event:goal"],+ "include" => %{"imports" => true}+ })++ resp = json_response(conn, 200)++ # Imported pageviews are aggregated by page only and cannot be checked+ # against the goal's custom props, so they must not be counted.+ assert resp["results"] == [%{"dimensions" => ["Visit /checkout"], "metrics" => [1]}]+ end+ endend