Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Filter out garbage props by `site.allowed_event_props` (#2907)
RobertJoonasdb359c2a6f83
diff --git a/lib/plausible/site.ex b/lib/plausible/site.exindex da226a02a4..661e83312c 100644--- a/lib/plausible/site.ex+++ b/lib/plausible/site.ex@@ -17,6 +17,7 @@ defmodule Plausible.Site dofield :locked, :booleanfield :stats_start_date, :datefield :native_stats_start_at, :naive_datetime+ field :allowed_event_props, {:array, :string}field :ingest_rate_limit_scale_seconds, :integer, default: 60# default is set via changeset/2@@ -160,6 +161,10 @@ defmodule Plausible.Site do)end+ def set_allowed_event_props(site, list) do+ change(site, allowed_event_props: list)+ end+def remove_imported_data(site) dochange(site, imported_data: nil)enddiff --git a/lib/plausible/site/admin.ex b/lib/plausible/site/admin.exindex 6333b1d7db..19f6994942 100644--- a/lib/plausible/site/admin.ex+++ b/lib/plausible/site/admin.ex@@ -41,6 +41,14 @@ defmodule Plausible.SiteAdmin dopublic: nil,owner: %{value: &get_owner_email/1},other_members: %{value: &get_other_members/1},+ allowed_event_props: %{+ value: fn site ->+ case site.allowed_event_props do+ nil -> ""+ list -> Enum.join(list, ", ")+ end+ end+ },limits: %{value: fn site ->case site.ingest_rate_limit_threshold do@@ -53,6 +61,40 @@ defmodule Plausible.SiteAdmin do]end+ def list_actions(_conn) do+ [+ set_allowed_event_props: %{+ inputs: [+ %{+ name: "props",+ title:+ "Insert comma separated property names (e.g: author, logged_in, url, ...). Submit a blank field to allow all property names",+ default: ""+ }+ ],+ name: "Allow only these custom properties",+ action: &set_allowed_props_for_site/3+ }+ ]+ end++ def set_allowed_props_for_site(_conn, [site], params) do+ props_list =+ case String.trim(params["props"]) do+ "" -> nil+ props -> String.split(props, ~r/\s*,\s*/)+ end++ Plausible.Site.set_allowed_event_props(site, props_list)+ |> Repo.update!()++ :ok+ end++ def set_allowed_props_for_site(_, _, _) do+ {:error, "Please select only one site for this action"}+ end+defp format_date(date) doTimex.format!(date, "{Mshort} {D}, {YYYY}")enddiff --git a/lib/plausible/stats/custom_props.ex b/lib/plausible/stats/custom_props.exindex ad87471b78..306bfa5fc0 100644--- a/lib/plausible/stats/custom_props.ex+++ b/lib/plausible/stats/custom_props.ex@@ -45,7 +45,14 @@ defmodule Plausible.Stats.CustomProps doselect: meta.key,distinct: true)+ |> maybe_allowed_props_only(site.allowed_event_props)|> ClickhouseRepo.all()endend++ def maybe_allowed_props_only(q, allowed_props) when is_list(allowed_props) do+ from [_e, m] in q, where: m.key in ^allowed_props+ end++ def maybe_allowed_props_only(q, nil), do: qenddiff --git a/lib/plausible/stats/filter_suggestions.ex b/lib/plausible/stats/filter_suggestions.exindex bc6701c0c7..078b798540 100644--- a/lib/plausible/stats/filter_suggestions.ex+++ b/lib/plausible/stats/filter_suggestions.ex@@ -133,18 +133,17 @@ defmodule Plausible.Stats.FilterSuggestions dodef filter_suggestions(site, query, "prop_key", filter_search) dofilter_query = if filter_search == nil, do: "%", else: "%#{filter_search}%"- q =- from(e in base_event_query(site, Query.remove_event_filters(query, [:props])),- inner_lateral_join: meta in "meta",- as: :meta,- select: meta.key,- where: fragment("? ilike ?", meta.key, ^filter_query),- group_by: meta.key,- order_by: [desc: fragment("count(*)")],- limit: 25- )-- ClickhouseRepo.all(q)+ from(e in base_event_query(site, Query.remove_event_filters(query, [:props])),+ inner_lateral_join: meta in "meta",+ as: :meta,+ select: meta.key,+ where: fragment("? ilike ?", meta.key, ^filter_query),+ group_by: meta.key,+ order_by: [desc: fragment("count(*)")],+ limit: 25+ )+ |> Plausible.Stats.CustomProps.maybe_allowed_props_only(site.allowed_event_props)+ |> ClickhouseRepo.all()|> wrap_suggestions()enddiff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exsindex a0d15e1fc3..102e6d7573 100644--- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs@@ -299,7 +299,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do]end- test "returns prop_names=nil when goal :member + property filter are applied at the same time",+ test "returns prop_names=[] when goal :member + property filter are applied at the same time",%{conn: conn,site: site@@ -331,6 +331,70 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest doassert [%{"prop_names" => []}] = json_response(conn, 200)end+ test "filters out garbage prop_names",+ %{+ conn: conn,+ site: site+ } do+ site =+ site+ |> Plausible.Site.set_allowed_event_props(["author"])+ |> Plausible.Repo.update!()++ populate_stats(site, [+ build(:event,+ name: "Payment",+ "meta.key": ["author"],+ "meta.value": ["Valdis"]+ ),+ build(:event,+ name: "Payment",+ "meta.key": ["Garbage"],+ "meta.value": ["321"]+ ),+ build(:event,+ name: "Payment",+ "meta.key": ["OnlyGarbage"],+ "meta.value": ["123"]+ )+ ])++ insert(:goal, %{site: site, event_name: "Payment"})++ filters = Jason.encode!(%{goal: "Payment"})+ conn = get(conn, "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}")++ assert [%{"prop_names" => ["author"]}] = json_response(conn, 200)+ end++ test "does not filter any prop names by default (when site.allowed_event_props is nil)",+ %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:event,+ name: "Payment",+ "meta.key": ["Garbage"],+ "meta.value": ["321"]+ ),+ build(:event,+ name: "Payment",+ "meta.key": ["OnlyGarbage"],+ "meta.value": ["123"]+ )+ ])++ insert(:goal, %{site: site, event_name: "Payment"})++ filters = Jason.encode!(%{goal: "Payment"})+ conn = get(conn, "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}")++ assert [%{"prop_names" => prop_names}] = json_response(conn, 200)+ assert "Garbage" in prop_names+ assert "OnlyGarbage" in prop_names+ end+test "can filter by multiple mixed goals", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview, pathname: "/"),diff --git a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exsindex be4004327f..47630c46e9 100644--- a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs@@ -278,6 +278,73 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do]end+ test "returns suggestions for prop key based on site.allowed_event_props list", %{+ conn: conn,+ site: site+ } do+ site =+ site+ |> Plausible.Site.set_allowed_event_props(["author"])+ |> Plausible.Repo.update!()++ populate_stats(site, [+ build(:pageview,+ "meta.key": ["author"],+ "meta.value": ["Uku Taht"],+ timestamp: ~N[2022-01-01 00:00:00]+ ),+ build(:pageview,+ "meta.key": ["garbage1"],+ "meta.value": ["somegarbage1"],+ timestamp: ~N[2022-01-01 00:00:00]+ ),+ build(:pageview,+ "meta.key": ["garbage2"],+ "meta.value": ["somegarbage2"],+ timestamp: ~N[2022-01-01 00:00:00]+ )+ ])++ conn =+ get(conn, "/api/stats/#{site.domain}/suggestions/prop_key?period=day&date=2022-01-01")++ assert json_response(conn, 200) == [+ %{"label" => "author", "value" => "author"}+ ]+ end++ test "does not filter out prop key suggestions by default (when site.allowed_event_props is nil)",+ %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview,+ "meta.key": ["author"],+ "meta.value": ["Uku Taht"],+ timestamp: ~N[2022-01-01 00:00:00]+ ),+ build(:pageview,+ "meta.key": ["garbage1"],+ "meta.value": ["somegarbage1"],+ timestamp: ~N[2022-01-01 00:00:00]+ ),+ build(:pageview,+ "meta.key": ["garbage2"],+ "meta.value": ["somegarbage2"],+ timestamp: ~N[2022-01-01 00:00:00]+ )+ ])++ conn =+ get(conn, "/api/stats/#{site.domain}/suggestions/prop_key?period=day&date=2022-01-01")++ suggestions = json_response(conn, 200)+ assert %{"label" => "author", "value" => "author"} in suggestions+ assert %{"label" => "garbage1", "value" => "garbage1"} in suggestions+ assert %{"label" => "garbage2", "value" => "garbage2"} in suggestions+ end+test "returns suggestions for prop value ordered by count, but (none) value is always first",%{conn: conn, site: site} dopopulate_stats(site, [