Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Limit Custom Properties access in Stats API (#3670)
RobertJoonas403f559b35cb
diff --git a/lib/plausible/props.ex b/lib/plausible/props.exindex 26ce57e840..919eb9e504 100644--- a/lib/plausible/props.ex+++ b/lib/plausible/props.ex@@ -16,6 +16,58 @@ defmodule Plausible.Props do@max_prop_value_length 2000def max_prop_value_length, do: @max_prop_value_length+ @internal_keys ~w(url path)+ @doc """+ Lists prop keys used internally.++ These props should be allowed by default, and should not be displayed in the+ props settings page. For example, `url` is a special prop key used for file+ downloads and outbound links. It doesn't make sense to remove this prop key+ from the allow list, or to suggest users to add this prop key.+ """+ def internal_keys, do: @internal_keys++ @doc """+ Returns the custom props allowed in queries for the given site. There are+ two factors deciding whether a custom property is allowed for a site.++ ### 1. Subscription plan including the props feature.++ Internally used keys (i.e. `#{inspect(@internal_keys)}`) are always allowed,+ even for plans that don't include props. For any other props, access to the+ Custom Properties feature is required.++ ### 2. The site having an `allowed_event_props` list configured.++ For customers with a configured `allowed_event_props` list, this function+ returns that list (+ internally used keys). That helps to filter out garbage+ props which people might not want to see in their dashboards.++ With the `bypass_setup?` boolean option you can override the requirement of+ the site having set up props in the `allowed_event_props` list. For example,+ this is currently used for fetching allowed properties in Stats API queries+ in order to ensure the props feature access.++ Since `allowed_event_props` was added after the props feature had already+ been used for a while, there are sites with `allowed_event_props = nil`. For+ those sites, all custom properties that exist in the database are allowed to+ be queried.+ """+ @spec allowed_for(Plausible.Site.t()) :: [prop()] | :all+ def allowed_for(site, opts \\ []) do+ site = Plausible.Repo.preload(site, :owner)+ internal_keys = Plausible.Props.internal_keys()+ props_enabled? = Plausible.Billing.Feature.Props.check_availability(site.owner) == :ok+ bypass_setup? = Keyword.get(opts, :bypass_setup?)++ cond do+ props_enabled? && is_nil(site.allowed_event_props) -> :all+ props_enabled? && bypass_setup? -> :all+ props_enabled? -> site.allowed_event_props ++ internal_keys+ true -> internal_keys+ end+ end+@spec allow(Plausible.Site.t(), [prop()] | prop()) ::{:ok, Plausible.Site.t()} | {:error, Ecto.Changeset.t()} | {:error, :upgrade_required}@doc """@@ -85,17 +137,6 @@ defmodule Plausible.Props doallow(site, props_to_allow)end- @internal_keys ~w(url path)- @doc """- Lists prop keys used internally.-- These props should be allowed by default, and should not be displayed in the- props settings page. For example, `url` is a special prop key used for file- downloads and outbound links. It doesn't make sense to remove this prop key- from the allow list, or to suggest users to add this prop key.- """- def internal_keys, do: @internal_keys-def ensure_prop_key_accessible(prop_key, user) doif prop_key in @internal_keys do:okdiff --git a/lib/plausible/stats/custom_props.ex b/lib/plausible/stats/custom_props.exindex d4bb9715b1..023b5c995f 100644--- a/lib/plausible/stats/custom_props.ex+++ b/lib/plausible/stats/custom_props.ex@@ -33,15 +33,15 @@ defmodule Plausible.Stats.CustomProps doselect: meta.key,distinct: true)- |> maybe_allowed_props_only(site.allowed_event_props)+ |> maybe_allowed_props_only(site)|> ClickhouseRepo.all()endend- def maybe_allowed_props_only(q, allowed_props) when is_list(allowed_props) do- props = allowed_props ++ Plausible.Props.internal_keys()- from [..., m] in q, where: m.key in ^props+ def maybe_allowed_props_only(q, site) do+ case Plausible.Props.allowed_for(site) do+ :all -> q+ allowed_props -> from [..., m] in q, where: m.key in ^allowed_props+ endend-- def maybe_allowed_props_only(q, nil), do: qenddiff --git a/lib/plausible/stats/filter_suggestions.ex b/lib/plausible/stats/filter_suggestions.exindex e701d84597..0e5d979867 100644--- a/lib/plausible/stats/filter_suggestions.ex+++ b/lib/plausible/stats/filter_suggestions.ex@@ -142,7 +142,7 @@ defmodule Plausible.Stats.FilterSuggestions doorder_by: [desc: fragment("count(*)")],limit: 25)- |> Plausible.Stats.CustomProps.maybe_allowed_props_only(site.allowed_event_props)+ |> Plausible.Stats.CustomProps.maybe_allowed_props_only(site)|> ClickhouseRepo.all()|> wrap_suggestions()enddiff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.exindex 3f19f0c6cb..aaeb395713 100644--- a/lib/plausible_web/controllers/api/external_stats_controller.ex+++ b/lib/plausible_web/controllers/api/external_stats_controller.ex@@ -5,18 +5,19 @@ defmodule PlausibleWeb.Api.ExternalStatsController doalias Plausible.Stats.{Query, CustomProps}def realtime_visitors(conn, _params) do- site = conn.assigns[:site]+ site = conn.assigns.sitequery = Query.from(site, %{"period" => "realtime"})json(conn, Plausible.Stats.Clickhouse.current_visitors(site, query))enddef aggregate(conn, params) do- site = conn.assigns[:site]+ site = Repo.preload(conn.assigns.site, :owner)with :ok <- validate_period(params),:ok <- validate_date(params),query <- Query.from(site, params),- {:ok, metrics} <- parse_and_validate_metrics(params, nil, query) do+ {:ok, metrics} <- parse_and_validate_metrics(params, nil, query),+ :ok <- ensure_custom_props_access(site, query) doresults =if params["compare"] == "previous_period" do{:ok, prev_query} = Plausible.Stats.Comparisons.compare(site, query, "previous_period")@@ -43,22 +44,20 @@ defmodule PlausibleWeb.Api.ExternalStatsController dojson(conn, %{results: results})else- {:error, msg} ->- conn- |> put_status(400)- |> json(%{error: msg})+ err_tuple -> send_json_error_response(conn, err_tuple)endenddef breakdown(conn, params) do- site = conn.assigns[:site]+ site = Repo.preload(conn.assigns.site, :owner)with :ok <- validate_period(params),:ok <- validate_date(params),{:ok, property} <- validate_property(params),query <- Query.from(site, params),{:ok, metrics} <- parse_and_validate_metrics(params, property, query),- {:ok, limit} <- validate_or_default_limit(params) do+ {:ok, limit} <- validate_or_default_limit(params),+ :ok <- ensure_custom_props_access(site, query, property) dopage = String.to_integer(Map.get(params, "page", "1"))results = Plausible.Stats.breakdown(site, query, property, metrics, {limit, page})@@ -75,10 +74,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController dojson(conn, %{results: results})else- {:error, msg} ->- conn- |> put_status(400)- |> json(%{error: msg})+ err_tuple -> send_json_error_response(conn, err_tuple)endend@@ -129,6 +125,35 @@ defmodule PlausibleWeb.Api.ExternalStatsController doendend+ @spec ensure_custom_props_access(Plausible.Site.t(), Query.t(), String.t() | nil) ::+ :ok | {:error, {402, String.t()}}+ defp ensure_custom_props_access(site, query, property \\ nil) do+ allowed_props = Plausible.Props.allowed_for(site, bypass_setup?: true)+ prop_filter = Query.get_filter_by_prefix(query, "event:props:")++ query_allowed? =+ case {prop_filter, property, allowed_props} do+ {_, _, :all} ->+ true++ {{"event:props:" <> prop, _}, _property, allowed_props} ->+ prop in allowed_props++ {_filter, "event:props:" <> prop, allowed_props} ->+ prop in allowed_props++ _ ->+ true+ end++ if query_allowed? do+ :ok+ else+ msg = "The owner of this site does not have access to the custom properties feature"+ {:error, {402, msg}}+ end+ end+defp validate_all_metrics(metrics, property, query) doEnum.reduce_while(metrics, [], fn metric, acc ->case validate_metric(metric, property, query) do@@ -172,21 +197,19 @@ defmodule PlausibleWeb.Api.ExternalStatsController doenddef timeseries(conn, params) do- site = conn.assigns[:site]+ site = Repo.preload(conn.assigns.site, :owner)with :ok <- validate_period(params),:ok <- validate_date(params),:ok <- validate_interval(params),query <- Query.from(site, params),- {:ok, metrics} <- parse_and_validate_metrics(params, nil, query) do+ {:ok, metrics} <- parse_and_validate_metrics(params, nil, query),+ :ok <- ensure_custom_props_access(site, query) dograph = Plausible.Stats.timeseries(site, query, metrics)json(conn, %{results: graph})else- {:error, msg} ->- conn- |> put_status(400)- |> json(%{error: msg})+ err_tuple -> send_json_error_response(conn, err_tuple)endend@@ -257,4 +280,16 @@ defmodule PlausibleWeb.Api.ExternalStatsController doenddefp validate_interval(_), do: :ok++ defp send_json_error_response(conn, {:error, {status, msg}}) do+ conn+ |> put_status(status)+ |> json(%{error: msg})+ end++ defp send_json_error_response(conn, {:error, msg}) do+ conn+ |> put_status(400)+ |> json(%{error: msg})+ endenddiff --git a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exsindex 8f7ff2ec78..38ecfc6441 100644--- a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs@@ -1,10 +1,48 @@defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest douse PlausibleWeb.ConnCaseimport Plausible.TestUtils+ alias Plausible.Billing.Featuresetup [:create_user, :create_new_site, :create_api_key, :use_api_key]@user_id 123+ describe "feature access" do+ test "cannot filter by a custom prop without access to the props feature", %{+ conn: conn,+ user: user,+ site: site+ } do+ ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)+ insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)++ conn =+ get(conn, "/api/v1/stats/aggregate", %{+ "site_id" => site.domain,+ "filters" => "event:props:author==Uku"+ })++ assert json_response(conn, 402)["error"] ==+ "The owner of this site does not have access to the custom properties feature"+ end++ test "can filter by an internal prop key without access to the props feature", %{+ conn: conn,+ user: user,+ site: site+ } do+ ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)+ insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)++ conn =+ get(conn, "/api/v1/stats/aggregate", %{+ "site_id" => site.domain,+ "filters" => "event:props:url==https://site.com"+ })++ assert json_response(conn, 200)["results"]+ end+ end+describe "param validation" dotest "validates that date can be parsed", %{conn: conn, site: site} doconn =diff --git a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exsindex eb603f1588..bde4094bd6 100644--- a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs@@ -1,10 +1,87 @@defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest douse PlausibleWeb.ConnCase+ alias Plausible.Billing.Feature@user_id 1231setup [:create_user, :create_new_site, :create_api_key, :use_api_key]+ describe "feature access" do+ test "cannot break down by a custom prop without access to the props feature", %{+ conn: conn,+ user: user,+ site: site+ } do+ ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)+ insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)++ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "site_id" => site.domain,+ "property" => "event:props:author"+ })++ assert json_response(conn, 402)["error"] ==+ "The owner of this site does not have access to the custom properties feature"+ end++ test "can break down by an internal prop key without access to the props feature", %{+ conn: conn,+ user: user,+ site: site+ } do+ ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)+ insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)++ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "site_id" => site.domain,+ "property" => "event:props:path"+ })++ assert json_response(conn, 200)["results"]+ end++ test "cannot filter by a custom prop without access to the props feature", %{+ conn: conn,+ user: user,+ site: site+ } do+ ep =+ insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)++ insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)++ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "site_id" => site.domain,+ "property" => "visit:source",+ "filters" => "event:props:author==Uku"+ })++ assert json_response(conn, 402)["error"] ==+ "The owner of this site does not have access to the custom properties feature"+ end++ test "can filter by an internal prop key without access to the props feature", %{+ conn: conn,+ user: user,+ site: site+ } do+ ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)+ insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)++ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "site_id" => site.domain,+ "property" => "visit:source",+ "filters" => "event:props:url==whatever"+ })++ assert json_response(conn, 200)["results"]+ end+ end+describe "param validation" dotest "validates that property is required", %{conn: conn, site: site} doconn =diff --git a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exsindex eb75e0c2a5..c086dfad26 100644--- a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs@@ -1,8 +1,50 @@defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest douse PlausibleWeb.ConnCase+ alias Plausible.Billing.Featuresetup [:create_user, :create_new_site, :create_api_key, :use_api_key]+ describe "feature access" do+ test "cannot filter by a custom prop without access to the props feature", %{+ conn: conn,+ user: user,+ site: site+ } do+ ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)+ insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)++ conn =+ get(conn, "/api/v1/stats/timeseries", %{+ "site_id" => site.domain,+ "period" => "month",+ "date" => "2021-01-01",+ "filters" => "event:props:author==Uku"+ })++ assert json_response(conn, 402)["error"] ==+ "The owner of this site does not have access to the custom properties feature"+ end++ test "can filter by an internal prop without access to the props feature", %{+ conn: conn,+ user: user,+ site: site+ } do+ ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)+ insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)++ conn =+ get(conn, "/api/v1/stats/timeseries", %{+ "site_id" => site.domain,+ "period" => "month",+ "date" => "2021-01-01",+ "filters" => "event:props:path==/404"+ })++ assert json_response(conn, 200)["results"]+ end+ end+describe "param validation" dotest "validates that date can be parsed", %{conn: conn, site: site} doconn =