Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Limit rate of requests for Stats/Sites API via Team directly (#5231)
Adrian Gruntkowskie07300e0c0c1
diff --git a/lib/plausible/auth/api_key.ex b/lib/plausible/auth/api_key.exindex 87d93cb064..b6834d32fb 100644--- a/lib/plausible/auth/api_key.ex+++ b/lib/plausible/auth/api_key.ex@@ -6,14 +6,13 @@ defmodule Plausible.Auth.ApiKey do@type t() :: %__MODULE__{}@required [:user_id, :name]- @optional [:key, :scopes, :hourly_request_limit]+ @optional [:key, :scopes]@hourly_request_limit on_ee(do: 600, else: 1_000_000)schema "api_keys" dofield :name, :stringfield :scopes, {:array, :string}, default: ["stats:read:*"]- field :hourly_request_limit, :integer, default: @hourly_request_limitfield :key, :string, virtual: truefield :key_hash, :string@@ -37,7 +36,7 @@ defmodule Plausible.Auth.ApiKey dodef update(schema, attrs \\ %{}) doschema- |> cast(attrs, [:name, :user_id, :scopes, :hourly_request_limit])+ |> cast(attrs, [:name, :user_id, :scopes])|> validate_required([:user_id, :name])enddiff --git a/lib/plausible/auth/api_key_admin.ex b/lib/plausible/auth/api_key_admin.exindex ff7f7094aa..d87d7ffdc3 100644--- a/lib/plausible/auth/api_key_admin.ex+++ b/lib/plausible/auth/api_key_admin.ex@@ -29,7 +29,6 @@ defmodule Plausible.Auth.ApiKeyAdmin doname: nil,key: %{create: :readonly, update: :hidden, help_text: @plaintext_key_help},key_prefix: %{create: :hidden, update: :readonly},- hourly_request_limit: %{default: 1000},scope: %{choices: [{"Stats API", ["stats:read:*"]}, {"Sites API", ["sites:provision:*"]}]},user_id: nil]diff --git a/lib/plausible/auth/auth.ex b/lib/plausible/auth/auth.exindex 02de141988..5771261bab 100644--- a/lib/plausible/auth/auth.ex+++ b/lib/plausible/auth/auth.ex@@ -175,8 +175,16 @@ defmodule Plausible.Auth doendend- @spec find_api_key(String.t()) :: {:ok, Auth.ApiKey.t()} | {:error, :invalid_api_key}- def find_api_key(raw_key) do+ @spec find_api_key(String.t(), Keyword.t()) ::+ {:ok, Auth.ApiKey.t() | %{api_key: Auth.ApiKey.t(), team: Teams.Team.t() | nil}}+ | {:error, :invalid_api_key | :missing_site_id}+ def find_api_key(raw_key, opts \\ []) do+ {team_scope, id} = Keyword.get(opts, :team_by, {nil, nil})++ find_api_key(raw_key, team_scope, id)+ end++ defp find_api_key(raw_key, nil, _) dohashed_key = Auth.ApiKey.do_hash(raw_key)query =@@ -193,6 +201,28 @@ defmodule Plausible.Auth doendend+ defp find_api_key(raw_key, :site, nil) do+ with {:ok, api_key} <- find_api_key(raw_key, nil, nil) do+ {:ok, %{api_key: api_key, team: nil}}+ end+ end++ defp find_api_key(raw_key, :site, domain) do+ with {:ok, api_key} <- find_api_key(raw_key, nil, nil) do+ team = find_team_by_site(domain)+ {:ok, %{api_key: api_key, team: team}}+ end+ end++ defp find_team_by_site(domain) do+ Repo.one(+ from s in Plausible.Site,+ inner_join: t in assoc(s, :team),+ where: s.domain == ^domain or s.domain_changed_from == ^domain,+ select: t+ )+ end+defp check_stats_api_available(user) docase Plausible.Teams.get_by_owner(user) do{:ok, team} ->diff --git a/lib/plausible/billing/billing.ex b/lib/plausible/billing/billing.exindex aedc193e0c..3931bbe580 100644--- a/lib/plausible/billing/billing.ex+++ b/lib/plausible/billing/billing.ex@@ -245,10 +245,6 @@ defmodule Plausible.Billing do)if plan do- owner_ids = Enum.map(team.owners, & &1.id)- api_keys = from(key in Plausible.Auth.ApiKey, where: key.user_id in ^owner_ids)- Repo.update_all(api_keys, set: [hourly_request_limit: plan.hourly_api_request_limit])-Repo.update_all(from(t in Teams.Team, where: t.id == ^team.id),set: [hourly_api_request_limit: plan.hourly_api_request_limit]diff --git a/lib/plausible_web/plugs/authorize_public_api.ex b/lib/plausible_web/plugs/authorize_public_api.exindex a9f094eb3a..4a7788d92b 100644--- a/lib/plausible_web/plugs/authorize_public_api.ex+++ b/lib/plausible_web/plugs/authorize_public_api.ex@@ -34,6 +34,8 @@ defmodule PlausibleWeb.Plugs.AuthorizePublicAPI doalias Plausible.Sitesalias PlausibleWeb.Api.Helpers, as: H+ require Logger+# Scopes permitted implicitly for every API key. Existing API keys# have _either_ `["stats:read:*"]` (the default) or `["sites:provision:*"]`# set as their valid scopes. We always consider implicit scopes as@@ -47,10 +49,11 @@ defmodule PlausibleWeb.Plugs.AuthorizePublicAPI dodef call(conn, _opts) dorequested_scope = Map.fetch!(conn.assigns, :api_scope)+ context = conn.assigns[:api_context]with {:ok, token} <- get_bearer_token(conn),- {:ok, api_key} <- Auth.find_api_key(token),- :ok <- check_api_key_rate_limit(api_key),+ {:ok, api_key, limit_key, hourly_limit} <- find_api_key(conn, token, context),+ :ok <- check_api_key_rate_limit(limit_key, hourly_limit),{:ok, conn} <- verify_by_scope(conn, api_key, requested_scope) doassign(conn, :current_user, api_key.user)else@@ -60,6 +63,53 @@ defmodule PlausibleWeb.Plugs.AuthorizePublicAPI do### Verification dispatched by scope+ defp find_api_key(conn, token, :site) do+ case Auth.find_api_key(token, team_by: {:site, conn.params["site_id"]}) do+ {:ok, %{api_key: api_key, team: nil}} ->+ {:ok, api_key, limit_key(api_key, nil), Auth.ApiKey.hourly_request_limit()}++ {:ok, %{api_key: api_key, team: team}} ->+ team_role_result = Plausible.Teams.Memberships.team_role(team, api_key.user)++ cond do+ Auth.is_super_admin?(api_key.user) ->+ :pass++ team_role_result == {:ok, :guest} ->+ Logger.warning(+ "[#{inspect(__MODULE__)}] API key #{api_key.id} user accessing #{conn.params["site_id"]} as a guest"+ )++ team_role_result == {:error, :not_a_member} ->+ Logger.warning(+ "[#{inspect(__MODULE__)}] API key #{api_key.id} user trying to access #{conn.params["site_id"]} as a non-member"+ )++ true ->+ :pass+ end++ {:ok, api_key, limit_key(api_key, team.identifier), team.hourly_api_request_limit}++ {:error, _} = error ->+ error+ end+ end++ defp find_api_key(_conn, token, _) do+ with {:ok, api_key} <- Auth.find_api_key(token) do+ {:ok, api_key, limit_key(api_key, nil), Auth.ApiKey.hourly_request_limit()}+ end+ end++ defp limit_key(api_key, nil) do+ "api_request:#{api_key.id}"+ end++ defp limit_key(api_key, team_id) do+ "api_request:#{api_key.id}:#{team_id}"+ end+defp verify_by_scope(conn, api_key, "stats:read:" <> _ = scope) dowith :ok <- check_scope(api_key, scope),{:ok, site} <- find_site(conn.params["site_id"]),@@ -107,14 +157,10 @@ defmodule PlausibleWeb.Plugs.AuthorizePublicAPI doendend- defp check_api_key_rate_limit(api_key) do- case RateLimit.check_rate(- "api_request:#{api_key.id}",- to_timeout(hour: 1),- api_key.hourly_request_limit- ) do+ defp check_api_key_rate_limit(limit_key, hourly_limit) do+ case RateLimit.check_rate(limit_key, to_timeout(hour: 1), hourly_limit) do{:allow, _} -> :ok- {:deny, _} -> {:error, :rate_limit, api_key.hourly_request_limit}+ {:deny, _} -> {:error, :rate_limit, hourly_limit}endenddiff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.exindex ebde84bfcd..a05ad16357 100644--- a/lib/plausible_web/router.ex+++ b/lib/plausible_web/router.ex@@ -227,7 +227,8 @@ defmodule PlausibleWeb.Router doendend- scope "/api/v1/stats", PlausibleWeb.Api, assigns: %{api_scope: "stats:read:*"} do+ scope "/api/v1/stats", PlausibleWeb.Api,+ assigns: %{api_scope: "stats:read:*", api_context: :site} dopipe_through [:public_api, PlausibleWeb.Plugs.AuthorizePublicAPI]get "/realtime/visitors", ExternalStatsController, :realtime_visitors@@ -236,7 +237,8 @@ defmodule PlausibleWeb.Router doget "/timeseries", ExternalStatsController, :timeseriesend- scope "/api/v2", PlausibleWeb.Api, assigns: %{api_scope: "stats:read:*", schema_type: :public} do+ scope "/api/v2", PlausibleWeb.Api,+ assigns: %{api_scope: "stats:read:*", api_context: :site, schema_type: :public} dopipe_through [:public_api, PlausibleWeb.Plugs.AuthorizePublicAPI]post "/query", ExternalQueryApiController, :query@@ -267,25 +269,31 @@ defmodule PlausibleWeb.Router doget "/", ExternalSitesController, :indexget "/teams", ExternalSitesController, :teams_index- get "/goals", ExternalSitesController, :goals_index- get "/guests", ExternalSitesController, :guests_index- get "/:site_id", ExternalSitesController, :get_site++ scope assigns: %{api_context: :site} do+ get "/goals", ExternalSitesController, :goals_index+ get "/guests", ExternalSitesController, :guests_index+ get "/:site_id", ExternalSitesController, :get_site+ endendscope assigns: %{api_scope: "sites:provision:*"} dopipe_through PlausibleWeb.Plugs.AuthorizePublicAPIpost "/", ExternalSitesController, :create_site- put "/shared-links", ExternalSitesController, :find_or_create_shared_link- put "/goals", ExternalSitesController, :find_or_create_goal- delete "/goals/:goal_id", ExternalSitesController, :delete_goal+ scope assigns: %{api_context: :site} do+ put "/shared-links", ExternalSitesController, :find_or_create_shared_link++ put "/goals", ExternalSitesController, :find_or_create_goal+ delete "/goals/:goal_id", ExternalSitesController, :delete_goal- put "/guests", ExternalSitesController, :find_or_create_guest- delete "/guests/:email", ExternalSitesController, :delete_guest+ put "/guests", ExternalSitesController, :find_or_create_guest+ delete "/guests/:email", ExternalSitesController, :delete_guest- put "/:site_id", ExternalSitesController, :update_site- delete "/:site_id", ExternalSitesController, :delete_site+ put "/:site_id", ExternalSitesController, :update_site+ delete "/:site_id", ExternalSitesController, :delete_site+ endendendenddiff --git a/test/plausible/auth/auth_test.exs b/test/plausible/auth/auth_test.exsindex 28914857d3..3d72c134a6 100644--- a/test/plausible/auth/auth_test.exs+++ b/test/plausible/auth/auth_test.exs@@ -25,26 +25,6 @@ defmodule Plausible.AuthTest doassert {:ok, %Auth.ApiKey{}} = Auth.create_api_key(user, "my new key", key)end- @tag :ee_only- test "defaults to 600 requests per hour limit in EE" do- user = new_user(trial_expiry_date: Date.utc_today())-- {:ok, %Auth.ApiKey{hourly_request_limit: hourly_request_limit}} =- Auth.create_api_key(user, "my new EE key", Ecto.UUID.generate())-- assert hourly_request_limit == 600- end-- @tag :ce_build_only- test "defaults to 1000000 requests per hour limit in CE" do- user = new_user(trial_expiry_date: Date.utc_today())-- {:ok, %Auth.ApiKey{hourly_request_limit: hourly_request_limit}} =- Auth.create_api_key(user, "my new CE key", Ecto.UUID.generate())-- assert hourly_request_limit == 1_000_000- end-test "errors when key already exists" dou1 = new_user(trial_expiry_date: Date.utc_today())u2 = new_user(trial_expiry_date: Date.utc_today())diff --git a/test/plausible/billing/billing_test.exs b/test/plausible/billing/billing_test.exsindex a5149d28ca..1a51b196b2 100644--- a/test/plausible/billing/billing_test.exs+++ b/test/plausible/billing/billing_test.exs@@ -289,14 +289,13 @@ defmodule Plausible.BillingTest doteam = team_of(user)- api_key = insert(:api_key, user: user, hourly_request_limit: 1)+ _api_key = insert(:api_key, user: user)assert team.hourly_api_request_limit != 10_000%{@subscription_created_params | "passthrough" => "ee:true;user:#{user.id};team:#{team.id}"}|> Billing.subscription_created()- assert Repo.reload!(api_key).hourly_request_limit == 10_000assert Repo.reload!(team).hourly_api_request_limit == 10_000endend@@ -420,7 +419,7 @@ defmodule Plausible.BillingTest doteam = team_of(user)- api_key = insert(:api_key, user: user, hourly_request_limit: 1)+ _api_key = insert(:api_key, user: user)assert team.hourly_api_request_limit != 10_000@@ -432,7 +431,6 @@ defmodule Plausible.BillingTest do})|> Billing.subscription_updated()- assert Repo.reload!(api_key).hourly_request_limit == 10_000assert Repo.reload!(team).hourly_api_request_limit == 10_000enddiff --git a/test/plausible_web/controllers/api/external_sites_controller_test.exs b/test/plausible_web/controllers/api/external_sites_controller_test.exsindex 5cc2d3e1f9..61fe3f1227 100644--- a/test/plausible_web/controllers/api/external_sites_controller_test.exs+++ b/test/plausible_web/controllers/api/external_sites_controller_test.exs@@ -235,6 +235,7 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest doassert json_response(conn, 404) == %{"error" => "Site could not be found"}end+ @tag :capture_logtest "cannot delete a site that the user does not own", %{conn: conn, user: user} dosite = new_site()add_guest(site, user: user, role: :editor)@@ -332,6 +333,7 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest doassert res["error"] == "Site could not be found"end+ @tag :capture_logtest "returns 404 when api key owner does not have permissions to create a shared link", %{conn: conn,user: user@@ -480,6 +482,7 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest doassert res["error"] == "Site could not be found"end+ @tag :capture_logtest "returns 404 when api key owner does not have permissions to create a goal", %{conn: conn,user: user@@ -586,6 +589,7 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest doassert json_response(conn, 404) == %{"error" => "Goal could not be found"}end+ @tag :capture_logtest "cannot delete a goal belongs to a site that the user does not own", %{conn: conn,user: user@@ -1097,6 +1101,7 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do}end+ @tag :capture_logtest "returns goals for site where user is viewer", %{site: site} doviewer = new_user()add_guest(site, user: viewer, role: :viewer)diff --git a/test/plausible_web/controllers/api/external_stats_controller/auth_test.exs b/test/plausible_web/controllers/api/external_stats_controller/auth_test.exsindex 016bee9d98..2783fee452 100644--- a/test/plausible_web/controllers/api/external_stats_controller/auth_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/auth_test.exs@@ -100,23 +100,30 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AuthTest doendtest "limits the rate of API requests", %{user: user} do- api_key = insert(:api_key, user_id: user.id, hourly_request_limit: 3)+ site = new_site(owner: user)++ user+ |> team_of()+ |> Ecto.Changeset.change(hourly_api_request_limit: 3)+ |> Plausible.Repo.update!()++ api_key = insert(:api_key, user_id: user.id)build_conn()|> with_api_key(api_key.key)- |> get("/api/v1/stats/aggregate")+ |> get("/api/v1/stats/aggregate", %{"site_id" => site.domain})build_conn()|> with_api_key(api_key.key)- |> get("/api/v1/stats/aggregate")+ |> get("/api/v1/stats/aggregate", %{"site_id" => site.domain})build_conn()|> with_api_key(api_key.key)- |> get("/api/v1/stats/aggregate")+ |> get("/api/v1/stats/aggregate", %{"site_id" => site.domain})build_conn()|> with_api_key(api_key.key)- |> get("/api/v1/stats/aggregate")+ |> get("/api/v1/stats/aggregate", %{"site_id" => site.domain})|> assert_error(429,"Too many API requests. Your API key is limited to 3 requests per hour."diff --git a/test/plausible_web/controllers/api/stats_controller/sources_test.exs b/test/plausible_web/controllers/api/stats_controller/sources_test.exsindex a8b1e761e3..1140906c7f 100644--- a/test/plausible_web/controllers/api/stats_controller/sources_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/sources_test.exs@@ -1805,7 +1805,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest doassert %{"error_code" => "unsupported_filters"} = json_response(conn, 422)end- @tag :capure_log+ @tag :capture_logtest "returns 502 when Google API responds with an unexpected error", %{conn: conn,site: sitediff --git a/test/plausible_web/plugs/authorize_public_api_test.exs b/test/plausible_web/plugs/authorize_public_api_test.exsindex d832df6681..ec6cb8c9a0 100644--- a/test/plausible_web/plugs/authorize_public_api_test.exs+++ b/test/plausible_web/plugs/authorize_public_api_test.exs@@ -2,6 +2,8 @@ defmodule PlausibleWeb.Plugs.AuthorizePublicAPITest douse PlausibleWeb.ConnCase, async: falseuse Plausible.Teams.Test+ import ExUnit.CaptureLog+alias PlausibleWeb.Plugs.AuthorizePublicAPIsetup %{conn: conn} do@@ -171,14 +173,17 @@ defmodule PlausibleWeb.Plugs.AuthorizePublicAPITest doendtest "halts with error when API rate limit hit", %{conn: conn} do- user = insert(:user)- api_key = insert(:api_key, user: user, hourly_request_limit: 1)+ user = new_user(team: [hourly_api_request_limit: 1])+ site = new_site(owner: user)++ api_key = insert(:api_key, user: user)conn =conn|> put_req_header("authorization", "Bearer #{api_key.key}")- |> get("/")+ |> get("/?site_id=#{site.domain}")|> assign(:api_scope, "sites:read:*")+ |> assign(:api_context, :site)first_resp = AuthorizePublicAPI.call(conn, nil)second_resp = AuthorizePublicAPI.call(conn, nil)@@ -188,6 +193,64 @@ defmodule PlausibleWeb.Plugs.AuthorizePublicAPITest doassert json_response(second_resp, 429)["error"] =~ "Too many API requests."end+ test "does not rate limit when not in applicable context", %{conn: conn} do+ user = new_user(team: [hourly_api_request_limit: 1])+ site = new_site(owner: user)++ api_key = insert(:api_key, user: user)++ conn =+ conn+ |> put_req_header("authorization", "Bearer #{api_key.key}")+ |> get("/?site_id=#{site.domain}")+ |> assign(:api_scope, "sites:read:*")++ first_resp = AuthorizePublicAPI.call(conn, nil)+ second_resp = AuthorizePublicAPI.call(conn, nil)++ refute first_resp.halted+ refute second_resp.halted+ end++ test "logs a warning on using API key against site with guest access", %{conn: conn} do+ user = new_user(team: [hourly_api_request_limit: 1])+ site = new_site()+ add_guest(site, user: user, role: :editor)++ api_key = insert(:api_key, user: user)++ capture_log(fn ->+ conn =+ conn+ |> put_req_header("authorization", "Bearer #{api_key.key}")+ |> get("/?site_id=#{site.domain}")+ |> assign(:api_scope, "sites:read:*")+ |> assign(:api_context, :site)+ |> AuthorizePublicAPI.call(nil)++ refute conn.halted+ end) =~ "API key #{api_key.id} user accessing #{site.domain} as a guest"+ end++ test "logs a warning on using API key against site with no access", %{conn: conn} do+ user = new_user(team: [hourly_api_request_limit: 1])+ site = new_site()++ api_key = insert(:api_key, user: user)++ capture_log(fn ->+ conn =+ conn+ |> put_req_header("authorization", "Bearer #{api_key.key}")+ |> get("/?site_id=#{site.domain}")+ |> assign(:api_scope, "stats:read:*")+ |> assign(:api_context, :site)+ |> AuthorizePublicAPI.call(nil)++ assert conn.halted+ end) =~ "API key #{api_key.id} user trying to access #{site.domain} as a non-member"+ end+test "passes and sets current user when valid API key with required scope provided", %{conn: conn} do