Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Refactor rate limit module (#6752)
Artur Pata535a4fc1b079
diff --git a/config/ce.exs b/config/ce.exsindex fb4cecbcfd..61c760924e 100644--- a/config/ce.exs+++ b/config/ce.exs@@ -16,7 +16,7 @@ config :esbuild,env: %{"NODE_PATH" => Path.expand("../deps", __DIR__)}]-config :plausible, Plausible.Auth.ApiKey,+config :plausible, PlausibleWeb.Api.RateLimit,legacy_per_user_hourly_request_limit: 1_000_000,burst_request_limit: 1_000_000,burst_period_seconds: 10diff --git a/config/config.exs b/config/config.exsindex cb0490676b..00ede68296 100644--- a/config/config.exs+++ b/config/config.exs@@ -88,7 +88,7 @@ config :sentry,config :prom_ex, :storage_adapter, Plausible.PromEx.StripedPeepconfig :peep, :bucket_calculator, Plausible.PromEx.Buckets-config :plausible, Plausible.Auth.ApiKey,+config :plausible, PlausibleWeb.Api.RateLimit,legacy_per_user_hourly_request_limit: 600,burst_request_limit: 60,burst_period_seconds: 10diff --git a/lib/plausible/auth/api_key.ex b/lib/plausible/auth/api_key.exindex 80334e38d0..c48709fa3c 100644--- a/lib/plausible/auth/api_key.ex+++ b/lib/plausible/auth/api_key.ex@@ -30,7 +30,6 @@ defmodule Plausible.Auth.ApiKey doOnly team members can use team-scoped keys."""- use Plausibleuse Ecto.Schemaimport Ecto.Changeset@@ -39,8 +38,6 @@ defmodule Plausible.Auth.ApiKey do@required [:user_id, :name]@optional [:key, :scopes]- @default_hourly_request_limit_per_team on_ee(do: 600, else: 1_000_000)-schema "api_keys" dofield :name, :stringfield :scopes, {:array, :string}, default: ["stats:read:*"]@@ -57,28 +54,6 @@ defmodule Plausible.Auth.ApiKey dotimestamps()end- defp config(), do: Application.fetch_env!(:plausible, __MODULE__)-- def default_hourly_request_limit(), do: @default_hourly_request_limit_per_team- def limit_key(team), do: "api_request:team:#{team.identifier}"-- def legacy_hourly_request_limit() do- config()- |> Keyword.fetch!(:legacy_per_user_hourly_request_limit)- end-- def legacy_limit_key(user), do: "api_request:legacy_user:#{user.id}"-- def burst_request_limit(),- do:- config()- |> Keyword.fetch!(:burst_request_limit)-- def burst_period_seconds(),- do:- config()- |> Keyword.fetch!(:burst_period_seconds)-def changeset(struct, team, attrs) when not is_nil(team) dostruct|> cast(attrs, @required ++ @optional)diff --git a/lib/plausible/teams/team.ex b/lib/plausible/teams/team.exindex 3eb615269b..829c16a4c9 100644--- a/lib/plausible/teams/team.ex+++ b/lib/plausible/teams/team.ex@@ -12,8 +12,6 @@ defmodule Plausible.Teams.Team douse Ecto.Schemause Plausible- alias Plausible.Auth-import Ecto.Changeset@type t() :: %__MODULE__{}@@ -28,6 +26,8 @@ defmodule Plausible.Teams.Team do@suggested_name_suffix "'s team"@suggested_name_fallback "My team"+ @default_hourly_api_request_limit on_ee(do: 600, else: 1_000_000)+on_ee do@derive {Plausible.Audit.Encoder,only: [@@ -57,7 +57,7 @@ defmodule Plausible.Teams.Team dofield :setup_at, :naive_datetime# Field synced from current subscription plan, if any. The value of this field is treated as the source of truth when out of sync- field :hourly_api_request_limit, :integer, default: Auth.ApiKey.default_hourly_request_limit()+ field :hourly_api_request_limit, :integer, default: @default_hourly_api_request_limit# Field for purely informational purposes in CRM contextfield :notes, :string@@ -230,6 +230,8 @@ defmodule Plausible.Teams.Team doendend+ def default_hourly_api_request_limit(), do: @default_hourly_api_request_limit+def trial_accept_traffic_until_offset_days(), do: @trial_accept_traffic_until_offset_daysdef subscription_accept_traffic_until_offset_days(),diff --git a/lib/plausible_web/controllers/api/rate_limit.ex b/lib/plausible_web/controllers/api/rate_limit.exnew file mode 100644index 0000000000..51dfd6d2c3--- /dev/null+++ b/lib/plausible_web/controllers/api/rate_limit.ex@@ -0,0 +1,68 @@+defmodule PlausibleWeb.Api.RateLimit do+ @moduledoc """+ Rate limits are imposed on API callers to prevent overloading the server.+ """++ defp config(), do: Application.fetch_env!(:plausible, __MODULE__)++ def limit_key(team), do: "api_request:team:#{team.identifier}"++ def legacy_hourly_request_limit() do+ config()+ |> Keyword.fetch!(:legacy_per_user_hourly_request_limit)+ end++ def legacy_limit_key(user), do: "api_request:legacy_user:#{user.id}"++ def burst_request_limit(),+ do:+ config()+ |> Keyword.fetch!(:burst_request_limit)++ def burst_period_seconds(),+ do:+ config()+ |> Keyword.fetch!(:burst_period_seconds)++ @doc """+ Checks a request against both limits.++ The message in `{:error, :rate_limit, message}` is meant to be shown to the+ caller.+ """+ @spec check_rate_limit(String.t(), pos_integer()) :: :ok | {:error, :rate_limit, String.t()}+ def check_rate_limit(limit_key, hourly_limit) do+ with :ok <- check_hourly_limit(limit_key, hourly_limit) do+ check_burst_limit(limit_key)+ end+ end++ defp check_hourly_limit(limit_key, hourly_limit) do+ case Plausible.RateLimit.check_rate(limit_key, to_timeout(hour: 1), hourly_limit) do+ {:allow, _} ->+ :ok++ {:deny, _} ->+ {:error, :rate_limit,+ "Too many API requests. The limit is #{hourly_limit} per hour. Please contact us to request more capacity."}+ end+ end++ defp check_burst_limit(limit_key) do+ burst_period_seconds = burst_period_seconds()+ burst_request_limit = burst_request_limit()++ case Plausible.RateLimit.check_rate(+ limit_key,+ to_timeout(second: burst_period_seconds),+ burst_request_limit+ ) do+ {:allow, _} ->+ :ok++ {:deny, _} ->+ {:error, :rate_limit,+ "Too many API requests in a short period of time. The limit is #{burst_request_limit} per #{burst_period_seconds} seconds. Please throttle your requests."}+ end+ end+enddiff --git a/lib/plausible_web/plugs/authorize_public_api.ex b/lib/plausible_web/plugs/authorize_public_api.exindex 9dde8e3b3b..142c91f5bd 100644--- a/lib/plausible_web/plugs/authorize_public_api.ex+++ b/lib/plausible_web/plugs/authorize_public_api.ex@@ -30,9 +30,9 @@ defmodule PlausibleWeb.Plugs.AuthorizePublicAPI doimport Plug.Connalias Plausible.Auth- alias Plausible.RateLimitalias Plausible.Teamsalias PlausibleWeb.Api.Helpers, as: H+ alias PlausibleWeb.Api.RateLimitrequire Logger@@ -53,8 +53,7 @@ defmodule PlausibleWeb.Plugs.AuthorizePublicAPI dowith {:ok, token} <- get_bearer_token(conn),{:ok, api_key, limit_key, hourly_limit} <- find_api_key(conn, token, context),- :ok <- check_api_key_rate_limit(limit_key, hourly_limit),- :ok <- check_api_key_burst_limit(limit_key),+ :ok <- RateLimit.check_rate_limit(limit_key, hourly_limit),{:ok, conn} <- verify_by_scope(conn, api_key, requested_scope) doconn|> assign(:current_user, api_key.user)@@ -69,8 +68,8 @@ defmodule PlausibleWeb.Plugs.AuthorizePublicAPI dodefp find_api_key(conn, token, :site) docase Auth.find_api_key_for_team_of_site(token, conn.params["site_id"]) do{:ok, %{api_key: api_key, team: nil}} ->- {:ok, api_key, Auth.ApiKey.legacy_limit_key(api_key.user),- Auth.ApiKey.legacy_hourly_request_limit()}+ {:ok, api_key, RateLimit.legacy_limit_key(api_key.user),+ RateLimit.legacy_hourly_request_limit()}{:ok, %{api_key: api_key, team: team}} ->team_role_result = Plausible.Teams.Memberships.team_role(team, api_key.user)@@ -93,7 +92,7 @@ defmodule PlausibleWeb.Plugs.AuthorizePublicAPI do:passend- {:ok, api_key, Auth.ApiKey.limit_key(team), team.hourly_api_request_limit}+ {:ok, api_key, RateLimit.limit_key(team), team.hourly_api_request_limit}{:error, _} = error ->error@@ -103,11 +102,11 @@ defmodule PlausibleWeb.Plugs.AuthorizePublicAPI dodefp find_api_key(_conn, token, _) docase Auth.find_api_key(token) do{:ok, %{api_key: api_key, team: nil}} ->- {:ok, api_key, Auth.ApiKey.legacy_limit_key(api_key.user),- Auth.ApiKey.legacy_hourly_request_limit()}+ {:ok, api_key, RateLimit.legacy_limit_key(api_key.user),+ RateLimit.legacy_hourly_request_limit()}{:ok, %{api_key: api_key, team: team}} ->- {:ok, api_key, Auth.ApiKey.limit_key(team), team.hourly_api_request_limit}+ {:ok, api_key, RateLimit.limit_key(team), team.hourly_api_request_limit}{:error, _} = error ->error@@ -184,35 +183,6 @@ defmodule PlausibleWeb.Plugs.AuthorizePublicAPI doendend- defp check_api_key_rate_limit(limit_key, hourly_limit) do- case RateLimit.check_rate(limit_key, to_timeout(hour: 1), hourly_limit) do- {:allow, _} ->- :ok-- {:deny, _} ->- {:error, :rate_limit,- "Too many API requests. The limit is #{hourly_limit} per hour. Please contact us to request more capacity."}- end- end-- defp check_api_key_burst_limit(limit_key) do- burst_period_seconds = Auth.ApiKey.burst_period_seconds()- burst_request_limit = Auth.ApiKey.burst_request_limit()-- case RateLimit.check_rate(- limit_key,- to_timeout(second: burst_period_seconds),- burst_request_limit- ) do- {:allow, _} ->- :ok-- {:deny, _} ->- {:error, :rate_limit,- "Too many API requests in a short period of time. The limit is #{burst_request_limit} per #{burst_period_seconds} seconds. Please throttle your requests."}- end- end-defp maybe_verify_site_access(conn, api_key, feature) docase find_site(conn.params["site_id"]) do{:ok, site} ->diff --git a/test/plausible/data_migration/backfill_teams_hourly_request_limit_test.exs b/test/plausible/data_migration/backfill_teams_hourly_request_limit_test.exsindex 400c972ddc..2f1a795952 100644--- a/test/plausible/data_migration/backfill_teams_hourly_request_limit_test.exs+++ b/test/plausible/data_migration/backfill_teams_hourly_request_limit_test.exs@@ -3,9 +3,9 @@ defmodule Plausible.DataMigration.BackfillTeamsHourlyRequestLimitTest doimport ExUnit.CaptureIO- alias Plausible.Authalias Plausible.DataMigration.BackfillTeamsHourlyRequestLimitalias Plausible.Repo+ alias Plausible.Teams.Teamdescribe "run/1" dotest "runs for empty dataset" do@@ -60,13 +60,13 @@ defmodule Plausible.DataMigration.BackfillTeamsHourlyRequestLimitTest doassert Repo.reload(team1).hourly_api_request_limit == 5000assert Repo.reload(team2).hourly_api_request_limit ==- Auth.ApiKey.default_hourly_request_limit()+ Team.default_hourly_api_request_limit()assert Repo.reload(team3).hourly_api_request_limit ==- Auth.ApiKey.default_hourly_request_limit()+ Team.default_hourly_api_request_limit()assert Repo.reload(team4).hourly_api_request_limit ==- Auth.ApiKey.default_hourly_request_limit()+ Team.default_hourly_api_request_limit()endendenddiff --git a/test/plausible_web/plugs/authorize_public_api_test.exs b/test/plausible_web/plugs/authorize_public_api_test.exsindex a7aa215d25..f9f17c7b2e 100644--- a/test/plausible_web/plugs/authorize_public_api_test.exs+++ b/test/plausible_web/plugs/authorize_public_api_test.exs@@ -212,7 +212,7 @@ defmodule PlausibleWeb.Plugs.AuthorizePublicAPITest douser = new_user()api_key = insert_api_key(unquote(key_type), user: user, scopes: ["sites:read:*"])- limit = Plausible.Auth.ApiKey.burst_request_limit()+ limit = PlausibleWeb.Api.RateLimit.burst_request_limit()# The limiter uses a fixed time window, so a client straddling a window# boundary can legitimately get up to `2 * limit` requests through in@@ -245,7 +245,7 @@ defmodule PlausibleWeb.Plugs.AuthorizePublicAPITest do:team_scope_api_key -> {1, 100}end- patch_env(Plausible.Auth.ApiKey,+ patch_env(PlausibleWeb.Api.RateLimit,legacy_per_user_hourly_request_limit: legacy_hourly_limit,# relax burst request limit to check hourly request limitburst_request_limit: 1_000,@@ -352,7 +352,7 @@ defmodule PlausibleWeb.Plugs.AuthorizePublicAPITest doendtest "legacy API key requests are rate limited to configured requests per hour _per user_, but their team's team-scoped keys are on an independent rate limit" do- patch_env(Plausible.Auth.ApiKey,+ patch_env(PlausibleWeb.Api.RateLimit,legacy_per_user_hourly_request_limit: 100,# relax burst request limit to check hourly request limitburst_request_limit: 200,@@ -363,7 +363,7 @@ defmodule PlausibleWeb.Plugs.AuthorizePublicAPITest do_site = new_site(owner: legacy_api_key_user)legacy_api_key = insert_api_key(:legacy_api_key, user: legacy_api_key_user)- 1..Plausible.Auth.ApiKey.legacy_hourly_request_limit()+ 1..PlausibleWeb.Api.RateLimit.legacy_hourly_request_limit()|> Enum.map(fn _ ->conn =get_fresh_conn()