A platform for community discussion. Free, open, simple.
FIX: Rate limit health check requests per backend
Jake Goldsboroughd948d26d2522
diff --git a/lib/middleware/request_tracker.rb b/lib/middleware/request_tracker.rbindex d20c8aaaaee..844d2513c22 100644--- a/lib/middleware/request_tracker.rb+++ b/lib/middleware/request_tracker.rb@@ -81,6 +81,7 @@ class Middleware::RequestTracker# Update the documentation for the `add_request_rate_limiter` plugin API if this list changes.default_rate_limiters = [RequestTracker::RateLimiters::User,+ RequestTracker::RateLimiters::HealthCheck,RequestTracker::RateLimiters::IP,]diff --git a/lib/plugin/instance.rb b/lib/plugin/instance.rbindex 2b2bcd3b8d6..37766811a28 100644--- a/lib/plugin/instance.rb+++ b/lib/plugin/instance.rb@@ -1511,6 +1511,7 @@ class Plugin::Instance# rate limiters:## `RequestTracker::RateLimiters::User` - Rate limits authenticated requests based on the user's id+ # `RequestTracker::RateLimiters::HealthCheck` - Rate limits health check requests per IP and backend hostname# `RequestTracker::RateLimiters::IP` - Rate limits requests based on the IP address## @param identifier [Symbol] A unique identifier for the rate limiter.diff --git a/lib/request_tracker/rate_limiters/health_check.rb b/lib/request_tracker/rate_limiters/health_check.rbnew file mode 100644index 00000000000..c3ad5d8d92a--- /dev/null+++ b/lib/request_tracker/rate_limiters/health_check.rb@@ -0,0 +1,33 @@+# frozen_string_literal: true++module RequestTracker+ module RateLimiters+ # Rate limits health check requests per backend instead of sharing the+ # per-IP budget.+ #+ # Load balancers check every backend they route to, and every backend+ # increments the same Redis-backed per-IP counter, so health check+ # volume scales with backend count while the budget does not. With+ # enough backends, the checks alone exceed the limit and all backends+ # get marked down with 429s. Including the backend hostname in the key+ # gives each backend its own budget, so the per-key rate stays+ # constant at any scale.+ #+ # Health checks from private addresses already skip rate limiting, so+ # most deployments see no change; this matters when checks arrive from+ # publicly-routable addresses (e.g. public IPv6 ranges).+ class HealthCheck < Base+ def rate_limit_key+ "health_check/#{@request.ip}/#{Discourse.os_hostname}"+ end++ def rate_limit_globally?+ true+ end++ def active?+ @request.path == "/srv/status"+ end+ end+ end+enddiff --git a/spec/lib/middleware/request_tracker_spec.rb b/spec/lib/middleware/request_tracker_spec.rbindex dd6a5084ff6..f096a354a4a 100644--- a/spec/lib/middleware/request_tracker_spec.rb+++ b/spec/lib/middleware/request_tracker_spec.rb@@ -1403,6 +1403,61 @@ RSpec.describe Middleware::RequestTracker doendend+ describe "health check rate limits" do+ def health_check_env(ip)+ env(:path => "/srv/status", "REMOTE_ADDR" => ip)+ end++ before do+ global_setting :max_reqs_per_ip_per_10_seconds, 1+ global_setting :max_reqs_per_ip_mode, "block"+ end++ it "rate limits health checks under their own key, not the per-IP budget" do+ status, _ = middleware.call(health_check_env("1.2.3.4"))+ expect(status).to eq(200)++ status, headers = middleware.call(health_check_env("1.2.3.4"))+ expect(status).to eq(429)+ expect(headers["Discourse-Rate-Limit-Error-Code"]).to eq("health_check_10_secs_limit")+ end++ it "gives each backend hostname its own budget" do+ # Load balancers health check every backend, so check volume scales+ # with backend count while a per-IP budget does not. Keying on the+ # backend hostname keeps the per-key rate constant at any scale.+ Discourse.stubs(:os_hostname).returns("backend-1")+ status, _ = middleware.call(health_check_env("1.2.3.4"))+ expect(status).to eq(200)++ Discourse.stubs(:os_hostname).returns("backend-2")+ status, _ = middleware.call(health_check_env("1.2.3.4"))+ expect(status).to eq(200)++ Discourse.stubs(:os_hostname).returns("backend-1")+ status, headers = middleware.call(health_check_env("1.2.3.4"))+ expect(status).to eq(429)+ expect(headers["Discourse-Rate-Limit-Error-Code"]).to eq("health_check_10_secs_limit")+ end++ it "does not affect health checks from private IPs, which skip rate limiting" do+ status, _ = middleware.call(health_check_env("10.0.1.2"))+ expect(status).to eq(200)++ status, _ = middleware.call(health_check_env("10.0.1.2"))+ expect(status).to eq(200)+ end++ it "does not affect rate limiting for other paths" do+ status, _ = middleware.call(env("REMOTE_ADDR" => "1.2.3.4"))+ expect(status).to eq(200)++ status, headers = middleware.call(env("REMOTE_ADDR" => "1.2.3.4"))+ expect(status).to eq(429)+ expect(headers["Discourse-Rate-Limit-Error-Code"]).to eq("ip_10_secs_limit")+ end+ end+describe "crawler rate limits" docontext "when there are multiple matching crawlers" dobefore { SiteSetting.slow_down_crawler_user_agents = "badcrawler2|badcrawler22" }diff --git a/spec/lib/plugin/instance_spec.rb b/spec/lib/plugin/instance_spec.rbindex e194adcc37d..fa5c3bb2f3c 100644--- a/spec/lib/plugin/instance_spec.rb+++ b/spec/lib/plugin/instance_spec.rb@@ -1294,7 +1294,7 @@ TEXT)}.to raise_error(ArgumentError,- "0 is not a valid value. Must be one of RequestTracker::RateLimiters::User, RequestTracker::RateLimiters::IP",+ "0 is not a valid value. Must be one of RequestTracker::RateLimiters::User, RequestTracker::RateLimiters::HealthCheck, RequestTracker::RateLimiters::IP",)end@@ -1310,7 +1310,7 @@ TEXT)}.to raise_error(ArgumentError,- "0 is not a valid value. Must be one of RequestTracker::RateLimiters::User, RequestTracker::RateLimiters::IP",+ "0 is not a valid value. Must be one of RequestTracker::RateLimiters::User, RequestTracker::RateLimiters::HealthCheck, RequestTracker::RateLimiters::IP",)end@@ -1342,11 +1342,15 @@ TEXTRequestTracker::RateLimiters::User,)- expect(Middleware::RequestTracker.rate_limiters_stack[1].superclass).to eq(+ expect(Middleware::RequestTracker.rate_limiters_stack[1]).to eq(+ RequestTracker::RateLimiters::HealthCheck,+ )++ expect(Middleware::RequestTracker.rate_limiters_stack[2].superclass).to eq(RequestTracker::RateLimiters::Base,)- expect(Middleware::RequestTracker.rate_limiters_stack[2]).to eq(+ expect(Middleware::RequestTracker.rate_limiters_stack[3]).to eq(RequestTracker::RateLimiters::IP,)end@@ -1366,10 +1370,14 @@ TEXT)expect(Middleware::RequestTracker.rate_limiters_stack[1]).to eq(+ RequestTracker::RateLimiters::HealthCheck,+ )++ expect(Middleware::RequestTracker.rate_limiters_stack[2]).to eq(RequestTracker::RateLimiters::IP,)- expect(Middleware::RequestTracker.rate_limiters_stack[2].superclass).to eq(+ expect(Middleware::RequestTracker.rate_limiters_stack[3].superclass).to eq(RequestTracker::RateLimiters::Base,)end