Skip to content

A platform for community discussion. Free, open, simple.

FIX: Rate limit health check requests per backend

Jake Goldsboroughd948d26d2522
diff --git a/lib/middleware/request_tracker.rb b/lib/middleware/request_tracker.rb
index d20c8aaaaee..844d2513c22 100644
--- a/lib/middleware/request_tracker.rb
+++ b/lib/middleware/request_tracker.rb
@@ -81,6 +81,7 @@ class Middleware::RequestTracker
# Update the documentation for the `add_request_rate_limiter` plugin API if this list changes.
default_rate_limiters = [
RequestTracker::RateLimiters::User,
+ RequestTracker::RateLimiters::HealthCheck,
RequestTracker::RateLimiters::IP,
]
diff --git a/lib/plugin/instance.rb b/lib/plugin/instance.rb
index 2b2bcd3b8d6..37766811a28 100644
--- a/lib/plugin/instance.rb
+++ b/lib/plugin/instance.rb
@@ -1511,6 +1511,7 @@ class Plugin::Instance
# rate limiters:
#
# `RequestTracker::RateLimiters::User` - Rate limits authenticated requests based on the user's id
+ # `RequestTracker::RateLimiters::HealthCheck` - Rate limits health check requests per IP and backend hostname
# `RequestTracker::RateLimiters::IP` - Rate limits requests based on the IP address
#
# @param identifier [Symbol] A unique identifier for the rate limiter.
diff --git a/lib/request_tracker/rate_limiters/health_check.rb b/lib/request_tracker/rate_limiters/health_check.rb
new file mode 100644
index 00000000000..c3ad5d8d92a
--- /dev/null
+++ b/lib/request_tracker/rate_limiters/health_check.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module RequestTracker
+ module RateLimiters
+ # Rate limits health check requests per backend instead of sharing the
+ # per-IP budget.
+ #
+ # Load balancers check every backend they route to, and every backend
+ # increments the same Redis-backed per-IP counter, so health check
+ # volume scales with backend count while the budget does not. With
+ # enough backends, the checks alone exceed the limit and all backends
+ # get marked down with 429s. Including the backend hostname in the key
+ # gives each backend its own budget, so the per-key rate stays
+ # constant at any scale.
+ #
+ # Health checks from private addresses already skip rate limiting, so
+ # most deployments see no change; this matters when checks arrive from
+ # publicly-routable addresses (e.g. public IPv6 ranges).
+ class HealthCheck < Base
+ def rate_limit_key
+ "health_check/#{@request.ip}/#{Discourse.os_hostname}"
+ end
+
+ def rate_limit_globally?
+ true
+ end
+
+ def active?
+ @request.path == "/srv/status"
+ end
+ end
+ end
+end
diff --git a/spec/lib/middleware/request_tracker_spec.rb b/spec/lib/middleware/request_tracker_spec.rb
index dd6a5084ff6..f096a354a4a 100644
--- a/spec/lib/middleware/request_tracker_spec.rb
+++ b/spec/lib/middleware/request_tracker_spec.rb
@@ -1403,6 +1403,61 @@ RSpec.describe Middleware::RequestTracker do
end
end
+ describe "health check rate limits" do
+ def health_check_env(ip)
+ env(:path => "/srv/status", "REMOTE_ADDR" => ip)
+ end
+
+ before do
+ global_setting :max_reqs_per_ip_per_10_seconds, 1
+ global_setting :max_reqs_per_ip_mode, "block"
+ end
+
+ it "rate limits health checks under their own key, not the per-IP budget" do
+ status, _ = middleware.call(health_check_env("1.2.3.4"))
+ expect(status).to eq(200)
+
+ status, headers = middleware.call(health_check_env("1.2.3.4"))
+ expect(status).to eq(429)
+ expect(headers["Discourse-Rate-Limit-Error-Code"]).to eq("health_check_10_secs_limit")
+ end
+
+ it "gives each backend hostname its own budget" do
+ # Load balancers health check every backend, so check volume scales
+ # with backend count while a per-IP budget does not. Keying on the
+ # backend hostname keeps the per-key rate constant at any scale.
+ Discourse.stubs(:os_hostname).returns("backend-1")
+ status, _ = middleware.call(health_check_env("1.2.3.4"))
+ expect(status).to eq(200)
+
+ Discourse.stubs(:os_hostname).returns("backend-2")
+ status, _ = middleware.call(health_check_env("1.2.3.4"))
+ expect(status).to eq(200)
+
+ Discourse.stubs(:os_hostname).returns("backend-1")
+ status, headers = middleware.call(health_check_env("1.2.3.4"))
+ expect(status).to eq(429)
+ expect(headers["Discourse-Rate-Limit-Error-Code"]).to eq("health_check_10_secs_limit")
+ end
+
+ it "does not affect health checks from private IPs, which skip rate limiting" do
+ status, _ = middleware.call(health_check_env("10.0.1.2"))
+ expect(status).to eq(200)
+
+ status, _ = middleware.call(health_check_env("10.0.1.2"))
+ expect(status).to eq(200)
+ end
+
+ it "does not affect rate limiting for other paths" do
+ status, _ = middleware.call(env("REMOTE_ADDR" => "1.2.3.4"))
+ expect(status).to eq(200)
+
+ status, headers = middleware.call(env("REMOTE_ADDR" => "1.2.3.4"))
+ expect(status).to eq(429)
+ expect(headers["Discourse-Rate-Limit-Error-Code"]).to eq("ip_10_secs_limit")
+ end
+ end
+
describe "crawler rate limits" do
context "when there are multiple matching crawlers" do
before { SiteSetting.slow_down_crawler_user_agents = "badcrawler2|badcrawler22" }
diff --git a/spec/lib/plugin/instance_spec.rb b/spec/lib/plugin/instance_spec.rb
index e194adcc37d..fa5c3bb2f3c 100644
--- a/spec/lib/plugin/instance_spec.rb
+++ b/spec/lib/plugin/instance_spec.rb
@@ -1294,7 +1294,7 @@ TEXT
)
}.to raise_error(
ArgumentError,
- "0 is not a valid value. Must be one of RequestTracker::RateLimiters::User, RequestTracker::RateLimiters::IP",
+ "0 is not a valid value. Must be one of RequestTracker::RateLimiters::User, RequestTracker::RateLimiters::HealthCheck, RequestTracker::RateLimiters::IP",
)
end
@@ -1310,7 +1310,7 @@ TEXT
)
}.to raise_error(
ArgumentError,
- "0 is not a valid value. Must be one of RequestTracker::RateLimiters::User, RequestTracker::RateLimiters::IP",
+ "0 is not a valid value. Must be one of RequestTracker::RateLimiters::User, RequestTracker::RateLimiters::HealthCheck, RequestTracker::RateLimiters::IP",
)
end
@@ -1342,11 +1342,15 @@ TEXT
RequestTracker::RateLimiters::User,
)
- expect(Middleware::RequestTracker.rate_limiters_stack[1].superclass).to eq(
+ expect(Middleware::RequestTracker.rate_limiters_stack[1]).to eq(
+ RequestTracker::RateLimiters::HealthCheck,
+ )
+
+ expect(Middleware::RequestTracker.rate_limiters_stack[2].superclass).to eq(
RequestTracker::RateLimiters::Base,
)
- expect(Middleware::RequestTracker.rate_limiters_stack[2]).to eq(
+ expect(Middleware::RequestTracker.rate_limiters_stack[3]).to eq(
RequestTracker::RateLimiters::IP,
)
end
@@ -1366,10 +1370,14 @@ TEXT
)
expect(Middleware::RequestTracker.rate_limiters_stack[1]).to eq(
+ RequestTracker::RateLimiters::HealthCheck,
+ )
+
+ expect(Middleware::RequestTracker.rate_limiters_stack[2]).to eq(
RequestTracker::RateLimiters::IP,
)
- expect(Middleware::RequestTracker.rate_limiters_stack[2].superclass).to eq(
+ expect(Middleware::RequestTracker.rate_limiters_stack[3].superclass).to eq(
RequestTracker::RateLimiters::Base,
)
end