Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[server] Add rate-limit rejection telemetry (#13182)

neeraj-pilote24d45e4be39
diff --git a/server/pkg/middleware/rate_limit.go b/server/pkg/middleware/rate_limit.go
index efc9e8a426..9a57da7962 100644
--- a/server/pkg/middleware/rate_limit.go
+++ b/server/pkg/middleware/rate_limit.go
@@ -13,11 +13,30 @@ import (
"github.com/ente/museum/pkg/utils/auth"
"github.com/ente/museum/pkg/utils/network"
+ "github.com/gin-contrib/requestid"
"github.com/gin-gonic/gin"
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promauto"
log "github.com/sirupsen/logrus"
"github.com/ulule/limiter/v3"
)
+type rateLimitScope string
+
+const (
+ rateLimitScopeIP rateLimitScope = "ip"
+ rateLimitScopeCollection rateLimitScope = "collection"
+ rateLimitScopeUser rateLimitScope = "user"
+ rateLimitScopeRouteGlobal rateLimitScope = "route_global"
+ rateLimitScopeProcessGlobal rateLimitScope = "process_global"
+ processGlobalRateLimitURL = "all"
+)
+
+var rateLimitRejections = promauto.NewCounterVec(prometheus.CounterOpts{
+ Name: "museum_rate_limit_rejections_total",
+ Help: "The number of requests rejected by Museum rate limiters",
+}, []string{"scope", "method", "url"})
+
type RateLimitMiddleware struct {
count int64
limit int64
@@ -69,6 +88,7 @@ func (r *RateLimitMiddleware) Stop() {
func (r *RateLimitMiddleware) GlobalRateLimiter() gin.HandlerFunc {
return func(c *gin.Context) {
if !r.Increment() {
+ recordRateLimitRejection(rateLimitScopeProcessGlobal, c.Request.Method, processGlobalRateLimitURL)
if r.count%100 == 0 {
go r.discordCtrl.NotifyPotentialAbuse(fmt.Sprintf("Global ratelimit (%d) breached %d", r.limit, r.count))
}
@@ -84,15 +104,15 @@ func (r *RateLimitMiddleware) APIRateLimitMiddleware(urlSanitizer func(_ *gin.Co
requestPath := urlSanitizer(c)
if globalRateLimiter := r.getGlobalLimiter(requestPath, c.Request.Method); globalRateLimiter != nil {
- if r.isRateLimited(c, globalRateLimiter, globalRateLimitKey(requestPath), fmt.Sprintf("🌍 Global rate limit: %s", requestPath)) {
+ if r.isRateLimited(c, globalRateLimiter, globalRateLimitKey(requestPath), requestPath, rateLimitScopeRouteGlobal, fmt.Sprintf("🌍 Global rate limit: %s", requestPath)) {
return
}
}
rateLimiter := r.getLimiter(requestPath, c.Request.Method)
if rateLimiter != nil {
- key := r.getRateLimitKey(c, requestPath)
- if r.isRateLimited(c, rateLimiter, key, fmt.Sprintf("🌐 IP rate limit: %s", requestPath)) {
+ key, scope := r.getRateLimitKey(c, requestPath)
+ if r.isRateLimited(c, rateLimiter, key, requestPath, scope, fmt.Sprintf("🌐 IP rate limit: %s", requestPath)) {
return
}
}
@@ -104,7 +124,7 @@ func (r *RateLimitMiddleware) APIRateLimitForUserMiddleware(urlSanitizer func(_
return func(c *gin.Context) {
requestPath := urlSanitizer(c)
if globalRateLimiter := r.getGlobalLimiter(requestPath, c.Request.Method); globalRateLimiter != nil {
- if r.isRateLimited(c, globalRateLimiter, globalRateLimitKey(requestPath), fmt.Sprintf("🌍 Global rate limit: %s", requestPath)) {
+ if r.isRateLimited(c, globalRateLimiter, globalRateLimitKey(requestPath), requestPath, rateLimitScopeRouteGlobal, fmt.Sprintf("🌍 Global rate limit: %s", requestPath)) {
return
}
}
@@ -116,7 +136,7 @@ func (r *RateLimitMiddleware) APIRateLimitForUserMiddleware(urlSanitizer func(_
log.Error("userID must be present in request header for applying rate-limit")
return
}
- if r.isRateLimited(c, rateLimiter, fmt.Sprintf("%d-%s", userID, requestPath), fmt.Sprintf("πŸ‘€ User rate limit: %s user=%d", requestPath, userID)) {
+ if r.isRateLimited(c, rateLimiter, fmt.Sprintf("%d-%s", userID, requestPath), requestPath, rateLimitScopeUser, fmt.Sprintf("πŸ‘€ User rate limit: %s user=%d", requestPath, userID)) {
return
}
}
@@ -124,7 +144,7 @@ func (r *RateLimitMiddleware) APIRateLimitForUserMiddleware(urlSanitizer func(_
}
}
-func (r *RateLimitMiddleware) isRateLimited(c *gin.Context, rateLimiter *limiter.Limiter, key string, message string) bool {
+func (r *RateLimitMiddleware) isRateLimited(c *gin.Context, rateLimiter *limiter.Limiter, key string, requestPath string, scope rateLimitScope, message string) bool {
limitContext, err := rateLimiter.Get(c, key)
if err != nil {
log.Error("Failed to check rate limit", err)
@@ -133,12 +153,22 @@ func (r *RateLimitMiddleware) isRateLimited(c *gin.Context, rateLimiter *limiter
if !limitContext.Reached {
return false
}
+ recordRateLimitRejection(scope, c.Request.Method, requestPath)
go r.discordCtrl.NotifyPotentialAbuse(message)
- log.Error(fmt.Sprintf("Rate limit breached %s", key))
+ log.WithFields(log.Fields{
+ "rate_limit_scope": scope,
+ "req_id": requestid.Get(c),
+ "req_method": c.Request.Method,
+ "req_uri": requestPath,
+ }).Error("Rate limit breached")
c.AbortWithStatusJSON(http.StatusTooManyRequests, gin.H{"error": "Rate limit breached, try later"})
return true
}
+func recordRateLimitRejection(scope rateLimitScope, method string, requestPath string) {
+ rateLimitRejections.WithLabelValues(string(scope), method, requestPath).Inc()
+}
+
func (r *RateLimitMiddleware) getGlobalLimiter(reqPath string, reqMethod string) *limiter.Limiter {
if isEventURLPath(reqPath) {
return r.limit120ReqPerHour
@@ -156,21 +186,21 @@ func globalRateLimitKey(reqPath string) string {
return reqPath
}
-func (r *RateLimitMiddleware) getRateLimitKey(c *gin.Context, reqPath string) string {
+func (r *RateLimitMiddleware) getRateLimitKey(c *gin.Context, reqPath string) (string, rateLimitScope) {
if !isPublicCollectionUploadURLPath(reqPath) {
- return fmt.Sprintf("%s-%s", network.GetClientIP(c), reqPath)
+ return fmt.Sprintf("%s-%s", network.GetClientIP(c), reqPath), rateLimitScopeIP
}
value, ok := c.Get(auth.PublicAccessKey)
if !ok {
log.WithField("path", reqPath).Warn("public access context missing for collection scoped rate limit")
- return fmt.Sprintf("%s-%s", network.GetClientIP(c), reqPath)
+ return fmt.Sprintf("%s-%s", network.GetClientIP(c), reqPath), rateLimitScopeIP
}
accessContext, ok := value.(ente.PublicAccessContext)
if !ok {
log.WithField("path", reqPath).Warn("invalid public access context for collection scoped rate limit")
- return fmt.Sprintf("%s-%s", network.GetClientIP(c), reqPath)
+ return fmt.Sprintf("%s-%s", network.GetClientIP(c), reqPath), rateLimitScopeIP
}
- return fmt.Sprintf("collection:%d-%s", accessContext.CollectionID, reqPath)
+ return fmt.Sprintf("collection:%d-%s", accessContext.CollectionID, reqPath), rateLimitScopeCollection
}
func isPublicCollectionUploadURLPath(reqPath string) bool {
diff --git a/server/pkg/middleware/rate_limit_metrics_test.go b/server/pkg/middleware/rate_limit_metrics_test.go
new file mode 100644
index 0000000000..6b62eb8688
--- /dev/null
+++ b/server/pkg/middleware/rate_limit_metrics_test.go
@@ -0,0 +1,69 @@
+package middleware
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/ente/museum/ente"
+ "github.com/ente/museum/pkg/utils/auth"
+ "github.com/gin-gonic/gin"
+ "github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+func TestRecordRateLimitRejection(t *testing.T) {
+ counter := rateLimitRejections.WithLabelValues(
+ string(rateLimitScopeIP),
+ http.MethodGet,
+ "/test/rate-limit",
+ )
+ before := testutil.ToFloat64(counter)
+
+ recordRateLimitRejection(rateLimitScopeIP, http.MethodGet, "/test/rate-limit")
+
+ after := testutil.ToFloat64(counter)
+ if after != before+1 {
+ t.Fatalf("rate limit rejection counter = %v, want %v", after, before+1)
+ }
+}
+
+func TestRateLimitScopeLabels(t *testing.T) {
+ tests := []struct {
+ scope rateLimitScope
+ want string
+ }{
+ {scope: rateLimitScopeIP, want: "ip"},
+ {scope: rateLimitScopeCollection, want: "collection"},
+ {scope: rateLimitScopeUser, want: "user"},
+ {scope: rateLimitScopeRouteGlobal, want: "route_global"},
+ {scope: rateLimitScopeProcessGlobal, want: "process_global"},
+ }
+
+ for _, tt := range tests {
+ if got := string(tt.scope); got != tt.want {
+ t.Errorf("rate limit scope label = %q, want %q", got, tt.want)
+ }
+ }
+}
+
+func TestGetRateLimitKeyReturnsScopeForSelectedKey(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ rateLimiter := &RateLimitMiddleware{}
+ check := func(path string, publicContext any, wantKey string, wantScope rateLimitScope) {
+ t.Helper()
+ context, _ := gin.CreateTestContext(httptest.NewRecorder())
+ context.Request = httptest.NewRequest(http.MethodPost, path, nil)
+ context.Request.RemoteAddr = "198.51.100.1:1234"
+ if publicContext != nil {
+ context.Set(auth.PublicAccessKey, publicContext)
+ }
+ gotKey, gotScope := rateLimiter.getRateLimitKey(context, path)
+ if gotKey != wantKey || gotScope != wantScope {
+ t.Errorf("getRateLimitKey(%q) = (%q, %q), want (%q, %q)", path, gotKey, gotScope, wantKey, wantScope)
+ }
+ }
+
+ check("/users/srp/attributes", nil, "198.51.100.1-/users/srp/attributes", rateLimitScopeIP)
+ check("/public-collection/upload-url", ente.PublicAccessContext{CollectionID: 42}, "collection:42-/public-collection/upload-url", rateLimitScopeCollection)
+ check("/public-collection/upload-url", nil, "198.51.100.1-/public-collection/upload-url", rateLimitScopeIP)
+}