π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[server] Add rate-limit rejection telemetry (#13182)
neeraj-pilote24d45e4be39
diff --git a/server/pkg/middleware/rate_limit.go b/server/pkg/middleware/rate_limit.goindex efc9e8a426..9a57da7962 100644--- a/server/pkg/middleware/rate_limit.go+++ b/server/pkg/middleware/rate_limit.go@@ -13,11 +13,30 @@ import ("github.com/ente/museum/pkg/utils/auth""github.com/ente/museum/pkg/utils/network"+ "github.com/gin-contrib/requestid""github.com/gin-gonic/gin"+ "github.com/prometheus/client_golang/prometheus"+ "github.com/prometheus/client_golang/prometheus/promauto"log "github.com/sirupsen/logrus""github.com/ulule/limiter/v3")+type rateLimitScope string++const (+ rateLimitScopeIP rateLimitScope = "ip"+ rateLimitScopeCollection rateLimitScope = "collection"+ rateLimitScopeUser rateLimitScope = "user"+ rateLimitScopeRouteGlobal rateLimitScope = "route_global"+ rateLimitScopeProcessGlobal rateLimitScope = "process_global"+ processGlobalRateLimitURL = "all"+)++var rateLimitRejections = promauto.NewCounterVec(prometheus.CounterOpts{+ Name: "museum_rate_limit_rejections_total",+ Help: "The number of requests rejected by Museum rate limiters",+}, []string{"scope", "method", "url"})+type RateLimitMiddleware struct {count int64limit int64@@ -69,6 +88,7 @@ func (r *RateLimitMiddleware) Stop() {func (r *RateLimitMiddleware) GlobalRateLimiter() gin.HandlerFunc {return func(c *gin.Context) {if !r.Increment() {+ recordRateLimitRejection(rateLimitScopeProcessGlobal, c.Request.Method, processGlobalRateLimitURL)if r.count%100 == 0 {go r.discordCtrl.NotifyPotentialAbuse(fmt.Sprintf("Global ratelimit (%d) breached %d", r.limit, r.count))}@@ -84,15 +104,15 @@ func (r *RateLimitMiddleware) APIRateLimitMiddleware(urlSanitizer func(_ *gin.CorequestPath := urlSanitizer(c)if globalRateLimiter := r.getGlobalLimiter(requestPath, c.Request.Method); globalRateLimiter != nil {- if r.isRateLimited(c, globalRateLimiter, globalRateLimitKey(requestPath), fmt.Sprintf("π Global rate limit: %s", requestPath)) {+ if r.isRateLimited(c, globalRateLimiter, globalRateLimitKey(requestPath), requestPath, rateLimitScopeRouteGlobal, fmt.Sprintf("π Global rate limit: %s", requestPath)) {return}}rateLimiter := r.getLimiter(requestPath, c.Request.Method)if rateLimiter != nil {- key := r.getRateLimitKey(c, requestPath)- if r.isRateLimited(c, rateLimiter, key, fmt.Sprintf("π IP rate limit: %s", requestPath)) {+ key, scope := r.getRateLimitKey(c, requestPath)+ if r.isRateLimited(c, rateLimiter, key, requestPath, scope, fmt.Sprintf("π IP rate limit: %s", requestPath)) {return}}@@ -104,7 +124,7 @@ func (r *RateLimitMiddleware) APIRateLimitForUserMiddleware(urlSanitizer func(_return func(c *gin.Context) {requestPath := urlSanitizer(c)if globalRateLimiter := r.getGlobalLimiter(requestPath, c.Request.Method); globalRateLimiter != nil {- if r.isRateLimited(c, globalRateLimiter, globalRateLimitKey(requestPath), fmt.Sprintf("π Global rate limit: %s", requestPath)) {+ if r.isRateLimited(c, globalRateLimiter, globalRateLimitKey(requestPath), requestPath, rateLimitScopeRouteGlobal, fmt.Sprintf("π Global rate limit: %s", requestPath)) {return}}@@ -116,7 +136,7 @@ func (r *RateLimitMiddleware) APIRateLimitForUserMiddleware(urlSanitizer func(_log.Error("userID must be present in request header for applying rate-limit")return}- if r.isRateLimited(c, rateLimiter, fmt.Sprintf("%d-%s", userID, requestPath), fmt.Sprintf("π€ User rate limit: %s user=%d", requestPath, userID)) {+ if r.isRateLimited(c, rateLimiter, fmt.Sprintf("%d-%s", userID, requestPath), requestPath, rateLimitScopeUser, fmt.Sprintf("π€ User rate limit: %s user=%d", requestPath, userID)) {return}}@@ -124,7 +144,7 @@ func (r *RateLimitMiddleware) APIRateLimitForUserMiddleware(urlSanitizer func(_}}-func (r *RateLimitMiddleware) isRateLimited(c *gin.Context, rateLimiter *limiter.Limiter, key string, message string) bool {+func (r *RateLimitMiddleware) isRateLimited(c *gin.Context, rateLimiter *limiter.Limiter, key string, requestPath string, scope rateLimitScope, message string) bool {limitContext, err := rateLimiter.Get(c, key)if err != nil {log.Error("Failed to check rate limit", err)@@ -133,12 +153,22 @@ func (r *RateLimitMiddleware) isRateLimited(c *gin.Context, rateLimiter *limiterif !limitContext.Reached {return false}+ recordRateLimitRejection(scope, c.Request.Method, requestPath)go r.discordCtrl.NotifyPotentialAbuse(message)- log.Error(fmt.Sprintf("Rate limit breached %s", key))+ log.WithFields(log.Fields{+ "rate_limit_scope": scope,+ "req_id": requestid.Get(c),+ "req_method": c.Request.Method,+ "req_uri": requestPath,+ }).Error("Rate limit breached")c.AbortWithStatusJSON(http.StatusTooManyRequests, gin.H{"error": "Rate limit breached, try later"})return true}+func recordRateLimitRejection(scope rateLimitScope, method string, requestPath string) {+ rateLimitRejections.WithLabelValues(string(scope), method, requestPath).Inc()+}+func (r *RateLimitMiddleware) getGlobalLimiter(reqPath string, reqMethod string) *limiter.Limiter {if isEventURLPath(reqPath) {return r.limit120ReqPerHour@@ -156,21 +186,21 @@ func globalRateLimitKey(reqPath string) string {return reqPath}-func (r *RateLimitMiddleware) getRateLimitKey(c *gin.Context, reqPath string) string {+func (r *RateLimitMiddleware) getRateLimitKey(c *gin.Context, reqPath string) (string, rateLimitScope) {if !isPublicCollectionUploadURLPath(reqPath) {- return fmt.Sprintf("%s-%s", network.GetClientIP(c), reqPath)+ return fmt.Sprintf("%s-%s", network.GetClientIP(c), reqPath), rateLimitScopeIP}value, ok := c.Get(auth.PublicAccessKey)if !ok {log.WithField("path", reqPath).Warn("public access context missing for collection scoped rate limit")- return fmt.Sprintf("%s-%s", network.GetClientIP(c), reqPath)+ return fmt.Sprintf("%s-%s", network.GetClientIP(c), reqPath), rateLimitScopeIP}accessContext, ok := value.(ente.PublicAccessContext)if !ok {log.WithField("path", reqPath).Warn("invalid public access context for collection scoped rate limit")- return fmt.Sprintf("%s-%s", network.GetClientIP(c), reqPath)+ return fmt.Sprintf("%s-%s", network.GetClientIP(c), reqPath), rateLimitScopeIP}- return fmt.Sprintf("collection:%d-%s", accessContext.CollectionID, reqPath)+ return fmt.Sprintf("collection:%d-%s", accessContext.CollectionID, reqPath), rateLimitScopeCollection}func isPublicCollectionUploadURLPath(reqPath string) bool {diff --git a/server/pkg/middleware/rate_limit_metrics_test.go b/server/pkg/middleware/rate_limit_metrics_test.gonew file mode 100644index 0000000000..6b62eb8688--- /dev/null+++ b/server/pkg/middleware/rate_limit_metrics_test.go@@ -0,0 +1,69 @@+package middleware++import (+ "net/http"+ "net/http/httptest"+ "testing"++ "github.com/ente/museum/ente"+ "github.com/ente/museum/pkg/utils/auth"+ "github.com/gin-gonic/gin"+ "github.com/prometheus/client_golang/prometheus/testutil"+)++func TestRecordRateLimitRejection(t *testing.T) {+ counter := rateLimitRejections.WithLabelValues(+ string(rateLimitScopeIP),+ http.MethodGet,+ "/test/rate-limit",+ )+ before := testutil.ToFloat64(counter)++ recordRateLimitRejection(rateLimitScopeIP, http.MethodGet, "/test/rate-limit")++ after := testutil.ToFloat64(counter)+ if after != before+1 {+ t.Fatalf("rate limit rejection counter = %v, want %v", after, before+1)+ }+}++func TestRateLimitScopeLabels(t *testing.T) {+ tests := []struct {+ scope rateLimitScope+ want string+ }{+ {scope: rateLimitScopeIP, want: "ip"},+ {scope: rateLimitScopeCollection, want: "collection"},+ {scope: rateLimitScopeUser, want: "user"},+ {scope: rateLimitScopeRouteGlobal, want: "route_global"},+ {scope: rateLimitScopeProcessGlobal, want: "process_global"},+ }++ for _, tt := range tests {+ if got := string(tt.scope); got != tt.want {+ t.Errorf("rate limit scope label = %q, want %q", got, tt.want)+ }+ }+}++func TestGetRateLimitKeyReturnsScopeForSelectedKey(t *testing.T) {+ gin.SetMode(gin.TestMode)+ rateLimiter := &RateLimitMiddleware{}+ check := func(path string, publicContext any, wantKey string, wantScope rateLimitScope) {+ t.Helper()+ context, _ := gin.CreateTestContext(httptest.NewRecorder())+ context.Request = httptest.NewRequest(http.MethodPost, path, nil)+ context.Request.RemoteAddr = "198.51.100.1:1234"+ if publicContext != nil {+ context.Set(auth.PublicAccessKey, publicContext)+ }+ gotKey, gotScope := rateLimiter.getRateLimitKey(context, path)+ if gotKey != wantKey || gotScope != wantScope {+ t.Errorf("getRateLimitKey(%q) = (%q, %q), want (%q, %q)", path, gotKey, gotScope, wantKey, wantScope)+ }+ }++ check("/users/srp/attributes", nil, "198.51.100.1-/users/srp/attributes", rateLimitScopeIP)+ check("/public-collection/upload-url", ente.PublicAccessContext{CollectionID: 42}, "collection:42-/public-collection/upload-url", rateLimitScopeCollection)+ check("/public-collection/upload-url", nil, "198.51.100.1-/public-collection/upload-url", rateLimitScopeIP)+}