π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[server] Separate thumbnail request limits
Neeraj Gupta4fccba4e0c8a
diff --git a/server/cmd/museum/main.go b/server/cmd/museum/main.goindex 6614ba9a18..f8b7e757bb 100644--- a/server/cmd/museum/main.go+++ b/server/cmd/museum/main.go@@ -228,7 +228,7 @@ func main() {collectionLinkRepo.Cache = accessTokenCachediscordController := discord.NewDiscordController(userRepo, hostName, environment)userLookupController := controller.NewUserLookupController(userRepo, discordController)- rateLimiter := middleware.NewRateLimitMiddleware(discordController, 1000, 1*time.Second)+ rateLimiter := middleware.NewRateLimitMiddleware(discordController, 1000, 1000, 1*time.Second)defer rateLimiter.Stop()emailNotificationCtrl := &email.EmailNotificationController{diff --git a/server/cmd/museum/main_test.go b/server/cmd/museum/main_test.goindex 1f5fb65265..4bb6b16e22 100644--- a/server/cmd/museum/main_test.go+++ b/server/cmd/museum/main_test.go@@ -71,7 +71,7 @@ func TestURLSanitizerSharesRateLimitsAcrossEncodings(t *testing.T) {{"/file-link/verify-password", "/file-link/verify-%70assword", "/file-link%2Fverify-password"},} {t.Run(paths[0], func(t *testing.T) {- rateLimiter := middleware.NewRateLimitMiddleware(discord.NewDiscordController(nil, "test", "test"), 1000, time.Minute)+ rateLimiter := middleware.NewRateLimitMiddleware(discord.NewDiscordController(nil, "test", "test"), 1000, 1000, time.Minute)t.Cleanup(rateLimiter.Stop)router := gin.New()router.POST(paths[0], rateLimiter.APIRateLimitMiddleware(urlSanitizer), func(c *gin.Context) {diff --git a/server/pkg/middleware/rate_limit.go b/server/pkg/middleware/rate_limit.goindex 9d7b4c5f4f..21bbb26c89 100644--- a/server/pkg/middleware/rate_limit.go+++ b/server/pkg/middleware/rate_limit.go@@ -24,12 +24,13 @@ import (type rateLimitScope stringconst (- rateLimitScopeIP rateLimitScope = "ip"- rateLimitScopeCollection rateLimitScope = "collection"- rateLimitScopeUser rateLimitScope = "user"- rateLimitScopeRouteGlobal rateLimitScope = "route_global"- rateLimitScopeProcessGlobal rateLimitScope = "process_global"- processGlobalRateLimitURL = "all"+ rateLimitScopeIP rateLimitScope = "ip"+ rateLimitScopeCollection rateLimitScope = "collection"+ rateLimitScopeUser rateLimitScope = "user"+ rateLimitScopeRouteGlobal rateLimitScope = "route_global"+ rateLimitScopeProcessGlobal rateLimitScope = "process_global"+ processGlobalRateLimitURL = "all"+ processThumbnailRateLimitURL = "thumbnails")var rateLimitRejections = promauto.NewCounterVec(prometheus.CounterOpts{@@ -39,7 +40,9 @@ var rateLimitRejections = promauto.NewCounterVec(prometheus.CounterOpts{type RateLimitMiddleware struct {count int64+ thumbnailCount int64limit int64+ thumbnailLimit int64reset time.Durationticker *time.Tickerlimit10ReqPerMin *limiter.Limiter@@ -53,7 +56,7 @@ type RateLimitMiddleware struct {discordCtrl *discord.DiscordController}-func NewRateLimitMiddleware(discordCtrl *discord.DiscordController, limit int64, reset time.Duration) *RateLimitMiddleware {+func NewRateLimitMiddleware(discordCtrl *discord.DiscordController, limit, thumbnailLimit int64, reset time.Duration) *RateLimitMiddleware {rl := &RateLimitMiddleware{limit10ReqPerMin: util.NewRateLimiter("10-M"),limit60ReqPerMin: util.NewRateLimiter("60-M"),@@ -65,32 +68,36 @@ func NewRateLimitMiddleware(discordCtrl *discord.DiscordController, limit int64,limit700ReqPerSec: util.NewRateLimiter("700-S"),discordCtrl: discordCtrl,limit: limit,+ thumbnailLimit: thumbnailLimit,reset: reset,ticker: time.NewTicker(reset),}go func() {for range rl.ticker.C {atomic.StoreInt64(&rl.count, 0)+ atomic.StoreInt64(&rl.thumbnailCount, 0)}}()return rl}-func (r *RateLimitMiddleware) Increment() bool {- newCount := atomic.AddInt64(&r.count, 1)- return newCount <= r.limit-}-func (r *RateLimitMiddleware) Stop() {r.ticker.Stop()}func (r *RateLimitMiddleware) GlobalRateLimiter() gin.HandlerFunc {return func(c *gin.Context) {- if !r.Increment() {- recordRateLimitRejection(rateLimitScopeProcessGlobal, c.Request.Method, processGlobalRateLimitURL)- if r.count%100 == 0 {- go r.discordCtrl.NotifyPotentialAbuse(fmt.Sprintf("Global ratelimit (%d) breached %d", r.limit, r.count))+ counter, limit := &r.count, r.limit+ metricURL, name := processGlobalRateLimitURL, "Global"+ if c.Request.Method == http.MethodGet && isThumbnailReadURLPath(c.FullPath()) {+ counter, limit = &r.thumbnailCount, r.thumbnailLimit+ metricURL, name = processThumbnailRateLimitURL, "Thumbnail"+ }+ count := atomic.AddInt64(counter, 1)+ if count > limit {+ recordRateLimitRejection(rateLimitScopeProcessGlobal, c.Request.Method, metricURL)+ if count%100 == 0 {+ go r.discordCtrl.NotifyPotentialAbuse(fmt.Sprintf("%s ratelimit (%d) breached %d", name, limit, count))}c.AbortWithStatusJSON(http.StatusTooManyRequests, gin.H{"error": "Rate limit breached, try later"})return@@ -99,6 +106,20 @@ func (r *RateLimitMiddleware) GlobalRateLimiter() gin.HandlerFunc {}}+// Match registered routes so unknown paths and writes retain the general budget.+func isThumbnailReadURLPath(reqPath string) bool {+ switch reqPath {+ case "/files/preview/:fileID", "/files/preview/v2/:fileID", "/files/thumbnail/v3/:fileID",+ "/public-collection/files/preview/:fileID", "/public-collection/files/thumbnail/v3/:fileID",+ "/public-memory/files/preview/:fileID", "/public-memory/files/thumbnail/v3/:fileID",+ "/cast/files/preview/:fileID", "/cast/files/thumbnail/v3/:fileID",+ "/file-link/thumbnail", "/file-link/thumbnail/v3":+ return true+ default:+ return false+ }+}+func (r *RateLimitMiddleware) APIRateLimitMiddleware(urlSanitizer func(_ *gin.Context) string) gin.HandlerFunc {return func(c *gin.Context) {requestPath := urlSanitizer(c)diff --git a/server/pkg/middleware/rate_limit_global_test.go b/server/pkg/middleware/rate_limit_global_test.gonew file mode 100644index 0000000000..a1a063bc4f--- /dev/null+++ b/server/pkg/middleware/rate_limit_global_test.go@@ -0,0 +1,149 @@+package middleware++import (+ "net/http"+ "net/http/httptest"+ "strings"+ "sync"+ "sync/atomic"+ "testing"+ "time"++ "github.com/gin-gonic/gin"+ "github.com/prometheus/client_golang/prometheus/testutil"+ "github.com/stretchr/testify/require"+)++func TestGlobalRateLimitBucketsAreIndependent(t *testing.T) {+ gin.SetMode(gin.TestMode)+ for _, thumbnailFirst := range []bool{true, false} {+ t.Run(map[bool]string{true: "thumbnail flood", false: "general flood"}[thumbnailFirst], func(t *testing.T) {+ rl := &RateLimitMiddleware{limit: 3, thumbnailLimit: 2}+ router := newGlobalRateLimitTestRouter(rl)+ paths := []string{"/collections", "/files/thumbnail/v3/1"}+ limits := []int{3, 2}+ if thumbnailFirst {+ paths[0], paths[1] = paths[1], paths[0]+ limits[0], limits[1] = limits[1], limits[0]+ }+ for i, path := range paths {+ for range limits[i] {+ require.Equal(t, http.StatusNoContent, globalRateLimitTestRequest(router, http.MethodGet, path))+ }+ for range 5 {+ require.Equal(t, http.StatusTooManyRequests, globalRateLimitTestRequest(router, http.MethodGet, path))+ }+ }+ })+ }+}++func TestGlobalRateLimitThumbnailRoutesShareBudget(t *testing.T) {+ gin.SetMode(gin.TestMode)+ rl := &RateLimitMiddleware{limit: 1, thumbnailLimit: 1}+ router := newGlobalRateLimitTestRouter(rl)+ require.Equal(t, http.StatusNoContent, globalRateLimitTestRequest(router, http.MethodGet, "/files/thumbnail/v3/1"))+ for _, route := range []string{+ "/files/preview/:fileID", "/files/preview/v2/:fileID",+ "/public-collection/files/preview/:fileID", "/public-collection/files/thumbnail/v3/:fileID",+ "/public-memory/files/preview/:fileID", "/public-memory/files/thumbnail/v3/:fileID",+ "/cast/files/preview/:fileID", "/cast/files/thumbnail/v3/:fileID",+ "/file-link/thumbnail", "/file-link/thumbnail/v3",+ } {+ router.GET(route, func(c *gin.Context) { c.Status(http.StatusNoContent) })+ path := strings.ReplaceAll(route, ":fileID", "2")+ require.Equal(t, http.StatusTooManyRequests, globalRateLimitTestRequest(router, http.MethodGet, path), route)+ }+ require.Equal(t, http.StatusNoContent, globalRateLimitTestRequest(router, http.MethodGet, "/collections"))+}++func TestGlobalRateLimitOtherRoutesRetainGeneralBudget(t *testing.T) {+ gin.SetMode(gin.TestMode)+ for _, tc := range []struct{ method, path string }{+ {http.MethodGet, "/files/download/1"},+ {http.MethodGet, "/files/data/preview"},+ {http.MethodPut, "/files/thumbnail"},+ {http.MethodPost, "/files/thumbnail/v3/1"},+ {http.MethodGet, "/files/thumbnail/unknown/1"},+ {http.MethodOptions, "/files/thumbnail/v3/1"},+ } {+ t.Run(tc.method+tc.path, func(t *testing.T) {+ rl := &RateLimitMiddleware{limit: 0, thumbnailLimit: 1}+ router := newGlobalRateLimitTestRouter(rl)+ router.GET("/files/download/:fileID", func(c *gin.Context) { c.Status(http.StatusNoContent) })+ router.GET("/files/data/preview", func(c *gin.Context) { c.Status(http.StatusNoContent) })+ router.PUT("/files/thumbnail", func(c *gin.Context) { c.Status(http.StatusNoContent) })+ require.Equal(t, http.StatusTooManyRequests, globalRateLimitTestRequest(router, tc.method, tc.path))+ require.Equal(t, http.StatusNoContent, globalRateLimitTestRequest(router, http.MethodGet, "/files/thumbnail/v3/1"))+ })+ }+}++func TestGlobalRateLimitResetsBothBuckets(t *testing.T) {+ gin.SetMode(gin.TestMode)+ rl := NewRateLimitMiddleware(nil, 1, 1, time.Hour)+ t.Cleanup(rl.Stop)+ router := newGlobalRateLimitTestRouter(rl)+ for _, path := range []string{"/collections", "/files/thumbnail/v3/1"} {+ require.Equal(t, http.StatusNoContent, globalRateLimitTestRequest(router, http.MethodGet, path))+ require.Equal(t, http.StatusTooManyRequests, globalRateLimitTestRequest(router, http.MethodGet, path))+ }+ rl.ticker.Reset(time.Millisecond)+ require.Eventually(t, func() bool {+ return atomic.LoadInt64(&rl.count) == 0 && atomic.LoadInt64(&rl.thumbnailCount) == 0+ }, time.Second, time.Millisecond)+ for _, path := range []string{"/collections", "/files/thumbnail/v3/1"} {+ require.Equal(t, http.StatusNoContent, globalRateLimitTestRequest(router, http.MethodGet, path))+ }+}++func TestGlobalRateLimitConcurrentRequests(t *testing.T) {+ gin.SetMode(gin.TestMode)+ rl := &RateLimitMiddleware{limit: 10, thumbnailLimit: 20}+ router := newGlobalRateLimitTestRouter(rl)+ var accepted [2]int64+ var wg sync.WaitGroup+ for i, path := range []string{"/collections", "/files/thumbnail/v3/1"} {+ for range 50 {+ wg.Add(1)+ go func() {+ defer wg.Done()+ if globalRateLimitTestRequest(router, http.MethodGet, path) == http.StatusNoContent {+ atomic.AddInt64(&accepted[i], 1)+ }+ }()+ }+ }+ wg.Wait()+ require.Equal(t, [2]int64{10, 20}, accepted)+}++func TestGlobalRateLimitRejectionMetricsIdentifyBucket(t *testing.T) {+ gin.SetMode(gin.TestMode)+ rl := &RateLimitMiddleware{}+ router := newGlobalRateLimitTestRouter(rl)+ for _, tc := range []struct{ path, label string }{+ {"/collections", processGlobalRateLimitURL},+ {"/files/thumbnail/v3/1", processThumbnailRateLimitURL},+ } {+ counter := rateLimitRejections.WithLabelValues(string(rateLimitScopeProcessGlobal), http.MethodGet, tc.label)+ before := testutil.ToFloat64(counter)+ require.Equal(t, http.StatusTooManyRequests, globalRateLimitTestRequest(router, http.MethodGet, tc.path))+ require.Equal(t, before+1, testutil.ToFloat64(counter))+ }+}++func newGlobalRateLimitTestRouter(rl *RateLimitMiddleware) *gin.Engine {+ router := gin.New()+ router.Use(rl.GlobalRateLimiter())+ for _, path := range []string{"/collections", "/files/thumbnail/v3/:fileID"} {+ router.GET(path, func(c *gin.Context) { c.Status(http.StatusNoContent) })+ }+ return router+}++func globalRateLimitTestRequest(router *gin.Engine, method, path string) int {+ response := httptest.NewRecorder()+ router.ServeHTTP(response, httptest.NewRequest(method, path, nil))+ return response.Code+}diff --git a/server/pkg/middleware/space_test.go b/server/pkg/middleware/space_test.goindex 1a8dc974f9..8a5f90e271 100644--- a/server/pkg/middleware/space_test.go+++ b/server/pkg/middleware/space_test.go@@ -13,7 +13,7 @@ import (func TestSpacePresignAllowsPhotoBatchesAfterProfileUploads(t *testing.T) {gin.SetMode(gin.TestMode)- rateLimiter := NewRateLimitMiddleware(discord.NewDiscordController(nil, "test", "test"), 1000, time.Minute)+ rateLimiter := NewRateLimitMiddleware(discord.NewDiscordController(nil, "test", "test"), 1000, 1000, time.Minute)t.Cleanup(rateLimiter.Stop)router := gin.New()router.POST("/spaces/:spaceID/uploads/presign", rateLimiter.APIRateLimitForUserMiddleware(func(c *gin.Context) string {