π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[server] Keep thumbnail rate limit independently configurable
Neeraj Gupta61cb531d43d0
diff --git a/server/cmd/museum/main.go b/server/cmd/museum/main.goindex 6614ba9a18..f8b7e757bb 100644--- a/server/cmd/museum/main.go+++ b/server/cmd/museum/main.go@@ -228,7 +228,7 @@ func main() {collectionLinkRepo.Cache = accessTokenCachediscordController := discord.NewDiscordController(userRepo, hostName, environment)userLookupController := controller.NewUserLookupController(userRepo, discordController)- rateLimiter := middleware.NewRateLimitMiddleware(discordController, 1000, 1*time.Second)+ rateLimiter := middleware.NewRateLimitMiddleware(discordController, 1000, 1000, 1*time.Second)defer rateLimiter.Stop()emailNotificationCtrl := &email.EmailNotificationController{diff --git a/server/cmd/museum/main_test.go b/server/cmd/museum/main_test.goindex 1f5fb65265..4bb6b16e22 100644--- a/server/cmd/museum/main_test.go+++ b/server/cmd/museum/main_test.go@@ -71,7 +71,7 @@ func TestURLSanitizerSharesRateLimitsAcrossEncodings(t *testing.T) {{"/file-link/verify-password", "/file-link/verify-%70assword", "/file-link%2Fverify-password"},} {t.Run(paths[0], func(t *testing.T) {- rateLimiter := middleware.NewRateLimitMiddleware(discord.NewDiscordController(nil, "test", "test"), 1000, time.Minute)+ rateLimiter := middleware.NewRateLimitMiddleware(discord.NewDiscordController(nil, "test", "test"), 1000, 1000, time.Minute)t.Cleanup(rateLimiter.Stop)router := gin.New()router.POST(paths[0], rateLimiter.APIRateLimitMiddleware(urlSanitizer), func(c *gin.Context) {diff --git a/server/pkg/middleware/rate_limit.go b/server/pkg/middleware/rate_limit.goindex 50e9c4f1c7..8528faeffc 100644--- a/server/pkg/middleware/rate_limit.go+++ b/server/pkg/middleware/rate_limit.go@@ -42,6 +42,7 @@ type RateLimitMiddleware struct {count int64thumbnailCount int64limit int64+ thumbnailLimit int64reset time.Durationticker *time.Tickerlimit10ReqPerMin *limiter.Limiter@@ -55,7 +56,7 @@ type RateLimitMiddleware struct {discordCtrl *discord.DiscordController}-func NewRateLimitMiddleware(discordCtrl *discord.DiscordController, limit int64, reset time.Duration) *RateLimitMiddleware {+func NewRateLimitMiddleware(discordCtrl *discord.DiscordController, limit, thumbnailLimit int64, reset time.Duration) *RateLimitMiddleware {rl := &RateLimitMiddleware{limit10ReqPerMin: util.NewRateLimiter("10-M"),limit60ReqPerMin: util.NewRateLimiter("60-M"),@@ -67,6 +68,7 @@ func NewRateLimitMiddleware(discordCtrl *discord.DiscordController, limit int64,limit700ReqPerSec: util.NewRateLimiter("700-S"),discordCtrl: discordCtrl,limit: limit,+ thumbnailLimit: thumbnailLimit,reset: reset,ticker: time.NewTicker(reset),}@@ -85,17 +87,17 @@ func (r *RateLimitMiddleware) Stop() {func (r *RateLimitMiddleware) GlobalRateLimiter() gin.HandlerFunc {return func(c *gin.Context) {- counter := &r.count+ counter, limit := &r.count, r.limitmetricURL, name := processGlobalRateLimitURL, "Global"if c.Request.Method == http.MethodGet && isThumbnailReadURLPath(c.FullPath()) {- counter = &r.thumbnailCount+ counter, limit = &r.thumbnailCount, r.thumbnailLimitmetricURL, name = processThumbnailRateLimitURL, "Thumbnail"}count := atomic.AddInt64(counter, 1)- if count > r.limit {+ if count > limit {recordRateLimitRejection(rateLimitScopeProcessGlobal, c.Request.Method, metricURL)if count%100 == 0 {- go r.discordCtrl.NotifyPotentialAbuse(fmt.Sprintf("%s ratelimit (%d) breached %d", name, r.limit, count))+ go r.discordCtrl.NotifyPotentialAbuse(fmt.Sprintf("%s ratelimit (%d) breached %d", name, limit, count))}c.AbortWithStatusJSON(http.StatusTooManyRequests, gin.H{"error": "Rate limit breached, try later"})returndiff --git a/server/pkg/middleware/rate_limit_global_test.go b/server/pkg/middleware/rate_limit_global_test.goindex 648ef8ecc8..6744a216ab 100644--- a/server/pkg/middleware/rate_limit_global_test.go+++ b/server/pkg/middleware/rate_limit_global_test.go@@ -18,18 +18,25 @@ func TestGlobalRateLimitBucketsAreIndependent(t *testing.T) {gin.SetMode(gin.TestMode)for _, firstPath := range []string{"/collections", "/files/thumbnail/v3/1"} {t.Run(firstPath, func(t *testing.T) {- rl := &RateLimitMiddleware{limit: 2}+ rl := NewRateLimitMiddleware(nil, 3, 2, time.Hour)+ t.Cleanup(rl.Stop)router := newGlobalRateLimitTestRouter(rl)- paths := []string{"/collections", "/files/thumbnail/v3/1"}- if firstPath != paths[0] {+ paths := []struct {+ path string+ limit int+ }{+ {"/collections", 3},+ {"/files/thumbnail/v3/1", 2},+ }+ if firstPath != paths[0].path {paths[0], paths[1] = paths[1], paths[0]}- for _, path := range paths {- for range rl.limit {- require.Equal(t, http.StatusNoContent, globalRateLimitTestRequest(router, http.MethodGet, path))+ for _, tc := range paths {+ for range tc.limit {+ require.Equal(t, http.StatusNoContent, globalRateLimitTestRequest(router, http.MethodGet, tc.path))}for range 5 {- require.Equal(t, http.StatusTooManyRequests, globalRateLimitTestRequest(router, http.MethodGet, path))+ require.Equal(t, http.StatusTooManyRequests, globalRateLimitTestRequest(router, http.MethodGet, tc.path))}}})@@ -38,7 +45,7 @@ func TestGlobalRateLimitBucketsAreIndependent(t *testing.T) {func TestGlobalRateLimitThumbnailRoutesShareBudget(t *testing.T) {gin.SetMode(gin.TestMode)- rl := &RateLimitMiddleware{limit: 1}+ rl := &RateLimitMiddleware{limit: 1, thumbnailLimit: 1}router := newGlobalRateLimitTestRouter(rl)require.Equal(t, http.StatusNoContent, globalRateLimitTestRequest(router, http.MethodGet, "/files/thumbnail/v3/1"))for _, route := range []string{@@ -66,7 +73,7 @@ func TestGlobalRateLimitOtherRoutesRetainGeneralBudget(t *testing.T) {{http.MethodOptions, "/files/thumbnail/v3/1"},} {t.Run(tc.method+tc.path, func(t *testing.T) {- rl := &RateLimitMiddleware{limit: 1}+ rl := &RateLimitMiddleware{limit: 1, thumbnailLimit: 1}router := newGlobalRateLimitTestRouter(rl)router.GET("/files/download/:fileID", func(c *gin.Context) { c.Status(http.StatusNoContent) })router.GET("/files/data/preview", func(c *gin.Context) { c.Status(http.StatusNoContent) })@@ -80,7 +87,7 @@ func TestGlobalRateLimitOtherRoutesRetainGeneralBudget(t *testing.T) {func TestGlobalRateLimitResetsBothBuckets(t *testing.T) {gin.SetMode(gin.TestMode)- rl := NewRateLimitMiddleware(nil, 1, time.Hour)+ rl := NewRateLimitMiddleware(nil, 1, 1, time.Hour)t.Cleanup(rl.Stop)router := newGlobalRateLimitTestRouter(rl)for _, path := range []string{"/collections", "/files/thumbnail/v3/1"} {@@ -98,7 +105,7 @@ func TestGlobalRateLimitResetsBothBuckets(t *testing.T) {func TestGlobalRateLimitConcurrentRequests(t *testing.T) {gin.SetMode(gin.TestMode)- rl := &RateLimitMiddleware{limit: 10}+ rl := &RateLimitMiddleware{limit: 10, thumbnailLimit: 20}router := newGlobalRateLimitTestRouter(rl)var accepted [2]int64var wg sync.WaitGroup@@ -114,7 +121,7 @@ func TestGlobalRateLimitConcurrentRequests(t *testing.T) {}}wg.Wait()- require.Equal(t, [2]int64{10, 10}, accepted)+ require.Equal(t, [2]int64{10, 20}, accepted)}func TestGlobalRateLimitRejectionMetricsIdentifyBucket(t *testing.T) {diff --git a/server/pkg/middleware/space_test.go b/server/pkg/middleware/space_test.goindex 1a8dc974f9..8a5f90e271 100644--- a/server/pkg/middleware/space_test.go+++ b/server/pkg/middleware/space_test.go@@ -13,7 +13,7 @@ import (func TestSpacePresignAllowsPhotoBatchesAfterProfileUploads(t *testing.T) {gin.SetMode(gin.TestMode)- rateLimiter := NewRateLimitMiddleware(discord.NewDiscordController(nil, "test", "test"), 1000, time.Minute)+ rateLimiter := NewRateLimitMiddleware(discord.NewDiscordController(nil, "test", "test"), 1000, 1000, time.Minute)t.Cleanup(rateLimiter.Stop)router := gin.New()router.POST("/spaces/:spaceID/uploads/presign", rateLimiter.APIRateLimitForUserMiddleware(func(c *gin.Context) string {