Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[server] Keep thumbnail rate limit independently configurable

Neeraj Gupta61cb531d43d0
diff --git a/server/cmd/museum/main.go b/server/cmd/museum/main.go
index 6614ba9a18..f8b7e757bb 100644
--- a/server/cmd/museum/main.go
+++ b/server/cmd/museum/main.go
@@ -228,7 +228,7 @@ func main() {
collectionLinkRepo.Cache = accessTokenCache
discordController := discord.NewDiscordController(userRepo, hostName, environment)
userLookupController := controller.NewUserLookupController(userRepo, discordController)
- rateLimiter := middleware.NewRateLimitMiddleware(discordController, 1000, 1*time.Second)
+ rateLimiter := middleware.NewRateLimitMiddleware(discordController, 1000, 1000, 1*time.Second)
defer rateLimiter.Stop()
emailNotificationCtrl := &email.EmailNotificationController{
diff --git a/server/cmd/museum/main_test.go b/server/cmd/museum/main_test.go
index 1f5fb65265..4bb6b16e22 100644
--- a/server/cmd/museum/main_test.go
+++ b/server/cmd/museum/main_test.go
@@ -71,7 +71,7 @@ func TestURLSanitizerSharesRateLimitsAcrossEncodings(t *testing.T) {
{"/file-link/verify-password", "/file-link/verify-%70assword", "/file-link%2Fverify-password"},
} {
t.Run(paths[0], func(t *testing.T) {
- rateLimiter := middleware.NewRateLimitMiddleware(discord.NewDiscordController(nil, "test", "test"), 1000, time.Minute)
+ rateLimiter := middleware.NewRateLimitMiddleware(discord.NewDiscordController(nil, "test", "test"), 1000, 1000, time.Minute)
t.Cleanup(rateLimiter.Stop)
router := gin.New()
router.POST(paths[0], rateLimiter.APIRateLimitMiddleware(urlSanitizer), func(c *gin.Context) {
diff --git a/server/pkg/middleware/rate_limit.go b/server/pkg/middleware/rate_limit.go
index 50e9c4f1c7..8528faeffc 100644
--- a/server/pkg/middleware/rate_limit.go
+++ b/server/pkg/middleware/rate_limit.go
@@ -42,6 +42,7 @@ type RateLimitMiddleware struct {
count int64
thumbnailCount int64
limit int64
+ thumbnailLimit int64
reset time.Duration
ticker *time.Ticker
limit10ReqPerMin *limiter.Limiter
@@ -55,7 +56,7 @@ type RateLimitMiddleware struct {
discordCtrl *discord.DiscordController
}
-func NewRateLimitMiddleware(discordCtrl *discord.DiscordController, limit int64, reset time.Duration) *RateLimitMiddleware {
+func NewRateLimitMiddleware(discordCtrl *discord.DiscordController, limit, thumbnailLimit int64, reset time.Duration) *RateLimitMiddleware {
rl := &RateLimitMiddleware{
limit10ReqPerMin: util.NewRateLimiter("10-M"),
limit60ReqPerMin: util.NewRateLimiter("60-M"),
@@ -67,6 +68,7 @@ func NewRateLimitMiddleware(discordCtrl *discord.DiscordController, limit int64,
limit700ReqPerSec: util.NewRateLimiter("700-S"),
discordCtrl: discordCtrl,
limit: limit,
+ thumbnailLimit: thumbnailLimit,
reset: reset,
ticker: time.NewTicker(reset),
}
@@ -85,17 +87,17 @@ func (r *RateLimitMiddleware) Stop() {
func (r *RateLimitMiddleware) GlobalRateLimiter() gin.HandlerFunc {
return func(c *gin.Context) {
- counter := &r.count
+ counter, limit := &r.count, r.limit
metricURL, name := processGlobalRateLimitURL, "Global"
if c.Request.Method == http.MethodGet && isThumbnailReadURLPath(c.FullPath()) {
- counter = &r.thumbnailCount
+ counter, limit = &r.thumbnailCount, r.thumbnailLimit
metricURL, name = processThumbnailRateLimitURL, "Thumbnail"
}
count := atomic.AddInt64(counter, 1)
- if count > r.limit {
+ if count > limit {
recordRateLimitRejection(rateLimitScopeProcessGlobal, c.Request.Method, metricURL)
if count%100 == 0 {
- go r.discordCtrl.NotifyPotentialAbuse(fmt.Sprintf("%s ratelimit (%d) breached %d", name, r.limit, count))
+ go r.discordCtrl.NotifyPotentialAbuse(fmt.Sprintf("%s ratelimit (%d) breached %d", name, limit, count))
}
c.AbortWithStatusJSON(http.StatusTooManyRequests, gin.H{"error": "Rate limit breached, try later"})
return
diff --git a/server/pkg/middleware/rate_limit_global_test.go b/server/pkg/middleware/rate_limit_global_test.go
index 648ef8ecc8..6744a216ab 100644
--- a/server/pkg/middleware/rate_limit_global_test.go
+++ b/server/pkg/middleware/rate_limit_global_test.go
@@ -18,18 +18,25 @@ func TestGlobalRateLimitBucketsAreIndependent(t *testing.T) {
gin.SetMode(gin.TestMode)
for _, firstPath := range []string{"/collections", "/files/thumbnail/v3/1"} {
t.Run(firstPath, func(t *testing.T) {
- rl := &RateLimitMiddleware{limit: 2}
+ rl := NewRateLimitMiddleware(nil, 3, 2, time.Hour)
+ t.Cleanup(rl.Stop)
router := newGlobalRateLimitTestRouter(rl)
- paths := []string{"/collections", "/files/thumbnail/v3/1"}
- if firstPath != paths[0] {
+ paths := []struct {
+ path string
+ limit int
+ }{
+ {"/collections", 3},
+ {"/files/thumbnail/v3/1", 2},
+ }
+ if firstPath != paths[0].path {
paths[0], paths[1] = paths[1], paths[0]
}
- for _, path := range paths {
- for range rl.limit {
- require.Equal(t, http.StatusNoContent, globalRateLimitTestRequest(router, http.MethodGet, path))
+ for _, tc := range paths {
+ for range tc.limit {
+ require.Equal(t, http.StatusNoContent, globalRateLimitTestRequest(router, http.MethodGet, tc.path))
}
for range 5 {
- require.Equal(t, http.StatusTooManyRequests, globalRateLimitTestRequest(router, http.MethodGet, path))
+ require.Equal(t, http.StatusTooManyRequests, globalRateLimitTestRequest(router, http.MethodGet, tc.path))
}
}
})
@@ -38,7 +45,7 @@ func TestGlobalRateLimitBucketsAreIndependent(t *testing.T) {
func TestGlobalRateLimitThumbnailRoutesShareBudget(t *testing.T) {
gin.SetMode(gin.TestMode)
- rl := &RateLimitMiddleware{limit: 1}
+ rl := &RateLimitMiddleware{limit: 1, thumbnailLimit: 1}
router := newGlobalRateLimitTestRouter(rl)
require.Equal(t, http.StatusNoContent, globalRateLimitTestRequest(router, http.MethodGet, "/files/thumbnail/v3/1"))
for _, route := range []string{
@@ -66,7 +73,7 @@ func TestGlobalRateLimitOtherRoutesRetainGeneralBudget(t *testing.T) {
{http.MethodOptions, "/files/thumbnail/v3/1"},
} {
t.Run(tc.method+tc.path, func(t *testing.T) {
- rl := &RateLimitMiddleware{limit: 1}
+ rl := &RateLimitMiddleware{limit: 1, thumbnailLimit: 1}
router := newGlobalRateLimitTestRouter(rl)
router.GET("/files/download/:fileID", func(c *gin.Context) { c.Status(http.StatusNoContent) })
router.GET("/files/data/preview", func(c *gin.Context) { c.Status(http.StatusNoContent) })
@@ -80,7 +87,7 @@ func TestGlobalRateLimitOtherRoutesRetainGeneralBudget(t *testing.T) {
func TestGlobalRateLimitResetsBothBuckets(t *testing.T) {
gin.SetMode(gin.TestMode)
- rl := NewRateLimitMiddleware(nil, 1, time.Hour)
+ rl := NewRateLimitMiddleware(nil, 1, 1, time.Hour)
t.Cleanup(rl.Stop)
router := newGlobalRateLimitTestRouter(rl)
for _, path := range []string{"/collections", "/files/thumbnail/v3/1"} {
@@ -98,7 +105,7 @@ func TestGlobalRateLimitResetsBothBuckets(t *testing.T) {
func TestGlobalRateLimitConcurrentRequests(t *testing.T) {
gin.SetMode(gin.TestMode)
- rl := &RateLimitMiddleware{limit: 10}
+ rl := &RateLimitMiddleware{limit: 10, thumbnailLimit: 20}
router := newGlobalRateLimitTestRouter(rl)
var accepted [2]int64
var wg sync.WaitGroup
@@ -114,7 +121,7 @@ func TestGlobalRateLimitConcurrentRequests(t *testing.T) {
}
}
wg.Wait()
- require.Equal(t, [2]int64{10, 10}, accepted)
+ require.Equal(t, [2]int64{10, 20}, accepted)
}
func TestGlobalRateLimitRejectionMetricsIdentifyBucket(t *testing.T) {
diff --git a/server/pkg/middleware/space_test.go b/server/pkg/middleware/space_test.go
index 1a8dc974f9..8a5f90e271 100644
--- a/server/pkg/middleware/space_test.go
+++ b/server/pkg/middleware/space_test.go
@@ -13,7 +13,7 @@ import (
func TestSpacePresignAllowsPhotoBatchesAfterProfileUploads(t *testing.T) {
gin.SetMode(gin.TestMode)
- rateLimiter := NewRateLimitMiddleware(discord.NewDiscordController(nil, "test", "test"), 1000, time.Minute)
+ rateLimiter := NewRateLimitMiddleware(discord.NewDiscordController(nil, "test", "test"), 1000, 1000, time.Minute)
t.Cleanup(rateLimiter.Stop)
router := gin.New()
router.POST("/spaces/:spaceID/uploads/presign", rateLimiter.APIRateLimitForUserMiddleware(func(c *gin.Context) string {