Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[server] Separate thumbnail request limits

Neeraj Gupta4fccba4e0c8a
diff --git a/server/cmd/museum/main.go b/server/cmd/museum/main.go
index 6614ba9a18..f8b7e757bb 100644
--- a/server/cmd/museum/main.go
+++ b/server/cmd/museum/main.go
@@ -228,7 +228,7 @@ func main() {
collectionLinkRepo.Cache = accessTokenCache
discordController := discord.NewDiscordController(userRepo, hostName, environment)
userLookupController := controller.NewUserLookupController(userRepo, discordController)
- rateLimiter := middleware.NewRateLimitMiddleware(discordController, 1000, 1*time.Second)
+ rateLimiter := middleware.NewRateLimitMiddleware(discordController, 1000, 1000, 1*time.Second)
defer rateLimiter.Stop()
emailNotificationCtrl := &email.EmailNotificationController{
diff --git a/server/cmd/museum/main_test.go b/server/cmd/museum/main_test.go
index 1f5fb65265..4bb6b16e22 100644
--- a/server/cmd/museum/main_test.go
+++ b/server/cmd/museum/main_test.go
@@ -71,7 +71,7 @@ func TestURLSanitizerSharesRateLimitsAcrossEncodings(t *testing.T) {
{"/file-link/verify-password", "/file-link/verify-%70assword", "/file-link%2Fverify-password"},
} {
t.Run(paths[0], func(t *testing.T) {
- rateLimiter := middleware.NewRateLimitMiddleware(discord.NewDiscordController(nil, "test", "test"), 1000, time.Minute)
+ rateLimiter := middleware.NewRateLimitMiddleware(discord.NewDiscordController(nil, "test", "test"), 1000, 1000, time.Minute)
t.Cleanup(rateLimiter.Stop)
router := gin.New()
router.POST(paths[0], rateLimiter.APIRateLimitMiddleware(urlSanitizer), func(c *gin.Context) {
diff --git a/server/pkg/middleware/rate_limit.go b/server/pkg/middleware/rate_limit.go
index 9d7b4c5f4f..21bbb26c89 100644
--- a/server/pkg/middleware/rate_limit.go
+++ b/server/pkg/middleware/rate_limit.go
@@ -24,12 +24,13 @@ import (
type rateLimitScope string
const (
- rateLimitScopeIP rateLimitScope = "ip"
- rateLimitScopeCollection rateLimitScope = "collection"
- rateLimitScopeUser rateLimitScope = "user"
- rateLimitScopeRouteGlobal rateLimitScope = "route_global"
- rateLimitScopeProcessGlobal rateLimitScope = "process_global"
- processGlobalRateLimitURL = "all"
+ rateLimitScopeIP rateLimitScope = "ip"
+ rateLimitScopeCollection rateLimitScope = "collection"
+ rateLimitScopeUser rateLimitScope = "user"
+ rateLimitScopeRouteGlobal rateLimitScope = "route_global"
+ rateLimitScopeProcessGlobal rateLimitScope = "process_global"
+ processGlobalRateLimitURL = "all"
+ processThumbnailRateLimitURL = "thumbnails"
)
var rateLimitRejections = promauto.NewCounterVec(prometheus.CounterOpts{
@@ -39,7 +40,9 @@ var rateLimitRejections = promauto.NewCounterVec(prometheus.CounterOpts{
type RateLimitMiddleware struct {
count int64
+ thumbnailCount int64
limit int64
+ thumbnailLimit int64
reset time.Duration
ticker *time.Ticker
limit10ReqPerMin *limiter.Limiter
@@ -53,7 +56,7 @@ type RateLimitMiddleware struct {
discordCtrl *discord.DiscordController
}
-func NewRateLimitMiddleware(discordCtrl *discord.DiscordController, limit int64, reset time.Duration) *RateLimitMiddleware {
+func NewRateLimitMiddleware(discordCtrl *discord.DiscordController, limit, thumbnailLimit int64, reset time.Duration) *RateLimitMiddleware {
rl := &RateLimitMiddleware{
limit10ReqPerMin: util.NewRateLimiter("10-M"),
limit60ReqPerMin: util.NewRateLimiter("60-M"),
@@ -65,32 +68,36 @@ func NewRateLimitMiddleware(discordCtrl *discord.DiscordController, limit int64,
limit700ReqPerSec: util.NewRateLimiter("700-S"),
discordCtrl: discordCtrl,
limit: limit,
+ thumbnailLimit: thumbnailLimit,
reset: reset,
ticker: time.NewTicker(reset),
}
go func() {
for range rl.ticker.C {
atomic.StoreInt64(&rl.count, 0)
+ atomic.StoreInt64(&rl.thumbnailCount, 0)
}
}()
return rl
}
-func (r *RateLimitMiddleware) Increment() bool {
- newCount := atomic.AddInt64(&r.count, 1)
- return newCount <= r.limit
-}
-
func (r *RateLimitMiddleware) Stop() {
r.ticker.Stop()
}
func (r *RateLimitMiddleware) GlobalRateLimiter() gin.HandlerFunc {
return func(c *gin.Context) {
- if !r.Increment() {
- recordRateLimitRejection(rateLimitScopeProcessGlobal, c.Request.Method, processGlobalRateLimitURL)
- if r.count%100 == 0 {
- go r.discordCtrl.NotifyPotentialAbuse(fmt.Sprintf("Global ratelimit (%d) breached %d", r.limit, r.count))
+ counter, limit := &r.count, r.limit
+ metricURL, name := processGlobalRateLimitURL, "Global"
+ if c.Request.Method == http.MethodGet && isThumbnailReadURLPath(c.FullPath()) {
+ counter, limit = &r.thumbnailCount, r.thumbnailLimit
+ metricURL, name = processThumbnailRateLimitURL, "Thumbnail"
+ }
+ count := atomic.AddInt64(counter, 1)
+ if count > limit {
+ recordRateLimitRejection(rateLimitScopeProcessGlobal, c.Request.Method, metricURL)
+ if count%100 == 0 {
+ go r.discordCtrl.NotifyPotentialAbuse(fmt.Sprintf("%s ratelimit (%d) breached %d", name, limit, count))
}
c.AbortWithStatusJSON(http.StatusTooManyRequests, gin.H{"error": "Rate limit breached, try later"})
return
@@ -99,6 +106,20 @@ func (r *RateLimitMiddleware) GlobalRateLimiter() gin.HandlerFunc {
}
}
+// Match registered routes so unknown paths and writes retain the general budget.
+func isThumbnailReadURLPath(reqPath string) bool {
+ switch reqPath {
+ case "/files/preview/:fileID", "/files/preview/v2/:fileID", "/files/thumbnail/v3/:fileID",
+ "/public-collection/files/preview/:fileID", "/public-collection/files/thumbnail/v3/:fileID",
+ "/public-memory/files/preview/:fileID", "/public-memory/files/thumbnail/v3/:fileID",
+ "/cast/files/preview/:fileID", "/cast/files/thumbnail/v3/:fileID",
+ "/file-link/thumbnail", "/file-link/thumbnail/v3":
+ return true
+ default:
+ return false
+ }
+}
+
func (r *RateLimitMiddleware) APIRateLimitMiddleware(urlSanitizer func(_ *gin.Context) string) gin.HandlerFunc {
return func(c *gin.Context) {
requestPath := urlSanitizer(c)
diff --git a/server/pkg/middleware/rate_limit_global_test.go b/server/pkg/middleware/rate_limit_global_test.go
new file mode 100644
index 0000000000..a1a063bc4f
--- /dev/null
+++ b/server/pkg/middleware/rate_limit_global_test.go
@@ -0,0 +1,149 @@
+package middleware
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/gin-gonic/gin"
+ "github.com/prometheus/client_golang/prometheus/testutil"
+ "github.com/stretchr/testify/require"
+)
+
+func TestGlobalRateLimitBucketsAreIndependent(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ for _, thumbnailFirst := range []bool{true, false} {
+ t.Run(map[bool]string{true: "thumbnail flood", false: "general flood"}[thumbnailFirst], func(t *testing.T) {
+ rl := &RateLimitMiddleware{limit: 3, thumbnailLimit: 2}
+ router := newGlobalRateLimitTestRouter(rl)
+ paths := []string{"/collections", "/files/thumbnail/v3/1"}
+ limits := []int{3, 2}
+ if thumbnailFirst {
+ paths[0], paths[1] = paths[1], paths[0]
+ limits[0], limits[1] = limits[1], limits[0]
+ }
+ for i, path := range paths {
+ for range limits[i] {
+ require.Equal(t, http.StatusNoContent, globalRateLimitTestRequest(router, http.MethodGet, path))
+ }
+ for range 5 {
+ require.Equal(t, http.StatusTooManyRequests, globalRateLimitTestRequest(router, http.MethodGet, path))
+ }
+ }
+ })
+ }
+}
+
+func TestGlobalRateLimitThumbnailRoutesShareBudget(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ rl := &RateLimitMiddleware{limit: 1, thumbnailLimit: 1}
+ router := newGlobalRateLimitTestRouter(rl)
+ require.Equal(t, http.StatusNoContent, globalRateLimitTestRequest(router, http.MethodGet, "/files/thumbnail/v3/1"))
+ for _, route := range []string{
+ "/files/preview/:fileID", "/files/preview/v2/:fileID",
+ "/public-collection/files/preview/:fileID", "/public-collection/files/thumbnail/v3/:fileID",
+ "/public-memory/files/preview/:fileID", "/public-memory/files/thumbnail/v3/:fileID",
+ "/cast/files/preview/:fileID", "/cast/files/thumbnail/v3/:fileID",
+ "/file-link/thumbnail", "/file-link/thumbnail/v3",
+ } {
+ router.GET(route, func(c *gin.Context) { c.Status(http.StatusNoContent) })
+ path := strings.ReplaceAll(route, ":fileID", "2")
+ require.Equal(t, http.StatusTooManyRequests, globalRateLimitTestRequest(router, http.MethodGet, path), route)
+ }
+ require.Equal(t, http.StatusNoContent, globalRateLimitTestRequest(router, http.MethodGet, "/collections"))
+}
+
+func TestGlobalRateLimitOtherRoutesRetainGeneralBudget(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ for _, tc := range []struct{ method, path string }{
+ {http.MethodGet, "/files/download/1"},
+ {http.MethodGet, "/files/data/preview"},
+ {http.MethodPut, "/files/thumbnail"},
+ {http.MethodPost, "/files/thumbnail/v3/1"},
+ {http.MethodGet, "/files/thumbnail/unknown/1"},
+ {http.MethodOptions, "/files/thumbnail/v3/1"},
+ } {
+ t.Run(tc.method+tc.path, func(t *testing.T) {
+ rl := &RateLimitMiddleware{limit: 0, thumbnailLimit: 1}
+ router := newGlobalRateLimitTestRouter(rl)
+ router.GET("/files/download/:fileID", func(c *gin.Context) { c.Status(http.StatusNoContent) })
+ router.GET("/files/data/preview", func(c *gin.Context) { c.Status(http.StatusNoContent) })
+ router.PUT("/files/thumbnail", func(c *gin.Context) { c.Status(http.StatusNoContent) })
+ require.Equal(t, http.StatusTooManyRequests, globalRateLimitTestRequest(router, tc.method, tc.path))
+ require.Equal(t, http.StatusNoContent, globalRateLimitTestRequest(router, http.MethodGet, "/files/thumbnail/v3/1"))
+ })
+ }
+}
+
+func TestGlobalRateLimitResetsBothBuckets(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ rl := NewRateLimitMiddleware(nil, 1, 1, time.Hour)
+ t.Cleanup(rl.Stop)
+ router := newGlobalRateLimitTestRouter(rl)
+ for _, path := range []string{"/collections", "/files/thumbnail/v3/1"} {
+ require.Equal(t, http.StatusNoContent, globalRateLimitTestRequest(router, http.MethodGet, path))
+ require.Equal(t, http.StatusTooManyRequests, globalRateLimitTestRequest(router, http.MethodGet, path))
+ }
+ rl.ticker.Reset(time.Millisecond)
+ require.Eventually(t, func() bool {
+ return atomic.LoadInt64(&rl.count) == 0 && atomic.LoadInt64(&rl.thumbnailCount) == 0
+ }, time.Second, time.Millisecond)
+ for _, path := range []string{"/collections", "/files/thumbnail/v3/1"} {
+ require.Equal(t, http.StatusNoContent, globalRateLimitTestRequest(router, http.MethodGet, path))
+ }
+}
+
+func TestGlobalRateLimitConcurrentRequests(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ rl := &RateLimitMiddleware{limit: 10, thumbnailLimit: 20}
+ router := newGlobalRateLimitTestRouter(rl)
+ var accepted [2]int64
+ var wg sync.WaitGroup
+ for i, path := range []string{"/collections", "/files/thumbnail/v3/1"} {
+ for range 50 {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ if globalRateLimitTestRequest(router, http.MethodGet, path) == http.StatusNoContent {
+ atomic.AddInt64(&accepted[i], 1)
+ }
+ }()
+ }
+ }
+ wg.Wait()
+ require.Equal(t, [2]int64{10, 20}, accepted)
+}
+
+func TestGlobalRateLimitRejectionMetricsIdentifyBucket(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ rl := &RateLimitMiddleware{}
+ router := newGlobalRateLimitTestRouter(rl)
+ for _, tc := range []struct{ path, label string }{
+ {"/collections", processGlobalRateLimitURL},
+ {"/files/thumbnail/v3/1", processThumbnailRateLimitURL},
+ } {
+ counter := rateLimitRejections.WithLabelValues(string(rateLimitScopeProcessGlobal), http.MethodGet, tc.label)
+ before := testutil.ToFloat64(counter)
+ require.Equal(t, http.StatusTooManyRequests, globalRateLimitTestRequest(router, http.MethodGet, tc.path))
+ require.Equal(t, before+1, testutil.ToFloat64(counter))
+ }
+}
+
+func newGlobalRateLimitTestRouter(rl *RateLimitMiddleware) *gin.Engine {
+ router := gin.New()
+ router.Use(rl.GlobalRateLimiter())
+ for _, path := range []string{"/collections", "/files/thumbnail/v3/:fileID"} {
+ router.GET(path, func(c *gin.Context) { c.Status(http.StatusNoContent) })
+ }
+ return router
+}
+
+func globalRateLimitTestRequest(router *gin.Engine, method, path string) int {
+ response := httptest.NewRecorder()
+ router.ServeHTTP(response, httptest.NewRequest(method, path, nil))
+ return response.Code
+}
diff --git a/server/pkg/middleware/space_test.go b/server/pkg/middleware/space_test.go
index 1a8dc974f9..8a5f90e271 100644
--- a/server/pkg/middleware/space_test.go
+++ b/server/pkg/middleware/space_test.go
@@ -13,7 +13,7 @@ import (
func TestSpacePresignAllowsPhotoBatchesAfterProfileUploads(t *testing.T) {
gin.SetMode(gin.TestMode)
- rateLimiter := NewRateLimitMiddleware(discord.NewDiscordController(nil, "test", "test"), 1000, time.Minute)
+ rateLimiter := NewRateLimitMiddleware(discord.NewDiscordController(nil, "test", "test"), 1000, 1000, time.Minute)
t.Cleanup(rateLimiter.Stop)
router := gin.New()
router.POST("/spaces/:spaceID/uploads/presign", rateLimiter.APIRateLimitForUserMiddleware(func(c *gin.Context) string {