π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Limit Museum request bodies to 4 MiB
Neeraj Gupta21650638e6dd
diff --git a/docs/docs/self-hosting/guides/photos-k8s-helm.md b/docs/docs/self-hosting/guides/photos-k8s-helm.mdindex 68fbf1748e..69abb0f8ee 100644--- a/docs/docs/self-hosting/guides/photos-k8s-helm.md+++ b/docs/docs/self-hosting/guides/photos-k8s-helm.md@@ -164,7 +164,7 @@ museum:className: nginxannotations:cert-manager.io/cluster-issuer: letsencrypt-prod- nginx.ingress.kubernetes.io/proxy-body-size: "50g"+ nginx.ingress.kubernetes.io/proxy-body-size: "4m"hosts:- host: api.photos.example.compaths:diff --git a/server/cmd/museum/main.go b/server/cmd/museum/main.goindex 344becb4b7..4a88d541c2 100644--- a/server/cmd/museum/main.go+++ b/server/cmd/museum/main.go@@ -537,7 +537,7 @@ func main() {return base.ServerReqID()},}),- middleware.Logger(urlSanitizer), cors(), cacheHeaders(),+ cors(), middleware.LimitRequestBody(), middleware.Logger(urlSanitizer), cacheHeaders(),gzip.Gzip(gzip.DefaultCompression), middleware.PanicRecover())publicAPI := server.Group("/")diff --git a/server/pkg/middleware/request_body.go b/server/pkg/middleware/request_body.gonew file mode 100644index 0000000000..85c801f007--- /dev/null+++ b/server/pkg/middleware/request_body.go@@ -0,0 +1,22 @@+package middleware++import (+ "net/http"++ "github.com/ente/museum/pkg/utils/handler"+ "github.com/gin-gonic/gin"+)++// Match client_max_body_size in scripts/deploy/museum.nginx.conf.+const maxRequestBodySize = 4 << 20++func LimitRequestBody() gin.HandlerFunc {+ return func(c *gin.Context) {+ if c.Request.ContentLength > maxRequestBodySize {+ handler.Error(c, &http.MaxBytesError{Limit: maxRequestBodySize})+ return+ }+ c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxRequestBodySize)+ c.Next()+ }+}diff --git a/server/pkg/middleware/request_body_test.go b/server/pkg/middleware/request_body_test.gonew file mode 100644index 0000000000..c45931bc0e--- /dev/null+++ b/server/pkg/middleware/request_body_test.go@@ -0,0 +1,97 @@+package middleware++import (+ "io"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"+ "testing/iotest"++ "github.com/gin-gonic/gin"+ "github.com/sirupsen/logrus"+ logtest "github.com/sirupsen/logrus/hooks/test"+ "github.com/stretchr/testify/require"+)++func TestLimitRequestBody(t *testing.T) {+ gin.SetMode(gin.TestMode)+ logger := logrus.StandardLogger()+ originalOutput := logger.Out+ logger.SetOutput(io.Discard)+ t.Cleanup(func() { logger.SetOutput(originalOutput) })++ for _, tc := range []struct {+ name string+ size int+ }{+ {"empty", 0},+ {"below limit", maxRequestBodySize - 1},+ {"at limit", maxRequestBodySize},+ {"above limit", maxRequestBodySize + 1},+ } {+ body := strings.Repeat("a", tc.size)+ for _, framing := range []string{"known length", "unknown length"} {+ t.Run(tc.name+"/"+framing, func(t *testing.T) {+ // Exercise both ordinary body logging and the redacted path.+ for _, path := range []string{"/test", "/events"} {+ t.Run(path, func(t *testing.T) {+ router := gin.New()+ router.Use(LimitRequestBody(), Logger(func(c *gin.Context) string { return c.FullPath() }))+ called := false+ router.POST(path, func(c *gin.Context) {+ called = true+ got, err := io.ReadAll(c.Request.Body)+ require.NoError(t, err)+ require.True(t, string(got) == body, "accepted request body changed")+ c.Status(http.StatusNoContent)+ })+ reader := strings.NewReader(body)+ req := httptest.NewRequest(http.MethodPost, path, reader)+ if framing == "unknown length" {+ req.ContentLength = -1+ req.TransferEncoding = []string{"chunked"}+ }+ response := httptest.NewRecorder()+ router.ServeHTTP(response, req)++ if tc.size > maxRequestBodySize {+ require.Equal(t, http.StatusRequestEntityTooLarge, response.Code)+ require.False(t, called)+ if framing == "known length" {+ require.Equal(t, tc.size, reader.Len(), "oversized declared body was read")+ }+ } else {+ require.Equal(t, http.StatusNoContent, response.Code)+ require.True(t, called)+ }+ })+ }+ })+ }+ }+}++func TestLoggerStopsOnBodyReadError(t *testing.T) {+ gin.SetMode(gin.TestMode)+ logger := logrus.StandardLogger()+ originalHooks := logger.ReplaceHooks(make(logrus.LevelHooks))+ t.Cleanup(func() { logger.ReplaceHooks(originalHooks) })+ hook := logtest.NewGlobal()++ router := gin.New()+ router.Use(LimitRequestBody(), Logger(func(c *gin.Context) string { return c.FullPath() }))+ router.POST("/test", func(c *gin.Context) {+ t.Fatal("handler called after body read error")+ })+ body := io.MultiReader(strings.NewReader("partial-body"), iotest.ErrReader(io.ErrUnexpectedEOF))+ req := httptest.NewRequest(http.MethodPost, "/test", body)+ response := httptest.NewRecorder()+ router.ServeHTTP(response, req)++ require.Equal(t, http.StatusBadRequest, response.Code)+ entries := hook.AllEntries()+ require.Len(t, entries, 1)+ require.Equal(t, "Request failed", entries[0].Message)+ require.NotContains(t, entries[0].Data, "req_body")+}diff --git a/server/pkg/middleware/request_logger.go b/server/pkg/middleware/request_logger.goindex 333b5d3d9c..7e332ae65e 100644--- a/server/pkg/middleware/request_logger.go+++ b/server/pkg/middleware/request_logger.go@@ -108,6 +108,7 @@ func Logger(urlSanitizer func(_ *gin.Context) string) gin.HandlerFunc {buf, err := io.ReadAll(c.Request.Body)if err != nil {handler.Error(c, err)+ return}rdr1 := io.NopCloser(bytes.NewBuffer(buf))rdr2 := io.NopCloser(bytes.NewBuffer(buf))diff --git a/server/pkg/utils/handler/handler.go b/server/pkg/utils/handler/handler.goindex d8d5fd1861..379683f4d7 100644--- a/server/pkg/utils/handler/handler.go+++ b/server/pkg/utils/handler/handler.go@@ -92,6 +92,7 @@ func requestFailureLogLevel(err error, apiErr *ente.ApiError) *log.Level {errors.Is(err, ente.ErrUserDeleted) ||errors.Is(err, sql.ErrNoRows) ||errors.Is(err, errBindJSON) ||+ isRequestBodyTooLarge(err) ||isRequestIOError(err) ||(apiErr != nil && apiErr.HttpStatusCode >= 400 && apiErr.HttpStatusCode < 500) {return logLevel(log.WarnLevel)@@ -109,8 +110,15 @@ func isRequestIOError(err error) bool {errors.Is(err, syscall.ECONNRESET)}+func isRequestBodyTooLarge(err error) bool {+ var sizeError *http.MaxBytesError+ return errors.As(err, &sizeError)+}+func httpStatusCode(err error) int {switch {+ case isRequestBodyTooLarge(err):+ return http.StatusRequestEntityTooLargecase errors.Is(err, ente.ErrNotFound) ||errors.Is(err, sql.ErrNoRows):return http.StatusNotFounddiff --git a/server/pkg/utils/handler/handler_test.go b/server/pkg/utils/handler/handler_test.goindex 0627c740d9..74e4b8e8fc 100644--- a/server/pkg/utils/handler/handler_test.go+++ b/server/pkg/utils/handler/handler_test.go@@ -82,6 +82,16 @@ func TestErrorLogsWarnForExpectedClientErrors(t *testing.T) {err: ente.ErrAuthenticationRequired,wantStatus: http.StatusUnauthorized,},+ {+ name: "request body too large",+ err: &http.MaxBytesError{Limit: 4 << 20},+ wantStatus: http.StatusRequestEntityTooLarge,+ },+ {+ name: "request body too large while binding JSON",+ err: bindJSONError{err: &http.MaxBytesError{Limit: 4 << 20}},+ wantStatus: http.StatusRequestEntityTooLarge,+ },} {t.Run(tt.name, func(t *testing.T) {recorder, ctx := testContext()