Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Limit Museum request bodies to 4 MiB

Neeraj Gupta21650638e6dd
diff --git a/docs/docs/self-hosting/guides/photos-k8s-helm.md b/docs/docs/self-hosting/guides/photos-k8s-helm.md
index 68fbf1748e..69abb0f8ee 100644
--- a/docs/docs/self-hosting/guides/photos-k8s-helm.md
+++ b/docs/docs/self-hosting/guides/photos-k8s-helm.md
@@ -164,7 +164,7 @@ museum:
className: nginx
annotations:
cert-manager.io/cluster-issuer: letsencrypt-prod
- nginx.ingress.kubernetes.io/proxy-body-size: "50g"
+ nginx.ingress.kubernetes.io/proxy-body-size: "4m"
hosts:
- host: api.photos.example.com
paths:
diff --git a/server/cmd/museum/main.go b/server/cmd/museum/main.go
index 344becb4b7..4a88d541c2 100644
--- a/server/cmd/museum/main.go
+++ b/server/cmd/museum/main.go
@@ -537,7 +537,7 @@ func main() {
return base.ServerReqID()
},
}),
- middleware.Logger(urlSanitizer), cors(), cacheHeaders(),
+ cors(), middleware.LimitRequestBody(), middleware.Logger(urlSanitizer), cacheHeaders(),
gzip.Gzip(gzip.DefaultCompression), middleware.PanicRecover())
publicAPI := server.Group("/")
diff --git a/server/pkg/middleware/request_body.go b/server/pkg/middleware/request_body.go
new file mode 100644
index 0000000000..85c801f007
--- /dev/null
+++ b/server/pkg/middleware/request_body.go
@@ -0,0 +1,22 @@
+package middleware
+
+import (
+ "net/http"
+
+ "github.com/ente/museum/pkg/utils/handler"
+ "github.com/gin-gonic/gin"
+)
+
+// Match client_max_body_size in scripts/deploy/museum.nginx.conf.
+const maxRequestBodySize = 4 << 20
+
+func LimitRequestBody() gin.HandlerFunc {
+ return func(c *gin.Context) {
+ if c.Request.ContentLength > maxRequestBodySize {
+ handler.Error(c, &http.MaxBytesError{Limit: maxRequestBodySize})
+ return
+ }
+ c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxRequestBodySize)
+ c.Next()
+ }
+}
diff --git a/server/pkg/middleware/request_body_test.go b/server/pkg/middleware/request_body_test.go
new file mode 100644
index 0000000000..c45931bc0e
--- /dev/null
+++ b/server/pkg/middleware/request_body_test.go
@@ -0,0 +1,97 @@
+package middleware
+
+import (
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "testing/iotest"
+
+ "github.com/gin-gonic/gin"
+ "github.com/sirupsen/logrus"
+ logtest "github.com/sirupsen/logrus/hooks/test"
+ "github.com/stretchr/testify/require"
+)
+
+func TestLimitRequestBody(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ logger := logrus.StandardLogger()
+ originalOutput := logger.Out
+ logger.SetOutput(io.Discard)
+ t.Cleanup(func() { logger.SetOutput(originalOutput) })
+
+ for _, tc := range []struct {
+ name string
+ size int
+ }{
+ {"empty", 0},
+ {"below limit", maxRequestBodySize - 1},
+ {"at limit", maxRequestBodySize},
+ {"above limit", maxRequestBodySize + 1},
+ } {
+ body := strings.Repeat("a", tc.size)
+ for _, framing := range []string{"known length", "unknown length"} {
+ t.Run(tc.name+"/"+framing, func(t *testing.T) {
+ // Exercise both ordinary body logging and the redacted path.
+ for _, path := range []string{"/test", "/events"} {
+ t.Run(path, func(t *testing.T) {
+ router := gin.New()
+ router.Use(LimitRequestBody(), Logger(func(c *gin.Context) string { return c.FullPath() }))
+ called := false
+ router.POST(path, func(c *gin.Context) {
+ called = true
+ got, err := io.ReadAll(c.Request.Body)
+ require.NoError(t, err)
+ require.True(t, string(got) == body, "accepted request body changed")
+ c.Status(http.StatusNoContent)
+ })
+ reader := strings.NewReader(body)
+ req := httptest.NewRequest(http.MethodPost, path, reader)
+ if framing == "unknown length" {
+ req.ContentLength = -1
+ req.TransferEncoding = []string{"chunked"}
+ }
+ response := httptest.NewRecorder()
+ router.ServeHTTP(response, req)
+
+ if tc.size > maxRequestBodySize {
+ require.Equal(t, http.StatusRequestEntityTooLarge, response.Code)
+ require.False(t, called)
+ if framing == "known length" {
+ require.Equal(t, tc.size, reader.Len(), "oversized declared body was read")
+ }
+ } else {
+ require.Equal(t, http.StatusNoContent, response.Code)
+ require.True(t, called)
+ }
+ })
+ }
+ })
+ }
+ }
+}
+
+func TestLoggerStopsOnBodyReadError(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ logger := logrus.StandardLogger()
+ originalHooks := logger.ReplaceHooks(make(logrus.LevelHooks))
+ t.Cleanup(func() { logger.ReplaceHooks(originalHooks) })
+ hook := logtest.NewGlobal()
+
+ router := gin.New()
+ router.Use(LimitRequestBody(), Logger(func(c *gin.Context) string { return c.FullPath() }))
+ router.POST("/test", func(c *gin.Context) {
+ t.Fatal("handler called after body read error")
+ })
+ body := io.MultiReader(strings.NewReader("partial-body"), iotest.ErrReader(io.ErrUnexpectedEOF))
+ req := httptest.NewRequest(http.MethodPost, "/test", body)
+ response := httptest.NewRecorder()
+ router.ServeHTTP(response, req)
+
+ require.Equal(t, http.StatusBadRequest, response.Code)
+ entries := hook.AllEntries()
+ require.Len(t, entries, 1)
+ require.Equal(t, "Request failed", entries[0].Message)
+ require.NotContains(t, entries[0].Data, "req_body")
+}
diff --git a/server/pkg/middleware/request_logger.go b/server/pkg/middleware/request_logger.go
index 333b5d3d9c..7e332ae65e 100644
--- a/server/pkg/middleware/request_logger.go
+++ b/server/pkg/middleware/request_logger.go
@@ -108,6 +108,7 @@ func Logger(urlSanitizer func(_ *gin.Context) string) gin.HandlerFunc {
buf, err := io.ReadAll(c.Request.Body)
if err != nil {
handler.Error(c, err)
+ return
}
rdr1 := io.NopCloser(bytes.NewBuffer(buf))
rdr2 := io.NopCloser(bytes.NewBuffer(buf))
diff --git a/server/pkg/utils/handler/handler.go b/server/pkg/utils/handler/handler.go
index d8d5fd1861..379683f4d7 100644
--- a/server/pkg/utils/handler/handler.go
+++ b/server/pkg/utils/handler/handler.go
@@ -92,6 +92,7 @@ func requestFailureLogLevel(err error, apiErr *ente.ApiError) *log.Level {
errors.Is(err, ente.ErrUserDeleted) ||
errors.Is(err, sql.ErrNoRows) ||
errors.Is(err, errBindJSON) ||
+ isRequestBodyTooLarge(err) ||
isRequestIOError(err) ||
(apiErr != nil && apiErr.HttpStatusCode >= 400 && apiErr.HttpStatusCode < 500) {
return logLevel(log.WarnLevel)
@@ -109,8 +110,15 @@ func isRequestIOError(err error) bool {
errors.Is(err, syscall.ECONNRESET)
}
+func isRequestBodyTooLarge(err error) bool {
+ var sizeError *http.MaxBytesError
+ return errors.As(err, &sizeError)
+}
+
func httpStatusCode(err error) int {
switch {
+ case isRequestBodyTooLarge(err):
+ return http.StatusRequestEntityTooLarge
case errors.Is(err, ente.ErrNotFound) ||
errors.Is(err, sql.ErrNoRows):
return http.StatusNotFound
diff --git a/server/pkg/utils/handler/handler_test.go b/server/pkg/utils/handler/handler_test.go
index 0627c740d9..74e4b8e8fc 100644
--- a/server/pkg/utils/handler/handler_test.go
+++ b/server/pkg/utils/handler/handler_test.go
@@ -82,6 +82,16 @@ func TestErrorLogsWarnForExpectedClientErrors(t *testing.T) {
err: ente.ErrAuthenticationRequired,
wantStatus: http.StatusUnauthorized,
},
+ {
+ name: "request body too large",
+ err: &http.MaxBytesError{Limit: 4 << 20},
+ wantStatus: http.StatusRequestEntityTooLarge,
+ },
+ {
+ name: "request body too large while binding JSON",
+ err: bindJSONError{err: &http.MaxBytesError{Limit: 4 << 20}},
+ wantStatus: http.StatusRequestEntityTooLarge,
+ },
} {
t.Run(tt.name, func(t *testing.T) {
recorder, ctx := testContext()