Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(middleware): stop buffering whole request bodies in the access log (#3927)

ordinary being6057a8101737
diff --git a/internal/middleware/logger.go b/internal/middleware/logger.go
index b05a02c35..a65ef0f98 100644
--- a/internal/middleware/logger.go
+++ b/internal/middleware/logger.go
@@ -18,8 +18,17 @@ import (
const (
maxBodySize = 1024 * 10 // 最大记录10KB的body内容
+ // bodyTruncatedMarker 是日志里跟在被截断的 body 后面的标记。
+ bodyTruncatedMarker = "... [内容过长,已截断]"
)
+// replayedBody 把日志窥探消费掉的字节还给 handler,让它读到的 body 与没有这层
+// 中间件时完全一致;Close 仍然关闭原始 body,连接得以释放。
+type replayedBody struct {
+ io.Reader
+ io.Closer
+}
+
// loggerResponseBodyWriter 自定义ResponseWriter用于捕获响应内容(用于logger中间件)
type loggerResponseBodyWriter struct {
gin.ResponseWriter
@@ -87,7 +96,7 @@ func sanitizeQuery(raw string) string {
return values.Encode()
}
-// readRequestBody 读取请求体(限制大小用于日志,但完整读取用于重置)
+// readRequestBody 读取请求体(限制大小用于日志,但把剩余字节原样交还后续handler)
func readRequestBody(c *gin.Context) string {
if c.Request.Body == nil {
return ""
@@ -101,26 +110,34 @@ func readRequestBody(c *gin.Context) string {
return "[非文本类型,已跳过]"
}
- // 完整读取body内容(不限制大小),因为需要完整重置给后续handler使用
- bodyBytes, err := io.ReadAll(c.Request.Body)
+ // 重置request body:把已窥探的前缀接回未读取的剩余部分,handler 仍能读到完整
+ // 数据,但中间件自身只驻留日志需要的那点字节。
+ //
+ // 这里原本是无上限的 io.ReadAll(c.Request.Body)。Logger 注册在 Auth 之前
+ // (router.go NewRouter),因此对所有请求执行,而任何路由的
+ // http.MaxBytesReader(limitJSONBody / limitUploadBody / browserskill 的 4 KiB
+ // 授权接口)都是在 handler 里才装上的:body 早已被整体读进内存,那些 cap 只剩
+ // 报错的意义,没有限住内存的作用。没有 cap 的路由(对话、检索、IM webhook、
+ // MCP 端点)则连上限都不存在。
+ bodyBytes, err := io.ReadAll(io.LimitReader(c.Request.Body, maxBodySize+1))
+ originalBody := c.Request.Body
+ c.Request.Body = &replayedBody{
+ Reader: io.MultiReader(bytes.NewReader(bodyBytes), originalBody),
+ Closer: originalBody,
+ }
if err != nil {
return "[读取请求体失败]"
}
- // 重置request body,使用完整内容,确保后续handler能读取到完整数据
- c.Request.Body = io.NopCloser(bytes.NewBuffer(bodyBytes))
-
// 用于日志的body(限制大小)
- var logBodyBytes []byte
- if len(bodyBytes) > maxBodySize {
- logBodyBytes = bodyBytes[:maxBodySize]
- } else {
- logBodyBytes = bodyBytes
+ truncated := len(bodyBytes) > maxBodySize
+ if truncated {
+ bodyBytes = bodyBytes[:maxBodySize]
}
- bodyStr := string(logBodyBytes)
- if len(bodyBytes) > maxBodySize {
- bodyStr += "... [内容过长,已截断]"
+ bodyStr := string(bodyBytes)
+ if truncated {
+ bodyStr += bodyTruncatedMarker
}
return sanitizeBody(bodyStr)
diff --git a/internal/middleware/logger_test.go b/internal/middleware/logger_test.go
index e44aa84db..79fe6d814 100644
--- a/internal/middleware/logger_test.go
+++ b/internal/middleware/logger_test.go
@@ -1,8 +1,14 @@
package middleware
import (
+ "errors"
+ "io"
+ "net/http"
+ "net/http/httptest"
"strings"
"testing"
+
+ "github.com/gin-gonic/gin"
)
func TestSanitizeBody(t *testing.T) {
@@ -84,6 +90,121 @@ func TestSanitizeBody(t *testing.T) {
}
}
+// The logging middleware is registered ahead of Auth and of every route
+// handler, so whatever it buffers is buffered before a route's own body cap can
+// refuse the request. These tests pin both halves of that: the middleware stays
+// inside its own log budget, and the handler still reads every byte.
+
+// countingBody records how many bytes a reader actually pulled, so a test can
+// tell "the middleware peeked" from "the middleware slurped the request".
+type countingBody struct {
+ io.Reader
+ read int
+}
+
+func (b *countingBody) Read(p []byte) (int, error) {
+ n, err := b.Reader.Read(p)
+ b.read += n
+ return n, err
+}
+
+func (b *countingBody) Close() error { return nil }
+
+// newJSONContext returns a context whose body is a countingBody, plus that
+// counter, so a test can observe how much the middleware consumed.
+func newJSONContext(t *testing.T, path, body string) (*gin.Context, *countingBody) {
+ t.Helper()
+ gin.SetMode(gin.TestMode)
+ c, _ := gin.CreateTestContext(httptest.NewRecorder())
+ c.Request = httptest.NewRequest(http.MethodPost, path, nil)
+ counter := &countingBody{Reader: strings.NewReader(body)}
+ c.Request.Body = counter
+ c.Request.ContentLength = int64(len(body))
+ c.Request.Header.Set("Content-Type", "application/json")
+ return c, counter
+}
+
+func TestReadRequestBodyKeepsShortBodyIntact(t *testing.T) {
+ payload := `{"query":"refund policy","apiKey":"sk-secret"}`
+
+ c, _ := newJSONContext(t, "/api/v1/knowledge-search", payload)
+ got := readRequestBody(c)
+
+ want := `{"query":"refund policy","apiKey":"***"}`
+ if got != want {
+ t.Fatalf("readRequestBody() = %q, want %q", got, want)
+ }
+ rest, err := io.ReadAll(c.Request.Body)
+ if err != nil {
+ t.Fatalf("handler read: %v", err)
+ }
+ if string(rest) != payload {
+ t.Fatalf("handler saw %q, want the untouched body %q", rest, payload)
+ }
+}
+
+// A request larger than the log budget must cost the middleware no more than the
+// log budget (plus one byte, to tell "exactly at the limit" from "over it").
+func TestReadRequestBodyStaysInsideItsLogBudget(t *testing.T) {
+ payload := strings.Repeat("a", maxBodySize*8)
+
+ c, counter := newJSONContext(t, "/api/v1/knowledge-search", payload)
+ got := readRequestBody(c)
+
+ if budget := maxBodySize + 1; counter.read > budget {
+ t.Fatalf("middleware read %d bytes of a %d byte request, want at most %d",
+ counter.read, len(payload), budget)
+ }
+ if want := maxBodySize + len(bodyTruncatedMarker); len(got) != want {
+ t.Fatalf("logged body is %d bytes, want %d (%d logged + %q)",
+ len(got), want, maxBodySize, bodyTruncatedMarker)
+ }
+ if !strings.HasSuffix(got, bodyTruncatedMarker) {
+ t.Fatalf("logged body %q does not end with the truncation marker", got)
+ }
+ rest, err := io.ReadAll(c.Request.Body)
+ if err != nil {
+ t.Fatalf("handler read: %v", err)
+ }
+ if len(rest) != len(payload) {
+ t.Fatalf("handler saw %d bytes, want all %d of the request", len(rest), len(payload))
+ }
+}
+
+// The peek must not become a cap the handler inherits: the route's own
+// http.MaxBytesReader still sees the whole request and still decides.
+func TestReadRequestBodyLeavesRouteBodyCapDeciding(t *testing.T) {
+ const routeCap = 64 << 10
+ payload := strings.Repeat("x", routeCap*2)
+
+ c, _ := newJSONContext(t, "/api/v1/skills/catalog", payload)
+ readRequestBody(c)
+ c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, routeCap)
+
+ _, err := io.ReadAll(c.Request.Body)
+ var tooLarge *http.MaxBytesError
+ if !errors.As(err, &tooLarge) {
+ t.Fatalf("MaxBytesReader error = %v, want *http.MaxBytesError", err)
+ }
+}
+
+func TestReadRequestBodySkipsNonTextBodies(t *testing.T) {
+ payload := "--x\r\nContent-Disposition: form-data; name=\"file\"\r\n\r\nbody\r\n--x--\r\n"
+
+ gin.SetMode(gin.TestMode)
+ c, _ := gin.CreateTestContext(httptest.NewRecorder())
+ c.Request = httptest.NewRequest(http.MethodPost, "/api/v1/knowledge/file", strings.NewReader(payload))
+ c.Request.Header.Set("Content-Type", "multipart/form-data; boundary=x")
+
+ if got := readRequestBody(c); got != "[非文本类型,已跳过]" {
+ t.Fatalf("readRequestBody() = %q, want the non-text marker", got)
+ }
+ rest, err := io.ReadAll(c.Request.Body)
+ if err != nil || string(rest) != payload {
+ t.Fatalf("non-text body was disturbed: %q (%v)", rest, err)
+ }
+}
+
func TestSanitizeQuery(t *testing.T) {
got := sanitizeQuery("code=secret-code&state=secret-state&next=%2Fsettings&state=second")
want := "code=%2A%2A%2A&next=%2Fsettings&state=%2A%2A%2A"