Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(middleware): stop buffering whole request bodies in the access log (#3927)
ordinary being6057a8101737
diff --git a/internal/middleware/logger.go b/internal/middleware/logger.goindex b05a02c35..a65ef0f98 100644--- a/internal/middleware/logger.go+++ b/internal/middleware/logger.go@@ -18,8 +18,17 @@ import (const (maxBodySize = 1024 * 10 // 最大记录10KB的body内容+ // bodyTruncatedMarker 是日志里跟在被截断的 body 后面的标记。+ bodyTruncatedMarker = "... [内容过长,已截断]")+// replayedBody 把日志窥探消费掉的字节还给 handler,让它读到的 body 与没有这层+// 中间件时完全一致;Close 仍然关闭原始 body,连接得以释放。+type replayedBody struct {+ io.Reader+ io.Closer+}+// loggerResponseBodyWriter 自定义ResponseWriter用于捕获响应内容(用于logger中间件)type loggerResponseBodyWriter struct {gin.ResponseWriter@@ -87,7 +96,7 @@ func sanitizeQuery(raw string) string {return values.Encode()}-// readRequestBody 读取请求体(限制大小用于日志,但完整读取用于重置)+// readRequestBody 读取请求体(限制大小用于日志,但把剩余字节原样交还后续handler)func readRequestBody(c *gin.Context) string {if c.Request.Body == nil {return ""@@ -101,26 +110,34 @@ func readRequestBody(c *gin.Context) string {return "[非文本类型,已跳过]"}- // 完整读取body内容(不限制大小),因为需要完整重置给后续handler使用- bodyBytes, err := io.ReadAll(c.Request.Body)+ // 重置request body:把已窥探的前缀接回未读取的剩余部分,handler 仍能读到完整+ // 数据,但中间件自身只驻留日志需要的那点字节。+ //+ // 这里原本是无上限的 io.ReadAll(c.Request.Body)。Logger 注册在 Auth 之前+ // (router.go NewRouter),因此对所有请求执行,而任何路由的+ // http.MaxBytesReader(limitJSONBody / limitUploadBody / browserskill 的 4 KiB+ // 授权接口)都是在 handler 里才装上的:body 早已被整体读进内存,那些 cap 只剩+ // 报错的意义,没有限住内存的作用。没有 cap 的路由(对话、检索、IM webhook、+ // MCP 端点)则连上限都不存在。+ bodyBytes, err := io.ReadAll(io.LimitReader(c.Request.Body, maxBodySize+1))+ originalBody := c.Request.Body+ c.Request.Body = &replayedBody{+ Reader: io.MultiReader(bytes.NewReader(bodyBytes), originalBody),+ Closer: originalBody,+ }if err != nil {return "[读取请求体失败]"}- // 重置request body,使用完整内容,确保后续handler能读取到完整数据- c.Request.Body = io.NopCloser(bytes.NewBuffer(bodyBytes))-// 用于日志的body(限制大小)- var logBodyBytes []byte- if len(bodyBytes) > maxBodySize {- logBodyBytes = bodyBytes[:maxBodySize]- } else {- logBodyBytes = bodyBytes+ truncated := len(bodyBytes) > maxBodySize+ if truncated {+ bodyBytes = bodyBytes[:maxBodySize]}- bodyStr := string(logBodyBytes)- if len(bodyBytes) > maxBodySize {- bodyStr += "... [内容过长,已截断]"+ bodyStr := string(bodyBytes)+ if truncated {+ bodyStr += bodyTruncatedMarker}return sanitizeBody(bodyStr)diff --git a/internal/middleware/logger_test.go b/internal/middleware/logger_test.goindex e44aa84db..79fe6d814 100644--- a/internal/middleware/logger_test.go+++ b/internal/middleware/logger_test.go@@ -1,8 +1,14 @@package middlewareimport (+ "errors"+ "io"+ "net/http"+ "net/http/httptest""strings""testing"++ "github.com/gin-gonic/gin")func TestSanitizeBody(t *testing.T) {@@ -84,6 +90,121 @@ func TestSanitizeBody(t *testing.T) {}}+// The logging middleware is registered ahead of Auth and of every route+// handler, so whatever it buffers is buffered before a route's own body cap can+// refuse the request. These tests pin both halves of that: the middleware stays+// inside its own log budget, and the handler still reads every byte.++// countingBody records how many bytes a reader actually pulled, so a test can+// tell "the middleware peeked" from "the middleware slurped the request".+type countingBody struct {+ io.Reader+ read int+}++func (b *countingBody) Read(p []byte) (int, error) {+ n, err := b.Reader.Read(p)+ b.read += n+ return n, err+}++func (b *countingBody) Close() error { return nil }++// newJSONContext returns a context whose body is a countingBody, plus that+// counter, so a test can observe how much the middleware consumed.+func newJSONContext(t *testing.T, path, body string) (*gin.Context, *countingBody) {+ t.Helper()+ gin.SetMode(gin.TestMode)+ c, _ := gin.CreateTestContext(httptest.NewRecorder())+ c.Request = httptest.NewRequest(http.MethodPost, path, nil)+ counter := &countingBody{Reader: strings.NewReader(body)}+ c.Request.Body = counter+ c.Request.ContentLength = int64(len(body))+ c.Request.Header.Set("Content-Type", "application/json")+ return c, counter+}++func TestReadRequestBodyKeepsShortBodyIntact(t *testing.T) {+ payload := `{"query":"refund policy","apiKey":"sk-secret"}`++ c, _ := newJSONContext(t, "/api/v1/knowledge-search", payload)+ got := readRequestBody(c)++ want := `{"query":"refund policy","apiKey":"***"}`+ if got != want {+ t.Fatalf("readRequestBody() = %q, want %q", got, want)+ }+ rest, err := io.ReadAll(c.Request.Body)+ if err != nil {+ t.Fatalf("handler read: %v", err)+ }+ if string(rest) != payload {+ t.Fatalf("handler saw %q, want the untouched body %q", rest, payload)+ }+}++// A request larger than the log budget must cost the middleware no more than the+// log budget (plus one byte, to tell "exactly at the limit" from "over it").+func TestReadRequestBodyStaysInsideItsLogBudget(t *testing.T) {+ payload := strings.Repeat("a", maxBodySize*8)++ c, counter := newJSONContext(t, "/api/v1/knowledge-search", payload)+ got := readRequestBody(c)++ if budget := maxBodySize + 1; counter.read > budget {+ t.Fatalf("middleware read %d bytes of a %d byte request, want at most %d",+ counter.read, len(payload), budget)+ }+ if want := maxBodySize + len(bodyTruncatedMarker); len(got) != want {+ t.Fatalf("logged body is %d bytes, want %d (%d logged + %q)",+ len(got), want, maxBodySize, bodyTruncatedMarker)+ }+ if !strings.HasSuffix(got, bodyTruncatedMarker) {+ t.Fatalf("logged body %q does not end with the truncation marker", got)+ }+ rest, err := io.ReadAll(c.Request.Body)+ if err != nil {+ t.Fatalf("handler read: %v", err)+ }+ if len(rest) != len(payload) {+ t.Fatalf("handler saw %d bytes, want all %d of the request", len(rest), len(payload))+ }+}++// The peek must not become a cap the handler inherits: the route's own+// http.MaxBytesReader still sees the whole request and still decides.+func TestReadRequestBodyLeavesRouteBodyCapDeciding(t *testing.T) {+ const routeCap = 64 << 10+ payload := strings.Repeat("x", routeCap*2)++ c, _ := newJSONContext(t, "/api/v1/skills/catalog", payload)+ readRequestBody(c)+ c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, routeCap)++ _, err := io.ReadAll(c.Request.Body)+ var tooLarge *http.MaxBytesError+ if !errors.As(err, &tooLarge) {+ t.Fatalf("MaxBytesReader error = %v, want *http.MaxBytesError", err)+ }+}++func TestReadRequestBodySkipsNonTextBodies(t *testing.T) {+ payload := "--x\r\nContent-Disposition: form-data; name=\"file\"\r\n\r\nbody\r\n--x--\r\n"++ gin.SetMode(gin.TestMode)+ c, _ := gin.CreateTestContext(httptest.NewRecorder())+ c.Request = httptest.NewRequest(http.MethodPost, "/api/v1/knowledge/file", strings.NewReader(payload))+ c.Request.Header.Set("Content-Type", "multipart/form-data; boundary=x")++ if got := readRequestBody(c); got != "[非文本类型,已跳过]" {+ t.Fatalf("readRequestBody() = %q, want the non-text marker", got)+ }+ rest, err := io.ReadAll(c.Request.Body)+ if err != nil || string(rest) != payload {+ t.Fatalf("non-text body was disturbed: %q (%v)", rest, err)+ }+}+func TestSanitizeQuery(t *testing.T) {got := sanitizeQuery("code=secret-code&state=secret-state&next=%2Fsettings&state=second")want := "code=%2A%2A%2A&next=%2Fsettings&state=%2A%2A%2A"