Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(knowledge): cap the knowledge-base definition body at 1 MiB (#4730)
yetuge6feabaa4df5c
diff --git a/internal/handler/knowledgebase.go b/internal/handler/knowledgebase.goindex ec2d444aa..e31919cd8 100644--- a/internal/handler/knowledgebase.go+++ b/internal/handler/knowledgebase.go@@ -474,8 +474,13 @@ func (h *KnowledgeBaseHandler) CreateKnowledgeBase(c *gin.Context) {logger.Info(ctx, "Start creating knowledge base")// Parse request body+ limitJSONBody(c, knowledgeBaseJSONMaxBytes)body, err := io.ReadAll(c.Request.Body)if err != nil {+ if isRequestBodyTooLarge(err) {+ _ = c.Error(apperrors.NewBadRequestError("request body is too large"))+ return+ }_ = c.Error(apperrors.NewBadRequestError("Invalid request parameters").WithDetails(err.Error()))return}@@ -1227,8 +1232,13 @@ func (h *KnowledgeBaseHandler) PatchKnowledgeBase(c *gin.Context) {_ = c.Error(apperrors.NewForbiddenError("No permission to update knowledge base"))return}+ limitJSONBody(c, knowledgeBaseJSONMaxBytes)body, err := io.ReadAll(c.Request.Body)if err != nil {+ if isRequestBodyTooLarge(err) {+ _ = c.Error(apperrors.NewBadRequestError("request body is too large"))+ return+ }_ = c.Error(apperrors.NewBadRequestError("Invalid request parameters").WithDetails(err.Error()))return}diff --git a/internal/handler/knowledgebase_body_limit_test.go b/internal/handler/knowledgebase_body_limit_test.gonew file mode 100644index 000000000..f3277f3dc--- /dev/null+++ b/internal/handler/knowledgebase_body_limit_test.go@@ -0,0 +1,75 @@+package handler++import (+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"++ "github.com/gin-gonic/gin"+ "github.com/stretchr/testify/require"++ "github.com/Tencent/WeKnora/internal/errors"+ "github.com/Tencent/WeKnora/internal/types"+)++func createKnowledgeBaseProbe(t *testing.T, body string) *gin.Context {+ t.Helper()+ gin.SetMode(gin.TestMode)+ w := httptest.NewRecorder()+ c, _ := gin.CreateTestContext(w)+ req := httptest.NewRequest(http.MethodPost, "/knowledge-bases", strings.NewReader(body))+ req.Header.Set("Content-Type", "application/json")+ c.Request = req+ c.Set(types.TenantIDContextKey.String(), uint64(1))++ h := &KnowledgeBaseHandler{}+ h.CreateKnowledgeBase(c)+ return c+}++func firstAppError(t *testing.T, c *gin.Context) *errors.AppError {+ t.Helper()+ require.NotEmpty(t, c.Errors, "the handler must record an error")+ appErr, ok := c.Errors[0].Err.(*errors.AppError)+ require.True(t, ok, "expected an AppError, got %T", c.Errors[0].Err)+ return appErr+}++// A knowledge-base definition carries scalars, ids and prompt instructions,+// whose per-field limits keep a legal body to a few hundred KiB. Without a cap+// the handler buffered whatever arrived — the Logger middleware retains only+// 10 KiB for its log line but still hands the rest of the stream to the+// handler — so one authenticated POST could hold an arbitrary body in memory+// before the JSON parser saw it.+func TestCreateKnowledgeBaseRejectsAnOversizedBody(t *testing.T) {+ // Valid JSON past the cap: without the cap the parser would accept it and+ // the handler would go on to validate the name. The cap must answer first.+ body := `{"name":"` + strings.Repeat("a", knowledgeBaseJSONMaxBytes) + `"}`++ c := createKnowledgeBaseProbe(t, body)++ appErr := firstAppError(t, c)+ require.Equal(t, errors.ErrBadRequest, appErr.Code)+ require.Equal(t, "request body is too large", appErr.Message)+}++// The cap must not change how a legal body is answered: an empty name is still+// the validation error, proving the read completed and the JSON parsed.+func TestCreateKnowledgeBaseKeepsALegalBodyPath(t *testing.T) {+ c := createKnowledgeBaseProbe(t, `{"name":" "}`)++ appErr := firstAppError(t, c)+ require.Equal(t, errors.ErrBadRequest, appErr.Code)+ require.Equal(t, "name cannot be empty", appErr.Message)+}++// A definition that fits the cap but fails to parse must still report the+// parse failure, not the size limit.+func TestCreateKnowledgeBaseStillReportsMalformedJSON(t *testing.T) {+ c := createKnowledgeBaseProbe(t, `{"name":`)++ appErr := firstAppError(t, c)+ require.Equal(t, errors.ErrBadRequest, appErr.Code)+ require.Contains(t, appErr.Message, "Invalid request parameters")+}diff --git a/internal/handler/upload_limit.go b/internal/handler/upload_limit.goindex 281e71216..9840ebcf4 100644--- a/internal/handler/upload_limit.go+++ b/internal/handler/upload_limit.go@@ -18,6 +18,17 @@ const uploadEnvelopeSlack = 1 << 20// GetMaxSkillBundleSize; this only needs to hold a locator URL.const skillSourceJSONMaxBytes = 64 << 10+// knowledgeBaseJSONMaxBytes caps a knowledge-base definition body.+//+// A definition carries scalars, model ids and prompt instructions, whose+// per-field limits are already enforced by ValidateKnowledgeBasePromptInstructions;+// the largest legal body is a few hundred KiB. Without a cap the handler+// buffered whatever arrived: the Logger middleware only retains 10 KiB, but it+// still reads up to 10 KiB+1 and then hands the rest of the stream to this+// handler, so an authenticated POST could make one request occupy as much+// memory as the process will surrender.+const knowledgeBaseJSONMaxBytes = 1 << 20+// limitUploadBody caps the request body before anything parses it.//// It has to run before FormFile rather than after: ParseMultipartForm buffers