Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(markdown): sanitize invalid UTF-8 and NUL bytes
十三分之五六七786d67609dc8
diff --git a/internal/application/repository/knowledge.go b/internal/application/repository/knowledge.goindex 687ff4713..065912c69 100644--- a/internal/application/repository/knowledge.go+++ b/internal/application/repository/knowledge.go@@ -6,6 +6,7 @@ import ("strings""time"+ "github.com/Tencent/WeKnora/internal/common""github.com/Tencent/WeKnora/internal/types""github.com/Tencent/WeKnora/internal/types/interfaces""gorm.io/gorm"@@ -53,6 +54,7 @@ func NewKnowledgeRepository(db *gorm.DB) interfaces.KnowledgeRepository {// CreateKnowledge creates knowledgefunc (r *knowledgeRepository) CreateKnowledge(ctx context.Context, knowledge *types.Knowledge) error {+ knowledge.ErrorMessage = common.CleanInvalidUTF8(knowledge.ErrorMessage)err := r.db.WithContext(ctx).Create(knowledge).Errorreturn err}@@ -306,6 +308,7 @@ func (r *knowledgeRepository) RenameKnowledgeFolderPath(// UpdateKnowledge updates knowledgefunc (r *knowledgeRepository) UpdateKnowledge(ctx context.Context, knowledge *types.Knowledge) error {+ knowledge.ErrorMessage = common.CleanInvalidUTF8(knowledge.ErrorMessage)omit := omitFieldsOnUpdate// Legacy/unit-test schemas created before custom_metadata should continue// to support unrelated updates when the caller did not provide the field.@@ -321,6 +324,11 @@ func (r *knowledgeRepository) UpdateKnowledgeBatch(ctx context.Context, knowledgif len(knowledgeList) == 0 {return nil}+ for _, knowledge := range knowledgeList {+ if knowledge != nil {+ knowledge.ErrorMessage = common.CleanInvalidUTF8(knowledge.ErrorMessage)+ }+ }return r.db.Debug().WithContext(ctx).Omit(omitFieldsOnUpdate...).Save(knowledgeList).Error}@@ -513,6 +521,14 @@ func (r *knowledgeRepository) UpdateKnowledgeColumn(column string,value interface{},) error {+ if column == "error_message" {+ switch v := value.(type) {+ case string:+ value = common.CleanInvalidUTF8(v)+ case []byte:+ value = common.CleanInvalidUTF8(string(v))+ }+ }err := r.db.WithContext(ctx).Model(&types.Knowledge{}).Where("id = ?", id).Update(column, value).Errorreturn err}@@ -529,6 +545,14 @@ func (r *knowledgeRepository) UpdateKnowledgeColumns(if len(values) == 0 {return nil}+ if value, ok := values["error_message"]; ok {+ switch v := value.(type) {+ case string:+ values["error_message"] = common.CleanInvalidUTF8(v)+ case []byte:+ values["error_message"] = common.CleanInvalidUTF8(string(v))+ }+ }return r.db.WithContext(ctx).Model(&types.Knowledge{}).Where("id = ?", id).Updates(values).Error}diff --git a/internal/application/repository/knowledge_finalize_test.go b/internal/application/repository/knowledge_finalize_test.goindex b27efd1b5..abcb68a97 100644--- a/internal/application/repository/knowledge_finalize_test.go+++ b/internal/application/repository/knowledge_finalize_test.go@@ -5,6 +5,7 @@ import ("sync""sync/atomic""testing"+ "unicode/utf8""github.com/Tencent/WeKnora/internal/types""github.com/google/uuid"@@ -99,6 +100,23 @@ func reloadKnowledgeErrorMessage(t *testing.T, db *gorm.DB, id string) string {return msg}+func TestKnowledgeRepository_UpdateKnowledgeColumnsSanitizesErrorMessage(t *testing.T) {+ db := setupKnowledgeTestDB(t)+ repo := NewKnowledgeRepository(db)+ id := insertProcessingKnowledge(t, db)+ invalid := "parse failed " + string([]byte{0xef, 0xbc, 0x2e})++ require.NoError(t, repo.UpdateKnowledgeColumns(context.Background(), id, map[string]interface{}{+ "error_message": invalid,+ }))++ got := reloadKnowledgeErrorMessage(t, db, id)+ if !utf8.ValidString(got) {+ t.Fatalf("persisted error_message is invalid UTF-8: % x", []byte(got))+ }+ assert.Equal(t, "parse failed .", got)+}+func insertKnowledgeWithStatus(t *testing.T, db *gorm.DB, status string, deleted bool) string {t.Helper()id := uuid.New().String()diff --git a/internal/application/repository/knowledge_span_repo.go b/internal/application/repository/knowledge_span_repo.goindex 1defb0f78..c610f1e9d 100644--- a/internal/application/repository/knowledge_span_repo.go+++ b/internal/application/repository/knowledge_span_repo.go@@ -5,6 +5,7 @@ import ("errors""time"+ "github.com/Tencent/WeKnora/internal/common""github.com/Tencent/WeKnora/internal/types""gorm.io/gorm""gorm.io/gorm/clause"@@ -61,6 +62,12 @@ func (r *knowledgeSpanRepository) Upsert(ctx context.Context, row *types.Knowledif row.Attempt == 0 {row.Attempt = 1}+ // Error fields may originate from third-party parser/model responses.+ // Normalize them at the persistence boundary so SQLite/PostgreSQL and the+ // tracing API never receive malformed UTF-8.+ row.ErrorCode = common.CleanInvalidUTF8(row.ErrorCode)+ row.ErrorMessage = common.CleanInvalidUTF8(row.ErrorMessage)+ row.ErrorDetail = common.CleanInvalidUTF8(row.ErrorDetail)// We let GORM populate created_at/updated_at via the autoCreate /// autoUpdate tags. ON CONFLICT updates only the fields that may// transition between calls — name/kind/parent are immutable once@@ -163,6 +170,7 @@ func (r *knowledgeSpanRepository) GetSpan(ctx context.Context, knowledgeID strin// Postgres-specific WITH RECURSIVE would be denser but harder to test on// the SQLite Lite backend. The iterative path stays portable.func (r *knowledgeSpanRepository) CancelDescendants(ctx context.Context, knowledgeID string, attempt int, parentSpanID, reason string) (int64, error) {+ reason = common.CleanInvalidUTF8(reason)frontier := []string{parentSpanID}var totalAffected int64for depth := 0; depth < 16 && len(frontier) > 0; depth++ {@@ -215,6 +223,8 @@ func (r *knowledgeSpanRepository) CancelDescendants(ctx context.Context, knowledfunc (r *knowledgeSpanRepository) CancelAllOpenSpans(ctx context.Context, knowledgeID string, attempt int, errorCode, reason string,) (int64, error) {+ errorCode = common.CleanInvalidUTF8(errorCode)+ reason = common.CleanInvalidUTF8(reason)now := time.Now()updates := map[string]any{"status": types.SpanStatusCancelled,@@ -240,6 +250,8 @@ func (r *knowledgeSpanRepository) CancelOpenSpansByName(if knowledgeID == "" || attempt <= 0 || name == "" {return 0, nil}+ errorCode = common.CleanInvalidUTF8(errorCode)+ reason = common.CleanInvalidUTF8(reason)now := time.Now()res := r.db.WithContext(ctx).Model(&types.KnowledgeProcessingSpan{}).Where("knowledge_id = ? AND attempt = ? AND name = ? AND status IN ?",diff --git a/internal/application/repository/knowledge_span_repo_test.go b/internal/application/repository/knowledge_span_repo_test.goindex 341a3d4fd..72b70adec 100644--- a/internal/application/repository/knowledge_span_repo_test.go+++ b/internal/application/repository/knowledge_span_repo_test.go@@ -4,6 +4,7 @@ import ("context""testing""time"+ "unicode/utf8""github.com/Tencent/WeKnora/internal/types""github.com/stretchr/testify/assert"@@ -85,6 +86,37 @@ func TestKnowledgeSpanRepo_UpsertAndList(t *testing.T) {assert.Equal(t, int64(2000), rows[0].DurationMs)}+func TestKnowledgeSpanRepo_UpsertSanitizesErrorFields(t *testing.T) {+ repo, _ := setupSpanTestRepo(t)+ now := time.Now()+ invalid := "prefix " + string([]byte{0xef, 0xbc, 0x2e}) + " suffix"++ require.NoError(t, repo.Upsert(context.Background(), &types.KnowledgeProcessingSpan{+ KnowledgeID: "kid-invalid-utf8",+ Attempt: 1,+ SpanID: "span-invalid",+ Name: types.StageDocReader,+ Kind: types.SpanKindStage,+ Status: types.SpanStatusFailed,+ ErrorCode: invalid,+ ErrorMessage: invalid,+ ErrorDetail: invalid,+ StartedAt: &now,+ }))++ rows, err := repo.ListByAttempt(context.Background(), "kid-invalid-utf8", 1)+ require.NoError(t, err)+ require.Len(t, rows, 1)+ for _, value := range []string{rows[0].ErrorCode, rows[0].ErrorMessage, rows[0].ErrorDetail} {+ if !utf8.ValidString(value) {+ t.Fatalf("persisted error field is invalid UTF-8: % x", []byte(value))+ }+ if value != "prefix . suffix" {+ t.Fatalf("persisted error field = %q, want %q", value, "prefix . suffix")+ }+ }+}+// TestKnowledgeSpanRepo_NextAttempt confirms that NextAttempt allocates// a fresh number per knowledge, isolating reparse history. Critical// because the API layer renders attempt history by this number.@@ -182,6 +214,45 @@ func TestKnowledgeSpanRepo_CancelOpenSpansByName(t *testing.T) {assert.Equal(t, types.SpanStatusRunning, statusBy["q-old"])}+func TestKnowledgeSpanRepo_CancelPathsSanitizeErrorFields(t *testing.T) {+ repo, _ := setupSpanTestRepo(t)+ ctx := context.Background()+ kid := "kid-cancel-invalid-utf8"+ now := time.Now()+ for _, r := range []*types.KnowledgeProcessingSpan{+ {KnowledgeID: kid, Attempt: 1, SpanID: "root", Name: "root", Kind: types.SpanKindRoot, Status: types.SpanStatusRunning, StartedAt: &now},+ {KnowledgeID: kid, Attempt: 1, SpanID: "descendant", ParentSpanID: "root", Name: "descendant", Kind: types.SpanKindSubSpan, Status: types.SpanStatusRunning, StartedAt: &now},+ {KnowledgeID: kid, Attempt: 1, SpanID: "bulk", Name: "bulk", Kind: types.SpanKindSubSpan, Status: types.SpanStatusRunning, StartedAt: &now},+ {KnowledgeID: kid, Attempt: 1, SpanID: "named", Name: "named", Kind: types.SpanKindSubSpan, Status: types.SpanStatusRunning, StartedAt: &now},+ } {+ require.NoError(t, repo.Upsert(ctx, r))+ }++ invalidCode := "CODE " + string([]byte{0xef, 0xbc, 0x2e})+ invalidReason := "reason " + string([]byte{0xef, 0xbc, 0x2e})+ _, err := repo.CancelOpenSpansByName(ctx, kid, 1, "named", invalidCode, invalidReason)+ require.NoError(t, err)+ _, err = repo.CancelDescendants(ctx, kid, 1, "root", invalidReason)+ require.NoError(t, err)+ _, err = repo.CancelAllOpenSpans(ctx, kid, 1, invalidCode, invalidReason)+ require.NoError(t, err)++ rows, err := repo.ListByAttempt(ctx, kid, 1)+ require.NoError(t, err)+ byID := make(map[string]types.KnowledgeProcessingSpan, len(rows))+ for _, row := range rows {+ byID[row.SpanID] = row+ }+ assert.Equal(t, "UPSTREAM_FAILED", byID["descendant"].ErrorCode)+ assert.Equal(t, "reason .", byID["descendant"].ErrorMessage)+ for _, id := range []string{"root", "bulk", "named"} {+ assert.Equal(t, "CODE .", byID[id].ErrorCode, id)+ assert.Equal(t, "reason .", byID[id].ErrorMessage, id)+ assert.True(t, utf8.ValidString(byID[id].ErrorCode), id)+ assert.True(t, utf8.ValidString(byID[id].ErrorMessage), id)+ }+}+// TestKnowledgeSpanRepo_ListAttemptIsolation guarantees that different// attempts of the same knowledge stay queryable independently — the// foundation for the "show history" UI navigation (?attempt=N).diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.goindex 6c2f7350b..ce9365fcd 100644--- a/internal/application/service/knowledge_process.go+++ b/internal/application/service/knowledge_process.go@@ -11,6 +11,7 @@ import ("time""github.com/Tencent/WeKnora/internal/application/service/retriever"+ "github.com/Tencent/WeKnora/internal/common"werrors "github.com/Tencent/WeKnora/internal/errors""github.com/Tencent/WeKnora/internal/infrastructure/chunker""github.com/Tencent/WeKnora/internal/infrastructure/docparser"@@ -251,6 +252,23 @@ func (s *knowledgeService) processChunks(ctx context.Context,options = opts[0]}+ // Parser output and manually supplied passages can contain malformed byte+ // sequences. Clean them before logging, chunk persistence, or embedding;+ // the embedding provider and tracing/database drivers expect valid UTF-8.+ for i := range chunks {+ chunks[i].Content = common.CleanInvalidUTF8(chunks[i].Content)+ chunks[i].ContextHeader = common.CleanInvalidUTF8(chunks[i].ContextHeader)+ for j := range chunks[i].Images {+ chunks[i].Images[j].URL = common.CleanInvalidUTF8(chunks[i].Images[j].URL)+ chunks[i].Images[j].Caption = common.CleanInvalidUTF8(chunks[i].Images[j].Caption)+ chunks[i].Images[j].OCRText = common.CleanInvalidUTF8(chunks[i].Images[j].OCRText)+ chunks[i].Images[j].OriginalURL = common.CleanInvalidUTF8(chunks[i].Images[j].OriginalURL)+ }+ }+ for i := range options.ParentChunks {+ options.ParentChunks[i].Content = common.CleanInvalidUTF8(options.ParentChunks[i].Content)+ }+// Check if knowledge is being deleted/cancelled before processing.// Both statuses short-circuit identically here — there's nothing to clean// up yet so the branch is purely "stop early".@@ -2896,6 +2914,7 @@ func (s *knowledgeService) UpdateImageInfo(chunkID string,imageInfo string,) error {+ imageInfo = common.CleanInvalidUTF8(imageInfo)var images []*types.ImageInfoif err := json.Unmarshal([]byte(imageInfo), &images); err != nil {logger.Errorf(ctx, "Failed to unmarshal image info: %v", err)@@ -3516,6 +3535,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e// Step 3: Split into chunks using Go chunker. Browser textareas normalize// pasted content to LF, so normalize uploaded source text before calculating// chunk boundaries as well.+ sanitizeReadResult(convertResult)convertResult.MarkdownContent = chunker.NormalizeLineEndings(convertResult.MarkdownContent)chunkCfg := buildSplitterConfigFromChunking(eff.ChunkingConfig)@@ -3572,6 +3592,28 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) ereturn nil}+// sanitizeReadResult protects every text field that can cross from a parser+// into the embedding, storage, or tracing layers. A parser may return a Go+// string containing arbitrary bytes even though the string type itself does+// not enforce UTF-8 validity.+func sanitizeReadResult(result *types.ReadResult) {+ if result == nil {+ return+ }+ result.MarkdownContent = common.CleanInvalidUTF8(result.MarkdownContent)+ result.ImageDirPath = common.CleanInvalidUTF8(result.ImageDirPath)+ result.Error = common.CleanInvalidUTF8(result.Error)+ for key, value := range result.Metadata {+ result.Metadata[key] = common.CleanInvalidUTF8(value)+ }+ for i := range result.ImageRefs {+ result.ImageRefs[i].Filename = common.CleanInvalidUTF8(result.ImageRefs[i].Filename)+ result.ImageRefs[i].OriginalRef = common.CleanInvalidUTF8(result.ImageRefs[i].OriginalRef)+ result.ImageRefs[i].MimeType = common.CleanInvalidUTF8(result.ImageRefs[i].MimeType)+ result.ImageRefs[i].StorageKey = common.CleanInvalidUTF8(result.ImageRefs[i].StorageKey)+ }+}+// convert handles both file and URL reading using a unified ReadRequest.func (s *knowledgeService) convert(ctx context.Context,@@ -3689,6 +3731,7 @@ func (s *knowledgeService) convert(code, "document read failed", err)return s.failKnowledge(ctx, knowledge, isLastRetry, "document read failed: %v", err)}+ sanitizeReadResult(result)if result.Error != "" {logger.Errorf(ctx, "[convert] parser returned error kb=%s knowledge=%s file=%q type=%s engine=%q: %s",kb.ID, knowledge.ID, req.FileName, fileType, parserEngine, result.Error)diff --git a/internal/common/tools_test.go b/internal/common/tools_test.goindex d3a2e6e26..53ec83d8b 100644--- a/internal/common/tools_test.go+++ b/internal/common/tools_test.go@@ -1,6 +1,9 @@package common-import "testing"+import (+ "testing"+ "unicode/utf8"+)func TestParseLLMJsonResponse(t *testing.T) {type payload struct {@@ -39,6 +42,18 @@ func TestParseLLMJsonResponse(t *testing.T) {}}+func TestCleanInvalidUTF8(t *testing.T) {+ input := "valid " + string([]byte{0xef, 0xbc, 0x2e}) + "\x00tail"++ got := CleanInvalidUTF8(input)+ if !utf8.ValidString(got) {+ t.Fatalf("CleanInvalidUTF8 returned invalid UTF-8: % x", []byte(got))+ }+ if want := "valid .tail"; got != want {+ t.Fatalf("CleanInvalidUTF8() = %q, want %q", got, want)+ }+}+func BenchmarkParseLLMJsonResponse_Fenced(b *testing.B) {const content = "```json\n{\"name\":\"Acme\",\"slug\":\"entity/acme\",\"aliases\":[\"A\",\"B\"]}\n```"type payload struct {