Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(markdown): sanitize invalid UTF-8 and NUL bytes

十三分之五六七786d67609dc8
diff --git a/internal/application/repository/knowledge.go b/internal/application/repository/knowledge.go
index 687ff4713..065912c69 100644
--- a/internal/application/repository/knowledge.go
+++ b/internal/application/repository/knowledge.go
@@ -6,6 +6,7 @@ import (
"strings"
"time"
+ "github.com/Tencent/WeKnora/internal/common"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"gorm.io/gorm"
@@ -53,6 +54,7 @@ func NewKnowledgeRepository(db *gorm.DB) interfaces.KnowledgeRepository {
// CreateKnowledge creates knowledge
func (r *knowledgeRepository) CreateKnowledge(ctx context.Context, knowledge *types.Knowledge) error {
+ knowledge.ErrorMessage = common.CleanInvalidUTF8(knowledge.ErrorMessage)
err := r.db.WithContext(ctx).Create(knowledge).Error
return err
}
@@ -306,6 +308,7 @@ func (r *knowledgeRepository) RenameKnowledgeFolderPath(
// UpdateKnowledge updates knowledge
func (r *knowledgeRepository) UpdateKnowledge(ctx context.Context, knowledge *types.Knowledge) error {
+ knowledge.ErrorMessage = common.CleanInvalidUTF8(knowledge.ErrorMessage)
omit := omitFieldsOnUpdate
// Legacy/unit-test schemas created before custom_metadata should continue
// to support unrelated updates when the caller did not provide the field.
@@ -321,6 +324,11 @@ func (r *knowledgeRepository) UpdateKnowledgeBatch(ctx context.Context, knowledg
if len(knowledgeList) == 0 {
return nil
}
+ for _, knowledge := range knowledgeList {
+ if knowledge != nil {
+ knowledge.ErrorMessage = common.CleanInvalidUTF8(knowledge.ErrorMessage)
+ }
+ }
return r.db.Debug().WithContext(ctx).Omit(omitFieldsOnUpdate...).Save(knowledgeList).Error
}
@@ -513,6 +521,14 @@ func (r *knowledgeRepository) UpdateKnowledgeColumn(
column string,
value interface{},
) error {
+ if column == "error_message" {
+ switch v := value.(type) {
+ case string:
+ value = common.CleanInvalidUTF8(v)
+ case []byte:
+ value = common.CleanInvalidUTF8(string(v))
+ }
+ }
err := r.db.WithContext(ctx).Model(&types.Knowledge{}).Where("id = ?", id).Update(column, value).Error
return err
}
@@ -529,6 +545,14 @@ func (r *knowledgeRepository) UpdateKnowledgeColumns(
if len(values) == 0 {
return nil
}
+ if value, ok := values["error_message"]; ok {
+ switch v := value.(type) {
+ case string:
+ values["error_message"] = common.CleanInvalidUTF8(v)
+ case []byte:
+ values["error_message"] = common.CleanInvalidUTF8(string(v))
+ }
+ }
return r.db.WithContext(ctx).Model(&types.Knowledge{}).Where("id = ?", id).Updates(values).Error
}
diff --git a/internal/application/repository/knowledge_finalize_test.go b/internal/application/repository/knowledge_finalize_test.go
index b27efd1b5..abcb68a97 100644
--- a/internal/application/repository/knowledge_finalize_test.go
+++ b/internal/application/repository/knowledge_finalize_test.go
@@ -5,6 +5,7 @@ import (
"sync"
"sync/atomic"
"testing"
+ "unicode/utf8"
"github.com/Tencent/WeKnora/internal/types"
"github.com/google/uuid"
@@ -99,6 +100,23 @@ func reloadKnowledgeErrorMessage(t *testing.T, db *gorm.DB, id string) string {
return msg
}
+func TestKnowledgeRepository_UpdateKnowledgeColumnsSanitizesErrorMessage(t *testing.T) {
+ db := setupKnowledgeTestDB(t)
+ repo := NewKnowledgeRepository(db)
+ id := insertProcessingKnowledge(t, db)
+ invalid := "parse failed " + string([]byte{0xef, 0xbc, 0x2e})
+
+ require.NoError(t, repo.UpdateKnowledgeColumns(context.Background(), id, map[string]interface{}{
+ "error_message": invalid,
+ }))
+
+ got := reloadKnowledgeErrorMessage(t, db, id)
+ if !utf8.ValidString(got) {
+ t.Fatalf("persisted error_message is invalid UTF-8: % x", []byte(got))
+ }
+ assert.Equal(t, "parse failed .", got)
+}
+
func insertKnowledgeWithStatus(t *testing.T, db *gorm.DB, status string, deleted bool) string {
t.Helper()
id := uuid.New().String()
diff --git a/internal/application/repository/knowledge_span_repo.go b/internal/application/repository/knowledge_span_repo.go
index 1defb0f78..c610f1e9d 100644
--- a/internal/application/repository/knowledge_span_repo.go
+++ b/internal/application/repository/knowledge_span_repo.go
@@ -5,6 +5,7 @@ import (
"errors"
"time"
+ "github.com/Tencent/WeKnora/internal/common"
"github.com/Tencent/WeKnora/internal/types"
"gorm.io/gorm"
"gorm.io/gorm/clause"
@@ -61,6 +62,12 @@ func (r *knowledgeSpanRepository) Upsert(ctx context.Context, row *types.Knowled
if row.Attempt == 0 {
row.Attempt = 1
}
+ // Error fields may originate from third-party parser/model responses.
+ // Normalize them at the persistence boundary so SQLite/PostgreSQL and the
+ // tracing API never receive malformed UTF-8.
+ row.ErrorCode = common.CleanInvalidUTF8(row.ErrorCode)
+ row.ErrorMessage = common.CleanInvalidUTF8(row.ErrorMessage)
+ row.ErrorDetail = common.CleanInvalidUTF8(row.ErrorDetail)
// We let GORM populate created_at/updated_at via the autoCreate /
// autoUpdate tags. ON CONFLICT updates only the fields that may
// transition between calls — name/kind/parent are immutable once
@@ -163,6 +170,7 @@ func (r *knowledgeSpanRepository) GetSpan(ctx context.Context, knowledgeID strin
// Postgres-specific WITH RECURSIVE would be denser but harder to test on
// the SQLite Lite backend. The iterative path stays portable.
func (r *knowledgeSpanRepository) CancelDescendants(ctx context.Context, knowledgeID string, attempt int, parentSpanID, reason string) (int64, error) {
+ reason = common.CleanInvalidUTF8(reason)
frontier := []string{parentSpanID}
var totalAffected int64
for depth := 0; depth < 16 && len(frontier) > 0; depth++ {
@@ -215,6 +223,8 @@ func (r *knowledgeSpanRepository) CancelDescendants(ctx context.Context, knowled
func (r *knowledgeSpanRepository) CancelAllOpenSpans(
ctx context.Context, knowledgeID string, attempt int, errorCode, reason string,
) (int64, error) {
+ errorCode = common.CleanInvalidUTF8(errorCode)
+ reason = common.CleanInvalidUTF8(reason)
now := time.Now()
updates := map[string]any{
"status": types.SpanStatusCancelled,
@@ -240,6 +250,8 @@ func (r *knowledgeSpanRepository) CancelOpenSpansByName(
if knowledgeID == "" || attempt <= 0 || name == "" {
return 0, nil
}
+ errorCode = common.CleanInvalidUTF8(errorCode)
+ reason = common.CleanInvalidUTF8(reason)
now := time.Now()
res := r.db.WithContext(ctx).Model(&types.KnowledgeProcessingSpan{}).
Where("knowledge_id = ? AND attempt = ? AND name = ? AND status IN ?",
diff --git a/internal/application/repository/knowledge_span_repo_test.go b/internal/application/repository/knowledge_span_repo_test.go
index 341a3d4fd..72b70adec 100644
--- a/internal/application/repository/knowledge_span_repo_test.go
+++ b/internal/application/repository/knowledge_span_repo_test.go
@@ -4,6 +4,7 @@ import (
"context"
"testing"
"time"
+ "unicode/utf8"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/assert"
@@ -85,6 +86,37 @@ func TestKnowledgeSpanRepo_UpsertAndList(t *testing.T) {
assert.Equal(t, int64(2000), rows[0].DurationMs)
}
+func TestKnowledgeSpanRepo_UpsertSanitizesErrorFields(t *testing.T) {
+ repo, _ := setupSpanTestRepo(t)
+ now := time.Now()
+ invalid := "prefix " + string([]byte{0xef, 0xbc, 0x2e}) + " suffix"
+
+ require.NoError(t, repo.Upsert(context.Background(), &types.KnowledgeProcessingSpan{
+ KnowledgeID: "kid-invalid-utf8",
+ Attempt: 1,
+ SpanID: "span-invalid",
+ Name: types.StageDocReader,
+ Kind: types.SpanKindStage,
+ Status: types.SpanStatusFailed,
+ ErrorCode: invalid,
+ ErrorMessage: invalid,
+ ErrorDetail: invalid,
+ StartedAt: &now,
+ }))
+
+ rows, err := repo.ListByAttempt(context.Background(), "kid-invalid-utf8", 1)
+ require.NoError(t, err)
+ require.Len(t, rows, 1)
+ for _, value := range []string{rows[0].ErrorCode, rows[0].ErrorMessage, rows[0].ErrorDetail} {
+ if !utf8.ValidString(value) {
+ t.Fatalf("persisted error field is invalid UTF-8: % x", []byte(value))
+ }
+ if value != "prefix . suffix" {
+ t.Fatalf("persisted error field = %q, want %q", value, "prefix . suffix")
+ }
+ }
+}
+
// TestKnowledgeSpanRepo_NextAttempt confirms that NextAttempt allocates
// a fresh number per knowledge, isolating reparse history. Critical
// because the API layer renders attempt history by this number.
@@ -182,6 +214,45 @@ func TestKnowledgeSpanRepo_CancelOpenSpansByName(t *testing.T) {
assert.Equal(t, types.SpanStatusRunning, statusBy["q-old"])
}
+func TestKnowledgeSpanRepo_CancelPathsSanitizeErrorFields(t *testing.T) {
+ repo, _ := setupSpanTestRepo(t)
+ ctx := context.Background()
+ kid := "kid-cancel-invalid-utf8"
+ now := time.Now()
+ for _, r := range []*types.KnowledgeProcessingSpan{
+ {KnowledgeID: kid, Attempt: 1, SpanID: "root", Name: "root", Kind: types.SpanKindRoot, Status: types.SpanStatusRunning, StartedAt: &now},
+ {KnowledgeID: kid, Attempt: 1, SpanID: "descendant", ParentSpanID: "root", Name: "descendant", Kind: types.SpanKindSubSpan, Status: types.SpanStatusRunning, StartedAt: &now},
+ {KnowledgeID: kid, Attempt: 1, SpanID: "bulk", Name: "bulk", Kind: types.SpanKindSubSpan, Status: types.SpanStatusRunning, StartedAt: &now},
+ {KnowledgeID: kid, Attempt: 1, SpanID: "named", Name: "named", Kind: types.SpanKindSubSpan, Status: types.SpanStatusRunning, StartedAt: &now},
+ } {
+ require.NoError(t, repo.Upsert(ctx, r))
+ }
+
+ invalidCode := "CODE " + string([]byte{0xef, 0xbc, 0x2e})
+ invalidReason := "reason " + string([]byte{0xef, 0xbc, 0x2e})
+ _, err := repo.CancelOpenSpansByName(ctx, kid, 1, "named", invalidCode, invalidReason)
+ require.NoError(t, err)
+ _, err = repo.CancelDescendants(ctx, kid, 1, "root", invalidReason)
+ require.NoError(t, err)
+ _, err = repo.CancelAllOpenSpans(ctx, kid, 1, invalidCode, invalidReason)
+ require.NoError(t, err)
+
+ rows, err := repo.ListByAttempt(ctx, kid, 1)
+ require.NoError(t, err)
+ byID := make(map[string]types.KnowledgeProcessingSpan, len(rows))
+ for _, row := range rows {
+ byID[row.SpanID] = row
+ }
+ assert.Equal(t, "UPSTREAM_FAILED", byID["descendant"].ErrorCode)
+ assert.Equal(t, "reason .", byID["descendant"].ErrorMessage)
+ for _, id := range []string{"root", "bulk", "named"} {
+ assert.Equal(t, "CODE .", byID[id].ErrorCode, id)
+ assert.Equal(t, "reason .", byID[id].ErrorMessage, id)
+ assert.True(t, utf8.ValidString(byID[id].ErrorCode), id)
+ assert.True(t, utf8.ValidString(byID[id].ErrorMessage), id)
+ }
+}
+
// TestKnowledgeSpanRepo_ListAttemptIsolation guarantees that different
// attempts of the same knowledge stay queryable independently — the
// foundation for the "show history" UI navigation (?attempt=N).
diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go
index 6c2f7350b..ce9365fcd 100644
--- a/internal/application/service/knowledge_process.go
+++ b/internal/application/service/knowledge_process.go
@@ -11,6 +11,7 @@ import (
"time"
"github.com/Tencent/WeKnora/internal/application/service/retriever"
+ "github.com/Tencent/WeKnora/internal/common"
werrors "github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/infrastructure/chunker"
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
@@ -251,6 +252,23 @@ func (s *knowledgeService) processChunks(ctx context.Context,
options = opts[0]
}
+ // Parser output and manually supplied passages can contain malformed byte
+ // sequences. Clean them before logging, chunk persistence, or embedding;
+ // the embedding provider and tracing/database drivers expect valid UTF-8.
+ for i := range chunks {
+ chunks[i].Content = common.CleanInvalidUTF8(chunks[i].Content)
+ chunks[i].ContextHeader = common.CleanInvalidUTF8(chunks[i].ContextHeader)
+ for j := range chunks[i].Images {
+ chunks[i].Images[j].URL = common.CleanInvalidUTF8(chunks[i].Images[j].URL)
+ chunks[i].Images[j].Caption = common.CleanInvalidUTF8(chunks[i].Images[j].Caption)
+ chunks[i].Images[j].OCRText = common.CleanInvalidUTF8(chunks[i].Images[j].OCRText)
+ chunks[i].Images[j].OriginalURL = common.CleanInvalidUTF8(chunks[i].Images[j].OriginalURL)
+ }
+ }
+ for i := range options.ParentChunks {
+ options.ParentChunks[i].Content = common.CleanInvalidUTF8(options.ParentChunks[i].Content)
+ }
+
// Check if knowledge is being deleted/cancelled before processing.
// Both statuses short-circuit identically here — there's nothing to clean
// up yet so the branch is purely "stop early".
@@ -2896,6 +2914,7 @@ func (s *knowledgeService) UpdateImageInfo(
chunkID string,
imageInfo string,
) error {
+ imageInfo = common.CleanInvalidUTF8(imageInfo)
var images []*types.ImageInfo
if err := json.Unmarshal([]byte(imageInfo), &images); err != nil {
logger.Errorf(ctx, "Failed to unmarshal image info: %v", err)
@@ -3516,6 +3535,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
// Step 3: Split into chunks using Go chunker. Browser textareas normalize
// pasted content to LF, so normalize uploaded source text before calculating
// chunk boundaries as well.
+ sanitizeReadResult(convertResult)
convertResult.MarkdownContent = chunker.NormalizeLineEndings(convertResult.MarkdownContent)
chunkCfg := buildSplitterConfigFromChunking(eff.ChunkingConfig)
@@ -3572,6 +3592,28 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
return nil
}
+// sanitizeReadResult protects every text field that can cross from a parser
+// into the embedding, storage, or tracing layers. A parser may return a Go
+// string containing arbitrary bytes even though the string type itself does
+// not enforce UTF-8 validity.
+func sanitizeReadResult(result *types.ReadResult) {
+ if result == nil {
+ return
+ }
+ result.MarkdownContent = common.CleanInvalidUTF8(result.MarkdownContent)
+ result.ImageDirPath = common.CleanInvalidUTF8(result.ImageDirPath)
+ result.Error = common.CleanInvalidUTF8(result.Error)
+ for key, value := range result.Metadata {
+ result.Metadata[key] = common.CleanInvalidUTF8(value)
+ }
+ for i := range result.ImageRefs {
+ result.ImageRefs[i].Filename = common.CleanInvalidUTF8(result.ImageRefs[i].Filename)
+ result.ImageRefs[i].OriginalRef = common.CleanInvalidUTF8(result.ImageRefs[i].OriginalRef)
+ result.ImageRefs[i].MimeType = common.CleanInvalidUTF8(result.ImageRefs[i].MimeType)
+ result.ImageRefs[i].StorageKey = common.CleanInvalidUTF8(result.ImageRefs[i].StorageKey)
+ }
+}
+
// convert handles both file and URL reading using a unified ReadRequest.
func (s *knowledgeService) convert(
ctx context.Context,
@@ -3689,6 +3731,7 @@ func (s *knowledgeService) convert(
code, "document read failed", err)
return s.failKnowledge(ctx, knowledge, isLastRetry, "document read failed: %v", err)
}
+ sanitizeReadResult(result)
if result.Error != "" {
logger.Errorf(ctx, "[convert] parser returned error kb=%s knowledge=%s file=%q type=%s engine=%q: %s",
kb.ID, knowledge.ID, req.FileName, fileType, parserEngine, result.Error)
diff --git a/internal/common/tools_test.go b/internal/common/tools_test.go
index d3a2e6e26..53ec83d8b 100644
--- a/internal/common/tools_test.go
+++ b/internal/common/tools_test.go
@@ -1,6 +1,9 @@
package common
-import "testing"
+import (
+ "testing"
+ "unicode/utf8"
+)
func TestParseLLMJsonResponse(t *testing.T) {
type payload struct {
@@ -39,6 +42,18 @@ func TestParseLLMJsonResponse(t *testing.T) {
}
}
+func TestCleanInvalidUTF8(t *testing.T) {
+ input := "valid " + string([]byte{0xef, 0xbc, 0x2e}) + "\x00tail"
+
+ got := CleanInvalidUTF8(input)
+ if !utf8.ValidString(got) {
+ t.Fatalf("CleanInvalidUTF8 returned invalid UTF-8: % x", []byte(got))
+ }
+ if want := "valid .tail"; got != want {
+ t.Fatalf("CleanInvalidUTF8() = %q, want %q", got, want)
+ }
+}
+
func BenchmarkParseLLMJsonResponse_Fenced(b *testing.B) {
const content = "```json\n{\"name\":\"Acme\",\"slug\":\"entity/acme\",\"aliases\":[\"A\",\"B\"]}\n```"
type payload struct {