Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(errors): truncate error messages at rune boundaries (#4609)
Sun Shihaoe11e919ea132
diff --git a/internal/application/service/knowledge_housekeeping.go b/internal/application/service/knowledge_housekeeping.goindex 3aa88a149..89d712c79 100644--- a/internal/application/service/knowledge_housekeeping.go+++ b/internal/application/service/knowledge_housekeeping.go@@ -35,6 +35,8 @@ import ("github.com/Tencent/WeKnora/internal/types/interfaces""github.com/robfig/cron/v3""gorm.io/gorm"++ "github.com/Tencent/WeKnora/internal/common")// HousekeepingService runs background sweeps to recover stuck rows.@@ -875,7 +877,7 @@ func (h *HousekeepingService) reapTerminalSpans(ctx context.Context) {msg = strings.TrimSpace(*r.ErrorMessage)}if len(msg) > 1024 {- msg = msg[:1024]+ msg = common.TruncateUTF8Bytes(msg, 1024)}res := h.db.WithContext(ctx).Model(&types.KnowledgeProcessingSpan{}).Where("id = ? AND status IN ?", r.ID, openSpanStatuses).diff --git a/internal/application/service/knowledge_span_tracker.go b/internal/application/service/knowledge_span_tracker.goindex f5bac889f..4eac57108 100644--- a/internal/application/service/knowledge_span_tracker.go+++ b/internal/application/service/knowledge_span_tracker.go@@ -38,6 +38,8 @@ import ("github.com/google/uuid""github.com/hibiken/asynq""gorm.io/gorm"++ "github.com/Tencent/WeKnora/internal/common")// maxSpanNameLen matches knowledge_processing_spans.name (varchar(255)).@@ -520,11 +522,11 @@ func (t *spanTracker) failSpan(if errorDetail != nil {detail = errorDetail.Error()if len(detail) > 8192 {- detail = detail[:8192]+ detail = common.TruncateUTF8Bytes(detail, 8192)}}if len(errorMessage) > 1024 {- errorMessage = errorMessage[:1024]+ errorMessage = common.TruncateUTF8Bytes(errorMessage, 1024)}row := &types.KnowledgeProcessingSpan{KnowledgeID: span.KnowledgeID,@@ -844,7 +846,7 @@ func (t *spanTracker) FinalizeAttempt(ctx context.Context, knowledgeID string, adur = now.Sub(started).Milliseconds()}if len(errorMessage) > 1024 {- errorMessage = errorMessage[:1024]+ errorMessage = common.TruncateUTF8Bytes(errorMessage, 1024)}row := &types.KnowledgeProcessingSpan{KnowledgeID: root.KnowledgeID,@@ -918,6 +920,7 @@ func (noopSpanTracker) LatestAttempt(_ context.Context, _ string) int { return 0func (noopSpanTracker) BeginStage(_ context.Context, _ string, _ int, _ string, _ types.JSONMap) *Span {return nil}+func (noopSpanTracker) BeginSubSpan(_ context.Context, _ *Span, _, _ string, _ types.JSONMap) *Span {return nil}@@ -926,9 +929,11 @@ func (noopSpanTracker) FailSpan(_ context.Context, _ *Span, _, _ string, _ errorfunc (noopSpanTracker) FailSpanRetryable(_ context.Context, _ *Span, _, _ string, _ error) {}func (noopSpanTracker) SkipSpan(_ context.Context, _ *Span, _ string) {}func (noopSpanTracker) LookupStage(_ context.Context, _ string, _ int, _ string) *Span { return nil }+func (noopSpanTracker) LookupSpanByName(_ context.Context, _ string, _ int, _ string) *Span {return nil}+func (noopSpanTracker) FinalizeAttempt(_ context.Context, _ string, _ int, _ string, _ types.JSONMap, _, _ string) {}func (noopSpanTracker) AbortAttempt(_ context.Context, _ string, _ int, _, _, _ string) {}diff --git a/internal/application/service/temporary_document_parse.go b/internal/application/service/temporary_document_parse.goindex 44cd2bbc2..1605136c7 100644--- a/internal/application/service/temporary_document_parse.go+++ b/internal/application/service/temporary_document_parse.go@@ -110,7 +110,7 @@ func (s *temporaryDocumentService) Process(ctx context.Context, task *asynq.Task}message := parseErr.Error()if len(message) > 2000 {- message = message[:2000]+ message = common.TruncateUTF8Bytes(message, 2000)}_ = s.repo.MarkFailed(ctx, payload.TenantID, payload.DocumentID, message)logger.Errorf(ctx, "temporary document parse failed: document_id=%s err=%v", payload.DocumentID, parseErr)diff --git a/internal/common/tools.go b/internal/common/tools.goindex 93db4158c..0355250a1 100644--- a/internal/common/tools.go+++ b/internal/common/tools.go@@ -291,3 +291,21 @@ func truncatePipelineValue(content string) string {func TruncateForLog(content string) string {return truncatePipelineValue(content)}++// TruncateUTF8Bytes shortens s to at most maxBytes bytes without splitting a+// multi-byte rune. Plain string slicing cuts UTF-8 in the middle of a rune,+// which leaves invalid text in error messages that are persisted or returned+// to API clients. maxBytes <= 0 returns an empty string.+func TruncateUTF8Bytes(s string, maxBytes int) string {+ if maxBytes <= 0 {+ return ""+ }+ if len(s) <= maxBytes {+ return s+ }+ end := maxBytes+ for end > 0 && !utf8.RuneStart(s[end]) {+ end--+ }+ return s[:end]+}diff --git a/internal/common/truncate_utf8_test.go b/internal/common/truncate_utf8_test.gonew file mode 100644index 000000000..6ecc76ebc--- /dev/null+++ b/internal/common/truncate_utf8_test.go@@ -0,0 +1,45 @@+package common++import (+ "strings"+ "testing"+ "unicode/utf8"+)++// Error messages are truncated before being persisted or returned to clients.+// A plain byte slice splits UTF-8 runes and leaves invalid text behind.+func TestTruncateUTF8BytesNeverSplitsARune(t *testing.T) {+ // 3 bytes per rune, so 1024 is not a rune boundary.+ msg := strings.Repeat("解析失败", 200)+ got := TruncateUTF8Bytes(msg, 1024)++ if !utf8.ValidString(got) {+ t.Fatalf("truncated message is not valid UTF-8: %q", got[max(0, len(got)-8):])+ }+ if len(got) > 1024 {+ t.Fatalf("truncated message is %d bytes, want at most 1024", len(got))+ }+ if !strings.HasPrefix(msg, got) {+ t.Fatal("truncation must keep a prefix of the original message")+ }+ if len(got) < 1020 {+ t.Fatalf("truncation dropped %d bytes, want at most 2", 1024-len(got))+ }+}++func TestTruncateUTF8BytesKeepsShortAndEmptyInput(t *testing.T) {+ for _, s := range []string{"", "ok", "解析失败", strings.Repeat("错", 341)} {+ if got := TruncateUTF8Bytes(s, 1024); got != s {+ t.Fatalf("TruncateUTF8Bytes(%q) = %q, want the input unchanged", s, got)+ }+ }+}++func TestTruncateUTF8BytesRejectsNonPositiveLimit(t *testing.T) {+ if got := TruncateUTF8Bytes("内容", 0); got != "" {+ t.Fatalf("limit 0 = %q, want an empty string", got)+ }+ if got := TruncateUTF8Bytes("内容", -1); got != "" {+ t.Fatalf("limit -1 = %q, want an empty string", got)+ }+}