Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(errors): truncate error messages at rune boundaries (#4609)

Sun Shihaoe11e919ea132
diff --git a/internal/application/service/knowledge_housekeeping.go b/internal/application/service/knowledge_housekeeping.go
index 3aa88a149..89d712c79 100644
--- a/internal/application/service/knowledge_housekeeping.go
+++ b/internal/application/service/knowledge_housekeeping.go
@@ -35,6 +35,8 @@ import (
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/robfig/cron/v3"
"gorm.io/gorm"
+
+ "github.com/Tencent/WeKnora/internal/common"
)
// HousekeepingService runs background sweeps to recover stuck rows.
@@ -875,7 +877,7 @@ func (h *HousekeepingService) reapTerminalSpans(ctx context.Context) {
msg = strings.TrimSpace(*r.ErrorMessage)
}
if len(msg) > 1024 {
- msg = msg[:1024]
+ msg = common.TruncateUTF8Bytes(msg, 1024)
}
res := h.db.WithContext(ctx).Model(&types.KnowledgeProcessingSpan{}).
Where("id = ? AND status IN ?", r.ID, openSpanStatuses).
diff --git a/internal/application/service/knowledge_span_tracker.go b/internal/application/service/knowledge_span_tracker.go
index f5bac889f..4eac57108 100644
--- a/internal/application/service/knowledge_span_tracker.go
+++ b/internal/application/service/knowledge_span_tracker.go
@@ -38,6 +38,8 @@ import (
"github.com/google/uuid"
"github.com/hibiken/asynq"
"gorm.io/gorm"
+
+ "github.com/Tencent/WeKnora/internal/common"
)
// maxSpanNameLen matches knowledge_processing_spans.name (varchar(255)).
@@ -520,11 +522,11 @@ func (t *spanTracker) failSpan(
if errorDetail != nil {
detail = errorDetail.Error()
if len(detail) > 8192 {
- detail = detail[:8192]
+ detail = common.TruncateUTF8Bytes(detail, 8192)
}
}
if len(errorMessage) > 1024 {
- errorMessage = errorMessage[:1024]
+ errorMessage = common.TruncateUTF8Bytes(errorMessage, 1024)
}
row := &types.KnowledgeProcessingSpan{
KnowledgeID: span.KnowledgeID,
@@ -844,7 +846,7 @@ func (t *spanTracker) FinalizeAttempt(ctx context.Context, knowledgeID string, a
dur = now.Sub(started).Milliseconds()
}
if len(errorMessage) > 1024 {
- errorMessage = errorMessage[:1024]
+ errorMessage = common.TruncateUTF8Bytes(errorMessage, 1024)
}
row := &types.KnowledgeProcessingSpan{
KnowledgeID: root.KnowledgeID,
@@ -918,6 +920,7 @@ func (noopSpanTracker) LatestAttempt(_ context.Context, _ string) int { return 0
func (noopSpanTracker) BeginStage(_ context.Context, _ string, _ int, _ string, _ types.JSONMap) *Span {
return nil
}
+
func (noopSpanTracker) BeginSubSpan(_ context.Context, _ *Span, _, _ string, _ types.JSONMap) *Span {
return nil
}
@@ -926,9 +929,11 @@ func (noopSpanTracker) FailSpan(_ context.Context, _ *Span, _, _ string, _ error
func (noopSpanTracker) FailSpanRetryable(_ context.Context, _ *Span, _, _ string, _ error) {}
func (noopSpanTracker) SkipSpan(_ context.Context, _ *Span, _ string) {}
func (noopSpanTracker) LookupStage(_ context.Context, _ string, _ int, _ string) *Span { return nil }
+
func (noopSpanTracker) LookupSpanByName(_ context.Context, _ string, _ int, _ string) *Span {
return nil
}
+
func (noopSpanTracker) FinalizeAttempt(_ context.Context, _ string, _ int, _ string, _ types.JSONMap, _, _ string) {
}
func (noopSpanTracker) AbortAttempt(_ context.Context, _ string, _ int, _, _, _ string) {}
diff --git a/internal/application/service/temporary_document_parse.go b/internal/application/service/temporary_document_parse.go
index 44cd2bbc2..1605136c7 100644
--- a/internal/application/service/temporary_document_parse.go
+++ b/internal/application/service/temporary_document_parse.go
@@ -110,7 +110,7 @@ func (s *temporaryDocumentService) Process(ctx context.Context, task *asynq.Task
}
message := parseErr.Error()
if len(message) > 2000 {
- message = message[:2000]
+ message = common.TruncateUTF8Bytes(message, 2000)
}
_ = s.repo.MarkFailed(ctx, payload.TenantID, payload.DocumentID, message)
logger.Errorf(ctx, "temporary document parse failed: document_id=%s err=%v", payload.DocumentID, parseErr)
diff --git a/internal/common/tools.go b/internal/common/tools.go
index 93db4158c..0355250a1 100644
--- a/internal/common/tools.go
+++ b/internal/common/tools.go
@@ -291,3 +291,21 @@ func truncatePipelineValue(content string) string {
func TruncateForLog(content string) string {
return truncatePipelineValue(content)
}
+
+// TruncateUTF8Bytes shortens s to at most maxBytes bytes without splitting a
+// multi-byte rune. Plain string slicing cuts UTF-8 in the middle of a rune,
+// which leaves invalid text in error messages that are persisted or returned
+// to API clients. maxBytes <= 0 returns an empty string.
+func TruncateUTF8Bytes(s string, maxBytes int) string {
+ if maxBytes <= 0 {
+ return ""
+ }
+ if len(s) <= maxBytes {
+ return s
+ }
+ end := maxBytes
+ for end > 0 && !utf8.RuneStart(s[end]) {
+ end--
+ }
+ return s[:end]
+}
diff --git a/internal/common/truncate_utf8_test.go b/internal/common/truncate_utf8_test.go
new file mode 100644
index 000000000..6ecc76ebc
--- /dev/null
+++ b/internal/common/truncate_utf8_test.go
@@ -0,0 +1,45 @@
+package common
+
+import (
+ "strings"
+ "testing"
+ "unicode/utf8"
+)
+
+// Error messages are truncated before being persisted or returned to clients.
+// A plain byte slice splits UTF-8 runes and leaves invalid text behind.
+func TestTruncateUTF8BytesNeverSplitsARune(t *testing.T) {
+ // 3 bytes per rune, so 1024 is not a rune boundary.
+ msg := strings.Repeat("解析失败", 200)
+ got := TruncateUTF8Bytes(msg, 1024)
+
+ if !utf8.ValidString(got) {
+ t.Fatalf("truncated message is not valid UTF-8: %q", got[max(0, len(got)-8):])
+ }
+ if len(got) > 1024 {
+ t.Fatalf("truncated message is %d bytes, want at most 1024", len(got))
+ }
+ if !strings.HasPrefix(msg, got) {
+ t.Fatal("truncation must keep a prefix of the original message")
+ }
+ if len(got) < 1020 {
+ t.Fatalf("truncation dropped %d bytes, want at most 2", 1024-len(got))
+ }
+}
+
+func TestTruncateUTF8BytesKeepsShortAndEmptyInput(t *testing.T) {
+ for _, s := range []string{"", "ok", "解析失败", strings.Repeat("错", 341)} {
+ if got := TruncateUTF8Bytes(s, 1024); got != s {
+ t.Fatalf("TruncateUTF8Bytes(%q) = %q, want the input unchanged", s, got)
+ }
+ }
+}
+
+func TestTruncateUTF8BytesRejectsNonPositiveLimit(t *testing.T) {
+ if got := TruncateUTF8Bytes("内容", 0); got != "" {
+ t.Fatalf("limit 0 = %q, want an empty string", got)
+ }
+ if got := TruncateUTF8Bytes("内容", -1); got != "" {
+ t.Fatalf("limit -1 = %q, want an empty string", got)
+ }
+}