Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(router): truncate dead-letter knowledge errors at rune boundary (#4554)
Clint757540a754cb
diff --git a/internal/router/task.go b/internal/router/task.goindex 6e9910675..b2fdffb51 100644--- a/internal/router/task.go+++ b/internal/router/task.go@@ -9,6 +9,7 @@ import ("strconv""strings""time"+ "unicode/utf8""github.com/Tencent/WeKnora/internal/application/service""github.com/Tencent/WeKnora/internal/common"@@ -426,11 +427,7 @@ func newDeadLetterKnowledgeFailer(ks interfaces.KnowledgeService, tracker servicprobe.Attempt, probe.KnowledgeID)return}- errMsg := "task " + t.Type() + " exhausted retries: " + taskErr.Error()- // 8KB is the same cap the dead-letter row uses for last_error.- if len(errMsg) > 8192 {- errMsg = errMsg[:8192]- }+ errMsg := truncateKnowledgeErrorMessage("task " + t.Type() + " exhausted retries: " + taskErr.Error())before, after := *row, *rowafter.ParseStatus = types.ParseStatusFailedafter.ErrorMessage = errMsg@@ -454,6 +451,22 @@ func newDeadLetterKnowledgeFailer(ks interfaces.KnowledgeService, tracker servic}}+// truncateKnowledgeErrorMessage caps msg at 8KB, the same cap the dead-letter+// row uses for last_error. The cut backs off to a rune boundary because a+// partial multi-byte rune is invalid UTF-8, which PostgreSQL rejects, leaving+// the knowledge stuck in its in-flight status.+func truncateKnowledgeErrorMessage(msg string) string {+ const maxLen = 8192+ if len(msg) <= maxLen {+ return msg+ }+ cut := maxLen+ for cut > 0 && !utf8.RuneStart(msg[cut]) {+ cut--+ }+ return msg[:cut]+}+func markKnowledgeListDeleteFailed(ctx context.Context,repo interfaces.KnowledgeRepository,@@ -483,10 +496,7 @@ func markKnowledgeListDeleteFailed(}}- errMsg := "delete task exhausted retries: " + taskErr.Error()- if len(errMsg) > 8192 {- errMsg = errMsg[:8192]- }+ errMsg := truncateKnowledgeErrorMessage("delete task exhausted retries: " + taskErr.Error())for _, knowledgeID := range payload.KnowledgeIDs {if knowledgeID == "" {continuediff --git a/internal/router/task_dead_letter_utf8_test.go b/internal/router/task_dead_letter_utf8_test.gonew file mode 100644index 000000000..ffac04e91--- /dev/null+++ b/internal/router/task_dead_letter_utf8_test.go@@ -0,0 +1,77 @@+package router++import (+ "context"+ "encoding/json"+ "errors"+ "path/filepath"+ "strings"+ "testing"+ "unicode/utf8"++ "github.com/Tencent/WeKnora/internal/application/repository"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/hibiken/asynq"+ "github.com/stretchr/testify/require"+ "gorm.io/driver/sqlite"+ "gorm.io/gorm"+)++// errorSplittingRuneAt8192 returns an error whose message, once prefixed,+// places byte 8192 inside a three-byte CJK rune.+func errorSplittingRuneAt8192(prefix string) error {+ return errors.New(strings.Repeat("a", 8192-len(prefix)-1) + strings.Repeat("中", 8))+}++func TestDeadLetterFailureMessageIsTruncatedAtRuneBoundary(t *testing.T) {+ db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "callback.db")), &gorm.Config{})+ require.NoError(t, err)+ sqlDB, err := db.DB()+ require.NoError(t, err)+ t.Cleanup(func() { require.NoError(t, sqlDB.Close()) })+ require.NoError(t, db.AutoMigrate(&types.Knowledge{}))+ repo := repository.NewKnowledgeRepository(db)++ t.Run("document process", func(t *testing.T) {+ require.NoError(t, db.Create(&types.Knowledge{+ ID: "processing", TenantID: 7, KnowledgeBaseID: "kb", ParseStatus: types.ParseStatusProcessing,+ }).Error)+ payload, err := json.Marshal(+ types.DocumentProcessPayload{TenantID: 7, KnowledgeBaseID: "kb", KnowledgeID: "processing"},+ )+ require.NoError(t, err)+ taskErr := errorSplittingRuneAt8192("task " + types.TypeDocumentProcess + " exhausted retries: ")++ callback := newDeadLetterKnowledgeFailer(transferFailureKnowledge{repo: repo}, nil)+ callback(context.Background(), asynq.NewTask(types.TypeDocumentProcess, payload), taskErr)++ row, err := repo.GetKnowledgeByID(context.Background(), 7, "processing")+ require.NoError(t, err)+ require.Equal(t, types.ParseStatusFailed, row.ParseStatus)+ require.True(t, utf8.ValidString(row.ErrorMessage), "error_message must be valid UTF-8")+ require.LessOrEqual(t, len(row.ErrorMessage), 8192)+ require.Len(t, row.ErrorMessage, 8191)+ })++ t.Run("knowledge list delete", func(t *testing.T) {+ require.NoError(t, db.Create(&types.Knowledge{+ ID: "deleting", TenantID: 7, KnowledgeBaseID: "kb", ParseStatus: types.ParseStatusDeleting,+ }).Error)+ payload, err := json.Marshal(types.KnowledgeListDeletePayload{+ TenantID: 7, KnowledgeBaseID: "kb", KnowledgeIDs: []string{"deleting"},+ })+ require.NoError(t, err)+ taskErr := errorSplittingRuneAt8192("delete task exhausted retries: ")++ markKnowledgeListDeleteFailed(+ context.Background(), repo, asynq.NewTask(types.TypeKnowledgeListDelete, payload), taskErr,+ )++ row, err := repo.GetKnowledgeByID(context.Background(), 7, "deleting")+ require.NoError(t, err)+ require.Equal(t, types.ParseStatusFailed, row.ParseStatus)+ require.True(t, utf8.ValidString(row.ErrorMessage), "error_message must be valid UTF-8")+ require.LessOrEqual(t, len(row.ErrorMessage), 8192)+ require.Len(t, row.ErrorMessage, 8191)+ })+}