Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(datasource): truncate failed sync detail at rune boundary (#4555)
Clinte80b2b66c293
diff --git a/internal/application/service/datasource_service.go b/internal/application/service/datasource_service.goindex 75461fd03..4e2c0c11b 100644--- a/internal/application/service/datasource_service.go+++ b/internal/application/service/datasource_service.go@@ -12,6 +12,7 @@ import ("slices""strings""time"+ "unicode/utf8""github.com/Tencent/WeKnora/internal/application/access""github.com/Tencent/WeKnora/internal/datasource"@@ -1325,7 +1326,13 @@ func allFetchedItemsFailedError(result *types.SyncResult) error {detail = result.Errors[0].Display()const maxDetailLen = 500if len(detail) > maxDetailLen {- detail = detail[:maxDetailLen] + "..."+ // Back off to a rune boundary: a partial multi-byte rune is invalid+ // UTF-8, which PostgreSQL rejects when the error is persisted.+ cut := maxDetailLen+ for cut > 0 && !utf8.RuneStart(detail[cut]) {+ cut--+ }+ detail = detail[:cut] + "..."}}if detail == "" {diff --git a/internal/application/service/datasource_service_test.go b/internal/application/service/datasource_service_test.goindex b8cb29ee5..4d256e07f 100644--- a/internal/application/service/datasource_service_test.go+++ b/internal/application/service/datasource_service_test.go@@ -8,6 +8,7 @@ import ("strings""testing""time"+ "unicode/utf8"apprepo "github.com/Tencent/WeKnora/internal/application/repository""github.com/Tencent/WeKnora/internal/datasource"@@ -221,6 +222,20 @@ func TestAllFetchedItemsFailedErrorTruncatesLongDetail(t *testing.T) {assert.Contains(t, err.Error(), "...")}+func TestAllFetchedItemsFailedErrorTruncatesChineseDetailAtRuneBoundary(t *testing.T) {+ // 499 ASCII bytes put byte 500 inside the first three-byte rune.+ message := strings.Repeat("x", 499) + strings.Repeat("导出失败", 50)+ err := allFetchedItemsFailedError(&types.SyncResult{+ Total: 1,+ Failed: 1,+ Errors: []types.SyncItemError{{Message: message}},+ })+ require.Error(t, err)+ assert.True(t, utf8.ValidString(err.Error()), "error message must be valid UTF-8")+ assert.LessOrEqual(t, len(err.Error()), 560)+ assert.True(t, strings.HasSuffix(err.Error(), strings.Repeat("x", 499)+"..."))+}+const deletedItemConnectorType = "test-sync-deletion"type deletedItemConnector struct{}