Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(datasource): truncate failed sync detail at rune boundary (#4555)

Clinte80b2b66c293
diff --git a/internal/application/service/datasource_service.go b/internal/application/service/datasource_service.go
index 75461fd03..4e2c0c11b 100644
--- a/internal/application/service/datasource_service.go
+++ b/internal/application/service/datasource_service.go
@@ -12,6 +12,7 @@ import (
"slices"
"strings"
"time"
+ "unicode/utf8"
"github.com/Tencent/WeKnora/internal/application/access"
"github.com/Tencent/WeKnora/internal/datasource"
@@ -1325,7 +1326,13 @@ func allFetchedItemsFailedError(result *types.SyncResult) error {
detail = result.Errors[0].Display()
const maxDetailLen = 500
if len(detail) > maxDetailLen {
- detail = detail[:maxDetailLen] + "..."
+ // Back off to a rune boundary: a partial multi-byte rune is invalid
+ // UTF-8, which PostgreSQL rejects when the error is persisted.
+ cut := maxDetailLen
+ for cut > 0 && !utf8.RuneStart(detail[cut]) {
+ cut--
+ }
+ detail = detail[:cut] + "..."
}
}
if detail == "" {
diff --git a/internal/application/service/datasource_service_test.go b/internal/application/service/datasource_service_test.go
index b8cb29ee5..4d256e07f 100644
--- a/internal/application/service/datasource_service_test.go
+++ b/internal/application/service/datasource_service_test.go
@@ -8,6 +8,7 @@ import (
"strings"
"testing"
"time"
+ "unicode/utf8"
apprepo "github.com/Tencent/WeKnora/internal/application/repository"
"github.com/Tencent/WeKnora/internal/datasource"
@@ -221,6 +222,20 @@ func TestAllFetchedItemsFailedErrorTruncatesLongDetail(t *testing.T) {
assert.Contains(t, err.Error(), "...")
}
+func TestAllFetchedItemsFailedErrorTruncatesChineseDetailAtRuneBoundary(t *testing.T) {
+ // 499 ASCII bytes put byte 500 inside the first three-byte rune.
+ message := strings.Repeat("x", 499) + strings.Repeat("导出失败", 50)
+ err := allFetchedItemsFailedError(&types.SyncResult{
+ Total: 1,
+ Failed: 1,
+ Errors: []types.SyncItemError{{Message: message}},
+ })
+ require.Error(t, err)
+ assert.True(t, utf8.ValidString(err.Error()), "error message must be valid UTF-8")
+ assert.LessOrEqual(t, len(err.Error()), 560)
+ assert.True(t, strings.HasSuffix(err.Error(), strings.Repeat("x", 499)+"..."))
+}
+
const deletedItemConnectorType = "test-sync-deletion"
type deletedItemConnector struct{}