Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(datasource): truncate connector error previews at rune boundaries (#4614)
yetugedebade7114b1
diff --git a/internal/datasource/connector/feishu/core/client.go b/internal/datasource/connector/feishu/core/client.goindex 52ec80fdd..2b162d3d4 100644--- a/internal/datasource/connector/feishu/core/client.go+++ b/internal/datasource/connector/feishu/core/client.go@@ -12,6 +12,7 @@ import ("strings""sync""time"+ "unicode/utf8""github.com/Tencent/WeKnora/internal/datasource""github.com/Tencent/WeKnora/internal/logger"@@ -349,7 +350,14 @@ func truncate(s string, maxLen int) string {if len(s) <= maxLen {return s}- return s[:maxLen] + "..."+ // Back off to a rune boundary: a partial multi-byte rune is invalid UTF-8,+ // and this preview reaches sync_logs / data_sources through the sync error+ // message, which PostgreSQL rejects. Same rule as #4555.+ cut := maxLen+ for cut > 0 && !utf8.RuneStart(s[cut]) {+ cut--+ }+ return s[:cut] + "..."}// ListWikiSpaces returns all wiki spaces accessible to the app.diff --git a/internal/datasource/connector/feishu/core/truncate_utf8_test.go b/internal/datasource/connector/feishu/core/truncate_utf8_test.gonew file mode 100644index 000000000..847131a2c--- /dev/null+++ b/internal/datasource/connector/feishu/core/truncate_utf8_test.go@@ -0,0 +1,39 @@+package core++import (+ "strings"+ "testing"+ "unicode/utf8"+)++// truncate caps a connector error preview with a plain byte slice. Connector error+// text is often Chinese, so the cut can land inside a multi-byte rune; these+// previews reach sync_logs and data_sources through the sync error message, and+// PostgreSQL rejects the invalid UTF-8 that a partial rune produces.+func TestTruncateKeepsValidUTF8AcrossCaps(t *testing.T) {+ body := strings.Repeat("错", 400)+ for _, maxLen := range []int{499, 500, 501, 1000, 1} {+ got := truncate(body, maxLen)++ if !utf8.ValidString(got) {+ t.Fatalf("%s(%d): preview is not valid UTF-8", "truncate", maxLen)+ }+ if len(got) > maxLen+len("...") {+ t.Fatalf("%s(%d): preview is %d bytes", "truncate", maxLen, len(got))+ }+ if !strings.HasPrefix(body, strings.TrimSuffix(got, "...")) {+ t.Fatalf("%s(%d): preview is not a prefix of the input", "truncate", maxLen)+ }+ }+}++func TestTruncateLeavesShortAndASCIIInputAlone(t *testing.T) {+ for _, s := range []string{"", "ok", "服务返回 500", "短"} {+ if got := truncate(s, 500); got != s {+ t.Fatalf("%s(%q, 500) = %q, want the input unchanged", "truncate", s, got)+ }+ }+ if got := truncate("abcdef", 3); got != "abc..." {+ t.Fatalf("truncate keeps the ASCII cut: %q", got)+ }+}diff --git a/internal/datasource/connector/ima/client.go b/internal/datasource/connector/ima/client.goindex e5d713f7a..dd9513c49 100644--- a/internal/datasource/connector/ima/client.go+++ b/internal/datasource/connector/ima/client.go@@ -13,6 +13,7 @@ import ("net/http""sync""time"+ "unicode/utf8""github.com/Tencent/WeKnora/internal/datasource""github.com/Tencent/WeKnora/internal/logger"@@ -415,7 +416,14 @@ func truncate(s string, maxLen int) string {if len(s) <= maxLen {return s}- return s[:maxLen] + "..."+ // Back off to a rune boundary: a partial multi-byte rune is invalid UTF-8,+ // and this preview reaches sync_logs / data_sources through the sync error+ // message, which PostgreSQL rejects. Same rule as #4555.+ cut := maxLen+ for cut > 0 && !utf8.RuneStart(s[cut]) {+ cut--+ }+ return s[:cut] + "..."}// redact returns a masked form of a credential for logging (never log full).diff --git a/internal/datasource/connector/ima/truncate_utf8_test.go b/internal/datasource/connector/ima/truncate_utf8_test.gonew file mode 100644index 000000000..86139be4c--- /dev/null+++ b/internal/datasource/connector/ima/truncate_utf8_test.go@@ -0,0 +1,39 @@+package ima++import (+ "strings"+ "testing"+ "unicode/utf8"+)++// truncate caps a connector error preview with a plain byte slice. Connector error+// text is often Chinese, so the cut can land inside a multi-byte rune; these+// previews reach sync_logs and data_sources through the sync error message, and+// PostgreSQL rejects the invalid UTF-8 that a partial rune produces.+func TestTruncateKeepsValidUTF8AcrossCaps(t *testing.T) {+ body := strings.Repeat("错", 400)+ for _, maxLen := range []int{499, 500, 501, 1000, 1} {+ got := truncate(body, maxLen)++ if !utf8.ValidString(got) {+ t.Fatalf("%s(%d): preview is not valid UTF-8", "truncate", maxLen)+ }+ if len(got) > maxLen+len("...") {+ t.Fatalf("%s(%d): preview is %d bytes", "truncate", maxLen, len(got))+ }+ if !strings.HasPrefix(body, strings.TrimSuffix(got, "...")) {+ t.Fatalf("%s(%d): preview is not a prefix of the input", "truncate", maxLen)+ }+ }+}++func TestTruncateLeavesShortAndASCIIInputAlone(t *testing.T) {+ for _, s := range []string{"", "ok", "服务返回 500", "短"} {+ if got := truncate(s, 500); got != s {+ t.Fatalf("%s(%q, 500) = %q, want the input unchanged", "truncate", s, got)+ }+ }+ if got := truncate("abcdef", 3); got != "abc..." {+ t.Fatalf("truncate keeps the ASCII cut: %q", got)+ }+}diff --git a/internal/datasource/connector/yuque/client.go b/internal/datasource/connector/yuque/client.goindex f51fae198..b3730da69 100644--- a/internal/datasource/connector/yuque/client.go+++ b/internal/datasource/connector/yuque/client.go@@ -10,6 +10,7 @@ import ("net/url""sync""time"+ "unicode/utf8""github.com/Tencent/WeKnora/internal/datasource""github.com/Tencent/WeKnora/internal/logger"@@ -229,7 +230,14 @@ func truncate(s string, maxLen int) string {if len(s) <= maxLen {return s}- return s[:maxLen] + "..."+ // Back off to a rune boundary: a partial multi-byte rune is invalid UTF-8,+ // and this preview reaches sync_logs / data_sources through the sync error+ // message, which PostgreSQL rejects. Same rule as #4555.+ cut := maxLen+ for cut > 0 && !utf8.RuneStart(s[cut]) {+ cut--+ }+ return s[:cut] + "..."}// buildQuery encodes query parameters, omitting empty values.diff --git a/internal/datasource/connector/yuque/truncate_utf8_test.go b/internal/datasource/connector/yuque/truncate_utf8_test.gonew file mode 100644index 000000000..3d2fdb6db--- /dev/null+++ b/internal/datasource/connector/yuque/truncate_utf8_test.go@@ -0,0 +1,39 @@+package yuque++import (+ "strings"+ "testing"+ "unicode/utf8"+)++// truncate caps a connector error preview with a plain byte slice. Connector error+// text is often Chinese, so the cut can land inside a multi-byte rune; these+// previews reach sync_logs and data_sources through the sync error message, and+// PostgreSQL rejects the invalid UTF-8 that a partial rune produces.+func TestTruncateKeepsValidUTF8AcrossCaps(t *testing.T) {+ body := strings.Repeat("错", 400)+ for _, maxLen := range []int{499, 500, 501, 1000, 1} {+ got := truncate(body, maxLen)++ if !utf8.ValidString(got) {+ t.Fatalf("%s(%d): preview is not valid UTF-8", "truncate", maxLen)+ }+ if len(got) > maxLen+len("...") {+ t.Fatalf("%s(%d): preview is %d bytes", "truncate", maxLen, len(got))+ }+ if !strings.HasPrefix(body, strings.TrimSuffix(got, "...")) {+ t.Fatalf("%s(%d): preview is not a prefix of the input", "truncate", maxLen)+ }+ }+}++func TestTruncateLeavesShortAndASCIIInputAlone(t *testing.T) {+ for _, s := range []string{"", "ok", "服务返回 500", "短"} {+ if got := truncate(s, 500); got != s {+ t.Fatalf("%s(%q, 500) = %q, want the input unchanged", "truncate", s, got)+ }+ }+ if got := truncate("abcdef", 3); got != "abc..." {+ t.Fatalf("truncate keeps the ASCII cut: %q", got)+ }+}diff --git a/internal/im/mattermost/client.go b/internal/im/mattermost/client.goindex a2727363e..9284bea5b 100644--- a/internal/im/mattermost/client.go+++ b/internal/im/mattermost/client.go@@ -10,6 +10,7 @@ import ("net/url""strings""time"+ "unicode/utf8"secutils "github.com/Tencent/WeKnora/internal/utils")@@ -239,7 +240,13 @@ func truncateForErr(b []byte) string {const max = 512s := string(b)if len(s) > max {- return s[:max] + "..."+ // Back off to a rune boundary: a partial multi-byte rune is invalid+ // UTF-8 in an error string that callers log and surface.+ cut := max+ for cut > 0 && !utf8.RuneStart(s[cut]) {+ cut--+ }+ return s[:cut] + "..."}return s}diff --git a/internal/im/mattermost/truncate_utf8_test.go b/internal/im/mattermost/truncate_utf8_test.gonew file mode 100644index 000000000..a06b1bcc3--- /dev/null+++ b/internal/im/mattermost/truncate_utf8_test.go@@ -0,0 +1,35 @@+package mattermost++import (+ "strings"+ "testing"+ "unicode/utf8"+)++// truncateForErr caps response bodies at 512 bytes by slicing. Mattermost error+// bodies are localised, so the cut can land inside a multi-byte rune and the+// error string callers log and surface becomes invalid UTF-8.+func TestTruncateForErrKeepsValidUTF8(t *testing.T) {+ // 511 ASCII bytes put byte 512 inside the next three-byte rune.+ body := strings.Repeat("x", 511) + strings.Repeat("错误", 50)++ got := truncateForErr([]byte(body))++ if !utf8.ValidString(got) {+ t.Fatalf("preview is not valid UTF-8: %q", got)+ }+ if len(got) > 512+len("...") {+ t.Fatalf("preview is %d bytes", len(got))+ }+ if !strings.HasPrefix(body, strings.TrimSuffix(got, "...")) {+ t.Fatalf("preview is not a prefix of the input")+ }+}++func TestTruncateForErrLeavesShortInputAlone(t *testing.T) {+ for _, s := range []string{"", "ok", "服务返回 500"} {+ if got := truncateForErr([]byte(s)); got != s {+ t.Fatalf("truncateForErr(%q) = %q, want the input unchanged", s, got)+ }+ }+}