Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(datasource): truncate connector error previews at rune boundaries (#4614)

yetugedebade7114b1
diff --git a/internal/datasource/connector/feishu/core/client.go b/internal/datasource/connector/feishu/core/client.go
index 52ec80fdd..2b162d3d4 100644
--- a/internal/datasource/connector/feishu/core/client.go
+++ b/internal/datasource/connector/feishu/core/client.go
@@ -12,6 +12,7 @@ import (
"strings"
"sync"
"time"
+ "unicode/utf8"
"github.com/Tencent/WeKnora/internal/datasource"
"github.com/Tencent/WeKnora/internal/logger"
@@ -349,7 +350,14 @@ func truncate(s string, maxLen int) string {
if len(s) <= maxLen {
return s
}
- return s[:maxLen] + "..."
+ // Back off to a rune boundary: a partial multi-byte rune is invalid UTF-8,
+ // and this preview reaches sync_logs / data_sources through the sync error
+ // message, which PostgreSQL rejects. Same rule as #4555.
+ cut := maxLen
+ for cut > 0 && !utf8.RuneStart(s[cut]) {
+ cut--
+ }
+ return s[:cut] + "..."
}
// ListWikiSpaces returns all wiki spaces accessible to the app.
diff --git a/internal/datasource/connector/feishu/core/truncate_utf8_test.go b/internal/datasource/connector/feishu/core/truncate_utf8_test.go
new file mode 100644
index 000000000..847131a2c
--- /dev/null
+++ b/internal/datasource/connector/feishu/core/truncate_utf8_test.go
@@ -0,0 +1,39 @@
+package core
+
+import (
+ "strings"
+ "testing"
+ "unicode/utf8"
+)
+
+// truncate caps a connector error preview with a plain byte slice. Connector error
+// text is often Chinese, so the cut can land inside a multi-byte rune; these
+// previews reach sync_logs and data_sources through the sync error message, and
+// PostgreSQL rejects the invalid UTF-8 that a partial rune produces.
+func TestTruncateKeepsValidUTF8AcrossCaps(t *testing.T) {
+ body := strings.Repeat("错", 400)
+ for _, maxLen := range []int{499, 500, 501, 1000, 1} {
+ got := truncate(body, maxLen)
+
+ if !utf8.ValidString(got) {
+ t.Fatalf("%s(%d): preview is not valid UTF-8", "truncate", maxLen)
+ }
+ if len(got) > maxLen+len("...") {
+ t.Fatalf("%s(%d): preview is %d bytes", "truncate", maxLen, len(got))
+ }
+ if !strings.HasPrefix(body, strings.TrimSuffix(got, "...")) {
+ t.Fatalf("%s(%d): preview is not a prefix of the input", "truncate", maxLen)
+ }
+ }
+}
+
+func TestTruncateLeavesShortAndASCIIInputAlone(t *testing.T) {
+ for _, s := range []string{"", "ok", "服务返回 500", "短"} {
+ if got := truncate(s, 500); got != s {
+ t.Fatalf("%s(%q, 500) = %q, want the input unchanged", "truncate", s, got)
+ }
+ }
+ if got := truncate("abcdef", 3); got != "abc..." {
+ t.Fatalf("truncate keeps the ASCII cut: %q", got)
+ }
+}
diff --git a/internal/datasource/connector/ima/client.go b/internal/datasource/connector/ima/client.go
index e5d713f7a..dd9513c49 100644
--- a/internal/datasource/connector/ima/client.go
+++ b/internal/datasource/connector/ima/client.go
@@ -13,6 +13,7 @@ import (
"net/http"
"sync"
"time"
+ "unicode/utf8"
"github.com/Tencent/WeKnora/internal/datasource"
"github.com/Tencent/WeKnora/internal/logger"
@@ -415,7 +416,14 @@ func truncate(s string, maxLen int) string {
if len(s) <= maxLen {
return s
}
- return s[:maxLen] + "..."
+ // Back off to a rune boundary: a partial multi-byte rune is invalid UTF-8,
+ // and this preview reaches sync_logs / data_sources through the sync error
+ // message, which PostgreSQL rejects. Same rule as #4555.
+ cut := maxLen
+ for cut > 0 && !utf8.RuneStart(s[cut]) {
+ cut--
+ }
+ return s[:cut] + "..."
}
// redact returns a masked form of a credential for logging (never log full).
diff --git a/internal/datasource/connector/ima/truncate_utf8_test.go b/internal/datasource/connector/ima/truncate_utf8_test.go
new file mode 100644
index 000000000..86139be4c
--- /dev/null
+++ b/internal/datasource/connector/ima/truncate_utf8_test.go
@@ -0,0 +1,39 @@
+package ima
+
+import (
+ "strings"
+ "testing"
+ "unicode/utf8"
+)
+
+// truncate caps a connector error preview with a plain byte slice. Connector error
+// text is often Chinese, so the cut can land inside a multi-byte rune; these
+// previews reach sync_logs and data_sources through the sync error message, and
+// PostgreSQL rejects the invalid UTF-8 that a partial rune produces.
+func TestTruncateKeepsValidUTF8AcrossCaps(t *testing.T) {
+ body := strings.Repeat("错", 400)
+ for _, maxLen := range []int{499, 500, 501, 1000, 1} {
+ got := truncate(body, maxLen)
+
+ if !utf8.ValidString(got) {
+ t.Fatalf("%s(%d): preview is not valid UTF-8", "truncate", maxLen)
+ }
+ if len(got) > maxLen+len("...") {
+ t.Fatalf("%s(%d): preview is %d bytes", "truncate", maxLen, len(got))
+ }
+ if !strings.HasPrefix(body, strings.TrimSuffix(got, "...")) {
+ t.Fatalf("%s(%d): preview is not a prefix of the input", "truncate", maxLen)
+ }
+ }
+}
+
+func TestTruncateLeavesShortAndASCIIInputAlone(t *testing.T) {
+ for _, s := range []string{"", "ok", "服务返回 500", "短"} {
+ if got := truncate(s, 500); got != s {
+ t.Fatalf("%s(%q, 500) = %q, want the input unchanged", "truncate", s, got)
+ }
+ }
+ if got := truncate("abcdef", 3); got != "abc..." {
+ t.Fatalf("truncate keeps the ASCII cut: %q", got)
+ }
+}
diff --git a/internal/datasource/connector/yuque/client.go b/internal/datasource/connector/yuque/client.go
index f51fae198..b3730da69 100644
--- a/internal/datasource/connector/yuque/client.go
+++ b/internal/datasource/connector/yuque/client.go
@@ -10,6 +10,7 @@ import (
"net/url"
"sync"
"time"
+ "unicode/utf8"
"github.com/Tencent/WeKnora/internal/datasource"
"github.com/Tencent/WeKnora/internal/logger"
@@ -229,7 +230,14 @@ func truncate(s string, maxLen int) string {
if len(s) <= maxLen {
return s
}
- return s[:maxLen] + "..."
+ // Back off to a rune boundary: a partial multi-byte rune is invalid UTF-8,
+ // and this preview reaches sync_logs / data_sources through the sync error
+ // message, which PostgreSQL rejects. Same rule as #4555.
+ cut := maxLen
+ for cut > 0 && !utf8.RuneStart(s[cut]) {
+ cut--
+ }
+ return s[:cut] + "..."
}
// buildQuery encodes query parameters, omitting empty values.
diff --git a/internal/datasource/connector/yuque/truncate_utf8_test.go b/internal/datasource/connector/yuque/truncate_utf8_test.go
new file mode 100644
index 000000000..3d2fdb6db
--- /dev/null
+++ b/internal/datasource/connector/yuque/truncate_utf8_test.go
@@ -0,0 +1,39 @@
+package yuque
+
+import (
+ "strings"
+ "testing"
+ "unicode/utf8"
+)
+
+// truncate caps a connector error preview with a plain byte slice. Connector error
+// text is often Chinese, so the cut can land inside a multi-byte rune; these
+// previews reach sync_logs and data_sources through the sync error message, and
+// PostgreSQL rejects the invalid UTF-8 that a partial rune produces.
+func TestTruncateKeepsValidUTF8AcrossCaps(t *testing.T) {
+ body := strings.Repeat("错", 400)
+ for _, maxLen := range []int{499, 500, 501, 1000, 1} {
+ got := truncate(body, maxLen)
+
+ if !utf8.ValidString(got) {
+ t.Fatalf("%s(%d): preview is not valid UTF-8", "truncate", maxLen)
+ }
+ if len(got) > maxLen+len("...") {
+ t.Fatalf("%s(%d): preview is %d bytes", "truncate", maxLen, len(got))
+ }
+ if !strings.HasPrefix(body, strings.TrimSuffix(got, "...")) {
+ t.Fatalf("%s(%d): preview is not a prefix of the input", "truncate", maxLen)
+ }
+ }
+}
+
+func TestTruncateLeavesShortAndASCIIInputAlone(t *testing.T) {
+ for _, s := range []string{"", "ok", "服务返回 500", "短"} {
+ if got := truncate(s, 500); got != s {
+ t.Fatalf("%s(%q, 500) = %q, want the input unchanged", "truncate", s, got)
+ }
+ }
+ if got := truncate("abcdef", 3); got != "abc..." {
+ t.Fatalf("truncate keeps the ASCII cut: %q", got)
+ }
+}
diff --git a/internal/im/mattermost/client.go b/internal/im/mattermost/client.go
index a2727363e..9284bea5b 100644
--- a/internal/im/mattermost/client.go
+++ b/internal/im/mattermost/client.go
@@ -10,6 +10,7 @@ import (
"net/url"
"strings"
"time"
+ "unicode/utf8"
secutils "github.com/Tencent/WeKnora/internal/utils"
)
@@ -239,7 +240,13 @@ func truncateForErr(b []byte) string {
const max = 512
s := string(b)
if len(s) > max {
- return s[:max] + "..."
+ // Back off to a rune boundary: a partial multi-byte rune is invalid
+ // UTF-8 in an error string that callers log and surface.
+ cut := max
+ for cut > 0 && !utf8.RuneStart(s[cut]) {
+ cut--
+ }
+ return s[:cut] + "..."
}
return s
}
diff --git a/internal/im/mattermost/truncate_utf8_test.go b/internal/im/mattermost/truncate_utf8_test.go
new file mode 100644
index 000000000..a06b1bcc3
--- /dev/null
+++ b/internal/im/mattermost/truncate_utf8_test.go
@@ -0,0 +1,35 @@
+package mattermost
+
+import (
+ "strings"
+ "testing"
+ "unicode/utf8"
+)
+
+// truncateForErr caps response bodies at 512 bytes by slicing. Mattermost error
+// bodies are localised, so the cut can land inside a multi-byte rune and the
+// error string callers log and surface becomes invalid UTF-8.
+func TestTruncateForErrKeepsValidUTF8(t *testing.T) {
+ // 511 ASCII bytes put byte 512 inside the next three-byte rune.
+ body := strings.Repeat("x", 511) + strings.Repeat("错误", 50)
+
+ got := truncateForErr([]byte(body))
+
+ if !utf8.ValidString(got) {
+ t.Fatalf("preview is not valid UTF-8: %q", got)
+ }
+ if len(got) > 512+len("...") {
+ t.Fatalf("preview is %d bytes", len(got))
+ }
+ if !strings.HasPrefix(body, strings.TrimSuffix(got, "...")) {
+ t.Fatalf("preview is not a prefix of the input")
+ }
+}
+
+func TestTruncateForErrLeavesShortInputAlone(t *testing.T) {
+ for _, s := range []string{"", "ok", "服务返回 500"} {
+ if got := truncateForErr([]byte(s)); got != s {
+ t.Fatalf("truncateForErr(%q) = %q, want the input unchanged", s, got)
+ }
+ }
+}