Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(skills): truncate a downloaded skill error body at a rune boundary (#4665)

yetuge6019fab4e8bd
diff --git a/internal/application/service/tenant_skill_source.go b/internal/application/service/tenant_skill_source.go
index 94b3b5503..776b94801 100644
--- a/internal/application/service/tenant_skill_source.go
+++ b/internal/application/service/tenant_skill_source.go
@@ -15,6 +15,7 @@ import (
"sync"
"time"
+ "github.com/Tencent/WeKnora/internal/common"
secutils "github.com/Tencent/WeKnora/internal/utils"
)
@@ -913,8 +914,9 @@ func looksLikeSkillMarkdown(body []byte) bool {
func truncateSkillError(msg string) string {
msg = strings.Join(strings.Fields(msg), " ")
- if len(msg) > 200 {
- return msg[:200]
- }
- return msg
+ // Back off to a rune boundary: the body comes from a third-party host, so a
+ // Chinese error page would otherwise be split mid-rune and the 400 response
+ // this produces would carry an invalid trailing byte. Same rule as the
+ // provider error previews and common.TruncateUTF8Bytes.
+ return common.TruncateUTF8Bytes(msg, 200)
}
diff --git a/internal/application/service/tenant_skill_source_utf8_test.go b/internal/application/service/tenant_skill_source_utf8_test.go
new file mode 100644
index 000000000..ddd10550d
--- /dev/null
+++ b/internal/application/service/tenant_skill_source_utf8_test.go
@@ -0,0 +1,51 @@
+package service
+
+import (
+ "strings"
+ "testing"
+ "unicode/utf8"
+
+ "github.com/stretchr/testify/require"
+)
+
+// truncateSkillError caps a remote error body at 200 bytes by slicing the
+// string. The body comes from a third-party host, so it can be Chinese; the cut
+// then lands inside a multi-byte rune and the 400 response this produces carries
+// an invalid trailing byte.
+//
+// The sibling helper for provider error previews already backs off to a rune
+// boundary (readErrorPreview / trimPartialUTF8 in internal/infrastructure/
+// web_search); this one is the remaining byte-slicing cap on a user-visible
+// error message.
+func TestTruncateSkillErrorKeepsValidUTF8(t *testing.T) {
+ // 199 ASCII bytes put byte 200 inside the next three-byte rune.
+ for _, msg := range []string{
+ strings.Repeat("错", 100),
+ strings.Repeat("x", 199) + strings.Repeat("失败", 50),
+ strings.Repeat("x", 198) + strings.Repeat("失败", 50),
+ } {
+ got := truncateSkillError(msg)
+
+ require.True(t, utf8.ValidString(got),
+ "truncateSkillError: result is not valid UTF-8 for a %d-byte body", len(msg))
+ require.LessOrEqual(t, len(got), 200, "truncateSkillError: result is %d bytes, want at most 200", len(got))
+ require.True(t, strings.HasPrefix(msg, got), "truncateSkillError: result is not a prefix of the body")
+ }
+}
+
+// Text that already fits, and whose whitespace collapses to something that
+// still fits, must come back unchanged apart from the whitespace normalization.
+func TestTruncateSkillErrorLeavesShortInputAlone(t *testing.T) {
+ for _, msg := range []string{
+ "",
+ "ok",
+ "下载失败",
+ "line one\n\nline two",
+ strings.Repeat("错", 66), // 198 bytes: under the cap
+ } {
+ got := truncateSkillError(msg)
+ want := strings.Join(strings.Fields(msg), " ")
+
+ require.Equal(t, want, got)
+ }
+}
diff --git a/internal/infrastructure/web_search/duckduckgo.go b/internal/infrastructure/web_search/duckduckgo.go
index 62f7e9010..99ba7d8e8 100644
--- a/internal/infrastructure/web_search/duckduckgo.go
+++ b/internal/infrastructure/web_search/duckduckgo.go
@@ -8,6 +8,7 @@ import (
"net/url"
"strings"
"time"
+ "unicode/utf8"
"github.com/PuerkitoBio/goquery"
"github.com/Tencent/WeKnora/internal/logger"
@@ -249,8 +250,16 @@ func extractTitle(text string) string {
lines := strings.Split(text, "\n")
if len(lines) > 0 {
title := strings.TrimSpace(lines[0])
- if len(title) > 100 {
- title = title[:100] + "..."
+ const maxTitleBytes = 100
+ if len(title) > maxTitleBytes {
+ // Back off to a rune boundary: titles are often Chinese, and a partial
+ // rune is invalid UTF-8 that the knowledge base's title column rejects.
+ // Same rule as #4555 / #4583.
+ cut := maxTitleBytes
+ for cut > 0 && !utf8.RuneStart(title[cut]) {
+ cut--
+ }
+ title = title[:cut] + "..."
}
return title
}
diff --git a/internal/infrastructure/web_search/duckduckgo_utf8_test.go b/internal/infrastructure/web_search/duckduckgo_utf8_test.go
new file mode 100644
index 000000000..d99c066cd
--- /dev/null
+++ b/internal/infrastructure/web_search/duckduckgo_utf8_test.go
@@ -0,0 +1,68 @@
+package web_search
+
+import (
+ "strings"
+ "testing"
+ "unicode/utf8"
+)
+
+// extractTitle caps the first line of a DuckDuckGo RelatedTopics/Results text at
+// 100 bytes. Those texts carry page titles, which are often Chinese, so the cut
+// can land inside a multi-byte rune; the truncated title then becomes a
+// WebSearchResult.Title, and the search results are indexed into a knowledge
+// base whose title column PostgreSQL validates as UTF-8 (the same rule #4555 and
+// #4583 applied elsewhere).
+func TestExtractTitleKeepsValidUTF8(t *testing.T) {
+ // 99 ASCII bytes put byte 100 inside the next three-byte rune.
+ for _, text := range []string{
+ strings.Repeat("错", 200),
+ strings.Repeat("x", 99) + strings.Repeat("错误", 50),
+ strings.Repeat("x", 98) + strings.Repeat("错误", 50),
+ } {
+ got := extractTitle(text)
+
+ if !utf8.ValidString(got) {
+ t.Fatalf("extractTitle: title is not valid UTF-8 for input %d bytes", len(text))
+ }
+ if len(got) > 100+len("...") {
+ t.Fatalf("extractTitle: title is %d bytes, want at most %d", len(got), 100+len("..."))
+ }
+ if !strings.HasPrefix(text, strings.TrimSuffix(got, "...")) {
+ t.Fatalf("extractTitle: title is not a prefix of the input")
+ }
+ }
+}
+
+// A long first line is what gets capped; the cap must not change behavior for
+// text that already fits, for ASCII, or for text whose first line is empty.
+func TestExtractTitleLeavesShortInputAlone(t *testing.T) {
+ for _, text := range []string{
+ "",
+ "ok",
+ "服务返回 500",
+ strings.Repeat("x", 100),
+ strings.Repeat("错", 33), // 99 bytes: under the cap
+ } {
+ if got := extractTitle(text); got != strings.TrimSpace(text) {
+ t.Fatalf("extractTitle(%q) = %q, want the input unchanged", text, got)
+ }
+ }
+}
+
+// Only the first line becomes the title, before the cap applies.
+func TestExtractTitleUsesOnlyTheFirstLine(t *testing.T) {
+ got := extractTitle("Title line\nSnippet line")
+
+ if got != "Title line" {
+ t.Fatalf("extractTitle = %q, want %q", got, "Title line")
+ }
+}
+
+// A title that is exactly at the cap is not truncated, so no ellipsis is added.
+func TestExtractTitleAtTheCapIsNotTruncated(t *testing.T) {
+ text := strings.Repeat("a", 100)
+
+ if got := extractTitle(text); got != text {
+ t.Fatalf("extractTitle(100 ASCII bytes) = %q, want it unchanged", got)
+ }
+}