Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(skills): truncate a downloaded skill error body at a rune boundary (#4665)
yetuge6019fab4e8bd
diff --git a/internal/application/service/tenant_skill_source.go b/internal/application/service/tenant_skill_source.goindex 94b3b5503..776b94801 100644--- a/internal/application/service/tenant_skill_source.go+++ b/internal/application/service/tenant_skill_source.go@@ -15,6 +15,7 @@ import ("sync""time"+ "github.com/Tencent/WeKnora/internal/common"secutils "github.com/Tencent/WeKnora/internal/utils")@@ -913,8 +914,9 @@ func looksLikeSkillMarkdown(body []byte) bool {func truncateSkillError(msg string) string {msg = strings.Join(strings.Fields(msg), " ")- if len(msg) > 200 {- return msg[:200]- }- return msg+ // Back off to a rune boundary: the body comes from a third-party host, so a+ // Chinese error page would otherwise be split mid-rune and the 400 response+ // this produces would carry an invalid trailing byte. Same rule as the+ // provider error previews and common.TruncateUTF8Bytes.+ return common.TruncateUTF8Bytes(msg, 200)}diff --git a/internal/application/service/tenant_skill_source_utf8_test.go b/internal/application/service/tenant_skill_source_utf8_test.gonew file mode 100644index 000000000..ddd10550d--- /dev/null+++ b/internal/application/service/tenant_skill_source_utf8_test.go@@ -0,0 +1,51 @@+package service++import (+ "strings"+ "testing"+ "unicode/utf8"++ "github.com/stretchr/testify/require"+)++// truncateSkillError caps a remote error body at 200 bytes by slicing the+// string. The body comes from a third-party host, so it can be Chinese; the cut+// then lands inside a multi-byte rune and the 400 response this produces carries+// an invalid trailing byte.+//+// The sibling helper for provider error previews already backs off to a rune+// boundary (readErrorPreview / trimPartialUTF8 in internal/infrastructure/+// web_search); this one is the remaining byte-slicing cap on a user-visible+// error message.+func TestTruncateSkillErrorKeepsValidUTF8(t *testing.T) {+ // 199 ASCII bytes put byte 200 inside the next three-byte rune.+ for _, msg := range []string{+ strings.Repeat("错", 100),+ strings.Repeat("x", 199) + strings.Repeat("失败", 50),+ strings.Repeat("x", 198) + strings.Repeat("失败", 50),+ } {+ got := truncateSkillError(msg)++ require.True(t, utf8.ValidString(got),+ "truncateSkillError: result is not valid UTF-8 for a %d-byte body", len(msg))+ require.LessOrEqual(t, len(got), 200, "truncateSkillError: result is %d bytes, want at most 200", len(got))+ require.True(t, strings.HasPrefix(msg, got), "truncateSkillError: result is not a prefix of the body")+ }+}++// Text that already fits, and whose whitespace collapses to something that+// still fits, must come back unchanged apart from the whitespace normalization.+func TestTruncateSkillErrorLeavesShortInputAlone(t *testing.T) {+ for _, msg := range []string{+ "",+ "ok",+ "下载失败",+ "line one\n\nline two",+ strings.Repeat("错", 66), // 198 bytes: under the cap+ } {+ got := truncateSkillError(msg)+ want := strings.Join(strings.Fields(msg), " ")++ require.Equal(t, want, got)+ }+}diff --git a/internal/infrastructure/web_search/duckduckgo.go b/internal/infrastructure/web_search/duckduckgo.goindex 62f7e9010..99ba7d8e8 100644--- a/internal/infrastructure/web_search/duckduckgo.go+++ b/internal/infrastructure/web_search/duckduckgo.go@@ -8,6 +8,7 @@ import ("net/url""strings""time"+ "unicode/utf8""github.com/PuerkitoBio/goquery""github.com/Tencent/WeKnora/internal/logger"@@ -249,8 +250,16 @@ func extractTitle(text string) string {lines := strings.Split(text, "\n")if len(lines) > 0 {title := strings.TrimSpace(lines[0])- if len(title) > 100 {- title = title[:100] + "..."+ const maxTitleBytes = 100+ if len(title) > maxTitleBytes {+ // Back off to a rune boundary: titles are often Chinese, and a partial+ // rune is invalid UTF-8 that the knowledge base's title column rejects.+ // Same rule as #4555 / #4583.+ cut := maxTitleBytes+ for cut > 0 && !utf8.RuneStart(title[cut]) {+ cut--+ }+ title = title[:cut] + "..."}return title}diff --git a/internal/infrastructure/web_search/duckduckgo_utf8_test.go b/internal/infrastructure/web_search/duckduckgo_utf8_test.gonew file mode 100644index 000000000..d99c066cd--- /dev/null+++ b/internal/infrastructure/web_search/duckduckgo_utf8_test.go@@ -0,0 +1,68 @@+package web_search++import (+ "strings"+ "testing"+ "unicode/utf8"+)++// extractTitle caps the first line of a DuckDuckGo RelatedTopics/Results text at+// 100 bytes. Those texts carry page titles, which are often Chinese, so the cut+// can land inside a multi-byte rune; the truncated title then becomes a+// WebSearchResult.Title, and the search results are indexed into a knowledge+// base whose title column PostgreSQL validates as UTF-8 (the same rule #4555 and+// #4583 applied elsewhere).+func TestExtractTitleKeepsValidUTF8(t *testing.T) {+ // 99 ASCII bytes put byte 100 inside the next three-byte rune.+ for _, text := range []string{+ strings.Repeat("错", 200),+ strings.Repeat("x", 99) + strings.Repeat("错误", 50),+ strings.Repeat("x", 98) + strings.Repeat("错误", 50),+ } {+ got := extractTitle(text)++ if !utf8.ValidString(got) {+ t.Fatalf("extractTitle: title is not valid UTF-8 for input %d bytes", len(text))+ }+ if len(got) > 100+len("...") {+ t.Fatalf("extractTitle: title is %d bytes, want at most %d", len(got), 100+len("..."))+ }+ if !strings.HasPrefix(text, strings.TrimSuffix(got, "...")) {+ t.Fatalf("extractTitle: title is not a prefix of the input")+ }+ }+}++// A long first line is what gets capped; the cap must not change behavior for+// text that already fits, for ASCII, or for text whose first line is empty.+func TestExtractTitleLeavesShortInputAlone(t *testing.T) {+ for _, text := range []string{+ "",+ "ok",+ "服务返回 500",+ strings.Repeat("x", 100),+ strings.Repeat("错", 33), // 99 bytes: under the cap+ } {+ if got := extractTitle(text); got != strings.TrimSpace(text) {+ t.Fatalf("extractTitle(%q) = %q, want the input unchanged", text, got)+ }+ }+}++// Only the first line becomes the title, before the cap applies.+func TestExtractTitleUsesOnlyTheFirstLine(t *testing.T) {+ got := extractTitle("Title line\nSnippet line")++ if got != "Title line" {+ t.Fatalf("extractTitle = %q, want %q", got, "Title line")+ }+}++// A title that is exactly at the cap is not truncated, so no ellipsis is added.+func TestExtractTitleAtTheCapIsNotTruncated(t *testing.T) {+ text := strings.Repeat("a", 100)++ if got := extractTitle(text); got != text {+ t.Fatalf("extractTitle(100 ASCII bytes) = %q, want it unchanged", got)+ }+}