Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(wiki): skip ambiguous single-Han auto-links (#2860)
JYao-Chene3d52e835301
diff --git a/internal/application/service/wiki_linkify.go b/internal/application/service/wiki_linkify.goindex 0f9d62e1c..8f05cb1d8 100644--- a/internal/application/service/wiki_linkify.go+++ b/internal/application/service/wiki_linkify.go@@ -47,6 +47,14 @@ func linkifyContent(content string, refs []linkRef, selfSlug string) (string, boif r.slug == selfSlug {continue}+ // A single Han character has no reliable token boundary in unsegmented+ // Chinese prose. Substring-linking it would turn words such as 风沙、+ // 栏目 and 核心 into broken links. Keep one-character Wiki pages and+ // any explicit [[...]] links authored by the generator, but never+ // inject a new cross-link for such a surface form.+ if isSingleHanRune(r.matchText) {+ continue+ }sorted = append(sorted, r)}sort.SliceStable(sorted, func(i, j int) bool {@@ -125,6 +133,15 @@ func hasASCIILetterEdge(s string) bool {return isASCIIWordRune(first) || isASCIIWordRune(last)}+// isSingleHanRune reports whether s consists of exactly one Han character.+func isSingleHanRune(s string) bool {+ if s == "" {+ return false+ }+ r, size := utf8.DecodeRuneInString(s)+ return size == len(s) && unicode.Is(unicode.Han, r)+}+func isASCIIWordRune(r rune) bool {if r > unicode.MaxASCII {return falsediff --git a/internal/application/service/wiki_linkify_test.go b/internal/application/service/wiki_linkify_test.goindex 2d81f08c2..9d914a5bc 100644--- a/internal/application/service/wiki_linkify_test.go+++ b/internal/application/service/wiki_linkify_test.go@@ -37,6 +37,34 @@ func TestLinkifyContent_LongerNameWinsOverSubstring(t *testing.T) {}}+func TestLinkifyContent_SkipsSingleHanRuneAutoLink(t *testing.T) {+ refs := []linkRef{+ {slug: "entity/feng", matchText: "风"},+ }+ in := "穿越河西走廊体验大漠风沙。"+ got, changed := linkifyContent(in, refs, "")+ if changed {+ t.Fatalf("single Han rune must not be auto-linked: %q", got)+ }+ if got != in {+ t.Fatalf("content changed: got %q, want %q", got, in)+ }+}++func TestLinkifyContent_PreservesExplicitSingleHanRuneLink(t *testing.T) {+ refs := []linkRef{+ {slug: "entity/feng", matchText: "风"},+ }+ in := "寓言中,[[entity/feng|风]]羡慕目。"+ got, changed := linkifyContent(in, refs, "")+ if changed {+ t.Fatalf("explicit single-character link must be preserved without reinjection: %q", got)+ }+ if got != in {+ t.Fatalf("content changed: got %q, want %q", got, in)+ }+}+func TestLinkifyContent_ASCIIWordBoundary(t *testing.T) {refs := []linkRef{{slug: "ai", matchText: "AI"},