Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(wiki): generate SQLite title trigrams per word (#4679)

Krisbb979f63b9aa
diff --git a/internal/application/repository/wiki_page.go b/internal/application/repository/wiki_page.go
index 83769973f..953c52ec7 100644
--- a/internal/application/repository/wiki_page.go
+++ b/internal/application/repository/wiki_page.go
@@ -8,6 +8,7 @@ import (
"sort"
"strings"
"time"
+ "unicode"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
@@ -1149,19 +1150,23 @@ func wikiPageLitePointers(rows []types.WikiPageLite) []*types.WikiPageLite {
return out
}
-// wikiTitleTrigrams returns the padded rune trigrams used by pg_trgm's text
-// similarity model. Padding lets short titles still share a meaningful signal
-// only when their beginnings or endings agree.
+// wikiTitleTrigrams follows pg_trgm's word boundaries: non-alphanumeric
+// characters separate words, and each word receives two leading spaces and
+// one trailing space before its rune trigrams are added to the shared set.
func wikiTitleTrigrams(value string) map[string]struct{} {
- value = strings.ToLower(strings.TrimSpace(value))
- if value == "" {
+ words := strings.FieldsFunc(strings.ToLower(value), func(r rune) bool {
+ return !unicode.IsLetter(r) && !unicode.IsNumber(r)
+ })
+ if len(words) == 0 {
return nil
}
- runes := append([]rune{' ', ' '}, []rune(value)...)
- runes = append(runes, ' ')
- grams := make(map[string]struct{}, len(runes)-2)
- for i := 0; i+3 <= len(runes); i++ {
- grams[string(runes[i:i+3])] = struct{}{}
+ grams := make(map[string]struct{})
+ for _, word := range words {
+ runes := append([]rune{' ', ' '}, []rune(word)...)
+ runes = append(runes, ' ')
+ for i := 0; i+3 <= len(runes); i++ {
+ grams[string(runes[i:i+3])] = struct{}{}
+ }
}
return grams
}
diff --git a/internal/application/repository/wiki_page_test.go b/internal/application/repository/wiki_page_test.go
index faf74f0a1..c3681e60b 100644
--- a/internal/application/repository/wiki_page_test.go
+++ b/internal/application/repository/wiki_page_test.go
@@ -949,3 +949,35 @@ func TestSearchAcross_SQLiteLikeEscapesWildcards(t *testing.T) {
require.Len(t, pages, 1)
assert.Equal(t, literalUnderscore.ID, pages[0].ID)
}
+
+func TestWikiTitleTrigramsWordBoundaries(t *testing.T) {
+ for _, tt := range []struct{ name, left, right string }{
+ {"punctuation", "a-b", "a b"},
+ {"word order", "a b", "b a"},
+ {"repeated whitespace", "a\t b", "a b"},
+ {"unicode words", "甲-乙", "甲 乙"},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ assert.Equal(t, wikiTitleTrigrams(tt.left), wikiTitleTrigrams(tt.right))
+ assert.Equal(t, 1.0, wikiTrigramJaccard(wikiTitleTrigrams(tt.left), wikiTitleTrigrams(tt.right)))
+ })
+ }
+ assert.Empty(t, wikiTitleTrigrams("--- !!!"))
+ assert.Equal(t, map[string]struct{}{
+ " f": {}, " fo": {}, "foo": {}, "oo ": {},
+ " b": {}, " ba": {}, "bar": {}, "ar ": {},
+ }, wikiTitleTrigrams("foo|bar"))
+}
+
+func TestFindSimilarPagesSQLitePunctuationVariants(t *testing.T) {
+ db := setupWikiPagesTestDB(t)
+ repo := NewWikiPageRepository(db)
+ page := makeWikiPage("kb-a", "entity/a-b", types.WikiPageTypeEntity, types.WikiPageStatusPublished)
+ page.Title = "A-B"
+ require.NoError(t, repo.Create(context.Background(), page))
+
+ pages, err := repo.FindSimilarPages(context.Background(), "kb-a", "A B", nil, 10)
+ require.NoError(t, err)
+ require.Len(t, pages, 1)
+ assert.Equal(t, page.Slug, pages[0].Slug)
+}