Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(wiki): generate SQLite title trigrams per word (#4679)
Krisbb979f63b9aa
diff --git a/internal/application/repository/wiki_page.go b/internal/application/repository/wiki_page.goindex 83769973f..953c52ec7 100644--- a/internal/application/repository/wiki_page.go+++ b/internal/application/repository/wiki_page.go@@ -8,6 +8,7 @@ import ("sort""strings""time"+ "unicode""github.com/Tencent/WeKnora/internal/types""github.com/Tencent/WeKnora/internal/types/interfaces"@@ -1149,19 +1150,23 @@ func wikiPageLitePointers(rows []types.WikiPageLite) []*types.WikiPageLite {return out}-// wikiTitleTrigrams returns the padded rune trigrams used by pg_trgm's text-// similarity model. Padding lets short titles still share a meaningful signal-// only when their beginnings or endings agree.+// wikiTitleTrigrams follows pg_trgm's word boundaries: non-alphanumeric+// characters separate words, and each word receives two leading spaces and+// one trailing space before its rune trigrams are added to the shared set.func wikiTitleTrigrams(value string) map[string]struct{} {- value = strings.ToLower(strings.TrimSpace(value))- if value == "" {+ words := strings.FieldsFunc(strings.ToLower(value), func(r rune) bool {+ return !unicode.IsLetter(r) && !unicode.IsNumber(r)+ })+ if len(words) == 0 {return nil}- runes := append([]rune{' ', ' '}, []rune(value)...)- runes = append(runes, ' ')- grams := make(map[string]struct{}, len(runes)-2)- for i := 0; i+3 <= len(runes); i++ {- grams[string(runes[i:i+3])] = struct{}{}+ grams := make(map[string]struct{})+ for _, word := range words {+ runes := append([]rune{' ', ' '}, []rune(word)...)+ runes = append(runes, ' ')+ for i := 0; i+3 <= len(runes); i++ {+ grams[string(runes[i:i+3])] = struct{}{}+ }}return grams}diff --git a/internal/application/repository/wiki_page_test.go b/internal/application/repository/wiki_page_test.goindex faf74f0a1..c3681e60b 100644--- a/internal/application/repository/wiki_page_test.go+++ b/internal/application/repository/wiki_page_test.go@@ -949,3 +949,35 @@ func TestSearchAcross_SQLiteLikeEscapesWildcards(t *testing.T) {require.Len(t, pages, 1)assert.Equal(t, literalUnderscore.ID, pages[0].ID)}++func TestWikiTitleTrigramsWordBoundaries(t *testing.T) {+ for _, tt := range []struct{ name, left, right string }{+ {"punctuation", "a-b", "a b"},+ {"word order", "a b", "b a"},+ {"repeated whitespace", "a\t b", "a b"},+ {"unicode words", "甲-乙", "甲 乙"},+ } {+ t.Run(tt.name, func(t *testing.T) {+ assert.Equal(t, wikiTitleTrigrams(tt.left), wikiTitleTrigrams(tt.right))+ assert.Equal(t, 1.0, wikiTrigramJaccard(wikiTitleTrigrams(tt.left), wikiTitleTrigrams(tt.right)))+ })+ }+ assert.Empty(t, wikiTitleTrigrams("--- !!!"))+ assert.Equal(t, map[string]struct{}{+ " f": {}, " fo": {}, "foo": {}, "oo ": {},+ " b": {}, " ba": {}, "bar": {}, "ar ": {},+ }, wikiTitleTrigrams("foo|bar"))+}++func TestFindSimilarPagesSQLitePunctuationVariants(t *testing.T) {+ db := setupWikiPagesTestDB(t)+ repo := NewWikiPageRepository(db)+ page := makeWikiPage("kb-a", "entity/a-b", types.WikiPageTypeEntity, types.WikiPageStatusPublished)+ page.Title = "A-B"+ require.NoError(t, repo.Create(context.Background(), page))++ pages, err := repo.FindSimilarPages(context.Background(), "kb-a", "A B", nil, 10)+ require.NoError(t, err)+ require.Len(t, pages, 1)+ assert.Equal(t, page.Slug, pages[0].Slug)+}