Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
perf(reranking): accelerate MMR selection with sparse postings
胡罡铭98e9dc078625
diff --git a/internal/reranking/mmr.go b/internal/reranking/mmr.goindex 124ef65d9..3120f654a 100644--- a/internal/reranking/mmr.go+++ b/internal/reranking/mmr.go@@ -18,18 +18,31 @@ func SelectMMR(ctx context.Context, results []*types.SearchResult, k int, lambdaif k <= 0 || len(results) == 0 {return nil}+ // The first pick has no redundancy term. Avoid passage cleaning and+ // tokenization when the caller only wants that pick.+ if k == 1 {+ best, bestScore := 0, math.Inf(-1)+ for i, r := range results {+ if score := lambda*r.Score - (1.0-lambda)*0; score > bestScore {+ best, bestScore = i, score+ }+ }+ return []int{best}+ }remaining := make([]int, len(results))- tokenSets := make([]map[string]struct{}, len(results))- for i, r := range results {+ for i := range results {remaining[i] = i- tokenSets[i] = searchutil.TokenizeSimple(EnrichedPassage(ctx, r))}+ tokens, offsets, postings := buildMMRPostings(ctx, results)- // Incremental form: maxRedundancy[i] caches result i's maximum jaccard- // against everything selected so far, so each round only needs one- // comparison per remaining result.+ // Cache each candidate's maximum Jaccard against the selected passages.+ // Postings count intersections only for candidates sharing a token with+ // the new pick; disjoint passages have zero redundancy and need no update.maxRedundancy := make([]float64, len(results))+ intersections := make([]int, len(results))+ taken := make([]bool, len(results))+ touched := make([]int, 0, len(results))selected := make([]int, 0, min(k, len(results)))for len(selected) < k && len(remaining) > 0 {bestPos := 0@@ -43,10 +56,69 @@ func SelectMMR(ctx context.Context, results []*types.SearchResult, k int, lambda}chosen := remaining[bestPos]selected = append(selected, chosen)+ taken[chosen] = trueremaining = append(remaining[:bestPos], remaining[bestPos+1:]...)- for _, i := range remaining {- maxRedundancy[i] = math.Max(maxRedundancy[i], searchutil.Jaccard(tokenSets[i], tokenSets[chosen]))+ if len(selected) == k || len(remaining) == 0 {+ break+ }+ touched = touched[:0]+ for _, token := range tokens[chosen] {+ for _, i := range postings[offsets[token]:offsets[token+1]] {+ if taken[i] {+ continue+ }+ if intersections[i] == 0 {+ touched = append(touched, i)+ }+ intersections[i]+++ }+ }+ for _, i := range touched {+ intersection := intersections[i]+ union := len(tokens[i]) + len(tokens[chosen]) - intersection+ maxRedundancy[i] = math.Max(maxRedundancy[i], float64(intersection)/float64(union))+ intersections[i] = 0}}return selected}++// buildMMRPostings interns each unique token and stores its candidate IDs in+// one flat array. offsets[t]:offsets[t+1] is token t's posting list. Each row+// comes from a token set, so one shared token contributes exactly once to a+// pair's intersection, regardless of its frequency in either passage.+func buildMMRPostings(+ ctx context.Context, results []*types.SearchResult,+) (tokens [][]int, offsets, postings []int) {+ termIDs := make(map[string]int)+ var counts []int+ tokens = make([][]int, len(results))+ for i, r := range results {+ set := searchutil.TokenizeSimple(EnrichedPassage(ctx, r))+ tokens[i] = make([]int, 0, len(set))+ for term := range set {+ id, exists := termIDs[term]+ if !exists {+ id = len(counts)+ termIDs[term] = id+ counts = append(counts, 0)+ }+ counts[id]+++ tokens[i] = append(tokens[i], id)+ }+ }+ offsets = make([]int, len(counts)+1)+ for id, count := range counts {+ offsets[id+1] = offsets[id] + count+ }+ postings = make([]int, offsets[len(counts)])+ // Reuse counts as insertion cursors; offsets stay unchanged for lookups.+ copy(counts, offsets)+ for i, row := range tokens {+ for _, id := range row {+ postings[counts[id]] = i+ counts[id]+++ }+ }+ return tokens, offsets, postings+}diff --git a/internal/reranking/mmr_test.go b/internal/reranking/mmr_test.goindex 44c714ed7..a7932ab52 100644--- a/internal/reranking/mmr_test.go+++ b/internal/reranking/mmr_test.go@@ -4,6 +4,8 @@ import ("context""fmt""math"+ "slices"+ "strings""testing""github.com/Tencent/WeKnora/internal/searchutil"@@ -116,3 +118,94 @@ func BenchmarkSelectMMR(b *testing.B) {SelectMMR(ctx, results, 250, DefaultMMRLambda)}}++func TestSelectMMR_preservesDiversityAndTies(t *testing.T) {+ results := []*types.SearchResult{+ {Content: "hybrid retrieval combines semantic vectors and keyword search", Score: 0.9},+ {Content: "**hybrid retrieval** combines semantic vectors and keyword search", Score: 0.88},+ {Content: "wiki construction extracts entities concepts and source citations", Score: 0.83},+ }+ if got := SelectMMR(context.Background(), results, 3, DefaultMMRLambda); !slices.Equal(got, []int{0, 2, 1}) {+ t.Fatalf("selection = %v, want relevant, diverse, then redundant", got)+ }+ for _, r := range results {+ r.Content, r.Score = "", 0.5+ }+ if got := SelectMMR(context.Background(), results, 10, DefaultMMRLambda); !slices.Equal(got, []int{0, 1, 2}) {+ t.Fatalf("empty tied passages = %v, want original order", got)+ }+}++func TestSelectMMR_multilingualEnrichedPassages(t *testing.T) {+ results := []*types.SearchResult{+ {Content: "混合检索结合语义向量和关键词索引,提高知识库检索覆盖率。", Score: 0.9},+ {Content: "混合检索结合语义向量和关键词索引,提高知识库检索覆盖率。", Score: 0.85},+ {Content: "Wiki 构建从文档抽取实体和概念,保留事实引用与来源分块。", Score: 0.8},+ {Content: "retrieval", ImageInfo: `[{"caption":"semantic vectors and keyword search"}]`, Score: 0.75},+ {+ Content: "sources", Score: 0.7,+ ChunkMetadata: types.JSON(`{"generated_questions":[+ {"id":"q1","question":"How does wiki construction preserve source citations?"}+ ]}`),+ },+ {Content: "", Score: 0.7},+ }+ for _, lambda := range []float64{0, 0.3, DefaultMMRLambda, 1} {+ for _, k := range []int{1, 3, len(results), len(results) + 1} {+ got := SelectMMR(context.Background(), results, k, lambda)+ want := selectMMRNaive(context.Background(), results, k, lambda)+ if !slices.Equal(got, want) {+ t.Fatalf("k=%d lambda=%v: selection = %v, naive = %v", k, lambda, got, want)+ }+ }+ }+}++// mmrPassageCorpus models a rerank pool with overlapping vocabulary across+// documents and less overlap across topics. It is synthetic and deterministic;+// the benchmarks measure MMR computation, not end-to-end retrieval quality.+func mmrPassageCorpus(n, words int, chinese bool) []*types.SearchResult {+ results := make([]*types.SearchResult, n)+ terms := []string{"检索", "知识库", "向量", "索引", "语义", "文档", "引用", "实体", "概念", "构建", "答案", "模型"}+ for i := range results {+ var content strings.Builder+ for j := 0; j < words; j++ {+ if chinese {+ fmt.Fprintf(&content, "%s ", terms[(i+j)%len(terms)])+ }+ // Different topics have disjoint bands; nearby documents within+ // a topic share most of their vocabulary.+ fmt.Fprintf(&content, "term%d ", (i%8)*1000+(j+i/8*7)%500)+ }+ results[i] = &types.SearchResult{+ ID: fmt.Sprintf("chunk-%03d", i), Content: content.String(),+ Score: 1 - float64(i%100)/200,+ }+ }+ return results+}++func BenchmarkSelectMMRPassages(b *testing.B) {+ ctx := context.Background()+ for _, tc := range []struct {+ name string+ n, words, k int+ chinese bool+ }{+ {name: "short", n: 50, words: 12, k: 10},+ {name: "english_top10", n: 200, words: 200, k: 10},+ {name: "english_top30", n: 200, words: 200, k: 30},+ {name: "mixed_top10", n: 200, words: 200, k: 10, chinese: true},+ {name: "mixed_top30", n: 200, words: 200, k: 30, chinese: true},+ {name: "top1", n: 200, words: 200, k: 1},+ } {+ b.Run(tc.name, func(b *testing.B) {+ results := mmrPassageCorpus(tc.n, tc.words, tc.chinese)+ b.ReportAllocs()+ b.ResetTimer()+ for i := 0; i < b.N; i++ {+ SelectMMR(ctx, results, tc.k, DefaultMMRLambda)+ }+ })+ }+}diff --git a/website-docs/02-architecture/04-rag-pipeline.md b/website-docs/02-architecture/04-rag-pipeline.mdindex bdda78b46..6abafa66c 100644--- a/website-docs/02-architecture/04-rag-pipeline.md+++ b/website-docs/02-architecture/04-rag-pipeline.md@@ -228,7 +228,7 @@ pipeline = types.NewPipelineBuilder().- 诊断里记录 `images_scored` / `images_kept`。4. **复合打分** `compositeScore`:`0.6*模型分 + 0.3*检索基础分 + 0.1*来源权重`(web_search 来源权重 0.95,其余 1.0),clamp 到 [0,1]。图谱实体检索命中的 chunk 没有检索分,基础分用模型分代替。基础分/模型分/复合分记录在 `Metadata["base_score"]` / `["model_score"]` / `["composite_score"]`。早期版本还会乘一个「越靠文档前部越高」的位置先验(±0.05),因为它与分块编辑后的偏移变化耦合且收益不明确,已被移除。5. **FAQ 加权**:`FAQPriorityEnabled` 且 `FAQScoreBoost > 1.0` 时,FAQ chunk 分数乘以 boost,记 `Metadata["faq_boosted"]`。结果不封顶到 1.0——封顶会让高分 FAQ 全部并列 1.0,彼此顺序退化为 tie-breaker。-6. **MMR 多样性选择** `applyMMR`(λ=0.7,k=`RerankTopK`):`mmr = 0.7*relevance - 0.3*max_jaccard_redundancy`,用 `searchutil.TokenizeSimple` + `Jaccard` 并行预计算 token 集合,迭代贪心选出 `RerankResult`。+6. **MMR 多样性选择** `reranking.SelectMMR`(λ=0.7,k=`RerankTopK`):`mmr = 0.7*relevance - 0.3*max_jaccard_redundancy`。对增强后的分块正文用 `searchutil.TokenizeSimple` 分词一次,再构建「token → 候选下标」倒排索引。每轮只累计与新选中分块共享 token 的候选交集,用 `交集 / (两侧 token 数之和 - 交集)` 计算 Jaccard,并更新最大冗余缓存;没有共享 token 的候选无需比较。索引使用连续数组存储,额外空间与候选 token 总数成正比,选择结果和平分时的输入顺序保持不变。`k=1` 时直接按相关性选择,跳过正文清洗、分词和索引构建。聊天、Agent 检索工具和检索 API 共用这一实现。**PluginMemoryAffinity**(`memory_affinity.go`)注册在链的最内层,同样先 `next()` 再后置处理:对该调用者过往回答中至少引用过 2 次的文档,按使用次数对数增长加权,最高 ×1.15,只用于在相近候选之间打破平局。随后才轮到 WikiBoost 的后置加权。