Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

perf(reranking): accelerate MMR selection with sparse postings

胡罡铭98e9dc078625
diff --git a/internal/reranking/mmr.go b/internal/reranking/mmr.go
index 124ef65d9..3120f654a 100644
--- a/internal/reranking/mmr.go
+++ b/internal/reranking/mmr.go
@@ -18,18 +18,31 @@ func SelectMMR(ctx context.Context, results []*types.SearchResult, k int, lambda
if k <= 0 || len(results) == 0 {
return nil
}
+ // The first pick has no redundancy term. Avoid passage cleaning and
+ // tokenization when the caller only wants that pick.
+ if k == 1 {
+ best, bestScore := 0, math.Inf(-1)
+ for i, r := range results {
+ if score := lambda*r.Score - (1.0-lambda)*0; score > bestScore {
+ best, bestScore = i, score
+ }
+ }
+ return []int{best}
+ }
remaining := make([]int, len(results))
- tokenSets := make([]map[string]struct{}, len(results))
- for i, r := range results {
+ for i := range results {
remaining[i] = i
- tokenSets[i] = searchutil.TokenizeSimple(EnrichedPassage(ctx, r))
}
+ tokens, offsets, postings := buildMMRPostings(ctx, results)
- // Incremental form: maxRedundancy[i] caches result i's maximum jaccard
- // against everything selected so far, so each round only needs one
- // comparison per remaining result.
+ // Cache each candidate's maximum Jaccard against the selected passages.
+ // Postings count intersections only for candidates sharing a token with
+ // the new pick; disjoint passages have zero redundancy and need no update.
maxRedundancy := make([]float64, len(results))
+ intersections := make([]int, len(results))
+ taken := make([]bool, len(results))
+ touched := make([]int, 0, len(results))
selected := make([]int, 0, min(k, len(results)))
for len(selected) < k && len(remaining) > 0 {
bestPos := 0
@@ -43,10 +56,69 @@ func SelectMMR(ctx context.Context, results []*types.SearchResult, k int, lambda
}
chosen := remaining[bestPos]
selected = append(selected, chosen)
+ taken[chosen] = true
remaining = append(remaining[:bestPos], remaining[bestPos+1:]...)
- for _, i := range remaining {
- maxRedundancy[i] = math.Max(maxRedundancy[i], searchutil.Jaccard(tokenSets[i], tokenSets[chosen]))
+ if len(selected) == k || len(remaining) == 0 {
+ break
+ }
+ touched = touched[:0]
+ for _, token := range tokens[chosen] {
+ for _, i := range postings[offsets[token]:offsets[token+1]] {
+ if taken[i] {
+ continue
+ }
+ if intersections[i] == 0 {
+ touched = append(touched, i)
+ }
+ intersections[i]++
+ }
+ }
+ for _, i := range touched {
+ intersection := intersections[i]
+ union := len(tokens[i]) + len(tokens[chosen]) - intersection
+ maxRedundancy[i] = math.Max(maxRedundancy[i], float64(intersection)/float64(union))
+ intersections[i] = 0
}
}
return selected
}
+
+// buildMMRPostings interns each unique token and stores its candidate IDs in
+// one flat array. offsets[t]:offsets[t+1] is token t's posting list. Each row
+// comes from a token set, so one shared token contributes exactly once to a
+// pair's intersection, regardless of its frequency in either passage.
+func buildMMRPostings(
+ ctx context.Context, results []*types.SearchResult,
+) (tokens [][]int, offsets, postings []int) {
+ termIDs := make(map[string]int)
+ var counts []int
+ tokens = make([][]int, len(results))
+ for i, r := range results {
+ set := searchutil.TokenizeSimple(EnrichedPassage(ctx, r))
+ tokens[i] = make([]int, 0, len(set))
+ for term := range set {
+ id, exists := termIDs[term]
+ if !exists {
+ id = len(counts)
+ termIDs[term] = id
+ counts = append(counts, 0)
+ }
+ counts[id]++
+ tokens[i] = append(tokens[i], id)
+ }
+ }
+ offsets = make([]int, len(counts)+1)
+ for id, count := range counts {
+ offsets[id+1] = offsets[id] + count
+ }
+ postings = make([]int, offsets[len(counts)])
+ // Reuse counts as insertion cursors; offsets stay unchanged for lookups.
+ copy(counts, offsets)
+ for i, row := range tokens {
+ for _, id := range row {
+ postings[counts[id]] = i
+ counts[id]++
+ }
+ }
+ return tokens, offsets, postings
+}
diff --git a/internal/reranking/mmr_test.go b/internal/reranking/mmr_test.go
index 44c714ed7..a7932ab52 100644
--- a/internal/reranking/mmr_test.go
+++ b/internal/reranking/mmr_test.go
@@ -4,6 +4,8 @@ import (
"context"
"fmt"
"math"
+ "slices"
+ "strings"
"testing"
"github.com/Tencent/WeKnora/internal/searchutil"
@@ -116,3 +118,94 @@ func BenchmarkSelectMMR(b *testing.B) {
SelectMMR(ctx, results, 250, DefaultMMRLambda)
}
}
+
+func TestSelectMMR_preservesDiversityAndTies(t *testing.T) {
+ results := []*types.SearchResult{
+ {Content: "hybrid retrieval combines semantic vectors and keyword search", Score: 0.9},
+ {Content: "**hybrid retrieval** combines semantic vectors and keyword search", Score: 0.88},
+ {Content: "wiki construction extracts entities concepts and source citations", Score: 0.83},
+ }
+ if got := SelectMMR(context.Background(), results, 3, DefaultMMRLambda); !slices.Equal(got, []int{0, 2, 1}) {
+ t.Fatalf("selection = %v, want relevant, diverse, then redundant", got)
+ }
+ for _, r := range results {
+ r.Content, r.Score = "", 0.5
+ }
+ if got := SelectMMR(context.Background(), results, 10, DefaultMMRLambda); !slices.Equal(got, []int{0, 1, 2}) {
+ t.Fatalf("empty tied passages = %v, want original order", got)
+ }
+}
+
+func TestSelectMMR_multilingualEnrichedPassages(t *testing.T) {
+ results := []*types.SearchResult{
+ {Content: "混合检索结合语义向量和关键词索引,提高知识库检索覆盖率。", Score: 0.9},
+ {Content: "混合检索结合语义向量和关键词索引,提高知识库检索覆盖率。", Score: 0.85},
+ {Content: "Wiki 构建从文档抽取实体和概念,保留事实引用与来源分块。", Score: 0.8},
+ {Content: "retrieval", ImageInfo: `[{"caption":"semantic vectors and keyword search"}]`, Score: 0.75},
+ {
+ Content: "sources", Score: 0.7,
+ ChunkMetadata: types.JSON(`{"generated_questions":[
+ {"id":"q1","question":"How does wiki construction preserve source citations?"}
+ ]}`),
+ },
+ {Content: "", Score: 0.7},
+ }
+ for _, lambda := range []float64{0, 0.3, DefaultMMRLambda, 1} {
+ for _, k := range []int{1, 3, len(results), len(results) + 1} {
+ got := SelectMMR(context.Background(), results, k, lambda)
+ want := selectMMRNaive(context.Background(), results, k, lambda)
+ if !slices.Equal(got, want) {
+ t.Fatalf("k=%d lambda=%v: selection = %v, naive = %v", k, lambda, got, want)
+ }
+ }
+ }
+}
+
+// mmrPassageCorpus models a rerank pool with overlapping vocabulary across
+// documents and less overlap across topics. It is synthetic and deterministic;
+// the benchmarks measure MMR computation, not end-to-end retrieval quality.
+func mmrPassageCorpus(n, words int, chinese bool) []*types.SearchResult {
+ results := make([]*types.SearchResult, n)
+ terms := []string{"检索", "知识库", "向量", "索引", "语义", "文档", "引用", "实体", "概念", "构建", "答案", "模型"}
+ for i := range results {
+ var content strings.Builder
+ for j := 0; j < words; j++ {
+ if chinese {
+ fmt.Fprintf(&content, "%s ", terms[(i+j)%len(terms)])
+ }
+ // Different topics have disjoint bands; nearby documents within
+ // a topic share most of their vocabulary.
+ fmt.Fprintf(&content, "term%d ", (i%8)*1000+(j+i/8*7)%500)
+ }
+ results[i] = &types.SearchResult{
+ ID: fmt.Sprintf("chunk-%03d", i), Content: content.String(),
+ Score: 1 - float64(i%100)/200,
+ }
+ }
+ return results
+}
+
+func BenchmarkSelectMMRPassages(b *testing.B) {
+ ctx := context.Background()
+ for _, tc := range []struct {
+ name string
+ n, words, k int
+ chinese bool
+ }{
+ {name: "short", n: 50, words: 12, k: 10},
+ {name: "english_top10", n: 200, words: 200, k: 10},
+ {name: "english_top30", n: 200, words: 200, k: 30},
+ {name: "mixed_top10", n: 200, words: 200, k: 10, chinese: true},
+ {name: "mixed_top30", n: 200, words: 200, k: 30, chinese: true},
+ {name: "top1", n: 200, words: 200, k: 1},
+ } {
+ b.Run(tc.name, func(b *testing.B) {
+ results := mmrPassageCorpus(tc.n, tc.words, tc.chinese)
+ b.ReportAllocs()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ SelectMMR(ctx, results, tc.k, DefaultMMRLambda)
+ }
+ })
+ }
+}
diff --git a/website-docs/02-architecture/04-rag-pipeline.md b/website-docs/02-architecture/04-rag-pipeline.md
index bdda78b46..6abafa66c 100644
--- a/website-docs/02-architecture/04-rag-pipeline.md
+++ b/website-docs/02-architecture/04-rag-pipeline.md
@@ -228,7 +228,7 @@ pipeline = types.NewPipelineBuilder().
- 诊断里记录 `images_scored` / `images_kept`。
4. **复合打分** `compositeScore`:`0.6*模型分 + 0.3*检索基础分 + 0.1*来源权重`(web_search 来源权重 0.95,其余 1.0),clamp 到 [0,1]。图谱实体检索命中的 chunk 没有检索分,基础分用模型分代替。基础分/模型分/复合分记录在 `Metadata["base_score"]` / `["model_score"]` / `["composite_score"]`。早期版本还会乘一个「越靠文档前部越高」的位置先验(±0.05),因为它与分块编辑后的偏移变化耦合且收益不明确,已被移除。
5. **FAQ 加权**:`FAQPriorityEnabled` 且 `FAQScoreBoost > 1.0` 时,FAQ chunk 分数乘以 boost,记 `Metadata["faq_boosted"]`。结果不封顶到 1.0——封顶会让高分 FAQ 全部并列 1.0,彼此顺序退化为 tie-breaker。
-6. **MMR 多样性选择** `applyMMR`(λ=0.7,k=`RerankTopK`):`mmr = 0.7*relevance - 0.3*max_jaccard_redundancy`,用 `searchutil.TokenizeSimple` + `Jaccard` 并行预计算 token 集合,迭代贪心选出 `RerankResult`。
+6. **MMR 多样性选择** `reranking.SelectMMR`(λ=0.7,k=`RerankTopK`):`mmr = 0.7*relevance - 0.3*max_jaccard_redundancy`。对增强后的分块正文用 `searchutil.TokenizeSimple` 分词一次,再构建「token → 候选下标」倒排索引。每轮只累计与新选中分块共享 token 的候选交集,用 `交集 / (两侧 token 数之和 - 交集)` 计算 Jaccard,并更新最大冗余缓存;没有共享 token 的候选无需比较。索引使用连续数组存储,额外空间与候选 token 总数成正比,选择结果和平分时的输入顺序保持不变。`k=1` 时直接按相关性选择,跳过正文清洗、分词和索引构建。聊天、Agent 检索工具和检索 API 共用这一实现。
**PluginMemoryAffinity**(`memory_affinity.go`)注册在链的最内层,同样先 `next()` 再后置处理:对该调用者过往回答中至少引用过 2 次的文档,按使用次数对数增长加权,最高 ×1.15,只用于在相近候选之间打破平局。随后才轮到 WikiBoost 的后置加权。