Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
Fix/hybrid search match count normalization (#2750)
Zhang GHaac4f1586b4d
diff --git a/internal/application/service/knowledgebase_search.go b/internal/application/service/knowledgebase_search.goindex ef21dbcac..c4f7510d6 100644--- a/internal/application/service/knowledgebase_search.go+++ b/internal/application/service/knowledgebase_search.go@@ -12,6 +12,12 @@ import (secutils "github.com/Tencent/WeKnora/internal/utils")+// maxRetrievalPoolSize bounds every retrieval depth derived from MatchCount:+// the over-retrieval pool below and the doubling TopK of the FAQ iterative+// path. Without it a caller-supplied MatchCount scales the vector-store query+// depth without limit.+const maxRetrievalPoolSize = 500+// GetQueryEmbedding computes the query embedding using the embedding model// associated with the given knowledge base. Callers can pre-compute and reuse// the result across multiple KBs that share the same embedding model to avoid@@ -88,6 +94,10 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,id string,params types.SearchParams,) ([]*types.SearchResult, error) {+ // Normalize once, before anything reads MatchCount. params is a value+ // copy, so this stays local to the call.+ params.MatchCount = normalizedMatchCount(params.MatchCount)+// Determine the set of KB IDs to search.searchKBIDs := params.KnowledgeBaseIDsif len(searchKBIDs) == 0 {@@ -147,10 +157,11 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,}// Over-retrieval (existing rule, preserved): 5x per-KB matchCount,- // floor of 50, capped at 500 across the whole search.- matchCount := max(params.MatchCount*5, 50) * len(searchKBIDs)- if matchCount > 500 {- matchCount = 500+ // floored at DefaultRetrievalTopK, capped at maxRetrievalPoolSize across+ // the whole search.+ matchCount := max(params.MatchCount*5, types.DefaultRetrievalTopK) * len(searchKBIDs)+ if matchCount > maxRetrievalPoolSize {+ matchCount = maxRetrievalPoolSize}// Compute the query embedding once before fan-out and propagate via@@ -248,17 +259,37 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,return nil, err}- // Limit to MatchCount; 0 means "no explicit limit" — use over-retrieval floor (50).- effectiveMatchCount := params.MatchCount- if effectiveMatchCount <= 0 {- effectiveMatchCount = 50- }- if len(deduplicatedChunks) > effectiveMatchCount {- deduplicatedChunks = deduplicatedChunks[:effectiveMatchCount]+ // Truncate to the primary-match cap. MatchCount is guaranteed positive by+ // the normalization at the top of this function; the slice bound below+ // depends on that.+ if len(deduplicatedChunks) > params.MatchCount {+ deduplicatedChunks = deduplicatedChunks[:params.MatchCount]}+return s.processSearchResults(ctx, deduplicatedChunks, params.SkipContextEnrichment)}+// normalizedMatchCount resolves the effective primary-match cap for a search.+//+// MatchCount arrives as Go's zero value whenever a caller omits it — JSON+// cannot tell an absent match_count from an explicit 0 — and three consumers+// inside HybridSearch read it: the over-retrieval floor, the FAQ+// iterative-retrieval trigger, and the final truncation. They must agree, so+// the value is resolved here rather than at each use site. A raw 0 truncates+// the deduplicated chunk list to [:0] and empties an otherwise successful+// search; a negative value panics on that same slice bound.+//+// The fallback is shared with RetrievalConfig.GetEffectiveEmbeddingTopK:+// internal callers (chat pipeline, agent tools) feed these results into+// reranking, which trims to RerankTopK afterwards, so a wide candidate pool is+// the right failure mode for them.+func normalizedMatchCount(requested int) int {+ if requested <= 0 {+ return types.DefaultRetrievalTopK+ }+ return requested+}+// pickPrimary returns the KB whose ID matches id, or nil if id is not in// scope. Callers map a nil result to NotFound; there is intentionally no// kbs[0] fallback because it would mask caller bugs and could leak andiff --git a/internal/application/service/knowledgebase_search_faq.go b/internal/application/service/knowledgebase_search_faq.goindex 124da2c00..e6f28a6fe 100644--- a/internal/application/service/knowledgebase_search_faq.go+++ b/internal/application/service/knowledgebase_search_faq.go@@ -77,8 +77,11 @@ func (s *knowledgeBaseService) iterativeRetrieveWithDeduplication(ctx context.Co) ([]*types.IndexWithScore, error) {maxIterations := 5// Start with a larger TopK since we're called when first retrieval wasn't enough- // The first retrieval already used matchCount*3, so start from there- currentTopK := matchCount * 3+ // The first retrieval already used matchCount*3, so start from there.+ // matchCount is caller-supplied, so both the seed and the per-iteration+ // doubling are bounded by maxRetrievalPoolSize — otherwise a single request+ // could drive the vector-store query depth arbitrarily deep.+ currentTopK := min(matchCount*3, maxRetrievalPoolSize)uniqueChunks := make(map[string]*types.IndexWithScore)// Cache chunk data to avoid repeated DB queries across iterationschunkDataCache := make(map[string]*types.Chunk)@@ -198,8 +201,13 @@ func (s *knowledgeBaseService) iterativeRetrieveWithDeduplication(ctx context.Cobreak}- // Increase TopK for next iteration- currentTopK *= 2+ // Increase TopK for next iteration. Once the cap is reached, another+ // round would re-issue an identical query, so stop instead.+ if currentTopK >= maxRetrievalPoolSize {+ logger.Infof(ctx, "Retrieval depth cap %d reached, stopping iteration", maxRetrievalPoolSize)+ break+ }+ currentTopK = min(currentTopK*2, maxRetrievalPoolSize)}// Convert map to slice and sort by scorediff --git a/internal/application/service/knowledgebase_search_matchcount_test.go b/internal/application/service/knowledgebase_search_matchcount_test.gonew file mode 100644index 000000000..da8df5f7b--- /dev/null+++ b/internal/application/service/knowledgebase_search_matchcount_test.go@@ -0,0 +1,73 @@+package service++import (+ "context"+ "testing"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"++ "github.com/Tencent/WeKnora/internal/types"+)++// TestNormalizedMatchCount pins the contract that a non-positive MatchCount+// resolves to the service-wide retrieval depth. The regression this guards is+// severe and silent: a caller that omitted match_count reached the truncation+// step with 0, which sliced the deduplicated chunk list to [:0] and turned a+// successful retrieval into an empty response. A negative value panicked on+// the same slice bound.+//+// The fallback shares DefaultRetrievalTopK with the over-retrieval floor on+// purpose: with no explicit MatchCount the `*5` amplification in HybridSearch+// collapses, so that floor alone decides the per-retriever depth and a larger+// truncation bound could never yield more results.+func TestNormalizedMatchCount(t *testing.T) {+ t.Parallel()+ tests := []struct {+ name string+ requested int+ want int+ }{+ {name: "omitted arrives as zero", requested: 0, want: types.DefaultRetrievalTopK},+ {name: "negative cannot index a slice", requested: -1, want: types.DefaultRetrievalTopK},+ {name: "explicit value is honored", requested: 3, want: 3},+ {name: "large explicit value is not clamped here", requested: 10000, want: 10000},+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ t.Parallel()+ assert.Equal(t, tt.want, normalizedMatchCount(tt.requested))+ })+ }+}++// TestIterativeRetrieve_CapsSeedTopK guards the FAQ iterative path against an+// unbounded caller-supplied MatchCount. The seed is MatchCount*3 and doubles+// each round, so without the cap a single request could ask a vector store for+// hundreds of thousands of rows.+func TestIterativeRetrieve_CapsSeedTopK(t *testing.T) {+ t.Parallel()+ // canned is nil, so the first iteration retrieves nothing and the loop+ // breaks — leaving group.TopK at exactly the seed value under test.+ empty := &fakeRetrieveEngineService{+ engineType: types.PostgresRetrieverEngineType,+ support: []types.RetrieverType{types.VectorRetrieverType},+ }+ groups := []*storeGroup{+ {+ Engine: buildBoundComposite(t, empty),+ BaseParams: vectorParams("q"),+ TopK: 50,+ KBIDs: []string{"kb-1"},+ },+ }+ s := &knowledgeBaseService{}+ ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(1))++ results, err := s.iterativeRetrieveWithDeduplication(ctx, groups, 100000, "q")+ require.NoError(t, err)+ assert.Empty(t, results)+ assert.Equal(t, maxRetrievalPoolSize, groups[0].TopK,+ "seed TopK must be capped at the retrieval pool bound")+}diff --git a/internal/types/retrieval_config.go b/internal/types/retrieval_config.goindex f62604a17..fe5ab23b4 100644--- a/internal/types/retrieval_config.go+++ b/internal/types/retrieval_config.go@@ -37,10 +37,16 @@ type RetrievalConfig struct {RRFKeywordWeight float64 `json:"rrf_keyword_weight,omitempty"`}+// DefaultRetrievalTopK is the retrieval depth used when a caller supplies no+// usable TopK / MatchCount. HybridSearch also floors its over-retrieval pool+// at this value, so the fallback and the pool it draws from stay in step by+// construction rather than by two independently maintained literals.+const DefaultRetrievalTopK = 50+// GetEffectiveEmbeddingTopK returns EmbeddingTopK with a fallback default.func (c *RetrievalConfig) GetEffectiveEmbeddingTopK() int {if c == nil || c.EmbeddingTopK <= 0 {- return 50+ return DefaultRetrievalTopK}return c.EmbeddingTopK}diff --git a/website-docs/02-architecture/04-rag-pipeline.md b/website-docs/02-architecture/04-rag-pipeline.mdindex 7e57c3838..c7dd2ccc1 100644--- a/website-docs/02-architecture/04-rag-pipeline.md+++ b/website-docs/02-architecture/04-rag-pipeline.md@@ -476,7 +476,7 @@ sequenceDiagram`internal/application/service/knowledgebase_search.go` 是所有检索的汇聚点(chat pipeline、Agent 工具、搜索 API 共用):1. **授权与校验**:批量加载 KB(含跨租户 Organization 共享库),逐库 `authorizeKBAccess`;`validateSameEmbeddingModel` 拒绝跨 embedding 空间的多库检索(wiki/graph 无向量库有豁免)。-2. **过召回**:`matchCount = max(MatchCount*5, 50) * len(KBs)`,上限 500。+2. **入参归一化 + 过召回**:`MatchCount <= 0`(调用方未传时 JSON 反序列化即为 0)先经 `normalizedMatchCount` 归一化为 `types.DefaultRetrievalTopK`(50),使过召回下限、FAQ 迭代触发条件、末尾截断三处读到同一个值——否则截断会把结果集切成 `[:0]`,负数还会越界 panic;随后 `matchCount = max(MatchCount*5, 50) * len(KBs)`,上限 `maxRetrievalPoolSize`(500)。3. **查询向量只算一次**,随 `params.QueryEmbedding` 传播到所有 store 组。4. **storeGroup 分组**(`knowledgebase_search_storegroup.go`):按 `(VectorStoreID, 属主租户)` 分组;每组经 `retriever.CreateRetrieveEngineForKB` 解析出 `CompositeRetrieveEngine`。`buildRetrievalParams` 按组内每个 KB 的类型路由:FAQ 库走 FAQ 向量索引(`KnowledgeType=faq`,无关键词索引),文档库走默认向量索引 + 关键词索引。5. **fan-out**(`knowledgebase_search_fanout.go`):单组直查零开销;多组用 `errgroup` 并发(上限 4),每组超时 `MULTI_STORE_RETRIEVE_TIMEOUT_SEC`(默认 30s),all-or-nothing 失败策略;结果跨引擎类型时用 `EngineAwareNormalizer` 把向量分归一化到 [0,1](详见检索引擎文档)。@@ -484,7 +484,7 @@ sequenceDiagram- 仅向量或仅关键词 → `deduplicateByScore`(按 chunk 保留最高分);- 混合 → **加权 RRF**:`score = vectorWeight/(k+vectorRank) + keywordWeight/(k+keywordRank)`,`k` 与权重来自租户 `RetrievalConfig`(有缺省值),rank 基于各自检索器返回顺序(1-indexed),对分数尺度免疫。7. **FAQ 命中策略**(`knowledgebase_search_faq.go`,仅 FAQ 类型 KB):- - **迭代检索**:去重后不足 `MatchCount` 且首轮已打满 → 从 `TopK*3` 起最多 5 轮翻倍扩大 TopK 重检索,跨 store 组统一生效,chunk 数据缓存避免重复回表;+ - **迭代检索**:去重后不足 `MatchCount` 且首轮已打满 → 从 `TopK*3` 起最多 5 轮翻倍扩大 TopK 重检索,跨 store 组统一生效,chunk 数据缓存避免重复回表;种子与每轮增长均封顶 `maxRetrievalPoolSize`,触顶即停(再迭代只会重发同一个查询);- **负例问题过滤**:查询与 FAQ 的 `NegativeQuestions` 精确匹配(小写去空格)即剔除该条——支持"这个问题不要用这条 FAQ 答"的运营配置。8. 截断到 `MatchCount` 后 `processSearchResults` 补全 chunk 元数据(管线场景 `SkipContextEnrichment=true`,上下文组装留给 merge 阶段)。diff --git a/website-docs/03-features/17-faq.md b/website-docs/03-features/17-faq.mdindex 643fc8ceb..42c5d673f 100644--- a/website-docs/03-features/17-faq.md+++ b/website-docs/03-features/17-faq.md@@ -187,7 +187,7 @@ type FAQSearchRequest struct {1. **混合召回**:查询文本归一化后做向量检索 + BM25 关键词检索,融合去重;2. **两级标签优先**:`FirstPriorityTagIDs` 命中的条目排最前,其次 `SecondPriorityTagIDs`;3. **负例过滤**(`filterByNegativeQuestions`):查询文本与某条目的任一反例问完全匹配(小写比较)→ 该条目从结果中剔除。典型场景:用户问"不支持 X 吗",避免返回"支持 X"的条目;-4. **迭代召回**(`applyFAQPostProcessing`):当过滤后的唯一条目数不足 `match_count` 且向量结果打满时触发 `iterativeRetrieveWithDeduplication`——最多迭代 5 次、每次 TopK 翻倍,带去重与负例过滤缓存,无新结果提前终止;+4. **迭代召回**(`applyFAQPostProcessing`):当过滤后的唯一条目数不足 `match_count` 且向量结果打满时触发 `iterativeRetrieveWithDeduplication`——最多迭代 5 次、每次 TopK 翻倍(种子 `MatchCount*3` 与每轮增长均封顶 500,触顶即停),带去重与负例过滤缓存,无新结果提前终止;5. 结果附带 `score`、`match_type`、`matched_question`(实际命中的是标准问还是哪个相似问),答案按 `answer_strategy`(all / random)返回。非 FAQ 类型 KB 直接跳过该后处理(`if kb.Type != types.KnowledgeBaseTypeFAQ { return chunks, nil }`),普通混合检索不受影响;agent 检索链在 FAQ 库上同样经过这条后处理路径。