Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

Fix/hybrid search match count normalization (#2750)

Zhang GHaac4f1586b4d
diff --git a/internal/application/service/knowledgebase_search.go b/internal/application/service/knowledgebase_search.go
index ef21dbcac..c4f7510d6 100644
--- a/internal/application/service/knowledgebase_search.go
+++ b/internal/application/service/knowledgebase_search.go
@@ -12,6 +12,12 @@ import (
secutils "github.com/Tencent/WeKnora/internal/utils"
)
+// maxRetrievalPoolSize bounds every retrieval depth derived from MatchCount:
+// the over-retrieval pool below and the doubling TopK of the FAQ iterative
+// path. Without it a caller-supplied MatchCount scales the vector-store query
+// depth without limit.
+const maxRetrievalPoolSize = 500
+
// GetQueryEmbedding computes the query embedding using the embedding model
// associated with the given knowledge base. Callers can pre-compute and reuse
// the result across multiple KBs that share the same embedding model to avoid
@@ -88,6 +94,10 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
id string,
params types.SearchParams,
) ([]*types.SearchResult, error) {
+ // Normalize once, before anything reads MatchCount. params is a value
+ // copy, so this stays local to the call.
+ params.MatchCount = normalizedMatchCount(params.MatchCount)
+
// Determine the set of KB IDs to search.
searchKBIDs := params.KnowledgeBaseIDs
if len(searchKBIDs) == 0 {
@@ -147,10 +157,11 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
}
// Over-retrieval (existing rule, preserved): 5x per-KB matchCount,
- // floor of 50, capped at 500 across the whole search.
- matchCount := max(params.MatchCount*5, 50) * len(searchKBIDs)
- if matchCount > 500 {
- matchCount = 500
+ // floored at DefaultRetrievalTopK, capped at maxRetrievalPoolSize across
+ // the whole search.
+ matchCount := max(params.MatchCount*5, types.DefaultRetrievalTopK) * len(searchKBIDs)
+ if matchCount > maxRetrievalPoolSize {
+ matchCount = maxRetrievalPoolSize
}
// Compute the query embedding once before fan-out and propagate via
@@ -248,17 +259,37 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
return nil, err
}
- // Limit to MatchCount; 0 means "no explicit limit" — use over-retrieval floor (50).
- effectiveMatchCount := params.MatchCount
- if effectiveMatchCount <= 0 {
- effectiveMatchCount = 50
- }
- if len(deduplicatedChunks) > effectiveMatchCount {
- deduplicatedChunks = deduplicatedChunks[:effectiveMatchCount]
+ // Truncate to the primary-match cap. MatchCount is guaranteed positive by
+ // the normalization at the top of this function; the slice bound below
+ // depends on that.
+ if len(deduplicatedChunks) > params.MatchCount {
+ deduplicatedChunks = deduplicatedChunks[:params.MatchCount]
}
+
return s.processSearchResults(ctx, deduplicatedChunks, params.SkipContextEnrichment)
}
+// normalizedMatchCount resolves the effective primary-match cap for a search.
+//
+// MatchCount arrives as Go's zero value whenever a caller omits it — JSON
+// cannot tell an absent match_count from an explicit 0 — and three consumers
+// inside HybridSearch read it: the over-retrieval floor, the FAQ
+// iterative-retrieval trigger, and the final truncation. They must agree, so
+// the value is resolved here rather than at each use site. A raw 0 truncates
+// the deduplicated chunk list to [:0] and empties an otherwise successful
+// search; a negative value panics on that same slice bound.
+//
+// The fallback is shared with RetrievalConfig.GetEffectiveEmbeddingTopK:
+// internal callers (chat pipeline, agent tools) feed these results into
+// reranking, which trims to RerankTopK afterwards, so a wide candidate pool is
+// the right failure mode for them.
+func normalizedMatchCount(requested int) int {
+ if requested <= 0 {
+ return types.DefaultRetrievalTopK
+ }
+ return requested
+}
+
// pickPrimary returns the KB whose ID matches id, or nil if id is not in
// scope. Callers map a nil result to NotFound; there is intentionally no
// kbs[0] fallback because it would mask caller bugs and could leak an
diff --git a/internal/application/service/knowledgebase_search_faq.go b/internal/application/service/knowledgebase_search_faq.go
index 124da2c00..e6f28a6fe 100644
--- a/internal/application/service/knowledgebase_search_faq.go
+++ b/internal/application/service/knowledgebase_search_faq.go
@@ -77,8 +77,11 @@ func (s *knowledgeBaseService) iterativeRetrieveWithDeduplication(ctx context.Co
) ([]*types.IndexWithScore, error) {
maxIterations := 5
// Start with a larger TopK since we're called when first retrieval wasn't enough
- // The first retrieval already used matchCount*3, so start from there
- currentTopK := matchCount * 3
+ // The first retrieval already used matchCount*3, so start from there.
+ // matchCount is caller-supplied, so both the seed and the per-iteration
+ // doubling are bounded by maxRetrievalPoolSize — otherwise a single request
+ // could drive the vector-store query depth arbitrarily deep.
+ currentTopK := min(matchCount*3, maxRetrievalPoolSize)
uniqueChunks := make(map[string]*types.IndexWithScore)
// Cache chunk data to avoid repeated DB queries across iterations
chunkDataCache := make(map[string]*types.Chunk)
@@ -198,8 +201,13 @@ func (s *knowledgeBaseService) iterativeRetrieveWithDeduplication(ctx context.Co
break
}
- // Increase TopK for next iteration
- currentTopK *= 2
+ // Increase TopK for next iteration. Once the cap is reached, another
+ // round would re-issue an identical query, so stop instead.
+ if currentTopK >= maxRetrievalPoolSize {
+ logger.Infof(ctx, "Retrieval depth cap %d reached, stopping iteration", maxRetrievalPoolSize)
+ break
+ }
+ currentTopK = min(currentTopK*2, maxRetrievalPoolSize)
}
// Convert map to slice and sort by score
diff --git a/internal/application/service/knowledgebase_search_matchcount_test.go b/internal/application/service/knowledgebase_search_matchcount_test.go
new file mode 100644
index 000000000..da8df5f7b
--- /dev/null
+++ b/internal/application/service/knowledgebase_search_matchcount_test.go
@@ -0,0 +1,73 @@
+package service
+
+import (
+ "context"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+// TestNormalizedMatchCount pins the contract that a non-positive MatchCount
+// resolves to the service-wide retrieval depth. The regression this guards is
+// severe and silent: a caller that omitted match_count reached the truncation
+// step with 0, which sliced the deduplicated chunk list to [:0] and turned a
+// successful retrieval into an empty response. A negative value panicked on
+// the same slice bound.
+//
+// The fallback shares DefaultRetrievalTopK with the over-retrieval floor on
+// purpose: with no explicit MatchCount the `*5` amplification in HybridSearch
+// collapses, so that floor alone decides the per-retriever depth and a larger
+// truncation bound could never yield more results.
+func TestNormalizedMatchCount(t *testing.T) {
+ t.Parallel()
+ tests := []struct {
+ name string
+ requested int
+ want int
+ }{
+ {name: "omitted arrives as zero", requested: 0, want: types.DefaultRetrievalTopK},
+ {name: "negative cannot index a slice", requested: -1, want: types.DefaultRetrievalTopK},
+ {name: "explicit value is honored", requested: 3, want: 3},
+ {name: "large explicit value is not clamped here", requested: 10000, want: 10000},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+ assert.Equal(t, tt.want, normalizedMatchCount(tt.requested))
+ })
+ }
+}
+
+// TestIterativeRetrieve_CapsSeedTopK guards the FAQ iterative path against an
+// unbounded caller-supplied MatchCount. The seed is MatchCount*3 and doubles
+// each round, so without the cap a single request could ask a vector store for
+// hundreds of thousands of rows.
+func TestIterativeRetrieve_CapsSeedTopK(t *testing.T) {
+ t.Parallel()
+ // canned is nil, so the first iteration retrieves nothing and the loop
+ // breaks — leaving group.TopK at exactly the seed value under test.
+ empty := &fakeRetrieveEngineService{
+ engineType: types.PostgresRetrieverEngineType,
+ support: []types.RetrieverType{types.VectorRetrieverType},
+ }
+ groups := []*storeGroup{
+ {
+ Engine: buildBoundComposite(t, empty),
+ BaseParams: vectorParams("q"),
+ TopK: 50,
+ KBIDs: []string{"kb-1"},
+ },
+ }
+ s := &knowledgeBaseService{}
+ ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(1))
+
+ results, err := s.iterativeRetrieveWithDeduplication(ctx, groups, 100000, "q")
+ require.NoError(t, err)
+ assert.Empty(t, results)
+ assert.Equal(t, maxRetrievalPoolSize, groups[0].TopK,
+ "seed TopK must be capped at the retrieval pool bound")
+}
diff --git a/internal/types/retrieval_config.go b/internal/types/retrieval_config.go
index f62604a17..fe5ab23b4 100644
--- a/internal/types/retrieval_config.go
+++ b/internal/types/retrieval_config.go
@@ -37,10 +37,16 @@ type RetrievalConfig struct {
RRFKeywordWeight float64 `json:"rrf_keyword_weight,omitempty"`
}
+// DefaultRetrievalTopK is the retrieval depth used when a caller supplies no
+// usable TopK / MatchCount. HybridSearch also floors its over-retrieval pool
+// at this value, so the fallback and the pool it draws from stay in step by
+// construction rather than by two independently maintained literals.
+const DefaultRetrievalTopK = 50
+
// GetEffectiveEmbeddingTopK returns EmbeddingTopK with a fallback default.
func (c *RetrievalConfig) GetEffectiveEmbeddingTopK() int {
if c == nil || c.EmbeddingTopK <= 0 {
- return 50
+ return DefaultRetrievalTopK
}
return c.EmbeddingTopK
}
diff --git a/website-docs/02-architecture/04-rag-pipeline.md b/website-docs/02-architecture/04-rag-pipeline.md
index 7e57c3838..c7dd2ccc1 100644
--- a/website-docs/02-architecture/04-rag-pipeline.md
+++ b/website-docs/02-architecture/04-rag-pipeline.md
@@ -476,7 +476,7 @@ sequenceDiagram
`internal/application/service/knowledgebase_search.go` 是所有检索的汇聚点(chat pipeline、Agent 工具、搜索 API 共用):
1. **授权与校验**:批量加载 KB(含跨租户 Organization 共享库),逐库 `authorizeKBAccess`;`validateSameEmbeddingModel` 拒绝跨 embedding 空间的多库检索(wiki/graph 无向量库有豁免)。
-2. **过召回**:`matchCount = max(MatchCount*5, 50) * len(KBs)`,上限 500。
+2. **入参归一化 + 过召回**:`MatchCount <= 0`(调用方未传时 JSON 反序列化即为 0)先经 `normalizedMatchCount` 归一化为 `types.DefaultRetrievalTopK`(50),使过召回下限、FAQ 迭代触发条件、末尾截断三处读到同一个值——否则截断会把结果集切成 `[:0]`,负数还会越界 panic;随后 `matchCount = max(MatchCount*5, 50) * len(KBs)`,上限 `maxRetrievalPoolSize`(500)。
3. **查询向量只算一次**,随 `params.QueryEmbedding` 传播到所有 store 组。
4. **storeGroup 分组**(`knowledgebase_search_storegroup.go`):按 `(VectorStoreID, 属主租户)` 分组;每组经 `retriever.CreateRetrieveEngineForKB` 解析出 `CompositeRetrieveEngine`。`buildRetrievalParams` 按组内每个 KB 的类型路由:FAQ 库走 FAQ 向量索引(`KnowledgeType=faq`,无关键词索引),文档库走默认向量索引 + 关键词索引。
5. **fan-out**(`knowledgebase_search_fanout.go`):单组直查零开销;多组用 `errgroup` 并发(上限 4),每组超时 `MULTI_STORE_RETRIEVE_TIMEOUT_SEC`(默认 30s),all-or-nothing 失败策略;结果跨引擎类型时用 `EngineAwareNormalizer` 把向量分归一化到 [0,1](详见检索引擎文档)。
@@ -484,7 +484,7 @@ sequenceDiagram
- 仅向量或仅关键词 → `deduplicateByScore`(按 chunk 保留最高分);
- 混合 → **加权 RRF**:`score = vectorWeight/(k+vectorRank) + keywordWeight/(k+keywordRank)`,`k` 与权重来自租户 `RetrievalConfig`(有缺省值),rank 基于各自检索器返回顺序(1-indexed),对分数尺度免疫。
7. **FAQ 命中策略**(`knowledgebase_search_faq.go`,仅 FAQ 类型 KB):
- - **迭代检索**:去重后不足 `MatchCount` 且首轮已打满 → 从 `TopK*3` 起最多 5 轮翻倍扩大 TopK 重检索,跨 store 组统一生效,chunk 数据缓存避免重复回表;
+ - **迭代检索**:去重后不足 `MatchCount` 且首轮已打满 → 从 `TopK*3` 起最多 5 轮翻倍扩大 TopK 重检索,跨 store 组统一生效,chunk 数据缓存避免重复回表;种子与每轮增长均封顶 `maxRetrievalPoolSize`,触顶即停(再迭代只会重发同一个查询);
- **负例问题过滤**:查询与 FAQ 的 `NegativeQuestions` 精确匹配(小写去空格)即剔除该条——支持"这个问题不要用这条 FAQ 答"的运营配置。
8. 截断到 `MatchCount` 后 `processSearchResults` 补全 chunk 元数据(管线场景 `SkipContextEnrichment=true`,上下文组装留给 merge 阶段)。
diff --git a/website-docs/03-features/17-faq.md b/website-docs/03-features/17-faq.md
index 643fc8ceb..42c5d673f 100644
--- a/website-docs/03-features/17-faq.md
+++ b/website-docs/03-features/17-faq.md
@@ -187,7 +187,7 @@ type FAQSearchRequest struct {
1. **混合召回**:查询文本归一化后做向量检索 + BM25 关键词检索,融合去重;
2. **两级标签优先**:`FirstPriorityTagIDs` 命中的条目排最前,其次 `SecondPriorityTagIDs`;
3. **负例过滤**(`filterByNegativeQuestions`):查询文本与某条目的任一反例问完全匹配(小写比较)→ 该条目从结果中剔除。典型场景:用户问"不支持 X 吗",避免返回"支持 X"的条目;
-4. **迭代召回**(`applyFAQPostProcessing`):当过滤后的唯一条目数不足 `match_count` 且向量结果打满时触发 `iterativeRetrieveWithDeduplication`——最多迭代 5 次、每次 TopK 翻倍,带去重与负例过滤缓存,无新结果提前终止;
+4. **迭代召回**(`applyFAQPostProcessing`):当过滤后的唯一条目数不足 `match_count` 且向量结果打满时触发 `iterativeRetrieveWithDeduplication`——最多迭代 5 次、每次 TopK 翻倍(种子 `MatchCount*3` 与每轮增长均封顶 500,触顶即停),带去重与负例过滤缓存,无新结果提前终止;
5. 结果附带 `score`、`match_type`、`matched_question`(实际命中的是标准问还是哪个相似问),答案按 `answer_strategy`(all / random)返回。
非 FAQ 类型 KB 直接跳过该后处理(`if kb.Type != types.KnowledgeBaseTypeFAQ { return chunks, nil }`),普通混合检索不受影响;agent 检索链在 FAQ 库上同样经过这条后处理路径。