Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
feat(retrieval): unify rerank across search APIs and expose diagnostics (#3675)
lyingbug9fc08c2079d8
diff --git a/client/knowledgebase.go b/client/knowledgebase.goindex dbf2186f9..be63e2288 100644--- a/client/knowledgebase.go+++ b/client/knowledgebase.go@@ -244,6 +244,8 @@ type SearchResult struct {type HybridSearchResponse struct {Success bool `json:"success"`Data []*SearchResult `json:"data"`+ // Meta is present when the request carried a rerank object.+ Meta *RetrievalMeta `json:"meta,omitempty"`}type CopyKnowledgeBaseRequest struct {@@ -405,6 +407,16 @@ type SearchParams struct {MatchCount int `json:"match_count"`DisableKeywordsMatch bool `json:"disable_keywords_match"`DisableVectorMatch bool `json:"disable_vector_match"`+ // KnowledgeBaseIDs searches several knowledge bases that share one+ // embedding model; the path ID must be one of them.+ KnowledgeBaseIDs []string `json:"knowledge_base_ids,omitempty"`+ // KnowledgeIDs limits the search to these documents.+ KnowledgeIDs []string `json:"knowledge_ids,omitempty"`+ // TagIDs limits the search to these tags.+ TagIDs []string `json:"tag_ids,omitempty"`+ // Rerank, when set and enabled, reranks the candidates before cutting to+ // MatchCount. Nil keeps the raw retrieval order.+ Rerank *RerankOptions `json:"rerank,omitempty"`}// HybridSearch performs hybrid search.@@ -415,6 +427,21 @@ func (c *Client) HybridSearch(params *SearchParams,opts ...ResourceURLOptions,) ([]*SearchResult, error) {+ response, err := c.HybridSearchDetailed(ctx, knowledgeBaseID, params, opts...)+ if err != nil {+ return nil, err+ }+ return response.Data, nil+}++// HybridSearchDetailed performs hybrid search and returns the whole+// response, including the rerank diagnostics in Meta.+func (c *Client) HybridSearchDetailed(+ ctx context.Context,+ knowledgeBaseID string,+ params *SearchParams,+ opts ...ResourceURLOptions,+) (*HybridSearchResponse, error) {path := fmt.Sprintf("/api/v1/knowledge-bases/%s/hybrid-search", knowledgeBaseID)queryParams := url.Values{}@@ -432,7 +459,7 @@ func (c *Client) HybridSearch(return nil, err}- return response.Data, nil+ return &response, nil}// TogglePinKnowledgeBase toggles the pin status of a knowledge base.diff --git a/client/retrieval.go b/client/retrieval.gonew file mode 100644index 000000000..0d90fc9b1--- /dev/null+++ b/client/retrieval.go@@ -0,0 +1,44 @@+package client++// RerankOptions controls the rerank stage of the knowledge-search and+// hybrid-search endpoints.+//+// On hybrid-search, a nil *RerankOptions means no rerank. On knowledge-search,+// nil reranks with the tenant retrieval config; set Enabled to false to turn+// rerank off.+type RerankOptions struct {+ // Enabled turns rerank off when explicitly false. Nil means on.+ Enabled *bool `json:"enabled,omitempty"`+ // ModelID selects the rerank model. Empty uses the tenant retrieval+ // config, then the tenant's first rerank model.+ ModelID string `json:"model_id,omitempty"`+ // TopK caps the reranked results. Zero uses the endpoint's result count.+ TopK int `json:"top_k,omitempty"`+ // Threshold is the minimum rerank model score. Nil uses the tenant+ // retrieval config.+ Threshold *float64 `json:"threshold,omitempty"`+}++// RerankDiagnostics reports what the rerank stage did with a request.+type RerankDiagnostics struct {+ // Applied is true when rerank scores decided the returned order.+ Applied bool `json:"applied"`+ // Outcome is one of ok, threshold_degraded, fallback_top1,+ // all_below_threshold, model_error, model_unavailable, no_model,+ // disabled, no_candidates.+ Outcome string `json:"outcome"`+ // ModelSource is request, tenant or auto.+ ModelID string `json:"model_id,omitempty"`+ ModelSource string `json:"model_source,omitempty"`+ Threshold float64 `json:"threshold"`+ EffectiveThreshold float64 `json:"effective_threshold"`+ TopScore float64 `json:"top_score"`+ CandidateCount int `json:"candidate_count"`+ ResultCount int `json:"result_count"`+ Error string `json:"error,omitempty"`+}++// RetrievalMeta carries the diagnostics returned beside retrieval results.+type RetrievalMeta struct {+ Rerank *RerankDiagnostics `json:"rerank,omitempty"`+}diff --git a/client/session.go b/client/session.goindex 39615efaa..ae54e440d 100644--- a/client/session.go+++ b/client/session.go@@ -479,12 +479,22 @@ type SearchKnowledgeRequest struct {KnowledgeIDs []string `json:"knowledge_ids,omitempty"` // Specific knowledge (file) IDsTagIDs []string `json:"tag_ids,omitempty"` // Tag IDs for filtering within a single KBMentionedItems []MentionedItem `json:"mentioned_items,omitempty"` // Optional scoped tag mentions++ // Optional overrides of the tenant retrieval config.+ VectorThreshold *float64 `json:"vector_threshold,omitempty"` // Minimum vector similarity+ KeywordThreshold *float64 `json:"keyword_threshold,omitempty"` // Minimum keyword score+ MatchCount int `json:"match_count,omitempty"` // Number of results to return+ DisableKeywordsMatch bool `json:"disable_keywords_match,omitempty"` // Vector recall only+ DisableVectorMatch bool `json:"disable_vector_match,omitempty"` // Keyword recall only+ Rerank *RerankOptions `json:"rerank,omitempty"` // Rerank override}// SearchKnowledgeResponse search results responsetype SearchKnowledgeResponse struct {Success bool `json:"success"`Data []*SearchResult `json:"data"`+ // Meta reports what the rerank stage did.+ Meta *RetrievalMeta `json:"meta,omitempty"`}// SearchKnowledge performs knowledge base search without LLM summarization.@@ -494,6 +504,20 @@ func (c *Client) SearchKnowledge(request *SearchKnowledgeRequest,opts ...ResourceURLOptions,) ([]*SearchResult, error) {+ response, err := c.SearchKnowledgeDetailed(ctx, request, opts...)+ if err != nil {+ return nil, err+ }+ return response.Data, nil+}++// SearchKnowledgeDetailed performs knowledge base search and returns the+// whole response, including the rerank diagnostics in Meta.+func (c *Client) SearchKnowledgeDetailed(+ ctx context.Context,+ request *SearchKnowledgeRequest,+ opts ...ResourceURLOptions,+) (*SearchKnowledgeResponse, error) {debugLogger.Debug("search_knowledge_start","knowledge_base_ids", request.KnowledgeBaseIDs,"knowledge_ids", request.KnowledgeIDs,@@ -526,5 +550,5 @@ func (c *Client) SearchKnowledge(}debugLogger.Debug("search_knowledge_completed", "result_count", len(response.Data))- return response.Data, nil+ return &response, nil}diff --git a/internal/agent/tools/query_knowledge_graph_test.go b/internal/agent/tools/query_knowledge_graph_test.goindex d2cc20897..c72d2eee6 100644--- a/internal/agent/tools/query_knowledge_graph_test.go+++ b/internal/agent/tools/query_knowledge_graph_test.go@@ -67,6 +67,13 @@ func (s *stubKnowledgeBaseService) HybridSearch(context.Context, string, types.Sreturn s.results, nil}+func (s *stubKnowledgeBaseService) HybridSearchWithRerank(+ ctx context.Context, id string, params types.SearchParams,+) (*types.RetrievalResult, error) {+ results, err := s.HybridSearch(ctx, id, params)+ return &types.RetrievalResult{Results: results}, err+}+func (s *stubKnowledgeBaseService) GetQueryEmbedding(context.Context, string, string) ([]float32, error) {return nil, nil}diff --git a/internal/agent/tools/search_knowledge.go b/internal/agent/tools/search_knowledge.goindex 933a67d1b..73533b8ef 100644--- a/internal/agent/tools/search_knowledge.go+++ b/internal/agent/tools/search_knowledge.go@@ -4,7 +4,6 @@ import ("context""encoding/json""fmt"- "math""regexp""sort""strings"@@ -13,6 +12,7 @@ import ("github.com/Tencent/WeKnora/internal/config""github.com/Tencent/WeKnora/internal/logger""github.com/Tencent/WeKnora/internal/models/rerank"+ "github.com/Tencent/WeKnora/internal/reranking""github.com/Tencent/WeKnora/internal/searchutil""github.com/Tencent/WeKnora/internal/types""github.com/Tencent/WeKnora/internal/types/interfaces"@@ -248,11 +248,7 @@ func (t *SearchKnowledgeTool) Execute(ctx context.Context, args json.RawMessage)}if len(ranked) > 0 {- mmrK := len(ranked)- if mmrK > retrievalLimit {- mmrK = retrievalLimit- }- if selected := t.applyMMR(ctx, ranked, mmrK, 0.7); len(selected) > 0 {+ if selected := selectMMR(ctx, ranked, min(len(ranked), retrievalLimit)); len(selected) > 0 {ranked = selected}}@@ -602,14 +598,13 @@ func (t *SearchKnowledgeTool) concurrentSearchByTargets(return allResults}-// rerankResults applies reranking to all search results (including FAQ entries)-// using the configured rerank model, then filters by threshold and applies-// composite scoring so MMR/sorting uses a single score scale.+// rerankResults scores all search results (including FAQ entries) with the+// configured rerank model through the shared rerank stage, keeping those+// that pass the threshold with composite scores, best first.//// A failed rerank call degrades to the raw retrieval order. An empty result-// after threshold filtering is kept empty: filterRerankRankResults already-// preserves the top candidate down to agentRerankFallbackMinScore, so reaching-// zero means even the best match is below that floor.+// is kept empty: the shared stage already preserves the top candidate down to+// its fallback floor, so reaching zero means even the best match is below it.func (t *SearchKnowledgeTool) rerankResults(ctx context.Context,query string,@@ -619,21 +614,29 @@ func (t *SearchKnowledgeTool) rerankResults(return results, nil}- rankResults, err := t.rerankScores(ctx, query, results)- if err != nil {- logger.Warnf(ctx, "[Tool][SearchKnowledge] Rerank model failed, using raw retrieval results: %v", err)+ rows := make([]*types.SearchResult, len(results))+ for i, r := range results {+ rows[i] = r.SearchResult+ }+ threshold := t.rerankThreshold()+ res := reranking.Rerank(ctx, t.rerankModel, query, rows, reranking.Options{+ Threshold: threshold,+ FallbackMinScore: reranking.FallbackMinScore(t.searchTargets.HasRecallThresholdOverride()),+ })+ if res.Diagnostics.Outcome == types.RerankOutcomeModelError {+ logger.Warnf(ctx, "[Tool][SearchKnowledge] Rerank model failed, using raw retrieval results: %s",+ res.Diagnostics.Error)return results, nil}- threshold := t.rerankThreshold()- reranked := t.applyModelRerankScores(- results,- rankResults,- threshold,- t.searchTargets.HasRecallThresholdOverride(),- )- logger.Infof(ctx, "[Tool][SearchKnowledge] Reranked %d/%d results above threshold %.2f",- len(reranked), len(results), threshold)+ reranked := make([]*searchResultWithMeta, 0, len(res.Results))+ for i, row := range res.Results {+ withMeta := *results[res.Indices[i]]+ withMeta.SearchResult = row+ reranked = append(reranked, &withMeta)+ }+ logger.Infof(ctx, "[Tool][SearchKnowledge] Reranked %d/%d results above threshold %.2f (%s)",+ len(reranked), len(results), threshold, res.Diagnostics.Outcome)return reranked, nil}@@ -666,98 +669,11 @@ func (t *SearchKnowledgeTool) getFAQMetadata(return meta, nil}-// rerankScores scores the candidates with the configured rerank model and-// returns the raw relevance scores, leaving threshold filtering and composite-// scoring to the caller.-func (t *SearchKnowledgeTool) rerankScores(- ctx context.Context,- query string,- results []*searchResultWithMeta,-) ([]rerank.RankResult, error) {- passages := make([]string, len(results))- for i, result := range results {- passages[i] = t.rerankPassage(ctx, result.SearchResult)- }- rerankResp, err := t.rerankModel.Rerank(ctx, query, passages)- if err != nil {- return nil, fmt.Errorf("rerank call failed: %w", err)- }- return rerankResp, nil-}--// rerankPassage is the text the rerank model scores: the document title-// followed by the enriched chunk. A chunk rarely restates what its document-// is about, so without the title a passage from "Show HN: Echo - ... using-// open-weight models" scored 0.002 against "Echo open-weight models reduce-// cost" and 0.45 with it. FAQ entries carry their own question instead.-func (t *SearchKnowledgeTool) rerankPassage(ctx context.Context, result *types.SearchResult) string {- passage := t.getEnrichedPassage(ctx, result)- title := strings.TrimSpace(result.KnowledgeTitle)- if title == "" || result.ChunkType == string(types.ChunkTypeFAQ) {- return passage- }- return title + "\n\n" + passage-}-func (t *SearchKnowledgeTool) rerankThreshold() float64 {if t.config != nil && t.config.Conversation != nil && t.config.Conversation.RerankThreshold > 0 {return t.config.Conversation.RerankThreshold}- return 0.3-}--const agentRerankFallbackMinScore = 0.15--func filterRerankRankResults(- rankResults []rerank.RankResult,- threshold float64,- preserveTop bool,-) []rerank.RankResult {- if len(rankResults) == 0 {- return nil- }- filtered := make([]rerank.RankResult, 0, len(rankResults))- for _, r := range rankResults {- if r.RelevanceScore >= threshold {- filtered = append(filtered, r)- }- }- if len(filtered) == 0 {- top := rankResults[0]- for _, r := range rankResults[1:] {- if r.RelevanceScore > top.RelevanceScore {- top = r- }- }- if preserveTop || top.RelevanceScore >= agentRerankFallbackMinScore {- return []rerank.RankResult{top}- }- }- return filtered-}--func (t *SearchKnowledgeTool) applyModelRerankScores(- originals []*searchResultWithMeta,- rankResults []rerank.RankResult,- threshold float64,- preserveTop bool,-) []*searchResultWithMeta {- filtered := filterRerankRankResults(rankResults, threshold, preserveTop)- out := make([]*searchResultWithMeta, 0, len(filtered))- for _, rr := range filtered {- if rr.Index < 0 || rr.Index >= len(originals) {- continue- }- newResult := *originals[rr.Index]- baseScore := newResult.Score- modelScore := rr.RelevanceScore- newResult.Score = t.compositeScore(&newResult, modelScore, baseScore)- out = append(out, &newResult)- }- sort.Slice(out, func(i, j int) bool {- return out[i].Score > out[j].Score- })- return out+ return reranking.DefaultThreshold}// deduplicateResults removes duplicate chunks, keeping the first occurrence.@@ -973,8 +889,9 @@ func (t *SearchKnowledgeTool) formatOutput(return &types.ToolResult{Success: true, Output: ob.String(), Data: data}}-// getEnrichedPassage merges chunk content with image captions / OCR text so-// the reranker and MMR see the same evidence the model will.+// getEnrichedPassage merges the raw chunk content with image captions / OCR+// text. The pattern filter matches against it, so unlike the rerank passage+// it keeps markdown, URLs and other literal text a grep may target.func (t *SearchKnowledgeTool) getEnrichedPassage(ctx context.Context, result *types.SearchResult) string {if result.ImageInfo == "" {return result.Content@@ -1006,90 +923,19 @@ func (t *SearchKnowledgeTool) getEnrichedPassage(ctx context.Context, result *tyreturn combinedText + strings.Join(imageTexts, "\n")}-// compositeScore calculates a composite score considering multiple factors-func (t *SearchKnowledgeTool) compositeScore(- result *searchResultWithMeta,- modelScore, baseScore float64,-) float64 {- // Source weight: web_search results get slightly lower weight- sourceWeight := 1.0- if strings.ToLower(result.KnowledgeSource) == "web_search" {- sourceWeight = 0.95- }-- // Position prior: slightly favor chunks earlier in the document- positionPrior := 1.0- if result.StartAt >= 0 && result.EndAt > result.StartAt {- positionRatio := 1.0 - float64(result.StartAt)/float64(result.EndAt+1)- positionPrior += t.clampFloat(positionRatio, -0.05, 0.05)+// selectMMR reduces redundancy among results through the shared MMR stage.+func selectMMR(ctx context.Context, results []*searchResultWithMeta, k int) []*searchResultWithMeta {+ rows := make([]*types.SearchResult, len(results))+ for i, r := range results {+ rows[i] = r.SearchResult}-- composite := 0.6*modelScore + 0.3*baseScore + 0.1*sourceWeight- composite *= positionPrior- if composite < 0 {- composite = 0- }- if composite > 1 {- composite = 1- }- return composite-}--func (t *SearchKnowledgeTool) clampFloat(v, minV, maxV float64) float64 {- return searchutil.ClampFloat(v, minV, maxV)-}--// applyMMR applies Maximal Marginal Relevance to reduce redundancy.-func (t *SearchKnowledgeTool) applyMMR(- ctx context.Context,- results []*searchResultWithMeta,- k int,- lambda float64,-) []*searchResultWithMeta {- if k <= 0 || len(results) == 0 {+ picks := reranking.SelectMMR(ctx, rows, k, reranking.DefaultMMRLambda)+ if picks == nil {return nil}-- selected := make([]*searchResultWithMeta, 0, k)- candidates := make([]*searchResultWithMeta, len(results))- copy(candidates, results)-- tokenSets := make([]map[string]struct{}, len(candidates))- for i, r := range candidates {- tokenSets[i] = t.tokenizeSimple(t.getEnrichedPassage(ctx, r.SearchResult))- }-- // Incremental form: maxRedundancy[i] caches candidate i's maximum jaccard- // against everything selected so far, so each round only needs one- // comparison per remaining candidate. Selection output is identical to- // the naive form, including tie-breaking.- maxRedundancy := make([]float64, len(candidates))- for len(selected) < k && len(candidates) > 0 {- bestIdx := 0- bestScore := -1.0- for i, r := range candidates {- mmr := lambda*r.Score - (1.0-lambda)*maxRedundancy[i]- if mmr > bestScore {- bestScore = mmr- bestIdx = i- }- }- selected = append(selected, candidates[bestIdx])- chosenTokens := tokenSets[bestIdx]- candidates = append(candidates[:bestIdx], candidates[bestIdx+1:]...)- tokenSets = append(tokenSets[:bestIdx], tokenSets[bestIdx+1:]...)- maxRedundancy = append(maxRedundancy[:bestIdx], maxRedundancy[bestIdx+1:]...)- for i := range candidates {- maxRedundancy[i] = math.Max(maxRedundancy[i], t.jaccard(tokenSets[i], chosenTokens))- }+ selected := make([]*searchResultWithMeta, len(picks))+ for i, p := range picks {+ selected[i] = results[p]}return selected}--func (t *SearchKnowledgeTool) tokenizeSimple(text string) map[string]struct{} {- return searchutil.TokenizeSimple(text)-}--func (t *SearchKnowledgeTool) jaccard(a, b map[string]struct{}) float64 {- return searchutil.Jaccard(a, b)-}diff --git a/internal/agent/tools/search_knowledge_mmr_test.go b/internal/agent/tools/search_knowledge_mmr_test.godeleted file mode 100644index 8491d7167..000000000--- a/internal/agent/tools/search_knowledge_mmr_test.go+++ /dev/null@@ -1,165 +0,0 @@-package tools--import (- "context"- "fmt"- "math"- "testing"-- "github.com/Tencent/WeKnora/internal/types"-)--// applyMMRNaive is the pre-optimization form of applyMMR: it recomputes the-// redundancy of every candidate against every already-selected result on each-// round, re-tokenizing the selected passages every time. It is kept here purely-// as the reference oracle for the incremental implementation.-func (t *SearchKnowledgeTool) applyMMRNaive(- ctx context.Context,- results []*searchResultWithMeta,- k int,- lambda float64,-) []*searchResultWithMeta {- if k <= 0 || len(results) == 0 {- return nil- }-- selected := make([]*searchResultWithMeta, 0, k)- candidates := make([]*searchResultWithMeta, len(results))- copy(candidates, results)-- tokenSets := make([]map[string]struct{}, len(candidates))- for i, r := range candidates {- tokenSets[i] = t.tokenizeSimple(t.getEnrichedPassage(ctx, r.SearchResult))- }-- for len(selected) < k && len(candidates) > 0 {- bestIdx := 0- bestScore := -1.0-- for i, r := range candidates {- relevance := r.Score- redundancy := 0.0- for _, s := range selected {- selectedTokens := t.tokenizeSimple(t.getEnrichedPassage(ctx, s.SearchResult))- redundancy = math.Max(redundancy, t.jaccard(tokenSets[i], selectedTokens))- }- mmr := lambda*relevance - (1.0-lambda)*redundancy- if mmr > bestScore {- bestScore = mmr- bestIdx = i- }- }-- selected = append(selected, candidates[bestIdx])- candidates = append(candidates[:bestIdx], candidates[bestIdx+1:]...)- tokenSets = append(tokenSets[:bestIdx], tokenSets[bestIdx+1:]...)- }-- return selected-}--// mmrTestCorpus builds a deterministic candidate set whose passages share-// vocabulary in overlapping bands, so redundancy actually drives selection-// instead of the score alone.-func mmrTestCorpus(n int) []*searchResultWithMeta {- vocab := []string{- "insurance", "policy", "claim", "premium", "deductible", "liability",- "coverage", "endorsement", "underwriting", "reinsurance", "subrogation",- "indemnity", "exclusion", "rider", "annuity",- }-- results := make([]*searchResultWithMeta, 0, n)- // Simple LCG so the corpus is identical on every run and every platform.- state := uint64(42)- next := func(mod int) int {- state = state*6364136223846793005 + 1442695040888963407- return int((state >> 33) % uint64(mod))- }-- for i := 0; i < n; i++ {- words := make([]byte, 0, 128)- for j := 0; j < 12; j++ {- words = append(words, vocab[next(len(vocab))]...)- words = append(words, ' ')- }- content := fmt.Sprintf("chunk %d %s", i, string(words))- results = append(results, &searchResultWithMeta{- SearchResult: &types.SearchResult{- ID: fmt.Sprintf("chunk-%03d", i),- Content: content,- KnowledgeID: fmt.Sprintf("doc-%d", i%7),- // Scores intentionally collide across candidates so the- // tie-breaking path is exercised too.- Score: float64(next(20)) / 20.0,- },- QueryType: "vector",- KnowledgeBaseID: "kb-1",- })- }- return results-}--func TestApplyMMR_matchesNaiveSelection(t *testing.T) {- t.Parallel()-- tool := &SearchKnowledgeTool{}- ctx := context.Background()- results := mmrTestCorpus(40)-- for _, tc := range []struct {- k int- lambda float64- }{- {k: 1, lambda: 0.7},- {k: 5, lambda: 0.7},- {k: 12, lambda: 0.3},- {k: 40, lambda: 0.9},- {k: 60, lambda: 0.5}, // k larger than the candidate count- } {- tc := tc- t.Run(fmt.Sprintf("k=%d/lambda=%.1f", tc.k, tc.lambda), func(t *testing.T) {- want := tool.applyMMRNaive(ctx, results, tc.k, tc.lambda)- got := tool.applyMMR(ctx, results, tc.k, tc.lambda)-- if len(got) != len(want) {- t.Fatalf("selected %d results, naive selected %d", len(got), len(want))- }- for i := range want {- if got[i].ID != want[i].ID {- t.Fatalf("rank %d: got %s, naive got %s", i, got[i].ID, want[i].ID)- }- }- })- }-}--func TestApplyMMR_emptyAndNonPositiveK(t *testing.T) {- t.Parallel()-- tool := &SearchKnowledgeTool{}- ctx := context.Background()-- if got := tool.applyMMR(ctx, mmrTestCorpus(3), 0, 0.7); got != nil {- t.Fatalf("expected nil for k=0, got %d results", len(got))- }- if got := tool.applyMMR(ctx, nil, 5, 0.7); got != nil {- t.Fatalf("expected nil for empty candidates, got %d results", len(got))- }-}--func BenchmarkApplyMMR(b *testing.B) {- tool := &SearchKnowledgeTool{}- ctx := context.Background()- results := mmrTestCorpus(250)-- b.Run("incremental", func(b *testing.B) {- for i := 0; i < b.N; i++ {- tool.applyMMR(ctx, results, 250, 0.7)- }- })- b.Run("naive", func(b *testing.B) {- for i := 0; i < b.N; i++ {- tool.applyMMRNaive(ctx, results, 250, 0.7)- }- })-}diff --git a/internal/agent/tools/search_knowledge_rerank_test.go b/internal/agent/tools/search_knowledge_rerank_test.goindex 3031643ee..9ce9f971b 100644--- a/internal/agent/tools/search_knowledge_rerank_test.go+++ b/internal/agent/tools/search_knowledge_rerank_test.go@@ -9,6 +9,7 @@ import ("github.com/Tencent/WeKnora/internal/config""github.com/Tencent/WeKnora/internal/models/rerank"+ "github.com/Tencent/WeKnora/internal/reranking""github.com/Tencent/WeKnora/internal/types")@@ -58,75 +59,6 @@ func newRerankTestResults() []*searchResultWithMeta {}}-func TestFilterRerankRankResults_thresholdAndFallback(t *testing.T) {- t.Parallel()- rankResults := []rerank.RankResult{- {Index: 0, RelevanceScore: 0.05},- {Index: 1, RelevanceScore: 0.02},- }- filtered := filterRerankRankResults(rankResults, 0.3, false)- if len(filtered) != 0 {- t.Fatalf("expected empty filter, got %#v", filtered)- }-- rankResults = []rerank.RankResult{- {Index: 0, RelevanceScore: 0.05},- {Index: 1, RelevanceScore: 0.20},- }- filtered = filterRerankRankResults(rankResults, 0.3, false)- if len(filtered) != 1 || filtered[0].Index != 1 {- t.Fatalf("expected fallback top score, got %#v", filtered)- }-- rankResults = []rerank.RankResult{- {Index: 0, RelevanceScore: 0.05},- {Index: 1, RelevanceScore: 0.02},- }- filtered = filterRerankRankResults(rankResults, 0.3, true)- if len(filtered) != 1 || filtered[0].Index != 0 {- t.Fatalf("expected explicit scope to preserve top result, got %#v", filtered)- }-- rankResults = []rerank.RankResult{- {Index: 0, RelevanceScore: 0.8},- {Index: 1, RelevanceScore: 0.4},- {Index: 2, RelevanceScore: 0.1},- }- filtered = filterRerankRankResults(rankResults, 0.3, false)- if len(filtered) != 2 {- t.Fatalf("expected 2 passing scores, got %#v", filtered)- }-}--func TestApplyModelRerankScores_faqUsesCompositeScale(t *testing.T) {- t.Parallel()- tool := &SearchKnowledgeTool{- config: &config.Config{- Conversation: &config.ConversationConfig{RerankThreshold: 0.3},- },- }- originals := []*searchResultWithMeta{- {- SearchResult: &types.SearchResult{ID: "faq-1", Content: "Q: WeKnora", Score: 0.011},- KnowledgeBaseType: types.KnowledgeBaseTypeFAQ,- },- {- SearchResult: &types.SearchResult{ID: "doc-1", Content: "swimming club", Score: 0.02},- },- }- rankResults := []rerank.RankResult{- {Index: 0, RelevanceScore: 0.05},- {Index: 1, RelevanceScore: 0.9},- }- out := tool.applyModelRerankScores(originals, rankResults, 0.3, false)- if len(out) != 1 || out[0].ID != "doc-1" {- t.Fatalf("weak FAQ should be filtered out, got %#v", out)- }- if out[0].Score <= 0.011 {- t.Fatalf("composite score should exceed raw retrieval score, got %.4f", out[0].Score)- }-}-// A rerank API failure must degrade to the raw retrieval order rather than// dropping the recall set or re-scoring it with a chat model.func TestRerankResults_modelErrorKeepsRawResults(t *testing.T) {@@ -194,51 +126,6 @@ func TestRerankResults_withoutModelIsPassthrough(t *testing.T) {}}-// A chunk rarely names its document's subject, so the rerank model must see-// the document title; FAQ entries are scored on their own question.-func TestRerankScores_prefixDocumentTitle(t *testing.T) {- t.Parallel()- model := &stubReranker{scores: []float64{0.5, 0.5, 0.5}}- tool := newRerankTestTool(model)- results := []*searchResultWithMeta{- {SearchResult: &types.SearchResult{- ID: "c1", Content: "allocate inference across open-weight models",- KnowledgeTitle: " Show HN: Echo ", ChunkType: string(types.ChunkTypeText),- }},- {SearchResult: &types.SearchResult{- ID: "c2", Content: "What is Echo?", KnowledgeTitle: "FAQ set", ChunkType: string(types.ChunkTypeFAQ),- }},- {SearchResult: &types.SearchResult{ID: "c3", Content: "untitled"}},- }-- if _, err := tool.rerankScores(context.Background(), "query", results); err != nil {- t.Fatalf("rerankScores returned error: %v", err)- }- want := []string{"Show HN: Echo\n\nallocate inference across open-weight models", "What is Echo?", "untitled"}- for i, w := range want {- if model.documents[i] != w {- t.Fatalf("passage %d = %q, want %q", i, model.documents[i], w)- }- }-}--// Title is shared by every chunk of a document, so a passage whose body never-// names the query subject still goes to the reranker as title + chunk.-func TestRerankPassage_prefixesTitleWhenChunkOmitsSubject(t *testing.T) {- t.Parallel()- got := (&SearchKnowledgeTool{}).rerankPassage(context.Background(), &types.SearchResult{- Content: "installation prerequisites and docker compose flags",- KnowledgeTitle: "Show HN: Echo – Fable-level results at 1/3 the cost using open-weight models",- ChunkType: string(types.ChunkTypeText),- })- if !strings.HasPrefix(got, "Show HN: Echo") {- t.Fatalf("expected document title prefix, got %q", got)- }- if !strings.Contains(got, "installation prerequisites") {- t.Fatalf("chunk body must still be present: %q", got)- }-}-// Execute must surface rerank_rejected through Data and the empty statement// when the model scores every retrieved candidate below the fallback floor.func TestExecuteReportsRerankRejection(t *testing.T) {@@ -286,7 +173,7 @@ func TestExecuteReportsRerankRejection(t *testing.T) {func TestRerankThreshold_default(t *testing.T) {t.Parallel()tool := &SearchKnowledgeTool{}- if got := tool.rerankThreshold(); got != 0.3 {- t.Fatalf("default threshold = %v, want 0.3", got)+ if got := tool.rerankThreshold(); got != reranking.DefaultThreshold {+ t.Fatalf("default threshold = %v, want %v", got, reranking.DefaultThreshold)}}diff --git a/internal/application/service/chat_pipeline/merge_history.go b/internal/application/service/chat_pipeline/merge_history.goindex 33d81b00a..8341028a5 100644--- a/internal/application/service/chat_pipeline/merge_history.go+++ b/internal/application/service/chat_pipeline/merge_history.go@@ -66,7 +66,9 @@ func filterHistoryResults(}r.MatchType = types.MatchTypeHistoryr.Score = r.Score * historyScoreDiscount- r.Metadata = ensureMetadata(r.Metadata)+ if r.Metadata == nil {+ r.Metadata = make(map[string]string)+ }r.Metadata["history_similarity"] = strings.TrimRight(strings.TrimRight(fmt.Sprintf("%.4f", sim), "0"), ".")filtered = append(filtered, r)diff --git a/internal/application/service/chat_pipeline/query_expansion.go b/internal/application/service/chat_pipeline/query_expansion.goindex 3324a9853..f5f1a6420 100644--- a/internal/application/service/chat_pipeline/query_expansion.go+++ b/internal/application/service/chat_pipeline/query_expansion.go@@ -67,8 +67,8 @@ func (p *PluginSearch) runQueryExpansion(ctx context.Context, chatManage *types.MatchCount: expTopK,TagIDs: t.TagIDs,ScopeTagIDs: t.ScopeTagIDs,- DisableVectorMatch: false,- DisableKeywordsMatch: false,+ DisableVectorMatch: chatManage.DisableVectorMatch,+ DisableKeywordsMatch: chatManage.DisableKeywordsMatch,SkipContextEnrichment: true, // Pipeline handles context assembly in merge stage}// Apply knowledge ID filter if this is a partial KB searchdiff --git a/internal/application/service/chat_pipeline/rerank.go b/internal/application/service/chat_pipeline/rerank.goindex 659a0c659..feca29bb0 100644--- a/internal/application/service/chat_pipeline/rerank.go+++ b/internal/application/service/chat_pipeline/rerank.go@@ -2,14 +2,9 @@ package chatpipelineimport ("context"- "encoding/json""fmt"- "math"- "regexp"- "strings"- "github.com/Tencent/WeKnora/internal/models/rerank"- "github.com/Tencent/WeKnora/internal/searchutil"+ "github.com/Tencent/WeKnora/internal/reranking""github.com/Tencent/WeKnora/internal/tracing/langfuse""github.com/Tencent/WeKnora/internal/types""github.com/Tencent/WeKnora/internal/types/interfaces"@@ -48,13 +43,23 @@ func (p *PluginRerank) OnEvent(ctx context.Context,"rerank_thresh": chatManage.RerankThreshold,"rewrite_query": chatManage.RewriteQuery,})+ diag := &types.RerankDiagnostics{+ ModelID: chatManage.RerankModelID,+ Threshold: chatManage.RerankThreshold,+ EffectiveThreshold: chatManage.RerankThreshold,+ CandidateCount: len(chatManage.SearchResult),+ ResultCount: len(chatManage.SearchResult),+ }+ chatManage.RerankDiagnostics = diagif len(chatManage.SearchResult) == 0 {+ diag.Outcome = types.RerankOutcomeNoCandidatespipelineInfo(ctx, "Rerank", "skip", map[string]interface{}{"reason": "empty_search_result",})return next()}if chatManage.RerankModelID == "" {+ diag.Outcome = types.RerankOutcomeNoModelpipelineWarn(ctx, "Rerank", "skip", map[string]interface{}{"reason": "empty_model_id",})@@ -64,6 +69,8 @@ func (p *PluginRerank) OnEvent(ctx context.Context,// Get rerank model from servicererankModel, err := p.modelService.GetRerankModel(ctx, chatManage.RerankModelID)if err != nil {+ diag.Outcome = types.RerankOutcomeModelUnavailable+ diag.Error = err.Error()pipelineError(ctx, "Rerank", "get_model", map[string]interface{}{"model_id": chatManage.RerankModelID,"error": err.Error(),@@ -71,209 +78,82 @@ func (p *PluginRerank) OnEvent(ctx context.Context,return ErrGetRerankModel.WithError(err)}- // Prepare passages for reranking.- var passages []string- var candidatesToRerank []*types.SearchResult-- for _, result := range chatManage.SearchResult {- passage := getEnrichedPassage(ctx, result)- if strings.TrimSpace(passage) == "" {- pipelineInfo(ctx, "Rerank", "empty_passage_skip", map[string]interface{}{- "chunk_id": result.ID,- })- continue- }- passages = append(passages, passage)- candidatesToRerank = append(candidatesToRerank, result)- }-- passagesPreview := langfuse.SummarizePassagePreviews(candidatesToRerank, passages, 25)rerankCtx, rerankSpan := langfuse.GetManager().StartSpan(ctx, langfuse.SpanOptions{Name: "rerank",Input: map[string]interface{}{- "query": chatManage.RewriteQuery,- "candidate_count": len(candidatesToRerank),- "rerank_model_id": chatManage.RerankModelID,- "threshold": chatManage.RerankThreshold,- "rerank_top_k": chatManage.RerankTopK,- "faq_priority": chatManage.FAQPriorityEnabled,- "faq_score_boost": chatManage.FAQScoreBoost,- "passages_preview": passagesPreview,+ "query": chatManage.RewriteQuery,+ "candidate_count": len(chatManage.SearchResult),+ "rerank_model_id": chatManage.RerankModelID,+ "threshold": chatManage.RerankThreshold,+ "rerank_top_k": chatManage.RerankTopK,+ "faq_priority": chatManage.FAQPriorityEnabled,+ "faq_score_boost": chatManage.FAQScoreBoost,},Metadata: map[string]interface{}{"session_id": chatManage.SessionID,},})ctx = rerankCtx- spanOutput := map[string]interface{}{}- var spanErr error- defer func() {- rerankSpan.Finish(spanOutput, nil, spanErr)- }()-- pipelineInfo(ctx, "Rerank", "build_passages", map[string]interface{}{- "total_cnt": len(chatManage.SearchResult),- "candidate_cnt": len(candidatesToRerank),- })-- var rerankResp []rerank.RankResult- var rawRerankResp []rerank.RankResult- thresholdDegraded := false-- // Only call rerank model if there are candidates- if len(candidatesToRerank) > 0 {- // Single rerank call with RewriteQuery, use threshold degradation if no results- originalThreshold := chatManage.RerankThreshold- var rerankErr error- rerankResp, rerankErr = p.rerank(ctx, chatManage, rerankModel, chatManage.RewriteQuery, passages, candidatesToRerank)-- if rerankErr != nil {- // Rerank API failed — fallback to original retrieval results so the- // pipeline can still return something useful to the caller.- pipelineWarn(ctx, "Rerank", "api_error_fallback", map[string]interface{}{- "error": rerankErr.Error(),- "candidate_cnt": len(candidatesToRerank),- })- chatManage.SearchResult = candidatesToRerank- spanOutput = map[string]interface{}{- "stage": "api_error_fallback",- "candidate_count": len(candidatesToRerank),- "error": rerankErr.Error(),- }- return next()- }- rawRerankResp = append([]rerank.RankResult(nil), rerankResp...)-- // If no results and threshold is high enough, try with lower threshold- if len(rerankResp) == 0 && originalThreshold > 0.3 {- thresholdDegraded = true- degradedThreshold := originalThreshold * 0.7- if degradedThreshold < 0.3 {- degradedThreshold = 0.3- }- pipelineWarn(ctx, "Rerank", "threshold_degrade", map[string]interface{}{- "original": originalThreshold,- "degraded": degradedThreshold,- "candidate_cnt": len(candidatesToRerank),- "reason": "no results above original threshold, retrying with lower threshold",- })- chatManage.RerankThreshold = degradedThreshold- rerankResp, rerankErr = p.rerank(ctx, chatManage, rerankModel, chatManage.RewriteQuery, passages, candidatesToRerank)- // Restore original threshold- chatManage.RerankThreshold = originalThreshold- if rerankErr != nil {- pipelineWarn(ctx, "Rerank", "api_error_fallback", map[string]interface{}{- "error": rerankErr.Error(),- "candidate_cnt": len(candidatesToRerank),- })- chatManage.SearchResult = candidatesToRerank- spanOutput = map[string]interface{}{- "stage": "api_error_fallback",- "candidate_count": len(candidatesToRerank),- "threshold_degraded": thresholdDegraded,- "error": rerankErr.Error(),- }- return next()- }- rawRerankResp = append([]rerank.RankResult(nil), rerankResp...)- }- }-- pipelineInfo(ctx, "Rerank", "model_response", map[string]interface{}{- "result_cnt": len(rerankResp),- })logRerankInputScoreSample(ctx, chatManage.SearchResult)- for i := range chatManage.SearchResult {- chatManage.SearchResult[i].Metadata = ensureMetadata(chatManage.SearchResult[i].Metadata)+ opts := reranking.Options{+ Threshold: chatManage.RerankThreshold,+ TopK: max(1, chatManage.RerankTopK),+ FallbackMinScore: reranking.FallbackMinScore(chatManage.SearchTargets.HasRecallThresholdOverride()),}- reranked := make([]*types.SearchResult, 0, len(rerankResp))-- // Process reranked results- for _, rr := range rerankResp {- if rr.Index >= len(candidatesToRerank) {- continue- }- sr := candidatesToRerank[rr.Index]- base := sr.Score- sr.Metadata["base_score"] = fmt.Sprintf("%.4f", base)- modelScore := rr.RelevanceScore- sr.Metadata["model_score"] = fmt.Sprintf("%.4f", modelScore)- sr.Score = compositeScore(sr, modelScore, base)-- // Apply FAQ score boost if enabled- if chatManage.FAQPriorityEnabled && chatManage.FAQScoreBoost > 1.0 &&- sr.ChunkType == string(types.ChunkTypeFAQ) {- originalScore := sr.Score- sr.Score = math.Min(sr.Score*chatManage.FAQScoreBoost, 1.0)- sr.Metadata["faq_boosted"] = "true"- sr.Metadata["faq_original_score"] = fmt.Sprintf("%.4f", originalScore)- pipelineInfo(ctx, "Rerank", "faq_boost", map[string]interface{}{- "chunk_id": sr.ID,- "original_score": fmt.Sprintf("%.4f", originalScore),- "boosted_score": fmt.Sprintf("%.4f", sr.Score),- "boost_factor": chatManage.FAQScoreBoost,- })- }-- reranked = append(reranked, sr)+ if chatManage.FAQPriorityEnabled {+ opts.FAQScoreBoost = chatManage.FAQScoreBoost}+ res := reranking.Rerank(ctx, rerankModel, chatManage.RewriteQuery, chatManage.SearchResult, opts)+ *diag = res.Diagnostics+ diag.ModelID = chatManage.RerankModelID- final := applyMMR(ctx, reranked, chatManage, min(len(reranked), max(1, chatManage.RerankTopK)), 0.7)- chatManage.RerankResult = final+ if diag.Outcome == types.RerankOutcomeModelError {+ // Rerank API failed — fall back to the retrieval results so the+ // pipeline can still return something useful to the caller.+ pipelineWarn(ctx, "Rerank", "api_error_fallback", map[string]interface{}{+ "error": diag.Error,+ "candidate_cnt": diag.CandidateCount,+ })+ chatManage.SearchResult = res.Results+ rerankSpan.Finish(map[string]interface{}{+ "stage": "api_error_fallback",+ "candidate_count": diag.CandidateCount,+ "error": diag.Error,+ }, nil, nil)+ return next()+ }- // Log composite top scores and MMR selection summary- topN := min(3, len(reranked))- for i := 0; i < topN; i++ {+ chatManage.RerankResult = res.Results+ for i := 0; i < min(3, len(res.Scored)); i++ {pipelineInfo(ctx, "Rerank", "composite_top", map[string]interface{}{"rank": i + 1,- "chunk_id": reranked[i].ID,- "base_score": reranked[i].Metadata["base_score"],- "final_score": fmt.Sprintf("%.4f", reranked[i].Score),+ "chunk_id": res.Scored[i].ID,+ "base_score": res.Scored[i].Metadata["base_score"],+ "final_score": fmt.Sprintf("%.4f", res.Scored[i].Score),})}+ rerankSpan.Finish(buildRerankSpanOutput(res, chatManage), nil, nil)if len(chatManage.RerankResult) == 0 {pipelineWarn(ctx, "Rerank", "output", map[string]interface{}{"filtered_cnt": 0,+ "outcome": diag.Outcome,+ "top_score": diag.TopScore,+ "threshold": diag.EffectiveThreshold,})- spanOutput = buildRerankSpanOutput(- candidatesToRerank,- passages,- rawRerankResp,- reranked,- nil,- chatManage,- thresholdDegraded,- )return ErrSearchNothing}-- spanOutput = buildRerankSpanOutput(- candidatesToRerank,- passages,- rawRerankResp,- reranked,- chatManage.RerankResult,- chatManage,- thresholdDegraded,- )pipelineInfo(ctx, "Rerank", "output", map[string]interface{}{"filtered_cnt": len(chatManage.RerankResult),+ "outcome": diag.Outcome,})return next()}-func buildRerankSpanOutput(- candidates []*types.SearchResult,- passages []string,- modelScores []rerank.RankResult,- composite []*types.SearchResult,- final []*types.SearchResult,- chatManage *types.ChatManage,- thresholdDegraded bool,-) map[string]interface{} {+func buildRerankSpanOutput(res *reranking.Result, chatManage *types.ChatManage) map[string]interface{} {+ candidates, passages, modelScores := res.Candidates, res.Passages, res.ModelScoresmodelRows := make([]map[string]interface{}, 0, len(modelScores))for i, rr := range modelScores {row := map[string]interface{}{@@ -295,16 +175,19 @@ func buildRerankSpanOutput(}out := map[string]interface{}{- "candidate_count": len(candidates),- "model_result_count": len(modelScores),- "composite_count": len(composite),- "final_count": len(final),- "threshold": chatManage.RerankThreshold,- "rerank_top_k": chatManage.RerankTopK,- "threshold_degraded": thresholdDegraded,- "model_scores": langfuse.SummarizeRankScores(modelRows, 50),- "composite_results": langfuse.SummarizeSearchResults(composite, 25),- "final_results": langfuse.SummarizeSearchResults(final, 25),+ "candidate_count": len(candidates),+ "model_result_count": len(modelScores),+ "composite_count": len(res.Scored),+ "final_count": len(res.Results),+ "threshold": chatManage.RerankThreshold,+ "effective_threshold": res.Diagnostics.EffectiveThreshold,+ "rerank_top_k": chatManage.RerankTopK,+ "outcome": res.Diagnostics.Outcome,+ "threshold_degraded": res.Diagnostics.Outcome == types.RerankOutcomeThresholdDegraded,+ "passages_preview": langfuse.SummarizePassagePreviews(candidates, passages, 25),+ "model_scores": langfuse.SummarizeRankScores(modelRows, 50),+ "composite_results": langfuse.SummarizeSearchResults(res.Scored, 25),+ "final_results": langfuse.SummarizeSearchResults(res.Results, 25),}if len(modelScores) > 50 {out["model_scores_truncated"] = len(modelScores) - 50@@ -312,387 +195,6 @@ func buildRerankSpanOutput(return out}-// rerank performs the actual reranking operation with given query and passages-func (p *PluginRerank) rerank(ctx context.Context,- chatManage *types.ChatManage, rerankModel rerank.Reranker, query string, passages []string,- candidates []*types.SearchResult,-) ([]rerank.RankResult, error) {- pipelineInfo(ctx, "Rerank", "model_call", map[string]interface{}{- "query_variant": query,- "passages": len(passages),- })-- // Filter out empty or whitespace-only passages before sending to the API- var cleanPassages []string- var cleanCandidates []*types.SearchResult- for i, p := range passages {- if strings.TrimSpace(p) != "" {- cleanPassages = append(cleanPassages, p)- if i < len(candidates) {- cleanCandidates = append(cleanCandidates, candidates[i])- }- }- }- if len(cleanPassages) == 0 {- pipelineInfo(ctx, "Rerank", "model_call_skip", map[string]interface{}{- "reason": "all_passages_empty",- })- return nil, nil- }- passages = cleanPassages- candidates = cleanCandidates-- rerankResp, err := rerankModel.Rerank(ctx, query, passages)- if err != nil {- pipelineError(ctx, "Rerank", "model_call", map[string]interface{}{- "query_variant": query,- "error": err.Error(),- })- return nil, err- }-- // Log top scores for debugging- pipelineInfo(ctx, "Rerank", "threshold", map[string]interface{}{- "threshold": chatManage.RerankThreshold,- })- logged := min(5, len(rerankResp))- for i := range logged {- if rerankResp[i].Index < len(candidates) {- pipelineInfo(ctx, "Rerank", "top_score", map[string]interface{}{- "rank": i + 1,- "score": rerankResp[i].RelevanceScore,- "chunk_id": candidates[rerankResp[i].Index].ID,- "match_type": candidates[rerankResp[i].Index].MatchType,- "chunk_type": candidates[rerankResp[i].Index].ChunkType,- "content_len": len(candidates[rerankResp[i].Index].Content),- })- }- }- if len(rerankResp) > logged {- pipelineInfo(ctx, "Rerank", "top_score_summary", map[string]interface{}{- "total": len(rerankResp),- "logged": logged,- "truncated": len(rerankResp) - logged,- })- }-- // Filter results based on threshold- rankFilter := []rerank.RankResult{}- for _, result := range rerankResp {- if result.Index >= len(candidates) {- continue- }- if result.RelevanceScore >= chatManage.RerankThreshold {- rankFilter = append(rankFilter, result)- }- }-- // Fallback: if threshold filtering removed all results but the top candidate- // still has a reasonable score, keep it as a safety net. Skip fallback entirely- // when the best score is too low — forcing irrelevant results is worse than- // returning nothing and letting the caller handle the empty-result case.- fallbackMinScore := rerankFallbackMinScore(chatManage.SearchTargets)- if len(rankFilter) == 0 && len(rerankResp) > 0 && rerankResp[0].RelevanceScore >= fallbackMinScore {- rankFilter = rerankResp[:1]- pipelineInfo(ctx, "Rerank", "fallback_top1", map[string]interface{}{- "reason": "all_below_threshold",- "threshold": chatManage.RerankThreshold,- "top_score": rerankResp[0].RelevanceScore,- })- } else if len(rankFilter) == 0 {- pipelineInfo(ctx, "Rerank", "fallback_skip", map[string]interface{}{- "reason": "top_score_too_low",- "threshold": chatManage.RerankThreshold,- "top_score": safeTopScore(rerankResp),- })- }-- return rankFilter, nil-}--func rerankFallbackMinScore(searchTargets types.SearchTargets) float64 {- if searchTargets.HasRecallThresholdOverride() {- // The user explicitly constrained this turn to a tag/document scope.- // Preserve its best candidate instead of letting a global rerank- // threshold erase the entire authoritative scope.- return 0- }- return 0.15-}--// ensureMetadata ensures the metadata is not nil-func ensureMetadata(m map[string]string) map[string]string {- if m == nil {- return make(map[string]string)- }- return m-}--func safeTopScore(results []rerank.RankResult) float64 {- if len(results) == 0 {- return 0- }- return results[0].RelevanceScore-}--// compositeScore calculates the composite score for a search result-func compositeScore(sr *types.SearchResult, modelScore, baseScore float64) float64 {- sourceWeight := 1.0- switch strings.ToLower(sr.KnowledgeSource) {- case "web_search":- sourceWeight = 0.95- default:- sourceWeight = 1.0- }- composite := 0.6*modelScore + 0.3*baseScore + 0.1*sourceWeight- if composite < 0 {- composite = 0- }- if composite > 1 {- composite = 1- }- return composite-}--// applyMMR applies the MMR algorithm to the search results with pre-computed token sets-func applyMMR(- ctx context.Context,- results []*types.SearchResult,- chatManage *types.ChatManage,- k int,- lambda float64,-) []*types.SearchResult {- if k <= 0 || len(results) == 0 {- return nil- }- pipelineInfo(ctx, "Rerank", "mmr_start", map[string]interface{}{- "lambda": lambda,- "k": k,- "candidates": len(results),- })-- // Pre-compute all token sets concurrently (CPU-bound tokenization)- allTokenSets := ParallelMap(results, 0, func(i int, r *types.SearchResult) map[string]struct{} {- return searchutil.TokenizeSimple(getEnrichedPassage(ctx, r))- })-- selected := make([]*types.SearchResult, 0, k)- selectedTokenSets := make([]map[string]struct{}, 0, k)- selectedIndices := make(map[int]struct{})-- for len(selected) < k && len(selectedIndices) < len(results) {- bestIdx := -1- bestScore := -1.0-- for i, r := range results {- if _, isSelected := selectedIndices[i]; isSelected {- continue- }-- relevance := r.Score- redundancy := 0.0-- // Use pre-computed token sets for redundancy calculation- for _, selTokens := range selectedTokenSets {- sim := searchutil.Jaccard(allTokenSets[i], selTokens)- if sim > redundancy {- redundancy = sim- }- }-- mmr := lambda*relevance - (1.0-lambda)*redundancy- if mmr > bestScore {- bestScore = mmr- bestIdx = i- }- }-- if bestIdx < 0 {- break- }-- selected = append(selected, results[bestIdx])- selectedTokenSets = append(selectedTokenSets, allTokenSets[bestIdx])- selectedIndices[bestIdx] = struct{}{}- }-- // Compute average redundancy among selected using pre-computed token sets- avgRed := 0.0- if len(selected) > 1 {- pairs := 0- for i := 0; i < len(selectedTokenSets); i++ {- for j := i + 1; j < len(selectedTokenSets); j++ {- avgRed += searchutil.Jaccard(selectedTokenSets[i], selectedTokenSets[j])- pairs++- }- }- if pairs > 0 {- avgRed /= float64(pairs)- }- }- pipelineInfo(ctx, "Rerank", "mmr_done", map[string]interface{}{- "selected": len(selected),- "avg_redundancy": fmt.Sprintf("%.4f", avgRed),- })- return selected-}--// --- Passage cleaning for rerank ----//-// Rerank models work on semantic text similarity. Markdown formatting, raw URLs,-// image references, table separators, and other structural syntax are noise that-// can dilute the semantic signal. The functions below strip this noise before-// passages are sent to the rerank model.--var (- // reMarkdownImage matches  — the entire construct is noise.- // URL group supports one level of balanced parentheses.- reMarkdownImage = regexp.MustCompile(`!\[[^\]]*\]\([^()\s]*(?:\([^)]*\)[^()\s]*)*\)`)- // reLinkedImage matches [](link_url) — unwrap to - // so that the subsequent reMarkdownImage pass can remove the image.- reLinkedImage = regexp.MustCompile(- `\[!\[([^\]]*)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)\]` +- `\([^()\s]*(?:\([^)]*\)[^()\s]*)*\)`,- )- // reMarkdownLink matches [text](url) — we keep the text, drop the URL.- // URL group supports one level of balanced parentheses.- reMarkdownLink = regexp.MustCompile(`\[([^\]]+)\]\([^()\s]*(?:\([^)]*\)[^()\s]*)*\)`)- // reRawURL matches standalone http(s) URLs.- reRawURL = regexp.MustCompile(`https?://[^\s)\]>]+`)- // reCodeBlock captures the semantic body of fenced code blocks.- reCodeBlock = regexp.MustCompile("(?s)```[^\\r\\n]*\\r?\\n(.*?)\\r?\\n?```")- // reLatexBlock captures the semantic body of block-level LaTeX ($$...$$).- reLatexBlock = regexp.MustCompile(`(?s)\$\$(.*?)\$\$`)- // reTableSep matches table separator rows like |---|---|.- // Uses [ \t] instead of \s to avoid consuming newlines across rows.- reTableSep = regexp.MustCompile(`(?m)^[ \t]*\|[ \t:|-]+\|[ \t]*$`)- // reTableRow matches markdown table data rows like | col1 | col2 |.- // Uses [ \t] instead of \s to avoid consuming newlines across rows.- reTableRow = regexp.MustCompile(`(?m)^[ \t]*\|(.+?)\|[ \t]*$`)- // reHeadingPrefix matches leading # markers in headings.- reHeadingPrefix = regexp.MustCompile(`(?m)^#{1,6}\s+`)- // reBlockquote matches leading > markers.- reBlockquote = regexp.MustCompile(`(?m)^>\s?`)- // reBoldItalic3 matches ***text*** wrappers (must come before 2 and 1).- reBoldItalic3 = regexp.MustCompile(`\*{3}(.+?)\*{3}`)- // reBoldItalic2 matches **text** wrappers.- reBoldItalic2 = regexp.MustCompile(`\*{2}(.+?)\*{2}`)- // reBoldItalic1 matches *text* wrappers.- reBoldItalic1 = regexp.MustCompile(`\*(.+?)\*`)- // reExcessiveNewlines collapses 3+ consecutive newlines into 2.- reExcessiveNewlines = regexp.MustCompile(`\n{3,}`)- // reListMarker matches unordered (- , * ) and ordered (1. ) list prefixes.- reListMarker = regexp.MustCompile(`(?m)^[\t ]*(?:[-*+]|\d+\.)\s+`)- // reHTMLTag matches HTML tags like <br>, <div class="...">, etc.- reHTMLTag = regexp.MustCompile(`</?[a-zA-Z][^>]*>`)-)--// cleanPassageForRerank strips markdown/structural noise from text to produce-// a clean semantic passage for the rerank model. The cleaning is designed to-// preserve all meaningful semantic content while removing formatting-// that would confuse text-similarity scoring.-func cleanPassageForRerank(text string) string {- // 1. Unwrap code blocks so code-only candidates remain rerankable.- text = reCodeBlock.ReplaceAllString(text, "$1")- // 2. Unwrap LaTeX blocks so formula-only candidates remain rerankable.- text = reLatexBlock.ReplaceAllString(text, "$1")- // 3. Remove HTML tags- text = reHTMLTag.ReplaceAllString(text, "")- // 3.5. Unwrap nested [](link_url) → - // so that the next step removes the full construct cleanly.- text = reLinkedImage.ReplaceAllString(text, "")- // 4. Remove markdown image references entirely- text = reMarkdownImage.ReplaceAllString(text, "")- // 5. Convert markdown links to just their display text- text = reMarkdownLink.ReplaceAllString(text, "$1")- // 6. Remove standalone raw URLs- text = reRawURL.ReplaceAllString(text, "")- // 7. Remove table separator rows- text = reTableSep.ReplaceAllString(text, "")- // 7.5. Convert table data rows to plain text (strip | delimiters)- text = reTableRow.ReplaceAllStringFunc(text, func(match string) string {- inner := reTableRow.FindStringSubmatch(match)- if len(inner) < 2 {- return match- }- cells := strings.Split(inner[1], "|")- var parts []string- for _, cell := range cells {- cell = strings.TrimSpace(cell)- if cell != "" {- parts = append(parts, cell)- }- }- return strings.Join(parts, ", ")- })- // 8. Strip heading markers but keep heading text- text = reHeadingPrefix.ReplaceAllString(text, "")- // 9. Strip blockquote markers- text = reBlockquote.ReplaceAllString(text, "")- // 10. Unwrap bold/italic markers, keeping inner text (order: *** before ** before *)- text = reBoldItalic3.ReplaceAllString(text, "$1")- text = reBoldItalic2.ReplaceAllString(text, "$1")- text = reBoldItalic1.ReplaceAllString(text, "$1")- // 11. Strip list markers- text = reListMarker.ReplaceAllString(text, "")- // 12. Collapse excessive newlines- text = reExcessiveNewlines.ReplaceAllString(text, "\n\n")-- return strings.TrimSpace(text)-}--// getEnrichedPassage 合并Content、ImageInfo和GeneratedQuestions的文本内容-func getEnrichedPassage(ctx context.Context, result *types.SearchResult) string {- combinedText := cleanPassageForRerank(result.Content)- var enrichments []string-- // 解析ImageInfo- if result.ImageInfo != "" {- var imageInfos []types.ImageInfo- err := json.Unmarshal([]byte(result.ImageInfo), &imageInfos)- if err != nil {- pipelineWarn(ctx, "Rerank", "image_info_parse", map[string]interface{}{- "error": err.Error(),- })- } else {- // 提取所有图片的描述和OCR文本- for _, img := range imageInfos {- if img.Caption != "" {- enrichments = append(enrichments, img.Caption)- }- if img.OCRText != "" {- enrichments = append(enrichments, img.OCRText)- }- }- }- }-- // 解析ChunkMetadata中的GeneratedQuestions- if len(result.ChunkMetadata) > 0 {- var docMeta types.DocumentChunkMetadata- err := json.Unmarshal(result.ChunkMetadata, &docMeta)- if err != nil {- pipelineWarn(ctx, "Rerank", "chunk_metadata_parse", map[string]interface{}{- "error": err.Error(),- })- } else if questionStrings := docMeta.GetQuestionStrings(); len(questionStrings) > 0 {- enrichments = append(enrichments, strings.Join(questionStrings, "; "))- }- }-- if len(enrichments) == 0 {- return combinedText- }-- // 组合内容和增强信息- if combinedText != "" {- combinedText += "\n\n"- }- combinedText += strings.Join(enrichments, "\n")-- return combinedText-}-func logRerankInputScoreSample(ctx context.Context, results []*types.SearchResult) {const maxLogRows = 8limit := min(maxLogRows, len(results))diff --git a/internal/application/service/chat_pipeline/rerank_scope_test.go b/internal/application/service/chat_pipeline/rerank_scope_test.godeleted file mode 100644index 6bdccdb31..000000000--- a/internal/application/service/chat_pipeline/rerank_scope_test.go+++ /dev/null@@ -1,18 +0,0 @@-package chatpipeline--import (- "testing"-- "github.com/Tencent/WeKnora/internal/types"-)--func TestRerankFallbackMinScoreForExplicitScope(t *testing.T) {- if got := rerankFallbackMinScore(nil); got != 0.15 {- t.Fatalf("default fallback minimum = %v, want 0.15", got)- }-- targets := types.SearchTargets{{DisableRecallThresholds: true}}- if got := rerankFallbackMinScore(targets); got != 0 {- t.Fatalf("explicit-scope fallback minimum = %v, want 0", got)- }-}diff --git a/internal/application/service/chat_pipeline/rerank_test.go b/internal/application/service/chat_pipeline/rerank_test.gonew file mode 100644index 000000000..aa428e76c--- /dev/null+++ b/internal/application/service/chat_pipeline/rerank_test.go@@ -0,0 +1,184 @@+package chatpipeline++import (+ "context"+ "sync"+ "testing"++ "github.com/Tencent/WeKnora/internal/models/rerank"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+)++type fixedReranker struct{ scores []float64 }++func (r *fixedReranker) Rerank(_ context.Context, _ string, documents []string) ([]rerank.RankResult, error) {+ out := make([]rerank.RankResult, len(documents))+ for i := range documents {+ out[i] = rerank.RankResult{Index: i, RelevanceScore: r.scores[i]}+ }+ return out, nil+}++func (r *fixedReranker) GetModelName() string { return "fixed" }+func (r *fixedReranker) GetModelID() string { return "fixed" }++type rerankOnlyModelService struct {+ interfaces.ModelService+ reranker rerank.Reranker+}++func (s *rerankOnlyModelService) GetRerankModel(context.Context, string) (rerank.Reranker, error) {+ return s.reranker, nil+}++func rerankChatManage(threshold float64) *types.ChatManage {+ return &types.ChatManage{+ PipelineRequest: types.PipelineRequest{RerankModelID: "rr-1", RerankTopK: 5, RerankThreshold: threshold},+ PipelineState: types.PipelineState{+ RewriteQuery: "q",+ SearchResult: []*types.SearchResult{+ {ID: "c1", Content: "first", Score: 0.5},+ {ID: "c2", Content: "second", Score: 0.4},+ },+ },+ }+}++func TestPluginRerankRecordsDiagnostics(t *testing.T) {+ plugin := &PluginRerank{modelService: &rerankOnlyModelService{+ reranker: &fixedReranker{scores: []float64{0.2, 0.8}},+ }}+ cm := rerankChatManage(0.3)++ next := func() *PluginError { return nil }+ if err := plugin.OnEvent(context.Background(), types.CHUNK_RERANK, cm, next); err != nil {+ t.Fatalf("OnEvent: %v", err)+ }+ if len(cm.RerankResult) != 1 || cm.RerankResult[0].ID != "c2" {+ t.Fatalf("rerank result = %+v", cm.RerankResult)+ }+ d := cm.RerankDiagnostics+ if d == nil || d.Outcome != types.RerankOutcomeOK || d.ModelID != "rr-1" ||+ d.TopScore != 0.8 || d.ResultCount != 1 {+ t.Fatalf("diagnostics = %+v", d)+ }+}++// An all-rejecting rerank ends the search with ErrSearchNothing, and the+// diagnostics say why so the knowledge-search API can explain the empty+// result instead of returning a bare [].+func TestPluginRerankExplainsEmptyResult(t *testing.T) {+ plugin := &PluginRerank{modelService: &rerankOnlyModelService{+ reranker: &fixedReranker{scores: []float64{0.05, 0.1}},+ }}+ cm := rerankChatManage(0.3)++ err := plugin.OnEvent(context.Background(), types.CHUNK_RERANK, cm, func() *PluginError { return nil })+ if err != ErrSearchNothing {+ t.Fatalf("expected ErrSearchNothing, got %v", err)+ }+ d := cm.RerankDiagnostics+ if d == nil || d.Outcome != types.RerankOutcomeAllBelowThreshold || d.TopScore != 0.1 || d.CandidateCount != 2 {+ t.Fatalf("diagnostics = %+v", d)+ }+}++func TestPluginRerankWithoutModelReportsNoModel(t *testing.T) {+ cm := rerankChatManage(0.3)+ cm.RerankModelID = ""+ nextCalled := false+ err := (&PluginRerank{}).OnEvent(context.Background(), types.CHUNK_RERANK, cm, func() *PluginError {+ nextCalled = true+ return nil+ })+ if err != nil || !nextCalled {+ t.Fatalf("err=%v next=%v", err, nextCalled)+ }+ if cm.RerankDiagnostics == nil || cm.RerankDiagnostics.Outcome != types.RerankOutcomeNoModel {+ t.Fatalf("diagnostics = %+v", cm.RerankDiagnostics)+ }+}++// recordingSearchKnowledgeBaseService records every HybridSearch call.+type recordingSearchKnowledgeBaseService struct {+ interfaces.KnowledgeBaseService+ mu sync.Mutex+ params []types.SearchParams+ embedCalls int+}++func (s *recordingSearchKnowledgeBaseService) GetKnowledgeBasesByIDsOnly(+ context.Context, []string,+) ([]*types.KnowledgeBase, error) {+ return []*types.KnowledgeBase{{+ ID: "kb-1", Type: types.KnowledgeBaseTypeDocument, EmbeddingModelID: "embedding-1",+ IndexingStrategy: types.DefaultIndexingStrategy(),+ }}, nil+}++func (s *recordingSearchKnowledgeBaseService) ResolveEmbeddingModelKeys(+ context.Context, []*types.KnowledgeBase,+) map[string]string {+ return map[string]string{"kb-1": "model|endpoint"}+}++func (s *recordingSearchKnowledgeBaseService) GetQueryEmbedding(context.Context, string, string) ([]float32, error) {+ s.mu.Lock()+ defer s.mu.Unlock()+ s.embedCalls+++ return []float32{0.1}, nil+}++func (s *recordingSearchKnowledgeBaseService) HybridSearch(+ _ context.Context, _ string, params types.SearchParams,+) ([]*types.SearchResult, error) {+ s.mu.Lock()+ defer s.mu.Unlock()+ s.params = append(s.params, params)+ return []*types.SearchResult{{ID: "c1", Content: "hit", Score: 0.5}}, nil+}++func TestSearchHonorsRecallPathSwitches(t *testing.T) {+ tests := []struct {+ name string+ disableVector bool+ disableKw bool+ wantEmbed int+ }{+ {name: "vector only", disableKw: true, wantEmbed: 1},+ {name: "keyword only", disableVector: true, wantEmbed: 0},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ svc := &recordingSearchKnowledgeBaseService{}+ plugin := &PluginSearch{knowledgeBaseService: svc}+ cm := &types.ChatManage{+ PipelineRequest: types.PipelineRequest{+ SearchTargets: types.SearchTargets{{+ Type: types.SearchTargetTypeKnowledgeBase, KnowledgeBaseID: "kb-1",+ }},+ EmbeddingTopK: 10,+ DisableVectorMatch: tt.disableVector,+ DisableKeywordsMatch: tt.disableKw,+ },+ PipelineState: types.PipelineState{RewriteQuery: "q"},+ }+ if err := plugin.OnEvent(context.Background(), types.CHUNK_SEARCH, cm, func() *PluginError {+ return nil+ }); err != nil {+ t.Fatalf("OnEvent: %v", err)+ }+ if len(svc.params) != 1 {+ t.Fatalf("expected one HybridSearch call, got %d", len(svc.params))+ }+ p := svc.params[0]+ if p.DisableVectorMatch != tt.disableVector || p.DisableKeywordsMatch != tt.disableKw {+ t.Fatalf("params = %+v", p)+ }+ if svc.embedCalls != tt.wantEmbed {+ t.Fatalf("embedding calls = %d, want %d", svc.embedCalls, tt.wantEmbed)+ }+ })+ }+}diff --git a/internal/application/service/chat_pipeline/search.go b/internal/application/service/chat_pipeline/search.goindex a3b2423ce..1e6a66daf 100644--- a/internal/application/service/chat_pipeline/search.go+++ b/internal/application/service/chat_pipeline/search.go@@ -412,9 +412,9 @@ func (p *PluginSearch) searchByTargets(// only targets that have a real keyword index; vector-only targets must// propagate the embedding failure instead of looking like empty recall.var queryEmbedding []float32- disableVector := false+ disableVector := chatManage.DisableVectorMatchsearchableTargets := targets- if modelKey != "" {+ if modelKey != "" && !disableVector {emb, err := p.knowledgeBaseService.GetQueryEmbedding(ctx, targets[0].KnowledgeBaseID, queryText)if err != nil {searchableTargets = make([]*types.SearchTarget, 0, len(targets))@@ -475,6 +475,7 @@ func (p *PluginSearch) searchByTargets(MatchCount: chatManage.EmbeddingTopK,SkipContextEnrichment: true,DisableVectorMatch: disableVector,+ DisableKeywordsMatch: chatManage.DisableKeywordsMatch,}res, err := p.knowledgeBaseService.HybridSearch(ctx, fullKBIDs[0], params)if err != nil {@@ -554,6 +555,7 @@ func (p *PluginSearch) searchSingleTarget(ScopeTagIDs: t.ScopeTagIDs,SkipContextEnrichment: true,DisableVectorMatch: disableVector,+ DisableKeywordsMatch: chatManage.DisableKeywordsMatch,}if t.Type == types.SearchTargetTypeKnowledge {params.KnowledgeIDs = t.KnowledgeIDsdiff --git a/internal/application/service/datasource_service_test.go b/internal/application/service/datasource_service_test.goindex bc90b9953..2e96a16e3 100644--- a/internal/application/service/datasource_service_test.go+++ b/internal/application/service/datasource_service_test.go@@ -105,6 +105,13 @@ func (s *processSyncKBService) HybridSearch(context.Context, string, types.Searcreturn nil, nil}+func (s *processSyncKBService) HybridSearchWithRerank(+ ctx context.Context, id string, params types.SearchParams,+) (*types.RetrievalResult, error) {+ results, err := s.HybridSearch(ctx, id, params)+ return &types.RetrievalResult{Results: results}, err+}+func (s *processSyncKBService) GetQueryEmbedding(context.Context, string, string) ([]float32, error) {return nil, nil}diff --git a/internal/application/service/knowledgebase_search.go b/internal/application/service/knowledgebase_search.goindex 5970777fd..bc03d388e 100644--- a/internal/application/service/knowledgebase_search.go+++ b/internal/application/service/knowledgebase_search.go@@ -130,6 +130,28 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,// copy, so this stays local to the call.params.MatchCount = normalizedMatchCount(params.MatchCount)+ chunks, err := s.hybridSearchCandidates(ctx, id, params)+ if err != nil || len(chunks) == 0 {+ return nil, err+ }++ // Truncate to the primary-match cap. MatchCount is guaranteed positive by+ // the normalization at the top of this function; the slice bound below+ // depends on that.+ if len(chunks) > params.MatchCount {+ chunks = chunks[:params.MatchCount]+ }++ return s.processSearchResults(ctx, chunks, params.SkipContextEnrichment)+}++// hybridSearchCandidates runs retrieval, fusion and FAQ post-processing for+// HybridSearch and returns the fused chunks, best first, before the+// MatchCount cut. params.MatchCount must already be normalized.+func (s *knowledgeBaseService) hybridSearchCandidates(ctx context.Context,+ id string,+ params types.SearchParams,+) ([]*types.IndexWithScore, error) {// Determine the set of KB IDs to search.searchKBIDs := params.KnowledgeBaseIDsif len(searchKBIDs) == 0 {@@ -284,20 +306,8 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,// AppError from inside the iterative fan-out path (e.g. a per-group// timeout surfaced as ErrVectorStoreUnavailable) must surface to the// caller rather than be silently converted to a truncated chunk list.- deduplicatedChunks, err = s.applyFAQPostProcessing(+ return s.applyFAQPostProcessing(ctx, kb, deduplicatedChunks, vectorResults, groups, params, matchCount)- if err != nil {- return nil, err- }-- // Truncate to the primary-match cap. MatchCount is guaranteed positive by- // the normalization at the top of this function; the slice bound below- // depends on that.- if len(deduplicatedChunks) > params.MatchCount {- deduplicatedChunks = deduplicatedChunks[:params.MatchCount]- }-- return s.processSearchResults(ctx, deduplicatedChunks, params.SkipContextEnrichment)}// normalizedMatchCount resolves the effective primary-match cap for a search.diff --git a/internal/application/service/knowledgebase_search_rerank.go b/internal/application/service/knowledgebase_search_rerank.gonew file mode 100644index 000000000..d9a7c8022--- /dev/null+++ b/internal/application/service/knowledgebase_search_rerank.go@@ -0,0 +1,230 @@+package service++import (+ "context"+ "fmt"++ apperrors "github.com/Tencent/WeKnora/internal/errors"+ "github.com/Tencent/WeKnora/internal/logger"+ "github.com/Tencent/WeKnora/internal/reranking"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+ secutils "github.com/Tencent/WeKnora/internal/utils"+)++// resolveRerankModelID picks the rerank model for a retrieval API request:+// the requested ID, else the tenant RetrievalConfig, else the tenant's first+// rerank model. It returns the ID and its source (types.RerankModelSource*),+// or an empty ID when the tenant has no rerank model.+//+// A requested ID must name an active rerank model of the tenant; anything+// else is the caller's mistake and fails with a bad request instead of+// silently falling back to another model.+func resolveRerankModelID(+ ctx context.Context,+ modelService interfaces.ModelService,+ requested string,+ rc *types.RetrievalConfig,+) (string, string, error) {+ if requested != "" {+ model, err := modelService.GetModelByID(ctx, requested)+ if err != nil || model == nil {+ return "", "", apperrors.NewBadRequestError(+ fmt.Sprintf("rerank model %q not found or not active", requested))+ }+ if model.Type != types.ModelTypeRerank {+ return "", "", apperrors.NewBadRequestError(+ fmt.Sprintf("model %q is a %s model, not a rerank model", requested, model.Type))+ }+ return model.ID, types.RerankModelSourceRequest, nil+ }+ if rc != nil && rc.RerankModelID != "" {+ return rc.RerankModelID, types.RerankModelSourceTenant, nil+ }+ models, err := modelService.ListModels(ctx)+ if err != nil {+ // Auto-detection is best effort: without a model the search still+ // answers, in retrieval order, and the diagnostics say why.+ logger.Warnf(ctx, "Rerank model auto-detection failed: %v", err)+ return "", "", nil+ }+ for _, model := range models {+ if model != nil && model.Type == types.ModelTypeRerank {+ return model.ID, types.RerankModelSourceAuto, nil+ }+ }+ return "", "", nil+}++// HybridSearchWithRerank runs HybridSearch and, when params.Rerank asks for+// it, reranks the fused candidates before cutting to the requested count.+//+// The rerank candidate pool is the top max(top_k, DefaultRetrievalTopK)+// fused chunks, so a small match_count still gives the model a meaningful+// pool to choose from. Context enrichment (parent / nearby / relation+// chunks) runs on the reranked rows only.+func (s *knowledgeBaseService) HybridSearchWithRerank(ctx context.Context,+ id string,+ params types.SearchParams,+) (*types.RetrievalResult, error) {+ if !params.Rerank.IsEnabled() {+ results, err := s.HybridSearch(ctx, id, params)+ if err != nil {+ return nil, err+ }+ out := &types.RetrievalResult{Results: results}+ if params.Rerank != nil {+ out.Meta.Rerank = &types.RerankDiagnostics{+ Outcome: types.RerankOutcomeDisabled,+ CandidateCount: len(results),+ ResultCount: len(results),+ }+ }+ return out, nil+ }++ opts := params.Rerank+ params.MatchCount = normalizedMatchCount(params.MatchCount)+ topK := opts.TopK+ if topK <= 0 {+ topK = params.MatchCount+ }+ topK = min(topK, maxRetrievalPoolSize)++ var rc *types.RetrievalConfig+ if tenantInfo, ok := types.TenantInfoFromContext(ctx); ok && tenantInfo != nil {+ rc = tenantInfo.RetrievalConfig+ }+ // Rerank models belong to the caller. A shared KB request executes in+ // the owner's tenant, so resolve models back in the caller's.+ modelCtx := ctx+ if caller := types.CallerFromContext(ctx); caller.TenantID != 0 {+ modelCtx = types.WithExecutionTenant(ctx, caller.TenantID)+ }+ // Resolve before retrieving so a bad model_id costs nothing.+ modelID, modelSource, err := resolveRerankModelID(modelCtx, s.modelService, opts.ModelID, rc)+ if err != nil {+ return nil, err+ }+ threshold := rc.GetEffectiveRerankThreshold()+ if opts.Threshold != nil {+ threshold = *opts.Threshold+ }+ diag := &types.RerankDiagnostics{+ ModelID: modelID,+ ModelSource: modelSource,+ Threshold: threshold,+ EffectiveThreshold: threshold,+ }+ out := &types.RetrievalResult{Meta: types.RetrievalMeta{Rerank: diag}}++ chunks, err := s.hybridSearchCandidates(ctx, id, params)+ if err != nil {+ return nil, err+ }+ if len(chunks) == 0 {+ diag.Outcome = types.RerankOutcomeNoCandidates+ return out, nil+ }+ pool := chunks[:min(len(chunks), max(topK, types.DefaultRetrievalTopK))]+ candidates, err := s.processSearchResults(ctx, pool, true)+ if err != nil {+ return nil, err+ }+ diag.CandidateCount = len(candidates)++ final := s.rerankCandidates(modelCtx, modelID, params.QueryText, candidates, threshold, topK, diag)+ diag.ResultCount = len(final)+ logger.Infof(ctx, "Hybrid search rerank: kb=%s model=%s(%s) outcome=%s candidates=%d results=%d",+ secutils.SanitizeForLog(id), modelID, modelSource, diag.Outcome, len(candidates), len(final))++ if params.SkipContextEnrichment || len(final) == 0 {+ out.Results = final+ return out, nil+ }+ out.Results, err = s.enrichRerankedResults(ctx, pool, final)+ if err != nil {+ return nil, err+ }+ return out, nil+}++// rerankCandidates reranks candidates with modelID and returns at most topK+// rows. When the model is missing or fails, it records why in diag and+// returns the retrieval order instead.+func (s *knowledgeBaseService) rerankCandidates(+ ctx context.Context,+ modelID, query string,+ candidates []*types.SearchResult,+ threshold float64,+ topK int,+ diag *types.RerankDiagnostics,+) []*types.SearchResult {+ retrievalOrder := candidates[:min(len(candidates), topK)]+ if len(candidates) == 0 {+ diag.Outcome = types.RerankOutcomeNoCandidates+ return nil+ }+ if modelID == "" {+ diag.Outcome = types.RerankOutcomeNoModel+ return retrievalOrder+ }+ model, err := s.modelService.GetRerankModel(ctx, modelID)+ if err != nil {+ logger.Warnf(ctx, "Rerank model %s unavailable, keeping retrieval order: %v", modelID, err)+ diag.Outcome = types.RerankOutcomeModelUnavailable+ diag.Error = err.Error()+ return retrievalOrder+ }++ res := reranking.Rerank(ctx, model, query, candidates, reranking.Options{+ Threshold: threshold,+ TopK: topK,+ FallbackMinScore: reranking.DefaultFallbackMinScore,+ })+ modelSource := diag.ModelSource+ *diag = res.Diagnostics+ diag.ModelID, diag.ModelSource = modelID, modelSource+ if diag.Outcome == types.RerankOutcomeModelError {+ return retrievalOrder+ }+ return res.Results+}++// enrichRerankedResults adds the parent / nearby / relation context chunks+// of the reranked rows. processSearchResults rebuilds the primary rows from+// storage, so they are swapped back for the reranked rows, which carry the+// rerank scores and metadata.+func (s *knowledgeBaseService) enrichRerankedResults(+ ctx context.Context,+ pool []*types.IndexWithScore,+ reranked []*types.SearchResult,+) ([]*types.SearchResult, error) {+ byChunkID := make(map[string]*types.IndexWithScore, len(pool))+ for _, c := range pool {+ byChunkID[c.ChunkID] = c+ }+ rerankedByID := make(map[string]*types.SearchResult, len(reranked))+ primary := make([]*types.IndexWithScore, 0, len(reranked))+ for _, r := range reranked {+ c, ok := byChunkID[r.ID]+ if !ok {+ continue+ }+ scored := *c+ scored.Score = r.Score+ primary = append(primary, &scored)+ rerankedByID[r.ID] = r+ }++ enriched, err := s.processSearchResults(ctx, primary, false)+ if err != nil {+ return nil, err+ }+ for i, r := range enriched {+ if rr, ok := rerankedByID[r.ID]; ok {+ enriched[i] = rr+ }+ }+ return enriched, nil+}diff --git a/internal/application/service/knowledgebase_search_rerank_test.go b/internal/application/service/knowledgebase_search_rerank_test.gonew file mode 100644index 000000000..b81b275bd--- /dev/null+++ b/internal/application/service/knowledgebase_search_rerank_test.go@@ -0,0 +1,191 @@+package service++import (+ "context"+ "errors"+ "testing"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"++ apperrors "github.com/Tencent/WeKnora/internal/errors"+ "github.com/Tencent/WeKnora/internal/models/rerank"+ "github.com/Tencent/WeKnora/internal/types"+)++// scoredReranker returns one canned score per passage, or an error.+type scoredReranker struct {+ scores []float64+ err error+}++func (r *scoredReranker) Rerank(_ context.Context, _ string, documents []string) ([]rerank.RankResult, error) {+ if r.err != nil {+ return nil, r.err+ }+ out := make([]rerank.RankResult, len(documents))+ for i := range documents {+ out[i] = rerank.RankResult{Index: i, RelevanceScore: r.scores[i]}+ }+ return out, nil+}++func (r *scoredReranker) GetModelName() string { return "scored" }+func (r *scoredReranker) GetModelID() string { return "scored" }++// rerankModelService serves one reranker (or a load error) on top of+// stubModelService's model lookups.+type rerankModelService struct {+ stubModelService+ reranker rerank.Reranker+ loadErr error+}++func (s *rerankModelService) GetRerankModel(context.Context, string) (rerank.Reranker, error) {+ return s.reranker, s.loadErr+}++func newRerankModelService() *rerankModelService {+ return &rerankModelService{stubModelService: stubModelService{+ modelsByID: map[string]*types.Model{+ "rr-1": {ID: "rr-1", Type: types.ModelTypeRerank},+ "chat-1": {ID: "chat-1", Type: types.ModelTypeKnowledgeQA},+ },+ availableModels: []*types.Model{+ {ID: "chat-1", Type: types.ModelTypeKnowledgeQA},+ {ID: "rr-auto", Type: types.ModelTypeRerank},+ },+ }}+}++func TestResolveRerankModelID(t *testing.T) {+ t.Parallel()+ ctx := context.Background()+ models := newRerankModelService()+ tenantRC := &types.RetrievalConfig{RerankModelID: "rr-tenant"}++ id, source, err := resolveRerankModelID(ctx, models, "rr-1", tenantRC)+ require.NoError(t, err)+ assert.Equal(t, "rr-1", id)+ assert.Equal(t, types.RerankModelSourceRequest, source)++ id, source, err = resolveRerankModelID(ctx, models, "", tenantRC)+ require.NoError(t, err)+ assert.Equal(t, "rr-tenant", id)+ assert.Equal(t, types.RerankModelSourceTenant, source)++ id, source, err = resolveRerankModelID(ctx, models, "", nil)+ require.NoError(t, err)+ assert.Equal(t, "rr-auto", id)+ assert.Equal(t, types.RerankModelSourceAuto, source)++ models.availableModels = nil+ id, source, err = resolveRerankModelID(ctx, models, "", nil)+ require.NoError(t, err)+ assert.Empty(t, id)+ assert.Empty(t, source)++ // A requested model never silently falls back to another one.+ for _, requested := range []string{"missing", "chat-1"} {+ _, _, err = resolveRerankModelID(ctx, models, requested, tenantRC)+ appErr, ok := apperrors.IsAppError(err)+ require.True(t, ok, "requested %q: %v", requested, err)+ assert.Equal(t, apperrors.ErrBadRequest, appErr.Code, "requested %q", requested)+ }+}++func rerankTestCandidates() []*types.SearchResult {+ return []*types.SearchResult{+ {ID: "c1", Content: "first body", Score: 0.9},+ {ID: "c2", Content: "second body", Score: 0.8},+ {ID: "c3", Content: "third body", Score: 0.7},+ }+}++func TestRerankCandidates_reranksAndKeepsModelSource(t *testing.T) {+ t.Parallel()+ models := newRerankModelService()+ models.reranker = &scoredReranker{scores: []float64{0.1, 0.9, 0.5}}+ s := &knowledgeBaseService{modelService: models}+ diag := &types.RerankDiagnostics{ModelSource: types.RerankModelSourceRequest}++ got := s.rerankCandidates(context.Background(), "rr-1", "q", rerankTestCandidates(), 0.3, 2, diag)++ require.Len(t, got, 2)+ assert.Equal(t, "c2", got[0].ID)+ assert.Equal(t, "c3", got[1].ID)+ assert.Equal(t, types.RerankOutcomeOK, diag.Outcome)+ assert.True(t, diag.Applied)+ assert.Equal(t, "rr-1", diag.ModelID)+ assert.Equal(t, types.RerankModelSourceRequest, diag.ModelSource)+ assert.Equal(t, 0.9, diag.TopScore)+}++func TestRerankCandidates_degradesToRetrievalOrder(t *testing.T) {+ t.Parallel()+ tests := []struct {+ name string+ modelID string+ model rerank.Reranker+ loadErr error+ want types.RerankOutcome+ }{+ {name: "no model", modelID: "", want: types.RerankOutcomeNoModel},+ {+ name: "model unavailable", modelID: "rr-1", loadErr: errors.New("bad credentials"),+ want: types.RerankOutcomeModelUnavailable,+ },+ {+ name: "model error", modelID: "rr-1", model: &scoredReranker{err: errors.New("upstream 500")},+ want: types.RerankOutcomeModelError,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ models := newRerankModelService()+ models.reranker, models.loadErr = tt.model, tt.loadErr+ s := &knowledgeBaseService{modelService: models}+ diag := &types.RerankDiagnostics{}++ got := s.rerankCandidates(context.Background(), tt.modelID, "q", rerankTestCandidates(), 0.3, 2, diag)++ require.Len(t, got, 2)+ assert.Equal(t, "c1", got[0].ID)+ assert.Equal(t, "c2", got[1].ID)+ assert.Equal(t, tt.want, diag.Outcome)+ assert.False(t, diag.Applied)+ })+ }+}++func TestApplyKnowledgeSearchOverrides(t *testing.T) {+ t.Parallel()+ base := func() *types.ChatManage {+ return &types.ChatManage{PipelineRequest: types.PipelineRequest{+ EmbeddingTopK: 50, VectorThreshold: 0.15, KeywordThreshold: 0.3, RerankTopK: 10, RerankThreshold: 0.2,+ }}+ }++ cm := base()+ applyKnowledgeSearchOverrides(cm, &types.KnowledgeSearchOptions{})+ assert.Equal(t, base().PipelineRequest, cm.PipelineRequest, "no overrides keep the tenant config")++ vector, keyword, threshold := 0.5, 0.6, -1.5+ cm = base()+ applyKnowledgeSearchOverrides(cm, &types.KnowledgeSearchOptions{+ VectorThreshold: &vector, KeywordThreshold: &keyword, MatchCount: 80,+ })+ assert.Equal(t, 0.5, cm.VectorThreshold)+ assert.Equal(t, 0.6, cm.KeywordThreshold)+ assert.Equal(t, 80, cm.RerankTopK, "match_count is the final result count")+ assert.Equal(t, 80, cm.EmbeddingTopK, "recall must be deep enough to reach match_count")++ cm = base()+ applyKnowledgeSearchOverrides(cm, &types.KnowledgeSearchOptions{+ MatchCount: 20,+ Rerank: &types.RerankOptions{TopK: 5, Threshold: &threshold},+ })+ assert.Equal(t, 5, cm.RerankTopK, "rerank.top_k wins over match_count")+ assert.Equal(t, -1.5, cm.RerankThreshold)+ assert.Equal(t, 50, cm.EmbeddingTopK)+}diff --git a/internal/application/service/session_knowledge_qa.go b/internal/application/service/session_knowledge_qa.goindex 2c468c5bc..b94307f2f 100644--- a/internal/application/service/session_knowledge_qa.go+++ b/internal/application/service/session_knowledge_qa.go@@ -9,6 +9,7 @@ import ("github.com/Tencent/WeKnora/internal/agent/tools"chatpipeline "github.com/Tencent/WeKnora/internal/application/service/chat_pipeline""github.com/Tencent/WeKnora/internal/common"+ apperrors "github.com/Tencent/WeKnora/internal/errors""github.com/Tencent/WeKnora/internal/event""github.com/Tencent/WeKnora/internal/logger""github.com/Tencent/WeKnora/internal/modelcontext"@@ -830,12 +831,20 @@ func (s *sessionService) KnowledgeQAByEvent(ctx context.Context,// SearchKnowledge performs knowledge base search without LLM summarization// knowledgeBaseIDs: list of knowledge base IDs to search (supports multi-KB)// knowledgeIDs: list of specific knowledge (file) IDs to search+// opts: caller overrides of the tenant RetrievalConfig; nil keeps itfunc (s *sessionService) SearchKnowledge(ctx context.Context,knowledgeBaseIDs []string, knowledgeIDs []string, tagScopes []types.TagScope, query string,-) ([]*types.SearchResult, error) {+ opts *types.KnowledgeSearchOptions,+) (*types.RetrievalResult, error) {logger.Info(ctx, "Start knowledge base search without LLM summary")logger.Infof(ctx, "Knowledge base search parameters, knowledge base IDs: %v, knowledge IDs: %v, tag scopes: %d, query: %s",knowledgeBaseIDs, knowledgeIDs, len(tagScopes), query)+ if opts == nil {+ opts = &types.KnowledgeSearchOptions{}+ }+ if opts.DisableVectorMatch && opts.DisableKeywordsMatch {+ return nil, apperrors.NewBadRequestError("disable_vector_match and disable_keywords_match cannot both be true")+ }// Get tenant ID from contexttenantID, ok := types.TenantIDFromContext(ctx)@@ -844,66 +853,146 @@ func (s *sessionService) SearchKnowledge(ctx context.Context,return nil, fmt.Errorf("workspace ID not found in context")}- // Build unified search targets (computed once, used throughout pipeline)- searchTargets, err := s.buildSearchTargets(ctx, tenantID, knowledgeBaseIDs, knowledgeIDs, tagScopes)- if err != nil {- return nil, fmt.Errorf("build search targets: %w", err)- }-- if len(searchTargets) == 0 {- logger.Warn(ctx, "No search targets available, returning empty results")- return []*types.SearchResult{}, nil- }-- // Create default retrieval parameters — prefer tenant RetrievalConfig, fallback to built-in defaults- userID := types.SessionOwnerIDFromContext(ctx)-// Load tenant-level retrieval config (nil is safe — GetEffective* methods handle nil receiver)var rc *types.RetrievalConfigif tenant, err2 := s.tenantService.GetTenantByID(ctx, tenantID); err2 == nil {rc = tenant.RetrievalConfig}+ // Resolve the rerank model before retrieving so a bad model_id costs+ // nothing. No rerank object means "rerank with the tenant config", which+ // is what this endpoint has always done.+ rerankDisabled := opts.Rerank != nil && !opts.Rerank.IsEnabled()+ var rerankModelID, rerankModelSource string+ if !rerankDisabled {+ requested := ""+ if opts.Rerank != nil {+ requested = opts.Rerank.ModelID+ }+ var err error+ rerankModelID, rerankModelSource, err = resolveRerankModelID(ctx, s.modelService, requested, rc)+ if err != nil {+ return nil, err+ }+ }+ // A model that cannot be loaded degrades to the retrieval order, as on+ // hybrid-search, instead of failing the search in the rerank stage.+ var unavailable *types.RerankDiagnostics+ if rerankModelID != "" {+ if _, err := s.modelService.GetRerankModel(ctx, rerankModelID); err != nil {+ logger.Warnf(ctx, "Rerank model %s unavailable, searching without rerank: %v", rerankModelID, err)+ unavailable = &types.RerankDiagnostics{+ Outcome: types.RerankOutcomeModelUnavailable,+ ModelID: rerankModelID,+ Error: err.Error(),+ }+ rerankModelID = ""+ }+ }++ // Build unified search targets (computed once, used throughout pipeline)+ searchTargets, err := s.buildSearchTargets(ctx, tenantID, knowledgeBaseIDs, knowledgeIDs, tagScopes)+ if err != nil {+ return nil, fmt.Errorf("build search targets: %w", err)+ }+chatManage := &types.ChatManage{PipelineRequest: types.PipelineRequest{- Query: query,- UserID: userID,- KnowledgeBaseIDs: knowledgeBaseIDs,- KnowledgeIDs: knowledgeIDs,- SearchTargets: searchTargets,- MaxRounds: s.cfg.Conversation.MaxRounds,- EmbeddingTopK: rc.GetEffectiveEmbeddingTopK(),- VectorThreshold: rc.GetEffectiveVectorThreshold(),- KeywordThreshold: rc.GetEffectiveKeywordThreshold(),- RerankTopK: rc.GetEffectiveRerankTopK(),- RerankThreshold: rc.GetEffectiveRerankThreshold(),+ Query: query,+ UserID: types.SessionOwnerIDFromContext(ctx),+ KnowledgeBaseIDs: knowledgeBaseIDs,+ KnowledgeIDs: knowledgeIDs,+ SearchTargets: searchTargets,+ MaxRounds: s.cfg.Conversation.MaxRounds,+ EmbeddingTopK: rc.GetEffectiveEmbeddingTopK(),+ VectorThreshold: rc.GetEffectiveVectorThreshold(),+ KeywordThreshold: rc.GetEffectiveKeywordThreshold(),+ DisableVectorMatch: opts.DisableVectorMatch,+ DisableKeywordsMatch: opts.DisableKeywordsMatch,+ RerankModelID: rerankModelID,+ RerankTopK: rc.GetEffectiveRerankTopK(),+ RerankThreshold: rc.GetEffectiveRerankThreshold(),},PipelineState: types.PipelineState{RewriteQuery: query,},}+ applyKnowledgeSearchOverrides(chatManage, opts)- // Get default models- models, err := s.modelService.ListModels(ctx)+ results, err := s.runKnowledgeSearchPipeline(ctx, chatManage, len(searchTargets))if err != nil {- logger.Errorf(ctx, "Failed to get models: %v", err)return nil, err}- // Use rerank model from RetrievalConfig if set, otherwise auto-select the first available- if rc != nil && rc.RerankModelID != "" {- chatManage.RerankModelID = rc.RerankModelID- } else {- for _, model := range models {- if model == nil {- continue- }- if model.Type == types.ModelTypeRerank {- chatManage.RerankModelID = model.ID- break- }+ diag := chatManage.RerankDiagnostics+ switch {+ case rerankDisabled:+ diag = &types.RerankDiagnostics{Outcome: types.RerankOutcomeDisabled}+ case unavailable != nil:+ diag = unavailable+ diag.Threshold, diag.EffectiveThreshold = chatManage.RerankThreshold, chatManage.RerankThreshold+ case diag == nil:+ // Retrieval found nothing, so the rerank stage never ran.+ diag = &types.RerankDiagnostics{+ Outcome: types.RerankOutcomeNoCandidates,+ ModelID: rerankModelID,+ Threshold: chatManage.RerankThreshold,+ EffectiveThreshold: chatManage.RerankThreshold,}}+ diag.ModelSource = rerankModelSource+ if rerankDisabled {+ diag.ModelSource = ""+ }+ if rerankDisabled || unavailable != nil {+ // The rerank stage did not run; the results are the retrieval order.+ diag.CandidateCount = len(results)+ }+ diag.ResultCount = len(results)++ logger.Infof(ctx, "Knowledge base search completed, found %d results, rerank outcome: %s",+ len(results), diag.Outcome)+ return &types.RetrievalResult{Results: results, Meta: types.RetrievalMeta{Rerank: diag}}, nil+}++// applyKnowledgeSearchOverrides applies the caller's knowledge-search+// overrides on top of the tenant RetrievalConfig defaults in chatManage.+func applyKnowledgeSearchOverrides(chatManage *types.ChatManage, opts *types.KnowledgeSearchOptions) {+ if opts.VectorThreshold != nil {+ chatManage.VectorThreshold = *opts.VectorThreshold+ }+ if opts.KeywordThreshold != nil {+ chatManage.KeywordThreshold = *opts.KeywordThreshold+ }+ // The final FILTER_TOP_K stage cuts to RerankTopK whether or not the+ // rerank stage runs, so it is the result count. Recall must be at least+ // as deep for the count to be reachable.+ if opts.MatchCount > 0 {+ chatManage.RerankTopK = opts.MatchCount+ }+ if opts.Rerank != nil {+ if opts.Rerank.TopK > 0 {+ chatManage.RerankTopK = opts.Rerank.TopK+ }+ if opts.Rerank.Threshold != nil {+ chatManage.RerankThreshold = *opts.Rerank.Threshold+ }+ }+ chatManage.EmbeddingTopK = max(chatManage.EmbeddingTopK, chatManage.RerankTopK)+}++// runKnowledgeSearchPipeline runs the retrieval-only pipeline stages and+// returns the merged results. An empty search or an all-rejecting rerank is+// an empty result, not an error.+func (s *sessionService) runKnowledgeSearchPipeline(+ ctx context.Context,+ chatManage *types.ChatManage,+ targetCount int,+) ([]*types.SearchResult, error) {+ if targetCount == 0 {+ logger.Warn(ctx, "No search targets available, returning empty results")+ return []*types.SearchResult{}, nil+ }// Use specific event list, only including retrieval-related events, not LLM summarizationsearchEvents := []types.EventType{@@ -943,8 +1032,6 @@ func (s *sessionService) SearchKnowledge(ctx context.Context,}logger.Infof(ctx, "Event %v triggered successfully", event)}-- logger.Infof(ctx, "Knowledge base search completed, found %d results", len(chatManage.MergeResult))return chatManage.MergeResult, nil}diff --git a/internal/application/service/session_knowledge_search_test.go b/internal/application/service/session_knowledge_search_test.gonew file mode 100644index 000000000..52d8b270d--- /dev/null+++ b/internal/application/service/session_knowledge_search_test.go@@ -0,0 +1,181 @@+package service++import (+ "context"+ "errors"+ "testing"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"++ chatpipeline "github.com/Tencent/WeKnora/internal/application/service/chat_pipeline"+ "github.com/Tencent/WeKnora/internal/config"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+)++type searchTenantService struct {+ interfaces.TenantService+ rc *types.RetrievalConfig+}++func (s *searchTenantService) GetTenantByID(context.Context, uint64) (*types.Tenant, error) {+ return &types.Tenant{ID: 1, RetrievalConfig: s.rc}, nil+}++type searchKBService struct {+ interfaces.KnowledgeBaseService+}++func (s *searchKBService) GetKnowledgeBasesByIDsOnly(_ context.Context, ids []string) ([]*types.KnowledgeBase, error) {+ kbs := make([]*types.KnowledgeBase, len(ids))+ for i, id := range ids {+ kbs[i] = &types.KnowledgeBase{ID: id, TenantID: 1}+ }+ return kbs, nil+}++// fakeRetrieval stands in for CHUNK_SEARCH and CHUNK_MERGE: it records the+// pipeline request it saw and serves canned retrieval results.+type fakeRetrieval struct {+ results []*types.SearchResult+ seen types.PipelineRequest+}++func (f *fakeRetrieval) ActivationEvents() []types.EventType {+ return []types.EventType{types.CHUNK_SEARCH, types.CHUNK_MERGE}+}++func (f *fakeRetrieval) OnEvent(+ _ context.Context, event types.EventType, cm *types.ChatManage, next func() *chatpipeline.PluginError,+) *chatpipeline.PluginError {+ switch event {+ case types.CHUNK_SEARCH:+ f.seen = cm.PipelineRequest+ if len(f.results) == 0 {+ return chatpipeline.ErrSearchNothing+ }+ cm.SearchResult = f.results+ case types.CHUNK_MERGE:+ cm.MergeResult = cm.RerankResult+ if len(cm.MergeResult) == 0 {+ cm.MergeResult = cm.SearchResult+ }+ }+ return next()+}++func newSearchKnowledgeService(+ t *testing.T, rc *types.RetrievalConfig, models *rerankModelService, retrieval *fakeRetrieval,+) *sessionService {+ t.Helper()+ events := chatpipeline.NewEventManager()+ events.Register(retrieval)+ chatpipeline.NewPluginRerank(events, models)+ chatpipeline.NewPluginFilterTopK(events)+ return &sessionService{+ cfg: &config.Config{Conversation: &config.ConversationConfig{}},+ tenantService: &searchTenantService{rc: rc},+ knowledgeBaseService: &searchKBService{},+ modelService: models,+ eventManager: events,+ }+}++func searchKnowledgeCtx() context.Context {+ return types.WithCaller(context.WithValue(context.Background(), types.TenantIDContextKey, uint64(1)),+ types.Caller{TenantID: 1, UserID: "u-1"})+}++func threeHits() []*types.SearchResult {+ return []*types.SearchResult{+ {ID: "c1", Content: "first", Score: 0.9, KnowledgeID: "k1"},+ {ID: "c2", Content: "second", Score: 0.8, KnowledgeID: "k2"},+ {ID: "c3", Content: "third", Score: 0.7, KnowledgeID: "k3"},+ }+}++func TestSearchKnowledge_defaultRerankReportsTenantModel(t *testing.T) {+ models := newRerankModelService()+ models.reranker = &scoredReranker{scores: []float64{0.1, 0.9, 0.6}}+ retrieval := &fakeRetrieval{results: threeHits()}+ svc := newSearchKnowledgeService(t, &types.RetrievalConfig{RerankModelID: "rr-1", RerankThreshold: 0.3},+ models, retrieval)++ got, err := svc.SearchKnowledge(searchKnowledgeCtx(), []string{"kb-1"}, nil, nil, "q", nil)+ require.NoError(t, err)++ require.Len(t, got.Results, 2)+ assert.Equal(t, "c2", got.Results[0].ID)+ d := got.Meta.Rerank+ require.NotNil(t, d)+ assert.Equal(t, types.RerankOutcomeOK, d.Outcome)+ assert.Equal(t, "rr-1", d.ModelID)+ assert.Equal(t, types.RerankModelSourceTenant, d.ModelSource)+ assert.Equal(t, 3, d.CandidateCount)+ assert.Equal(t, 2, d.ResultCount)+}++func TestSearchKnowledge_overridesReachThePipeline(t *testing.T) {+ models := newRerankModelService()+ retrieval := &fakeRetrieval{results: threeHits()}+ svc := newSearchKnowledgeService(t, nil, models, retrieval)+ vector := 0.42+ disabled := false++ got, err := svc.SearchKnowledge(searchKnowledgeCtx(), []string{"kb-1"}, nil, nil, "q",+ &types.KnowledgeSearchOptions{+ VectorThreshold: &vector, MatchCount: 2, DisableKeywordsMatch: true,+ Rerank: &types.RerankOptions{Enabled: &disabled},+ })+ require.NoError(t, err)++ assert.Equal(t, 0.42, retrieval.seen.VectorThreshold)+ assert.True(t, retrieval.seen.DisableKeywordsMatch)+ assert.Empty(t, retrieval.seen.RerankModelID, "a disabled rerank must not reach the pipeline")+ require.Len(t, got.Results, 2, "match_count cuts the final results")+ assert.Equal(t, types.RerankOutcomeDisabled, got.Meta.Rerank.Outcome)+}++func TestSearchKnowledge_explainsEmptyResults(t *testing.T) {+ models := newRerankModelService()+ models.reranker = &scoredReranker{scores: []float64{0.01, 0.02, 0.03}}+ svc := newSearchKnowledgeService(t, nil, models, &fakeRetrieval{results: threeHits()})++ got, err := svc.SearchKnowledge(searchKnowledgeCtx(), []string{"kb-1"}, nil, nil, "q",+ &types.KnowledgeSearchOptions{Rerank: &types.RerankOptions{ModelID: "rr-1"}})+ require.NoError(t, err)+ assert.Empty(t, got.Results)+ assert.Equal(t, types.RerankOutcomeAllBelowThreshold, got.Meta.Rerank.Outcome)+ assert.Equal(t, types.RerankModelSourceRequest, got.Meta.Rerank.ModelSource)+ assert.InDelta(t, 0.03, got.Meta.Rerank.TopScore, 1e-9)++ svc = newSearchKnowledgeService(t, nil, models, &fakeRetrieval{})+ got, err = svc.SearchKnowledge(searchKnowledgeCtx(), []string{"kb-1"}, nil, nil, "q", nil)+ require.NoError(t, err)+ assert.Empty(t, got.Results)+ assert.Equal(t, types.RerankOutcomeNoCandidates, got.Meta.Rerank.Outcome)+}++func TestSearchKnowledge_unavailableModelDegradesToRetrievalOrder(t *testing.T) {+ models := newRerankModelService()+ models.loadErr = errors.New("bad credentials")+ svc := newSearchKnowledgeService(t, &types.RetrievalConfig{RerankModelID: "rr-1"}, models,+ &fakeRetrieval{results: threeHits()})++ got, err := svc.SearchKnowledge(searchKnowledgeCtx(), []string{"kb-1"}, nil, nil, "q", nil)+ require.NoError(t, err)+ require.Len(t, got.Results, 3)+ assert.Equal(t, "c1", got.Results[0].ID)+ d := got.Meta.Rerank+ assert.Equal(t, types.RerankOutcomeModelUnavailable, d.Outcome)+ assert.Equal(t, "bad credentials", d.Error)+ assert.Equal(t, types.RerankModelSourceTenant, d.ModelSource)+}++func TestSearchKnowledge_rejectsUnknownRequestedModel(t *testing.T) {+ svc := newSearchKnowledgeService(t, nil, newRerankModelService(), &fakeRetrieval{results: threeHits()})+ _, err := svc.SearchKnowledge(searchKnowledgeCtx(), []string{"kb-1"}, nil, nil, "q",+ &types.KnowledgeSearchOptions{Rerank: &types.RerankOptions{ModelID: "missing"}})+ require.Error(t, err)+}diff --git a/internal/application/service/tenant_skill_install_test.go b/internal/application/service/tenant_skill_install_test.goindex 05b0d64f8..20cd34bb8 100644--- a/internal/application/service/tenant_skill_install_test.go+++ b/internal/application/service/tenant_skill_install_test.go@@ -3307,9 +3307,9 @@ func (s *installSessionService) KnowledgeQAByEvent(context.Context, *types.ChatM}func (s *installSessionService) SearchKnowledge(- context.Context, []string, []string, []types.TagScope, string,-) ([]*types.SearchResult, error) {- return nil, nil+ context.Context, []string, []string, []types.TagScope, string, *types.KnowledgeSearchOptions,+) (*types.RetrievalResult, error) {+ return &types.RetrievalResult{}, nil}func (s *installSessionService) AgentQA(context.Context, *types.QARequest, *event.EventBus) error {diff --git a/internal/handler/knowledgebase.go b/internal/handler/knowledgebase.goindex 1ae74d9b0..38878415c 100644--- a/internal/handler/knowledgebase.go+++ b/internal/handler/knowledgebase.go@@ -305,7 +305,7 @@ func (h *KnowledgeBaseHandler) resolveKBStoreView(// HybridSearch godoc// @Summary 混合搜索-// @Description 在知识库中执行向量和关键词混合搜索。推荐使用 POST;GET 携带 JSON 请求体仍受支持(兼容旧客户端)。+// @Description 底层召回:向量+关键词混合检索,默认不 rerank(可用 rerank 字段开启);一般检索请用 /knowledge-search。推荐 POST,GET 带 JSON 体仅兼容旧客户端。// @Tags 知识库// @Accept json// @Produce json@@ -342,6 +342,15 @@ func (h *KnowledgeBaseHandler) HybridSearch(c *gin.Context) {_ = c.Error(apperrors.NewBadRequestError("query_text is required"))return}+ if err := req.Rerank.Validate(); err != nil {+ _ = c.Error(apperrors.NewBadRequestError(err.Error()))+ return+ }+ if req.Rerank.IsEnabled() && strings.TrimSpace(req.QueryText) == "" {+ // The rerank model scores passages against the query text.+ _ = c.Error(apperrors.NewBadRequestError("query_text is required when rerank is enabled"))+ return+ }logger.Infof(ctx, "Executing hybrid search, knowledge base ID: %s, query: %s, effectiveTenantID: %d",secutils.SanitizeForLog(id), secutils.SanitizeForLog(req.QueryText), effectiveTenantID)@@ -354,9 +363,17 @@ func (h *KnowledgeBaseHandler) HybridSearch(c *gin.Context) {return}- // Execute hybrid search with default search parameters+ // Execute hybrid search. Without a rerank object this is the raw recall+ // primitive it has always been; with one, the response carries meta.// Note: For shared KBs, the service uses effectiveTenantID internally via context- results, err := h.service.HybridSearch(c.Request.Context(), id, req)+ var retrieval *types.RetrievalResult+ if req.Rerank != nil {+ retrieval, err = h.service.HybridSearchWithRerank(c.Request.Context(), id, req)+ } else {+ var results []*types.SearchResult+ results, err = h.service.HybridSearch(c.Request.Context(), id, req)+ retrieval = &types.RetrievalResult{Results: results}+ }if err != nil {// Service-layer typed AppErrors (e.g. ErrVectorStoreBindingInvalid,// ErrVectorStoreUnavailable, BadRequest from multi-store fan-out)@@ -373,11 +390,15 @@ func (h *KnowledgeBaseHandler) HybridSearch(c *gin.Context) {}logger.Infof(ctx, "Hybrid search completed, knowledge base ID: %s, result count: %d",- secutils.SanitizeForLog(id), len(results))- c.JSON(http.StatusOK, gin.H{+ secutils.SanitizeForLog(id), len(retrieval.Results))+ response := gin.H{"success": true,- "data": rewriter.CopyReferences(ctx, results),- })+ "data": rewriter.CopyReferences(ctx, retrieval.Results),+ }+ if retrieval.Meta.Rerank != nil {+ response["meta"] = retrieval.Meta+ }+ c.JSON(http.StatusOK, response)}// CreateKnowledgeBase godocdiff --git a/internal/handler/knowledgebase_hybrid_search_test.go b/internal/handler/knowledgebase_hybrid_search_test.goindex 4ded29b65..eb83f5c55 100644--- a/internal/handler/knowledgebase_hybrid_search_test.go+++ b/internal/handler/knowledgebase_hybrid_search_test.go@@ -17,6 +17,7 @@ import (type hybridSearchTestService struct {interfaces.KnowledgeBaseServicesearchCalls int+ rerankCalls intsearchParams types.SearchParamsresults []*types.SearchResult}@@ -38,6 +39,21 @@ func (s *hybridSearchTestService) HybridSearch(return []*types.SearchResult{}, nil}+func (s *hybridSearchTestService) HybridSearchWithRerank(+ _ context.Context,+ _ string,+ params types.SearchParams,+) (*types.RetrievalResult, error) {+ s.rerankCalls+++ s.searchParams = params+ return &types.RetrievalResult{+ Results: []*types.SearchResult{{ID: "c1", Score: 0.8}},+ Meta: types.RetrievalMeta{Rerank: &types.RerankDiagnostics{+ Applied: true, Outcome: types.RerankOutcomeOK, ModelID: "rr-1",+ }},+ }, nil+}+func newHybridSearchTestRouter(svc interfaces.KnowledgeBaseService) *gin.Engine {gin.SetMode(gin.TestMode)router := gin.New()@@ -86,6 +102,56 @@ func TestHybridSearchRejectsMissingQueryText(t *testing.T) {}}+func TestHybridSearchRejectsInvalidRerank(t *testing.T) {+ for name, body := range map[string]string{+ "negative top_k": `{"query_text":"q","rerank":{"top_k":-1}}`,+ "rerank without query_text": `{"query_embedding":[0.1],"disable_keywords_match":true,` ++ `"rerank":{"model_id":"rr-1"}}`,+ } {+ t.Run(name, func(t *testing.T) {+ svc := &hybridSearchTestService{}+ response := performHybridSearchRequest(svc, body)+ if response.Code != http.StatusBadRequest {+ t.Fatalf("expected 400, got %d body=%s", response.Code, response.Body.String())+ }+ if svc.searchCalls+svc.rerankCalls != 0 {+ t.Fatal("invalid request reached the service")+ }+ })+ }+}++func TestHybridSearchWithoutRerankKeepsResponseShape(t *testing.T) {+ svc := &hybridSearchTestService{}+ response := performHybridSearchRequest(svc, `{"query_text":"q"}`)+ if response.Code != http.StatusOK || svc.searchCalls != 1 || svc.rerankCalls != 0 {+ t.Fatalf("code=%d search=%d rerank=%d", response.Code, svc.searchCalls, svc.rerankCalls)+ }+ if strings.Contains(response.Body.String(), `"meta"`) {+ t.Fatalf("plain hybrid search must not grow a meta field: %s", response.Body.String())+ }+}++func TestHybridSearchWithRerankReturnsMeta(t *testing.T) {+ svc := &hybridSearchTestService{}+ response := performHybridSearchRequest(svc,+ `{"query_text":"q","match_count":5,"rerank":{"model_id":"rr-1","top_k":3,"threshold":0}}`)+ if response.Code != http.StatusOK {+ t.Fatalf("expected 200, got %d body=%s", response.Code, response.Body.String())+ }+ if svc.rerankCalls != 1 || svc.searchCalls != 0 {+ t.Fatalf("search=%d rerank=%d", svc.searchCalls, svc.rerankCalls)+ }+ rr := svc.searchParams.Rerank+ if rr == nil || rr.ModelID != "rr-1" || rr.TopK != 3 || rr.Threshold == nil || *rr.Threshold != 0 {+ t.Fatalf("rerank options not passed through: %+v", rr)+ }+ body := response.Body.String()+ if !strings.Contains(body, `"meta":{"rerank":{"applied":true,"outcome":"ok","model_id":"rr-1"`) {+ t.Fatalf("expected rerank meta, got %s", body)+ }+}+func TestHybridSearchAcceptsQueryText(t *testing.T) {svc := &hybridSearchTestService{}response := performHybridSearchRequest(svc, `{"query_text":"MiniMax","match_count":3}`)diff --git a/internal/handler/session/qa.go b/internal/handler/session/qa.goindex 46b7a2aa8..e96f13525 100644--- a/internal/handler/session/qa.go+++ b/internal/handler/session/qa.go@@ -814,7 +814,7 @@ func (h *Handler) setupSSEStream(reqCtx *qaRequestContext, generateTitle bool, m// SearchKnowledge godoc// @Summary 知识搜索-// @Description 在知识库中搜索(不使用LLM总结)+// @Description 在知识库中搜索(不使用LLM总结)。与产品内问答使用同一检索流程(召回、rerank、合并),外部检索首选;可覆盖召回参数与 rerank// @Tags 问答// @Accept json// @Produce json@@ -887,6 +887,11 @@ func (h *Handler) SearchKnowledge(c *gin.Context) {c.Error(err)return}+ opts, err := knowledgeSearchOptions(&request)+ if err != nil {+ _ = c.Error(errors.NewBadRequestError(err.Error()))+ return+ }logger.Infof(ctx,@@ -898,20 +903,50 @@ func (h *Handler) SearchKnowledge(c *gin.Context) {)// Directly call knowledge retrieval service without LLM summarization- searchResults, err := h.sessionService.SearchKnowledge(ctx, knowledgeBaseIDs, request.KnowledgeIDs, tagScopes, request.Query)+ retrieval, err := h.sessionService.SearchKnowledge(+ ctx, knowledgeBaseIDs, request.KnowledgeIDs, tagScopes, request.Query, opts,+ )if err != nil {+ // Typed AppErrors (e.g. an unknown rerank model_id) keep their code.+ if appErr, ok := errors.IsAppError(err); ok {+ _ = c.Error(appErr)+ return+ }logger.ErrorWithFields(ctx, err, nil)c.Error(errors.NewInternalServerError(err.Error()))return}- logger.Infof(ctx, "Knowledge search completed, found %d results", len(searchResults))+ logger.Infof(ctx, "Knowledge search completed, found %d results", len(retrieval.Results))c.JSON(http.StatusOK, gin.H{"success": true,- "data": rewriter.CopyReferences(ctx, searchResults),+ "data": rewriter.CopyReferences(ctx, retrieval.Results),+ "meta": retrieval.Meta,})}+// knowledgeSearchOptions validates the retrieval overrides of a+// knowledge-search request.+func knowledgeSearchOptions(request *SearchKnowledgeRequest) (*types.KnowledgeSearchOptions, error) {+ if request.MatchCount < 0 {+ return nil, fmt.Errorf("match_count must not be negative")+ }+ if request.DisableVectorMatch && request.DisableKeywordsMatch {+ return nil, fmt.Errorf("disable_vector_match and disable_keywords_match cannot both be true")+ }+ if err := request.Rerank.Validate(); err != nil {+ return nil, err+ }+ return &types.KnowledgeSearchOptions{+ VectorThreshold: request.VectorThreshold,+ KeywordThreshold: request.KeywordThreshold,+ MatchCount: request.MatchCount,+ DisableKeywordsMatch: request.DisableKeywordsMatch,+ DisableVectorMatch: request.DisableVectorMatch,+ Rerank: request.Rerank,+ }, nil+}+// KnowledgeQA godoc// @Summary 知识问答// @Description 基于知识库的问答(使用LLM总结),支持SSE流式响应diff --git a/internal/handler/session/search_knowledge_resource_urls_test.go b/internal/handler/session/search_knowledge_resource_urls_test.goindex b114042dd..2d399606e 100644--- a/internal/handler/session/search_knowledge_resource_urls_test.go+++ b/internal/handler/session/search_knowledge_resource_urls_test.go@@ -19,15 +19,24 @@ import (type stubSearchSessionService struct {interfaces.SessionService+ calls int+ lastOpts *types.KnowledgeSearchOptions}func (s *stubSearchSessionService) SearchKnowledge(- _ context.Context, _ []string, _ []string, _ []types.TagScope, _ string,-) ([]*types.SearchResult, error) {- return []*types.SearchResult{{- Content: "chunk ",- ImageInfo: `[{"url":"` + testResourceHandle + `"}]`,- }}, nil+ _ context.Context, _ []string, _ []string, _ []types.TagScope, _ string, opts *types.KnowledgeSearchOptions,+) (*types.RetrievalResult, error) {+ s.calls+++ s.lastOpts = opts+ return &types.RetrievalResult{+ Results: []*types.SearchResult{{+ Content: "chunk ",+ ImageInfo: `[{"url":"` + testResourceHandle + `"}]`,+ }},+ Meta: types.RetrievalMeta{Rerank: &types.RerankDiagnostics{+ Applied: true, Outcome: types.RerankOutcomeOK, ModelID: "rr-1", ModelSource: types.RerankModelSourceTenant,+ }},+ }, nil}func TestSearchKnowledge_PublicResourceURLs(t *testing.T) {@@ -98,3 +107,69 @@ func TestSearchKnowledge_DefaultKeepsHandles(t *testing.T) {require.Len(t, resp.Data, 1)assert.Contains(t, resp.Data[0].Content, testResourceHandle)}++func performKnowledgeSearch(t *testing.T, svc *stubSearchSessionService, body string) *httptest.ResponseRecorder {+ t.Helper()+ gin.SetMode(gin.TestMode)+ r := gin.New()+ r.Use(middleware.ErrorHandler())+ h := &Handler{sessionService: svc, fileService: &stubResourceFileService{}}+ r.POST("/knowledge-search", h.SearchKnowledge)+ req := httptest.NewRequest(http.MethodPost, "/knowledge-search", bytes.NewBufferString(body))+ req.Header.Set("Content-Type", "application/json")+ w := httptest.NewRecorder()+ r.ServeHTTP(w, req)+ return w+}++func TestSearchKnowledge_PassesRetrievalOverridesAndReturnsMeta(t *testing.T) {+ svc := &stubSearchSessionService{}+ w := performKnowledgeSearch(t, svc, `{+ "query":"q","knowledge_base_ids":["kb-1"],+ "vector_threshold":0.4,"keyword_threshold":0,"match_count":7,"disable_keywords_match":true,+ "rerank":{"enabled":false}+ }`)+ require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String())++ opts := svc.lastOpts+ require.NotNil(t, opts)+ require.NotNil(t, opts.VectorThreshold)+ require.NotNil(t, opts.KeywordThreshold, "an explicit 0 must survive as an override")+ assert.Equal(t, 0.4, *opts.VectorThreshold)+ assert.Equal(t, 0.0, *opts.KeywordThreshold)+ assert.Equal(t, 7, opts.MatchCount)+ assert.True(t, opts.DisableKeywordsMatch)+ require.NotNil(t, opts.Rerank)+ assert.False(t, opts.Rerank.IsEnabled())++ var body struct {+ Meta types.RetrievalMeta `json:"meta"`+ }+ require.NoError(t, json.Unmarshal(w.Body.Bytes(), &body))+ require.NotNil(t, body.Meta.Rerank)+ assert.Equal(t, types.RerankOutcomeOK, body.Meta.Rerank.Outcome)+ assert.Equal(t, types.RerankModelSourceTenant, body.Meta.Rerank.ModelSource)+}++func TestSearchKnowledge_OmittedOverridesKeepTenantConfig(t *testing.T) {+ svc := &stubSearchSessionService{}+ w := performKnowledgeSearch(t, svc, `{"query":"q","knowledge_base_ids":["kb-1"]}`)+ require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String())+ require.NotNil(t, svc.lastOpts)+ assert.Equal(t, types.KnowledgeSearchOptions{}, *svc.lastOpts)+}++func TestSearchKnowledge_RejectsInvalidOverrides(t *testing.T) {+ for name, extra := range map[string]string{+ "negative match_count": `"match_count":-1`,+ "both recall paths off": `"disable_keywords_match":true,"disable_vector_match":true`,+ "negative rerank top_k": `"rerank":{"top_k":-2}`,+ } {+ t.Run(name, func(t *testing.T) {+ svc := &stubSearchSessionService{}+ w := performKnowledgeSearch(t, svc, `{"query":"q","knowledge_base_ids":["kb-1"],`+extra+`}`)+ require.Equal(t, http.StatusBadRequest, w.Code, "body=%s", w.Body.String())+ assert.Zero(t, svc.calls)+ })+ }+}diff --git a/internal/handler/session/types.go b/internal/handler/session/types.goindex 4d9e046a0..21349bfe7 100644--- a/internal/handler/session/types.go+++ b/internal/handler/session/types.go@@ -87,6 +87,14 @@ type SearchKnowledgeRequest struct {KnowledgeIDs []string `json:"knowledge_ids"` // IDs of specific knowledge (files) to searchTagIDs []string `json:"tag_ids"` // Tag IDs for filtering within a single KBMentionedItems []MentionedItemRequest `json:"mentioned_items"` // Optional scoped tag mentions++ // Optional overrides of the tenant retrieval config. Omitted fields keep it.+ VectorThreshold *float64 `json:"vector_threshold,omitempty"` // Minimum vector similarity+ KeywordThreshold *float64 `json:"keyword_threshold,omitempty"` // Minimum keyword score+ MatchCount int `json:"match_count,omitempty"` // Number of results to return+ DisableKeywordsMatch bool `json:"disable_keywords_match,omitempty"` // Vector recall only+ DisableVectorMatch bool `json:"disable_vector_match,omitempty"` // Keyword recall only+ Rerank *types.RerankOptions `json:"rerank,omitempty"` // Rerank override}// StopSessionRequest represents the stop session requestdiff --git a/internal/im/cmd_search.go b/internal/im/cmd_search.goindex 81d8a58fe..3cca05eeb 100644--- a/internal/im/cmd_search.go+++ b/internal/im/cmd_search.go@@ -84,10 +84,11 @@ func (c *SearchCommand) Execute(ctx context.Context, cmdCtx *CommandContext, arg}}- results, err := c.sessionService.SearchKnowledge(ctx, kbIDs, nil, nil, query)+ retrieval, err := c.sessionService.SearchKnowledge(ctx, kbIDs, nil, nil, query, nil)if err != nil {return nil, fmt.Errorf("search knowledge: %w", err)}+ results := retrieval.Resultsif len(results) == 0 {return &CommandResult{diff --git a/internal/reranking/mmr.go b/internal/reranking/mmr.gonew file mode 100644index 000000000..124ef65d9--- /dev/null+++ b/internal/reranking/mmr.go@@ -0,0 +1,52 @@+package reranking++import (+ "context"+ "math"++ "github.com/Tencent/WeKnora/internal/searchutil"+ "github.com/Tencent/WeKnora/internal/types"+)++// DefaultMMRLambda weighs relevance against redundancy in SelectMMR.+const DefaultMMRLambda = 0.7++// SelectMMR picks up to k results by Maximal Marginal Relevance over their+// Score and the token overlap of their enriched passages, and returns the+// indices of the picks in selection order. Ties go to the earlier result.+func SelectMMR(ctx context.Context, results []*types.SearchResult, k int, lambda float64) []int {+ if k <= 0 || len(results) == 0 {+ return nil+ }++ remaining := make([]int, len(results))+ tokenSets := make([]map[string]struct{}, len(results))+ for i, r := range results {+ remaining[i] = i+ tokenSets[i] = searchutil.TokenizeSimple(EnrichedPassage(ctx, r))+ }++ // Incremental form: maxRedundancy[i] caches result i's maximum jaccard+ // against everything selected so far, so each round only needs one+ // comparison per remaining result.+ maxRedundancy := make([]float64, len(results))+ selected := make([]int, 0, min(k, len(results)))+ for len(selected) < k && len(remaining) > 0 {+ bestPos := 0+ bestScore := math.Inf(-1)+ for pos, i := range remaining {+ mmr := lambda*results[i].Score - (1.0-lambda)*maxRedundancy[i]+ if mmr > bestScore {+ bestScore = mmr+ bestPos = pos+ }+ }+ chosen := remaining[bestPos]+ selected = append(selected, chosen)+ remaining = append(remaining[:bestPos], remaining[bestPos+1:]...)+ for _, i := range remaining {+ maxRedundancy[i] = math.Max(maxRedundancy[i], searchutil.Jaccard(tokenSets[i], tokenSets[chosen]))+ }+ }+ return selected+}diff --git a/internal/reranking/mmr_test.go b/internal/reranking/mmr_test.gonew file mode 100644index 000000000..44c714ed7--- /dev/null+++ b/internal/reranking/mmr_test.go@@ -0,0 +1,118 @@+package reranking++import (+ "context"+ "fmt"+ "math"+ "testing"++ "github.com/Tencent/WeKnora/internal/searchutil"+ "github.com/Tencent/WeKnora/internal/types"+)++// selectMMRNaive recomputes the redundancy of every candidate against every+// already-selected result on each round. It is the reference oracle for the+// incremental SelectMMR.+func selectMMRNaive(ctx context.Context, results []*types.SearchResult, k int, lambda float64) []int {+ if k <= 0 || len(results) == 0 {+ return nil+ }+ var selected []int+ taken := make(map[int]bool)+ for len(selected) < k && len(selected) < len(results) {+ best, bestScore := -1, math.Inf(-1)+ for i, r := range results {+ if taken[i] {+ continue+ }+ redundancy := 0.0+ for _, s := range selected {+ redundancy = math.Max(redundancy, searchutil.Jaccard(+ searchutil.TokenizeSimple(EnrichedPassage(ctx, r)),+ searchutil.TokenizeSimple(EnrichedPassage(ctx, results[s])),+ ))+ }+ if mmr := lambda*r.Score - (1.0-lambda)*redundancy; mmr > bestScore {+ best, bestScore = i, mmr+ }+ }+ selected = append(selected, best)+ taken[best] = true+ }+ return selected+}++// mmrTestCorpus builds a deterministic candidate set whose passages share+// vocabulary in overlapping bands, so redundancy actually drives selection+// instead of the score alone.+func mmrTestCorpus(n int) []*types.SearchResult {+ vocab := []string{+ "insurance", "policy", "claim", "premium", "deductible", "liability",+ "coverage", "endorsement", "underwriting", "reinsurance", "subrogation",+ "indemnity", "exclusion", "rider", "annuity",+ }+ // Simple LCG so the corpus is identical on every run and every platform.+ state := uint64(42)+ next := func(mod int) int {+ state = state*6364136223846793005 + 1442695040888963407+ return int((state >> 33) % uint64(mod))+ }+ results := make([]*types.SearchResult, 0, n)+ for i := 0; i < n; i++ {+ words := make([]byte, 0, 128)+ for j := 0; j < 12; j++ {+ words = append(words, vocab[next(len(vocab))]...)+ words = append(words, ' ')+ }+ results = append(results, &types.SearchResult{+ ID: fmt.Sprintf("chunk-%03d", i),+ Content: fmt.Sprintf("chunk %d %s", i, string(words)),+ // Scores intentionally collide so tie-breaking is exercised too.+ Score: float64(next(20)) / 20.0,+ })+ }+ return results+}++func TestSelectMMR_matchesNaiveSelection(t *testing.T) {+ t.Parallel()+ ctx := context.Background()+ results := mmrTestCorpus(40)+ for _, tc := range []struct {+ k int+ lambda float64+ }{+ {k: 1, lambda: 0.7},+ {k: 5, lambda: 0.7},+ {k: 12, lambda: 0.3},+ {k: 40, lambda: 0.9},+ {k: 60, lambda: 0.5}, // k larger than the candidate count+ } {+ t.Run(fmt.Sprintf("k=%d/lambda=%.1f", tc.k, tc.lambda), func(t *testing.T) {+ want := selectMMRNaive(ctx, results, tc.k, tc.lambda)+ got := SelectMMR(ctx, results, tc.k, tc.lambda)+ if fmt.Sprint(got) != fmt.Sprint(want) {+ t.Fatalf("SelectMMR = %v, naive = %v", got, want)+ }+ })+ }+}++func TestSelectMMR_emptyAndNonPositiveK(t *testing.T) {+ t.Parallel()+ ctx := context.Background()+ if got := SelectMMR(ctx, mmrTestCorpus(3), 0, DefaultMMRLambda); got != nil {+ t.Fatalf("expected nil for k=0, got %v", got)+ }+ if got := SelectMMR(ctx, nil, 5, DefaultMMRLambda); got != nil {+ t.Fatalf("expected nil for empty candidates, got %v", got)+ }+}++func BenchmarkSelectMMR(b *testing.B) {+ ctx := context.Background()+ results := mmrTestCorpus(250)+ for i := 0; i < b.N; i++ {+ SelectMMR(ctx, results, 250, DefaultMMRLambda)+ }+}diff --git a/internal/reranking/passage.go b/internal/reranking/passage.gonew file mode 100644index 000000000..d705216a2--- /dev/null+++ b/internal/reranking/passage.go@@ -0,0 +1,173 @@+package reranking++import (+ "context"+ "encoding/json"+ "regexp"+ "strings"++ "github.com/Tencent/WeKnora/internal/logger"+ "github.com/Tencent/WeKnora/internal/types"+)++// --- Passage cleaning for rerank ---+//+// Rerank models work on semantic text similarity. Markdown formatting, raw URLs,+// image references, table separators, and other structural syntax are noise that+// can dilute the semantic signal. The functions below strip this noise before+// passages are sent to the rerank model.++var (+ // reMarkdownImage matches  — the entire construct is noise.+ // URL group supports one level of balanced parentheses.+ reMarkdownImage = regexp.MustCompile(`!\[[^\]]*\]\([^()\s]*(?:\([^)]*\)[^()\s]*)*\)`)+ // reLinkedImage matches [](link_url) — unwrap to + // so that the subsequent reMarkdownImage pass can remove the image.+ reLinkedImage = regexp.MustCompile(+ `\[!\[([^\]]*)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)\]` ++ `\([^()\s]*(?:\([^)]*\)[^()\s]*)*\)`,+ )+ // reMarkdownLink matches [text](url) — we keep the text, drop the URL.+ // URL group supports one level of balanced parentheses.+ reMarkdownLink = regexp.MustCompile(`\[([^\]]+)\]\([^()\s]*(?:\([^)]*\)[^()\s]*)*\)`)+ // reRawURL matches standalone http(s) URLs.+ reRawURL = regexp.MustCompile(`https?://[^\s)\]>]+`)+ // reCodeBlock captures the semantic body of fenced code blocks.+ reCodeBlock = regexp.MustCompile("(?s)```[^\\r\\n]*\\r?\\n(.*?)\\r?\\n?```")+ // reLatexBlock captures the semantic body of block-level LaTeX ($$...$$).+ reLatexBlock = regexp.MustCompile(`(?s)\$\$(.*?)\$\$`)+ // reTableSep matches table separator rows like |---|---|.+ // Uses [ \t] instead of \s to avoid consuming newlines across rows.+ reTableSep = regexp.MustCompile(`(?m)^[ \t]*\|[ \t:|-]+\|[ \t]*$`)+ // reTableRow matches markdown table data rows like | col1 | col2 |.+ // Uses [ \t] instead of \s to avoid consuming newlines across rows.+ reTableRow = regexp.MustCompile(`(?m)^[ \t]*\|(.+?)\|[ \t]*$`)+ // reHeadingPrefix matches leading # markers in headings.+ reHeadingPrefix = regexp.MustCompile(`(?m)^#{1,6}\s+`)+ // reBlockquote matches leading > markers.+ reBlockquote = regexp.MustCompile(`(?m)^>\s?`)+ // reBoldItalic3 matches ***text*** wrappers (must come before 2 and 1).+ reBoldItalic3 = regexp.MustCompile(`\*{3}(.+?)\*{3}`)+ // reBoldItalic2 matches **text** wrappers.+ reBoldItalic2 = regexp.MustCompile(`\*{2}(.+?)\*{2}`)+ // reBoldItalic1 matches *text* wrappers.+ reBoldItalic1 = regexp.MustCompile(`\*(.+?)\*`)+ // reExcessiveNewlines collapses 3+ consecutive newlines into 2.+ reExcessiveNewlines = regexp.MustCompile(`\n{3,}`)+ // reListMarker matches unordered (- , * ) and ordered (1. ) list prefixes.+ reListMarker = regexp.MustCompile(`(?m)^[\t ]*(?:[-*+]|\d+\.)\s+`)+ // reHTMLTag matches HTML tags like <br>, <div class="...">, etc.+ reHTMLTag = regexp.MustCompile(`</?[a-zA-Z][^>]*>`)+)++// CleanPassage strips markdown/structural noise from text to produce a clean+// semantic passage for the rerank model. The cleaning is designed to preserve+// all meaningful semantic content while removing formatting that would+// confuse text-similarity scoring.+func CleanPassage(text string) string {+ // 1. Unwrap code blocks so code-only candidates remain rerankable.+ text = reCodeBlock.ReplaceAllString(text, "$1")+ // 2. Unwrap LaTeX blocks so formula-only candidates remain rerankable.+ text = reLatexBlock.ReplaceAllString(text, "$1")+ // 3. Remove HTML tags+ text = reHTMLTag.ReplaceAllString(text, "")+ // 3.5. Unwrap nested [](link_url) → + // so that the next step removes the full construct cleanly.+ text = reLinkedImage.ReplaceAllString(text, "")+ // 4. Remove markdown image references entirely+ text = reMarkdownImage.ReplaceAllString(text, "")+ // 5. Convert markdown links to just their display text+ text = reMarkdownLink.ReplaceAllString(text, "$1")+ // 6. Remove standalone raw URLs+ text = reRawURL.ReplaceAllString(text, "")+ // 7. Remove table separator rows+ text = reTableSep.ReplaceAllString(text, "")+ // 7.5. Convert table data rows to plain text (strip | delimiters)+ text = reTableRow.ReplaceAllStringFunc(text, func(match string) string {+ inner := reTableRow.FindStringSubmatch(match)+ if len(inner) < 2 {+ return match+ }+ cells := strings.Split(inner[1], "|")+ var parts []string+ for _, cell := range cells {+ cell = strings.TrimSpace(cell)+ if cell != "" {+ parts = append(parts, cell)+ }+ }+ return strings.Join(parts, ", ")+ })+ // 8. Strip heading markers but keep heading text+ text = reHeadingPrefix.ReplaceAllString(text, "")+ // 9. Strip blockquote markers+ text = reBlockquote.ReplaceAllString(text, "")+ // 10. Unwrap bold/italic markers, keeping inner text (order: *** before ** before *)+ text = reBoldItalic3.ReplaceAllString(text, "$1")+ text = reBoldItalic2.ReplaceAllString(text, "$1")+ text = reBoldItalic1.ReplaceAllString(text, "$1")+ // 11. Strip list markers+ text = reListMarker.ReplaceAllString(text, "")+ // 12. Collapse excessive newlines+ text = reExcessiveNewlines.ReplaceAllString(text, "\n\n")++ return strings.TrimSpace(text)+}++// EnrichedPassage is the cleaned chunk body plus the image captions, OCR text+// and generated questions attached to it. MMR compares candidates on this+// text, so chunks of one document are not made similar by a shared title.+func EnrichedPassage(ctx context.Context, result *types.SearchResult) string {+ combinedText := CleanPassage(result.Content)+ var enrichments []string++ if result.ImageInfo != "" {+ var imageInfos []types.ImageInfo+ if err := json.Unmarshal([]byte(result.ImageInfo), &imageInfos); err != nil {+ logger.Warnf(ctx, "[Rerank] Failed to parse image info of chunk %s: %v", result.ID, err)+ } else {+ for _, img := range imageInfos {+ if img.Caption != "" {+ enrichments = append(enrichments, img.Caption)+ }+ if img.OCRText != "" {+ enrichments = append(enrichments, img.OCRText)+ }+ }+ }+ }++ if len(result.ChunkMetadata) > 0 {+ var docMeta types.DocumentChunkMetadata+ if err := json.Unmarshal(result.ChunkMetadata, &docMeta); err != nil {+ logger.Warnf(ctx, "[Rerank] Failed to parse chunk metadata of chunk %s: %v", result.ID, err)+ } else if questionStrings := docMeta.GetQuestionStrings(); len(questionStrings) > 0 {+ enrichments = append(enrichments, strings.Join(questionStrings, "; "))+ }+ }++ if len(enrichments) == 0 {+ return combinedText+ }+ if combinedText != "" {+ combinedText += "\n\n"+ }+ return combinedText + strings.Join(enrichments, "\n")+}++// ModelPassage is the text the rerank model scores: the document title+// followed by the enriched chunk. A chunk rarely restates what its document+// is about, so without the title a passage from "Show HN: Echo - ... using+// open-weight models" scored 0.002 against "Echo open-weight models reduce+// cost" and 0.45 with it. FAQ entries carry their own question instead.+//+// An empty enriched body stays empty: a title alone is not evidence that+// the chunk answers anything.+func ModelPassage(ctx context.Context, result *types.SearchResult) string {+ passage := EnrichedPassage(ctx, result)+ title := strings.TrimSpace(result.KnowledgeTitle)+ if strings.TrimSpace(passage) == "" || title == "" || result.ChunkType == string(types.ChunkTypeFAQ) {+ return passage+ }+ return title + "\n\n" + passage+}diff --git a/internal/application/service/chat_pipeline/rerank_clean_test.go b/internal/reranking/passage_test.gosimilarity index 93%rename from internal/application/service/chat_pipeline/rerank_clean_test.gorename to internal/reranking/passage_test.goindex 0fccdc151..a8c1b9fd8 100644--- a/internal/application/service/chat_pipeline/rerank_clean_test.go+++ b/internal/reranking/passage_test.go@@ -1,4 +1,4 @@-package chatpipeline+package rerankingimport ("context"@@ -20,7 +20,7 @@ func TestGetEnrichedPassageKeepsQuestionsFromEarlierRevision(t *testing.T) {t.Fatal(err)}- passage := getEnrichedPassage(context.Background(), &types.SearchResult{+ passage := EnrichedPassage(context.Background(), &types.SearchResult{Content: "edited chunk body",ChunkMetadata: types.JSON(metadata),})@@ -49,7 +49,7 @@ func TestGetEnrichedPassageKeepsCodeAndMathCandidates(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- passage := getEnrichedPassage(context.Background(), &types.SearchResult{Content: tt.content})+ passage := EnrichedPassage(context.Background(), &types.SearchResult{Content: tt.content})if strings.TrimSpace(passage) == "" {t.Fatal("semantic-only candidate was removed from the rerank passage")}@@ -171,9 +171,9 @@ func TestCleanPassageForRerank(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- got := cleanPassageForRerank(tt.input)+ got := CleanPassage(tt.input)if got != tt.expect {- t.Errorf("cleanPassageForRerank():\ngot: %q\nexpect: %q", got, tt.expect)+ t.Errorf("CleanPassage():\ngot: %q\nexpect: %q", got, tt.expect)}})}diff --git a/internal/reranking/rerank.go b/internal/reranking/rerank.gonew file mode 100644index 000000000..4b7e9e743--- /dev/null+++ b/internal/reranking/rerank.go@@ -0,0 +1,266 @@+// Package reranking is the single rerank stage shared by the chat pipeline,+// the agent search_knowledge tool and the retrieval APIs: passage building,+// model scoring, threshold filtering with degradation, composite scoring and+// MMR selection.+package reranking++import (+ "context"+ "maps"+ "math"+ "sort"+ "strconv"+ "strings"++ "github.com/Tencent/WeKnora/internal/logger"+ "github.com/Tencent/WeKnora/internal/models/rerank"+ "github.com/Tencent/WeKnora/internal/types"+)++const (+ // DefaultThreshold is the minimum rerank score used when no configuration+ // supplies one. It matches RetrievalConfig.GetEffectiveRerankThreshold.+ DefaultThreshold = 0.2+ // DefaultFallbackMinScore is the score the best candidate needs to be kept+ // when nothing passes the threshold. Below it, returning nothing beats+ // forcing an irrelevant result on the caller.+ DefaultFallbackMinScore = 0.15++ // A threshold above degradeFloor that rejects every candidate is retried+ // at degradeFactor times itself, but never below degradeFloor.+ degradeFloor = 0.3+ degradeFactor = 0.7+)++// FallbackMinScore returns the score the best candidate needs to survive an+// all-rejecting threshold. When the caller explicitly scoped the search to a+// tag or document set, its best candidate is kept regardless, rather than+// letting a global threshold erase the whole authoritative scope. Model+// scores can be negative, so "regardless" is -Inf rather than 0.+func FallbackMinScore(explicitScope bool) float64 {+ if explicitScope {+ return math.Inf(-1)+ }+ return DefaultFallbackMinScore+}++// Options configures one rerank run.+type Options struct {+ // Threshold is the minimum model score a candidate needs.+ Threshold float64+ // TopK, when positive, MMR-selects at most TopK results. Otherwise every+ // candidate that passed the threshold is returned, best first.+ TopK int+ // FallbackMinScore is the score the best candidate needs to be kept when+ // nothing passes the threshold. See FallbackMinScore().+ FallbackMinScore float64+ // FAQScoreBoost, when above 1, multiplies the composite score of FAQ+ // entries (capped at 1).+ FAQScoreBoost float64+}++// Result is the outcome of a rerank run.+type Result struct {+ // Results are the returned rows, best first. They are copies carrying the+ // composite score and base_score / model_score metadata; the input rows+ // are never modified. On a model error they are the input rows unchanged.+ Results []*types.SearchResult+ // Indices[i] is the position of Results[i] in the input slice.+ Indices []int+ // Diagnostics summarises the run for API callers and logs.+ Diagnostics types.RerankDiagnostics++ // Candidates are the input rows with a non-empty passage, in input+ // order; Passages are what the model scored for them. ModelScores index+ // into Candidates. Scored are the rows that passed the threshold, with+ // composite scores, before MMR. Kept for tracing.+ Candidates []*types.SearchResult+ Passages []string+ ModelScores []rerank.RankResult+ Scored []*types.SearchResult+}++// Rerank scores results against query with model and returns the rows that+// pass opts, best first. A failed model call is not an error: the result+// carries the input rows unchanged and a model_error outcome, so callers can+// degrade to the retrieval order.+func Rerank(+ ctx context.Context,+ model rerank.Reranker,+ query string,+ results []*types.SearchResult,+ opts Options,+) *Result {+ res := &Result{+ Diagnostics: types.RerankDiagnostics{+ Threshold: opts.Threshold,+ EffectiveThreshold: opts.Threshold,+ },+ }++ candidateIdx := make([]int, 0, len(results))+ for i, r := range results {+ if r == nil {+ continue+ }+ passage := ModelPassage(ctx, r)+ if strings.TrimSpace(passage) == "" {+ continue+ }+ candidateIdx = append(candidateIdx, i)+ res.Candidates = append(res.Candidates, r)+ res.Passages = append(res.Passages, passage)+ }+ res.Diagnostics.CandidateCount = len(res.Candidates)+ if len(res.Candidates) == 0 {+ res.Diagnostics.Outcome = types.RerankOutcomeNoCandidates+ return res+ }++ scores, err := model.Rerank(ctx, query, res.Passages)+ if err != nil {+ logger.Warnf(ctx, "[Rerank] Model call failed, keeping retrieval order: %v", err)+ res.Diagnostics.Outcome = types.RerankOutcomeModelError+ res.Diagnostics.Error = err.Error()+ res.Results = results+ res.Indices = make([]int, len(results))+ for i := range results {+ res.Indices[i] = i+ }+ res.Diagnostics.ResultCount = len(results)+ return res+ }+ // Drop out-of-range indices once so nothing below has to re-check them.+ valid := scores[:0:0]+ for _, s := range scores {+ if s.Index >= 0 && s.Index < len(res.Candidates) {+ valid = append(valid, s)+ }+ }+ res.ModelScores = valid+ res.Diagnostics.Applied = true+ top, hasTop := bestScore(valid)+ if hasTop {+ res.Diagnostics.TopScore = top.RelevanceScore+ }++ passing, outcome, effective := applyThreshold(valid, opts)+ res.Diagnostics.Outcome = outcome+ res.Diagnostics.EffectiveThreshold = effective++ scored := make([]*types.SearchResult, 0, len(passing))+ scoredIdx := make([]int, 0, len(passing))+ for _, rr := range passing {+ scored = append(scored, scoredCopy(res.Candidates[rr.Index], rr.RelevanceScore, opts.FAQScoreBoost))+ scoredIdx = append(scoredIdx, candidateIdx[rr.Index])+ }+ order := make([]int, len(scored))+ for i := range order {+ order[i] = i+ }+ sort.SliceStable(order, func(a, b int) bool { return scored[order[a]].Score > scored[order[b]].Score })+ res.Scored = make([]*types.SearchResult, len(order))+ sortedIdx := make([]int, len(order))+ for i, o := range order {+ res.Scored[i] = scored[o]+ sortedIdx[i] = scoredIdx[o]+ }++ picks := make([]int, len(res.Scored))+ for i := range picks {+ picks[i] = i+ }+ if opts.TopK > 0 {+ picks = SelectMMR(ctx, res.Scored, min(opts.TopK, len(res.Scored)), DefaultMMRLambda)+ }+ res.Results = make([]*types.SearchResult, 0, len(picks))+ res.Indices = make([]int, 0, len(picks))+ for _, p := range picks {+ res.Results = append(res.Results, res.Scored[p])+ res.Indices = append(res.Indices, sortedIdx[p])+ }+ res.Diagnostics.ResultCount = len(res.Results)++ logger.Infof(ctx, "[Rerank] %d candidates -> %d results, outcome=%s threshold=%.3f effective=%.3f top=%.4f",+ len(res.Candidates), len(res.Results), outcome, opts.Threshold, effective, res.Diagnostics.TopScore)+ return res+}++// applyThreshold keeps the scores at or above opts.Threshold. When none+// pass, a threshold above degradeFloor is lowered once; when still none+// pass, the best candidate is kept if it reaches opts.FallbackMinScore.+func applyThreshold(+ scores []rerank.RankResult,+ opts Options,+) ([]rerank.RankResult, types.RerankOutcome, float64) {+ threshold := opts.Threshold+ if passing := filterScores(scores, threshold); len(passing) > 0 {+ return passing, types.RerankOutcomeOK, threshold+ }+ if threshold > degradeFloor {+ threshold = math.Max(threshold*degradeFactor, degradeFloor)+ if passing := filterScores(scores, threshold); len(passing) > 0 {+ return passing, types.RerankOutcomeThresholdDegraded, threshold+ }+ }+ if top, ok := bestScore(scores); ok && top.RelevanceScore >= opts.FallbackMinScore {+ return []rerank.RankResult{top}, types.RerankOutcomeFallbackTop1, threshold+ }+ return nil, types.RerankOutcomeAllBelowThreshold, threshold+}++func filterScores(scores []rerank.RankResult, threshold float64) []rerank.RankResult {+ var out []rerank.RankResult+ for _, s := range scores {+ if s.RelevanceScore >= threshold {+ out = append(out, s)+ }+ }+ return out+}++// bestScore returns the highest score. Providers usually return scores+// sorted, but nothing guarantees it.+func bestScore(scores []rerank.RankResult) (rerank.RankResult, bool) {+ if len(scores) == 0 {+ return rerank.RankResult{}, false+ }+ top := scores[0]+ for _, s := range scores[1:] {+ if s.RelevanceScore > top.RelevanceScore {+ top = s+ }+ }+ return top, true+}++// scoredCopy returns a copy of r whose Score is the composite of the model+// score and its retrieval score, recording both in Metadata.+func scoredCopy(r *types.SearchResult, modelScore, faqBoost float64) *types.SearchResult {+ c := *r+ c.Metadata = maps.Clone(r.Metadata)+ if c.Metadata == nil {+ c.Metadata = make(map[string]string, 2)+ }+ base := r.Score+ c.Metadata["base_score"] = strconv.FormatFloat(base, 'f', 4, 64)+ c.Metadata["model_score"] = strconv.FormatFloat(modelScore, 'f', 4, 64)+ c.Score = CompositeScore(&c, modelScore, base)+ if faqBoost > 1.0 && c.ChunkType == string(types.ChunkTypeFAQ) {+ c.Metadata["faq_boosted"] = "true"+ c.Metadata["faq_original_score"] = strconv.FormatFloat(c.Score, 'f', 4, 64)+ c.Score = math.Min(c.Score*faqBoost, 1.0)+ }+ return &c+}++// CompositeScore blends the rerank model score with the retrieval score and a+// source weight, clamped to [0, 1].+func CompositeScore(r *types.SearchResult, modelScore, baseScore float64) float64 {+ sourceWeight := 1.0+ if strings.EqualFold(r.KnowledgeSource, "web_search") {+ sourceWeight = 0.95+ }+ composite := 0.6*modelScore + 0.3*baseScore + 0.1*sourceWeight+ return math.Min(math.Max(composite, 0), 1)+}diff --git a/internal/reranking/rerank_test.go b/internal/reranking/rerank_test.gonew file mode 100644index 000000000..98b833e84--- /dev/null+++ b/internal/reranking/rerank_test.go@@ -0,0 +1,202 @@+package reranking++import (+ "context"+ "errors"+ "math"+ "strings"+ "testing"++ "github.com/Tencent/WeKnora/internal/models/rerank"+ "github.com/Tencent/WeKnora/internal/types"+)++// stubReranker returns canned scores (or an error) without any network call.+type stubReranker struct {+ scores []float64+ err error+ calls int+ documents []string+}++func (s *stubReranker) Rerank(_ context.Context, _ string, documents []string) ([]rerank.RankResult, error) {+ s.calls+++ s.documents = documents+ if s.err != nil {+ return nil, s.err+ }+ out := make([]rerank.RankResult, 0, len(documents))+ for i := range documents {+ score := 0.0+ if i < len(s.scores) {+ score = s.scores[i]+ }+ out = append(out, rerank.RankResult{Index: i, RelevanceScore: score})+ }+ return out, nil+}++func (s *stubReranker) GetModelName() string { return "stub-rerank" }+func (s *stubReranker) GetModelID() string { return "stub-rerank-id" }++func rows(contents ...string) []*types.SearchResult {+ out := make([]*types.SearchResult, len(contents))+ for i, c := range contents {+ out[i] = &types.SearchResult{ID: "c" + string(rune('1'+i)), Content: c, Score: 0.5}+ }+ return out+}++func ids(results []*types.SearchResult) string {+ parts := make([]string, len(results))+ for i, r := range results {+ parts[i] = r.ID+ }+ return strings.Join(parts, ",")+}++func TestRerank_thresholdKeepsPassingBestFirst(t *testing.T) {+ t.Parallel()+ in := rows("alpha", "beta", "gamma")+ res := Rerank(context.Background(), &stubReranker{scores: []float64{0.4, 0.9, 0.1}}, "q", in,+ Options{Threshold: 0.3, FallbackMinScore: DefaultFallbackMinScore})++ if got := ids(res.Results); got != "c2,c1" {+ t.Fatalf("results = %s, want c2,c1", got)+ }+ if res.Indices[0] != 1 || res.Indices[1] != 0 {+ t.Fatalf("indices = %v, want [1 0]", res.Indices)+ }+ d := res.Diagnostics+ if !d.Applied || d.Outcome != types.RerankOutcomeOK || d.TopScore != 0.9 ||+ d.CandidateCount != 3 || d.ResultCount != 2 {+ t.Fatalf("diagnostics = %+v", d)+ }+ if res.Results[0].Metadata["model_score"] != "0.9000" || res.Results[0].Metadata["base_score"] != "0.5000" {+ t.Fatalf("metadata = %v", res.Results[0].Metadata)+ }+ if want := 0.6*0.9 + 0.3*0.5 + 0.1; math.Abs(res.Results[0].Score-want) > 1e-9 {+ t.Fatalf("composite = %v, want %v", res.Results[0].Score, want)+ }+ // Inputs are never modified.+ if in[1].Score != 0.5 || in[1].Metadata != nil {+ t.Fatalf("input row was mutated: %+v", in[1])+ }+}++func TestRerank_degradesHighThresholdBeforeFallback(t *testing.T) {+ t.Parallel()+ // 0.5 rejects everything; degraded threshold is max(0.5*0.7, 0.3) = 0.35.+ res := Rerank(context.Background(), &stubReranker{scores: []float64{0.36, 0.2}}, "q", rows("a", "b"),+ Options{Threshold: 0.5, FallbackMinScore: DefaultFallbackMinScore})+ if got := ids(res.Results); got != "c1" {+ t.Fatalf("results = %s, want c1", got)+ }+ if res.Diagnostics.Outcome != types.RerankOutcomeThresholdDegraded ||+ math.Abs(res.Diagnostics.EffectiveThreshold-0.35) > 1e-9 {+ t.Fatalf("diagnostics = %+v", res.Diagnostics)+ }+}++func TestRerank_fallbackTop1AndEmpty(t *testing.T) {+ t.Parallel()+ ctx := context.Background()++ res := Rerank(ctx, &stubReranker{scores: []float64{0.05, 0.20}}, "q", rows("a", "b"),+ Options{Threshold: 0.3, FallbackMinScore: DefaultFallbackMinScore})+ if ids(res.Results) != "c2" || res.Diagnostics.Outcome != types.RerankOutcomeFallbackTop1 {+ t.Fatalf("expected best candidate fallback, got %s %+v", ids(res.Results), res.Diagnostics)+ }++ res = Rerank(ctx, &stubReranker{scores: []float64{0.10, 0.04}}, "q", rows("a", "b"),+ Options{Threshold: 0.3, FallbackMinScore: DefaultFallbackMinScore})+ if len(res.Results) != 0 || res.Diagnostics.Outcome != types.RerankOutcomeAllBelowThreshold ||+ res.Diagnostics.TopScore != 0.10 {+ t.Fatalf("expected empty result, got %s %+v", ids(res.Results), res.Diagnostics)+ }++ // An explicit scope keeps its best candidate even with a negative score.+ res = Rerank(ctx, &stubReranker{scores: []float64{-2, -1}}, "q", rows("a", "b"),+ Options{Threshold: 0.3, FallbackMinScore: FallbackMinScore(true)})+ if ids(res.Results) != "c2" {+ t.Fatalf("explicit scope must keep its best candidate, got %s", ids(res.Results))+ }+}++func TestRerank_modelErrorReturnsInputUnchanged(t *testing.T) {+ t.Parallel()+ in := rows("a", "b")+ model := &stubReranker{err: errors.New("upstream 500")}+ res := Rerank(context.Background(), model, "q", in, Options{Threshold: 0.3})+ if res.Diagnostics.Outcome != types.RerankOutcomeModelError || res.Diagnostics.Applied ||+ res.Diagnostics.Error != "upstream 500" {+ t.Fatalf("diagnostics = %+v", res.Diagnostics)+ }+ if len(res.Results) != 2 || res.Results[0] != in[0] || res.Results[1] != in[1] {+ t.Fatalf("expected the input rows, got %#v", res.Results)+ }+ if model.calls != 1 {+ t.Fatalf("expected one model call, got %d", model.calls)+ }+}++func TestRerank_skipsEmptyPassagesAndReportsNoCandidates(t *testing.T) {+ t.Parallel()+ model := &stubReranker{scores: []float64{0.9}}+ res := Rerank(context.Background(), model, "q", rows(" ", "body"), Options{Threshold: 0.3})+ if len(model.documents) != 1 || ids(res.Results) != "c2" || res.Indices[0] != 1 {+ t.Fatalf("documents=%q results=%s indices=%v", model.documents, ids(res.Results), res.Indices)+ }++ model = &stubReranker{}+ res = Rerank(context.Background(), model, "q", rows(""), Options{Threshold: 0.3})+ if model.calls != 0 || res.Diagnostics.Outcome != types.RerankOutcomeNoCandidates {+ t.Fatalf("calls=%d diagnostics=%+v", model.calls, res.Diagnostics)+ }+}++func TestRerank_topKAppliesMMRAndFAQBoost(t *testing.T) {+ t.Parallel()+ in := rows("alpha beta", "gamma delta", "epsilon zeta")+ in[2].ChunkType = string(types.ChunkTypeFAQ)+ res := Rerank(context.Background(), &stubReranker{scores: []float64{0.9, 0.8, 0.7}}, "q", in,+ Options{Threshold: 0.3, TopK: 2, FAQScoreBoost: 1.5})+ if len(res.Results) != 2 || len(res.Scored) != 3 {+ t.Fatalf("results=%d scored=%d", len(res.Results), len(res.Scored))+ }+ // The FAQ entry is boosted from 0.67 to 1.0 and ranks first.+ if res.Results[0].ID != "c3" || res.Results[0].Metadata["faq_boosted"] != "true" || res.Results[0].Score != 1 {+ t.Fatalf("FAQ boost not applied: %+v", res.Results[0])+ }+}++// A chunk rarely names its document's subject, so the rerank model sees the+// document title; FAQ entries are scored on their own question.+func TestModelPassage_prefixesDocumentTitle(t *testing.T) {+ t.Parallel()+ model := &stubReranker{scores: []float64{0.5, 0.5, 0.5}}+ Rerank(context.Background(), model, "q", []*types.SearchResult{+ {+ ID: "c1", Content: "allocate inference across **open-weight** models",+ KnowledgeTitle: " Show HN: Echo ", ChunkType: string(types.ChunkTypeText),+ },+ {ID: "c2", Content: "What is Echo?", KnowledgeTitle: "FAQ set", ChunkType: string(types.ChunkTypeFAQ)},+ {ID: "c3", Content: "untitled"},+ }, Options{})+ want := []string{"Show HN: Echo\n\nallocate inference across open-weight models", "What is Echo?", "untitled"}+ for i, w := range want {+ if model.documents[i] != w {+ t.Fatalf("passage %d = %q, want %q", i, model.documents[i], w)+ }+ }+}++func TestFallbackMinScore(t *testing.T) {+ t.Parallel()+ if got := FallbackMinScore(false); got != DefaultFallbackMinScore {+ t.Fatalf("default fallback minimum = %v", got)+ }+ if got := FallbackMinScore(true); !math.IsInf(got, -1) {+ t.Fatalf("explicit-scope fallback minimum = %v, want -Inf", got)+ }+}diff --git a/internal/types/chat_manage.go b/internal/types/chat_manage.goindex a74cbec89..9e6c49d4c 100644--- a/internal/types/chat_manage.go+++ b/internal/types/chat_manage.go@@ -17,6 +17,10 @@ type PipelineRequest struct {KeywordThreshold float64 `json:"keyword_threshold"`EmbeddingTopK int `json:"embedding_top_k"`VectorDatabase string `json:"vector_database"`+ // DisableVectorMatch / DisableKeywordsMatch turn off one recall path for+ // every search target. Set by the knowledge-search API.+ DisableVectorMatch bool `json:"disable_vector_match,omitempty"`+ DisableKeywordsMatch bool `json:"disable_keywords_match,omitempty"`// Rerank parametersRerankModelID string `json:"rerank_model_id"`@@ -134,6 +138,8 @@ type PipelineState struct {// UsedMemories mirrors MemoryPrompt in structured form so the answer can// tell the user which memories it saw.UsedMemories UsedMemories `json:"-"`+ // RerankDiagnostics records what the rerank stage did this turn.+ RerankDiagnostics *RerankDiagnostics `json:"-"`}// PipelineContext holds runtime context for the current pipeline execution.@@ -217,6 +223,8 @@ func (c *ChatManage) Clone() *ChatManage {KeywordThreshold: c.KeywordThreshold,EmbeddingTopK: c.EmbeddingTopK,VectorDatabase: c.VectorDatabase,+ DisableVectorMatch: c.DisableVectorMatch,+ DisableKeywordsMatch: c.DisableKeywordsMatch,RerankModelID: c.RerankModelID,RerankTopK: c.RerankTopK,RerankThreshold: c.RerankThreshold,diff --git a/internal/types/interfaces/knowledgebase.go b/internal/types/interfaces/knowledgebase.goindex 387c5676f..ddf5071e0 100644--- a/internal/types/interfaces/knowledgebase.go+++ b/internal/types/interfaces/knowledgebase.go@@ -95,6 +95,12 @@ type KnowledgeBaseService interface {// - Possible errors such as not existing, insufficient permissions, search engine errors, etc.HybridSearch(ctx context.Context, id string, params types.SearchParams) ([]*types.SearchResult, error)+ // HybridSearchWithRerank is HybridSearch plus the optional rerank stage+ // requested by params.Rerank, returning rerank diagnostics beside the+ // results. It serves the hybrid-search API; internal callers rerank on+ // their own and use HybridSearch.+ HybridSearchWithRerank(ctx context.Context, id string, params types.SearchParams) (*types.RetrievalResult, error)+// GetQueryEmbedding computes the query embedding using the embedding model// associated with the given knowledge base. This allows callers to pre-compute// and reuse embeddings across multiple KBs that share the same model.diff --git a/internal/types/interfaces/session.go b/internal/types/interfaces/session.goindex f109e5dbd..6dcc23c05 100644--- a/internal/types/interfaces/session.go+++ b/internal/types/interfaces/session.go@@ -64,7 +64,15 @@ type SessionService interface {// SearchKnowledge performs knowledge-based search, without summarization// knowledgeBaseIDs: list of knowledge base IDs to search (supports multi-KB)// knowledgeIDs: list of specific knowledge (file) IDs to search- SearchKnowledge(ctx context.Context, knowledgeBaseIDs []string, knowledgeIDs []string, tagScopes []types.TagScope, query string) ([]*types.SearchResult, error)+ // opts: caller overrides of the tenant retrieval config; nil keeps it+ SearchKnowledge(+ ctx context.Context,+ knowledgeBaseIDs []string,+ knowledgeIDs []string,+ tagScopes []types.TagScope,+ query string,+ opts *types.KnowledgeSearchOptions,+ ) (*types.RetrievalResult, error)// AgentQA performs agent-based question answering with conversation history and streaming support.AgentQA(ctx context.Context, req *types.QARequest, eventBus *event.EventBus) error}diff --git a/internal/types/rerank.go b/internal/types/rerank.gonew file mode 100644index 000000000..8f270ba1f--- /dev/null+++ b/internal/types/rerank.go@@ -0,0 +1,120 @@+package types++import "errors"++// RerankOptions is the per-request rerank control shared by the retrieval+// APIs (knowledge-search and hybrid-search).+//+// On hybrid-search, an absent object means "no rerank" (the endpoint is a raw+// recall primitive). On knowledge-search, an absent object means "rerank with+// the tenant configuration", which is what the endpoint has always done.+type RerankOptions struct {+ // Enabled turns the rerank stage off when explicitly false. Nil means on.+ Enabled *bool `json:"enabled,omitempty"`+ // ModelID selects the rerank model. Empty falls back to the tenant+ // RetrievalConfig, then to the first rerank model of the tenant.+ ModelID string `json:"model_id,omitempty"`+ // TopK caps the reranked results. Zero uses the endpoint's result count.+ TopK int `json:"top_k,omitempty"`+ // Threshold is the minimum rerank model score. Nil uses the tenant+ // RetrievalConfig. A pointer because 0 and negative scores are valid.+ Threshold *float64 `json:"threshold,omitempty"`+}++// IsEnabled reports whether the options ask for a rerank stage. A nil+// receiver is not enabled; callers decide what an absent object means.+func (o *RerankOptions) IsEnabled() bool {+ return o != nil && (o.Enabled == nil || *o.Enabled)+}++// Validate rejects option values no request can mean. A nil receiver is valid.+func (o *RerankOptions) Validate() error {+ if o != nil && o.TopK < 0 {+ return errors.New("rerank.top_k must not be negative")+ }+ return nil+}++// RerankOutcome says what the rerank stage did with a request.+type RerankOutcome string++const (+ // RerankOutcomeOK means results passed the threshold and were reranked.+ RerankOutcomeOK RerankOutcome = "ok"+ // RerankOutcomeThresholdDegraded means nothing passed the requested+ // threshold, so a lower one was applied.+ RerankOutcomeThresholdDegraded RerankOutcome = "threshold_degraded"+ // RerankOutcomeFallbackTop1 means nothing passed any threshold and only the+ // best candidate was kept.+ RerankOutcomeFallbackTop1 RerankOutcome = "fallback_top1"+ // RerankOutcomeAllBelowThreshold means the model rejected every candidate+ // and the result is empty.+ RerankOutcomeAllBelowThreshold RerankOutcome = "all_below_threshold"+ // RerankOutcomeModelError means the rerank call failed and the retrieval+ // order was returned instead.+ RerankOutcomeModelError RerankOutcome = "model_error"+ // RerankOutcomeModelUnavailable means the configured model could not be+ // loaded and the retrieval order was returned instead.+ RerankOutcomeModelUnavailable RerankOutcome = "model_unavailable"+ // RerankOutcomeNoModel means rerank was wanted but the tenant has no rerank+ // model, so the retrieval order was returned.+ RerankOutcomeNoModel RerankOutcome = "no_model"+ // RerankOutcomeDisabled means the request turned rerank off.+ RerankOutcomeDisabled RerankOutcome = "disabled"+ // RerankOutcomeNoCandidates means retrieval returned nothing to rerank.+ RerankOutcomeNoCandidates RerankOutcome = "no_candidates"+)++// Rerank model ID sources reported in RerankDiagnostics.ModelSource.+const (+ RerankModelSourceRequest = "request"+ RerankModelSourceTenant = "tenant"+ RerankModelSourceAuto = "auto"+)++// RerankDiagnostics reports what the rerank stage did, so an empty or+// unexpectedly ordered result can be explained to API callers.+type RerankDiagnostics struct {+ // Applied is true when rerank scores decided the returned order.+ Applied bool `json:"applied"`+ Outcome RerankOutcome `json:"outcome"`+ // ModelID and ModelSource (request / tenant / auto) identify the model.+ ModelID string `json:"model_id,omitempty"`+ ModelSource string `json:"model_source,omitempty"`+ // Threshold is the requested minimum score; EffectiveThreshold is the one+ // applied after degradation.+ Threshold float64 `json:"threshold"`+ EffectiveThreshold float64 `json:"effective_threshold"`+ // TopScore is the best rerank model score among the candidates.+ TopScore float64 `json:"top_score"`+ CandidateCount int `json:"candidate_count"`+ ResultCount int `json:"result_count"`+ // Error carries the reason for model_error / model_unavailable.+ Error string `json:"error,omitempty"`+}++// RetrievalMeta is the diagnostic envelope returned beside retrieval API+// results.+type RetrievalMeta struct {+ Rerank *RerankDiagnostics `json:"rerank,omitempty"`+}++// RetrievalResult is a retrieval API result with its diagnostics.+type RetrievalResult struct {+ Results []*SearchResult+ Meta RetrievalMeta+}++// KnowledgeSearchOptions are the caller overrides accepted by the+// knowledge-search API. Zero values keep the tenant RetrievalConfig.+type KnowledgeSearchOptions struct {+ // VectorThreshold and KeywordThreshold override the recall thresholds.+ VectorThreshold *float64+ KeywordThreshold *float64+ // MatchCount is the number of results to return.+ MatchCount int+ DisableKeywordsMatch bool+ DisableVectorMatch bool+ // Rerank overrides the rerank stage. Nil reranks with the tenant config.+ Rerank *RerankOptions+}diff --git a/internal/types/search.go b/internal/types/search.goindex daceb9094..595cb4f2d 100644--- a/internal/types/search.go+++ b/internal/types/search.go@@ -249,6 +249,10 @@ type SearchParams struct {// in processSearchResults. Used by the chat pipeline where context assembly// is handled separately in the merge stage.SkipContextEnrichment bool `json:"skip_context_enrichment,omitempty"`+ // Rerank, when present and enabled, reranks the fused candidates before+ // truncating to MatchCount. Only the hybrid-search API reads it; internal+ // callers rerank on their own.+ Rerank *RerankOptions `json:"rerank,omitempty"`}// Value implements the driver.Valuer interface, used to convert SearchResult to database valuediff --git a/website-docs/04-api/01-api-overview.md b/website-docs/04-api/01-api-overview.mdindex 3fab558cd..4bfe4bda1 100644--- a/website-docs/04-api/01-api-overview.md+++ b/website-docs/04-api/01-api-overview.md@@ -211,6 +211,111 @@ X-Accel-Buffering: no各渠道(Web / IM / 嵌入挂件 / API)分别拿到哪种形式、以及图片加载不出来时怎么排查,见[图片与文件的对外访问](../03-features/21-file-access.md)。+## 检索接口怎么选 {#retrieval-api}++对外有两个检索接口,都要求 API Key 有 `retrieve`(或 full)权限,都返回 `SearchResult` 列表。++**默认用 `POST /knowledge-search`**。它和产品内的问答走同一条检索流程(召回 → rerank → 合并 → 截断),返回的就是页面上问答会用到的那些片段。`POST /knowledge-bases/{id}/hybrid-search` 是更底层的召回接口:默认不做 rerank,分数就是召回分,适合需要看到或控制召回原始结果的场景。++### 按场景选++| 我想…… | 用哪个 | 请求体要点 |+| --- | --- | --- |+| 给自己的 RAG / 智能体拿检索结果,排序和页面问答一致 | `knowledge-search` | `query` + `knowledge_base_ids`,其余不填 |+| 同时搜多个知识库,且它们用的 embedding 模型不同 | `knowledge-search` | `knowledge_base_ids` |+| 只在某几个文档或标签里搜 | `knowledge-search` | `knowledge_ids` / `tag_ids` |+| 调整返回条数或召回阈值,但仍然要 rerank | `knowledge-search` | `match_count`、`vector_threshold`、`keyword_threshold` |+| 换一个 rerank 模型,或改 rerank 阈值 | `knowledge-search` | `rerank.model_id`、`rerank.threshold` |+| 不要 rerank,直接拿召回结果 | `knowledge-search` 或 `hybrid-search` | 前者传 `"rerank":{"enabled":false}`;后者不传 `rerank` |+| 结果为空,想知道原因 | `knowledge-search` | 看响应里的 `meta.rerank.outcome` |+| 已经自己算好了查询向量 | `hybrid-search` | `query_embedding` + `disable_keywords_match: true` |+| 评测召回质量:固定一个库、固定参数,看原始召回分 | `hybrid-search` | 不传 `rerank` |+| 在上面的评测基础上,再对比加 rerank 后的效果 | `hybrid-search` | 同一请求加上 `rerank` |+| 父块、相邻块要作为单独的结果行返回,而不是拼进正文 | `hybrid-search` | 默认如此,`skip_context_enrichment: true` 可关 |++### 两者的差别++| | `knowledge-search` | `hybrid-search` |+| --- | --- | --- |+| rerank | 默认开(用空间配置),`rerank` 对象可以覆盖或关闭 | 默认关,传 `rerank` 对象才开 |+| 多知识库 | 可以,embedding 模型可以不同 | `knowledge_base_ids` 可以,但 embedding 模型必须相同,路径上的 `{id}` 也要在其中 |+| 预计算向量 | 不支持 | `query_embedding` |+| 上下文块 | 合并进结果的 `content` | 作为额外的结果行返回 |+| `match_count` 省略时 | 空间配置的 `rerank_top_k`(默认 10) | 50 |+| `meta.rerank` | 每次都返回 | 带了 `rerank` 才返回 |++两个接口的召回参数(`vector_threshold`、`keyword_threshold`、`match_count`、`disable_keywords_match`、`disable_vector_match`)和 `rerank` 对象含义相同;`knowledge-search` 省略的参数沿用空间的检索配置(`GET /tenants/kv/retrieval-config`)。++### rerank 对象++两个接口的 `rerank` 字段结构相同:++| 字段 | 类型 | 默认值 | 说明 |+| --- | --- | --- | --- |+| `enabled` | bool | `true` | 设为 `false` 关闭 rerank,结果保持召回顺序 |+| `model_id` | string | 见下文 | rerank 模型 ID(`GET /models` 里 `type` 为 `Rerank` 的模型)。ID 不存在、未激活或不是 rerank 模型时返回 400,不会悄悄换成别的模型 |+| `top_k` | int | 接口的返回条数 | rerank 后最多保留几条;负数返回 400 |+| `threshold` | float | 空间检索配置里的 `rerank_threshold`(未配置时 0.2) | 模型分数下限;`0` 和负数都是合法值 |++不传 `model_id` 时依次使用:空间检索配置里的 `rerank_model_id` → 空间里第一个 rerank 模型。都没有时不做 rerank,按召回顺序返回,`meta.rerank.outcome` 为 `no_model`。++rerank 的过程和问答链路、智能推理的 `search_knowledge` 工具共用同一套实现(`internal/reranking`):++1. 送给模型打分的文本 = 文档标题 + 去掉 Markdown 标记的分块正文 + 图片描述与 OCR 文本 + 生成的问题。FAQ 条目不加标题。+2. 保留分数不低于 `threshold` 的结果。如果一条都没有、而 `threshold` 高于 0.3,就把阈值降到 `max(threshold×0.7, 0.3)` 再筛一次。还是没有的话,最高分不低于 0.15 时只保留这一条,否则返回空列表。+3. 排序分 = `0.6×模型分 + 0.3×召回分 + 0.1×来源权重`;结果的 `metadata` 里带 `model_score` 和 `base_score`。+4. 用 MMR(λ=0.7)从中挑出 `top_k` 条,降低内容重复。++`hybrid-search` 开启 rerank 时,候选池是融合后排名前 `max(top_k, 50)` 的分块,所以 `match_count` 很小也有足够的候选给模型挑。++rerank 模型加载失败或调用出错时,请求不会失败,而是按召回顺序返回,并在 `meta.rerank` 里写明原因。++### meta.rerank 诊断++`knowledge-search` 的响应总是带 `meta.rerank`;`hybrid-search` 只在请求里带了 `rerank` 对象时才带。++```json+{+ "success": true,+ "data": [],+ "meta": {+ "rerank": {+ "applied": true,+ "outcome": "all_below_threshold",+ "model_id": "rr-1",+ "model_source": "tenant",+ "threshold": 0.3,+ "effective_threshold": 0.3,+ "top_score": 0.08,+ "candidate_count": 24,+ "result_count": 0+ }+ }+}+```++| 字段 | 说明 |+| --- | --- |+| `applied` | rerank 分数是否决定了返回顺序 |+| `outcome` | 见下表 |+| `model_id` / `model_source` | 使用的模型,以及它从哪来:`request`(请求指定)、`tenant`(空间配置)、`auto`(自动选择) |+| `threshold` / `effective_threshold` | 请求的阈值,以及降级后实际使用的阈值 |+| `top_score` | 候选中的最高模型分 |+| `candidate_count` / `result_count` | 送去 rerank 的候选数 / 返回条数 |+| `error` | `model_error`、`model_unavailable` 时的错误信息 |++| `outcome` | 含义 |+| --- | --- |+| `ok` | 有结果达到阈值 |+| `threshold_degraded` | 原阈值下没有结果,降低阈值后才有 |+| `fallback_top1` | 所有阈值都没过,只保留了最高分的一条 |+| `all_below_threshold` | 模型认为没有候选相关,结果为空。可以换个说法重新提问,或者调低 `rerank.threshold` |+| `model_error` | 模型调用失败,按召回顺序返回 |+| `model_unavailable` | 模型加载失败(凭证、地址等配置问题),按召回顺序返回 |+| `no_model` | 空间里没有 rerank 模型,按召回顺序返回 |+| `disabled` | 请求里 `rerank.enabled` 为 `false` |+| `no_candidates` | 召回阶段没有结果 |+## 限流说明| 面 | 限制 | 来源 |diff --git a/website-docs/04-api/02-api-chat.md b/website-docs/04-api/02-api-chat.mdindex 4f9a24e6c..73aebbb5e 100644--- a/website-docs/04-api/02-api-chat.md+++ b/website-docs/04-api/02-api-chat.md@@ -417,7 +417,7 @@ curl -N -X POST $BASE/api/v1/agent-chat/s-1 -H "Authorization: Bearer $TOKEN" \### POST /api/v1/knowledge-search-用途:无会话知识检索(非流式)。Handler: `internal/handler/session/qa.go` 的 `SearchKnowledge`。+用途:无会话知识检索(非流式),外部系统取检索结果的首选接口。和产品内问答走同一条检索流程(召回 → rerank → 合并 → 截断),排序与页面问答一致。和 `hybrid-search` 怎么选见[检索接口怎么选](./01-api-overview.md#retrieval-api)。Handler: `internal/handler/session/qa.go` 的 `SearchKnowledge`。| 字段 | 类型 | 必填 | 说明 || --- | --- | --- | --- |@@ -427,12 +427,23 @@ curl -N -X POST $BASE/api/v1/agent-chat/s-1 -H "Authorization: Bearer $TOKEN" \| `knowledge_ids` | []string | 否 | 限定文件 || `tag_ids` | []string | 否 | 标签过滤 || `mentioned_items` | []object | 否 | 带 KB 范围的标签提及 |+| `vector_threshold` / `keyword_threshold` | float | 否 | 召回阈值;省略时用空间检索配置(默认 0.15 / 0.3) |+| `match_count` | int | 否 | 返回条数;省略时用空间配置的 `rerank_top_k`(默认 10)。召回深度会自动加大到不小于它 |+| `disable_keywords_match` / `disable_vector_match` | bool | 否 | 关闭某一路召回;两个都为 `true` 返回 400 |+| `rerank` | object | 否 | 覆盖 rerank 设置;`{"enabled":false}` 关闭 rerank。字段见 [rerank 对象](./01-api-overview.md#retrieval-api)。`rerank.top_k` 同时给出时优先于 `match_count` |-响应:200 `{"success":true,"data":[SearchResult]}`(`id,content,knowledge_id,knowledge_title,score,chunk_type,knowledge_base_id,...`)+省略的字段都沿用空间的检索配置(`GET /tenants/kv/retrieval-config`),不传任何新字段时行为和以前一样。++响应:200 `{"success":true,"data":[SearchResult],"meta":{"rerank":{...}}}`。`SearchResult` 含 `id,content,knowledge_id,knowledge_title,score,chunk_type,knowledge_base_id,...`;经过 rerank 的结果 `metadata` 带 `model_score` 和 `base_score`。`data` 为空时看 `meta.rerank.outcome` 判断原因,见 [meta.rerank 诊断](./01-api-overview.md#retrieval-api)。```bashcurl -X POST $BASE/api/v1/knowledge-search -H "X-API-Key: $API_KEY" \-H 'Content-Type: application/json' -d '{"query":"部署要求","knowledge_base_ids":["kb-1"]}'++# 只用向量召回、取 5 条,并关闭 rerank+curl -X POST $BASE/api/v1/knowledge-search -H "X-API-Key: $API_KEY" \+ -H 'Content-Type: application/json' \+ -d '{"query":"部署要求","knowledge_base_ids":["kb-1"],"disable_keywords_match":true,"match_count":5,"rerank":{"enabled":false}}'```## 消息(/api/v1/messages)diff --git a/website-docs/04-api/02-api-knowledge.md b/website-docs/04-api/02-api-knowledge.mdindex 265b69723..2e97f7f2c 100644--- a/website-docs/04-api/02-api-knowledge.md+++ b/website-docs/04-api/02-api-knowledge.md@@ -131,7 +131,7 @@ curl -X PUT $BASE/api/v1/knowledge-bases/kb-1/pin -H "Authorization: Bearer $TOK### POST /api/v1/knowledge-bases/:id/hybrid-search(兼容 GET)-用途:KB 内混合检索(向量+关键词)。权限:Viewer+,KB read;API key `retrieve`/full。GET 携带 JSON body 仅为向后兼容(#1727),推荐 POST。+用途:KB 内的底层召回(向量+关键词),默认不做 rerank,返回召回分;可选开启 rerank。适合评测召回、传预计算向量等需要控制原始召回的场景,一般的检索请用 [`knowledge-search`](./02-api-chat.md),选择方法见[检索接口怎么选](./01-api-overview.md#retrieval-api)。权限:Viewer+,KB read;API key `retrieve`/full。GET 携带 JSON body 仅为向后兼容(#1727),推荐 POST。查询参数:`resource_urls=handle|public`(`public` 把结果 `content` / `image_info` 里的 `resource://` 换成可加载直链,详见 [API 总览](./01-api-overview.md))。@@ -139,21 +139,28 @@ curl -X PUT $BASE/api/v1/knowledge-bases/kb-1/pin -H "Authorization: Bearer $TOK| 字段 | 类型 | 必填 | 说明 || --- | --- | --- | --- |-| `query_text` | string | 条件必填 | 查询文本(除非提供 `query_embedding`) |+| `query_text` | string | 条件必填 | 查询文本(除非提供 `query_embedding`;开启 rerank 时必填) || `query_embedding` | []float32 | 否 | 预计算向量 || `vector_threshold` / `keyword_threshold` | float64 | 否 | 匹配阈值 |-| `match_count` | int | 否 | 返回条数上限 |+| `match_count` | int | 否 | 返回条数上限(默认 50) || `disable_keywords_match` / `disable_vector_match` | bool | 否 | 关闭某一路召回 |+| `knowledge_base_ids` | []string | 否 | 一次检索多个知识库,路径上的 `:id` 必须在其中;这些知识库的 embedding 模型必须相同,否则返回 400 || `knowledge_ids` | []string | 否 | 限定知识条目 || `tag_ids` | []string | 否 | 标签过滤(OR) || `only_recommended` | bool | 否 | FAQ 仅推荐条目 || `skip_context_enrichment` | bool | 否 | 跳过父块/上下文补齐 |+| `rerank` | object | 否 | 传入即开启 rerank(`{}` 使用空间配置的模型),字段见 [rerank 对象](./01-api-overview.md#retrieval-api) |-响应:200 `{"success":true,"data":[SearchResult]}`+响应:200 `{"success":true,"data":[SearchResult]}`;带 `rerank` 时多一个 `meta.rerank`(见 [meta.rerank 诊断](./01-api-overview.md#retrieval-api))。```bashcurl -X POST "$BASE/api/v1/knowledge-bases/kb-1/hybrid-search?resource_urls=public" -H "X-API-Key: $API_KEY" \-H 'Content-Type: application/json' -d '{"query_text":"退款流程","match_count":5}'++# 固定召回参数,再用指定模型 rerank+curl -X POST $BASE/api/v1/knowledge-bases/kb-1/hybrid-search -H "X-API-Key: $API_KEY" \+ -H 'Content-Type: application/json' \+ -d '{"query_text":"退款流程","vector_threshold":0.3,"match_count":5,"rerank":{"model_id":"rr-1","threshold":0.2}}'```### POST /api/v1/knowledge-bases/copy