Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(agent): score search_knowledge reranks with the document title
wizardchen62bb9da270f7
diff --git a/config/prompt_templates/agent_system_prompt.yaml b/config/prompt_templates/agent_system_prompt.yamlindex 871dd8ca2..85cc8e52b 100644--- a/config/prompt_templates/agent_system_prompt.yaml+++ b/config/prompt_templates/agent_system_prompt.yaml@@ -45,7 +45,7 @@ templates:You are WeKnora, an assistant developed by Tencent, specializing in questions and work involving the user's knowledge bases.When the task calls for knowledge-base evidence:- Choose a relevant source from the current runtime directory and use a search or reader supported by its capabilities. A knowledge base's <description> states what it is for and its <generated_profile> (gist, topics, typical questions) states what it actually contains; when several are bound, prefer the ones whose profile matches the question and pass their ids to the search tool instead of searching all of them.- - Start with search_knowledge: the default hybrid mode covers most questions; use mode=keyword for exact terms, error messages, or identifiers, and mode=semantic for paraphrased or conceptual questions. Rephrase or switch mode rather than repeating an unproductive query.+ - Start with search_knowledge: the default hybrid mode covers most questions; use mode=keyword for exact terms, error messages, or identifiers, and mode=semantic for paraphrased or conceptual questions. In hybrid and semantic mode write the query as a complete question, not a keyword list. Rephrase rather than repeating an unproductive query.- Use read_document(id=dN) when a returned passage is incomplete: page with offset, or pass query to find a passage inside the document. Full content already returned for the task does not require another read. list_documents shows what a knowledge base contains when the search surface is unclear.- For exact quotes, numbers, or code, verify the underlying source passage rather than relying on document metadata or a generated overview.Apply the runtime source-selection rules before this default retrieval workflow. Other requested sources and deliverables are part of the user's task.diff --git a/internal/agent/tools/search_knowledge.go b/internal/agent/tools/search_knowledge.goindex a0d0354b1..58693ffa6 100644--- a/internal/agent/tools/search_knowledge.go+++ b/internal/agent/tools/search_knowledge.go@@ -37,9 +37,12 @@ var searchKnowledgeTool = BaseTool{"tolerates paraphrase. \"keyword\" matches the literal terms and is the right choice for identifiers, " +"error codes, product names and exact phrases. A base without the requested index is searched with the " +"index it has; the result reports that as a mode fallback.\n" +- "Write query as one natural-language question or a short phrase; in keyword mode write the exact terms. " +- "Pass knowledge_base_ids to focus on the bases whose profile fits the question. Run the tool again with a " +- "different query or mode when the results are thin.\n" ++ "In hybrid and semantic mode write query as one complete natural-language question or statement that " ++ "names the subject and what you want to know (\"Why does Echo use open-weight models to cut cost?\"), " ++ "not a keyword list (\"Echo open-weight cost\"): results are ranked by a relevance model that scores " ++ "meaning. In keyword mode write the exact terms. Pass knowledge_base_ids to focus on the bases whose " ++ "profile fits the question. When nothing passes the relevance check, rephrase the question; switching " ++ "mode alone does not help.\n" +"Every chunk carries a cN handle and belongs to a dN document. Use read_document(id=dN) to read the " +"surrounding context or the whole document. Searches match chunk text only; to find a document by its " +"title or file name use list_documents(keyword=...).",@@ -48,7 +51,7 @@ var searchKnowledgeTool = BaseTool{"properties": {"query": {"type": "string",- "description": "A natural-language question or short phrase; exact terms when mode is keyword",+ "description": "A complete natural-language question, not a keyword list; exact terms when mode is keyword","minLength": 1},"mode": {@@ -230,12 +233,16 @@ func (t *SearchKnowledgeTool) Execute(ctx context.Context, args json.RawMessage)deduplicated := t.deduplicateResults(allResults)ranked := deduplicated+ rerankRejected := 0if t.rerankModel != nil && len(deduplicated) > 0 {reranked, err := t.rerankResults(ctx, query, deduplicated)if err != nil {logger.Warnf(ctx, "[Tool][SearchKnowledge] Rerank failed, using retrieval order: %v", err)} else {ranked = reranked+ if len(reranked) == 0 {+ rerankRejected = len(deduplicated)+ }}}@@ -286,6 +293,9 @@ func (t *SearchKnowledgeTool) Execute(ctx context.Context, args json.RawMessage)result := t.formatOutput(ctx, final, kbIDs, query, mode)annotateModeFallback(result.Data, mode, kbModes)+ if rerankRejected > 0 {+ result.Data["rerank_rejected"] = rerankRejected+ }if len(final) == 0 {result.Output = emptySearchStatement(query, result.Data, len(kbIDs))}@@ -294,10 +304,19 @@ func (t *SearchKnowledgeTool) Execute(ctx context.Context, args json.RawMessage)// emptySearchStatement describes an empty result, including any mode// fallback, so the model does not read "no semantic neighbours" as "the-// exact term does not occur".+// exact term does not occur". When retrieval found candidates and the rerank+// model rejected all of them, it says so: every mode shares that relevance+// check, so the model should rephrase instead of cycling through modes.func emptySearchStatement(query string, data map[string]interface{}, kbCount int) string {mode, _ := data["mode"].(string)msg := fmt.Sprintf("No matching chunks for %q (mode=%s) in %d knowledge base(s).", query, mode, kbCount)+ if rejected, _ := data["rerank_rejected"].(int); rejected > 0 {+ msg = fmt.Sprintf("No chunk passed the relevance check for %q (mode=%s) in %d knowledge base(s): "++ "retrieval found %d candidate chunks but the relevance model scored none of them as answering the "++ "query. Every mode applies the same check, so switching mode will not help; rephrase as a complete "++ "question that names the subject, or use terms the documents themselves would use.",+ query, mode, kbCount, rejected)+ }fallbacks, _ := data["mode_fallbacks"].([]map[string]interface{})for _, fb := range fallbacks {msg += fmt.Sprintf(" Knowledge base %v was searched with mode=%v (%v).",@@ -655,7 +674,7 @@ func (t *SearchKnowledgeTool) rerankScores() ([]rerank.RankResult, error) {passages := make([]string, len(results))for i, result := range results {- passages[i] = t.getEnrichedPassage(ctx, result.SearchResult)+ passages[i] = t.rerankPassage(ctx, result.SearchResult)}rerankResp, err := t.rerankModel.Rerank(ctx, query, passages)if err != nil {@@ -664,6 +683,20 @@ func (t *SearchKnowledgeTool) rerankScores(return rerankResp, nil}+// rerankPassage is the text the rerank model scores: the document title+// followed by the enriched chunk. A chunk rarely restates what its document+// is about, so without the title a passage from "Show HN: Echo - ... using+// open-weight models" scored 0.002 against "Echo open-weight models reduce+// cost" and 0.45 with it. FAQ entries carry their own question instead.+func (t *SearchKnowledgeTool) rerankPassage(ctx context.Context, result *types.SearchResult) string {+ passage := t.getEnrichedPassage(ctx, result)+ title := strings.TrimSpace(result.KnowledgeTitle)+ if title == "" || result.ChunkType == string(types.ChunkTypeFAQ) {+ return passage+ }+ return title + "\n\n" + passage+}+func (t *SearchKnowledgeTool) rerankThreshold() float64 {if t.config != nil && t.config.Conversation != nil && t.config.Conversation.RerankThreshold > 0 {return t.config.Conversation.RerankThresholddiff --git a/internal/agent/tools/search_knowledge_rerank_test.go b/internal/agent/tools/search_knowledge_rerank_test.goindex d601a8acd..918efab2b 100644--- a/internal/agent/tools/search_knowledge_rerank_test.go+++ b/internal/agent/tools/search_knowledge_rerank_test.go@@ -12,15 +12,17 @@ import (// stubReranker returns canned scores (or an error) without any network call.type stubReranker struct {- scores []float64- err error- calls int+ scores []float64+ err error+ calls int+ documents []string}func (s *stubReranker) Rerank(_ context.Context, _ string, documents []string,) ([]rerank.RankResult, error) {s.calls+++ s.documents = documentsif s.err != nil {return nil, s.err}@@ -190,6 +192,34 @@ func TestRerankResults_withoutModelIsPassthrough(t *testing.T) {}}+// A chunk rarely names its document's subject, so the rerank model must see+// the document title; FAQ entries are scored on their own question.+func TestRerankScores_prefixDocumentTitle(t *testing.T) {+ t.Parallel()+ model := &stubReranker{scores: []float64{0.5, 0.5, 0.5}}+ tool := newRerankTestTool(model)+ results := []*searchResultWithMeta{+ {SearchResult: &types.SearchResult{+ ID: "c1", Content: "allocate inference across open-weight models",+ KnowledgeTitle: " Show HN: Echo ", ChunkType: string(types.ChunkTypeText),+ }},+ {SearchResult: &types.SearchResult{+ ID: "c2", Content: "What is Echo?", KnowledgeTitle: "FAQ set", ChunkType: string(types.ChunkTypeFAQ),+ }},+ {SearchResult: &types.SearchResult{ID: "c3", Content: "untitled"}},+ }++ if _, err := tool.rerankScores(context.Background(), "query", results); err != nil {+ t.Fatalf("rerankScores returned error: %v", err)+ }+ want := []string{"Show HN: Echo\n\nallocate inference across open-weight models", "What is Echo?", "untitled"}+ for i, w := range want {+ if model.documents[i] != w {+ t.Fatalf("passage %d = %q, want %q", i, model.documents[i], w)+ }+ }+}+func TestRerankThreshold_default(t *testing.T) {t.Parallel()tool := &SearchKnowledgeTool{}diff --git a/internal/agent/tools/search_knowledge_test.go b/internal/agent/tools/search_knowledge_test.goindex 6576c6eae..1521591a3 100644--- a/internal/agent/tools/search_knowledge_test.go+++ b/internal/agent/tools/search_knowledge_test.go@@ -88,6 +88,22 @@ func TestResolveKBSearchModesFallsBackPerKnowledgeBase(t *testing.T) {}}+// When the rerank model rejected every candidate, the statement must not+// read as "nothing was found": switching mode cannot get past that check.+func TestEmptySearchStatementReportsRerankRejection(t *testing.T) {+ data := map[string]interface{}{"mode": SearchModeHybrid, "rerank_rejected": 18}+ msg := emptySearchStatement("Echo open-weight cost", data, 20)+ for _, want := range []string{"found 18 candidate chunks", "switching mode will not help", "complete question"} {+ if !strings.Contains(msg, want) {+ t.Fatalf("statement missing %q: %q", want, msg)+ }+ }+ if plain := emptySearchStatement("x", map[string]interface{}{"mode": SearchModeHybrid}, 1); !strings.HasPrefix(+ plain, "No matching chunks") {+ t.Fatalf("no rerank rejection keeps the plain statement: %q", plain)+ }+}+func TestAnnotateModeFallbackReportsEffectiveMode(t *testing.T) {all := map[string]kbSearchMode{"kb-faq": {mode: SearchModeSemantic, fallback: true, reason: "FAQ"}}data := map[string]interface{}{"mode": SearchModeKeyword}