Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(agent): score search_knowledge reranks with the document title

wizardchen62bb9da270f7
diff --git a/config/prompt_templates/agent_system_prompt.yaml b/config/prompt_templates/agent_system_prompt.yaml
index 871dd8ca2..85cc8e52b 100644
--- a/config/prompt_templates/agent_system_prompt.yaml
+++ b/config/prompt_templates/agent_system_prompt.yaml
@@ -45,7 +45,7 @@ templates:
You are WeKnora, an assistant developed by Tencent, specializing in questions and work involving the user's knowledge bases.
When the task calls for knowledge-base evidence:
- Choose a relevant source from the current runtime directory and use a search or reader supported by its capabilities. A knowledge base's <description> states what it is for and its <generated_profile> (gist, topics, typical questions) states what it actually contains; when several are bound, prefer the ones whose profile matches the question and pass their ids to the search tool instead of searching all of them.
- - Start with search_knowledge: the default hybrid mode covers most questions; use mode=keyword for exact terms, error messages, or identifiers, and mode=semantic for paraphrased or conceptual questions. Rephrase or switch mode rather than repeating an unproductive query.
+ - Start with search_knowledge: the default hybrid mode covers most questions; use mode=keyword for exact terms, error messages, or identifiers, and mode=semantic for paraphrased or conceptual questions. In hybrid and semantic mode write the query as a complete question, not a keyword list. Rephrase rather than repeating an unproductive query.
- Use read_document(id=dN) when a returned passage is incomplete: page with offset, or pass query to find a passage inside the document. Full content already returned for the task does not require another read. list_documents shows what a knowledge base contains when the search surface is unclear.
- For exact quotes, numbers, or code, verify the underlying source passage rather than relying on document metadata or a generated overview.
Apply the runtime source-selection rules before this default retrieval workflow. Other requested sources and deliverables are part of the user's task.
diff --git a/internal/agent/tools/search_knowledge.go b/internal/agent/tools/search_knowledge.go
index a0d0354b1..58693ffa6 100644
--- a/internal/agent/tools/search_knowledge.go
+++ b/internal/agent/tools/search_knowledge.go
@@ -37,9 +37,12 @@ var searchKnowledgeTool = BaseTool{
"tolerates paraphrase. \"keyword\" matches the literal terms and is the right choice for identifiers, " +
"error codes, product names and exact phrases. A base without the requested index is searched with the " +
"index it has; the result reports that as a mode fallback.\n" +
- "Write query as one natural-language question or a short phrase; in keyword mode write the exact terms. " +
- "Pass knowledge_base_ids to focus on the bases whose profile fits the question. Run the tool again with a " +
- "different query or mode when the results are thin.\n" +
+ "In hybrid and semantic mode write query as one complete natural-language question or statement that " +
+ "names the subject and what you want to know (\"Why does Echo use open-weight models to cut cost?\"), " +
+ "not a keyword list (\"Echo open-weight cost\"): results are ranked by a relevance model that scores " +
+ "meaning. In keyword mode write the exact terms. Pass knowledge_base_ids to focus on the bases whose " +
+ "profile fits the question. When nothing passes the relevance check, rephrase the question; switching " +
+ "mode alone does not help.\n" +
"Every chunk carries a cN handle and belongs to a dN document. Use read_document(id=dN) to read the " +
"surrounding context or the whole document. Searches match chunk text only; to find a document by its " +
"title or file name use list_documents(keyword=...).",
@@ -48,7 +51,7 @@ var searchKnowledgeTool = BaseTool{
"properties": {
"query": {
"type": "string",
- "description": "A natural-language question or short phrase; exact terms when mode is keyword",
+ "description": "A complete natural-language question, not a keyword list; exact terms when mode is keyword",
"minLength": 1
},
"mode": {
@@ -230,12 +233,16 @@ func (t *SearchKnowledgeTool) Execute(ctx context.Context, args json.RawMessage)
deduplicated := t.deduplicateResults(allResults)
ranked := deduplicated
+ rerankRejected := 0
if t.rerankModel != nil && len(deduplicated) > 0 {
reranked, err := t.rerankResults(ctx, query, deduplicated)
if err != nil {
logger.Warnf(ctx, "[Tool][SearchKnowledge] Rerank failed, using retrieval order: %v", err)
} else {
ranked = reranked
+ if len(reranked) == 0 {
+ rerankRejected = len(deduplicated)
+ }
}
}
@@ -286,6 +293,9 @@ func (t *SearchKnowledgeTool) Execute(ctx context.Context, args json.RawMessage)
result := t.formatOutput(ctx, final, kbIDs, query, mode)
annotateModeFallback(result.Data, mode, kbModes)
+ if rerankRejected > 0 {
+ result.Data["rerank_rejected"] = rerankRejected
+ }
if len(final) == 0 {
result.Output = emptySearchStatement(query, result.Data, len(kbIDs))
}
@@ -294,10 +304,19 @@ func (t *SearchKnowledgeTool) Execute(ctx context.Context, args json.RawMessage)
// emptySearchStatement describes an empty result, including any mode
// fallback, so the model does not read "no semantic neighbours" as "the
-// exact term does not occur".
+// exact term does not occur". When retrieval found candidates and the rerank
+// model rejected all of them, it says so: every mode shares that relevance
+// check, so the model should rephrase instead of cycling through modes.
func emptySearchStatement(query string, data map[string]interface{}, kbCount int) string {
mode, _ := data["mode"].(string)
msg := fmt.Sprintf("No matching chunks for %q (mode=%s) in %d knowledge base(s).", query, mode, kbCount)
+ if rejected, _ := data["rerank_rejected"].(int); rejected > 0 {
+ msg = fmt.Sprintf("No chunk passed the relevance check for %q (mode=%s) in %d knowledge base(s): "+
+ "retrieval found %d candidate chunks but the relevance model scored none of them as answering the "+
+ "query. Every mode applies the same check, so switching mode will not help; rephrase as a complete "+
+ "question that names the subject, or use terms the documents themselves would use.",
+ query, mode, kbCount, rejected)
+ }
fallbacks, _ := data["mode_fallbacks"].([]map[string]interface{})
for _, fb := range fallbacks {
msg += fmt.Sprintf(" Knowledge base %v was searched with mode=%v (%v).",
@@ -655,7 +674,7 @@ func (t *SearchKnowledgeTool) rerankScores(
) ([]rerank.RankResult, error) {
passages := make([]string, len(results))
for i, result := range results {
- passages[i] = t.getEnrichedPassage(ctx, result.SearchResult)
+ passages[i] = t.rerankPassage(ctx, result.SearchResult)
}
rerankResp, err := t.rerankModel.Rerank(ctx, query, passages)
if err != nil {
@@ -664,6 +683,20 @@ func (t *SearchKnowledgeTool) rerankScores(
return rerankResp, nil
}
+// rerankPassage is the text the rerank model scores: the document title
+// followed by the enriched chunk. A chunk rarely restates what its document
+// is about, so without the title a passage from "Show HN: Echo - ... using
+// open-weight models" scored 0.002 against "Echo open-weight models reduce
+// cost" and 0.45 with it. FAQ entries carry their own question instead.
+func (t *SearchKnowledgeTool) rerankPassage(ctx context.Context, result *types.SearchResult) string {
+ passage := t.getEnrichedPassage(ctx, result)
+ title := strings.TrimSpace(result.KnowledgeTitle)
+ if title == "" || result.ChunkType == string(types.ChunkTypeFAQ) {
+ return passage
+ }
+ return title + "\n\n" + passage
+}
+
func (t *SearchKnowledgeTool) rerankThreshold() float64 {
if t.config != nil && t.config.Conversation != nil && t.config.Conversation.RerankThreshold > 0 {
return t.config.Conversation.RerankThreshold
diff --git a/internal/agent/tools/search_knowledge_rerank_test.go b/internal/agent/tools/search_knowledge_rerank_test.go
index d601a8acd..918efab2b 100644
--- a/internal/agent/tools/search_knowledge_rerank_test.go
+++ b/internal/agent/tools/search_knowledge_rerank_test.go
@@ -12,15 +12,17 @@ import (
// stubReranker returns canned scores (or an error) without any network call.
type stubReranker struct {
- scores []float64
- err error
- calls int
+ scores []float64
+ err error
+ calls int
+ documents []string
}
func (s *stubReranker) Rerank(
_ context.Context, _ string, documents []string,
) ([]rerank.RankResult, error) {
s.calls++
+ s.documents = documents
if s.err != nil {
return nil, s.err
}
@@ -190,6 +192,34 @@ func TestRerankResults_withoutModelIsPassthrough(t *testing.T) {
}
}
+// A chunk rarely names its document's subject, so the rerank model must see
+// the document title; FAQ entries are scored on their own question.
+func TestRerankScores_prefixDocumentTitle(t *testing.T) {
+ t.Parallel()
+ model := &stubReranker{scores: []float64{0.5, 0.5, 0.5}}
+ tool := newRerankTestTool(model)
+ results := []*searchResultWithMeta{
+ {SearchResult: &types.SearchResult{
+ ID: "c1", Content: "allocate inference across open-weight models",
+ KnowledgeTitle: " Show HN: Echo ", ChunkType: string(types.ChunkTypeText),
+ }},
+ {SearchResult: &types.SearchResult{
+ ID: "c2", Content: "What is Echo?", KnowledgeTitle: "FAQ set", ChunkType: string(types.ChunkTypeFAQ),
+ }},
+ {SearchResult: &types.SearchResult{ID: "c3", Content: "untitled"}},
+ }
+
+ if _, err := tool.rerankScores(context.Background(), "query", results); err != nil {
+ t.Fatalf("rerankScores returned error: %v", err)
+ }
+ want := []string{"Show HN: Echo\n\nallocate inference across open-weight models", "What is Echo?", "untitled"}
+ for i, w := range want {
+ if model.documents[i] != w {
+ t.Fatalf("passage %d = %q, want %q", i, model.documents[i], w)
+ }
+ }
+}
+
func TestRerankThreshold_default(t *testing.T) {
t.Parallel()
tool := &SearchKnowledgeTool{}
diff --git a/internal/agent/tools/search_knowledge_test.go b/internal/agent/tools/search_knowledge_test.go
index 6576c6eae..1521591a3 100644
--- a/internal/agent/tools/search_knowledge_test.go
+++ b/internal/agent/tools/search_knowledge_test.go
@@ -88,6 +88,22 @@ func TestResolveKBSearchModesFallsBackPerKnowledgeBase(t *testing.T) {
}
}
+// When the rerank model rejected every candidate, the statement must not
+// read as "nothing was found": switching mode cannot get past that check.
+func TestEmptySearchStatementReportsRerankRejection(t *testing.T) {
+ data := map[string]interface{}{"mode": SearchModeHybrid, "rerank_rejected": 18}
+ msg := emptySearchStatement("Echo open-weight cost", data, 20)
+ for _, want := range []string{"found 18 candidate chunks", "switching mode will not help", "complete question"} {
+ if !strings.Contains(msg, want) {
+ t.Fatalf("statement missing %q: %q", want, msg)
+ }
+ }
+ if plain := emptySearchStatement("x", map[string]interface{}{"mode": SearchModeHybrid}, 1); !strings.HasPrefix(
+ plain, "No matching chunks") {
+ t.Fatalf("no rerank rejection keeps the plain statement: %q", plain)
+ }
+}
+
func TestAnnotateModeFallbackReportsEffectiveMode(t *testing.T) {
all := map[string]kbSearchMode{"kb-faq": {mode: SearchModeSemantic, fallback: true, reason: "FAQ"}}
data := map[string]interface{}{"mode": SearchModeKeyword}