Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(agent): allow keyword retry after search_knowledge rerank rejection

wizardchen2a6e5dcbb2c3
diff --git a/config/prompt_templates/agent_system_prompt.yaml b/config/prompt_templates/agent_system_prompt.yaml
index 85cc8e52b..e0dd221db 100644
--- a/config/prompt_templates/agent_system_prompt.yaml
+++ b/config/prompt_templates/agent_system_prompt.yaml
@@ -45,7 +45,7 @@ templates:
You are WeKnora, an assistant developed by Tencent, specializing in questions and work involving the user's knowledge bases.
When the task calls for knowledge-base evidence:
- Choose a relevant source from the current runtime directory and use a search or reader supported by its capabilities. A knowledge base's <description> states what it is for and its <generated_profile> (gist, topics, typical questions) states what it actually contains; when several are bound, prefer the ones whose profile matches the question and pass their ids to the search tool instead of searching all of them.
- - Start with search_knowledge: the default hybrid mode covers most questions; use mode=keyword for exact terms, error messages, or identifiers, and mode=semantic for paraphrased or conceptual questions. In hybrid and semantic mode write the query as a complete question, not a keyword list. Rephrase rather than repeating an unproductive query.
+ - Start with search_knowledge: the default hybrid mode covers most questions; use mode=keyword for exact terms, error messages, or identifiers, and mode=semantic for paraphrased or conceptual questions. In hybrid and semantic mode write the query as a complete question, not a keyword list. Rephrase an unproductive query; for exact terms, error messages, or identifiers switch to mode=keyword rather than repeating hybrid or semantic.
- Use read_document(id=dN) when a returned passage is incomplete: page with offset, or pass query to find a passage inside the document. Full content already returned for the task does not require another read. list_documents shows what a knowledge base contains when the search surface is unclear.
- For exact quotes, numbers, or code, verify the underlying source passage rather than relying on document metadata or a generated overview.
Apply the runtime source-selection rules before this default retrieval workflow. Other requested sources and deliverables are part of the user's task.
diff --git a/internal/agent/tools/search_knowledge.go b/internal/agent/tools/search_knowledge.go
index 58693ffa6..933a67d1b 100644
--- a/internal/agent/tools/search_knowledge.go
+++ b/internal/agent/tools/search_knowledge.go
@@ -41,11 +41,12 @@ var searchKnowledgeTool = BaseTool{
"names the subject and what you want to know (\"Why does Echo use open-weight models to cut cost?\"), " +
"not a keyword list (\"Echo open-weight cost\"): results are ranked by a relevance model that scores " +
"meaning. In keyword mode write the exact terms. Pass knowledge_base_ids to focus on the bases whose " +
- "profile fits the question. When nothing passes the relevance check, rephrase the question; switching " +
- "mode alone does not help.\n" +
+ "profile fits the question. When nothing passes the relevance check, rephrase with terms the documents " +
+ "would use; repeating the same query in another mode usually does not help. For identifiers, error " +
+ "codes or exact names retry with mode=keyword.\n" +
"Every chunk carries a cN handle and belongs to a dN document. Use read_document(id=dN) to read the " +
- "surrounding context or the whole document. Searches match chunk text only; to find a document by its " +
- "title or file name use list_documents(keyword=...).",
+ "surrounding context or the whole document. Retrieval matches chunk text; the relevance model also " +
+ "sees the document title. To find a document by its title or file name use list_documents(keyword=...).",
schema: json.RawMessage(`{
"type": "object",
"properties": {
@@ -305,16 +306,17 @@ func (t *SearchKnowledgeTool) Execute(ctx context.Context, args json.RawMessage)
// emptySearchStatement describes an empty result, including any mode
// fallback, so the model does not read "no semantic neighbours" as "the
// exact term does not occur". When retrieval found candidates and the rerank
-// model rejected all of them, it says so: every mode shares that relevance
-// check, so the model should rephrase instead of cycling through modes.
+// model rejected all of them, it says so: repeating the same query in
+// another mode is usually wasted, but keyword mode can still surface a
+// different candidate set for identifiers and exact terms.
func emptySearchStatement(query string, data map[string]interface{}, kbCount int) string {
mode, _ := data["mode"].(string)
msg := fmt.Sprintf("No matching chunks for %q (mode=%s) in %d knowledge base(s).", query, mode, kbCount)
if rejected, _ := data["rerank_rejected"].(int); rejected > 0 {
msg = fmt.Sprintf("No chunk passed the relevance check for %q (mode=%s) in %d knowledge base(s): "+
"retrieval found %d candidate chunks but the relevance model scored none of them as answering the "+
- "query. Every mode applies the same check, so switching mode will not help; rephrase as a complete "+
- "question that names the subject, or use terms the documents themselves would use.",
+ "query. Repeating the same query in another mode usually will not help; rephrase with terms the "+
+ "documents would use. For identifiers, error codes or exact names retry with mode=keyword.",
query, mode, kbCount, rejected)
}
fallbacks, _ := data["mode_fallbacks"].([]map[string]interface{})
diff --git a/internal/agent/tools/search_knowledge_rerank_test.go b/internal/agent/tools/search_knowledge_rerank_test.go
index 918efab2b..3031643ee 100644
--- a/internal/agent/tools/search_knowledge_rerank_test.go
+++ b/internal/agent/tools/search_knowledge_rerank_test.go
@@ -2,7 +2,9 @@ package tools
import (
"context"
+ "encoding/json"
"errors"
+ "strings"
"testing"
"github.com/Tencent/WeKnora/internal/config"
@@ -220,6 +222,67 @@ func TestRerankScores_prefixDocumentTitle(t *testing.T) {
}
}
+// Title is shared by every chunk of a document, so a passage whose body never
+// names the query subject still goes to the reranker as title + chunk.
+func TestRerankPassage_prefixesTitleWhenChunkOmitsSubject(t *testing.T) {
+ t.Parallel()
+ got := (&SearchKnowledgeTool{}).rerankPassage(context.Background(), &types.SearchResult{
+ Content: "installation prerequisites and docker compose flags",
+ KnowledgeTitle: "Show HN: Echo – Fable-level results at 1/3 the cost using open-weight models",
+ ChunkType: string(types.ChunkTypeText),
+ })
+ if !strings.HasPrefix(got, "Show HN: Echo") {
+ t.Fatalf("expected document title prefix, got %q", got)
+ }
+ if !strings.Contains(got, "installation prerequisites") {
+ t.Fatalf("chunk body must still be present: %q", got)
+ }
+}
+
+// Execute must surface rerank_rejected through Data and the empty statement
+// when the model scores every retrieved candidate below the fallback floor.
+func TestExecuteReportsRerankRejection(t *testing.T) {
+ t.Parallel()
+ model := &stubReranker{scores: []float64{0.10, 0.04}}
+ svc := &stubKnowledgeBaseService{
+ results: []*types.SearchResult{
+ {
+ ID: "c1", Content: "comments section", KnowledgeID: "d1", KnowledgeBaseID: "kb-1",
+ KnowledgeTitle: "Show HN: Echo", Score: 0.8,
+ },
+ {
+ ID: "c2", Content: "unrelated notes", KnowledgeID: "d2", KnowledgeBaseID: "kb-1",
+ KnowledgeTitle: "Other", Score: 0.7,
+ },
+ },
+ }
+ tool := NewSearchKnowledgeTool(
+ svc, nil, nil,
+ types.SearchTargets{{
+ Type: types.SearchTargetTypeKnowledgeBase, KnowledgeBaseID: "kb-1", TenantID: 1,
+ }},
+ model,
+ &config.Config{Conversation: &config.ConversationConfig{RerankThreshold: 0.3}},
+ )
+ res, err := tool.Execute(context.Background(), json.RawMessage(
+ `{"query":"Why does Echo use open-weight models to cut cost?"}`,
+ ))
+ if err != nil || res == nil || !res.Success {
+ t.Fatalf("Execute: res=%+v err=%v", res, err)
+ }
+ if got, _ := res.Data["rerank_rejected"].(int); got != 2 {
+ t.Fatalf("rerank_rejected = %v, want 2; data=%+v", res.Data["rerank_rejected"], res.Data)
+ }
+ if !strings.Contains(res.Output, "found 2 candidate chunks") ||
+ !strings.Contains(res.Output, "Repeating the same query") ||
+ !strings.Contains(res.Output, "mode=keyword") {
+ t.Fatalf("empty statement = %q", res.Output)
+ }
+ if strings.Contains(res.Output, "switching mode will not help") {
+ t.Fatalf("statement must not forbid a keyword retry: %q", res.Output)
+ }
+}
+
func TestRerankThreshold_default(t *testing.T) {
t.Parallel()
tool := &SearchKnowledgeTool{}
diff --git a/internal/agent/tools/search_knowledge_test.go b/internal/agent/tools/search_knowledge_test.go
index 1521591a3..3c73329a4 100644
--- a/internal/agent/tools/search_knowledge_test.go
+++ b/internal/agent/tools/search_knowledge_test.go
@@ -89,15 +89,19 @@ func TestResolveKBSearchModesFallsBackPerKnowledgeBase(t *testing.T) {
}
// When the rerank model rejected every candidate, the statement must not
-// read as "nothing was found": switching mode cannot get past that check.
+// read as "nothing was found": repeating the same query in another mode is
+// usually wasted, but keyword mode remains the retry for exact terms.
func TestEmptySearchStatementReportsRerankRejection(t *testing.T) {
data := map[string]interface{}{"mode": SearchModeHybrid, "rerank_rejected": 18}
msg := emptySearchStatement("Echo open-weight cost", data, 20)
- for _, want := range []string{"found 18 candidate chunks", "switching mode will not help", "complete question"} {
+ for _, want := range []string{"found 18 candidate chunks", "Repeating the same query", "mode=keyword"} {
if !strings.Contains(msg, want) {
t.Fatalf("statement missing %q: %q", want, msg)
}
}
+ if strings.Contains(msg, "switching mode will not help") {
+ t.Fatalf("statement must not forbid switching to keyword: %q", msg)
+ }
if plain := emptySearchStatement("x", map[string]interface{}{"mode": SearchModeHybrid}, 1); !strings.HasPrefix(
plain, "No matching chunks") {
t.Fatalf("no rerank rejection keeps the plain statement: %q", plain)