RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Port Agentic RAG to Golang (#19563)
Yingfeng4e54fe13388b
diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.goindex bdee103835..310b557d9a 100644--- a/cmd/ragflow_server.go+++ b/cmd/ragflow_server.go@@ -949,6 +949,13 @@ func startServer(ctx context.Context) {Model: model,ModelName: resolvedModelName,Outer: outerModel,+ // OriginalQuestion mirrors Python RAGTools(original_user_question=...):+ // the user's own, unrewritten question as received from the chat+ // layer. The outer model's `rag(question=...)` argument is+ // model-generated and often compresses a multi-hop question to its+ // first hop; resolveEffectiveQuestion (agentic_rag.py:865) prefers+ // this original over that rewrite when both describe the same turn.+ OriginalQuestion: req.Question,// OuterSupportsTools gates the outer react loop on tool capability// (Python is_tools); false → fall back to direct RunAgenticRAG.OuterSupportsTools: outerSupportsTools,@@ -982,11 +989,48 @@ func startServer(ctx context.Context) {WebSearch: harnessWebSearcher(req.WebSearch),KBs: kbs,HasEmbedder: hasEmbedder,+ // Embedder backs claim recall's KNN leg (Python+ // recall_dataset_claims, navigation.py:1837-1909, which embeds the+ // query with tools.embed_mdl — the FIRST dataset's embedding model,+ // dialog_service.py:362-366) and the structure-drill seed vector.+ // Without it the claim leg silently degrades to BM25-only and+ // paraphrase-phrased claims are never recalled. Bound to+ // kbs[0].EmbdID (validateDatasetEmbeddingModels above guarantees+ // every bound dataset shares it), query-side encoded.+ Embedder: embedderForDatasets(kbs, modelProviderService),// Tagger is the Go equivalent of Python's label_question// (agentic_rag.py:668): classifies the query into question-type// tags the retriever boosts on. metadataService implements it// (service.MetadataService.LabelQuestion).Tagger: metadataService,+ // SystemPrompt mirrors Python RAGTools(system_prompt=+ // _render_reasoning_system_prompt(dialog, prompt_config, kwargs),+ // dialog_service.py:2084) — the dialog-level UI configuration the+ // final-answer compose appends after the agentic contract+ // (agentic_rag_graph.py:923-933).+ SystemPrompt: req.SystemPrompt,+ // CiteRules and EvidenceMaxTokens deliberately stay unset: Python's+ // dialog path constructs RAGTools WITHOUT user_defined_prompts+ // (dialog_service.py:2072-2090), so citation_prompt defaults apply,+ // and Python has no evidence-token override (the compose always+ // uses min(chat_mdl.max_length, _EVIDENCE_BUDGET_TOKENS=8000),+ // which EvidenceMaxTokens<=0 reproduces).+ }+ // Diagnose WHY compiled expansion is disabled: NewCompiledExpander+ // returns nil for three reasons (store==nil / no datasets / no tenant)+ // and RAGTools.Expand==nil silences the whole channel with no other+ // trace — the observed "Compiled expansion enabled = 0" was invisible.+ if deps.Expand == nil {+ switch {+ case docEngine == nil:+ common.Warn("compiled expansion disabled: document engine unavailable (store==nil)")+ case len(req.DatasetIDs) == 0:+ common.Warn("compiled expansion disabled: no bound dataset (DatasetIDs empty)")+ case strings.TrimSpace(req.TenantID) == "":+ common.Warn("compiled expansion disabled: no tenant (TenantID empty)")+ default:+ common.Warn("compiled expansion disabled: unknown reason")+ }}if req.AnswerSink != nil {deps.AnswerSink = &advanced_rag.AnswerSink{@@ -1410,6 +1454,22 @@ func hasEmbedderFor(kbs []*entity.Knowledgebase) bool {return len(kbs) > 0 && kbs[0] != nil && kbs[0].EmbdID != ""}+// embedderForDatasets builds the embedding handle the agentic harness uses for+// query-side encoding outside the main retrieval leg: claim recall's KNN leg+// (Python recall_dataset_claims, navigation.py:1837-1909, embedding with+// tools.embed_mdl) and the structure-drill seed vector. The model is the FIRST+// dataset's embedding (Python dialog_service.py:362-366 resolves+// kbs[0].embd_id under kbs[0].tenant_id; validateDatasetEmbeddingModels+// guarantees the rest share it). Nil when no dataset carries an embedding+// model — the claim leg then degrades to BM25-only, matching a Python run+// without embed_mdl.+func embedderForDatasets(kbs []*entity.Knowledgebase, modelSvc *service.ModelProviderService) nlp.NavEmbedder {+ if len(kbs) == 0 || kbs[0] == nil || kbs[0].EmbdID == "" {+ return nil+ }+ return service.NewNavEmbedder(modelSvc, kbs[0].EmbdID)+}+// validateDatasetEmbeddingModels mirrors Python validate_dataset_embedding_models// (knowledgebase_service.py:62-94): every bound dataset must use the same embedding// model, or none at all. Upstream this runs before embd_mdl is resolveddiff --git a/internal/agent/component/llm.go b/internal/agent/component/llm.goindex aeeb78bf75..46e39f04c5 100644--- a/internal/agent/component/llm.go+++ b/internal/agent/component/llm.go@@ -1401,6 +1401,12 @@ func validateFittedMessages(msgFit []schema.Message) string {return ""}+// FitMessages exposes message_fit_in semantics (LLM.fit_messages, PR #16413)+// for packages that compose final-answer prompts outside the agent loop.+func FitMessages(systemPrompt string, msgs []schema.Message, maxLength int) ([]schema.Message, string) {+ return fitMessages(systemPrompt, msgs, maxLength)+}+// fitMessages calls message_fit_in semantics on the given messages and// validates that the result ends with a non-empty user turn. Returns the// fitted messages and an error string (empty on success).diff --git a/internal/agent/retrievalbridge/runtime_adapter.go b/internal/agent/retrievalbridge/runtime_adapter.goindex 2feae1dc84..a2d20ff3ca 100644--- a/internal/agent/retrievalbridge/runtime_adapter.go+++ b/internal/agent/retrievalbridge/runtime_adapter.go@@ -56,12 +56,20 @@ func (a *RuntimeAdapter) Search(ctx context.Context, db *gorm.DB, req agentrunt.db = dao.DB}toolReq := agenttool.RetrievalRequest{- Query: req.Query,- DatasetIDs: req.DatasetIDs,- MemoryIDs: req.MemoryIDs,- TopN: req.TopN,- RerankCandidatesCount: req.RerankCandidatesCount,- TopK: req.TopK,+ Query: req.Query,+ DatasetIDs: req.DatasetIDs,+ MemoryIDs: req.MemoryIDs,+ TopN: req.TopN,+ RerankCandidatesCount: req.RerankCandidatesCount,+ TopK: req.TopK,+ // VectorSimilarityWeight is the VECTOR weight (Python+ // vector_similarity_weight) and is forwarded to the nlp layer+ // UN-inverted. KeywordsSimilarityWeight (canvas semantics, keyword+ // weight) stays on its own inversion path in nlpRequestFromRetrieval —+ // conflating the two turned the agentic hybrid leg vector-dominant and+ // the BM25 legs into pure-vector searches.+ VectorSimilarityWeight: req.VectorSimilarityWeight,+ DisableVectorLeg: req.DisableVectorLeg,KeywordsSimilarityWeight: req.KeywordsSimilarityWeight,UseKG: req.UseKG,SimilarityThreshold: req.SimilarityThreshold,diff --git a/internal/agent/runtime/retrieval.go b/internal/agent/runtime/retrieval.goindex e7fe3c05bd..f0646cf73c 100644--- a/internal/agent/runtime/retrieval.go+++ b/internal/agent/runtime/retrieval.go@@ -59,12 +59,20 @@ type RetrievalChunk struct {// RetrievalRequest is the input to RetrievalService.Search.type RetrievalRequest struct {- Query string- DatasetIDs []string- MemoryIDs []string- TopN int- RerankCandidatesCount int- TopK int+ Query string+ DatasetIDs []string+ MemoryIDs []string+ TopN int+ RerankCandidatesCount int+ TopK int+ // VectorSimilarityWeight is the VECTOR leg's weight (Python+ // vector_similarity_weight), forwarded verbatim by the agentic harness. The+ // canvas path keeps using KeywordsSimilarityWeight (the keyword weight the+ // adapter inverts); see retrievalbridge/runtime_adapter.go.+ VectorSimilarityWeight *float64+ // DisableVectorLeg mirrors Python passing embd_mdl=None: the backend runs+ // the keyword-only branch with NO dense leg (not even a weight-0 one).+ DisableVectorLeg boolKeywordsSimilarityWeight *float64UseKG boolSimilarityThreshold *float64diff --git a/internal/agent/tool/dataset_navigation.go b/internal/agent/tool/dataset_navigation.goindex d302679c59..6dd1c4d8fe 100644--- a/internal/agent/tool/dataset_navigation.go+++ b/internal/agent/tool/dataset_navigation.go@@ -20,14 +20,31 @@ import ("context""encoding/json""fmt"+ "log"+ "sort""strings"einotool "github.com/cloudwego/eino/components/tool""github.com/cloudwego/eino/schema"+ "ragflow/internal/dao""ragflow/internal/service/nav")+// Content-recall fallback tunables, mirroring Python _content_recall_docs+// (navigation.py:387-394, :539, :547-549).+const (+ // datasetNavRecallTopN is _NAV_RECALL_TOP_N: chunk candidates fetched+ // before doc aggregation.+ datasetNavRecallTopN = 40+ // datasetNavRecallMinScore is the literal similarity_threshold Python+ // passes (:548).+ datasetNavRecallMinScore = 0.2+ // datasetNavRecallVectorWeight is the hybrid vector blend Python hardcodes+ // when an embedder exists (:539): `vector_weight = 0.3 if embd_mdl else 0`.+ datasetNavRecallVectorWeight = 0.3+)+// datasetNavigationToolName mirrors Python's dataset_navigation_by_tree router// tool. It navigates the dataset nav tree and returns the doc_ids to read.const datasetNavigationToolName = "dataset_navigation_by_tree"@@ -194,10 +211,60 @@ func (d *DatasetNavigationByTree) InvokableRun(ctx context.Context, argumentsInJ}}- // Fallback: if semantic routing found nothing (e.g. no embedder), walk the- // root clusters so the tool still returns a useful (if coarse) doc set. The- // scope still applies — collect() filters these leaves, standing in for- // Python's _content_recall_docs(tools, query, doc_scope).+ // Fallback 1 — content recall (Python _content_recall_docs,+ // navigation.py:514-565, the miss-tier of dataset_navigation_by_tree): when+ // no compiled tree routed — the nav rows do not exist or matched nothing —+ // recall documents by chunk CONTENT. A plain hybrid retrieval runs over the+ // datasets' chunk index and the hits aggregate to docs most-hit-first: a+ // question matching detail that only lives in a document BODY never appears+ // in the tree, so the retrieval that reads real chunk text is what catches+ // it. The caller's doc scope is forwarded as DocScope (Python forwards it+ // as doc_ids to the retrieval), and collect()'s inScope still applies.+ if len(docs) == 0 {+ threshold := datasetNavRecallMinScore+ w := datasetNavRecallVectorWeight+ chunks, err := GetRetrievalService().Search(ctx, dao.DB, RetrievalRequest{+ Query: query,+ DatasetIDs: datasetIDs,+ TopN: datasetNavRecallTopN,+ SimilarityThreshold: &threshold,+ VectorSimilarityWeight: &w,+ TenantID: tenantID,+ DocScope: docScope,+ RetrievalFrom: "dataset",+ })+ if err != nil {+ log.Printf("[Dataset navigation] content-recall retrieval failed: %v", err)+ } else {+ // Python :557-563 — the retrieval's doc_aggs read in order; the ES+ // aggregation orders by hit count descending, so the same order is+ // derived from the flat chunk hits here.+ order := []string{}+ counts := map[string]int{}+ for _, c := range chunks {+ did := strings.TrimSpace(c.DocumentID)+ if did == "" {+ continue+ }+ if _, ok := counts[did]; !ok {+ order = append(order, did)+ }+ counts[did]+++ }+ sort.SliceStable(order, func(i, j int) bool { return counts[order[i]] > counts[order[j]] })+ log.Printf("[Dataset navigation] Content recall found %d candidate doc(s).", len(order))+ for _, did := range order {+ collect(did)+ }+ }+ }++ // Fallback 2 — Go-only last resort: if even content recall found nothing+ // (e.g. no retrieval service wired), walk the root clusters so the tool+ // still returns a useful (if coarse) doc set. Python has no such tier — its+ // walk ends at content recall — and this one applies no relevance signal+ // beyond cluster order, so it must stay BEHIND the recall tier. The scope+ // still applies — collect() filters these leaves.if len(docs) == 0 {for _, datasetID := range datasetIDs {clusters, _, err := ns.ListClusters(ctx, tenantID, datasetID, 0, 100)diff --git a/internal/agent/tool/retrieval_nlp.go b/internal/agent/tool/retrieval_nlp.goindex d4ae6c70d9..5ad301e62b 100644--- a/internal/agent/tool/retrieval_nlp.go+++ b/internal/agent/tool/retrieval_nlp.go@@ -353,7 +353,20 @@ func nlpRequestFromRetrieval(if req.SimilarityThreshold != nil {nlpReq.SimilarityThreshold = req.SimilarityThreshold}- if req.KeywordsSimilarityWeight != nil {+ if req.DisableVectorLeg {+ // Python embd_mdl=None (bm25_search / grep_search / retrieve with+ // using_embedding=False): keyword-only search, NO dense leg at all —+ // not even a weight-0 one, which would still constrain the candidate+ // pool through the KNN similarity option.+ nlpReq.EmbeddingModel = nil+ }+ if req.VectorSimilarityWeight != nil {+ // Agentic harness path: the vector weight arrives already in vector+ // semantics (Python vector_similarity_weight) — forward verbatim.+ nlpReq.VectorSimilarityWeight = req.VectorSimilarityWeight+ } else if req.KeywordsSimilarityWeight != nil {+ // Canvas path: keywords_similarity_weight is the KEYWORD weight+ // (user-facing config); the vector weight is its complement.vectorSimilarityWeight := 1 - *req.KeywordsSimilarityWeightnlpReq.VectorSimilarityWeight = &vectorSimilarityWeight}diff --git a/internal/agent/tool/retrieval_service.go b/internal/agent/tool/retrieval_service.goindex a064f89bc9..a6b1d670f6 100644--- a/internal/agent/tool/retrieval_service.go+++ b/internal/agent/tool/retrieval_service.go@@ -59,11 +59,18 @@ type RetrievalRequest struct {RerankCandidatesCount intTopK intKeywordsSimilarityWeight *float64- UseKG bool- SimilarityThreshold *float64- RerankID string- CrossLanguages []string- TOCEnhance bool+ // VectorSimilarityWeight is the VECTOR leg's weight forwarded DIRECTLY (no+ // 1-w inversion) by the agentic harness bridge. When set it wins over the+ // canvas KeywordsSimilarityWeight inversion in nlpRequestFromRetrieval.+ VectorSimilarityWeight *float64+ // DisableVectorLeg mirrors Python embd_mdl=None: the nlp layer runs the+ // keyword-only branch with NO dense leg at all.+ DisableVectorLeg bool+ UseKG bool+ SimilarityThreshold *float64+ RerankID string+ CrossLanguages []string+ TOCEnhance bool// RankFeature mirrors Python RAGTools.retrieve's rank_feature argument:// question-type tags (from label_question) the retriever uses to boost// matching chunks. The Go engine consumes it as a tag → weight map, so itdiff --git a/internal/common/logger.go b/internal/common/logger.goindex f52e2c0f9e..3d7e99ef11 100644--- a/internal/common/logger.go+++ b/internal/common/logger.go@@ -229,16 +229,31 @@ func Warn(msg string, fields ...zap.Field) {// project's structured logs. When the project logger has not been initialized// yet (e.g. before InitLogger runs or in standalone tests) it falls back to the// standard-library default. The returned logger writes at Info level.+//+// The returned *log.Logger resolves the write target LAZILY on every write.+// Package-level variables like `var _LOG = common.StdLogger()` are evaluated+// during package init, long before InitLogger runs; a logger captured eagerly+// at that moment would be log.Default() forever and its output would vanish+// into stderr, never reaching the structured log files.func StdLogger() *log.Logger {- if Logger == nil {- return log.Default()- }stdLogOnce.Do(func() {- stdLogger = zap.NewStdLog(Logger)+ stdLogger = log.New(stdLogRouter{}, "", 0)})return stdLogger}+// stdLogRouter dispatches *log.Logger writes to the current global logger on+// every write (see StdLogger).+type stdLogRouter struct{}++func (stdLogRouter) Write(p []byte) (int, error) {+ if Logger == nil {+ return os.Stderr.Write(p)+ }+ Logger.Info(strings.TrimRight(string(p), "\n"))+ return len(p), nil+}+// IsDebugEnabled returns true if debug logging is enabled.func IsDebugEnabled() bool {return atomicLevel.Enabled(zapcore.DebugLevel)diff --git a/internal/entity/models/chat_tools.go b/internal/entity/models/chat_tools.goindex c3ea0b1ddc..f84eccdf24 100644--- a/internal/entity/models/chat_tools.go+++ b/internal/entity/models/chat_tools.go@@ -20,6 +20,7 @@ import ("context""encoding/json""fmt"+ "strings""sync""ragflow/internal/tokenizer"@@ -157,10 +158,12 @@ func runToolLoop(ctx context.Context, cm *ChatModel, history []Message, toolsLis// Execute the round's tool calls and fold their results into history.// If one of them is a terminal tool and succeeded, its result is already// the final answer — return it directly instead of re-invoking the model- // (Python chat_model.py:619-627).+ // (Python chat_model.py:619-627). Non-streaming matches Python's+ // all-empty fallthrough: no qualifying terminal result and the loop+ // runs another round (chat_model.py:692-704).var hit boolvar toolAnswer string- history, toolAnswer, hit = appendToolResults(history, resp.ToolCalls, cm.ToolConfig.ToolCallSession, terminalSet(cm))+ history, toolAnswer, hit = appendToolResults(history, resp.ToolCalls, cm.ToolConfig.ToolCallSession, terminalSet(cm), false)if hit {// This round's usage was already folded in by addRoundUsage above// (it accumulates into aggUsage and forwards a per-round delta to the@@ -363,9 +366,14 @@ func runStreamToolLoop(ctx context.Context, cm *ChatModel, history []Message, to// A terminal tool's successful result is already the final answer:// stream it to the caller and stop instead of asking the model again// (Python chat_model.py:2574-2582). sendTerminal streams the result.+ // Streaming passes emptyTerminalIsHit=true: a Go streaming tool returns+ // "" to say "already delivered through the sink", and the loop must+ // stop even when the folded content is empty — unlike Python, whose rag+ // tool returns the full text and whose all-empty fallthrough would only+ // buy an extra model round whose output the mux drops.var termAnswer stringvar termHit bool- history, termAnswer, termHit = appendToolResults(history, toolCalls, cm.ToolConfig.ToolCallSession, terminalSet(cm))+ history, termAnswer, termHit = appendToolResults(history, toolCalls, cm.ToolConfig.ToolCallSession, terminalSet(cm), true)if termHit {if err := sendTerminal(sender, &termAnswer); err != nil {return totalTokens, err@@ -401,9 +409,22 @@ func runStreamToolLoop(ctx context.Context, cm *ChatModel, history []Message, to// When terminal is non-empty, a successful call to one of those tools// short-circuits: the loop returns (history, that result, true) so the caller// treats it as the final answer instead of re-invoking the model (mirrors-// Python chat_model.py:619-627 / :2574-2582). When terminal is empty or no-// terminal tool fired, it returns (history, "", false).-func appendToolResults(history []Message, toolCalls []map[string]interface{}, session ToolCallSession, terminal map[string]struct{}) ([]Message, string, bool) {+// Python chat_model.py:619-627 / :2574-2582). An EMPTY terminal result does+// not win on its own: Python's fold only returns on a non-empty string+// (`if out:`, chat_model.py:696), so an empty result is skipped in favour of+// a later non-empty sibling. When NO terminal result is non-empty,+// emptyTerminalIsHit decides:+// - false (non-streaming, Python :692-704 fallthrough): no hit — the tool+// responses are in history and the loop runs another model round, which+// then answers from them or re-calls the tool;+// - true (streaming): hit with the first successful call's (empty) content.+// The Go streaming tool returns "" to say "already delivered through the+// sink", and the loop must stop (sendTerminal stays silent) instead of+// paying another model round whose output the mux would only drop.+//+// When terminal is empty or no terminal tool fired, it returns+// (history, "", false) either way.+func appendToolResults(history []Message, toolCalls []map[string]interface{}, session ToolCallSession, terminal map[string]struct{}, emptyTerminalIsHit bool) ([]Message, string, bool) {if session == nil {history = append(history, Message{Role: "assistant",@@ -474,21 +495,37 @@ func appendToolResults(history []Message, toolCalls []map[string]interface{}, seToolCalls: toolCalls,})- // A successful terminal tool is the final answer; remember its result but- // still append every tool message so history stays well-formed if the- // caller ignores the short-circuit (defensive).- var terminalAnswer string- var terminalHit bool+ // Every tool_call the assistant declared gets its matching tool message,+ // well-formed history whichever path the caller takes. Then two passes over+ // the (tiny) results slice pick the fold: the FIRST non-empty successful+ // terminal result wins (Python chat_model.py:696 `if out:` — an empty+ // terminal result never ships while a non-empty sibling exists, trimmed+ // whitespace counts as empty), then the empty fallthrough per+ // emptyTerminalIsHit (see the doc comment above).for _, r := range results {history = append(history, Message{Role: "tool",Content: r.content,ToolCallID: r.tcID,})- if !terminalHit && r.err == nil && r.name != "" {+ }+ var terminalAnswer string+ var terminalHit bool+ for _, r := range results {+ if r.err == nil && r.name != "" && strings.TrimSpace(r.content) != "" {if _, ok := terminal[r.name]; ok {- terminalAnswer = r.content- terminalHit = true+ terminalAnswer, terminalHit = r.content, true+ break+ }+ }+ }+ if !terminalHit && emptyTerminalIsHit {+ for _, r := range results {+ if r.err == nil && r.name != "" {+ if _, ok := terminal[r.name]; ok {+ terminalAnswer, terminalHit = r.content, true+ break+ }}}}diff --git a/internal/entity/models/chat_tools_test.go b/internal/entity/models/chat_tools_test.goindex beaeb57efe..e6e0ae2c2c 100644--- a/internal/entity/models/chat_tools_test.go+++ b/internal/entity/models/chat_tools_test.go@@ -53,7 +53,7 @@ func TestAppendToolResultsTerminalHit(t *testing.T) {hist, answer, hit := appendToolResults(history, []map[string]interface{}{toolCallMsg("call-1", "summarize_document", `{"doc_id":"abc"}`),- }, sess, term)+ }, sess, term, false)if !hit {t.Fatal("terminal tool success must short-circuit")@@ -75,7 +75,7 @@ func TestAppendToolResultsTerminalErrorDoesNotShortCircuit(t *testing.T) {_, answer, hit := appendToolResults(history, []map[string]interface{}{toolCallMsg("call-1", "nonterminal", `{}`),- }, &stubSession{}, term)+ }, &stubSession{}, term, false)if hit || answer != "" {t.Fatal("non-terminal call must not short-circuit")}@@ -86,7 +86,7 @@ func TestAppendToolResultsNoTerminalConfigured(t *testing.T) {history := []Message{{Role: "user", Content: "q"}}hist, answer, hit := appendToolResults(history, []map[string]interface{}{toolCallMsg("call-1", "rag", `{}`),- }, &stubSession{}, nil)+ }, &stubSession{}, nil, false)if hit || answer != "" {t.Fatal("no terminal set must never short-circuit")}@@ -101,7 +101,7 @@ func TestAppendToolResultsNonTerminalIgnoresSet(t *testing.T) {term := map[string]struct{}{"rag": {}}_, answer, hit := appendToolResults(history, []map[string]interface{}{toolCallMsg("call-1", "retrieve", `{}`),- }, &stubSession{}, term)+ }, &stubSession{}, term, false)if hit || answer != "" {t.Fatal("non-terminal tool must not short-circuit even with a terminal set present")}@@ -135,8 +135,11 @@ func TestSetTerminalToolsWithoutToolsIsNoop(t *testing.T) {}// A terminal tool that already streamed its answer returns "" to say "nothing-// more to emit". It must still short-circuit, so the loop stops instead of-// asking the model again.+// more to emit". On the STREAMING path (emptyTerminalIsHit=true) it must still+// short-circuit, so the loop stops instead of asking the model again — the Go+// streaming tool's "" means "already delivered", unlike Python's rag tool,+// which returns the full text and whose all-empty fold falls through+// (chat_model.py:692-704).func TestAppendToolResultsTerminalHitWithEmptyResult(t *testing.T) {history := []Message{{Role: "user", Content: "q"}}sess := &stubSession{results: map[string]string{"rag": ""}}@@ -144,10 +147,10 @@ func TestAppendToolResultsTerminalHitWithEmptyResult(t *testing.T) {hist, answer, hit := appendToolResults(history, []map[string]interface{}{toolCallMsg("call-1", "rag", `{"question":"q"}`),- }, sess, term)+ }, sess, term, true)if !hit {- t.Fatal("a successful terminal tool must short-circuit even with an empty result")+ t.Fatal("a successful terminal tool must short-circuit even with an empty result (streaming contract)")}if answer != "" {t.Fatalf("answer = %q, want empty", answer)@@ -157,6 +160,73 @@ func TestAppendToolResultsTerminalHitWithEmptyResult(t *testing.T) {}}+// questionKeyedSession returns a canned result per question argument, so two+// same-name tool calls can be told apart regardless of the goroutine order+// appendToolResults executes them in.+type questionKeyedSession struct {+ results map[string]string+}++func (s *questionKeyedSession) ToolCall(_ string, args map[string]interface{}) (string, error) {+ if q, ok := args["question"].(string); ok {+ if r, ok := s.results[q]; ok {+ return r, nil+ }+ }+ return "", nil+}++// TestAppendToolResultsSkipsEmptyTerminalForNonEmptySibling pins the fold+// alignment with Python chat_model.py:696 (`if out:`): an EMPTY (or+// whitespace-only) terminal result never ships while a later non-empty+// sibling exists — the second call's answer is the final one.+func TestAppendToolResultsSkipsEmptyTerminalForNonEmptySibling(t *testing.T) {+ history := []Message{{Role: "user", Content: "q"}}+ sess := &questionKeyedSession{results: map[string]string{+ "first": "",+ "second": " \n\t",+ "third": "103 years",+ }}+ term := map[string]struct{}{"rag": {}}++ _, answer, hit := appendToolResults(history, []map[string]interface{}{+ toolCallMsg("call-1", "rag", `{"question":"first"}`),+ toolCallMsg("call-2", "rag", `{"question":"second"}`),+ toolCallMsg("call-3", "rag", `{"question":"third"}`),+ }, sess, term, false)++ if !hit {+ t.Fatal("the non-empty terminal sibling must short-circuit")+ }+ if answer != "103 years" {+ t.Fatalf("answer = %q, want the non-empty sibling's result", answer)+ }+}++// TestAppendToolResultsAllEmptyTerminalFallsThrough pins the non-streaming+// all-empty case against Python chat_model.py:692-704 (no else branch: an+// unqualified terminal result is skipped, the tool responses land in history+// and the loop runs another model round): hit=false so the caller re-invokes+// the model instead of shipping a blank answer.+func TestAppendToolResultsAllEmptyTerminalFallsThrough(t *testing.T) {+ history := []Message{{Role: "user", Content: "q"}}+ sess := &stubSession{results: map[string]string{"rag": ""}}+ term := map[string]struct{}{"rag": {}}++ hist, answer, hit := appendToolResults(history, []map[string]interface{}{+ toolCallMsg("call-1", "rag", `{"question":"q"}`),+ }, sess, term, false)++ if hit || answer != "" {+ t.Fatal("all-empty terminal results must fall through to the next model round")+ }+ // History still carries the assistant tool_calls + tool result, well-formed+ // for the next round.+ if len(hist) != 3 {+ t.Fatalf("history length = %d, want 3 (user + assistant + tool)", len(hist))+ }+}+// sendTerminal must stay silent for an empty answer, so a streamed terminal// result is not emitted a second time by the short-circuit.func TestSendTerminalEmptyAnswerIsSilent(t *testing.T) {diff --git a/internal/rag/advanced_rag/agentic_rag.go b/internal/rag/advanced_rag/agentic_rag.goindex 3e8cb71b2b..5f08d4284f 100644--- a/internal/rag/advanced_rag/agentic_rag.go+++ b/internal/rag/advanced_rag/agentic_rag.go@@ -168,9 +168,19 @@ type RAGTools struct {// Finalize runs the terminal composition (Python _compose_answer_from_evidence).// It is wired by Rag so the graph can invoke it FROM INSIDE its last node —// Python's agentic and low graphs both end in a formalize_answer node that- // composes and streams the answer. Nil means "the caller- // composes afterwards", which is only correct for the naive path.- Finalize func(ctx context.Context)+ // composes and streams the answer. The graph forwards the node's state+ // values the compose prompt reads: partialAnswer and emptyResult (Python+ // agentic_rag_graph.py:1397-1400 / orchestrator/direct.py) plus question —+ // the graph state's FORMALIZED question (Python :834+ // `question = state.get("question")`), which the formalize_question node+ // wrote. The compose prompt must be built from the formalized question,+ // not the outer tool argument: the graph researched the formalized+ // multi-hop question, so composing from the compressed outer argument+ // collapses the final answer to the first completed sub-answer. Empty+ // question means "the caller composes afterwards" (post-graph fallback) —+ // composeFinalAnswer then falls back to req.Question. Nil Finalize means+ // the caller composes afterwards, which is only correct for the naive path.+ Finalize func(ctx context.Context, partialAnswer, emptyResult bool, question string)// Messages is the conversation history used by Formalize to resolve// pronouns/ellipses into a standalone question. The agentic and low graphs// formalize it as their first node (Python build_agentic_graph and@@ -950,17 +960,18 @@ func (c *RAGCache) noteVerdict(verdict string) {// researchStatusTrailer mirrors Python rag (:902-929): for every non-SUFFICIENT// verdict it returns the trailing sentence folded into the "[Research status]"-// note. After two consecutive unsatisfying rag() calls (ConsecutiveUnanswerable-// >= 2 on the shared *RAGCache) it tells the outer agent to STOP calling rag-// again; otherwise it invites a focused re-ask. It returns "" when there is-// nothing to annotate — a SUFFICIENT verdict, an empty answer, or no SCA-// feedback.+// note — worded exactly as agentic_rag.py:927/:929. After two consecutive+// unsatisfying rag() calls (ConsecutiveUnanswerable >= 2 on the shared+// *RAGCache, Python _consecutive_unanswerable >= _GUA=2) it tells the outer+// agent to STOP calling rag again; otherwise it invites a focused re-ask. It+// returns "" when there is nothing to annotate — a SUFFICIENT verdict, an empty+// answer, or no SCA feedback.func researchStatusTrailer(cache *RAGCache, resp *RunResponse) string {if resp.Verdict != VerdictInsufficient || resp.SCAFeedback == "" || resp.Answer == "" {return ""}if cache != nil && cache.ConsecutiveUnanswerable() >= 2 {- return " STOP calling rag again: the same gaps remain."+ return fmt.Sprintf(" STOP calling rag again: %d consecutive research rounds returned insufficient evidence. The sources likely lack the required data. Give your best answer from the evidence already gathered; do not re-run rag.", cache.ConsecutiveUnanswerable())}return " If these gaps are material, call rag again with a question focused on them."}@@ -1262,12 +1273,12 @@ func Rag(ctx context.Context, deps RAGTools, req harness.RunRequest) *RunRespons// which Python composes under naiveAnswerSystem instead) still gets an// answer here.composed := false- compose := func(ctx context.Context) {+ compose := func(ctx context.Context, partialAnswer, emptyResult bool, question string) {if composed {return}composed = true- composeFinalAnswer(ctx, deps, req, kb, resp, logger)+ composeFinalAnswer(ctx, deps, req, kb, resp, logger, partialAnswer, emptyResult, question)}deps.Finalize = compose@@ -1290,7 +1301,12 @@ func Rag(ctx context.Context, deps RAGTools, req harness.RunRequest) *RunRespons// evidence, so composing here would replace that with the// FinalAnswerSystem / kb_prompt answer.if spec.Label != "naive" {- compose(ctx)+ // A run that never reached the graph's last node produced neither a+ // partial answer nor empty-result state flags (Python's fallback+ // compose reads the same defaults: no partial preamble, no-evidence+ // hedge only when the pool is actually empty). The question is empty —+ // the graph never formalized, so compose falls back to req.Question.+ compose(ctx, false, false, "")}// Python rag (:902-929) appends a "[Research status]" note for EVERY@@ -1301,7 +1317,9 @@ func Rag(ctx context.Context, deps RAGTools, req harness.RunRequest) *RunRespons// the SCA verdict was known. Skip the naive/sufficient paths: an empty// answer or a SUFFICIENT verdict has nothing to annotate.if t := researchStatusTrailer(deps.Cache, resp); t != "" {- resp.Answer += "\n\n[Research status] " + resp.SCAFeedback + t+ // Python: f"...[Research status] {status_hint}{...}. {trailer}" — the+ // period closes the hint clause before the trailer sentence.+ resp.Answer += "\n\n[Research status] " + resp.SCAFeedback + "." + t}// Cache the freshly produced answer for later near-identical questions, and@@ -1320,10 +1338,29 @@ func Rag(ctx context.Context, deps RAGTools, req harness.RunRequest) *RunRespons// When no model is configured this is a no-op: the caller still receives the// evidence and can answer with it, matching Python's behaviour of returning an// empty kbinfos rather than failing.-func composeFinalAnswer(ctx context.Context, deps RAGTools, req harness.RunRequest, kb *harness.Kbinfos, resp *RunResponse, logger *log.Logger) {+//+// partialAnswer / emptyResult are the compose prompt inputs Python reads off+// the graph state (_compose_answer_from_evidence: `partial_answer` drives the+// partial-information preamble, `empty_result` is the always-true-in-graph+// term of `no_evidence = abstain or empty_result or not chunks`). question is+// the graph state's FORMALIZED question the last node forwarded (Python :834+// `question = state.get("question")`); empty falls back to req.Question —+// only the post-graph fallback composes after a run that never formalized.+// The graph's Finalize forwards the formalize_answer node's own state values;+// the Go-only post-graph fallback passes the response flags instead.+func composeFinalAnswer(ctx context.Context, deps RAGTools, req harness.RunRequest, kb *harness.Kbinfos, resp *RunResponse, logger *log.Logger, partialAnswer, emptyResult bool, question string) {if deps.Model == nil {return}+ // Python composes from the graph state's formalized question+ // (agentic_rag_graph.py:834): the graph researched the full multi-hop+ // question, so the compose prompt must carry it — not the outer tool+ // argument, which compresses multi-hop questions to their first hop and+ // collapses the final answer to the first completed sub-answer.+ composeQuestion := question+ if composeQuestion == "" {+ composeQuestion = req.Question+ }// Python tags the terminal node @in_phase("finalize")// (formalize_answer); without it every call made here is// attributed to "unknown" in the usage table.@@ -1351,7 +1388,7 @@ func composeFinalAnswer(ctx context.Context, deps RAGTools, req harness.RunRequeif deps.AnswerSink != nil {if streamer, ok := deps.Model.(harness.StreamingSessionModel); ok {deps.AnswerSink.reset()- streamed, err := ComposeAnswerStream(ctx, adeps, streamer, kb, req.Question, resp.Partial, func(delta string, isThink bool) error {+ streamed, err := ComposeAnswerStream(ctx, adeps, streamer, kb, composeQuestion, partialAnswer, emptyResult, func(delta string, isThink bool) error {deps.AnswerSink.deliver(delta, isThink)return nil})@@ -1372,12 +1409,15 @@ func composeFinalAnswer(ctx context.Context, deps RAGTools, req harness.RunReque// by the wrapped invoker above (Python relies on @in_phase alone).// empty_result is Python's third no-evidence term (_compose_answer_from_evidence) — the loop's own// "nothing was found" signal, distinct from abstain and from an empty pool.- res := ComposeAnswerWith(ctx, adeps, kb, req.Question, resp.Partial, false, resp.EmptyResult)+ res := ComposeAnswerWith(ctx, adeps, kb, composeQuestion, partialAnswer, false, emptyResult)if deps.Stats != nil && res.Failed {deps.Stats.RecordFailed(harness.PhaseFinalize)}resp.Answer = res.Answer- if res.Partial {+ if res.Partial || partialAnswer {+ // Reflect the graph state's partial flag back on the response even when+ // the composed text itself predates the flag (Python reads+ // partial_answer straight from the compose-time state).resp.Partial = true}if deps.AnswerSink != nil {@@ -1679,9 +1719,22 @@ func runOuterReact(ctx context.Context, deps RAGTools, req harness.RunRequest, lanswer, _, err := outer.ChatWithTools(ctx, p.system, p.history, &models.ChatConfig{})if err != nil {logger.Printf("[Agentic RAG] outer react failed: %v; falling back to direct graph", err)- // Fall back to the inner graph directly so the user still gets an answer.+ // Fall back to the inner graph directly so the user still gets an+ // answer. The graph composes inside its last node with the FORMALIZED+ // question (Python formalize_answer reads state["question"]); the+ // guarded direct call below only fires when the graph never composed.+ composed := false+ deps.Finalize = func(fctx context.Context, partial, empty bool, question string) {+ if composed {+ return+ }+ composed = true+ composeFinalAnswer(fctx, deps, req, p.kb, p.resp, logger, partial, empty, question)+ }RunAgenticRAG(ctx, deps, req, p.sd, p.kb, p.resp, logger, p.spec)- composeFinalAnswer(ctx, deps, req, p.kb, p.resp, logger)+ if !composed {+ composeFinalAnswer(ctx, deps, req, p.kb, p.resp, logger, p.resp.Partial, true, "")+ }return p.resp}// The fold returned the LOWEST-INDEX terminal rag call's answer: keep that@@ -1737,8 +1790,21 @@ func runOuterReactStream(ctx context.Context, deps RAGTools, req harness.RunRequif err != nil {logger.Printf("[Agentic RAG] outer react stream failed: %v; falling back to direct graph", err)// Fall back with the caller's own sink so the answer still streams out.+ // Same guarded compose as runOuterReact: the graph composes inside its+ // last node with the FORMALIZED question; the direct call only fires+ // when the graph never reached it.+ composed := false+ deps.Finalize = func(fctx context.Context, partial, empty bool, question string) {+ if composed {+ return+ }+ composed = true+ composeFinalAnswer(fctx, deps, req, p.kb, p.resp, logger, partial, empty, question)+ }RunAgenticRAG(ctx, deps, req, p.sd, p.kb, p.resp, logger, p.spec)- composeFinalAnswer(ctx, deps, req, p.kb, p.resp, logger)+ if !composed {+ composeFinalAnswer(ctx, deps, req, p.kb, p.resp, logger, p.resp.Partial, true, "")+ }return p.resp}// A tool-less reply is the answer (Python rag_agent returns the model's@@ -1780,6 +1846,59 @@ type outerReactSession struct {// selectEvidence can match the terminal fold's winning answer back to its own// pool. Guarded by mu.calls []ragCallResult++ // flightMu guards inflight, the single-flight registry for CONCURRENT+ // IDENTICAL rag calls.+ flightMu sync.Mutex+ // inflight maps the rag question argument to its in-progress flight.+ // Guarded by flightMu.+ inflight map[string]*ragFlight+}++// ragFlight is ONE in-progress rag execution shared by every concurrent+// identical tool call. Go-only robustness (Python has no single-flight): the+// outer model sometimes emits the SAME rag call 2-3 times in one round+// (observed with MiniMax-M3 in frame_benchmark 20260912_091520 — two graph+// runs started at the same millisecond, three planner runs for one question),+// and both Python (asyncio.gather, chat_model.py:682) and Go+// (models.appendToolResults) execute them all concurrently while the terminal+// fold keeps only the FIRST result. The duplicates each burn a full graph run+// (double provider load → the SCA deadline overruns in the same log) and+// their — sometimes better — answers are discarded. The first caller executes+// and publishes; the rest block on done and return the same answer.+type ragFlight struct {+ done chan struct{}+ answer string+}++// beginRagFlight registers the caller as the OWNER of question's execution, or+// returns the existing flight to wait on (owner=nil, wait!=nil). The key is+// the literal question argument — Python's own cache key+// (agentic_rag.py:889); near-identical NON-identical re-asks stay on the+// RAGCache similarity path (Rag's Lookup), exactly as in Python.+func (s *outerReactSession) beginRagFlight(question string) (owner, wait *ragFlight) {+ s.flightMu.Lock()+ defer s.flightMu.Unlock()+ if existing, ok := s.inflight[question]; ok {+ return nil, existing+ }+ flight := &ragFlight{done: make(chan struct{})}+ if s.inflight == nil {+ s.inflight = make(map[string]*ragFlight)+ }+ s.inflight[question] = flight+ return flight, nil+}++// endRagFlight removes the flight and releases its waiters. The answer MUST be+// stored on the flight before endRagFlight runs: closing done happens-before+// every waiter's receive, so their subsequent read of answer sees the write+// (Go memory model, channel close).+func (s *outerReactSession) endRagFlight(question string, flight *ragFlight) {+ s.flightMu.Lock()+ delete(s.inflight, question)+ s.flightMu.Unlock()+ close(flight.done)}// ToolCall routes the two outer tools to their Go implementations.@@ -1798,6 +1917,17 @@ func (s *outerReactSession) ToolCall(name string, arguments map[string]interfaceif q, ok := arguments["question"].(string); ok && q != "" {req.Question = q}+ // Python agentic_rag.py:865 — prefer the user's ORIGINAL, complete+ // question over the outer model's rewritten `question` argument when+ // both clearly describe the same turn. The outer rewrite often drops+ // the final target of a multi-hop question, and no later stage can+ // recover a deleted answer-attribute. This is the Python defense line's+ // position: inside the `rag` tool itself, i.e. per outer tool call.+ if effective := resolveEffectiveQuestion(req.Question, s.deps.OriginalQuestion); effective != req.Question {+ s.logger.Printf("[Agentic RAG] using original user question over outer rewrite (original=%q → rewrite=%q)",+ trunc(s.deps.OriginalQuestion, 80), trunc(req.Question, 80))+ req.Question = effective+ }// Python's inner _compose_answer_from_evidence is text-only: the outer// model already saw the images via multimodal history, and the rephrased// question carries no images. Drop the original images so the inner@@ -1805,6 +1935,16 @@ func (s *outerReactSession) ToolCall(name string, arguments map[string]interface// receive them — matching Python's inner compose.req.Images = nil+ // Single-flight (see ragFlight): the FIRST caller owns the execution;+ // identical concurrent calls block until it finishes and return the+ // same answer instead of re-running the whole graph.+ flight, wait := s.beginRagFlight(req.Question)+ if wait != nil {+ <-wait.done+ return wait.answer, nil+ }+ defer s.endRagFlight(req.Question, flight)+// Per-call evidence, response and the projections that point at them:// Python gives every rag() invocation its own graph state and shares only// the tools object (agentic_rag.py:877 run_agentic_rag(self, messages)).@@ -1815,30 +1955,71 @@ func (s *outerReactSession) ToolCall(name string, arguments map[string]interfaceinner := s.depsinner.KB = kb+ // Python composes INSIDE the graph with the formalize_answer state+ // (partial_answer from the node, empty_result always true there, and+ // question = state["question"], the FORMALIZED multi-hop question —+ // agentic_rag_graph.py:834). Wire the per-call Finalize so the graph's+ // last node composes itself; the guarded direct call below only fires+ // when the graph never reached that node.+ composed := false+ inner.Finalize = func(fctx context.Context, partial, empty bool, question string) {+ if composed {+ return+ }+ composed = true+ composeFinalAnswer(fctx, inner, req, kb, resp, s.logger, partial, empty, question)+ }+if s.mux != nil {// Streaming: s.deps already routes the inner research log and the// composed answer into the caller's sink, so return "" — the// terminal short-circuit stops the loop and sendTerminal stays// silent, so the answer is not streamed twice.s.mux.markTerminal()- RunAgenticRAG(s.ctx, inner, req, sd, kb, resp, s.logger, s.spec)- composeFinalAnswer(s.ctx, inner, req, kb, resp, s.logger)- s.publish(resp, kb)- return "", nil+ } else {+ // Non-streaming: disable the sink so the answer is not emitted+ // twice, and hand this call's own answer back for the terminal+ // short-circuit.+ inner.AnswerSink = nil}- // Non-streaming: disable the sink so the answer is not emitted twice,- // and hand this call's own answer back for the terminal short-circuit.- inner.AnswerSink = nilRunAgenticRAG(s.ctx, inner, req, sd, kb, resp, s.logger, s.spec)- composeFinalAnswer(s.ctx, inner, req, kb, resp, s.logger)+ if !composed {+ composeFinalAnswer(s.ctx, inner, req, kb, resp, s.logger, resp.Partial, true, "")+ }+ // Python rag (:902-929) appends a "[Research status]" note to the TOOL+ // RESULT for every non-SUFFICIENT verdict, so the outer model can+ // decide whether to re-run `rag` from the reported gaps. It does NOT+ // change the streamed answer (the compose already streamed); after two+ // consecutive unsatisfying rounds it tells the outer agent to STOP.+ if t := researchStatusTrailer(s.deps.Cache, resp); t != "" {+ // Same fold as Rag's direct path: the period closes the hint+ // clause before the trailer sentence.+ resp.Answer += "\n\n[Research status] " + resp.SCAFeedback + "." + t+ }s.publish(resp, kb)+ if s.mux != nil {+ // The answer already streamed through the shared mux; waiters+ // return "" too — sendTerminal's empty guard keeps the fold from+ // re-streaming it (markTerminal already fired on this call).+ flight.answer = ""+ return "", nil+ }+ // Waiters replay this exact answer; publish already ran once above, so+ // the evidence pool is not unioned twice and selectEvidence keeps a+ // single unambiguous call record.+ flight.answer = resp.Answerreturn resp.Answer, nilcase "summarize_document":docID, _ := arguments["doc_id"].(string)if docID == "" {return "Error: missing doc_id argument.", nil}- blocks := harness.SummarizeDocument(s.ctx, s.sd, docID, s.deps.EvidenceMaxTokens)+ // Python budgets the document read and its kb_prompt at+ // chat_mdl.max_length — the FULL window, not a fixed cap+ // (agentic_rag.py:952). No citation header: Python's dialog path+ // constructs RAGTools with do_refer=False (dialog_service.py:2081), so+ // summarize_document returns bare blocks there.+ blocks := harness.SummarizeDocument(s.ctx, s.sd, docID, s.deps.MaxLength)if len(blocks) == 0 {return "The document is unavailable or has no readable chunks.", nil}diff --git a/internal/rag/advanced_rag/agentic_rag_graph.go b/internal/rag/advanced_rag/agentic_rag_graph.goindex fa6ae929ae..0c49d3854f 100644--- a/internal/rag/advanced_rag/agentic_rag_graph.go+++ b/internal/rag/advanced_rag/agentic_rag_graph.go@@ -52,16 +52,15 @@ import ("unicode""unicode/utf8"- "github.com/cespare/xxhash/v2""github.com/cloudwego/eino/compose""github.com/cloudwego/eino/schema""ragflow/internal/agent/chat""ragflow/internal/common"- "ragflow/internal/engine/redis""ragflow/internal/rag/advanced_rag/harness""ragflow/internal/rag/advanced_rag/harness/orchestrator""ragflow/internal/rag/prompts"+ "ragflow/internal/tokenizer")// ---------------------------------------------------------------------------@@ -379,13 +378,30 @@ func SelectSCAView(chunks []map[string]any, focusTerms []string) ([]map[string]aranked = append(ranked, scored{i, c, rel*0.45 + min(covRatio, 1.0)*0.45 + fresh})}sort.SliceStable(ranked, func(i, j int) bool { return ranked[i].score > ranked[j].score })+ // Evidence rows first (Python :146-154): an atomic proposition carrying a+ // verbatim quote is what the reviewer should read before wading through raw+ // passages. Both groups keep their relevance order; only the grouping is+ // lifted before the view cap is applied, so claim evidence can no longer be+ // pushed out of the view by later, loosely-related raw chunks.+ var evidence, rest []map[string]any+ for _, r := range ranked {+ if strings.HasPrefix(harness.ChunkIDOf(r.chunk), evidenceChunkPrefix) {+ evidence = append(evidence, r.chunk)+ } else {+ rest = append(rest, r.chunk)+ }+ }limit := min(len(ranked), SCAViewCap)view := make([]map[string]any, 0, limit)+ view = append(view, evidence...)+ view = append(view, rest...)+ view = view[:min(len(view), limit)]ids := make([]string, 0, limit)- for _, r := range ranked[:limit] {- view = append(view, r.chunk)- ids = append(ids, harness.ChunkIDOf(r.chunk))+ for _, c := range view {+ ids = append(ids, harness.ChunkIDOf(c))}+ // identity is a hash of the SORTED id set, so this reordering cannot break+ // the unproductive-round detector (Python :155-158).sort.Strings(ids)return view, joinHash(ids)}@@ -673,6 +689,13 @@ func SplitThinkStream(ctx context.Context, deltas <-chan string) <-chan ThinkChureturn out}+// Slot-expansion bounds for the query_rewrite DECOMPOSE (Python+// _MAX_SLOT_DEPTH = 3 / _MAX_SLOTS_TOTAL = 8).+const (+ maxSlotDepth = 3+ maxSlotsTotal = 8+)+// SCAGapsToRewrite mirrors Python _sca_gaps_to_rewrite.//// Preference order:@@ -968,6 +991,10 @@ const (fanoutHybridTopN = 30// fanoutSemanticQuota caps narrow-BYPASS hits admitted per fan-out.fanoutSemanticQuota = 4+ // evidenceTopUp caps how many of the chunks cited by the channel-0 claim+ // rows to pull in verbatim (Python _EVIDENCE_TOP_UP). A directed fetch by+ // id, not another recall.+ evidenceTopUp = 8// Narrowing budget for the exact leg (Python narrow_by_terms kwargs).fanoutNarrowMaxOutPerChunk = 1200fanoutNarrowMaxOutTotal = 16000@@ -1056,23 +1083,108 @@ func FanoutSearch(ctx context.Context, deps RAGTools, st *AgenticState, queries}added := 0+ rawAdded := 0admit := func(batch []map[string]any) bool {for _, c := range batch {if added >= room {return true}id := harness.ChunkIDOf(c)+ isEvidence := strings.HasPrefix(id, "claim_")if id != "" {if seen[id] {continue}seen[id] = true}+ // Raw passages get their own budget (Python _RAW_SNIPPET_QUOTA);+ // anything left above it stays free for evidence rows, which are+ // far denser answer material. Evidence rows (claim_ prefix) bypass+ // the quota and never count against it.+ if !isEvidence && rawAdded >= rawSnippetQuota {+ continue+ }st.KB.Chunks = append(st.KB.Chunks, c)added+++ if !isEvidence {+ rawAdded+++ }}return added >= room}+ // Channel 0 (Python _collect_evidence): claim/evidence rows lead the pool —+ // they are the compact, verbatim-bearing proxy for the chunks they source.+ // Gated on the dataset having compiled rows at all; capped at+ // evidencePoolQuota rows across the whole prefetch; best effort.+ channel0 := [][]map[string]any{}+ channel0Rows := 0+ for _, q := range qs {+ select {+ case <-ctx.Done():+ return 0+ default:+ }+ if !harness.DatasetHasCompilation(ctx, sd) {+ break+ }+ // Python :703 — top_n=max(2, top_n) per query, so a single-fanout+ // question still recalls at least two claim rows.+ recallN := capPerQuery+ if recallN < 2 {+ recallN = 2+ }+ hits := harness.RecallDatasetClaims(ctx, sd, q, recallN)+ if len(hits) == 0 {+ continue+ }+ pseudo := harness.ClaimPseudoChunks(hits)+ kept := make([]map[string]any, 0, len(pseudo))+ for _, pc := range pseudo {+ if channel0Rows >= evidencePoolQuota {+ break+ }+ kept = append(kept, pc)+ channel0Rows+++ }+ if len(kept) == 0 {+ break+ }+ channel0 = append(channel0, kept)+ if admit(kept) {+ return added+ }+ }+ // Evidence top-up (Python _fanout_search:782-799): an evidence row carries a+ // verbatim quote but not its surrounding passage, so pull exactly the chunks+ // it cites instead of running another global recall. Deduped against the+ // pool (a chunk the claim already quotes verbatim adds nothing new) and+ // capped at evidenceTopUp ids. Best effort.+ if len(channel0) > 0 {+ var wanted []string+ inWanted := map[string]bool{}+ for _, pseudo := range channel0 {+ for _, c := range pseudo {+ ids, _ := c["source_chunk_ids"].([]string)+ for _, cid := range ids {+ cid = strings.TrimSpace(cid)+ if cid == "" || seen[cid] || inWanted[cid] {+ continue+ }+ inWanted[cid] = true+ wanted = append(wanted, cid)+ }+ }+ }+ if len(wanted) > evidenceTopUp {+ wanted = wanted[:evidenceTopUp]+ }+ if len(wanted) > 0 {+ fetched := harness.LoadChunksForIDs(ctx, sd, wanted)+ if len(fetched) > 0 && admit(fetched) {+ return added+ }+ }+ }// Channel A across every fan-out first, then channel B.for _, p := range pairs {if admit(p.exact) {@@ -1189,7 +1301,14 @@ func BuildLowGraph(ctx context.Context, deps RAGTools, req harness.RunRequest, s// composition node, so the answer is produced inside the graph.addNode("formalize_answer", func(c context.Context, r *harness.RunRequest) (*harness.RunRequest, error) {if deps.Finalize != nil {- deps.Finalize(c)+ // Python low-graph state at this node: partial_answer=False+ // (build_low_graph formalize_question) and empty_result=True+ // (orchestrator/direct.py direct_search) — the values+ // _compose_answer_from_evidence reads from the state. The question+ // is the one formalize_question wrote into this same RunRequest+ // (Python state["question"], agentic_rag_graph.py:834): the low+ // graph's compose must use the formalized question too.+ deps.Finalize(c, false, true, r.Question)}return r, nil})@@ -1396,7 +1515,7 @@ func scaNode(ctx context.Context, deps RAGTools, st *AgenticState, logger *log.Lres := orchestrator.SufficientContextAgent(callCtx, orchestrator.SCADeps{KB: st.KB,- Model: &jsonModelAdapter{inner: deps.Model, maxLength: deps.MaxLength, modelName: deps.ModelName, cache: redisGenJSONCache{}},+ Model: &jsonModelAdapter{inner: deps.Model, maxLength: deps.MaxLength},Prompts: deps.SCAPrompts,}, st.Question, claims)@@ -1476,7 +1595,7 @@ func queryRewriteNode(ctx context.Context, deps RAGTools, st *AgenticState, loggdefer cancel()rewritten := orchestrator.RewriteGapToQuery(callCtx, orchestrator.RewriteDeps{- Model: &jsonModelAdapter{inner: deps.Model, maxLength: deps.MaxLength, modelName: deps.ModelName, cache: redisGenJSONCache{}},+ Model: &jsonModelAdapter{inner: deps.Model, maxLength: deps.MaxLength},Prompts: deps.RewritePrompts,ResearchContext: researchContext,}, st.Question, gaps)@@ -1522,6 +1641,53 @@ func queryRewriteNode(ctx context.Context, deps RAGTools, st *AgenticState, loggreturn}+ // DECOMPOSE (Python query_rewrite:1349-1375): promote SCA gaps to new+ // slots so the next research pass gets a typed unknown with its own+ // action session — that is how the plan actually expands. No extra LLM+ // call: the SCA already told us what is missing (missing_fact + hint).+ //+ // ORDER (Python :1306-1375): promotion runs AFTER the rewrite LLM call, the+ // empty-query early return (:1326-1328) and the saturation early-exit+ // (:1337-1339) — both of which DISCARD the promotion by returning before it.+ // Promoting before the rewrite (the earlier Go port's order) mutated the slot+ // table even on rounds that were then dropped, desyncing the table from the+ // queries actually pursued.+ if st.SlotTable.Depth < maxSlotDepth {+ slots := st.SlotTable.State+ known := map[string]bool{}+ nextID := 0+ for _, v := range slots {+ for _, c := range v.QuestionClues {+ known[strings.ToLower(strings.TrimSpace(c))] = true+ }+ if v.ID >= nextID {+ nextID = v.ID + 1+ }+ }+ promoted := 0+ for _, g := range gaps {+ if len(slots) >= maxSlotsTotal {+ break+ }+ key := strings.ToLower(strings.TrimSpace(g.What))+ if key == "" || known[key] {+ continue+ }+ clues := []string{truncateRunes(g.What, 200)}+ if strings.TrimSpace(g.SearchHint) != "" {+ clues = append(clues, truncateRunes(g.SearchHint, 200))+ }+ slots = append(slots, harness.Variable{ID: nextID, Type: "entity", QuestionClues: clues})+ known[key] = true+ nextID+++ promoted+++ }+ if promoted > 0 {+ st.SlotTable.State = slots+ logger.Printf("[QueryRewriter] decompose: %d gap(s) promoted to slots (depth=%d)", promoted, st.SlotTable.Depth)+ }+ }+logger.Printf("[QueryRewriter] insufficient round %d → %d targeted query(s): %v",st.SearchRounds+1, len(queries), queries)st.NoProgress = false@@ -1546,8 +1712,19 @@ func formalizeAnswerNode(ctx context.Context, deps RAGTools, st *AgenticState, llogger.Printf("[Finalize] partial=%v empty=%v chunks=%d", st.PartialAnswer, st.EmptyResult, len(st.KB.Chunks))// formalize_answer — the node itself composes and streams the answer// (_compose_answer_from_evidence); it does not just flag the state.+ // Python composes with THIS node's state values: partial_answer was set+ // right above (agentic_rag_graph.py:1397-1400) and empty_result is still+ // the True formalize_question wrote (agentic_rag_graph.py:1042 — never+ // reset anywhere in the graph), so the compose prompt carries the+ // no-evidence hedge on every round and, when INSUFFICIENT, the partial+ // preamble. Forwarding the state beats the caller's response flags,+ // which are only copied after the graph returns.if deps.Finalize != nil {- deps.Finalize(ctx)+ // question = state["question"] (Python :834): the FORMALIZED question+ // the formalize_question node wrote — composing from the outer tool+ // argument instead collapses the final answer to the first completed+ // sub-answer of a multi-hop question.+ deps.Finalize(ctx, st.PartialAnswer, true, st.Question)}}@@ -1681,71 +1858,25 @@ func viewChunkIDs(view []map[string]any) []string {// genJSONMaxRetry mirrors Python gen_json's default max_retry=2: the first// call, plus one corrective round that feeds the malformed answer and the parse// error back to the model.-const genJSONMaxRetry = 2--// LLMCache is the gen_json reply-cache seam (Python-// rag.graphrag.utils.get_llm_cache / set_llm_cache). A nil seam disables-// caching, which is also what a Redis-less deployment gets.-type LLMCache interface {- Get(ctx context.Context, key string) (string, bool)- Set(ctx context.Context, key, value string)-}--// genJSONCacheTTL mirrors Python set_llm_cache's `24 * 3600` second expiry.-const genJSONCacheTTL = 24 * time.Hour--// genJSONCacheKey mirrors gen_json's cache key byte for byte://-// xxhash.xxh64(str(llm_name) + str(system_prompt) + str(user_prompt) + str(gen_conf)).hexdigest()-//-// `str(gen_conf)` of the default {} is "{}" — the two agentic callers (SCA-// review, gap→query rewrite) pass no gen_conf, so the literal is exact. Go's-// zero-padded %016x over xxhash.Sum64String prints the same 16 lowercase hex-// digits Python's hexdigest() does (plain %x drops leading zeros), so a Go and-// a Python process sharing one Redis share one cache. An empty modelName yields-// "" and the caller must then SKIP the cache: keying on an empty name would-// collapse every model onto one bucket.-func genJSONCacheKey(modelName, systemPrompt, userPrompt string) string {- if modelName == "" {- return ""- }- return fmt.Sprintf("%016x", xxhash.Sum64String(modelName+systemPrompt+userPrompt+"{}"))-}--// redisGenJSONCache is the production LLMCache: the process-wide Redis client.-// With no Redis configured redis.Get() is nil and every call degrades to a miss-// — the same "no cache" behaviour Python has when REDIS_CONN is absent.-type redisGenJSONCache struct{}--func (redisGenJSONCache) Get(ctx context.Context, key string) (string, bool) {- client := redis.Get()- if client == nil || key == "" {- return "", false- }- v, err := client.Get(ctx, key)- if err != nil || v == "" {- return "", false- }- return v, true-}--func (redisGenJSONCache) Set(ctx context.Context, key, value string) {- client := redis.Get()- if client == nil || key == "" || value == "" {- return- }- client.Set(ctx, key, value, genJSONCacheTTL)-}+// Deliberately NO reply cache: Python's gen_json carries Redis cache code, but+// its set/get keys never match — get_llm_cache hashes (llm_name, system,+// user_prompt, gen_conf) (generator.py:569) while set_llm_cache hashes+// (llm_name, system, ans, user_prompt, gen_conf) (generator.py:582, the value+// `ans` is hashed INTO the key) — so the lookup always misses and every call+// reaches the model. The observable contract is "same prompt still calls the+// LLM", which also rules out replaying a verdict computed against evidence a+// later round has already superseded.+const genJSONMaxRetry = 2// jsonModelAdapter adapts harness.SessionModel to orchestrator.JSONModel.//-// Mirrors Python's gen_json(prompt, "Output:\n", chat_mdl): a 24h reply cache is-// consulted first, the rendered prompt is the system turn and "Output:\n" the-// user turn (fitted ONCE to the model's context window), then the first JSON-// value is parsed out of the reply. Malformed JSON is retried (up to-// genJSONMaxRetry calls) with the model's own bad answer and the parse error-// appended to the user turn, so a single formatting hiccup does not abort the-// SCA review or the query rewrite.+// Mirrors Python's gen_json(prompt, "Output:\n", chat_mdl): the rendered prompt+// is the system turn and "Output:\n" the user turn (fitted ONCE to the model's+// context window), then the first JSON value is parsed out of the reply.+// Malformed JSON is retried (up to genJSONMaxRetry calls) with the model's own+// bad answer and the parse error appended to the user turn, so a single+// formatting hiccup does not abort the SCA review or the query rewrite.type jsonModelAdapter struct {inner harness.SessionModel// maxLength is the chat model's context window (Python@@ -1753,11 +1884,6 @@ type jsonModelAdapter struct {// prompt is trimmed instead of rejected by the provider. <=0 falls back to// chat.EffectiveContextLength's 8192 default.maxLength int- // modelName is the resolved chat model identity (Python chat_mdl.llm_name).- // Empty disables the reply cache (see genJSONCacheKey).- modelName string- // cache is the gen_json reply cache; nil disables caching.- cache LLMCache}// parseGenJSONReply parses a cleaned model reply with gen_json's tolerance: a@@ -1800,22 +1926,10 @@ func (a *jsonModelAdapter) GenJSON(ctx context.Context, prompt string) (any, errif a.inner == nil {return nil, fmt.Errorf("agentic: no model configured")}- // Python gen_json consults the 24h reply cache FIRST, keyed on the ORIGINAL- // system/user prompts (not the fitted ones), and returns without any model- // call on a hit:- // cached = get_llm_cache(chat_mdl.llm_name, system_prompt, user_prompt, gen_conf)- // if cached: return json_repair.loads(cached)+ // No reply cache: see the genJSONMaxRetry note — Python's own cache never+ // hits (its set/get keys never match), so every call reaches the model.const userPrompt = "Output:\n"- cacheKey := ""- if a.cache != nil {- cacheKey = genJSONCacheKey(a.modelName, prompt, userPrompt)- if cached, hit := a.cache.Get(ctx, cacheKey); hit {- if v, ok := parseGenJSONReply(cached); ok {- return v, nil- }- }- }- // Python gen_json then fits ONCE, before the retry loop:+ // Python gen_json fits ONCE, before the retry loop:// _, msg = message_fit_in(form_message(system_prompt, user_prompt), max_length)// and sends msg[0] as the system turn and msg[1:] as history, appending the// corrective text to the LAST user turn WITHOUT re-fitting. A zero/negative@@ -1857,11 +1971,6 @@ func (a *jsonModelAdapter) GenJSON(ctx context.Context, prompt string) (any, err// raw reply with its fences / think block.lastAns = cleanedif v, ok := parseGenJSONReply(cleaned); ok {- // Python caches the CLEANED answer on the first successful parse- // (set_llm_cache(..., ans, ...)), and only there.- if a.cache != nil && cacheKey != "" {- a.cache.Set(ctx, cacheKey, cleaned)- }return v, nil}// Deliberately log length, not content: the reply may embed retrieved@@ -1969,7 +2078,14 @@ func renderResearchContext(st *AgenticState) string {if i >= PoolHeadLines {break}- first := harness.ChunkTextOf(c)+ // Python reads `c.get("content") or c.get("content_with_weight")`+ // here (query_rewrite node) — content FIRST, unlike _chunk_text's+ // content_with_weight-first order used elsewhere. The pool-head+ // lines are rewriter prompt content, so keep Python's order.+ first := anyString(c["content"])+ if first == "" {+ first = anyString(c["content_with_weight"])+ }if idx := strings.IndexByte(first, '\n'); idx >= 0 {first = first[:idx]}@@ -1995,8 +2111,13 @@ func routeSCA(st *AgenticState, enableSCA bool, scaMaxRounds int) agenticNode {}// Evidence pool saturated at the SCA view cap: any further chunk lands// beyond what the SCA can read, so another search round cannot flip the- // sufficiency verdict. Short-circuit straight to finalize.- if st.KB != nil && len(st.KB.Chunks) >= SCAViewCap {+ // sufficiency verdict. Gated to SECOND-or-later reviews on THIS branch:+ // claim pseudo-chunks bypass the pool cap (direct append in the claim+ // prefetch), so a rewrite round can still add evidence — the FIRST SCA+ // review must always get its rewrite round when insufficient, or hard+ // multi-hop questions lose their only refinement pass and the retry moves+ // to the outer agent as a whole new graph run (Python _route_sca:1411).+ if st.KB != nil && len(st.KB.Chunks) >= SCAViewCap && st.SearchRounds >= 1 {_LOG.Printf("[SCA] evidence pool FULL (%d chunks >= SCA view cap %d); early-stopping to finalize_answer.", len(st.KB.Chunks), SCAViewCap)return nodeFormalizeAnswer}@@ -2223,6 +2344,387 @@ func buildSlotTableFrom(ctx context.Context, deps harness.SessionDeps, questionreturn init.Root, init.FirstQueries, nil}+// ── Evidence-guided batched answer generation (APT-RAG) ─────────────────────+// Mirrors Python rag/advanced_rag/agentic_rag_graph.py:1563-1749.++// evidenceChunkPrefix mirrors Python _EVIDENCE_CHUNK_PREFIX(agentic_rag_graph.py:537):+// only the claim pseudo-chunks are atomic evidence rows.+const evidenceChunkPrefix = "claim_"++// Evidence-guided batching constants(Python _EVIDENCE_BATCH_*,:1574-1577).+//+// The threshold is deliberately conservative: APT-RAG ships sim_threshold=0.0+// (any overlap), which merges weakly related slots and, worse, lets a single+// parse failure blank every answer in the batch at once. Both conditions must+// hold. The absolute floor stops two tiny evidence sets from merging on a+// single coincidental hit; the ratio gate stays LOW enough to actually fire —+// measured FRAMES overlap was 0.03-0.2 mean, so the old 0.5 never triggered at+// all. (APT-RAG ships 0.0; that is too loose for us because one parse failure+// would blank every answer in the batch.)+const (+ EvidenceBatchMinSim = 0.15+ EvidenceBatchMinShared = 2+ EvidenceBatchMaxSlots = 4+ EvidenceBatchMaxChars = 12000+)++// EvidenceBatchPrompt mirrors Python _EVIDENCE_BATCH_PROMPT(:1579-1585),+// verbatim.+const EvidenceBatchPrompt = "Answer each listed sub-question using ONLY the shared evidence below.\n" ++ "These sub-questions share this evidence, so read it once and answer all of them.\n" ++ "Return JSON only, mapping each sub-question id to its answer: " ++ `{"<id>": "<answer>", ...}. Use an empty string when the evidence does not ` ++ "answer that sub-question. Never invent facts."++// intersectionSize counts the shared members of two evidence-id sets.+func intersectionSize(a, b map[string]bool) int {+ n := 0+ for id := range a {+ if b[id] {+ n+++ }+ }+ return n+}++// BatchFillSlots mirrors Python _batch_fill_slots(:1588-1694): answer several+// unresolved slots in ONE call when their evidence overlaps.+//+// Pure efficiency: neither the slot structure nor the evidence semantics+// change — only the number of generation calls made over the same passages.+//+// Python wraps the call site in try/except(:1890-1895)and each model call in+// its own try/except(:1672-1683); the Go port is best-effort inside: a+// missing model, a failed call, or an unparsable reply skips that cluster and+// the remaining clusters still run.+func BatchFillSlots(ctx context.Context, deps harness.SessionDeps, slotTable *harness.State, slotEvidence map[string]SlotEvidence) int {+ // Python :1596 — unresolved slots only; :1637 — slot_by_id over them.+ unresolvedN := 0+ slotByID := map[int]*harness.Variable{}+ evIDs := map[int][]string{} // ordered evidence ids (Python ev[v.id] set)+ evSet := map[int]map[string]bool{}+ for i := range slotTable.State {+ v := &slotTable.State[i]+ if v.Filled() {+ continue+ }+ unresolvedN+++ slotByID[v.ID] = v+ // Python :1599 — evidence ids recorded for this slot by its session,+ // keyed by str(slot id), blanks dropped.+ ids := map[string]bool{}+ var ordered []string+ for _, id := range slotEvidence[fmt.Sprint(v.ID)].EvidenceIDs {+ if id == "" || ids[id] {+ continue+ }+ ids[id] = true+ ordered = append(ordered, id)+ }+ if len(ordered) > 0 {+ evIDs[v.ID] = ordered+ evSet[v.ID] = ids+ }+ }+ if len(evSet) < 2 {+ // Diagnostics (Python :1602-1611): batching needs TWO slots that are+ // both unresolved AND carrying evidence. Log why it did not happen,+ // otherwise a never-firing path is indistinguishable from a working one.+ _LOG.Printf("[SlotResearch] batching skipped: unresolved=%d with_evidence=%d", unresolvedN, len(evSet))+ return 0+ }++ ids := make([]int, 0, len(evSet))+ for id := range evSet {+ ids = append(ids, id)+ }+ // Python iterates the `ev` SET, whose order is arbitrary; sort first so the+ // stable ordering below is deterministic across runs.+ sort.Ints(ids)++ // sim mirrors Python _sim(:1615-1617): Jaccard over evidence-id sets.+ sim := func(a, b int) float64 {+ inter := intersectionSize(evSet[a], evSet[b])+ union := len(evSet[a]) + len(evSet[b]) - inter+ if union == 0 {+ return 0.0+ }+ return float64(inter) / float64(union)+ }++ // Largest-incompatible-first (APT-RAG, Python :1619-1621): place the least+ // compatible slot first, so it is not left without a cluster at the end.+ // Counts are precomputed — Python's sort key is fixed before sorting.+ incompatible := make(map[int]int, len(ids))+ for _, i := range ids {+ n := 0+ for _, j := range ids {+ if j != i && sim(i, j) < EvidenceBatchMinSim {+ n+++ }+ }+ incompatible[i] = n+ }+ order := append([]int(nil), ids...)+ sort.SliceStable(order, func(x, y int) bool { return incompatible[order[x]] > incompatible[order[y]] })++ // Greedy clustering(Python :1622-1629): a slot joins the first cluster+ // that is under the size cap AND shares ≥MIN_SHARED ids AND ≥MIN_SIM with+ // EVERY member; otherwise it starts its own cluster.+ clusters := [][]int{}+ for _, i := range order {+ placed := false+ for ci := range clusters {+ cl := clusters[ci]+ if len(cl) >= EvidenceBatchMaxSlots {+ continue+ }+ compatible := true+ for _, j := range cl {+ if intersectionSize(evSet[i], evSet[j]) < EvidenceBatchMinShared || sim(i, j) < EvidenceBatchMinSim {+ compatible = false+ break+ }+ }+ if compatible {+ clusters[ci] = append(cl, i)+ placed = true+ break+ }+ }+ if !placed {+ clusters = append(clusters, []int{i})+ }+ }++ // by_chunk_id(Python :1631-1636)over the whole pool.+ byChunkID := map[string]map[string]any{}+ if deps.KB != nil {+ for _, c := range deps.KB.Chunks {+ if cid := harness.ChunkIDOf(c); cid != "" {+ byChunkID[cid] = c+ }+ }+ }++ filled := 0+ for _, cl := range clusters {+ if len(cl) < 2 {+ continue+ }+ // Python :1643-1645 builds union_ids as a set (arbitrary order); the+ // port keeps first-seen order across the cluster for determinism.+ unionIDs := []string{}+ seenID := map[string]bool{}+ for _, i := range cl {+ for _, id := range evIDs[i] {+ if !seenID[id] {+ seenID[id] = true+ unionIDs = append(unionIDs, id)+ }+ }+ }+ body, total := []string{}, 0+ for _, cid := range unionIDs {+ c := byChunkID[cid]+ if c == nil {+ continue+ }+ // Python :1651 — content_with_weight first, content fallback.+ text := anyString(c["content_with_weight"])+ if text == "" {+ text = anyString(c["content"])+ }+ text = strings.TrimSpace(text)+ if text == "" {+ continue+ }+ // Python len() counts code points, so the MAX_CHARS cap is+ // character-based, not byte-based.+ n := utf8.RuneCountInString(text)+ if total+n > EvidenceBatchMaxChars {+ break+ }+ body = append(body, text)+ total += n+ }+ if len(body) == 0 {+ continue+ }++ lines := []string{}+ for _, sid := range cl {+ v := slotByID[sid]+ if v == nil {+ continue+ }+ // Python :1666-1667 — join ALL clues, then cut the JOINED text+ // to 300 code points.+ clues := truncateRunes(strings.Join(v.QuestionClues, "; "), 300)+ lines = append(lines, fmt.Sprintf("- %d: %s", sid, clues))+ }+ if len(lines) < 2 {+ continue+ }++ // Python :1671.+ user := "Sub-questions:\n" + strings.Join(lines, "\n") + "\n\nShared evidence:\n" + strings.Join(body, "\n---\n")+ // Python :1673-1675 — no model: stop batching entirely.+ if deps.Model == nil {+ return filled+ }+ // Python :1676-1680 — async_chat(system_prompt, [user], answer_conf);+ // the Go SessionModel takes the system message inline.+ reply, err := deps.Model.Complete(ctx, []schema.Message{+ *schema.SystemMessage(EvidenceBatchPrompt),+ *schema.UserMessage(user),+ }, nil)+ if err != nil {+ // Python :1681-1683 — one failed call must not cost the other+ // clusters.+ _LOG.Printf("[SlotResearch] batched answer call failed: %v", err)+ continue+ }++ // Python :1685-1687 — _extract_json_object; a non-object reply+ // skips the cluster.+ data, ok := extractJSONObject(reply.Content).(map[string]any)+ if !ok {+ continue+ }+ for _, sid := range cl {+ v := slotByID[sid]+ if v == nil {+ continue+ }+ // Python :1690-1693 — the reply maps str(slot id) to its answer;+ // never overwrite an already-filled slot, and a blank answer+ // counts as "evidence does not answer this".+ val, isStr := data[fmt.Sprint(sid)].(string)+ if !isStr {+ continue+ }+ val = strings.TrimSpace(val)+ if val == "" || v.Filled() {+ continue+ }+ cand := truncateRunes(val, 400)+ v.Candidate = &cand+ filled+++ }+ }+ return filled+}++// Word-level coverage a pooled evidence row must reach before it is allowed to+// answer a slot on its own. Deliberately strict: a wrong prefill costs+// accuracy, while a missed prefill only costs one session (which still runs).+// (Python _EVIDENCE_PREFILL_COVERAGE,:1697-1700)+const EvidencePrefillCoverage = 0.6++// PrefillSlotsFromEvidence mirrors Python _prefill_slots_from_evidence(:1703-1749):+// answer slots that an already-pooled evidence row directly answers.+//+// An evidence row is an atomic proposition carrying a verbatim quote, so when+// it already covers a slot's question there is nothing for an action session+// to research — skipping it removes a WHOLE session (the dominant cost), not+// just tokens inside one. Saves calls, uses real evidence, and a wrong guess+// is still caught later by the SCA.+func PrefillSlotsFromEvidence(slotTable *harness.State, kb *harness.Kbinfos) int {+ // Python :1714-1715 — evidence rows are the claim pseudo-chunks.+ evRows := []map[string]any{}+ if kb != nil {+ for _, c := range kb.Chunks {+ if strings.HasPrefix(harness.ChunkIDOf(c), evidenceChunkPrefix) {+ evRows = append(evRows, c)+ }+ }+ }+ if len(evRows) == 0 {+ return 0+ }++ filled := 0+ for i := range slotTable.State {+ v := &slotTable.State[i]+ if v.Filled() {+ continue+ }+ clues := []string{}+ for _, c := range v.QuestionClues {+ if strings.TrimSpace(c) != "" {+ clues = append(clues, c)+ }+ }+ if len(clues) == 0 {+ continue+ }+ // Python :1724-1726 — lowercase terms of ≥3 code points over all clues.+ terms := map[string]bool{}+ for _, c := range clues {+ for _, t := range harness.QueryToTerms(c) {+ if utf8.RuneCountInString(t) >= 3 {+ terms[strings.ToLower(t)] = true+ }+ }+ }+ if len(terms) == 0 {+ continue+ }++ best, bestCov := -1, 0.0+ for j, e := range evRows {+ // Python :1732-1734 — content_with_weight ONLY (not content).+ text := strings.ToLower(anyString(e["content_with_weight"]))+ if text == "" {+ continue+ }+ cov := 0+ for t := range terms {+ if strings.Contains(text, t) {+ cov+++ }+ }+ covF := float64(cov) / float64(len(terms))+ // Python :1736-1737 — strictly greater, so ties keep the FIRST row.+ if covF > bestCov {+ best, bestCov = j, covF+ }+ }+ if best < 0 || bestCov < EvidencePrefillCoverage {+ continue+ }++ // Python :1741-1743 — the row renders as+ // "[evidence] <name> — <desc>\nEvidence (verbatim): ...".+ //+ // Format provenance (Python's TWO deliberate claim formats): the+ // fan-out channel-0 rows this prefill consumes are rendered by+ // harness.ClaimPseudoChunks with the "[evidence] " prefix (Python+ // _collect_evidence, agentic_rag_graph.py:718). The OTHER producers —+ // action-session _claim_prefetch (action_session.py:741-747) and the+ // navigate_structure publish (_publish_claim_hits, navigation.py:1585)+ // — use "[claim #N] <name>", and Python's prefill never sees "[evidence]"+ // on those either; the replace below simply leaves their marker intact,+ // exactly as Python does. Go matches both producers one-to-one.+ head := anyString(evRows[best]["content_with_weight"])+ if idx := strings.IndexByte(head, '\n'); idx >= 0 {+ head = head[:idx]+ }+ name := strings.TrimSpace(strings.Trim(strings.ReplaceAll(head, "[evidence]", ""), " —-"))+ if name == "" {+ continue+ }+ // Python :1746-1748 — candidate = name[:400], strength = coverage.+ cand := truncateRunes(name, 400)+ v.Candidate = &cand+ strength := bestCov+ v.CandidateStrength = &strength+ filled+++ }+ return filled+}+// RunSlotResearchPass mirrors Python _run_slot_research_pass: drive ONE// research round with slot-aware action sessions.//@@ -2246,6 +2748,20 @@ func RunSlotResearchPass(ctx context.Context, deps harness.SessionDeps, questionreturn nil}+ // Evidence-row prefill(Python :1782-1790): slots the pooled evidence+ // already answers cost no action session at all — this is where whole+ // calls get removed. Python guards the call with try/except; the port is+ // pure (no model, no I/O) and cannot raise.+ kb := deps.KB+ if kb == nil {+ // Python :1779 — `tools.kbinfos or state.kbinfos`.+ kb = st.KB+ }+ if prefillN := PrefillSlotsFromEvidence(&slotTable, kb); prefillN > 0 {+ _LOG.Printf("[SlotResearch] evidence prefill answered %d slot(s) with no session", prefillN)+ unresolved = slotTable.Unresolved()+ }+// Shared across sessions so duplicate retrievals are served from cache.sharedToolCache := harness.NewToolCache()var sharedSearchQueries []string@@ -2340,6 +2856,14 @@ func RunSlotResearchPass(ctx context.Context, deps harness.SessionDeps, question_LOG.Printf("[SlotResearch] slot evidence bound: %v", bounds)}+ // Evidence-guided batching(Python :1888-1895): slots that retrieved the+ // same passages get answered together instead of one generation call each.+ // Python guards the call with try/except; BatchFillSlots is best-effort+ // internally (per-cluster failures are logged and skipped).+ if batched := BatchFillSlots(ctx, deps, &slotTable, sessionEvidence); batched > 0 {+ _LOG.Printf("[SlotResearch] batched generation filled %d slot(s)", batched)+ }+unresolvedOut := make([]map[string]any, 0, len(unresolved))for _, v := range slotTable.Unresolved() {clues := v.DiscoveredClues@@ -3215,6 +3739,13 @@ const (// evidenceBudgetTokens is the token ceiling of the evidence block// (Python agentic_rag._EVIDENCE_BUDGET_TOKENS).evidenceBudgetTokens = 8000+ // evidencePoolQuota mirrors Python _EVIDENCE_POOL_QUOTA: claim pseudo-chunk+ // cap across the whole first prefetch.+ evidencePoolQuota = 24+ // rawSnippetQuota mirrors Python _RAW_SNIPPET_QUOTA: the raw-chunk+ // admission budget inside the fan-out, so chunk channels cannot crowd out+ // the denser evidence rows.+ rawSnippetQuota = 30// citeChunkCap caps chunks rendered as citation reference// (Python _CITE_CHUNK_CAP).citeChunkCap = 6@@ -3376,18 +3907,27 @@ func ComposeAnswerWith(ctx context.Context, deps AnswerDeps, kb *harness.KbinfoscallCtx, cancel := context.WithTimeout(ctx, deadlineToDuration(answerTimeoutS))defer cancel()- logger.Printf("[Formalize][pre_summary] question=%q pre_summary_len=%d evidence_len=%d",- trunc(question, 160), len(preSummary), len(prompt.user))+ logger.Printf("[Formalize][pre_summary] question=%q pre_summary_len=%d evidence_len=%d\npre_summary=%q",+ trunc(question, 160), len(preSummary), len(prompt.user), truncateRunes(preSummary, 3000))- userMsg := schema.UserMessage(prompt.user)+ // Python fits the composed prompt ONCE before the call:+ // message_fit_in(form_message(system, user), min(chat_mdl.max_length, 8000))+ // (agentic_rag_graph.py:946) — msg[0] is the system turn, msg[-1] the user+ // turn; form_message (generator.py:495) is exactly this two-message shape,+ // so the fit must not synthesize an extra system turn.+ systemTurn, userTurn := fitComposePrompt(prompt.system, prompt.user, deps.MaxLength)+ userMsg := schema.UserMessage(userTurn)if len(deps.UserImages) > 0 {// Mirror Python's direct async_chat fallback, which is called with the// original multimodal messages: attach the vision-gated images to the// final-answer user message so the compose model can see them.userMsg = multimodalUserMsg(prompt.user, deps.UserImages)}- reply, err := deps.Model.Complete(callCtx, []schema.Message{- *schema.SystemMessage(prompt.system),+ // Python samples the compose call at answer_conf — gen_conf or the default+ // {"temperature": 0.3} (build_agentic_graph :962/:1021); the graph runs with+ // gen_conf unset, so 0.3 always applies.+ reply, err := modelWithTemperature(deps.Model, answerTemperature).Complete(callCtx, []schema.Message{+ *schema.SystemMessage(systemTurn),*userMsg,}, nil)if err != nil {@@ -3397,6 +3937,56 @@ func ComposeAnswerWith(ctx context.Context, deps AnswerDeps, kb *harness.Kbinfosreturn AnswerResult{Answer: cleanAnswer(reply.Content), Partial: partial}}+// answerTemperature mirrors Python answer_conf's default sampling temperature:+// build_agentic_graph uses gen_conf or {"temperature": 0.3} (graph.py:962/:1021)+// and run_agentic_rag invokes the graph with gen_conf unset, so the compose+// call always samples at 0.3.+const answerTemperature = 0.3++// fitComposePrompt mirrors Python's compose-time fit+// (agentic_rag_graph.py:946): message_fit_in(form_message(system, user),+// min(chat_mdl.max_length, _EVIDENCE_BUDGET_TOKENS)). message_fit_in+// (generator.py:69-137) normalizes a non-positive budget to 8192, returns the+// pair untouched when it already fits, then trims by TOKENS (trim_content):+// the system share branch (>0.8 of the total) preserves the user turn first,+// otherwise the system turn is preserved first and the user turn gets the+// remainder. form_message (generator.py:495) is exactly the [system, user]+// pair, so no extra system turn is synthesized here.+func fitComposePrompt(system, user string, maxLength int) (string, string) {+ budget := maxLength+ if budget > evidenceBudgetTokens {+ budget = evidenceBudgetTokens+ }+ if budget <= 0 {+ // message_fit_in normalizes a non-positive max_length to 8192+ // (generator.py:69-72).+ budget = 8192+ }+ ll := tokenizer.NumTokensFromString(system)+ ll2 := tokenizer.NumTokensFromString(user)+ if ll+ll2 < budget {+ return system, user+ }+ if ll+ll2 <= 0 {+ // message_fit_in's degenerate branch: token counts are zero — keep the+ // content unchanged rather than trimming blindly.+ return system, user+ }+ if float64(ll)/float64(ll+ll2) > 0.8 {+ // System-dominated prompt: the USER turn is preserved first.+ preservedLast := min(ll2, budget)+ user = tokenizer.TrimContentToTokenLimit(user, preservedLast)+ remaining := max(0, budget-preservedLast)+ system = tokenizer.TrimContentToTokenLimit(system, remaining)+ return system, user+ }+ preservedSystem := min(ll, budget)+ system = tokenizer.TrimContentToTokenLimit(system, preservedSystem)+ remaining := max(0, budget-preservedSystem)+ user = tokenizer.TrimContentToTokenLimit(user, remaining)+ return system, user+}+// answerPrompt is the terminal node's input: the ranked evidence under its token// budget plus the question and any research findings. Shared by the one-shot and// the streaming compose so both render exactly the same prompt.@@ -3498,7 +4088,11 @@ func (d AnswerDeps) answerPromptWithEvidence(kb *harness.Kbinfos, question strin// it renders the same prompt, forwards each piece as it arrives, and returns the// assembled answer. A streaming failure is returned so the caller can fall back// to the one-shot call.-func ComposeAnswerStream(ctx context.Context, deps AnswerDeps, model harness.StreamingSessionModel, kb *harness.Kbinfos, question string, partial bool, onDelta func(delta string, isThink bool) error) (AnswerResult, error) {+//+// emptyResult is Python's state["empty_result"] term of+// `no_evidence = abstain or empty_result or not chunks` — the graph compose+// path forwards the state's value (always True there; see formalizeAnswerNode).+func ComposeAnswerStream(ctx context.Context, deps AnswerDeps, model harness.StreamingSessionModel, kb *harness.Kbinfos, question string, partial, emptyResult bool, onDelta func(delta string, isThink bool) error) (AnswerResult, error) {logger := deps.Loggerif logger == nil {logger = _LOG@@ -3512,23 +4106,49 @@ func ComposeAnswerStream(ctx context.Context, deps AnswerDeps, model harness.Str}// Same no-evidence rule as the one-shot path (_compose_answer_from_evidence): the prompt must carry// the degradation instruction even when no empty_response short-circuits.- noEvidence := len(chunks) == 0+ noEvidence := emptyResult || len(chunks) == 0+ // Python _compose_answer_from_evidence:840 — the compose kickoff line. The+ // streaming path has no separate `abstain` signal (Go threads only+ // partial/emptyResult), so the abstain note term cannot fire here.+ note := ""+ if partial {+ note = " — partial answer, some gaps remain"+ }+ logger.Printf("[Composing the answer] Writing the final answer to %q from %d gathered passage(s)%s.",+ trunc(question, 60), len(chunks), note)prompt := deps.answerPromptWithEvidence(kb, question, partial, noEvidence)if noEvidence && deps.EmptyResponse != "" {return AnswerResult{Answer: deps.EmptyResponse, NoEvidence: true}, nil}+ // Python _compose_answer_from_evidence:938-944 — the pre_summary the compose+ // prompt actually carries, content included (first 3000 chars), so a run+ // that answered without the slot facts is diagnosable from the log alone.+ preSummary := ""+ if kb != nil {+ preSummary = kb.PreSummary+ }+ logger.Printf("[Formalize][pre_summary] question=%q pre_summary_len=%d evidence_len=%d\npre_summary=%q",+ trunc(question, 160), len(preSummary), len(prompt.user), truncateRunes(preSummary, 3000))callCtx, cancel := context.WithTimeout(ctx, deadlineToDuration(answerTimeoutS))defer cancel()- userMsg := schema.UserMessage(prompt.user)+ // Same message_fit_in as the one-shot path (agentic_rag_graph.py:946 —+ // Python composes once and streams from the fitted messages).+ systemTurn, userTurn := fitComposePrompt(prompt.system, prompt.user, deps.MaxLength)+ userMsg := schema.UserMessage(userTurn)if len(deps.UserImages) > 0 {// Same as ComposeAnswerWith: attach the vision-gated images so the// compose model sees them on the non-outer path.userMsg = multimodalUserMsg(prompt.user, deps.UserImages)}+ // Temperature: the production streaming carrier+ // (harness.InvokerSessionModel.StreamComplete) samples at 0.3 internally —+ // the same value answer_conf carries here; SessionModel's streaming+ // surface has no per-call temperature, so other carriers run at their own+ // default (Python-parity limitation, flagged in the port notes).reply, err := model.StreamComplete(callCtx, []schema.Message{- *schema.SystemMessage(prompt.system),+ *schema.SystemMessage(systemTurn),*userMsg,}, nil, func(delta string, isThink bool) error {if onDelta == nil || delta == "" {@@ -3582,10 +4202,13 @@ func rankBySimilarity(chunks []map[string]any) []map[string]any {}func similarityOf(c map[string]any) float64 {- if v, ok := toFloat(c["similarity"]); ok {+ // Python: `float(c.get("similarity", 0.0) or c.get("score", 0.0) or 0.0)` —+ // a FALSY similarity (0.0 or missing) falls through to score, so this is+ // not "first key present wins" (same rule SelectSCAView/chunkScore follow).+ if v, ok := toFloat(c["similarity"]); ok && v != 0 {return v}- if v, ok := toFloat(c["score"]); ok {+ if v, ok := toFloat(c["score"]); ok && v != 0 {return v}return 0.0diff --git a/internal/rag/advanced_rag/agentic_rag_graph_test.go b/internal/rag/advanced_rag/agentic_rag_graph_test.goindex c07ac3e8a4..300f0172bb 100644--- a/internal/rag/advanced_rag/agentic_rag_graph_test.go+++ b/internal/rag/advanced_rag/agentic_rag_graph_test.go@@ -15,6 +15,8 @@ import ("github.com/cloudwego/eino/compose""github.com/cloudwego/eino/schema"+ "ragflow/internal/engine"+ "ragflow/internal/engine/types""ragflow/internal/entity""ragflow/internal/rag/advanced_rag/harness")@@ -169,12 +171,15 @@ func TestRouteSCADisabledSCAClosesOut(t *testing.T) {// TestRouteSCAFullPoolShortCircuits pins the PR: a pool at/over the SCA view cap// cannot flip the verdict, so routeSCA finalizes instead of researching — even-// when the verdict is insufficient and headroom remains.+// when the verdict is insufficient and headroom remains. Gated to SECOND-or-later+// reviews (SearchRounds >= 1): claim pseudo-chunks bypass the pool cap, so the+// FIRST review must always get its rewrite round when insufficient (Python+// _route_sca:1411).func TestRouteSCAFullPoolShortCircuits(t *testing.T) {st := &AgenticState{Verdict: VerdictInsufficient,MaxLoops: 3,- SearchRounds: 0,+ SearchRounds: 1,KB: &harness.Kbinfos{Chunks: make([]map[string]any, SCAViewCap)},}st.Deadline = time.Now().Add(120 * time.Second)@@ -186,6 +191,18 @@ func TestRouteSCAFullPoolShortCircuits(t *testing.T) {if n := routeSCA(st, true, 3); n != nodeQueryRewrite {t.Fatalf("below cap: expected rewrite, got %v", n)}+ // FIRST review (SearchRounds == 0) with a full pool: claim pseudo-chunks+ // bypass the pool cap, so the rewrite round must still run.+ first := &AgenticState{+ Verdict: VerdictInsufficient,+ MaxLoops: 3,+ SearchRounds: 0,+ KB: &harness.Kbinfos{Chunks: make([]map[string]any, SCAViewCap)},+ }+ first.Deadline = time.Now().Add(120 * time.Second)+ if n := routeSCA(first, true, 3); n != nodeQueryRewrite {+ t.Fatalf("first review full pool: expected rewrite, got %v", n)+ }}func TestRouteSCAInsufficientStartsRewrite(t *testing.T) {@@ -681,8 +698,8 @@ func (r *channelRetriever) Retrieve(_ context.Context, req harness.RetrieveReque// Each leg names its weight explicitly; nil (the control was not supplied)// is recorded as -1 so the assertions below catch a leg that drops it.weight := -1.0- if req.KeywordsSimilarityWeight != nil {- weight = *req.KeywordsSimilarityWeight+ if req.VectorSimilarityWeight != nil {+ weight = *req.VectorSimilarityWeight}r.weights = append(r.weights, weight)r.topN = append(r.topN, req.TopN)@@ -771,6 +788,87 @@ func (r *fanoutHitRetriever) Retrieve(_ context.Context, req harness.RetrieveReq}, nil}+// claimTopUpEngine stands in for the doc engine across the three store reads+// the channel-0 path makes: the has-compilation probe, the claim recall legs,+// and the directed source-chunk fetch.+type claimTopUpEngine struct {+ engine.DocEngine+}++func (e *claimTopUpEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {+ if _, ok := req.Filter["compile_kwd"]; ok {+ // DatasetHasCompilation probe: the dataset IS compiled.+ return &types.SearchResult{Chunks: []map[string]interface{}{{"id": "row"}}}, nil+ }+ if ids, ok := req.Filter["id"].([]string); ok {+ // LoadChunksForIDs: the source chunks behind the admitted claims.+ rows := make([]map[string]interface{}, 0, len(ids))+ for _, id := range ids {+ rows = append(rows, map[string]interface{}{+ "id": id,+ "content_with_weight": "full text of " + id,+ "doc_id": "doc-1",+ })+ }+ return &types.SearchResult{Chunks: rows}, nil+ }+ // Claim recall leg: one claim citing src-1.+ payload, _ := json.Marshal(map[string]any{+ "name": "The tower is 330m tall",+ "description": "height",+ "evidence": []any{map[string]any{"quote": "330 metres"}},+ "source_chunk_ids": []string{"src-1"},+ })+ return &types.SearchResult{Chunks: []map[string]interface{}{{+ "content_with_weight": string(payload),+ "source_chunk_ids": []string{"src-1"},+ "doc_id": "doc-1",+ "similarity": 0.9,+ }}}, nil+}++// TestFanoutSearchEvidenceTopUp pins the channel-0 directional top-up (Python+// _fanout_search:782-799): after a claim pseudo chunk is admitted, the source+// chunk it cites is fetched by id and admitted too — the verbatim quote alone+// does not carry the surrounding passage.+func TestFanoutSearchEvidenceTopUp(t *testing.T) {+ r := &channelRetriever{}+ de := &claimTopUpEngine{}+ // Unique tenant per run: the harness claim caches are process-global with a+ // 300s TTL, and this test must pay the store reads itself.+ tenant := fmt.Sprintf("tenant-topup-%d", time.Now().UnixNano())+ deps := RAGTools{Search: harness.SearchDeps{+ Backend: r,+ DocEngine: de,+ TenantID: tenant,+ KbIDs: []string{"kb-1"},+ }}+ st := &AgenticState{KB: &harness.Kbinfos{}}+ added := FanoutSearch(context.Background(), deps, st, []string{"tower height topup"}, 8, 60)+ if added != 2 {+ t.Fatalf("added = %d, want 2 (the claim row + its source chunk)", added)+ }+ var claimEntry, srcEntry map[string]any+ for _, c := range st.KB.Chunks {+ id, _ := c["chunk_id"].(string)+ switch {+ case strings.HasPrefix(id, "claim_"):+ claimEntry = c+ case id == "src-1":+ srcEntry = c+ }+ }+ if claimEntry == nil {+ t.Fatalf("claim pseudo chunk not admitted; pool = %v", st.KB.Chunks)+ }+ if srcEntry == nil {+ t.Fatalf("source chunk behind the claim was not top-upped; pool = %v", st.KB.Chunks)+ }+ if got, _ := srcEntry["content_with_weight"].(string); got != "full text of src-1" {+ t.Errorf("top-up content = %q, want the fetched source chunk text", got)+ }+}+// draftModel returns a canned draft and records the prompt it was given.type draftModel struct {reply string@@ -1227,12 +1325,20 @@ func TestFinalizeRunsOnceFromInsideTheGraph(t *testing.T) {// Mirror Rag: the hook is idempotent, so the graph's node and the// post-graph safety net cannot both compose.composed := false- deps.Finalize = func(context.Context) {+ deps.Finalize = func(_ context.Context, _ bool, _ bool, question string) {if composed {return}composed = truefinalized+++ // The low graph's formalize_answer must forward the FORMALIZED+ // question the formalize_question node wrote into the RunRequest+ // (Python state["question"], agentic_rag_graph.py:834) — composing+ // from the outer tool argument collapses a multi-hop answer to its+ // first sub-answer.+ if question != "when was it made?" {+ t.Errorf("Finalize question = %q, want the formalized RunRequest question", question)+ }}var buf bytes.Buffer@@ -1244,7 +1350,7 @@ func TestFinalizeRunsOnceFromInsideTheGraph(t *testing.T) {t.Fatalf("formalize_answer node composed %d time(s), want 1; log:\n%s", finalized, buf.String())}// ...and the post-graph call is then a no-op.- deps.Finalize(ctx)+ deps.Finalize(ctx, false, false, "")if finalized != 1 {t.Errorf("composition ran %d time(s) after the post-graph call, want 1", finalized)}@@ -1481,13 +1587,14 @@ func (rfTagger) LabelQuestion(_ context.Context, _ string, _ []*entity.Knowledgereturn map[string]float64{"location": 1.0}}-// TestAgenticLoopProjectsRankFeature guards the projection that Python gets for-// free: tools.retrieve reads every setting (incl.-// rank_feature=label_question(question, self.kbs), agentic_rag.py:668) off the-// RAGTools instance, so the agentic loop's fan-out and action-session retrieval-// must see the same Tagger/KBs. A partial SearchDeps projection silently drops-// the tag boost and degrades ranking with no error.-func TestAgenticLoopProjectsRankFeature(t *testing.T) {+// TestAgenticLoopRankFeaturePolicy pins the rank_feature policy: Python's+// search.py legs (the fan-out channels, search_chunks, retrieve) call+// retriever.retrieval WITHOUT rank_feature (:158-173/:223-238/:260-275), and+// only RAGTools.retrieve — the low-mode direct pass — passes+// rank_feature=label_question(question, self.kbs) (agentic_rag.py:668). The+// agentic loop's research retrievals must therefore carry NO tag boost, even+// though the Tagger/KBs projection is wired through.+func TestAgenticLoopRankFeaturePolicy(t *testing.T) {SetAgenticLoop(NewAgenticLoop())defer SetAgenticLoop(nil)@@ -1515,13 +1622,12 @@ func TestAgenticLoopProjectsRankFeature(t *testing.T) {t.Fatal("the agentic loop performed no retrieval")}// Scope to the fan-out / slot research retrievals — the ones Python issues- // through tools.retrieve / search_chunks and therefore the ones that must- // carry rank_feature.+ // through search.py's hybrid/bm25 legs, none of which pass rank_feature.//- // Deliberately exempt: the navigation tools' own recalls (chunk-agg routing+ // Also exempt: the navigation tools' own recalls (chunk-agg routing// via chunkAggRetrieveFrom, and _recall_chunk_ids_in_doc) which mirror// Python's _search_layers_nav_chunk_agg / _recall_chunk_ids_in_doc- // (navigation.py:1399) — neither passes rank_feature there.+ // (navigation.py:1399) — neither passes rank_feature there either.research := map[string]bool{"when was it opened": true, "when opened": true}checked := 0for i, q := range r.queries {@@ -1529,8 +1635,8 @@ func TestAgenticLoopProjectsRankFeature(t *testing.T) {continue}checked++- if r.features[i]["location"] != 1.0 {- t.Errorf("retrieve %q RankFeature = %v, want the label_question tag boost", q, r.features[i])+ if len(r.features[i]) != 0 {+ t.Errorf("retrieve %q RankFeature = %v, want nil (search.py legs never pass rank_feature)", q, r.features[i])}}if checked == 0 {@@ -2176,90 +2282,6 @@ func TestGenJSONStripsThinkAndFenceOnFirstTry(t *testing.T) {}}-// fakeLLMCache is an in-memory LLMCache double.-type fakeLLMCache struct {- entries map[string]string- gets int- sets int-}--func newFakeLLMCache() *fakeLLMCache { return &fakeLLMCache{entries: map[string]string{}} }--func (c *fakeLLMCache) Get(_ context.Context, key string) (string, bool) {- c.gets++- v, ok := c.entries[key]- return v, ok-}--func (c *fakeLLMCache) Set(_ context.Context, key, value string) {- c.sets++- c.entries[key] = value-}--// TestGenJSONCachesFirstSuccessfulReply pins gen_json's 24h reply cache-// (Python get_llm_cache / set_llm_cache): the CLEANED answer of the first-// successful parse is stored, and an identical (model, system, user) triple is-// served from the cache WITHOUT a second model call.-func TestGenJSONCachesFirstSuccessfulReply(t *testing.T) {- cache := newFakeLLMCache()- mdl := &scriptedModel{}- mdl.push("Sure.\n```json\n{\"ok\": true}\n```\n")-- adapter := &jsonModelAdapter{inner: mdl, modelName: "gpt-4o", cache: cache}- first, err := adapter.GenJSON(context.Background(), "Render JSON.")- if err != nil {- t.Fatalf("first GenJSON: %v", err)- }- if m, ok := first.(map[string]any); !ok || m["ok"] != true {- t.Fatalf("first GenJSON = %#v", first)- }- if len(mdl.seen) != 1 || cache.sets != 1 {- t.Fatalf("first call: model calls = %d, cache sets = %d, want 1/1", len(mdl.seen), cache.sets)- }- wantKey := genJSONCacheKey("gpt-4o", "Render JSON.", "Output:\n")- // The cleanup regex removes only the ```json fence and the trailing fence:- // the "Sure.\n" prose prefix has no </think> terminator, so it survives —- // exactly as it does in Python.- if got := cache.entries[wantKey]; got != "Sure.\n{\"ok\": true}\n" {- t.Errorf("cached value = %q, want the cleaned answer", got)- }-- second, err := adapter.GenJSON(context.Background(), "Render JSON.")- if err != nil {- t.Fatalf("second GenJSON: %v", err)- }- if m, ok := second.(map[string]any); !ok || m["ok"] != true {- t.Fatalf("second GenJSON = %#v", second)- }- if len(mdl.seen) != 1 {- t.Errorf("cache hit still called the model: %d calls", len(mdl.seen))- }-}--// TestGenJSONCacheKeyMatchesPython locks the key SHAPE: %x over the xxh64 of-// model+system+user+"{}" (str(gen_conf) of the default {} is "{}"), i.e. the 16-// lowercase hex digits Python's xxhash.xxh64(...).hexdigest() produces. An empty-// model name yields "" so the cache is skipped rather than collapsing models.-func TestGenJSONCacheKeyMatchesPython(t *testing.T) {- got := genJSONCacheKey("gpt-4o", "SYS", "Output:\n")- // Reference value produced by Python itself:- // xxhash.xxh64(('gpt-4o' + 'SYS' + 'Output:\n' + '{}').encode('utf-8')).hexdigest()- // -> '02dffaad8465c0c9'- // Note the LEADING ZERO: a plain %x prints 15 digits ('2dffaad8465c0c9'),- // which would silently stop sharing the cache with Python.- if got != "02dffaad8465c0c9" {- t.Fatalf("key = %q, want 02dffaad8465c0c9 (Python xxh64 hexdigest)", got)- }- // Empty model name yields "" so callers SKIP the cache rather than collapse- // every model onto one bucket.- if genJSONCacheKey("", "SYS", "Output:\n") != "" {- t.Error("empty model name must disable the cache key")- }- if genJSONCacheKey("a", "SYS", "u") == genJSONCacheKey("b", "SYS", "u") {- t.Error("distinct model names produced the same key")- }-}-// TestGenJSONFitsPromptToContextWindow pins Python gen_json's// message_fit_in(form_message(system_prompt, user_prompt), chat_mdl.max_length):// an oversized system prompt must be trimmed to the model window BEFORE thediff --git a/internal/rag/advanced_rag/agentic_rag_test.go b/internal/rag/advanced_rag/agentic_rag_test.goindex 8fb771d8ae..758d73344b 100644--- a/internal/rag/advanced_rag/agentic_rag_test.go+++ b/internal/rag/advanced_rag/agentic_rag_test.go@@ -25,6 +25,7 @@ import ("strings""sync""testing"+ "time""github.com/cloudwego/eino/schema""ragflow/internal/rag/advanced_rag/harness"@@ -386,7 +387,7 @@ func (m *streamingModel) StreamComplete(_ context.Context, _ []schema.Message, _func TestComposeAnswerStreamForwardsDeltas(t *testing.T) {m := &streamingModel{pieces: []string{"Hello ", "world"}}var got []string- res, err := ComposeAnswerStream(context.Background(), AnswerDeps{Model: m}, m, nil, "q", false,+ res, err := ComposeAnswerStream(context.Background(), AnswerDeps{Model: m}, m, nil, "q", false, false,func(delta string, _ bool) error {got = append(got, delta)return nil@@ -406,7 +407,7 @@ func TestComposeAnswerStreamForwardsDeltas(t *testing.T) {func TestComposeAnswerStreamReturnsErrorOnFailure(t *testing.T) {m := &streamingModel{fail: true}- if _, err := ComposeAnswerStream(context.Background(), AnswerDeps{Model: m}, m, nil, "q", false, nil); err == nil {+ if _, err := ComposeAnswerStream(context.Background(), AnswerDeps{Model: m}, m, nil, "q", false, false, nil); err == nil {t.Fatal("want an error so the caller can fall back to one-shot")}}@@ -450,10 +451,15 @@ func TestGetCitationGuidelinesUsesDefaultWithoutOverride(t *testing.T) {}func TestGetCitationGuidelinesHonoursOverride(t *testing.T) {- // Python renders the user's template; Go takes the rendered result verbatim.+ // Python renders the user's template and STILL appends the+ // illustrative-IDs caveat after it (generator.py:227-228 concatenates the+ // suffix onto whatever template rendered).got := GetCitationGuidelines("Cite as [n].")- if got != "Cite as [n]." {- t.Fatalf("got %q, want the override", got)+ if !strings.HasPrefix(got, "Cite as [n].") {+ t.Fatalf("got %q, want the override first", got)+ }+ if !strings.Contains(got, "IMPORTANT: The example IDs above") {+ t.Fatalf("got %q, want the illustrative-IDs caveat appended", got)}}@@ -590,6 +596,113 @@ func TestOuterReactSessionToolCallKeepsSharedRequestIntact(t *testing.T) {}}+// TestRagFlightSharesConcurrentIdenticalCalls pins the single-flight contract:+// a caller arriving while the flight is open waits on it and replays its+// answer; a caller arriving AFTER the window closed owns a fresh execution+// (window-only dedup) and — like production's defer — must end it. The main+// goroutine holds the window open long enough that the waiters normally take+// the wait path, but the assertions tolerate both arrivals. Run with -race.+func TestRagFlightSharesConcurrentIdenticalCalls(t *testing.T) {+ session := &outerReactSession{}++ owner, wait := session.beginRagFlight("same question")+ if owner == nil || wait != nil {+ t.Fatalf("first caller: owner=%v wait=%v, want (flight, nil)", owner, wait)+ }++ const waiters = 3+ var wg sync.WaitGroup+ got := make([]string, waiters)+ for i := 0; i < waiters; i++ {+ wg.Add(1)+ go func(i int) {+ defer wg.Done()+ o, w := session.beginRagFlight("same question")+ if o != nil {+ // Arrived after the window closed: this caller owns a fresh+ // execution and must end it (production's ToolCall defers+ // endRagFlight), or later callers would block forever.+ session.endRagFlight("same question", o)+ return+ }+ <-w.done+ got[i] = w.answer+ }(i)+ }++ // Hold the window open so the waiters observe it (scheduling latency is+ // microseconds; the tolerant assertions above cover the pathological case).+ time.Sleep(50 * time.Millisecond)+ owner.answer = "shared answer"+ session.endRagFlight("same question", owner)+ wg.Wait()++ for i, g := range got {+ if g != "shared answer" && g != "" {+ t.Errorf("waiter %d replayed %q, want %q or the owner path (\"\")", i, g, "shared answer")+ }+ }++ // After the flight ends a NEW call owns a fresh execution (window-only+ // dedup: a later round must genuinely re-run).+ owner2, wait2 := session.beginRagFlight("same question")+ if owner2 == nil || wait2 != nil {+ t.Fatalf("post-window caller: owner=%v wait=%v, want (flight, nil)", owner2, wait2)+ }+ session.endRagFlight("same question", owner2)++ // A different question never shares a flight.+ other, otherWait := session.beginRagFlight("different question")+ if other == nil || otherWait != nil {+ t.Fatalf("different question: owner=%v wait=%v, want (flight, nil)", other, otherWait)+ }+ session.endRagFlight("different question", other)+}++// TestOuterReactSessionToolCallWaitsOnInFlightRag pins the ToolCall-level+// behavior: a rag call whose question already has an in-progress execution+// blocks and replays that execution's answer WITHOUT running its own graph —+// no second publish, no second call record (Python's asyncio.gather runs both+// duplicates fully and the terminal fold discards the loser; this wait is the+// approved Go-side single-flight, see ragFlight).+func TestOuterReactSessionToolCallWaitsOnInFlightRag(t *testing.T) {+ spec := harness.GetMode("naive")+ session := &outerReactSession{+ ctx: context.Background(),+ spec: spec,+ kb: &harness.Kbinfos{},+ resp: &RunResponse{Mode: spec},+ logger: log.New(io.Discard, "", 0),+ req: harness.RunRequest{Question: "original question"},+ }++ owner, wait := session.beginRagFlight("the question")+ if owner == nil || wait != nil {+ t.Fatalf("pre-registered flight: owner=%v wait=%v, want (flight, nil)", owner, wait)+ }+ go func() {+ time.Sleep(20 * time.Millisecond)+ owner.answer = "shared answer"+ session.endRagFlight("the question", owner)+ }()++ start := time.Now()+ got, err := session.ToolCall("rag", map[string]interface{}{"question": "the question"})+ if err != nil {+ t.Fatalf("ToolCall: %v", err)+ }+ if got != "shared answer" {+ t.Errorf("ToolCall replayed %q, want %q", got, "shared answer")+ }+ if elapsed := time.Since(start); elapsed < 15*time.Millisecond {+ t.Errorf("ToolCall returned after %v — it did NOT wait for the in-flight execution", elapsed)+ }+ // The waiter must not have published or recorded its own call result.+ if len(session.calls) != 0 {+ t.Errorf("calls = %d, want 0 (a waiting call must not publish)", len(session.calls))+ }+}+// ragTestChunkIDs lists a chunk pool's ids in order.func ragTestChunkIDs(chunks []map[string]any) []string {out := make([]string, 0, len(chunks))diff --git a/internal/rag/advanced_rag/evidence_fill_test.go b/internal/rag/advanced_rag/evidence_fill_test.gonew file mode 100644index 0000000000..7e61aed276--- /dev/null+++ b/internal/rag/advanced_rag/evidence_fill_test.go@@ -0,0 +1,395 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package advanced_rag++import (+ "context"+ "errors"+ "fmt"+ "strings"+ "sync"+ "testing"++ "github.com/cloudwego/eino/schema"+ "ragflow/internal/rag/advanced_rag/harness"+)++// ---------------------------------------------------------------------------+// Test doubles+// ---------------------------------------------------------------------------++// failingModel errors on every Complete call — the batched-answer LLM seam.+type failingModel struct {+ mu sync.Mutex+ calls int+}++func (m *failingModel) Complete(context.Context, []schema.Message, []harness.ToolSpec) (*harness.ModelReply, error) {+ m.mu.Lock()+ defer m.mu.Unlock()+ m.calls+++ return nil, errors.New("model unavailable")+}++// Calls reports how many times Complete was invoked.+func (m *failingModel) Calls() int {+ m.mu.Lock()+ defer m.mu.Unlock()+ return m.calls+}++// claimRow builds one claim pseudo-chunk (Python "[evidence] <name> — <desc>"+// row) the way the pool stores it.+func claimRow(id, content string) map[string]any {+ return map[string]any{+ "chunk_id": id,+ "content_with_weight": content,+ "doc_id": "d1",+ }+}++// ---------------------------------------------------------------------------+// PrefillSlotsFromEvidence(Python _prefill_slots_from_evidence)+// ---------------------------------------------------------------------------++// TestPrefillSlotsFromEvidence_CoverageBoundary pins the 0.6 word-coverage+// gate(Python _EVIDENCE_PREFILL_COVERAGE=0.6, :1738): exactly-at and above+// prefill, below does not; the candidate is the de-prefixed first line and the+// strength is the coverage itself.+func TestPrefillSlotsFromEvidence_CoverageBoundary(t *testing.T) {+ // Clue terms: {alpha, tower, opened, paris, 1889} (>=3 code points only).+ clue := "alpha tower opened paris 1889"+ // Covers tower/opened/paris = 3/5 = exactly 0.6.+ atGate := claimRow("claim_a", "[evidence] Eiffel Tower — landmark\nEvidence (verbatim): \"the tower opened in paris near the river\"")+ // Covers tower/opened = 2/5 = 0.4.+ below := claimRow("claim_b", "[evidence] Eiffel Tower — landmark\nEvidence (verbatim): \"the tower opened here long ago\"")++ newState := func() harness.State {+ return harness.NewState([]harness.Variable{+ {ID: 1, Type: "aspect", QuestionClues: []string{clue}},+ }, 0, nil)+ }++ // At the gate: prefilled.+ st := newState()+ if n := PrefillSlotsFromEvidence(&st, &harness.Kbinfos{Chunks: []map[string]any{atGate}}); n != 1 {+ t.Fatalf("coverage exactly 0.6 must prefill, filled = %d", n)+ }+ v := st.State[0]+ if v.Candidate == nil || *v.Candidate != "Eiffel Tower — landmark" {+ t.Fatalf("candidate = %v, want the de-prefixed first line", v.Candidate)+ }+ if v.CandidateStrength == nil || *v.CandidateStrength != 0.6 {+ t.Fatalf("strength = %v, want 0.6", v.CandidateStrength)+ }++ // Below the gate: untouched.+ st = newState()+ if n := PrefillSlotsFromEvidence(&st, &harness.Kbinfos{Chunks: []map[string]any{below}}); n != 0 {+ t.Fatalf("coverage 0.4 must NOT prefill, filled = %d", n)+ }+ if st.State[0].Candidate != nil {+ t.Fatalf("candidate = %v, want nil", st.State[0].Candidate)+ }+}++// TestPrefillSlotsFromEvidence_Guards mirrors the skips(Python :1714-1728,+// :1744): no claim rows in the pool, already-filled slots, blank clues, and a+// first line that strips to nothing.+func TestPrefillSlotsFromEvidence_Guards(t *testing.T) {+ st := harness.NewState([]harness.Variable{+ {ID: 1, Type: "aspect", QuestionClues: []string{"eiffel tower paris"}},+ {ID: 2, Type: "aspect", Candidate: strPtr("known")},+ {ID: 3, Type: "aspect", QuestionClues: []string{" "}},+ }, 0, nil)++ // No claim rows: 0 regardless of content.+ if n := PrefillSlotsFromEvidence(&st, &harness.Kbinfos{Chunks: []map[string]any{+ claimRow("c1", "plain chunk"),+ }}); n != 0 {+ t.Fatalf("no claim_ rows must yield 0, got %d", n)+ }+ // nil KB is safe.+ if n := PrefillSlotsFromEvidence(&st, nil); n != 0 {+ t.Fatalf("nil KB must yield 0, got %d", n)+ }++ // Row whose head strips to nothing: even full coverage fills nothing.+ if n := PrefillSlotsFromEvidence(&st, &harness.Kbinfos{Chunks: []map[string]any{+ claimRow("claim_x", "[evidence] — \nEvidence (verbatim): \"eiffel tower paris\""),+ }}); n != 0 {+ t.Fatalf("empty name must yield 0, got %d", n)+ }+ if st.State[0].Candidate != nil || st.State[2].Candidate != nil {+ t.Fatal("unfilled slots must stay untouched")+ }+ if st.State[1].Candidate == nil || *st.State[1].Candidate != "known" {+ t.Fatal("a filled slot must never be overwritten")+ }+}++// TestRunSlotResearchPass_PrefilledSlotsSkipSession pins the wiring(Python+// :1782-1790): when the pooled evidence covers every slot at >= 0.6, the+// pass fills them WITHOUT running a single action session (zero model calls).+func TestRunSlotResearchPass_PrefilledSlotsSkipSession(t *testing.T) {+ kb := &harness.Kbinfos{Chunks: []map[string]any{+ claimRow("claim_a", "[evidence] Eiffel Tower — built in Paris by Gustave Eiffel, opened in 1889\nEvidence (verbatim): \"...\""),+ }}+ st := &AgenticState{+ Question: "what is the tower",+ SlotTable: harness.NewState([]harness.Variable{+ {ID: 0, Type: "aspect", QuestionClues: []string{"eiffel tower opened paris 1889"}},+ {ID: 1, Type: "aspect", QuestionClues: []string{"gustave eiffel designed paris tower"}},+ }, 0, nil),+ }+ mdl := &scriptedModel{}+ deps := harness.SessionDeps{Model: mdl, KB: kb}++ res := RunSlotResearchPass(context.Background(), deps, st.Question, st, 120.0)+ if res == nil {+ t.Fatal("expected a result")+ }+ // The prefill removed BOTH sessions: the model was never called.+ if got := len(mdl.seen); got != 0 {+ t.Fatalf("model calls = %d, want 0 (prefill must skip every session)", got)+ }+ for i, want := range []string{+ "Eiffel Tower — built in Paris by Gustave Eiffel, opened in 1889",+ "Eiffel Tower — built in Paris by Gustave Eiffel, opened in 1889",+ } {+ if st.SlotTable.State[i].Candidate == nil || *st.SlotTable.State[i].Candidate != want {+ t.Fatalf("slot %d candidate = %v, want %q", i, st.SlotTable.State[i].Candidate, want)+ }+ }+ if len(res.UnresolvedSlots) != 0 {+ t.Fatalf("unresolved = %d, want 0", len(res.UnresolvedSlots))+ }+}++// ---------------------------------------------------------------------------+// BatchFillSlots(Python _batch_fill_slots)+// ---------------------------------------------------------------------------++// TestBatchFillSlots_ClustersOverlappingEvidence: two slots sharing >= 2+// evidence ids with Jaccard >= 0.15 are answered in ONE call; a slot with+// disjoint evidence stays in its own (size-1) cluster and is never asked.+func TestBatchFillSlots_ClustersOverlappingEvidence(t *testing.T) {+ chunks := []map[string]any{}+ for _, id := range []string{"c1", "c2", "c3", "c4", "c9", "c10"} {+ chunks = append(chunks, map[string]any{+ "chunk_id": id,+ "content_with_weight": "content of " + id,+ "doc_id": "d1",+ })+ }+ slotTable := harness.NewState([]harness.Variable{+ {ID: 1, Type: "aspect", QuestionClues: []string{"who built it"}},+ {ID: 2, Type: "aspect", QuestionClues: []string{"when opened"}},+ {ID: 3, Type: "aspect", QuestionClues: []string{"unrelated"}},+ }, 0, nil)+ slotEvidence := map[string]SlotEvidence{+ "1": {EvidenceIDs: []string{"c1", "c2", "c3"}}, // vs 2: inter=2, union=4, sim=0.5+ "2": {EvidenceIDs: []string{"c1", "c2", "c4"}},+ "3": {EvidenceIDs: []string{"c9", "c10"}},+ }+ mdl := &scriptedModel{}+ mdl.push(`{"1": "Gustave Eiffel", "2": "1889", "3": "invented"}`)+ deps := harness.SessionDeps{Model: mdl, KB: &harness.Kbinfos{Chunks: chunks}}++ if got := BatchFillSlots(context.Background(), deps, &slotTable, slotEvidence); got != 2 {+ t.Fatalf("filled = %d, want 2", got)+ }+ // Exactly ONE generation call served slots 1 and 2.+ if len(mdl.seen) != 1 {+ t.Fatalf("model calls = %d, want 1", len(mdl.seen))+ }+ if got := *slotTable.State[0].Candidate; got != "Gustave Eiffel" {+ t.Fatalf("slot 1 candidate = %q", got)+ }+ if got := *slotTable.State[1].Candidate; got != "1889" {+ t.Fatalf("slot 2 candidate = %q", got)+ }+ // The disjoint slot was never asked, never filled.+ if slotTable.State[2].Candidate != nil {+ t.Fatalf("slot 3 candidate = %v, want nil", slotTable.State[2].Candidate)+ }+ // The prompt carries both sub-questions and the shared evidence body.+ user := mdl.seen[0][1].Content+ for _, want := range []string{"- 1: who built it", "- 2: when opened", "content of c1", "content of c4"} {+ if !strings.Contains(user, want) {+ t.Fatalf("prompt missing %q:\n%s", want, user)+ }+ }+}++// TestBatchFillSlots_ThresholdsBlockMerging pins both gates(Python+// :1574-1575): fewer than MIN_SHARED shared ids, or a Jaccard below+// MIN_SIM, keeps the slots in separate size-1 clusters — no LLM call at all.+func TestBatchFillSlots_ThresholdsBlockMerging(t *testing.T) {+ mk := func(e1, e2 []string) (harness.State, map[string]SlotEvidence) {+ st := harness.NewState([]harness.Variable{+ {ID: 1, Type: "aspect", QuestionClues: []string{"a"}},+ {ID: 2, Type: "aspect", QuestionClues: []string{"b"}},+ }, 0, nil)+ ev := map[string]SlotEvidence{+ "1": {EvidenceIDs: e1},+ "2": {EvidenceIDs: e2},+ }+ return st, ev+ }+ ids := func(n int) []string {+ out := make([]string, 0, n)+ for i := 1; i <= n; i++ {+ out = append(out, fmt.Sprintf("c%02d", i))+ }+ return out+ }++ cases := []struct {+ name string+ e1 []string+ e2 []string+ }{+ // inter=1 < MIN_SHARED=2.+ {"shared-below-floor", []string{"c01", "c02"}, []string{"c02", "c03"}},+ // inter=2, union=14 → sim≈0.143 < MIN_SIM=0.15.+ {"sim-below-gate", ids(14), []string{"c13", "c14"}},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ st, ev := mk(tc.e1, tc.e2)+ mdl := &scriptedModel{}+ deps := harness.SessionDeps{Model: mdl, KB: &harness.Kbinfos{}}+ if got := BatchFillSlots(context.Background(), deps, &st, ev); got != 0 {+ t.Fatalf("filled = %d, want 0", got)+ }+ if len(mdl.seen) != 0 {+ t.Fatalf("model calls = %d, want 0 (below the merge gates)", len(mdl.seen))+ }+ })+ }+}++// TestBatchFillSlots_MaxSlotsPerBatch pins _EVIDENCE_BATCH_MAX_SLOTS=4+// (Python :1576, :1625): the fifth identical-evidence slot cannot join the+// full batch and is left unfilled this round.+func TestBatchFillSlots_MaxSlotsPerBatch(t *testing.T) {+ vars := make([]harness.Variable, 0, 5)+ ev := map[string]SlotEvidence{}+ for i := 1; i <= 5; i++ {+ vars = append(vars, harness.Variable{ID: i, Type: "aspect", QuestionClues: []string{"same clue"}})+ ev[fmt.Sprint(i)] = SlotEvidence{EvidenceIDs: []string{"c1", "c2"}}+ }+ st := harness.NewState(vars, 0, nil)+ kb := &harness.Kbinfos{Chunks: []map[string]any{+ {"chunk_id": "c1", "content_with_weight": "shared text one", "doc_id": "d1"},+ {"chunk_id": "c2", "content_with_weight": "shared text two", "doc_id": "d1"},+ }}+ mdl := &scriptedModel{}+ mdl.push(`{"1": "a", "2": "b", "3": "c", "4": "d", "5": "e"}`)+ deps := harness.SessionDeps{Model: mdl, KB: kb}++ if got := BatchFillSlots(context.Background(), deps, &st, ev); got != 4 {+ t.Fatalf("filled = %d, want 4 (MAX_SLOTS)", got)+ }+ if len(mdl.seen) != 1 {+ t.Fatalf("model calls = %d, want 1", len(mdl.seen))+ }+ for i := 0; i < 4; i++ {+ if st.State[i].Candidate == nil {+ t.Fatalf("slot %d must be filled", i+1)+ }+ }+ if st.State[4].Candidate != nil {+ t.Fatalf("slot 5 candidate = %v, want nil (batch cap 4)", st.State[4].Candidate)+ }+}++// TestBatchFillSlots_LLMFailureBestEffort mirrors Python :1681-1683: a failed+// model call (or an unparsable reply, or no model at all) fills nothing and+// never panics — the pass continues without the batch.+func TestBatchFillSlots_LLMFailureBestEffort(t *testing.T) {+ mk := func() (harness.State, map[string]SlotEvidence) {+ st := harness.NewState([]harness.Variable{+ {ID: 1, Type: "aspect", QuestionClues: []string{"a"}},+ {ID: 2, Type: "aspect", QuestionClues: []string{"b"}},+ }, 0, nil)+ ev := map[string]SlotEvidence{+ "1": {EvidenceIDs: []string{"c1", "c2"}},+ "2": {EvidenceIDs: []string{"c1", "c2"}},+ }+ return st, ev+ }+ kb := &harness.Kbinfos{Chunks: []map[string]any{+ {"chunk_id": "c1", "content_with_weight": "text one", "doc_id": "d1"},+ {"chunk_id": "c2", "content_with_weight": "text two", "doc_id": "d1"},+ }}++ t.Run("model-error", func(t *testing.T) {+ st, ev := mk()+ mdl := &failingModel{}+ deps := harness.SessionDeps{Model: mdl, KB: kb}+ if got := BatchFillSlots(context.Background(), deps, &st, ev); got != 0 {+ t.Fatalf("filled = %d, want 0", got)+ }+ if mdl.Calls() != 1 {+ t.Fatalf("model calls = %d, want 1 (the cluster is attempted)", mdl.Calls())+ }+ for _, v := range st.State {+ if v.Candidate != nil {+ t.Fatal("no slot may be filled after a failed call")+ }+ }+ })+ t.Run("unparsable-reply", func(t *testing.T) {+ st, ev := mk()+ mdl := &scriptedModel{}+ mdl.push("sorry, I cannot answer that") // no JSON object+ deps := harness.SessionDeps{Model: mdl, KB: kb}+ if got := BatchFillSlots(context.Background(), deps, &st, ev); got != 0 {+ t.Fatalf("filled = %d, want 0", got)+ }+ })+ t.Run("no-model", func(t *testing.T) {+ st, ev := mk()+ deps := harness.SessionDeps{KB: kb} // Model nil — Python :1674 `return filled`+ if got := BatchFillSlots(context.Background(), deps, &st, ev); got != 0 {+ t.Fatalf("filled = %d, want 0", got)+ }+ })+}++// TestBatchFillSlots_SkipDiagnostics pins the never-firing diagnostics+// (Python :1606-1610): fewer than two evidence-carrying unresolved slots+// logs the skip line and short-circuits before any model call.+func TestBatchFillSlots_SkipDiagnostics(t *testing.T) {+ st := harness.NewState([]harness.Variable{+ {ID: 1, Type: "aspect", QuestionClues: []string{"a"}},+ {ID: 2, Type: "aspect", QuestionClues: []string{"b"}},+ }, 0, nil)+ ev := map[string]SlotEvidence{+ "1": {EvidenceIDs: []string{"c1"}}, // only ONE slot carries evidence+ }+ mdl := &scriptedModel{}+ deps := harness.SessionDeps{Model: mdl, KB: &harness.Kbinfos{}}+ if got := BatchFillSlots(context.Background(), deps, &st, ev); got != 0 {+ t.Fatalf("filled = %d, want 0", got)+ }+ if len(mdl.seen) != 0 {+ t.Fatalf("model calls = %d, want 0", len(mdl.seen))+ }+}diff --git a/internal/rag/advanced_rag/harness/action_session.go b/internal/rag/advanced_rag/harness/action_session.goindex dfac24ed50..333eab5afa 100644--- a/internal/rag/advanced_rag/harness/action_session.go+++ b/internal/rag/advanced_rag/harness/action_session.go@@ -28,6 +28,7 @@ import ("strings""sync""time"+ "unicode/utf8""github.com/RealAlexandreAI/json-repair""github.com/cloudwego/eino/compose"@@ -960,10 +961,12 @@ var (Type: "function",Function: ToolFunction{Name: "search_chunks",- Description: `WHEN TO CALL: Primary semantic recall. Use when exact retrieve returns nothing useful, when the corpus is large and you are unsure which document holds the answer, or when the answer passage shares no surface words with your query. Send 1-2 queries.` ++ Description: `WHEN TO CALL: Primary semantic recall. Use when exact retrieve returns nothing useful, when the corpus is large and you are unsure which document holds the answer, or when the answer passage shares no surface words with your query. Send 1-2 queries; compiled-structure expansion is automatic (a no-op without compiled structure). ` +`DO NOT CALL: When you already have a doc_id and want to read that document (use list_chunks); when a single exact passage would be found faster by grep-style retrieve.` +- `ARGUMENTS: query — array of 1-2 strings. Compiled-structure expansion is automatic and a no-op on datasets without compiled structure, so no extra argument is needed.` +- `OUTPUT: Relevance-ranked snippet chunks, possibly with structural neighbours (parent/child headings, sibling pages) appended. ok = new evidence; redundant = already seen.` ++ `ARGUMENTS: query — array of 1-2 strings.` ++ `OUTPUT: Relevance-ranked snippet chunks, possibly with structural neighbours appended. ` ++ `Results may LEAD with [claim score=...] entries — the dataset's compiled atomic facts carrying VERBATIM source quotes. If a claim directly answers the query, cite it and answer WITHOUT further searching; deep-read its listed chunk only for missing context or numbers. ` ++ `ok = new evidence; redundant = already seen.` +`IF IT FAILS: miss means this query matched nothing — change the angle or fall back to retrieve or navigate_tree. Re-issuing a near-duplicate query is skipped as redundant, so vary the query instead of paraphrasing it.`,Parameters: arrayParam("", 1, 2),},@@ -1017,7 +1020,8 @@ var (Description: `WHEN TO CALL: You know the doc_id and need to PINPOINT where the answer lives inside that one document, without reading every chunk. The in-document counterpart of navigate_tree.` +`DO NOT CALL: When you have no doc_id yet; when the document has no compiled structure (use list_chunks to read the full document).` +`ARGUMENTS: doc_id — string, required. query — string, what to locate within the document. kind — enum catalog / mindmap / graph, default catalog (compiled-structure kind).` +- `OUTPUT: The structure outline annotated with matching chunk_ids, reading-order aware. ok = useful hits; poor (chunk_ptrs = 0) means it drilled to nothing usable.` ++ `OUTPUT: The structure outline annotated with matching chunk_ids, reading-order aware. [claim] lines carry VERBATIM quotes from the document — cite them and answer WITHOUT calling list_chunks when they directly answer the query (deep-read the claimed chunk ids only for surrounding context or numbers the quotes lack). ` ++ `ok = useful hits; poor (chunk_ptrs = 0) means it drilled to nothing usable.` +`IF IT FAILS: empty (no_structure) — try another doc_id or kind, or fall back to list_chunks / search_chunks. poor — read the full document via list_chunks(doc_id). A second empty disables the tool for the session.`,Parameters: map[string]any{"type": "object",@@ -1288,10 +1292,10 @@ func ReasonStatus(reason string) string {//// ONE structural difference, isolated to one seam: Python calls the provider's native// tool-calling API (`tools=[...]`, `tool_choice="auto"`) and parses-// `message.tool_calls`; the Go chat seam (internal/agent/chat) has no tools field, so-// invocation goes through the SessionModel interface, whose default implementation asks-// the model to emit the call as a JSON block and parses it back. Downstream session-// logic is identical either way.+// `message.tool_calls`; the Go chat seam exposes the same capability+// (chat.Request.Tools / Response.ToolCalls), and the SessionModel implementation+// (InvokerSessionModel) forwards the harness ToolSpec surface through it. There is+// no prompt-based fallback: downstream session logic is identical either way.const (// initTimeoutS bounds the slot-table decomposition call.@@ -1348,6 +1352,34 @@ func searchTokens(q string) map[string]struct{} {return out}+// argQueryString mirrors Python `str(args.get("query") or "").strip()`+// (action_session.py:1430): a string passes through, a nil/empty value yields+// "", and any other value — typically the []any query list retrieve accepts —+// is stringified so it still participates in near-dup detection and the+// seen_queries ledger.+func argQueryString(v any) string {+ switch t := v.(type) {+ case nil:+ return ""+ case string:+ if t == "" {+ return ""+ }+ return strings.TrimSpace(t)+ case []any:+ if len(t) == 0 {+ return ""+ }+ parts := make([]string, 0, len(t))+ for _, e := range t {+ parts = append(parts, fmt.Sprintf("%v", e))+ }+ return strings.TrimSpace("[" + strings.Join(parts, ", ") + "]")+ default:+ return strings.TrimSpace(fmt.Sprintf("%v", t))+ }+}+// IsNearDup mirrors Python _is_near_dup: true when q shares >=// nearDupJaccard of its tokens with any query in seen.func IsNearDup(q string, seen []string) bool {@@ -1383,6 +1415,11 @@ func IsNearDup(q string, seen []string) bool {// The model seam// ---------------------------------------------------------------------------+// The production carrier always supports per-call temperatures: the pinned+// Python temperatures (keywords 0.1 / structure_qa 0.2 / compute 0.0) must+// never silently fall back to a model default.+var _ TemperatureModel = (*InvokerSessionModel)(nil)+// CompleteWithTemperature implements TemperatureModel.func (m *InvokerSessionModel) CompleteWithTemperature(ctx context.Context, messages []schema.Message, tools []ToolSpec, temp float64) (*ModelReply, error) {if m == nil || m.Invoker == nil {@@ -1823,8 +1860,12 @@ func (s *SessionState) toolNode(ctx context.Context) error {// Near-duplicate retrieval suppression: if the model re-issues the same// intent as an earlier search (paraphrase), do NOT re-run the index —// return a nudge so it patches / reframes instead of burning turns.- q, _ := c.Args["query"].(string)- q = strings.TrimSpace(q)+ // Python :1430 — q = str(args.get("query") or "").strip(): the query may+ // be a LIST (retrieve takes up to 3), and its string form still counts+ // for near-dup detection and the seen_queries ledger. A type assertion+ // here degrades every array query to "" — seen_queries stays empty, and+ // the skipped-dup convergence can never trigger.+ q := argQueryString(c.Args["query"])if retrievalTools[c.Name] && q != "" && IsNearDup(q, seenQueries) {skipped++_LOG.Printf("[Action Session] skipping near-duplicate retrieval %q (already searched)", trunc(q, 80))@@ -1892,14 +1933,17 @@ func (s *SessionState) toolNode(ctx context.Context) error {payload := marshalPassages(chunks)// If the session is already heavy, cut this payload proportionally.- if used+len(payload) > budgetChars {+ // Python :1509-1514 counts len() of a str — CODE POINTS, not bytes — so+ // the budget and the cut must be rune-based too; a byte cap would hit+ // CJK payloads ~3x early and shrink the evidence the model sees.+ if used+utf8.RuneCountInString(payload) > budgetChars {keep := budgetChars - usedif keep < 800 {keep = 800}- payload = payload[:min(len(payload), keep)]+ payload = truncateRunes(payload, keep)}- used += len(payload)+ used += utf8.RuneCountInString(payload)s.Messages = appendMessages(s.Messages, *schema.ToolMessage(payload, c.ID))// Ladder continuation (Python action_session.py:_tool_node): when the@@ -1940,7 +1984,8 @@ func (s *SessionState) toolNode(ctx context.Context) error {pendingRule = ex.PendingRulefor _, m := range ex.Messages {if m.Role == schema.Tool {- used += len(m.Content)+ // Python :1556 counts len() of a str — code points.+ used += utf8.RuneCountInString(m.Content)}s.Messages = appendMessages(s.Messages, m)}@@ -2313,9 +2358,19 @@ func ParseTerminal(content string, parent State) ([]State, *string, *string, mapif data == nil {data = map[string]any{}}+ // Python _parse_terminal:1282 — `answer = str(data.get("answer",+ // "")).strip() or None`: an empty/whitespace answer is NOT a found+ // answer. Returning a non-nil empty string here terminated the session+ // with FoundAnswer="" — the slot research pass then reported+ // collected_answer=false for a session that never answered, and the+ // model never got the chance to re-emit a proper patch/answer.ans := ""if raw, ok := data["answer"]; ok && raw != nil {- ans = fmt.Sprint(raw)+ ans = strings.TrimSpace(fmt.Sprint(raw))+ }+ var found *string+ if ans != "" {+ found = &ans}patches := toPatchList(data["new_state"])var branches []State@@ -2323,7 +2378,7 @@ func ParseTerminal(content string, parent State) ([]State, *string, *string, mapbranches = append(branches, *ns)}tt := "answer"- return branches, &ans, &tt, data+ return branches, found, &tt, data}return nil, nil, nil, nil}@@ -2550,14 +2605,16 @@ var NavRules = []NavRule{Mode: ModeAuto,Args: func(nav *NavContext) map[string]any { return map[string]any{"query": nav.Direction} },// Tree missed => no routed hints to merge against; the only useful step- // is an unscoped search.+ // is an unscoped search. Python :1895 — {OK, MISS, EMPTY, POOR, ERROR};+ // REDUNDANT is deliberately absent: a redundant locate changed nothing,+ // and widening it to global would re-run the same corpus search that+ // produced the duplicates.Next: map[string]string{- StatusOK: "drill",- StatusMiss: "global",- StatusEmpty: "global",- StatusPoor: "global",- StatusError: "global",- StatusRedundant: "global",+ StatusOK: "drill",+ StatusMiss: "global",+ StatusEmpty: "global",+ StatusPoor: "global",+ StatusError: "global",},},{@@ -2568,12 +2625,12 @@ var NavRules = []NavRule{Args: func(nav *NavContext) map[string]any { return map[string]any{"query": nav.Direction} },// drill returns OK with the merged, re-ranked evidence; only an empty// whole-corpus result (MISS) or an infra failure falls through to global.+ // Python :1905 — {OK, MISS, EMPTY, ERROR}; no POOR and no REDUNDANT.Next: map[string]string{- StatusOK: "",- StatusMiss: "global",- StatusEmpty: "global",- StatusError: "global",- StatusRedundant: "global",+ StatusOK: "",+ StatusMiss: "global",+ StatusEmpty: "global",+ StatusError: "global",},},{@@ -2868,8 +2925,11 @@ func (e *NavExchange) consumeExchange(ruleID, tool string, args map[string]any,rawArgs = []byte("{}")}payload := marshalPassages(oc.Payload)- if maxChars > 0 && len(payload) > maxChars {- payload = payload[:max(maxChars, 800)]+ // Python :1948 slices a str — CODE POINTS, not bytes — so the cap must be+ // rune-based too; a byte cut would hit CJK payloads ~3x early AND could+ // split a UTF-8 sequence, corrupting the JSON tool response.+ if maxChars > 0 && utf8.RuneCountInString(payload) > maxChars {+ payload = truncateRunes(payload, max(maxChars, 800))}e.Messages = append(e.Messages,*schema.AssistantMessage("", []schema.ToolCall{{@@ -3174,10 +3234,12 @@ func InitializeState(ctx context.Context, deps SessionDeps, question string, fansystem := loadPrompt(deps.Prompts, "action_initialize_state")user := "Question: " + questionif len(fanoutHint) > 0 {- user += "\n\nCandidate aspects already identified:\n"+ lines := make([]string, 0, len(fanoutHint))for _, h := range fanoutHint {- user += "- " + h + "\n"+ lines = append(lines, "- "+h)}+ // Python :2225 — "\n".join(...), so the block has NO trailing newline.+ user += "\n\nCandidate aspects already identified:\n" + strings.Join(lines, "\n")}// Python :2106 — `min(_INIT_TIMEOUT_S, deadline_left or _INIT_TIMEOUT_S)`:// 0 means "unset" (falls back to the full budget) while a NEGATIVE deadline —@@ -3393,10 +3455,10 @@ type ContextLengthModel interface {// SessionModel is ONE model turn with THIS mode's tool surface.//-// Python calls this via _llm_once_with_tools → _acompletion. The Go chat seam-// has no native tools field, so the default implementation (see-// InvokerSessionModel) asks for the call as a JSON block. Swap in a native-// implementation when the chat seam grows a tools field.+// Python calls this via _llm_once_with_tools → _acompletion. The production+// implementation (InvokerSessionModel) binds the tool schemas natively through+// the chat seam (chat.Request.Tools + ToolChoiceAuto) and reads the calls back+// from Response.ToolCalls — the same protocol Python uses.type SessionModel interface {Complete(ctx context.Context, messages []schema.Message, tools []ToolSpec) (*ModelReply, error)}@@ -3526,11 +3588,10 @@ func extractRelevantEvidence(kb *Kbinfos, direction string, maxChunks int) strinif content == "" {continue}- // Mirror Python: hard-cut at 300 (no ellipsis) and flatten newlines so- // the digest stays single-line per chunk and matches Python output.- if len(content) > 300 {- content = content[:300]- }+ // Mirror Python: hard-cut at 300 CODE POINTS (:2082 `[:300]`, no+ // ellipsis) and flatten newlines so the digest stays single-line per+ // chunk and matches Python output.+ content = truncateRunes(content, 300)content = strings.ReplaceAll(content, "\n", " ")// Mirror Python f"[{cid}] {text}" so the model can cite the chunk id.b.WriteString("[")diff --git a/internal/rag/advanced_rag/harness/action_session_test.go b/internal/rag/advanced_rag/harness/action_session_test.goindex 8658f5cb4d..676956e1cf 100644--- a/internal/rag/advanced_rag/harness/action_session_test.go+++ b/internal/rag/advanced_rag/harness/action_session_test.go@@ -1188,3 +1188,50 @@ func TestActionRunPromptHasPlaybook(t *testing.T) {}}}++// TestParseTerminalEmptyAnswerNotFound pins Python _parse_terminal:1282:+// `answer = str(data.get("answer", "")).strip() or None` — an empty or+// whitespace <answer> payload is NOT a found answer. Python's _run_action_node+// then only ends the session when the <answer> carried a new_state patch; with+// neither an answer nor a patch it NUDGES and the session continues, so a bare+// <answer> block can never end a session with collected_answer="".+func TestParseTerminalEmptyAnswerNotFound(t *testing.T) {+ parent := NewState([]Variable{{ID: 0, Type: "answer"}}, 0, nil)++ // Whitespace answer, no patch: no found answer, no branches — the session+ // keeps running (runActionNode falls through to the nudge).+ states, found, tt, _ := ParseTerminal("<answer>{\"answer\": \" \"}</answer>", parent)+ if found != nil {+ t.Errorf("whitespace answer: FoundAnswer = %q, want nil", *found)+ }+ if len(states) != 0 {+ t.Errorf("whitespace answer: %d branch(es), want 0", len(states))+ }+ if tt == nil || *tt != "answer" {+ t.Errorf("whitespace answer: terminal type = %v, want answer", tt)+ }++ // Missing answer field, no patch: same.+ states, found, _, _ = ParseTerminal("<answer>{}</answer>", parent)+ if found != nil || len(states) != 0 {+ t.Errorf("missing answer: FoundAnswer=%v branches=%d, want nil/0", found, len(states))+ }++ // A real answer still parses.+ _, found, _, _ = ParseTerminal("<answer>{\"answer\": \" 74 \"}</answer>", parent)+ if found == nil || *found != "74" {+ t.Errorf("real answer: FoundAnswer = %v, want 74 (stripped)", found)+ }++ // Empty answer WITH a new_state patch: the patch is returned (Python+ // :1283-1288 final_state), terminal type stays "answer", found stays nil.+ states, found, _, _ = ParseTerminal(+ "<answer>{\"answer\": \"\", \"new_state\": [{\"id\": 0, \"candidate\": \"74\", \"candidate_strength\": 0.9}]}</answer>",+ parent)+ if found != nil {+ t.Errorf("empty answer with patch: FoundAnswer = %q, want nil", *found)+ }+ if len(states) != 1 || states[0].State[0].Candidate == nil || *states[0].State[0].Candidate != "74" {+ t.Errorf("empty answer with patch: branches = %+v, want one state with candidate 74", states)+ }+}diff --git a/internal/rag/advanced_rag/harness/kbinfos.go b/internal/rag/advanced_rag/harness/kbinfos.goindex 605e37545c..28e747f17c 100644--- a/internal/rag/advanced_rag/harness/kbinfos.go+++ b/internal/rag/advanced_rag/harness/kbinfos.go@@ -19,6 +19,7 @@ package harnessimport ("context""fmt"+ "strings""sync")@@ -194,6 +195,39 @@ func (p *PoolAdmitter) Index(c map[string]any) int {return indexOfChunk(p.k.Chunks, chunkKey(c))}+// ClaimCoveredIDs returns the chunk ids already represented VERBATIM by a claim+// pseudo-chunk in the LIVE pool (Python _claim_covered_ids,+// action_session.py:619-631): a claim carries its own verbatim quote plus the+// ids of the chunks it was distilled from, so admitting those passages again+// is duplicate payload — the answer material is already in the pool at a+// fraction of the size. Call it ONCE per Admit batch and test each chunk's id+// against the result (Python computes the set once per _admit_evidence call).+func (p *PoolAdmitter) ClaimCoveredIDs() map[string]bool {+ if p.k == nil {+ return nil+ }+ covered := map[string]bool{}+ for _, c := range p.k.Chunks {+ if id, ok := c["chunk_id"].(string); !ok || !strings.HasPrefix(id, "claim_") {+ continue+ }+ for _, cid := range toStringList(c["source_chunk_ids"]) {+ if cid != "" {+ covered[cid] = true+ }+ }+ }+ return covered+}++// CoveredByClaim mirrors the _admit_evidence skip (action_session.py:672-676):+// a non-table chunk whose id a pooled claim already quotes verbatim must NOT+// enter the pool again. Table chunks are exempt: their answer rows survive+// only in full text.+func (p *PoolAdmitter) CoveredByClaim(cid string, covered map[string]bool, isTable bool) bool {+ return !isTable && cid != "" && covered[cid]+}+// Merge appends the given chunks/aggs, deduplicating by chunkKey, and returns the// GLOBAL index (position in k.Chunks) of every contributed chunk — deduped ones// included.@@ -218,6 +252,50 @@ func (k *Kbinfos) Merge(chunks, aggs []map[string]any) []int {return added}+// RetireClaimsCoveredBy removes claim pseudo-chunks whose source_chunk_ids+// intersect the given read ids (Python _exec_list_chunks:928-934): a deep read+// COVERS its claims — once the full chunk text is in the pool, the claim's+// 1200-char quote of the same passage is duplicated tokens in every later+// prompt. The claim already did its job (it pointed here). Returns how many+// entries were retired. The claim_ prefix and the "source_chunk_ids listed+// under the read ids" test mirror Python verbatim.+func (k *Kbinfos) RetireClaimsCoveredBy(readIDs []string) int {+ if k == nil || len(readIDs) == 0 {+ return 0+ }+ read := make(map[string]bool, len(readIDs))+ for _, id := range readIDs {+ if id != "" {+ read[id] = true+ }+ }+ k.mu.Lock()+ defer k.mu.Unlock()+ removed := 0+ out := make([]map[string]any, 0, len(k.Chunks))+ for _, c := range k.Chunks {+ id, ok := c["chunk_id"].(string)+ if ok && strings.HasPrefix(id, "claim_") {+ retired := false+ for _, src := range toStringList(c["source_chunk_ids"]) {+ if read[src] {+ retired = true+ break+ }+ }+ if retired {+ removed+++ continue+ }+ }+ out = append(out, c)+ }+ if removed > 0 {+ k.Chunks = out+ }+ return removed+}+// MergeDocAggs appends doc aggregations, deduplicating by doc_id: the first agg// per doc_id wins (Python _merge_kbinfos' dseen set).//diff --git a/internal/rag/advanced_rag/harness/keywords.go b/internal/rag/advanced_rag/harness/keywords.goindex 283da15aed..14b364f020 100644--- a/internal/rag/advanced_rag/harness/keywords.go+++ b/internal/rag/advanced_rag/harness/keywords.go@@ -51,6 +51,9 @@ const (keywordQualifierRepeat = 3// keywordMaxChars caps both strings (Python: _KEYWORD_MAX_CHARS).keywordMaxChars = 400+ // keywordExtractionTemperature pins Python keywords.py:141's+ // {"temperature": 0.1} for the extraction call.+ keywordExtractionTemperature = 0.1)// keywordAspects is the aspect order. Entity and qualifiers are the weighted@@ -220,21 +223,18 @@ func ExtractWeightedKeywords(ctx context.Context, model SessionModel, question s*schema.SystemMessage(systemPrompt),*schema.UserMessage(userContent),}- // Temperature (one intentional seam vs Python): Python hardcodes- // {"temperature": 0.1} — a mechanical rewrite, not a reasoning task, so- // it must be stable. Go mirrors that exactly WHEN the model implements- // TemperatureModel by calling CompleteWithTemperature(ctx, msgs, nil, 0.1).- // Models that do NOT support per-call temperature fall back to the plain- // Complete and therefore run at the model's own default temperature. We- // deliberately prefer widest compatibility over a literal 0.1: forcing- // 0.1 unconditionally would make keyword extraction error out on models- // without temperature control. (Documented seam, not a bug — see- // HARNESS.md.)+ // Python hardcodes {"temperature": 0.1} (keywords.py:141) — a mechanical+ // rewrite, not a reasoning task, so it must be stable. The value is a+ // pinned constant: every production carrier implements TemperatureModel+ // (compile-time assertion on InvokerSessionModel), so the temperature is+ // always sent; a carrier without per-call temperature support falls back+ // to its own default (Go-only provider limitation, flagged).var reply *ModelReplyvar err errorif tm, ok := model.(TemperatureModel); ok {- reply, err = tm.CompleteWithTemperature(ctx, msgs, nil, 0.1)+ reply, err = tm.CompleteWithTemperature(ctx, msgs, nil, keywordExtractionTemperature)} else {+ _LOG.Printf("[Keywords] model %T cannot carry per-call temperature; using its default (Python would send %v)", model, keywordExtractionTemperature)reply, err = model.Complete(ctx, msgs, nil)}if err == nil {diff --git a/internal/rag/advanced_rag/harness/porter_stemmer.go b/internal/rag/advanced_rag/harness/porter_stemmer.gonew file mode 100644index 0000000000..1b25fb7761--- /dev/null+++ b/internal/rag/advanced_rag/harness/porter_stemmer.go@@ -0,0 +1,403 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package harness++import "strings"++// Porter stemming, a faithful port of nltk.stem.porter.PorterStemmer in its+// default NLTK_EXTENSIONS mode — the mode Python's text_processing.py:221+// exercises in production (`PorterStemmer().stem`, no mode argument). Python's+// _stem calls it for every stemmable keyword token, so the keyword narrowing in+// this package must produce the SAME stems (the previous suffix-stripping+// fallback diverged: "nominations"→"nominat" vs "nomin", "flies"→"fle" vs+// "fli", "dies"→"dy" vs "die", ...).+//+// The port follows the 1980 paper's five steps plus NLTK's extensions: the+// irregular-form pool consulted first, the <=2-letter circuit breaker, the+// 4-letter "ies"/"ied" reductions, the consonant-conditional step-1c, the+// "bli"/"fulli"/recursive-"alli" step-2 rules, the "logi" measure quirk and+// the 2-letter *o exception.++// porterPool mirrors nltk's _pool: irregular forms keyed by their restored+// form (NLTK_EXTENSIONS only). Consulted before any rule runs.+var porterPool = map[string][]string{+ "sky": {"sky", "skies"},+ "die": {"dying"},+ "lie": {"lying"},+ "tie": {"tying"},+ "news": {"news"},+ "inning": {"innings", "inning"},+ "outing": {"outings", "outing"},+ "canning": {"cannings", "canning"},+ "howe": {"howe"},+ "proceed": {"proceed"},+ "exceed": {"exceed"},+ "succeed": {"succeed"},+}++func porterStem(word string) string {+ if word == "" {+ return word+ }+ for restored, forms := range porterPool {+ for _, form := range forms {+ if word == form {+ return restored+ }+ }+ }+ // NLTK_EXTENSIONS short-word circuit breaker (nltk stem(): length <= 2 is+ // returned unchanged; the paper never mentions it).+ if len(word) <= 2 {+ return word+ }+ w := porterStep1a(word)+ w = porterStep1b(w)+ w = porterStep1c(w)+ w = porterStep2(w)+ w = porterStep3(w)+ w = porterStep4(w)+ w = porterStep5a(w)+ w = porterStep5b(w)+ return w+}++// porterIsConsonant mirrors nltk _is_consonant with the iterative y-chain+// resolution: 'y' at position 0 is a consonant; a 'y' counts as a consonant+// when the character before it is one (y-runs resolved iteratively).+func porterIsConsonant(word string, i int) bool {+ if i < 0 || i >= len(word) {+ return false+ }+ switch c := word[i]; c {+ case 'a', 'e', 'i', 'o', 'u':+ return false+ case 'y':+ negate := false+ for i > 0 && word[i] == 'y' {+ negate = !negate+ i--+ }+ if i == 0 {+ // The y-run reaches position 0, where 'y' is a consonant.+ return !negate+ }+ return (strings.IndexByte("aeiou", word[i]) < 0) != negate+ default:+ return true+ }+}++// porterConsonantFlags classifies the whole word once (nltk+// _consonant_flags): true = consonant. 'y' is a consonant at position 0 and+// otherwise the complement of the previous character's classification.+func porterConsonantFlags(word string) []bool {+ flags := make([]bool, len(word))+ for i := range word {+ switch {+ case i == 0:+ flags[i] = porterIsConsonant(word, 0)+ case word[i] == 'y':+ flags[i] = !flags[i-1]+ default:+ flags[i] = porterIsConsonant(word, i)+ }+ }+ return flags+}++// porterMeasure mirrors nltk _measure: the number of VC sequences in the+// word's consonant/vowel shape ("cvccvc" → 2, "cvc" → 1).+func porterMeasure(stem string) int {+ flags := porterConsonantFlags(stem)+ m := 0+ for i := 1; i < len(flags); i++ {+ if flags[i] && !flags[i-1] {+ m+++ }+ }+ return m+}++// porterContainsVowel mirrors nltk _contains_vowel.+func porterContainsVowel(stem string) bool {+ for _, cons := range porterConsonantFlags(stem) {+ if !cons {+ return true+ }+ }+ return false+}++// porterEndsDoubleConsonant mirrors nltk _ends_double_consonant (*d).+func porterEndsDoubleConsonant(word string) bool {+ return len(word) >= 2 && word[len(word)-1] == word[len(word)-2] &&+ porterIsConsonant(word, len(word)-1)+}++// porterEndsCVC mirrors nltk _ends_cvc (*o): consonant-vowel-consonant with+// the final consonant not w/x/y — plus NLTK's 2-letter cv special case.+func porterEndsCVC(word string) bool {+ if len(word) >= 3 &&+ porterIsConsonant(word, len(word)-3) &&+ !porterIsConsonant(word, len(word)-2) &&+ porterIsConsonant(word, len(word)-1) &&+ word[len(word)-1] != 'w' && word[len(word)-1] != 'x' && word[len(word)-1] != 'y' {+ return true+ }+ return len(word) == 2 && !porterIsConsonant(word, 0) && porterIsConsonant(word, 1)+}++// porterRule is one (suffix, replacement, condition) rule; condition nil means+// unconditional. A condition receives the word with the suffix already+// removed. The replacement replaces the SUFFIX (the stem keeps its letters);+// the two special step-1b rules append to the stem instead, marked by the+// replacement markers below.+type porterRule struct {+ suffix string+ replacement string+ condition func(stem string) bool+}++// porterAppendE marks the step-1b (m=1 and *o) rule: append "e" to the stem.+const porterAppendE = "\x00e"++// porterApplyRuleList mirrors nltk _apply_rule_list: the FIRST suffix that+// matches wins — even when its condition fails (the walk stops and the word is+// returned unchanged). The one exception is nltk's "*d" rule: when the word+// ends in a doubled consonant the stem is word[:-2] and a FAILED condition+// falls through to the next rule instead of stopping (which is how+// "falling"→"fall" passes the *d branch and reaches the +e branch's guard).+func porterApplyRuleList(word string, rules []porterRule) string {+ for _, r := range rules {+ if r.suffix == "*d" {+ if !porterEndsDoubleConsonant(word) {+ continue+ }+ stem := word[:len(word)-2]+ if r.condition(stem) {+ return stem + word[len(word)-1:]+ }+ continue+ }+ if !strings.HasSuffix(word, r.suffix) {+ continue+ }+ stem := strings.TrimSuffix(word, r.suffix)+ if r.condition != nil && !r.condition(stem) {+ return word+ }+ if r.replacement == porterAppendE {+ return stem + "e"+ }+ return stem + r.replacement+ }+ return word+}++func porterHasPositiveMeasure(stem string) bool { return porterMeasure(stem) > 0 }++func porterStep1a(word string) string {+ // NLTK_EXTENSIONS: a 4-letter "ies" reduces to "ie" (dies→die, ties→tie).+ if strings.HasSuffix(word, "ies") && len(word) == 4 {+ return strings.TrimSuffix(word, "ies") + "ie"+ }+ return porterApplyRuleList(word, []porterRule{+ {suffix: "sses", replacement: "ss"},+ {suffix: "ies", replacement: "i"},+ {suffix: "ss", replacement: "ss"},+ {suffix: "s", replacement: ""},+ })+}++// porterStep1b mirrors nltk _step1b.+func porterStep1b(word string) string {+ // NLTK_EXTENSIONS: "ied" reduces by one letter for a 4-letter word+ // (died→die), by two otherwise (spied→spi) — before the eed/ed/ing walk.+ if strings.HasSuffix(word, "ied") {+ if len(word) == 4 {+ return strings.TrimSuffix(word, "ied") + "ie"+ }+ return strings.TrimSuffix(word, "ied") + "i"+ }+ if strings.HasSuffix(word, "eed") {+ if porterHasPositiveMeasure(strings.TrimSuffix(word, "eed")) {+ return strings.TrimSuffix(word, "eed") + "ee"+ }+ return word+ }+ intermediate := ""+ if strings.HasSuffix(word, "ed") && porterContainsVowel(strings.TrimSuffix(word, "ed")) {+ intermediate = strings.TrimSuffix(word, "ed")+ } else if strings.HasSuffix(word, "ing") && porterContainsVowel(strings.TrimSuffix(word, "ing")) {+ intermediate = strings.TrimSuffix(word, "ing")+ } else {+ return word+ }+ // nltk's _step1b_helper: the four post-removal rules are an if/else CHAIN+ // (first hit wins, and the *d branch is exclusive with the +e branch —+ // "falling"→"fall" passes the *d guard on l/s/z and then fails the +e+ // guard, staying "fall").+ switch {+ case strings.HasSuffix(intermediate, "at"):+ return intermediate + "e"+ case strings.HasSuffix(intermediate, "bl"):+ return intermediate + "e"+ case strings.HasSuffix(intermediate, "iz"):+ return intermediate + "e"+ case porterEndsDoubleConsonant(intermediate) &&+ intermediate[len(intermediate)-1] != 'l' &&+ intermediate[len(intermediate)-1] != 's' &&+ intermediate[len(intermediate)-1] != 'z':+ // *d and not (*L or *S or *Z): collapse the doubled consonant+ // (hopp→hop, tann→tan).+ return intermediate[:len(intermediate)-1]+ case porterMeasure(intermediate) == 1 && porterEndsCVC(intermediate):+ // (m=1 and *o) → + E (fil→file; fail→fail stays).+ return intermediate + "e"+ }+ return intermediate+}++// porterStep1c mirrors nltk _step1c in NLTK_EXTENSIONS mode: y→i when the word+// (minus y) is longer than one letter AND the character before the y is a+// consonant — so happy→happi but enjoy→enjoy, and spy/fly/try merge with their+// -ied/-ies forms (spy→spi, spied→spi).+func porterStep1c(word string) string {+ if !strings.HasSuffix(word, "y") {+ return word+ }+ stem := strings.TrimSuffix(word, "y")+ if len(stem) > 1 && porterIsConsonant(stem, len(stem)-1) {+ return stem + "i"+ }+ return word+}++// porterStep2 mirrors nltk _step2: map double suffixes to single ones when the+// stem has positive measure. NLTK_EXTENSIONS first unwraps a trailing "alli"+// (radicalli→radical→radic) by recursing.+func porterStep2(word string) string {+ if strings.HasSuffix(word, "alli") {+ stem := strings.TrimSuffix(word, "alli")+ if porterHasPositiveMeasure(stem) {+ return porterStep2(stem + "al")+ }+ }+ measure := func(stem string) bool { return porterHasPositiveMeasure(stem) }+ rules := []porterRule{+ {suffix: "ational", replacement: "ate", condition: measure},+ {suffix: "tional", replacement: "tion", condition: measure},+ {suffix: "enci", replacement: "ence", condition: measure},+ {suffix: "anci", replacement: "ance", condition: measure},+ {suffix: "izer", replacement: "ize", condition: measure},+ {suffix: "bli", replacement: "ble", condition: measure},+ {suffix: "alli", replacement: "al", condition: measure},+ {suffix: "entli", replacement: "ent", condition: measure},+ {suffix: "eli", replacement: "e", condition: measure},+ {suffix: "ousli", replacement: "ous", condition: measure},+ {suffix: "ization", replacement: "ize", condition: measure},+ {suffix: "ation", replacement: "ate", condition: measure},+ {suffix: "ator", replacement: "ate", condition: measure},+ {suffix: "alism", replacement: "al", condition: measure},+ {suffix: "iveness", replacement: "ive", condition: measure},+ {suffix: "fulness", replacement: "ful", condition: measure},+ {suffix: "ousness", replacement: "ous", condition: measure},+ {suffix: "aliti", replacement: "al", condition: measure},+ {suffix: "iviti", replacement: "ive", condition: measure},+ {suffix: "biliti", replacement: "ble", condition: measure},+ // NLTK_EXTENSIONS additions: "fulli", and the "logi" quirk whose+ // condition measures the stem INCLUDING its trailing "l" (word minus+ // "log"), so geo/theo endings (archaeologi, philologi) reduce.+ {suffix: "fulli", replacement: "ful", condition: measure},+ {suffix: "logi", replacement: "log", condition: func(stem string) bool {+ return len(stem) >= 1 && porterHasPositiveMeasure(stem[:len(stem)-1])+ }},+ }+ return porterApplyRuleList(word, rules)+}++// porterStep3 mirrors nltk _step3: -ic-, -full, -ness etc. (all measure > 0).+func porterStep3(word string) string {+ measure := func(stem string) bool { return porterHasPositiveMeasure(stem) }+ return porterApplyRuleList(word, []porterRule{+ {suffix: "icate", replacement: "ic", condition: measure},+ {suffix: "ative", replacement: "", condition: measure},+ {suffix: "alize", replacement: "al", condition: measure},+ {suffix: "iciti", replacement: "ic", condition: measure},+ {suffix: "ical", replacement: "ic", condition: measure},+ {suffix: "ful", replacement: "", condition: measure},+ {suffix: "ness", replacement: "", condition: measure},+ })+}++// porterStep4 mirrors nltk _step4: drop -ant/-ence etc. when the stem's+// measure exceeds 1 ("ion" additionally requires an s/t before the suffix).+func porterStep4(word string) string {+ m := func(stem string) bool { return porterMeasure(stem) > 1 }+ rules := []porterRule{+ {suffix: "al", condition: m},+ {suffix: "ance", condition: m},+ {suffix: "ence", condition: m},+ {suffix: "er", condition: m},+ {suffix: "ic", condition: m},+ {suffix: "able", condition: m},+ {suffix: "ible", condition: m},+ {suffix: "ant", condition: m},+ {suffix: "ement", condition: m},+ {suffix: "ment", condition: m},+ {suffix: "ent", condition: m},+ {suffix: "ion", condition: func(stem string) bool {+ return porterMeasure(stem) > 1 && len(stem) > 0 &&+ (stem[len(stem)-1] == 's' || stem[len(stem)-1] == 't')+ }},+ {suffix: "ou", condition: m},+ {suffix: "ism", condition: m},+ {suffix: "ate", condition: m},+ {suffix: "iti", condition: m},+ {suffix: "ous", condition: m},+ {suffix: "ive", condition: m},+ {suffix: "ize", condition: m},+ }+ return porterApplyRuleList(word, rules)+}++// porterStep5a mirrors nltk _step5a — deliberately NOT a rule list (the paper+// lets BOTH conditions apply in turn here, unlike every other step):+// - (m>1) E → probate→probat+// - (m=1 and not *o) E → cease→ceas+func porterStep5a(word string) string {+ if strings.HasSuffix(word, "e") {+ stem := strings.TrimSuffix(word, "e")+ if porterMeasure(stem) > 1 {+ return stem+ }+ if porterMeasure(stem) == 1 && !porterEndsCVC(stem) {+ return stem+ }+ }+ return word+}++// porterStep5b mirrors nltk _step5b: (m>1, measured on word minus its last+// letter) LL → L (controll→control; roll→roll stays).+func porterStep5b(word string) string {+ if strings.HasSuffix(word, "ll") && porterMeasure(word[:len(word)-1]) > 1 {+ return strings.TrimSuffix(word, "l")+ }+ return word+}diff --git a/internal/rag/advanced_rag/harness/porter_stemmer_test.go b/internal/rag/advanced_rag/harness/porter_stemmer_test.gonew file mode 100644index 0000000000..0675a7dd5d--- /dev/null+++ b/internal/rag/advanced_rag/harness/porter_stemmer_test.go@@ -0,0 +1,96 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package harness++import "testing"++// TestPorterStemMatchesNLTK pins porterStem against nltk.stem.porter's+// NLTK_EXTENSIONS output (PorterStemmer().stem), collected with+//+// from nltk.stem import PorterStemmer+// print(PorterStemmer().stem(word))+//+// in production's environment. Every pair below is ground truth, not+// aspiration: the keyword-narrowing stem match (keywordForms/_sentence_stems)+// must produce Python-identical stems or the narrow keeps different sentences+// than Python does.+func TestPorterStemMatchesNLTK(t *testing.T) {+ cases := []struct{ word, want string }{+ // Step-1a/1b regulars.+ {"nominations", "nomin"}, {"nominated", "nomin"}, {"companies", "compani"},+ {"running", "run"}, {"caresses", "caress"}, {"ponies", "poni"},+ {"caress", "caress"}, {"cats", "cat"}, {"meetings", "meet"}, {"stems", "stem"},+ {"cries", "cri"}, {"flies", "fli"},+ // Irregular pool.+ {"sky", "sky"}, {"skies", "sky"}, {"dying", "die"}, {"lying", "lie"},+ {"tying", "tie"}, {"news", "news"}, {"innings", "inning"},+ {"outings", "outing"}, {"cannings", "canning"}, {"howe", "howe"},+ {"proceed", "proceed"}, {"exceed", "exceed"}, {"succeed", "succeed"},+ // 4-letter ies/ied reductions.+ {"ties", "tie"}, {"dies", "die"}, {"died", "die"}, {"spied", "spi"},+ // eed / *d / *o step-1b branches.+ {"agreed", "agre"}, {"feed", "feed"}, {"plastered", "plaster"},+ {"bled", "bled"}, {"motoring", "motor"}, {"sing", "sing"},+ {"conflated", "conflat"}, {"troubled", "troubl"}, {"sized", "size"},+ {"hopping", "hop"}, {"falling", "fall"}, {"filing", "file"},+ // Step-1c y-condition (consonant before the y; len(stem) > 1).+ {"happy", "happi"}, {"enjoy", "enjoy"}, {"spy", "spi"}, {"try", "tri"},+ // Step-2/3/4 table from the paper, NLTK mode.+ {"relation", "relat"}, {"conditional", "condit"}, {"rational", "ration"},+ {"valenci", "valenc"}, {"hesitanci", "hesit"}, {"digitizer", "digit"},+ {"conformabli", "conform"}, {"radicalli", "radic"},+ {"differentli", "differ"}, {"vileli", "vile"},+ {"analogousli", "analog"}, {"vietnamization", "vietnam"},+ {"predication", "predic"}, {"operator", "oper"},+ {"feudalism", "feudal"}, {"decisiveness", "decis"},+ {"hopefulness", "hope"}, {"callousness", "callous"},+ {"formaliti", "formal"}, {"sensitiviti", "sensit"},+ {"sensibiliti", "sensibl"}, {"triplicate", "triplic"},+ {"formative", "form"}, {"formalize", "formal"},+ {"electriciti", "electr"}, {"electrical", "electr"},+ {"hopeful", "hope"}, {"goodness", "good"},+ {"revival", "reviv"}, {"allowance", "allow"}, {"inference", "infer"},+ {"airliner", "airlin"}, {"gyroscopic", "gyroscop"},+ {"adjustable", "adjust"}, {"defensible", "defens"},+ {"irritant", "irrit"}, {"replacement", "replac"},+ {"adjustment", "adjust"}, {"dependent", "depend"},+ {"adoption", "adopt"}, {"homologou", "homolog"},+ {"communism", "commun"}, {"activate", "activ"},+ {"angulariti", "angular"}, {"homologous", "homolog"},+ {"effective", "effect"}, {"bowdlerize", "bowdler"},+ // Step-5a/5b.+ {"probate", "probat"}, {"rate", "rate"}, {"cease", "ceas"},+ {"controll", "control"}, {"roll", "roll"},+ }+ for _, c := range cases {+ if got := porterStem(c.word); got != c.want {+ t.Errorf("porterStem(%q) = %q, want %q", c.word, got, c.want)+ }+ }+}++// TestStemUsesPorter pins the seam: the package's stem() (the one+// keywordForms/_sentence_stems call) is the Porter stemmer, matching Python+// text_processing._stem's nltk branch.+func TestStemUsesPorter(t *testing.T) {+ if stem("nominations") != "nomin" {+ t.Errorf("stem(nominations) = %q, want the Porter stem \"nomin\"", stem("nominations"))+ }+ if stem("flies") != "fli" {+ t.Errorf("stem(flies) = %q, want \"fli\"", stem("flies"))+ }+}diff --git a/internal/rag/advanced_rag/harness/stats.go b/internal/rag/advanced_rag/harness/stats.goindex 96d05a968c..efd8d8138f 100644--- a/internal/rag/advanced_rag/harness/stats.go+++ b/internal/rag/advanced_rag/harness/stats.go@@ -85,14 +85,6 @@ const (PhaseFinalize = "finalize")-var phaseRank = func() map[string]int {- m := make(map[string]int, len(phaseOrder))- for i, n := range phaseOrder {- m[n] = i- }- return m-}()-// phaseRoundKey identifies a phase within one orchestrator round.type phaseRoundKey struct {phase stringdiff --git a/internal/rag/advanced_rag/harness/structure_qa.go b/internal/rag/advanced_rag/harness/structure_qa.goindex 50e7116490..113bb6c7ba 100644--- a/internal/rag/advanced_rag/harness/structure_qa.go+++ b/internal/rag/advanced_rag/harness/structure_qa.go@@ -39,6 +39,10 @@ import (// store access, and because both callers would otherwise import it from a file// that in turn imports them.+// structureQATemperature pins Python structure_qa.py:79's+// {"temperature": 0.2} for the outline-verdict call.+const structureQATemperature = 0.2+// RenderStructure renders a compiled structure (entities + relations) as a// compact outline for the prompt. Mirrors Python _render_structure://@@ -140,16 +144,18 @@ func AskStructure(ctx context.Context, model SessionModel, topic, noun, label stmsgs := make([]schema.Message, 0, 1+len(fitted))msgs = append(msgs, *schema.SystemMessage(system))msgs = append(msgs, fitted...)- // Python hardcodes {"temperature": 0.2} for this node — a stable verdict on- // a mechanical render, so it must not sample hot. Go mirrors that exactly- // when the model implements TemperatureModel; models without per-call- // temperature fall back to the plain Complete and run at their own default.- // (Documented seam, not a bug — see HARNESS.md and keywords.go.)+ // Python hardcodes {"temperature": 0.2} for this node (structure_qa.py:79)+ // — a stable verdict on a mechanical render, so it must not sample hot. The+ // value is a pinned constant: every production carrier implements+ // TemperatureModel (compile-time assertion on InvokerSessionModel), so the+ // temperature is always sent; a carrier without per-call temperature+ // support falls back to its own default (Go-only provider limitation).var resp *ModelReplyvar err errorif tm, ok := model.(TemperatureModel); ok {- resp, err = tm.CompleteWithTemperature(ctx, msgs, nil, 0.2)+ resp, err = tm.CompleteWithTemperature(ctx, msgs, nil, structureQATemperature)} else {+ _LOG.Printf("[%s] model %T cannot carry per-call temperature; using its default (Python would send %v)", label, model, structureQATemperature)resp, err = model.Complete(ctx, msgs, nil)}if err != nil {diff --git a/internal/rag/advanced_rag/harness/tool_claim_router.go b/internal/rag/advanced_rag/harness/tool_claim_router.gonew file mode 100644index 0000000000..30ea5e1c0e--- /dev/null+++ b/internal/rag/advanced_rag/harness/tool_claim_router.go@@ -0,0 +1,182 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package harness++import (+ "context"+ "math"+ "sort"+ "strings"++ "ragflow/internal/service/nav"+)++// The claim_agg navigation-tree router: the Go mirror of Python+// dataset_api_service.py `_search_layers_navigation_tree(router="claim_agg")`.+//+// The claim leg runs first and decides the ranking outright when it hits — a+// claim is an atomic proposition carrying its own vector and verbatim+// evidence, so matching one means the document asserts that fact. No claim+// rows, or an empty claim leg, falls back to the chunk leg (PageIndex style:+// hybrid chunk recall rolled up per document).+//+// Like chunk_agg (internal/service/nav), the algorithm consumes the agentic+// claim-recall capability through the NavTreeRouter interface and keeps the+// summarizer injected, so the harness stays free of the service layer.++// claimAggPool mirrors Python _NAV_CLAIM_POOL: the per-leg store limit for the+// claim routing leg, wide enough for a document to collect several claim hits+// before the per-document roll-up (the aggregation rewards multi-hit docs).+const claimAggPool = 256++// claimDocScore mirrors _nav_doc_score (PageIndex): hit scores summed, damped+// by hit count. The sum rewards a document matching several claims; the+// square-root denominator damps it so a document cannot win on volume alone.+func claimDocScore(total float64, hits int) float64 {+ return total / math.Sqrt(float64(hits)+1)+}++// ClaimAggRouter is a NavTreeRouter that routes documents through the+// compilers' claim rows first, falling back to the chunk_agg router when the+// claim leg comes back empty (a raptor-less or pre-claim dataset is a+// legitimate empty leg, not a failure).+type ClaimAggRouter struct {+ // Deps carries the claim-recall wiring (doc engine, embedder, index).+ // Route overrides only the per-call tenant/dataset.+ Deps SearchDeps+ // Fallback is the chunk_agg router used when the claim leg is empty.+ Fallback NavTreeRouter+ // Summarize loads the nav_doc summary per routed document (Python+ // _nav_doc_summaries). Nil leaves the summaries empty.+ Summarize nav.DocSummarizer+}++// claimAggBucket is the per-document roll-up of claim hits (Python+// _nav_bucket_compiled_rows): total, best and hit count per doc.+type claimAggBucket struct {+ docID string+ total float64+ best float64+ hits int+}++// Route implements the agentic NavTreeRouter contract with the claim-first+// strategy. It returns (nil, nil) when the fallback router reports no compiled+// tree, an empty non-nil slice when a structure exists but nothing routed, and+// a non-nil error when the fallback's retrieval itself failed — the claim leg+// never errors (recall failures read as an empty leg, which is exactly the+// fallback trigger).+func (r *ClaimAggRouter) Route(ctx context.Context, tenantID, kbID, query string, docScope []string, topK int) ([][2]string, error) {+ fallback := func() ([][2]string, error) {+ if r.Fallback == nil {+ return nil, nil+ }+ return r.Fallback.Route(ctx, tenantID, kbID, query, docScope, topK)+ }+ deps := r.Deps+ deps.TenantID = tenantID+ deps.KbIDs = []string{kbID}+ // Python queries the two compilers SEPARATELY, never as one mixed+ // condition (:4230-4239): tree first — it is the compiled benchmark path —+ // then page_index only when the tree pass came back empty. Each pass pins+ // compile_kwd and row_types=("claim",).+ claims := recallDatasetClaimsFiltered(ctx, deps, query, claimAggPool, []string{"tree"}, []string{"claim"})+ if len(claims) == 0 {+ claims = recallDatasetClaimsFiltered(ctx, deps, query, claimAggPool, []string{"page_index", "pageindex"}, []string{"claim"})+ }+ if len(claims) == 0 {+ // No claim rows (or the recall failed): a legitimate empty leg — the+ // chunk leg decides (Python _search_layers_navigation_tree:4053-4057).+ return fallback()+ }++ // Roll the claim hits up per document (Python _nav_bucket_compiled_rows):+ // the doc score lives on each claim's similarity.+ agg := map[string]*claimAggBucket{}+ for _, c := range claims {+ did := strings.TrimSpace(c.DocID)+ if did == "" {+ continue+ }+ b, ok := agg[did]+ if !ok {+ agg[did] = &claimAggBucket{docID: did, total: c.Score, best: c.Score, hits: 1}+ continue+ }+ b.total += c.Score+ b.hits+++ if c.Score > b.best {+ b.best = c.Score+ }+ }++ // doc_scope: Python filters the store condition before the recall+ // (:4186-4187); post-filtering the rolled-up buckets is equivalent here+ // because the recall is threshold-free.+ scope := make(map[string]bool, len(docScope))+ for _, d := range docScope {+ if d = strings.TrimSpace(d); d != "" {+ scope[d] = true+ }+ }+ ranked := make([]*claimAggBucket, 0, len(agg))+ for _, b := range agg {+ if len(scope) > 0 && !scope[b.docID] {+ continue+ }+ ranked = append(ranked, b)+ }+ if len(ranked) == 0 {+ // A scoped claim leg that reaches nothing is still an empty leg — the+ // chunk leg (which applies the same scope) may reach further.+ return fallback()+ }+ // Order by the damped DocScore (Python _nav_rank_compiled_buckets), best+ // similarity as the tie-break.+ sort.SliceStable(ranked, func(i, j int) bool {+ si := claimDocScore(ranked[i].total, ranked[i].hits)+ sj := claimDocScore(ranked[j].total, ranked[j].hits)+ if si != sj {+ return si > sj+ }+ return ranked[i].best > ranked[j].best+ })+ if topK > 0 && len(ranked) > topK {+ ranked = ranked[:topK]+ }+ // Same focus cap as the sibling routers (Python _NAV_DOC_FOCUS_LIMIT = 3):+ // routing to many documents makes the agent carry that many times the+ // evidence in every later round.+ const navDocFocusLimit = 3+ if navDocFocusLimit > 0 && len(ranked) > navDocFocusLimit {+ ranked = ranked[:navDocFocusLimit]+ }++ ids := make([]string, 0, len(ranked))+ for _, b := range ranked {+ ids = append(ids, b.docID)+ }+ summaries := map[string]string{}+ if r.Summarize != nil {+ summaries = r.Summarize(ctx, tenantID, kbID, ids)+ }+ routed := make([][2]string, 0, len(ranked))+ for _, b := range ranked {+ routed = append(routed, [2]string{b.docID, strings.TrimSpace(summaries[b.docID])})+ }+ return routed, nil+}diff --git a/internal/rag/advanced_rag/harness/tool_claims.go b/internal/rag/advanced_rag/harness/tool_claims.gonew file mode 100644index 0000000000..3c513fc9dd--- /dev/null+++ b/internal/rag/advanced_rag/harness/tool_claims.go@@ -0,0 +1,770 @@+// Package-level claim recall for the agentic harness: a KB-wide FLAT search+// over claim/evidence rows (entity_type_kwd="claim"), mirroring Python+// harness/tools/navigation.py::recall_dataset_claims and its prefetch wiring+// (action_session.py::_claim_prefetch + agentic_rag_graph.py channel 0).+//+// Two legs per search — BM25 over content_ltks/content_sm_ltks and KNN over+// the claim rows' q_<dim>_vec — fused by reciprocal rank. No similarity+// threshold anywhere: the store ranks, we never scan. The BM25 leg alone+// keeps recall alive without an embedder, so claim recall can only ever ADD+// to a search result.++package harness++import (+ "context"+ "crypto/md5"+ "encoding/json"+ "fmt"+ "sort"+ "strings"+ "sync"+ "time"++ "ragflow/internal/engine"+ "ragflow/internal/engine/types"+ nlp "ragflow/internal/service/nlp"+)++const (+ // ClaimPrefetchTopN mirrors Python _CLAIM_PREFETCH_TOP_N.+ ClaimPrefetchTopN = 6+ // ClaimEvidenceChars caps the rendered verbatim quote (Python+ // _STRUCT_CLAIM_EVIDENCE_CHARS). 1200 over 600: the quote must be+ // self-sufficient in ONE shot — a thin quote makes the SCA judge the+ // context insufficient, and one extra re-search turn costs a full+ // 10k+-token prompt.+ ClaimEvidenceChars = 1200+ // claimRecallLegFloor is the per-leg store limit floor (Python+ // `limit = max(top_n, 32)`).+ claimRecallLegFloor = 32+ // claimPrefetchCacheTTL / claimPrefetchCacheCap mirror Python+ // _CLAIM_PREFETCH_CACHE_TTL/_CAP: the agent re-issues near-identical+ // queries across turns and each miss costs one KNN round-trip per KB.+ claimPrefetchCacheTTL = 300.0+ claimPrefetchCacheCap = 64+ // EvidenceQuoteChars caps the verbatim quote in the GRAPH fan-out's+ // channel-0 pseudo chunks (Python agentic_rag_graph.py:723 `[:400]`). The+ // action-session prefetch uses the longer ClaimEvidenceChars (1200,+ // _STRUCT_CLAIM_EVIDENCE_CHARS) — the two conventions are deliberately+ // different on the Python side and must not be conflated.+ EvidenceQuoteChars = 400+)++// claimRowTypes lists the entity_type_kwd values the agentic claim search+// matches: Python _evidence_row_types resolves to ("claim",) for every+// compile kind (page_index's pre-rename fact/conclusion spellings are not+// searched — a recompile retypes them).+var _ = []string{"claim", "fact", "conclusion"} // legacy spellings, API-side only (Python CLAIM_ROW_TYPES)++// claimRowTypes lists the entity_type_kwd values that carry a claim (Python+// _EVIDENCE_ROW_TYPES_BY_COMPILE flattened: every compiler writes "claim"+// now; the legacy spellings keep pre-rename rows visible).+var claimRowTypes = []string{"claim"}++// compilationKwds mirrors Python _COMPILATION_KWDS — the compile_kwd values+// the knowledge-compilation paths write, used by the has-compilation probe.+var compilationKwds = []string{"tree", "page_index", "pageindex", "timeline", "dataset_nav"}++// ClaimHit is one recalled claim, in Python's recall_dataset_claims shape.+type ClaimHit struct {+ Name string+ Description string+ Quote string+ DocID string+ ChunkIDs []string+ Score float64+ Rank int+}++type claimCacheEntry struct {+ at time.Time+ hits []*ClaimHit+}++var (+ claimRecallMu sync.Mutex+ claimRecallCache = map[string]claimCacheEntry{}+ compilationMu sync.Mutex+ compilationCache = map[string]compilationProbe{}+)++type compilationProbe struct {+ at time.Time+ has bool+}++// claimEngine resolves the doc engine: the deps' handle first, the process+// singleton as fallback (mirrors runSearch's DocEngine resolution).+func claimEngine(deps SearchDeps) engine.DocEngine {+ if deps.DocEngine != nil {+ return deps.DocEngine+ }+ return engine.Get()+}++// claimIndexName resolves the tenant index (deps override, then the default).+func claimIndexName(deps SearchDeps) string {+ if v := strings.TrimSpace(deps.IndexName); v != "" {+ return v+ }+ return fmt.Sprintf("ragflow_%s", deps.TenantID)+}++// DatasetHasCompilation reports whether any in-scope KB carries compiled+// rows (Python dataset_has_compilation + dataset_compilation_kinds): one+// cheap probe per KB set, TTL-cached. Probe failures FAIL OPEN (true) so a+// glitch never silently disables claim recall.+func DatasetHasCompilation(ctx context.Context, deps SearchDeps) bool {+ de := claimEngine(deps)+ if de == nil || len(deps.KbIDs) == 0 {+ return false+ }+ key := deps.TenantID + "|" + strings.Join(deps.KbIDs, ",")+ now := time.Now()++ compilationMu.Lock()+ if p, ok := compilationCache[key]; ok && now.Sub(p.at).Seconds() < claimPrefetchCacheTTL {+ compilationMu.Unlock()+ return p.has+ }+ compilationMu.Unlock()++ has := false+ res, err := de.Search(ctx, &types.SearchRequest{+ IndexNames: []string{claimIndexName(deps)},+ KbIDs: deps.KbIDs,+ Offset: 0,+ Limit: 1,+ SelectFields: []string{"id"},+ Filter: map[string]interface{}{"compile_kwd": compilationKwds},+ })+ if err != nil {+ has = true // fail open, mirroring Python+ } else {+ has = res != nil && len(res.Chunks) > 0+ }++ compilationMu.Lock()+ compilationCache[key] = compilationProbe{at: now, has: has}+ compilationMu.Unlock()+ return has+}++// RecallDatasetClaims runs the two-leg hybrid claim recall and returns up to+// topN fused hits. Empty (never an error) on any failure — the prefetch can+// only ever add to a search result.+func RecallDatasetClaims(ctx context.Context, deps SearchDeps, query string, topN int) []*ClaimHit {+ return recallDatasetClaimsFiltered(ctx, deps, query, topN, nil, nil)+}++// recallDatasetClaimsFiltered is the recall core with optional compile_kwd /+// entity_type_kwd pinning (Python claim_agg queries ("tree","claim") and+// ("page_index","claim") as two separate passes; the session-level recall+// passes nil/nil and matches row_types=("claim",) unconditionally).+func recallDatasetClaimsFiltered(ctx context.Context, deps SearchDeps, query string, topN int, compileKwds, rowTypes []string) []*ClaimHit {+ de := claimEngine(deps)+ if de == nil || deps.TenantID == "" || len(deps.KbIDs) == 0 {+ return nil+ }+ query = strings.TrimSpace(query)+ if query == "" || topN <= 0 {+ return nil+ }+ key := deps.TenantID + "|" + strings.Join(deps.KbIDs, ",") + "|" + strings.Join(compileKwds, ",") + "|" + strings.ToLower(query)+ now := time.Now()++ claimRecallMu.Lock()+ if cached, ok := claimRecallCache[key]; ok && now.Sub(cached.at).Seconds() < claimPrefetchCacheTTL {+ claimRecallMu.Unlock()+ return cached.hits+ }+ if len(claimRecallCache) >= claimPrefetchCacheCap {+ claimRecallCache = map[string]claimCacheEntry{}+ }+ claimRecallMu.Unlock()++ limit := topN+ if limit < claimRecallLegFloor {+ limit = claimRecallLegFloor+ }+ rowTypesEffective := rowTypes+ if len(rowTypesEffective) == 0 {+ rowTypesEffective = claimRowTypes+ }+ filter := map[string]interface{}{+ "entity_type_kwd": rowTypesEffective,+ "scope_kwd": []string{"doc"},+ }+ if len(compileKwds) > 0 {+ filter["compile_kwd"] = compileKwds+ }+ fields := []string{"content_with_weight", "source_chunk_ids", "doc_id"}++ // KNN leg (needs an embedder); BM25 leg always runs.+ var denseLeg, textLeg []*ClaimHit+ if deps.HasEmbedder && deps.Embedder != nil {+ if vecs := mustEncodeQueries(deps, ctx, query); len(vecs) > 0 && len(vecs[0]) > 0 {+ qvec := vecs[0]+ dense := make([]float64, len(qvec))+ for i, v := range qvec {+ dense[i] = float64(v)+ }+ exprs := []interface{}{&types.MatchDenseExpr{+ VectorColumnName: fmt.Sprintf("q_%d_vec", len(qvec)),+ EmbeddingData: dense,+ EmbeddingDataType: "float",+ DistanceType: "cosine",+ TopN: limit,+ }}+ denseLeg = claimSearchLeg(ctx, de, deps, filter, fields, exprs, limit)+ }+ }+ exprs := []interface{}{&types.MatchTextExpr{+ Fields: []string{"content_ltks", "content_sm_ltks"},+ MatchingText: query,+ TopN: limit,+ }}+ textLeg = claimSearchLeg(ctx, de, deps, filter, fields, exprs, limit)++ if len(denseLeg) == 0 && len(textLeg) == 0 {+ return nil+ }+ out := rrfFuseClaims(denseLeg, textLeg)+ if len(out) > topN {+ out = out[:topN]+ }++ claimRecallMu.Lock()+ claimRecallCache[key] = claimCacheEntry{at: now, hits: out}+ claimRecallMu.Unlock()+ return out+}++// claimSearchLeg runs one store search and parses the claim hits.+func claimSearchLeg(ctx context.Context, de engine.DocEngine, deps SearchDeps, filter map[string]interface{}, fields []string, exprs []interface{}, limit int) []*ClaimHit {+ res, err := de.Search(ctx, &types.SearchRequest{+ IndexNames: []string{claimIndexName(deps)},+ KbIDs: deps.KbIDs,+ Offset: 0,+ Limit: limit,+ SelectFields: fields,+ Filter: filter,+ MatchExprs: exprs,+ })+ if err != nil || res == nil {+ return nil+ }+ hits := make([]*ClaimHit, 0, len(res.Chunks))+ for _, row := range res.Chunks {+ if hit := parseClaimHit(row); hit != nil {+ hits = append(hits, hit)+ }+ }+ return hits+}++// parseClaimHit extracts one claim hit from a store row. Nil when the row is+// not a usable claim (bad payload, unnamed, or no chunk pointer).+func parseClaimHit(row map[string]interface{}) *ClaimHit {+ raw, _ := row["content_with_weight"].(string)+ var payload map[string]interface{}+ if err := json.Unmarshal([]byte(raw), &payload); err != nil || payload == nil {+ return nil+ }+ name := strings.TrimSpace(claimStr(payload["name"]))+ if name == "" {+ return nil+ }+ chunkIDs := claimChunkIDs(row["source_chunk_ids"], payload["source_chunk_ids"])+ if len(chunkIDs) == 0 {+ return nil+ }+ quote := ""+ if ev, ok := payload["evidence"].([]interface{}); ok {+ for _, item := range ev {+ m, ok := item.(map[string]interface{})+ if !ok {+ continue+ }+ if q := strings.TrimSpace(claimStr(m["quote"])); q != "" {+ quote = q+ break+ }+ }+ }+ hit := &ClaimHit{+ Name: name,+ Description: strings.TrimSpace(claimStr(payload["description"])),+ Quote: quote,+ DocID: strings.TrimSpace(claimStr(row["doc_id"])),+ ChunkIDs: chunkIDs,+ }+ if v, ok := row["similarity"].(float64); ok {+ hit.Score = v+ }+ return hit+}++// rrfFuseClaims fuses the retrieval legs by reciprocal rank (Python+// _rrf_fuse, k=60): each fused hit keeps its best leg score and gains a+// 1-based rank.+func rrfFuseClaims(legs ...[]*ClaimHit) []*ClaimHit {+ const k = 60+ type key struct {+ docID string+ name string+ }+ best := map[key]*ClaimHit{}+ fused := map[key]float64{}+ for _, leg := range legs {+ for rank, hit := range leg {+ hk := key{docID: hit.DocID, name: hit.Name}+ fused[hk] += 1.0 / float64(k+rank+1)+ if cur := best[hk]; cur == nil || hit.Score > cur.Score {+ best[hk] = hit+ }+ }+ }+ out := make([]*ClaimHit, 0, len(best))+ for hk, hit := range best {+ hit.Rank = 0 // set below, after sorting+ _ = fused[hk]+ out = append(out, hit)+ }+ sort.Slice(out, func(i, j int) bool {+ ki := key{docID: out[i].DocID, name: out[i].Name}+ kj := key{docID: out[j].DocID, name: out[j].Name}+ return fused[ki] > fused[kj]+ })+ for i, hit := range out {+ hit.Rank = i + 1+ }+ return out+}++// ClaimPseudoChunks renders recalled claims as the pool-shaped pseudo chunks+// the GRAPH fan-out's channel 0 admits (Python agentic_rag_graph.py+// _collect_evidence :712-733): the row leads with the literal "[evidence]"+// prefix, the quote is a LITERAL quoted span capped at EvidenceQuoteChars+// (400), and the whole content is capped at 1200. source_chunk_ids ride along+// so the directional top-up and _prefill_slots_from_evidence can find the+// underlying chunks.+//+// This is NOT the action-session prefetch's format — Python's _claim_prefetch+// renders "[claim #rank]" with a 1200-char quote (action_session.py:741-747),+// which ClaimPrefetch builds inline. The two formats are Python-exact and+// deliberately distinct.+func ClaimPseudoChunks(claims []*ClaimHit) []map[string]interface{} {+ out := make([]map[string]interface{}, 0, len(claims))+ for _, c := range claims {+ cid := claimHitID(c)+ content := "[evidence] " + c.Name+ if c.Description != "" && c.Description != c.Name {+ content += " — " + c.Description+ }+ if c.Quote != "" {+ // Python [:_STRUCT_CLAIM_EVIDENCE_CHARS] counts CODE POINTS.+ quote := truncateRunes(c.Quote, EvidenceQuoteChars)+ content += "\nEvidence (verbatim): \"" + quote + "\""+ }+ // Python content[:1200] (:728) counts CODE POINTS, not bytes.+ content = truncateRunes(content, 1200)+ out = append(out, map[string]interface{}{+ "chunk_id": cid,+ "content_with_weight": content,+ "doc_id": c.DocID,+ "source_chunk_ids": c.ChunkIDs,+ })+ }+ return out+}++// claimHitID mirrors Python's "claim_" + md5(f"{doc_id}:{name}")[:12] pool id.+func claimHitID(c *ClaimHit) string {+ return "claim_" + fmt.Sprintf("%x", md5.Sum([]byte(c.DocID+":"+c.Name)))[:12]+}++// ClaimPrefetch runs the gated prefetch for one corpus search (Python+// action_session._claim_prefetch): the has-compilation gate, the recall, and+// the exclusive passage build. ok is false when the caller must fall through+// to the plain chunk search.+func ClaimPrefetch(ctx context.Context, deps SearchDeps, query string, seen map[string]bool) ([]map[string]any, []string, []map[string]interface{}, bool) {+ if strings.TrimSpace(query) == "" {+ return nil, nil, nil, false+ }+ if !DatasetHasCompilation(ctx, deps) {+ return nil, nil, nil, false+ }+ claims := RecallDatasetClaims(ctx, deps, query, ClaimPrefetchTopN)+ if len(claims) == 0 {+ return nil, nil, nil, false+ }+ var payload []map[string]any+ var ids []string+ var pseudo []map[string]interface{}+ for _, c := range claims {+ cid := claimHitID(c)+ if seen[cid] {+ continue+ }+ content := fmt.Sprintf("[claim #%d] %s", c.Rank, c.Name)+ if c.Description != "" && c.Description != c.Name {+ content += " — " + c.Description+ }+ if c.Quote != "" {+ // Python [:_STRUCT_CLAIM_EVIDENCE_CHARS] counts CODE POINTS.+ quote := truncateRunes(c.Quote, ClaimEvidenceChars)+ content += fmt.Sprintf("\nEvidence (verbatim): %q", quote)+ }+ // Python content[:1200] (:747) counts CODE POINTS, not bytes.+ content = truncateRunes(content, 1200)+ docID := c.DocID+ payload = append(payload, map[string]any{"id": cid, "content": content, "doc_id": docID})+ ids = append(ids, cid)+ pseudo = append(pseudo, map[string]interface{}{+ "chunk_id": cid,+ "content_with_weight": content,+ "doc_id": docID,+ "source_chunk_ids": c.ChunkIDs,+ })+ }+ if len(payload) == 0 {+ return nil, nil, nil, false+ }+ return payload, ids, pseudo, true+}++// claimChunkIDs merges the row column and the payload's own ids.+func claimChunkIDs(values ...interface{}) []string {+ for _, v := range values {+ switch x := v.(type) {+ case []string:+ if out := claimTrimIDs(x); len(out) > 0 {+ return out+ }+ case []interface{}:+ out := make([]string, 0, len(x))+ for _, item := range x {+ if s := strings.TrimSpace(claimStr(item)); s != "" {+ out = append(out, s)+ }+ }+ if len(out) > 0 {+ return out+ }+ case string:+ if s := strings.TrimSpace(x); s != "" {+ return []string{s}+ }+ }+ }+ return nil+}++func claimTrimIDs(ids []string) []string {+ out := make([]string, 0, len(ids))+ for _, id := range ids {+ if s := strings.TrimSpace(id); s != "" {+ out = append(out, s)+ }+ }+ return out+}++// claimStr coerces a JSON value to its trimmed string form.+func claimStr(v interface{}) string {+ switch x := v.(type) {+ case string:+ return x+ case fmt.Stringer:+ return x.String()+ default:+ if v == nil {+ return ""+ }+ return fmt.Sprintf("%v", v)+ }+}++// ---------------------------------------------------------------------------+// Document-level claim recall (Python navigation.py::_recall_claim_hits)+// ---------------------------------------------------------------------------++// DocClaimHit is one claim recalled for a SINGLE document, in the shape+// Python _recall_claim_hits returns to _render_toc_drilldown: the source+// chunk pointer, the fused rank/score, the statement and its verbatim+// evidence ([{"quote": ...}] when the claim carries one, else empty).+type DocClaimHit struct {+ ChunkID string+ Score float64+ Rank int+ Name string+ Description string+ Evidence []map[string]any+}++// evidenceRowTypes mirrors Python _evidence_row_types: the entity_type_kwd+// values carrying evidence for the given compile kinds. Every compiler writes+// its evidence rows as "claim", so any known kind yields ["claim"]; with no+// kinds the drill searches ["claim"] outright. Unknown kinds contribute+// nothing (Python: the same map lookup misses).+func evidenceRowTypes(kinds []string) []string {+ if len(kinds) == 0 {+ return []string{"claim"}+ }+ for _, k := range kinds {+ switch k {+ case "tree", "raptor", "page_index":+ return []string{"claim"}+ }+ }+ return nil+}++// docClaimCacheEntry memoizes one (doc_id, query) claim recall. The drill calls+// it once per routed doc and the model re-issues navigate_structure, so the+// same pair recurs within a session; each miss costs two store round-trips.+type docClaimCacheEntry struct {+ at time.Time+ hits []DocClaimHit+}++var (+ docClaimMu sync.Mutex+ docClaimCache = map[string]docClaimCacheEntry{}+)++// RecallDocClaimHits runs the two-leg hybrid claim recall for ONE document and+// returns up to topN fused hits (Python navigation.py:_recall_claim_hits):+// a KNN leg over the claim rows' q_<dim>_vec (when qvec is available) and a+// BM25 leg over content_ltks/content_sm_ltks, fused by reciprocal rank. No+// similarity threshold: the store ranks, the top-N ARE the hit set. Empty —+// never an error — when the document has no compiled claims or the store+// fails; a failed recall simply leaves the drill-down in charge.+func RecallDocClaimHits(ctx context.Context, deps SearchDeps, query, docID string, kinds []string, qvec []float64, topN int) []DocClaimHit {+ if docID == "" || topN <= 0 {+ return nil+ }+ de := claimEngine(deps)+ if de == nil || deps.TenantID == "" {+ return nil+ }+ query = strings.TrimSpace(query)+ key := docID + "|" + strings.ToLower(query)+ now := time.Now()++ docClaimMu.Lock()+ if cached, ok := docClaimCache[key]; ok && now.Sub(cached.at).Seconds() < claimPrefetchCacheTTL {+ docClaimMu.Unlock()+ return cached.hits+ }+ if len(docClaimCache) >= claimPrefetchCacheCap {+ docClaimCache = map[string]docClaimCacheEntry{}+ }+ docClaimMu.Unlock()++ limit := topN+ if limit < claimRecallLegFloor {+ limit = claimRecallLegFloor+ }+ rowTypes := evidenceRowTypes(kinds)+ filter := map[string]interface{}{+ "doc_id": []string{docID},+ "entity_type_kwd": rowTypes,+ // Doc-scope rows only: the Build button's KB-wide merged rows carry a+ // doc_id too, and without this filter they would answer for a single+ // document. Mirrors the dataset-level claim leg.+ "scope_kwd": []string{"doc"},+ }+ if len(kinds) > 0 {+ sorted := make([]string, 0, len(kinds))+ for _, k := range kinds {+ if k = strings.TrimSpace(k); k != "" {+ sorted = append(sorted, k)+ }+ }+ sort.Strings(sorted)+ filter["compile_kwd"] = sorted+ }+ fields := []string{"content_with_weight", "source_chunk_ids", "entity_type_kwd", "compile_kwd"}++ // KNN leg (needs a query vector); BM25 leg always runs.+ var denseLeg, textLeg []*ClaimHit+ if len(qvec) > 0 {+ exprs := []interface{}{&types.MatchDenseExpr{+ VectorColumnName: fmt.Sprintf("q_%d_vec", len(qvec)),+ EmbeddingData: qvec,+ EmbeddingDataType: "float",+ DistanceType: "cosine",+ TopN: limit,+ }}+ denseLeg = claimSearchLeg(ctx, de, deps, filter, fields, exprs, limit)+ }+ exprs := []interface{}{&types.MatchTextExpr{+ Fields: []string{"content_ltks", "content_sm_ltks"},+ MatchingText: query,+ TopN: limit,+ }}+ textLeg = claimSearchLeg(ctx, de, deps, filter, fields, exprs, limit)++ if len(denseLeg) == 0 && len(textLeg) == 0 {+ return nil+ }+ fused := rrfFuseClaims(denseLeg, textLeg)+ if len(fused) > topN {+ fused = fused[:topN]+ }+ out := make([]DocClaimHit, 0, len(fused))+ for _, h := range fused {+ hit := DocClaimHit{+ ChunkID: "",+ Score: h.Score,+ Rank: h.Rank,+ Name: h.Name,+ Description: h.Description,+ Evidence: []map[string]any{},+ }+ if len(h.ChunkIDs) > 0 {+ hit.ChunkID = h.ChunkIDs[0]+ }+ // Python: description = h["description"] or h["name"].+ if hit.Description == "" {+ hit.Description = h.Name+ }+ if h.Quote != "" {+ hit.Evidence = []map[string]any{{"quote": h.Quote}}+ }+ out = append(out, hit)+ }++ docClaimMu.Lock()+ docClaimCache[key] = docClaimCacheEntry{at: now, hits: out}+ docClaimMu.Unlock()+ return out+}++// docClaimQuote returns the first non-empty verbatim quote of a hit's evidence+// list (Python: the `for ev in hit["evidence"]` loop).+func docClaimQuote(h DocClaimHit) string {+ for _, ev := range h.Evidence {+ if q, ok := ev["quote"].(string); ok {+ if s := strings.TrimSpace(q); s != "" {+ return s+ }+ }+ }+ return ""+}++// publishClaimHits mirrors the rendered claims of ONE document into the shared+// evidence pool (Python navigation.py::_publish_claim_hits): the SCA, the slot+// prefill and the final compose all read ONLY the pool, so a claim that+// already states the fact must land there. Pure addition: no retrieval is+// suppressed, entries dedup by the SAME "claim_"+md5(doc_id:name) id the+// session prefetch writes, so a claim found by either path is one entry.+// Returns how many NEW entries the pool gained.+func publishClaimHits(deps SearchDeps, hits []DocClaimHit, docID string) int {+ if deps.KB == nil || len(hits) == 0 {+ return 0+ }+ added := 0+ deps.KB.Admit(func(p *PoolAdmitter) {+ for _, h := range hits {+ name := strings.TrimSpace(h.Name)+ if name == "" {+ continue+ }+ cid := claimHitID(&ClaimHit{DocID: docID, Name: name})+ rank := "?"+ if h.Rank > 0 {+ rank = fmt.Sprint(h.Rank)+ }+ content := fmt.Sprintf("[claim #%s] %s", rank, name)+ if desc := strings.TrimSpace(h.Description); desc != "" && desc != name {+ content += " — " + desc+ }+ if quote := docClaimQuote(h); quote != "" {+ // Python [:_STRUCT_CLAIM_EVIDENCE_CHARS] counts CODE POINTS.+ quote := truncateRunes(docClaimQuote(h), ClaimEvidenceChars)+ content += "\nEvidence (verbatim): \"" + quote + "\""+ }+ // Python content[:1200] (navigation.py:1585-1600) counts CODE POINTS.+ content = truncateRunes(content, 1200)+ src := []string{}+ if h.ChunkID != "" {+ src = []string{h.ChunkID}+ }+ if p.Add(map[string]interface{}{+ "chunk_id": cid,+ "content_with_weight": content,+ "doc_id": docID,+ "source_chunk_ids": src,+ }) {+ added+++ }+ }+ })+ if added > 0 {+ _LOG.Printf("[navigate_structure] published %d claim(s) to the evidence pool (doc=%s)", added, docID)+ }+ return added+}++// LoadChunksForIDs fetches source chunks by id, any owning document (Python+// navigation.py::_load_chunks_for_ids). Zero LLM, zero recall — a directed+// fetch used by the fan-out evidence top-up to pull the passages an admitted+// claim row cites. Empty on any failure: best effort by contract.+func LoadChunksForIDs(ctx context.Context, deps SearchDeps, ids []string) []map[string]any {+ ids = claimTrimIDs(ids)+ if len(ids) == 0 {+ return nil+ }+ de := claimEngine(deps)+ if de == nil {+ return nil+ }+ res, err := de.Search(ctx, &types.SearchRequest{+ IndexNames: []string{claimIndexName(deps)},+ KbIDs: deps.KbIDs,+ Offset: 0,+ Limit: len(ids),+ SelectFields: []string{"content_with_weight", "docnm_kwd", "doc_id"},+ Filter: map[string]interface{}{"id": ids},+ })+ if err != nil || res == nil {+ return nil+ }+ out := make([]map[string]any, 0, len(res.Chunks))+ for _, row := range res.Chunks {+ out = append(out, map[string]any{+ "chunk_id": row["id"],+ "content_with_weight": row["content_with_weight"],+ "docnm_kwd": row["docnm_kwd"],+ "doc_id": row["doc_id"],+ })+ }+ return out+}++// mustEncodeQueries embeds one search query: query-side encoding when the+// embedder is asymmetric (Python tools.embed_mdl.encode_queries; NavQueryEmbedder+// for Cohere/Voyage/Jina/NVIDIA), plain Encode otherwise. Returns nil on+// failure — the BM25 leg alone keeps claim recall alive.+func mustEncodeQueries(deps SearchDeps, ctx context.Context, query string) [][]float32 {+ if deps.Embedder == nil {+ return nil+ }+ if qe, ok := deps.Embedder.(nlp.NavQueryEmbedder); ok {+ vecs, err := qe.EncodeQueries(ctx, deps.TenantID, []string{query})+ if err != nil {+ return nil+ }+ return vecs+ }+ vecs, err := deps.Embedder.Encode(ctx, deps.TenantID, []string{query})+ if err != nil {+ return nil+ }+ return vecs+}diff --git a/internal/rag/advanced_rag/harness/tool_claims_test.go b/internal/rag/advanced_rag/harness/tool_claims_test.gonew file mode 100644index 0000000000..41951df06c--- /dev/null+++ b/internal/rag/advanced_rag/harness/tool_claims_test.go@@ -0,0 +1,396 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package harness++import (+ "context"+ "encoding/json"+ "reflect"+ "strings"+ "testing"++ "ragflow/internal/engine"+ "ragflow/internal/engine/types"+)++// recordingClaimEngine implements engine.DocEngine by serving Search from a+// fixed row set and recording every request; the rest of the interface is+// promoted from a nil field and never called by these tests.+type recordingClaimEngine struct {+ engine.DocEngine+ requests []*types.SearchRequest+ rows []map[string]interface{}+}++func (e *recordingClaimEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {+ e.requests = append(e.requests, req)+ return &types.SearchResult{Chunks: e.rows}, nil+}++// claimRow builds one claim store row in the shape the compilers write.+func claimRow(docID, name, description, quote string, similarity float64, chunkIDs []string) map[string]interface{} {+ payload := map[string]any{"name": name, "type": "claim"}+ if description != "" {+ payload["description"] = description+ }+ if quote != "" {+ payload["evidence"] = []any{map[string]any{"quote": quote}}+ }+ if chunkIDs != nil {+ payload["source_chunk_ids"] = chunkIDs+ }+ raw, _ := json.Marshal(payload)+ return map[string]interface{}{+ "content_with_weight": string(raw),+ "source_chunk_ids": chunkIDs,+ "doc_id": docID,+ "similarity": similarity,+ }+}++func resetClaimCaches() {+ claimRecallMu.Lock()+ claimRecallCache = map[string]claimCacheEntry{}+ claimRecallMu.Unlock()+ docClaimMu.Lock()+ docClaimCache = map[string]docClaimCacheEntry{}+ docClaimMu.Unlock()+ compilationMu.Lock()+ compilationCache = map[string]compilationProbe{}+ compilationMu.Unlock()+}++// TestRecallDocClaimHitsFiltersAndMaps pins the store contract and the output+// mapping of the per-document claim leg (Python navigation.py:_recall_claim_hits):+// doc-scope + claim-row filters, sorted compile_kwd when kinds are given, and+// hits shaped as {chunk_id, score, rank, name, description-or-name, evidence}.+func TestRecallDocClaimHitsFiltersAndMaps(t *testing.T) {+ resetClaimCaches()+ de := &recordingClaimEngine{rows: []map[string]interface{}{+ claimRow("doc-1", "The tower opened in 1889", "", "The tower opened in 1889.", 0.9, []string{"c1", "c2"}),+ }}+ deps := SearchDeps{TenantID: "tenant-1", IndexName: "idx", DocEngine: de, KbIDs: []string{"kb-1"}}++ hits := RecallDocClaimHits(context.Background(), deps, "When did the tower open?", "doc-1", []string{"tree", "raptor"}, nil, 8)+ if len(hits) != 1 {+ t.Fatalf("hits = %d, want 1", len(hits))+ }+ h := hits[0]+ if h.ChunkID != "c1" {+ t.Errorf("ChunkID = %q, want the first source chunk c1", h.ChunkID)+ }+ if h.Score != 0.9 || h.Rank != 1 {+ t.Errorf("Score/Rank = %v/%d, want 0.9/1", h.Score, h.Rank)+ }+ if h.Name != "The tower opened in 1889" {+ t.Errorf("Name = %q", h.Name)+ }+ // Python: description = h["description"] or h["name"].+ if h.Description != h.Name {+ t.Errorf("Description = %q, want the name fallback", h.Description)+ }+ wantEv := []map[string]any{{"quote": "The tower opened in 1889."}}+ if !reflect.DeepEqual(h.Evidence, wantEv) {+ t.Errorf("Evidence = %v, want %v", h.Evidence, wantEv)+ }++ // Store contract: one BM25 leg (no qvec), doc-scope claim filters, sorted+ // compile_kwd, the wide field list, limit floored at 32.+ if len(de.requests) != 1 {+ t.Fatalf("store calls = %d, want 1 (BM25 leg only)", len(de.requests))+ }+ req := de.requests[0]+ wantFilter := map[string]interface{}{+ "doc_id": []string{"doc-1"},+ "entity_type_kwd": []string{"claim"},+ "scope_kwd": []string{"doc"},+ "compile_kwd": []string{"raptor", "tree"},+ }+ if !reflect.DeepEqual(req.Filter, wantFilter) {+ t.Errorf("Filter = %v, want %v", req.Filter, wantFilter)+ }+ for _, f := range []string{"content_with_weight", "source_chunk_ids", "entity_type_kwd", "compile_kwd"} {+ if !sliceContains(req.SelectFields, f) {+ t.Errorf("SelectFields missing %q: %v", f, req.SelectFields)+ }+ }+ if req.Limit != claimRecallLegFloor {+ t.Errorf("Limit = %d, want the %d floor", req.Limit, claimRecallLegFloor)+ }+ if len(req.MatchExprs) != 1 {+ t.Fatalf("MatchExprs = %d, want the single BM25 leg", len(req.MatchExprs))+ }+ text, ok := req.MatchExprs[0].(*types.MatchTextExpr)+ if !ok {+ t.Fatalf("MatchExprs[0] is %T, want *types.MatchTextExpr", req.MatchExprs[0])+ }+ if text.MatchingText != "When did the tower open?" {+ t.Errorf("BM25 leg text = %q, want the raw query", text.MatchingText)+ }+}++// TestRecallDocClaimHitsDenseLeg pins the KNN leg: a query vector turns into+// one MatchDenseExpr over q_<dim>_vec ahead of the BM25 leg.+func TestRecallDocClaimHitsDenseLeg(t *testing.T) {+ resetClaimCaches()+ de := &recordingClaimEngine{}+ deps := SearchDeps{TenantID: "tenant-1", DocEngine: de}+ qvec := []float64{0.1, 0.2, 0.3}+ RecallDocClaimHits(context.Background(), deps, "q", "doc-1", nil, qvec, 8)+ if len(de.requests) != 2 {+ t.Fatalf("store calls = %d, want 2 (KNN + BM25 legs)", len(de.requests))+ }+ dense, ok := de.requests[0].MatchExprs[0].(*types.MatchDenseExpr)+ if !ok {+ t.Fatalf("first leg is %T, want *types.MatchDenseExpr", de.requests[0].MatchExprs[0])+ }+ if dense.VectorColumnName != "q_3_vec" {+ t.Errorf("vector column = %q, want q_3_vec", dense.VectorColumnName)+ }+ if !reflect.DeepEqual(dense.EmbeddingData, qvec) {+ t.Errorf("embedding data = %v, want %v", dense.EmbeddingData, qvec)+ }+ // No kinds: no compile_kwd filter, row types default to claim.+ if _, ok := de.requests[0].Filter["compile_kwd"]; ok {+ t.Errorf("compile_kwd filter present without kinds: %v", de.requests[0].Filter)+ }+ if !reflect.DeepEqual(de.requests[0].Filter["entity_type_kwd"], []string{"claim"}) {+ t.Errorf("entity_type_kwd = %v, want [claim]", de.requests[0].Filter["entity_type_kwd"])+ }+}++// TestRecallDocClaimHitsMemo pins the (doc_id, query) memo: the drill re-issues+// navigate_structure, so the same pair must not cost two more store+// round-trips, while a different query or doc must miss. Empty results are NOT+// cached (Python returns before the cache write).+func TestRecallDocClaimHitsMemo(t *testing.T) {+ resetClaimCaches()+ de := &recordingClaimEngine{rows: []map[string]interface{}{+ claimRow("doc-1", "claim a", "", "", 0.5, []string{"c1"}),+ }}+ deps := SearchDeps{TenantID: "tenant-1", DocEngine: de}+ ctx := context.Background()++ RecallDocClaimHits(ctx, deps, "same query", "doc-1", nil, nil, 8)+ RecallDocClaimHits(ctx, deps, "Same Query ", "doc-1", nil, nil, 8)+ if len(de.requests) != 1 {+ t.Fatalf("store calls after repeat = %d, want 1 (memo hit)", len(de.requests))+ }+ RecallDocClaimHits(ctx, deps, "other query", "doc-1", nil, nil, 8)+ if len(de.requests) != 2 {+ t.Fatalf("store calls after new query = %d, want 2", len(de.requests))+ }+ RecallDocClaimHits(ctx, deps, "same query", "doc-2", nil, nil, 8)+ if len(de.requests) != 3 {+ t.Fatalf("store calls after new doc = %d, want 3", len(de.requests))+ }++ empty := &recordingClaimEngine{}+ emptyDeps := SearchDeps{TenantID: "tenant-1", DocEngine: empty}+ RecallDocClaimHits(ctx, emptyDeps, "nothing here", "doc-9", nil, nil, 8)+ RecallDocClaimHits(ctx, emptyDeps, "nothing here", "doc-9", nil, nil, 8)+ if len(empty.requests) != 2 {+ t.Fatalf("empty result must not be memoized; store calls = %d, want 2", len(empty.requests))+ }+}++// TestClaimAggRouterClaimsDecide pins the claim-first routing order: documents+// ranked by the damped claim DocScore with the best claim similarity as the+// reported tie-break, nav summaries attached, and the chunk_agg fallback NEVER+// consulted while the claim leg hits.+func TestClaimAggRouterClaimsDecide(t *testing.T) {+ resetClaimCaches()+ de := &recordingClaimEngine{rows: []map[string]interface{}{+ // doc-a: one strong claim -> DocScore 0.9/sqrt(2) ≈ 0.636.+ claimRow("doc-a", "claim a", "", "quote a", 0.9, []string{"c1"}),+ // doc-b: two weaker claims -> DocScore 0.9/sqrt(3) ≈ 0.520.+ claimRow("doc-b", "claim b1", "", "", 0.5, []string{"c2"}),+ claimRow("doc-b", "claim b2", "", "", 0.4, []string{"c3"}),+ }}+ fallback := &fallbackRouter{}+ summaries := func(_ context.Context, _, _ string, docIDs []string) map[string]string {+ out := map[string]string{}+ for _, d := range docIDs {+ out[d] = "summary of " + d+ }+ return out+ }+ r := &ClaimAggRouter{+ Deps: SearchDeps{TenantID: "tenant-1", DocEngine: de},+ Fallback: fallback,+ Summarize: summaries,+ }+ routed, err := r.Route(context.Background(), "tenant-1", "kb-1", "tower height", nil, 8)+ if err != nil {+ t.Fatalf("Route: %v", err)+ }+ if fallback.called {+ t.Fatal("chunk_agg fallback must not run while the claim leg hits")+ }+ if len(routed) != 2 {+ t.Fatalf("routed = %v, want 2 docs", routed)+ }+ if routed[0][0] != "doc-a" || routed[1][0] != "doc-b" {+ t.Errorf("order = %v, want [doc-a doc-b] (damped DocScore)", routed)+ }+ if routed[0][1] != "summary of doc-a" {+ t.Errorf("summary = %q, want the nav_doc summary", routed[0][1])+ }+}++// TestClaimAggRouterDocScope pins the doc_scope filter: out-of-scope documents+// never surface, and a scope that empties the claim leg falls back to the+// chunk leg (Python filters the store condition, so a scoped miss is an empty+// leg, not a routing verdict).+func TestClaimAggRouterDocScope(t *testing.T) {+ resetClaimCaches()+ de := &recordingClaimEngine{rows: []map[string]interface{}{+ claimRow("doc-a", "claim a", "", "", 0.9, []string{"c1"}),+ claimRow("doc-b", "claim b", "", "", 0.8, []string{"c2"}),+ }}+ fallback := &fallbackRouter{next: [][2]string{{"doc-fallback", ""}}}+ r := &ClaimAggRouter{+ Deps: SearchDeps{TenantID: "tenant-1", DocEngine: de},+ Fallback: fallback,+ }+ routed, err := r.Route(context.Background(), "tenant-1", "kb-1", "q", []string{"doc-b"}, 8)+ if err != nil {+ t.Fatalf("Route: %v", err)+ }+ if len(routed) != 1 || routed[0][0] != "doc-b" {+ t.Errorf("routed = %v, want only doc-b", routed)+ }+ if fallback.called {+ t.Fatal("fallback must not run when the scoped claim leg still hits")+ }++ fallback2 := &fallbackRouter{next: [][2]string{{"doc-fallback", ""}}}+ r2 := &ClaimAggRouter{Deps: SearchDeps{TenantID: "tenant-1", DocEngine: de}, Fallback: fallback2}+ routed2, err := r2.Route(context.Background(), "tenant-1", "kb-1", "q", []string{"doc-elsewhere"}, 8)+ if err != nil {+ t.Fatalf("Route: %v", err)+ }+ if !fallback2.called || len(routed2) != 1 || routed2[0][0] != "doc-fallback" {+ t.Errorf("emptied scoped leg must fall back, got routed=%v called=%v", routed2, fallback2.called)+ }+}++// TestClaimAggRouterFallsBackOnEmpty pins the empty-leg contract: no claim+// rows (a raptor-less or pre-claim dataset) is a legitimate empty leg, so the+// chunk_agg fallback decides.+func TestClaimAggRouterFallsBackOnEmpty(t *testing.T) {+ resetClaimCaches()+ de := &recordingClaimEngine{}+ fallback := &fallbackRouter{next: [][2]string{{"doc-chunk", "chunk summary"}}}+ r := &ClaimAggRouter{+ Deps: SearchDeps{TenantID: "tenant-1", DocEngine: de},+ Fallback: fallback,+ }+ routed, err := r.Route(context.Background(), "tenant-1", "kb-1", "q", nil, 8)+ if err != nil {+ t.Fatalf("Route: %v", err)+ }+ if !fallback.called {+ t.Fatal("empty claim leg must consult the chunk_agg fallback")+ }+ if len(routed) != 1 || routed[0][0] != "doc-chunk" {+ t.Errorf("routed = %v, want the fallback's result", routed)+ }+}++// fallbackRouter stands in for the chunk_agg router.+type fallbackRouter struct {+ called bool+ next [][2]string+}++func (r *fallbackRouter) Route(_ context.Context, _, _, _ string, _ []string, _ int) ([][2]string, error) {+ r.called = true+ return r.next, nil+}++// TestPublishClaimHits pins the evidence-pool publish (Python+// _publish_claim_hits): one pseudo chunk per claim under the SAME+// "claim_"+md5(doc:name) id the session prefetch writes, verbatim evidence+// capped at ClaimEvidenceChars, the claim's own chunk pointer as+// source_chunk_ids, deduped against the live pool.+func TestPublishClaimHits(t *testing.T) {+ kb := &Kbinfos{}+ deps := SearchDeps{KB: kb}+ hits := []DocClaimHit{+ {ChunkID: "c1", Rank: 2, Name: "Claim A", Description: "Claim A", Evidence: []map[string]any{{"quote": strings.Repeat("q", 2000)}}},+ {ChunkID: "", Rank: 3, Name: "Claim B", Description: "a different description"},+ }+ if got := publishClaimHits(deps, hits, "doc-1"); got != 2 {+ t.Fatalf("published = %d, want 2", got)+ }+ if got := publishClaimHits(deps, hits, "doc-1"); got != 0 {+ t.Fatalf("re-publish must dedup to 0, got %d", got)+ }+ if len(kb.Chunks) != 2 {+ t.Fatalf("pool = %d entries, want 2", len(kb.Chunks))+ }+ a := kb.Chunks[0]+ if a["chunk_id"] != claimHitID(&ClaimHit{DocID: "doc-1", Name: "Claim A"}) {+ t.Errorf("chunk_id = %v, want the shared claim id scheme", a["chunk_id"])+ }+ content, _ := a["content_with_weight"].(string)+ if !strings.HasPrefix(content, "[claim #2] Claim A") {+ t.Errorf("content = %q, want the [claim #rank] prefix", content)+ }+ if !strings.Contains(content, "\nEvidence (verbatim): \"") {+ t.Errorf("content missing the verbatim evidence line:\n%s", content)+ }+ if len(content) > 1200 {+ t.Errorf("content length = %d, want the 1200 cap", len(content))+ }+ if !reflect.DeepEqual(a["source_chunk_ids"], []string{"c1"}) {+ t.Errorf("source_chunk_ids = %v, want [c1]", a["source_chunk_ids"])+ }+ // Rank-3 claim with no chunk pointer and no quote: empty source list,+ // description rendered when it differs from the name.+ b := kb.Chunks[1]+ if !reflect.DeepEqual(b["source_chunk_ids"], []string{}) {+ t.Errorf("source_chunk_ids = %v, want []", b["source_chunk_ids"])+ }+ if !strings.Contains(b["content_with_weight"].(string), "Claim B — a different description") {+ t.Errorf("content = %q, want the description dash form", b["content_with_weight"])+ }+}++// TestLoadChunksForIDs pins the directed source-chunk fetch (Python+// _load_chunks_for_ids): an id-filtered store read that maps rows back to+// pool-shaped chunks.+func TestLoadChunksForIDs(t *testing.T) {+ de := &recordingClaimEngine{rows: []map[string]interface{}{+ {"id": "c1", "content_with_weight": "full text one", "doc_id": "doc-1", "docnm_kwd": "a.pdf"},+ }}+ deps := SearchDeps{TenantID: "tenant-1", IndexName: "idx", DocEngine: de, KbIDs: []string{"kb-1"}}+ out := LoadChunksForIDs(context.Background(), deps, []string{"c1", " "})+ if len(out) != 1 {+ t.Fatalf("fetched = %d, want 1", len(out))+ }+ if out[0]["chunk_id"] != "c1" || out[0]["content_with_weight"] != "full text one" {+ t.Errorf("fetched row = %v", out[0])+ }+ if len(de.requests) != 1 || !reflect.DeepEqual(de.requests[0].Filter["id"], []string{"c1"}) {+ t.Errorf("request = %+v, want one id-filtered read for [c1]", de.requests[0])+ }+ if LoadChunksForIDs(context.Background(), deps, nil) != nil {+ t.Error("empty ids must fetch nothing")+ }+}diff --git a/internal/rag/advanced_rag/harness/tool_compiled_expansion.go b/internal/rag/advanced_rag/harness/tool_compiled_expansion.goindex cebae3a427..5ca213249b 100644--- a/internal/rag/advanced_rag/harness/tool_compiled_expansion.go+++ b/internal/rag/advanced_rag/harness/tool_compiled_expansion.go@@ -18,6 +18,7 @@ package harnessimport ("context"+ "crypto/md5""encoding/json""fmt""math"@@ -25,6 +26,7 @@ import ("strings""ragflow/internal/entity"+ "ragflow/internal/tokenizer")// Dense-match parameters for compiled-row search, mirroring Python@@ -179,6 +181,18 @@ func (e *compiledExpander) Expand(ctx context.Context, kb *Kbinfos, query, keywomatch = strings.TrimSpace(keywords)}+ // The hybrid hits themselves — the claim-neighbour leg keys on them (the+ // passages hybrid_search deemed relevant), so capture them BEFORE any+ // expansion appends its own rows (Python :60-68).+ hitChunks := make([]hitChunk, 0, len(kb.Chunks))+ for _, c := range kb.Chunks {+ hitChunks = append(hitChunks, hitChunk{+ id: ChunkIDOf(c),+ sim: similarityOf(c),+ docID: DocIDOf(c),+ })+ }+// Admit one expansion batch. UNCAPPED, like Python's expansion (it appends// straight to kbinfos, which is what pushes the pool past _EVIDENCE_POOL_CAP),// but under the pool lock and deduped against the LIVE pool — the per-call@@ -211,11 +225,19 @@ func (e *compiledExpander) Expand(ctx context.Context, kb *Kbinfos, query, keywo}}// Tree structure graph, selected by compile_kwd (Python L91-104):- // template_kind empty, compile_kwd="tree".+ // template_kind empty, compile_kwd="tree". When the per-row strategy+ // comes back empty, fall back to the LEGACY graph-blob strategy —+ // documents compiled before the per-row migration persist ONE+ // knowledge_graph_kwd="graph" blob per document and no raw rows+ // (Python L110-125 _expand_tree_blob_strategy); it disappears once+ // those docs are recompiled.chunks, err := e.expandCompiledStrategy(ctx, sc, match, seen, "tree", "", 5)if err != nil {return err}+ if len(chunks) == 0 {+ chunks = e.expandTreeBlobStrategy(ctx, sc, match, seen, 5)+ }if n := admitExpansion(chunks); n > 0 {_LOG.Printf("[Compiled expand] tree: +%d chunks", n)}@@ -230,6 +252,15 @@ func (e *compiledExpander) Expand(ctx context.Context, kb *Kbinfos, query, keywo_LOG.Printf("[Compiled expand] %s: +%d chunks", ck.label, n)}}+ // Claims adjacent to the passages hybrid_search just hit (Python+ // L154-169 _expand_claim_neighbor_strategy). Non-redundant with the+ // query-keyed claim prefetch: when the prefetch hits, the exclusive+ // takeover means this expansion never runs; when it misses for the+ // QUERY, chunk hits are a different key and their sibling claims may+ // still match.+ if n := admitExpansion(e.expandClaimNeighborStrategy(ctx, sc, hitChunks, seen, 6)); n > 0 {+ _LOG.Printf("[Compiled expand] claim neighbours: +%d chunks", n)+ }}// Re-sort so compiled-expansion chunks blend by similarity with regular ones@@ -613,3 +644,276 @@ func cosine(a, b []float64) float64 {}return dot / (math.Sqrt(na) * math.Sqrt(nb))}++// ---------------------------------------------------------------------------+// Legacy tree-blob expansion (Python _search_tree_blobs / _score_tree_entities+// / _expand_tree_blob_strategy, compiled_expansion.py:579-718) and the+// claim-neighbour expansion (Python _expand_claim_neighbor_strategy, :724-829).+// ---------------------------------------------------------------------------++// hitChunk is one hybrid_search hit the claim-neighbour leg keys on+// (Python :60-68's (chunk_id, similarity, doc_id) triples).+type hitChunk struct {+ id string+ sim float64+ docID string+}++// expandTreeBlobStrategy mirrors Python _expand_tree_blob_strategy: expand a+// tree-compiled document from its pre-migration graph blob, in-process. Same+// contract as expandCompiledStrategy — seed entities, 1-hop neighbours, and+// the neighbours' source_chunk_ids loaded back as real chunks — except seeds,+// relations, and addressing all come from the blob's JSON, so the walk costs+// zero extra store round-trips. Entity payloads are compiled summaries: only+// the loaded chunks reach the model.+func (e *compiledExpander) expandTreeBlobStrategy(ctx context.Context, sc compiledScope, match string, seen map[string]bool, maxChunks int) []map[string]any {+ // Python _search_tree_blobs: compile_kwd=["tree"] + knowledge_graph_kwd=+ // ["graph"], payload-only, capped at 16 rows.+ filters := map[string][]string{"compile_kwd": {"tree"}, "knowledge_graph_kwd": {"graph"}}+ rows := e.searchCompiledRows(ctx, sc, "", 16, filters)+ if len(rows) == 0 {+ return nil+ }++ byDoc := map[string][]string{}+ var order []string+ add := func(docID, cid string) {+ for _, existing := range byDoc[docID] {+ if existing == cid {+ return+ }+ }+ byDoc[docID] = append(byDoc[docID], cid)+ kept := false+ for _, d := range order {+ if d == docID {+ kept = true+ break+ }+ }+ if !kept {+ order = append(order, docID)+ }+ }+ for _, row := range rows {+ docID := compiledDocID(row)+ if docID == "" {+ continue+ }+ graph, ok := decodeJSONObject(asString(row["content_with_weight"]))+ if !ok {+ continue+ }+ // Seeds: query-matched entities, then 1-hop neighbours through the+ // blob's own relations (parent/child edges) — no store round-trips.+ scored := scoreTreeEntities(graph, match)+ if len(scored) == 0 {+ continue+ }+ if len(scored) > 5 {+ scored = scored[:5]+ }+ seedNames := map[string]bool{}+ for _, ent := range scored {+ if name := strings.TrimSpace(asString(ent["name"])); name != "" {+ seedNames[name] = true+ }+ }+ neighbours := map[string]bool{}+ for _, rel := range objectList(graph["relations"]) {+ from := strings.TrimSpace(compiledRelationEndpoint(rel, "from"))+ to := strings.TrimSpace(compiledRelationEndpoint(rel, "to"))+ if from != "" && seedNames[from] && to != "" {+ neighbours[to] = true+ }+ if to != "" && seedNames[to] && from != "" {+ neighbours[from] = true+ }+ }+ // Address chunks: every entity at or adjacent to a seed contributes+ // its source_chunk_ids (the same contract the raw-row strategy applies+ // to neighbour entities).+ wanted := func(name string) bool { return seedNames[name] || neighbours[name] }+ for _, ent := range objectList(graph["entities"]) {+ if !wanted(strings.TrimSpace(asString(ent["name"]))) {+ continue+ }+ for _, cid := range compiledSourceChunkIDs(ent) {+ if cid != "" && !seen[cid] {+ add(docID, cid)+ }+ }+ }+ }+ loaded, err := e.loadByDoc(ctx, sc, order, byDoc, seen, maxChunks)+ if err != nil {+ // Python's blob strategy lets the store error escape into+ // _expand_with_compiled's caller; Expand already reports it the same+ // way for the other strategies.+ _LOG.Printf("[Compiled expand] tree blob chunk load failed: %v", err)+ return nil+ }+ return loaded+}++// scoreTreeEntities mirrors Python _score_tree_entities: rank blob entities by+// query-term overlap over name + description. Lexical on purpose — the raw-row+// strategy seeds the same way (BM25 over content_ltks), and the blob's vectors+// are one shared row vector with nothing per-entity to cosine against.+func scoreTreeEntities(graph map[string]any, query string) []map[string]any {+ terms := treeEntityTerms(query)+ if len(terms) == 0 {+ return nil+ }+ type scored struct {+ ent map[string]any+ score int+ }+ var out []scored+ for _, ent := range objectList(graph["entities"]) {+ text := strings.ToLower(asString(ent["name"]) + " " + asString(ent["description"]))+ if strings.TrimSpace(text) == "" {+ continue+ }+ score := 0+ for _, t := range terms {+ if strings.Contains(text, t) {+ score+++ }+ }+ if score > 0 {+ out = append(out, scored{ent: ent, score: score})+ }+ }+ // Stable sort: ties keep the blob's own entity order (Python's list.sort+ // is stable, :632).+ sort.SliceStable(out, func(i, j int) bool { return out[i].score > out[j].score })+ ents := make([]map[string]any, 0, len(out))+ for _, s := range out {+ ents = append(ents, s.ent)+ }+ return ents+}++// treeEntityTerms mirrors Python `_tokenize(query).split()` — the coarse+// RAGFlow tokenizer (knowlege_compile/dataset_nav.py:518) — lowercased for the+// containment match.+func treeEntityTerms(query string) []string {+ toks, err := tokenizer.Tokenize(query)+ if err != nil {+ return nil+ }+ var out []string+ seen := map[string]bool{}+ for _, t := range strings.Fields(strings.ToLower(toks)) {+ if t != "" && !seen[t] {+ seen[t] = true+ out = append(out, t)+ }+ }+ return out+}++// expandClaimNeighborStrategy mirrors Python _expand_claim_neighbor_strategy:+// inject claims adjacent to the passages hybrid_search just hit. A claim whose+// source_chunk_ids intersect a hit chunk is a verified atomic statement about a+// passage the retriever already deemed relevant — even when the QUERY never+// matched it. Emitted as pseudo-chunks (name + verbatim quote, "[claim] "+// prefix, ASCII " -- " description separator, no per-quote cap beyond the+// overall 1200) with the hit chunk's similarity, so they blend into the+// ranking. Deliberately NOT 1-hop expanded and deliberately keyed on the HIT+// chunks, not the query (Python :742-746).+func (e *compiledExpander) expandClaimNeighborStrategy(ctx context.Context, sc compiledScope, hits []hitChunk, seen map[string]bool, maxChunks int) []map[string]any {+ live := make([]hitChunk, 0, len(hits))+ for _, h := range hits {+ if h.id != "" && !strings.HasPrefix(h.id, "claim_") {+ live = append(live, h)+ }+ }+ if len(live) == 0 {+ return nil+ }+ hitIDs := make(map[string]bool, len(live))+ hitDocs := []string{}+ docSeen := map[string]bool{}+ for _, h := range live {+ hitIDs[h.id] = true+ if h.docID != "" && !docSeen[h.docID] {+ docSeen[h.docID] = true+ hitDocs = append(hitDocs, h.docID)+ }+ }+ sort.Strings(hitDocs)++ // Python :763-780: claim rows of the hit documents, payload-only, 256 rows.+ filters := map[string][]string{"entity_type_kwd": {"claim"}, "scope_kwd": {"doc"}}+ if len(hitDocs) > 0 {+ filters["doc_id"] = hitDocs+ }+ rows := e.searchCompiledRows(ctx, sc, "", 256, filters)++ out := make([]map[string]any, 0, maxChunks)+ for _, row := range rows {+ payload, ok := decodeJSONObject(asString(row["content_with_weight"]))+ if !ok {+ continue+ }+ name := strings.TrimSpace(asString(payload["name"]))+ if name == "" {+ continue+ }+ srcIDs := compiledSourceChunkIDs(row)+ if len(srcIDs) == 0 {+ srcIDs = compiledSourceChunkIDs(payload)+ }+ overlap := map[string]bool{}+ for _, cid := range srcIDs {+ if hitIDs[cid] {+ overlap[cid] = true+ }+ }+ if len(overlap) == 0 {+ continue+ }+ rowDoc := compiledDocID(row)+ cid := "claim_" + fmt.Sprintf("%x", md5.Sum([]byte(rowDoc+":"+name)))[:12]+ if seen[cid] {+ continue+ }+ description := strings.TrimSpace(asString(payload["description"]))+ quote := ""+ for _, ev := range objectList(payload["evidence"]) {+ if q := strings.TrimSpace(asString(ev["quote"])); q != "" {+ quote = q+ break+ }+ }+ content := "[claim] " + name+ if description != "" && description != name {+ content += " -- " + description+ }+ if quote != "" {+ content += "\nEvidence (verbatim): \"" + quote + "\""+ }+ // Inherit the best-matching hit's similarity so the pseudo-chunk lands+ // beside the passage that spawned it in the final ranking.+ similarity := 0.0+ for _, h := range live {+ if overlap[h.id] && h.sim > similarity {+ similarity = h.sim+ }+ }+ seen[cid] = true+ out = append(out, map[string]any{+ "chunk_id": cid,+ "content_with_weight": truncateRunes(content, 1200),+ "doc_id": rowDoc,+ "source_chunk_ids": srcIDs,+ "similarity": similarity,+ })+ if len(out) >= maxChunks {+ break+ }+ }+ return out+}diff --git a/internal/rag/advanced_rag/harness/tool_executor.go b/internal/rag/advanced_rag/harness/tool_executor.goindex 36bde2d7b8..172366221d 100644--- a/internal/rag/advanced_rag/harness/tool_executor.go+++ b/internal/rag/advanced_rag/harness/tool_executor.go@@ -242,21 +242,7 @@ func (e *searchExecutor) navigateTree(ctx context.Context, args map[string]any)if query == "" {return ToolOutcome{Payload: []any{}, Status: StatusError, Reason: ReasonBadArgs}, nil}- router := e.deps.NavRouter- if router == nil {- // Default navigation-tree router mirrors Python _NAV_TREE_ROUTER = "chunk_agg":- // route documents by aggregating raw-chunk retrieval rather than by a nav-row- // KNN descent (nav.NewNavServiceRouter). A nil retrieval backend degrades to the- // nav-row router, which needs no backend.- if e.deps.Backend != nil {- router = nav.NewChunkAggRouter(- chunkAggRetrieveFrom(e.deps.Backend),- nav.ChunkAggSummarize(),- )- } else {- router = nav.NewNavServiceRouter()- }- }+ router := e.navRouter()// The tool argument is NOT threaded (Python _exec_navigate_tree,// action_session.py:_exec_navigate_tree, calls _navigate_tree_impl(query, keywords=...) with// no doc_scope), but the SESSION scope still applies: the impl routes through@@ -264,8 +250,11 @@ func (e *searchExecutor) navigateTree(ctx context.Context, args map[string]any)// — the session doc_scope (navigation.py:_nav_search_titled). The router must therefore// receive the session ceiling, never the tool argument.res := NavigateTree(ctx, router, NavTreeInput{- Query: query,- Keywords: e.req.Keywords,+ Query: query,+ // Python :985 threads ONLY the tool argument's keywords (usually absent,+ // so "") — the run-level request keywords are a Go-only invention that+ // re-biased every tree routing toward the original question keywords.+ Keywords: argString(args, "keywords"),DocScope: e.deps.DocScope,TenantID: e.deps.TenantID,KbIDs: e.deps.KbIDs,@@ -275,28 +264,24 @@ func (e *searchExecutor) navigateTree(ctx context.Context, args map[string]any)case ReasonNoStructure:return ToolOutcome{Payload: []any{map[string]any{- "kind": "navigate_tree",- "note": "This dataset has no compiled document-navigation tree. Use search_chunks / retrieve instead.",- "query": query,- "doc_ids": []string{},+ // Python action_session.py:_exec_navigate_tree (dataset_has_compilation+ // gate) — note text verbatim; the payload carries kind+note only.+ "kind": "navigate_tree",+ "note": "This dataset has no compiled document-navigation structure; use search_chunks / retrieve instead.",}},- Status: StatusEmpty,- Reason: ReasonNoStructure,- Metrics: map[string]any{"docs": 0, "routed_docs": [][2]string{}},+ Status: StatusEmpty,+ Reason: ReasonNoStructure,}, nilcase ReasonNoDoc:// Structure exists, this query reached nothing — a MISS, not an EMPTY.+ // Python :990 note verbatim; the payload carries kind+note only.return ToolOutcome{Payload: []any{map[string]any{- "kind": "navigate_tree",- "content": res.Text,- "doc_ids": []string{},- "note": "The navigation tree exists but this query routed to no document. Try a different topic/entity phrasing, or use search_chunks.",- "query": query,+ "kind": "navigate_tree",+ "note": "navigate_tree routed to no document for this query. Rephrase the query, or use search_chunks / retrieve instead.",}},- Status: StatusMiss,- Reason: ReasonNoDoc,- Metrics: map[string]any{"docs": 0},+ Status: StatusMiss,+ Reason: ReasonNoDoc,}, nilcase ReasonInfra, ReasonBadArgs:return ToolOutcome{@@ -306,10 +291,15 @@ func (e *searchExecutor) navigateTree(ctx context.Context, args map[string]any)}, nil}- // Routed: expose the summary-bearing payload the ladder consumes.+ // Routed: expose the summary-bearing payload the ladder consumes. Python+ // :995 caps the content at 8000 chars.+ content := res.Text+ if len(content) > 8000 {+ content = content[:8000]+ }payload := []any{map[string]any{"kind": "navigate_tree",- "content": res.Text,+ "content": content,"doc_ids": res.DocIDs,}}return ToolOutcome{@@ -407,13 +397,13 @@ func (e *searchExecutor) navigateStructure(ctx context.Context, args map[string]docIDs = docIDs[:navTreeMaxDocs]}if len(docIDs) == 0 {- // Routing reached no document (Python: empty_reason="no_doc").+ // Routing reached no document (Python: empty_reason="no_doc"). Python+ // :1037 emits the SAME note for every empty_reason — text verbatim.return ToolOutcome{Payload: []any{map[string]any{- "kind": "navigate_structure",- "note": "The query routed to no document in this dataset. Try a different topic/entity phrasing, pass doc_id, or use search_chunks.",- "query": query,- "doc_ids": []string{},+ "kind": "navigate_structure",+ "doc_id": docID,+ "note": fmt.Sprintf("No compiled structure of kind='%s' reachable for this document. Try another doc_id or kind, or use search_chunks / retrieve / list_chunks.", kind),}},Status: StatusMiss,Reason: ReasonNoDoc,@@ -431,13 +421,16 @@ func (e *searchExecutor) navigateStructure(ctx context.Context, args map[string]"chunk_ptrs": drill.chunkPtrs,"top_score": drill.topScore,"chunk_paths": drill.chunkPaths,+ "claim_hits": drill.claimHits,}if res.EmptyReason != "" {+ // Python :1037 — one note for every empty_reason; kind renders with+ // Python repr() quoting (kind!r).return ToolOutcome{Payload: []any{map[string]any{- "kind": "navigate_structure",- "note": fmt.Sprintf("No compiled structure of kind=%q reachable for the located document(s). Try another kind, or use search_chunks / retrieve / list_chunks.", kind),- "doc_ids": docIDs,+ "kind": "navigate_structure",+ "doc_id": docID,+ "note": fmt.Sprintf("No compiled structure of kind='%s' reachable for this document. Try another doc_id or kind, or use search_chunks / retrieve / list_chunks.", kind),}},Status: ReasonStatus(res.EmptyReason),Reason: res.EmptyReason,@@ -445,9 +438,11 @@ func (e *searchExecutor) navigateStructure(ctx context.Context, args map[string]}, nil}// Reached structures but drilled to nothing usable: not an error, just weak —- // the orchestrator falls back to retrieval on status == poor.+ // the orchestrator falls back to retrieval on status == poor. Claim-first+ // hits are the exception: the claims carry the answer material and load no+ // chunk snippets, so a zero chunk_ptr count with matched claims is a success.status := StatusOK- if drill.chunkPtrs == 0 {+ if drill.chunkPtrs == 0 && drill.claimHits == 0 {status = StatusPoor}content := res.Text@@ -468,19 +463,29 @@ func (e *searchExecutor) navigateStructure(ctx context.Context, args map[string]}// navRouter returns the navigation-tree router navigateTree and navigateStructure-// share (mirrors Python's _NAV_TREE_ROUTER = "chunk_agg"; a nil retrieval backend-// degrades to the nav-row router, which needs no backend).+// share. It mirrors Python's agentic router choice: the navigation-tree route+// asks for router="claim_agg" (dataset_api_service.search_dataset_layers) — the+// claim leg runs first and decides the ranking when it hits, and raw-chunk+// aggregation (chunk_agg) is the fallback. A nil retrieval backend degrades to+// the nav-row router, which needs no backend.func (e *searchExecutor) navRouter() NavTreeRouter {if e.deps.NavRouter != nil {return e.deps.NavRouter}- if e.deps.Backend != nil {- return nav.NewChunkAggRouter(- chunkAggRetrieveFrom(e.deps.Backend),- nav.ChunkAggSummarize(),- )+ return defaultNavRouter(e.deps)+}++// defaultNavRouter builds the claim_agg router over the chunk_agg fallback.+// See ClaimAggRouter and nav.NewChunkAggRouter.+func defaultNavRouter(deps SearchDeps) NavTreeRouter {+ if deps.Backend == nil {+ return nav.NewNavServiceRouter()+ }+ return &ClaimAggRouter{+ Deps: deps,+ Fallback: nav.NewChunkAggRouter(chunkAggRetrieveFrom(deps.Backend), nav.ChunkAggSummarize()),+ Summarize: nav.ChunkAggSummarize(),}- return nav.NewNavServiceRouter()}func firstOr(xs []string, def string) string {@@ -506,11 +511,17 @@ func argString(args map[string]any, key string) string {}// evidencePoolCap is the hard cap on the shared evidence pool-// (kbinfos["chunks"]). Mirrors Python _EVIDENCE_POOL_CAP (=60, the same ceiling-// as _MAX_SNIPPET_POOL / _SCA_VIEW_CAP): once the pool is saturated, further-// admits cannot reach the SCA view or improve the answer, so the action session-// stops admitting chunks (observed pools otherwise grew to ~106).-const evidencePoolCap = 60+// (kbinfos["chunks"]). Mirrors Python _EVIDENCE_POOL_CAP (=120,+// action_session.py:62): deliberately LARGER than _SCA_VIEW_CAP (=60) so+// storage and review stay DECOUPLED — the pool accumulates while the SCA reads+// a ranked top-60 view. Coupling them at 60 starved the raw-evidence channel in+// 42% of rounds (every admit rejected -> status REDUNDANT -> the model+// re-searched for nothing).+//+// Claim pseudo-chunks BYPASS this cap: Python's _claim_prefetch appends them+// directly to kbinfos["chunks"] (:755), and the cap check (:657) only guards+// _admit_evidence's regular chunks.+const evidencePoolCap = 120// The cap check and its "pool FULL" line now live on PoolAdmitter.Full, where// the pool lock is held (see kbinfos.go): the check must not read len(Chunks)@@ -567,6 +578,51 @@ func (e *searchExecutor) search(ctx context.Context, name string, args map[strin// pool — a per-call snapshot of it (Python's `kb_seen`) is exact only while// nothing can interleave, and another session appending makes it stale.seen := map[string]bool{}+ // Claim-first, MUTUALLY EXCLUSIVE (Python _run_search: _claim_prefetch ++ // _CLAIM_PREFETCH_EXCLUSIVE): when claim rows hit, their verbatim evidence+ // IS the answer material — chunk snippets on top would echo the same+ // passages and burn tokens. Claims carry chunk pointers, so deep-reading+ // stays one list_chunks away. No hits → the chunk search runs exactly as+ // before. Applies to the whole retrieve family + search_chunks, matching+ // Python's _exec_retrieve/_exec_search_chunks (both route through+ // _run_search). Best effort: any failure falls through without claims.+ if name == "retrieve" || name == "search_chunks" || strings.HasPrefix(name, "grep") {+ if len(queries) > 0 {+ if claimPayload, _, claimPseudo, ok := ClaimPrefetch(ctx, e.deps, queries[0], seen); ok {+ newEvidence := 0+ e.deps.KB.Admit(func(p *PoolAdmitter) {+ for _, pc := range claimPseudo {+ // Claim pseudo-chunks BYPASS the pool cap: Python+ // _claim_prefetch appends them directly to+ // kbinfos["chunks"] (:755) and the cap check (:657)+ // only guards regular chunk admits — the verbatim+ // evidence must land in the pool even when it is FULL.+ cid := ChunkIDOf(pc)+ if seen[cid] {+ continue+ }+ seen[cid] = true+ evidenceIDs = append(evidenceIDs, cid)+ payload = append(payload, passageFromChunk(pc))+ if p.Add(pc) {+ newEvidence+++ }+ }+ })+ status := StatusOK+ if newEvidence == 0 {+ status = StatusRedundant+ }+ return ToolOutcome{+ Payload: payload,+ EvidenceIDs: evidenceIDs,+ Status: status,+ Reason: ReasonNone,+ Metrics: map[string]any{"hits": len(claimPayload), "new_evidence": newEvidence, "claims": len(claimPayload)},+ }, nil+ }+ }+ }for _, q := range queries {// Per-tool top_n (mirrors Python action_session: retrieve=10,// search_chunks=20). They were previously collapsed onto e.req.TopN,@@ -645,6 +701,10 @@ func (e *searchExecutor) search(ctx context.Context, name string, args map[strin// cannot interleave two sessions' batches. Locking per chunk would let// them interleave into pool orders Python can never produce.e.deps.KB.Admit(func(p *PoolAdmitter) {+ // Python _admit_evidence computes _claim_covered_ids(kbinfos) from the+ // LIVE pool once per call; compute it once per batch under the same+ // critical section.+ covered := p.ClaimCoveredIDs()for _, c := range chunks {// Python _admit_evidence early-stops at the top once the shared pool// reaches the cap, BEFORE the per-call dedup.@@ -655,6 +715,12 @@ func (e *searchExecutor) search(ctx context.Context, name string, args map[strinif seen[cid] {continue}+ // Python :672-676 — already quoted verbatim by a pooled claim →+ // skip the full passage (table chunks exempt: their answer rows+ // survive only in full text). Not pooled, not passed to the model.+ if p.CoveredByClaim(cid, covered, IsTableChunk(c)) {+ continue+ }seen[cid] = trueevidenceIDs = append(evidenceIDs, cid)payload = append(payload, passageFromChunk(c))@@ -951,10 +1017,11 @@ func (r *RuntimeRetriever) Retrieve(ctx context.Context, req RetrieveRequest) ([// stay nil so the retrieval service keeps its own default, while a zero// is a real override. Taking the address of a zero value here forced// "threshold 0 / full vector weight" onto every caller that omitted them.- SimilarityThreshold: req.SimilarityThreshold,- KeywordsSimilarityWeight: req.KeywordsSimilarityWeight,- TenantID: req.TenantID,- RankFeature: req.RankFeature,+ SimilarityThreshold: req.SimilarityThreshold,+ VectorSimilarityWeight: req.VectorSimilarityWeight,+ DisableVectorLeg: req.DisableVectorLeg,+ TenantID: req.TenantID,+ RankFeature: req.RankFeature,// ExcludeCompiled maps Python hybrid_search's// must_not={"exists":"compile_kwd"} onto the runtime request's// OnlyOriginalText (the "no compile_kwd" exclusion).diff --git a/internal/rag/advanced_rag/harness/tool_executor_test.go b/internal/rag/advanced_rag/harness/tool_executor_test.goindex 50e93107a1..ed444bf245 100644--- a/internal/rag/advanced_rag/harness/tool_executor_test.go+++ b/internal/rag/advanced_rag/harness/tool_executor_test.go@@ -73,24 +73,24 @@ func TestRuntimeRetrieverPreservesUnsetControls(t *testing.T) {if _, err := r.Retrieve(context.Background(), RetrieveRequest{Query: "q", DatasetIDs: []string{"kb-1"}}); err != nil {t.Fatalf("Retrieve: %v", err)}- if got.SimilarityThreshold != nil || got.KeywordsSimilarityWeight != nil {+ if got.SimilarityThreshold != nil || got.VectorSimilarityWeight != nil {t.Errorf("omitted controls = %v / %v, want nil so the service keeps its defaults",- got.SimilarityThreshold, got.KeywordsSimilarityWeight)+ got.SimilarityThreshold, got.VectorSimilarityWeight)}threshold, weight := 0.35, 0.3if _, err := r.Retrieve(context.Background(), RetrieveRequest{- Query: "q",- SimilarityThreshold: &threshold,- KeywordsSimilarityWeight: &weight,+ Query: "q",+ SimilarityThreshold: &threshold,+ VectorSimilarityWeight: &weight,}); err != nil {t.Fatalf("Retrieve: %v", err)}if got.SimilarityThreshold == nil || *got.SimilarityThreshold != threshold {t.Errorf("SimilarityThreshold = %v, want %v", got.SimilarityThreshold, threshold)}- if got.KeywordsSimilarityWeight == nil || *got.KeywordsSimilarityWeight != weight {- t.Errorf("KeywordsSimilarityWeight = %v, want %v", got.KeywordsSimilarityWeight, weight)+ if got.VectorSimilarityWeight == nil || *got.VectorSimilarityWeight != weight {+ t.Errorf("VectorSimilarityWeight = %v, want %v", got.VectorSimilarityWeight, weight)}}@@ -108,9 +108,9 @@ func TestChunkAggRetrieveLeavesControlsUnset(t *testing.T) {t.Fatalf("chunks = %d, want the retrieved chunk", len(chunks))}req := r.lastReq(t)- if req.SimilarityThreshold != nil || req.KeywordsSimilarityWeight != nil {+ if req.SimilarityThreshold != nil || req.VectorSimilarityWeight != nil {t.Errorf("controls = %v / %v, want nil (zero is a valid value, not an unset marker)",- req.SimilarityThreshold, req.KeywordsSimilarityWeight)+ req.SimilarityThreshold, req.VectorSimilarityWeight)}}@@ -461,7 +461,9 @@ func TestEvidencePoolCapStopsAdmitting(t *testing.T) {func TestWebSearchAdmitsToPool(t *testing.T) {deps, kb := newTestSearchDeps(&stubRetriever{})deps.WebSearch = stubWebSearch{results: []string{"web answer one", "web answer two", "web answer three"}}- oc, err := WebSearchTool(context.Background(), deps, map[string]any{"": []any{"q1", "q2"}})+ // Python action_session.execute_tool reads the query list from+ // args["query"] (_arg_query_list, :938-944, :1159).+ oc, err := WebSearchTool(context.Background(), deps, map[string]any{"query": []any{"q1", "q2"}})if err != nil {t.Fatalf("web_search: %v", err)}@@ -491,7 +493,8 @@ func TestWebSearchAdmitsToPool(t *testing.T) {func TestWebSearchDedupsAcrossQueries(t *testing.T) {deps, kb := newTestSearchDeps(&stubRetriever{})deps.WebSearch = stubWebSearch{results: []string{"dup passage", "unique one", "dup passage"}}- oc, err := WebSearchTool(context.Background(), deps, map[string]any{"": []any{"q1", "q2"}})+ // Same args["query"] contract as above (Python _arg_query_list).+ oc, err := WebSearchTool(context.Background(), deps, map[string]any{"query": []any{"q1", "q2"}})if err != nil {t.Fatalf("web_search: %v", err)}diff --git a/internal/rag/advanced_rag/harness/tool_navigation.go b/internal/rag/advanced_rag/harness/tool_navigation.goindex 0dbae84d48..7014f6f26e 100644--- a/internal/rag/advanced_rag/harness/tool_navigation.go+++ b/internal/rag/advanced_rag/harness/tool_navigation.go@@ -43,7 +43,11 @@ import (// Structure navigation (ontology_navigate / mindmap_navigate)// ----------------------------------------------------------------------------var catalogKinds = map[string]bool{"tree": true, "timeline": true, "raptor": true, "page_index": true, "pageindex": true}+// Python _CATALOG_KINDS (navigation.py:49): {"tree", "timeline", "page_index",+// "pageindex"} — raptor is NOT a catalog kind (its stale docstring comment+// notwithstanding, the set itself excludes it; upper RAPTOR clusters carry no+// per-node vectors and drill into them is the claim leg's job).+var catalogKinds = map[string]bool{"tree": true, "timeline": true, "page_index": true, "pageindex": true}var mindmapKinds = map[string]bool{"mindmap": true, "mind_map": true}const navSystemPrompt = `You are given the {noun} of one or more documents — an outline of entities and their relations — and a question.@@ -476,6 +480,13 @@ func NavigateTree(ctx context.Context, router NavTreeRouter, in NavTreeInput) Na// read found zero entities (:1046).return navEmpty(ReasonInfra, "nav tree descent failed")}+ // No compiled tree is a DATASET-level fact: Python's nav-tree route+ // deliberately has NO fallback here (navigation.py:902-912 — the generic+ // retriever re-inventing dataset_nav's work was the most expensive thing+ // that path could do). When routing misses, the CALLER falls back to+ // retrieve/search_chunks — the same work, done once and owned by the+ // orchestrator instead of hidden inside a "route" call. The tool's empty+ // verdict lets the session disable the tool.// Every dataset lacks a compiled tree — a DATASET-level fact, so the// caller may disable the tool for the session.return navEmpty(ReasonNoStructure, "no compiled navigation tree")@@ -483,8 +494,11 @@ func NavigateTree(ctx context.Context, router NavTreeRouter, in NavTreeInput) Naif len(ordered) == 0 {// Structure exists but THIS query reached nothing: a query-level miss.// The dataset may still have a tree a better-formed query would hit.- // No error attribute: Python's query-level miss carries none- // (navigation.py:899), unlike infra/bad_args.+ // Python deliberately does NOT back this miss with a retrieval fallback+ // (navigation.py:902-912): it returns count="0" / no_doc and leaves the+ // fallback to the caller's retrieve/search_chunks.+ // No error attribute on the empty verdict: Python's query-level miss+ // carries none (navigation.py:912), unlike infra/bad_args.return navEmpty(ReasonNoDoc, "")}@@ -659,15 +673,16 @@ func objectList(v any) []map[string]any {// navigate-tree / navigate-structure algorithm, mirroring how navigation.py owns// its orchestration while importing its primitives.-// Structure drill-down bounds (mirror navigation.py's module-level limits).+// Structure drill-down bounds (mirror navigation.py's module-level limits,+// navigation.py:1084-1097).const (- structMaxDepth = 12 // _STRUCT_MAX_DEPTH: stop walking past this depth- structMaxNodes = 300 // _STRUCT_MAX_NODES: cap on nodes kept per level- structBranchK = 12 // _STRUCT_BRANCH_K: children kept per node- structVecBeamRatio = 0.65 // _STRUCT_VEC_BEAM_RATIO: score floor relative to best- structRelevanceMin = 1 // min keyword relevance to keep a node without a vector- structDescSnippet = 300 // snippet length for a node's description in the outline- structMaxChunks = 6 // _STRUCT_MAX_CHUNKS: top chunks exposed per drilled doc+ structMaxDepth = 3 // _STRUCT_MAX_DEPTH: max TOC levels drilled+ structMaxNodes = 10 // _STRUCT_MAX_NODES: cap on nodes rendered in the outline+ structBranchK = 2 // _STRUCT_BRANCH_K: keep top-K most relevant nodes per level+ structVecBeamRatio = 0.5 // _STRUCT_VEC_BEAM_RATIO: score floor relative to best+ structRelevanceMin = 1 // a node must match at least this many query terms to descend+ structDescSnippet = 180 // _STRUCT_DESC_SNIPPET: cap on a node's description snippet+ structMaxChunks = 4 // _STRUCT_MAX_CHUNKS: cap on chunk snippets returned in the outlinestructCatalogKinds = "tree_node|page_index|section" // kinds eligible for catalog outline// structTocMaxDepth bounds the ancestor walk in renderTocDrilldown's@@ -681,6 +696,32 @@ const (// (RAPTOR blob path). Mirror _STRUCT_RECALL_TOP_N and _STRUCT_MAX_CHUNK_HITS.structRecallTopN = 24 // _STRUCT_RECALL_TOP_N: hybrid chunk hits recalled per documentstructMaxChunkHits = 8 // _STRUCT_MAX_CHUNK_HITS: retrieved chunks shown per drilled doc++ // Claim/evidence leg for the drill-down (Python navigation.py:1129-1146).+ // Claim rows are scored FLAT against the query (hybrid BM25+KNN, RRF), so a+ // bad early pick cannot prune the subtree holding the answer. Claim-first:+ // when claims hit they REPLACE the drill-down — a claim is an atomic+ // proposition carrying its own verbatim evidence, so a hit already is the+ // answer material; the tree only labels where each claim sits.+ structClaimLeg = true // _STRUCT_CLAIM_LEG+ structClaimTopN = 8 // _STRUCT_CLAIM_TOP_N: claims rendered per document+ structClaimFirst = true // _STRUCT_CLAIM_FIRST+ structClaimFirstMinHits = 1 // _STRUCT_CLAIM_FIRST_MIN_HITS+ // structClaimSufficientHits is _STRUCT_CLAIM_SUFFICIENT_HITS (Python+ // navigation.py:1153): enough distinct claims matching the query means the+ // model may cite them without deep-reading chunks. The drillout emits+ // <claims_sufficient/> when a document's claim hits reach it.+ structClaimSufficientHits = 3++ // Per-node-vector flat selection (Python navigation.py:1307-1315 ++ // :1912-1936): page_index spreads facts across TOC levels, so descending+ // level by level can prune the branch holding the answer before it is ever+ // scored. Scoring EVERY node against the query at once costs nothing extra+ // (their embeddings are already loaded) and cannot prune — this is the+ // DEFAULT primary path for page_index structures.+ structTocLLMSelect = false // _STRUCT_TOC_LLM_SELECT (navigation.py:1111): the one-shot LLM TOC selection is OFF in Python — not ported+ structFlatNodeSelect = true // _STRUCT_FLAT_NODE_SELECT (navigation.py:1157)+ structFlatTopN = 10 // _STRUCT_FLAT_TOP_N (navigation.py:1158))// queryTerms mirrors navigation.py's regex tokenizer: lowercase coarse word@@ -896,6 +937,43 @@ func hasDistinctNodeVectors(nodes []structureNode) bool {return false}+// flatSelectNodes mirrors navigation.py _flat_select_nodes: score EVERY node+// against the query at once and return the top-N names. page_index spreads+// facts across TOC levels, so descending level by level can prune the branch+// holding the answer before it is ever scored; scoring all nodes costs nothing+// extra (their embeddings are already loaded) and cannot prune. Returns nil+// when no query vector is available: the caller then falls back to the beam+// descent, which also works from keyword overlap.+func flatSelectNodes(qvec []float64, nodes []structureNode, topN int) []string {+ if len(qvec) == 0 || len(nodes) == 0 || topN <= 0 {+ return nil+ }+ type scored struct {+ name string+ score float64+ }+ var candidates []scored+ for _, n := range nodes {+ name := strings.TrimSpace(n.name)+ if name == "" || len(n.vec) == 0 {+ continue+ }+ candidates = append(candidates, scored{name: name, score: cosine(qvec, n.vec)})+ }+ if len(candidates) == 0 {+ return nil+ }+ sort.Slice(candidates, func(i, j int) bool { return candidates[i].score > candidates[j].score })+ if len(candidates) > topN {+ candidates = candidates[:topN]+ }+ out := make([]string, 0, len(candidates))+ for _, c := range candidates {+ out = append(out, c.name)+ }+ return out+}+// nodesCoveringChunks returns the names of nodes whose source_chunk_ids cover any// of chunkIDs, mirroring navigation.py _nodes_covering_chunks. Used only to give// retrieved chunks a structural home; a chunk no node covers is still shown.@@ -1168,6 +1246,9 @@ type structureDrillout struct {topScore float64chunkPaths map[string]stringselector string+ // claimHits is the count of claim rows that matched the query for this+ // drill (Python stats["claim_hits"], set by the caller when non-empty).+ claimHits int}// chunkHit is a retrieved chunk id with its retrieval score, the input shape of@@ -1219,8 +1300,13 @@ func drillWithAncestors(names map[string]bool, parents map[string]string) map[st// renderTocDrilldown mirrors navigation.py _render_toc_drilldown: it renders a// query-focused outline of one document's compiled structure, choosing the nodes-// to show by one of three strategies in priority order:+// to show by one of four strategies in priority order://+// - claimHits (claim-first, non-empty and >= structClaimFirstMinHits): claims+// matched the query directly. They are rendered as statement + verbatim+// evidence and take over the selection outright — nothing is drilled, no+// chunk snippets are loaded, and the tree only labels where each claim sits+// — selector "claim".// - selected (non-empty): node names picked by the whole-TOC LLM pass, kept// whole with their ancestors — selector "llm_toc".// - chunkHits (non-empty): [(id, score)] from chunk retrieval (RAPTOR blob path);@@ -1232,7 +1318,7 @@ func drillWithAncestors(names map[string]bool, parents map[string]string) map[st// When no terms/vector and neither selection is provided it falls back to the flat// outline. loader, when non-nil, fetches chunk texts so short snippets can be// appended; a nil loader skips the snippet pass.-func renderTocDrilldown(query string, qvec []float64, nodes []structureNode, rels []structureRel, loader func(ids []string) []chunkWithText, chunkHits []chunkHit, selected []string) structureDrillout {+func renderTocDrilldown(query string, qvec []float64, nodes []structureNode, rels []structureRel, loader func(ids []string) []chunkWithText, chunkHits []chunkHit, selected []string, claimHits []DocClaimHit) structureDrillout {flat := func() structureDrillout {out := renderOutline(nodes, rels)n, p := outlineStats(nodes)@@ -1252,8 +1338,39 @@ func renderTocDrilldown(query string, qvec []float64, nodes []structureNode, relvar selectedIDs []stringbest := 0.0selector := "beam"+ claimFirst := structClaimFirst && len(claimHits) >= structClaimFirstMinHitsswitch {+ case claimFirst:+ // Claims already carry the answer material (statement + verbatim+ // evidence), so nothing is drilled: the tree's only remaining job is to+ // show where each claim sits. Descending would re-derive the same facts+ // and risk pruning past them (Python navigation.py:2183-2193).+ var claimCIDs []string+ for _, h := range claimHits {+ if cid := strings.TrimSpace(h.ChunkID); cid != "" && !sliceContains(claimCIDs, cid) {+ claimCIDs = append(claimCIDs, cid)+ }+ }+ covering := nodesCoveringChunks(nodes, claimCIDs)+ names := make(map[string]bool)+ for _, n := range covering {+ if _, ok := byName[n]; ok {+ names[n] = true+ }+ }+ keptNames = drillWithAncestors(names, parents)+ for _, n := range sortedKeptNames(keptNames) {+ if e, ok := byName[n]; ok {+ kept = append(kept, e)+ }+ }+ for _, h := range claimHits {+ if h.Score > best {+ best = h.Score+ }+ }+ selector = "claim"case len(selected) > 0:names := make(map[string]bool)for _, n := range selected {@@ -1300,8 +1417,11 @@ func renderTocDrilldown(query string, qvec []float64, nodes []structureNode, relkept, parents, keptNames, best = drillKeptNodes(qvec, terms, nodes, rels)selector = "beam"}- if len(kept) == 0 && len(selectedIDs) == 0 {+ if len(kept) == 0 && len(selectedIDs) == 0 && !(structClaimFirst && len(claimHits) > 0) {// A strategy was chosen but kept nothing; still report which one ran.+ // Claim-first is the exception (Python navigation.py:2215-2219): upper+ // RAPTOR clusters carry no source_chunk_ids, so no node may cover a hit+ // claim — bailing out here would throw the claims away.out := flat()out.selector = selectorreturn out@@ -1360,12 +1480,59 @@ func renderTocDrilldown(query string, qvec []float64, nodes []structureNode, relindent := strings.Repeat(" ", depthOf[name])lines = append(lines, navOutlineLine(indent, name, e.nodeType, e.desc, e.sourceChunkIDs))}+ // Claim/evidence lines come before the chunk snippets (Python+ // navigation.py:2264-2283): a gated claim is a verified atomic fact carrying+ // its verbatim quote, so it reads better than a truncated chunk and needs no+ // chunk load at all. Its chunk is dropped from the snippet list below so the+ // same content is not sent twice.+ claimed := map[string]bool{}+ for _, hit := range claimHits {+ cid := strings.TrimSpace(hit.ChunkID)+ if cid == "" || claimed[cid] {+ continue+ }+ claimed[cid] = true+ line := "- [claim] " + strings.TrimSpace(hit.Name)+ if quote := docClaimQuote(hit); quote != "" {+ line += " | Evidence: \"" + Snippet(quote, ClaimEvidenceChars) + "\""+ }+ line += " [chunks: " + cid + "]"+ lines = append(lines, line)+ }+ // A claim's chunk still deserves a structural label — annotate it from the+ // node that covers it, falling back to nothing when no node does (upper+ // RAPTOR clusters carry no source_chunk_ids, so this stays honest rather+ // than inventing a path) (Python navigation.py:2285-2294).+ for cid := range claimed {+ if _, ok := chunkPaths[cid]; ok {+ continue+ }+ if owners := nodesCoveringChunks(nodes, []string{cid}); len(owners) > 0 {+ chunkPaths[cid] = owners[0]+ }+ }// Chunk snippets. When chunk retrieval did the selecting, the hits are shown// as-is (filtering them back through the nodes would drop chunks no node- // covers); otherwise take the chunks behind the drilled nodes.- wanted := collectChunkIDs(kept, 32)- if len(selectedIDs) > 0 {+ // covers); otherwise take the chunks behind the drilled nodes. Under+ // claim-first a matched claim already renders its verbatim evidence, so+ // nothing is loaded — loading the chunk behind it would echo the same+ // passage a second time (Python navigation.py:2300-2303).+ var wanted []string+ if claimFirst {+ wanted = nil+ } else if len(selectedIDs) > 0 {wanted = selectedIDs+ } else {+ wanted = collectChunkIDs(kept, 32)+ }+ if len(wanted) > 0 && len(claimed) > 0 {+ filtered := make([]string, 0, len(wanted))+ for _, c := range wanted {+ if !claimed[c] {+ filtered = append(filtered, c)+ }+ }+ wanted = filtered}if len(wanted) > 0 && loader != nil {chunks := loader(wanted)@@ -1405,6 +1572,7 @@ func renderTocDrilldown(query string, qvec []float64, nodes []structureNode, reltopScore: best,chunkPaths: chunkPaths,selector: selector,+ claimHits: len(claimHits),}}@@ -1515,14 +1683,49 @@ func readStructureDocCore(ctx context.Context, tenantID, query, docID, kind stri}loader := func(ids []string) []chunkWithText { return structureNodeLoader(ctx, indexName, ids) }+ // Claim leg (Python _read_structures:1294-1306): flat hybrid claim recall+ // for this document. Selection priority: claims when they hit, then+ // whatever the tree's own shape supports.+ var claimHits []DocClaimHit+ if structClaimLeg {+ kindList := make([]string, 0, len(kinds))+ for k := range kinds {+ kindList = append(kindList, k)+ }+ sort.Strings(kindList)+ claimHits = RecallDocClaimHits(ctx, deps, query, docID, kindList, qvec, structClaimTopN)+ }+ // Claim-first: the claims decide, so nothing is drilled and no chunks are+ // recalled to choose (Python _read_structures:1303-1306).+ claimFirst := structClaimFirst && len(claimHits) >= structClaimFirstMinHits+// RAPTOR / shared-vector blobs cannot score their nodes against the query, so// they defer to chunk retrieval for the drill (mirrors Python routing them to- // _recall_chunk_ids_in_doc). Per-node-vector structures keep the beam drill.+ // _recall_chunk_ids_in_doc). Per-node-vector structures take the FLAT node+ // selection instead of the beam drill (Python navigation.py:1307-1315).+ var selected []stringvar chunkHits []chunkHit- if !hasDistinctNodeVectors(nodes) {+ if !claimFirst && hasDistinctNodeVectors(nodes) {+ // Python :1312-1315 — _STRUCT_TOC_LLM_SELECT is False (navigation.py:1111),+ // so the one-shot LLM TOC selection never runs and the flat vector scoring+ // is the default primary path. It renders under selector="llm_toc"+ // (renderTocDrilldown's selected branch); an empty result (no query vector)+ // falls back to the beam descent, which also works from keyword overlap.+ if structFlatNodeSelect {+ selected = flatSelectNodes(qvec, nodes, structFlatTopN)+ }+ } else if !claimFirst {chunkHits = recallChunkIDsInDoc(ctx, deps.Backend, query, docID, tenantID, deps.KbIDs, 0)}- drill := renderTocDrilldown(query, qvec, nodes, rels, loader, chunkHits, nil)+ drill := renderTocDrilldown(query, qvec, nodes, rels, loader, chunkHits, selected, claimHits)+ if len(claimHits) > 0 {+ // Mirror the rendered claims into the shared evidence pool (Python+ // _publish_claim_hits): the SCA, the slot prefill and the final compose+ // all read ONLY the pool, so a claim that already states the fact must+ // land there.+ drill.claimHits = len(claimHits)+ publishClaimHits(deps, claimHits, docID)+ }return drill, nodes, rels, ""}@@ -1536,6 +1739,11 @@ func structureDocSegment(docID, query, kind string, rank int, nodes []structureNif drill.outline != "" {b.WriteString("\n <structure>" + esc(drill.outline) + "</structure>")}+ // Python navigation.py:1052-1053 — enough matched claims mark the document's+ // evidence as citable without a list_chunks deep-read.+ if drill.claimHits >= structClaimSufficientHits {+ b.WriteString("\n <claims_sufficient/>")+ }b.WriteString("\n </doc>")return b.String()}@@ -1604,6 +1812,7 @@ func navigateStructures(ctx context.Context, tenantID, query string, docIDs []stentities = append(entities, entityMaps(nodes)...)agg.nodes += drill.nodesagg.chunkPtrs += drill.chunkPtrs+ agg.claimHits += drill.claimHitsif drill.topScore > agg.topScore {agg.topScore = drill.topScore}diff --git a/internal/rag/advanced_rag/harness/tool_navigation_test.go b/internal/rag/advanced_rag/harness/tool_navigation_test.goindex a6c9374092..7552e79b97 100644--- a/internal/rag/advanced_rag/harness/tool_navigation_test.go+++ b/internal/rag/advanced_rag/harness/tool_navigation_test.go@@ -915,7 +915,7 @@ func drillLoader(ids []string) []chunkWithText {func TestRenderTocDrilloutLLMSelect(t *testing.T) {nodes, rels := renderDrillNodes()- out := renderTocDrilldown("some query", nil, nodes, rels, drillLoader, nil, []string{"Leaf"})+ out := renderTocDrilldown("some query", nil, nodes, rels, drillLoader, nil, []string{"Leaf"}, nil)if out.selector != "llm_toc" {t.Errorf("selector = %q, want llm_toc", out.selector)}@@ -931,7 +931,7 @@ func TestRenderTocDrilloutLLMSelect(t *testing.T) {func TestRenderTocDrilloutChunkRetrieval(t *testing.T) {nodes, rels := renderDrillNodes()hits := []chunkHit{{id: "r1", score: 0.9}, {id: "l1", score: 0.7}}- out := renderTocDrilldown("q", nil, nodes, rels, drillLoader, hits, nil)+ out := renderTocDrilldown("q", nil, nodes, rels, drillLoader, hits, nil, nil)if out.selector != "chunk_retrieval" {t.Errorf("selector = %q, want chunk_retrieval", out.selector)}@@ -948,7 +948,7 @@ func TestRenderTocDrilloutBeamFallback(t *testing.T) {nodes, rels := renderDrillNodes()// No selection and no hits, but a query: beam picks by keyword relevance. The// blank query yields no terms, so it must still return a (flat) outline, not panic.- out := renderTocDrilldown("leaf", nil, nodes, rels, drillLoader, nil, nil)+ out := renderTocDrilldown("leaf", nil, nodes, rels, drillLoader, nil, nil, nil)if out.outline == "" {t.Fatal("beam path produced an empty outline")}@@ -1014,7 +1014,7 @@ func TestTocDrilldownKeptOrderIsDeterministic(t *testing.T) {t.Run("llm_toc", func(t *testing.T) {// The model's picks arrive in ITS order; it must not leak into the output.got := stableDrillout(t, func() structureDrillout {- return renderTocDrilldown("q", nil, nodes, rels, drillLoader, nil, []string{"Gamma", "Alpha", "Beta"})+ return renderTocDrilldown("q", nil, nodes, rels, drillLoader, nil, []string{"Gamma", "Alpha", "Beta"}, nil)})assertKeptNodeOrder(t, got.outline)// chunkPaths follows the same order: "shared" is covered by both Gamma and@@ -1028,9 +1028,10 @@ func TestTocDrilldownKeptOrderIsDeterministic(t *testing.T) {if !reflect.DeepEqual(got.chunkPaths, want) {t.Errorf("chunkPaths = %v, want %v", got.chunkPaths, want)}- // Snippets are capped at structMaxChunks in kept order, so "r1" (Root sorts- // last) sits outside a stable, repeatable window.- if !strings.Contains(got.outline, "[chunk g1]") || strings.Contains(got.outline, "[chunk r1]") {+ // Snippets are capped at structMaxChunks (=4, Python _STRUCT_MAX_CHUNKS)+ // in kept order, so "g1" (Gamma) and "r1" (Root sorts last) sit outside+ // the stable, repeatable window.+ if !strings.Contains(got.outline, "[chunk a1]") || strings.Contains(got.outline, "[chunk g1]") || strings.Contains(got.outline, "[chunk r1]") {t.Errorf("outline's snippet window is not the first %d chunks in kept order:\n%s",structMaxChunks, got.outline)}@@ -1039,14 +1040,14 @@ func TestTocDrilldownKeptOrderIsDeterministic(t *testing.T) {t.Run("chunk_retrieval", func(t *testing.T) {got := stableDrillout(t, func() structureDrillout {return renderTocDrilldown("q", nil, nodes, rels, drillLoader,- []chunkHit{{id: "g1", score: 0.9}, {id: "a1", score: 0.7}}, nil)+ []chunkHit{{id: "g1", score: 0.9}, {id: "a1", score: 0.7}}, nil, nil)})assertKeptNodeOrder(t, got.outline)})t.Run("beam", func(t *testing.T) {got := stableDrillout(t, func() structureDrillout {- return renderTocDrilldown("alpha beta gamma root", nil, nodes, rels, drillLoader, nil, nil)+ return renderTocDrilldown("alpha beta gamma root", nil, nodes, rels, drillLoader, nil, nil, nil)})assertKeptNodeOrder(t, got.outline)})diff --git a/internal/rag/advanced_rag/harness/tool_search.go b/internal/rag/advanced_rag/harness/tool_search.goindex ce68acaabc..df9fa80913 100644--- a/internal/rag/advanced_rag/harness/tool_search.go+++ b/internal/rag/advanced_rag/harness/tool_search.go@@ -176,15 +176,26 @@ type RetrieveRequest struct {TopN intTopK intRerankCandidatesCount int- // SimilarityThreshold and KeywordsSimilarityWeight are pointers because ZERO- // is a valid explicit setting (threshold 0 = no floor; weight 0 = the vector- // leg gets the whole weight). nil means "not supplied", so the retriever- // keeps its own default instead of being handed a zero override.+ // SimilarityThreshold is a pointer because ZERO is a valid explicit setting+ // (threshold 0 = no floor); nil means "not supplied", so the retriever keeps+ // its own default instead of being handed a zero override.SimilarityThreshold *float64- // KeywordsSimilarityWeight is named for keywords but carries the VECTOR- // weight, as in Python's vector_similarity_weight.- KeywordsSimilarityWeight *float64- TenantID string+ // VectorSimilarityWeight carries the VECTOR leg's weight directly (Python+ // vector_similarity_weight): 0.3 hybrid, 1.0 vector-only, 0.0 keyword-only.+ // It is forwarded verbatim to the retrieval backend — NO inversion. The+ // canvas-facing adapter field of the same name family+ // (agent/tool RetrievalRequest.KeywordsSimilarityWeight) means the KEYWORD+ // weight and is inverted there; conflating the two silently turned the+ // agentic hybrid leg vector-dominant (0.3 -> 0.7) and the BM25 legs into+ // pure-vector searches.+ VectorSimilarityWeight *float64+ // DisableVectorLeg mirrors Python passing embd_mdl=None: bm25_search /+ // grep_search (search.py:260-275) and RAGTools.retrieve with+ // using_embedding=False (agentic_rag.py:retrieve) run NO dense leg at all —+ // not even a weight-0 one, which would still constrain the candidate pool+ // through the KNN similarity option.+ DisableVectorLeg bool+ TenantID string// MetaDataFilter restricts retrieval to chunks whose metadata matches// (Python tools.meta_data_filter). Nil means no filtering.MetaDataFilter map[string]any@@ -577,6 +588,19 @@ type searchOpts struct {// vector_search / bm25_search pass their display labels "Vector search"// (:248) / "BM25 search" (:288). Both forms are reproduced verbatim.narrowLabel string+ // disableVector mirrors Python passing embd_mdl=None: when set the request+ // carries DisableVectorLeg and the backend runs the keyword-only branch.+ // Grep (bm25_search) always sets it; hybrid/retrieve set it from their own+ // embedder gates (HasEmbedder / UsingEmbedding).+ disableVector bool+ // rankFeature opts the entry point into the question-type tag boost+ // (rank_feature). Python: ONLY RAGTools.retrieve passes+ // rank_feature=label_question(question, self.kbs) (agentic_rag.py:668);+ // search.py's three legs call retriever.retrieval WITHOUT rank_feature+ // (:158-173 hybrid, :223-238 vector, :260-275 bm25 — grep_search delegates+ // to bm25_search, :428). Before this gate every Go leg shipped the boost,+ // re-ranking search_chunks/retrieve results differently from Python.+ rankFeature bool}// searchLogLine renders a per-leg "searching" line (search.py:hybrid_search/215/252/418):@@ -688,12 +712,14 @@ func runSearch(ctx context.Context, deps SearchDeps, p SearchParams, opts search}// 4. Retrieve.- // rank_feature (Python retrieve: rank_feature=label_question(question,- // self.kbs), agentic_rag.py:retrieve). Go computes it from the KB objects via the- // injected Tagger; nil Tagger ⇒ no boost (Python's label_question returns- // None). The result is a tag → weight map the engine understands.+ // rank_feature ONLY on the retrieve leg (Python RAGTools.retrieve:+ // rank_feature=label_question(question, self.kbs), agentic_rag.py:668).+ // search.py's hybrid/vector/bm25 legs never pass it (:158-173/:223-238/+ // :260-275), so the other entry points leave the request's RankFeature nil.+ // Go computes it from the KB objects via the injected Tagger; nil Tagger ⇒+ // no boost (Python's label_question returns None).var rankFeature map[string]float64- if deps.Tagger != nil {+ if opts.rankFeature && deps.Tagger != nil {rankFeature = deps.Tagger.LabelQuestion(ctx, effectiveQuery, deps.KBs)}// The per-query progress line is NOT emitted here: runSearch already logs@@ -708,17 +734,16 @@ func runSearch(ctx context.Context, deps SearchDeps, p SearchParams, opts searchTopK: intOrDef(deps.TopK, DefaultTopK),RerankCandidatesCount: max(intOrDef(deps.RerankCandidatesCount, DefaultRerankCandidatesCount), topN),SimilarityThreshold: &opts.threshold,- // The field is named for keywords but carries the VECTOR weight, as in- // Python's vector_similarity_weight. Whether the vector leg runs at all- // — and its default — is decided by the calling entry-point function- // (see searchOpts.weight), NOT by a shared channel flag. ExcludeCompiled- // mirrors Python hybrid_search's must_not={"exists":"compile_kwd"}- // (search.py:hybrid_search), which keeps compiled products out of plain retrieval.- KeywordsSimilarityWeight: &opts.weight,- TenantID: deps.TenantID,- MetaDataFilter: deps.MetaDataFilter,- RankFeature: rankFeature,- ExcludeCompiled: opts.excludeCompiled,+ // VectorSimilarityWeight carries the VECTOR weight verbatim (Python+ // vector_similarity_weight); the canvas adapter's keyword-weight+ // inversion does NOT apply to this field. DisableVectorLeg mirrors+ // Python passing embd_mdl=None (no dense leg at all).+ VectorSimilarityWeight: &opts.weight,+ DisableVectorLeg: opts.disableVector,+ TenantID: deps.TenantID,+ MetaDataFilter: deps.MetaDataFilter,+ RankFeature: rankFeature,+ ExcludeCompiled: opts.excludeCompiled,})if err != nil {// Go-only line: Python lets the retriever's exception propagate untagged.@@ -791,11 +816,15 @@ func HybridSearch(ctx context.Context, deps SearchDeps, p SearchParams) ([]map[sthreshold: floatOrDef(deps.SimilarityThreshold, DefaultSimilarityThreshold),excludeCompiled: true,promoteChildren: true,- logLabel: "Hybrid search",- logVerb: "Searching the knowledge base for",- logKeywords: true,- logExtra: true, // Python's dedup/compiled/progress lines live here only- cache: true, // Python's search_cache is read/written here only (:136/:207)+ // Python hybrid_search passes the real embd_mdl even at weight 0; the+ // dense leg is dropped only when no embedder is configured+ // (search.py:143-145 `vector_weight = ... if embd_mdl else 0`).+ disableVector: !deps.HasEmbedder,+ logLabel: "Hybrid search",+ logVerb: "Searching the knowledge base for",+ logKeywords: true,+ logExtra: true, // Python's dedup/compiled/progress lines live here only+ cache: true, // Python's search_cache is read/written here only (:136/:207)// Python passes the SNAKE_CASE tag to _narrow_or_keep on this leg// (search.py:hybrid_search) although its own lines say "Hybrid search" — the// inconsistency is Python's and is reproduced verbatim.@@ -833,11 +862,14 @@ func BM25Search(ctx context.Context, deps SearchDeps, p SearchParams) ([]map[strthreshold: BM25SearchDefaultSimilarityThreshold,excludeCompiled: true,promoteChildren: true,- logLabel: "BM25 search",- logVerb: "Searching by keyword for",- logKeywords: true,- logExtra: false, // Python bm25_search logs nothing but its searching line (:252)- narrowLabel: "BM25 search",+ // Python bm25_search passes embd_mdl=None (search.py:260-275): keyword+ // only, NO dense leg at all — not even a weight-0 one.+ disableVector: true,+ logLabel: "BM25 search",+ logVerb: "Searching by keyword for",+ logKeywords: true,+ logExtra: false, // Python bm25_search logs nothing but its searching line (:252)+ narrowLabel: "BM25 search",})}@@ -929,6 +961,9 @@ func RetrieveSearch(ctx context.Context, deps SearchDeps, p SearchParams) ([]mapthreshold: floatOrDef(deps.SimilarityThreshold, DefaultSimilarityThreshold),excludeCompiled: false,promoteChildren: true,+ // Python RAGTools.retrieve: embd_mdl = self.embed_mdl if using_embedding+ // else None (agentic_rag.py:retrieve) — no dense leg without the flag.+ disableVector: !deps.UsingEmbedding,// Go-only identity: Python's L1 RAGTools.retrieve// logs nothing of its own, so there is no Python string to mirror. The// tag stays human-readable and the extra lines stay on, because this is@@ -938,6 +973,9 @@ func RetrieveSearch(ctx context.Context, deps SearchDeps, p SearchParams) ([]maplogKeywords: true,logExtra: true,narrowLabel: "retrieve",+ // Python RAGTools.retrieve is the ONLY entry point that passes+ // rank_feature=label_question(question, self.kbs) (agentic_rag.py:668).+ rankFeature: true,})}@@ -962,20 +1000,18 @@ func WebSearchTool(ctx context.Context, deps SearchDeps, args map[string]any) (Treturn ToolOutcome{Payload: []any{map[string]any{"kind": "web_search",- "note": "web_search is not configured in this deployment. Do not use this tool again; use the corpus tools (retrieve / search_chunks / navigate_*) instead.",+ "note": "Web search is NOT configured for this session. Do not use this tool again; use the corpus tools (retrieve / search_chunks / navigate_*) instead.",}},Status: StatusError,Reason: ReasonInfra,Metrics: map[string]any{},}, nil}- queries := toolStringList(args, "")- if len(queries) == 0 {- // Fall back to a single positional element pushed under the empty key.- if raw, ok := args[""]; ok {- queries = toolStringList(map[string]any{"q": raw}, "q")- }- }+ // Python execute_tool (:1159) — _exec_web_search(tools, _arg_query_list(args, 2)):+ // the query list is read from args["query"] (_arg_query_list, :938-944). The+ // old ""-key read never found anything, so every call with a proper+ // {"query": [...]} payload was rejected as bad_args.+ queries := toolStringList(args, "query")if len(queries) == 0 {return ToolOutcome{Payload: []any{}, Status: StatusError, Reason: ReasonBadArgs}, nil}@@ -1015,6 +1051,9 @@ func WebSearchTool(ctx context.Context, deps SearchDeps, args map[string]any) (T// has no await (the retrieval itself is awaited above it), so two sessions can// never interleave here.deps.KB.Admit(func(p *PoolAdmitter) {+ // Python's claim-coverage skip (:672-676) is provably unreachable here:+ // web passages carry synthetic "web_N" ids, which no claim's+ // source_chunk_ids can reference.for i, r := range results {if r == "" || seen[r] {continue@@ -1024,8 +1063,8 @@ func WebSearchTool(ctx context.Context, deps SearchDeps, args map[string]any) (Tif p.Full() {continue}- seen[r] = truechunkID := fmt.Sprintf("web_%d", i)+ seen[r] = truec := map[string]any{"chunk_id": chunkID,"content": r,@@ -1143,6 +1182,7 @@ func (e *searchExecutor) listChunks(ctx context.Context, args map[string]any) (T// has no await, so two sessions can never interleave here, and the pool-side// dedup must see the LIVE pool rather than a snapshot taken upfront.e.deps.KB.Admit(func(p *PoolAdmitter) {+ covered := p.ClaimCoveredIDs()for _, c := range admit {cid := ChunkIDOf(c)if cid == "" {@@ -1158,6 +1198,11 @@ func (e *searchExecutor) listChunks(ctx context.Context, args map[string]any) (Tif seen[cid] {continue}+ // Python :672-676 — already quoted verbatim by a pooled claim →+ // skip the full passage (table chunks exempt).+ if p.CoveredByClaim(cid, covered, IsTableChunk(c)) {+ continue+ }seen[cid] = trueevidenceIDs = append(evidenceIDs, cid)// Shape like _admit_evidence(include_doc_id=False): {"id","content"} with@@ -1176,6 +1221,13 @@ func (e *searchExecutor) listChunks(ctx context.Context, args map[string]any) (Tif len(payload) == 0 {return ToolOutcome{Payload: []any{}, Status: StatusMiss, Reason: ReasonNoDoc, Metrics: map[string]any{"hits": 0, "new_evidence": 0}}, nil}+ // Python _exec_list_chunks (:928-934) — a deep read COVERS its claims: once+ // the full chunk text is in the pool, the claim's 1200-char quote of the+ // same passage is duplicated tokens in every later prompt. Retire claim+ // pseudo-chunks whose source chunk was just read.+ if retired := e.deps.KB.RetireClaimsCoveredBy(evidenceIDs); retired > 0 {+ searchLogger(e.deps).Printf("[list_chunks] retired %d covered claim(s) from the evidence pool", retired)+ }status := StatusOKif newChunks == 0 {status = StatusRedundantdiff --git a/internal/rag/advanced_rag/harness/tool_search_test.go b/internal/rag/advanced_rag/harness/tool_search_test.goindex ca8ec959bf..ce6815c18f 100644--- a/internal/rag/advanced_rag/harness/tool_search_test.go+++ b/internal/rag/advanced_rag/harness/tool_search_test.go@@ -145,32 +145,54 @@ func TestHybridSearchEffectiveQueryCapsCodePoints(t *testing.T) {}}-func TestHybridSearchPassesRankFeature(t *testing.T) {- // label_question -> rank_feature must flow into the underlying retrieve- // request (Python retrieve: rank_feature=label_question(question, self.kbs),- // agentic_rag.py:retrieve). A nil Tagger must leave RankFeature empty.+func TestRankFeatureOnlyOnRetrieveLeg(t *testing.T) {+ // Python passes rank_feature ONLY from RAGTools.retrieve+ // (agentic_rag.py:668 rank_feature=label_question(question, self.kbs)).+ // search.py's three legs call retriever.retrieval WITHOUT rank_feature+ // (:158-173 hybrid, :223-238 vector, :260-275 bm25; grep_search delegates to+ // bm25_search, :428), so those requests must stay nil even with a Tagger.r := &stubRetriever{chunks: []map[string]any{{"content": "hit"}}}deps, _ := newTestSearchDeps(r)deps.KBs = []*entity.Knowledgebase{{}}deps.Tagger = stubTagger{t: t}+HybridSearch(context.Background(), deps, SearchParams{Question: "who made it?"})- if len(r.requests) != 1 {- t.Fatalf("requests = %d, want 1", len(r.requests))- }+ VectorSearch(context.Background(), deps, SearchParams{Question: "who made it?"})+ BM25Search(context.Background(), deps, SearchParams{Question: "who made it?"})+ GrepSearch(context.Background(), deps, SearchParams{Question: "who made it?"})+want := map[string]float64{"definition": 1.0, "entity": 1.0}- if got := r.requests[0].RankFeature; !reflect.DeepEqual(got, want) {- t.Errorf("RankFeature = %v, want %v", got, want)+ for i, req := range r.requests {+ if req.RankFeature != nil {+ t.Errorf("request %d (%s) RankFeature = %v, want nil (search.py legs never pass rank_feature)", i, req.Query, req.RankFeature)+ }}- // Nil Tagger -> empty rank feature (Python label_question None).+ // RAGTools.retrieve is the only leg that carries the label_question boost.r2 := &stubRetriever{chunks: []map[string]any{{"content": "hit"}}}deps2, _ := newTestSearchDeps(r2)- HybridSearch(context.Background(), deps2, SearchParams{Question: "unique query no rf"})+ deps2.KBs = []*entity.Knowledgebase{{}}+ deps2.Tagger = stubTagger{t: t}+ deps2.UsingEmbedding = true+ RetrieveSearch(context.Background(), deps2, SearchParams{Question: "who made it?"})if len(r2.requests) != 1 {t.Fatalf("requests = %d, want 1", len(r2.requests))}- if len(r2.requests[0].RankFeature) != 0 {- t.Errorf("RankFeature = %v, want empty for nil Tagger", r2.requests[0].RankFeature)+ if got := r2.requests[0].RankFeature; !reflect.DeepEqual(got, want) {+ t.Errorf("RetrieveSearch RankFeature = %v, want %v", got, want)+ }++ // Nil Tagger -> empty rank feature even on the retrieve leg (Python+ // label_question returning None).+ r3 := &stubRetriever{chunks: []map[string]any{{"content": "hit"}}}+ deps3, _ := newTestSearchDeps(r3)+ deps3.UsingEmbedding = true+ RetrieveSearch(context.Background(), deps3, SearchParams{Question: "unique query no rf"})+ if len(r3.requests) != 1 {+ t.Fatalf("requests = %d, want 1", len(r3.requests))+ }+ if len(r3.requests[0].RankFeature) != 0 {+ t.Errorf("RankFeature = %v, want empty for nil Tagger", r3.requests[0].RankFeature)}}@@ -538,7 +560,7 @@ func TestVectorSearchWeightIsOne(t *testing.T) {deps.HasEmbedder = trueVectorSearch(context.Background(), deps, SearchParams{Question: "q"})req := r.lastReq(t)- if got := ptrFloat(t, req.KeywordsSimilarityWeight); got != 1.0 {+ if got := ptrFloat(t, req.VectorSimilarityWeight); got != 1.0 {t.Errorf("vector search weight = %v, want 1.0", got)}if !req.ExcludeCompiled {@@ -553,9 +575,13 @@ func TestBM25SearchUsesZeroWeight(t *testing.T) {deps, _ := newTestSearchDeps(r)BM25Search(context.Background(), deps, SearchParams{Question: "q"})req := r.lastReq(t)- if got := ptrFloat(t, req.KeywordsSimilarityWeight); got != 0 {+ if got := ptrFloat(t, req.VectorSimilarityWeight); got != 0 {t.Errorf("bm25 search weight = %v, want 0", got)}+ // Python bm25_search passes embd_mdl=None: no dense leg at all.+ if !req.DisableVectorLeg {+ t.Error("bm25 search must disable the vector leg (embd_mdl=None)")+ }if got := ptrFloat(t, req.SimilarityThreshold); got != 0 {t.Errorf("bm25 search threshold = %v, want 0", got)}@@ -571,9 +597,12 @@ func TestGrepSearchDelegatesToBM25(t *testing.T) {deps, _ := newTestSearchDeps(r)GrepSearch(context.Background(), deps, SearchParams{Question: "q", Keywords: "kw"})req := r.lastReq(t)- if got := ptrFloat(t, req.KeywordsSimilarityWeight); got != 0 {+ if got := ptrFloat(t, req.VectorSimilarityWeight); got != 0 {t.Errorf("grep search weight = %v, want 0", got)}+ if !req.DisableVectorLeg {+ t.Error("grep search must disable the vector leg (embd_mdl=None)")+ }if !req.ExcludeCompiled {t.Error("grep search must exclude compiled rows")}@@ -692,7 +721,7 @@ func TestGrepSearchDerivesKeywordsHint(t *testing.T) {if want := "who made Culdcept? who made Culdcept"; req.Query != want {t.Errorf("query = %q, want %q (question + derived terms)", req.Query, want)}- if ptrFloat(t, req.KeywordsSimilarityWeight) != 0 || !req.ExcludeCompiled {+ if ptrFloat(t, req.VectorSimilarityWeight) != 0 || !req.ExcludeCompiled {t.Error("grep must stay keyword-only (weight 0) and exclude compiled rows")}// The derived hint also drives the narrowing stage: a prose candidate whose@@ -747,9 +776,14 @@ func TestHybridSearchExcludesCompiledAndWeightsThreeTenths(t *testing.T) {deps.HasEmbedder = trueHybridSearch(context.Background(), deps, SearchParams{Question: "q"})req := r.lastReq(t)- if got := ptrFloat(t, req.KeywordsSimilarityWeight); got != HybridSearchDefaultVectorWeight {+ if got := ptrFloat(t, req.VectorSimilarityWeight); got != HybridSearchDefaultVectorWeight {t.Errorf("hybrid search weight = %v, want %v", got, HybridSearchDefaultVectorWeight)}+ // With an embedder configured Python hybrid_search passes the real+ // embd_mdl: the dense leg RUNS at weight 0.3.+ if req.DisableVectorLeg {+ t.Error("hybrid search with an embedder must keep the vector leg")+ }if !req.ExcludeCompiled {t.Error("hybrid search must exclude compiled rows")}@@ -767,7 +801,7 @@ func TestRetrieveSearchDoesNotExcludeCompiled(t *testing.T) {if req.ExcludeCompiled {t.Error("retrieve search must NOT exclude compiled rows")}- if got := ptrFloat(t, req.KeywordsSimilarityWeight); got != DefaultHybridVectorWeight {+ if got := ptrFloat(t, req.VectorSimilarityWeight); got != DefaultHybridVectorWeight {t.Errorf("retrieve search weight = %v, want %v", got, DefaultHybridVectorWeight)}}diff --git a/internal/rag/advanced_rag/harness/tool_text_processing.go b/internal/rag/advanced_rag/harness/tool_text_processing.goindex 8ae6ba1ce4..1d1a0bc08e 100644--- a/internal/rag/advanced_rag/harness/tool_text_processing.go+++ b/internal/rag/advanced_rag/harness/tool_text_processing.go@@ -168,12 +168,6 @@ func containedInPhrase(low string, phrases map[string]struct{}) bool {return false}-// stemSuffixes mirrors Python _STEM_SUFFIXES (longest-first; first match wins).-var stemSuffixes = [][2]string{- {"ations", ""}, {"ation", ""}, {"ated", ""}, {"ates", ""}, {"ate", ""},- {"ings", ""}, {"ing", ""}, {"ies", "i"}, {"ied", "i"}, {"ed", ""}, {"es", ""}, {"s", ""},-}-func isAlphaOnly(s string) bool {for _, r := range s {if !((r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z')) {@@ -186,41 +180,13 @@ func isAlphaOnly(s string) bool {// stemmable mirrors Python _stemmable: len>=4 and purely ASCII letters.func stemmable(token string) bool { return len(token) >= 4 && isAlphaOnly(token) }-func isVowelRune(r rune) bool {- switch r {- case 'a', 'e', 'i', 'o', 'u', 'A', 'E', 'I', 'O', 'U':- return true- }- return false-}--// fallbackStem mirrors Python _fallback_stem.-func fallbackStem(word string) string {- w := word- for _, suf := range stemSuffixes {- s, rep := suf[0], suf[1]- if strings.HasSuffix(w, s) && len(w)-len(s) >= 3 {- w = w[:len(w)-len(s)] + rep- break- }- }- if len(w) > 3 && strings.HasSuffix(w, "y") {- w = w[:len(w)-1] + "i"- }- if len(w) > 3 && strings.HasSuffix(w, "e") {- w = w[:len(w)-1]- }- if len(w) > 3 {- rb := []rune(w)- if rb[len(rb)-1] == rb[len(rb)-2] && !isVowelRune(rb[len(rb)-1]) {- w = string(rb[:len(rb)-1])- }- }- return w-}--// stem is Python _stem: nltk PorterStemmer when available, else fallbackStem.-func stem(word string) string { return fallbackStem(word) }+// stem is Python _stem: nltk's PorterStemmer (text_processing.py:218-222+// uses it whenever nltk is importable, which production guarantees via+// rag/nlp/synonym.py). porterStem is a faithful port of that exact+// implementation (NLTK_EXTENSIONS mode), so Go matches Python's stems+// word-for-word. The old suffix-stripping fallback is gone: it diverged from+// nltk on exactly the words that matter for keyword narrowing.+func stem(word string) string { return porterStem(word) }// keywordForms mirrors Python _keyword_forms: verbatim keeps forms containing// any non-stemmable token (matched by substring); stemmed holds all-ASCII-letterdiff --git a/internal/rag/prompts/generator.go b/internal/rag/prompts/generator.goindex 2b1a9e4c00..8078dda229 100644--- a/internal/rag/prompts/generator.go+++ b/internal/rag/prompts/generator.go@@ -116,14 +116,14 @@ var DefaultCiteRules = CitationPrompt("")const citationIDSuffix = "\n\nIMPORTANT: The example IDs above (45, 46, 78, etc.) are illustrative only. Use the actual chunk IDs from the provided knowledge blocks."// CitationPrompt mirrors Python rag/prompts/generator.citation_prompt: return-// the citation-rules text the final answer must follow. With an empty-// userDefined it returns the embedded citation_prompt.md plus the illustrative-// IDs caveat; a non-empty userDefined (the rendered user_defined_prompts)-// replaces it verbatim, exactly as Python returns-// citation_prompt(self.user_defined_prompts).+// the citation-rules text the final answer must follow. The user-defined+// override (citation_guidelines) REPLACES the template but Python still+// appends the illustrative-IDs caveat to it (generator.py:227-228 — the+// suffix is concatenated after whatever template rendered), so the same+// caveat is appended here in both cases.func CitationPrompt(userDefined string) string {if strings.TrimSpace(userDefined) != "" {- return userDefined+ return userDefined + citationIDSuffix}data, err := templatesFS.ReadFile("citation_prompt.md")if err != nil {diff --git a/internal/rag/prompts/generator_test.go b/internal/rag/prompts/generator_test.goindex 40d813effe..962676095d 100644--- a/internal/rag/prompts/generator_test.go+++ b/internal/rag/prompts/generator_test.go@@ -140,8 +140,13 @@ func TestCitationPromptCarriesIllustrativeIDCaveat(t *testing.T) {}func TestCitationPromptHonoursOverride(t *testing.T) {- if got := CitationPrompt("Cite as [n]."); got != "Cite as [n]." {- t.Errorf("CitationPrompt(override) = %q, want the override verbatim", got)+ // Python generator.py:227-228 renders the override (citation_guidelines)+ // template and STILL appends the illustrative-IDs caveat unconditionally —+ // the suffix is concatenated after whatever template rendered, so the+ // override replaces the template but never the caveat.+ want := "Cite as [n]." + citationIDSuffix+ if got := CitationPrompt("Cite as [n]."); got != want {+ t.Errorf("CitationPrompt(override) = %q, want override + caveat %q", got, want)}}diff --git a/internal/service/chat_pipeline.go b/internal/service/chat_pipeline.goindex e81ce7788a..ba67310463 100644--- a/internal/service/chat_pipeline.go+++ b/internal/service/chat_pipeline.go@@ -739,7 +739,24 @@ func (s *ChatPipelineService) AsyncChat(if s.shouldUseWebSearch(chat, kwargs["internet"]) {webSearch = s.harnessWebSearchFunc(chat.PromptConfig)}- hk, harnessAnswer, hErr := s.retrieveViaHarness(ctx, question, kbs, docIDs, imageFiles, attachments, thinkingMode, chat.TenantID, chat.LLMID, chat.ID, webSearch, sink)+ // Python passes system_prompt=_render_reasoning_system_prompt(+ // dialog, prompt_config, kwargs) to RAGTools+ // (dialog_service.py:2084): the dialog system prompt rendered+ // with the caller kwargs, a UTC date, and {knowledge} defaulted+ // to "" — the agentic graph supplies evidence itself.+ harnessSystemPrompt := ""+ if sp, ok := chat.PromptConfig["system"].(string); ok && sp != "" {+ kws := make(map[string]interface{}, len(kwargs)+2)+ for k, v := range kwargs {+ kws[k] = v+ }+ kws["date"] = time.Now().UTC().Format("2006-01-02 15:04:05")+ if _, ok := kws["knowledge"]; !ok {+ kws["knowledge"] = ""+ }+ harnessSystemPrompt = s.formatPrompt(sp, kws)+ }+ hk, harnessAnswer, hErr := s.retrieveViaHarness(ctx, question, kbs, docIDs, imageFiles, attachments, thinkingMode, chat.TenantID, chat.LLMID, chat.ID, webSearch, sink, harnessSystemPrompt)// The harness streams think-then-answer inside ONE compose call.// Close the block here, once that call (and its trailing// narration line) has returned: a reasoning-only run would@@ -4676,6 +4693,13 @@ type HarnessRequest struct {// the tool from the agentic surface, mirroring Python's provider gate// (action_session.py:463 discards web_search when tools.web_search is None).WebSearch func(ctx context.Context, queries []string) ([]string, error)+ // SystemPrompt is the dialog-level system prompt rendered the way Python's+ // _render_reasoning_system_prompt (dialog_service.py:1887-1917) renders it+ // for RAGTools(system_prompt=...): caller kwargs + a UTC date, with+ // {knowledge} defaulted to "" (the agentic graph supplies evidence through+ // its own evidence block). Empty when the dialog configures none — Python+ // then composes without the "# Assistant configuration" block.+ SystemPrompt string}// HarnessResult is the evidence the harness returns, normalized to the map@@ -4742,7 +4766,7 @@ func toolLoopLine(sink func(delta string, isThink bool), line string) {sink(line+thinkLineBreak, true)}-func (s *ChatPipelineService) retrieveViaHarness(ctx context.Context, question string, kbs []*entity.Knowledgebase, docIDs []string, images []string, textAttachments string, thinkingMode, tenantID, modelID, sessionID string, webSearch func(context.Context, []string) ([]string, error), answerSink func(delta string, isThink bool)) (map[string]interface{}, string, error) {+func (s *ChatPipelineService) retrieveViaHarness(ctx context.Context, question string, kbs []*entity.Knowledgebase, docIDs []string, images []string, textAttachments string, thinkingMode, tenantID, modelID, sessionID string, webSearch func(context.Context, []string) ([]string, error), answerSink func(delta string, isThink bool), dialogSystemPrompt string) (map[string]interface{}, string, error) {if harnessRetriever == nil {return nil, "", fmt.Errorf("harness retriever not wired at bootstrap")}@@ -4766,6 +4790,7 @@ func (s *ChatPipelineService) retrieveViaHarness(ctx context.Context, question sImages: images,TextAttachments: textAttachments,WebSearch: webSearch,+ SystemPrompt: dialogSystemPrompt,})if err != nil {return nil, "", errdiff --git a/internal/service/chat_pipeline_test.go b/internal/service/chat_pipeline_test.goindex fe34071638..aa167dd8d2 100644--- a/internal/service/chat_pipeline_test.go+++ b/internal/service/chat_pipeline_test.go@@ -1569,7 +1569,7 @@ func TestRetrieveViaHarnessEmitsToolLoopLines(t *testing.T) {var got []stringvar thinks []bools := &ChatPipelineService{}- _, answer, err := s.retrieveViaHarness(context.Background(), "q", nil, nil, nil, "", "high", "t", "m", "sess", nil, collectSink(&got, &thinks))+ _, answer, err := s.retrieveViaHarness(context.Background(), "q", nil, nil, nil, "", "high", "t", "m", "sess", nil, collectSink(&got, &thinks), "")if err != nil {t.Fatalf("retrieveViaHarness: %v", err)}@@ -1611,7 +1611,7 @@ func TestRetrieveViaHarnessToolLoopObservation(t *testing.T) {var got []stringvar thinks []bools := &ChatPipelineService{}- if _, _, err := s.retrieveViaHarness(context.Background(), "q", nil, nil, nil, "", "high", "t", "m", "sess", nil, collectSink(&got, &thinks)); err != nil {+ if _, _, err := s.retrieveViaHarness(context.Background(), "q", nil, nil, nil, "", "high", "t", "m", "sess", nil, collectSink(&got, &thinks), ""); err != nil {t.Fatalf("retrieveViaHarness: %v", err)}joined := strings.Join(got, "")@@ -1631,7 +1631,7 @@ func TestRetrieveViaHarnessNaiveSkipsToolLoop(t *testing.T) {var got []stringvar thinks []bools := &ChatPipelineService{}- if _, _, err := s.retrieveViaHarness(context.Background(), "q", nil, nil, nil, "", "naive", "t", "m", "sess", nil, collectSink(&got, &thinks)); err != nil {+ if _, _, err := s.retrieveViaHarness(context.Background(), "q", nil, nil, nil, "", "naive", "t", "m", "sess", nil, collectSink(&got, &thinks), ""); err != nil {t.Fatalf("retrieveViaHarness: %v", err)}if len(got) != 0 {diff --git a/internal/service/nlp/retrieval.go b/internal/service/nlp/retrieval.goindex 62c70f529f..d560207aeb 100644--- a/internal/service/nlp/retrieval.go+++ b/internal/service/nlp/retrieval.go@@ -541,7 +541,7 @@ func buildRetrievalFusionExpr(docEngineType string, topn int, vectorSimilarityWereturn &types.FusionExpr{Method: "weighted_sum",TopN: topn,- FusionParams: map[string]interface{}{"weights": "0.05,0.95"},+ FusionParams: map[string]interface{}{"weights": "0.001,1"},}}diff --git a/internal/service/nlp/retrieval_test.go b/internal/service/nlp/retrieval_test.goindex 505a0eb1e6..61afe6a909 100644--- a/internal/service/nlp/retrieval_test.go+++ b/internal/service/nlp/retrieval_test.go@@ -311,15 +311,16 @@ func assertFusionWeights(t *testing.T, request *types.SearchRequest, want string}}-func TestBuildRetrievalFusionExprKeepsLegacyWeightsOutsideInfinity(t *testing.T) {+func TestBuildRetrievalFusionExprKeepsPythonWeightsOutsideInfinity(t *testing.T) {expr := buildRetrievalFusionExpr(string(engine.EngineElasticsearch), 10, float64Ptr(0.8))- if got := expr.FusionParams["weights"]; got != "0.05,0.95" {- t.Fatalf("expected Elasticsearch weights=0.05,0.95, got %v", got)+ // Python Dealer.search's non-Infinity branch (rag/nlp/search.py:265).+ if got := expr.FusionParams["weights"]; got != "0.001,1" {+ t.Fatalf("expected Elasticsearch weights=0.001,1, got %v", got)}}-func TestSearchKeepsLegacyFusionWeightForElasticsearch(t *testing.T) {+func TestSearchKeepsPythonFusionWeightForElasticsearch(t *testing.T) {if GetQueryBuilder() == nil {globalQueryBuilder = NewQueryBuilder()}@@ -350,7 +351,7 @@ func TestSearchKeepsLegacyFusionWeightForElasticsearch(t *testing.T) {if !ok {t.Fatalf("expected third match expression to be FusionExpr, got %T", docEngine.searchRequest.MatchExprs[2])}- if got := fusionExpr.FusionParams["weights"]; got != "0.05,0.95" {- t.Fatalf("expected Elasticsearch weights=0.05,0.95, got %v", got)+ if got := fusionExpr.FusionParams["weights"]; got != "0.001,1" {+ t.Fatalf("expected Elasticsearch weights=0.001,1, got %v", got)}}